1//===- SIInsertWaitcnts.cpp - Insert Wait Instructions --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert wait instructions for memory reads and writes.
11///
12/// Memory reads and writes are issued asynchronously, so we need to insert
13/// S_WAITCNT instructions when we want to access any of their results or
14/// overwrite any register that's used asynchronously.
15///
16/// TODO: This pass currently keeps one timeline per hardware counter. A more
17/// finely-grained approach that keeps one timeline per event type could
18/// sometimes get away with generating weaker s_waitcnt instructions. For
19/// example, when both SMEM and LDS are in flight and we need to wait for
20/// the i-th-last LDS instruction, then an lgkmcnt(i) is actually sufficient,
21/// but the pass will currently generate a conservative lgkmcnt(0) because
22/// multiple event types are in flight.
23//
24//===----------------------------------------------------------------------===//
25
26#include "AMDGPU.h"
27#include "AMDGPUHWEvents.h"
28#include "AMDGPUWaitcntUtils.h"
29#include "GCNSubtarget.h"
30#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
31#include "SIMachineFunctionInfo.h"
32#include "Utils/AMDGPUBaseInfo.h"
33#include "llvm/ADT/MapVector.h"
34#include "llvm/ADT/PostOrderIterator.h"
35#include "llvm/ADT/Sequence.h"
36#include "llvm/Analysis/AliasAnalysis.h"
37#include "llvm/CodeGen/MachineFrameInfo.h"
38#include "llvm/CodeGen/MachineLoopInfo.h"
39#include "llvm/CodeGen/MachinePassManager.h"
40#include "llvm/CodeGen/MachinePostDominators.h"
41#include "llvm/IR/Dominators.h"
42#include "llvm/InitializePasses.h"
43#include "llvm/TargetParser/AMDGPUTargetParser.h"
44
45using namespace llvm;
46
47using HWEvents = AMDGPU::HWEvents;
48
49#define DEBUG_TYPE "si-insert-waitcnts"
50
51static cl::opt<bool>
52 ForceEmitZeroFlag("amdgpu-waitcnt-forcezero",
53 cl::desc("Force all waitcnt instrs to be emitted as "
54 "s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
55 cl::init(Val: false), cl::Hidden);
56
57static cl::opt<bool> ForceEmitZeroLoadFlag(
58 "amdgpu-waitcnt-load-forcezero",
59 cl::desc("Force all waitcnt load counters to wait until 0"),
60 cl::init(Val: false), cl::Hidden);
61
62static cl::opt<bool> ExpertSchedulingModeFlag(
63 "amdgpu-expert-scheduling-mode",
64 cl::desc("Enable expert scheduling mode 2 for all functions (GFX12+ only)"),
65 cl::init(Val: false), cl::Hidden);
66
67namespace {
68
69template <typename EmitWaitcntFn>
70static void EmitExpandedWaitcnt(unsigned Outstanding, unsigned Target,
71 EmitWaitcntFn &&EmitWaitcnt) {
72 // Emit waitcnts from (Outstanding - 1) down to Target.
73 for (unsigned I = Outstanding - 1; I > Target && I != ~0u; --I)
74 EmitWaitcnt(I);
75 EmitWaitcnt(Target);
76}
77
78/// Integer IDs used to track vector memory locations we may have to wait on.
79/// Encoded as u16 chunks:
80///
81/// [0, REGUNITS_END ): MCRegUnit
82/// [LDSDMA_BEGIN, LDSDMA_END ) : LDS DMA IDs
83///
84/// NOTE: The choice of encoding these as "u16 chunks" is arbitrary.
85/// It gives (2 << 16) - 1 entries per category which is more than enough
86/// for all register units. MCPhysReg is u16 so we don't even support >u16
87/// physical register numbers at this time, let alone >u16 register units.
88/// In any case, an assertion in "WaitcntBrackets" ensures REGUNITS_END
89/// is enough for all register units.
90using VMEMID = uint32_t;
91
92enum : VMEMID {
93 TRACKINGID_RANGE_LEN = (1 << 16),
94
95 // Important: MCRegUnits must always be tracked starting from 0, as we
96 // need to be able to convert between a MCRegUnit and a VMEMID freely.
97 REGUNITS_BEGIN = 0,
98 REGUNITS_END = REGUNITS_BEGIN + TRACKINGID_RANGE_LEN,
99
100 // Note for LDSDMA: LDSDMA_BEGIN corresponds to the "common"
101 // entry, which is updated for all LDS DMA operations encountered.
102 // Specific LDS DMA IDs start at LDSDMA_BEGIN + 1.
103 NUM_LDSDMA = TRACKINGID_RANGE_LEN,
104 LDSDMA_BEGIN = REGUNITS_END,
105 LDSDMA_END = LDSDMA_BEGIN + NUM_LDSDMA,
106};
107
108/// Convert a MCRegUnit to a VMEMID.
109static constexpr VMEMID toVMEMID(MCRegUnit RU) {
110 return static_cast<unsigned>(RU);
111}
112
113} // namespace
114
115namespace {
116
117// Maps values of InstCounterType to the instruction that waits on that
118// counter. Only used if GCNSubtarget::hasExtendedWaitCounts()
119// returns true, and does not cover VA_VDST or VM_VSRC.
120static const unsigned
121 instrsForExtendedCounterTypes[AMDGPU::NUM_EXTENDED_INST_CNTS] = {
122 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT,
123 AMDGPU::S_WAIT_EXPCNT, AMDGPU::S_WAIT_STORECNT,
124 AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
125 AMDGPU::S_WAIT_KMCNT, AMDGPU::S_WAIT_XCNT,
126 AMDGPU::S_WAIT_ASYNCCNT, AMDGPU::S_WAIT_TENSORCNT};
127
128// ASYNCMARK and WAIT_ASYNCMARK are meta instructions that emit no hardware
129// code but still need to be processed by this pass for async vmcnt tracking.
130static bool isNonWaitcntMetaInst(const MachineInstr &MI) {
131 switch (MI.getOpcode()) {
132 case AMDGPU::ASYNCMARK:
133 case AMDGPU::WAIT_ASYNCMARK:
134 return false;
135 default:
136 return MI.isMetaInstruction();
137 }
138}
139
140static bool updateVMCntOnly(const MachineInstr &Inst) {
141 return (SIInstrInfo::isVMEM(MI: Inst) && !SIInstrInfo::isFLAT(MI: Inst)) ||
142 SIInstrInfo::isFLATGlobal(MI: Inst) || SIInstrInfo::isFLATScratch(MI: Inst);
143}
144
145#ifndef NDEBUG
146static bool isNormalMode(AMDGPU::InstCounterType MaxCounter) {
147 return MaxCounter == AMDGPU::NUM_NORMAL_INST_CNTS;
148}
149#endif // NDEBUG
150
151class WaitcntBrackets;
152
153// This abstracts the logic for generating and updating S_WAIT* instructions
154// away from the analysis that determines where they are needed. This was
155// done because the set of counters and instructions for waiting on them
156// underwent a major shift with gfx12, sufficiently so that having this
157// abstraction allows the main analysis logic to be simpler than it would
158// otherwise have had to become.
159class WaitcntGenerator {
160protected:
161 const GCNSubtarget &ST;
162 const SIInstrInfo &TII;
163 AMDGPU::IsaVersion IV;
164 AMDGPU::InstCounterType MaxCounter;
165 bool OptNone;
166 bool ExpandWaitcntProfiling = false;
167 const AMDGPU::HardwareLimits &Limits;
168
169public:
170 WaitcntGenerator() = delete;
171 WaitcntGenerator(const WaitcntGenerator &) = delete;
172 WaitcntGenerator(const MachineFunction &MF,
173 AMDGPU::InstCounterType MaxCounter,
174 const AMDGPU::HardwareLimits &Limits)
175 : ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
176 IV(AMDGPU::getIsaVersion(GPU: ST.getCPU())), MaxCounter(MaxCounter),
177 OptNone(MF.getFunction().hasOptNone() ||
178 MF.getTarget().getOptLevel() == CodeGenOptLevel::None),
179 ExpandWaitcntProfiling(
180 MF.getFunction().hasFnAttribute(Kind: "amdgpu-expand-waitcnt-profiling")),
181 Limits(Limits) {}
182
183 // Return true if the current function should be compiled with no
184 // optimization.
185 bool isOptNone() const { return OptNone; }
186
187 unsigned getLimit(AMDGPU::InstCounterType E) const { return Limits.get(T: E); }
188
189 // Edits an existing sequence of wait count instructions according
190 // to an incoming Waitcnt value, which is itself updated to reflect
191 // any new wait count instructions which may need to be generated by
192 // WaitcntGenerator::createNewWaitcnt(). It will return true if any edits
193 // were made.
194 //
195 // This editing will usually be merely updated operands, but it may also
196 // delete instructions if the incoming Wait value indicates they are not
197 // needed. It may also remove existing instructions for which a wait
198 // is needed if it can be determined that it is better to generate new
199 // instructions later, as can happen on gfx12.
200 virtual bool
201 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
202 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
203 MachineBasicBlock::instr_iterator It) const = 0;
204
205 // Transform a soft waitcnt into a normal one.
206 bool promoteSoftWaitCnt(MachineInstr *Waitcnt) const;
207
208 // Generates new wait count instructions according to the value of
209 // Wait, returning true if any new instructions were created.
210 // ScoreBrackets is used for profiling expansion.
211 virtual bool createNewWaitcnt(MachineBasicBlock &Block,
212 MachineBasicBlock::instr_iterator It,
213 AMDGPU::Waitcnt Wait,
214 const WaitcntBrackets &ScoreBrackets) = 0;
215
216 // Returns the set of HWEvents that corresponds to counter \p T.
217 virtual HWEvents getWaitEvents(AMDGPU::InstCounterType T) const = 0;
218
219 /// \returns the counter that corresponds to event \p E.
220 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
221 assert(E.size() == 1 && "Cannot handle a mask of events!");
222 for (auto T : AMDGPU::inst_counter_types()) {
223 if (getWaitEvents(T) & E)
224 return T;
225 }
226 llvm_unreachable("event type has no associated counter");
227 }
228
229 // Returns a new waitcnt with all counters except VScnt set to 0. If
230 // IncludeVSCnt is true, VScnt is set to 0, otherwise it is set to ~0u.
231 // AsyncCnt and TensorCnt always default to ~0u (don't wait for it). They
232 // are only updated when a call to @llvm.amdgcn.wait.asyncmark() is
233 // processed.
234 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const = 0;
235
236 virtual ~WaitcntGenerator() = default;
237};
238
239class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
240 static constexpr const HWEvents
241 WaitEventMaskForInstPreGFX12[AMDGPU::NUM_INST_CNTS] = {
242 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_SAMPLER_READ_ACCESS |
243 HWEvents::VMEM_BVH_READ_ACCESS,
244 HWEvents::SMEM_ACCESS | HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS |
245 HWEvents::SQ_MESSAGE,
246 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
247 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
248 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
249 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
250 HWEvents::NONE,
251 HWEvents::NONE,
252 HWEvents::NONE,
253 HWEvents::NONE,
254 HWEvents::NONE,
255 HWEvents::NONE,
256 HWEvents::NONE,
257 HWEvents::NONE,
258 HWEvents::NONE};
259
260public:
261 using WaitcntGenerator::WaitcntGenerator;
262 bool
263 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
264 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
265 MachineBasicBlock::instr_iterator It) const override;
266
267 bool createNewWaitcnt(MachineBasicBlock &Block,
268 MachineBasicBlock::instr_iterator It,
269 AMDGPU::Waitcnt Wait,
270 const WaitcntBrackets &ScoreBrackets) override;
271
272 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
273 HWEvents EVs = WaitEventMaskForInstPreGFX12[T];
274 if (T == AMDGPU::LOAD_CNT && !ST.hasVscnt())
275 EVs |= WaitEventMaskForInstPreGFX12[AMDGPU::STORE_CNT];
276 return EVs;
277 }
278
279 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
280};
281
282class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
283protected:
284 bool IsExpertMode;
285 static constexpr const HWEvents
286 WaitEventMaskForInstGFX12Plus[AMDGPU::NUM_INST_CNTS] = {
287 HWEvents::VMEM_READ_ACCESS | HWEvents::GLOBAL_INV_ACCESS,
288 HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS,
289 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
290 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
291 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
292
293 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
294 HWEvents::VMEM_SAMPLER_READ_ACCESS,
295 HWEvents::VMEM_BVH_READ_ACCESS,
296
297 HWEvents::SMEM_ACCESS | HWEvents::SQ_MESSAGE | HWEvents::SCC_WRITE,
298 HWEvents::VMEM_GROUP | HWEvents::SMEM_GROUP,
299 HWEvents::ASYNC_ACCESS,
300 HWEvents::TENSOR_ACCESS,
301 HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_DPMACC_READ |
302 HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_XDL_READ,
303 HWEvents::VGPR_CSMACC_WRITE | HWEvents::VGPR_DPMACC_WRITE |
304 HWEvents::VGPR_TRANS_WRITE | HWEvents::VGPR_XDL_WRITE,
305 HWEvents::VGPR_LDS_READ | HWEvents::VGPR_FLAT_READ |
306 HWEvents::VGPR_VMEM_READ};
307
308public:
309 WaitcntGeneratorGFX12Plus() = delete;
310 WaitcntGeneratorGFX12Plus(const MachineFunction &MF,
311 AMDGPU::InstCounterType MaxCounter,
312 const AMDGPU::HardwareLimits &Limits,
313 bool IsExpertMode)
314 : WaitcntGenerator(MF, MaxCounter, Limits), IsExpertMode(IsExpertMode) {}
315
316 bool
317 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
318 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
319 MachineBasicBlock::instr_iterator It) const override;
320
321 bool createNewWaitcnt(MachineBasicBlock &Block,
322 MachineBasicBlock::instr_iterator It,
323 AMDGPU::Waitcnt Wait,
324 const WaitcntBrackets &ScoreBrackets) override;
325
326 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
327 return WaitEventMaskForInstGFX12Plus[T];
328 }
329
330 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
331};
332
333// Flags indicating which counters should be flushed in a loop preheader.
334struct PreheaderFlushFlags {
335 bool FlushVmCnt = false;
336 bool FlushDsCnt = false;
337};
338
339class SIInsertWaitcnts {
340 DenseMap<const Value *, MachineBasicBlock *> SLoadAddresses;
341 DenseMap<MachineBasicBlock *, PreheaderFlushFlags> PreheadersToFlush;
342 MachineLoopInfo &MLI;
343 MachinePostDominatorTree &PDT;
344 AliasAnalysis *AA = nullptr;
345 MachineFunction &MF;
346
347 struct BlockInfo {
348 std::unique_ptr<WaitcntBrackets> Incoming;
349 bool Dirty = true;
350 BlockInfo() = default;
351 BlockInfo(BlockInfo &&) = default;
352 BlockInfo &operator=(BlockInfo &&) = default;
353 ~BlockInfo();
354 };
355
356 MapVector<MachineBasicBlock *, BlockInfo> BlockInfos;
357
358 bool ForceEmitWaitcnt[AMDGPU::NUM_INST_CNTS] = {};
359
360 std::unique_ptr<WaitcntGenerator> WCG;
361
362 // Remember call and return instructions in the function.
363 DenseSet<MachineInstr *> CallInsts;
364 DenseSet<MachineInstr *> ReturnInsts;
365
366 // Remember all S_ENDPGM instructions. The boolean flag is true if there might
367 // be outstanding stores but definitely no outstanding scratch stores, to help
368 // with insertion of DEALLOC_VGPRS messages.
369 DenseMap<MachineInstr *, bool> EndPgmInsts;
370
371 AMDGPU::HardwareLimits Limits;
372
373public:
374 const GCNSubtarget &ST;
375 const SIInstrInfo &TII;
376 const SIRegisterInfo &TRI;
377 const MachineRegisterInfo &MRI;
378 AMDGPU::InstCounterType SmemAccessCounter;
379 AMDGPU::InstCounterType MaxCounter;
380 bool IsExpertMode = false;
381 const bool TgSplit;
382
383 SIInsertWaitcnts(MachineLoopInfo &MLI, MachinePostDominatorTree &PDT,
384 AliasAnalysis *AA, MachineFunction &MF)
385 : MLI(MLI), PDT(PDT), AA(AA), MF(MF), ST(MF.getSubtarget<GCNSubtarget>()),
386 TII(*ST.getInstrInfo()), TRI(TII.getRegisterInfo()),
387 MRI(MF.getRegInfo()),
388 TgSplit(ST.hasTgSplitSupport() &&
389 AMDGPU::isTgSplitEnabled(F: MF.getFunction())) {}
390
391 const AMDGPU::HardwareLimits &getLimits() const { return Limits; }
392
393 PreheaderFlushFlags getPreheaderFlushFlags(MachineLoop *ML,
394 const WaitcntBrackets &Brackets);
395 PreheaderFlushFlags isPreheaderToFlush(MachineBasicBlock &MBB,
396 const WaitcntBrackets &ScoreBrackets);
397 bool isVMEMOrFlatVMEM(const MachineInstr &MI) const;
398 bool isDSRead(const MachineInstr &MI) const;
399 bool mayStoreIncrementingDSCNT(const MachineInstr &MI) const;
400 bool run();
401
402 bool isAsync(const MachineInstr &MI) const {
403 if (!SIInstrInfo::isLDSDMA(MI))
404 return false;
405 if (SIInstrInfo::usesASYNC_CNT(MI))
406 return true;
407 const MachineOperand *Async =
408 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::IsAsync);
409 return Async && (Async->getImm());
410 }
411
412 bool isNonAsyncLdsDmaWrite(const MachineInstr &MI) const {
413 return SIInstrInfo::mayWriteLDSThroughDMA(MI) && !isAsync(MI);
414 }
415
416 bool isAsyncLdsDmaWrite(const MachineInstr &MI) const {
417 return SIInstrInfo::mayWriteLDSThroughDMA(MI) && isAsync(MI);
418 }
419
420 bool shouldUpdateAsyncMark(const MachineInstr &MI,
421 AMDGPU::InstCounterType T) const {
422 if (SIInstrInfo::usesTENSOR_CNT(MI))
423 return T == AMDGPU::TENSOR_CNT;
424 if (!isAsyncLdsDmaWrite(MI))
425 return false;
426 if (SIInstrInfo::usesASYNC_CNT(MI))
427 return T == AMDGPU::ASYNC_CNT;
428 return T == AMDGPU::LOAD_CNT;
429 }
430
431 bool isVmemAccess(const MachineInstr &MI) const;
432 bool generateWaitcntInstBefore(MachineInstr &MI,
433 WaitcntBrackets &ScoreBrackets,
434 MachineInstr *OldWaitcntInstr,
435 PreheaderFlushFlags FlushFlags);
436 bool generateWaitcnt(AMDGPU::Waitcnt Wait,
437 MachineBasicBlock::instr_iterator It,
438 MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
439 MachineInstr *OldWaitcntInstr);
440 void updateEventWaitcntAfter(MachineInstr &Inst,
441 WaitcntBrackets *ScoreBrackets);
442 bool isNextENDPGM(MachineBasicBlock::instr_iterator It,
443 MachineBasicBlock *Block) const;
444 bool insertForcedWaitAfter(MachineInstr &Inst, MachineBasicBlock &Block,
445 WaitcntBrackets &ScoreBrackets);
446 bool insertWaitcntInBlock(MachineFunction &MF, MachineBasicBlock &Block,
447 WaitcntBrackets &ScoreBrackets);
448 /// Removes redundant Soft Xcnt Waitcnts in \p Block emitted by the Memory
449 /// Legalizer. Returns true if block was modified.
450 bool removeRedundantSoftXcnts(MachineBasicBlock &Block);
451 void setSchedulingMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
452 bool ExpertMode) const;
453 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const {
454 return WCG->getWaitEvents(T);
455 }
456 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
457 return WCG->getCounterFromEvent(E);
458 }
459};
460
461// This objects maintains the current score brackets of each wait counter, and
462// a per-register scoreboard for each wait counter.
463//
464// We also maintain the latest score for every event type that can change the
465// waitcnt in order to know if there are multiple types of events within
466// the brackets. When multiple types of event happen in the bracket,
467// wait count may get decreased out of order, therefore we need to put in
468// "s_waitcnt 0" before use.
469class WaitcntBrackets {
470public:
471 WaitcntBrackets(const SIInsertWaitcnts *Context) : Context(Context) {
472 assert(Context->TRI.getNumRegUnits() < REGUNITS_END);
473 }
474
475#ifndef NDEBUG
476 ~WaitcntBrackets() {
477 unsigned NumUnusedVmem = 0, NumUnusedSGPRs = 0;
478 for (auto &[ID, Val] : VMem) {
479 if (Val.empty())
480 ++NumUnusedVmem;
481 }
482 for (auto &[ID, Val] : SGPRs) {
483 if (Val.empty())
484 ++NumUnusedSGPRs;
485 }
486
487 if (NumUnusedVmem || NumUnusedSGPRs) {
488 errs() << "WaitcntBracket had unused entries at destruction time: "
489 << NumUnusedVmem << " VMem and " << NumUnusedSGPRs
490 << " SGPR unused entries\n";
491 std::abort();
492 }
493 }
494#endif
495
496 bool isSmemCounter(AMDGPU::InstCounterType T) const {
497 return T == Context->SmemAccessCounter || T == AMDGPU::X_CNT;
498 }
499
500 unsigned getOutstanding(AMDGPU::InstCounterType T) const {
501 return ScoreUBs[T] - ScoreLBs[T];
502 }
503
504 bool hasPendingVMEM(VMEMID ID, AMDGPU::InstCounterType T) const {
505 return getVMemScore(TID: ID, T) > getScoreLB(T);
506 }
507
508 /// \Return true if we have no score entries for counter \p T.
509 bool empty(AMDGPU::InstCounterType T) const { return getScoreRange(T) == 0; }
510
511private:
512 unsigned getScoreLB(AMDGPU::InstCounterType T) const {
513 assert(T < AMDGPU::NUM_INST_CNTS);
514 return ScoreLBs[T];
515 }
516
517 unsigned getScoreUB(AMDGPU::InstCounterType T) const {
518 assert(T < AMDGPU::NUM_INST_CNTS);
519 return ScoreUBs[T];
520 }
521
522 unsigned getScoreRange(AMDGPU::InstCounterType T) const {
523 return getScoreUB(T) - getScoreLB(T);
524 }
525
526 unsigned getSGPRScore(MCRegUnit RU, AMDGPU::InstCounterType T) const {
527 auto It = SGPRs.find(Val: RU);
528 return It != SGPRs.end() ? It->second.get(T) : 0;
529 }
530
531 unsigned getVMemScore(VMEMID TID, AMDGPU::InstCounterType T) const {
532 auto It = VMem.find(Val: TID);
533 return It != VMem.end() ? It->second.Scores[T] : 0;
534 }
535
536public:
537 // Do some internal consistency checks.
538 void verify() {
539 // These track the same underlying hardware counter so the score ranges
540 // should be identical.
541 assert(getScoreLB(AMDGPU::VA_VDST_RD) == getScoreLB(AMDGPU::VA_VDST_WR));
542 assert(getScoreUB(AMDGPU::VA_VDST_RD) == getScoreUB(AMDGPU::VA_VDST_WR));
543 }
544
545 bool merge(const WaitcntBrackets &Other);
546
547 bool counterOutOfOrder(AMDGPU::InstCounterType T) const;
548 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const {
549 simplifyWaitcnt(CheckWait: Wait, UpdateWait&: Wait);
550 }
551 void simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
552 AMDGPU::Waitcnt &UpdateWait) const;
553 void simplifyWaitcnt(AMDGPU::InstCounterType T, unsigned &Count) const;
554 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T) const;
555 void simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
556 AMDGPU::Waitcnt &UpdateWait) const;
557 void simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
558 AMDGPU::Waitcnt &UpdateWait) const;
559
560 void determineWaitForPhysReg(AMDGPU::InstCounterType T, MCPhysReg Reg,
561 AMDGPU::Waitcnt &Wait,
562 const MachineInstr &MI) const;
563 MCPhysReg determineVGPR16Dependency(const MachineInstr &MI,
564 AMDGPU::InstCounterType T,
565 MCPhysReg Reg) const;
566 void determineWaitForLDSDMA(AMDGPU::InstCounterType T, VMEMID TID,
567 AMDGPU::Waitcnt &Wait) const;
568 AMDGPU::Waitcnt determineAsyncWait(unsigned N);
569 void tryClearSCCWriteEvent(MachineInstr *Inst);
570
571 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
572 void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
573 void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
574 void updateByEvent(HWEvents E, MachineInstr &MI);
575 void recordAsyncMark(MachineInstr &MI);
576
577 HWEvents getPendingEvents() const { return PendingEvents; }
578 bool hasPendingEvent() const { return PendingEvents.any(); }
579 bool hasPendingEvent(HWEvents E) const { return PendingEvents.contains(Other: E); }
580 bool hasPendingEvent(AMDGPU::InstCounterType T) const {
581 bool HasPending = (PendingEvents & Context->getWaitEvents(T)).any();
582 assert(HasPending == !empty(T) &&
583 "Expected pending events iff scoreboard is not empty");
584 return HasPending;
585 }
586
587 bool hasMixedPendingEvents(AMDGPU::InstCounterType T) const {
588 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
589 // Return true if more than one bit is set in Events.
590 return Events.size() > 1;
591 }
592
593 bool hasPendingFlat() const {
594 return ((LastFlatDsCnt > ScoreLBs[AMDGPU::DS_CNT] &&
595 LastFlatDsCnt <= ScoreUBs[AMDGPU::DS_CNT]) ||
596 (LastFlatLoadCnt > ScoreLBs[AMDGPU::LOAD_CNT] &&
597 LastFlatLoadCnt <= ScoreUBs[AMDGPU::LOAD_CNT]));
598 }
599
600 void setPendingFlat() {
601 LastFlatLoadCnt = ScoreUBs[AMDGPU::LOAD_CNT];
602 LastFlatDsCnt = ScoreUBs[AMDGPU::DS_CNT];
603 }
604
605 bool hasPendingGDS() const {
606 return LastGDS > ScoreLBs[AMDGPU::DS_CNT] &&
607 LastGDS <= ScoreUBs[AMDGPU::DS_CNT];
608 }
609
610 unsigned getPendingGDSWait() const {
611 return std::min(a: getScoreUB(T: AMDGPU::DS_CNT) - LastGDS,
612 b: getLimit(T: AMDGPU::DS_CNT) - 1);
613 }
614
615 void setPendingGDS() { LastGDS = ScoreUBs[AMDGPU::DS_CNT]; }
616
617 // Return true if there might be pending writes to the vgpr-interval by VMEM
618 // instructions where the HWEvents in VGPRContext are not contained in E.
619 bool hasDifferentVGPRPendingEvents(MCPhysReg Reg, HWEvents E) const {
620 for (MCRegUnit RU : regunits(Reg)) {
621 auto It = VMem.find(Val: toVMEMID(RU));
622 if (It != VMem.end() && (It->second.VGPRPendingEvents & ~E).any())
623 return true;
624 }
625 return false;
626 }
627
628 void clearVGPRPendingEvents(MCPhysReg Reg) {
629 for (MCRegUnit RU : regunits(Reg)) {
630 if (auto It = VMem.find(Val: toVMEMID(RU)); It != VMem.end()) {
631 It->second.VGPRPendingEvents = HWEvents::NONE;
632 if (It->second.empty())
633 VMem.erase(I: It);
634 }
635 }
636 }
637
638 void setStateOnFunctionEntryOrReturn() {
639 setScoreUB(T: AMDGPU::STORE_CNT,
640 Val: getScoreUB(T: AMDGPU::STORE_CNT) + getLimit(T: AMDGPU::STORE_CNT));
641 PendingEvents |= Context->getWaitEvents(T: AMDGPU::STORE_CNT);
642 }
643
644 ArrayRef<const MachineInstr *> getLDSDMAStores() const {
645 return LDSDMAStores;
646 }
647
648 bool hasPointSampleAccel(const MachineInstr &MI) const;
649 bool hasPointSamplePendingVmemTypes(const MachineInstr &MI,
650 MCPhysReg RU) const;
651
652 void print(raw_ostream &) const;
653 void dump() const { print(dbgs()); }
654
655 // Free up memory by removing empty entries from the DenseMap that track event
656 // scores.
657 void purgeEmptyTrackingData();
658
659private:
660 unsigned getLimit(AMDGPU::InstCounterType T) const {
661 return Context->getLimits().get(T);
662 }
663
664 struct MergeInfo {
665 unsigned OldLB;
666 unsigned OtherLB;
667 unsigned MyShift;
668 unsigned OtherShift;
669 };
670
671 using CounterValueArray = std::array<unsigned, AMDGPU::NUM_INST_CNTS>;
672
673 void determineWaitForScore(AMDGPU::InstCounterType T, unsigned Score,
674 AMDGPU::Waitcnt &Wait) const;
675
676 static bool mergeScore(const MergeInfo &M, unsigned &Score,
677 unsigned OtherScore);
678 bool mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
679 ArrayRef<CounterValueArray> OtherMarks);
680
681 iterator_range<MCRegUnitIterator> regunits(MCPhysReg Reg) const {
682 assert(Reg != AMDGPU::SCC && "Shouldn't be used on SCC");
683 if (!Context->TRI.isInAllocatableClass(RegNo: Reg))
684 return {{}, {}};
685 return Context->TRI.regunits(Reg);
686 }
687
688 void setScoreLB(AMDGPU::InstCounterType T, unsigned Val) {
689 assert(T < AMDGPU::NUM_INST_CNTS);
690 ScoreLBs[T] = Val;
691 }
692
693 void setScoreUB(AMDGPU::InstCounterType T, unsigned Val) {
694 assert(T < AMDGPU::NUM_INST_CNTS);
695 ScoreUBs[T] = Val;
696
697 if (T != AMDGPU::EXP_CNT)
698 return;
699
700 if (getScoreRange(T: AMDGPU::EXP_CNT) > getLimit(T: AMDGPU::EXP_CNT))
701 ScoreLBs[AMDGPU::EXP_CNT] =
702 ScoreUBs[AMDGPU::EXP_CNT] - getLimit(T: AMDGPU::EXP_CNT);
703 }
704
705 void setRegScore(MCPhysReg Reg, AMDGPU::InstCounterType T, unsigned Val) {
706 const SIRegisterInfo &TRI = Context->TRI;
707 if (Reg == AMDGPU::SCC) {
708 SCCScore = Val;
709 } else if (TRI.isVectorRegister(MRI: Context->MRI, Reg)) {
710 for (MCRegUnit RU : regunits(Reg))
711 VMem[toVMEMID(RU)].Scores[T] = Val;
712 } else if (TRI.isSGPRReg(MRI: Context->MRI, Reg)) {
713 for (MCRegUnit RU : regunits(Reg))
714 SGPRs[RU].get(T) = Val;
715 } else {
716 llvm_unreachable("Register cannot be tracked/unknown register!");
717 }
718 }
719
720 void setVMemScore(VMEMID TID, AMDGPU::InstCounterType T, unsigned Val) {
721 VMem[TID].Scores[T] = Val;
722 }
723
724 void setScoreByOperand(const MachineOperand &Op,
725 AMDGPU::InstCounterType CntTy, unsigned Val);
726
727 const SIInsertWaitcnts *Context;
728
729 unsigned ScoreLBs[AMDGPU::NUM_INST_CNTS] = {0};
730 unsigned ScoreUBs[AMDGPU::NUM_INST_CNTS] = {0};
731 HWEvents PendingEvents;
732 // Remember the last flat memory operation.
733 unsigned LastFlatDsCnt = 0;
734 unsigned LastFlatLoadCnt = 0;
735 // Remember the last GDS operation.
736 unsigned LastGDS = 0;
737
738 // The score tracking logic is fragmented as follows:
739 // - VMem: VGPR RegUnits and LDS DMA IDs, see the VMEMID encoding.
740 // - SGPRs: SGPR RegUnits
741 // - SCC: Non-allocatable and not general purpose: not a SGPR.
742 //
743 // For the VMem case, if the key is within the range of LDS DMA IDs,
744 // then the corresponding index into the `LDSDMAStores` vector below is:
745 // Key - LDSDMA_BEGIN - 1
746 // This is because LDSDMA_BEGIN is a generic entry and does not have an
747 // associated MachineInstr.
748 //
749 // TODO: Could we track SCC alongside SGPRs so it's not longer a special case?
750
751 struct VMEMInfo {
752 // Scores for all instruction counters. Zero-initialized.
753 CounterValueArray Scores{};
754 // For VGPRs, we need to track an additional fine-grained set of pending
755 // events.
756 HWEvents VGPRPendingEvents;
757
758 bool empty() const {
759 return all_of(Range: Scores, P: equal_to(Arg: 0)) && !VGPRPendingEvents;
760 }
761 };
762
763 /// Wait cnt scores for every sgpr, the DS_CNT (corresponding to LGKMcnt
764 /// pre-gfx12) or KM_CNT (gfx12+ only), and X_CNT (gfx1250) are relevant.
765 class SGPRInfo {
766 /// Either DS_CNT or KM_CNT score.
767 unsigned ScoreDsKmCnt = 0;
768 unsigned ScoreXCnt = 0;
769
770 public:
771 unsigned get(AMDGPU::InstCounterType T) const {
772 assert(
773 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
774 "Invalid counter");
775 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
776 }
777 unsigned &get(AMDGPU::InstCounterType T) {
778 assert(
779 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
780 "Invalid counter");
781 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
782 }
783
784 bool empty() const { return !ScoreDsKmCnt && !ScoreXCnt; }
785 };
786
787 DenseMap<VMEMID, VMEMInfo> VMem; // VGPR + LDS DMA
788 DenseMap<MCRegUnit, SGPRInfo> SGPRs;
789
790 // Reg score for SCC.
791 unsigned SCCScore = 0;
792 // The unique instruction that has an SCC write pending, if there is one.
793 const MachineInstr *PendingSCCWrite = nullptr;
794
795 // Store representative LDS DMA operations. The only useful info here is
796 // alias info. One store is kept per unique AAInfo.
797 SmallVector<const MachineInstr *> LDSDMAStores;
798
799 // State of all counters at each async mark encountered so far.
800 SmallVector<CounterValueArray> AsyncMarks;
801
802 // But in the rare pathological case, a nest of loops that pushes marks
803 // without waiting on any mark can cause AsyncMarks to grow very large. We cap
804 // it to a reasonable limit. We can tune this later or potentially introduce a
805 // user option to control the value.
806 static constexpr unsigned MaxAsyncMarks = 16;
807
808 // Track the upper bound score for async operations that are not part of a
809 // mark yet. Initialized to all zeros.
810 CounterValueArray AsyncScore{};
811};
812
813SIInsertWaitcnts::BlockInfo::~BlockInfo() = default;
814
815class SIInsertWaitcntsLegacy : public MachineFunctionPass {
816public:
817 static char ID;
818 SIInsertWaitcntsLegacy() : MachineFunctionPass(ID) {}
819
820 bool runOnMachineFunction(MachineFunction &MF) override;
821
822 StringRef getPassName() const override {
823 return "SI insert wait instructions";
824 }
825
826 void getAnalysisUsage(AnalysisUsage &AU) const override {
827 AU.setPreservesCFG();
828 AU.addRequired<MachineLoopInfoWrapperPass>();
829 AU.addRequired<MachinePostDominatorTreeWrapperPass>();
830 AU.addUsedIfAvailable<AAResultsWrapperPass>();
831 AU.addPreserved<AAResultsWrapperPass>();
832 MachineFunctionPass::getAnalysisUsage(AU);
833 }
834};
835
836} // end anonymous namespace
837
838void WaitcntBrackets::setScoreByOperand(const MachineOperand &Op,
839 AMDGPU::InstCounterType CntTy,
840 unsigned Score) {
841 setRegScore(Reg: Op.getReg().asMCReg(), T: CntTy, Val: Score);
842}
843
844// Return true if the subtarget is one that enables Point Sample Acceleration
845// and the MachineInstr passed in is one to which it might be applied (the
846// hardware makes this decision based on several factors, but we can't determine
847// this at compile time, so we have to assume it might be applied if the
848// instruction supports it).
849bool WaitcntBrackets::hasPointSampleAccel(const MachineInstr &MI) const {
850 if (!Context->ST.hasPointSampleAccel() || !SIInstrInfo::isMIMG(MI))
851 return false;
852
853 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: MI.getOpcode());
854 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
855 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
856 return BaseInfo->PointSampleAccel;
857}
858
859// Return true if the subtarget enables Point Sample Acceleration, the supplied
860// MachineInstr is one to which it might be applied and the supplied interval is
861// one that has outstanding writes to vmem-types different than VMEM_NOSAMPLER
862// (this is the type that a point sample accelerated instruction effectively
863// becomes)
864bool WaitcntBrackets::hasPointSamplePendingVmemTypes(const MachineInstr &MI,
865 MCPhysReg Reg) const {
866 if (!hasPointSampleAccel(MI))
867 return false;
868
869 return hasDifferentVGPRPendingEvents(Reg, E: HWEvents::VMEM_READ_ACCESS);
870}
871
872void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
873 assert(E.size() == 1 && "Expected singular event!");
874 AMDGPU::InstCounterType T = Context->getCounterFromEvent(E);
875 assert(T < Context->MaxCounter);
876
877 unsigned UB = getScoreUB(T);
878 unsigned Increment = 1;
879 if ((T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) &&
880 AMDGPU::getHasMatrixScale(Opc: Inst.getOpcode()) &&
881 Context->ST.hasVOP3PX2IncrementsVaVdstTwice()) {
882 // V_WMMA_SCALE instructions use VOP3PX2 encoding. Hardware treats this as
883 // two VOP3P instructions and increments VA_VDST twice.
884 Increment = 2;
885 }
886 unsigned CurrScore = UB + Increment;
887 if (CurrScore == 0)
888 report_fatal_error(reason: "InsertWaitcnt score wraparound");
889 // PendingEvents and ScoreUB need to be update regardless if this event
890 // changes the score of a register or not.
891 // Examples including vm_cnt when buffer-store or lgkm_cnt when send-message.
892 PendingEvents |= E;
893 setScoreUB(T, Val: CurrScore);
894
895 const SIRegisterInfo &TRI = Context->TRI;
896 const MachineRegisterInfo &MRI = Context->MRI;
897 const SIInstrInfo &TII = Context->TII;
898
899 if (T == AMDGPU::EXP_CNT) {
900 // Put score on the source vgprs. If this is a store, just use those
901 // specific register(s).
902 if (TII.isDS(MI: Inst) && Inst.mayLoadOrStore()) {
903 // All GDS operations must protect their address register (same as
904 // export.)
905 if (const auto *AddrOp = TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::addr))
906 setScoreByOperand(Op: *AddrOp, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
907
908 if (Inst.mayStore()) {
909 if (const auto *Data0 =
910 TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data0))
911 setScoreByOperand(Op: *Data0, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
912 if (const auto *Data1 =
913 TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data1))
914 setScoreByOperand(Op: *Data1, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
915 } else if (SIInstrInfo::isAtomicRet(MI: Inst) && !SIInstrInfo::isGWS(MI: Inst) &&
916 Inst.getOpcode() != AMDGPU::DS_APPEND &&
917 Inst.getOpcode() != AMDGPU::DS_CONSUME &&
918 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
919 for (const MachineOperand &Op : Inst.all_uses()) {
920 if (TRI.isVectorRegister(MRI, Reg: Op.getReg()))
921 setScoreByOperand(Op, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
922 }
923 }
924 } else if (TII.isFLAT(MI: Inst)) {
925 if (Inst.mayStore()) {
926 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
927 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
928 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
929 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
930 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
931 }
932 } else if (TII.isMIMG(MI: Inst)) {
933 if (Inst.mayStore()) {
934 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
935 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
936 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
937 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
938 }
939 } else if (TII.isMTBUF(MI: Inst)) {
940 if (Inst.mayStore())
941 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
942 } else if (TII.isMUBUF(MI: Inst)) {
943 if (Inst.mayStore()) {
944 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
945 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
946 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
947 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
948 }
949 } else if (TII.isLDSDIR(MI: Inst)) {
950 // LDSDIR instructions attach the score to the destination.
951 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::vdst),
952 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
953 } else {
954 if (TII.isEXP(MI: Inst)) {
955 // For export the destination registers are really temps that
956 // can be used as the actual source after export patching, so
957 // we need to treat them like sources and set the EXP_CNT
958 // score.
959 for (MachineOperand &DefMO : Inst.all_defs()) {
960 if (TRI.isVGPR(MRI, Reg: DefMO.getReg())) {
961 setScoreByOperand(Op: DefMO, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
962 }
963 }
964 }
965 for (const MachineOperand &Op : Inst.all_uses()) {
966 if (TRI.isVectorRegister(MRI, Reg: Op.getReg()))
967 setScoreByOperand(Op, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
968 }
969 }
970 } else if (T == AMDGPU::X_CNT) {
971 HWEvents OtherEvent =
972 E == HWEvents::SMEM_GROUP ? HWEvents::VMEM_GROUP : HWEvents::SMEM_GROUP;
973 if (PendingEvents.contains(Other: OtherEvent)) {
974 // Hardware inserts an implicit xcnt between interleaved
975 // SMEM and VMEM operations. So there will never be
976 // outstanding address translations for both SMEM and
977 // VMEM at the same time.
978 setScoreLB(T, Val: getScoreUB(T) - 1);
979 PendingEvents -= OtherEvent;
980 }
981 for (const MachineOperand &Op : Inst.all_uses())
982 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
983 } else if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR ||
984 T == AMDGPU::VM_VSRC) {
985 // Match the score to the VGPR destination or source registers as
986 // appropriate
987 for (const MachineOperand &Op : Inst.operands()) {
988 if (!Op.isReg())
989 continue;
990
991 // Skip based on counter type and operand type
992 if (T == AMDGPU::VA_VDST_RD && Op.isDef())
993 continue; // RD tracks reads only
994 if (T == AMDGPU::VA_VDST_WR && Op.isUse())
995 continue; // WR tracks writes only
996 if (T == AMDGPU::VM_VSRC && Op.isDef())
997 continue;
998
999 if (TRI.isVectorRegister(MRI: Context->MRI, Reg: Op.getReg()))
1000 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
1001 }
1002 } else /* LGKM_CNT || EXP_CNT || VS_CNT || NUM_INST_CNTS */ {
1003 // Match the score to the destination registers.
1004 //
1005 // Check only explicit operands. Stores, especially spill stores, include
1006 // implicit uses and defs of their super registers which would create an
1007 // artificial dependency, while these are there only for register liveness
1008 // accounting purposes.
1009 //
1010 // Special cases where implicit register defs exists, such as M0 or VCC,
1011 // but none with memory instructions.
1012 for (const MachineOperand &Op : Inst.defs()) {
1013 if (T == AMDGPU::LOAD_CNT || T == AMDGPU::SAMPLE_CNT ||
1014 T == AMDGPU::BVH_CNT) {
1015 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg())) // TODO: add wrapper
1016 continue;
1017 if (updateVMCntOnly(Inst)) {
1018 // updateVMCntOnly should only leave us with VGPRs
1019 // MUBUF, MTBUF, MIMG, FlatGlobal, and FlatScratch only have VGPR/AGPR
1020 // defs.
1021 assert(TRI.isVectorRegister(MRI, Op.getReg()));
1022 HWEvents VGPRContext =
1023 AMDGPU::getSimplifiedVMEMEventsFor(Inst, TII: Context->TII);
1024 // If instruction can have Point Sample Accel applied, we have to flag
1025 // this with another potential dependency
1026 if (hasPointSampleAccel(MI: Inst))
1027 VGPRContext |= HWEvents::VMEM_READ_ACCESS;
1028 for (MCRegUnit RU : regunits(Reg: Op.getReg().asMCReg()))
1029 VMem[toVMEMID(RU)].VGPRPendingEvents |= VGPRContext;
1030 }
1031 }
1032 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
1033 }
1034 if (Inst.mayStore() &&
1035 (TII.isDS(MI: Inst) || Context->isNonAsyncLdsDmaWrite(MI: Inst))) {
1036 // MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS
1037 // written can be accessed. A load from LDS to VMEM does not need a wait.
1038 //
1039 // The "Slot" is the offset from LDSDMA_BEGIN. If it's non-zero, then
1040 // there is a MachineInstr in LDSDMAStores used to track this LDSDMA
1041 // store. The "Slot" is the index into LDSDMAStores + 1.
1042 unsigned Slot = 0;
1043 for (const auto *MemOp : Inst.memoperands()) {
1044 if (!MemOp->isStore() ||
1045 MemOp->getAddrSpace() != AMDGPUAS::LOCAL_ADDRESS)
1046 continue;
1047 // Comparing just AA info does not guarantee memoperands are equal
1048 // in general, but this is so for LDS DMA in practice.
1049 auto AAI = MemOp->getAAInfo();
1050 // Alias scope information gives a way to definitely identify an
1051 // original memory object and practically produced in the module LDS
1052 // lowering pass. If there is no scope available we will not be able
1053 // to disambiguate LDS aliasing as after the module lowering all LDS
1054 // is squashed into a single big object.
1055 if (!AAI || !AAI.Scope)
1056 break;
1057 for (unsigned I = 0, E = LDSDMAStores.size(); I != E && !Slot; ++I) {
1058 for (const auto *MemOp : LDSDMAStores[I]->memoperands()) {
1059 if (MemOp->isStore() && AAI == MemOp->getAAInfo()) {
1060 Slot = I + 1;
1061 break;
1062 }
1063 }
1064 }
1065 if (Slot)
1066 break;
1067 // The slot may not be valid because it can be >= NUM_LDSDMA which
1068 // means the scoreboard cannot track it. We still want to preserve the
1069 // MI in order to check alias information, though.
1070 LDSDMAStores.push_back(Elt: &Inst);
1071 Slot = LDSDMAStores.size();
1072 break;
1073 }
1074 setVMemScore(TID: LDSDMA_BEGIN, T, Val: CurrScore);
1075 if (Slot && Slot < NUM_LDSDMA)
1076 setVMemScore(TID: LDSDMA_BEGIN + Slot, T, Val: CurrScore);
1077 }
1078
1079 if (Context->shouldUpdateAsyncMark(MI: Inst, T)) {
1080 AsyncScore[T] = CurrScore;
1081 }
1082
1083 if (SIInstrInfo::isSBarrierSCCWrite(Opcode: Inst.getOpcode())) {
1084 setRegScore(Reg: AMDGPU::SCC, T, Val: CurrScore);
1085 PendingSCCWrite = &Inst;
1086 }
1087 }
1088}
1089
1090void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
1091 // In the absence of loops, AsyncMarks can grow linearly with the program
1092 // until we encounter an ASYNCMARK_WAIT. We could drop the oldest mark above a
1093 // limit every time we push a new mark, but that seems like unnecessary work
1094 // in practical cases. We do separately truncate the array when processing a
1095 // loop, which should be sufficient.
1096 AsyncMarks.push_back(Elt: AsyncScore);
1097 LLVM_DEBUG({
1098 dbgs() << "recordAsyncMark:\n" << Inst;
1099 for (const auto &Mark : AsyncMarks) {
1100 llvm::interleaveComma(Mark, dbgs());
1101 dbgs() << '\n';
1102 }
1103 });
1104}
1105
1106void WaitcntBrackets::print(raw_ostream &OS) const {
1107 const GCNSubtarget &ST = Context->ST;
1108
1109 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter)) {
1110 unsigned SR = getScoreRange(T);
1111 switch (T) {
1112 case AMDGPU::LOAD_CNT:
1113 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM") << "_CNT("
1114 << SR << "):";
1115 break;
1116 case AMDGPU::DS_CNT:
1117 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM") << "_CNT("
1118 << SR << "):";
1119 break;
1120 case AMDGPU::EXP_CNT:
1121 OS << " EXP_CNT(" << SR << "):";
1122 break;
1123 case AMDGPU::STORE_CNT:
1124 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS") << "_CNT("
1125 << SR << "):";
1126 break;
1127 case AMDGPU::SAMPLE_CNT:
1128 OS << " SAMPLE_CNT(" << SR << "):";
1129 break;
1130 case AMDGPU::BVH_CNT:
1131 OS << " BVH_CNT(" << SR << "):";
1132 break;
1133 case AMDGPU::KM_CNT:
1134 OS << " KM_CNT(" << SR << "):";
1135 break;
1136 case AMDGPU::X_CNT:
1137 OS << " X_CNT(" << SR << "):";
1138 break;
1139 case AMDGPU::ASYNC_CNT:
1140 OS << " ASYNC_CNT(" << SR << "):";
1141 break;
1142 case AMDGPU::VA_VDST_RD:
1143 OS << " VA_VDST_RD(" << SR << "): ";
1144 break;
1145 case AMDGPU::VA_VDST_WR:
1146 OS << " VA_VDST_WR(" << SR << "): ";
1147 break;
1148 case AMDGPU::VM_VSRC:
1149 OS << " VM_VSRC(" << SR << "): ";
1150 break;
1151 default:
1152 OS << " UNKNOWN(" << SR << "):";
1153 break;
1154 }
1155
1156 if (SR != 0) {
1157 // Print vgpr scores.
1158 unsigned LB = getScoreLB(T);
1159
1160 SmallVector<VMEMID> SortedVMEMIDs(VMem.keys());
1161 sort(C&: SortedVMEMIDs);
1162
1163 for (auto ID : SortedVMEMIDs) {
1164 unsigned RegScore = VMem.at(Val: ID).Scores[T];
1165 if (RegScore <= LB)
1166 continue;
1167 unsigned RelScore = RegScore - LB - 1;
1168 if (ID < REGUNITS_END) {
1169 OS << ' ' << RelScore << ':'
1170 << printRegUnit(Unit: static_cast<MCRegUnit>(ID), TRI: &Context->TRI);
1171 } else {
1172 assert(ID >= LDSDMA_BEGIN && ID < LDSDMA_END &&
1173 "Unhandled/unexpected ID value!");
1174 OS << ' ' << RelScore << ":LDSDMA" << ID;
1175 }
1176 }
1177
1178 // Also need to print sgpr scores for lgkm_cnt or xcnt.
1179 if (isSmemCounter(T)) {
1180 SmallVector<MCRegUnit> SortedSMEMIDs(SGPRs.keys());
1181 sort(C&: SortedSMEMIDs);
1182 for (auto ID : SortedSMEMIDs) {
1183 unsigned RegScore = SGPRs.at(Val: ID).get(T);
1184 if (RegScore <= LB)
1185 continue;
1186 unsigned RelScore = RegScore - LB - 1;
1187 OS << ' ' << RelScore << ':'
1188 << printRegUnit(Unit: static_cast<MCRegUnit>(ID), TRI: &Context->TRI);
1189 }
1190 }
1191
1192 if (T == AMDGPU::KM_CNT && SCCScore > 0)
1193 OS << ' ' << SCCScore << ":scc";
1194 }
1195 OS << '\n';
1196 }
1197
1198 OS << "Pending Events: ";
1199 if (hasPendingEvent()) {
1200 OS << getPendingEvents();
1201 } else {
1202 OS << "none";
1203 }
1204 OS << '\n';
1205
1206 OS << "Async score: ";
1207 if (AsyncScore.empty())
1208 OS << "none";
1209 else
1210 llvm::interleaveComma(c: AsyncScore, os&: OS);
1211 OS << '\n';
1212
1213 OS << "Async marks: " << AsyncMarks.size() << '\n';
1214
1215 for (const auto &Mark : AsyncMarks) {
1216 for (auto T : AMDGPU::inst_counter_types()) {
1217 unsigned MarkedScore = Mark[T];
1218 switch (T) {
1219 case AMDGPU::LOAD_CNT:
1220 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM")
1221 << "_CNT: " << MarkedScore;
1222 break;
1223 case AMDGPU::DS_CNT:
1224 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM")
1225 << "_CNT: " << MarkedScore;
1226 break;
1227 case AMDGPU::EXP_CNT:
1228 OS << " EXP_CNT: " << MarkedScore;
1229 break;
1230 case AMDGPU::STORE_CNT:
1231 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS")
1232 << "_CNT: " << MarkedScore;
1233 break;
1234 case AMDGPU::SAMPLE_CNT:
1235 OS << " SAMPLE_CNT: " << MarkedScore;
1236 break;
1237 case AMDGPU::BVH_CNT:
1238 OS << " BVH_CNT: " << MarkedScore;
1239 break;
1240 case AMDGPU::KM_CNT:
1241 OS << " KM_CNT: " << MarkedScore;
1242 break;
1243 case AMDGPU::X_CNT:
1244 OS << " X_CNT: " << MarkedScore;
1245 break;
1246 case AMDGPU::ASYNC_CNT:
1247 OS << " ASYNC_CNT: " << MarkedScore;
1248 break;
1249 default:
1250 OS << " UNKNOWN: " << MarkedScore;
1251 break;
1252 }
1253 }
1254 OS << '\n';
1255 }
1256 OS << '\n';
1257}
1258
1259/// Simplify \p UpdateWait by removing waits that are redundant based on the
1260/// current WaitcntBrackets and any other waits specified in \p CheckWait.
1261void WaitcntBrackets::simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
1262 AMDGPU::Waitcnt &UpdateWait) const {
1263 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::LOAD_CNT);
1264 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::EXP_CNT);
1265 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::DS_CNT);
1266 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::STORE_CNT);
1267 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::SAMPLE_CNT);
1268 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::BVH_CNT);
1269 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::KM_CNT);
1270 simplifyXcnt(CheckWait, UpdateWait);
1271 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VA_VDST_RD);
1272 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VA_VDST_WR);
1273 simplifyVmVsrc(CheckWait, UpdateWait);
1274 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::ASYNC_CNT);
1275}
1276
1277void WaitcntBrackets::simplifyWaitcnt(AMDGPU::InstCounterType T,
1278 unsigned &Count) const {
1279 // The number of outstanding events for this type, T, can be calculated
1280 // as (UB - LB). If the current Count is greater than or equal to the number
1281 // of outstanding events, then the wait for this counter is redundant.
1282 if (Count >= getScoreRange(T))
1283 Count = ~0u;
1284}
1285
1286void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait,
1287 AMDGPU::InstCounterType T) const {
1288 unsigned Cnt = Wait.get(T);
1289 simplifyWaitcnt(T, Count&: Cnt);
1290 Wait.set(T, Val: Cnt);
1291}
1292
1293void WaitcntBrackets::simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
1294 AMDGPU::Waitcnt &UpdateWait) const {
1295 // Try to simplify xcnt further by checking for joint kmcnt and loadcnt
1296 // optimizations. On entry to a block with multiple predescessors, there may
1297 // be pending SMEM and VMEM events active at the same time.
1298 // In such cases, only clear one active event at a time.
1299 // TODO: Revisit xcnt optimizations for gfx1250.
1300 // Wait on XCNT is redundant if we are already waiting for a load to complete.
1301 // SMEM can return out of order, so only omit XCNT wait if we are waiting till
1302 // zero.
1303 if (CheckWait.get(T: AMDGPU::KM_CNT) == 0 &&
1304 hasPendingEvent(E: HWEvents::SMEM_GROUP))
1305 UpdateWait.set(T: AMDGPU::X_CNT, Val: ~0u);
1306 // If we have pending store we cannot optimize XCnt because we do not wait for
1307 // stores. VMEM loads retun in order, so if we only have loads XCnt is
1308 // decremented to the same number as LOADCnt.
1309 if (CheckWait.get(T: AMDGPU::LOAD_CNT) != ~0u &&
1310 hasPendingEvent(E: HWEvents::VMEM_GROUP) &&
1311 !hasPendingEvent(T: AMDGPU::STORE_CNT) &&
1312 CheckWait.get(T: AMDGPU::X_CNT) >= CheckWait.get(T: AMDGPU::LOAD_CNT))
1313 UpdateWait.set(T: AMDGPU::X_CNT, Val: ~0u);
1314 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::X_CNT);
1315}
1316
1317void WaitcntBrackets::simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
1318 AMDGPU::Waitcnt &UpdateWait) const {
1319 // Waiting for a VMEM counter (like LOAD_CNT) implies an equivalent wait for
1320 // VM_VSRC, because if the VMEM operation has completed then it must surely
1321 // have read its VGPR sources, but only if there are no other outstanding VMEM
1322 // operations that use a different counter (like SAMPLE_CNT).
1323 static constexpr AMDGPU::InstCounterType VmemCounters[] = {
1324 AMDGPU::LOAD_CNT, AMDGPU::STORE_CNT, AMDGPU::SAMPLE_CNT, AMDGPU::BVH_CNT,
1325 AMDGPU::DS_CNT, AMDGPU::ASYNC_CNT};
1326 HWEvents VmemEvents = llvm::accumulate(
1327 Range: VmemCounters, Init: HWEvents(), Op: [&](HWEvents Acc, AMDGPU::InstCounterType T) {
1328 return Acc | Context->getWaitEvents(T);
1329 });
1330 HWEvents PendingVmemEvents = PendingEvents & VmemEvents;
1331 for (AMDGPU::InstCounterType T : VmemCounters) {
1332 unsigned CheckCount = CheckWait.get(T);
1333 if (UpdateWait.get(T: AMDGPU::VM_VSRC) >= CheckCount &&
1334 (CheckCount == 0 || !counterOutOfOrder(T)) &&
1335 (PendingVmemEvents & ~Context->getWaitEvents(T)) == 0)
1336 UpdateWait.set(T: AMDGPU::VM_VSRC, Val: ~0u);
1337 }
1338
1339 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VM_VSRC);
1340}
1341
1342void WaitcntBrackets::purgeEmptyTrackingData() {
1343 VMem.remove_if(Pred: [](const auto &P) { return P.second.empty(); });
1344 SGPRs.remove_if(Pred: [](const auto &P) { return P.second.empty(); });
1345}
1346
1347void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
1348 unsigned ScoreToWait,
1349 AMDGPU::Waitcnt &Wait) const {
1350 const unsigned LB = getScoreLB(T);
1351 const unsigned UB = getScoreUB(T);
1352
1353 // If the score falls within the bracket, we need a waitcnt.
1354 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1355 if ((T == AMDGPU::LOAD_CNT || T == AMDGPU::DS_CNT) && hasPendingFlat() &&
1356 !Context->ST.hasFlatLgkmVMemCountInOrder()) {
1357 // If there is a pending FLAT operation, and this is a VMem or LGKM
1358 // waitcnt and the target can report early completion, then we need
1359 // to force a waitcnt 0.
1360 Wait.add(T, Count: 0);
1361 } else if (counterOutOfOrder(T)) {
1362 // Counter can get decremented out-of-order when there
1363 // are multiple types event in the bracket. Also emit an s_wait counter
1364 // with a conservative value of 0 for the counter.
1365 Wait.add(T, Count: 0);
1366 } else {
1367 // If a counter has been maxed out avoid overflow by waiting for
1368 // MAX(CounterType) - 1 instead.
1369 unsigned NeededWait = std::min(a: UB - ScoreToWait, b: getLimit(T) - 1);
1370 Wait.add(T, Count: NeededWait);
1371 }
1372 }
1373}
1374
1375AMDGPU::Waitcnt WaitcntBrackets::determineAsyncWait(unsigned N) {
1376 LLVM_DEBUG({
1377 dbgs() << "Need " << N << " async marks. Found " << AsyncMarks.size()
1378 << ":\n";
1379 for (const auto &Mark : AsyncMarks) {
1380 llvm::interleaveComma(Mark, dbgs());
1381 dbgs() << '\n';
1382 }
1383 });
1384
1385 if (AsyncMarks.size() == MaxAsyncMarks) {
1386 // Enforcing MaxAsyncMarks here is unnecessary work because the size of
1387 // MaxAsyncMarks is linear when traversing straightline code. But we do
1388 // need to check if truncation may have occured at a merge, and adjust N
1389 // to ensure that a wait is generated.
1390 LLVM_DEBUG(dbgs() << "Possible truncation. Ensuring a non-trivial wait.\n");
1391 N = std::min(a: N, b: (unsigned)MaxAsyncMarks - 1);
1392 }
1393
1394 AMDGPU::Waitcnt Wait;
1395 if (AsyncMarks.size() <= N) {
1396 LLVM_DEBUG(dbgs() << "No additional wait for async mark.\n");
1397 return Wait;
1398 }
1399
1400 size_t MarkIndex = AsyncMarks.size() - N - 1;
1401 const auto &RequiredMark = AsyncMarks[MarkIndex];
1402 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types())
1403 determineWaitForScore(T, ScoreToWait: RequiredMark[T], Wait);
1404
1405 // Immediately remove the waited mark and all older ones
1406 // This happens BEFORE the wait is actually inserted, which is fine
1407 // because we've already extracted the wait requirements
1408 LLVM_DEBUG({
1409 dbgs() << "Removing " << (MarkIndex + 1)
1410 << " async marks after determining wait\n";
1411 });
1412 AsyncMarks.erase(CS: AsyncMarks.begin(), CE: AsyncMarks.begin() + MarkIndex + 1);
1413
1414 LLVM_DEBUG(dbgs() << "Waits to add: " << Wait);
1415 return Wait;
1416}
1417
1418// With D16Write32BitVgpr, D16 inst might be clobbered by events running on the
1419// other half 16bit.
1420//
1421// Replace VGPR16 to VGPR32 for wait check if:
1422// 1. MI is a VALU, and there is a wait event on the other half
1423// 2. MI is a LdSt, and there is a wait event on the other half from different
1424// order group
1425MCPhysReg WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
1426 AMDGPU::InstCounterType T,
1427 MCPhysReg Reg) const {
1428 const TargetRegisterClass *RC = Context->TRI.getPhysRegBaseClass(Reg);
1429 unsigned Size = Context->TRI.getRegSizeInBits(RC: *RC);
1430
1431 if (Size != 16 || !Context->ST.hasD16Writes32BitVgpr())
1432 return Reg;
1433
1434 // With D16Writes32BitVgpr, D16 Inst might clobber the whole vgpr32
1435 // check dependency on the other half
1436 Register Reg32 = Context->TRI.get32BitRegister(Reg);
1437 Register OtherHalf = Context->TRI.getSubReg(
1438 Reg: Reg32,
1439 Idx: AMDGPU::isHi16Reg(Reg, MRI: Context->TRI) ? AMDGPU::lo16 : AMDGPU::hi16);
1440
1441 AMDGPU::Waitcnt Wait;
1442 for (MCRegUnit RU : regunits(Reg: OtherHalf))
1443 determineWaitForScore(T, ScoreToWait: getVMemScore(TID: toVMEMID(RU), T), Wait);
1444
1445 // No wait on otherhalf
1446 if (!Wait.hasWait())
1447 return Reg;
1448
1449 if (Context->TII.isVALU(MI, /*AllowLDSDMA=*/true))
1450 return Reg32;
1451
1452 // If hi/lo16 mixed events
1453 HWEvents MIEvents = AMDGPU::getEventsFor(
1454 Inst: MI, ST: Context->ST, IsExpertMode: Context->IsExpertMode, TgSplit: Context->TgSplit);
1455 HWEvents OtherHalfEvents = Context->getWaitEvents(T);
1456 HWEvents Events = MIEvents & OtherHalfEvents;
1457 if (Events.size() > 1)
1458 return Reg32;
1459 return Reg;
1460}
1461
1462void WaitcntBrackets::determineWaitForPhysReg(AMDGPU::InstCounterType T,
1463 MCPhysReg Reg,
1464 AMDGPU::Waitcnt &Wait,
1465 const MachineInstr &MI) const {
1466 if (Reg == AMDGPU::SCC) {
1467 determineWaitForScore(T, ScoreToWait: SCCScore, Wait);
1468 } else {
1469 bool IsVGPR = Context->TRI.isVectorRegister(MRI: Context->MRI, Reg);
1470 if (IsVGPR)
1471 Reg = determineVGPR16Dependency(MI, T, Reg);
1472 for (MCRegUnit RU : regunits(Reg))
1473 determineWaitForScore(
1474 T, ScoreToWait: IsVGPR ? getVMemScore(TID: toVMEMID(RU), T) : getSGPRScore(RU, T),
1475 Wait);
1476 }
1477}
1478
1479void WaitcntBrackets::determineWaitForLDSDMA(AMDGPU::InstCounterType T,
1480 VMEMID TID,
1481 AMDGPU::Waitcnt &Wait) const {
1482 assert(TID >= LDSDMA_BEGIN && TID < LDSDMA_END);
1483 determineWaitForScore(T, ScoreToWait: getVMemScore(TID, T), Wait);
1484}
1485
1486void WaitcntBrackets::tryClearSCCWriteEvent(MachineInstr *Inst) {
1487 // S_BARRIER_WAIT on the same barrier guarantees that the pending write to
1488 // SCC has landed
1489 if (PendingSCCWrite &&
1490 PendingSCCWrite->getOpcode() == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM &&
1491 PendingSCCWrite->getOperand(i: 0).getImm() == Inst->getOperand(i: 0).getImm()) {
1492 HWEvents SCC_WRITE_PendingEvent = HWEvents::SCC_WRITE;
1493 // If this SCC_WRITE is the only pending KM_CNT event, clear counter.
1494 if ((PendingEvents & Context->getWaitEvents(T: AMDGPU::KM_CNT)) ==
1495 SCC_WRITE_PendingEvent) {
1496 setScoreLB(T: AMDGPU::KM_CNT, Val: getScoreUB(T: AMDGPU::KM_CNT));
1497 }
1498
1499 PendingEvents -= SCC_WRITE_PendingEvent;
1500 PendingSCCWrite = nullptr;
1501 }
1502}
1503
1504void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) {
1505 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
1506 unsigned Cnt;
1507 if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) {
1508 Cnt =
1509 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
1510 } else {
1511 Cnt = Wait.get(T);
1512 }
1513 applyWaitcnt(T, Count: Cnt);
1514 }
1515}
1516
1517void WaitcntBrackets::applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count) {
1518 const unsigned UB = getScoreUB(T);
1519 if (Count >= UB)
1520 return;
1521 if (Count != 0) {
1522 if (counterOutOfOrder(T))
1523 return;
1524 setScoreLB(T, Val: std::max(a: getScoreLB(T), b: UB - Count));
1525 } else {
1526 setScoreLB(T, Val: UB);
1527 PendingEvents -= Context->getWaitEvents(T);
1528 }
1529
1530 if (T == AMDGPU::KM_CNT && Count == 0 &&
1531 hasPendingEvent(E: HWEvents::SMEM_GROUP)) {
1532 if (!hasMixedPendingEvents(T: AMDGPU::X_CNT))
1533 applyWaitcnt(T: AMDGPU::X_CNT, Count: 0);
1534 else
1535 PendingEvents -= HWEvents::SMEM_GROUP;
1536 }
1537 if (T == AMDGPU::LOAD_CNT && hasPendingEvent(E: HWEvents::VMEM_GROUP) &&
1538 !hasPendingEvent(T: AMDGPU::STORE_CNT)) {
1539 if (!hasMixedPendingEvents(T: AMDGPU::X_CNT))
1540 applyWaitcnt(T: AMDGPU::X_CNT, Count);
1541 else if (Count == 0)
1542 PendingEvents -= HWEvents::VMEM_GROUP;
1543 }
1544}
1545
1546void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait,
1547 AMDGPU::InstCounterType T) {
1548 unsigned Cnt = Wait.get(T);
1549 applyWaitcnt(T, Count: Cnt);
1550}
1551
1552// Where there are multiple types of event in the bracket of a counter,
1553// the decrement may go out of order.
1554bool WaitcntBrackets::counterOutOfOrder(AMDGPU::InstCounterType T) const {
1555 // Scalar memory read always can go out of order.
1556 if ((T == Context->SmemAccessCounter &&
1557 hasPendingEvent(E: HWEvents::SMEM_ACCESS)) ||
1558 (T == AMDGPU::X_CNT && hasPendingEvent(E: HWEvents::SMEM_GROUP)))
1559 return true;
1560
1561 if (T == AMDGPU::LOAD_CNT) {
1562
1563 // On targets without VScnt, LOAD_CNT includes all of STORE_CNT as well.
1564 // All these events use one counter and do not go out of order with respect
1565 // to each other.
1566 if (!Context->ST.hasVscnt())
1567 return false;
1568
1569 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
1570
1571 // If the target does not have extended counters, VMEM_BVH/SAMPLE_READ
1572 // events are equivalent to VMEM_READ_ACCESS. We do not go out of order in
1573 // such cases.
1574 static constexpr HWEvents ExtendedImageEvents =
1575 HWEvents::VMEM_SAMPLER_READ_ACCESS | HWEvents::VMEM_BVH_READ_ACCESS;
1576 if (!Context->ST.hasExtendedWaitCounts() &&
1577 (Events & ExtendedImageEvents).any()) {
1578 Events -= ExtendedImageEvents;
1579 Events |= HWEvents::VMEM_READ_ACCESS;
1580 }
1581
1582 // GLOBAL_INV completes in-order with other LOAD_CNT events,
1583 // so having GLOBAL_INV_ACCESS mixed with other LOAD_CNT
1584 // events doesn't cause out-of-order completion.
1585 Events -= HWEvents::GLOBAL_INV_ACCESS;
1586
1587 // Return true only if there are still multiple event types after removing
1588 // GLOBAL_INV
1589 return Events.size() > 1;
1590 }
1591
1592 return hasMixedPendingEvents(T);
1593}
1594
1595INITIALIZE_PASS_BEGIN(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1596 false, false)
1597INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
1598INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass)
1599INITIALIZE_PASS_END(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1600 false, false)
1601
1602char SIInsertWaitcntsLegacy::ID = 0;
1603
1604char &llvm::SIInsertWaitcntsID = SIInsertWaitcntsLegacy::ID;
1605
1606FunctionPass *llvm::createSIInsertWaitcntsPass() {
1607 return new SIInsertWaitcntsLegacy();
1608}
1609
1610static bool updateOperandIfDifferent(MachineInstr &MI, AMDGPU::OpName OpName,
1611 unsigned NewEnc) {
1612 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpName);
1613 assert(OpIdx >= 0);
1614
1615 MachineOperand &MO = MI.getOperand(i: OpIdx);
1616
1617 if (NewEnc == MO.getImm())
1618 return false;
1619
1620 MO.setImm(NewEnc);
1621 return true;
1622}
1623
1624bool WaitcntGenerator::promoteSoftWaitCnt(MachineInstr *Waitcnt) const {
1625 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: Waitcnt->getOpcode());
1626 if (Opcode == Waitcnt->getOpcode())
1627 return false;
1628
1629 Waitcnt->setDesc(TII.get(Opcode));
1630 return true;
1631}
1632
1633/// Combine consecutive S_WAITCNT and S_WAITCNT_VSCNT instructions that
1634/// precede \p It and follow \p OldWaitcntInstr and apply any extra waits
1635/// from \p Wait that were added by previous passes. Currently this pass
1636/// conservatively assumes that these preexisting waits are required for
1637/// correctness.
1638bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1639 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1640 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) const {
1641 assert(isNormalMode(MaxCounter));
1642
1643 bool Modified = false;
1644 MachineInstr *WaitcntInstr = nullptr;
1645 MachineInstr *WaitcntVsCntInstr = nullptr;
1646
1647 LLVM_DEBUG({
1648 dbgs() << "PreGFX12::applyPreexistingWaitcnt at: ";
1649 if (It.isEnd())
1650 dbgs() << "end of block\n";
1651 else
1652 dbgs() << *It;
1653 });
1654
1655 for (auto &II :
1656 make_early_inc_range(Range: make_range(x: OldWaitcntInstr.getIterator(), y: It))) {
1657 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
1658 if (isNonWaitcntMetaInst(MI: II)) {
1659 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
1660 continue;
1661 }
1662
1663 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: II.getOpcode());
1664 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1665
1666 // Update required wait count. If this is a soft waitcnt (= it was added
1667 // by an earlier pass), it may be entirely removed.
1668 if (Opcode == AMDGPU::S_WAITCNT) {
1669 unsigned IEnc = II.getOperand(i: 0).getImm();
1670 AMDGPU::Waitcnt OldWait = AMDGPU::decodeWaitcnt(Version: IV, Encoded: IEnc);
1671 if (TrySimplify)
1672 ScoreBrackets.simplifyWaitcnt(Wait&: OldWait);
1673 Wait = Wait.combined(Other: OldWait);
1674
1675 // Merge consecutive waitcnt of the same type by erasing multiples.
1676 if (WaitcntInstr || (!Wait.hasWaitExceptStoreCnt() && TrySimplify)) {
1677 II.eraseFromParent();
1678 Modified = true;
1679 } else
1680 WaitcntInstr = &II;
1681 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
1682 assert(ST.hasVMemToLDSLoad());
1683 LLVM_DEBUG(dbgs() << "Processing S_WAITCNT_lds_direct: " << II
1684 << "Before: " << Wait << '\n';);
1685 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID: LDSDMA_BEGIN,
1686 Wait);
1687 LLVM_DEBUG(dbgs() << "After: " << Wait << '\n';);
1688
1689 // It is possible (but unlikely) that this is the only wait instruction,
1690 // in which case, we exit this loop without a WaitcntInstr to consume
1691 // `Wait`. But that works because `Wait` was passed in by reference, and
1692 // the callee eventually calls createNewWaitcnt on it. We test this
1693 // possibility in an articial MIR test since such a situation cannot be
1694 // recreated by running the memory legalizer.
1695 II.eraseFromParent();
1696 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
1697 unsigned N = II.getOperand(i: 0).getImm();
1698 LLVM_DEBUG(dbgs() << "Processing WAIT_ASYNCMARK: " << II << '\n';);
1699 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
1700 Wait = Wait.combined(Other: OldWait);
1701 } else {
1702 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1703 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1704
1705 unsigned OldVSCnt =
1706 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1707 if (TrySimplify)
1708 ScoreBrackets.simplifyWaitcnt(T: AMDGPU::STORE_CNT, Count&: OldVSCnt);
1709 Wait.set(T: AMDGPU::STORE_CNT,
1710 Val: std::min(a: Wait.get(T: AMDGPU::STORE_CNT), b: OldVSCnt));
1711
1712 if (WaitcntVsCntInstr || (!Wait.hasWaitStoreCnt() && TrySimplify)) {
1713 II.eraseFromParent();
1714 Modified = true;
1715 } else
1716 WaitcntVsCntInstr = &II;
1717 }
1718 }
1719
1720 if (WaitcntInstr) {
1721 Modified |= updateOperandIfDifferent(MI&: *WaitcntInstr, OpName: AMDGPU::OpName::simm16,
1722 NewEnc: AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait));
1723 Modified |= promoteSoftWaitCnt(Waitcnt: WaitcntInstr);
1724
1725 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::LOAD_CNT);
1726 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::EXP_CNT);
1727 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
1728 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
1729 Wait.set(T: AMDGPU::EXP_CNT, Val: ~0u);
1730 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
1731
1732 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
1733 << "New Instr at block end: "
1734 << *WaitcntInstr << '\n'
1735 : dbgs() << "applied pre-existing waitcnt\n"
1736 << "Old Instr: " << *It
1737 << "New Instr: " << *WaitcntInstr << '\n');
1738 }
1739
1740 if (WaitcntVsCntInstr) {
1741 Modified |=
1742 updateOperandIfDifferent(MI&: *WaitcntVsCntInstr, OpName: AMDGPU::OpName::simm16,
1743 NewEnc: Wait.get(T: AMDGPU::STORE_CNT));
1744 Modified |= promoteSoftWaitCnt(Waitcnt: WaitcntVsCntInstr);
1745
1746 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::STORE_CNT);
1747 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
1748
1749 LLVM_DEBUG(It.isEnd()
1750 ? dbgs() << "applied pre-existing waitcnt\n"
1751 << "New Instr at block end: " << *WaitcntVsCntInstr
1752 << '\n'
1753 : dbgs() << "applied pre-existing waitcnt\n"
1754 << "Old Instr: " << *It
1755 << "New Instr: " << *WaitcntVsCntInstr << '\n');
1756 }
1757
1758 return Modified;
1759}
1760
1761/// Generate S_WAITCNT and/or S_WAITCNT_VSCNT instructions for any
1762/// required counters in \p Wait
1763bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1764 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
1765 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
1766 assert(isNormalMode(MaxCounter));
1767
1768 bool Modified = false;
1769 const DebugLoc &DL = Block.findDebugLoc(MBBI: It);
1770
1771 // Waits for VMcnt, LKGMcnt and/or EXPcnt are encoded together into a
1772 // single instruction while VScnt has its own instruction.
1773 if (Wait.hasWaitExceptStoreCnt()) {
1774 // If profiling expansion is enabled, emit an expanded sequence
1775 if (ExpandWaitcntProfiling) {
1776 // Check if any of the counters to be waited on are out-of-order.
1777 // If so, fall back to normal (non-expanded) behavior since expansion
1778 // would provide misleading profiling information.
1779 bool AnyOutOfOrder = false;
1780 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1781 unsigned WaitCnt = Wait.get(T: CT);
1782 if (WaitCnt != ~0u && ScoreBrackets.counterOutOfOrder(T: CT)) {
1783 AnyOutOfOrder = true;
1784 break;
1785 }
1786 }
1787
1788 if (AnyOutOfOrder) {
1789 // Fall back to non-expanded wait
1790 unsigned Enc = AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait);
1791 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: Enc);
1792 Modified = true;
1793 } else {
1794 // All counters are in-order, safe to expand
1795 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1796 unsigned WaitCnt = Wait.get(T: CT);
1797 if (WaitCnt == ~0u)
1798 continue;
1799
1800 unsigned Outstanding =
1801 std::min(a: ScoreBrackets.getOutstanding(T: CT), b: getLimit(E: CT) - 1);
1802 EmitExpandedWaitcnt(Outstanding, Target: WaitCnt, EmitWaitcnt: [&](unsigned Count) {
1803 AMDGPU::Waitcnt W;
1804 W.set(T: CT, Val: Count);
1805 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT))
1806 .addImm(Val: AMDGPU::encodeWaitcnt(Version: IV, Decoded: W));
1807 });
1808 Modified = true;
1809 }
1810 }
1811 } else {
1812 // Normal behavior: emit single combined waitcnt
1813 unsigned Enc = AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait);
1814 [[maybe_unused]] auto SWaitInst =
1815 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: Enc);
1816 Modified = true;
1817
1818 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1819 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1820 dbgs() << "New Instr: " << *SWaitInst << '\n');
1821 }
1822 }
1823
1824 if (Wait.hasWaitStoreCnt()) {
1825 assert(ST.hasVscnt());
1826
1827 if (ExpandWaitcntProfiling && Wait.get(T: AMDGPU::STORE_CNT) != ~0u &&
1828 !ScoreBrackets.counterOutOfOrder(T: AMDGPU::STORE_CNT)) {
1829 // Only expand if counter is not out-of-order
1830 unsigned Outstanding =
1831 std::min(a: ScoreBrackets.getOutstanding(T: AMDGPU::STORE_CNT),
1832 b: getLimit(E: AMDGPU::STORE_CNT) - 1);
1833 EmitExpandedWaitcnt(
1834 Outstanding, Target: Wait.get(T: AMDGPU::STORE_CNT), EmitWaitcnt: [&](unsigned Count) {
1835 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
1836 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
1837 .addImm(Val: Count);
1838 });
1839 Modified = true;
1840 } else {
1841 [[maybe_unused]] auto SWaitInst =
1842 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
1843 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
1844 .addImm(Val: Wait.get(T: AMDGPU::STORE_CNT));
1845 Modified = true;
1846
1847 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1848 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1849 dbgs() << "New Instr: " << *SWaitInst << '\n');
1850 }
1851 }
1852
1853 return Modified;
1854}
1855
1856AMDGPU::Waitcnt
1857WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1858 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt && ST.hasVscnt() ? 0 : ~0u);
1859}
1860
1861AMDGPU::Waitcnt
1862WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1863 unsigned ExpertVal = IsExpertMode ? 0 : ~0u;
1864 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0,
1865 ~0u /* XCNT */, ~0u /* ASYNC_CNT */,
1866 ~0u /* TENSOR_CNT */, ExpertVal, ExpertVal, ExpertVal);
1867}
1868
1869/// Combine consecutive S_WAIT_*CNT instructions that precede \p It and
1870/// follow \p OldWaitcntInstr and apply any extra waits from \p Wait that
1871/// were added by previous passes. Currently this pass conservatively
1872/// assumes that these preexisting waits are required for correctness.
1873bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1874 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1875 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) const {
1876 assert(!isNormalMode(MaxCounter));
1877
1878 bool Modified = false;
1879 MachineInstr *CombinedLoadDsCntInstr = nullptr;
1880 MachineInstr *CombinedStoreDsCntInstr = nullptr;
1881 MachineInstr *WaitcntDepctrInstr = nullptr;
1882 MachineInstr *WaitInstrs[AMDGPU::NUM_EXTENDED_INST_CNTS] = {};
1883
1884 LLVM_DEBUG({
1885 dbgs() << "GFX12Plus::applyPreexistingWaitcnt at: ";
1886 if (It.isEnd())
1887 dbgs() << "end of block\n";
1888 else
1889 dbgs() << *It;
1890 });
1891
1892 // Accumulate waits that should not be simplified.
1893 AMDGPU::Waitcnt RequiredWait;
1894
1895 for (auto &II :
1896 make_early_inc_range(Range: make_range(x: OldWaitcntInstr.getIterator(), y: It))) {
1897 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
1898 if (isNonWaitcntMetaInst(MI: II)) {
1899 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
1900 continue;
1901 }
1902
1903 // Update required wait count. If this is a soft waitcnt (= it was added
1904 // by an earlier pass), it may be entirely removed.
1905
1906 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: II.getOpcode());
1907 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1908
1909 // Don't crash if the programmer used legacy waitcnt intrinsics, but don't
1910 // attempt to do more than that either.
1911 if (Opcode == AMDGPU::S_WAITCNT)
1912 continue;
1913
1914 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
1915 unsigned OldEnc =
1916 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1917 AMDGPU::Waitcnt OldWait = AMDGPU::decodeLoadcntDscnt(Version: IV, LoadcntDscnt: OldEnc);
1918 if (TrySimplify)
1919 Wait = Wait.combined(Other: OldWait);
1920 else
1921 RequiredWait = RequiredWait.combined(Other: OldWait);
1922 // Keep the first wait_loadcnt, erase the rest.
1923 if (CombinedLoadDsCntInstr == nullptr) {
1924 CombinedLoadDsCntInstr = &II;
1925 } else {
1926 II.eraseFromParent();
1927 Modified = true;
1928 }
1929 } else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
1930 unsigned OldEnc =
1931 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1932 AMDGPU::Waitcnt OldWait = AMDGPU::decodeStorecntDscnt(Version: IV, StorecntDscnt: OldEnc);
1933 if (TrySimplify)
1934 Wait = Wait.combined(Other: OldWait);
1935 else
1936 RequiredWait = RequiredWait.combined(Other: OldWait);
1937 // Keep the first wait_storecnt, erase the rest.
1938 if (CombinedStoreDsCntInstr == nullptr) {
1939 CombinedStoreDsCntInstr = &II;
1940 } else {
1941 II.eraseFromParent();
1942 Modified = true;
1943 }
1944 } else if (Opcode == AMDGPU::S_WAITCNT_DEPCTR) {
1945 unsigned OldEnc =
1946 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1947 AMDGPU::Waitcnt OldWait;
1948 // Set both counters to the decoded value from the single hardware field
1949 unsigned VaVdst = AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: OldEnc);
1950 OldWait.set(T: AMDGPU::VA_VDST_RD, Val: VaVdst);
1951 OldWait.set(T: AMDGPU::VA_VDST_WR, Val: VaVdst);
1952 OldWait.set(T: AMDGPU::VM_VSRC, Val: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: OldEnc));
1953 if (TrySimplify)
1954 ScoreBrackets.simplifyWaitcnt(Wait&: OldWait);
1955 Wait = Wait.combined(Other: OldWait);
1956 if (WaitcntDepctrInstr == nullptr) {
1957 WaitcntDepctrInstr = &II;
1958 } else {
1959 // S_WAITCNT_DEPCTR requires special care. Don't remove a
1960 // duplicate if it is waiting on things other than VA_VDST or
1961 // VM_VSRC. If that is the case, just make sure the VA_VDST and
1962 // VM_VSRC subfields of the operand are set to the "no wait"
1963 // values.
1964
1965 unsigned Enc =
1966 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1967 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: ~0u);
1968 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
1969 unsigned VaVdst = std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD),
1970 b: Wait.get(T: AMDGPU::VA_VDST_WR));
1971 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
1972
1973 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: ST)) {
1974 Modified |= updateOperandIfDifferent(MI&: II, OpName: AMDGPU::OpName::simm16, NewEnc: Enc);
1975 Modified |= promoteSoftWaitCnt(Waitcnt: &II);
1976 } else {
1977 II.eraseFromParent();
1978 Modified = true;
1979 }
1980 }
1981 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
1982 // Architectures higher than GFX10 do not have direct loads to
1983 // LDS, so no work required here yet.
1984 II.eraseFromParent();
1985 Modified = true;
1986 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
1987 // Update the Waitcnt, but don't erase the wait.asyncmark() itself. It
1988 // shows up in the assembly as a comment with the original parameter N.
1989 unsigned N = II.getOperand(i: 0).getImm();
1990 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
1991 Wait = Wait.combined(Other: OldWait);
1992 } else {
1993 std::optional<AMDGPU::InstCounterType> CT =
1994 AMDGPU::counterTypeForInstr(Opcode);
1995 assert(CT.has_value());
1996 unsigned OldCnt =
1997 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1998 if (TrySimplify)
1999 Wait.add(T: CT.value(), Count: OldCnt);
2000 else
2001 RequiredWait.add(T: CT.value(), Count: OldCnt);
2002 // Keep the first wait of its kind, erase the rest.
2003 if (WaitInstrs[CT.value()] == nullptr) {
2004 WaitInstrs[CT.value()] = &II;
2005 } else {
2006 II.eraseFromParent();
2007 Modified = true;
2008 }
2009 }
2010 }
2011
2012 ScoreBrackets.simplifyWaitcnt(CheckWait: Wait.combined(Other: RequiredWait), UpdateWait&: Wait);
2013 Wait = Wait.combined(Other: RequiredWait);
2014
2015 if (CombinedLoadDsCntInstr) {
2016 // Only keep an S_WAIT_LOADCNT_DSCNT if both counters actually need
2017 // to be waited for. Otherwise, let the instruction be deleted so
2018 // the appropriate single counter wait instruction can be inserted
2019 // instead, when new S_WAIT_*CNT instructions are inserted by
2020 // createNewWaitcnt(). As a side effect, resetting the wait counts will
2021 // cause any redundant S_WAIT_LOADCNT or S_WAIT_DSCNT to be removed by
2022 // the loop below that deals with single counter instructions.
2023 //
2024 // A wait for LOAD_CNT or DS_CNT implies a wait for VM_VSRC, since
2025 // instructions that have decremented LOAD_CNT or DS_CNT on completion
2026 // will have needed to wait for their register sources to be available
2027 // first.
2028 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u && Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2029 unsigned NewEnc = AMDGPU::encodeLoadcntDscnt(Version: IV, Decoded: Wait);
2030 Modified |= updateOperandIfDifferent(MI&: *CombinedLoadDsCntInstr,
2031 OpName: AMDGPU::OpName::simm16, NewEnc);
2032 Modified |= promoteSoftWaitCnt(Waitcnt: CombinedLoadDsCntInstr);
2033 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::LOAD_CNT);
2034 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
2035 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2036 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2037
2038 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2039 << "New Instr at block end: "
2040 << *CombinedLoadDsCntInstr << '\n'
2041 : dbgs() << "applied pre-existing waitcnt\n"
2042 << "Old Instr: " << *It << "New Instr: "
2043 << *CombinedLoadDsCntInstr << '\n');
2044 } else {
2045 CombinedLoadDsCntInstr->eraseFromParent();
2046 Modified = true;
2047 }
2048 }
2049
2050 if (CombinedStoreDsCntInstr) {
2051 // Similarly for S_WAIT_STORECNT_DSCNT.
2052 if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u && Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2053 unsigned NewEnc = AMDGPU::encodeStorecntDscnt(Version: IV, Decoded: Wait);
2054 Modified |= updateOperandIfDifferent(MI&: *CombinedStoreDsCntInstr,
2055 OpName: AMDGPU::OpName::simm16, NewEnc);
2056 Modified |= promoteSoftWaitCnt(Waitcnt: CombinedStoreDsCntInstr);
2057 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::STORE_CNT);
2058 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
2059 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
2060 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2061
2062 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2063 << "New Instr at block end: "
2064 << *CombinedStoreDsCntInstr << '\n'
2065 : dbgs() << "applied pre-existing waitcnt\n"
2066 << "Old Instr: " << *It << "New Instr: "
2067 << *CombinedStoreDsCntInstr << '\n');
2068 } else {
2069 CombinedStoreDsCntInstr->eraseFromParent();
2070 Modified = true;
2071 }
2072 }
2073
2074 // Look for an opportunity to convert existing S_WAIT_LOADCNT,
2075 // S_WAIT_STORECNT and S_WAIT_DSCNT into new S_WAIT_LOADCNT_DSCNT
2076 // or S_WAIT_STORECNT_DSCNT. This is achieved by selectively removing
2077 // instructions so that createNewWaitcnt() will create new combined
2078 // instructions to replace them.
2079
2080 if (Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2081 // This is a vector of addresses in WaitInstrs pointing to instructions
2082 // that should be removed if they are present.
2083 SmallVector<MachineInstr **, 2> WaitsToErase;
2084
2085 // If it's known that both DScnt and either LOADcnt or STOREcnt (but not
2086 // both) need to be waited for, ensure that there are no existing
2087 // individual wait count instructions for these.
2088
2089 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u) {
2090 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::LOAD_CNT]);
2091 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::DS_CNT]);
2092 } else if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u) {
2093 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::STORE_CNT]);
2094 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::DS_CNT]);
2095 }
2096
2097 for (MachineInstr **WI : WaitsToErase) {
2098 if (!*WI)
2099 continue;
2100
2101 (*WI)->eraseFromParent();
2102 *WI = nullptr;
2103 Modified = true;
2104 }
2105 }
2106
2107 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2108 if (!WaitInstrs[CT])
2109 continue;
2110
2111 unsigned NewCnt = Wait.get(T: CT);
2112 if (NewCnt != ~0u) {
2113 Modified |= updateOperandIfDifferent(MI&: *WaitInstrs[CT],
2114 OpName: AMDGPU::OpName::simm16, NewEnc: NewCnt);
2115 Modified |= promoteSoftWaitCnt(Waitcnt: WaitInstrs[CT]);
2116
2117 ScoreBrackets.applyWaitcnt(T: CT, Count: NewCnt);
2118 Wait.clear(T: CT);
2119
2120 LLVM_DEBUG(It.isEnd()
2121 ? dbgs() << "applied pre-existing waitcnt\n"
2122 << "New Instr at block end: " << *WaitInstrs[CT]
2123 << '\n'
2124 : dbgs() << "applied pre-existing waitcnt\n"
2125 << "Old Instr: " << *It
2126 << "New Instr: " << *WaitInstrs[CT] << '\n');
2127 } else {
2128 WaitInstrs[CT]->eraseFromParent();
2129 Modified = true;
2130 }
2131 }
2132
2133 if (WaitcntDepctrInstr) {
2134 // Get the encoded Depctr immediate and override the VA_VDST and VM_VSRC
2135 // subfields with the new required values.
2136 unsigned Enc =
2137 TII.getNamedOperand(MI&: *WaitcntDepctrInstr, OperandName: AMDGPU::OpName::simm16)
2138 ->getImm();
2139 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: Wait.get(T: AMDGPU::VM_VSRC));
2140 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2141 unsigned VaVdst =
2142 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
2143 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2144
2145 ScoreBrackets.applyWaitcnt(T: AMDGPU::VA_VDST_RD, Count: VaVdst);
2146 ScoreBrackets.applyWaitcnt(T: AMDGPU::VA_VDST_WR, Count: VaVdst);
2147 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::VM_VSRC);
2148 Wait.set(T: AMDGPU::VA_VDST_RD, Val: ~0u);
2149 Wait.set(T: AMDGPU::VA_VDST_WR, Val: ~0u);
2150 Wait.set(T: AMDGPU::VM_VSRC, Val: ~0u);
2151
2152 // If that new encoded Depctr immediate would actually still wait
2153 // for anything, update the instruction's operand. Otherwise it can
2154 // just be deleted.
2155 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: ST)) {
2156 Modified |= updateOperandIfDifferent(MI&: *WaitcntDepctrInstr,
2157 OpName: AMDGPU::OpName::simm16, NewEnc: Enc);
2158 LLVM_DEBUG(It.isEnd() ? dbgs() << "applyPreexistingWaitcnt\n"
2159 << "New Instr at block end: "
2160 << *WaitcntDepctrInstr << '\n'
2161 : dbgs() << "applyPreexistingWaitcnt\n"
2162 << "Old Instr: " << *It << "New Instr: "
2163 << *WaitcntDepctrInstr << '\n');
2164 } else {
2165 WaitcntDepctrInstr->eraseFromParent();
2166 Modified = true;
2167 }
2168 }
2169
2170 return Modified;
2171}
2172
2173/// Generate S_WAIT_*CNT instructions for any required counters in \p Wait
2174bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
2175 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
2176 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
2177 assert(!isNormalMode(MaxCounter));
2178
2179 bool Modified = false;
2180 const DebugLoc &DL = Block.findDebugLoc(MBBI: It);
2181
2182 // For GFX12+, we use separate wait instructions, which makes expansion
2183 // simpler
2184 if (ExpandWaitcntProfiling) {
2185 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2186 unsigned Count = Wait.get(T: CT);
2187 if (Count == ~0u)
2188 continue;
2189
2190 // Skip expansion for out-of-order counters - emit normal wait instead
2191 if (ScoreBrackets.counterOutOfOrder(T: CT)) {
2192 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2193 .addImm(Val: Count);
2194 Modified = true;
2195 continue;
2196 }
2197
2198 unsigned Outstanding =
2199 std::min(a: ScoreBrackets.getOutstanding(T: CT), b: getLimit(E: CT) - 1);
2200 EmitExpandedWaitcnt(Outstanding, Target: Count, EmitWaitcnt: [&](unsigned Val) {
2201 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2202 .addImm(Val);
2203 });
2204 Modified = true;
2205 }
2206 return Modified;
2207 }
2208
2209 // Normal behavior (no expansion)
2210 // Check for opportunities to use combined wait instructions.
2211 if (Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2212 MachineInstr *SWaitInst = nullptr;
2213
2214 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u) {
2215 unsigned Enc = AMDGPU::encodeLoadcntDscnt(Version: IV, Decoded: Wait);
2216
2217 SWaitInst = BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAIT_LOADCNT_DSCNT))
2218 .addImm(Val: Enc);
2219
2220 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2221 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2222 } else if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u) {
2223 unsigned Enc = AMDGPU::encodeStorecntDscnt(Version: IV, Decoded: Wait);
2224
2225 SWaitInst = BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAIT_STORECNT_DSCNT))
2226 .addImm(Val: Enc);
2227
2228 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
2229 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2230 }
2231
2232 if (SWaitInst) {
2233 Modified = true;
2234
2235 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2236 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2237 dbgs() << "New Instr: " << *SWaitInst << '\n');
2238 }
2239 }
2240
2241 // Generate an instruction for any remaining counter that needs
2242 // waiting for.
2243
2244 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2245 unsigned Count = Wait.get(T: CT);
2246 if (Count == ~0u)
2247 continue;
2248
2249 [[maybe_unused]] auto SWaitInst =
2250 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2251 .addImm(Val: Count);
2252
2253 Modified = true;
2254
2255 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2256 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2257 dbgs() << "New Instr: " << *SWaitInst << '\n');
2258 }
2259
2260 if (Wait.hasWaitDepctr()) {
2261 assert(IsExpertMode);
2262 unsigned Enc =
2263 AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: Wait.get(T: AMDGPU::VM_VSRC), STI: ST);
2264 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2265 unsigned VaVdst =
2266 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
2267 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2268
2269 [[maybe_unused]] auto SWaitInst =
2270 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR)).addImm(Val: Enc);
2271
2272 Modified = true;
2273
2274 LLVM_DEBUG(dbgs() << "generateWaitcnt\n";
2275 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2276 dbgs() << "New Instr: " << *SWaitInst << '\n');
2277 }
2278
2279 return Modified;
2280}
2281
2282/// Generate s_waitcnt instruction to be placed before cur_Inst.
2283/// Instructions of a given type are returned in order,
2284/// but instructions of different types can complete out of order.
2285/// We rely on this in-order completion
2286/// and simply assign a score to the memory access instructions.
2287/// We keep track of the active "score bracket" to determine
2288/// if an access of a memory read requires an s_waitcnt
2289/// and if so what the value of each counter is.
2290/// The "score bracket" is bound by the lower bound and upper bound
2291/// scores (*_score_LB and *_score_ub respectively).
2292/// If FlushFlags.FlushVmCnt is true, we want to flush the vmcnt counter here.
2293/// If FlushFlags.FlushDsCnt is true, we want to flush the dscnt counter here
2294/// (GFX12+ only, where DS_CNT is a separate counter).
2295bool SIInsertWaitcnts::generateWaitcntInstBefore(
2296 MachineInstr &MI, WaitcntBrackets &ScoreBrackets,
2297 MachineInstr *OldWaitcntInstr, PreheaderFlushFlags FlushFlags) {
2298 LLVM_DEBUG(dbgs() << "\n*** GenerateWaitcntInstBefore: "; MI.print(dbgs()););
2299
2300 assert(!isNonWaitcntMetaInst(MI));
2301
2302 AMDGPU::Waitcnt Wait;
2303 const unsigned Opc = MI.getOpcode();
2304
2305 switch (Opc) {
2306 case AMDGPU::BUFFER_WBINVL1:
2307 case AMDGPU::BUFFER_WBINVL1_SC:
2308 case AMDGPU::BUFFER_WBINVL1_VOL:
2309 case AMDGPU::BUFFER_GL0_INV:
2310 case AMDGPU::BUFFER_GL1_INV: {
2311 // FIXME: This should have already been handled by the memory legalizer.
2312 // Removing this currently doesn't affect any lit tests, but we need to
2313 // verify that nothing was relying on this. The number of buffer invalidates
2314 // being handled here should not be expanded.
2315 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2316 break;
2317 }
2318 case AMDGPU::SI_RETURN_TO_EPILOG:
2319 case AMDGPU::SI_RETURN:
2320 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
2321 case AMDGPU::S_SETPC_B64_return: {
2322 // All waits must be resolved at call return.
2323 // NOTE: this could be improved with knowledge of all call sites or
2324 // with knowledge of the called routines.
2325 ReturnInsts.insert(V: &MI);
2326 AMDGPU::Waitcnt AllZeroWait =
2327 WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2328 // On GFX12+, if LOAD_CNT is pending but no VGPRs are waiting for loads
2329 // (e.g., only GLOBAL_INV is pending), we can skip waiting on loadcnt.
2330 // GLOBAL_INV increments loadcnt but doesn't write to VGPRs, so there's
2331 // no need to wait for it at function boundaries.
2332 if (ST.hasExtendedWaitCounts() &&
2333 !ScoreBrackets.hasPendingEvent(E: HWEvents::VMEM_READ_ACCESS))
2334 AllZeroWait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2335 Wait = AllZeroWait;
2336 break;
2337 }
2338 case AMDGPU::S_ENDPGM:
2339 case AMDGPU::S_ENDPGM_SAVED: {
2340 // In dynamic VGPR mode, we want to release the VGPRs before the wave exits.
2341 // Technically the hardware will do this on its own if we don't, but that
2342 // might cost extra cycles compared to doing it explicitly.
2343 // When not in dynamic VGPR mode, identify S_ENDPGM instructions which may
2344 // have to wait for outstanding VMEM stores. In this case it can be useful
2345 // to send a message to explicitly release all VGPRs before the stores have
2346 // completed, but it is only safe to do this if there are no outstanding
2347 // scratch stores.
2348 EndPgmInsts[&MI] =
2349 !ScoreBrackets.empty(T: AMDGPU::STORE_CNT) &&
2350 !ScoreBrackets.hasPendingEvent(E: HWEvents::SCRATCH_WRITE_ACCESS);
2351 break;
2352 }
2353 case AMDGPU::S_SENDMSG:
2354 case AMDGPU::S_SENDMSGHALT: {
2355 if (ST.hasLegacyGeometry() &&
2356 ((MI.getOperand(i: 0).getImm() & AMDGPU::SendMsg::ID_MASK_PreGFX11_) ==
2357 AMDGPU::SendMsg::ID_GS_DONE_PreGFX11)) {
2358 // Resolve vm waits before gs-done.
2359 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2360 break;
2361 }
2362 [[fallthrough]];
2363 }
2364 default: {
2365
2366 // Export & GDS instructions do not read the EXEC mask until after the
2367 // export is granted (which can occur well after the instruction is issued).
2368 // The shader program must flush all EXP operations on the export-count
2369 // before overwriting the EXEC mask.
2370 if (MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI)) {
2371 // Export and GDS are tracked individually, either may trigger a waitcnt
2372 // for EXEC.
2373 if (ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_GPR_LOCK) ||
2374 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_PARAM_ACCESS) ||
2375 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_POS_ACCESS) ||
2376 ScoreBrackets.hasPendingEvent(E: HWEvents::GDS_GPR_LOCK)) {
2377 Wait.set(T: AMDGPU::EXP_CNT, Val: 0);
2378 }
2379 }
2380
2381 // Wait for any pending GDS instruction to complete before any
2382 // "Always GDS" instruction.
2383 if (TII.isAlwaysGDS(Opcode: Opc) && ScoreBrackets.hasPendingGDS())
2384 Wait.add(T: AMDGPU::DS_CNT, Count: ScoreBrackets.getPendingGDSWait());
2385
2386 if (MI.isCall()) {
2387 // The function is going to insert a wait on everything in its prolog.
2388 // This still needs to be careful if the call target is a load (e.g. a GOT
2389 // load). We also need to check WAW dependency with saved PC.
2390 CallInsts.insert(V: &MI);
2391 Wait = AMDGPU::Waitcnt();
2392
2393 const MachineOperand &CallAddrOp = TII.getCalleeOperand(MI);
2394 if (CallAddrOp.isReg()) {
2395 ScoreBrackets.determineWaitForPhysReg(
2396 T: SmemAccessCounter, Reg: CallAddrOp.getReg().asMCReg(), Wait, MI);
2397
2398 if (const auto *RtnAddrOp =
2399 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::dst)) {
2400 ScoreBrackets.determineWaitForPhysReg(
2401 T: SmemAccessCounter, Reg: RtnAddrOp->getReg().asMCReg(), Wait, MI);
2402 }
2403 }
2404 } else if (Opc == AMDGPU::S_BARRIER_WAIT) {
2405 ScoreBrackets.tryClearSCCWriteEvent(Inst: &MI);
2406 } else {
2407 // FIXME: Should not be relying on memoperands.
2408 // Look at the source operands of every instruction to see if
2409 // any of them results from a previous memory operation that affects
2410 // its current usage. If so, an s_waitcnt instruction needs to be
2411 // emitted.
2412 // If the source operand was defined by a load, add the s_waitcnt
2413 // instruction.
2414 //
2415 // Two cases are handled for destination operands:
2416 // 1) If the destination operand was defined by a load, add the s_waitcnt
2417 // instruction to guarantee the right WAW order.
2418 // 2) If a destination operand that was used by a recent export/store ins,
2419 // add s_waitcnt on exp_cnt to guarantee the WAR order.
2420
2421 for (const MachineMemOperand *Memop : MI.memoperands()) {
2422 const Value *Ptr = Memop->getValue();
2423 if (Memop->isStore()) {
2424 if (auto It = SLoadAddresses.find(Val: Ptr); It != SLoadAddresses.end()) {
2425 Wait.add(T: SmemAccessCounter, Count: 0);
2426 if (PDT.dominates(A: MI.getParent(), B: It->second))
2427 SLoadAddresses.erase(I: It);
2428 }
2429 }
2430 unsigned AS = Memop->getAddrSpace();
2431 if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::FLAT_ADDRESS)
2432 continue;
2433 // No need to wait before load from VMEM to LDS.
2434 if (TII.mayWriteLDSThroughDMA(MI))
2435 continue;
2436
2437 // LOAD_CNT is only relevant to vgpr or LDS.
2438 unsigned TID = LDSDMA_BEGIN;
2439 if (Ptr && Memop->getAAInfo()) {
2440 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
2441 for (unsigned I = 0, E = LDSDMAStores.size(); I != E; ++I) {
2442 if (MI.mayAlias(AA, Other: *LDSDMAStores[I], UseTBAA: true)) {
2443 if ((I + 1) >= NUM_LDSDMA) {
2444 // We didn't have enough slot to track this LDS DMA store, it
2445 // has been tracked using the common RegNo (FIRST_LDS_VGPR).
2446 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID,
2447 Wait);
2448 break;
2449 }
2450
2451 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT,
2452 TID: TID + I + 1, Wait);
2453 }
2454 }
2455 } else {
2456 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID, Wait);
2457 }
2458 if (Memop->isStore()) {
2459 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::EXP_CNT, TID, Wait);
2460 }
2461 }
2462
2463 // Loop over use and def operands.
2464 for (const MachineOperand &Op : MI.operands()) {
2465 if (!Op.isReg())
2466 continue;
2467
2468 // If the instruction does not read tied source, skip the operand.
2469 if (Op.isTied() && Op.isUse() && TII.doesNotReadTiedSource(MI))
2470 continue;
2471
2472 MCPhysReg Reg = Op.getReg().asMCReg();
2473
2474 const bool IsVGPR = TRI.isVectorRegister(MRI, Reg: Op.getReg());
2475 if (IsVGPR) {
2476 // Implicit VGPR defs and uses are never a part of the memory
2477 // instructions description and usually present to account for
2478 // super-register liveness.
2479 // TODO: Most of the other instructions also have implicit uses
2480 // for the liveness accounting only.
2481 if (Op.isImplicit() && MI.mayLoadOrStore())
2482 continue;
2483
2484 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VA_VDST_WR, Reg, Wait,
2485 MI);
2486 if (Op.isDef()) {
2487 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VA_VDST_RD, Reg, Wait,
2488 MI);
2489 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VM_VSRC, Reg, Wait,
2490 MI);
2491 }
2492
2493 // RAW always needs an s_waitcnt. WAW needs an s_waitcnt unless the
2494 // previous write and this write are the same type of VMEM
2495 // instruction, in which case they are (in some architectures)
2496 // guaranteed to write their results in order anyway.
2497 // Additionally check instructions where Point Sample Acceleration
2498 // might be applied.
2499 if (Op.isUse() || !updateVMCntOnly(Inst: MI) ||
2500 ScoreBrackets.hasDifferentVGPRPendingEvents(
2501 Reg, E: AMDGPU::getSimplifiedVMEMEventsFor(Inst: MI, TII)) ||
2502 ScoreBrackets.hasPointSamplePendingVmemTypes(MI, Reg) ||
2503 !ST.hasVmemWriteVgprInOrder()) {
2504 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::LOAD_CNT, Reg, Wait,
2505 MI);
2506 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::SAMPLE_CNT, Reg, Wait,
2507 MI);
2508 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::BVH_CNT, Reg, Wait,
2509 MI);
2510 ScoreBrackets.clearVGPRPendingEvents(Reg);
2511 }
2512
2513 if (Op.isDef() ||
2514 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_LDS_ACCESS)) {
2515 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::EXP_CNT, Reg, Wait,
2516 MI);
2517 }
2518 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::DS_CNT, Reg, Wait, MI);
2519 } else if (Op.getReg() == AMDGPU::SCC) {
2520 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::KM_CNT, Reg, Wait, MI);
2521 } else {
2522 ScoreBrackets.determineWaitForPhysReg(T: SmemAccessCounter, Reg, Wait,
2523 MI);
2524 }
2525
2526 if (ST.hasWaitXcnt() && Op.isDef())
2527 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::X_CNT, Reg, Wait, MI);
2528 }
2529 }
2530 }
2531 }
2532
2533 // Ensure safety against exceptions from outstanding memory operations while
2534 // waiting for a barrier:
2535 //
2536 // * Some subtargets safely handle backing off the barrier in hardware
2537 // when an exception occurs.
2538 // * Some subtargets have an implicit S_WAITCNT 0 before barriers, so that
2539 // there can be no outstanding memory operations during the wait.
2540 // * Subtargets with split barriers don't need to back off the barrier; it
2541 // is up to the trap handler to preserve the user barrier state correctly.
2542 //
2543 // In all other cases, ensure safety by ensuring that there are no outstanding
2544 // memory operations.
2545 if (Opc == AMDGPU::S_BARRIER && !ST.hasAutoWaitcntBeforeBarrier() &&
2546 !ST.hasBackOffBarrier()) {
2547 Wait = Wait.combined(Other: WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/true));
2548 }
2549
2550 // TODO: Remove this work-around, enable the assert for Bug 457939
2551 // after fixing the scheduler. Also, the Shader Compiler code is
2552 // independent of target.
2553 if (SIInstrInfo::isCBranchVCCZRead(MI) && ST.hasReadVCCZBug() &&
2554 ScoreBrackets.hasPendingEvent(E: HWEvents::SMEM_ACCESS)) {
2555 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2556 }
2557
2558 // Verify that the wait is actually needed.
2559 ScoreBrackets.simplifyWaitcnt(Wait);
2560
2561 // It is only necessary to insert an S_WAITCNT_DEPCTR instruction that
2562 // waits on VA_VDST if the instruction it would precede is not a VALU
2563 // instruction, since hardware handles VALU->VGPR->VALU hazards in
2564 // expert scheduling mode.
2565 if (TII.isVALU(MI, /*AllowLDSDMA=*/false)) {
2566 Wait.set(T: AMDGPU::VA_VDST_RD, Val: ~0u);
2567 Wait.set(T: AMDGPU::VA_VDST_WR, Val: ~0u);
2568 }
2569
2570 // Since the translation for VMEM addresses occur in-order, we can apply the
2571 // XCnt if the current instruction is of VMEM type and has a memory
2572 // dependency with another VMEM instruction in flight.
2573 if (Wait.get(T: AMDGPU::X_CNT) != ~0u && isVmemAccess(MI)) {
2574 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::X_CNT);
2575 Wait.set(T: AMDGPU::X_CNT, Val: ~0u);
2576 }
2577
2578 // When forcing emit, we need to skip terminators because that would break the
2579 // terminators of the MBB if we emit a waitcnt between terminators.
2580 if (ForceEmitZeroFlag && !MI.isTerminator())
2581 Wait = WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2582
2583 // If we force waitcnt then update Wait accordingly.
2584 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
2585 if (!ForceEmitWaitcnt[T])
2586 continue;
2587 Wait.set(T, Val: 0);
2588 }
2589
2590 if (FlushFlags.FlushVmCnt) {
2591 for (AMDGPU::InstCounterType T :
2592 {AMDGPU::LOAD_CNT, AMDGPU::SAMPLE_CNT, AMDGPU::BVH_CNT})
2593 Wait.set(T, Val: 0);
2594 }
2595
2596 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(T: AMDGPU::DS_CNT))
2597 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2598
2599 if (ForceEmitZeroLoadFlag && Wait.get(T: AMDGPU::LOAD_CNT) != ~0u)
2600 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2601
2602 return generateWaitcnt(Wait, It: MI.getIterator(), Block&: *MI.getParent(), ScoreBrackets,
2603 OldWaitcntInstr);
2604}
2605
2606bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
2607 MachineBasicBlock::instr_iterator It,
2608 MachineBasicBlock &Block,
2609 WaitcntBrackets &ScoreBrackets,
2610 MachineInstr *OldWaitcntInstr) {
2611 bool Modified = false;
2612
2613 if (OldWaitcntInstr)
2614 // Try to merge the required wait with preexisting waitcnt instructions.
2615 // Also erase redundant waitcnt.
2616 Modified =
2617 WCG->applyPreexistingWaitcnt(ScoreBrackets, OldWaitcntInstr&: *OldWaitcntInstr, Wait, It);
2618
2619 // ExpCnt can be merged into VINTERP.
2620 if (Wait.get(T: AMDGPU::EXP_CNT) != ~0u && It != Block.instr_end() &&
2621 SIInstrInfo::isVINTERP(MI: *It)) {
2622 MachineOperand *WaitExp = TII.getNamedOperand(MI&: *It, OperandName: AMDGPU::OpName::waitexp);
2623 if (Wait.get(T: AMDGPU::EXP_CNT) < WaitExp->getImm()) {
2624 WaitExp->setImm(Wait.get(T: AMDGPU::EXP_CNT));
2625 Modified = true;
2626 }
2627 // Apply ExpCnt before resetting it, so applyWaitcnt below sees all counts.
2628 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::EXP_CNT);
2629 Wait.set(T: AMDGPU::EXP_CNT, Val: ~0u);
2630
2631 LLVM_DEBUG(dbgs() << "generateWaitcnt\n"
2632 << "Update Instr: " << *It);
2633 }
2634
2635 if (WCG->createNewWaitcnt(Block, It, Wait, ScoreBrackets))
2636 Modified = true;
2637
2638 // Any counts that could have been applied to any existing waitcnt
2639 // instructions will have been done so, now deal with any remaining.
2640 ScoreBrackets.applyWaitcnt(Wait);
2641
2642 return Modified;
2643}
2644
2645bool SIInsertWaitcnts::isVmemAccess(const MachineInstr &MI) const {
2646 return (TII.isFLAT(MI) && TII.mayAccessVMEMThroughFlat(MI)) ||
2647 (TII.isVMEM(MI) && !AMDGPU::getMUBUFIsBufferInv(Opc: MI.getOpcode()));
2648}
2649
2650// Return true if the next instruction is S_ENDPGM, following fallthrough
2651// blocks if necessary.
2652bool SIInsertWaitcnts::isNextENDPGM(MachineBasicBlock::instr_iterator It,
2653 MachineBasicBlock *Block) const {
2654 auto BlockEnd = Block->getParent()->end();
2655 auto BlockIter = Block->getIterator();
2656
2657 while (true) {
2658 if (It.isEnd()) {
2659 if (++BlockIter != BlockEnd) {
2660 It = BlockIter->instr_begin();
2661 continue;
2662 }
2663
2664 return false;
2665 }
2666
2667 if (!It->isMetaInstruction())
2668 break;
2669
2670 It++;
2671 }
2672
2673 assert(!It.isEnd());
2674
2675 return It->getOpcode() == AMDGPU::S_ENDPGM;
2676}
2677
2678// Add a wait after an instruction if architecture requirements mandate one.
2679bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
2680 MachineBasicBlock &Block,
2681 WaitcntBrackets &ScoreBrackets) {
2682 AMDGPU::Waitcnt Wait;
2683 bool NeedsEndPGMCheck = false;
2684
2685 if (ST.isPreciseMemoryEnabled() && Inst.mayLoadOrStore())
2686 Wait = WCG->getAllZeroWaitcnt(IncludeVSCnt: Inst.mayStore() &&
2687 !SIInstrInfo::isAtomicRet(MI: Inst));
2688
2689 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode())) {
2690 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2691 NeedsEndPGMCheck = true;
2692 }
2693
2694 ScoreBrackets.simplifyWaitcnt(Wait);
2695
2696 auto SuccessorIt = std::next(x: Inst.getIterator());
2697 bool Result = generateWaitcnt(Wait, It: SuccessorIt, Block, ScoreBrackets,
2698 /*OldWaitcntInstr=*/nullptr);
2699
2700 if (Result && NeedsEndPGMCheck && isNextENDPGM(It: SuccessorIt, Block: &Block)) {
2701 BuildMI(BB&: Block, I: SuccessorIt, MIMD: Inst.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_NOP))
2702 .addImm(Val: 0);
2703 }
2704
2705 return Result;
2706}
2707
2708void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
2709 WaitcntBrackets *ScoreBrackets) {
2710
2711 HWEvents InstEvents = AMDGPU::getEventsFor(Inst, ST, IsExpertMode, TgSplit);
2712 for (HWEvents E : InstEvents)
2713 ScoreBrackets->updateByEvent(E, Inst);
2714
2715 if (TII.isDS(MI: Inst) && TII.usesLGKM_CNT(MI: Inst)) {
2716 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode()) ||
2717 TII.hasModifiersSet(MI: Inst, OpName: AMDGPU::OpName::gds)) {
2718 ScoreBrackets->setPendingGDS();
2719 }
2720 } else if (TII.isFLAT(MI: Inst)) {
2721 if (Inst.mayLoadOrStore() && TII.mayAccessVMEMThroughFlat(MI: Inst) &&
2722 TII.mayAccessLDSThroughFlat(MI: Inst, TgSplit) &&
2723 !SIInstrInfo::isLDSDMA(MI: Inst)) {
2724 // Async/LDSDMA operations have FLAT encoding but do not actually use flat
2725 // pointers. They do have two operands that each access global and LDS,
2726 // thus making it appear at this point that they are using a flat pointer.
2727 // Filter them out, and for the rest, generate a dependency on flat
2728 // pointers so that both VM and LGKM counters are flushed.
2729 ScoreBrackets->setPendingFlat();
2730 }
2731 } else if (Inst.isCall()) {
2732 // Act as a wait on everything, but AsyncCnt and TensorCnt are never
2733 // included in such blanket waits.
2734 ScoreBrackets->applyWaitcnt(Wait: WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false));
2735 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2736 } else if (TII.isVINTERP(MI: Inst)) {
2737 int64_t Imm = TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::waitexp)->getImm();
2738 ScoreBrackets->applyWaitcnt(T: AMDGPU::EXP_CNT, Count: Imm);
2739 }
2740
2741 // Set XCNT to zero in the bracket for instructions that implicitly drain
2742 // XCNT.
2743 if (ST.hasWaitXcnt() && SIInstrInfo::isXcntDrain(MI: Inst))
2744 ScoreBrackets->applyWaitcnt(T: AMDGPU::X_CNT, Count: 0);
2745}
2746
2747bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
2748 unsigned OtherScore) {
2749 unsigned MyShifted = Score <= M.OldLB ? 0 : Score + M.MyShift;
2750 unsigned OtherShifted =
2751 OtherScore <= M.OtherLB ? 0 : OtherScore + M.OtherShift;
2752 Score = std::max(a: MyShifted, b: OtherShifted);
2753 return OtherShifted > MyShifted;
2754}
2755
2756bool WaitcntBrackets::mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
2757 ArrayRef<CounterValueArray> OtherMarks) {
2758 bool StrictDom = false;
2759
2760 LLVM_DEBUG(dbgs() << "Merging async marks ...");
2761 // Early exit: nothing to merge when both sides are empty.
2762 if (AsyncMarks.empty() && OtherMarks.empty()) {
2763 LLVM_DEBUG(dbgs() << " nothing to merge\n");
2764 return false;
2765 }
2766 LLVM_DEBUG(dbgs() << '\n');
2767
2768 // Determine maximum length needed after merging
2769 auto MaxSize = (unsigned)std::max(a: AsyncMarks.size(), b: OtherMarks.size());
2770 MaxSize = std::min(a: MaxSize, b: MaxAsyncMarks);
2771
2772 // Keep only the most recent marks within our limit.
2773 if (AsyncMarks.size() > MaxSize)
2774 AsyncMarks.erase(CS: AsyncMarks.begin(),
2775 CE: AsyncMarks.begin() + (AsyncMarks.size() - MaxSize));
2776
2777 // Pad with zero-filled marks if our list is shorter. Zero represents "no
2778 // pending async operations at this checkpoint" and acts as the identity
2779 // element for max() during merging. We pad at the beginning since the marks
2780 // need to be aligned in most-recent order.
2781 constexpr CounterValueArray ZeroMark{};
2782 AsyncMarks.insert(I: AsyncMarks.begin(), NumToInsert: MaxSize - AsyncMarks.size(), Elt: ZeroMark);
2783
2784 LLVM_DEBUG({
2785 dbgs() << "Before merge:\n";
2786 for (const auto &Mark : AsyncMarks) {
2787 llvm::interleaveComma(Mark, dbgs());
2788 dbgs() << '\n';
2789 }
2790 dbgs() << "Other marks:\n";
2791 for (const auto &Mark : OtherMarks) {
2792 llvm::interleaveComma(Mark, dbgs());
2793 dbgs() << '\n';
2794 }
2795 });
2796
2797 // Merge element-wise using the existing mergeScore function and the
2798 // appropriate MergeInfo for each counter type. Rebase and merge EVERY
2799 // surviving mark into the new (widened) frame, aligned by recency from the
2800 // back. A mark with no counterpart on the other side merges against the
2801 // zero/identity mark: mergeScore still applies MyShift, so the mark is
2802 // re-expressed in the new frame instead of being left with a stale
2803 // (pre-merge) score.
2804 const unsigned OtherSize = OtherMarks.size();
2805 const unsigned OurSize = AsyncMarks.size();
2806 // After the both-empty early-return above, max(AsyncMarks, OtherMarks) >= 1,
2807 // and the erase/pad steps normalize AsyncMarks to exactly MaxSize. Hence
2808 // OurSize == MaxSize >= 1 (as long as MaxAsyncMarks != 0), so the
2809 // seq_inclusive(1, OurSize) below never trips its "Begin <= End" assertion
2810 // the way seq_inclusive(1, MergeCount) could when OtherSize == 0.
2811 assert(OurSize >= 1 &&
2812 "AsyncMarks padded to MaxSize >= 1 (needs MaxAsyncMarks != 0)");
2813
2814 for (auto Idx : seq_inclusive<unsigned>(Begin: 1, End: OurSize)) {
2815 const CounterValueArray &OtherMark =
2816 Idx <= OtherSize ? OtherMarks[OtherSize - Idx] : ZeroMark;
2817 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter))
2818 StrictDom |=
2819 mergeScore(M: MergeInfos[T], Score&: AsyncMarks[OurSize - Idx][T], OtherScore: OtherMark[T]);
2820 }
2821
2822 LLVM_DEBUG({
2823 dbgs() << "After merge:\n";
2824 for (const auto &Mark : AsyncMarks) {
2825 llvm::interleaveComma(Mark, dbgs());
2826 dbgs() << '\n';
2827 }
2828 });
2829
2830 return StrictDom;
2831}
2832
2833/// Merge the pending events and associater score brackets of \p Other into
2834/// this brackets status.
2835///
2836/// Returns whether the merge resulted in a change that requires tighter waits
2837/// (i.e. the merged brackets strictly dominate the original brackets).
2838bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
2839 bool StrictDom = false;
2840
2841 // Check if "other" has keys we don't have, and create default entries for
2842 // those. If they remain empty after merging, we will clean it up after.
2843 for (auto K : Other.VMem.keys())
2844 VMem.try_emplace(Key: K);
2845 for (auto K : Other.SGPRs.keys())
2846 SGPRs.try_emplace(Key: K);
2847
2848 // Array to store MergeInfo for each counter type
2849 MergeInfo MergeInfos[AMDGPU::NUM_INST_CNTS];
2850
2851 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter)) {
2852 // Merge event flags for this counter
2853 const HWEvents &EventsForT = Context->getWaitEvents(T);
2854 const HWEvents OldEvents = PendingEvents & EventsForT;
2855 const HWEvents OtherEvents = Other.PendingEvents & EventsForT;
2856 if (!OldEvents.contains(Other: OtherEvents))
2857 StrictDom = true;
2858 PendingEvents |= OtherEvents;
2859
2860 // Merge scores for this counter
2861 const unsigned MyPending = ScoreUBs[T] - ScoreLBs[T];
2862 const unsigned OtherPending = Other.ScoreUBs[T] - Other.ScoreLBs[T];
2863 const unsigned NewUB = ScoreLBs[T] + std::max(a: MyPending, b: OtherPending);
2864 if (NewUB < ScoreLBs[T])
2865 report_fatal_error(reason: "waitcnt score overflow");
2866
2867 MergeInfo &M = MergeInfos[T];
2868 M.OldLB = ScoreLBs[T];
2869 M.OtherLB = Other.ScoreLBs[T];
2870 M.MyShift = NewUB - ScoreUBs[T];
2871 M.OtherShift = NewUB - Other.ScoreUBs[T];
2872
2873 ScoreUBs[T] = NewUB;
2874
2875 if (T == AMDGPU::LOAD_CNT)
2876 StrictDom |= mergeScore(M, Score&: LastFlatLoadCnt, OtherScore: Other.LastFlatLoadCnt);
2877
2878 if (T == AMDGPU::DS_CNT) {
2879 StrictDom |= mergeScore(M, Score&: LastFlatDsCnt, OtherScore: Other.LastFlatDsCnt);
2880 StrictDom |= mergeScore(M, Score&: LastGDS, OtherScore: Other.LastGDS);
2881 }
2882
2883 if (T == AMDGPU::KM_CNT) {
2884 StrictDom |= mergeScore(M, Score&: SCCScore, OtherScore: Other.SCCScore);
2885 if (Other.hasPendingEvent(E: HWEvents::SCC_WRITE)) {
2886 if (!(OldEvents & HWEvents::SCC_WRITE)) {
2887 PendingSCCWrite = Other.PendingSCCWrite;
2888 } else if (PendingSCCWrite != Other.PendingSCCWrite) {
2889 PendingSCCWrite = nullptr;
2890 }
2891 }
2892 }
2893
2894 for (auto &[RegID, Info] : VMem)
2895 StrictDom |= mergeScore(M, Score&: Info.Scores[T], OtherScore: Other.getVMemScore(TID: RegID, T));
2896
2897 if (isSmemCounter(T)) {
2898 for (auto &[RegID, Info] : SGPRs) {
2899 auto It = Other.SGPRs.find(Val: RegID);
2900 unsigned OtherScore = (It != Other.SGPRs.end()) ? It->second.get(T) : 0;
2901 StrictDom |= mergeScore(M, Score&: Info.get(T), OtherScore);
2902 }
2903 }
2904 }
2905
2906 for (auto &[TID, Info] : VMem) {
2907 if (auto It = Other.VMem.find(Val: TID); It != Other.VMem.end()) {
2908 HWEvents NewVGPRContext =
2909 Info.VGPRPendingEvents | It->second.VGPRPendingEvents;
2910 StrictDom |= NewVGPRContext != Info.VGPRPendingEvents;
2911 Info.VGPRPendingEvents = NewVGPRContext;
2912 }
2913 }
2914
2915 StrictDom |= mergeAsyncMarks(MergeInfos, OtherMarks: Other.AsyncMarks);
2916 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter))
2917 StrictDom |= mergeScore(M: MergeInfos[T], Score&: AsyncScore[T], OtherScore: Other.AsyncScore[T]);
2918
2919 purgeEmptyTrackingData();
2920 return StrictDom;
2921}
2922
2923static bool isWaitInstr(MachineInstr &Inst) {
2924 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: Inst.getOpcode());
2925 return Opcode == AMDGPU::S_WAITCNT ||
2926 (Opcode == AMDGPU::S_WAITCNT_VSCNT && Inst.getOperand(i: 0).isReg() &&
2927 Inst.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL) ||
2928 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
2929 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
2930 Opcode == AMDGPU::S_WAITCNT_lds_direct ||
2931 Opcode == AMDGPU::WAIT_ASYNCMARK ||
2932 AMDGPU::counterTypeForInstr(Opcode).has_value();
2933}
2934
2935void SIInsertWaitcnts::setSchedulingMode(MachineBasicBlock &MBB,
2936 MachineBasicBlock::iterator I,
2937 bool ExpertMode) const {
2938 const unsigned EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
2939 Values: AMDGPU::Hwreg::ID_SCHED_MODE, Values: AMDGPU::Hwreg::HwregOffset::Default, Values: 2);
2940 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
2941 .addImm(Val: ExpertMode ? 2 : 0)
2942 .addImm(Val: EncodedReg);
2943}
2944
2945namespace {
2946// TODO: Remove this work-around after fixing the scheduler.
2947// There are two reasons why vccz might be incorrect; see ST.hasReadVCCZBug()
2948// and ST.partialVCCWritesUpdateVCCZ().
2949// i. VCCZBug: There is a hardware bug on CI/SI where SMRD instruction may
2950// corrupt vccz bit, so when we detect that an instruction may read from
2951// a corrupt vccz bit, we need to:
2952// 1. Insert s_waitcnt lgkm(0) to wait for all outstanding SMRD
2953// operations to complete.
2954// 2. Recompute the correct value of vccz by writing the current value
2955// of vcc back to vcc.
2956// ii. Partial writes to vcc don't update vccz, so we need to recompute the
2957// correct value of vccz by reading vcc and writing it back to vcc.
2958// No waitcnt is needed in this case.
2959class VCCZWorkaround {
2960 const WaitcntBrackets &ScoreBrackets;
2961 const GCNSubtarget &ST;
2962 const SIInstrInfo &TII;
2963 const SIRegisterInfo &TRI;
2964 bool VCCZCorruptionBug = false;
2965 bool VCCZNotUpdatedByPartialWrites = false;
2966 /// vccz could be incorrect at a basic block boundary if a predecessor wrote
2967 /// to vcc and then issued an smem load, so initialize to true.
2968 bool MustRecomputeVCCZ = true;
2969
2970public:
2971 VCCZWorkaround(const WaitcntBrackets &ScoreBrackets, const GCNSubtarget &ST,
2972 const SIInstrInfo &TII, const SIRegisterInfo &TRI)
2973 : ScoreBrackets(ScoreBrackets), ST(ST), TII(TII), TRI(TRI) {
2974 VCCZCorruptionBug = ST.hasReadVCCZBug();
2975 VCCZNotUpdatedByPartialWrites = !ST.partialVCCWritesUpdateVCCZ();
2976 }
2977 /// If \p MI reads vccz and we must recompute it based on MustRecomputeVCCZ,
2978 /// then emit a vccz recompute instruction before \p MI. This needs to be
2979 /// called on every instruction in the basic block because it also tracks the
2980 /// state and updates MustRecomputeVCCZ accordingly. Returns true if it
2981 /// modified the IR.
2982 bool tryRecomputeVCCZ(MachineInstr &MI) {
2983 // No need to run this if neither bug is present.
2984 if (!VCCZCorruptionBug && !VCCZNotUpdatedByPartialWrites)
2985 return false;
2986
2987 // If MI is an SMEM and it can corrupt vccz on this target, then we need
2988 // both to emit a waitcnt and to recompute vccz.
2989 // But we don't actually emit a waitcnt here. This is done in
2990 // generateWaitcntInstBefore() because it tracks all the necessary waitcnt
2991 // state, and can either skip emitting a waitcnt if there is already one in
2992 // the IR, or emit an "optimized" combined waitcnt.
2993 // If this is an smem read, it could complete and clobber vccz at any time.
2994 MustRecomputeVCCZ |= VCCZCorruptionBug && TII.isSMRD(MI);
2995
2996 // If the target partial vcc writes don't update vccz, and MI is such an
2997 // instruction then we must recompute vccz.
2998 // Note: We are using PartiallyWritesToVCCOpt optional to avoid calling
2999 // `definesRegister()` more than needed, because it's not very cheap.
3000 std::optional<bool> PartiallyWritesToVCCOpt;
3001 auto PartiallyWritesToVCC = [](MachineInstr &MI) {
3002 return MI.definesRegister(Reg: AMDGPU::VCC_LO, /*TRI=*/nullptr) ||
3003 MI.definesRegister(Reg: AMDGPU::VCC_HI, /*TRI=*/nullptr);
3004 };
3005 if (VCCZNotUpdatedByPartialWrites) {
3006 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
3007 // If this is a partial VCC write but won't update vccz, then we must
3008 // recompute vccz.
3009 MustRecomputeVCCZ |= *PartiallyWritesToVCCOpt;
3010 }
3011
3012 // If MI is a vcc write with no pending smem, or there is a pending smem
3013 // but the target does not suffer from the vccz corruption bug, then we
3014 // don't need to recompute vccz as this write will recompute it anyway.
3015 if (!ScoreBrackets.hasPendingEvent(E: HWEvents::SMEM_ACCESS) ||
3016 !VCCZCorruptionBug) {
3017 // Compute PartiallyWritesToVCCOpt if we haven't done so already.
3018 if (!PartiallyWritesToVCCOpt)
3019 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
3020 bool FullyWritesToVCC = !*PartiallyWritesToVCCOpt &&
3021 MI.definesRegister(Reg: AMDGPU::VCC, /*TRI=*/nullptr);
3022 // If we write to the full vcc or we write partially and the target
3023 // updates vccz on partial writes, then vccz will be updated correctly.
3024 bool UpdatesVCCZ = FullyWritesToVCC || (!VCCZNotUpdatedByPartialWrites &&
3025 *PartiallyWritesToVCCOpt);
3026 if (UpdatesVCCZ)
3027 MustRecomputeVCCZ = false;
3028 }
3029
3030 // If MI is a branch that reads VCCZ then emit a waitcnt and a vccz
3031 // restore instruction if either is needed.
3032 if (SIInstrInfo::isCBranchVCCZRead(MI) && MustRecomputeVCCZ) {
3033 // Recompute the vccz bit. Any time a value is written to vcc, the vccz
3034 // bit is updated, so we can restore the bit by reading the value of vcc
3035 // and then writing it back to the register.
3036 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
3037 MCID: TII.get(Opcode: ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
3038 DestReg: TRI.getVCC())
3039 .addReg(RegNo: TRI.getVCC());
3040 MustRecomputeVCCZ = false;
3041 return true;
3042 }
3043 return false;
3044 }
3045};
3046
3047} // namespace
3048
3049// Generate s_waitcnt instructions where needed.
3050bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
3051 MachineBasicBlock &Block,
3052 WaitcntBrackets &ScoreBrackets) {
3053 bool Modified = false;
3054
3055 LLVM_DEBUG({
3056 dbgs() << "*** Begin Block: ";
3057 Block.printName(dbgs());
3058 ScoreBrackets.dump();
3059 });
3060 VCCZWorkaround VCCZW(ScoreBrackets, ST, TII, TRI);
3061
3062 // Walk over the instructions.
3063 MachineInstr *OldWaitcntInstr = nullptr;
3064
3065 // NOTE: We may append instrs after Inst while iterating.
3066 ScoreBrackets.verify();
3067 for (MachineBasicBlock::instr_iterator Iter = Block.instr_begin(),
3068 E = Block.instr_end();
3069 Iter != E; ++Iter) {
3070 MachineInstr &Inst = *Iter;
3071 if (isNonWaitcntMetaInst(MI: Inst))
3072 continue;
3073 // Track pre-existing waitcnts that were added in earlier iterations or by
3074 // the memory legalizer.
3075 if (isWaitInstr(Inst) ||
3076 (IsExpertMode && Inst.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR)) {
3077 if (!OldWaitcntInstr)
3078 OldWaitcntInstr = &Inst;
3079 continue;
3080 }
3081
3082 PreheaderFlushFlags FlushFlags;
3083 if (Block.getFirstTerminator() == Inst)
3084 FlushFlags = isPreheaderToFlush(MBB&: Block, ScoreBrackets);
3085
3086 // Generate an s_waitcnt instruction to be placed before Inst, if needed.
3087 Modified |= generateWaitcntInstBefore(MI&: Inst, ScoreBrackets, OldWaitcntInstr,
3088 FlushFlags);
3089 OldWaitcntInstr = nullptr;
3090
3091 if (Inst.getOpcode() == AMDGPU::ASYNCMARK) {
3092 // Asyncmarks record the current wait state and so should not allow
3093 // waitcnts that occur after them to be merged into waitcnts that occur
3094 // before.
3095 ScoreBrackets.recordAsyncMark(Inst);
3096 continue;
3097 }
3098
3099 if (TII.isSMRD(MI: Inst)) {
3100 for (const MachineMemOperand *Memop : Inst.memoperands()) {
3101 // No need to handle invariant loads when avoiding WAR conflicts, as
3102 // there cannot be a vector store to the same memory location.
3103 if (!Memop->isInvariant()) {
3104 const Value *Ptr = Memop->getValue();
3105 SLoadAddresses.insert(KV: std::pair(Ptr, Inst.getParent()));
3106 }
3107 }
3108 }
3109
3110 updateEventWaitcntAfter(Inst, ScoreBrackets: &ScoreBrackets);
3111
3112 // Note: insertForcedWaitAfter() may add instrs after Iter that need to be
3113 // visited by the loop.
3114 Modified |= insertForcedWaitAfter(Inst, Block, ScoreBrackets);
3115
3116 LLVM_DEBUG({
3117 Inst.print(dbgs());
3118 ScoreBrackets.dump();
3119 });
3120
3121 // If the target suffers from the vccz bugs, this may emit the necessary
3122 // vccz recompute instruction before \p Inst if needed.
3123 Modified |= VCCZW.tryRecomputeVCCZ(MI&: Inst);
3124
3125 ScoreBrackets.verify();
3126 }
3127
3128 // Flush counters at the end of the block if needed (for preheaders with no
3129 // terminator).
3130 AMDGPU::Waitcnt Wait;
3131 if (Block.getFirstTerminator() == Block.end()) {
3132 PreheaderFlushFlags FlushFlags = isPreheaderToFlush(MBB&: Block, ScoreBrackets);
3133 if (FlushFlags.FlushVmCnt) {
3134 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::LOAD_CNT))
3135 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
3136 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::SAMPLE_CNT))
3137 Wait.set(T: AMDGPU::SAMPLE_CNT, Val: 0);
3138 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::BVH_CNT))
3139 Wait.set(T: AMDGPU::BVH_CNT, Val: 0);
3140 }
3141 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(T: AMDGPU::DS_CNT))
3142 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
3143 }
3144
3145 // Combine or remove any redundant waitcnts at the end of the block.
3146 Modified |= generateWaitcnt(Wait, It: Block.instr_end(), Block, ScoreBrackets,
3147 OldWaitcntInstr);
3148
3149 LLVM_DEBUG({
3150 dbgs() << "*** End Block: ";
3151 Block.printName(dbgs());
3152 ScoreBrackets.dump();
3153 });
3154
3155 return Modified;
3156}
3157
3158bool SIInsertWaitcnts::removeRedundantSoftXcnts(MachineBasicBlock &Block) {
3159 if (Block.size() <= 1)
3160 return false;
3161 // The Memory Legalizer conservatively inserts a soft xcnt before each
3162 // atomic RMW operation. However, for sequences of back-to-back atomic
3163 // RMWs, only the first s_wait_xcnt insertion is necessary. Optimize away
3164 // the redundant soft xcnts.
3165 bool Modified = false;
3166 // Remember the last atomic with a soft xcnt right before it.
3167 MachineInstr *LastAtomicWithSoftXcnt = nullptr;
3168
3169 for (MachineInstr &MI : drop_begin(RangeOrContainer&: Block)) {
3170 // Ignore last atomic if non-LDS VMEM and SMEM.
3171 bool IsLDS = TII.isDS(MI) ||
3172 (TII.isFLAT(MI) && TII.mayAccessLDSThroughFlat(MI, TgSplit));
3173 if (!IsLDS && (MI.mayLoad() ^ MI.mayStore()))
3174 LastAtomicWithSoftXcnt = nullptr;
3175
3176 bool IsAtomicRMW =
3177 SIInstrFlags::isMaybeAtomic(O: MI) && MI.mayLoad() && MI.mayStore();
3178 MachineInstr &PrevMI = *MI.getPrevNode();
3179 // This is an atomic with a soft xcnt.
3180 if (PrevMI.getOpcode() == AMDGPU::S_WAIT_XCNT_soft && IsAtomicRMW) {
3181 // If we have already found an atomic with a soft xcnt, remove this soft
3182 // xcnt as it's redundant.
3183 if (LastAtomicWithSoftXcnt) {
3184 PrevMI.eraseFromParent();
3185 Modified = true;
3186 }
3187 LastAtomicWithSoftXcnt = &MI;
3188 }
3189 }
3190 return Modified;
3191}
3192
3193// Return flags indicating which counters should be flushed in the preheader.
3194PreheaderFlushFlags
3195SIInsertWaitcnts::isPreheaderToFlush(MachineBasicBlock &MBB,
3196 const WaitcntBrackets &ScoreBrackets) {
3197 auto [Iterator, IsInserted] =
3198 PreheadersToFlush.try_emplace(Key: &MBB, Args: PreheaderFlushFlags());
3199 if (!IsInserted)
3200 return Iterator->second;
3201
3202 MachineBasicBlock *Succ = MBB.getSingleSuccessor();
3203 if (!Succ)
3204 return PreheaderFlushFlags();
3205
3206 MachineLoop *Loop = MLI.getLoopFor(BB: Succ);
3207 if (!Loop)
3208 return PreheaderFlushFlags();
3209
3210 if (Loop->getLoopPreheader() == &MBB) {
3211 Iterator->second = getPreheaderFlushFlags(ML: Loop, Brackets: ScoreBrackets);
3212 return Iterator->second;
3213 }
3214
3215 return PreheaderFlushFlags();
3216}
3217
3218bool SIInsertWaitcnts::isVMEMOrFlatVMEM(const MachineInstr &MI) const {
3219 if (SIInstrInfo::isFLAT(MI))
3220 return TII.mayAccessVMEMThroughFlat(MI);
3221 return SIInstrInfo::isVMEM(MI);
3222}
3223
3224bool SIInsertWaitcnts::isDSRead(const MachineInstr &MI) const {
3225 return SIInstrInfo::isDS(MI) && MI.mayLoad() && !MI.mayStore();
3226}
3227
3228// Check if instruction is a store to LDS that is counted via DSCNT
3229// (where that counter exists).
3230bool SIInsertWaitcnts::mayStoreIncrementingDSCNT(const MachineInstr &MI) const {
3231 return MI.mayStore() && SIInstrInfo::isDS(MI);
3232}
3233
3234// Return flags indicating which counters should be flushed in the preheader of
3235// the given loop. We currently decide to flush in the following situations:
3236// For VMEM (FlushVmCnt):
3237// 1. The loop contains vmem store(s), no vmem load and at least one use of a
3238// vgpr containing a value that is loaded outside of the loop. (Only on
3239// targets with no vscnt counter).
3240// 2. The loop contains vmem load(s), but the loaded values are not used in the
3241// loop, and at least one use of a vgpr containing a value that is loaded
3242// outside of the loop.
3243// For DS (FlushDsCnt, GFX12+ only):
3244// 3. The loop contains no DS reads, and at least one use of a vgpr containing
3245// a value that is DS read outside of the loop.
3246// 4. The loop contains DS read(s), loaded values are not used in the same
3247// iteration but in the next iteration (prefetch pattern), and at least one
3248// use of a vgpr containing a value that is DS read outside of the loop.
3249// Flushing in preheader reduces wait overhead if the wait requirement in
3250// iteration 1 would otherwise be more strict (but unfortunately preheader
3251// flush decision is taken before knowing that).
3252// 5. (Single-block loops only) The loop has DS prefetch reads with flush point
3253// tracking. Some DS reads may be used in the same iteration (creating
3254// "flush points"), but others remain unflushed at the backedge. When a DS
3255// read is consumed in the same iteration, it and all prior reads are
3256// "flushed" (FIFO order). No DS writes are allowed in the loop.
3257// TODO: Find a way to extend to multi-block loops.
3258PreheaderFlushFlags
3259SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
3260 const WaitcntBrackets &Brackets) {
3261 PreheaderFlushFlags Flags;
3262 bool HasVMemLoad = false;
3263 bool HasVMemStore = false;
3264 bool UsesVgprVMEMLoadedOutside = false;
3265 bool UsesVgprDSReadOutside = false;
3266 bool VMemInvalidated = false;
3267 // DS optimization only applies to GFX12+ where DS_CNT is separate.
3268 // Tracking status for "no DS read in loop" or "pure DS prefetch
3269 // (use only in next iteration)".
3270 bool TrackSimpleDSOpt = ST.hasExtendedWaitCounts();
3271 DenseSet<MCRegUnit> VgprUse;
3272 DenseSet<MCRegUnit> VgprDefVMEM;
3273 DenseSet<MCRegUnit> VgprDefDS;
3274
3275 // Track DS reads for prefetch pattern with flush points (single-block only).
3276 // Keeps track of the last DS read (position counted from the top of the loop)
3277 // to each VGPR. Read is considered consumed (and thus needs flushing) if
3278 // the dest register has a use or is overwritten (by any later opertions).
3279 DenseMap<MCRegUnit, unsigned> LastDSReadPositionMap;
3280 unsigned DSReadPosition = 0;
3281 bool IsSingleBlock = ML->getNumBlocks() == 1;
3282 bool TrackDSFlushPoint = ST.hasExtendedWaitCounts() && IsSingleBlock;
3283 unsigned LastDSFlushPosition = 0;
3284
3285 for (MachineBasicBlock *MBB : ML->blocks()) {
3286 for (MachineInstr &MI : *MBB) {
3287 if (isVMEMOrFlatVMEM(MI)) {
3288 HasVMemLoad |= MI.mayLoad();
3289 HasVMemStore |= MI.mayStore();
3290 }
3291 // TODO: Can we relax DSStore check? There may be cases where
3292 // these DS stores are drained prior to the end of MBB (or loop).
3293 if (mayStoreIncrementingDSCNT(MI)) {
3294 // Early exit if none of the optimizations are feasible.
3295 // Otherwise, set tracking status appropriately and continue.
3296 if (VMemInvalidated)
3297 return Flags;
3298 TrackSimpleDSOpt = false;
3299 TrackDSFlushPoint = false;
3300 }
3301 bool IsDSRead = isDSRead(MI);
3302 if (IsDSRead)
3303 ++DSReadPosition;
3304
3305 // Helper: if RU has a pending DS read, update LastDSFlushPosition
3306 auto updateDSReadFlushTracking = [&](MCRegUnit RU) {
3307 if (!TrackDSFlushPoint)
3308 return;
3309 if (auto It = LastDSReadPositionMap.find(Val: RU);
3310 It != LastDSReadPositionMap.end()) {
3311 // RU defined by DSRead is used or overwritten. Need to complete
3312 // the read, if not already implied by a later DSRead (to any RU)
3313 // needing to complete in FIFO order.
3314 LastDSFlushPosition = std::max(a: LastDSFlushPosition, b: It->second);
3315 }
3316 };
3317
3318 for (const MachineOperand &Op : MI.all_uses()) {
3319 if (Op.isDebug() || !TRI.isVectorRegister(MRI, Reg: Op.getReg()))
3320 continue;
3321 // Vgpr use
3322 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3323 // If we find a register that is loaded inside the loop, 1. and 2.
3324 // are invalidated.
3325 if (VgprDefVMEM.contains(V: RU))
3326 VMemInvalidated = true;
3327
3328 // Check for DS reads used inside the loop
3329 if (VgprDefDS.contains(V: RU))
3330 TrackSimpleDSOpt = false;
3331
3332 // Early exit if all optimizations are invalidated
3333 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3334 return Flags;
3335
3336 // Check for flush points (DS read used in same iteration)
3337 updateDSReadFlushTracking(RU);
3338
3339 VgprUse.insert(V: RU);
3340 // Check if this register has a pending VMEM load from outside the
3341 // loop (value loaded outside and used inside).
3342 VMEMID ID = toVMEMID(RU);
3343 if (Brackets.hasPendingVMEM(ID, T: AMDGPU::LOAD_CNT) ||
3344 Brackets.hasPendingVMEM(ID, T: AMDGPU::SAMPLE_CNT) ||
3345 Brackets.hasPendingVMEM(ID, T: AMDGPU::BVH_CNT))
3346 UsesVgprVMEMLoadedOutside = true;
3347 // Check if loaded outside the loop via DS (not VMEM/FLAT).
3348 // Only consider it a DS read if there's no pending VMEM load for
3349 // this register, since FLAT can set both counters.
3350 else if (Brackets.hasPendingVMEM(ID, T: AMDGPU::DS_CNT))
3351 UsesVgprDSReadOutside = true;
3352 }
3353 }
3354
3355 // VMem load vgpr def
3356 if (isVMEMOrFlatVMEM(MI) && MI.mayLoad()) {
3357 for (const MachineOperand &Op : MI.all_defs()) {
3358 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3359 // If we find a register that is loaded inside the loop, 1. and 2.
3360 // are invalidated.
3361 if (VgprUse.contains(V: RU))
3362 VMemInvalidated = true;
3363 VgprDefVMEM.insert(V: RU);
3364 }
3365 }
3366 // Early exit if all optimizations are invalidated
3367 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3368 return Flags;
3369 }
3370
3371 // DS read vgpr def
3372 // Note: Unlike VMEM, we DON'T invalidate when VgprUse.contains(RegNo).
3373 // If USE comes before DEF, it's the prefetch pattern (use value from
3374 // previous iteration, read for next iteration). We should still flush
3375 // in preheader so iteration 1 doesn't need to wait inside the loop.
3376 // Only invalidate when DEF comes before USE (same-iteration consumption,
3377 // checked above when processing uses).
3378 if (IsDSRead || TrackDSFlushPoint) {
3379 for (const MachineOperand &Op : MI.all_defs()) {
3380 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg()))
3381 continue;
3382 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3383 // Check for overwrite of pending DS read (flush point) by any
3384 // instruction
3385 updateDSReadFlushTracking(RU);
3386 if (IsDSRead) {
3387 VgprDefDS.insert(V: RU);
3388 if (TrackDSFlushPoint)
3389 LastDSReadPositionMap[RU] = DSReadPosition;
3390 }
3391 }
3392 }
3393 }
3394 }
3395 }
3396
3397 // VMEM flush decision
3398 if (!VMemInvalidated && UsesVgprVMEMLoadedOutside &&
3399 ((!ST.hasVscnt() && HasVMemStore && !HasVMemLoad) ||
3400 (HasVMemLoad && ST.hasVmemWriteVgprInOrder())))
3401 Flags.FlushVmCnt = true;
3402
3403 // DS flush decision:
3404 // Simple DS Opt: flush if loop uses DS read values from outside
3405 // and either has no DS reads in the loop, or DS reads whose results
3406 // are not used in the loop.
3407 bool SimpleDSOpt = TrackSimpleDSOpt && UsesVgprDSReadOutside;
3408 // Prefetch with flush points: some DS reads used in same iteration,
3409 // but unflushed reads remain at backedge
3410 bool HasUnflushedDSReads = DSReadPosition > LastDSFlushPosition;
3411 bool DSFlushPointPrefetch =
3412 TrackDSFlushPoint && UsesVgprDSReadOutside && HasUnflushedDSReads;
3413
3414 if (SimpleDSOpt || DSFlushPointPrefetch)
3415 Flags.FlushDsCnt = true;
3416
3417 return Flags;
3418}
3419
3420bool SIInsertWaitcntsLegacy::runOnMachineFunction(MachineFunction &MF) {
3421 auto &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
3422 auto &PDT =
3423 getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
3424 AliasAnalysis *AA = nullptr;
3425 if (auto *AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
3426 AA = &AAR->getAAResults();
3427
3428 return SIInsertWaitcnts(MLI, PDT, AA, MF).run();
3429}
3430
3431PreservedAnalyses
3432SIInsertWaitcntsPass::run(MachineFunction &MF,
3433 MachineFunctionAnalysisManager &MFAM) {
3434 auto &MLI = MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
3435 auto &PDT = MFAM.getResult<MachinePostDominatorTreeAnalysis>(IR&: MF);
3436 auto *AA = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
3437 .getManager()
3438 .getCachedResult<AAManager>(IR&: MF.getFunction());
3439
3440 if (!SIInsertWaitcnts(MLI, PDT, AA, MF).run())
3441 return PreservedAnalyses::all();
3442
3443 return getMachineFunctionPassPreservedAnalyses()
3444 .preserveSet<CFGAnalyses>()
3445 .preserve<AAManager>();
3446}
3447
3448bool SIInsertWaitcnts::run() {
3449 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3450
3451 AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: ST.getCPU());
3452
3453 // Initialize hardware limits first, as they're needed by the generators.
3454 Limits = AMDGPU::HardwareLimits(IV);
3455
3456 if (ST.hasExtendedWaitCounts()) {
3457 IsExpertMode = ST.hasExpertSchedulingMode() &&
3458 (ExpertSchedulingModeFlag.getNumOccurrences()
3459 ? ExpertSchedulingModeFlag
3460 : MF.getFunction()
3461 .getFnAttribute(Kind: "amdgpu-expert-scheduling-mode")
3462 .getValueAsBool());
3463 MaxCounter = IsExpertMode ? AMDGPU::NUM_EXPERT_INST_CNTS
3464 : AMDGPU::NUM_EXTENDED_INST_CNTS;
3465 // Initialize WCG per MF. It contains state that depends on MF attributes.
3466 WCG = std::make_unique<WaitcntGeneratorGFX12Plus>(args&: MF, args&: MaxCounter, args&: Limits,
3467 args&: IsExpertMode);
3468 } else {
3469 MaxCounter = AMDGPU::NUM_NORMAL_INST_CNTS;
3470 // Initialize WCG per MF. It contains state that depends on MF attributes.
3471 WCG = std::make_unique<WaitcntGeneratorPreGFX12>(
3472 args&: MF, args: AMDGPU::NUM_NORMAL_INST_CNTS, args&: Limits);
3473 }
3474
3475 SmemAccessCounter = getCounterFromEvent(E: HWEvents::SMEM_ACCESS);
3476
3477 bool Modified = false;
3478
3479 MachineBasicBlock &EntryBB = MF.front();
3480
3481 if (!MFI->isEntryFunction() &&
3482 !MF.getFunction().hasFnAttribute(Kind: Attribute::Naked)) {
3483 // Wait for any outstanding memory operations that the input registers may
3484 // depend on. We can't track them and it's better to do the wait after the
3485 // costly call sequence.
3486
3487 // TODO: Could insert earlier and schedule more liberally with operations
3488 // that only use caller preserved registers.
3489 MachineBasicBlock::iterator I = EntryBB.begin();
3490 while (I != EntryBB.end() && I->isMetaInstruction())
3491 ++I;
3492
3493 if (ST.hasExtendedWaitCounts()) {
3494 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAIT_LOADCNT_DSCNT))
3495 .addImm(Val: 0);
3496 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
3497 if (CT == AMDGPU::LOAD_CNT || CT == AMDGPU::DS_CNT ||
3498 CT == AMDGPU::STORE_CNT || CT == AMDGPU::X_CNT ||
3499 CT == AMDGPU::ASYNC_CNT || CT == AMDGPU::TENSOR_CNT)
3500 continue;
3501
3502 if (!ST.hasImageInsts() &&
3503 (CT == AMDGPU::EXP_CNT || CT == AMDGPU::SAMPLE_CNT ||
3504 CT == AMDGPU::BVH_CNT))
3505 continue;
3506
3507 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(),
3508 MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
3509 .addImm(Val: 0);
3510 }
3511 if (IsExpertMode) {
3512 unsigned Enc = AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST);
3513 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: 0);
3514 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
3515 .addImm(Val: Enc);
3516 }
3517 } else {
3518 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: 0);
3519 }
3520
3521 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(args: this);
3522 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
3523 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
3524
3525 Modified = true;
3526 }
3527
3528 // Keep iterating over the blocks in reverse post order, inserting and
3529 // updating s_waitcnt where needed, until a fix point is reached.
3530 for (auto *MBB : ReversePostOrderTraversal<MachineFunction *>(&MF))
3531 BlockInfos.try_emplace(Key: MBB);
3532
3533 std::unique_ptr<WaitcntBrackets> Brackets;
3534 bool Repeat;
3535 do {
3536 Repeat = false;
3537
3538 for (auto BII = BlockInfos.begin(), BIE = BlockInfos.end(); BII != BIE;
3539 ++BII) {
3540 MachineBasicBlock *MBB = BII->first;
3541 BlockInfo &BI = BII->second;
3542 if (!BI.Dirty)
3543 continue;
3544
3545 if (BI.Incoming) {
3546 if (!Brackets)
3547 Brackets = std::make_unique<WaitcntBrackets>(args&: *BI.Incoming);
3548 else
3549 *Brackets = *BI.Incoming;
3550 } else {
3551 if (!Brackets) {
3552 Brackets = std::make_unique<WaitcntBrackets>(args: this);
3553 } else {
3554 // Reinitialize in-place. N.B. do not do this by assigning from a
3555 // temporary because the WaitcntBrackets class is large and it could
3556 // cause this function to use an unreasonable amount of stack space.
3557 Brackets->~WaitcntBrackets();
3558 new (Brackets.get()) WaitcntBrackets(this);
3559 }
3560 }
3561
3562 if (ST.hasWaitXcnt())
3563 Modified |= removeRedundantSoftXcnts(Block&: *MBB);
3564 Modified |= insertWaitcntInBlock(MF, Block&: *MBB, ScoreBrackets&: *Brackets);
3565 BI.Dirty = false;
3566
3567 if (Brackets->hasPendingEvent()) {
3568 BlockInfo *MoveBracketsToSucc = nullptr;
3569 for (MachineBasicBlock *Succ : MBB->successors()) {
3570 auto *SuccBII = BlockInfos.find(Key: Succ);
3571 BlockInfo &SuccBI = SuccBII->second;
3572 if (!SuccBI.Incoming) {
3573 SuccBI.Dirty = true;
3574 if (SuccBII <= BII) {
3575 LLVM_DEBUG(dbgs() << "Repeat on backedge without merge\n");
3576 Repeat = true;
3577 }
3578 if (!MoveBracketsToSucc) {
3579 MoveBracketsToSucc = &SuccBI;
3580 } else {
3581 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(args&: *Brackets);
3582 }
3583 } else {
3584 LLVM_DEBUG({
3585 dbgs() << "Try to merge ";
3586 MBB->printName(dbgs());
3587 dbgs() << " into ";
3588 Succ->printName(dbgs());
3589 dbgs() << '\n';
3590 });
3591 if (SuccBI.Incoming->merge(Other: *Brackets)) {
3592 SuccBI.Dirty = true;
3593 if (SuccBII <= BII) {
3594 LLVM_DEBUG(dbgs() << "Repeat on backedge with merge\n");
3595 Repeat = true;
3596 }
3597 }
3598 }
3599 }
3600 if (MoveBracketsToSucc)
3601 MoveBracketsToSucc->Incoming = std::move(Brackets);
3602 }
3603 }
3604 } while (Repeat);
3605
3606 if (ST.hasScalarStores()) {
3607 SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;
3608 bool HaveScalarStores = false;
3609
3610 for (MachineBasicBlock &MBB : MF) {
3611 for (MachineInstr &MI : MBB) {
3612 if (!HaveScalarStores && TII.isScalarStore(MI))
3613 HaveScalarStores = true;
3614
3615 if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
3616 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
3617 EndPgmBlocks.push_back(Elt: &MBB);
3618 }
3619 }
3620
3621 if (HaveScalarStores) {
3622 // If scalar writes are used, the cache must be flushed or else the next
3623 // wave to reuse the same scratch memory can be clobbered.
3624 //
3625 // Insert s_dcache_wb at wave termination points if there were any scalar
3626 // stores, and only if the cache hasn't already been flushed. This could
3627 // be improved by looking across blocks for flushes in postdominating
3628 // blocks from the stores but an explicitly requested flush is probably
3629 // very rare.
3630 for (MachineBasicBlock *MBB : EndPgmBlocks) {
3631 bool SeenDCacheWB = false;
3632
3633 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
3634 I != E; ++I) {
3635 if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
3636 SeenDCacheWB = true;
3637 else if (TII.isScalarStore(MI: *I))
3638 SeenDCacheWB = false;
3639
3640 // FIXME: It would be better to insert this before a waitcnt if any.
3641 if ((I->getOpcode() == AMDGPU::S_ENDPGM ||
3642 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
3643 !SeenDCacheWB) {
3644 Modified = true;
3645 BuildMI(BB&: *MBB, I, MIMD: I->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_DCACHE_WB));
3646 }
3647 }
3648 }
3649 }
3650 }
3651
3652 if (IsExpertMode) {
3653 // Enable expert scheduling on function entry. To satisfy ABI requirements
3654 // and to allow calls between function with different expert scheduling
3655 // settings, disable it around calls and before returns.
3656
3657 MachineBasicBlock::iterator I = EntryBB.begin();
3658 while (I != EntryBB.end() && I->isMetaInstruction())
3659 ++I;
3660 setSchedulingMode(MBB&: EntryBB, I, ExpertMode: true);
3661
3662 for (MachineInstr *MI : CallInsts) {
3663 MachineBasicBlock &MBB = *MI->getParent();
3664 setSchedulingMode(MBB, I: MI, ExpertMode: false);
3665 setSchedulingMode(MBB, I: std::next(x: MI->getIterator()), ExpertMode: true);
3666 }
3667
3668 for (MachineInstr *MI : ReturnInsts)
3669 setSchedulingMode(MBB&: *MI->getParent(), I: MI, ExpertMode: false);
3670
3671 Modified = true;
3672 }
3673
3674 // Deallocate the VGPRs before previously identified S_ENDPGM instructions.
3675 // This is done in different ways depending on how the VGPRs were allocated
3676 // (i.e. whether we're in dynamic VGPR mode or not).
3677 // Skip deallocation if kernel is waveslot limited vs VGPR limited. A short
3678 // waveslot limited kernel runs slower with the deallocation.
3679 if (!WCG->isOptNone() && MFI->isDynamicVGPREnabled()) {
3680 for (auto [MI, _] : EndPgmInsts) {
3681 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3682 MCID: TII.get(Opcode: AMDGPU::S_ALLOC_VGPR))
3683 .addImm(Val: 0);
3684 Modified = true;
3685 }
3686 } else if (!WCG->isOptNone() &&
3687 ST.getGeneration() >= AMDGPUSubtarget::GFX11 &&
3688 (MF.getFrameInfo().hasCalls() ||
3689 ST.getOccupancyWithNumVGPRs(
3690 VGPRs: TRI.getNumUsedPhysRegs(MRI, RC: AMDGPU::VGPR_32RegClass),
3691 /*IsDynamicVGPR=*/DynamicVGPRBlockSize: false) < ST.getMaxWavesPerEU())) {
3692 for (auto [MI, Flag] : EndPgmInsts) {
3693 if (Flag) {
3694 if (ST.requiresNopBeforeDeallocVGPRs()) {
3695 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3696 MCID: TII.get(Opcode: AMDGPU::S_NOP))
3697 .addImm(Val: 0);
3698 }
3699 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3700 MCID: TII.get(Opcode: AMDGPU::S_SENDMSG))
3701 .addImm(Val: AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus);
3702 Modified = true;
3703 }
3704 }
3705 }
3706
3707 if (MFI->isEntryFunction()) {
3708 MachineBasicBlock::iterator InsertPt = EntryBB.begin();
3709
3710 if (ST.hasWaitXcnt()) {
3711 // Set REPLAY_MODE (bit 25) in MODE register to enable multi-group XNACK
3712 // replay. This aligns hardware behavior with the compiler's s_wait_xcnt
3713 // insertion logic, which assumes multi-group mode by default.
3714 unsigned RegEncoding =
3715 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 25, Values: 1);
3716 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(),
3717 MCID: TII.get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
3718 .addImm(Val: 1)
3719 .addImm(Val: RegEncoding);
3720 Modified = true;
3721 }
3722
3723 if (ST.hasRequiresInitialUnclausedVmem()) {
3724 // Hardware entrypoints must begin with a specific sequence:
3725 // S_MOV_B64 S[64:65], 0
3726 // V_NOP
3727 // GLOBAL_PREFETCH_B8 V0, S[64:65] SCOPE:SCOPE_SE TH:TH_LOAD_RT
3728 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B64),
3729 DestReg: AMDGPU::SGPR64_SGPR65)
3730 .addImm(Val: 0);
3731 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
3732 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(),
3733 MCID: TII.get(Opcode: AMDGPU::GLOBAL_PREFETCH_B8_SADDR))
3734 .addReg(RegNo: AMDGPU::SGPR64_SGPR65)
3735 .addReg(RegNo: AMDGPU::VGPR0, Flags: RegState::Undef)
3736 .addImm(Val: 0)
3737 .addImm(Val: AMDGPU::CPol::SCOPE_SE | AMDGPU::CPol::TH_RT);
3738 Modified = true;
3739 }
3740 }
3741
3742 return Modified;
3743}
3744