1//===- SIInsertWaitcnts.cpp - Insert Wait Instructions --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert wait instructions for memory reads and writes.
11///
12/// Memory reads and writes are issued asynchronously, so we need to insert
13/// S_WAITCNT instructions when we want to access any of their results or
14/// overwrite any register that's used asynchronously.
15///
16/// TODO: This pass currently keeps one timeline per hardware counter. A more
17/// finely-grained approach that keeps one timeline per event type could
18/// sometimes get away with generating weaker s_waitcnt instructions. For
19/// example, when both SMEM and LDS are in flight and we need to wait for
20/// the i-th-last LDS instruction, then an lgkmcnt(i) is actually sufficient,
21/// but the pass will currently generate a conservative lgkmcnt(0) because
22/// multiple event types are in flight.
23//
24//===----------------------------------------------------------------------===//
25
26#include "AMDGPU.h"
27#include "AMDGPUHWEvents.h"
28#include "AMDGPUWaitcntUtils.h"
29#include "GCNSubtarget.h"
30#include "SIMachineFunctionInfo.h"
31#include "Utils/AMDGPUBaseInfo.h"
32#include "llvm/Analysis/AliasAnalysis.h"
33#include "llvm/CodeGen/MachineFrameInfo.h"
34#include "llvm/CodeGen/MachineLoopInfo.h"
35#include "llvm/CodeGen/MachinePassManager.h"
36#include "llvm/CodeGen/MachinePostDominators.h"
37#include "llvm/IR/Dominators.h"
38#include "llvm/InitializePasses.h"
39#include "llvm/Support/AMDGPUAsyncStages.h"
40#include "llvm/TargetParser/AMDGPUTargetParser.h"
41
42using namespace llvm;
43
44using HWEvents = AMDGPU::HWEvents;
45
46#define DEBUG_TYPE "si-insert-waitcnts"
47
48namespace {
49
50template <typename EmitWaitcntFn>
51static void EmitExpandedWaitcnt(unsigned Outstanding, unsigned Target,
52 EmitWaitcntFn &&EmitWaitcnt) {
53 // Emit waitcnts from (Outstanding - 1) down to Target.
54 for (unsigned I = Outstanding - 1; I > Target && I != ~0u; --I)
55 EmitWaitcnt(I);
56 EmitWaitcnt(Target);
57}
58
59/// Integer IDs used to track vector memory locations we may have to wait on.
60/// Encoded as u16 chunks:
61///
62/// [0, REGUNITS_END ): MCRegUnit
63/// [LDSDMA_BEGIN, LDSDMA_END ) : LDS DMA IDs
64///
65/// NOTE: The choice of encoding these as "u16 chunks" is arbitrary.
66/// It gives (2 << 16) - 1 entries per category which is more than enough
67/// for all register units. MCPhysReg is u16 so we don't even support >u16
68/// physical register numbers at this time, let alone >u16 register units.
69/// In any case, an assertion in "WaitcntBrackets" ensures REGUNITS_END
70/// is enough for all register units.
71using VMEMID = uint32_t;
72
73enum : VMEMID {
74 TRACKINGID_RANGE_LEN = (1 << 16),
75
76 // Important: MCRegUnits must always be tracked starting from 0, as we
77 // need to be able to convert between a MCRegUnit and a VMEMID freely.
78 REGUNITS_BEGIN = 0,
79 REGUNITS_END = REGUNITS_BEGIN + TRACKINGID_RANGE_LEN,
80
81 // Note for LDSDMA: LDSDMA_BEGIN corresponds to the "common"
82 // entry, which is updated for all LDS DMA operations encountered.
83 // Specific LDS DMA IDs start at LDSDMA_BEGIN + 1.
84 NUM_LDSDMA = TRACKINGID_RANGE_LEN,
85 LDSDMA_BEGIN = REGUNITS_END,
86 LDSDMA_END = LDSDMA_BEGIN + NUM_LDSDMA,
87};
88
89/// Convert a MCRegUnit to a VMEMID.
90static constexpr VMEMID toVMEMID(MCRegUnit RU) {
91 return static_cast<unsigned>(RU);
92}
93
94} // namespace
95
96namespace {
97
98// Maps values of InstCounterType to the instruction that waits on that
99// counter. Only used if GCNSubtarget::hasExtendedWaitCounts()
100// returns true, and does not cover VA_VDST or VM_VSRC.
101static const unsigned
102 instrsForExtendedCounterTypes[AMDGPU::NUM_EXTENDED_INST_CNTS] = {
103 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT,
104 AMDGPU::S_WAIT_EXPCNT, AMDGPU::S_WAIT_STORECNT,
105 AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
106 AMDGPU::S_WAIT_KMCNT, AMDGPU::S_WAIT_XCNT,
107 AMDGPU::S_WAIT_ASYNCCNT, AMDGPU::S_WAIT_TENSORCNT};
108
109// ASYNCMARK and WAIT_ASYNCMARK are meta instructions that emit no hardware
110// code but still need to be processed by this pass for async vmcnt tracking.
111static bool isNonWaitcntMetaInst(const MachineInstr &MI) {
112 switch (MI.getOpcode()) {
113 case AMDGPU::ASYNCMARK:
114 case AMDGPU::WAIT_ASYNCMARK:
115 return false;
116 default:
117 return MI.isMetaInstruction();
118 }
119}
120
121static bool updateVMCntOnly(const MachineInstr &Inst) {
122 return (SIInstrInfo::isVMEM(MI: Inst) && !SIInstrInfo::isFLAT(MI: Inst)) ||
123 SIInstrInfo::isFLATGlobal(MI: Inst) || SIInstrInfo::isFLATScratch(MI: Inst);
124}
125
126#ifndef NDEBUG
127static bool isNormalMode(AMDGPU::InstCounterType MaxCounter) {
128 return MaxCounter == AMDGPU::NUM_NORMAL_INST_CNTS;
129}
130#endif // NDEBUG
131
132class WaitcntBrackets;
133
134// This abstracts the logic for generating and updating S_WAIT* instructions
135// away from the analysis that determines where they are needed. This was
136// done because the set of counters and instructions for waiting on them
137// underwent a major shift with gfx12, sufficiently so that having this
138// abstraction allows the main analysis logic to be simpler than it would
139// otherwise have had to become.
140class WaitcntGenerator {
141protected:
142 const GCNSubtarget &ST;
143 const SIInstrInfo &TII;
144 AMDGPU::IsaVersion IV;
145 AMDGPU::InstCounterType MaxCounter;
146 bool OptNone;
147 bool ExpandWaitcntProfiling = false;
148 const AMDGPU::HardwareLimits &Limits;
149
150 // Soft waitcnts whose erasures are postponed so we can confirm if they're
151 // truly redundant or if they're waiting on a loop-carried dependency.
152 DenseSet<MachineInstr *> DeferredErasures;
153
154public:
155 WaitcntGenerator() = delete;
156 WaitcntGenerator(const WaitcntGenerator &) = delete;
157 WaitcntGenerator(const MachineFunction &MF,
158 AMDGPU::InstCounterType MaxCounter,
159 const AMDGPU::HardwareLimits &Limits)
160 : ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
161 IV(AMDGPU::getIsaVersion(GPU: ST.getCPU())), MaxCounter(MaxCounter),
162 OptNone(MF.getFunction().hasOptNone() ||
163 MF.getTarget().getOptLevel() == CodeGenOptLevel::None),
164 ExpandWaitcntProfiling(
165 MF.getFunction().hasFnAttribute(Kind: "amdgpu-expand-waitcnt-profiling")),
166 Limits(Limits) {}
167
168 // Return true if the current function should be compiled with no
169 // optimization.
170 bool isOptNone() const { return OptNone; }
171
172 unsigned getLimit(AMDGPU::InstCounterType E) const { return Limits.get(T: E); }
173
174 // Edits an existing sequence of wait count instructions according
175 // to an incoming Waitcnt value, which is itself updated to reflect
176 // any new wait count instructions which may need to be generated by
177 // WaitcntGenerator::createNewWaitcnt(). It will return true if any edits
178 // were made.
179 //
180 // This editing will usually be merely updated operands, but it may also
181 // delete instructions if the incoming Wait value indicates they are not
182 // needed. It may also remove existing instructions for which a wait
183 // is needed if it can be determined that it is better to generate new
184 // instructions later, as can happen on gfx12.
185 virtual bool
186 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
187 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
188 MachineBasicBlock::instr_iterator It) = 0;
189
190 bool eraseDeferredWaitcnts() {
191 bool Modified = false;
192 for (MachineInstr *MI : DeferredErasures) {
193 MI->eraseFromParent();
194 Modified = true;
195 }
196 DeferredErasures.clear();
197 return Modified;
198 }
199
200 // Transform a soft waitcnt into a normal one.
201 bool promoteSoftWaitCnt(MachineInstr *Waitcnt) const;
202
203 // Generates new wait count instructions according to the value of
204 // Wait, returning true if any new instructions were created.
205 // ScoreBrackets is used for profiling expansion.
206 virtual bool createNewWaitcnt(MachineBasicBlock &Block,
207 MachineBasicBlock::instr_iterator It,
208 AMDGPU::Waitcnt Wait,
209 const WaitcntBrackets &ScoreBrackets) = 0;
210
211 // Returns the set of HWEvents that corresponds to counter \p T.
212 virtual HWEvents getWaitEvents(AMDGPU::InstCounterType T) const = 0;
213
214 /// \returns the counter that corresponds to event \p E.
215 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
216 assert(E.size() == 1 && "Cannot handle a mask of events!");
217 for (auto T : AMDGPU::inst_counter_types()) {
218 if (getWaitEvents(T) & E)
219 return T;
220 }
221 llvm_unreachable("event type has no associated counter");
222 }
223
224 // Returns a new waitcnt with all counters except VScnt set to 0. If
225 // IncludeVSCnt is true, VScnt is set to 0, otherwise it is set to ~0u.
226 // AsyncCnt and TensorCnt always default to ~0u (don't wait for it). They
227 // are only updated when a call to @llvm.amdgcn.wait.asyncmark() is
228 // processed.
229 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const = 0;
230
231 virtual ~WaitcntGenerator() = default;
232};
233
234class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
235 static constexpr const HWEvents
236 WaitEventMaskForInstWithoutVscnt[AMDGPU::NUM_INST_CNTS] = {
237 // LOAD_CNT
238 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_SAMPLER_READ_ACCESS |
239 HWEvents::VMEM_BVH_READ_ACCESS | HWEvents::VMEM_INV_ACCESS |
240 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
241 // DS_CNT
242 HWEvents::SMEM_ACCESS | HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS |
243 HWEvents::SQ_MESSAGE,
244 // EXP_CNT
245 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
246 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
247 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
248 HWEvents::NONE, HWEvents::NONE, HWEvents::NONE, HWEvents::NONE,
249 HWEvents::NONE, HWEvents::NONE, HWEvents::NONE, HWEvents::NONE,
250 HWEvents::NONE, HWEvents::NONE};
251
252 // Hardware with VScnt has non-empty STORE_CNT wait events.
253 static constexpr const HWEvents
254 WaitEventMaskForInstWithVscnt[AMDGPU::NUM_INST_CNTS] = {
255 // LOAD_CNT
256 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_SAMPLER_READ_ACCESS |
257 HWEvents::VMEM_BVH_READ_ACCESS | HWEvents::VMEM_INV_ACCESS,
258 // DS_CNT
259 HWEvents::SMEM_ACCESS | HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS |
260 HWEvents::SQ_MESSAGE,
261 // EXP_CNT
262 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
263 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
264 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
265 // STORE_CNT
266 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
267 HWEvents::NONE, HWEvents::NONE, HWEvents::NONE, HWEvents::NONE,
268 HWEvents::NONE, HWEvents::NONE, HWEvents::NONE, HWEvents::NONE,
269 HWEvents::NONE};
270
271public:
272 using WaitcntGenerator::WaitcntGenerator;
273 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
274 MachineInstr &OldWaitcntInstr,
275 AMDGPU::Waitcnt &Wait,
276 MachineBasicBlock::instr_iterator It) override;
277
278 bool createNewWaitcnt(MachineBasicBlock &Block,
279 MachineBasicBlock::instr_iterator It,
280 AMDGPU::Waitcnt Wait,
281 const WaitcntBrackets &ScoreBrackets) override;
282
283 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
284 return ST.hasVscnt() ? WaitEventMaskForInstWithVscnt[T]
285 : WaitEventMaskForInstWithoutVscnt[T];
286 }
287
288 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
289};
290
291class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
292protected:
293 bool IsExpertMode;
294 static constexpr const HWEvents
295 WaitEventMaskForInstGFX12Plus[AMDGPU::NUM_INST_CNTS] = {
296 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_INV_ACCESS,
297 HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS,
298 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
299 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
300 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
301
302 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
303 HWEvents::VMEM_SAMPLER_READ_ACCESS,
304 HWEvents::VMEM_BVH_READ_ACCESS,
305
306 HWEvents::SMEM_ACCESS | HWEvents::SQ_MESSAGE | HWEvents::SCC_WRITE,
307 HWEvents::VMEM_GROUP | HWEvents::SMEM_GROUP,
308 HWEvents::ASYNC_ACCESS,
309 HWEvents::TENSOR_ACCESS,
310 HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_DPMACC_READ |
311 HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_XDL_READ,
312 HWEvents::VGPR_CSMACC_WRITE | HWEvents::VGPR_DPMACC_WRITE |
313 HWEvents::VGPR_TRANS_WRITE | HWEvents::VGPR_XDL_WRITE,
314 HWEvents::VGPR_LDS_READ | HWEvents::VGPR_FLAT_READ |
315 HWEvents::VGPR_VMEM_READ};
316
317public:
318 WaitcntGeneratorGFX12Plus() = delete;
319 WaitcntGeneratorGFX12Plus(const MachineFunction &MF,
320 AMDGPU::InstCounterType MaxCounter,
321 const AMDGPU::HardwareLimits &Limits,
322 bool IsExpertMode)
323 : WaitcntGenerator(MF, MaxCounter, Limits), IsExpertMode(IsExpertMode) {}
324
325 bool applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
326 MachineInstr &OldWaitcntInstr,
327 AMDGPU::Waitcnt &Wait,
328 MachineBasicBlock::instr_iterator It) override;
329
330 bool createNewWaitcnt(MachineBasicBlock &Block,
331 MachineBasicBlock::instr_iterator It,
332 AMDGPU::Waitcnt Wait,
333 const WaitcntBrackets &ScoreBrackets) override;
334
335 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
336 return WaitEventMaskForInstGFX12Plus[T];
337 }
338
339 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
340};
341
342// Flags indicating which counters should be flushed in a loop preheader.
343struct PreheaderFlushFlags {
344 bool FlushVmCnt = false;
345 bool FlushDsCnt = false;
346};
347
348class SIInsertWaitcnts {
349 DenseMap<const Value *, MachineBasicBlock *> SLoadAddresses;
350 DenseMap<MachineBasicBlock *, PreheaderFlushFlags> PreheadersToFlush;
351 MachineLoopInfo &MLI;
352 MachinePostDominatorTree &PDT;
353 AliasAnalysis *AA = nullptr;
354 MachineFunction &MF;
355
356 struct BlockInfo {
357 std::unique_ptr<WaitcntBrackets> Incoming;
358 bool Dirty = true;
359 BlockInfo() = default;
360 BlockInfo(BlockInfo &&) = default;
361 BlockInfo &operator=(BlockInfo &&) = default;
362 ~BlockInfo();
363 };
364
365 MapVector<MachineBasicBlock *, BlockInfo> BlockInfos;
366
367 bool ForceEmitWaitcnt[AMDGPU::NUM_INST_CNTS] = {};
368
369 std::unique_ptr<WaitcntGenerator> WCG;
370
371 // Remember call and return instructions in the function.
372 DenseSet<MachineInstr *> CallInsts;
373 DenseSet<MachineInstr *> ReturnInsts;
374
375 // Remember all S_ENDPGM instructions. The boolean flag is true if there might
376 // be outstanding stores but definitely no outstanding scratch stores, to help
377 // with insertion of DEALLOC_VGPRS messages.
378 DenseMap<MachineInstr *, bool> EndPgmInsts;
379
380 AMDGPU::HardwareLimits Limits;
381
382public:
383 const GCNSubtarget &ST;
384 const SIInstrInfo &TII;
385 const SIRegisterInfo &TRI;
386 const MachineRegisterInfo &MRI;
387 AMDGPU::InstCounterType SmemAccessCounter;
388 AMDGPU::InstCounterType MaxCounter;
389 bool IsExpertMode = false;
390 const bool TgSplit;
391
392 SIInsertWaitcnts(MachineLoopInfo &MLI, MachinePostDominatorTree &PDT,
393 AliasAnalysis *AA, MachineFunction &MF)
394 : MLI(MLI), PDT(PDT), AA(AA), MF(MF), ST(MF.getSubtarget<GCNSubtarget>()),
395 TII(*ST.getInstrInfo()), TRI(TII.getRegisterInfo()),
396 MRI(MF.getRegInfo()),
397 TgSplit(ST.hasTgSplitSupport() &&
398 AMDGPU::isTgSplitEnabled(F: MF.getFunction())) {}
399
400 const AMDGPU::HardwareLimits &getLimits() const { return Limits; }
401
402 PreheaderFlushFlags getPreheaderFlushFlags(MachineLoop *ML,
403 const WaitcntBrackets &Brackets);
404 PreheaderFlushFlags isPreheaderToFlush(MachineBasicBlock &MBB,
405 const WaitcntBrackets &ScoreBrackets);
406 bool isVMEMOrFlatVMEM(const MachineInstr &MI) const;
407 bool isDSRead(const MachineInstr &MI) const;
408 bool mayStoreIncrementingDSCNT(const MachineInstr &MI) const;
409 bool run();
410
411 bool isVmemAccess(const MachineInstr &MI) const;
412 bool generateWaitcntInstBefore(MachineInstr &MI,
413 WaitcntBrackets &ScoreBrackets,
414 MachineInstr *OldWaitcntInstr,
415 PreheaderFlushFlags FlushFlags);
416 bool generateWaitcnt(AMDGPU::Waitcnt Wait,
417 MachineBasicBlock::instr_iterator It,
418 MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
419 MachineInstr *OldWaitcntInstr);
420 void updateEventWaitcntAfter(MachineInstr &Inst,
421 WaitcntBrackets *ScoreBrackets);
422 bool isNextENDPGM(MachineBasicBlock::instr_iterator It,
423 MachineBasicBlock *Block) const;
424 bool insertForcedWaitAfter(MachineInstr &Inst, MachineBasicBlock &Block,
425 WaitcntBrackets &ScoreBrackets);
426 bool insertWaitcntInBlock(MachineFunction &MF, MachineBasicBlock &Block,
427 WaitcntBrackets &ScoreBrackets);
428 /// Removes redundant Soft Xcnt Waitcnts in \p Block emitted by the Memory
429 /// Legalizer. Returns true if block was modified.
430 bool removeRedundantSoftXcnts(MachineBasicBlock &Block);
431 void setSchedulingMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
432 bool ExpertMode) const;
433 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const {
434 return WCG->getWaitEvents(T);
435 }
436 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
437 return WCG->getCounterFromEvent(E);
438 }
439};
440
441// This objects maintains the current score brackets of each wait counter, and
442// a per-register scoreboard for each wait counter.
443//
444// We also maintain the latest score for every event type that can change the
445// waitcnt in order to know if there are multiple types of events within
446// the brackets. When multiple types of event happen in the bracket,
447// wait count may get decreased out of order, therefore we need to put in
448// "s_waitcnt 0" before use.
449class WaitcntBrackets {
450public:
451 WaitcntBrackets(const SIInsertWaitcnts *Context) : Context(Context) {
452 assert(Context->TRI.getNumRegUnits() < REGUNITS_END);
453 }
454
455#ifndef NDEBUG
456 ~WaitcntBrackets() {
457 unsigned NumUnusedVmem = 0, NumUnusedSGPRs = 0;
458 for (auto &[ID, Val] : VMem) {
459 if (Val.empty())
460 ++NumUnusedVmem;
461 }
462 for (auto &[ID, Val] : SGPRs) {
463 if (Val.empty())
464 ++NumUnusedSGPRs;
465 }
466
467 if (NumUnusedVmem || NumUnusedSGPRs) {
468 errs() << "WaitcntBracket had unused entries at destruction time: "
469 << NumUnusedVmem << " VMem and " << NumUnusedSGPRs
470 << " SGPR unused entries\n";
471 std::abort();
472 }
473 }
474#endif
475
476 bool isSmemCounter(AMDGPU::InstCounterType T) const {
477 return T == Context->SmemAccessCounter || T == AMDGPU::X_CNT;
478 }
479
480 unsigned getOutstanding(AMDGPU::InstCounterType T) const {
481 return ScoreUBs[T] - ScoreLBs[T];
482 }
483
484 bool hasPendingVMEM(VMEMID ID, AMDGPU::InstCounterType T) const {
485 return getVMemScore(TID: ID, T) > getScoreLB(T);
486 }
487
488 /// \Return true if we have no score entries for counter \p T.
489 bool empty(AMDGPU::InstCounterType T) const { return getScoreRange(T) == 0; }
490
491private:
492 unsigned getScoreLB(AMDGPU::InstCounterType T) const {
493 assert(T < AMDGPU::NUM_INST_CNTS);
494 return ScoreLBs[T];
495 }
496
497 unsigned getScoreUB(AMDGPU::InstCounterType T) const {
498 assert(T < AMDGPU::NUM_INST_CNTS);
499 return ScoreUBs[T];
500 }
501
502 unsigned getScoreRange(AMDGPU::InstCounterType T) const {
503 return getScoreUB(T) - getScoreLB(T);
504 }
505
506 unsigned getSGPRScore(MCRegUnit RU, AMDGPU::InstCounterType T) const {
507 auto It = SGPRs.find(Val: RU);
508 return It != SGPRs.end() ? It->second.get(T) : 0;
509 }
510
511 unsigned getVMemScore(VMEMID TID, AMDGPU::InstCounterType T) const {
512 auto It = VMem.find(Val: TID);
513 return It != VMem.end() ? It->second.Scores[T] : 0;
514 }
515
516 // Number of ordered XDL writes issued after the VA_VDST write tracked for
517 // TID. Only meaningful while that write is pending.
518 unsigned getNumOrderedXDLsAfterWrite(VMEMID TID) const {
519 return std::min(a: OrderedXDLCount - VMem.at(Val: TID).OrderedXDLCountAtWrite,
520 b: getLimit(T: AMDGPU::VA_VDST_WR) - 1);
521 }
522
523public:
524 // Do some internal consistency checks.
525 void verify() {
526 // These track the same underlying hardware counter so the score ranges
527 // should be identical.
528 assert(getScoreLB(AMDGPU::VA_VDST_RD) == getScoreLB(AMDGPU::VA_VDST_WR));
529 assert(getScoreUB(AMDGPU::VA_VDST_RD) == getScoreUB(AMDGPU::VA_VDST_WR));
530 }
531
532 bool merge(const WaitcntBrackets &Other);
533
534 bool counterOutOfOrder(AMDGPU::InstCounterType T) const;
535 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const {
536 simplifyWaitcnt(CheckWait: Wait, UpdateWait&: Wait);
537 }
538 void simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
539 AMDGPU::Waitcnt &UpdateWait) const;
540 void simplifyWaitcnt(AMDGPU::InstCounterType T, unsigned &Count) const;
541 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T) const;
542 void simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
543 AMDGPU::Waitcnt &UpdateWait) const;
544 void simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
545 AMDGPU::Waitcnt &UpdateWait) const;
546 // GLOBAL_INV/BUFFER_INV increment loadcnt/vmcnt but do not write VGPRs, so
547 // waits that only drain these events can be omitted.
548 void simplifyVmemInvOnlyWait(AMDGPU::Waitcnt &Wait) const {
549 // LOAD_CNT also includes store events on targets without VScnt.
550 HWEvents LoadEvents =
551 PendingEvents & Context->getWaitEvents(T: AMDGPU::LOAD_CNT);
552 if (LoadEvents == HWEvents::VMEM_INV_ACCESS)
553 Wait.clear(T: AMDGPU::LOAD_CNT);
554 }
555
556 void determineWaitForPhysReg(AMDGPU::InstCounterType T, MCRegister Reg,
557 AMDGPU::Waitcnt &Wait,
558 const MachineInstr &MI) const;
559 MCRegister determineVGPR16Dependency(const MachineInstr &MI,
560 AMDGPU::InstCounterType T,
561 MCRegister Reg) const;
562 void determineWaitForLDSDMA(AMDGPU::InstCounterType T, VMEMID TID,
563 AMDGPU::Waitcnt &Wait) const;
564 AMDGPU::Waitcnt determineAsyncWait(unsigned N, AMDGPU::AsyncStages Stages);
565 void tryClearSCCWriteEvent(MachineInstr *Inst);
566
567 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
568 void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
569 void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
570 void updateByEvent(HWEvents E, MachineInstr &MI);
571 void recordAsyncMark(MachineInstr &MI, AMDGPU::AsyncStages Stages);
572
573 HWEvents getPendingEvents() const { return PendingEvents; }
574 bool hasPendingEvent() const { return PendingEvents.any(); }
575 bool hasPendingEvent(HWEvents E) const { return PendingEvents.contains(Other: E); }
576 bool hasPendingEvent(AMDGPU::InstCounterType T) const {
577 bool HasPending = (PendingEvents & Context->getWaitEvents(T)).any();
578 assert(HasPending == !empty(T) &&
579 "Expected pending events iff scoreboard is not empty");
580 return HasPending;
581 }
582
583 bool hasMixedPendingEvents(AMDGPU::InstCounterType T) const {
584 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
585 // Return true if more than one bit is set in Events.
586 return Events.size() > 1;
587 }
588
589 bool hasPendingFlat() const {
590 return ((LastFlatDsCnt > ScoreLBs[AMDGPU::DS_CNT] &&
591 LastFlatDsCnt <= ScoreUBs[AMDGPU::DS_CNT]) ||
592 (LastFlatLoadCnt > ScoreLBs[AMDGPU::LOAD_CNT] &&
593 LastFlatLoadCnt <= ScoreUBs[AMDGPU::LOAD_CNT]));
594 }
595
596 void setPendingFlat() {
597 LastFlatLoadCnt = ScoreUBs[AMDGPU::LOAD_CNT];
598 LastFlatDsCnt = ScoreUBs[AMDGPU::DS_CNT];
599 }
600
601 bool hasPendingXcntOnlyVmem() const {
602 return LastXcntOnlyVmem > ScoreLBs[AMDGPU::X_CNT] &&
603 LastXcntOnlyVmem <= ScoreUBs[AMDGPU::X_CNT];
604 }
605
606 void setPendingXcntOnlyVmem() { LastXcntOnlyVmem = ScoreUBs[AMDGPU::X_CNT]; }
607
608 // Return true if an xcnt event not tracked by asynccnt may still be
609 // outstanding after \p Wait. Waiting for a counter to reach zero completes
610 // its operations, which also drains their xcnt.
611 bool hasPendingNonAsyncXcnt(
612 const AMDGPU::Waitcnt &Wait = AMDGPU::Waitcnt()) const {
613 if (hasPendingXcntOnlyVmem() || (hasPendingEvent(E: HWEvents::SMEM_GROUP) &&
614 Wait.get(T: AMDGPU::KM_CNT) != 0))
615 return true;
616 for (AMDGPU::InstCounterType T : {AMDGPU::LOAD_CNT, AMDGPU::STORE_CNT,
617 AMDGPU::SAMPLE_CNT, AMDGPU::BVH_CNT}) {
618 if (Wait.get(T) != 0 && hasPendingEvent(T))
619 return true;
620 }
621 return false;
622 }
623
624 bool hasPendingGDS() const {
625 return LastGDS > ScoreLBs[AMDGPU::DS_CNT] &&
626 LastGDS <= ScoreUBs[AMDGPU::DS_CNT];
627 }
628
629 unsigned getPendingGDSWait() const {
630 return std::min(a: getScoreUB(T: AMDGPU::DS_CNT) - LastGDS,
631 b: getLimit(T: AMDGPU::DS_CNT) - 1);
632 }
633
634 void setPendingGDS() { LastGDS = ScoreUBs[AMDGPU::DS_CNT]; }
635
636 // Return true if there might be pending writes to the vgpr-interval by VMEM
637 // instructions where the HWEvents in VGPRContext are not contained in E.
638 bool hasDifferentVGPRPendingEvents(MCRegister Reg, HWEvents E) const {
639 for (MCRegUnit RU : regunits(Reg)) {
640 auto It = VMem.find(Val: toVMEMID(RU));
641 if (It != VMem.end() && (It->second.VGPRPendingEvents & ~E).any())
642 return true;
643 }
644 return false;
645 }
646
647 void clearVGPRPendingEvents(MCRegister Reg) {
648 for (MCRegUnit RU : regunits(Reg)) {
649 if (auto It = VMem.find(Val: toVMEMID(RU)); It != VMem.end()) {
650 It->second.VGPRPendingEvents = HWEvents::NONE;
651 if (It->second.empty())
652 VMem.erase(I: It);
653 }
654 }
655 }
656
657 void setStateOnFunctionEntryOrReturn() {
658 setScoreUB(T: AMDGPU::STORE_CNT,
659 Val: getScoreUB(T: AMDGPU::STORE_CNT) + getLimit(T: AMDGPU::STORE_CNT));
660 PendingEvents |= Context->getWaitEvents(T: AMDGPU::STORE_CNT);
661 }
662
663 ArrayRef<const MachineInstr *> getLDSDMAStores() const {
664 return LDSDMAStores;
665 }
666
667 bool hasPointSampleAccel(const MachineInstr &MI) const;
668 bool hasPointSamplePendingVmemTypes(const MachineInstr &MI,
669 MCRegister RU) const;
670
671 void print(raw_ostream &) const;
672 void dump() const { print(dbgs()); }
673
674 // Free up memory by removing empty entries from the DenseMap that track event
675 // scores.
676 void purgeEmptyTrackingData();
677
678private:
679 unsigned getLimit(AMDGPU::InstCounterType T) const {
680 return Context->getLimits().get(T);
681 }
682
683 struct MergeInfo {
684 unsigned OldLB;
685 unsigned OtherLB;
686 unsigned MyShift;
687 unsigned OtherShift;
688 };
689
690 using CounterValueArray = std::array<unsigned, AMDGPU::NUM_INST_CNTS>;
691
692 unsigned getWaitForScore(AMDGPU::InstCounterType T, unsigned Score) const;
693 void determineWaitForScore(AMDGPU::InstCounterType T, unsigned Score,
694 AMDGPU::Waitcnt &Wait) const;
695 void determineWaitForVAVDSTWrite(VMEMID ID, AMDGPU::Waitcnt &Wait) const;
696
697 static bool mergeScore(const MergeInfo &M, unsigned &Score,
698 unsigned OtherScore);
699 bool mergeAsyncMarks(AMDGPU::AsyncStages Stage,
700 ArrayRef<MergeInfo> MergeInfos,
701 ArrayRef<CounterValueArray> OtherMarks);
702
703 iterator_range<MCRegUnitIterator> regunits(MCRegister Reg) const {
704 assert(Reg != AMDGPU::SCC && "Shouldn't be used on SCC");
705 if (!Context->TRI.isInAllocatableClass(RegNo: Reg))
706 return {{}, {}};
707 return Context->TRI.regunits(Reg);
708 }
709
710 void setScoreLB(AMDGPU::InstCounterType T, unsigned Val) {
711 assert(T < AMDGPU::NUM_INST_CNTS);
712 ScoreLBs[T] = Val;
713 }
714
715 void setScoreUB(AMDGPU::InstCounterType T, unsigned Val) {
716 assert(T < AMDGPU::NUM_INST_CNTS);
717 ScoreUBs[T] = Val;
718
719 if (T != AMDGPU::EXP_CNT)
720 return;
721
722 if (getScoreRange(T: AMDGPU::EXP_CNT) > getLimit(T: AMDGPU::EXP_CNT))
723 ScoreLBs[AMDGPU::EXP_CNT] =
724 ScoreUBs[AMDGPU::EXP_CNT] - getLimit(T: AMDGPU::EXP_CNT);
725 }
726
727 void setRegScore(MCRegister Reg, AMDGPU::InstCounterType T, unsigned Val) {
728 const SIRegisterInfo &TRI = Context->TRI;
729 if (Reg == AMDGPU::SCC) {
730 SCCScore = Val;
731 } else if (TRI.isVectorRegister(MRI: Context->MRI, Reg)) {
732 for (MCRegUnit RU : regunits(Reg)) {
733 auto &Info = VMem[toVMEMID(RU)];
734 Info.Scores[T] = Val;
735 if (T == AMDGPU::VA_VDST_WR)
736 Info.OrderedXDLCountAtWrite = OrderedXDLCount;
737 }
738 } else if (TRI.isSGPRReg(MRI: Context->MRI, Reg)) {
739 for (MCRegUnit RU : regunits(Reg))
740 SGPRs[RU].get(T) = Val;
741 } else {
742 llvm_unreachable("Register cannot be tracked/unknown register!");
743 }
744 }
745
746 void setVMemScore(VMEMID TID, AMDGPU::InstCounterType T, unsigned Val) {
747 VMem[TID].Scores[T] = Val;
748 }
749
750 void setScoreByOperand(const MachineOperand &Op,
751 AMDGPU::InstCounterType CntTy, unsigned Val);
752
753 const SIInsertWaitcnts *Context;
754
755 unsigned ScoreLBs[AMDGPU::NUM_INST_CNTS] = {0};
756 unsigned ScoreUBs[AMDGPU::NUM_INST_CNTS] = {0};
757 HWEvents PendingEvents;
758 // Remember the last flat memory operation.
759 unsigned LastFlatDsCnt = 0;
760 unsigned LastFlatLoadCnt = 0;
761 // Remember the last VMEM operation that increments xcnt but neither loadcnt
762 // nor storecnt, such as a prefetch or an LDS-only FLAT. Async LDS DMA is
763 // excluded: it is covered by ASYNC_ACCESS instead.
764 unsigned LastXcntOnlyVmem = 0;
765 // Remember the last GDS operation.
766 unsigned LastGDS = 0;
767 // Number of VA_VDST ordered XDL writes issued so far. Only differences
768 // between values are meaningful, so unsigned wraparound is harmless.
769 unsigned OrderedXDLCount = 0;
770
771 // The score tracking logic is fragmented as follows:
772 // - VMem: VGPR RegUnits and LDS DMA IDs, see the VMEMID encoding.
773 // - SGPRs: SGPR RegUnits
774 // - SCC: Non-allocatable and not general purpose: not a SGPR.
775 //
776 // For the VMem case, if the key is within the range of LDS DMA IDs,
777 // then the corresponding index into the `LDSDMAStores` vector below is:
778 // Key - LDSDMA_BEGIN - 1
779 // This is because LDSDMA_BEGIN is a generic entry and does not have an
780 // associated MachineInstr.
781 //
782 // TODO: Could we track SCC alongside SGPRs so it's not longer a special case?
783
784 struct VMEMInfo {
785 // Scores for all instruction counters. Zero-initialized.
786 CounterValueArray Scores{};
787 // OrderedXDLCount when the VA_VDST_WR score was set. Every ordered XDL
788 // write issued since then remains outstanding while this write is pending.
789 unsigned OrderedXDLCountAtWrite = 0;
790 // For VGPRs, we need to track an additional fine-grained set of pending
791 // events.
792 HWEvents VGPRPendingEvents;
793
794 bool empty() const {
795 return all_of(Range: Scores, P: equal_to(Arg: 0)) && !VGPRPendingEvents;
796 }
797 };
798
799 /// Wait cnt scores for every sgpr, the DS_CNT (corresponding to LGKMcnt
800 /// pre-gfx12) or KM_CNT (gfx12+ only), and X_CNT (gfx1250) are relevant.
801 class SGPRInfo {
802 /// Either DS_CNT or KM_CNT score.
803 unsigned ScoreDsKmCnt = 0;
804 unsigned ScoreXCnt = 0;
805
806 public:
807 unsigned get(AMDGPU::InstCounterType T) const {
808 assert(
809 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
810 "Invalid counter");
811 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
812 }
813 unsigned &get(AMDGPU::InstCounterType T) {
814 assert(
815 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
816 "Invalid counter");
817 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
818 }
819
820 bool empty() const { return !ScoreDsKmCnt && !ScoreXCnt; }
821 };
822
823 DenseMap<VMEMID, VMEMInfo> VMem; // VGPR + LDS DMA
824 DenseMap<MCRegUnit, SGPRInfo> SGPRs;
825
826 // Reg score for SCC.
827 unsigned SCCScore = 0;
828 // The unique instruction that has an SCC write pending, if there is one.
829 const MachineInstr *PendingSCCWrite = nullptr;
830
831 // Store representative LDS DMA operations. The only useful info here is
832 // alias info. One store is kept per unique AAInfo.
833 SmallVector<const MachineInstr *> LDSDMAStores;
834
835 // State of all counters at each async mark encountered so far, per stage.
836 std::array<SmallVector<CounterValueArray, 0>, AMDGPU::NumAsyncStages>
837 AsyncMarks;
838
839 // But in the rare pathological case, a nest of loops that pushes marks
840 // without waiting on any mark can cause AsyncMarks to grow very large. We cap
841 // it to a reasonable limit. We can tune this later or potentially introduce a
842 // user option to control the value.
843 static constexpr unsigned MaxAsyncMarks = 16;
844
845 // Track the upper bound score for async operations that are not part of a
846 // mark yet, per stage. Initialized to all zeros.
847 std::array<CounterValueArray, AMDGPU::NumAsyncStages> AsyncScore{};
848};
849
850SIInsertWaitcnts::BlockInfo::~BlockInfo() = default;
851
852class SIInsertWaitcntsLegacy : public MachineFunctionPass {
853public:
854 static char ID;
855 SIInsertWaitcntsLegacy() : MachineFunctionPass(ID) {}
856
857 bool runOnMachineFunction(MachineFunction &MF) override;
858
859 StringRef getPassName() const override {
860 return "SI insert wait instructions";
861 }
862
863 void getAnalysisUsage(AnalysisUsage &AU) const override {
864 AU.setPreservesCFG();
865 AU.addRequired<MachineLoopInfoWrapperPass>();
866 AU.addRequired<MachinePostDominatorTreeWrapperPass>();
867 AU.addUsedIfAvailable<AAResultsWrapperPass>();
868 AU.addPreserved<AAResultsWrapperPass>();
869 MachineFunctionPass::getAnalysisUsage(AU);
870 }
871};
872
873} // end anonymous namespace
874
875void WaitcntBrackets::setScoreByOperand(const MachineOperand &Op,
876 AMDGPU::InstCounterType CntTy,
877 unsigned Score) {
878 setRegScore(Reg: Op.getReg().asMCReg(), T: CntTy, Val: Score);
879}
880
881// Return true if the subtarget is one that enables Point Sample Acceleration
882// and the MachineInstr passed in is one to which it might be applied (the
883// hardware makes this decision based on several factors, but we can't determine
884// this at compile time, so we have to assume it might be applied if the
885// instruction supports it).
886bool WaitcntBrackets::hasPointSampleAccel(const MachineInstr &MI) const {
887 if (!Context->ST.hasPointSampleAccel() || !SIInstrInfo::isMIMG(MI))
888 return false;
889
890 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: MI.getOpcode());
891 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
892 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
893 return BaseInfo->PointSampleAccel;
894}
895
896// Return true if the subtarget enables Point Sample Acceleration, the supplied
897// MachineInstr is one to which it might be applied and the supplied interval is
898// one that has outstanding writes to vmem-types different than VMEM_NOSAMPLER
899// (this is the type that a point sample accelerated instruction effectively
900// becomes)
901bool WaitcntBrackets::hasPointSamplePendingVmemTypes(const MachineInstr &MI,
902 MCRegister Reg) const {
903 if (!hasPointSampleAccel(MI))
904 return false;
905
906 return hasDifferentVGPRPendingEvents(Reg, E: HWEvents::VMEM_READ_ACCESS);
907}
908
909void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
910 assert(E.size() == 1 && "Expected singular event!");
911 AMDGPU::InstCounterType T = Context->getCounterFromEvent(E);
912 assert(T < Context->MaxCounter);
913
914 unsigned UB = getScoreUB(T);
915 unsigned Increment = 1;
916 if ((T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) &&
917 AMDGPU::getHasMatrixScale(Opc: Inst.getOpcode()) &&
918 Context->ST.hasVOP3PX2IncrementsVaVdstTwice()) {
919 // V_WMMA_SCALE instructions use VOP3PX2 encoding. Hardware treats this as
920 // two VOP3P instructions and increments VA_VDST twice.
921 Increment = 2;
922 }
923 unsigned CurrScore = UB + Increment;
924 if (CurrScore <= UB)
925 report_fatal_error(reason: "InsertWaitcnt score wraparound");
926 // PendingEvents and ScoreUB need to be update regardless if this event
927 // changes the score of a register or not.
928 // Examples including vm_cnt when buffer-store or lgkm_cnt when send-message.
929 PendingEvents |= E;
930 setScoreUB(T, Val: CurrScore);
931
932 // Count this write before recording its own destination, so the
933 // destination does not count itself. A VOP3PX2 WMMA counts once even though
934 // it increments VA_VDST twice: only its WMMA half is an ordered XDL write.
935 if (E == HWEvents::VGPR_XDL_WRITE && Context->TII.isVAVDSTOrderedXDL(MI: Inst))
936 ++OrderedXDLCount;
937
938 const SIRegisterInfo &TRI = Context->TRI;
939 const MachineRegisterInfo &MRI = Context->MRI;
940 const SIInstrInfo &TII = Context->TII;
941
942 if (T == AMDGPU::EXP_CNT) {
943 // Put score on the source vgprs. If this is a store, just use those
944 // specific register(s).
945 if (TII.isDS(MI: Inst) && Inst.mayLoadOrStore()) {
946 // All GDS operations must protect their address register (same as
947 // export.)
948 if (const auto *AddrOp = TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::addr))
949 setScoreByOperand(Op: *AddrOp, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
950
951 if (Inst.mayStore()) {
952 if (const auto *Data0 =
953 TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data0))
954 setScoreByOperand(Op: *Data0, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
955 if (const auto *Data1 =
956 TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data1))
957 setScoreByOperand(Op: *Data1, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
958 } else if (SIInstrInfo::isAtomicRet(MI: Inst) && !SIInstrInfo::isGWS(MI: Inst) &&
959 Inst.getOpcode() != AMDGPU::DS_APPEND &&
960 Inst.getOpcode() != AMDGPU::DS_CONSUME &&
961 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
962 for (const MachineOperand &Op : Inst.all_uses()) {
963 if (TRI.isVectorRegister(MRI, Reg: Op.getReg()))
964 setScoreByOperand(Op, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
965 }
966 }
967 } else if (TII.isFLAT(MI: Inst)) {
968 if (Inst.mayStore()) {
969 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
970 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
971 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
972 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
973 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
974 }
975 } else if (TII.isMIMG(MI: Inst)) {
976 if (Inst.mayStore()) {
977 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
978 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
979 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
980 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
981 }
982 } else if (TII.isMTBUF(MI: Inst)) {
983 if (Inst.mayStore())
984 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
985 } else if (TII.isMUBUF(MI: Inst)) {
986 if (Inst.mayStore()) {
987 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
988 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
989 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
990 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
991 }
992 } else if (TII.isLDSDIR(MI: Inst)) {
993 // LDSDIR instructions attach the score to the destination.
994 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::vdst),
995 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
996 } else {
997 if (TII.isEXP(MI: Inst)) {
998 // For export the destination registers are really temps that
999 // can be used as the actual source after export patching, so
1000 // we need to treat them like sources and set the EXP_CNT
1001 // score.
1002 for (MachineOperand &DefMO : Inst.all_defs()) {
1003 if (TRI.isVGPR(MRI, Reg: DefMO.getReg())) {
1004 setScoreByOperand(Op: DefMO, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
1005 }
1006 }
1007 }
1008 for (const MachineOperand &Op : Inst.all_uses()) {
1009 if (TRI.isVectorRegister(MRI, Reg: Op.getReg()))
1010 setScoreByOperand(Op, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
1011 }
1012 }
1013 } else if (T == AMDGPU::X_CNT) {
1014 HWEvents OtherEvent =
1015 E == HWEvents::SMEM_GROUP ? HWEvents::VMEM_GROUP : HWEvents::SMEM_GROUP;
1016 if (PendingEvents.contains(Other: OtherEvent)) {
1017 // Hardware inserts an implicit xcnt between interleaved
1018 // SMEM and VMEM operations. So there will never be
1019 // outstanding address translations for both SMEM and
1020 // VMEM at the same time.
1021 setScoreLB(T, Val: getScoreUB(T) - 1);
1022 PendingEvents -= OtherEvent;
1023 }
1024 for (const MachineOperand &Op : Inst.all_uses()) {
1025 // Async ops have an implicit use of ASYNCcnt, which is not a real
1026 // register.
1027 if (Op.getReg() == AMDGPU::ASYNCcnt)
1028 continue;
1029 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
1030 }
1031 } else if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR ||
1032 T == AMDGPU::VM_VSRC) {
1033 // Match the score to the VGPR destination or source registers as
1034 // appropriate
1035 for (const MachineOperand &Op : Inst.operands()) {
1036 if (!Op.isReg())
1037 continue;
1038
1039 // Skip based on counter type and operand type
1040 if (T == AMDGPU::VA_VDST_RD && Op.isDef())
1041 continue; // RD tracks reads only
1042 if (T == AMDGPU::VA_VDST_WR && Op.isUse())
1043 continue; // WR tracks writes only
1044 if (T == AMDGPU::VM_VSRC && Op.isDef())
1045 continue;
1046
1047 if (TRI.isVectorRegister(MRI: Context->MRI, Reg: Op.getReg()))
1048 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
1049 }
1050 } else /* LGKM_CNT || EXP_CNT || VS_CNT || NUM_INST_CNTS */ {
1051 // Match the score to the destination registers.
1052 //
1053 // Check only explicit operands. Stores, especially spill stores, include
1054 // implicit uses and defs of their super registers which would create an
1055 // artificial dependency, while these are there only for register liveness
1056 // accounting purposes.
1057 //
1058 // Special cases where implicit register defs exists, such as M0 or VCC,
1059 // but none with memory instructions.
1060 for (const MachineOperand &Op : Inst.defs()) {
1061 if (T == AMDGPU::LOAD_CNT || T == AMDGPU::SAMPLE_CNT ||
1062 T == AMDGPU::BVH_CNT) {
1063 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg())) // TODO: add wrapper
1064 continue;
1065 if (updateVMCntOnly(Inst)) {
1066 // updateVMCntOnly should only leave us with VGPRs
1067 // MUBUF, MTBUF, MIMG, FlatGlobal, and FlatScratch only have VGPR/AGPR
1068 // defs.
1069 assert(TRI.isVectorRegister(MRI, Op.getReg()));
1070 HWEvents VGPRContext =
1071 AMDGPU::getSimplifiedVMEMEventsFor(Inst, TII: Context->TII);
1072 // If instruction can have Point Sample Accel applied, we have to flag
1073 // this with another potential dependency
1074 if (hasPointSampleAccel(MI: Inst))
1075 VGPRContext |= HWEvents::VMEM_READ_ACCESS;
1076 for (MCRegUnit RU : regunits(Reg: Op.getReg().asMCReg()))
1077 VMem[toVMEMID(RU)].VGPRPendingEvents |= VGPRContext;
1078 }
1079 }
1080 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
1081 }
1082 if (Inst.mayStore() &&
1083 (TII.isDS(MI: Inst) || TII.isNonAsyncLdsDmaWrite(MI: Inst))) {
1084 // MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS
1085 // written can be accessed. A load from LDS to VMEM does not need a wait.
1086 //
1087 // The "Slot" is the offset from LDSDMA_BEGIN. If it's non-zero, then
1088 // there is a MachineInstr in LDSDMAStores used to track this LDSDMA
1089 // store. The "Slot" is the index into LDSDMAStores + 1.
1090 size_t Slot = 0;
1091 for (const auto *MemOp : Inst.memoperands()) {
1092 if (!MemOp->isStore() ||
1093 MemOp->getAddrSpace() != AMDGPUAS::LOCAL_ADDRESS)
1094 continue;
1095 // Comparing just AA info does not guarantee memoperands are equal
1096 // in general, but this is so for LDS DMA in practice.
1097 auto AAI = MemOp->getAAInfo();
1098 // Alias scope information gives a way to definitely identify an
1099 // original memory object and practically produced in the module LDS
1100 // lowering pass. If there is no scope available we will not be able
1101 // to disambiguate LDS aliasing as after the module lowering all LDS
1102 // is squashed into a single big object.
1103 if (!AAI || !AAI.Scope)
1104 break;
1105 for (size_t I = 0, E = LDSDMAStores.size(); I != E && !Slot; ++I) {
1106 for (const auto *MemOp : LDSDMAStores[I]->memoperands()) {
1107 if (MemOp->isStore() && AAI == MemOp->getAAInfo()) {
1108 Slot = I + 1;
1109 break;
1110 }
1111 }
1112 }
1113 if (Slot)
1114 break;
1115 // The slot may not be valid because it can be >= NUM_LDSDMA which
1116 // means the scoreboard cannot track it. We still want to preserve the
1117 // MI in order to check alias information, though.
1118 LDSDMAStores.push_back(Elt: &Inst);
1119 Slot = LDSDMAStores.size();
1120 break;
1121 }
1122 setVMemScore(TID: LDSDMA_BEGIN, T, Val: CurrScore);
1123 if (Slot && Slot < NUM_LDSDMA)
1124 setVMemScore(TID: LDSDMA_BEGIN + static_cast<VMEMID>(Slot), T, Val: CurrScore);
1125 }
1126
1127 AMDGPU::AsyncStages Stage = TII.getAsyncStage(MI: Inst);
1128 if (any(Val: Stage))
1129 AsyncScore[AMDGPU::getAsyncStageIndex(Stage)][T] = CurrScore;
1130
1131 if (SIInstrInfo::isSBarrierSCCWrite(Opcode: Inst.getOpcode())) {
1132 setRegScore(Reg: AMDGPU::SCC, T, Val: CurrScore);
1133 PendingSCCWrite = &Inst;
1134 }
1135 }
1136}
1137
1138void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst,
1139 AMDGPU::AsyncStages Stages) {
1140 // In the absence of loops, AsyncMarks can grow linearly with the program
1141 // until we encounter an ASYNCMARK_WAIT. We could drop the oldest mark above a
1142 // limit every time we push a new mark, but that seems like unnecessary work
1143 // in practical cases. We do separately truncate the array when processing a
1144 // loop, which should be sufficient.
1145 //
1146 // A mark is inserted in every stage named. Each stage gets a copy of its
1147 // corresponding score.
1148 LLVM_DEBUG(dbgs() << "recordAsyncMark(stages=" << Stages << "):\n" << Inst);
1149 for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(Mask: Stages)) {
1150 unsigned SI = AMDGPU::getAsyncStageIndex(Stage: S);
1151 AsyncMarks[SI].push_back(Elt: AsyncScore[SI]);
1152 LLVM_DEBUG({
1153 dbgs() << " stage " << S << ":\n";
1154 for (const auto &Mark : AsyncMarks[SI]) {
1155 dbgs() << " ";
1156 llvm::interleaveComma(Mark, dbgs());
1157 dbgs() << '\n';
1158 }
1159 });
1160 }
1161}
1162
1163void WaitcntBrackets::print(raw_ostream &OS) const {
1164 const GCNSubtarget &ST = Context->ST;
1165
1166 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter)) {
1167 unsigned SR = getScoreRange(T);
1168 switch (T) {
1169 case AMDGPU::LOAD_CNT:
1170 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM") << "_CNT("
1171 << SR << "):";
1172 break;
1173 case AMDGPU::DS_CNT:
1174 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM") << "_CNT("
1175 << SR << "):";
1176 break;
1177 case AMDGPU::EXP_CNT:
1178 OS << " EXP_CNT(" << SR << "):";
1179 break;
1180 case AMDGPU::STORE_CNT:
1181 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS") << "_CNT("
1182 << SR << "):";
1183 break;
1184 case AMDGPU::SAMPLE_CNT:
1185 OS << " SAMPLE_CNT(" << SR << "):";
1186 break;
1187 case AMDGPU::BVH_CNT:
1188 OS << " BVH_CNT(" << SR << "):";
1189 break;
1190 case AMDGPU::KM_CNT:
1191 OS << " KM_CNT(" << SR << "):";
1192 break;
1193 case AMDGPU::X_CNT:
1194 OS << " X_CNT(" << SR << "):";
1195 break;
1196 case AMDGPU::ASYNC_CNT:
1197 OS << " ASYNC_CNT(" << SR << "):";
1198 break;
1199 case AMDGPU::VA_VDST_RD:
1200 OS << " VA_VDST_RD(" << SR << "): ";
1201 break;
1202 case AMDGPU::VA_VDST_WR:
1203 OS << " VA_VDST_WR(" << SR << "): ";
1204 break;
1205 case AMDGPU::VM_VSRC:
1206 OS << " VM_VSRC(" << SR << "): ";
1207 break;
1208 default:
1209 OS << " UNKNOWN(" << SR << "):";
1210 break;
1211 }
1212
1213 if (SR != 0) {
1214 // Print vgpr scores.
1215 unsigned LB = getScoreLB(T);
1216
1217 SmallVector<VMEMID> SortedVMEMIDs(VMem.keys());
1218 sort(C&: SortedVMEMIDs);
1219
1220 for (auto ID : SortedVMEMIDs) {
1221 unsigned RegScore = VMem.at(Val: ID).Scores[T];
1222 if (RegScore <= LB)
1223 continue;
1224 unsigned RelScore = RegScore - LB - 1;
1225 if (ID < REGUNITS_END) {
1226 OS << ' ' << RelScore << ':'
1227 << printRegUnit(Unit: static_cast<MCRegUnit>(ID), TRI: &Context->TRI);
1228 } else {
1229 assert(ID >= LDSDMA_BEGIN && ID < LDSDMA_END &&
1230 "Unhandled/unexpected ID value!");
1231 OS << ' ' << RelScore << ":LDSDMA" << ID;
1232 }
1233 }
1234
1235 // Also need to print sgpr scores for lgkm_cnt or xcnt.
1236 if (isSmemCounter(T)) {
1237 SmallVector<MCRegUnit> SortedSMEMIDs(SGPRs.keys());
1238 sort(C&: SortedSMEMIDs);
1239 for (auto ID : SortedSMEMIDs) {
1240 unsigned RegScore = SGPRs.at(Val: ID).get(T);
1241 if (RegScore <= LB)
1242 continue;
1243 unsigned RelScore = RegScore - LB - 1;
1244 OS << ' ' << RelScore << ':'
1245 << printRegUnit(Unit: static_cast<MCRegUnit>(ID), TRI: &Context->TRI);
1246 }
1247 }
1248
1249 if (T == AMDGPU::KM_CNT && SCCScore > 0)
1250 OS << ' ' << SCCScore << ":scc";
1251 }
1252 OS << '\n';
1253 }
1254
1255 OS << "Pending Events: ";
1256 if (hasPendingEvent()) {
1257 OS << getPendingEvents();
1258 } else {
1259 OS << "none";
1260 }
1261 OS << '\n';
1262
1263 for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(Mask: AMDGPU::AsyncStages::ALL)) {
1264 unsigned SI = AMDGPU::getAsyncStageIndex(Stage: S);
1265 const CounterValueArray &Score = AsyncScore[SI];
1266 OS << "Async score (stage " << S << "): ";
1267 if (llvm::all_of(Range: Score, P: [](unsigned V) { return V == 0; }))
1268 OS << "none";
1269 else
1270 llvm::interleaveComma(c: Score, os&: OS);
1271 OS << '\n';
1272
1273 OS << "Async marks (stage " << S << "): " << AsyncMarks[SI].size() << '\n';
1274
1275 for (const auto &Mark : AsyncMarks[SI]) {
1276 for (auto T : AMDGPU::inst_counter_types()) {
1277 unsigned MarkedScore = Mark[T];
1278 switch (T) {
1279 case AMDGPU::LOAD_CNT:
1280 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM")
1281 << "_CNT: " << MarkedScore;
1282 break;
1283 case AMDGPU::DS_CNT:
1284 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM")
1285 << "_CNT: " << MarkedScore;
1286 break;
1287 case AMDGPU::EXP_CNT:
1288 OS << " EXP_CNT: " << MarkedScore;
1289 break;
1290 case AMDGPU::STORE_CNT:
1291 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS")
1292 << "_CNT: " << MarkedScore;
1293 break;
1294 case AMDGPU::SAMPLE_CNT:
1295 OS << " SAMPLE_CNT: " << MarkedScore;
1296 break;
1297 case AMDGPU::BVH_CNT:
1298 OS << " BVH_CNT: " << MarkedScore;
1299 break;
1300 case AMDGPU::KM_CNT:
1301 OS << " KM_CNT: " << MarkedScore;
1302 break;
1303 case AMDGPU::X_CNT:
1304 OS << " X_CNT: " << MarkedScore;
1305 break;
1306 case AMDGPU::ASYNC_CNT:
1307 OS << " ASYNC_CNT: " << MarkedScore;
1308 break;
1309 default:
1310 OS << " UNKNOWN: " << MarkedScore;
1311 break;
1312 }
1313 }
1314 OS << '\n';
1315 }
1316 }
1317 OS << '\n';
1318}
1319
1320/// Simplify \p UpdateWait by removing waits that are redundant based on the
1321/// current WaitcntBrackets and any other waits specified in \p CheckWait.
1322void WaitcntBrackets::simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
1323 AMDGPU::Waitcnt &UpdateWait) const {
1324 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::LOAD_CNT);
1325 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::EXP_CNT);
1326 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::DS_CNT);
1327 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::STORE_CNT);
1328 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::SAMPLE_CNT);
1329 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::BVH_CNT);
1330 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::KM_CNT);
1331 simplifyXcnt(CheckWait, UpdateWait);
1332 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VA_VDST_RD);
1333 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VA_VDST_WR);
1334 simplifyVmVsrc(CheckWait, UpdateWait);
1335 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::ASYNC_CNT);
1336}
1337
1338void WaitcntBrackets::simplifyWaitcnt(AMDGPU::InstCounterType T,
1339 unsigned &Count) const {
1340 // The number of outstanding events for this type, T, can be calculated
1341 // as (UB - LB). If the current Count is greater than or equal to the number
1342 // of outstanding events, then the wait for this counter is redundant.
1343 if (Count >= getScoreRange(T))
1344 Count = ~0u;
1345}
1346
1347void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait,
1348 AMDGPU::InstCounterType T) const {
1349 unsigned Cnt = Wait.get(T);
1350 simplifyWaitcnt(T, Count&: Cnt);
1351 Wait.set(T, Val: Cnt);
1352}
1353
1354void WaitcntBrackets::simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
1355 AMDGPU::Waitcnt &UpdateWait) const {
1356 // Try to simplify xcnt further by checking for joint kmcnt and loadcnt
1357 // optimizations. On entry to a block with multiple predescessors, there may
1358 // be pending SMEM and VMEM events active at the same time.
1359 // In such cases, only clear one active event at a time.
1360 // TODO: Revisit xcnt optimizations for gfx1250.
1361 // Wait on XCNT is redundant if we are already waiting for a load to complete.
1362 // SMEM can return out of order, so only omit XCNT wait if we are waiting till
1363 // zero.
1364 if (CheckWait.get(T: AMDGPU::KM_CNT) == 0 &&
1365 hasPendingEvent(E: HWEvents::SMEM_GROUP))
1366 UpdateWait.set(T: AMDGPU::X_CNT, Val: ~0u);
1367 // If we have pending store we cannot optimize XCnt because we do not wait for
1368 // stores. VMEM loads retun in order, so if we only have loads XCnt is
1369 // decremented to the same number as LOADCnt. That does not hold while a
1370 // VMEM operation that LOADCnt does not count is in flight.
1371 if (CheckWait.get(T: AMDGPU::LOAD_CNT) != ~0u &&
1372 hasPendingEvent(E: HWEvents::VMEM_GROUP) &&
1373 !hasPendingEvent(T: AMDGPU::STORE_CNT) && !hasPendingXcntOnlyVmem() &&
1374 !hasPendingEvent(E: HWEvents::ASYNC_ACCESS) &&
1375 CheckWait.get(T: AMDGPU::X_CNT) >= CheckWait.get(T: AMDGPU::LOAD_CNT))
1376 UpdateWait.set(T: AMDGPU::X_CNT, Val: ~0u);
1377 // asynccnt==0 completes every async op, which also drains their xcnt.
1378 if (CheckWait.get(T: AMDGPU::ASYNC_CNT) == 0 &&
1379 hasPendingEvent(E: HWEvents::ASYNC_ACCESS) &&
1380 !hasPendingNonAsyncXcnt(Wait: CheckWait))
1381 UpdateWait.set(T: AMDGPU::X_CNT, Val: ~0u);
1382 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::X_CNT);
1383}
1384
1385void WaitcntBrackets::simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
1386 AMDGPU::Waitcnt &UpdateWait) const {
1387 // Waiting for a VMEM counter (like LOAD_CNT) implies an equivalent wait for
1388 // VM_VSRC, because if the VMEM operation has completed then it must surely
1389 // have read its VGPR sources, but only if there are no other outstanding VMEM
1390 // operations that use a different counter (like SAMPLE_CNT).
1391 static constexpr AMDGPU::InstCounterType VmemCounters[] = {
1392 AMDGPU::LOAD_CNT, AMDGPU::STORE_CNT, AMDGPU::SAMPLE_CNT, AMDGPU::BVH_CNT,
1393 AMDGPU::DS_CNT, AMDGPU::ASYNC_CNT};
1394 HWEvents VmemEvents = llvm::accumulate(
1395 Range: VmemCounters, Init: HWEvents(), Op: [&](HWEvents Acc, AMDGPU::InstCounterType T) {
1396 return Acc | Context->getWaitEvents(T);
1397 });
1398 HWEvents PendingVmemEvents = PendingEvents & VmemEvents;
1399 for (AMDGPU::InstCounterType T : VmemCounters) {
1400 unsigned CheckCount = CheckWait.get(T);
1401 if (UpdateWait.get(T: AMDGPU::VM_VSRC) >= CheckCount &&
1402 (CheckCount == 0 || !counterOutOfOrder(T)) &&
1403 (PendingVmemEvents & ~Context->getWaitEvents(T)) == 0)
1404 UpdateWait.set(T: AMDGPU::VM_VSRC, Val: ~0u);
1405 }
1406
1407 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VM_VSRC);
1408}
1409
1410void WaitcntBrackets::purgeEmptyTrackingData() {
1411 VMem.remove_if(Pred: [](const auto &P) { return P.second.empty(); });
1412 SGPRs.remove_if(Pred: [](const auto &P) { return P.second.empty(); });
1413}
1414
1415// Returns the wait needed for \p ScoreToWait, or ~0u if no wait is needed.
1416unsigned WaitcntBrackets::getWaitForScore(AMDGPU::InstCounterType T,
1417 unsigned ScoreToWait) const {
1418 const unsigned LB = getScoreLB(T);
1419 const unsigned UB = getScoreUB(T);
1420
1421 // If the score falls within the bracket, we need a waitcnt.
1422 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1423 if ((T == AMDGPU::LOAD_CNT || T == AMDGPU::DS_CNT) && hasPendingFlat() &&
1424 !Context->ST.hasFlatLgkmVMemCountInOrder()) {
1425 // If there is a pending FLAT operation, and this is a VMem or LGKM
1426 // waitcnt and the target can report early completion, then we need
1427 // to force a waitcnt 0.
1428 return 0;
1429 }
1430 if (counterOutOfOrder(T)) {
1431 // Counter can get decremented out-of-order when there
1432 // are multiple types event in the bracket. Also emit an s_wait counter
1433 // with a conservative value of 0 for the counter.
1434 return 0;
1435 }
1436 // If a counter has been maxed out avoid overflow by waiting for
1437 // MAX(CounterType) - 1 instead.
1438 return std::min(a: UB - ScoreToWait, b: getLimit(T) - 1);
1439 }
1440 return ~0u;
1441}
1442
1443void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
1444 unsigned ScoreToWait,
1445 AMDGPU::Waitcnt &Wait) const {
1446 Wait.add(T, Count: getWaitForScore(T, ScoreToWait));
1447}
1448
1449void WaitcntBrackets::determineWaitForVAVDSTWrite(VMEMID ID,
1450 AMDGPU::Waitcnt &Wait) const {
1451 unsigned Count =
1452 getWaitForScore(T: AMDGPU::VA_VDST_WR, ScoreToWait: getVMemScore(TID: ID, T: AMDGPU::VA_VDST_WR));
1453 if (Count == ~0u)
1454 return;
1455
1456 // The ordered-XDL count is a wait bound independent of exact event-class
1457 // ordering. Combine the bounds for this register unit before Wait.add()
1458 // takes the minimum across all required register-unit dependencies.
1459 Wait.add(T: AMDGPU::VA_VDST_WR,
1460 Count: std::max(a: Count, b: getNumOrderedXDLsAfterWrite(TID: ID)));
1461}
1462
1463AMDGPU::Waitcnt
1464WaitcntBrackets::determineAsyncWait(unsigned N, AMDGPU::AsyncStages Stages) {
1465 // Trim each named stage to at most N marks. The dropped marks imply waits
1466 // that get accumulated into one Waitcnt,
1467 // so a wait covering several stages is the union of their requirements.
1468 AMDGPU::Waitcnt Wait;
1469 for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(Mask: Stages)) {
1470 auto &StageMarks = AsyncMarks[AMDGPU::getAsyncStageIndex(Stage: S)];
1471 LLVM_DEBUG({
1472 dbgs() << "Need " << N << " async marks in stage " << S << ". Found "
1473 << StageMarks.size() << ":\n";
1474 for (const auto &Mark : StageMarks) {
1475 llvm::interleaveComma(Mark, dbgs());
1476 dbgs() << '\n';
1477 }
1478 });
1479
1480 unsigned StageN = N;
1481 if (StageMarks.size() == MaxAsyncMarks) {
1482 // Enforcing MaxAsyncMarks here is unnecessary work because the size of
1483 // MaxAsyncMarks is linear when traversing straightline code. But we do
1484 // need to check if truncation may have occured at a merge, and adjust N
1485 // to ensure that a wait is generated.
1486 LLVM_DEBUG(
1487 dbgs() << "Possible truncation. Ensuring a non-trivial wait.\n");
1488 StageN = std::min(a: StageN, b: (unsigned)MaxAsyncMarks - 1);
1489 }
1490
1491 if (StageMarks.size() <= StageN) {
1492 LLVM_DEBUG(dbgs() << "No additional wait for async mark.\n");
1493 continue;
1494 }
1495
1496 size_t MarkIndex = StageMarks.size() - StageN - 1;
1497 const auto &RequiredMark = StageMarks[MarkIndex];
1498 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
1499 determineWaitForScore(T, ScoreToWait: RequiredMark[T], Wait);
1500 }
1501
1502 // Immediately remove the waited mark and all older ones
1503 // This happens BEFORE the wait is actually inserted, which is fine
1504 // because we've already extracted the wait requirements
1505 LLVM_DEBUG({
1506 dbgs() << "Removing " << (MarkIndex + 1)
1507 << " async marks after determining wait\n";
1508 });
1509 StageMarks.erase(CS: StageMarks.begin(), CE: StageMarks.begin() + MarkIndex + 1);
1510 }
1511
1512 LLVM_DEBUG(dbgs() << "Waits to add: " << Wait);
1513 return Wait;
1514}
1515
1516// With D16Write32BitVgpr, D16 inst might be clobbered by events running on the
1517// other half 16bit.
1518//
1519// Replace VGPR16 to VGPR32 for wait check if:
1520// 1. MI is a VALU, and there is a wait event on the other half
1521// 2. MI is a LdSt, and there is a wait event on the other half from different
1522// order group
1523MCRegister WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
1524 AMDGPU::InstCounterType T,
1525 MCRegister Reg) const {
1526 const TargetRegisterClass *RC = Context->TRI.getPhysRegBaseClass(Reg);
1527 uint64_t Size = Context->TRI.getRegSizeInBits(RC: *RC);
1528
1529 if (Size != 16 || !Context->ST.hasD16Writes32BitVgpr())
1530 return Reg;
1531
1532 // With D16Writes32BitVgpr, D16 Inst might clobber the whole vgpr32
1533 // check dependency on the other half
1534 Register Reg32 = Context->TRI.get32BitRegister(Reg);
1535 Register OtherHalf = Context->TRI.getSubReg(
1536 Reg: Reg32,
1537 Idx: AMDGPU::isHi16Reg(Reg, MRI: Context->TRI) ? AMDGPU::lo16 : AMDGPU::hi16);
1538
1539 AMDGPU::Waitcnt Wait;
1540 for (MCRegUnit RU : regunits(Reg: OtherHalf)) {
1541 if (T == AMDGPU::VA_VDST_WR)
1542 determineWaitForVAVDSTWrite(ID: toVMEMID(RU), Wait);
1543 else
1544 determineWaitForScore(T, ScoreToWait: getVMemScore(TID: toVMEMID(RU), T), Wait);
1545 }
1546
1547 // No wait on otherhalf
1548 if (!Wait.hasWait())
1549 return Reg;
1550
1551 if (Context->TII.isVALU(MI, /*AllowLDSDMA=*/false))
1552 return Reg32;
1553
1554 // If hi/lo16 mixed events
1555 HWEvents MIEvents = AMDGPU::getEventsFor(
1556 Inst: MI, ST: Context->ST, IsExpertMode: Context->IsExpertMode, TgSplit: Context->TgSplit);
1557 HWEvents OtherHalfEvents = Context->getWaitEvents(T);
1558 HWEvents Events = MIEvents & OtherHalfEvents;
1559 if (Events.size() > 1)
1560 return Reg32;
1561 return Reg;
1562}
1563
1564void WaitcntBrackets::determineWaitForPhysReg(AMDGPU::InstCounterType T,
1565 MCRegister Reg,
1566 AMDGPU::Waitcnt &Wait,
1567 const MachineInstr &MI) const {
1568 if (Reg == AMDGPU::SCC) {
1569 determineWaitForScore(T, ScoreToWait: SCCScore, Wait);
1570 } else {
1571 bool IsVGPR = Context->TRI.isVectorRegister(MRI: Context->MRI, Reg);
1572 if (IsVGPR)
1573 Reg = determineVGPR16Dependency(MI, T, Reg);
1574 for (MCRegUnit RU : regunits(Reg)) {
1575 if (IsVGPR && T == AMDGPU::VA_VDST_WR) {
1576 determineWaitForVAVDSTWrite(ID: toVMEMID(RU), Wait);
1577 } else {
1578 determineWaitForScore(
1579 T, ScoreToWait: IsVGPR ? getVMemScore(TID: toVMEMID(RU), T) : getSGPRScore(RU, T),
1580 Wait);
1581 }
1582 }
1583 }
1584}
1585
1586void WaitcntBrackets::determineWaitForLDSDMA(AMDGPU::InstCounterType T,
1587 VMEMID TID,
1588 AMDGPU::Waitcnt &Wait) const {
1589 assert(TID >= LDSDMA_BEGIN && TID < LDSDMA_END);
1590 determineWaitForScore(T, ScoreToWait: getVMemScore(TID, T), Wait);
1591}
1592
1593void WaitcntBrackets::tryClearSCCWriteEvent(MachineInstr *Inst) {
1594 // S_BARRIER_WAIT on the same barrier guarantees that the pending write to
1595 // SCC has landed
1596 if (PendingSCCWrite &&
1597 PendingSCCWrite->getOpcode() == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM &&
1598 PendingSCCWrite->getOperand(i: 0).getImm() == Inst->getOperand(i: 0).getImm()) {
1599 HWEvents SCC_WRITE_PendingEvent = HWEvents::SCC_WRITE;
1600 // If this SCC_WRITE is the only pending KM_CNT event, clear counter.
1601 if ((PendingEvents & Context->getWaitEvents(T: AMDGPU::KM_CNT)) ==
1602 SCC_WRITE_PendingEvent) {
1603 setScoreLB(T: AMDGPU::KM_CNT, Val: getScoreUB(T: AMDGPU::KM_CNT));
1604 }
1605
1606 PendingEvents -= SCC_WRITE_PendingEvent;
1607 PendingSCCWrite = nullptr;
1608 }
1609}
1610
1611void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) {
1612 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
1613 unsigned Cnt;
1614 if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) {
1615 Cnt =
1616 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
1617 } else {
1618 Cnt = Wait.get(T);
1619 }
1620 applyWaitcnt(T, Count: Cnt);
1621 }
1622}
1623
1624void WaitcntBrackets::applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count) {
1625 const unsigned UB = getScoreUB(T);
1626 if (Count >= UB)
1627 return;
1628 if (Count != 0) {
1629 if (counterOutOfOrder(T))
1630 return;
1631 setScoreLB(T, Val: std::max(a: getScoreLB(T), b: UB - Count));
1632 } else {
1633 setScoreLB(T, Val: UB);
1634 PendingEvents -= Context->getWaitEvents(T);
1635 }
1636
1637 if (T == AMDGPU::KM_CNT && Count == 0 &&
1638 hasPendingEvent(E: HWEvents::SMEM_GROUP)) {
1639 if (!hasMixedPendingEvents(T: AMDGPU::X_CNT))
1640 applyWaitcnt(T: AMDGPU::X_CNT, Count: 0);
1641 else
1642 PendingEvents -= HWEvents::SMEM_GROUP;
1643 }
1644 if (T == AMDGPU::LOAD_CNT && hasPendingEvent(E: HWEvents::VMEM_GROUP) &&
1645 !hasPendingEvent(T: AMDGPU::STORE_CNT) && !hasPendingXcntOnlyVmem() &&
1646 !hasPendingEvent(E: HWEvents::ASYNC_ACCESS)) {
1647 if (!hasMixedPendingEvents(T: AMDGPU::X_CNT))
1648 applyWaitcnt(T: AMDGPU::X_CNT, Count);
1649 else if (Count == 0)
1650 PendingEvents -= HWEvents::VMEM_GROUP;
1651 }
1652 if (T == AMDGPU::ASYNC_CNT && Count == 0 && !hasPendingNonAsyncXcnt())
1653 applyWaitcnt(T: AMDGPU::X_CNT, Count: 0);
1654}
1655
1656void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait,
1657 AMDGPU::InstCounterType T) {
1658 unsigned Cnt = Wait.get(T);
1659 applyWaitcnt(T, Count: Cnt);
1660}
1661
1662// Where there are multiple types of event in the bracket of a counter,
1663// the decrement may go out of order.
1664bool WaitcntBrackets::counterOutOfOrder(AMDGPU::InstCounterType T) const {
1665 // Scalar memory read always can go out of order.
1666 if ((T == Context->SmemAccessCounter &&
1667 hasPendingEvent(E: HWEvents::SMEM_ACCESS)) ||
1668 (T == AMDGPU::X_CNT && hasPendingEvent(E: HWEvents::SMEM_GROUP)))
1669 return true;
1670
1671 if (T == AMDGPU::LOAD_CNT) {
1672
1673 // On targets without VScnt, LOAD_CNT includes all of STORE_CNT as well.
1674 // All these events use one counter and do not go out of order with respect
1675 // to each other.
1676 if (!Context->ST.hasVscnt())
1677 return false;
1678
1679 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
1680
1681 // If the target does not have extended counters, VMEM_BVH/SAMPLE_READ
1682 // events are equivalent to VMEM_READ_ACCESS. We do not go out of order in
1683 // such cases.
1684 static constexpr HWEvents ExtendedImageEvents =
1685 HWEvents::VMEM_SAMPLER_READ_ACCESS | HWEvents::VMEM_BVH_READ_ACCESS;
1686 if (!Context->ST.hasExtendedWaitCounts() &&
1687 (Events & ExtendedImageEvents).any()) {
1688 Events -= ExtendedImageEvents;
1689 Events |= HWEvents::VMEM_READ_ACCESS;
1690 }
1691
1692 // GLOBAL_INV/BUFFER_INV complete in-order with VMEM_READ_ACCESS, so mixing
1693 // VMEM_INV_ACCESS with other LOAD_CNT events does not cause out-of-order
1694 // completion.
1695 Events -= HWEvents::VMEM_INV_ACCESS;
1696
1697 return Events.size() > 1;
1698 }
1699
1700 return hasMixedPendingEvents(T);
1701}
1702
1703INITIALIZE_PASS_BEGIN(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1704 false, false)
1705INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
1706INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass)
1707INITIALIZE_PASS_END(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1708 false, false)
1709
1710char SIInsertWaitcntsLegacy::ID = 0;
1711
1712char &llvm::SIInsertWaitcntsID = SIInsertWaitcntsLegacy::ID;
1713
1714FunctionPass *llvm::createSIInsertWaitcntsPass() {
1715 return new SIInsertWaitcntsLegacy();
1716}
1717
1718static bool updateOperandIfDifferent(MachineInstr &MI, AMDGPU::OpName OpName,
1719 unsigned NewEnc) {
1720 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpName);
1721 assert(OpIdx >= 0);
1722
1723 MachineOperand &MO = MI.getOperand(i: OpIdx);
1724
1725 if (NewEnc == MO.getImm())
1726 return false;
1727
1728 MO.setImm(NewEnc);
1729 return true;
1730}
1731
1732bool WaitcntGenerator::promoteSoftWaitCnt(MachineInstr *Waitcnt) const {
1733 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: Waitcnt->getOpcode());
1734 if (Opcode == Waitcnt->getOpcode())
1735 return false;
1736
1737 Waitcnt->setDesc(TII.get(Opcode));
1738 return true;
1739}
1740
1741/// Combine consecutive S_WAITCNT and S_WAITCNT_VSCNT instructions that
1742/// precede \p It and follow \p OldWaitcntInstr and apply any extra waits
1743/// from \p Wait that were added by previous passes. Currently this pass
1744/// conservatively assumes that these preexisting waits are required for
1745/// correctness.
1746bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1747 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1748 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) {
1749 assert(isNormalMode(MaxCounter));
1750
1751 bool Modified = false;
1752 MachineInstr *WaitcntInstr = nullptr;
1753 MachineInstr *WaitcntVsCntInstr = nullptr;
1754
1755 LLVM_DEBUG({
1756 dbgs() << "PreGFX12::applyPreexistingWaitcnt at: ";
1757 if (It.isEnd())
1758 dbgs() << "end of block\n";
1759 else
1760 dbgs() << *It;
1761 });
1762
1763 for (auto &II :
1764 make_early_inc_range(Range: make_range(x: OldWaitcntInstr.getIterator(), y: It))) {
1765 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
1766 if (isNonWaitcntMetaInst(MI: II)) {
1767 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
1768 continue;
1769 }
1770
1771 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: II.getOpcode());
1772 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1773
1774 // Update required wait count. If this is a soft waitcnt (= it was added
1775 // by an earlier pass), it may be entirely removed.
1776 if (Opcode == AMDGPU::S_WAITCNT) {
1777 unsigned IEnc = static_cast<unsigned>(II.getOperand(i: 0).getImm());
1778 AMDGPU::Waitcnt OldWait = AMDGPU::decodeWaitcnt(Version: IV, Encoded: IEnc);
1779 if (TrySimplify) {
1780 ScoreBrackets.simplifyWaitcnt(Wait&: OldWait);
1781 ScoreBrackets.simplifyVmemInvOnlyWait(Wait&: OldWait);
1782 }
1783 Wait = Wait.combined(Other: OldWait);
1784
1785 // Merge consecutive waitcnt of the same type by erasing multiples.
1786 if (WaitcntInstr) {
1787 DeferredErasures.erase(V: &II);
1788 II.eraseFromParent();
1789 Modified = true;
1790 } else if (!Wait.hasWaitExceptStoreCnt() && TrySimplify) {
1791 // Might or might not be redundant, depending on if there's a loop
1792 // carried dependency. Stash it away so we can erase it if nothing saves
1793 // it.
1794 DeferredErasures.insert(V: &II);
1795 } else {
1796 WaitcntInstr = &II;
1797 DeferredErasures.erase(V: &II);
1798 }
1799 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
1800 assert(ST.hasVMemToLDSLoad());
1801 LLVM_DEBUG(dbgs() << "Processing S_WAITCNT_lds_direct: " << II
1802 << "Before: " << Wait << '\n';);
1803 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID: LDSDMA_BEGIN,
1804 Wait);
1805 LLVM_DEBUG(dbgs() << "After: " << Wait << '\n';);
1806
1807 // It is possible (but unlikely) that this is the only wait instruction,
1808 // in which case, we exit this loop without a WaitcntInstr to consume
1809 // `Wait`. But that works because `Wait` was passed in by reference, and
1810 // the callee eventually calls createNewWaitcnt on it. We test this
1811 // possibility in an articial MIR test since such a situation cannot be
1812 // recreated by running the memory legalizer.
1813 II.eraseFromParent();
1814 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
1815 unsigned N = static_cast<unsigned>(II.getOperand(i: 0).getImm());
1816 LLVM_DEBUG(dbgs() << "Processing WAIT_ASYNCMARK: " << II << '\n';);
1817 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
1818 N, Stages: AMDGPU::asyncStagesFromMaskOperand(Mask: II.getOperand(i: 1).getImm()));
1819 Wait = Wait.combined(Other: OldWait);
1820 } else {
1821 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1822 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1823
1824 unsigned OldVSCnt = static_cast<unsigned>(
1825 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
1826 if (TrySimplify)
1827 ScoreBrackets.simplifyWaitcnt(T: AMDGPU::STORE_CNT, Count&: OldVSCnt);
1828 Wait.set(T: AMDGPU::STORE_CNT,
1829 Val: std::min(a: Wait.get(T: AMDGPU::STORE_CNT), b: OldVSCnt));
1830
1831 if (WaitcntVsCntInstr) {
1832 DeferredErasures.erase(V: &II);
1833 II.eraseFromParent();
1834 Modified = true;
1835 } else if (!Wait.hasWaitStoreCnt() && TrySimplify) {
1836 // Loop-carried dependencies guard, see S_WAITCNT case above.
1837 DeferredErasures.insert(V: &II);
1838 } else {
1839 WaitcntVsCntInstr = &II;
1840 DeferredErasures.erase(V: &II);
1841 }
1842 }
1843 }
1844
1845 if (WaitcntInstr) {
1846 Modified |= updateOperandIfDifferent(MI&: *WaitcntInstr, OpName: AMDGPU::OpName::simm16,
1847 NewEnc: AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait));
1848 Modified |= promoteSoftWaitCnt(Waitcnt: WaitcntInstr);
1849
1850 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::LOAD_CNT);
1851 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::EXP_CNT);
1852 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
1853 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
1854 Wait.set(T: AMDGPU::EXP_CNT, Val: ~0u);
1855 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
1856
1857 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
1858 << "New Instr at block end: "
1859 << *WaitcntInstr << '\n'
1860 : dbgs() << "applied pre-existing waitcnt\n"
1861 << "Old Instr: " << *It
1862 << "New Instr: " << *WaitcntInstr << '\n');
1863 }
1864
1865 if (WaitcntVsCntInstr) {
1866 Modified |=
1867 updateOperandIfDifferent(MI&: *WaitcntVsCntInstr, OpName: AMDGPU::OpName::simm16,
1868 NewEnc: Wait.get(T: AMDGPU::STORE_CNT));
1869 Modified |= promoteSoftWaitCnt(Waitcnt: WaitcntVsCntInstr);
1870
1871 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::STORE_CNT);
1872 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
1873
1874 LLVM_DEBUG(It.isEnd()
1875 ? dbgs() << "applied pre-existing waitcnt\n"
1876 << "New Instr at block end: " << *WaitcntVsCntInstr
1877 << '\n'
1878 : dbgs() << "applied pre-existing waitcnt\n"
1879 << "Old Instr: " << *It
1880 << "New Instr: " << *WaitcntVsCntInstr << '\n');
1881 }
1882
1883 return Modified;
1884}
1885
1886/// Generate S_WAITCNT and/or S_WAITCNT_VSCNT instructions for any
1887/// required counters in \p Wait
1888bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1889 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
1890 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
1891 assert(isNormalMode(MaxCounter));
1892
1893 bool Modified = false;
1894 const DebugLoc &DL = Block.findDebugLoc(MBBI: It);
1895
1896 // Waits for VMcnt, LKGMcnt and/or EXPcnt are encoded together into a
1897 // single instruction while VScnt has its own instruction.
1898 if (Wait.hasWaitExceptStoreCnt()) {
1899 // If profiling expansion is enabled, emit an expanded sequence
1900 if (ExpandWaitcntProfiling) {
1901 // Check if any of the counters to be waited on are out-of-order.
1902 // If so, fall back to normal (non-expanded) behavior since expansion
1903 // would provide misleading profiling information.
1904 bool AnyOutOfOrder = false;
1905 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1906 unsigned WaitCnt = Wait.get(T: CT);
1907 if (WaitCnt != ~0u && ScoreBrackets.counterOutOfOrder(T: CT)) {
1908 AnyOutOfOrder = true;
1909 break;
1910 }
1911 }
1912
1913 if (AnyOutOfOrder) {
1914 // Fall back to non-expanded wait
1915 unsigned Enc = AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait);
1916 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: Enc);
1917 Modified = true;
1918 } else {
1919 // All counters are in-order, safe to expand
1920 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1921 unsigned WaitCnt = Wait.get(T: CT);
1922 if (WaitCnt == ~0u)
1923 continue;
1924
1925 unsigned Outstanding =
1926 std::min(a: ScoreBrackets.getOutstanding(T: CT), b: getLimit(E: CT) - 1);
1927 EmitExpandedWaitcnt(Outstanding, Target: WaitCnt, EmitWaitcnt: [&](unsigned Count) {
1928 AMDGPU::Waitcnt W;
1929 W.set(T: CT, Val: Count);
1930 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT))
1931 .addImm(Val: AMDGPU::encodeWaitcnt(Version: IV, Decoded: W));
1932 });
1933 Modified = true;
1934 }
1935 }
1936 } else {
1937 // Normal behavior: emit single combined waitcnt
1938 unsigned Enc = AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait);
1939 [[maybe_unused]] auto SWaitInst =
1940 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: Enc);
1941 Modified = true;
1942
1943 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1944 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1945 dbgs() << "New Instr: " << *SWaitInst << '\n');
1946 }
1947 }
1948
1949 if (Wait.hasWaitStoreCnt()) {
1950 assert(ST.hasVscnt());
1951
1952 if (ExpandWaitcntProfiling && Wait.get(T: AMDGPU::STORE_CNT) != ~0u &&
1953 !ScoreBrackets.counterOutOfOrder(T: AMDGPU::STORE_CNT)) {
1954 // Only expand if counter is not out-of-order
1955 unsigned Outstanding =
1956 std::min(a: ScoreBrackets.getOutstanding(T: AMDGPU::STORE_CNT),
1957 b: getLimit(E: AMDGPU::STORE_CNT) - 1);
1958 EmitExpandedWaitcnt(
1959 Outstanding, Target: Wait.get(T: AMDGPU::STORE_CNT), EmitWaitcnt: [&](unsigned Count) {
1960 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
1961 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
1962 .addImm(Val: Count);
1963 });
1964 Modified = true;
1965 } else {
1966 [[maybe_unused]] auto SWaitInst =
1967 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
1968 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
1969 .addImm(Val: Wait.get(T: AMDGPU::STORE_CNT));
1970 Modified = true;
1971
1972 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1973 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1974 dbgs() << "New Instr: " << *SWaitInst << '\n');
1975 }
1976 }
1977
1978 return Modified;
1979}
1980
1981AMDGPU::Waitcnt
1982WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1983 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt && ST.hasVscnt() ? 0 : ~0u);
1984}
1985
1986AMDGPU::Waitcnt
1987WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1988 unsigned ExpertVal = IsExpertMode ? 0 : ~0u;
1989 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0,
1990 ~0u /* XCNT */, ~0u /* ASYNC_CNT */,
1991 ~0u /* TENSOR_CNT */, ExpertVal, ExpertVal, ExpertVal);
1992}
1993
1994/// Combine consecutive S_WAIT_*CNT instructions that precede \p It and
1995/// follow \p OldWaitcntInstr and apply any extra waits from \p Wait that
1996/// were added by previous passes. Currently this pass conservatively
1997/// assumes that these preexisting waits are required for correctness.
1998bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1999 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
2000 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) {
2001 assert(!isNormalMode(MaxCounter));
2002
2003 bool Modified = false;
2004 MachineInstr *CombinedLoadDsCntInstr = nullptr;
2005 MachineInstr *CombinedStoreDsCntInstr = nullptr;
2006 MachineInstr *WaitcntDepctrInstr = nullptr;
2007 MachineInstr *WaitInstrs[AMDGPU::NUM_EXTENDED_INST_CNTS] = {};
2008
2009 LLVM_DEBUG({
2010 dbgs() << "GFX12Plus::applyPreexistingWaitcnt at: ";
2011 if (It.isEnd())
2012 dbgs() << "end of block\n";
2013 else
2014 dbgs() << *It;
2015 });
2016
2017 // Accumulate waits that should not be simplified.
2018 AMDGPU::Waitcnt RequiredWait;
2019
2020 for (auto &II :
2021 make_early_inc_range(Range: make_range(x: OldWaitcntInstr.getIterator(), y: It))) {
2022 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
2023 if (isNonWaitcntMetaInst(MI: II)) {
2024 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
2025 continue;
2026 }
2027
2028 // Update required wait count. If this is a soft waitcnt (= it was added
2029 // by an earlier pass), it may be entirely removed.
2030
2031 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: II.getOpcode());
2032 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
2033
2034 // Don't crash if the programmer used legacy waitcnt intrinsics, but don't
2035 // attempt to do more than that either.
2036 if (Opcode == AMDGPU::S_WAITCNT)
2037 continue;
2038
2039 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
2040 unsigned OldEnc = static_cast<unsigned>(
2041 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
2042 AMDGPU::Waitcnt OldWait = AMDGPU::decodeLoadcntDscnt(Version: IV, LoadcntDscnt: OldEnc);
2043 if (TrySimplify)
2044 Wait = Wait.combined(Other: OldWait);
2045 else
2046 RequiredWait = RequiredWait.combined(Other: OldWait);
2047 // Keep the first wait_loadcnt, erase the rest.
2048 if (CombinedLoadDsCntInstr == nullptr) {
2049 CombinedLoadDsCntInstr = &II;
2050 } else {
2051 II.eraseFromParent();
2052 Modified = true;
2053 }
2054 } else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
2055 unsigned OldEnc = static_cast<unsigned>(
2056 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
2057 AMDGPU::Waitcnt OldWait = AMDGPU::decodeStorecntDscnt(Version: IV, StorecntDscnt: OldEnc);
2058 if (TrySimplify)
2059 Wait = Wait.combined(Other: OldWait);
2060 else
2061 RequiredWait = RequiredWait.combined(Other: OldWait);
2062 // Keep the first wait_storecnt, erase the rest.
2063 if (CombinedStoreDsCntInstr == nullptr) {
2064 CombinedStoreDsCntInstr = &II;
2065 } else {
2066 II.eraseFromParent();
2067 Modified = true;
2068 }
2069 } else if (Opcode == AMDGPU::S_WAITCNT_DEPCTR) {
2070 unsigned OldEnc = static_cast<unsigned>(
2071 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
2072 AMDGPU::Waitcnt OldWait;
2073 // Set both counters to the decoded value from the single hardware field
2074 unsigned VaVdst = AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: OldEnc);
2075 OldWait.set(T: AMDGPU::VA_VDST_RD, Val: VaVdst);
2076 OldWait.set(T: AMDGPU::VA_VDST_WR, Val: VaVdst);
2077 OldWait.set(T: AMDGPU::VM_VSRC, Val: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: OldEnc));
2078 if (TrySimplify)
2079 ScoreBrackets.simplifyWaitcnt(Wait&: OldWait);
2080 Wait = Wait.combined(Other: OldWait);
2081 if (WaitcntDepctrInstr == nullptr) {
2082 WaitcntDepctrInstr = &II;
2083 } else {
2084 // S_WAITCNT_DEPCTR requires special care. Don't remove a
2085 // duplicate if it is waiting on things other than VA_VDST or
2086 // VM_VSRC. If that is the case, just make sure the VA_VDST and
2087 // VM_VSRC subfields of the operand are set to the "no wait"
2088 // values.
2089
2090 unsigned Enc = static_cast<unsigned>(
2091 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
2092 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: ~0u);
2093 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2094 unsigned VaVdst = std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD),
2095 b: Wait.get(T: AMDGPU::VA_VDST_WR));
2096 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2097
2098 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: ST)) {
2099 Modified |= updateOperandIfDifferent(MI&: II, OpName: AMDGPU::OpName::simm16, NewEnc: Enc);
2100 Modified |= promoteSoftWaitCnt(Waitcnt: &II);
2101 } else {
2102 II.eraseFromParent();
2103 Modified = true;
2104 }
2105 }
2106 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
2107 // Architectures higher than GFX10 do not have direct loads to
2108 // LDS, so no work required here yet.
2109 II.eraseFromParent();
2110 Modified = true;
2111 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
2112 // Update the Waitcnt, but don't erase the wait.asyncmark() itself. It
2113 // shows up in the assembly as a comment with the original parameter N.
2114 unsigned N = static_cast<unsigned>(II.getOperand(i: 0).getImm());
2115 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(
2116 N, Stages: AMDGPU::asyncStagesFromMaskOperand(Mask: II.getOperand(i: 1).getImm()));
2117 Wait = Wait.combined(Other: OldWait);
2118 } else {
2119 std::optional<AMDGPU::InstCounterType> CT =
2120 AMDGPU::counterTypeForInstr(Opcode);
2121 assert(CT.has_value());
2122 unsigned OldCnt = static_cast<unsigned>(
2123 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm());
2124 if (TrySimplify)
2125 Wait.add(T: CT.value(), Count: OldCnt);
2126 else
2127 RequiredWait.add(T: CT.value(), Count: OldCnt);
2128 // Keep the first wait of its kind, erase the rest.
2129 if (WaitInstrs[CT.value()] == nullptr) {
2130 WaitInstrs[CT.value()] = &II;
2131 } else {
2132 II.eraseFromParent();
2133 Modified = true;
2134 }
2135 }
2136 }
2137
2138 ScoreBrackets.simplifyWaitcnt(CheckWait: Wait.combined(Other: RequiredWait), UpdateWait&: Wait);
2139 Wait = Wait.combined(Other: RequiredWait);
2140
2141 if (CombinedLoadDsCntInstr) {
2142 // Only keep an S_WAIT_LOADCNT_DSCNT if both counters actually need
2143 // to be waited for. Otherwise, let the instruction be deleted so
2144 // the appropriate single counter wait instruction can be inserted
2145 // instead, when new S_WAIT_*CNT instructions are inserted by
2146 // createNewWaitcnt(). As a side effect, resetting the wait counts will
2147 // cause any redundant S_WAIT_LOADCNT or S_WAIT_DSCNT to be removed by
2148 // the loop below that deals with single counter instructions.
2149 //
2150 // A wait for LOAD_CNT or DS_CNT implies a wait for VM_VSRC, since
2151 // instructions that have decremented LOAD_CNT or DS_CNT on completion
2152 // will have needed to wait for their register sources to be available
2153 // first.
2154 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u && Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2155 unsigned NewEnc = AMDGPU::encodeLoadcntDscnt(Version: IV, Decoded: Wait);
2156 Modified |= updateOperandIfDifferent(MI&: *CombinedLoadDsCntInstr,
2157 OpName: AMDGPU::OpName::simm16, NewEnc);
2158 Modified |= promoteSoftWaitCnt(Waitcnt: CombinedLoadDsCntInstr);
2159 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::LOAD_CNT);
2160 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
2161 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2162 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2163
2164 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2165 << "New Instr at block end: "
2166 << *CombinedLoadDsCntInstr << '\n'
2167 : dbgs() << "applied pre-existing waitcnt\n"
2168 << "Old Instr: " << *It << "New Instr: "
2169 << *CombinedLoadDsCntInstr << '\n');
2170 } else {
2171 CombinedLoadDsCntInstr->eraseFromParent();
2172 Modified = true;
2173 }
2174 }
2175
2176 if (CombinedStoreDsCntInstr) {
2177 // Similarly for S_WAIT_STORECNT_DSCNT.
2178 if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u && Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2179 unsigned NewEnc = AMDGPU::encodeStorecntDscnt(Version: IV, Decoded: Wait);
2180 Modified |= updateOperandIfDifferent(MI&: *CombinedStoreDsCntInstr,
2181 OpName: AMDGPU::OpName::simm16, NewEnc);
2182 Modified |= promoteSoftWaitCnt(Waitcnt: CombinedStoreDsCntInstr);
2183 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::STORE_CNT);
2184 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
2185 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
2186 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2187
2188 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2189 << "New Instr at block end: "
2190 << *CombinedStoreDsCntInstr << '\n'
2191 : dbgs() << "applied pre-existing waitcnt\n"
2192 << "Old Instr: " << *It << "New Instr: "
2193 << *CombinedStoreDsCntInstr << '\n');
2194 } else {
2195 CombinedStoreDsCntInstr->eraseFromParent();
2196 Modified = true;
2197 }
2198 }
2199
2200 // Look for an opportunity to convert existing S_WAIT_LOADCNT,
2201 // S_WAIT_STORECNT and S_WAIT_DSCNT into new S_WAIT_LOADCNT_DSCNT
2202 // or S_WAIT_STORECNT_DSCNT. This is achieved by selectively removing
2203 // instructions so that createNewWaitcnt() will create new combined
2204 // instructions to replace them.
2205
2206 if (Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2207 // This is a vector of addresses in WaitInstrs pointing to instructions
2208 // that should be removed if they are present.
2209 SmallVector<MachineInstr **, 2> WaitsToErase;
2210
2211 // If it's known that both DScnt and either LOADcnt or STOREcnt (but not
2212 // both) need to be waited for, ensure that there are no existing
2213 // individual wait count instructions for these.
2214
2215 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u) {
2216 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::LOAD_CNT]);
2217 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::DS_CNT]);
2218 } else if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u) {
2219 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::STORE_CNT]);
2220 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::DS_CNT]);
2221 }
2222
2223 for (MachineInstr **WI : WaitsToErase) {
2224 if (!*WI)
2225 continue;
2226
2227 DeferredErasures.erase(V: *WI);
2228 (*WI)->eraseFromParent();
2229 *WI = nullptr;
2230 Modified = true;
2231 }
2232 }
2233
2234 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2235 if (!WaitInstrs[CT])
2236 continue;
2237
2238 unsigned NewCnt = Wait.get(T: CT);
2239 if (NewCnt != ~0u) {
2240 Modified |= updateOperandIfDifferent(MI&: *WaitInstrs[CT],
2241 OpName: AMDGPU::OpName::simm16, NewEnc: NewCnt);
2242 Modified |= promoteSoftWaitCnt(Waitcnt: WaitInstrs[CT]);
2243 DeferredErasures.erase(V: WaitInstrs[CT]);
2244
2245 ScoreBrackets.applyWaitcnt(T: CT, Count: NewCnt);
2246 Wait.clear(T: CT);
2247
2248 LLVM_DEBUG(It.isEnd()
2249 ? dbgs() << "applied pre-existing waitcnt\n"
2250 << "New Instr at block end: " << *WaitInstrs[CT]
2251 << '\n'
2252 : dbgs() << "applied pre-existing waitcnt\n"
2253 << "Old Instr: " << *It
2254 << "New Instr: " << *WaitInstrs[CT] << '\n');
2255 } else {
2256 // Defer erasure in case there's a loop-carried dependency.
2257 DeferredErasures.insert(V: WaitInstrs[CT]);
2258 }
2259 }
2260
2261 if (WaitcntDepctrInstr) {
2262 // Get the encoded Depctr immediate and override the VA_VDST and VM_VSRC
2263 // subfields with the new required values.
2264 unsigned Enc = static_cast<unsigned>(
2265 TII.getNamedOperand(MI&: *WaitcntDepctrInstr, OperandName: AMDGPU::OpName::simm16)
2266 ->getImm());
2267 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: Wait.get(T: AMDGPU::VM_VSRC));
2268 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2269 unsigned VaVdst =
2270 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
2271 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2272
2273 ScoreBrackets.applyWaitcnt(T: AMDGPU::VA_VDST_RD, Count: VaVdst);
2274 ScoreBrackets.applyWaitcnt(T: AMDGPU::VA_VDST_WR, Count: VaVdst);
2275 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::VM_VSRC);
2276 Wait.set(T: AMDGPU::VA_VDST_RD, Val: ~0u);
2277 Wait.set(T: AMDGPU::VA_VDST_WR, Val: ~0u);
2278 Wait.set(T: AMDGPU::VM_VSRC, Val: ~0u);
2279
2280 // If that new encoded Depctr immediate would actually still wait
2281 // for anything, update the instruction's operand. Otherwise it can
2282 // just be deleted.
2283 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: ST)) {
2284 Modified |= updateOperandIfDifferent(MI&: *WaitcntDepctrInstr,
2285 OpName: AMDGPU::OpName::simm16, NewEnc: Enc);
2286 LLVM_DEBUG(It.isEnd() ? dbgs() << "applyPreexistingWaitcnt\n"
2287 << "New Instr at block end: "
2288 << *WaitcntDepctrInstr << '\n'
2289 : dbgs() << "applyPreexistingWaitcnt\n"
2290 << "Old Instr: " << *It << "New Instr: "
2291 << *WaitcntDepctrInstr << '\n');
2292 } else {
2293 WaitcntDepctrInstr->eraseFromParent();
2294 Modified = true;
2295 }
2296 }
2297
2298 return Modified;
2299}
2300
2301/// Generate S_WAIT_*CNT instructions for any required counters in \p Wait
2302bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
2303 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
2304 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
2305 assert(!isNormalMode(MaxCounter));
2306
2307 bool Modified = false;
2308 const DebugLoc &DL = Block.findDebugLoc(MBBI: It);
2309
2310 // For GFX12+, we use separate wait instructions, which makes expansion
2311 // simpler
2312 if (ExpandWaitcntProfiling) {
2313 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2314 unsigned Count = Wait.get(T: CT);
2315 if (Count == ~0u)
2316 continue;
2317
2318 // Skip expansion for out-of-order counters - emit normal wait instead
2319 if (ScoreBrackets.counterOutOfOrder(T: CT)) {
2320 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2321 .addImm(Val: Count);
2322 Modified = true;
2323 continue;
2324 }
2325
2326 unsigned Outstanding =
2327 std::min(a: ScoreBrackets.getOutstanding(T: CT), b: getLimit(E: CT) - 1);
2328 EmitExpandedWaitcnt(Outstanding, Target: Count, EmitWaitcnt: [&](unsigned Val) {
2329 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2330 .addImm(Val);
2331 });
2332 Modified = true;
2333 }
2334 return Modified;
2335 }
2336
2337 // Normal behavior (no expansion)
2338 // Check for opportunities to use combined wait instructions.
2339 if (Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2340 MachineInstr *SWaitInst = nullptr;
2341
2342 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u) {
2343 unsigned Enc = AMDGPU::encodeLoadcntDscnt(Version: IV, Decoded: Wait);
2344
2345 SWaitInst = BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAIT_LOADCNT_DSCNT))
2346 .addImm(Val: Enc);
2347
2348 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2349 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2350 } else if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u) {
2351 unsigned Enc = AMDGPU::encodeStorecntDscnt(Version: IV, Decoded: Wait);
2352
2353 SWaitInst = BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAIT_STORECNT_DSCNT))
2354 .addImm(Val: Enc);
2355
2356 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
2357 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2358 }
2359
2360 if (SWaitInst) {
2361 Modified = true;
2362
2363 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2364 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2365 dbgs() << "New Instr: " << *SWaitInst << '\n');
2366 }
2367 }
2368
2369 // Generate an instruction for any remaining counter that needs
2370 // waiting for.
2371
2372 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2373 unsigned Count = Wait.get(T: CT);
2374 if (Count == ~0u)
2375 continue;
2376
2377 [[maybe_unused]] auto SWaitInst =
2378 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2379 .addImm(Val: Count);
2380
2381 Modified = true;
2382
2383 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2384 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2385 dbgs() << "New Instr: " << *SWaitInst << '\n');
2386 }
2387
2388 if (Wait.hasWaitDepctr()) {
2389 assert(IsExpertMode);
2390 unsigned Enc =
2391 AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: Wait.get(T: AMDGPU::VM_VSRC), STI: ST);
2392 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2393 unsigned VaVdst =
2394 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
2395 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2396
2397 [[maybe_unused]] auto SWaitInst =
2398 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR)).addImm(Val: Enc);
2399
2400 Modified = true;
2401
2402 LLVM_DEBUG(dbgs() << "generateWaitcnt\n";
2403 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2404 dbgs() << "New Instr: " << *SWaitInst << '\n');
2405 }
2406
2407 return Modified;
2408}
2409
2410/// Generate s_waitcnt instruction to be placed before cur_Inst.
2411/// Instructions of a given type are returned in order,
2412/// but instructions of different types can complete out of order.
2413/// We rely on this in-order completion
2414/// and simply assign a score to the memory access instructions.
2415/// We keep track of the active "score bracket" to determine
2416/// if an access of a memory read requires an s_waitcnt
2417/// and if so what the value of each counter is.
2418/// The "score bracket" is bound by the lower bound and upper bound
2419/// scores (*_score_LB and *_score_ub respectively).
2420/// If FlushFlags.FlushVmCnt is true, we want to flush the vmcnt counter here.
2421/// If FlushFlags.FlushDsCnt is true, we want to flush the dscnt counter here
2422/// (GFX12+ only, where DS_CNT is a separate counter).
2423bool SIInsertWaitcnts::generateWaitcntInstBefore(
2424 MachineInstr &MI, WaitcntBrackets &ScoreBrackets,
2425 MachineInstr *OldWaitcntInstr, PreheaderFlushFlags FlushFlags) {
2426 LLVM_DEBUG(dbgs() << "\n*** GenerateWaitcntInstBefore: "; MI.print(dbgs()););
2427
2428 assert(!isNonWaitcntMetaInst(MI));
2429
2430 AMDGPU::Waitcnt Wait;
2431 const unsigned Opc = MI.getOpcode();
2432
2433 switch (Opc) {
2434 case AMDGPU::BUFFER_WBINVL1:
2435 case AMDGPU::BUFFER_WBINVL1_SC:
2436 case AMDGPU::BUFFER_WBINVL1_VOL:
2437 case AMDGPU::BUFFER_GL0_INV:
2438 case AMDGPU::BUFFER_GL1_INV: {
2439 // FIXME: This should have already been handled by the memory legalizer.
2440 // Removing this currently doesn't affect any lit tests, but we need to
2441 // verify that nothing was relying on this. The number of buffer invalidates
2442 // being handled here should not be expanded.
2443 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2444 break;
2445 }
2446 case AMDGPU::SI_RETURN_TO_EPILOG:
2447 case AMDGPU::SI_RETURN:
2448 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
2449 case AMDGPU::S_SETPC_B64_return: {
2450 // All waits must be resolved at call return.
2451 // NOTE: this could be improved with knowledge of all call sites or
2452 // with knowledge of the called routines.
2453 ReturnInsts.insert(V: &MI);
2454 AMDGPU::Waitcnt AllZeroWait =
2455 WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2456 ScoreBrackets.simplifyVmemInvOnlyWait(Wait&: AllZeroWait);
2457 Wait = AllZeroWait;
2458 break;
2459 }
2460 case AMDGPU::S_ENDPGM:
2461 case AMDGPU::S_ENDPGM_SAVED: {
2462 // In dynamic VGPR mode, we want to release the VGPRs before the wave exits.
2463 // Technically the hardware will do this on its own if we don't, but that
2464 // might cost extra cycles compared to doing it explicitly.
2465 // When not in dynamic VGPR mode, identify S_ENDPGM instructions which may
2466 // have to wait for outstanding VMEM stores. In this case it can be useful
2467 // to send a message to explicitly release all VGPRs before the stores have
2468 // completed, but it is only safe to do this if there are no outstanding
2469 // scratch stores.
2470 EndPgmInsts[&MI] =
2471 !ScoreBrackets.empty(T: AMDGPU::STORE_CNT) &&
2472 !ScoreBrackets.hasPendingEvent(E: HWEvents::SCRATCH_WRITE_ACCESS);
2473 break;
2474 }
2475 case AMDGPU::S_SENDMSG:
2476 case AMDGPU::S_SENDMSGHALT: {
2477 if (ST.hasLegacyGeometry() &&
2478 ((MI.getOperand(i: 0).getImm() & AMDGPU::SendMsg::ID_MASK_PreGFX11_) ==
2479 AMDGPU::SendMsg::ID_GS_DONE_PreGFX11)) {
2480 // Resolve vm waits before gs-done.
2481 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2482 break;
2483 }
2484 [[fallthrough]];
2485 }
2486 default: {
2487
2488 // Export & GDS instructions do not read the EXEC mask until after the
2489 // export is granted (which can occur well after the instruction is issued).
2490 // The shader program must flush all EXP operations on the export-count
2491 // before overwriting the EXEC mask.
2492 if (MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI)) {
2493 // Export and GDS are tracked individually, either may trigger a waitcnt
2494 // for EXEC.
2495 if (ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_GPR_LOCK) ||
2496 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_PARAM_ACCESS) ||
2497 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_POS_ACCESS) ||
2498 ScoreBrackets.hasPendingEvent(E: HWEvents::GDS_GPR_LOCK)) {
2499 Wait.set(T: AMDGPU::EXP_CNT, Val: 0);
2500 }
2501 }
2502
2503 // Wait for any pending GDS instruction to complete before any
2504 // "Always GDS" instruction.
2505 if (TII.isAlwaysGDS(Opcode: Opc) && ScoreBrackets.hasPendingGDS())
2506 Wait.add(T: AMDGPU::DS_CNT, Count: ScoreBrackets.getPendingGDSWait());
2507
2508 if (MI.isCall()) {
2509 // The function is going to insert a wait on everything in its prolog.
2510 // This still needs to be careful if the call target is a load (e.g. a GOT
2511 // load). We also need to check WAW dependency with saved PC.
2512 CallInsts.insert(V: &MI);
2513 Wait = AMDGPU::Waitcnt();
2514
2515 const MachineOperand &CallAddrOp = TII.getCalleeOperand(MI);
2516 if (CallAddrOp.isReg()) {
2517 ScoreBrackets.determineWaitForPhysReg(
2518 T: SmemAccessCounter, Reg: CallAddrOp.getReg().asMCReg(), Wait, MI);
2519
2520 if (const auto *RtnAddrOp =
2521 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::dst)) {
2522 ScoreBrackets.determineWaitForPhysReg(
2523 T: SmemAccessCounter, Reg: RtnAddrOp->getReg().asMCReg(), Wait, MI);
2524 }
2525 }
2526 } else if (Opc == AMDGPU::S_BARRIER_WAIT) {
2527 ScoreBrackets.tryClearSCCWriteEvent(Inst: &MI);
2528 } else {
2529 // FIXME: Should not be relying on memoperands.
2530 // Look at the source operands of every instruction to see if
2531 // any of them results from a previous memory operation that affects
2532 // its current usage. If so, an s_waitcnt instruction needs to be
2533 // emitted.
2534 // If the source operand was defined by a load, add the s_waitcnt
2535 // instruction.
2536 //
2537 // Two cases are handled for destination operands:
2538 // 1) If the destination operand was defined by a load, add the s_waitcnt
2539 // instruction to guarantee the right WAW order.
2540 // 2) If a destination operand that was used by a recent export/store ins,
2541 // add s_waitcnt on exp_cnt to guarantee the WAR order.
2542
2543 for (const MachineMemOperand *Memop : MI.memoperands()) {
2544 const Value *Ptr = Memop->getValue();
2545 if (Memop->isStore()) {
2546 if (auto It = SLoadAddresses.find(Val: Ptr); It != SLoadAddresses.end()) {
2547 Wait.add(T: SmemAccessCounter, Count: 0);
2548 if (PDT.dominates(A: MI.getParent(), B: It->second))
2549 SLoadAddresses.erase(I: It);
2550 }
2551 }
2552 unsigned AS = Memop->getAddrSpace();
2553 if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::FLAT_ADDRESS)
2554 continue;
2555 // No need to wait before load from VMEM to LDS.
2556 if (TII.mayWriteLDSThroughDMA(MI))
2557 continue;
2558
2559 // LOAD_CNT is only relevant to vgpr or LDS.
2560 VMEMID TID = LDSDMA_BEGIN;
2561 if (Ptr && Memop->getAAInfo()) {
2562 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
2563 for (size_t I = 0, E = LDSDMAStores.size(); I != E; ++I) {
2564 if (MI.mayAlias(AA, Other: *LDSDMAStores[I], UseTBAA: true)) {
2565 size_t Slot = I + 1;
2566 if (Slot >= NUM_LDSDMA) {
2567 // We didn't have enough slot to track this LDS DMA store, it
2568 // has been tracked using the common RegNo (FIRST_LDS_VGPR).
2569 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID,
2570 Wait);
2571 break;
2572 }
2573
2574 ScoreBrackets.determineWaitForLDSDMA(
2575 T: AMDGPU::LOAD_CNT, TID: TID + static_cast<VMEMID>(Slot), Wait);
2576 }
2577 }
2578 } else {
2579 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID, Wait);
2580 }
2581 if (Memop->isStore()) {
2582 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::EXP_CNT, TID, Wait);
2583 }
2584 }
2585
2586 // Loop over use and def operands.
2587 for (const MachineOperand &Op : MI.operands()) {
2588 if (!Op.isReg())
2589 continue;
2590
2591 // If the instruction does not read tied source, skip the operand.
2592 if (Op.isTied() && Op.isUse() && TII.doesNotReadTiedSource(MI))
2593 continue;
2594
2595 MCRegister Reg = Op.getReg().asMCReg();
2596
2597 const bool IsVGPR = TRI.isVectorRegister(MRI, Reg: Op.getReg());
2598 if (IsVGPR) {
2599 // Implicit VGPR defs and uses are never a part of the memory
2600 // instructions description and usually present to account for
2601 // super-register liveness.
2602 // TODO: Most of the other instructions also have implicit uses
2603 // for the liveness accounting only.
2604 if (Op.isImplicit() && MI.mayLoadOrStore())
2605 continue;
2606
2607 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VA_VDST_WR, Reg, Wait,
2608 MI);
2609 if (Op.isDef()) {
2610 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VA_VDST_RD, Reg, Wait,
2611 MI);
2612 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VM_VSRC, Reg, Wait,
2613 MI);
2614 }
2615
2616 // RAW always needs an s_waitcnt. WAW needs an s_waitcnt unless the
2617 // previous write and this write are the same type of VMEM
2618 // instruction, in which case they are (in some architectures)
2619 // guaranteed to write their results in order anyway.
2620 // Additionally check instructions where Point Sample Acceleration
2621 // might be applied.
2622 if (Op.isUse() || !updateVMCntOnly(Inst: MI) ||
2623 ScoreBrackets.hasDifferentVGPRPendingEvents(
2624 Reg, E: AMDGPU::getSimplifiedVMEMEventsFor(Inst: MI, TII)) ||
2625 ScoreBrackets.hasPointSamplePendingVmemTypes(MI, Reg) ||
2626 !ST.hasVmemWriteVgprInOrder()) {
2627 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::LOAD_CNT, Reg, Wait,
2628 MI);
2629 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::SAMPLE_CNT, Reg, Wait,
2630 MI);
2631 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::BVH_CNT, Reg, Wait,
2632 MI);
2633 ScoreBrackets.clearVGPRPendingEvents(Reg);
2634 }
2635
2636 if (Op.isDef() ||
2637 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_LDS_ACCESS)) {
2638 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::EXP_CNT, Reg, Wait,
2639 MI);
2640 }
2641 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::DS_CNT, Reg, Wait, MI);
2642 } else if (Op.getReg() == AMDGPU::SCC) {
2643 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::KM_CNT, Reg, Wait, MI);
2644 } else {
2645 ScoreBrackets.determineWaitForPhysReg(T: SmemAccessCounter, Reg, Wait,
2646 MI);
2647 }
2648
2649 if (ST.hasWaitXcnt() && ST.isXNACKEnabled() && Op.isDef())
2650 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::X_CNT, Reg, Wait, MI);
2651 }
2652 }
2653 }
2654 }
2655
2656 // Ensure safety against exceptions from outstanding memory operations while
2657 // waiting for a barrier:
2658 //
2659 // * Some subtargets safely handle backing off the barrier in hardware
2660 // when an exception occurs.
2661 // * Some subtargets have an implicit S_WAITCNT 0 before barriers, so that
2662 // there can be no outstanding memory operations during the wait.
2663 // * Subtargets with split barriers don't need to back off the barrier; it
2664 // is up to the trap handler to preserve the user barrier state correctly.
2665 //
2666 // In all other cases, ensure safety by ensuring that there are no outstanding
2667 // memory operations.
2668 if (Opc == AMDGPU::S_BARRIER && !ST.hasAutoWaitcntBeforeBarrier() &&
2669 !ST.hasBackOffBarrier()) {
2670 Wait = Wait.combined(Other: WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/true));
2671 }
2672
2673 // TODO: Remove this work-around, enable the assert for Bug 457939
2674 // after fixing the scheduler. Also, the Shader Compiler code is
2675 // independent of target.
2676 if (SIInstrInfo::isCBranchVCCZRead(MI) && ST.hasReadVCCZBug() &&
2677 ScoreBrackets.hasPendingEvent(E: HWEvents::SMEM_ACCESS)) {
2678 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2679 }
2680
2681 // Verify that the wait is actually needed.
2682 ScoreBrackets.simplifyWaitcnt(Wait);
2683
2684 // It is only necessary to insert an S_WAITCNT_DEPCTR instruction that
2685 // waits on VA_VDST if the instruction it would precede is not a VALU
2686 // instruction, since hardware handles VALU->VGPR->VALU hazards in
2687 // expert scheduling mode.
2688 if (TII.isVALU(MI, /*AllowLDSDMA=*/false)) {
2689 Wait.set(T: AMDGPU::VA_VDST_RD, Val: ~0u);
2690 Wait.set(T: AMDGPU::VA_VDST_WR, Val: ~0u);
2691 }
2692
2693 // Since the translation for VMEM addresses occur in-order, we can apply the
2694 // XCnt if the current instruction is of VMEM type and has a memory
2695 // dependency with another VMEM instruction in flight.
2696 if (Wait.get(T: AMDGPU::X_CNT) != ~0u && isVmemAccess(MI)) {
2697 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::X_CNT);
2698 Wait.set(T: AMDGPU::X_CNT, Val: ~0u);
2699 }
2700
2701 // When forcing emit, we need to skip terminators because that would break the
2702 // terminators of the MBB if we emit a waitcnt between terminators.
2703 if (ST.getCLOpts().waitcnt_forcezero && !MI.isTerminator())
2704 Wait = WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2705
2706 // If we force waitcnt then update Wait accordingly.
2707 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
2708 if (!ForceEmitWaitcnt[T])
2709 continue;
2710 Wait.set(T, Val: 0);
2711 }
2712
2713 if (FlushFlags.FlushVmCnt) {
2714 for (AMDGPU::InstCounterType T :
2715 {AMDGPU::LOAD_CNT, AMDGPU::SAMPLE_CNT, AMDGPU::BVH_CNT})
2716 Wait.set(T, Val: 0);
2717 }
2718
2719 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(T: AMDGPU::DS_CNT))
2720 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2721
2722 if (ST.getCLOpts().waitcnt_load_forcezero &&
2723 Wait.get(T: AMDGPU::LOAD_CNT) != ~0u)
2724 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2725
2726 return generateWaitcnt(Wait, It: MI.getIterator(), Block&: *MI.getParent(), ScoreBrackets,
2727 OldWaitcntInstr);
2728}
2729
2730bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
2731 MachineBasicBlock::instr_iterator It,
2732 MachineBasicBlock &Block,
2733 WaitcntBrackets &ScoreBrackets,
2734 MachineInstr *OldWaitcntInstr) {
2735 bool Modified = false;
2736
2737 if (OldWaitcntInstr)
2738 // Try to merge the required wait with preexisting waitcnt instructions.
2739 // Also erase redundant waitcnt.
2740 Modified =
2741 WCG->applyPreexistingWaitcnt(ScoreBrackets, OldWaitcntInstr&: *OldWaitcntInstr, Wait, It);
2742
2743 // ExpCnt can be merged into VINTERP.
2744 if (Wait.get(T: AMDGPU::EXP_CNT) != ~0u && It != Block.instr_end() &&
2745 SIInstrInfo::isVINTERP(MI: *It)) {
2746 MachineOperand *WaitExp = TII.getNamedOperand(MI&: *It, OperandName: AMDGPU::OpName::waitexp);
2747 if (Wait.get(T: AMDGPU::EXP_CNT) < WaitExp->getImm()) {
2748 WaitExp->setImm(Wait.get(T: AMDGPU::EXP_CNT));
2749 Modified = true;
2750 }
2751 // Apply ExpCnt before resetting it, so applyWaitcnt below sees all counts.
2752 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::EXP_CNT);
2753 Wait.set(T: AMDGPU::EXP_CNT, Val: ~0u);
2754
2755 LLVM_DEBUG(dbgs() << "generateWaitcnt\n"
2756 << "Update Instr: " << *It);
2757 }
2758
2759 if (WCG->createNewWaitcnt(Block, It, Wait, ScoreBrackets))
2760 Modified = true;
2761
2762 // Any counts that could have been applied to any existing waitcnt
2763 // instructions will have been done so, now deal with any remaining.
2764 ScoreBrackets.applyWaitcnt(Wait);
2765
2766 return Modified;
2767}
2768
2769bool SIInsertWaitcnts::isVmemAccess(const MachineInstr &MI) const {
2770 return (TII.isFLAT(MI) && TII.mayAccessVMEMThroughFlat(MI)) ||
2771 (TII.isVMEM(MI) && !AMDGPU::getMUBUFIsBufferInv(Opc: MI.getOpcode()));
2772}
2773
2774// Return true if the next instruction is S_ENDPGM, following fallthrough
2775// blocks if necessary.
2776bool SIInsertWaitcnts::isNextENDPGM(MachineBasicBlock::instr_iterator It,
2777 MachineBasicBlock *Block) const {
2778 auto BlockEnd = Block->getParent()->end();
2779 auto BlockIter = Block->getIterator();
2780
2781 while (true) {
2782 if (It.isEnd()) {
2783 if (++BlockIter != BlockEnd) {
2784 It = BlockIter->instr_begin();
2785 continue;
2786 }
2787
2788 return false;
2789 }
2790
2791 if (!It->isMetaInstruction())
2792 break;
2793
2794 It++;
2795 }
2796
2797 assert(!It.isEnd());
2798
2799 return It->getOpcode() == AMDGPU::S_ENDPGM;
2800}
2801
2802// Add a wait after an instruction if architecture requirements mandate one.
2803bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
2804 MachineBasicBlock &Block,
2805 WaitcntBrackets &ScoreBrackets) {
2806 AMDGPU::Waitcnt Wait;
2807 bool NeedsEndPGMCheck = false;
2808
2809 if (ST.isPreciseMemoryEnabled() && Inst.mayLoadOrStore())
2810 Wait = WCG->getAllZeroWaitcnt(IncludeVSCnt: Inst.mayStore() &&
2811 !SIInstrInfo::isAtomicRet(MI: Inst));
2812
2813 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode())) {
2814 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2815 NeedsEndPGMCheck = true;
2816 }
2817
2818 ScoreBrackets.simplifyWaitcnt(Wait);
2819
2820 auto SuccessorIt = next_nodbg(It: Inst.getIterator(), End: Block.instr_end());
2821 bool Result = generateWaitcnt(Wait, It: SuccessorIt, Block, ScoreBrackets,
2822 /*OldWaitcntInstr=*/nullptr);
2823
2824 if (Result && NeedsEndPGMCheck && isNextENDPGM(It: SuccessorIt, Block: &Block)) {
2825 BuildMI(BB&: Block, I: SuccessorIt, MIMD: Inst.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_NOP))
2826 .addImm(Val: 0);
2827 }
2828
2829 return Result;
2830}
2831
2832void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
2833 WaitcntBrackets *ScoreBrackets) {
2834
2835 HWEvents InstEvents = AMDGPU::getEventsFor(Inst, ST, IsExpertMode, TgSplit);
2836 for (HWEvents E : InstEvents)
2837 ScoreBrackets->updateByEvent(E, Inst);
2838
2839 if (TII.isDS(MI: Inst) && TII.usesLGKM_CNT(MI: Inst)) {
2840 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode()) ||
2841 TII.hasModifiersSet(MI: Inst, OpName: AMDGPU::OpName::gds)) {
2842 ScoreBrackets->setPendingGDS();
2843 }
2844 } else if (TII.isFLAT(MI: Inst)) {
2845 if (InstEvents.contains(Other: HWEvents::VMEM_GROUP) &&
2846 !TII.mayAccessVMEMThroughFlat(MI: Inst) &&
2847 !InstEvents.contains(Other: HWEvents::ASYNC_ACCESS))
2848 ScoreBrackets->setPendingXcntOnlyVmem();
2849
2850 if (Inst.mayLoadOrStore() && TII.mayAccessVMEMThroughFlat(MI: Inst) &&
2851 TII.mayAccessLDSThroughFlat(MI: Inst, TgSplit) &&
2852 !SIInstrInfo::isLDSDMA(MI: Inst)) {
2853 // Async/LDSDMA operations have FLAT encoding but do not actually use flat
2854 // pointers. They do have two operands that each access global and LDS,
2855 // thus making it appear at this point that they are using a flat pointer.
2856 // Filter them out, and for the rest, generate a dependency on flat
2857 // pointers so that both VM and LGKM counters are flushed.
2858 ScoreBrackets->setPendingFlat();
2859 }
2860 } else if (Inst.isCall()) {
2861 // Act as a wait on everything, but AsyncCnt and TensorCnt are never
2862 // included in such blanket waits.
2863 ScoreBrackets->applyWaitcnt(Wait: WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false));
2864 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2865 } else if (TII.isVINTERP(MI: Inst)) {
2866 int64_t Imm = TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::waitexp)->getImm();
2867 ScoreBrackets->applyWaitcnt(T: AMDGPU::EXP_CNT, Count: static_cast<unsigned>(Imm));
2868 }
2869
2870 // Set XCNT to zero in the bracket for instructions that implicitly drain
2871 // XCNT.
2872 if (ST.hasWaitXcnt() && ST.isXNACKEnabled() && SIInstrInfo::isXcntDrain(MI: Inst))
2873 ScoreBrackets->applyWaitcnt(T: AMDGPU::X_CNT, Count: 0);
2874}
2875
2876bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
2877 unsigned OtherScore) {
2878 unsigned MyShifted = Score <= M.OldLB ? 0 : Score + M.MyShift;
2879 unsigned OtherShifted =
2880 OtherScore <= M.OtherLB ? 0 : OtherScore + M.OtherShift;
2881 Score = std::max(a: MyShifted, b: OtherShifted);
2882 return OtherShifted > MyShifted;
2883}
2884
2885bool WaitcntBrackets::mergeAsyncMarks(AMDGPU::AsyncStages Stage,
2886 ArrayRef<MergeInfo> MergeInfos,
2887 ArrayRef<CounterValueArray> OtherMarks) {
2888 bool StrictDom = false;
2889 auto &StageMarks = AsyncMarks[AMDGPU::getAsyncStageIndex(Stage)];
2890
2891 LLVM_DEBUG(dbgs() << "Merging async marks of stage " << Stage << " ...");
2892 // Early exit: nothing to merge when both sides are empty.
2893 if (StageMarks.empty() && OtherMarks.empty()) {
2894 LLVM_DEBUG(dbgs() << " nothing to merge\n");
2895 return false;
2896 }
2897 LLVM_DEBUG(dbgs() << '\n');
2898
2899 // Determine maximum length needed after merging
2900 auto MaxSize = (unsigned)std::max(a: StageMarks.size(), b: OtherMarks.size());
2901 MaxSize = std::min(a: MaxSize, b: MaxAsyncMarks);
2902
2903 // Keep only the most recent marks within our limit.
2904 if (StageMarks.size() > MaxSize)
2905 StageMarks.erase(CS: StageMarks.begin(),
2906 CE: StageMarks.begin() + (StageMarks.size() - MaxSize));
2907
2908 // Pad with zero-filled marks if our list is shorter. Zero represents "no
2909 // pending async operations at this checkpoint" and acts as the identity
2910 // element for max() during merging. We pad at the beginning since the marks
2911 // need to be aligned in most-recent order.
2912 constexpr CounterValueArray ZeroMark{};
2913 StageMarks.insert(I: StageMarks.begin(), NumToInsert: MaxSize - StageMarks.size(), Elt: ZeroMark);
2914
2915 LLVM_DEBUG({
2916 dbgs() << "Before merge:\n";
2917 for (const auto &Mark : StageMarks) {
2918 llvm::interleaveComma(Mark, dbgs());
2919 dbgs() << '\n';
2920 }
2921 dbgs() << "Other marks:\n";
2922 for (const auto &Mark : OtherMarks) {
2923 llvm::interleaveComma(Mark, dbgs());
2924 dbgs() << '\n';
2925 }
2926 });
2927
2928 // Merge element-wise using the existing mergeScore function and the
2929 // appropriate MergeInfo for each counter type. Rebase and merge EVERY
2930 // surviving mark into the new (widened) frame, aligned by recency from the
2931 // back. A mark with no counterpart on the other side merges against the
2932 // zero/identity mark: mergeScore still applies MyShift, so the mark is
2933 // re-expressed in the new frame instead of being left with a stale
2934 // (pre-merge) score.
2935 const size_t OtherSize = OtherMarks.size();
2936 const size_t OurSize = StageMarks.size();
2937 // After the both-empty early-return above, max(StageMarks, OtherMarks) >= 1,
2938 // and the erase/pad steps normalize StageMarks to exactly MaxSize. Hence
2939 // OurSize == MaxSize >= 1 (as long as MaxAsyncMarks != 0), so the
2940 // seq_inclusive(1, OurSize) below never trips its "Begin <= End" assertion
2941 // the way seq_inclusive(1, MergeCount) could when OtherSize == 0.
2942 assert(OurSize >= 1 &&
2943 "StageMarks padded to MaxSize >= 1 (needs MaxAsyncMarks != 0)");
2944
2945 for (auto Idx : seq_inclusive<size_t>(Begin: 1, End: OurSize)) {
2946 const CounterValueArray &OtherMark =
2947 Idx <= OtherSize ? OtherMarks[OtherSize - Idx] : ZeroMark;
2948 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter))
2949 StrictDom |=
2950 mergeScore(M: MergeInfos[T], Score&: StageMarks[OurSize - Idx][T], OtherScore: OtherMark[T]);
2951 }
2952
2953 LLVM_DEBUG({
2954 dbgs() << "After merge:\n";
2955 for (const auto &Mark : StageMarks) {
2956 llvm::interleaveComma(Mark, dbgs());
2957 dbgs() << '\n';
2958 }
2959 });
2960
2961 return StrictDom;
2962}
2963
2964/// Merge the pending events and associater score brackets of \p Other into
2965/// this brackets status.
2966///
2967/// Returns whether the merge resulted in a change that requires tighter waits
2968/// (i.e. the merged brackets strictly dominate the original brackets).
2969bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
2970 bool StrictDom = false;
2971
2972 // Check if "other" has keys we don't have, and create default entries for
2973 // those. If they remain empty after merging, we will clean it up after.
2974 for (auto K : Other.VMem.keys())
2975 VMem.try_emplace(Key: K);
2976 for (auto K : Other.SGPRs.keys())
2977 SGPRs.try_emplace(Key: K);
2978
2979 // Array to store MergeInfo for each counter type
2980 MergeInfo MergeInfos[AMDGPU::NUM_INST_CNTS];
2981
2982 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter)) {
2983 // Merge event flags for this counter
2984 const HWEvents &EventsForT = Context->getWaitEvents(T);
2985 const HWEvents OldEvents = PendingEvents & EventsForT;
2986 const HWEvents OtherEvents = Other.PendingEvents & EventsForT;
2987 if (!OldEvents.contains(Other: OtherEvents))
2988 StrictDom = true;
2989 PendingEvents |= OtherEvents;
2990
2991 // Merge scores for this counter
2992 const unsigned MyPending = ScoreUBs[T] - ScoreLBs[T];
2993 const unsigned OtherPending = Other.ScoreUBs[T] - Other.ScoreLBs[T];
2994 const unsigned NewUB = ScoreLBs[T] + std::max(a: MyPending, b: OtherPending);
2995 if (NewUB < ScoreLBs[T])
2996 report_fatal_error(reason: "waitcnt score overflow");
2997
2998 MergeInfo &M = MergeInfos[T];
2999 M.OldLB = ScoreLBs[T];
3000 M.OtherLB = Other.ScoreLBs[T];
3001 M.MyShift = NewUB - ScoreUBs[T];
3002 M.OtherShift = NewUB - Other.ScoreUBs[T];
3003
3004 ScoreUBs[T] = NewUB;
3005
3006 if (T == AMDGPU::LOAD_CNT)
3007 StrictDom |= mergeScore(M, Score&: LastFlatLoadCnt, OtherScore: Other.LastFlatLoadCnt);
3008
3009 if (T == AMDGPU::X_CNT)
3010 StrictDom |= mergeScore(M, Score&: LastXcntOnlyVmem, OtherScore: Other.LastXcntOnlyVmem);
3011
3012 if (T == AMDGPU::DS_CNT) {
3013 StrictDom |= mergeScore(M, Score&: LastFlatDsCnt, OtherScore: Other.LastFlatDsCnt);
3014 StrictDom |= mergeScore(M, Score&: LastGDS, OtherScore: Other.LastGDS);
3015 }
3016
3017 if (T == AMDGPU::KM_CNT) {
3018 StrictDom |= mergeScore(M, Score&: SCCScore, OtherScore: Other.SCCScore);
3019 if (Other.hasPendingEvent(E: HWEvents::SCC_WRITE)) {
3020 if (!(OldEvents & HWEvents::SCC_WRITE)) {
3021 PendingSCCWrite = Other.PendingSCCWrite;
3022 } else if (PendingSCCWrite != Other.PendingSCCWrite) {
3023 PendingSCCWrite = nullptr;
3024 }
3025 }
3026 }
3027
3028 for (auto &[RegID, Info] : VMem) {
3029 unsigned OtherScore = Other.getVMemScore(TID: RegID, T);
3030 if (T == AMDGPU::VA_VDST_WR) {
3031 // A fence count is usable only when every path carrying the dependency
3032 // has at least that many later ordered XDL writes. Keep this path's
3033 // OrderedXDLCount and rebase each write onto it.
3034 bool MyPending = Info.Scores[T] > M.OldLB;
3035 bool OtherPending = OtherScore > M.OtherLB;
3036 unsigned MyCount = MyPending ? getNumOrderedXDLsAfterWrite(TID: RegID) : 0;
3037 unsigned OtherCount =
3038 OtherPending ? Other.getNumOrderedXDLsAfterWrite(TID: RegID) : 0;
3039 unsigned NewCount = MyPending && OtherPending
3040 ? std::min(a: MyCount, b: OtherCount)
3041 : (MyPending ? MyCount : OtherCount);
3042 StrictDom |= MyPending && NewCount < MyCount;
3043 Info.OrderedXDLCountAtWrite = OrderedXDLCount - NewCount;
3044 }
3045 StrictDom |= mergeScore(M, Score&: Info.Scores[T], OtherScore);
3046 }
3047
3048 if (isSmemCounter(T)) {
3049 for (auto &[RegID, Info] : SGPRs) {
3050 auto It = Other.SGPRs.find(Val: RegID);
3051 unsigned OtherScore = (It != Other.SGPRs.end()) ? It->second.get(T) : 0;
3052 StrictDom |= mergeScore(M, Score&: Info.get(T), OtherScore);
3053 }
3054 }
3055 }
3056
3057 for (auto &[TID, Info] : VMem) {
3058 if (auto It = Other.VMem.find(Val: TID); It != Other.VMem.end()) {
3059 HWEvents NewVGPRContext =
3060 Info.VGPRPendingEvents | It->second.VGPRPendingEvents;
3061 StrictDom |= NewVGPRContext != Info.VGPRPendingEvents;
3062 Info.VGPRPendingEvents = NewVGPRContext;
3063 }
3064 }
3065
3066 for (AMDGPU::AsyncStages S : AMDGPU::asyncStages(Mask: AMDGPU::AsyncStages::ALL)) {
3067 unsigned SI = AMDGPU::getAsyncStageIndex(Stage: S);
3068 StrictDom |= mergeAsyncMarks(Stage: S, MergeInfos, OtherMarks: Other.AsyncMarks[SI]);
3069 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter))
3070 StrictDom |=
3071 mergeScore(M: MergeInfos[T], Score&: AsyncScore[SI][T], OtherScore: Other.AsyncScore[SI][T]);
3072 }
3073
3074 purgeEmptyTrackingData();
3075 return StrictDom;
3076}
3077
3078static bool isWaitInstr(MachineInstr &Inst) {
3079 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: Inst.getOpcode());
3080 return Opcode == AMDGPU::S_WAITCNT ||
3081 (Opcode == AMDGPU::S_WAITCNT_VSCNT && Inst.getOperand(i: 0).isReg() &&
3082 Inst.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL) ||
3083 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
3084 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
3085 Opcode == AMDGPU::S_WAITCNT_lds_direct ||
3086 Opcode == AMDGPU::WAIT_ASYNCMARK ||
3087 AMDGPU::counterTypeForInstr(Opcode).has_value();
3088}
3089
3090void SIInsertWaitcnts::setSchedulingMode(MachineBasicBlock &MBB,
3091 MachineBasicBlock::iterator I,
3092 bool ExpertMode) const {
3093 const uint64_t EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
3094 Values: AMDGPU::Hwreg::ID_SCHED_MODE, Values: AMDGPU::Hwreg::HwregOffset::Default, Values: 2);
3095 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
3096 .addImm(Val: ExpertMode ? 2 : 0)
3097 .addImm(Val: EncodedReg);
3098}
3099
3100namespace {
3101// TODO: Remove this work-around after fixing the scheduler.
3102// There are two reasons why vccz might be incorrect; see ST.hasReadVCCZBug()
3103// and ST.partialVCCWritesUpdateVCCZ().
3104// i. VCCZBug: There is a hardware bug on CI/SI where SMRD instruction may
3105// corrupt vccz bit, so when we detect that an instruction may read from
3106// a corrupt vccz bit, we need to:
3107// 1. Insert s_waitcnt lgkm(0) to wait for all outstanding SMRD
3108// operations to complete.
3109// 2. Recompute the correct value of vccz by writing the current value
3110// of vcc back to vcc.
3111// ii. Partial writes to vcc don't update vccz, so we need to recompute the
3112// correct value of vccz by reading vcc and writing it back to vcc.
3113// No waitcnt is needed in this case.
3114class VCCZWorkaround {
3115 const WaitcntBrackets &ScoreBrackets;
3116 const GCNSubtarget &ST;
3117 const SIInstrInfo &TII;
3118 const SIRegisterInfo &TRI;
3119 bool VCCZCorruptionBug = false;
3120 bool VCCZNotUpdatedByPartialWrites = false;
3121 /// vccz could be incorrect at a basic block boundary if a predecessor wrote
3122 /// to vcc and then issued an smem load, so initialize to true.
3123 bool MustRecomputeVCCZ = true;
3124
3125public:
3126 VCCZWorkaround(const WaitcntBrackets &ScoreBrackets, const GCNSubtarget &ST,
3127 const SIInstrInfo &TII, const SIRegisterInfo &TRI)
3128 : ScoreBrackets(ScoreBrackets), ST(ST), TII(TII), TRI(TRI) {
3129 VCCZCorruptionBug = ST.hasReadVCCZBug();
3130 VCCZNotUpdatedByPartialWrites = !ST.partialVCCWritesUpdateVCCZ();
3131 }
3132 /// If \p MI reads vccz and we must recompute it based on MustRecomputeVCCZ,
3133 /// then emit a vccz recompute instruction before \p MI. This needs to be
3134 /// called on every instruction in the basic block because it also tracks the
3135 /// state and updates MustRecomputeVCCZ accordingly. Returns true if it
3136 /// modified the IR.
3137 bool tryRecomputeVCCZ(MachineInstr &MI) {
3138 // No need to run this if neither bug is present.
3139 if (!VCCZCorruptionBug && !VCCZNotUpdatedByPartialWrites)
3140 return false;
3141
3142 // If MI is an SMEM and it can corrupt vccz on this target, then we need
3143 // both to emit a waitcnt and to recompute vccz.
3144 // But we don't actually emit a waitcnt here. This is done in
3145 // generateWaitcntInstBefore() because it tracks all the necessary waitcnt
3146 // state, and can either skip emitting a waitcnt if there is already one in
3147 // the IR, or emit an "optimized" combined waitcnt.
3148 // If this is an smem read, it could complete and clobber vccz at any time.
3149 MustRecomputeVCCZ |= VCCZCorruptionBug && TII.isSMRD(MI);
3150
3151 // If the target partial vcc writes don't update vccz, and MI is such an
3152 // instruction then we must recompute vccz.
3153 // Note: We are using PartiallyWritesToVCCOpt optional to avoid calling
3154 // `definesRegister()` more than needed, because it's not very cheap.
3155 std::optional<bool> PartiallyWritesToVCCOpt;
3156 auto PartiallyWritesToVCC = [](MachineInstr &MI) {
3157 return MI.definesRegister(Reg: AMDGPU::VCC_LO, /*TRI=*/nullptr) ||
3158 MI.definesRegister(Reg: AMDGPU::VCC_HI, /*TRI=*/nullptr);
3159 };
3160 if (VCCZNotUpdatedByPartialWrites) {
3161 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
3162 // If this is a partial VCC write but won't update vccz, then we must
3163 // recompute vccz.
3164 MustRecomputeVCCZ |= *PartiallyWritesToVCCOpt;
3165 }
3166
3167 // If MI is a vcc write with no pending smem, or there is a pending smem
3168 // but the target does not suffer from the vccz corruption bug, then we
3169 // don't need to recompute vccz as this write will recompute it anyway.
3170 if (!ScoreBrackets.hasPendingEvent(E: HWEvents::SMEM_ACCESS) ||
3171 !VCCZCorruptionBug) {
3172 // Compute PartiallyWritesToVCCOpt if we haven't done so already.
3173 if (!PartiallyWritesToVCCOpt)
3174 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
3175 bool FullyWritesToVCC = !*PartiallyWritesToVCCOpt &&
3176 MI.definesRegister(Reg: AMDGPU::VCC, /*TRI=*/nullptr);
3177 // If we write to the full vcc or we write partially and the target
3178 // updates vccz on partial writes, then vccz will be updated correctly.
3179 bool UpdatesVCCZ = FullyWritesToVCC || (!VCCZNotUpdatedByPartialWrites &&
3180 *PartiallyWritesToVCCOpt);
3181 if (UpdatesVCCZ)
3182 MustRecomputeVCCZ = false;
3183 }
3184
3185 // If MI is a branch that reads VCCZ then emit a waitcnt and a vccz
3186 // restore instruction if either is needed.
3187 if (SIInstrInfo::isCBranchVCCZRead(MI) && MustRecomputeVCCZ) {
3188 // Recompute the vccz bit. Any time a value is written to vcc, the vccz
3189 // bit is updated, so we can restore the bit by reading the value of vcc
3190 // and then writing it back to the register.
3191 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
3192 MCID: TII.get(Opcode: ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
3193 DestReg: TRI.getVCC())
3194 .addReg(RegNo: TRI.getVCC());
3195 MustRecomputeVCCZ = false;
3196 return true;
3197 }
3198 return false;
3199 }
3200};
3201
3202} // namespace
3203
3204// Generate s_waitcnt instructions where needed.
3205bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
3206 MachineBasicBlock &Block,
3207 WaitcntBrackets &ScoreBrackets) {
3208 bool Modified = false;
3209
3210 LLVM_DEBUG({
3211 dbgs() << "*** Begin Block: ";
3212 Block.printName(dbgs());
3213 ScoreBrackets.dump();
3214 });
3215 VCCZWorkaround VCCZW(ScoreBrackets, ST, TII, TRI);
3216
3217 // Walk over the instructions.
3218 MachineInstr *OldWaitcntInstr = nullptr;
3219
3220 // NOTE: We may append instrs after Inst while iterating.
3221 ScoreBrackets.verify();
3222 for (MachineBasicBlock::instr_iterator Iter = Block.instr_begin(),
3223 E = Block.instr_end();
3224 Iter != E; ++Iter) {
3225 MachineInstr &Inst = *Iter;
3226 if (isNonWaitcntMetaInst(MI: Inst))
3227 continue;
3228 // Track pre-existing waitcnts that were added in earlier iterations or by
3229 // the memory legalizer.
3230 if (isWaitInstr(Inst) ||
3231 (IsExpertMode && Inst.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR)) {
3232 if (!OldWaitcntInstr)
3233 OldWaitcntInstr = &Inst;
3234 continue;
3235 }
3236
3237 PreheaderFlushFlags FlushFlags;
3238 if (Block.getFirstTerminator() == Inst)
3239 FlushFlags = isPreheaderToFlush(MBB&: Block, ScoreBrackets);
3240
3241 // Generate an s_waitcnt instruction to be placed before Inst, if needed.
3242 Modified |= generateWaitcntInstBefore(MI&: Inst, ScoreBrackets, OldWaitcntInstr,
3243 FlushFlags);
3244 OldWaitcntInstr = nullptr;
3245
3246 if (Inst.getOpcode() == AMDGPU::ASYNCMARK) {
3247 // Asyncmarks record the current wait state and so should not allow
3248 // waitcnts that occur after them to be merged into waitcnts that occur
3249 // before.
3250 ScoreBrackets.recordAsyncMark(Inst, Stages: AMDGPU::asyncStagesFromMaskOperand(
3251 Mask: Inst.getOperand(i: 0).getImm()));
3252 continue;
3253 }
3254
3255 if (TII.isSMRD(MI: Inst)) {
3256 for (const MachineMemOperand *Memop : Inst.memoperands()) {
3257 // No need to handle invariant loads when avoiding WAR conflicts, as
3258 // there cannot be a vector store to the same memory location.
3259 if (!Memop->isInvariant()) {
3260 const Value *Ptr = Memop->getValue();
3261 SLoadAddresses.insert(KV: std::pair(Ptr, Inst.getParent()));
3262 }
3263 }
3264 }
3265
3266 updateEventWaitcntAfter(Inst, ScoreBrackets: &ScoreBrackets);
3267
3268 // Note: insertForcedWaitAfter() may add instrs after Iter that need to be
3269 // visited by the loop.
3270 Modified |= insertForcedWaitAfter(Inst, Block, ScoreBrackets);
3271
3272 LLVM_DEBUG({
3273 Inst.print(dbgs());
3274 ScoreBrackets.dump();
3275 });
3276
3277 // If the target suffers from the vccz bugs, this may emit the necessary
3278 // vccz recompute instruction before \p Inst if needed.
3279 Modified |= VCCZW.tryRecomputeVCCZ(MI&: Inst);
3280
3281 ScoreBrackets.verify();
3282 }
3283
3284 // Flush counters at the end of the block if needed (for preheaders with no
3285 // terminator).
3286 AMDGPU::Waitcnt Wait;
3287 if (Block.getFirstTerminator() == Block.end()) {
3288 PreheaderFlushFlags FlushFlags = isPreheaderToFlush(MBB&: Block, ScoreBrackets);
3289 if (FlushFlags.FlushVmCnt) {
3290 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::LOAD_CNT))
3291 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
3292 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::SAMPLE_CNT))
3293 Wait.set(T: AMDGPU::SAMPLE_CNT, Val: 0);
3294 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::BVH_CNT))
3295 Wait.set(T: AMDGPU::BVH_CNT, Val: 0);
3296 }
3297 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(T: AMDGPU::DS_CNT))
3298 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
3299 }
3300
3301 // Combine or remove any redundant waitcnts at the end of the block.
3302 Modified |= generateWaitcnt(Wait, It: Block.instr_end(), Block, ScoreBrackets,
3303 OldWaitcntInstr);
3304
3305 LLVM_DEBUG({
3306 dbgs() << "*** End Block: ";
3307 Block.printName(dbgs());
3308 ScoreBrackets.dump();
3309 });
3310
3311 return Modified;
3312}
3313
3314bool SIInsertWaitcnts::removeRedundantSoftXcnts(MachineBasicBlock &Block) {
3315 if (Block.size() <= 1)
3316 return false;
3317 // The Memory Legalizer conservatively inserts a soft xcnt before each
3318 // atomic RMW operation. However, for sequences of back-to-back atomic
3319 // RMWs, only the first s_wait_xcnt insertion is necessary. Optimize away
3320 // the redundant soft xcnts.
3321 bool Modified = false;
3322 // Remember the last atomic with a soft xcnt right before it.
3323 MachineInstr *LastAtomicWithSoftXcnt = nullptr;
3324
3325 for (MachineInstr &MI : drop_begin(RangeOrContainer&: Block)) {
3326 // Ignore last atomic if non-LDS VMEM and SMEM.
3327 bool IsLDS = TII.isDS(MI) ||
3328 (TII.isFLAT(MI) && TII.mayAccessLDSThroughFlat(MI, TgSplit));
3329 if (!IsLDS && (MI.mayLoad() ^ MI.mayStore()))
3330 LastAtomicWithSoftXcnt = nullptr;
3331
3332 bool IsAtomicRMW =
3333 SIInstrFlags::isMaybeAtomic(O: MI) && MI.mayLoad() && MI.mayStore();
3334 MachineInstr &PrevMI = *MI.getPrevNode();
3335 // This is an atomic with a soft xcnt.
3336 if (PrevMI.getOpcode() == AMDGPU::S_WAIT_XCNT_soft && IsAtomicRMW) {
3337 // If we have already found an atomic with a soft xcnt, remove this soft
3338 // xcnt as it's redundant.
3339 if (LastAtomicWithSoftXcnt) {
3340 PrevMI.eraseFromParent();
3341 Modified = true;
3342 }
3343 LastAtomicWithSoftXcnt = &MI;
3344 }
3345 }
3346 return Modified;
3347}
3348
3349// Return flags indicating which counters should be flushed in the preheader.
3350PreheaderFlushFlags
3351SIInsertWaitcnts::isPreheaderToFlush(MachineBasicBlock &MBB,
3352 const WaitcntBrackets &ScoreBrackets) {
3353 auto [Iterator, IsInserted] =
3354 PreheadersToFlush.try_emplace(Key: &MBB, Args: PreheaderFlushFlags());
3355 if (!IsInserted)
3356 return Iterator->second;
3357
3358 MachineBasicBlock *Succ = MBB.getSingleSuccessor();
3359 if (!Succ)
3360 return PreheaderFlushFlags();
3361
3362 MachineLoop *Loop = MLI.getLoopFor(BB: Succ);
3363 if (!Loop)
3364 return PreheaderFlushFlags();
3365
3366 if (Loop->getLoopPreheader() == &MBB) {
3367 Iterator->second = getPreheaderFlushFlags(ML: Loop, Brackets: ScoreBrackets);
3368 return Iterator->second;
3369 }
3370
3371 return PreheaderFlushFlags();
3372}
3373
3374bool SIInsertWaitcnts::isVMEMOrFlatVMEM(const MachineInstr &MI) const {
3375 if (SIInstrInfo::isFLAT(MI))
3376 return TII.mayAccessVMEMThroughFlat(MI);
3377 return SIInstrInfo::isVMEM(MI);
3378}
3379
3380bool SIInsertWaitcnts::isDSRead(const MachineInstr &MI) const {
3381 return SIInstrInfo::isDS(MI) && MI.mayLoad() && !MI.mayStore();
3382}
3383
3384// Check if instruction is a store to LDS that is counted via DSCNT
3385// (where that counter exists).
3386bool SIInsertWaitcnts::mayStoreIncrementingDSCNT(const MachineInstr &MI) const {
3387 return MI.mayStore() && SIInstrInfo::isDS(MI);
3388}
3389
3390// Return flags indicating which counters should be flushed in the preheader of
3391// the given loop. We currently decide to flush in the following situations:
3392// For VMEM (FlushVmCnt):
3393// 1. The loop contains vmem store(s), no vmem load and at least one use of a
3394// vgpr containing a value that is loaded outside of the loop. (Only on
3395// targets with no vscnt counter).
3396// 2. The loop contains vmem load(s), but the loaded values are not used in the
3397// loop, and at least one use of a vgpr containing a value that is loaded
3398// outside of the loop.
3399// For DS (FlushDsCnt, GFX12+ only):
3400// 3. The loop contains no DS reads, and at least one use of a vgpr containing
3401// a value that is DS read outside of the loop.
3402// 4. The loop contains DS read(s), loaded values are not used in the same
3403// iteration but in the next iteration (prefetch pattern), and at least one
3404// use of a vgpr containing a value that is DS read outside of the loop.
3405// Flushing in preheader reduces wait overhead if the wait requirement in
3406// iteration 1 would otherwise be more strict (but unfortunately preheader
3407// flush decision is taken before knowing that).
3408// 5. (Single-block loops only) The loop has DS prefetch reads with flush point
3409// tracking. Some DS reads may be used in the same iteration (creating
3410// "flush points"), but others remain unflushed at the backedge. When a DS
3411// read is consumed in the same iteration, it and all prior reads are
3412// "flushed" (FIFO order). No DS writes are allowed in the loop.
3413// TODO: Find a way to extend to multi-block loops.
3414PreheaderFlushFlags
3415SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
3416 const WaitcntBrackets &Brackets) {
3417 PreheaderFlushFlags Flags;
3418 bool HasVMemLoad = false;
3419 bool HasVMemStore = false;
3420 bool UsesVgprVMEMLoadedOutside = false;
3421 bool UsesVgprDSReadOutside = false;
3422 bool VMemInvalidated = false;
3423 // DS optimization only applies to GFX12+ where DS_CNT is separate.
3424 // Tracking status for "no DS read in loop" or "pure DS prefetch
3425 // (use only in next iteration)".
3426 bool TrackSimpleDSOpt = ST.hasExtendedWaitCounts();
3427 DenseSet<MCRegUnit> VgprUse;
3428 DenseSet<MCRegUnit> VgprDefVMEM;
3429 DenseSet<MCRegUnit> VgprDefDS;
3430
3431 // Track DS reads for prefetch pattern with flush points (single-block only).
3432 // Keeps track of the last DS read (position counted from the top of the loop)
3433 // to each VGPR. Read is considered consumed (and thus needs flushing) if
3434 // the dest register has a use or is overwritten (by any later opertions).
3435 DenseMap<MCRegUnit, unsigned> LastDSReadPositionMap;
3436 unsigned DSReadPosition = 0;
3437 bool IsSingleBlock = ML->getNumBlocks() == 1;
3438 bool TrackDSFlushPoint = ST.hasExtendedWaitCounts() && IsSingleBlock;
3439 unsigned LastDSFlushPosition = 0;
3440
3441 for (MachineBasicBlock *MBB : ML->blocks()) {
3442 for (MachineInstr &MI : *MBB) {
3443 if (isVMEMOrFlatVMEM(MI)) {
3444 HasVMemLoad |= MI.mayLoad();
3445 HasVMemStore |= MI.mayStore();
3446 }
3447 // TODO: Can we relax DSStore check? There may be cases where
3448 // these DS stores are drained prior to the end of MBB (or loop).
3449 if (mayStoreIncrementingDSCNT(MI)) {
3450 // Early exit if none of the optimizations are feasible.
3451 // Otherwise, set tracking status appropriately and continue.
3452 if (VMemInvalidated)
3453 return Flags;
3454 TrackSimpleDSOpt = false;
3455 TrackDSFlushPoint = false;
3456 }
3457 bool IsDSRead = isDSRead(MI);
3458 if (IsDSRead)
3459 ++DSReadPosition;
3460
3461 // Helper: if RU has a pending DS read, update LastDSFlushPosition
3462 auto updateDSReadFlushTracking = [&](MCRegUnit RU) {
3463 if (!TrackDSFlushPoint)
3464 return;
3465 if (auto It = LastDSReadPositionMap.find(Val: RU);
3466 It != LastDSReadPositionMap.end()) {
3467 // RU defined by DSRead is used or overwritten. Need to complete
3468 // the read, if not already implied by a later DSRead (to any RU)
3469 // needing to complete in FIFO order.
3470 LastDSFlushPosition = std::max(a: LastDSFlushPosition, b: It->second);
3471 }
3472 };
3473
3474 for (const MachineOperand &Op : MI.all_uses()) {
3475 if (Op.isDebug() || !TRI.isVectorRegister(MRI, Reg: Op.getReg()))
3476 continue;
3477 // Vgpr use
3478 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3479 // If we find a register that is loaded inside the loop, 1. and 2.
3480 // are invalidated.
3481 if (VgprDefVMEM.contains(V: RU))
3482 VMemInvalidated = true;
3483
3484 // Check for DS reads used inside the loop
3485 if (VgprDefDS.contains(V: RU))
3486 TrackSimpleDSOpt = false;
3487
3488 // Early exit if all optimizations are invalidated
3489 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3490 return Flags;
3491
3492 // Check for flush points (DS read used in same iteration)
3493 updateDSReadFlushTracking(RU);
3494
3495 VgprUse.insert(V: RU);
3496 // Check if this register has a pending VMEM load from outside the
3497 // loop (value loaded outside and used inside).
3498 VMEMID ID = toVMEMID(RU);
3499 if (Brackets.hasPendingVMEM(ID, T: AMDGPU::LOAD_CNT) ||
3500 Brackets.hasPendingVMEM(ID, T: AMDGPU::SAMPLE_CNT) ||
3501 Brackets.hasPendingVMEM(ID, T: AMDGPU::BVH_CNT))
3502 UsesVgprVMEMLoadedOutside = true;
3503 // Check if loaded outside the loop via DS (not VMEM/FLAT).
3504 // Only consider it a DS read if there's no pending VMEM load for
3505 // this register, since FLAT can set both counters.
3506 else if (Brackets.hasPendingVMEM(ID, T: AMDGPU::DS_CNT))
3507 UsesVgprDSReadOutside = true;
3508 }
3509 }
3510
3511 // VMem load vgpr def
3512 if (isVMEMOrFlatVMEM(MI) && MI.mayLoad()) {
3513 for (const MachineOperand &Op : MI.all_defs()) {
3514 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3515 // If we find a register that is loaded inside the loop, 1. and 2.
3516 // are invalidated.
3517 if (VgprUse.contains(V: RU))
3518 VMemInvalidated = true;
3519 VgprDefVMEM.insert(V: RU);
3520 }
3521 }
3522 // Early exit if all optimizations are invalidated
3523 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3524 return Flags;
3525 }
3526
3527 // DS read vgpr def
3528 // Note: Unlike VMEM, we DON'T invalidate when VgprUse.contains(RegNo).
3529 // If USE comes before DEF, it's the prefetch pattern (use value from
3530 // previous iteration, read for next iteration). We should still flush
3531 // in preheader so iteration 1 doesn't need to wait inside the loop.
3532 // Only invalidate when DEF comes before USE (same-iteration consumption,
3533 // checked above when processing uses).
3534 if (IsDSRead || TrackDSFlushPoint) {
3535 for (const MachineOperand &Op : MI.all_defs()) {
3536 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg()))
3537 continue;
3538 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3539 // Check for overwrite of pending DS read (flush point) by any
3540 // instruction
3541 updateDSReadFlushTracking(RU);
3542 if (IsDSRead) {
3543 VgprDefDS.insert(V: RU);
3544 if (TrackDSFlushPoint)
3545 LastDSReadPositionMap[RU] = DSReadPosition;
3546 }
3547 }
3548 }
3549 }
3550 }
3551 }
3552
3553 // VMEM flush decision
3554 if (!VMemInvalidated && UsesVgprVMEMLoadedOutside &&
3555 ((!ST.hasVscnt() && HasVMemStore && !HasVMemLoad) ||
3556 (HasVMemLoad && ST.hasVmemWriteVgprInOrder())))
3557 Flags.FlushVmCnt = true;
3558
3559 // DS flush decision:
3560 // Simple DS Opt: flush if loop uses DS read values from outside
3561 // and either has no DS reads in the loop, or DS reads whose results
3562 // are not used in the loop.
3563 bool SimpleDSOpt = TrackSimpleDSOpt && UsesVgprDSReadOutside;
3564 // Prefetch with flush points: some DS reads used in same iteration,
3565 // but unflushed reads remain at backedge
3566 bool HasUnflushedDSReads = DSReadPosition > LastDSFlushPosition;
3567 bool DSFlushPointPrefetch =
3568 TrackDSFlushPoint && UsesVgprDSReadOutside && HasUnflushedDSReads;
3569
3570 if (SimpleDSOpt || DSFlushPointPrefetch)
3571 Flags.FlushDsCnt = true;
3572
3573 return Flags;
3574}
3575
3576bool SIInsertWaitcntsLegacy::runOnMachineFunction(MachineFunction &MF) {
3577 auto &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
3578 auto &PDT =
3579 getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
3580 AliasAnalysis *AA = nullptr;
3581 if (auto *AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
3582 AA = &AAR->getAAResults();
3583
3584 return SIInsertWaitcnts(MLI, PDT, AA, MF).run();
3585}
3586
3587PreservedAnalyses
3588SIInsertWaitcntsPass::run(MachineFunction &MF,
3589 MachineFunctionAnalysisManager &MFAM) {
3590 auto &MLI = MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
3591 auto &PDT = MFAM.getResult<MachinePostDominatorTreeAnalysis>(IR&: MF);
3592 auto *AA = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
3593 .getManager()
3594 .getCachedResult<AAManager>(IR&: MF.getFunction());
3595
3596 if (!SIInsertWaitcnts(MLI, PDT, AA, MF).run())
3597 return PreservedAnalyses::all();
3598
3599 return getMachineFunctionPassPreservedAnalyses()
3600 .preserveSet<CFGAnalyses>()
3601 .preserve<AAManager>();
3602}
3603
3604bool SIInsertWaitcnts::run() {
3605 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3606
3607 AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: ST.getCPU());
3608
3609 // Initialize hardware limits first, as they're needed by the generators.
3610 Limits = AMDGPU::HardwareLimits(IV);
3611
3612 if (ST.hasExtendedWaitCounts()) {
3613 IsExpertMode = AMDGPU::isExpertSchedulingMode(ST, F: MF.getFunction());
3614 MaxCounter = IsExpertMode ? AMDGPU::NUM_EXPERT_INST_CNTS
3615 : AMDGPU::NUM_EXTENDED_INST_CNTS;
3616 // Initialize WCG per MF. It contains state that depends on MF attributes.
3617 WCG = std::make_unique<WaitcntGeneratorGFX12Plus>(args&: MF, args&: MaxCounter, args&: Limits,
3618 args&: IsExpertMode);
3619 } else {
3620 MaxCounter = AMDGPU::NUM_NORMAL_INST_CNTS;
3621 // Initialize WCG per MF. It contains state that depends on MF attributes.
3622 WCG = std::make_unique<WaitcntGeneratorPreGFX12>(
3623 args&: MF, args: AMDGPU::NUM_NORMAL_INST_CNTS, args&: Limits);
3624 }
3625
3626 SmemAccessCounter = getCounterFromEvent(E: HWEvents::SMEM_ACCESS);
3627
3628 bool Modified = false;
3629
3630 MachineBasicBlock &EntryBB = MF.front();
3631
3632 if (!MFI->isEntryFunction() &&
3633 !MF.getFunction().hasFnAttribute(Kind: Attribute::Naked)) {
3634 // Wait for any outstanding memory operations that the input registers may
3635 // depend on. We can't track them and it's better to do the wait after the
3636 // costly call sequence.
3637
3638 // TODO: Could insert earlier and schedule more liberally with operations
3639 // that only use caller preserved registers.
3640 MachineBasicBlock::iterator I = EntryBB.begin();
3641 while (I != EntryBB.end() && I->isMetaInstruction())
3642 ++I;
3643
3644 if (ST.hasExtendedWaitCounts()) {
3645 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAIT_LOADCNT_DSCNT))
3646 .addImm(Val: 0);
3647 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
3648 if (CT == AMDGPU::LOAD_CNT || CT == AMDGPU::DS_CNT ||
3649 CT == AMDGPU::STORE_CNT || CT == AMDGPU::X_CNT ||
3650 CT == AMDGPU::ASYNC_CNT || CT == AMDGPU::TENSOR_CNT)
3651 continue;
3652
3653 if (!ST.hasImageInsts() &&
3654 (CT == AMDGPU::EXP_CNT || CT == AMDGPU::SAMPLE_CNT ||
3655 CT == AMDGPU::BVH_CNT))
3656 continue;
3657
3658 if (!ST.hasAnyBVHInsts() && CT == AMDGPU::BVH_CNT)
3659 continue;
3660
3661 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(),
3662 MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
3663 .addImm(Val: 0);
3664 }
3665 if (IsExpertMode) {
3666 unsigned Enc = AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST);
3667 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: 0);
3668 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
3669 .addImm(Val: Enc);
3670 }
3671 } else {
3672 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: 0);
3673 }
3674
3675 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(args: this);
3676 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
3677 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
3678
3679 Modified = true;
3680 }
3681
3682 // Keep iterating over the blocks in reverse post order, inserting and
3683 // updating s_waitcnt where needed, until a fix point is reached.
3684 for (auto *MBB : ReversePostOrderTraversal<MachineFunction *>(&MF))
3685 BlockInfos.try_emplace(Key: MBB);
3686
3687 std::unique_ptr<WaitcntBrackets> Brackets;
3688 bool Repeat;
3689 do {
3690 Repeat = false;
3691
3692 for (auto BII = BlockInfos.begin(), BIE = BlockInfos.end(); BII != BIE;
3693 ++BII) {
3694 MachineBasicBlock *MBB = BII->first;
3695 BlockInfo &BI = BII->second;
3696 if (!BI.Dirty)
3697 continue;
3698
3699 if (BI.Incoming) {
3700 if (!Brackets)
3701 Brackets = std::make_unique<WaitcntBrackets>(args&: *BI.Incoming);
3702 else
3703 *Brackets = *BI.Incoming;
3704 } else {
3705 if (!Brackets) {
3706 Brackets = std::make_unique<WaitcntBrackets>(args: this);
3707 } else {
3708 // Reinitialize in-place. N.B. do not do this by assigning from a
3709 // temporary because the WaitcntBrackets class is large and it could
3710 // cause this function to use an unreasonable amount of stack space.
3711 Brackets->~WaitcntBrackets();
3712 new (Brackets.get()) WaitcntBrackets(this);
3713 }
3714 }
3715
3716 if (ST.hasWaitXcnt() && ST.isXNACKEnabled())
3717 Modified |= removeRedundantSoftXcnts(Block&: *MBB);
3718 Modified |= insertWaitcntInBlock(MF, Block&: *MBB, ScoreBrackets&: *Brackets);
3719 BI.Dirty = false;
3720
3721 if (Brackets->hasPendingEvent()) {
3722 BlockInfo *MoveBracketsToSucc = nullptr;
3723 for (MachineBasicBlock *Succ : MBB->successors()) {
3724 auto *SuccBII = BlockInfos.find(Key: Succ);
3725 BlockInfo &SuccBI = SuccBII->second;
3726 if (!SuccBI.Incoming) {
3727 SuccBI.Dirty = true;
3728 if (SuccBII <= BII) {
3729 LLVM_DEBUG(dbgs() << "Repeat on backedge without merge\n");
3730 Repeat = true;
3731 }
3732 if (!MoveBracketsToSucc) {
3733 MoveBracketsToSucc = &SuccBI;
3734 } else {
3735 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(args&: *Brackets);
3736 }
3737 } else {
3738 LLVM_DEBUG({
3739 dbgs() << "Try to merge ";
3740 MBB->printName(dbgs());
3741 dbgs() << " into ";
3742 Succ->printName(dbgs());
3743 dbgs() << '\n';
3744 });
3745 if (SuccBI.Incoming->merge(Other: *Brackets)) {
3746 SuccBI.Dirty = true;
3747 if (SuccBII <= BII) {
3748 LLVM_DEBUG(dbgs() << "Repeat on backedge with merge\n");
3749 Repeat = true;
3750 }
3751 }
3752 }
3753 }
3754 if (MoveBracketsToSucc)
3755 MoveBracketsToSucc->Incoming = std::move(Brackets);
3756 }
3757 }
3758 } while (Repeat);
3759
3760 Modified |= WCG->eraseDeferredWaitcnts();
3761
3762 if (ST.hasScalarStores()) {
3763 SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;
3764 bool HaveScalarStores = false;
3765
3766 for (MachineBasicBlock &MBB : MF) {
3767 for (MachineInstr &MI : MBB) {
3768 if (!HaveScalarStores && TII.isScalarStore(MI))
3769 HaveScalarStores = true;
3770
3771 if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
3772 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
3773 EndPgmBlocks.push_back(Elt: &MBB);
3774 }
3775 }
3776
3777 if (HaveScalarStores) {
3778 // If scalar writes are used, the cache must be flushed or else the next
3779 // wave to reuse the same scratch memory can be clobbered.
3780 //
3781 // Insert s_dcache_wb at wave termination points if there were any scalar
3782 // stores, and only if the cache hasn't already been flushed. This could
3783 // be improved by looking across blocks for flushes in postdominating
3784 // blocks from the stores but an explicitly requested flush is probably
3785 // very rare.
3786 for (MachineBasicBlock *MBB : EndPgmBlocks) {
3787 bool SeenDCacheWB = false;
3788
3789 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
3790 I != E; ++I) {
3791 if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
3792 SeenDCacheWB = true;
3793 else if (TII.isScalarStore(MI: *I))
3794 SeenDCacheWB = false;
3795
3796 // FIXME: It would be better to insert this before a waitcnt if any.
3797 if ((I->getOpcode() == AMDGPU::S_ENDPGM ||
3798 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
3799 !SeenDCacheWB) {
3800 Modified = true;
3801 BuildMI(BB&: *MBB, I, MIMD: I->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_DCACHE_WB));
3802 }
3803 }
3804 }
3805 }
3806 }
3807
3808 if (IsExpertMode) {
3809 // Enable expert scheduling on function entry. To satisfy ABI requirements
3810 // and to allow calls between function with different expert scheduling
3811 // settings, disable it around calls and before returns.
3812
3813 MachineBasicBlock::iterator I = EntryBB.begin();
3814 while (I != EntryBB.end() && I->isMetaInstruction())
3815 ++I;
3816 setSchedulingMode(MBB&: EntryBB, I, ExpertMode: true);
3817
3818 for (MachineInstr *MI : CallInsts) {
3819 MachineBasicBlock &MBB = *MI->getParent();
3820 setSchedulingMode(MBB, I: MI, ExpertMode: false);
3821 setSchedulingMode(MBB, I: std::next(x: MI->getIterator()), ExpertMode: true);
3822 }
3823
3824 for (MachineInstr *MI : ReturnInsts)
3825 setSchedulingMode(MBB&: *MI->getParent(), I: MI, ExpertMode: false);
3826
3827 Modified = true;
3828 }
3829
3830 // Deallocate the VGPRs before previously identified S_ENDPGM instructions.
3831 // This is done in different ways depending on how the VGPRs were allocated
3832 // (i.e. whether we're in dynamic VGPR mode or not).
3833 // Skip deallocation if kernel is waveslot limited vs VGPR limited. A short
3834 // waveslot limited kernel runs slower with the deallocation.
3835 if (!WCG->isOptNone() && MFI->isDynamicVGPREnabled()) {
3836 for (auto [MI, _] : EndPgmInsts) {
3837 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3838 MCID: TII.get(Opcode: AMDGPU::S_ALLOC_VGPR))
3839 .addImm(Val: 0);
3840 Modified = true;
3841 }
3842 } else if (!WCG->isOptNone() &&
3843 ST.getGeneration() >= AMDGPUSubtarget::GFX11 &&
3844 (MF.getFrameInfo().hasCalls() ||
3845 ST.getOccupancyWithNumVGPRs(
3846 VGPRs: TRI.getNumUsedPhysRegs(MRI, RC: AMDGPU::VGPR_32RegClass),
3847 /*IsDynamicVGPR=*/DynamicVGPRBlockSize: false) < ST.getMaxWavesPerEU())) {
3848 for (auto [MI, Flag] : EndPgmInsts) {
3849 if (Flag) {
3850 if (ST.requiresNopBeforeDeallocVGPRs()) {
3851 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3852 MCID: TII.get(Opcode: AMDGPU::S_NOP))
3853 .addImm(Val: 0);
3854 }
3855 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3856 MCID: TII.get(Opcode: AMDGPU::S_SENDMSG))
3857 .addImm(Val: AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus);
3858 Modified = true;
3859 }
3860 }
3861 }
3862
3863 if (MFI->isEntryFunction()) {
3864 MachineBasicBlock::iterator InsertPt = EntryBB.begin();
3865
3866 if (ST.hasWaitXcnt() && ST.isXNACKEnabled()) {
3867 // Set REPLAY_MODE (bit 25) in MODE register to enable multi-group XNACK
3868 // replay. This aligns hardware behavior with the compiler's s_wait_xcnt
3869 // insertion logic, which assumes multi-group mode by default.
3870 uint64_t RegEncoding =
3871 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 25, Values: 1);
3872 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(),
3873 MCID: TII.get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
3874 .addImm(Val: 1)
3875 .addImm(Val: RegEncoding);
3876 Modified = true;
3877 }
3878
3879 if (ST.hasRequiresInitialUnclausedVmem()) {
3880 // Hardware entrypoints must begin with a specific sequence:
3881 // S_MOV_B64 S[64:65], 0
3882 // V_NOP
3883 // GLOBAL_PREFETCH_B8 V0, S[64:65] SCOPE:SCOPE_SE TH:TH_LOAD_RT
3884 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B64),
3885 DestReg: AMDGPU::SGPR64_SGPR65)
3886 .addImm(Val: 0);
3887 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
3888 BuildMI(BB&: EntryBB, I: InsertPt, MIMD: DebugLoc(),
3889 MCID: TII.get(Opcode: AMDGPU::GLOBAL_PREFETCH_B8_SADDR))
3890 .addReg(RegNo: AMDGPU::SGPR64_SGPR65)
3891 .addReg(RegNo: AMDGPU::VGPR0, Flags: RegState::Undef)
3892 .addImm(Val: 0)
3893 .addImm(Val: AMDGPU::CPol::SCOPE_SE | AMDGPU::CPol::TH_RT);
3894 Modified = true;
3895 }
3896 }
3897
3898 return Modified;
3899}
3900