1//===- SIInsertWaitcnts.cpp - Insert Wait Instructions --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert wait instructions for memory reads and writes.
11///
12/// Memory reads and writes are issued asynchronously, so we need to insert
13/// S_WAITCNT instructions when we want to access any of their results or
14/// overwrite any register that's used asynchronously.
15///
16/// TODO: This pass currently keeps one timeline per hardware counter. A more
17/// finely-grained approach that keeps one timeline per event type could
18/// sometimes get away with generating weaker s_waitcnt instructions. For
19/// example, when both SMEM and LDS are in flight and we need to wait for
20/// the i-th-last LDS instruction, then an lgkmcnt(i) is actually sufficient,
21/// but the pass will currently generate a conservative lgkmcnt(0) because
22/// multiple event types are in flight.
23//
24//===----------------------------------------------------------------------===//
25
26#include "AMDGPU.h"
27#include "AMDGPUHWEvents.h"
28#include "AMDGPUWaitcntUtils.h"
29#include "GCNSubtarget.h"
30#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
31#include "SIMachineFunctionInfo.h"
32#include "Utils/AMDGPUBaseInfo.h"
33#include "llvm/ADT/MapVector.h"
34#include "llvm/ADT/PostOrderIterator.h"
35#include "llvm/ADT/Sequence.h"
36#include "llvm/Analysis/AliasAnalysis.h"
37#include "llvm/CodeGen/MachineFrameInfo.h"
38#include "llvm/CodeGen/MachineLoopInfo.h"
39#include "llvm/CodeGen/MachinePassManager.h"
40#include "llvm/CodeGen/MachinePostDominators.h"
41#include "llvm/IR/Dominators.h"
42#include "llvm/InitializePasses.h"
43#include "llvm/TargetParser/AMDGPUTargetParser.h"
44
45using namespace llvm;
46
47using HWEvents = AMDGPU::HWEvents;
48
49#define DEBUG_TYPE "si-insert-waitcnts"
50
51static cl::opt<bool>
52 ForceEmitZeroFlag("amdgpu-waitcnt-forcezero",
53 cl::desc("Force all waitcnt instrs to be emitted as "
54 "s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
55 cl::init(Val: false), cl::Hidden);
56
57static cl::opt<bool> ForceEmitZeroLoadFlag(
58 "amdgpu-waitcnt-load-forcezero",
59 cl::desc("Force all waitcnt load counters to wait until 0"),
60 cl::init(Val: false), cl::Hidden);
61
62static cl::opt<bool> ExpertSchedulingModeFlag(
63 "amdgpu-expert-scheduling-mode",
64 cl::desc("Enable expert scheduling mode 2 for all functions (GFX12+ only)"),
65 cl::init(Val: false), cl::Hidden);
66
67namespace {
68
69template <typename EmitWaitcntFn>
70static void EmitExpandedWaitcnt(unsigned Outstanding, unsigned Target,
71 EmitWaitcntFn &&EmitWaitcnt) {
72 // Emit waitcnts from (Outstanding - 1) down to Target.
73 for (unsigned I = Outstanding - 1; I > Target && I != ~0u; --I)
74 EmitWaitcnt(I);
75 EmitWaitcnt(Target);
76}
77
78/// Integer IDs used to track vector memory locations we may have to wait on.
79/// Encoded as u16 chunks:
80///
81/// [0, REGUNITS_END ): MCRegUnit
82/// [LDSDMA_BEGIN, LDSDMA_END ) : LDS DMA IDs
83///
84/// NOTE: The choice of encoding these as "u16 chunks" is arbitrary.
85/// It gives (2 << 16) - 1 entries per category which is more than enough
86/// for all register units. MCPhysReg is u16 so we don't even support >u16
87/// physical register numbers at this time, let alone >u16 register units.
88/// In any case, an assertion in "WaitcntBrackets" ensures REGUNITS_END
89/// is enough for all register units.
90using VMEMID = uint32_t;
91
92enum : VMEMID {
93 TRACKINGID_RANGE_LEN = (1 << 16),
94
95 // Important: MCRegUnits must always be tracked starting from 0, as we
96 // need to be able to convert between a MCRegUnit and a VMEMID freely.
97 REGUNITS_BEGIN = 0,
98 REGUNITS_END = REGUNITS_BEGIN + TRACKINGID_RANGE_LEN,
99
100 // Note for LDSDMA: LDSDMA_BEGIN corresponds to the "common"
101 // entry, which is updated for all LDS DMA operations encountered.
102 // Specific LDS DMA IDs start at LDSDMA_BEGIN + 1.
103 NUM_LDSDMA = TRACKINGID_RANGE_LEN,
104 LDSDMA_BEGIN = REGUNITS_END,
105 LDSDMA_END = LDSDMA_BEGIN + NUM_LDSDMA,
106};
107
108/// Convert a MCRegUnit to a VMEMID.
109static constexpr VMEMID toVMEMID(MCRegUnit RU) {
110 return static_cast<unsigned>(RU);
111}
112
113} // namespace
114
115namespace {
116
117// Maps values of InstCounterType to the instruction that waits on that
118// counter. Only used if GCNSubtarget::hasExtendedWaitCounts()
119// returns true, and does not cover VA_VDST or VM_VSRC.
120static const unsigned
121 instrsForExtendedCounterTypes[AMDGPU::NUM_EXTENDED_INST_CNTS] = {
122 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT,
123 AMDGPU::S_WAIT_EXPCNT, AMDGPU::S_WAIT_STORECNT,
124 AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
125 AMDGPU::S_WAIT_KMCNT, AMDGPU::S_WAIT_XCNT,
126 AMDGPU::S_WAIT_ASYNCCNT, AMDGPU::S_WAIT_TENSORCNT};
127
128// ASYNCMARK and WAIT_ASYNCMARK are meta instructions that emit no hardware
129// code but still need to be processed by this pass for async vmcnt tracking.
130static bool isNonWaitcntMetaInst(const MachineInstr &MI) {
131 switch (MI.getOpcode()) {
132 case AMDGPU::ASYNCMARK:
133 case AMDGPU::WAIT_ASYNCMARK:
134 return false;
135 default:
136 return MI.isMetaInstruction();
137 }
138}
139
140static bool updateVMCntOnly(const MachineInstr &Inst) {
141 return (SIInstrInfo::isVMEM(MI: Inst) && !SIInstrInfo::isFLAT(MI: Inst)) ||
142 SIInstrInfo::isFLATGlobal(MI: Inst) || SIInstrInfo::isFLATScratch(MI: Inst);
143}
144
145#ifndef NDEBUG
146static bool isNormalMode(AMDGPU::InstCounterType MaxCounter) {
147 return MaxCounter == AMDGPU::NUM_NORMAL_INST_CNTS;
148}
149#endif // NDEBUG
150
151class WaitcntBrackets;
152
153// This abstracts the logic for generating and updating S_WAIT* instructions
154// away from the analysis that determines where they are needed. This was
155// done because the set of counters and instructions for waiting on them
156// underwent a major shift with gfx12, sufficiently so that having this
157// abstraction allows the main analysis logic to be simpler than it would
158// otherwise have had to become.
159class WaitcntGenerator {
160protected:
161 const GCNSubtarget &ST;
162 const SIInstrInfo &TII;
163 AMDGPU::IsaVersion IV;
164 AMDGPU::InstCounterType MaxCounter;
165 bool OptNone;
166 bool ExpandWaitcntProfiling = false;
167 const AMDGPU::HardwareLimits &Limits;
168
169public:
170 WaitcntGenerator() = delete;
171 WaitcntGenerator(const WaitcntGenerator &) = delete;
172 WaitcntGenerator(const MachineFunction &MF,
173 AMDGPU::InstCounterType MaxCounter,
174 const AMDGPU::HardwareLimits &Limits)
175 : ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
176 IV(AMDGPU::getIsaVersion(GPU: ST.getCPU())), MaxCounter(MaxCounter),
177 OptNone(MF.getFunction().hasOptNone() ||
178 MF.getTarget().getOptLevel() == CodeGenOptLevel::None),
179 ExpandWaitcntProfiling(
180 MF.getFunction().hasFnAttribute(Kind: "amdgpu-expand-waitcnt-profiling")),
181 Limits(Limits) {}
182
183 // Return true if the current function should be compiled with no
184 // optimization.
185 bool isOptNone() const { return OptNone; }
186
187 unsigned getLimit(AMDGPU::InstCounterType E) const { return Limits.get(T: E); }
188
189 // Edits an existing sequence of wait count instructions according
190 // to an incoming Waitcnt value, which is itself updated to reflect
191 // any new wait count instructions which may need to be generated by
192 // WaitcntGenerator::createNewWaitcnt(). It will return true if any edits
193 // were made.
194 //
195 // This editing will usually be merely updated operands, but it may also
196 // delete instructions if the incoming Wait value indicates they are not
197 // needed. It may also remove existing instructions for which a wait
198 // is needed if it can be determined that it is better to generate new
199 // instructions later, as can happen on gfx12.
200 virtual bool
201 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
202 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
203 MachineBasicBlock::instr_iterator It) const = 0;
204
205 // Transform a soft waitcnt into a normal one.
206 bool promoteSoftWaitCnt(MachineInstr *Waitcnt) const;
207
208 // Generates new wait count instructions according to the value of
209 // Wait, returning true if any new instructions were created.
210 // ScoreBrackets is used for profiling expansion.
211 virtual bool createNewWaitcnt(MachineBasicBlock &Block,
212 MachineBasicBlock::instr_iterator It,
213 AMDGPU::Waitcnt Wait,
214 const WaitcntBrackets &ScoreBrackets) = 0;
215
216 // Returns the set of HWEvents that corresponds to counter \p T.
217 virtual HWEvents getWaitEvents(AMDGPU::InstCounterType T) const = 0;
218
219 /// \returns the counter that corresponds to event \p E.
220 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
221 assert(E.size() == 1 && "Cannot handle a mask of events!");
222 for (auto T : AMDGPU::inst_counter_types()) {
223 if (getWaitEvents(T) & E)
224 return T;
225 }
226 llvm_unreachable("event type has no associated counter");
227 }
228
229 // Returns a new waitcnt with all counters except VScnt set to 0. If
230 // IncludeVSCnt is true, VScnt is set to 0, otherwise it is set to ~0u.
231 // AsyncCnt and TensorCnt always default to ~0u (don't wait for it). They
232 // are only updated when a call to @llvm.amdgcn.wait.asyncmark() is
233 // processed.
234 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const = 0;
235
236 virtual ~WaitcntGenerator() = default;
237};
238
239class WaitcntGeneratorPreGFX12 final : public WaitcntGenerator {
240 static constexpr const HWEvents
241 WaitEventMaskForInstPreGFX12[AMDGPU::NUM_INST_CNTS] = {
242 HWEvents::VMEM_READ_ACCESS | HWEvents::VMEM_SAMPLER_READ_ACCESS |
243 HWEvents::VMEM_BVH_READ_ACCESS,
244 HWEvents::SMEM_ACCESS | HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS |
245 HWEvents::SQ_MESSAGE,
246 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
247 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
248 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
249 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
250 HWEvents::NONE,
251 HWEvents::NONE,
252 HWEvents::NONE,
253 HWEvents::NONE,
254 HWEvents::NONE,
255 HWEvents::NONE,
256 HWEvents::NONE,
257 HWEvents::NONE,
258 HWEvents::NONE};
259
260public:
261 using WaitcntGenerator::WaitcntGenerator;
262 bool
263 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
264 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
265 MachineBasicBlock::instr_iterator It) const override;
266
267 bool createNewWaitcnt(MachineBasicBlock &Block,
268 MachineBasicBlock::instr_iterator It,
269 AMDGPU::Waitcnt Wait,
270 const WaitcntBrackets &ScoreBrackets) override;
271
272 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
273 HWEvents EVs = WaitEventMaskForInstPreGFX12[T];
274 if (T == AMDGPU::LOAD_CNT && !ST.hasVscnt())
275 EVs |= WaitEventMaskForInstPreGFX12[AMDGPU::STORE_CNT];
276 return EVs;
277 }
278
279 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
280};
281
282class WaitcntGeneratorGFX12Plus final : public WaitcntGenerator {
283protected:
284 bool IsExpertMode;
285 static constexpr const HWEvents
286 WaitEventMaskForInstGFX12Plus[AMDGPU::NUM_INST_CNTS] = {
287 HWEvents::VMEM_READ_ACCESS | HWEvents::GLOBAL_INV_ACCESS,
288 HWEvents::LDS_ACCESS | HWEvents::GDS_ACCESS,
289 HWEvents::EXP_GPR_LOCK | HWEvents::GDS_GPR_LOCK |
290 HWEvents::VMW_GPR_LOCK | HWEvents::EXP_PARAM_ACCESS |
291 HWEvents::EXP_POS_ACCESS | HWEvents::EXP_LDS_ACCESS,
292
293 HWEvents::VMEM_WRITE_ACCESS | HWEvents::SCRATCH_WRITE_ACCESS,
294 HWEvents::VMEM_SAMPLER_READ_ACCESS,
295 HWEvents::VMEM_BVH_READ_ACCESS,
296
297 HWEvents::SMEM_ACCESS | HWEvents::SQ_MESSAGE | HWEvents::SCC_WRITE,
298 HWEvents::VMEM_GROUP | HWEvents::SMEM_GROUP,
299 HWEvents::ASYNC_ACCESS,
300 HWEvents::TENSOR_ACCESS,
301 HWEvents::VGPR_CSMACC_READ | HWEvents::VGPR_DPMACC_READ |
302 HWEvents::VGPR_TRANS_READ | HWEvents::VGPR_XDL_READ,
303 HWEvents::VGPR_CSMACC_WRITE | HWEvents::VGPR_DPMACC_WRITE |
304 HWEvents::VGPR_TRANS_WRITE | HWEvents::VGPR_XDL_WRITE,
305 HWEvents::VGPR_LDS_READ | HWEvents::VGPR_FLAT_READ |
306 HWEvents::VGPR_VMEM_READ};
307
308public:
309 WaitcntGeneratorGFX12Plus() = delete;
310 WaitcntGeneratorGFX12Plus(const MachineFunction &MF,
311 AMDGPU::InstCounterType MaxCounter,
312 const AMDGPU::HardwareLimits &Limits,
313 bool IsExpertMode)
314 : WaitcntGenerator(MF, MaxCounter, Limits), IsExpertMode(IsExpertMode) {}
315
316 bool
317 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
318 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
319 MachineBasicBlock::instr_iterator It) const override;
320
321 bool createNewWaitcnt(MachineBasicBlock &Block,
322 MachineBasicBlock::instr_iterator It,
323 AMDGPU::Waitcnt Wait,
324 const WaitcntBrackets &ScoreBrackets) override;
325
326 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const override {
327 return WaitEventMaskForInstGFX12Plus[T];
328 }
329
330 AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
331};
332
333// Flags indicating which counters should be flushed in a loop preheader.
334struct PreheaderFlushFlags {
335 bool FlushVmCnt = false;
336 bool FlushDsCnt = false;
337};
338
339class SIInsertWaitcnts {
340 DenseMap<const Value *, MachineBasicBlock *> SLoadAddresses;
341 DenseMap<MachineBasicBlock *, PreheaderFlushFlags> PreheadersToFlush;
342 MachineLoopInfo &MLI;
343 MachinePostDominatorTree &PDT;
344 AliasAnalysis *AA = nullptr;
345 MachineFunction &MF;
346
347 struct BlockInfo {
348 std::unique_ptr<WaitcntBrackets> Incoming;
349 bool Dirty = true;
350 BlockInfo() = default;
351 BlockInfo(BlockInfo &&) = default;
352 BlockInfo &operator=(BlockInfo &&) = default;
353 ~BlockInfo();
354 };
355
356 MapVector<MachineBasicBlock *, BlockInfo> BlockInfos;
357
358 bool ForceEmitWaitcnt[AMDGPU::NUM_INST_CNTS] = {};
359
360 std::unique_ptr<WaitcntGenerator> WCG;
361
362 // Remember call and return instructions in the function.
363 DenseSet<MachineInstr *> CallInsts;
364 DenseSet<MachineInstr *> ReturnInsts;
365
366 // Remember all S_ENDPGM instructions. The boolean flag is true if there might
367 // be outstanding stores but definitely no outstanding scratch stores, to help
368 // with insertion of DEALLOC_VGPRS messages.
369 DenseMap<MachineInstr *, bool> EndPgmInsts;
370
371 AMDGPU::HardwareLimits Limits;
372
373public:
374 const GCNSubtarget &ST;
375 const SIInstrInfo &TII;
376 const SIRegisterInfo &TRI;
377 const MachineRegisterInfo &MRI;
378 AMDGPU::InstCounterType SmemAccessCounter;
379 AMDGPU::InstCounterType MaxCounter;
380 bool IsExpertMode = false;
381 const bool TgSplit;
382
383 SIInsertWaitcnts(MachineLoopInfo &MLI, MachinePostDominatorTree &PDT,
384 AliasAnalysis *AA, MachineFunction &MF)
385 : MLI(MLI), PDT(PDT), AA(AA), MF(MF), ST(MF.getSubtarget<GCNSubtarget>()),
386 TII(*ST.getInstrInfo()), TRI(TII.getRegisterInfo()),
387 MRI(MF.getRegInfo()),
388 TgSplit(ST.hasTgSplitSupport() &&
389 AMDGPU::isTgSplitEnabled(F: MF.getFunction())) {}
390
391 const AMDGPU::HardwareLimits &getLimits() const { return Limits; }
392
393 PreheaderFlushFlags getPreheaderFlushFlags(MachineLoop *ML,
394 const WaitcntBrackets &Brackets);
395 PreheaderFlushFlags isPreheaderToFlush(MachineBasicBlock &MBB,
396 const WaitcntBrackets &ScoreBrackets);
397 bool isVMEMOrFlatVMEM(const MachineInstr &MI) const;
398 bool isDSRead(const MachineInstr &MI) const;
399 bool mayStoreIncrementingDSCNT(const MachineInstr &MI) const;
400 bool run();
401
402 bool isAsync(const MachineInstr &MI) const {
403 if (!SIInstrInfo::isLDSDMA(MI))
404 return false;
405 if (SIInstrInfo::usesASYNC_CNT(MI))
406 return true;
407 const MachineOperand *Async =
408 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::IsAsync);
409 return Async && (Async->getImm());
410 }
411
412 bool isNonAsyncLdsDmaWrite(const MachineInstr &MI) const {
413 return SIInstrInfo::mayWriteLDSThroughDMA(MI) && !isAsync(MI);
414 }
415
416 bool isAsyncLdsDmaWrite(const MachineInstr &MI) const {
417 return SIInstrInfo::mayWriteLDSThroughDMA(MI) && isAsync(MI);
418 }
419
420 bool shouldUpdateAsyncMark(const MachineInstr &MI,
421 AMDGPU::InstCounterType T) const {
422 if (SIInstrInfo::usesTENSOR_CNT(MI))
423 return T == AMDGPU::TENSOR_CNT;
424 if (!isAsyncLdsDmaWrite(MI))
425 return false;
426 if (SIInstrInfo::usesASYNC_CNT(MI))
427 return T == AMDGPU::ASYNC_CNT;
428 return T == AMDGPU::LOAD_CNT;
429 }
430
431 bool isVmemAccess(const MachineInstr &MI) const;
432 bool generateWaitcntInstBefore(MachineInstr &MI,
433 WaitcntBrackets &ScoreBrackets,
434 MachineInstr *OldWaitcntInstr,
435 PreheaderFlushFlags FlushFlags);
436 bool generateWaitcnt(AMDGPU::Waitcnt Wait,
437 MachineBasicBlock::instr_iterator It,
438 MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
439 MachineInstr *OldWaitcntInstr);
440 void updateEventWaitcntAfter(MachineInstr &Inst,
441 WaitcntBrackets *ScoreBrackets);
442 bool isNextENDPGM(MachineBasicBlock::instr_iterator It,
443 MachineBasicBlock *Block) const;
444 bool insertForcedWaitAfter(MachineInstr &Inst, MachineBasicBlock &Block,
445 WaitcntBrackets &ScoreBrackets);
446 bool insertWaitcntInBlock(MachineFunction &MF, MachineBasicBlock &Block,
447 WaitcntBrackets &ScoreBrackets);
448 /// Removes redundant Soft Xcnt Waitcnts in \p Block emitted by the Memory
449 /// Legalizer. Returns true if block was modified.
450 bool removeRedundantSoftXcnts(MachineBasicBlock &Block);
451 void setSchedulingMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
452 bool ExpertMode) const;
453 HWEvents getWaitEvents(AMDGPU::InstCounterType T) const {
454 return WCG->getWaitEvents(T);
455 }
456 AMDGPU::InstCounterType getCounterFromEvent(HWEvents E) const {
457 return WCG->getCounterFromEvent(E);
458 }
459};
460
461// This objects maintains the current score brackets of each wait counter, and
462// a per-register scoreboard for each wait counter.
463//
464// We also maintain the latest score for every event type that can change the
465// waitcnt in order to know if there are multiple types of events within
466// the brackets. When multiple types of event happen in the bracket,
467// wait count may get decreased out of order, therefore we need to put in
468// "s_waitcnt 0" before use.
469class WaitcntBrackets {
470public:
471 WaitcntBrackets(const SIInsertWaitcnts *Context) : Context(Context) {
472 assert(Context->TRI.getNumRegUnits() < REGUNITS_END);
473 }
474
475#ifndef NDEBUG
476 ~WaitcntBrackets() {
477 unsigned NumUnusedVmem = 0, NumUnusedSGPRs = 0;
478 for (auto &[ID, Val] : VMem) {
479 if (Val.empty())
480 ++NumUnusedVmem;
481 }
482 for (auto &[ID, Val] : SGPRs) {
483 if (Val.empty())
484 ++NumUnusedSGPRs;
485 }
486
487 if (NumUnusedVmem || NumUnusedSGPRs) {
488 errs() << "WaitcntBracket had unused entries at destruction time: "
489 << NumUnusedVmem << " VMem and " << NumUnusedSGPRs
490 << " SGPR unused entries\n";
491 std::abort();
492 }
493 }
494#endif
495
496 bool isSmemCounter(AMDGPU::InstCounterType T) const {
497 return T == Context->SmemAccessCounter || T == AMDGPU::X_CNT;
498 }
499
500 unsigned getOutstanding(AMDGPU::InstCounterType T) const {
501 return ScoreUBs[T] - ScoreLBs[T];
502 }
503
504 bool hasPendingVMEM(VMEMID ID, AMDGPU::InstCounterType T) const {
505 return getVMemScore(TID: ID, T) > getScoreLB(T);
506 }
507
508 /// \Return true if we have no score entries for counter \p T.
509 bool empty(AMDGPU::InstCounterType T) const { return getScoreRange(T) == 0; }
510
511private:
512 unsigned getScoreLB(AMDGPU::InstCounterType T) const {
513 assert(T < AMDGPU::NUM_INST_CNTS);
514 return ScoreLBs[T];
515 }
516
517 unsigned getScoreUB(AMDGPU::InstCounterType T) const {
518 assert(T < AMDGPU::NUM_INST_CNTS);
519 return ScoreUBs[T];
520 }
521
522 unsigned getScoreRange(AMDGPU::InstCounterType T) const {
523 return getScoreUB(T) - getScoreLB(T);
524 }
525
526 unsigned getSGPRScore(MCRegUnit RU, AMDGPU::InstCounterType T) const {
527 auto It = SGPRs.find(Val: RU);
528 return It != SGPRs.end() ? It->second.get(T) : 0;
529 }
530
531 unsigned getVMemScore(VMEMID TID, AMDGPU::InstCounterType T) const {
532 auto It = VMem.find(Val: TID);
533 return It != VMem.end() ? It->second.Scores[T] : 0;
534 }
535
536public:
537 bool merge(const WaitcntBrackets &Other);
538
539 bool counterOutOfOrder(AMDGPU::InstCounterType T) const;
540 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const {
541 simplifyWaitcnt(CheckWait: Wait, UpdateWait&: Wait);
542 }
543 void simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
544 AMDGPU::Waitcnt &UpdateWait) const;
545 void simplifyWaitcnt(AMDGPU::InstCounterType T, unsigned &Count) const;
546 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T) const;
547 void simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
548 AMDGPU::Waitcnt &UpdateWait) const;
549 void simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
550 AMDGPU::Waitcnt &UpdateWait) const;
551
552 void determineWaitForPhysReg(AMDGPU::InstCounterType T, MCPhysReg Reg,
553 AMDGPU::Waitcnt &Wait,
554 const MachineInstr &MI) const;
555 MCPhysReg determineVGPR16Dependency(const MachineInstr &MI,
556 AMDGPU::InstCounterType T,
557 MCPhysReg Reg) const;
558 void determineWaitForLDSDMA(AMDGPU::InstCounterType T, VMEMID TID,
559 AMDGPU::Waitcnt &Wait) const;
560 AMDGPU::Waitcnt determineAsyncWait(unsigned N);
561 void tryClearSCCWriteEvent(MachineInstr *Inst);
562
563 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
564 void applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count);
565 void applyWaitcnt(const AMDGPU::Waitcnt &Wait, AMDGPU::InstCounterType T);
566 void updateByEvent(HWEvents E, MachineInstr &MI);
567 void recordAsyncMark(MachineInstr &MI);
568
569 HWEvents getPendingEvents() const { return PendingEvents; }
570 bool hasPendingEvent() const { return PendingEvents.any(); }
571 bool hasPendingEvent(HWEvents E) const { return PendingEvents.contains(Other: E); }
572 bool hasPendingEvent(AMDGPU::InstCounterType T) const {
573 bool HasPending = (PendingEvents & Context->getWaitEvents(T)).any();
574 assert(HasPending == !empty(T) &&
575 "Expected pending events iff scoreboard is not empty");
576 return HasPending;
577 }
578
579 bool hasMixedPendingEvents(AMDGPU::InstCounterType T) const {
580 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
581 // Return true if more than one bit is set in Events.
582 return Events.size() > 1;
583 }
584
585 bool hasPendingFlat() const {
586 return ((LastFlatDsCnt > ScoreLBs[AMDGPU::DS_CNT] &&
587 LastFlatDsCnt <= ScoreUBs[AMDGPU::DS_CNT]) ||
588 (LastFlatLoadCnt > ScoreLBs[AMDGPU::LOAD_CNT] &&
589 LastFlatLoadCnt <= ScoreUBs[AMDGPU::LOAD_CNT]));
590 }
591
592 void setPendingFlat() {
593 LastFlatLoadCnt = ScoreUBs[AMDGPU::LOAD_CNT];
594 LastFlatDsCnt = ScoreUBs[AMDGPU::DS_CNT];
595 }
596
597 bool hasPendingGDS() const {
598 return LastGDS > ScoreLBs[AMDGPU::DS_CNT] &&
599 LastGDS <= ScoreUBs[AMDGPU::DS_CNT];
600 }
601
602 unsigned getPendingGDSWait() const {
603 return std::min(a: getScoreUB(T: AMDGPU::DS_CNT) - LastGDS,
604 b: getLimit(T: AMDGPU::DS_CNT) - 1);
605 }
606
607 void setPendingGDS() { LastGDS = ScoreUBs[AMDGPU::DS_CNT]; }
608
609 // Return true if there might be pending writes to the vgpr-interval by VMEM
610 // instructions where the HWEvents in VGPRContext are not contained in E.
611 bool hasDifferentVGPRPendingEvents(MCPhysReg Reg, HWEvents E) const {
612 for (MCRegUnit RU : regunits(Reg)) {
613 auto It = VMem.find(Val: toVMEMID(RU));
614 if (It != VMem.end() && (It->second.VGPRPendingEvents & ~E).any())
615 return true;
616 }
617 return false;
618 }
619
620 void clearVGPRPendingEvents(MCPhysReg Reg) {
621 for (MCRegUnit RU : regunits(Reg)) {
622 if (auto It = VMem.find(Val: toVMEMID(RU)); It != VMem.end()) {
623 It->second.VGPRPendingEvents = HWEvents::NONE;
624 if (It->second.empty())
625 VMem.erase(I: It);
626 }
627 }
628 }
629
630 void setStateOnFunctionEntryOrReturn() {
631 setScoreUB(T: AMDGPU::STORE_CNT,
632 Val: getScoreUB(T: AMDGPU::STORE_CNT) + getLimit(T: AMDGPU::STORE_CNT));
633 PendingEvents |= Context->getWaitEvents(T: AMDGPU::STORE_CNT);
634 }
635
636 ArrayRef<const MachineInstr *> getLDSDMAStores() const {
637 return LDSDMAStores;
638 }
639
640 bool hasPointSampleAccel(const MachineInstr &MI) const;
641 bool hasPointSamplePendingVmemTypes(const MachineInstr &MI,
642 MCPhysReg RU) const;
643
644 void print(raw_ostream &) const;
645 void dump() const { print(dbgs()); }
646
647 // Free up memory by removing empty entries from the DenseMap that track event
648 // scores.
649 void purgeEmptyTrackingData();
650
651private:
652 unsigned getLimit(AMDGPU::InstCounterType T) const {
653 return Context->getLimits().get(T);
654 }
655
656 struct MergeInfo {
657 unsigned OldLB;
658 unsigned OtherLB;
659 unsigned MyShift;
660 unsigned OtherShift;
661 };
662
663 using CounterValueArray = std::array<unsigned, AMDGPU::NUM_INST_CNTS>;
664
665 void determineWaitForScore(AMDGPU::InstCounterType T, unsigned Score,
666 AMDGPU::Waitcnt &Wait) const;
667
668 static bool mergeScore(const MergeInfo &M, unsigned &Score,
669 unsigned OtherScore);
670 bool mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
671 ArrayRef<CounterValueArray> OtherMarks);
672
673 iterator_range<MCRegUnitIterator> regunits(MCPhysReg Reg) const {
674 assert(Reg != AMDGPU::SCC && "Shouldn't be used on SCC");
675 if (!Context->TRI.isInAllocatableClass(RegNo: Reg))
676 return {{}, {}};
677 return Context->TRI.regunits(Reg);
678 }
679
680 void setScoreLB(AMDGPU::InstCounterType T, unsigned Val) {
681 assert(T < AMDGPU::NUM_INST_CNTS);
682 ScoreLBs[T] = Val;
683 }
684
685 void setScoreUB(AMDGPU::InstCounterType T, unsigned Val) {
686 assert(T < AMDGPU::NUM_INST_CNTS);
687 ScoreUBs[T] = Val;
688
689 if (T != AMDGPU::EXP_CNT)
690 return;
691
692 if (getScoreRange(T: AMDGPU::EXP_CNT) > getLimit(T: AMDGPU::EXP_CNT))
693 ScoreLBs[AMDGPU::EXP_CNT] =
694 ScoreUBs[AMDGPU::EXP_CNT] - getLimit(T: AMDGPU::EXP_CNT);
695 }
696
697 void setRegScore(MCPhysReg Reg, AMDGPU::InstCounterType T, unsigned Val) {
698 const SIRegisterInfo &TRI = Context->TRI;
699 if (Reg == AMDGPU::SCC) {
700 SCCScore = Val;
701 } else if (TRI.isVectorRegister(MRI: Context->MRI, Reg)) {
702 for (MCRegUnit RU : regunits(Reg))
703 VMem[toVMEMID(RU)].Scores[T] = Val;
704 } else if (TRI.isSGPRReg(MRI: Context->MRI, Reg)) {
705 for (MCRegUnit RU : regunits(Reg))
706 SGPRs[RU].get(T) = Val;
707 } else {
708 llvm_unreachable("Register cannot be tracked/unknown register!");
709 }
710 }
711
712 void setVMemScore(VMEMID TID, AMDGPU::InstCounterType T, unsigned Val) {
713 VMem[TID].Scores[T] = Val;
714 }
715
716 void setScoreByOperand(const MachineOperand &Op,
717 AMDGPU::InstCounterType CntTy, unsigned Val);
718
719 const SIInsertWaitcnts *Context;
720
721 unsigned ScoreLBs[AMDGPU::NUM_INST_CNTS] = {0};
722 unsigned ScoreUBs[AMDGPU::NUM_INST_CNTS] = {0};
723 HWEvents PendingEvents;
724 // Remember the last flat memory operation.
725 unsigned LastFlatDsCnt = 0;
726 unsigned LastFlatLoadCnt = 0;
727 // Remember the last GDS operation.
728 unsigned LastGDS = 0;
729
730 // The score tracking logic is fragmented as follows:
731 // - VMem: VGPR RegUnits and LDS DMA IDs, see the VMEMID encoding.
732 // - SGPRs: SGPR RegUnits
733 // - SCC: Non-allocatable and not general purpose: not a SGPR.
734 //
735 // For the VMem case, if the key is within the range of LDS DMA IDs,
736 // then the corresponding index into the `LDSDMAStores` vector below is:
737 // Key - LDSDMA_BEGIN - 1
738 // This is because LDSDMA_BEGIN is a generic entry and does not have an
739 // associated MachineInstr.
740 //
741 // TODO: Could we track SCC alongside SGPRs so it's not longer a special case?
742
743 struct VMEMInfo {
744 // Scores for all instruction counters. Zero-initialized.
745 CounterValueArray Scores{};
746 // For VGPRs, we need to track an additional fine-grained set of pending
747 // events.
748 HWEvents VGPRPendingEvents;
749
750 bool empty() const {
751 return all_of(Range: Scores, P: equal_to(Arg: 0)) && !VGPRPendingEvents;
752 }
753 };
754
755 /// Wait cnt scores for every sgpr, the DS_CNT (corresponding to LGKMcnt
756 /// pre-gfx12) or KM_CNT (gfx12+ only), and X_CNT (gfx1250) are relevant.
757 class SGPRInfo {
758 /// Either DS_CNT or KM_CNT score.
759 unsigned ScoreDsKmCnt = 0;
760 unsigned ScoreXCnt = 0;
761
762 public:
763 unsigned get(AMDGPU::InstCounterType T) const {
764 assert(
765 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
766 "Invalid counter");
767 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
768 }
769 unsigned &get(AMDGPU::InstCounterType T) {
770 assert(
771 (T == AMDGPU::DS_CNT || T == AMDGPU::KM_CNT || T == AMDGPU::X_CNT) &&
772 "Invalid counter");
773 return T == AMDGPU::X_CNT ? ScoreXCnt : ScoreDsKmCnt;
774 }
775
776 bool empty() const { return !ScoreDsKmCnt && !ScoreXCnt; }
777 };
778
779 DenseMap<VMEMID, VMEMInfo> VMem; // VGPR + LDS DMA
780 DenseMap<MCRegUnit, SGPRInfo> SGPRs;
781
782 // Reg score for SCC.
783 unsigned SCCScore = 0;
784 // The unique instruction that has an SCC write pending, if there is one.
785 const MachineInstr *PendingSCCWrite = nullptr;
786
787 // Store representative LDS DMA operations. The only useful info here is
788 // alias info. One store is kept per unique AAInfo.
789 SmallVector<const MachineInstr *> LDSDMAStores;
790
791 // State of all counters at each async mark encountered so far.
792 SmallVector<CounterValueArray> AsyncMarks;
793
794 // But in the rare pathological case, a nest of loops that pushes marks
795 // without waiting on any mark can cause AsyncMarks to grow very large. We cap
796 // it to a reasonable limit. We can tune this later or potentially introduce a
797 // user option to control the value.
798 static constexpr unsigned MaxAsyncMarks = 16;
799
800 // Track the upper bound score for async operations that are not part of a
801 // mark yet. Initialized to all zeros.
802 CounterValueArray AsyncScore{};
803};
804
805SIInsertWaitcnts::BlockInfo::~BlockInfo() = default;
806
807class SIInsertWaitcntsLegacy : public MachineFunctionPass {
808public:
809 static char ID;
810 SIInsertWaitcntsLegacy() : MachineFunctionPass(ID) {}
811
812 bool runOnMachineFunction(MachineFunction &MF) override;
813
814 StringRef getPassName() const override {
815 return "SI insert wait instructions";
816 }
817
818 void getAnalysisUsage(AnalysisUsage &AU) const override {
819 AU.setPreservesCFG();
820 AU.addRequired<MachineLoopInfoWrapperPass>();
821 AU.addRequired<MachinePostDominatorTreeWrapperPass>();
822 AU.addUsedIfAvailable<AAResultsWrapperPass>();
823 AU.addPreserved<AAResultsWrapperPass>();
824 MachineFunctionPass::getAnalysisUsage(AU);
825 }
826};
827
828} // end anonymous namespace
829
830void WaitcntBrackets::setScoreByOperand(const MachineOperand &Op,
831 AMDGPU::InstCounterType CntTy,
832 unsigned Score) {
833 setRegScore(Reg: Op.getReg().asMCReg(), T: CntTy, Val: Score);
834}
835
836// Return true if the subtarget is one that enables Point Sample Acceleration
837// and the MachineInstr passed in is one to which it might be applied (the
838// hardware makes this decision based on several factors, but we can't determine
839// this at compile time, so we have to assume it might be applied if the
840// instruction supports it).
841bool WaitcntBrackets::hasPointSampleAccel(const MachineInstr &MI) const {
842 if (!Context->ST.hasPointSampleAccel() || !SIInstrInfo::isMIMG(MI))
843 return false;
844
845 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: MI.getOpcode());
846 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
847 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
848 return BaseInfo->PointSampleAccel;
849}
850
851// Return true if the subtarget enables Point Sample Acceleration, the supplied
852// MachineInstr is one to which it might be applied and the supplied interval is
853// one that has outstanding writes to vmem-types different than VMEM_NOSAMPLER
854// (this is the type that a point sample accelerated instruction effectively
855// becomes)
856bool WaitcntBrackets::hasPointSamplePendingVmemTypes(const MachineInstr &MI,
857 MCPhysReg Reg) const {
858 if (!hasPointSampleAccel(MI))
859 return false;
860
861 return hasDifferentVGPRPendingEvents(Reg, E: HWEvents::VMEM_READ_ACCESS);
862}
863
864void WaitcntBrackets::updateByEvent(HWEvents E, MachineInstr &Inst) {
865 assert(E.size() == 1 && "Expected singular event!");
866 AMDGPU::InstCounterType T = Context->getCounterFromEvent(E);
867 assert(T < Context->MaxCounter);
868
869 unsigned UB = getScoreUB(T);
870 unsigned Increment = 1;
871 if ((T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR) &&
872 AMDGPU::getHasMatrixScale(Opc: Inst.getOpcode()) &&
873 Context->ST.hasVOP3PX2IncrementsVaVdstTwice()) {
874 // V_WMMA_SCALE instructions use VOP3PX2 encoding. Hardware treats this as
875 // two VOP3P instructions and increments VA_VDST twice.
876 Increment = 2;
877 }
878 unsigned CurrScore = UB + Increment;
879 if (CurrScore == 0)
880 report_fatal_error(reason: "InsertWaitcnt score wraparound");
881 // PendingEvents and ScoreUB need to be update regardless if this event
882 // changes the score of a register or not.
883 // Examples including vm_cnt when buffer-store or lgkm_cnt when send-message.
884 PendingEvents |= E;
885 setScoreUB(T, Val: CurrScore);
886
887 const SIRegisterInfo &TRI = Context->TRI;
888 const MachineRegisterInfo &MRI = Context->MRI;
889 const SIInstrInfo &TII = Context->TII;
890
891 if (T == AMDGPU::EXP_CNT) {
892 // Put score on the source vgprs. If this is a store, just use those
893 // specific register(s).
894 if (TII.isDS(MI: Inst) && Inst.mayLoadOrStore()) {
895 // All GDS operations must protect their address register (same as
896 // export.)
897 if (const auto *AddrOp = TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::addr))
898 setScoreByOperand(Op: *AddrOp, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
899
900 if (Inst.mayStore()) {
901 if (const auto *Data0 =
902 TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data0))
903 setScoreByOperand(Op: *Data0, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
904 if (const auto *Data1 =
905 TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data1))
906 setScoreByOperand(Op: *Data1, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
907 } else if (SIInstrInfo::isAtomicRet(MI: Inst) && !SIInstrInfo::isGWS(MI: Inst) &&
908 Inst.getOpcode() != AMDGPU::DS_APPEND &&
909 Inst.getOpcode() != AMDGPU::DS_CONSUME &&
910 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
911 for (const MachineOperand &Op : Inst.all_uses()) {
912 if (TRI.isVectorRegister(MRI, Reg: Op.getReg()))
913 setScoreByOperand(Op, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
914 }
915 }
916 } else if (TII.isFLAT(MI: Inst)) {
917 if (Inst.mayStore()) {
918 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
919 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
920 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
921 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
922 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
923 }
924 } else if (TII.isMIMG(MI: Inst)) {
925 if (Inst.mayStore()) {
926 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
927 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
928 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
929 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
930 }
931 } else if (TII.isMTBUF(MI: Inst)) {
932 if (Inst.mayStore())
933 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
934 } else if (TII.isMUBUF(MI: Inst)) {
935 if (Inst.mayStore()) {
936 setScoreByOperand(Op: Inst.getOperand(i: 0), CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
937 } else if (SIInstrInfo::isAtomicRet(MI: Inst)) {
938 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::data),
939 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
940 }
941 } else if (TII.isLDSDIR(MI: Inst)) {
942 // LDSDIR instructions attach the score to the destination.
943 setScoreByOperand(Op: *TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::vdst),
944 CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
945 } else {
946 if (TII.isEXP(MI: Inst)) {
947 // For export the destination registers are really temps that
948 // can be used as the actual source after export patching, so
949 // we need to treat them like sources and set the EXP_CNT
950 // score.
951 for (MachineOperand &DefMO : Inst.all_defs()) {
952 if (TRI.isVGPR(MRI, Reg: DefMO.getReg())) {
953 setScoreByOperand(Op: DefMO, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
954 }
955 }
956 }
957 for (const MachineOperand &Op : Inst.all_uses()) {
958 if (TRI.isVectorRegister(MRI, Reg: Op.getReg()))
959 setScoreByOperand(Op, CntTy: AMDGPU::EXP_CNT, Score: CurrScore);
960 }
961 }
962 } else if (T == AMDGPU::X_CNT) {
963 HWEvents OtherEvent =
964 E == HWEvents::SMEM_GROUP ? HWEvents::VMEM_GROUP : HWEvents::SMEM_GROUP;
965 if (PendingEvents.contains(Other: OtherEvent)) {
966 // Hardware inserts an implicit xcnt between interleaved
967 // SMEM and VMEM operations. So there will never be
968 // outstanding address translations for both SMEM and
969 // VMEM at the same time.
970 setScoreLB(T, Val: getScoreUB(T) - 1);
971 PendingEvents -= OtherEvent;
972 }
973 for (const MachineOperand &Op : Inst.all_uses())
974 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
975 } else if (T == AMDGPU::VA_VDST_RD || T == AMDGPU::VA_VDST_WR ||
976 T == AMDGPU::VM_VSRC) {
977 // Match the score to the VGPR destination or source registers as
978 // appropriate
979 for (const MachineOperand &Op : Inst.operands()) {
980 if (!Op.isReg())
981 continue;
982
983 // Skip based on counter type and operand type
984 if (T == AMDGPU::VA_VDST_RD && Op.isDef())
985 continue; // RD tracks reads only
986 if (T == AMDGPU::VA_VDST_WR && Op.isUse())
987 continue; // WR tracks writes only
988 if (T == AMDGPU::VM_VSRC && Op.isDef())
989 continue;
990
991 if (TRI.isVectorRegister(MRI: Context->MRI, Reg: Op.getReg()))
992 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
993 }
994 } else /* LGKM_CNT || EXP_CNT || VS_CNT || NUM_INST_CNTS */ {
995 // Match the score to the destination registers.
996 //
997 // Check only explicit operands. Stores, especially spill stores, include
998 // implicit uses and defs of their super registers which would create an
999 // artificial dependency, while these are there only for register liveness
1000 // accounting purposes.
1001 //
1002 // Special cases where implicit register defs exists, such as M0 or VCC,
1003 // but none with memory instructions.
1004 for (const MachineOperand &Op : Inst.defs()) {
1005 if (T == AMDGPU::LOAD_CNT || T == AMDGPU::SAMPLE_CNT ||
1006 T == AMDGPU::BVH_CNT) {
1007 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg())) // TODO: add wrapper
1008 continue;
1009 if (updateVMCntOnly(Inst)) {
1010 // updateVMCntOnly should only leave us with VGPRs
1011 // MUBUF, MTBUF, MIMG, FlatGlobal, and FlatScratch only have VGPR/AGPR
1012 // defs.
1013 assert(TRI.isVectorRegister(MRI, Op.getReg()));
1014 HWEvents VGPRContext =
1015 AMDGPU::getSimplifiedVMEMEventsFor(Inst, TII: Context->TII);
1016 // If instruction can have Point Sample Accel applied, we have to flag
1017 // this with another potential dependency
1018 if (hasPointSampleAccel(MI: Inst))
1019 VGPRContext |= HWEvents::VMEM_READ_ACCESS;
1020 for (MCRegUnit RU : regunits(Reg: Op.getReg().asMCReg()))
1021 VMem[toVMEMID(RU)].VGPRPendingEvents |= VGPRContext;
1022 }
1023 }
1024 setScoreByOperand(Op, CntTy: T, Score: CurrScore);
1025 }
1026 if (Inst.mayStore() &&
1027 (TII.isDS(MI: Inst) || Context->isNonAsyncLdsDmaWrite(MI: Inst))) {
1028 // MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS
1029 // written can be accessed. A load from LDS to VMEM does not need a wait.
1030 //
1031 // The "Slot" is the offset from LDSDMA_BEGIN. If it's non-zero, then
1032 // there is a MachineInstr in LDSDMAStores used to track this LDSDMA
1033 // store. The "Slot" is the index into LDSDMAStores + 1.
1034 unsigned Slot = 0;
1035 for (const auto *MemOp : Inst.memoperands()) {
1036 if (!MemOp->isStore() ||
1037 MemOp->getAddrSpace() != AMDGPUAS::LOCAL_ADDRESS)
1038 continue;
1039 // Comparing just AA info does not guarantee memoperands are equal
1040 // in general, but this is so for LDS DMA in practice.
1041 auto AAI = MemOp->getAAInfo();
1042 // Alias scope information gives a way to definitely identify an
1043 // original memory object and practically produced in the module LDS
1044 // lowering pass. If there is no scope available we will not be able
1045 // to disambiguate LDS aliasing as after the module lowering all LDS
1046 // is squashed into a single big object.
1047 if (!AAI || !AAI.Scope)
1048 break;
1049 for (unsigned I = 0, E = LDSDMAStores.size(); I != E && !Slot; ++I) {
1050 for (const auto *MemOp : LDSDMAStores[I]->memoperands()) {
1051 if (MemOp->isStore() && AAI == MemOp->getAAInfo()) {
1052 Slot = I + 1;
1053 break;
1054 }
1055 }
1056 }
1057 if (Slot)
1058 break;
1059 // The slot may not be valid because it can be >= NUM_LDSDMA which
1060 // means the scoreboard cannot track it. We still want to preserve the
1061 // MI in order to check alias information, though.
1062 LDSDMAStores.push_back(Elt: &Inst);
1063 Slot = LDSDMAStores.size();
1064 break;
1065 }
1066 setVMemScore(TID: LDSDMA_BEGIN, T, Val: CurrScore);
1067 if (Slot && Slot < NUM_LDSDMA)
1068 setVMemScore(TID: LDSDMA_BEGIN + Slot, T, Val: CurrScore);
1069 }
1070
1071 if (Context->shouldUpdateAsyncMark(MI: Inst, T)) {
1072 AsyncScore[T] = CurrScore;
1073 }
1074
1075 if (SIInstrInfo::isSBarrierSCCWrite(Opcode: Inst.getOpcode())) {
1076 setRegScore(Reg: AMDGPU::SCC, T, Val: CurrScore);
1077 PendingSCCWrite = &Inst;
1078 }
1079 }
1080}
1081
1082void WaitcntBrackets::recordAsyncMark(MachineInstr &Inst) {
1083 // In the absence of loops, AsyncMarks can grow linearly with the program
1084 // until we encounter an ASYNCMARK_WAIT. We could drop the oldest mark above a
1085 // limit every time we push a new mark, but that seems like unnecessary work
1086 // in practical cases. We do separately truncate the array when processing a
1087 // loop, which should be sufficient.
1088 AsyncMarks.push_back(Elt: AsyncScore);
1089 AsyncScore = {};
1090 LLVM_DEBUG({
1091 dbgs() << "recordAsyncMark:\n" << Inst;
1092 for (const auto &Mark : AsyncMarks) {
1093 llvm::interleaveComma(Mark, dbgs());
1094 dbgs() << '\n';
1095 }
1096 });
1097}
1098
1099void WaitcntBrackets::print(raw_ostream &OS) const {
1100 const GCNSubtarget &ST = Context->ST;
1101
1102 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter)) {
1103 unsigned SR = getScoreRange(T);
1104 switch (T) {
1105 case AMDGPU::LOAD_CNT:
1106 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM") << "_CNT("
1107 << SR << "):";
1108 break;
1109 case AMDGPU::DS_CNT:
1110 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM") << "_CNT("
1111 << SR << "):";
1112 break;
1113 case AMDGPU::EXP_CNT:
1114 OS << " EXP_CNT(" << SR << "):";
1115 break;
1116 case AMDGPU::STORE_CNT:
1117 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS") << "_CNT("
1118 << SR << "):";
1119 break;
1120 case AMDGPU::SAMPLE_CNT:
1121 OS << " SAMPLE_CNT(" << SR << "):";
1122 break;
1123 case AMDGPU::BVH_CNT:
1124 OS << " BVH_CNT(" << SR << "):";
1125 break;
1126 case AMDGPU::KM_CNT:
1127 OS << " KM_CNT(" << SR << "):";
1128 break;
1129 case AMDGPU::X_CNT:
1130 OS << " X_CNT(" << SR << "):";
1131 break;
1132 case AMDGPU::ASYNC_CNT:
1133 OS << " ASYNC_CNT(" << SR << "):";
1134 break;
1135 case AMDGPU::VA_VDST_RD:
1136 OS << " VA_VDST_RD(" << SR << "): ";
1137 break;
1138 case AMDGPU::VA_VDST_WR:
1139 OS << " VA_VDST_WR(" << SR << "): ";
1140 break;
1141 case AMDGPU::VM_VSRC:
1142 OS << " VM_VSRC(" << SR << "): ";
1143 break;
1144 default:
1145 OS << " UNKNOWN(" << SR << "):";
1146 break;
1147 }
1148
1149 if (SR != 0) {
1150 // Print vgpr scores.
1151 unsigned LB = getScoreLB(T);
1152
1153 SmallVector<VMEMID> SortedVMEMIDs(VMem.keys());
1154 sort(C&: SortedVMEMIDs);
1155
1156 for (auto ID : SortedVMEMIDs) {
1157 unsigned RegScore = VMem.at(Val: ID).Scores[T];
1158 if (RegScore <= LB)
1159 continue;
1160 unsigned RelScore = RegScore - LB - 1;
1161 if (ID < REGUNITS_END) {
1162 OS << ' ' << RelScore << ':'
1163 << printRegUnit(Unit: static_cast<MCRegUnit>(ID), TRI: &Context->TRI);
1164 } else {
1165 assert(ID >= LDSDMA_BEGIN && ID < LDSDMA_END &&
1166 "Unhandled/unexpected ID value!");
1167 OS << ' ' << RelScore << ":LDSDMA" << ID;
1168 }
1169 }
1170
1171 // Also need to print sgpr scores for lgkm_cnt or xcnt.
1172 if (isSmemCounter(T)) {
1173 SmallVector<MCRegUnit> SortedSMEMIDs(SGPRs.keys());
1174 sort(C&: SortedSMEMIDs);
1175 for (auto ID : SortedSMEMIDs) {
1176 unsigned RegScore = SGPRs.at(Val: ID).get(T);
1177 if (RegScore <= LB)
1178 continue;
1179 unsigned RelScore = RegScore - LB - 1;
1180 OS << ' ' << RelScore << ':'
1181 << printRegUnit(Unit: static_cast<MCRegUnit>(ID), TRI: &Context->TRI);
1182 }
1183 }
1184
1185 if (T == AMDGPU::KM_CNT && SCCScore > 0)
1186 OS << ' ' << SCCScore << ":scc";
1187 }
1188 OS << '\n';
1189 }
1190
1191 OS << "Pending Events: ";
1192 if (hasPendingEvent()) {
1193 OS << getPendingEvents();
1194 } else {
1195 OS << "none";
1196 }
1197 OS << '\n';
1198
1199 OS << "Async score: ";
1200 if (AsyncScore.empty())
1201 OS << "none";
1202 else
1203 llvm::interleaveComma(c: AsyncScore, os&: OS);
1204 OS << '\n';
1205
1206 OS << "Async marks: " << AsyncMarks.size() << '\n';
1207
1208 for (const auto &Mark : AsyncMarks) {
1209 for (auto T : AMDGPU::inst_counter_types()) {
1210 unsigned MarkedScore = Mark[T];
1211 switch (T) {
1212 case AMDGPU::LOAD_CNT:
1213 OS << " " << (ST.hasExtendedWaitCounts() ? "LOAD" : "VM")
1214 << "_CNT: " << MarkedScore;
1215 break;
1216 case AMDGPU::DS_CNT:
1217 OS << " " << (ST.hasExtendedWaitCounts() ? "DS" : "LGKM")
1218 << "_CNT: " << MarkedScore;
1219 break;
1220 case AMDGPU::EXP_CNT:
1221 OS << " EXP_CNT: " << MarkedScore;
1222 break;
1223 case AMDGPU::STORE_CNT:
1224 OS << " " << (ST.hasExtendedWaitCounts() ? "STORE" : "VS")
1225 << "_CNT: " << MarkedScore;
1226 break;
1227 case AMDGPU::SAMPLE_CNT:
1228 OS << " SAMPLE_CNT: " << MarkedScore;
1229 break;
1230 case AMDGPU::BVH_CNT:
1231 OS << " BVH_CNT: " << MarkedScore;
1232 break;
1233 case AMDGPU::KM_CNT:
1234 OS << " KM_CNT: " << MarkedScore;
1235 break;
1236 case AMDGPU::X_CNT:
1237 OS << " X_CNT: " << MarkedScore;
1238 break;
1239 case AMDGPU::ASYNC_CNT:
1240 OS << " ASYNC_CNT: " << MarkedScore;
1241 break;
1242 default:
1243 OS << " UNKNOWN: " << MarkedScore;
1244 break;
1245 }
1246 }
1247 OS << '\n';
1248 }
1249 OS << '\n';
1250}
1251
1252/// Simplify \p UpdateWait by removing waits that are redundant based on the
1253/// current WaitcntBrackets and any other waits specified in \p CheckWait.
1254void WaitcntBrackets::simplifyWaitcnt(const AMDGPU::Waitcnt &CheckWait,
1255 AMDGPU::Waitcnt &UpdateWait) const {
1256 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::LOAD_CNT);
1257 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::EXP_CNT);
1258 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::DS_CNT);
1259 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::STORE_CNT);
1260 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::SAMPLE_CNT);
1261 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::BVH_CNT);
1262 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::KM_CNT);
1263 simplifyXcnt(CheckWait, UpdateWait);
1264 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VA_VDST_RD);
1265 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VA_VDST_WR);
1266 simplifyVmVsrc(CheckWait, UpdateWait);
1267 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::ASYNC_CNT);
1268}
1269
1270void WaitcntBrackets::simplifyWaitcnt(AMDGPU::InstCounterType T,
1271 unsigned &Count) const {
1272 // The number of outstanding events for this type, T, can be calculated
1273 // as (UB - LB). If the current Count is greater than or equal to the number
1274 // of outstanding events, then the wait for this counter is redundant.
1275 if (Count >= getScoreRange(T))
1276 Count = ~0u;
1277}
1278
1279void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait,
1280 AMDGPU::InstCounterType T) const {
1281 unsigned Cnt = Wait.get(T);
1282 simplifyWaitcnt(T, Count&: Cnt);
1283 Wait.set(T, Val: Cnt);
1284}
1285
1286void WaitcntBrackets::simplifyXcnt(const AMDGPU::Waitcnt &CheckWait,
1287 AMDGPU::Waitcnt &UpdateWait) const {
1288 // Try to simplify xcnt further by checking for joint kmcnt and loadcnt
1289 // optimizations. On entry to a block with multiple predescessors, there may
1290 // be pending SMEM and VMEM events active at the same time.
1291 // In such cases, only clear one active event at a time.
1292 // TODO: Revisit xcnt optimizations for gfx1250.
1293 // Wait on XCNT is redundant if we are already waiting for a load to complete.
1294 // SMEM can return out of order, so only omit XCNT wait if we are waiting till
1295 // zero.
1296 if (CheckWait.get(T: AMDGPU::KM_CNT) == 0 &&
1297 hasPendingEvent(E: HWEvents::SMEM_GROUP))
1298 UpdateWait.set(T: AMDGPU::X_CNT, Val: ~0u);
1299 // If we have pending store we cannot optimize XCnt because we do not wait for
1300 // stores. VMEM loads retun in order, so if we only have loads XCnt is
1301 // decremented to the same number as LOADCnt.
1302 if (CheckWait.get(T: AMDGPU::LOAD_CNT) != ~0u &&
1303 hasPendingEvent(E: HWEvents::VMEM_GROUP) &&
1304 !hasPendingEvent(T: AMDGPU::STORE_CNT) &&
1305 CheckWait.get(T: AMDGPU::X_CNT) >= CheckWait.get(T: AMDGPU::LOAD_CNT))
1306 UpdateWait.set(T: AMDGPU::X_CNT, Val: ~0u);
1307 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::X_CNT);
1308}
1309
1310void WaitcntBrackets::simplifyVmVsrc(const AMDGPU::Waitcnt &CheckWait,
1311 AMDGPU::Waitcnt &UpdateWait) const {
1312 // Waiting for some counters implies waiting for VM_VSRC, since an
1313 // instruction that decrements a counter on completion would have
1314 // decremented VM_VSRC once its VGPR operands had been read.
1315 if (CheckWait.get(T: AMDGPU::VM_VSRC) >=
1316 std::min(l: {CheckWait.get(T: AMDGPU::LOAD_CNT),
1317 CheckWait.get(T: AMDGPU::STORE_CNT),
1318 CheckWait.get(T: AMDGPU::SAMPLE_CNT),
1319 CheckWait.get(T: AMDGPU::BVH_CNT), CheckWait.get(T: AMDGPU::DS_CNT)}))
1320 UpdateWait.set(T: AMDGPU::VM_VSRC, Val: ~0u);
1321 simplifyWaitcnt(Wait&: UpdateWait, T: AMDGPU::VM_VSRC);
1322}
1323
1324void WaitcntBrackets::purgeEmptyTrackingData() {
1325 VMem.remove_if(Pred: [](const auto &P) { return P.second.empty(); });
1326 SGPRs.remove_if(Pred: [](const auto &P) { return P.second.empty(); });
1327}
1328
1329void WaitcntBrackets::determineWaitForScore(AMDGPU::InstCounterType T,
1330 unsigned ScoreToWait,
1331 AMDGPU::Waitcnt &Wait) const {
1332 const unsigned LB = getScoreLB(T);
1333 const unsigned UB = getScoreUB(T);
1334
1335 // If the score falls within the bracket, we need a waitcnt.
1336 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1337 if ((T == AMDGPU::LOAD_CNT || T == AMDGPU::DS_CNT) && hasPendingFlat() &&
1338 !Context->ST.hasFlatLgkmVMemCountInOrder()) {
1339 // If there is a pending FLAT operation, and this is a VMem or LGKM
1340 // waitcnt and the target can report early completion, then we need
1341 // to force a waitcnt 0.
1342 Wait.add(T, Count: 0);
1343 } else if (counterOutOfOrder(T)) {
1344 // Counter can get decremented out-of-order when there
1345 // are multiple types event in the bracket. Also emit an s_wait counter
1346 // with a conservative value of 0 for the counter.
1347 Wait.add(T, Count: 0);
1348 } else {
1349 // If a counter has been maxed out avoid overflow by waiting for
1350 // MAX(CounterType) - 1 instead.
1351 unsigned NeededWait = std::min(a: UB - ScoreToWait, b: getLimit(T) - 1);
1352 Wait.add(T, Count: NeededWait);
1353 }
1354 }
1355}
1356
1357AMDGPU::Waitcnt WaitcntBrackets::determineAsyncWait(unsigned N) {
1358 LLVM_DEBUG({
1359 dbgs() << "Need " << N << " async marks. Found " << AsyncMarks.size()
1360 << ":\n";
1361 for (const auto &Mark : AsyncMarks) {
1362 llvm::interleaveComma(Mark, dbgs());
1363 dbgs() << '\n';
1364 }
1365 });
1366
1367 if (AsyncMarks.size() == MaxAsyncMarks) {
1368 // Enforcing MaxAsyncMarks here is unnecessary work because the size of
1369 // MaxAsyncMarks is linear when traversing straightline code. But we do
1370 // need to check if truncation may have occured at a merge, and adjust N
1371 // to ensure that a wait is generated.
1372 LLVM_DEBUG(dbgs() << "Possible truncation. Ensuring a non-trivial wait.\n");
1373 N = std::min(a: N, b: (unsigned)MaxAsyncMarks - 1);
1374 }
1375
1376 AMDGPU::Waitcnt Wait;
1377 if (AsyncMarks.size() <= N) {
1378 LLVM_DEBUG(dbgs() << "No additional wait for async mark.\n");
1379 return Wait;
1380 }
1381
1382 size_t MarkIndex = AsyncMarks.size() - N - 1;
1383 const auto &RequiredMark = AsyncMarks[MarkIndex];
1384 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types())
1385 determineWaitForScore(T, ScoreToWait: RequiredMark[T], Wait);
1386
1387 // Immediately remove the waited mark and all older ones
1388 // This happens BEFORE the wait is actually inserted, which is fine
1389 // because we've already extracted the wait requirements
1390 LLVM_DEBUG({
1391 dbgs() << "Removing " << (MarkIndex + 1)
1392 << " async marks after determining wait\n";
1393 });
1394 AsyncMarks.erase(CS: AsyncMarks.begin(), CE: AsyncMarks.begin() + MarkIndex + 1);
1395
1396 LLVM_DEBUG(dbgs() << "Waits to add: " << Wait);
1397 return Wait;
1398}
1399
1400// With D16Write32BitVgpr, D16 inst might be clobbered by events running on the
1401// other half 16bit.
1402//
1403// Replace VGPR16 to VGPR32 for wait check if:
1404// 1. MI is a VALU, and there is a wait event on the other half
1405// 2. MI is a LdSt, and there is a wait event on the other half from different
1406// order group
1407MCPhysReg WaitcntBrackets::determineVGPR16Dependency(const MachineInstr &MI,
1408 AMDGPU::InstCounterType T,
1409 MCPhysReg Reg) const {
1410 const TargetRegisterClass *RC = Context->TRI.getPhysRegBaseClass(Reg);
1411 unsigned Size = Context->TRI.getRegSizeInBits(RC: *RC);
1412
1413 if (Size != 16 || !Context->ST.hasD16Writes32BitVgpr())
1414 return Reg;
1415
1416 // With D16Writes32BitVgpr, D16 Inst might clobber the whole vgpr32
1417 // check dependency on the other half
1418 Register Reg32 = Context->TRI.get32BitRegister(Reg);
1419 Register OtherHalf = Context->TRI.getSubReg(
1420 Reg: Reg32,
1421 Idx: AMDGPU::isHi16Reg(Reg, MRI: Context->TRI) ? AMDGPU::lo16 : AMDGPU::hi16);
1422
1423 AMDGPU::Waitcnt Wait;
1424 for (MCRegUnit RU : regunits(Reg: OtherHalf))
1425 determineWaitForScore(T, ScoreToWait: getVMemScore(TID: toVMEMID(RU), T), Wait);
1426
1427 // No wait on otherhalf
1428 if (!Wait.hasWait())
1429 return Reg;
1430
1431 if (Context->TII.isVALU(MI, /*AllowLDSDMA=*/true))
1432 return Reg32;
1433
1434 // If hi/lo16 mixed events
1435 HWEvents MIEvents = AMDGPU::getEventsFor(
1436 Inst: MI, ST: Context->ST, IsExpertMode: Context->IsExpertMode, TgSplit: Context->TgSplit);
1437 HWEvents OtherHalfEvents = Context->getWaitEvents(T);
1438 HWEvents Events = MIEvents & OtherHalfEvents;
1439 if (Events.size() > 1)
1440 return Reg32;
1441 return Reg;
1442}
1443
1444void WaitcntBrackets::determineWaitForPhysReg(AMDGPU::InstCounterType T,
1445 MCPhysReg Reg,
1446 AMDGPU::Waitcnt &Wait,
1447 const MachineInstr &MI) const {
1448 if (Reg == AMDGPU::SCC) {
1449 determineWaitForScore(T, ScoreToWait: SCCScore, Wait);
1450 } else {
1451 bool IsVGPR = Context->TRI.isVectorRegister(MRI: Context->MRI, Reg);
1452 if (IsVGPR)
1453 Reg = determineVGPR16Dependency(MI, T, Reg);
1454 for (MCRegUnit RU : regunits(Reg))
1455 determineWaitForScore(
1456 T, ScoreToWait: IsVGPR ? getVMemScore(TID: toVMEMID(RU), T) : getSGPRScore(RU, T),
1457 Wait);
1458 }
1459}
1460
1461void WaitcntBrackets::determineWaitForLDSDMA(AMDGPU::InstCounterType T,
1462 VMEMID TID,
1463 AMDGPU::Waitcnt &Wait) const {
1464 assert(TID >= LDSDMA_BEGIN && TID < LDSDMA_END);
1465 determineWaitForScore(T, ScoreToWait: getVMemScore(TID, T), Wait);
1466}
1467
1468void WaitcntBrackets::tryClearSCCWriteEvent(MachineInstr *Inst) {
1469 // S_BARRIER_WAIT on the same barrier guarantees that the pending write to
1470 // SCC has landed
1471 if (PendingSCCWrite &&
1472 PendingSCCWrite->getOpcode() == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM &&
1473 PendingSCCWrite->getOperand(i: 0).getImm() == Inst->getOperand(i: 0).getImm()) {
1474 HWEvents SCC_WRITE_PendingEvent = HWEvents::SCC_WRITE;
1475 // If this SCC_WRITE is the only pending KM_CNT event, clear counter.
1476 if ((PendingEvents & Context->getWaitEvents(T: AMDGPU::KM_CNT)) ==
1477 SCC_WRITE_PendingEvent) {
1478 setScoreLB(T: AMDGPU::KM_CNT, Val: getScoreUB(T: AMDGPU::KM_CNT));
1479 }
1480
1481 PendingEvents -= SCC_WRITE_PendingEvent;
1482 PendingSCCWrite = nullptr;
1483 }
1484}
1485
1486void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) {
1487 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types())
1488 applyWaitcnt(Wait, T);
1489}
1490
1491void WaitcntBrackets::applyWaitcnt(AMDGPU::InstCounterType T, unsigned Count) {
1492 const unsigned UB = getScoreUB(T);
1493 if (Count >= UB)
1494 return;
1495 if (Count != 0) {
1496 if (counterOutOfOrder(T))
1497 return;
1498 setScoreLB(T, Val: std::max(a: getScoreLB(T), b: UB - Count));
1499 } else {
1500 setScoreLB(T, Val: UB);
1501 PendingEvents -= Context->getWaitEvents(T);
1502 }
1503
1504 if (T == AMDGPU::KM_CNT && Count == 0 &&
1505 hasPendingEvent(E: HWEvents::SMEM_GROUP)) {
1506 if (!hasMixedPendingEvents(T: AMDGPU::X_CNT))
1507 applyWaitcnt(T: AMDGPU::X_CNT, Count: 0);
1508 else
1509 PendingEvents -= HWEvents::SMEM_GROUP;
1510 }
1511 if (T == AMDGPU::LOAD_CNT && hasPendingEvent(E: HWEvents::VMEM_GROUP) &&
1512 !hasPendingEvent(T: AMDGPU::STORE_CNT)) {
1513 if (!hasMixedPendingEvents(T: AMDGPU::X_CNT))
1514 applyWaitcnt(T: AMDGPU::X_CNT, Count);
1515 else if (Count == 0)
1516 PendingEvents -= HWEvents::VMEM_GROUP;
1517 }
1518}
1519
1520void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait,
1521 AMDGPU::InstCounterType T) {
1522 unsigned Cnt = Wait.get(T);
1523 applyWaitcnt(T, Count: Cnt);
1524}
1525
1526// Where there are multiple types of event in the bracket of a counter,
1527// the decrement may go out of order.
1528bool WaitcntBrackets::counterOutOfOrder(AMDGPU::InstCounterType T) const {
1529 // Scalar memory read always can go out of order.
1530 if ((T == Context->SmemAccessCounter &&
1531 hasPendingEvent(E: HWEvents::SMEM_ACCESS)) ||
1532 (T == AMDGPU::X_CNT && hasPendingEvent(E: HWEvents::SMEM_GROUP)))
1533 return true;
1534
1535 if (T == AMDGPU::LOAD_CNT) {
1536
1537 // On targets without VScnt, LOAD_CNT includes all of STORE_CNT as well.
1538 // All these events use one counter and do not go out of order with respect
1539 // to each other.
1540 if (!Context->ST.hasVscnt())
1541 return false;
1542
1543 HWEvents Events = PendingEvents & Context->getWaitEvents(T);
1544
1545 // If the target does not have extended counters, VMEM_BVH/SAMPLE_READ
1546 // events are equivalent to VMEM_READ_ACCESS. We do not go out of order in
1547 // such cases.
1548 static constexpr HWEvents ExtendedImageEvents =
1549 HWEvents::VMEM_SAMPLER_READ_ACCESS | HWEvents::VMEM_BVH_READ_ACCESS;
1550 if (!Context->ST.hasExtendedWaitCounts() &&
1551 (Events & ExtendedImageEvents).any()) {
1552 Events -= ExtendedImageEvents;
1553 Events |= HWEvents::VMEM_READ_ACCESS;
1554 }
1555
1556 // GLOBAL_INV completes in-order with other LOAD_CNT events,
1557 // so having GLOBAL_INV_ACCESS mixed with other LOAD_CNT
1558 // events doesn't cause out-of-order completion.
1559 Events -= HWEvents::GLOBAL_INV_ACCESS;
1560
1561 // Return true only if there are still multiple event types after removing
1562 // GLOBAL_INV
1563 return Events.size() > 1;
1564 }
1565
1566 return hasMixedPendingEvents(T);
1567}
1568
1569INITIALIZE_PASS_BEGIN(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1570 false, false)
1571INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
1572INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass)
1573INITIALIZE_PASS_END(SIInsertWaitcntsLegacy, DEBUG_TYPE, "SI Insert Waitcnts",
1574 false, false)
1575
1576char SIInsertWaitcntsLegacy::ID = 0;
1577
1578char &llvm::SIInsertWaitcntsID = SIInsertWaitcntsLegacy::ID;
1579
1580FunctionPass *llvm::createSIInsertWaitcntsPass() {
1581 return new SIInsertWaitcntsLegacy();
1582}
1583
1584static bool updateOperandIfDifferent(MachineInstr &MI, AMDGPU::OpName OpName,
1585 unsigned NewEnc) {
1586 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpName);
1587 assert(OpIdx >= 0);
1588
1589 MachineOperand &MO = MI.getOperand(i: OpIdx);
1590
1591 if (NewEnc == MO.getImm())
1592 return false;
1593
1594 MO.setImm(NewEnc);
1595 return true;
1596}
1597
1598bool WaitcntGenerator::promoteSoftWaitCnt(MachineInstr *Waitcnt) const {
1599 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: Waitcnt->getOpcode());
1600 if (Opcode == Waitcnt->getOpcode())
1601 return false;
1602
1603 Waitcnt->setDesc(TII.get(Opcode));
1604 return true;
1605}
1606
1607/// Combine consecutive S_WAITCNT and S_WAITCNT_VSCNT instructions that
1608/// precede \p It and follow \p OldWaitcntInstr and apply any extra waits
1609/// from \p Wait that were added by previous passes. Currently this pass
1610/// conservatively assumes that these preexisting waits are required for
1611/// correctness.
1612bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1613 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1614 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) const {
1615 assert(isNormalMode(MaxCounter));
1616
1617 bool Modified = false;
1618 MachineInstr *WaitcntInstr = nullptr;
1619 MachineInstr *WaitcntVsCntInstr = nullptr;
1620
1621 LLVM_DEBUG({
1622 dbgs() << "PreGFX12::applyPreexistingWaitcnt at: ";
1623 if (It.isEnd())
1624 dbgs() << "end of block\n";
1625 else
1626 dbgs() << *It;
1627 });
1628
1629 for (auto &II :
1630 make_early_inc_range(Range: make_range(x: OldWaitcntInstr.getIterator(), y: It))) {
1631 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
1632 if (isNonWaitcntMetaInst(MI: II)) {
1633 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
1634 continue;
1635 }
1636
1637 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: II.getOpcode());
1638 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1639
1640 // Update required wait count. If this is a soft waitcnt (= it was added
1641 // by an earlier pass), it may be entirely removed.
1642 if (Opcode == AMDGPU::S_WAITCNT) {
1643 unsigned IEnc = II.getOperand(i: 0).getImm();
1644 AMDGPU::Waitcnt OldWait = AMDGPU::decodeWaitcnt(Version: IV, Encoded: IEnc);
1645 if (TrySimplify)
1646 ScoreBrackets.simplifyWaitcnt(Wait&: OldWait);
1647 Wait = Wait.combined(Other: OldWait);
1648
1649 // Merge consecutive waitcnt of the same type by erasing multiples.
1650 if (WaitcntInstr || (!Wait.hasWaitExceptStoreCnt() && TrySimplify)) {
1651 II.eraseFromParent();
1652 Modified = true;
1653 } else
1654 WaitcntInstr = &II;
1655 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
1656 assert(ST.hasVMemToLDSLoad());
1657 LLVM_DEBUG(dbgs() << "Processing S_WAITCNT_lds_direct: " << II
1658 << "Before: " << Wait << '\n';);
1659 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID: LDSDMA_BEGIN,
1660 Wait);
1661 LLVM_DEBUG(dbgs() << "After: " << Wait << '\n';);
1662
1663 // It is possible (but unlikely) that this is the only wait instruction,
1664 // in which case, we exit this loop without a WaitcntInstr to consume
1665 // `Wait`. But that works because `Wait` was passed in by reference, and
1666 // the callee eventually calls createNewWaitcnt on it. We test this
1667 // possibility in an articial MIR test since such a situation cannot be
1668 // recreated by running the memory legalizer.
1669 II.eraseFromParent();
1670 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
1671 unsigned N = II.getOperand(i: 0).getImm();
1672 LLVM_DEBUG(dbgs() << "Processing WAIT_ASYNCMARK: " << II << '\n';);
1673 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
1674 Wait = Wait.combined(Other: OldWait);
1675 } else {
1676 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1677 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1678
1679 unsigned OldVSCnt =
1680 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1681 if (TrySimplify)
1682 ScoreBrackets.simplifyWaitcnt(T: AMDGPU::STORE_CNT, Count&: OldVSCnt);
1683 Wait.set(T: AMDGPU::STORE_CNT,
1684 Val: std::min(a: Wait.get(T: AMDGPU::STORE_CNT), b: OldVSCnt));
1685
1686 if (WaitcntVsCntInstr || (!Wait.hasWaitStoreCnt() && TrySimplify)) {
1687 II.eraseFromParent();
1688 Modified = true;
1689 } else
1690 WaitcntVsCntInstr = &II;
1691 }
1692 }
1693
1694 if (WaitcntInstr) {
1695 Modified |= updateOperandIfDifferent(MI&: *WaitcntInstr, OpName: AMDGPU::OpName::simm16,
1696 NewEnc: AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait));
1697 Modified |= promoteSoftWaitCnt(Waitcnt: WaitcntInstr);
1698
1699 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::LOAD_CNT);
1700 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::EXP_CNT);
1701 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
1702 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
1703 Wait.set(T: AMDGPU::EXP_CNT, Val: ~0u);
1704 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
1705
1706 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
1707 << "New Instr at block end: "
1708 << *WaitcntInstr << '\n'
1709 : dbgs() << "applied pre-existing waitcnt\n"
1710 << "Old Instr: " << *It
1711 << "New Instr: " << *WaitcntInstr << '\n');
1712 }
1713
1714 if (WaitcntVsCntInstr) {
1715 Modified |=
1716 updateOperandIfDifferent(MI&: *WaitcntVsCntInstr, OpName: AMDGPU::OpName::simm16,
1717 NewEnc: Wait.get(T: AMDGPU::STORE_CNT));
1718 Modified |= promoteSoftWaitCnt(Waitcnt: WaitcntVsCntInstr);
1719
1720 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::STORE_CNT);
1721 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
1722
1723 LLVM_DEBUG(It.isEnd()
1724 ? dbgs() << "applied pre-existing waitcnt\n"
1725 << "New Instr at block end: " << *WaitcntVsCntInstr
1726 << '\n'
1727 : dbgs() << "applied pre-existing waitcnt\n"
1728 << "Old Instr: " << *It
1729 << "New Instr: " << *WaitcntVsCntInstr << '\n');
1730 }
1731
1732 return Modified;
1733}
1734
1735/// Generate S_WAITCNT and/or S_WAITCNT_VSCNT instructions for any
1736/// required counters in \p Wait
1737bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1738 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
1739 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
1740 assert(isNormalMode(MaxCounter));
1741
1742 bool Modified = false;
1743 const DebugLoc &DL = Block.findDebugLoc(MBBI: It);
1744
1745 // Waits for VMcnt, LKGMcnt and/or EXPcnt are encoded together into a
1746 // single instruction while VScnt has its own instruction.
1747 if (Wait.hasWaitExceptStoreCnt()) {
1748 // If profiling expansion is enabled, emit an expanded sequence
1749 if (ExpandWaitcntProfiling) {
1750 // Check if any of the counters to be waited on are out-of-order.
1751 // If so, fall back to normal (non-expanded) behavior since expansion
1752 // would provide misleading profiling information.
1753 bool AnyOutOfOrder = false;
1754 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1755 unsigned WaitCnt = Wait.get(T: CT);
1756 if (WaitCnt != ~0u && ScoreBrackets.counterOutOfOrder(T: CT)) {
1757 AnyOutOfOrder = true;
1758 break;
1759 }
1760 }
1761
1762 if (AnyOutOfOrder) {
1763 // Fall back to non-expanded wait
1764 unsigned Enc = AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait);
1765 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: Enc);
1766 Modified = true;
1767 } else {
1768 // All counters are in-order, safe to expand
1769 for (auto CT : {AMDGPU::LOAD_CNT, AMDGPU::DS_CNT, AMDGPU::EXP_CNT}) {
1770 unsigned WaitCnt = Wait.get(T: CT);
1771 if (WaitCnt == ~0u)
1772 continue;
1773
1774 unsigned Outstanding =
1775 std::min(a: ScoreBrackets.getOutstanding(T: CT), b: getLimit(E: CT) - 1);
1776 EmitExpandedWaitcnt(Outstanding, Target: WaitCnt, EmitWaitcnt: [&](unsigned Count) {
1777 AMDGPU::Waitcnt W;
1778 W.set(T: CT, Val: Count);
1779 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT))
1780 .addImm(Val: AMDGPU::encodeWaitcnt(Version: IV, Decoded: W));
1781 });
1782 Modified = true;
1783 }
1784 }
1785 } else {
1786 // Normal behavior: emit single combined waitcnt
1787 unsigned Enc = AMDGPU::encodeWaitcnt(Version: IV, Decoded: Wait);
1788 [[maybe_unused]] auto SWaitInst =
1789 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: Enc);
1790 Modified = true;
1791
1792 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1793 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1794 dbgs() << "New Instr: " << *SWaitInst << '\n');
1795 }
1796 }
1797
1798 if (Wait.hasWaitStoreCnt()) {
1799 assert(ST.hasVscnt());
1800
1801 if (ExpandWaitcntProfiling && Wait.get(T: AMDGPU::STORE_CNT) != ~0u &&
1802 !ScoreBrackets.counterOutOfOrder(T: AMDGPU::STORE_CNT)) {
1803 // Only expand if counter is not out-of-order
1804 unsigned Outstanding =
1805 std::min(a: ScoreBrackets.getOutstanding(T: AMDGPU::STORE_CNT),
1806 b: getLimit(E: AMDGPU::STORE_CNT) - 1);
1807 EmitExpandedWaitcnt(
1808 Outstanding, Target: Wait.get(T: AMDGPU::STORE_CNT), EmitWaitcnt: [&](unsigned Count) {
1809 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
1810 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
1811 .addImm(Val: Count);
1812 });
1813 Modified = true;
1814 } else {
1815 [[maybe_unused]] auto SWaitInst =
1816 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_VSCNT))
1817 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
1818 .addImm(Val: Wait.get(T: AMDGPU::STORE_CNT));
1819 Modified = true;
1820
1821 LLVM_DEBUG(dbgs() << "PreGFX12::createNewWaitcnt\n";
1822 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1823 dbgs() << "New Instr: " << *SWaitInst << '\n');
1824 }
1825 }
1826
1827 return Modified;
1828}
1829
1830AMDGPU::Waitcnt
1831WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1832 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt && ST.hasVscnt() ? 0 : ~0u);
1833}
1834
1835AMDGPU::Waitcnt
1836WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1837 unsigned ExpertVal = IsExpertMode ? 0 : ~0u;
1838 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0,
1839 ~0u /* XCNT */, ~0u /* ASYNC_CNT */,
1840 ~0u /* TENSOR_CNT */, ExpertVal, ExpertVal, ExpertVal);
1841}
1842
1843/// Combine consecutive S_WAIT_*CNT instructions that precede \p It and
1844/// follow \p OldWaitcntInstr and apply any extra waits from \p Wait that
1845/// were added by previous passes. Currently this pass conservatively
1846/// assumes that these preexisting waits are required for correctness.
1847bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1848 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1849 AMDGPU::Waitcnt &Wait, MachineBasicBlock::instr_iterator It) const {
1850 assert(!isNormalMode(MaxCounter));
1851
1852 bool Modified = false;
1853 MachineInstr *CombinedLoadDsCntInstr = nullptr;
1854 MachineInstr *CombinedStoreDsCntInstr = nullptr;
1855 MachineInstr *WaitcntDepctrInstr = nullptr;
1856 MachineInstr *WaitInstrs[AMDGPU::NUM_EXTENDED_INST_CNTS] = {};
1857
1858 LLVM_DEBUG({
1859 dbgs() << "GFX12Plus::applyPreexistingWaitcnt at: ";
1860 if (It.isEnd())
1861 dbgs() << "end of block\n";
1862 else
1863 dbgs() << *It;
1864 });
1865
1866 // Accumulate waits that should not be simplified.
1867 AMDGPU::Waitcnt RequiredWait;
1868
1869 for (auto &II :
1870 make_early_inc_range(Range: make_range(x: OldWaitcntInstr.getIterator(), y: It))) {
1871 LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
1872 if (isNonWaitcntMetaInst(MI: II)) {
1873 LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
1874 continue;
1875 }
1876
1877 // Update required wait count. If this is a soft waitcnt (= it was added
1878 // by an earlier pass), it may be entirely removed.
1879
1880 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: II.getOpcode());
1881 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1882
1883 // Don't crash if the programmer used legacy waitcnt intrinsics, but don't
1884 // attempt to do more than that either.
1885 if (Opcode == AMDGPU::S_WAITCNT)
1886 continue;
1887
1888 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
1889 unsigned OldEnc =
1890 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1891 AMDGPU::Waitcnt OldWait = AMDGPU::decodeLoadcntDscnt(Version: IV, LoadcntDscnt: OldEnc);
1892 if (TrySimplify)
1893 Wait = Wait.combined(Other: OldWait);
1894 else
1895 RequiredWait = RequiredWait.combined(Other: OldWait);
1896 // Keep the first wait_loadcnt, erase the rest.
1897 if (CombinedLoadDsCntInstr == nullptr) {
1898 CombinedLoadDsCntInstr = &II;
1899 } else {
1900 II.eraseFromParent();
1901 Modified = true;
1902 }
1903 } else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
1904 unsigned OldEnc =
1905 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1906 AMDGPU::Waitcnt OldWait = AMDGPU::decodeStorecntDscnt(Version: IV, StorecntDscnt: OldEnc);
1907 if (TrySimplify)
1908 Wait = Wait.combined(Other: OldWait);
1909 else
1910 RequiredWait = RequiredWait.combined(Other: OldWait);
1911 // Keep the first wait_storecnt, erase the rest.
1912 if (CombinedStoreDsCntInstr == nullptr) {
1913 CombinedStoreDsCntInstr = &II;
1914 } else {
1915 II.eraseFromParent();
1916 Modified = true;
1917 }
1918 } else if (Opcode == AMDGPU::S_WAITCNT_DEPCTR) {
1919 unsigned OldEnc =
1920 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1921 AMDGPU::Waitcnt OldWait;
1922 // Set both counters to the decoded value from the single hardware field
1923 unsigned VaVdst = AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: OldEnc);
1924 OldWait.set(T: AMDGPU::VA_VDST_RD, Val: VaVdst);
1925 OldWait.set(T: AMDGPU::VA_VDST_WR, Val: VaVdst);
1926 OldWait.set(T: AMDGPU::VM_VSRC, Val: AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: OldEnc));
1927 if (TrySimplify)
1928 ScoreBrackets.simplifyWaitcnt(Wait&: OldWait);
1929 Wait = Wait.combined(Other: OldWait);
1930 if (WaitcntDepctrInstr == nullptr) {
1931 WaitcntDepctrInstr = &II;
1932 } else {
1933 // S_WAITCNT_DEPCTR requires special care. Don't remove a
1934 // duplicate if it is waiting on things other than VA_VDST or
1935 // VM_VSRC. If that is the case, just make sure the VA_VDST and
1936 // VM_VSRC subfields of the operand are set to the "no wait"
1937 // values.
1938
1939 unsigned Enc =
1940 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1941 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: ~0u);
1942 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
1943 unsigned VaVdst = std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD),
1944 b: Wait.get(T: AMDGPU::VA_VDST_WR));
1945 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
1946
1947 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: ST)) {
1948 Modified |= updateOperandIfDifferent(MI&: II, OpName: AMDGPU::OpName::simm16, NewEnc: Enc);
1949 Modified |= promoteSoftWaitCnt(Waitcnt: &II);
1950 } else {
1951 II.eraseFromParent();
1952 Modified = true;
1953 }
1954 }
1955 } else if (Opcode == AMDGPU::S_WAITCNT_lds_direct) {
1956 // Architectures higher than GFX10 do not have direct loads to
1957 // LDS, so no work required here yet.
1958 II.eraseFromParent();
1959 Modified = true;
1960 } else if (Opcode == AMDGPU::WAIT_ASYNCMARK) {
1961 // Update the Waitcnt, but don't erase the wait.asyncmark() itself. It
1962 // shows up in the assembly as a comment with the original parameter N.
1963 unsigned N = II.getOperand(i: 0).getImm();
1964 AMDGPU::Waitcnt OldWait = ScoreBrackets.determineAsyncWait(N);
1965 Wait = Wait.combined(Other: OldWait);
1966 } else {
1967 std::optional<AMDGPU::InstCounterType> CT =
1968 AMDGPU::counterTypeForInstr(Opcode);
1969 assert(CT.has_value());
1970 unsigned OldCnt =
1971 TII.getNamedOperand(MI&: II, OperandName: AMDGPU::OpName::simm16)->getImm();
1972 if (TrySimplify)
1973 Wait.add(T: CT.value(), Count: OldCnt);
1974 else
1975 RequiredWait.add(T: CT.value(), Count: OldCnt);
1976 // Keep the first wait of its kind, erase the rest.
1977 if (WaitInstrs[CT.value()] == nullptr) {
1978 WaitInstrs[CT.value()] = &II;
1979 } else {
1980 II.eraseFromParent();
1981 Modified = true;
1982 }
1983 }
1984 }
1985
1986 ScoreBrackets.simplifyWaitcnt(CheckWait: Wait.combined(Other: RequiredWait), UpdateWait&: Wait);
1987 Wait = Wait.combined(Other: RequiredWait);
1988
1989 if (CombinedLoadDsCntInstr) {
1990 // Only keep an S_WAIT_LOADCNT_DSCNT if both counters actually need
1991 // to be waited for. Otherwise, let the instruction be deleted so
1992 // the appropriate single counter wait instruction can be inserted
1993 // instead, when new S_WAIT_*CNT instructions are inserted by
1994 // createNewWaitcnt(). As a side effect, resetting the wait counts will
1995 // cause any redundant S_WAIT_LOADCNT or S_WAIT_DSCNT to be removed by
1996 // the loop below that deals with single counter instructions.
1997 //
1998 // A wait for LOAD_CNT or DS_CNT implies a wait for VM_VSRC, since
1999 // instructions that have decremented LOAD_CNT or DS_CNT on completion
2000 // will have needed to wait for their register sources to be available
2001 // first.
2002 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u && Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2003 unsigned NewEnc = AMDGPU::encodeLoadcntDscnt(Version: IV, Decoded: Wait);
2004 Modified |= updateOperandIfDifferent(MI&: *CombinedLoadDsCntInstr,
2005 OpName: AMDGPU::OpName::simm16, NewEnc);
2006 Modified |= promoteSoftWaitCnt(Waitcnt: CombinedLoadDsCntInstr);
2007 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::LOAD_CNT);
2008 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
2009 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2010 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2011
2012 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2013 << "New Instr at block end: "
2014 << *CombinedLoadDsCntInstr << '\n'
2015 : dbgs() << "applied pre-existing waitcnt\n"
2016 << "Old Instr: " << *It << "New Instr: "
2017 << *CombinedLoadDsCntInstr << '\n');
2018 } else {
2019 CombinedLoadDsCntInstr->eraseFromParent();
2020 Modified = true;
2021 }
2022 }
2023
2024 if (CombinedStoreDsCntInstr) {
2025 // Similarly for S_WAIT_STORECNT_DSCNT.
2026 if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u && Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2027 unsigned NewEnc = AMDGPU::encodeStorecntDscnt(Version: IV, Decoded: Wait);
2028 Modified |= updateOperandIfDifferent(MI&: *CombinedStoreDsCntInstr,
2029 OpName: AMDGPU::OpName::simm16, NewEnc);
2030 Modified |= promoteSoftWaitCnt(Waitcnt: CombinedStoreDsCntInstr);
2031 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::STORE_CNT);
2032 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::DS_CNT);
2033 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
2034 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2035
2036 LLVM_DEBUG(It.isEnd() ? dbgs() << "applied pre-existing waitcnt\n"
2037 << "New Instr at block end: "
2038 << *CombinedStoreDsCntInstr << '\n'
2039 : dbgs() << "applied pre-existing waitcnt\n"
2040 << "Old Instr: " << *It << "New Instr: "
2041 << *CombinedStoreDsCntInstr << '\n');
2042 } else {
2043 CombinedStoreDsCntInstr->eraseFromParent();
2044 Modified = true;
2045 }
2046 }
2047
2048 // Look for an opportunity to convert existing S_WAIT_LOADCNT,
2049 // S_WAIT_STORECNT and S_WAIT_DSCNT into new S_WAIT_LOADCNT_DSCNT
2050 // or S_WAIT_STORECNT_DSCNT. This is achieved by selectively removing
2051 // instructions so that createNewWaitcnt() will create new combined
2052 // instructions to replace them.
2053
2054 if (Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2055 // This is a vector of addresses in WaitInstrs pointing to instructions
2056 // that should be removed if they are present.
2057 SmallVector<MachineInstr **, 2> WaitsToErase;
2058
2059 // If it's known that both DScnt and either LOADcnt or STOREcnt (but not
2060 // both) need to be waited for, ensure that there are no existing
2061 // individual wait count instructions for these.
2062
2063 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u) {
2064 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::LOAD_CNT]);
2065 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::DS_CNT]);
2066 } else if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u) {
2067 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::STORE_CNT]);
2068 WaitsToErase.push_back(Elt: &WaitInstrs[AMDGPU::DS_CNT]);
2069 }
2070
2071 for (MachineInstr **WI : WaitsToErase) {
2072 if (!*WI)
2073 continue;
2074
2075 (*WI)->eraseFromParent();
2076 *WI = nullptr;
2077 Modified = true;
2078 }
2079 }
2080
2081 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2082 if (!WaitInstrs[CT])
2083 continue;
2084
2085 unsigned NewCnt = Wait.get(T: CT);
2086 if (NewCnt != ~0u) {
2087 Modified |= updateOperandIfDifferent(MI&: *WaitInstrs[CT],
2088 OpName: AMDGPU::OpName::simm16, NewEnc: NewCnt);
2089 Modified |= promoteSoftWaitCnt(Waitcnt: WaitInstrs[CT]);
2090
2091 ScoreBrackets.applyWaitcnt(T: CT, Count: NewCnt);
2092 Wait.clear(T: CT);
2093
2094 LLVM_DEBUG(It.isEnd()
2095 ? dbgs() << "applied pre-existing waitcnt\n"
2096 << "New Instr at block end: " << *WaitInstrs[CT]
2097 << '\n'
2098 : dbgs() << "applied pre-existing waitcnt\n"
2099 << "Old Instr: " << *It
2100 << "New Instr: " << *WaitInstrs[CT] << '\n');
2101 } else {
2102 WaitInstrs[CT]->eraseFromParent();
2103 Modified = true;
2104 }
2105 }
2106
2107 if (WaitcntDepctrInstr) {
2108 // Get the encoded Depctr immediate and override the VA_VDST and VM_VSRC
2109 // subfields with the new required values.
2110 unsigned Enc =
2111 TII.getNamedOperand(MI&: *WaitcntDepctrInstr, OperandName: AMDGPU::OpName::simm16)
2112 ->getImm();
2113 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: Wait.get(T: AMDGPU::VM_VSRC));
2114 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2115 unsigned VaVdst =
2116 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
2117 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2118
2119 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::VA_VDST_RD);
2120 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::VA_VDST_WR);
2121 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::VM_VSRC);
2122 Wait.set(T: AMDGPU::VA_VDST_RD, Val: ~0u);
2123 Wait.set(T: AMDGPU::VA_VDST_WR, Val: ~0u);
2124 Wait.set(T: AMDGPU::VM_VSRC, Val: ~0u);
2125
2126 // If that new encoded Depctr immediate would actually still wait
2127 // for anything, update the instruction's operand. Otherwise it can
2128 // just be deleted.
2129 if (Enc != (unsigned)AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI: ST)) {
2130 Modified |= updateOperandIfDifferent(MI&: *WaitcntDepctrInstr,
2131 OpName: AMDGPU::OpName::simm16, NewEnc: Enc);
2132 LLVM_DEBUG(It.isEnd() ? dbgs() << "applyPreexistingWaitcnt\n"
2133 << "New Instr at block end: "
2134 << *WaitcntDepctrInstr << '\n'
2135 : dbgs() << "applyPreexistingWaitcnt\n"
2136 << "Old Instr: " << *It << "New Instr: "
2137 << *WaitcntDepctrInstr << '\n');
2138 } else {
2139 WaitcntDepctrInstr->eraseFromParent();
2140 Modified = true;
2141 }
2142 }
2143
2144 return Modified;
2145}
2146
2147/// Generate S_WAIT_*CNT instructions for any required counters in \p Wait
2148bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
2149 MachineBasicBlock &Block, MachineBasicBlock::instr_iterator It,
2150 AMDGPU::Waitcnt Wait, const WaitcntBrackets &ScoreBrackets) {
2151 assert(!isNormalMode(MaxCounter));
2152
2153 bool Modified = false;
2154 const DebugLoc &DL = Block.findDebugLoc(MBBI: It);
2155
2156 // For GFX12+, we use separate wait instructions, which makes expansion
2157 // simpler
2158 if (ExpandWaitcntProfiling) {
2159 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2160 unsigned Count = Wait.get(T: CT);
2161 if (Count == ~0u)
2162 continue;
2163
2164 // Skip expansion for out-of-order counters - emit normal wait instead
2165 if (ScoreBrackets.counterOutOfOrder(T: CT)) {
2166 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2167 .addImm(Val: Count);
2168 Modified = true;
2169 continue;
2170 }
2171
2172 unsigned Outstanding =
2173 std::min(a: ScoreBrackets.getOutstanding(T: CT), b: getLimit(E: CT) - 1);
2174 EmitExpandedWaitcnt(Outstanding, Target: Count, EmitWaitcnt: [&](unsigned Val) {
2175 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2176 .addImm(Val);
2177 });
2178 Modified = true;
2179 }
2180 return Modified;
2181 }
2182
2183 // Normal behavior (no expansion)
2184 // Check for opportunities to use combined wait instructions.
2185 if (Wait.get(T: AMDGPU::DS_CNT) != ~0u) {
2186 MachineInstr *SWaitInst = nullptr;
2187
2188 if (Wait.get(T: AMDGPU::LOAD_CNT) != ~0u) {
2189 unsigned Enc = AMDGPU::encodeLoadcntDscnt(Version: IV, Decoded: Wait);
2190
2191 SWaitInst = BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAIT_LOADCNT_DSCNT))
2192 .addImm(Val: Enc);
2193
2194 Wait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2195 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2196 } else if (Wait.get(T: AMDGPU::STORE_CNT) != ~0u) {
2197 unsigned Enc = AMDGPU::encodeStorecntDscnt(Version: IV, Decoded: Wait);
2198
2199 SWaitInst = BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAIT_STORECNT_DSCNT))
2200 .addImm(Val: Enc);
2201
2202 Wait.set(T: AMDGPU::STORE_CNT, Val: ~0u);
2203 Wait.set(T: AMDGPU::DS_CNT, Val: ~0u);
2204 }
2205
2206 if (SWaitInst) {
2207 Modified = true;
2208
2209 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2210 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2211 dbgs() << "New Instr: " << *SWaitInst << '\n');
2212 }
2213 }
2214
2215 // Generate an instruction for any remaining counter that needs
2216 // waiting for.
2217
2218 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
2219 unsigned Count = Wait.get(T: CT);
2220 if (Count == ~0u)
2221 continue;
2222
2223 [[maybe_unused]] auto SWaitInst =
2224 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
2225 .addImm(Val: Count);
2226
2227 Modified = true;
2228
2229 LLVM_DEBUG(dbgs() << "GFX12Plus::createNewWaitcnt\n";
2230 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2231 dbgs() << "New Instr: " << *SWaitInst << '\n');
2232 }
2233
2234 if (Wait.hasWaitDepctr()) {
2235 assert(IsExpertMode);
2236 unsigned Enc =
2237 AMDGPU::DepCtr::encodeFieldVmVsrc(VmVsrc: Wait.get(T: AMDGPU::VM_VSRC), STI: ST);
2238 // Encode min(VA_VDST_RD, VA_VDST_WR) into the single hardware field
2239 unsigned VaVdst =
2240 std::min(a: Wait.get(T: AMDGPU::VA_VDST_RD), b: Wait.get(T: AMDGPU::VA_VDST_WR));
2241 Enc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Enc, VaVdst);
2242
2243 [[maybe_unused]] auto SWaitInst =
2244 BuildMI(BB&: Block, I: It, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR)).addImm(Val: Enc);
2245
2246 Modified = true;
2247
2248 LLVM_DEBUG(dbgs() << "generateWaitcnt\n";
2249 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
2250 dbgs() << "New Instr: " << *SWaitInst << '\n');
2251 }
2252
2253 return Modified;
2254}
2255
2256/// Generate s_waitcnt instruction to be placed before cur_Inst.
2257/// Instructions of a given type are returned in order,
2258/// but instructions of different types can complete out of order.
2259/// We rely on this in-order completion
2260/// and simply assign a score to the memory access instructions.
2261/// We keep track of the active "score bracket" to determine
2262/// if an access of a memory read requires an s_waitcnt
2263/// and if so what the value of each counter is.
2264/// The "score bracket" is bound by the lower bound and upper bound
2265/// scores (*_score_LB and *_score_ub respectively).
2266/// If FlushFlags.FlushVmCnt is true, we want to flush the vmcnt counter here.
2267/// If FlushFlags.FlushDsCnt is true, we want to flush the dscnt counter here
2268/// (GFX12+ only, where DS_CNT is a separate counter).
2269bool SIInsertWaitcnts::generateWaitcntInstBefore(
2270 MachineInstr &MI, WaitcntBrackets &ScoreBrackets,
2271 MachineInstr *OldWaitcntInstr, PreheaderFlushFlags FlushFlags) {
2272 LLVM_DEBUG(dbgs() << "\n*** GenerateWaitcntInstBefore: "; MI.print(dbgs()););
2273
2274 assert(!isNonWaitcntMetaInst(MI));
2275
2276 AMDGPU::Waitcnt Wait;
2277 const unsigned Opc = MI.getOpcode();
2278
2279 switch (Opc) {
2280 case AMDGPU::BUFFER_WBINVL1:
2281 case AMDGPU::BUFFER_WBINVL1_SC:
2282 case AMDGPU::BUFFER_WBINVL1_VOL:
2283 case AMDGPU::BUFFER_GL0_INV:
2284 case AMDGPU::BUFFER_GL1_INV: {
2285 // FIXME: This should have already been handled by the memory legalizer.
2286 // Removing this currently doesn't affect any lit tests, but we need to
2287 // verify that nothing was relying on this. The number of buffer invalidates
2288 // being handled here should not be expanded.
2289 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2290 break;
2291 }
2292 case AMDGPU::SI_RETURN_TO_EPILOG:
2293 case AMDGPU::SI_RETURN:
2294 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
2295 case AMDGPU::S_SETPC_B64_return: {
2296 // All waits must be resolved at call return.
2297 // NOTE: this could be improved with knowledge of all call sites or
2298 // with knowledge of the called routines.
2299 ReturnInsts.insert(V: &MI);
2300 AMDGPU::Waitcnt AllZeroWait =
2301 WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2302 // On GFX12+, if LOAD_CNT is pending but no VGPRs are waiting for loads
2303 // (e.g., only GLOBAL_INV is pending), we can skip waiting on loadcnt.
2304 // GLOBAL_INV increments loadcnt but doesn't write to VGPRs, so there's
2305 // no need to wait for it at function boundaries.
2306 if (ST.hasExtendedWaitCounts() &&
2307 !ScoreBrackets.hasPendingEvent(E: HWEvents::VMEM_READ_ACCESS))
2308 AllZeroWait.set(T: AMDGPU::LOAD_CNT, Val: ~0u);
2309 Wait = AllZeroWait;
2310 break;
2311 }
2312 case AMDGPU::S_ENDPGM:
2313 case AMDGPU::S_ENDPGM_SAVED: {
2314 // In dynamic VGPR mode, we want to release the VGPRs before the wave exits.
2315 // Technically the hardware will do this on its own if we don't, but that
2316 // might cost extra cycles compared to doing it explicitly.
2317 // When not in dynamic VGPR mode, identify S_ENDPGM instructions which may
2318 // have to wait for outstanding VMEM stores. In this case it can be useful
2319 // to send a message to explicitly release all VGPRs before the stores have
2320 // completed, but it is only safe to do this if there are no outstanding
2321 // scratch stores.
2322 EndPgmInsts[&MI] =
2323 !ScoreBrackets.empty(T: AMDGPU::STORE_CNT) &&
2324 !ScoreBrackets.hasPendingEvent(E: HWEvents::SCRATCH_WRITE_ACCESS);
2325 break;
2326 }
2327 case AMDGPU::S_SENDMSG:
2328 case AMDGPU::S_SENDMSGHALT: {
2329 if (ST.hasLegacyGeometry() &&
2330 ((MI.getOperand(i: 0).getImm() & AMDGPU::SendMsg::ID_MASK_PreGFX11_) ==
2331 AMDGPU::SendMsg::ID_GS_DONE_PreGFX11)) {
2332 // Resolve vm waits before gs-done.
2333 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2334 break;
2335 }
2336 [[fallthrough]];
2337 }
2338 default: {
2339
2340 // Export & GDS instructions do not read the EXEC mask until after the
2341 // export is granted (which can occur well after the instruction is issued).
2342 // The shader program must flush all EXP operations on the export-count
2343 // before overwriting the EXEC mask.
2344 if (MI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: &TRI)) {
2345 // Export and GDS are tracked individually, either may trigger a waitcnt
2346 // for EXEC.
2347 if (ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_GPR_LOCK) ||
2348 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_PARAM_ACCESS) ||
2349 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_POS_ACCESS) ||
2350 ScoreBrackets.hasPendingEvent(E: HWEvents::GDS_GPR_LOCK)) {
2351 Wait.set(T: AMDGPU::EXP_CNT, Val: 0);
2352 }
2353 }
2354
2355 // Wait for any pending GDS instruction to complete before any
2356 // "Always GDS" instruction.
2357 if (TII.isAlwaysGDS(Opcode: Opc) && ScoreBrackets.hasPendingGDS())
2358 Wait.add(T: AMDGPU::DS_CNT, Count: ScoreBrackets.getPendingGDSWait());
2359
2360 if (MI.isCall()) {
2361 // The function is going to insert a wait on everything in its prolog.
2362 // This still needs to be careful if the call target is a load (e.g. a GOT
2363 // load). We also need to check WAW dependency with saved PC.
2364 CallInsts.insert(V: &MI);
2365 Wait = AMDGPU::Waitcnt();
2366
2367 const MachineOperand &CallAddrOp = TII.getCalleeOperand(MI);
2368 if (CallAddrOp.isReg()) {
2369 ScoreBrackets.determineWaitForPhysReg(
2370 T: SmemAccessCounter, Reg: CallAddrOp.getReg().asMCReg(), Wait, MI);
2371
2372 if (const auto *RtnAddrOp =
2373 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::dst)) {
2374 ScoreBrackets.determineWaitForPhysReg(
2375 T: SmemAccessCounter, Reg: RtnAddrOp->getReg().asMCReg(), Wait, MI);
2376 }
2377 }
2378 } else if (Opc == AMDGPU::S_BARRIER_WAIT) {
2379 ScoreBrackets.tryClearSCCWriteEvent(Inst: &MI);
2380 } else {
2381 // FIXME: Should not be relying on memoperands.
2382 // Look at the source operands of every instruction to see if
2383 // any of them results from a previous memory operation that affects
2384 // its current usage. If so, an s_waitcnt instruction needs to be
2385 // emitted.
2386 // If the source operand was defined by a load, add the s_waitcnt
2387 // instruction.
2388 //
2389 // Two cases are handled for destination operands:
2390 // 1) If the destination operand was defined by a load, add the s_waitcnt
2391 // instruction to guarantee the right WAW order.
2392 // 2) If a destination operand that was used by a recent export/store ins,
2393 // add s_waitcnt on exp_cnt to guarantee the WAR order.
2394
2395 for (const MachineMemOperand *Memop : MI.memoperands()) {
2396 const Value *Ptr = Memop->getValue();
2397 if (Memop->isStore()) {
2398 if (auto It = SLoadAddresses.find(Val: Ptr); It != SLoadAddresses.end()) {
2399 Wait.add(T: SmemAccessCounter, Count: 0);
2400 if (PDT.dominates(A: MI.getParent(), B: It->second))
2401 SLoadAddresses.erase(I: It);
2402 }
2403 }
2404 unsigned AS = Memop->getAddrSpace();
2405 if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::FLAT_ADDRESS)
2406 continue;
2407 // No need to wait before load from VMEM to LDS.
2408 if (TII.mayWriteLDSThroughDMA(MI))
2409 continue;
2410
2411 // LOAD_CNT is only relevant to vgpr or LDS.
2412 unsigned TID = LDSDMA_BEGIN;
2413 if (Ptr && Memop->getAAInfo()) {
2414 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
2415 for (unsigned I = 0, E = LDSDMAStores.size(); I != E; ++I) {
2416 if (MI.mayAlias(AA, Other: *LDSDMAStores[I], UseTBAA: true)) {
2417 if ((I + 1) >= NUM_LDSDMA) {
2418 // We didn't have enough slot to track this LDS DMA store, it
2419 // has been tracked using the common RegNo (FIRST_LDS_VGPR).
2420 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID,
2421 Wait);
2422 break;
2423 }
2424
2425 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT,
2426 TID: TID + I + 1, Wait);
2427 }
2428 }
2429 } else {
2430 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::LOAD_CNT, TID, Wait);
2431 }
2432 if (Memop->isStore()) {
2433 ScoreBrackets.determineWaitForLDSDMA(T: AMDGPU::EXP_CNT, TID, Wait);
2434 }
2435 }
2436
2437 // Loop over use and def operands.
2438 for (const MachineOperand &Op : MI.operands()) {
2439 if (!Op.isReg())
2440 continue;
2441
2442 // If the instruction does not read tied source, skip the operand.
2443 if (Op.isTied() && Op.isUse() && TII.doesNotReadTiedSource(MI))
2444 continue;
2445
2446 MCPhysReg Reg = Op.getReg().asMCReg();
2447
2448 const bool IsVGPR = TRI.isVectorRegister(MRI, Reg: Op.getReg());
2449 if (IsVGPR) {
2450 // Implicit VGPR defs and uses are never a part of the memory
2451 // instructions description and usually present to account for
2452 // super-register liveness.
2453 // TODO: Most of the other instructions also have implicit uses
2454 // for the liveness accounting only.
2455 if (Op.isImplicit() && MI.mayLoadOrStore())
2456 continue;
2457
2458 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VA_VDST_WR, Reg, Wait,
2459 MI);
2460 if (Op.isDef()) {
2461 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VA_VDST_RD, Reg, Wait,
2462 MI);
2463 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::VM_VSRC, Reg, Wait,
2464 MI);
2465 }
2466
2467 // RAW always needs an s_waitcnt. WAW needs an s_waitcnt unless the
2468 // previous write and this write are the same type of VMEM
2469 // instruction, in which case they are (in some architectures)
2470 // guaranteed to write their results in order anyway.
2471 // Additionally check instructions where Point Sample Acceleration
2472 // might be applied.
2473 if (Op.isUse() || !updateVMCntOnly(Inst: MI) ||
2474 ScoreBrackets.hasDifferentVGPRPendingEvents(
2475 Reg, E: AMDGPU::getSimplifiedVMEMEventsFor(Inst: MI, TII)) ||
2476 ScoreBrackets.hasPointSamplePendingVmemTypes(MI, Reg) ||
2477 !ST.hasVmemWriteVgprInOrder()) {
2478 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::LOAD_CNT, Reg, Wait,
2479 MI);
2480 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::SAMPLE_CNT, Reg, Wait,
2481 MI);
2482 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::BVH_CNT, Reg, Wait,
2483 MI);
2484 ScoreBrackets.clearVGPRPendingEvents(Reg);
2485 }
2486
2487 if (Op.isDef() ||
2488 ScoreBrackets.hasPendingEvent(E: HWEvents::EXP_LDS_ACCESS)) {
2489 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::EXP_CNT, Reg, Wait,
2490 MI);
2491 }
2492 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::DS_CNT, Reg, Wait, MI);
2493 } else if (Op.getReg() == AMDGPU::SCC) {
2494 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::KM_CNT, Reg, Wait, MI);
2495 } else {
2496 ScoreBrackets.determineWaitForPhysReg(T: SmemAccessCounter, Reg, Wait,
2497 MI);
2498 }
2499
2500 if (ST.hasWaitXcnt() && Op.isDef())
2501 ScoreBrackets.determineWaitForPhysReg(T: AMDGPU::X_CNT, Reg, Wait, MI);
2502 }
2503 }
2504 }
2505 }
2506
2507 // Ensure safety against exceptions from outstanding memory operations while
2508 // waiting for a barrier:
2509 //
2510 // * Some subtargets safely handle backing off the barrier in hardware
2511 // when an exception occurs.
2512 // * Some subtargets have an implicit S_WAITCNT 0 before barriers, so that
2513 // there can be no outstanding memory operations during the wait.
2514 // * Subtargets with split barriers don't need to back off the barrier; it
2515 // is up to the trap handler to preserve the user barrier state correctly.
2516 //
2517 // In all other cases, ensure safety by ensuring that there are no outstanding
2518 // memory operations.
2519 if (Opc == AMDGPU::S_BARRIER && !ST.hasAutoWaitcntBeforeBarrier() &&
2520 !ST.hasBackOffBarrier()) {
2521 Wait = Wait.combined(Other: WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/true));
2522 }
2523
2524 // TODO: Remove this work-around, enable the assert for Bug 457939
2525 // after fixing the scheduler. Also, the Shader Compiler code is
2526 // independent of target.
2527 if (SIInstrInfo::isCBranchVCCZRead(MI) && ST.hasReadVCCZBug() &&
2528 ScoreBrackets.hasPendingEvent(E: HWEvents::SMEM_ACCESS)) {
2529 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2530 }
2531
2532 // Verify that the wait is actually needed.
2533 ScoreBrackets.simplifyWaitcnt(Wait);
2534
2535 // It is only necessary to insert an S_WAITCNT_DEPCTR instruction that
2536 // waits on VA_VDST if the instruction it would precede is not a VALU
2537 // instruction, since hardware handles VALU->VGPR->VALU hazards in
2538 // expert scheduling mode.
2539 if (TII.isVALU(MI, /*AllowLDSDMA=*/false)) {
2540 Wait.set(T: AMDGPU::VA_VDST_RD, Val: ~0u);
2541 Wait.set(T: AMDGPU::VA_VDST_WR, Val: ~0u);
2542 }
2543
2544 // Since the translation for VMEM addresses occur in-order, we can apply the
2545 // XCnt if the current instruction is of VMEM type and has a memory
2546 // dependency with another VMEM instruction in flight.
2547 if (Wait.get(T: AMDGPU::X_CNT) != ~0u && isVmemAccess(MI)) {
2548 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::X_CNT);
2549 Wait.set(T: AMDGPU::X_CNT, Val: ~0u);
2550 }
2551
2552 // When forcing emit, we need to skip terminators because that would break the
2553 // terminators of the MBB if we emit a waitcnt between terminators.
2554 if (ForceEmitZeroFlag && !MI.isTerminator())
2555 Wait = WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
2556
2557 // If we force waitcnt then update Wait accordingly.
2558 for (AMDGPU::InstCounterType T : AMDGPU::inst_counter_types()) {
2559 if (!ForceEmitWaitcnt[T])
2560 continue;
2561 Wait.set(T, Val: 0);
2562 }
2563
2564 if (FlushFlags.FlushVmCnt) {
2565 for (AMDGPU::InstCounterType T :
2566 {AMDGPU::LOAD_CNT, AMDGPU::SAMPLE_CNT, AMDGPU::BVH_CNT})
2567 Wait.set(T, Val: 0);
2568 }
2569
2570 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(T: AMDGPU::DS_CNT))
2571 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2572
2573 if (ForceEmitZeroLoadFlag && Wait.get(T: AMDGPU::LOAD_CNT) != ~0u)
2574 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
2575
2576 return generateWaitcnt(Wait, It: MI.getIterator(), Block&: *MI.getParent(), ScoreBrackets,
2577 OldWaitcntInstr);
2578}
2579
2580bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
2581 MachineBasicBlock::instr_iterator It,
2582 MachineBasicBlock &Block,
2583 WaitcntBrackets &ScoreBrackets,
2584 MachineInstr *OldWaitcntInstr) {
2585 bool Modified = false;
2586
2587 if (OldWaitcntInstr)
2588 // Try to merge the required wait with preexisting waitcnt instructions.
2589 // Also erase redundant waitcnt.
2590 Modified =
2591 WCG->applyPreexistingWaitcnt(ScoreBrackets, OldWaitcntInstr&: *OldWaitcntInstr, Wait, It);
2592
2593 // ExpCnt can be merged into VINTERP.
2594 if (Wait.get(T: AMDGPU::EXP_CNT) != ~0u && It != Block.instr_end() &&
2595 SIInstrInfo::isVINTERP(MI: *It)) {
2596 MachineOperand *WaitExp = TII.getNamedOperand(MI&: *It, OperandName: AMDGPU::OpName::waitexp);
2597 if (Wait.get(T: AMDGPU::EXP_CNT) < WaitExp->getImm()) {
2598 WaitExp->setImm(Wait.get(T: AMDGPU::EXP_CNT));
2599 Modified = true;
2600 }
2601 // Apply ExpCnt before resetting it, so applyWaitcnt below sees all counts.
2602 ScoreBrackets.applyWaitcnt(Wait, T: AMDGPU::EXP_CNT);
2603 Wait.set(T: AMDGPU::EXP_CNT, Val: ~0u);
2604
2605 LLVM_DEBUG(dbgs() << "generateWaitcnt\n"
2606 << "Update Instr: " << *It);
2607 }
2608
2609 if (WCG->createNewWaitcnt(Block, It, Wait, ScoreBrackets))
2610 Modified = true;
2611
2612 // Any counts that could have been applied to any existing waitcnt
2613 // instructions will have been done so, now deal with any remaining.
2614 ScoreBrackets.applyWaitcnt(Wait);
2615
2616 return Modified;
2617}
2618
2619bool SIInsertWaitcnts::isVmemAccess(const MachineInstr &MI) const {
2620 return (TII.isFLAT(MI) && TII.mayAccessVMEMThroughFlat(MI)) ||
2621 (TII.isVMEM(MI) && !AMDGPU::getMUBUFIsBufferInv(Opc: MI.getOpcode()));
2622}
2623
2624// Return true if the next instruction is S_ENDPGM, following fallthrough
2625// blocks if necessary.
2626bool SIInsertWaitcnts::isNextENDPGM(MachineBasicBlock::instr_iterator It,
2627 MachineBasicBlock *Block) const {
2628 auto BlockEnd = Block->getParent()->end();
2629 auto BlockIter = Block->getIterator();
2630
2631 while (true) {
2632 if (It.isEnd()) {
2633 if (++BlockIter != BlockEnd) {
2634 It = BlockIter->instr_begin();
2635 continue;
2636 }
2637
2638 return false;
2639 }
2640
2641 if (!It->isMetaInstruction())
2642 break;
2643
2644 It++;
2645 }
2646
2647 assert(!It.isEnd());
2648
2649 return It->getOpcode() == AMDGPU::S_ENDPGM;
2650}
2651
2652// Add a wait after an instruction if architecture requirements mandate one.
2653bool SIInsertWaitcnts::insertForcedWaitAfter(MachineInstr &Inst,
2654 MachineBasicBlock &Block,
2655 WaitcntBrackets &ScoreBrackets) {
2656 AMDGPU::Waitcnt Wait;
2657 bool NeedsEndPGMCheck = false;
2658
2659 if (ST.isPreciseMemoryEnabled() && Inst.mayLoadOrStore())
2660 Wait = WCG->getAllZeroWaitcnt(IncludeVSCnt: Inst.mayStore() &&
2661 !SIInstrInfo::isAtomicRet(MI: Inst));
2662
2663 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode())) {
2664 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
2665 NeedsEndPGMCheck = true;
2666 }
2667
2668 ScoreBrackets.simplifyWaitcnt(Wait);
2669
2670 auto SuccessorIt = std::next(x: Inst.getIterator());
2671 bool Result = generateWaitcnt(Wait, It: SuccessorIt, Block, ScoreBrackets,
2672 /*OldWaitcntInstr=*/nullptr);
2673
2674 if (Result && NeedsEndPGMCheck && isNextENDPGM(It: SuccessorIt, Block: &Block)) {
2675 BuildMI(BB&: Block, I: SuccessorIt, MIMD: Inst.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_NOP))
2676 .addImm(Val: 0);
2677 }
2678
2679 return Result;
2680}
2681
2682void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
2683 WaitcntBrackets *ScoreBrackets) {
2684
2685 HWEvents InstEvents = AMDGPU::getEventsFor(Inst, ST, IsExpertMode, TgSplit);
2686 for (HWEvents E : InstEvents)
2687 ScoreBrackets->updateByEvent(E, Inst);
2688
2689 if (TII.isDS(MI: Inst) && TII.usesLGKM_CNT(MI: Inst)) {
2690 if (TII.isAlwaysGDS(Opcode: Inst.getOpcode()) ||
2691 TII.hasModifiersSet(MI: Inst, OpName: AMDGPU::OpName::gds)) {
2692 ScoreBrackets->setPendingGDS();
2693 }
2694 } else if (TII.isFLAT(MI: Inst)) {
2695 if (Inst.mayLoadOrStore() && TII.mayAccessVMEMThroughFlat(MI: Inst) &&
2696 TII.mayAccessLDSThroughFlat(MI: Inst, TgSplit) &&
2697 !SIInstrInfo::isLDSDMA(MI: Inst)) {
2698 // Async/LDSDMA operations have FLAT encoding but do not actually use flat
2699 // pointers. They do have two operands that each access global and LDS,
2700 // thus making it appear at this point that they are using a flat pointer.
2701 // Filter them out, and for the rest, generate a dependency on flat
2702 // pointers so that both VM and LGKM counters are flushed.
2703 ScoreBrackets->setPendingFlat();
2704 }
2705 } else if (Inst.isCall()) {
2706 // Act as a wait on everything, but AsyncCnt and TensorCnt are never
2707 // included in such blanket waits.
2708 ScoreBrackets->applyWaitcnt(Wait: WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false));
2709 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2710 } else if (TII.isVINTERP(MI: Inst)) {
2711 int64_t Imm = TII.getNamedOperand(MI&: Inst, OperandName: AMDGPU::OpName::waitexp)->getImm();
2712 ScoreBrackets->applyWaitcnt(T: AMDGPU::EXP_CNT, Count: Imm);
2713 }
2714
2715 // Set XCNT to zero in the bracket for instructions that implicitly drain
2716 // XCNT.
2717 if (ST.hasWaitXcnt() && SIInstrInfo::isXcntDrain(MI: Inst))
2718 ScoreBrackets->applyWaitcnt(T: AMDGPU::X_CNT, Count: 0);
2719}
2720
2721bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
2722 unsigned OtherScore) {
2723 unsigned MyShifted = Score <= M.OldLB ? 0 : Score + M.MyShift;
2724 unsigned OtherShifted =
2725 OtherScore <= M.OtherLB ? 0 : OtherScore + M.OtherShift;
2726 Score = std::max(a: MyShifted, b: OtherShifted);
2727 return OtherShifted > MyShifted;
2728}
2729
2730bool WaitcntBrackets::mergeAsyncMarks(ArrayRef<MergeInfo> MergeInfos,
2731 ArrayRef<CounterValueArray> OtherMarks) {
2732 bool StrictDom = false;
2733
2734 LLVM_DEBUG(dbgs() << "Merging async marks ...");
2735 // Early exit: nothing to merge when both sides are empty.
2736 if (AsyncMarks.empty() && OtherMarks.empty()) {
2737 LLVM_DEBUG(dbgs() << " nothing to merge\n");
2738 return false;
2739 }
2740 LLVM_DEBUG(dbgs() << '\n');
2741
2742 // Determine maximum length needed after merging
2743 auto MaxSize = (unsigned)std::max(a: AsyncMarks.size(), b: OtherMarks.size());
2744 MaxSize = std::min(a: MaxSize, b: MaxAsyncMarks);
2745
2746 // Keep only the most recent marks within our limit.
2747 if (AsyncMarks.size() > MaxSize)
2748 AsyncMarks.erase(CS: AsyncMarks.begin(),
2749 CE: AsyncMarks.begin() + (AsyncMarks.size() - MaxSize));
2750
2751 // Pad with zero-filled marks if our list is shorter. Zero represents "no
2752 // pending async operations at this checkpoint" and acts as the identity
2753 // element for max() during merging. We pad at the beginning since the marks
2754 // need to be aligned in most-recent order.
2755 constexpr CounterValueArray ZeroMark{};
2756 AsyncMarks.insert(I: AsyncMarks.begin(), NumToInsert: MaxSize - AsyncMarks.size(), Elt: ZeroMark);
2757
2758 LLVM_DEBUG({
2759 dbgs() << "Before merge:\n";
2760 for (const auto &Mark : AsyncMarks) {
2761 llvm::interleaveComma(Mark, dbgs());
2762 dbgs() << '\n';
2763 }
2764 dbgs() << "Other marks:\n";
2765 for (const auto &Mark : OtherMarks) {
2766 llvm::interleaveComma(Mark, dbgs());
2767 dbgs() << '\n';
2768 }
2769 });
2770
2771 // Merge element-wise using the existing mergeScore function and the
2772 // appropriate MergeInfo for each counter type. Iterate only while we have
2773 // elements in both vectors.
2774 unsigned OtherSize = OtherMarks.size();
2775 unsigned OurSize = AsyncMarks.size();
2776 unsigned MergeCount = std::min(a: OtherSize, b: OurSize);
2777 // OtherMarks is empty -> OtherSize == 0 -> MergeCount == 0.
2778 // Our existing marks are the conservative result; return early to avoid
2779 // passing MergeCount == 0 to seq_inclusive which asserts Begin <= End.
2780 if (MergeCount == 0)
2781 return StrictDom;
2782 for (auto Idx : seq_inclusive<unsigned>(Begin: 1, End: MergeCount)) {
2783 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter)) {
2784 StrictDom |= mergeScore(M: MergeInfos[T], Score&: AsyncMarks[OurSize - Idx][T],
2785 OtherScore: OtherMarks[OtherSize - Idx][T]);
2786 }
2787 }
2788
2789 LLVM_DEBUG({
2790 dbgs() << "After merge:\n";
2791 for (const auto &Mark : AsyncMarks) {
2792 llvm::interleaveComma(Mark, dbgs());
2793 dbgs() << '\n';
2794 }
2795 });
2796
2797 return StrictDom;
2798}
2799
2800/// Merge the pending events and associater score brackets of \p Other into
2801/// this brackets status.
2802///
2803/// Returns whether the merge resulted in a change that requires tighter waits
2804/// (i.e. the merged brackets strictly dominate the original brackets).
2805bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
2806 bool StrictDom = false;
2807
2808 // Check if "other" has keys we don't have, and create default entries for
2809 // those. If they remain empty after merging, we will clean it up after.
2810 for (auto K : Other.VMem.keys())
2811 VMem.try_emplace(Key: K);
2812 for (auto K : Other.SGPRs.keys())
2813 SGPRs.try_emplace(Key: K);
2814
2815 // Array to store MergeInfo for each counter type
2816 MergeInfo MergeInfos[AMDGPU::NUM_INST_CNTS];
2817
2818 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter)) {
2819 // Merge event flags for this counter
2820 const HWEvents &EventsForT = Context->getWaitEvents(T);
2821 const HWEvents OldEvents = PendingEvents & EventsForT;
2822 const HWEvents OtherEvents = Other.PendingEvents & EventsForT;
2823 if (!OldEvents.contains(Other: OtherEvents))
2824 StrictDom = true;
2825 PendingEvents |= OtherEvents;
2826
2827 // Merge scores for this counter
2828 const unsigned MyPending = ScoreUBs[T] - ScoreLBs[T];
2829 const unsigned OtherPending = Other.ScoreUBs[T] - Other.ScoreLBs[T];
2830 const unsigned NewUB = ScoreLBs[T] + std::max(a: MyPending, b: OtherPending);
2831 if (NewUB < ScoreLBs[T])
2832 report_fatal_error(reason: "waitcnt score overflow");
2833
2834 MergeInfo &M = MergeInfos[T];
2835 M.OldLB = ScoreLBs[T];
2836 M.OtherLB = Other.ScoreLBs[T];
2837 M.MyShift = NewUB - ScoreUBs[T];
2838 M.OtherShift = NewUB - Other.ScoreUBs[T];
2839
2840 ScoreUBs[T] = NewUB;
2841
2842 if (T == AMDGPU::LOAD_CNT)
2843 StrictDom |= mergeScore(M, Score&: LastFlatLoadCnt, OtherScore: Other.LastFlatLoadCnt);
2844
2845 if (T == AMDGPU::DS_CNT) {
2846 StrictDom |= mergeScore(M, Score&: LastFlatDsCnt, OtherScore: Other.LastFlatDsCnt);
2847 StrictDom |= mergeScore(M, Score&: LastGDS, OtherScore: Other.LastGDS);
2848 }
2849
2850 if (T == AMDGPU::KM_CNT) {
2851 StrictDom |= mergeScore(M, Score&: SCCScore, OtherScore: Other.SCCScore);
2852 if (Other.hasPendingEvent(E: HWEvents::SCC_WRITE)) {
2853 if (!(OldEvents & HWEvents::SCC_WRITE)) {
2854 PendingSCCWrite = Other.PendingSCCWrite;
2855 } else if (PendingSCCWrite != Other.PendingSCCWrite) {
2856 PendingSCCWrite = nullptr;
2857 }
2858 }
2859 }
2860
2861 for (auto &[RegID, Info] : VMem)
2862 StrictDom |= mergeScore(M, Score&: Info.Scores[T], OtherScore: Other.getVMemScore(TID: RegID, T));
2863
2864 if (isSmemCounter(T)) {
2865 for (auto &[RegID, Info] : SGPRs) {
2866 auto It = Other.SGPRs.find(Val: RegID);
2867 unsigned OtherScore = (It != Other.SGPRs.end()) ? It->second.get(T) : 0;
2868 StrictDom |= mergeScore(M, Score&: Info.get(T), OtherScore);
2869 }
2870 }
2871 }
2872
2873 for (auto &[TID, Info] : VMem) {
2874 if (auto It = Other.VMem.find(Val: TID); It != Other.VMem.end()) {
2875 HWEvents NewVGPRContext =
2876 Info.VGPRPendingEvents | It->second.VGPRPendingEvents;
2877 StrictDom |= NewVGPRContext != Info.VGPRPendingEvents;
2878 Info.VGPRPendingEvents = NewVGPRContext;
2879 }
2880 }
2881
2882 StrictDom |= mergeAsyncMarks(MergeInfos, OtherMarks: Other.AsyncMarks);
2883 for (auto T : inst_counter_types(MaxCounter: Context->MaxCounter))
2884 StrictDom |= mergeScore(M: MergeInfos[T], Score&: AsyncScore[T], OtherScore: Other.AsyncScore[T]);
2885
2886 purgeEmptyTrackingData();
2887 return StrictDom;
2888}
2889
2890static bool isWaitInstr(MachineInstr &Inst) {
2891 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode: Inst.getOpcode());
2892 return Opcode == AMDGPU::S_WAITCNT ||
2893 (Opcode == AMDGPU::S_WAITCNT_VSCNT && Inst.getOperand(i: 0).isReg() &&
2894 Inst.getOperand(i: 0).getReg() == AMDGPU::SGPR_NULL) ||
2895 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
2896 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
2897 Opcode == AMDGPU::S_WAITCNT_lds_direct ||
2898 Opcode == AMDGPU::WAIT_ASYNCMARK ||
2899 AMDGPU::counterTypeForInstr(Opcode).has_value();
2900}
2901
2902void SIInsertWaitcnts::setSchedulingMode(MachineBasicBlock &MBB,
2903 MachineBasicBlock::iterator I,
2904 bool ExpertMode) const {
2905 const unsigned EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
2906 Values: AMDGPU::Hwreg::ID_SCHED_MODE, Values: AMDGPU::Hwreg::HwregOffset::Default, Values: 2);
2907 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
2908 .addImm(Val: ExpertMode ? 2 : 0)
2909 .addImm(Val: EncodedReg);
2910}
2911
2912namespace {
2913// TODO: Remove this work-around after fixing the scheduler.
2914// There are two reasons why vccz might be incorrect; see ST.hasReadVCCZBug()
2915// and ST.partialVCCWritesUpdateVCCZ().
2916// i. VCCZBug: There is a hardware bug on CI/SI where SMRD instruction may
2917// corrupt vccz bit, so when we detect that an instruction may read from
2918// a corrupt vccz bit, we need to:
2919// 1. Insert s_waitcnt lgkm(0) to wait for all outstanding SMRD
2920// operations to complete.
2921// 2. Recompute the correct value of vccz by writing the current value
2922// of vcc back to vcc.
2923// ii. Partial writes to vcc don't update vccz, so we need to recompute the
2924// correct value of vccz by reading vcc and writing it back to vcc.
2925// No waitcnt is needed in this case.
2926class VCCZWorkaround {
2927 const WaitcntBrackets &ScoreBrackets;
2928 const GCNSubtarget &ST;
2929 const SIInstrInfo &TII;
2930 const SIRegisterInfo &TRI;
2931 bool VCCZCorruptionBug = false;
2932 bool VCCZNotUpdatedByPartialWrites = false;
2933 /// vccz could be incorrect at a basic block boundary if a predecessor wrote
2934 /// to vcc and then issued an smem load, so initialize to true.
2935 bool MustRecomputeVCCZ = true;
2936
2937public:
2938 VCCZWorkaround(const WaitcntBrackets &ScoreBrackets, const GCNSubtarget &ST,
2939 const SIInstrInfo &TII, const SIRegisterInfo &TRI)
2940 : ScoreBrackets(ScoreBrackets), ST(ST), TII(TII), TRI(TRI) {
2941 VCCZCorruptionBug = ST.hasReadVCCZBug();
2942 VCCZNotUpdatedByPartialWrites = !ST.partialVCCWritesUpdateVCCZ();
2943 }
2944 /// If \p MI reads vccz and we must recompute it based on MustRecomputeVCCZ,
2945 /// then emit a vccz recompute instruction before \p MI. This needs to be
2946 /// called on every instruction in the basic block because it also tracks the
2947 /// state and updates MustRecomputeVCCZ accordingly. Returns true if it
2948 /// modified the IR.
2949 bool tryRecomputeVCCZ(MachineInstr &MI) {
2950 // No need to run this if neither bug is present.
2951 if (!VCCZCorruptionBug && !VCCZNotUpdatedByPartialWrites)
2952 return false;
2953
2954 // If MI is an SMEM and it can corrupt vccz on this target, then we need
2955 // both to emit a waitcnt and to recompute vccz.
2956 // But we don't actually emit a waitcnt here. This is done in
2957 // generateWaitcntInstBefore() because it tracks all the necessary waitcnt
2958 // state, and can either skip emitting a waitcnt if there is already one in
2959 // the IR, or emit an "optimized" combined waitcnt.
2960 // If this is an smem read, it could complete and clobber vccz at any time.
2961 MustRecomputeVCCZ |= VCCZCorruptionBug && TII.isSMRD(MI);
2962
2963 // If the target partial vcc writes don't update vccz, and MI is such an
2964 // instruction then we must recompute vccz.
2965 // Note: We are using PartiallyWritesToVCCOpt optional to avoid calling
2966 // `definesRegister()` more than needed, because it's not very cheap.
2967 std::optional<bool> PartiallyWritesToVCCOpt;
2968 auto PartiallyWritesToVCC = [](MachineInstr &MI) {
2969 return MI.definesRegister(Reg: AMDGPU::VCC_LO, /*TRI=*/nullptr) ||
2970 MI.definesRegister(Reg: AMDGPU::VCC_HI, /*TRI=*/nullptr);
2971 };
2972 if (VCCZNotUpdatedByPartialWrites) {
2973 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
2974 // If this is a partial VCC write but won't update vccz, then we must
2975 // recompute vccz.
2976 MustRecomputeVCCZ |= *PartiallyWritesToVCCOpt;
2977 }
2978
2979 // If MI is a vcc write with no pending smem, or there is a pending smem
2980 // but the target does not suffer from the vccz corruption bug, then we
2981 // don't need to recompute vccz as this write will recompute it anyway.
2982 if (!ScoreBrackets.hasPendingEvent(E: HWEvents::SMEM_ACCESS) ||
2983 !VCCZCorruptionBug) {
2984 // Compute PartiallyWritesToVCCOpt if we haven't done so already.
2985 if (!PartiallyWritesToVCCOpt)
2986 PartiallyWritesToVCCOpt = PartiallyWritesToVCC(MI);
2987 bool FullyWritesToVCC = !*PartiallyWritesToVCCOpt &&
2988 MI.definesRegister(Reg: AMDGPU::VCC, /*TRI=*/nullptr);
2989 // If we write to the full vcc or we write partially and the target
2990 // updates vccz on partial writes, then vccz will be updated correctly.
2991 bool UpdatesVCCZ = FullyWritesToVCC || (!VCCZNotUpdatedByPartialWrites &&
2992 *PartiallyWritesToVCCOpt);
2993 if (UpdatesVCCZ)
2994 MustRecomputeVCCZ = false;
2995 }
2996
2997 // If MI is a branch that reads VCCZ then emit a waitcnt and a vccz
2998 // restore instruction if either is needed.
2999 if (SIInstrInfo::isCBranchVCCZRead(MI) && MustRecomputeVCCZ) {
3000 // Recompute the vccz bit. Any time a value is written to vcc, the vccz
3001 // bit is updated, so we can restore the bit by reading the value of vcc
3002 // and then writing it back to the register.
3003 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
3004 MCID: TII.get(Opcode: ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
3005 DestReg: TRI.getVCC())
3006 .addReg(RegNo: TRI.getVCC());
3007 MustRecomputeVCCZ = false;
3008 return true;
3009 }
3010 return false;
3011 }
3012};
3013
3014} // namespace
3015
3016// Generate s_waitcnt instructions where needed.
3017bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
3018 MachineBasicBlock &Block,
3019 WaitcntBrackets &ScoreBrackets) {
3020 bool Modified = false;
3021
3022 LLVM_DEBUG({
3023 dbgs() << "*** Begin Block: ";
3024 Block.printName(dbgs());
3025 ScoreBrackets.dump();
3026 });
3027 VCCZWorkaround VCCZW(ScoreBrackets, ST, TII, TRI);
3028
3029 // Walk over the instructions.
3030 MachineInstr *OldWaitcntInstr = nullptr;
3031
3032 // NOTE: We may append instrs after Inst while iterating.
3033 for (MachineBasicBlock::instr_iterator Iter = Block.instr_begin(),
3034 E = Block.instr_end();
3035 Iter != E; ++Iter) {
3036 MachineInstr &Inst = *Iter;
3037 if (isNonWaitcntMetaInst(MI: Inst))
3038 continue;
3039 // Track pre-existing waitcnts that were added in earlier iterations or by
3040 // the memory legalizer.
3041 if (isWaitInstr(Inst) ||
3042 (IsExpertMode && Inst.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR)) {
3043 if (!OldWaitcntInstr)
3044 OldWaitcntInstr = &Inst;
3045 continue;
3046 }
3047
3048 PreheaderFlushFlags FlushFlags;
3049 if (Block.getFirstTerminator() == Inst)
3050 FlushFlags = isPreheaderToFlush(MBB&: Block, ScoreBrackets);
3051
3052 // Generate an s_waitcnt instruction to be placed before Inst, if needed.
3053 Modified |= generateWaitcntInstBefore(MI&: Inst, ScoreBrackets, OldWaitcntInstr,
3054 FlushFlags);
3055 OldWaitcntInstr = nullptr;
3056
3057 if (Inst.getOpcode() == AMDGPU::ASYNCMARK) {
3058 // Asyncmarks record the current wait state and so should not allow
3059 // waitcnts that occur after them to be merged into waitcnts that occur
3060 // before.
3061 ScoreBrackets.recordAsyncMark(Inst);
3062 continue;
3063 }
3064
3065 if (TII.isSMRD(MI: Inst)) {
3066 for (const MachineMemOperand *Memop : Inst.memoperands()) {
3067 // No need to handle invariant loads when avoiding WAR conflicts, as
3068 // there cannot be a vector store to the same memory location.
3069 if (!Memop->isInvariant()) {
3070 const Value *Ptr = Memop->getValue();
3071 SLoadAddresses.insert(KV: std::pair(Ptr, Inst.getParent()));
3072 }
3073 }
3074 }
3075
3076 updateEventWaitcntAfter(Inst, ScoreBrackets: &ScoreBrackets);
3077
3078 // Note: insertForcedWaitAfter() may add instrs after Iter that need to be
3079 // visited by the loop.
3080 Modified |= insertForcedWaitAfter(Inst, Block, ScoreBrackets);
3081
3082 LLVM_DEBUG({
3083 Inst.print(dbgs());
3084 ScoreBrackets.dump();
3085 });
3086
3087 // If the target suffers from the vccz bugs, this may emit the necessary
3088 // vccz recompute instruction before \p Inst if needed.
3089 Modified |= VCCZW.tryRecomputeVCCZ(MI&: Inst);
3090 }
3091
3092 // Flush counters at the end of the block if needed (for preheaders with no
3093 // terminator).
3094 AMDGPU::Waitcnt Wait;
3095 if (Block.getFirstTerminator() == Block.end()) {
3096 PreheaderFlushFlags FlushFlags = isPreheaderToFlush(MBB&: Block, ScoreBrackets);
3097 if (FlushFlags.FlushVmCnt) {
3098 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::LOAD_CNT))
3099 Wait.set(T: AMDGPU::LOAD_CNT, Val: 0);
3100 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::SAMPLE_CNT))
3101 Wait.set(T: AMDGPU::SAMPLE_CNT, Val: 0);
3102 if (ScoreBrackets.hasPendingEvent(T: AMDGPU::BVH_CNT))
3103 Wait.set(T: AMDGPU::BVH_CNT, Val: 0);
3104 }
3105 if (FlushFlags.FlushDsCnt && ScoreBrackets.hasPendingEvent(T: AMDGPU::DS_CNT))
3106 Wait.set(T: AMDGPU::DS_CNT, Val: 0);
3107 }
3108
3109 // Combine or remove any redundant waitcnts at the end of the block.
3110 Modified |= generateWaitcnt(Wait, It: Block.instr_end(), Block, ScoreBrackets,
3111 OldWaitcntInstr);
3112
3113 LLVM_DEBUG({
3114 dbgs() << "*** End Block: ";
3115 Block.printName(dbgs());
3116 ScoreBrackets.dump();
3117 });
3118
3119 return Modified;
3120}
3121
3122bool SIInsertWaitcnts::removeRedundantSoftXcnts(MachineBasicBlock &Block) {
3123 if (Block.size() <= 1)
3124 return false;
3125 // The Memory Legalizer conservatively inserts a soft xcnt before each
3126 // atomic RMW operation. However, for sequences of back-to-back atomic
3127 // RMWs, only the first s_wait_xcnt insertion is necessary. Optimize away
3128 // the redundant soft xcnts.
3129 bool Modified = false;
3130 // Remember the last atomic with a soft xcnt right before it.
3131 MachineInstr *LastAtomicWithSoftXcnt = nullptr;
3132
3133 for (MachineInstr &MI : drop_begin(RangeOrContainer&: Block)) {
3134 // Ignore last atomic if non-LDS VMEM and SMEM.
3135 bool IsLDS = TII.isDS(MI) ||
3136 (TII.isFLAT(MI) && TII.mayAccessLDSThroughFlat(MI, TgSplit));
3137 if (!IsLDS && (MI.mayLoad() ^ MI.mayStore()))
3138 LastAtomicWithSoftXcnt = nullptr;
3139
3140 bool IsAtomicRMW =
3141 SIInstrFlags::isMaybeAtomic(O: MI) && MI.mayLoad() && MI.mayStore();
3142 MachineInstr &PrevMI = *MI.getPrevNode();
3143 // This is an atomic with a soft xcnt.
3144 if (PrevMI.getOpcode() == AMDGPU::S_WAIT_XCNT_soft && IsAtomicRMW) {
3145 // If we have already found an atomic with a soft xcnt, remove this soft
3146 // xcnt as it's redundant.
3147 if (LastAtomicWithSoftXcnt) {
3148 PrevMI.eraseFromParent();
3149 Modified = true;
3150 }
3151 LastAtomicWithSoftXcnt = &MI;
3152 }
3153 }
3154 return Modified;
3155}
3156
3157// Return flags indicating which counters should be flushed in the preheader.
3158PreheaderFlushFlags
3159SIInsertWaitcnts::isPreheaderToFlush(MachineBasicBlock &MBB,
3160 const WaitcntBrackets &ScoreBrackets) {
3161 auto [Iterator, IsInserted] =
3162 PreheadersToFlush.try_emplace(Key: &MBB, Args: PreheaderFlushFlags());
3163 if (!IsInserted)
3164 return Iterator->second;
3165
3166 MachineBasicBlock *Succ = MBB.getSingleSuccessor();
3167 if (!Succ)
3168 return PreheaderFlushFlags();
3169
3170 MachineLoop *Loop = MLI.getLoopFor(BB: Succ);
3171 if (!Loop)
3172 return PreheaderFlushFlags();
3173
3174 if (Loop->getLoopPreheader() == &MBB) {
3175 Iterator->second = getPreheaderFlushFlags(ML: Loop, Brackets: ScoreBrackets);
3176 return Iterator->second;
3177 }
3178
3179 return PreheaderFlushFlags();
3180}
3181
3182bool SIInsertWaitcnts::isVMEMOrFlatVMEM(const MachineInstr &MI) const {
3183 if (SIInstrInfo::isFLAT(MI))
3184 return TII.mayAccessVMEMThroughFlat(MI);
3185 return SIInstrInfo::isVMEM(MI);
3186}
3187
3188bool SIInsertWaitcnts::isDSRead(const MachineInstr &MI) const {
3189 return SIInstrInfo::isDS(MI) && MI.mayLoad() && !MI.mayStore();
3190}
3191
3192// Check if instruction is a store to LDS that is counted via DSCNT
3193// (where that counter exists).
3194bool SIInsertWaitcnts::mayStoreIncrementingDSCNT(const MachineInstr &MI) const {
3195 return MI.mayStore() && SIInstrInfo::isDS(MI);
3196}
3197
3198// Return flags indicating which counters should be flushed in the preheader of
3199// the given loop. We currently decide to flush in the following situations:
3200// For VMEM (FlushVmCnt):
3201// 1. The loop contains vmem store(s), no vmem load and at least one use of a
3202// vgpr containing a value that is loaded outside of the loop. (Only on
3203// targets with no vscnt counter).
3204// 2. The loop contains vmem load(s), but the loaded values are not used in the
3205// loop, and at least one use of a vgpr containing a value that is loaded
3206// outside of the loop.
3207// For DS (FlushDsCnt, GFX12+ only):
3208// 3. The loop contains no DS reads, and at least one use of a vgpr containing
3209// a value that is DS read outside of the loop.
3210// 4. The loop contains DS read(s), loaded values are not used in the same
3211// iteration but in the next iteration (prefetch pattern), and at least one
3212// use of a vgpr containing a value that is DS read outside of the loop.
3213// Flushing in preheader reduces wait overhead if the wait requirement in
3214// iteration 1 would otherwise be more strict (but unfortunately preheader
3215// flush decision is taken before knowing that).
3216// 5. (Single-block loops only) The loop has DS prefetch reads with flush point
3217// tracking. Some DS reads may be used in the same iteration (creating
3218// "flush points"), but others remain unflushed at the backedge. When a DS
3219// read is consumed in the same iteration, it and all prior reads are
3220// "flushed" (FIFO order). No DS writes are allowed in the loop.
3221// TODO: Find a way to extend to multi-block loops.
3222PreheaderFlushFlags
3223SIInsertWaitcnts::getPreheaderFlushFlags(MachineLoop *ML,
3224 const WaitcntBrackets &Brackets) {
3225 PreheaderFlushFlags Flags;
3226 bool HasVMemLoad = false;
3227 bool HasVMemStore = false;
3228 bool UsesVgprVMEMLoadedOutside = false;
3229 bool UsesVgprDSReadOutside = false;
3230 bool VMemInvalidated = false;
3231 // DS optimization only applies to GFX12+ where DS_CNT is separate.
3232 // Tracking status for "no DS read in loop" or "pure DS prefetch
3233 // (use only in next iteration)".
3234 bool TrackSimpleDSOpt = ST.hasExtendedWaitCounts();
3235 DenseSet<MCRegUnit> VgprUse;
3236 DenseSet<MCRegUnit> VgprDefVMEM;
3237 DenseSet<MCRegUnit> VgprDefDS;
3238
3239 // Track DS reads for prefetch pattern with flush points (single-block only).
3240 // Keeps track of the last DS read (position counted from the top of the loop)
3241 // to each VGPR. Read is considered consumed (and thus needs flushing) if
3242 // the dest register has a use or is overwritten (by any later opertions).
3243 DenseMap<MCRegUnit, unsigned> LastDSReadPositionMap;
3244 unsigned DSReadPosition = 0;
3245 bool IsSingleBlock = ML->getNumBlocks() == 1;
3246 bool TrackDSFlushPoint = ST.hasExtendedWaitCounts() && IsSingleBlock;
3247 unsigned LastDSFlushPosition = 0;
3248
3249 for (MachineBasicBlock *MBB : ML->blocks()) {
3250 for (MachineInstr &MI : *MBB) {
3251 if (isVMEMOrFlatVMEM(MI)) {
3252 HasVMemLoad |= MI.mayLoad();
3253 HasVMemStore |= MI.mayStore();
3254 }
3255 // TODO: Can we relax DSStore check? There may be cases where
3256 // these DS stores are drained prior to the end of MBB (or loop).
3257 if (mayStoreIncrementingDSCNT(MI)) {
3258 // Early exit if none of the optimizations are feasible.
3259 // Otherwise, set tracking status appropriately and continue.
3260 if (VMemInvalidated)
3261 return Flags;
3262 TrackSimpleDSOpt = false;
3263 TrackDSFlushPoint = false;
3264 }
3265 bool IsDSRead = isDSRead(MI);
3266 if (IsDSRead)
3267 ++DSReadPosition;
3268
3269 // Helper: if RU has a pending DS read, update LastDSFlushPosition
3270 auto updateDSReadFlushTracking = [&](MCRegUnit RU) {
3271 if (!TrackDSFlushPoint)
3272 return;
3273 if (auto It = LastDSReadPositionMap.find(Val: RU);
3274 It != LastDSReadPositionMap.end()) {
3275 // RU defined by DSRead is used or overwritten. Need to complete
3276 // the read, if not already implied by a later DSRead (to any RU)
3277 // needing to complete in FIFO order.
3278 LastDSFlushPosition = std::max(a: LastDSFlushPosition, b: It->second);
3279 }
3280 };
3281
3282 for (const MachineOperand &Op : MI.all_uses()) {
3283 if (Op.isDebug() || !TRI.isVectorRegister(MRI, Reg: Op.getReg()))
3284 continue;
3285 // Vgpr use
3286 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3287 // If we find a register that is loaded inside the loop, 1. and 2.
3288 // are invalidated.
3289 if (VgprDefVMEM.contains(V: RU))
3290 VMemInvalidated = true;
3291
3292 // Check for DS reads used inside the loop
3293 if (VgprDefDS.contains(V: RU))
3294 TrackSimpleDSOpt = false;
3295
3296 // Early exit if all optimizations are invalidated
3297 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3298 return Flags;
3299
3300 // Check for flush points (DS read used in same iteration)
3301 updateDSReadFlushTracking(RU);
3302
3303 VgprUse.insert(V: RU);
3304 // Check if this register has a pending VMEM load from outside the
3305 // loop (value loaded outside and used inside).
3306 VMEMID ID = toVMEMID(RU);
3307 if (Brackets.hasPendingVMEM(ID, T: AMDGPU::LOAD_CNT) ||
3308 Brackets.hasPendingVMEM(ID, T: AMDGPU::SAMPLE_CNT) ||
3309 Brackets.hasPendingVMEM(ID, T: AMDGPU::BVH_CNT))
3310 UsesVgprVMEMLoadedOutside = true;
3311 // Check if loaded outside the loop via DS (not VMEM/FLAT).
3312 // Only consider it a DS read if there's no pending VMEM load for
3313 // this register, since FLAT can set both counters.
3314 else if (Brackets.hasPendingVMEM(ID, T: AMDGPU::DS_CNT))
3315 UsesVgprDSReadOutside = true;
3316 }
3317 }
3318
3319 // VMem load vgpr def
3320 if (isVMEMOrFlatVMEM(MI) && MI.mayLoad()) {
3321 for (const MachineOperand &Op : MI.all_defs()) {
3322 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3323 // If we find a register that is loaded inside the loop, 1. and 2.
3324 // are invalidated.
3325 if (VgprUse.contains(V: RU))
3326 VMemInvalidated = true;
3327 VgprDefVMEM.insert(V: RU);
3328 }
3329 }
3330 // Early exit if all optimizations are invalidated
3331 if (VMemInvalidated && !TrackSimpleDSOpt && !TrackDSFlushPoint)
3332 return Flags;
3333 }
3334
3335 // DS read vgpr def
3336 // Note: Unlike VMEM, we DON'T invalidate when VgprUse.contains(RegNo).
3337 // If USE comes before DEF, it's the prefetch pattern (use value from
3338 // previous iteration, read for next iteration). We should still flush
3339 // in preheader so iteration 1 doesn't need to wait inside the loop.
3340 // Only invalidate when DEF comes before USE (same-iteration consumption,
3341 // checked above when processing uses).
3342 if (IsDSRead || TrackDSFlushPoint) {
3343 for (const MachineOperand &Op : MI.all_defs()) {
3344 if (!TRI.isVectorRegister(MRI, Reg: Op.getReg()))
3345 continue;
3346 for (MCRegUnit RU : TRI.regunits(Reg: Op.getReg().asMCReg())) {
3347 // Check for overwrite of pending DS read (flush point) by any
3348 // instruction
3349 updateDSReadFlushTracking(RU);
3350 if (IsDSRead) {
3351 VgprDefDS.insert(V: RU);
3352 if (TrackDSFlushPoint)
3353 LastDSReadPositionMap[RU] = DSReadPosition;
3354 }
3355 }
3356 }
3357 }
3358 }
3359 }
3360
3361 // VMEM flush decision
3362 if (!VMemInvalidated && UsesVgprVMEMLoadedOutside &&
3363 ((!ST.hasVscnt() && HasVMemStore && !HasVMemLoad) ||
3364 (HasVMemLoad && ST.hasVmemWriteVgprInOrder())))
3365 Flags.FlushVmCnt = true;
3366
3367 // DS flush decision:
3368 // Simple DS Opt: flush if loop uses DS read values from outside
3369 // and either has no DS reads in the loop, or DS reads whose results
3370 // are not used in the loop.
3371 bool SimpleDSOpt = TrackSimpleDSOpt && UsesVgprDSReadOutside;
3372 // Prefetch with flush points: some DS reads used in same iteration,
3373 // but unflushed reads remain at backedge
3374 bool HasUnflushedDSReads = DSReadPosition > LastDSFlushPosition;
3375 bool DSFlushPointPrefetch =
3376 TrackDSFlushPoint && UsesVgprDSReadOutside && HasUnflushedDSReads;
3377
3378 if (SimpleDSOpt || DSFlushPointPrefetch)
3379 Flags.FlushDsCnt = true;
3380
3381 return Flags;
3382}
3383
3384bool SIInsertWaitcntsLegacy::runOnMachineFunction(MachineFunction &MF) {
3385 auto &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
3386 auto &PDT =
3387 getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
3388 AliasAnalysis *AA = nullptr;
3389 if (auto *AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
3390 AA = &AAR->getAAResults();
3391
3392 return SIInsertWaitcnts(MLI, PDT, AA, MF).run();
3393}
3394
3395PreservedAnalyses
3396SIInsertWaitcntsPass::run(MachineFunction &MF,
3397 MachineFunctionAnalysisManager &MFAM) {
3398 auto &MLI = MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
3399 auto &PDT = MFAM.getResult<MachinePostDominatorTreeAnalysis>(IR&: MF);
3400 auto *AA = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
3401 .getManager()
3402 .getCachedResult<AAManager>(IR&: MF.getFunction());
3403
3404 if (!SIInsertWaitcnts(MLI, PDT, AA, MF).run())
3405 return PreservedAnalyses::all();
3406
3407 return getMachineFunctionPassPreservedAnalyses()
3408 .preserveSet<CFGAnalyses>()
3409 .preserve<AAManager>();
3410}
3411
3412bool SIInsertWaitcnts::run() {
3413 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3414
3415 AMDGPU::IsaVersion IV = AMDGPU::getIsaVersion(GPU: ST.getCPU());
3416
3417 // Initialize hardware limits first, as they're needed by the generators.
3418 Limits = AMDGPU::HardwareLimits(IV);
3419
3420 if (ST.hasExtendedWaitCounts()) {
3421 IsExpertMode = ST.hasExpertSchedulingMode() &&
3422 (ExpertSchedulingModeFlag.getNumOccurrences()
3423 ? ExpertSchedulingModeFlag
3424 : MF.getFunction()
3425 .getFnAttribute(Kind: "amdgpu-expert-scheduling-mode")
3426 .getValueAsBool());
3427 MaxCounter = IsExpertMode ? AMDGPU::NUM_EXPERT_INST_CNTS
3428 : AMDGPU::NUM_EXTENDED_INST_CNTS;
3429 // Initialize WCG per MF. It contains state that depends on MF attributes.
3430 WCG = std::make_unique<WaitcntGeneratorGFX12Plus>(args&: MF, args&: MaxCounter, args&: Limits,
3431 args&: IsExpertMode);
3432 } else {
3433 MaxCounter = AMDGPU::NUM_NORMAL_INST_CNTS;
3434 // Initialize WCG per MF. It contains state that depends on MF attributes.
3435 WCG = std::make_unique<WaitcntGeneratorPreGFX12>(
3436 args&: MF, args: AMDGPU::NUM_NORMAL_INST_CNTS, args&: Limits);
3437 }
3438
3439 SmemAccessCounter = getCounterFromEvent(E: HWEvents::SMEM_ACCESS);
3440
3441 bool Modified = false;
3442
3443 MachineBasicBlock &EntryBB = MF.front();
3444
3445 if (!MFI->isEntryFunction() &&
3446 !MF.getFunction().hasFnAttribute(Kind: Attribute::Naked)) {
3447 // Wait for any outstanding memory operations that the input registers may
3448 // depend on. We can't track them and it's better to do the wait after the
3449 // costly call sequence.
3450
3451 // TODO: Could insert earlier and schedule more liberally with operations
3452 // that only use caller preserved registers.
3453 MachineBasicBlock::iterator I = EntryBB.begin();
3454 while (I != EntryBB.end() && I->isMetaInstruction())
3455 ++I;
3456
3457 if (ST.hasExtendedWaitCounts()) {
3458 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAIT_LOADCNT_DSCNT))
3459 .addImm(Val: 0);
3460 for (auto CT : inst_counter_types(MaxCounter: AMDGPU::NUM_EXTENDED_INST_CNTS)) {
3461 if (CT == AMDGPU::LOAD_CNT || CT == AMDGPU::DS_CNT ||
3462 CT == AMDGPU::STORE_CNT || CT == AMDGPU::X_CNT ||
3463 CT == AMDGPU::ASYNC_CNT || CT == AMDGPU::TENSOR_CNT)
3464 continue;
3465
3466 if (!ST.hasImageInsts() &&
3467 (CT == AMDGPU::EXP_CNT || CT == AMDGPU::SAMPLE_CNT ||
3468 CT == AMDGPU::BVH_CNT))
3469 continue;
3470
3471 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(),
3472 MCID: TII.get(Opcode: instrsForExtendedCounterTypes[CT]))
3473 .addImm(Val: 0);
3474 }
3475 if (IsExpertMode) {
3476 unsigned Enc = AMDGPU::DepCtr::encodeFieldVaVdst(VaVdst: 0, STI: ST);
3477 Enc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Enc, VmVsrc: 0);
3478 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAITCNT_DEPCTR))
3479 .addImm(Val: Enc);
3480 }
3481 } else {
3482 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_WAITCNT)).addImm(Val: 0);
3483 }
3484
3485 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(args: this);
3486 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
3487 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
3488
3489 Modified = true;
3490 }
3491
3492 // Keep iterating over the blocks in reverse post order, inserting and
3493 // updating s_waitcnt where needed, until a fix point is reached.
3494 for (auto *MBB : ReversePostOrderTraversal<MachineFunction *>(&MF))
3495 BlockInfos.try_emplace(Key: MBB);
3496
3497 std::unique_ptr<WaitcntBrackets> Brackets;
3498 bool Repeat;
3499 do {
3500 Repeat = false;
3501
3502 for (auto BII = BlockInfos.begin(), BIE = BlockInfos.end(); BII != BIE;
3503 ++BII) {
3504 MachineBasicBlock *MBB = BII->first;
3505 BlockInfo &BI = BII->second;
3506 if (!BI.Dirty)
3507 continue;
3508
3509 if (BI.Incoming) {
3510 if (!Brackets)
3511 Brackets = std::make_unique<WaitcntBrackets>(args&: *BI.Incoming);
3512 else
3513 *Brackets = *BI.Incoming;
3514 } else {
3515 if (!Brackets) {
3516 Brackets = std::make_unique<WaitcntBrackets>(args: this);
3517 } else {
3518 // Reinitialize in-place. N.B. do not do this by assigning from a
3519 // temporary because the WaitcntBrackets class is large and it could
3520 // cause this function to use an unreasonable amount of stack space.
3521 Brackets->~WaitcntBrackets();
3522 new (Brackets.get()) WaitcntBrackets(this);
3523 }
3524 }
3525
3526 if (ST.hasWaitXcnt())
3527 Modified |= removeRedundantSoftXcnts(Block&: *MBB);
3528 Modified |= insertWaitcntInBlock(MF, Block&: *MBB, ScoreBrackets&: *Brackets);
3529 BI.Dirty = false;
3530
3531 if (Brackets->hasPendingEvent()) {
3532 BlockInfo *MoveBracketsToSucc = nullptr;
3533 for (MachineBasicBlock *Succ : MBB->successors()) {
3534 auto *SuccBII = BlockInfos.find(Key: Succ);
3535 BlockInfo &SuccBI = SuccBII->second;
3536 if (!SuccBI.Incoming) {
3537 SuccBI.Dirty = true;
3538 if (SuccBII <= BII) {
3539 LLVM_DEBUG(dbgs() << "Repeat on backedge without merge\n");
3540 Repeat = true;
3541 }
3542 if (!MoveBracketsToSucc) {
3543 MoveBracketsToSucc = &SuccBI;
3544 } else {
3545 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(args&: *Brackets);
3546 }
3547 } else {
3548 LLVM_DEBUG({
3549 dbgs() << "Try to merge ";
3550 MBB->printName(dbgs());
3551 dbgs() << " into ";
3552 Succ->printName(dbgs());
3553 dbgs() << '\n';
3554 });
3555 if (SuccBI.Incoming->merge(Other: *Brackets)) {
3556 SuccBI.Dirty = true;
3557 if (SuccBII <= BII) {
3558 LLVM_DEBUG(dbgs() << "Repeat on backedge with merge\n");
3559 Repeat = true;
3560 }
3561 }
3562 }
3563 }
3564 if (MoveBracketsToSucc)
3565 MoveBracketsToSucc->Incoming = std::move(Brackets);
3566 }
3567 }
3568 } while (Repeat);
3569
3570 if (ST.hasScalarStores()) {
3571 SmallVector<MachineBasicBlock *, 4> EndPgmBlocks;
3572 bool HaveScalarStores = false;
3573
3574 for (MachineBasicBlock &MBB : MF) {
3575 for (MachineInstr &MI : MBB) {
3576 if (!HaveScalarStores && TII.isScalarStore(MI))
3577 HaveScalarStores = true;
3578
3579 if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
3580 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
3581 EndPgmBlocks.push_back(Elt: &MBB);
3582 }
3583 }
3584
3585 if (HaveScalarStores) {
3586 // If scalar writes are used, the cache must be flushed or else the next
3587 // wave to reuse the same scratch memory can be clobbered.
3588 //
3589 // Insert s_dcache_wb at wave termination points if there were any scalar
3590 // stores, and only if the cache hasn't already been flushed. This could
3591 // be improved by looking across blocks for flushes in postdominating
3592 // blocks from the stores but an explicitly requested flush is probably
3593 // very rare.
3594 for (MachineBasicBlock *MBB : EndPgmBlocks) {
3595 bool SeenDCacheWB = false;
3596
3597 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
3598 I != E; ++I) {
3599 if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
3600 SeenDCacheWB = true;
3601 else if (TII.isScalarStore(MI: *I))
3602 SeenDCacheWB = false;
3603
3604 // FIXME: It would be better to insert this before a waitcnt if any.
3605 if ((I->getOpcode() == AMDGPU::S_ENDPGM ||
3606 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
3607 !SeenDCacheWB) {
3608 Modified = true;
3609 BuildMI(BB&: *MBB, I, MIMD: I->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_DCACHE_WB));
3610 }
3611 }
3612 }
3613 }
3614 }
3615
3616 if (IsExpertMode) {
3617 // Enable expert scheduling on function entry. To satisfy ABI requirements
3618 // and to allow calls between function with different expert scheduling
3619 // settings, disable it around calls and before returns.
3620
3621 MachineBasicBlock::iterator I = EntryBB.begin();
3622 while (I != EntryBB.end() && I->isMetaInstruction())
3623 ++I;
3624 setSchedulingMode(MBB&: EntryBB, I, ExpertMode: true);
3625
3626 for (MachineInstr *MI : CallInsts) {
3627 MachineBasicBlock &MBB = *MI->getParent();
3628 setSchedulingMode(MBB, I: MI, ExpertMode: false);
3629 setSchedulingMode(MBB, I: std::next(x: MI->getIterator()), ExpertMode: true);
3630 }
3631
3632 for (MachineInstr *MI : ReturnInsts)
3633 setSchedulingMode(MBB&: *MI->getParent(), I: MI, ExpertMode: false);
3634
3635 Modified = true;
3636 }
3637
3638 // Deallocate the VGPRs before previously identified S_ENDPGM instructions.
3639 // This is done in different ways depending on how the VGPRs were allocated
3640 // (i.e. whether we're in dynamic VGPR mode or not).
3641 // Skip deallocation if kernel is waveslot limited vs VGPR limited. A short
3642 // waveslot limited kernel runs slower with the deallocation.
3643 if (!WCG->isOptNone() && MFI->isDynamicVGPREnabled()) {
3644 for (auto [MI, _] : EndPgmInsts) {
3645 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3646 MCID: TII.get(Opcode: AMDGPU::S_ALLOC_VGPR))
3647 .addImm(Val: 0);
3648 Modified = true;
3649 }
3650 } else if (!WCG->isOptNone() &&
3651 ST.getGeneration() >= AMDGPUSubtarget::GFX11 &&
3652 (MF.getFrameInfo().hasCalls() ||
3653 ST.getOccupancyWithNumVGPRs(
3654 VGPRs: TRI.getNumUsedPhysRegs(MRI, RC: AMDGPU::VGPR_32RegClass),
3655 /*IsDynamicVGPR=*/DynamicVGPRBlockSize: false) <
3656 AMDGPU::IsaInfo::getMaxWavesPerEU(STI: ST))) {
3657 for (auto [MI, Flag] : EndPgmInsts) {
3658 if (Flag) {
3659 if (ST.requiresNopBeforeDeallocVGPRs()) {
3660 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3661 MCID: TII.get(Opcode: AMDGPU::S_NOP))
3662 .addImm(Val: 0);
3663 }
3664 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
3665 MCID: TII.get(Opcode: AMDGPU::S_SENDMSG))
3666 .addImm(Val: AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus);
3667 Modified = true;
3668 }
3669 }
3670 }
3671
3672 if (MFI->isEntryFunction() && ST.hasRequiresInitialUnclausedVmem()) {
3673 // Hardware entrypoints must begin with a specific sequence:
3674 // GLOBAL_WB SCOPE:SCOPE_CU
3675 // V_NOP
3676 MachineBasicBlock::iterator I = EntryBB.begin();
3677 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::GLOBAL_WB))
3678 .addImm(Val: AMDGPU::CPol::SCOPE_CU);
3679 BuildMI(BB&: EntryBB, I, MIMD: DebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_NOP_e32));
3680 Modified = true;
3681 }
3682
3683 return Modified;
3684}
3685