1//===- SIMemoryLegalizer.cpp ----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Memory legalizer - implements memory model. More information can be
11/// found here:
12/// http://llvm.org/docs/AMDGPUUsage.html#memory-model
13//
14//===----------------------------------------------------------------------===//
15
16#include "AMDGPU.h"
17#include "AMDGPUMachineModuleInfo.h"
18#include "AMDGPUMemoryUtils.h"
19#include "GCNSubtarget.h"
20#include "llvm/ADT/BitmaskEnum.h"
21#include "llvm/ADT/StringExtras.h"
22#include "llvm/CodeGen/MachineBasicBlock.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/MachinePassManager.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
27#include "llvm/IR/PassManager.h"
28#include "llvm/Support/AtomicOrdering.h"
29#include "llvm/Support/Debug.h"
30#include "llvm/TargetParser/AMDGPUTargetParser.h"
31
32using namespace llvm;
33using namespace llvm::AMDGPU;
34
35#define DEBUG_TYPE "si-memory-legalizer"
36#define PASS_NAME "SI Memory Legalizer"
37
38namespace {
39
40LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
41
42/// Memory operation flags. Can be ORed together.
43enum class SIMemOp {
44 NONE = 0u,
45 LOAD = 1u << 0,
46 STORE = 1u << 1,
47 LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ STORE)
48};
49
50/// Position to insert a new instruction relative to an existing
51/// instruction.
52enum class Position {
53 BEFORE,
54 AFTER
55};
56
57/// The atomic synchronization scopes supported by the AMDGPU target.
58enum class SIAtomicScope {
59 NONE,
60 SINGLETHREAD,
61 WAVEFRONT,
62 WORKGROUP,
63 CLUSTER, // Promoted to AGENT on targets without workgroup clusters.
64 AGENT,
65 SYSTEM
66};
67
68#ifndef NDEBUG
69static StringRef toString(SIAtomicScope S) {
70 switch (S) {
71 case SIAtomicScope::NONE:
72 return "none";
73 case SIAtomicScope::SINGLETHREAD:
74 return "singlethread";
75 case SIAtomicScope::WAVEFRONT:
76 return "wavefront";
77 case SIAtomicScope::WORKGROUP:
78 return "workgroup";
79 case SIAtomicScope::CLUSTER:
80 return "cluster";
81 case SIAtomicScope::AGENT:
82 return "agent";
83 case SIAtomicScope::SYSTEM:
84 return "system";
85 }
86 llvm_unreachable("unknown atomic scope");
87}
88
89static raw_ostream &operator<<(raw_ostream &OS, SIAtomicAddrSpace AS) {
90 if (!AS) {
91 OS << "none";
92 return OS;
93 }
94 ListSeparator LS("|");
95 if (any(AS & SIAtomicAddrSpace::GLOBAL))
96 OS << LS << "global";
97 if (any(AS & SIAtomicAddrSpace::LDS))
98 OS << LS << "lds";
99 if (any(AS & SIAtomicAddrSpace::SCRATCH))
100 OS << LS << "scratch";
101 if (any(AS & SIAtomicAddrSpace::GDS))
102 OS << LS << "gds";
103 if (any(AS & SIAtomicAddrSpace::OTHER))
104 OS << LS << "other";
105 return OS;
106}
107#endif
108
109class SIMemOpInfo final {
110private:
111
112 friend class SIMemOpAccess;
113
114 AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
115 AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic;
116 SIAtomicScope Scope = SIAtomicScope::SYSTEM;
117 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE;
118 SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::NONE;
119 bool IsCrossAddressSpaceOrdering = false;
120 bool IsVolatile = false;
121 bool IsNonTemporal = false;
122 bool IsLastUse = false;
123 bool IsCooperative = false;
124 bool IsAVNone = false;
125 unsigned CFSBits = 0;
126
127 // TODO: Should we assume Cooperative=true if no MMO is present?
128 SIMemOpInfo(
129 const GCNSubtarget &ST,
130 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent,
131 SIAtomicScope Scope = SIAtomicScope::SYSTEM,
132 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::ATOMIC,
133 SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::ALL,
134 bool IsCrossAddressSpaceOrdering = true,
135 AtomicOrdering FailureOrdering = AtomicOrdering::SequentiallyConsistent,
136 bool IsVolatile = false, bool IsNonTemporal = false,
137 bool IsLastUse = false, bool IsCooperative = false, bool IsAVNone = false,
138 bool CanDemoteWorkgroupToWavefront = false, unsigned CFSBits = 0)
139 : Ordering(Ordering), FailureOrdering(FailureOrdering), Scope(Scope),
140 OrderingAddrSpace(OrderingAddrSpace), InstrAddrSpace(InstrAddrSpace),
141 IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering),
142 IsVolatile(IsVolatile), IsNonTemporal(IsNonTemporal),
143 IsLastUse(IsLastUse), IsCooperative(IsCooperative), IsAVNone(IsAVNone),
144 CFSBits(CFSBits) {
145
146 if (Ordering == AtomicOrdering::NotAtomic) {
147 assert(!IsCooperative && "Cannot be cooperative & non-atomic!");
148 assert(Scope == SIAtomicScope::NONE && !any(OrderingAddrSpace) &&
149 !IsCrossAddressSpaceOrdering &&
150 FailureOrdering == AtomicOrdering::NotAtomic);
151 return;
152 }
153
154 assert(Scope != SIAtomicScope::NONE &&
155 any(OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) &&
156 any(InstrAddrSpace & SIAtomicAddrSpace::ATOMIC));
157
158 // There is also no cross address space ordering if the ordering
159 // address space is the same as the instruction address space and
160 // only contains a single address space.
161 if ((OrderingAddrSpace == InstrAddrSpace) &&
162 isPowerOf2_32(Value: uint32_t(InstrAddrSpace)))
163 this->IsCrossAddressSpaceOrdering = false;
164
165 // Limit the scope to the maximum supported by the instruction's address
166 // spaces.
167 if (!(InstrAddrSpace & ~SIAtomicAddrSpace::SCRATCH)) {
168 this->Scope = std::min(a: Scope, b: SIAtomicScope::SINGLETHREAD);
169 } else if (!(InstrAddrSpace &
170 ~(SIAtomicAddrSpace::SCRATCH | SIAtomicAddrSpace::LDS))) {
171 this->Scope = std::min(a: Scope, b: SIAtomicScope::WORKGROUP);
172 } else if (!(InstrAddrSpace &
173 ~(SIAtomicAddrSpace::SCRATCH | SIAtomicAddrSpace::LDS |
174 SIAtomicAddrSpace::GDS))) {
175 this->Scope = std::min(a: Scope, b: SIAtomicScope::AGENT);
176 }
177
178 // On targets that have no concept of a workgroup cluster, use
179 // AGENT scope as a conservatively correct alternative.
180 if (this->Scope == SIAtomicScope::CLUSTER && !ST.hasClusters())
181 this->Scope = SIAtomicScope::AGENT;
182
183 if (CanDemoteWorkgroupToWavefront &&
184 this->Scope == SIAtomicScope::WORKGROUP)
185 this->Scope = SIAtomicScope::WAVEFRONT;
186 }
187
188public:
189 /// \returns Atomic synchronization scope of the machine instruction used to
190 /// create this SIMemOpInfo.
191 SIAtomicScope getScope() const {
192 return Scope;
193 }
194
195 /// \returns Ordering constraint of the machine instruction used to
196 /// create this SIMemOpInfo.
197 AtomicOrdering getOrdering() const {
198 return Ordering;
199 }
200
201 /// \returns Failure ordering constraint of the machine instruction used to
202 /// create this SIMemOpInfo.
203 AtomicOrdering getFailureOrdering() const {
204 return FailureOrdering;
205 }
206
207 /// \returns The address spaces be accessed by the machine
208 /// instruction used to create this SIMemOpInfo.
209 SIAtomicAddrSpace getInstrAddrSpace() const {
210 return InstrAddrSpace;
211 }
212
213 /// \returns The address spaces that must be ordered by the machine
214 /// instruction used to create this SIMemOpInfo.
215 SIAtomicAddrSpace getOrderingAddrSpace() const {
216 return OrderingAddrSpace;
217 }
218
219 /// \returns Return true iff memory ordering of operations on
220 /// different address spaces is required.
221 bool getIsCrossAddressSpaceOrdering() const {
222 return IsCrossAddressSpaceOrdering;
223 }
224
225 /// \returns True if memory access of the machine instruction used to
226 /// create this SIMemOpInfo is volatile, false otherwise.
227 bool isVolatile() const {
228 return IsVolatile;
229 }
230
231 /// \returns True if memory access of the machine instruction used to
232 /// create this SIMemOpInfo is nontemporal, false otherwise.
233 bool isNonTemporal() const {
234 return IsNonTemporal;
235 }
236
237 /// \returns True if memory access of the machine instruction used to
238 /// create this SIMemOpInfo is last use, false otherwise.
239 bool isLastUse() const { return IsLastUse; }
240
241 /// \returns True if this is a cooperative load or store atomic.
242 bool isCooperative() const { return IsCooperative; }
243
244 /// \returns True if MakeAvailable/MakeVisible should be suppressed.
245 bool isAVNone() const { return IsAVNone; }
246
247 /// \returns True if ordering constraint of the machine instruction used to
248 /// create this SIMemOpInfo is unordered or higher, false otherwise.
249 bool isAtomic() const {
250 return Ordering != AtomicOrdering::NotAtomic;
251 }
252
253 /// \returns cache fill size bits
254 unsigned getCFS() const { return CFSBits; }
255};
256
257class SIMemOpAccess final {
258private:
259 const AMDGPUMachineModuleInfo *MMI = nullptr;
260 const GCNSubtarget &ST;
261 const bool CanDemoteWorkgroupToWavefront;
262
263 /// Reports unsupported message \p Msg for \p MI to LLVM context.
264 void reportUnsupported(const MachineBasicBlock::iterator &MI,
265 const char *Msg) const;
266
267 /// Inspects the target synchronization scope \p SSID and determines
268 /// the SI atomic scope it corresponds to, the address spaces it
269 /// covers, and whether the memory ordering applies between address
270 /// spaces.
271 std::optional<std::tuple<SIAtomicScope, SIAtomicAddrSpace, bool>>
272 toSIAtomicScope(SyncScope::ID SSID, SIAtomicAddrSpace InstrAddrSpace) const;
273
274 /// \returns Info constructed from \p MI, which has at least machine memory
275 /// operand.
276 std::optional<SIMemOpInfo>
277 constructFromMIWithMMO(const MachineBasicBlock::iterator &MI) const;
278
279public:
280 /// Construct class to support accessing the machine memory operands
281 /// of instructions.
282 SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI, const GCNSubtarget &ST,
283 const MachineFunction &MF);
284
285 /// \returns Load info if \p MI is a load operation, "std::nullopt" otherwise.
286 std::optional<SIMemOpInfo>
287 getLoadInfo(const MachineBasicBlock::iterator &MI) const;
288
289 /// \returns Store info if \p MI is a store operation, "std::nullopt"
290 /// otherwise.
291 std::optional<SIMemOpInfo>
292 getStoreInfo(const MachineBasicBlock::iterator &MI) const;
293
294 /// \returns Atomic fence info if \p MI is an atomic fence operation,
295 /// "std::nullopt" otherwise.
296 std::optional<SIMemOpInfo>
297 getAtomicFenceInfo(const MachineBasicBlock::iterator &MI) const;
298
299 /// \returns Atomic cmpxchg/rmw info if \p MI is an atomic cmpxchg or
300 /// rmw operation, "std::nullopt" otherwise.
301 std::optional<SIMemOpInfo>
302 getAtomicCmpxchgOrRmwInfo(const MachineBasicBlock::iterator &MI) const;
303
304 /// \returns DMA to LDS info if \p MI is as a direct-to/from-LDS load/store,
305 /// along with an indication of whether this is a load or store. If it is not
306 /// a direct-to-LDS operation, returns std::nullopt.
307 std::optional<SIMemOpInfo>
308 getLDSDMAInfo(const MachineBasicBlock::iterator &MI) const;
309};
310
311class SICacheControl {
312protected:
313
314 /// AMDGPU subtarget info.
315 const GCNSubtarget &ST;
316
317 /// Instruction info.
318 const SIInstrInfo *TII = nullptr;
319
320 IsaVersion IV;
321
322 /// Whether to insert cache invalidating instructions.
323 bool InsertCacheInv;
324
325 /// Cached value of whether tgsplit is enabled for this function.
326 bool TgSplitEnabled;
327
328 SICacheControl(const GCNSubtarget &ST, bool TgSplit);
329
330 /// Sets CPol \p Bits to "true" if present in instruction \p MI.
331 /// \returns Returns true if \p MI is modified, false otherwise.
332 bool enableCPolBits(const MachineBasicBlock::iterator MI,
333 unsigned Bits) const;
334
335 /// Check if any atomic operation on AS can affect memory accessible via the
336 /// global address space.
337 bool canAffectGlobalAddrSpace(SIAtomicAddrSpace AS) const;
338
339public:
340 using CPol = AMDGPU::CPol::CPol;
341
342 /// Create a cache control for the subtarget \p ST.
343 static std::unique_ptr<SICacheControl> create(const GCNSubtarget &ST,
344 bool TgSplit);
345
346 /// Update \p MI memory load instruction to bypass any caches up to
347 /// the \p Scope memory scope for address spaces \p
348 /// AddrSpace. Return true iff the instruction was modified.
349 virtual bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
350 SIAtomicScope Scope,
351 SIAtomicAddrSpace AddrSpace) const = 0;
352
353 /// Update \p MI memory store instruction to bypass any caches up to
354 /// the \p Scope memory scope for address spaces \p
355 /// AddrSpace. Return true iff the instruction was modified.
356 virtual bool enableStoreCacheBypass(const MachineBasicBlock::iterator &MI,
357 SIAtomicScope Scope,
358 SIAtomicAddrSpace AddrSpace) const = 0;
359
360 /// Update \p MI memory read-modify-write instruction to bypass any caches up
361 /// to the \p Scope memory scope for address spaces \p AddrSpace. Return true
362 /// iff the instruction was modified.
363 virtual bool enableRMWCacheBypass(const MachineBasicBlock::iterator &MI,
364 SIAtomicScope Scope,
365 SIAtomicAddrSpace AddrSpace) const = 0;
366
367 /// Update \p MI memory instruction of kind \p Op associated with address
368 /// spaces \p AddrSpace to indicate it is volatile and/or
369 /// nontemporal/last-use. Return true iff the instruction was modified.
370 virtual bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI,
371 SIAtomicAddrSpace AddrSpace,
372 SIMemOp Op, bool IsVolatile,
373 bool IsNonTemporal,
374 bool IsLastUse = false) const = 0;
375
376 /// Add final touches to a `mayStore` instruction \p MI, which may be a
377 /// Store or RMW instruction.
378 /// FIXME: This takes a MI because iterators aren't handled properly. When
379 /// this is called, they often point to entirely different insts. Thus we back
380 /// up the inst early and pass it here instead.
381 virtual bool finalizeStore(MachineInstr &MI, bool Atomic) const {
382 return false;
383 };
384
385 /// Add final touches to a `mayLoad` instruction \p MI.
386 virtual bool finalizeLoad(MachineBasicBlock::iterator &MI) const {
387 return false;
388 }
389
390 /// Handle cooperative load/store atomics.
391 virtual bool handleCooperativeAtomic(MachineInstr &MI) const {
392 llvm_unreachable(
393 "cooperative atomics are not available on this architecture");
394 }
395
396 /// Inserts any necessary instructions at position \p Pos relative
397 /// to instruction \p MI to ensure memory instructions before \p Pos of kind
398 /// \p Op associated with address spaces \p AddrSpace have completed. Used
399 /// between memory instructions to enforce the order they become visible as
400 /// observed by other memory instructions executing in memory scope \p Scope.
401 /// \p IsCrossAddrSpaceOrdering indicates if the memory ordering is between
402 /// address spaces. If \p AtomicsOnly is true, only insert waits for counters
403 /// that are used by atomic instructions.
404 /// Returns true iff any instructions inserted.
405 virtual bool insertWait(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
406 SIAtomicAddrSpace AddrSpace, SIMemOp Op,
407 bool IsCrossAddrSpaceOrdering, Position Pos,
408 AtomicOrdering Order, bool AtomicsOnly) const = 0;
409
410 /// Inserts any necessary instructions at position \p Pos relative to
411 /// instruction \p MI to ensure any subsequent memory instructions of this
412 /// thread with address spaces \p AddrSpace will observe the previous memory
413 /// operations by any thread for memory scopes up to memory scope \p Scope .
414 /// Returns true iff any instructions inserted.
415 virtual bool insertAcquire(MachineBasicBlock::iterator &MI,
416 SIAtomicScope Scope,
417 SIAtomicAddrSpace AddrSpace,
418 Position Pos) const = 0;
419
420 /// Inserts any necessary writeback instructions at position \p Pos relative
421 /// to instruction \p MI to make previous memory operations by this thread
422 /// with address spaces \p AddrSpace available to other threads in memory
423 /// scope \p Scope. Does not insert waits; callers must call insertWait
424 /// separately. Returns true iff any instructions inserted.
425 virtual bool insertWriteback(MachineBasicBlock::iterator &MI,
426 SIAtomicScope Scope, SIAtomicAddrSpace AddrSpace,
427 Position Pos) const = 0;
428
429 /// Inserts writeback (unless \p IsAVNone) followed by an unconditional wait.
430 bool insertRelease(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
431 SIAtomicAddrSpace AddrSpace, bool IsCrossAddrSpaceOrdering,
432 Position Pos, bool IsAVNone) const {
433 bool Changed = !IsAVNone && insertWriteback(MI, Scope, AddrSpace, Pos);
434 Changed |= insertWait(MI, Scope, AddrSpace, Op: SIMemOp::LOAD | SIMemOp::STORE,
435 IsCrossAddrSpaceOrdering, Pos,
436 Order: AtomicOrdering::Release, /*AtomicsOnly=*/false);
437 return Changed;
438 }
439
440 /// Handle operations that are considered non-volatile.
441 /// See \ref isNonVolatileMemoryAccess
442 virtual bool handleNonVolatile(MachineInstr &MI) const { return false; }
443
444 virtual bool setCFS(const MachineBasicBlock::iterator &MI,
445 unsigned CFSBits = 0) const {
446 return false;
447 }
448
449 /// Virtual destructor to allow derivations to be deleted.
450 virtual ~SICacheControl() = default;
451};
452
453/// Generates code sequences for the memory model of all GFX targets below
454/// GFX10.
455class SIGfx6CacheControl final : public SICacheControl {
456public:
457 SIGfx6CacheControl(const GCNSubtarget &ST, bool TgSplit)
458 : SICacheControl(ST, TgSplit) {}
459
460 bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
461 SIAtomicScope Scope,
462 SIAtomicAddrSpace AddrSpace) const override;
463
464 bool enableStoreCacheBypass(const MachineBasicBlock::iterator &MI,
465 SIAtomicScope Scope,
466 SIAtomicAddrSpace AddrSpace) const override;
467
468 bool enableRMWCacheBypass(const MachineBasicBlock::iterator &MI,
469 SIAtomicScope Scope,
470 SIAtomicAddrSpace AddrSpace) const override;
471
472 bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI,
473 SIAtomicAddrSpace AddrSpace, SIMemOp Op,
474 bool IsVolatile, bool IsNonTemporal,
475 bool IsLastUse) const override;
476
477 bool insertWait(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
478 SIAtomicAddrSpace AddrSpace, SIMemOp Op,
479 bool IsCrossAddrSpaceOrdering, Position Pos,
480 AtomicOrdering Order, bool AtomicsOnly) const override;
481
482 bool insertAcquire(MachineBasicBlock::iterator &MI,
483 SIAtomicScope Scope,
484 SIAtomicAddrSpace AddrSpace,
485 Position Pos) const override;
486
487 bool insertWriteback(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
488 SIAtomicAddrSpace AddrSpace,
489 Position Pos) const override;
490};
491
492/// Generates code sequences for the memory model of GFX10/11.
493class SIGfx10CacheControl final : public SICacheControl {
494public:
495 SIGfx10CacheControl(const GCNSubtarget &ST, bool TgSplit)
496 : SICacheControl(ST, TgSplit) {}
497
498 bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
499 SIAtomicScope Scope,
500 SIAtomicAddrSpace AddrSpace) const override;
501
502 bool enableStoreCacheBypass(const MachineBasicBlock::iterator &MI,
503 SIAtomicScope Scope,
504 SIAtomicAddrSpace AddrSpace) const override {
505 return false;
506 }
507
508 bool enableRMWCacheBypass(const MachineBasicBlock::iterator &MI,
509 SIAtomicScope Scope,
510 SIAtomicAddrSpace AddrSpace) const override {
511 return false;
512 }
513
514 bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI,
515 SIAtomicAddrSpace AddrSpace, SIMemOp Op,
516 bool IsVolatile, bool IsNonTemporal,
517 bool IsLastUse) const override;
518
519 bool insertWait(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
520 SIAtomicAddrSpace AddrSpace, SIMemOp Op,
521 bool IsCrossAddrSpaceOrdering, Position Pos,
522 AtomicOrdering Order, bool AtomicsOnly) const override;
523
524 bool insertAcquire(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
525 SIAtomicAddrSpace AddrSpace, Position Pos) const override;
526
527 bool insertWriteback(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
528 SIAtomicAddrSpace AddrSpace,
529 Position Pos) const override {
530 return false;
531 }
532};
533
534class SIGfx12CacheControl final : public SICacheControl {
535protected:
536 // Sets TH policy to \p Value if CPol operand is present in instruction \p MI.
537 // \returns Returns true if \p MI is modified, false otherwise.
538 bool setTH(const MachineBasicBlock::iterator MI,
539 AMDGPU::CPol::CPol Value) const;
540
541 // Sets Scope policy to \p Value if CPol operand is present in instruction \p
542 // MI. \returns Returns true if \p MI is modified, false otherwise.
543 bool setScope(const MachineBasicBlock::iterator MI,
544 AMDGPU::CPol::CPol Value) const;
545
546 // Stores with system scope (SCOPE_SYS) need to wait for:
547 // - loads or atomics(returning) - wait for {LOAD|SAMPLE|BVH|KM}CNT==0
548 // - non-returning-atomics - wait for STORECNT==0
549 // TODO: SIInsertWaitcnts will not always be able to remove STORECNT waits
550 // since it does not distinguish atomics-with-return from regular stores.
551 // There is no need to wait if memory is cached (mtype != UC).
552 bool
553 insertWaitsBeforeSystemScopeStore(const MachineBasicBlock::iterator MI) const;
554
555 bool setAtomicScope(const MachineBasicBlock::iterator &MI,
556 SIAtomicScope Scope, SIAtomicAddrSpace AddrSpace) const;
557
558public:
559 SIGfx12CacheControl(const GCNSubtarget &ST, bool TgSplit)
560 : SICacheControl(ST, TgSplit) {
561 // GFX120x and GFX125x memory models greatly overlap, and in some cases
562 // the behavior is the same if assuming GFX120x in CU mode.
563 assert(!ST.hasGFX1250Insts() || ST.hasGFX13Insts() || ST.isCuModeEnabled());
564 }
565
566 bool insertWait(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
567 SIAtomicAddrSpace AddrSpace, SIMemOp Op,
568 bool IsCrossAddrSpaceOrdering, Position Pos,
569 AtomicOrdering Order, bool AtomicsOnly) const override;
570
571 bool insertAcquire(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
572 SIAtomicAddrSpace AddrSpace, Position Pos) const override;
573
574 bool enableVolatileAndOrNonTemporal(MachineBasicBlock::iterator &MI,
575 SIAtomicAddrSpace AddrSpace, SIMemOp Op,
576 bool IsVolatile, bool IsNonTemporal,
577 bool IsLastUse) const override;
578
579 bool finalizeStore(MachineInstr &MI, bool Atomic) const override;
580
581 bool finalizeLoad(MachineBasicBlock::iterator &MI) const override;
582
583 bool handleCooperativeAtomic(MachineInstr &MI) const override;
584
585 bool insertWriteback(MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
586 SIAtomicAddrSpace AddrSpace,
587 Position Pos) const override;
588
589 bool enableLoadCacheBypass(const MachineBasicBlock::iterator &MI,
590 SIAtomicScope Scope,
591 SIAtomicAddrSpace AddrSpace) const override {
592 return setAtomicScope(MI, Scope, AddrSpace);
593 }
594
595 bool enableStoreCacheBypass(const MachineBasicBlock::iterator &MI,
596 SIAtomicScope Scope,
597 SIAtomicAddrSpace AddrSpace) const override {
598 return setAtomicScope(MI, Scope, AddrSpace);
599 }
600
601 bool enableRMWCacheBypass(const MachineBasicBlock::iterator &MI,
602 SIAtomicScope Scope,
603 SIAtomicAddrSpace AddrSpace) const override {
604 return setAtomicScope(MI, Scope, AddrSpace);
605 }
606
607 bool handleNonVolatile(MachineInstr &MI) const override;
608
609 bool setCFS(const MachineBasicBlock::iterator &MI,
610 unsigned CFSBits) const override;
611};
612
613class SIMemoryLegalizer final {
614private:
615 const MachineModuleInfo &MMI;
616 /// Cache Control.
617 std::unique_ptr<SICacheControl> CC = nullptr;
618
619 /// List of atomic pseudo instructions.
620 std::list<MachineBasicBlock::iterator> AtomicPseudoMIs;
621
622 /// Return true iff instruction \p MI is a atomic instruction that
623 /// returns a result.
624 bool isAtomicRet(const MachineInstr &MI) const {
625 return SIInstrInfo::isAtomicRet(MI);
626 }
627
628 /// Removes all processed atomic pseudo instructions from the current
629 /// function. Returns true if current function is modified, false otherwise.
630 bool removeAtomicPseudoMIs();
631
632 /// Expands load operation \p MI. Returns true if instructions are
633 /// added/deleted or \p MI is modified, false otherwise.
634 bool expandLoad(const SIMemOpInfo &MOI,
635 MachineBasicBlock::iterator &MI);
636 /// Expands store operation \p MI. Returns true if instructions are
637 /// added/deleted or \p MI is modified, false otherwise.
638 bool expandStore(const SIMemOpInfo &MOI,
639 MachineBasicBlock::iterator &MI);
640 /// Expands atomic fence operation \p MI. Returns true if
641 /// instructions are added/deleted or \p MI is modified, false otherwise.
642 bool expandAtomicFence(const SIMemOpInfo &MOI,
643 MachineBasicBlock::iterator &MI);
644 /// Expands atomic cmpxchg or rmw operation \p MI. Returns true if
645 /// instructions are added/deleted or \p MI is modified, false otherwise.
646 bool expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI,
647 MachineBasicBlock::iterator &MI);
648 /// Expands LDS DMA operation \p MI. Returns true if instructions are
649 /// added/deleted or \p MI is modified, false otherwise.
650 bool expandLDSDMA(const SIMemOpInfo &MOI, MachineBasicBlock::iterator &MI);
651
652public:
653 SIMemoryLegalizer(const MachineModuleInfo &MMI) : MMI(MMI) {};
654 bool run(MachineFunction &MF);
655};
656
657class SIMemoryLegalizerLegacy final : public MachineFunctionPass {
658public:
659 static char ID;
660
661 SIMemoryLegalizerLegacy() : MachineFunctionPass(ID) {}
662
663 void getAnalysisUsage(AnalysisUsage &AU) const override {
664 AU.setPreservesCFG();
665 MachineFunctionPass::getAnalysisUsage(AU);
666 }
667
668 StringRef getPassName() const override {
669 return PASS_NAME;
670 }
671
672 bool runOnMachineFunction(MachineFunction &MF) override;
673};
674
675/// Reads \p MI's MMRAs to parse the "amdgpu-synchronize-as" MMRA.
676/// If this tag isn't present, or if it has no meaningful values, returns
677/// \p ATOMIC, otherwise returns the address spaces specified by the MD.
678/// Issues a diagnostic if an unexpected suffix is observed for the MMRA tag.
679static SIAtomicAddrSpace getSynchronizeAddrSpaceMD(const MachineInstr &MI) {
680 return getSynchronizeAddrSpace(MMRA: MMRAMetadata(MI.getMMRAMetadata()),
681 DiagFn: &MI.getMF()->getFunction(), DL: MI.getDebugLoc());
682}
683
684static void diagnoseUnknownAVMetadata(const MachineInstr &MI,
685 StringRef Suffix) {
686 const MachineFunction *MF = MI.getMF();
687 const Function &Fn = MF->getFunction();
688 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
689 Fn, Twine("unknown amdgcn-av metadata '") + Suffix + Twine('\''),
690 MI.getDebugLoc(), DS_Warning));
691}
692
693static bool hasAVNoneMMRA(const MachineInstr &MI) {
694 MMRAMetadata MMRA(MI.getMMRAMetadata());
695 if (!MMRA)
696 return false;
697 bool TagFound = false;
698 for (const auto &[Prefix, Suffix] : MMRA) {
699 if (Prefix != "amdgcn-av")
700 continue;
701 if (Suffix == "none")
702 TagFound = true;
703 else
704 diagnoseUnknownAVMetadata(MI, Suffix);
705 }
706 return TagFound;
707}
708
709} // end anonymous namespace
710
711void SIMemOpAccess::reportUnsupported(const MachineBasicBlock::iterator &MI,
712 const char *Msg) const {
713 const Function &Func = MI->getMF()->getFunction();
714 Func.getContext().diagnose(
715 DI: DiagnosticInfoUnsupported(Func, Msg, MI->getDebugLoc()));
716}
717
718std::optional<std::tuple<SIAtomicScope, SIAtomicAddrSpace, bool>>
719SIMemOpAccess::toSIAtomicScope(SyncScope::ID SSID,
720 SIAtomicAddrSpace InstrAddrSpace) const {
721 if (SSID == SyncScope::System)
722 return std::tuple(SIAtomicScope::SYSTEM, SIAtomicAddrSpace::ATOMIC, true);
723 if (SSID == MMI->getAgentSSID())
724 return std::tuple(SIAtomicScope::AGENT, SIAtomicAddrSpace::ATOMIC, true);
725 if (SSID == MMI->getClusterSSID())
726 return std::tuple(SIAtomicScope::CLUSTER, SIAtomicAddrSpace::ATOMIC, true);
727 if (SSID == MMI->getWorkgroupSSID())
728 return std::tuple(SIAtomicScope::WORKGROUP, SIAtomicAddrSpace::ATOMIC,
729 true);
730 if (SSID == MMI->getWavefrontSSID())
731 return std::tuple(SIAtomicScope::WAVEFRONT, SIAtomicAddrSpace::ATOMIC,
732 true);
733 if (SSID == SyncScope::SingleThread)
734 return std::tuple(SIAtomicScope::SINGLETHREAD, SIAtomicAddrSpace::ATOMIC,
735 true);
736 if (SSID == MMI->getSystemOneAddressSpaceSSID())
737 return std::tuple(SIAtomicScope::SYSTEM,
738 SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false);
739 if (SSID == MMI->getAgentOneAddressSpaceSSID())
740 return std::tuple(SIAtomicScope::AGENT,
741 SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false);
742 if (SSID == MMI->getClusterOneAddressSpaceSSID())
743 return std::tuple(SIAtomicScope::CLUSTER,
744 SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false);
745 if (SSID == MMI->getWorkgroupOneAddressSpaceSSID())
746 return std::tuple(SIAtomicScope::WORKGROUP,
747 SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false);
748 if (SSID == MMI->getWavefrontOneAddressSpaceSSID())
749 return std::tuple(SIAtomicScope::WAVEFRONT,
750 SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false);
751 if (SSID == MMI->getSingleThreadOneAddressSpaceSSID())
752 return std::tuple(SIAtomicScope::SINGLETHREAD,
753 SIAtomicAddrSpace::ATOMIC & InstrAddrSpace, false);
754 return std::nullopt;
755}
756
757// TODO: Consider moving single-wave workgroup->wavefront scope relaxation to an
758// IR pass (and extending it to other scoped operations), so middle-end
759// optimizations see wavefront scope earlier.
760SIMemOpAccess::SIMemOpAccess(const AMDGPUMachineModuleInfo &MMI_,
761 const GCNSubtarget &ST, const MachineFunction &MF)
762 : MMI(&MMI_), ST(ST),
763 CanDemoteWorkgroupToWavefront(
764 ST.isSingleWavefrontWorkgroup(F: MF.getFunction()) &&
765 MF.getFunction().hasFnAttribute(Kind: "amdgpu-no-lds-dma")) {}
766
767std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
768 const MachineBasicBlock::iterator &MI) const {
769 assert(MI->getNumMemOperands() > 0);
770
771 std::optional<SyncScope::ID> MergedSSID;
772 AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
773 AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic;
774 SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::NONE;
775 bool IsNonTemporal = true;
776 bool IsVolatile = false;
777 bool IsLastUse = false;
778 bool IsCooperative = false;
779 unsigned CFSBits = 0;
780
781 // Validator should check whether or not MMOs cover the entire set of
782 // locations accessed by the memory instruction.
783 for (const auto &MMO : MI->memoperands()) {
784 IsNonTemporal &= MMO->isNonTemporal();
785 IsVolatile |= MMO->isVolatile();
786 IsLastUse |= MMO->getFlags() & MOLastUse;
787 IsCooperative |= MMO->getFlags() & MOCooperative;
788 CFSBits = (MMO->getFlags() / MOCFSB0) & AMDGPU::CPol::CFS_MASK;
789 InstrAddrSpace |= toSIAtomicAddrSpace(AS: MMO->getPointerInfo().getAddrSpace());
790 AtomicOrdering OpOrdering = MMO->getSuccessOrdering();
791 if (OpOrdering != AtomicOrdering::NotAtomic) {
792 // Merge the accumulated scope with the new one to get the smallest scope
793 // inclusive of both.
794 SyncScope::ID CurSSID = MergedSSID.value_or(u: MMO->getSyncScopeID());
795 const auto &Merged =
796 MMI->getMergedSyncScopeID(A: CurSSID, B: MMO->getSyncScopeID());
797 if (!Merged) {
798 reportUnsupported(MI, Msg: "unsupported atomic synchronization scope");
799 return std::nullopt;
800 }
801 MergedSSID = *Merged;
802 Ordering = getMergedAtomicOrdering(AO: Ordering, Other: OpOrdering);
803 assert(MMO->getFailureOrdering() != AtomicOrdering::Release &&
804 MMO->getFailureOrdering() != AtomicOrdering::AcquireRelease);
805 FailureOrdering =
806 getMergedAtomicOrdering(AO: FailureOrdering, Other: MMO->getFailureOrdering());
807 }
808 }
809 SyncScope::ID SSID = MergedSSID.value_or(u: SyncScope::SingleThread);
810
811 // FIXME: The MMO of buffer atomic instructions does not always have an atomic
812 // ordering. We only need to handle VBUFFER atomics on GFX12+ so we can fix it
813 // here, but the lowering should really be cleaned up at some point.
814 if ((ST.getGeneration() >= GCNSubtarget::GFX12) && SIInstrInfo::isBUF(MI: *MI) &&
815 SIInstrInfo::isAtomic(MI: *MI) && Ordering == AtomicOrdering::NotAtomic)
816 Ordering = AtomicOrdering::Monotonic;
817
818 SIAtomicScope Scope = SIAtomicScope::NONE;
819 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE;
820 bool IsCrossAddressSpaceOrdering = false;
821 if (Ordering != AtomicOrdering::NotAtomic) {
822 auto ScopeOrNone = toSIAtomicScope(SSID, InstrAddrSpace);
823 if (!ScopeOrNone) {
824 reportUnsupported(MI, Msg: "unsupported atomic synchronization scope");
825 return std::nullopt;
826 }
827 std::tie(args&: Scope, args&: OrderingAddrSpace, args&: IsCrossAddressSpaceOrdering) =
828 *ScopeOrNone;
829 if (!OrderingAddrSpace ||
830 ((OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) !=
831 OrderingAddrSpace) ||
832 !(InstrAddrSpace & SIAtomicAddrSpace::ATOMIC)) {
833 reportUnsupported(MI, Msg: "Unsupported atomic address space");
834 return std::nullopt;
835 }
836 }
837 return SIMemOpInfo(ST, Ordering, Scope, OrderingAddrSpace, InstrAddrSpace,
838 IsCrossAddressSpaceOrdering, FailureOrdering, IsVolatile,
839 IsNonTemporal, IsLastUse, IsCooperative,
840 hasAVNoneMMRA(MI: *MI), CanDemoteWorkgroupToWavefront,
841 CFSBits);
842}
843
844std::optional<SIMemOpInfo>
845SIMemOpAccess::getLoadInfo(const MachineBasicBlock::iterator &MI) const {
846 assert(SIInstrFlags::isMaybeAtomic(*MI));
847
848 if (!(MI->mayLoad() && !MI->mayStore()))
849 return std::nullopt;
850
851 // Be conservative if there are no memory operands.
852 if (MI->getNumMemOperands() == 0)
853 return SIMemOpInfo(ST);
854
855 return constructFromMIWithMMO(MI);
856}
857
858std::optional<SIMemOpInfo>
859SIMemOpAccess::getStoreInfo(const MachineBasicBlock::iterator &MI) const {
860 assert(SIInstrFlags::isMaybeAtomic(*MI));
861
862 if (!(!MI->mayLoad() && MI->mayStore()))
863 return std::nullopt;
864
865 // Be conservative if there are no memory operands.
866 if (MI->getNumMemOperands() == 0)
867 return SIMemOpInfo(ST);
868
869 return constructFromMIWithMMO(MI);
870}
871
872std::optional<SIMemOpInfo>
873SIMemOpAccess::getAtomicFenceInfo(const MachineBasicBlock::iterator &MI) const {
874 assert(SIInstrFlags::isMaybeAtomic(*MI));
875
876 if (MI->getOpcode() != AMDGPU::ATOMIC_FENCE)
877 return std::nullopt;
878
879 AtomicOrdering Ordering =
880 static_cast<AtomicOrdering>(MI->getOperand(i: 0).getImm());
881
882 SyncScope::ID SSID = static_cast<SyncScope::ID>(MI->getOperand(i: 1).getImm());
883 auto ScopeOrNone = toSIAtomicScope(SSID, InstrAddrSpace: SIAtomicAddrSpace::ATOMIC);
884 if (!ScopeOrNone) {
885 reportUnsupported(MI, Msg: "unsupported atomic synchronization scope");
886 return std::nullopt;
887 }
888
889 SIAtomicScope Scope = SIAtomicScope::NONE;
890 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE;
891 bool IsCrossAddressSpaceOrdering = false;
892 std::tie(args&: Scope, args&: OrderingAddrSpace, args&: IsCrossAddressSpaceOrdering) =
893 *ScopeOrNone;
894
895 if (OrderingAddrSpace != SIAtomicAddrSpace::ATOMIC) {
896 // We currently expect refineOrderingAS to be the only place that
897 // can refine the AS ordered by the fence.
898 // If that changes, we need to review the semantics of that function
899 // in case it needs to preserve certain address spaces.
900 reportUnsupported(MI, Msg: "Unsupported atomic address space");
901 return std::nullopt;
902 }
903
904 OrderingAddrSpace = getSynchronizeAddrSpaceMD(MI: *MI);
905
906 return SIMemOpInfo(ST, Ordering, Scope, OrderingAddrSpace,
907 SIAtomicAddrSpace::ATOMIC, IsCrossAddressSpaceOrdering,
908 AtomicOrdering::NotAtomic, false, false, false, false,
909 hasAVNoneMMRA(MI: *MI), CanDemoteWorkgroupToWavefront);
910}
911
912std::optional<SIMemOpInfo> SIMemOpAccess::getAtomicCmpxchgOrRmwInfo(
913 const MachineBasicBlock::iterator &MI) const {
914 assert(SIInstrFlags::isMaybeAtomic(*MI));
915
916 if (!(MI->mayLoad() && MI->mayStore()))
917 return std::nullopt;
918
919 // Be conservative if there are no memory operands.
920 if (MI->getNumMemOperands() == 0)
921 return SIMemOpInfo(ST);
922
923 return constructFromMIWithMMO(MI);
924}
925
926std::optional<SIMemOpInfo>
927SIMemOpAccess::getLDSDMAInfo(const MachineBasicBlock::iterator &MI) const {
928 assert(SIInstrFlags::isMaybeAtomic(*MI));
929
930 if (!SIInstrInfo::isLDSDMA(MI: *MI))
931 return std::nullopt;
932
933 return constructFromMIWithMMO(MI);
934}
935
936/// \returns true if \p MI has one or more MMO, and all of them are fit for
937/// being marked as non-volatile. This means that either they are accessing the
938/// constant address space, are accessing a known invariant memory location, or
939/// that they are marked with the non-volatile metadata/MMO flag.
940static bool isNonVolatileMemoryAccess(const MachineInstr &MI) {
941 if (MI.getNumMemOperands() == 0)
942 return false;
943 return all_of(Range: MI.memoperands(), P: [&](const MachineMemOperand *MMO) {
944 return MMO->getFlags() & (MOThreadPrivate | MachineMemOperand::MOInvariant);
945 });
946}
947
948SICacheControl::SICacheControl(const GCNSubtarget &ST, bool TgSplit) : ST(ST) {
949 TII = ST.getInstrInfo();
950 IV = getIsaVersion(GPU: ST.getCPU());
951 InsertCacheInv = !ST.getCLOpts().skip_cache_invalidations;
952 TgSplitEnabled = TgSplit;
953}
954
955bool SICacheControl::enableCPolBits(const MachineBasicBlock::iterator MI,
956 unsigned Bits) const {
957 MachineOperand *CPol = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::cpol);
958 if (!CPol)
959 return false;
960
961 CPol->setImm(CPol->getImm() | Bits);
962 return true;
963}
964
965bool SICacheControl::canAffectGlobalAddrSpace(SIAtomicAddrSpace AS) const {
966 assert((!ST.hasGloballyAddressableScratch() ||
967 any(AS & SIAtomicAddrSpace::GLOBAL) ||
968 !(AS & SIAtomicAddrSpace::SCRATCH)) &&
969 "scratch instructions should already be replaced by flat "
970 "instructions if GloballyAddressableScratch is enabled");
971 return any(Val: AS & SIAtomicAddrSpace::GLOBAL);
972}
973
974/* static */
975std::unique_ptr<SICacheControl> SICacheControl::create(const GCNSubtarget &ST,
976 bool TgSplit) {
977 GCNSubtarget::Generation Generation = ST.getGeneration();
978 if (Generation < AMDGPUSubtarget::GFX10)
979 return std::make_unique<SIGfx6CacheControl>(args: ST, args&: TgSplit);
980 if (Generation < AMDGPUSubtarget::GFX12)
981 return std::make_unique<SIGfx10CacheControl>(args: ST, args&: TgSplit);
982 return std::make_unique<SIGfx12CacheControl>(args: ST, args&: TgSplit);
983}
984
985bool SIGfx6CacheControl::enableLoadCacheBypass(
986 const MachineBasicBlock::iterator &MI,
987 SIAtomicScope Scope,
988 SIAtomicAddrSpace AddrSpace) const {
989 assert(MI->mayLoad() && !MI->mayStore());
990
991 if (!canAffectGlobalAddrSpace(AS: AddrSpace)) {
992 /// The scratch address space does not need the global memory caches
993 /// to be bypassed as all memory operations by the same thread are
994 /// sequentially consistent, and no other thread can access scratch
995 /// memory.
996
997 /// Other address spaces do not have a cache.
998 return false;
999 }
1000
1001 bool Changed = false;
1002 switch (Scope) {
1003 case SIAtomicScope::SYSTEM:
1004 if (ST.hasGFX940Insts()) {
1005 // Set SC bits to indicate system scope.
1006 Changed |= enableCPolBits(MI, Bits: CPol::SC0 | CPol::SC1);
1007 break;
1008 }
1009 [[fallthrough]];
1010 case SIAtomicScope::AGENT:
1011 if (ST.hasGFX940Insts()) {
1012 // Set SC bits to indicate agent scope.
1013 Changed |= enableCPolBits(MI, Bits: CPol::SC1);
1014 } else {
1015 // Set L1 cache policy to MISS_EVICT.
1016 // Note: there is no L2 cache bypass policy at the ISA level.
1017 Changed |= enableCPolBits(MI, Bits: CPol::GLC);
1018 }
1019 break;
1020 case SIAtomicScope::WORKGROUP:
1021 if (ST.hasGFX940Insts()) {
1022 // In threadgroup split mode the waves of a work-group can be executing
1023 // on different CUs. Therefore need to bypass the L1 which is per CU.
1024 // Otherwise in non-threadgroup split mode all waves of a work-group are
1025 // on the same CU, and so the L1 does not need to be bypassed. Setting
1026 // SC bits to indicate work-group scope will do this automatically.
1027 Changed |= enableCPolBits(MI, Bits: CPol::SC0);
1028 } else if (ST.hasGFX90AInsts()) {
1029 // In threadgroup split mode the waves of a work-group can be executing
1030 // on different CUs. Therefore need to bypass the L1 which is per CU.
1031 // Otherwise in non-threadgroup split mode all waves of a work-group are
1032 // on the same CU, and so the L1 does not need to be bypassed.
1033 if (TgSplitEnabled)
1034 Changed |= enableCPolBits(MI, Bits: CPol::GLC);
1035 }
1036 break;
1037 case SIAtomicScope::WAVEFRONT:
1038 case SIAtomicScope::SINGLETHREAD:
1039 // No cache to bypass.
1040 break;
1041 default:
1042 llvm_unreachable("Unsupported synchronization scope");
1043 }
1044
1045 return Changed;
1046}
1047
1048bool SIGfx6CacheControl::enableStoreCacheBypass(
1049 const MachineBasicBlock::iterator &MI,
1050 SIAtomicScope Scope,
1051 SIAtomicAddrSpace AddrSpace) const {
1052 assert(!MI->mayLoad() && MI->mayStore());
1053 bool Changed = false;
1054
1055 /// For targets other than GFX940, the L1 cache is write through so does not
1056 /// need to be bypassed. There is no bypass control for the L2 cache at the
1057 /// isa level.
1058
1059 if (ST.hasGFX940Insts() && canAffectGlobalAddrSpace(AS: AddrSpace)) {
1060 switch (Scope) {
1061 case SIAtomicScope::SYSTEM:
1062 // Set SC bits to indicate system scope.
1063 Changed |= enableCPolBits(MI, Bits: CPol::SC0 | CPol::SC1);
1064 break;
1065 case SIAtomicScope::AGENT:
1066 // Set SC bits to indicate agent scope.
1067 Changed |= enableCPolBits(MI, Bits: CPol::SC1);
1068 break;
1069 case SIAtomicScope::WORKGROUP:
1070 // Set SC bits to indicate workgroup scope.
1071 Changed |= enableCPolBits(MI, Bits: CPol::SC0);
1072 break;
1073 case SIAtomicScope::WAVEFRONT:
1074 case SIAtomicScope::SINGLETHREAD:
1075 // Leave SC bits unset to indicate wavefront scope.
1076 break;
1077 default:
1078 llvm_unreachable("Unsupported synchronization scope");
1079 }
1080
1081 /// The scratch address space does not need the global memory caches
1082 /// to be bypassed as all memory operations by the same thread are
1083 /// sequentially consistent, and no other thread can access scratch
1084 /// memory.
1085
1086 /// Other address spaces do not have a cache.
1087 }
1088
1089 return Changed;
1090}
1091
1092bool SIGfx6CacheControl::enableRMWCacheBypass(
1093 const MachineBasicBlock::iterator &MI,
1094 SIAtomicScope Scope,
1095 SIAtomicAddrSpace AddrSpace) const {
1096 assert(MI->mayLoad() && MI->mayStore());
1097 bool Changed = false;
1098
1099 /// For targets other than GFX940, do not set GLC for RMW atomic operations as
1100 /// L0/L1 cache is automatically bypassed, and the GLC bit is instead used to
1101 /// indicate if they are return or no-return. Note: there is no L2 cache
1102 /// coherent bypass control at the ISA level.
1103 /// For GFX90A+, RMW atomics implicitly bypass the L1 cache.
1104
1105 if (ST.hasGFX940Insts() && canAffectGlobalAddrSpace(AS: AddrSpace)) {
1106 switch (Scope) {
1107 case SIAtomicScope::SYSTEM:
1108 // Set SC1 bit to indicate system scope.
1109 Changed |= enableCPolBits(MI, Bits: CPol::SC1);
1110 break;
1111 case SIAtomicScope::AGENT:
1112 case SIAtomicScope::WORKGROUP:
1113 case SIAtomicScope::WAVEFRONT:
1114 case SIAtomicScope::SINGLETHREAD:
1115 // RMW atomic operations implicitly bypass the L1 cache and only use SC1
1116 // to indicate system or agent scope. The SC0 bit is used to indicate if
1117 // they are return or no-return. Leave SC1 bit unset to indicate agent
1118 // scope.
1119 break;
1120 default:
1121 llvm_unreachable("Unsupported synchronization scope");
1122 }
1123 }
1124
1125 return Changed;
1126}
1127
1128bool SIGfx6CacheControl::enableVolatileAndOrNonTemporal(
1129 MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op,
1130 bool IsVolatile, bool IsNonTemporal, bool IsLastUse = false) const {
1131 // Only handle load and store, not atomic read-modify-write instructions. The
1132 // latter use glc to indicate if the atomic returns a result and so must not
1133 // be used for cache control.
1134 assert((MI->mayLoad() ^ MI->mayStore()) || SIInstrInfo::isLDSDMA(*MI));
1135
1136 // Only update load and store, not LLVM IR atomic read-modify-write
1137 // instructions. The latter are always marked as volatile so cannot sensibly
1138 // handle it as do not want to pessimize all atomics. Also they do not support
1139 // the nontemporal attribute.
1140 assert(Op == SIMemOp::LOAD || Op == SIMemOp::STORE);
1141
1142 bool Changed = false;
1143
1144 if (IsVolatile) {
1145 if (ST.hasGFX940Insts()) {
1146 // Set SC bits to indicate system scope.
1147 Changed |= enableCPolBits(MI, Bits: CPol::SC0 | CPol::SC1);
1148 } else if (Op == SIMemOp::LOAD) {
1149 // Set L1 cache policy to be MISS_EVICT for load instructions
1150 // and MISS_LRU for store instructions.
1151 // Note: there is no L2 cache bypass policy at the ISA level.
1152 Changed |= enableCPolBits(MI, Bits: CPol::GLC);
1153 }
1154
1155 // Ensure operation has completed at system scope to cause all volatile
1156 // operations to be visible outside the program in a global order. Do not
1157 // request cross address space as only the global address space can be
1158 // observable outside the program, so no need to cause a waitcnt for LDS
1159 // address space operations.
1160 Changed |= insertWait(MI, Scope: SIAtomicScope::SYSTEM, AddrSpace, Op, IsCrossAddrSpaceOrdering: false,
1161 Pos: Position::AFTER, Order: AtomicOrdering::Unordered,
1162 /*AtomicsOnly=*/false);
1163
1164 return Changed;
1165 }
1166
1167 if (IsNonTemporal) {
1168 if (ST.hasGFX940Insts()) {
1169 Changed |= enableCPolBits(MI, Bits: CPol::NT);
1170 } else {
1171 // Setting both GLC and SLC configures L1 cache policy to MISS_EVICT
1172 // for both loads and stores, and the L2 cache policy to STREAM.
1173 Changed |= enableCPolBits(MI, Bits: CPol::SLC | CPol::GLC);
1174 }
1175 return Changed;
1176 }
1177
1178 return Changed;
1179}
1180
1181bool SIGfx6CacheControl::insertWait(MachineBasicBlock::iterator &MI,
1182 SIAtomicScope Scope,
1183 SIAtomicAddrSpace AddrSpace, SIMemOp Op,
1184 bool IsCrossAddrSpaceOrdering, Position Pos,
1185 AtomicOrdering Order,
1186 bool AtomicsOnly) const {
1187 bool Changed = false;
1188
1189 MachineBasicBlock &MBB = *MI->getParent();
1190 const DebugLoc &DL = MI->getDebugLoc();
1191
1192 if (Pos == Position::AFTER)
1193 ++MI;
1194
1195 // GFX90A+
1196 if (ST.hasGFX90AInsts() && TgSplitEnabled) {
1197 // In threadgroup split mode the waves of a work-group can be executing on
1198 // different CUs. Therefore need to wait for global or GDS memory operations
1199 // to complete to ensure they are visible to waves in the other CUs.
1200 // Otherwise in non-threadgroup split mode all waves of a work-group are on
1201 // the same CU, so no need to wait for global memory as all waves in the
1202 // work-group access the same the L1, nor wait for GDS as access are ordered
1203 // on a CU.
1204 if (any(Val: AddrSpace &
1205 (SIAtomicAddrSpace::GLOBAL | SIAtomicAddrSpace::SCRATCH |
1206 SIAtomicAddrSpace::GDS)) &&
1207 (Scope == SIAtomicScope::WORKGROUP)) {
1208 // Same as <GFX90A at AGENT scope;
1209 Scope = SIAtomicScope::AGENT;
1210 }
1211 // In threadgroup split mode LDS cannot be allocated so no need to wait for
1212 // LDS memory operations.
1213 AddrSpace &= ~SIAtomicAddrSpace::LDS;
1214 }
1215
1216 bool VMCnt = false;
1217 bool LGKMCnt = false;
1218
1219 if (any(Val: AddrSpace &
1220 (SIAtomicAddrSpace::GLOBAL | SIAtomicAddrSpace::SCRATCH))) {
1221 switch (Scope) {
1222 case SIAtomicScope::SYSTEM:
1223 case SIAtomicScope::AGENT:
1224 VMCnt |= true;
1225 break;
1226 case SIAtomicScope::WORKGROUP:
1227 case SIAtomicScope::WAVEFRONT:
1228 case SIAtomicScope::SINGLETHREAD:
1229 // The L1 cache keeps all memory operations in order for
1230 // wavefronts in the same work-group.
1231 break;
1232 default:
1233 llvm_unreachable("Unsupported synchronization scope");
1234 }
1235 }
1236
1237 if (any(Val: AddrSpace & SIAtomicAddrSpace::LDS)) {
1238 switch (Scope) {
1239 case SIAtomicScope::SYSTEM:
1240 case SIAtomicScope::AGENT:
1241 case SIAtomicScope::WORKGROUP:
1242 // If no cross address space ordering then an "S_WAITCNT lgkmcnt(0)" is
1243 // not needed as LDS operations for all waves are executed in a total
1244 // global ordering as observed by all waves. Required if also
1245 // synchronizing with global/GDS memory as LDS operations could be
1246 // reordered with respect to later global/GDS memory operations of the
1247 // same wave.
1248 LGKMCnt |= IsCrossAddrSpaceOrdering;
1249 break;
1250 case SIAtomicScope::WAVEFRONT:
1251 case SIAtomicScope::SINGLETHREAD:
1252 // The LDS keeps all memory operations in order for
1253 // the same wavefront.
1254 break;
1255 default:
1256 llvm_unreachable("Unsupported synchronization scope");
1257 }
1258 }
1259
1260 if (any(Val: AddrSpace & SIAtomicAddrSpace::GDS)) {
1261 switch (Scope) {
1262 case SIAtomicScope::SYSTEM:
1263 case SIAtomicScope::AGENT:
1264 // If no cross address space ordering then an GDS "S_WAITCNT lgkmcnt(0)"
1265 // is not needed as GDS operations for all waves are executed in a total
1266 // global ordering as observed by all waves. Required if also
1267 // synchronizing with global/LDS memory as GDS operations could be
1268 // reordered with respect to later global/LDS memory operations of the
1269 // same wave.
1270 LGKMCnt |= IsCrossAddrSpaceOrdering;
1271 break;
1272 case SIAtomicScope::WORKGROUP:
1273 case SIAtomicScope::WAVEFRONT:
1274 case SIAtomicScope::SINGLETHREAD:
1275 // The GDS keeps all memory operations in order for
1276 // the same work-group.
1277 break;
1278 default:
1279 llvm_unreachable("Unsupported synchronization scope");
1280 }
1281 }
1282
1283 if (VMCnt || LGKMCnt) {
1284 unsigned WaitCntImmediate =
1285 AMDGPU::encodeWaitcnt(Version: IV,
1286 Vmcnt: VMCnt ? 0 : getVmcntBitMask(Version: IV),
1287 Expcnt: getExpcntBitMask(Version: IV),
1288 Lgkmcnt: LGKMCnt ? 0 : getLgkmcntBitMask(Version: IV));
1289 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_soft))
1290 .addImm(Val: WaitCntImmediate);
1291 Changed = true;
1292 }
1293
1294 // On architectures that support direct loads to LDS, emit an unknown waitcnt
1295 // at workgroup-scoped release operations that specify the LDS address space.
1296 // SIInsertWaitcnts will later replace this with a vmcnt().
1297 if (ST.hasVMemToLDSLoad() && isReleaseOrStronger(AO: Order) &&
1298 Scope == SIAtomicScope::WORKGROUP &&
1299 any(Val: AddrSpace & SIAtomicAddrSpace::LDS)) {
1300 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_lds_direct));
1301 Changed = true;
1302 }
1303
1304 if (Pos == Position::AFTER)
1305 --MI;
1306
1307 return Changed;
1308}
1309
1310static bool canUseBUFFER_WBINVL1_VOL(const GCNSubtarget &ST) {
1311 if (ST.getGeneration() <= AMDGPUSubtarget::SOUTHERN_ISLANDS)
1312 return false;
1313 return !ST.isAmdPalOS() && !ST.isMesa3DOS();
1314}
1315
1316bool SIGfx6CacheControl::insertAcquire(MachineBasicBlock::iterator &MI,
1317 SIAtomicScope Scope,
1318 SIAtomicAddrSpace AddrSpace,
1319 Position Pos) const {
1320 if (!InsertCacheInv)
1321 return false;
1322
1323 bool Changed = false;
1324
1325 MachineBasicBlock &MBB = *MI->getParent();
1326 const DebugLoc &DL = MI->getDebugLoc();
1327
1328 if (Pos == Position::AFTER)
1329 ++MI;
1330
1331 const unsigned InvalidateL1 = canUseBUFFER_WBINVL1_VOL(ST)
1332 ? AMDGPU::BUFFER_WBINVL1_VOL
1333 : AMDGPU::BUFFER_WBINVL1;
1334
1335 if (canAffectGlobalAddrSpace(AS: AddrSpace)) {
1336 switch (Scope) {
1337 case SIAtomicScope::SYSTEM:
1338 if (ST.hasBufferInvInst()) {
1339 // Ensures that following loads will not see stale remote VMEM data or
1340 // stale local VMEM data with MTYPE NC. Local VMEM data with MTYPE RW
1341 // and CC will never be stale due to the local memory probes.
1342 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::BUFFER_INV))
1343 // Set SC bits to indicate system scope.
1344 .addImm(Val: AMDGPU::CPol::SC0 | AMDGPU::CPol::SC1);
1345 // Inserting a "S_WAITCNT vmcnt(0)" after is not required because the
1346 // hardware does not reorder memory operations by the same wave with
1347 // respect to a preceding "BUFFER_INV". The invalidate is guaranteed to
1348 // remove any cache lines of earlier writes by the same wave and ensures
1349 // later reads by the same wave will refetch the cache lines.
1350 Changed = true;
1351 break;
1352 }
1353
1354 if (ST.hasGFX90AInsts()) {
1355 // Ensures that following loads will not see stale remote VMEM data or
1356 // stale local VMEM data with MTYPE NC. Local VMEM data with MTYPE RW
1357 // and CC will never be stale due to the local memory probes.
1358 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::BUFFER_INVL2));
1359 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: InvalidateL1));
1360 // Inserting a "S_WAITCNT vmcnt(0)" after is not required because the
1361 // hardware does not reorder memory operations by the same wave with
1362 // respect to a preceding "BUFFER_INVL2". The invalidate is guaranteed
1363 // to remove any cache lines of earlier writes by the same wave and
1364 // ensures later reads by the same wave will refetch the cache lines.
1365 Changed = true;
1366 break;
1367 }
1368 [[fallthrough]];
1369 case SIAtomicScope::AGENT:
1370 if (ST.hasBufferInvInst()) {
1371 // Ensures that following loads will not see stale remote date or local
1372 // MTYPE NC global data. Local MTYPE RW and CC memory will never be
1373 // stale due to the memory probes.
1374 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::BUFFER_INV))
1375 // Set SC bits to indicate agent scope.
1376 .addImm(Val: AMDGPU::CPol::SC1);
1377 // Inserting "S_WAITCNT vmcnt(0)" is not required because the hardware
1378 // does not reorder memory operations with respect to preceeding buffer
1379 // invalidate. The invalidate is guaranteed to remove any cache lines of
1380 // earlier writes and ensures later writes will refetch the cache lines.
1381 } else
1382 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: InvalidateL1));
1383 Changed = true;
1384 break;
1385 case SIAtomicScope::WORKGROUP:
1386 if (TgSplitEnabled) {
1387 if (ST.hasBufferInvInst()) {
1388 // In threadgroup split mode the waves of a work-group can be
1389 // executing on different CUs. Therefore need to invalidate the L1
1390 // which is per CU. Otherwise in non-threadgroup split mode all waves
1391 // of a work-group are on the same CU, and so the L1 does not need to
1392 // be invalidated.
1393
1394 // Ensures L1 is invalidated if in threadgroup split mode. In
1395 // non-threadgroup split mode it is a NOP, but no point generating it
1396 // in that case if know not in that mode.
1397 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::BUFFER_INV))
1398 // Set SC bits to indicate work-group scope.
1399 .addImm(Val: AMDGPU::CPol::SC0);
1400 // Inserting "S_WAITCNT vmcnt(0)" is not required because the hardware
1401 // does not reorder memory operations with respect to preceeding
1402 // buffer invalidate. The invalidate is guaranteed to remove any cache
1403 // lines of earlier writes and ensures later writes will refetch the
1404 // cache lines.
1405 Changed = true;
1406 } else if (ST.hasGFX90AInsts()) {
1407 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: InvalidateL1));
1408 Changed = true;
1409 }
1410 }
1411 break;
1412 case SIAtomicScope::WAVEFRONT:
1413 case SIAtomicScope::SINGLETHREAD:
1414 // For GFX940, we could generate "BUFFER_INV" but it would do nothing as
1415 // there are no caches to invalidate. All other targets have no cache to
1416 // invalidate.
1417 break;
1418 default:
1419 llvm_unreachable("Unsupported synchronization scope");
1420 }
1421 }
1422
1423 /// The scratch address space does not need the global memory cache
1424 /// to be flushed as all memory operations by the same thread are
1425 /// sequentially consistent, and no other thread can access scratch
1426 /// memory.
1427
1428 /// Other address spaces do not have a cache.
1429
1430 if (Pos == Position::AFTER)
1431 --MI;
1432
1433 return Changed;
1434}
1435
1436bool SIGfx6CacheControl::insertWriteback(MachineBasicBlock::iterator &MI,
1437 SIAtomicScope Scope,
1438 SIAtomicAddrSpace AddrSpace,
1439 Position Pos) const {
1440 if (!ST.hasGFX90AInsts())
1441 return false;
1442
1443 bool Changed = false;
1444 MachineBasicBlock &MBB = *MI->getParent();
1445 const DebugLoc &DL = MI->getDebugLoc();
1446
1447 if (Pos == Position::AFTER)
1448 ++MI;
1449
1450 if (canAffectGlobalAddrSpace(AS: AddrSpace)) {
1451 switch (Scope) {
1452 case SIAtomicScope::SYSTEM:
1453 // Inserting a "S_WAITCNT vmcnt(0)" before is not required because the
1454 // hardware does not reorder memory operations by the same wave with
1455 // respect to a following "BUFFER_WBL2". The "BUFFER_WBL2" is guaranteed
1456 // to initiate writeback of any dirty cache lines of earlier writes by
1457 // the same wave. A "S_WAITCNT vmcnt(0)" is needed after to ensure the
1458 // writeback has completed.
1459 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::BUFFER_WBL2))
1460 // Set SC bits to indicate system scope.
1461 .addImm(Val: AMDGPU::CPol::SC0 | AMDGPU::CPol::SC1);
1462 Changed = true;
1463 break;
1464 case SIAtomicScope::AGENT:
1465 if (ST.hasGFX940Insts()) {
1466 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::BUFFER_WBL2))
1467 // Set SC bits to indicate agent scope.
1468 .addImm(Val: AMDGPU::CPol::SC1);
1469 Changed = true;
1470 }
1471 break;
1472 case SIAtomicScope::WORKGROUP:
1473 case SIAtomicScope::WAVEFRONT:
1474 case SIAtomicScope::SINGLETHREAD:
1475 // For GFX940, do not generate "BUFFER_WBL2" as there are no caches it
1476 // would writeback, and would require an otherwise unnecessary
1477 // "S_WAITCNT vmcnt(0)".
1478 break;
1479 default:
1480 llvm_unreachable("Unsupported synchronization scope");
1481 }
1482 }
1483
1484 if (Pos == Position::AFTER)
1485 --MI;
1486
1487 return Changed;
1488}
1489
1490bool SIGfx10CacheControl::enableLoadCacheBypass(
1491 const MachineBasicBlock::iterator &MI, SIAtomicScope Scope,
1492 SIAtomicAddrSpace AddrSpace) const {
1493 assert(MI->mayLoad() && !MI->mayStore());
1494 bool Changed = false;
1495
1496 if (canAffectGlobalAddrSpace(AS: AddrSpace)) {
1497 switch (Scope) {
1498 case SIAtomicScope::SYSTEM:
1499 case SIAtomicScope::AGENT:
1500 // Set the L0 and L1 cache policies to MISS_EVICT.
1501 // Note: there is no L2 cache coherent bypass control at the ISA level.
1502 // For GFX10, set GLC+DLC, for GFX11, only set GLC.
1503 Changed |=
1504 enableCPolBits(MI, Bits: CPol::GLC | (AMDGPU::isGFX10(STI: ST) ? CPol::DLC : 0));
1505 break;
1506 case SIAtomicScope::WORKGROUP:
1507 // In WGP mode the waves of a work-group can be executing on either CU of
1508 // the WGP. Therefore need to bypass the L0 which is per CU. Otherwise in
1509 // CU mode all waves of a work-group are on the same CU, and so the L0
1510 // does not need to be bypassed.
1511 if (!ST.isCuModeEnabled())
1512 Changed |= enableCPolBits(MI, Bits: CPol::GLC);
1513 break;
1514 case SIAtomicScope::WAVEFRONT:
1515 case SIAtomicScope::SINGLETHREAD:
1516 // No cache to bypass.
1517 break;
1518 default:
1519 llvm_unreachable("Unsupported synchronization scope");
1520 }
1521 }
1522
1523 /// The scratch address space does not need the global memory caches
1524 /// to be bypassed as all memory operations by the same thread are
1525 /// sequentially consistent, and no other thread can access scratch
1526 /// memory.
1527
1528 /// Other address spaces do not have a cache.
1529
1530 return Changed;
1531}
1532
1533bool SIGfx10CacheControl::enableVolatileAndOrNonTemporal(
1534 MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op,
1535 bool IsVolatile, bool IsNonTemporal, bool IsLastUse = false) const {
1536
1537 // Only handle load and store, not atomic read-modify-write instructions. The
1538 // latter use glc to indicate if the atomic returns a result and so must not
1539 // be used for cache control.
1540 assert((MI->mayLoad() ^ MI->mayStore()) || SIInstrInfo::isLDSDMA(*MI));
1541
1542 // Only update load and store, not LLVM IR atomic read-modify-write
1543 // instructions. The latter are always marked as volatile so cannot sensibly
1544 // handle it as do not want to pessimize all atomics. Also they do not support
1545 // the nontemporal attribute.
1546 assert(Op == SIMemOp::LOAD || Op == SIMemOp::STORE);
1547
1548 bool Changed = false;
1549
1550 if (IsVolatile) {
1551 // Set L0 and L1 cache policy to be MISS_EVICT for load instructions
1552 // and MISS_LRU for store instructions.
1553 // Note: there is no L2 cache coherent bypass control at the ISA level.
1554 if (Op == SIMemOp::LOAD) {
1555 Changed |= enableCPolBits(MI, Bits: CPol::GLC | CPol::DLC);
1556 }
1557
1558 // GFX11: Set MALL NOALLOC for both load and store instructions.
1559 if (AMDGPU::isGFX11(STI: ST))
1560 Changed |= enableCPolBits(MI, Bits: CPol::DLC);
1561
1562 // Ensure operation has completed at system scope to cause all volatile
1563 // operations to be visible outside the program in a global order. Do not
1564 // request cross address space as only the global address space can be
1565 // observable outside the program, so no need to cause a waitcnt for LDS
1566 // address space operations.
1567 Changed |= insertWait(MI, Scope: SIAtomicScope::SYSTEM, AddrSpace, Op, IsCrossAddrSpaceOrdering: false,
1568 Pos: Position::AFTER, Order: AtomicOrdering::Unordered,
1569 /*AtomicsOnly=*/false);
1570 return Changed;
1571 }
1572
1573 if (IsNonTemporal) {
1574 // For loads setting SLC configures L0 and L1 cache policy to HIT_EVICT
1575 // and L2 cache policy to STREAM.
1576 // For stores setting both GLC and SLC configures L0 and L1 cache policy
1577 // to MISS_EVICT and the L2 cache policy to STREAM.
1578 if (Op == SIMemOp::STORE)
1579 Changed |= enableCPolBits(MI, Bits: CPol::GLC);
1580 Changed |= enableCPolBits(MI, Bits: CPol::SLC);
1581
1582 // GFX11: Set MALL NOALLOC for both load and store instructions.
1583 if (AMDGPU::isGFX11(STI: ST))
1584 Changed |= enableCPolBits(MI, Bits: CPol::DLC);
1585
1586 return Changed;
1587 }
1588
1589 return Changed;
1590}
1591
1592bool SIGfx10CacheControl::insertWait(MachineBasicBlock::iterator &MI,
1593 SIAtomicScope Scope,
1594 SIAtomicAddrSpace AddrSpace, SIMemOp Op,
1595 bool IsCrossAddrSpaceOrdering,
1596 Position Pos, AtomicOrdering Order,
1597 bool AtomicsOnly) const {
1598 bool Changed = false;
1599
1600 MachineBasicBlock &MBB = *MI->getParent();
1601 const DebugLoc &DL = MI->getDebugLoc();
1602
1603 if (Pos == Position::AFTER)
1604 ++MI;
1605
1606 bool VMCnt = false;
1607 bool VSCnt = false;
1608 bool LGKMCnt = false;
1609
1610 if (any(Val: AddrSpace &
1611 (SIAtomicAddrSpace::GLOBAL | SIAtomicAddrSpace::SCRATCH))) {
1612 switch (Scope) {
1613 case SIAtomicScope::SYSTEM:
1614 case SIAtomicScope::AGENT:
1615 if (any(Val: Op & SIMemOp::LOAD))
1616 VMCnt |= true;
1617 if (any(Val: Op & SIMemOp::STORE))
1618 VSCnt |= true;
1619 break;
1620 case SIAtomicScope::WORKGROUP:
1621 // In WGP mode the waves of a work-group can be executing on either CU of
1622 // the WGP. Therefore need to wait for operations to complete to ensure
1623 // they are visible to waves in the other CU as the L0 is per CU.
1624 // Otherwise in CU mode and all waves of a work-group are on the same CU
1625 // which shares the same L0. Note that we still need to wait when
1626 // performing a release in this mode to respect the transitivity of
1627 // happens-before, e.g. other waves of the workgroup must be able to
1628 // release the memory from another wave at a wider scope.
1629 if (!ST.isCuModeEnabled() || isReleaseOrStronger(AO: Order)) {
1630 if (any(Val: Op & SIMemOp::LOAD))
1631 VMCnt |= true;
1632 if (any(Val: Op & SIMemOp::STORE))
1633 VSCnt |= true;
1634 }
1635 break;
1636 case SIAtomicScope::WAVEFRONT:
1637 case SIAtomicScope::SINGLETHREAD:
1638 // The L0 cache keeps all memory operations in order for
1639 // work-items in the same wavefront.
1640 break;
1641 default:
1642 llvm_unreachable("Unsupported synchronization scope");
1643 }
1644 }
1645
1646 if (any(Val: AddrSpace & SIAtomicAddrSpace::LDS)) {
1647 switch (Scope) {
1648 case SIAtomicScope::SYSTEM:
1649 case SIAtomicScope::AGENT:
1650 case SIAtomicScope::WORKGROUP:
1651 // If no cross address space ordering then an "S_WAITCNT lgkmcnt(0)" is
1652 // not needed as LDS operations for all waves are executed in a total
1653 // global ordering as observed by all waves. Required if also
1654 // synchronizing with global/GDS memory as LDS operations could be
1655 // reordered with respect to later global/GDS memory operations of the
1656 // same wave.
1657 LGKMCnt |= IsCrossAddrSpaceOrdering;
1658 break;
1659 case SIAtomicScope::WAVEFRONT:
1660 case SIAtomicScope::SINGLETHREAD:
1661 // The LDS keeps all memory operations in order for
1662 // the same wavefront.
1663 break;
1664 default:
1665 llvm_unreachable("Unsupported synchronization scope");
1666 }
1667 }
1668
1669 if (any(Val: AddrSpace & SIAtomicAddrSpace::GDS)) {
1670 switch (Scope) {
1671 case SIAtomicScope::SYSTEM:
1672 case SIAtomicScope::AGENT:
1673 // If no cross address space ordering then an GDS "S_WAITCNT lgkmcnt(0)"
1674 // is not needed as GDS operations for all waves are executed in a total
1675 // global ordering as observed by all waves. Required if also
1676 // synchronizing with global/LDS memory as GDS operations could be
1677 // reordered with respect to later global/LDS memory operations of the
1678 // same wave.
1679 LGKMCnt |= IsCrossAddrSpaceOrdering;
1680 break;
1681 case SIAtomicScope::WORKGROUP:
1682 case SIAtomicScope::WAVEFRONT:
1683 case SIAtomicScope::SINGLETHREAD:
1684 // The GDS keeps all memory operations in order for
1685 // the same work-group.
1686 break;
1687 default:
1688 llvm_unreachable("Unsupported synchronization scope");
1689 }
1690 }
1691
1692 if (VMCnt || LGKMCnt) {
1693 unsigned WaitCntImmediate =
1694 AMDGPU::encodeWaitcnt(Version: IV,
1695 Vmcnt: VMCnt ? 0 : getVmcntBitMask(Version: IV),
1696 Expcnt: getExpcntBitMask(Version: IV),
1697 Lgkmcnt: LGKMCnt ? 0 : getLgkmcntBitMask(Version: IV));
1698 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_soft))
1699 .addImm(Val: WaitCntImmediate);
1700 Changed = true;
1701 }
1702
1703 // On architectures that support direct loads to LDS, emit an unknown waitcnt
1704 // at workgroup-scoped release operations that specify the LDS address space.
1705 // SIInsertWaitcnts will later replace this with a vmcnt().
1706 if (ST.hasVMemToLDSLoad() && isReleaseOrStronger(AO: Order) &&
1707 Scope == SIAtomicScope::WORKGROUP &&
1708 any(Val: AddrSpace & SIAtomicAddrSpace::LDS)) {
1709 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_lds_direct));
1710 Changed = true;
1711 }
1712
1713 if (VSCnt) {
1714 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_WAITCNT_VSCNT_soft))
1715 .addReg(RegNo: AMDGPU::SGPR_NULL, Flags: RegState::Undef)
1716 .addImm(Val: 0);
1717 Changed = true;
1718 }
1719
1720 if (Pos == Position::AFTER)
1721 --MI;
1722
1723 return Changed;
1724}
1725
1726bool SIGfx10CacheControl::insertAcquire(MachineBasicBlock::iterator &MI,
1727 SIAtomicScope Scope,
1728 SIAtomicAddrSpace AddrSpace,
1729 Position Pos) const {
1730 if (!InsertCacheInv)
1731 return false;
1732
1733 bool Changed = false;
1734
1735 MachineBasicBlock &MBB = *MI->getParent();
1736 const DebugLoc &DL = MI->getDebugLoc();
1737
1738 if (Pos == Position::AFTER)
1739 ++MI;
1740
1741 if (canAffectGlobalAddrSpace(AS: AddrSpace)) {
1742 switch (Scope) {
1743 case SIAtomicScope::SYSTEM:
1744 case SIAtomicScope::AGENT:
1745 // The order of invalidates matter here. We must invalidate "outer in"
1746 // so L1 -> L0 to avoid L0 pulling in stale data from L1 when it is
1747 // invalidated.
1748 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::BUFFER_GL1_INV));
1749 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::BUFFER_GL0_INV));
1750 Changed = true;
1751 break;
1752 case SIAtomicScope::WORKGROUP:
1753 // In WGP mode the waves of a work-group can be executing on either CU of
1754 // the WGP. Therefore need to invalidate the L0 which is per CU. Otherwise
1755 // in CU mode and all waves of a work-group are on the same CU, and so the
1756 // L0 does not need to be invalidated.
1757 if (!ST.isCuModeEnabled()) {
1758 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::BUFFER_GL0_INV));
1759 Changed = true;
1760 }
1761 break;
1762 case SIAtomicScope::WAVEFRONT:
1763 case SIAtomicScope::SINGLETHREAD:
1764 // No cache to invalidate.
1765 break;
1766 default:
1767 llvm_unreachable("Unsupported synchronization scope");
1768 }
1769 }
1770
1771 /// The scratch address space does not need the global memory cache
1772 /// to be flushed as all memory operations by the same thread are
1773 /// sequentially consistent, and no other thread can access scratch
1774 /// memory.
1775
1776 /// Other address spaces do not have a cache.
1777
1778 if (Pos == Position::AFTER)
1779 --MI;
1780
1781 return Changed;
1782}
1783
1784bool SIGfx12CacheControl::setTH(const MachineBasicBlock::iterator MI,
1785 AMDGPU::CPol::CPol Value) const {
1786 MachineOperand *CPol = TII->getNamedOperand(MI&: *MI, OperandName: OpName::cpol);
1787 if (!CPol)
1788 return false;
1789
1790 uint64_t NewTH = Value & AMDGPU::CPol::TH;
1791 if ((CPol->getImm() & AMDGPU::CPol::TH) != NewTH) {
1792 CPol->setImm((CPol->getImm() & ~AMDGPU::CPol::TH) | NewTH);
1793 return true;
1794 }
1795
1796 return false;
1797}
1798
1799bool SIGfx12CacheControl::setScope(const MachineBasicBlock::iterator MI,
1800 AMDGPU::CPol::CPol Value) const {
1801 MachineOperand *CPol = TII->getNamedOperand(MI&: *MI, OperandName: OpName::cpol);
1802 if (!CPol)
1803 return false;
1804
1805 uint64_t NewScope = Value & AMDGPU::CPol::SCOPE;
1806 if ((CPol->getImm() & AMDGPU::CPol::SCOPE) != NewScope) {
1807 CPol->setImm((CPol->getImm() & ~AMDGPU::CPol::SCOPE) | NewScope);
1808 return true;
1809 }
1810
1811 return false;
1812}
1813
1814bool SIGfx12CacheControl::insertWaitsBeforeSystemScopeStore(
1815 const MachineBasicBlock::iterator MI) const {
1816 // TODO: implement flag for frontend to give us a hint not to insert waits.
1817
1818 MachineBasicBlock &MBB = *MI->getParent();
1819 const DebugLoc &DL = MI->getDebugLoc();
1820
1821 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: S_WAIT_LOADCNT_soft)).addImm(Val: 0);
1822 if (ST.hasImageInsts()) {
1823 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: S_WAIT_SAMPLECNT_soft)).addImm(Val: 0);
1824 if (ST.hasAnyBVHInsts())
1825 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: S_WAIT_BVHCNT_soft)).addImm(Val: 0);
1826 }
1827 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: S_WAIT_KMCNT_soft)).addImm(Val: 0);
1828 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: S_WAIT_STORECNT_soft)).addImm(Val: 0);
1829
1830 return true;
1831}
1832
1833bool SIGfx12CacheControl::insertWait(MachineBasicBlock::iterator &MI,
1834 SIAtomicScope Scope,
1835 SIAtomicAddrSpace AddrSpace, SIMemOp Op,
1836 bool IsCrossAddrSpaceOrdering,
1837 Position Pos, AtomicOrdering Order,
1838 bool AtomicsOnly) const {
1839 bool Changed = false;
1840
1841 MachineBasicBlock &MBB = *MI->getParent();
1842 const DebugLoc &DL = MI->getDebugLoc();
1843
1844 bool LOADCnt = false;
1845 bool DSCnt = false;
1846 bool STORECnt = false;
1847
1848 if (Pos == Position::AFTER)
1849 ++MI;
1850
1851 if (any(Val: AddrSpace &
1852 (SIAtomicAddrSpace::GLOBAL | SIAtomicAddrSpace::SCRATCH))) {
1853 switch (Scope) {
1854 case SIAtomicScope::SYSTEM:
1855 case SIAtomicScope::AGENT:
1856 case SIAtomicScope::CLUSTER:
1857 if (any(Val: Op & SIMemOp::LOAD))
1858 LOADCnt |= true;
1859 if (any(Val: Op & SIMemOp::STORE))
1860 STORECnt |= true;
1861 break;
1862 case SIAtomicScope::WORKGROUP:
1863 // GFX12.0:
1864 // In WGP mode the waves of a work-group can be executing on either CU
1865 // of the WGP. Therefore need to wait for operations to complete to
1866 // ensure they are visible to waves in the other CU as the L0 is per CU.
1867 //
1868 // Otherwise in CU mode and all waves of a work-group are on the same CU
1869 // which shares the same L0. Note that we still need to wait when
1870 // performing a release in this mode to respect the transitivity of
1871 // happens-before, e.g. other waves of the workgroup must be able to
1872 // release the memory from another wave at a wider scope.
1873 //
1874 // GFX12.5:
1875 // CU$ has two ports. To ensure operations are visible at the workgroup
1876 // level, we need to ensure all operations in this port have completed
1877 // so the other SIMDs in the WG can see them. There is no ordering
1878 // guarantee between the ports.
1879 if (!ST.isCuModeEnabled() || ST.hasGFX1250Insts() ||
1880 isReleaseOrStronger(AO: Order)) {
1881 if (any(Val: Op & SIMemOp::LOAD))
1882 LOADCnt |= true;
1883 if (any(Val: Op & SIMemOp::STORE))
1884 STORECnt |= true;
1885 }
1886 break;
1887 case SIAtomicScope::WAVEFRONT:
1888 case SIAtomicScope::SINGLETHREAD:
1889 // The L0 cache keeps all memory operations in order for
1890 // work-items in the same wavefront.
1891 break;
1892 default:
1893 llvm_unreachable("Unsupported synchronization scope");
1894 }
1895 }
1896
1897 if (any(Val: AddrSpace & SIAtomicAddrSpace::LDS)) {
1898 switch (Scope) {
1899 case SIAtomicScope::SYSTEM:
1900 case SIAtomicScope::AGENT:
1901 case SIAtomicScope::CLUSTER:
1902 case SIAtomicScope::WORKGROUP:
1903 // If no cross address space ordering then an "S_WAITCNT lgkmcnt(0)" is
1904 // not needed as LDS operations for all waves are executed in a total
1905 // global ordering as observed by all waves. Required if also
1906 // synchronizing with global/GDS memory as LDS operations could be
1907 // reordered with respect to later global/GDS memory operations of the
1908 // same wave.
1909 DSCnt |= IsCrossAddrSpaceOrdering;
1910 break;
1911 case SIAtomicScope::WAVEFRONT:
1912 case SIAtomicScope::SINGLETHREAD:
1913 // The LDS keeps all memory operations in order for
1914 // the same wavefront.
1915 break;
1916 default:
1917 llvm_unreachable("Unsupported synchronization scope");
1918 }
1919 }
1920
1921 if (LOADCnt) {
1922 // Acquire sequences only need to wait on the previous atomic operation.
1923 // e.g. a typical sequence looks like
1924 // atomic load
1925 // (wait)
1926 // global_inv
1927 //
1928 // We do not have BVH or SAMPLE atomics, so the atomic load is always going
1929 // to be tracked using loadcnt.
1930 //
1931 // This also applies to fences. Fences cannot pair with an instruction
1932 // tracked with bvh/samplecnt as we don't have any atomics that do that.
1933 if (!AtomicsOnly && ST.hasImageInsts()) {
1934 if (ST.hasAnyBVHInsts())
1935 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_WAIT_BVHCNT_soft)).addImm(Val: 0);
1936 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_WAIT_SAMPLECNT_soft)).addImm(Val: 0);
1937 }
1938 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_WAIT_LOADCNT_soft)).addImm(Val: 0);
1939 Changed = true;
1940 }
1941
1942 if (STORECnt) {
1943 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_WAIT_STORECNT_soft)).addImm(Val: 0);
1944 Changed = true;
1945 }
1946
1947 if (DSCnt) {
1948 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_WAIT_DSCNT_soft)).addImm(Val: 0);
1949 Changed = true;
1950 }
1951
1952 if (Pos == Position::AFTER)
1953 --MI;
1954
1955 return Changed;
1956}
1957
1958bool SIGfx12CacheControl::insertAcquire(MachineBasicBlock::iterator &MI,
1959 SIAtomicScope Scope,
1960 SIAtomicAddrSpace AddrSpace,
1961 Position Pos) const {
1962 if (!InsertCacheInv)
1963 return false;
1964
1965 MachineBasicBlock &MBB = *MI->getParent();
1966 const DebugLoc &DL = MI->getDebugLoc();
1967
1968 /// The scratch address space does not need the global memory cache
1969 /// to be flushed as all memory operations by the same thread are
1970 /// sequentially consistent, and no other thread can access scratch
1971 /// memory.
1972
1973 /// Other address spaces do not have a cache.
1974 if (!canAffectGlobalAddrSpace(AS: AddrSpace))
1975 return false;
1976
1977 AMDGPU::CPol::CPol ScopeImm = AMDGPU::CPol::SCOPE_DEV;
1978 switch (Scope) {
1979 case SIAtomicScope::SYSTEM:
1980 // If MTYPE NC is not used, HW ensures coherence in L2, so we only need to
1981 // invalidate up to SE scope.
1982 ScopeImm =
1983 ST.hasNoMTypeNC() ? AMDGPU::CPol::SCOPE_SE : AMDGPU::CPol::SCOPE_SYS;
1984 break;
1985 case SIAtomicScope::AGENT:
1986 // If MTYPE NC is not used, HW ensures coherence in L2, so we only need to
1987 // invalidate up to SE scope.
1988 ScopeImm =
1989 ST.hasNoMTypeNC() ? AMDGPU::CPol::SCOPE_SE : AMDGPU::CPol::SCOPE_DEV;
1990 break;
1991 case SIAtomicScope::CLUSTER:
1992 ScopeImm = AMDGPU::CPol::SCOPE_SE;
1993 break;
1994 case SIAtomicScope::WORKGROUP:
1995 // GFX12.0:
1996 // In WGP mode the waves of a work-group can be executing on either CU of
1997 // the WGP. Therefore we need to invalidate the L0 which is per CU.
1998 // Otherwise in CU mode all waves of a work-group are on the same CU, and
1999 // so the L0 does not need to be invalidated.
2000 //
2001 // GFX12.5 has a shared WGP$, so no invalidates are required.
2002 if (ST.isCuModeEnabled())
2003 return false;
2004
2005 ScopeImm = AMDGPU::CPol::SCOPE_SE;
2006 break;
2007 case SIAtomicScope::WAVEFRONT:
2008 case SIAtomicScope::SINGLETHREAD:
2009 // No cache to invalidate.
2010 return false;
2011 default:
2012 llvm_unreachable("Unsupported synchronization scope");
2013 }
2014
2015 if (Pos == Position::AFTER)
2016 ++MI;
2017
2018 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::GLOBAL_INV)).addImm(Val: ScopeImm);
2019
2020 if (Pos == Position::AFTER)
2021 --MI;
2022
2023 // Target requires a waitcnt to ensure that the proceeding INV has completed
2024 // as it may get reorded with following load instructions.
2025 if (ST.hasINVWBL2WaitCntRequirement() && Scope > SIAtomicScope::CLUSTER) {
2026 insertWait(MI, Scope, AddrSpace, Op: SIMemOp::LOAD,
2027 /*IsCrossAddrSpaceOrdering=*/false, Pos, Order: AtomicOrdering::Acquire,
2028 /*AtomicsOnly=*/false);
2029
2030 if (Pos == Position::AFTER)
2031 --MI;
2032 }
2033
2034 return true;
2035}
2036
2037bool SIGfx12CacheControl::insertWriteback(MachineBasicBlock::iterator &MI,
2038 SIAtomicScope Scope,
2039 SIAtomicAddrSpace AddrSpace,
2040 Position Pos) const {
2041 // The scratch address space does not need the global memory cache
2042 // writeback as all memory operations by the same thread are
2043 // sequentially consistent, and no other thread can access scratch
2044 // memory.
2045 if (!canAffectGlobalAddrSpace(AS: AddrSpace))
2046 return false;
2047
2048 bool Changed = false;
2049 MachineBasicBlock &MBB = *MI->getParent();
2050 const DebugLoc &DL = MI->getDebugLoc();
2051
2052 if (Pos == Position::AFTER)
2053 ++MI;
2054
2055 // global_wb is only necessary at system scope for GFX12.0.
2056 // They're also necessary at device scope for GFX12.5 as stores
2057 // cannot report completion earlier than L2.
2058 // global_wb is not necessary if the driver guarantees that MTYPE NC
2059 // is not used, since the hardware keeps the L2s coherent in that
2060 // case.
2061 //
2062 // Emitting it for lower scopes is a slow no-op, so we omit it
2063 // for performance.
2064 std::optional<AMDGPU::CPol::CPol> NeedsWB;
2065 switch (Scope) {
2066 case SIAtomicScope::SYSTEM:
2067 if (!ST.hasNoMTypeNC())
2068 NeedsWB = AMDGPU::CPol::SCOPE_SYS;
2069 break;
2070 case SIAtomicScope::AGENT:
2071 // GFX12.5 may have >1 L2 per device so we must emit a device scope WB,
2072 // unless MTYPE NC is not used.
2073 if (ST.hasGFX1250Insts() && !ST.hasNoMTypeNC())
2074 NeedsWB = AMDGPU::CPol::SCOPE_DEV;
2075 break;
2076 case SIAtomicScope::CLUSTER:
2077 case SIAtomicScope::WORKGROUP:
2078 case SIAtomicScope::WAVEFRONT:
2079 case SIAtomicScope::SINGLETHREAD:
2080 break;
2081 case SIAtomicScope::NONE:
2082 llvm_unreachable("Unsupported synchronization scope");
2083 break;
2084 }
2085
2086 if (NeedsWB) {
2087 // Target requires a waitcnt to ensure that the proceeding store
2088 // proceeding store/rmw operations have completed in L2 so their data will
2089 // be written back by the WB instruction.
2090 if (ST.hasINVWBL2WaitCntRequirement()) {
2091 insertWait(MI, Scope, AddrSpace, Op: SIMemOp::LOAD | SIMemOp::STORE,
2092 /*IsCrossAddrSpaceOrdering=*/false, Pos,
2093 Order: AtomicOrdering::Release,
2094 /*AtomicsOnly=*/false);
2095 }
2096
2097 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::GLOBAL_WB)).addImm(Val: *NeedsWB);
2098 Changed = true;
2099 }
2100
2101 if (Pos == Position::AFTER)
2102 --MI;
2103
2104 return Changed;
2105}
2106
2107bool SIGfx12CacheControl::handleNonVolatile(MachineInstr &MI) const {
2108 // On GFX12.5, set the NV CPol bit.
2109 if (!ST.hasGFX1250Insts())
2110 return false;
2111 MachineOperand *CPol = TII->getNamedOperand(MI, OperandName: OpName::cpol);
2112 if (!CPol)
2113 return false;
2114 CPol->setImm(CPol->getImm() | AMDGPU::CPol::NV);
2115 return true;
2116}
2117
2118bool SIGfx12CacheControl::enableVolatileAndOrNonTemporal(
2119 MachineBasicBlock::iterator &MI, SIAtomicAddrSpace AddrSpace, SIMemOp Op,
2120 bool IsVolatile, bool IsNonTemporal, bool IsLastUse = false) const {
2121
2122 // Only handle load and store, not atomic read-modify-write instructions.
2123 assert((MI->mayLoad() ^ MI->mayStore()) || SIInstrInfo::isLDSDMA(*MI));
2124
2125 // Only update load and store, not LLVM IR atomic read-modify-write
2126 // instructions. The latter are always marked as volatile so cannot sensibly
2127 // handle it as do not want to pessimize all atomics. Also they do not support
2128 // the nontemporal attribute.
2129 assert(Op == SIMemOp::LOAD || Op == SIMemOp::STORE);
2130
2131 bool Changed = false;
2132
2133 if (IsLastUse) {
2134 // Set last-use hint.
2135 Changed |= setTH(MI, Value: AMDGPU::CPol::TH_LU);
2136 } else if (IsNonTemporal) {
2137 // Set non-temporal hint for all cache levels.
2138 Changed |= setTH(MI, Value: AMDGPU::CPol::TH_NT);
2139 }
2140
2141 if (IsVolatile) {
2142 Changed |= setScope(MI, Value: AMDGPU::CPol::SCOPE_SYS);
2143
2144 if (ST.requiresWaitXCntForSingleAccessInstructions() &&
2145 SIInstrInfo::isVMEM(MI: *MI)) {
2146 MachineBasicBlock &MBB = *MI->getParent();
2147 BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: S_WAIT_XCNT_soft)).addImm(Val: 0);
2148 Changed = true;
2149 }
2150
2151 // Ensure operation has completed at system scope to cause all volatile
2152 // operations to be visible outside the program in a global order. Do not
2153 // request cross address space as only the global address space can be
2154 // observable outside the program, so no need to cause a waitcnt for LDS
2155 // address space operations.
2156 Changed |= insertWait(MI, Scope: SIAtomicScope::SYSTEM, AddrSpace, Op, IsCrossAddrSpaceOrdering: false,
2157 Pos: Position::AFTER, Order: AtomicOrdering::Unordered,
2158 /*AtomicsOnly=*/false);
2159 }
2160
2161 return Changed;
2162}
2163
2164bool SIGfx12CacheControl::finalizeStore(MachineInstr &MI, bool Atomic) const {
2165 assert(MI.mayStore() && "Not a Store inst");
2166 const bool IsRMW = (MI.mayLoad() && MI.mayStore());
2167 bool Changed = false;
2168
2169 if (Atomic && ST.requiresWaitXCntForSingleAccessInstructions() &&
2170 SIInstrInfo::isVMEM(MI)) {
2171 MachineBasicBlock &MBB = *MI.getParent();
2172 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: S_WAIT_XCNT_soft)).addImm(Val: 0);
2173 Changed = true;
2174 }
2175
2176 // Remaining fixes do not apply to RMWs.
2177 if (IsRMW)
2178 return Changed;
2179
2180 MachineOperand *CPol = TII->getNamedOperand(MI, OperandName: OpName::cpol);
2181 if (!CPol) // Some vmem operations do not have a scope and are not concerned.
2182 return Changed;
2183 const unsigned Scope = CPol->getImm() & CPol::SCOPE;
2184
2185 // GFX12.0 only: Extra waits needed before system scope stores.
2186 if (ST.requiresWaitsBeforeSystemScopeStores() && !Atomic &&
2187 Scope == CPol::SCOPE_SYS)
2188 Changed |= insertWaitsBeforeSystemScopeStore(MI: MI.getIterator());
2189
2190 return Changed;
2191}
2192
2193bool SIGfx12CacheControl::finalizeLoad(MachineBasicBlock::iterator &MI) const {
2194 if (!SIInstrInfo::isLoadMonitor(Opc: MI->getOpcode()))
2195 return false;
2196
2197 // load_monitor instructions need at least SCOPE_SE to ensure L2 is hit.
2198 MachineOperand *CPol = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::cpol);
2199 assert(CPol && "load_monitor must have a cpol operand");
2200 if ((CPol->getImm() & AMDGPU::CPol::SCOPE) < AMDGPU::CPol::SCOPE_SE)
2201 return setScope(MI, Value: AMDGPU::CPol::SCOPE_SE);
2202 return false;
2203}
2204
2205bool SIGfx12CacheControl::handleCooperativeAtomic(MachineInstr &MI) const {
2206 if (!ST.hasGFX1250Insts())
2207 return false;
2208
2209 // Cooperative atomics need to be SCOPE_DEV or higher.
2210 MachineOperand *CPol = TII->getNamedOperand(MI, OperandName: OpName::cpol);
2211 assert(CPol && "No CPol operand?");
2212 const unsigned Scope = CPol->getImm() & CPol::SCOPE;
2213 if (Scope < CPol::SCOPE_DEV)
2214 return setScope(MI, Value: CPol::SCOPE_DEV);
2215 return false;
2216}
2217
2218bool SIGfx12CacheControl::setAtomicScope(const MachineBasicBlock::iterator &MI,
2219 SIAtomicScope Scope,
2220 SIAtomicAddrSpace AddrSpace) const {
2221 bool Changed = false;
2222
2223 if (canAffectGlobalAddrSpace(AS: AddrSpace)) {
2224 switch (Scope) {
2225 case SIAtomicScope::SYSTEM:
2226 Changed |= setScope(MI, Value: AMDGPU::CPol::SCOPE_SYS);
2227 break;
2228 case SIAtomicScope::AGENT:
2229 Changed |= setScope(MI, Value: AMDGPU::CPol::SCOPE_DEV);
2230 break;
2231 case SIAtomicScope::CLUSTER:
2232 Changed |= setScope(MI, Value: AMDGPU::CPol::SCOPE_SE);
2233 break;
2234 case SIAtomicScope::WORKGROUP:
2235 // In workgroup mode, SCOPE_SE is needed as waves can executes on
2236 // different CUs that access different L0s.
2237 if (!ST.isCuModeEnabled())
2238 Changed |= setScope(MI, Value: AMDGPU::CPol::SCOPE_SE);
2239 break;
2240 case SIAtomicScope::WAVEFRONT:
2241 case SIAtomicScope::SINGLETHREAD:
2242 // No cache to bypass.
2243 break;
2244 default:
2245 llvm_unreachable("Unsupported synchronization scope");
2246 }
2247 }
2248
2249 // The scratch address space does not need the global memory caches
2250 // to be bypassed as all memory operations by the same thread are
2251 // sequentially consistent, and no other thread can access scratch
2252 // memory.
2253
2254 // Other address spaces do not have a cache.
2255
2256 return Changed;
2257}
2258
2259bool SIGfx12CacheControl::setCFS(const MachineBasicBlock::iterator &MI,
2260 unsigned CFSBits) const {
2261 if (!ST.hasCacheFillSize())
2262 return false;
2263 auto CoreMI = &*MI;
2264
2265 MachineOperand *CPol = TII->getNamedOperand(MI&: *CoreMI, OperandName: OpName::cpol);
2266 if (!CPol)
2267 return false;
2268
2269 uint64_t NewCFS = CFSBits << AMDGPU::CPol::CFS_SHIFT & AMDGPU::CPol::CFS;
2270 if (CFSBits) {
2271 if ((CPol->getImm() & AMDGPU::CPol::CFS) != NewCFS) {
2272 CPol->setImm((CPol->getImm() & ~AMDGPU::CPol::CFS) | NewCFS);
2273 return true;
2274 }
2275 }
2276
2277 return false;
2278}
2279
2280bool SIMemoryLegalizer::removeAtomicPseudoMIs() {
2281 if (AtomicPseudoMIs.empty())
2282 return false;
2283
2284 for (auto &MI : AtomicPseudoMIs)
2285 MI->eraseFromParent();
2286
2287 AtomicPseudoMIs.clear();
2288 return true;
2289}
2290
2291bool SIMemoryLegalizer::expandLoad(const SIMemOpInfo &MOI,
2292 MachineBasicBlock::iterator &MI) {
2293 assert(MI->mayLoad() && !MI->mayStore());
2294
2295 LLVM_DEBUG(dbgs() << "Expanding load: " << *MI);
2296
2297 bool Changed = false;
2298
2299 if (MOI.isAtomic()) {
2300 LLVM_DEBUG(dbgs() << " Atomic: ordering=" << toIRString(MOI.getOrdering())
2301 << ", scope=" << toString(MOI.getScope())
2302 << ", ordering-AS=" << MOI.getOrderingAddrSpace()
2303 << ", instr-AS=" << MOI.getInstrAddrSpace() << "\n");
2304 const AtomicOrdering Order = MOI.getOrdering();
2305 if (Order == AtomicOrdering::Monotonic ||
2306 Order == AtomicOrdering::Acquire ||
2307 Order == AtomicOrdering::SequentiallyConsistent) {
2308 Changed |= CC->enableLoadCacheBypass(MI, Scope: MOI.getScope(),
2309 AddrSpace: MOI.getOrderingAddrSpace());
2310 }
2311
2312 // Handle cooperative atomics after cache bypass step, as it may override
2313 // the scope of the instruction to a greater scope.
2314 if (MOI.isCooperative())
2315 Changed |= CC->handleCooperativeAtomic(MI&: *MI);
2316
2317 Changed |= CC->finalizeLoad(MI);
2318 Changed |= CC->setCFS(MI, CFSBits: MOI.getCFS());
2319
2320 if (Order == AtomicOrdering::SequentiallyConsistent)
2321 Changed |= CC->insertWait(MI, Scope: MOI.getScope(), AddrSpace: MOI.getOrderingAddrSpace(),
2322 Op: SIMemOp::LOAD | SIMemOp::STORE,
2323 IsCrossAddrSpaceOrdering: MOI.getIsCrossAddressSpaceOrdering(),
2324 Pos: Position::BEFORE, Order, /*AtomicsOnly=*/false);
2325
2326 if (Order == AtomicOrdering::Acquire ||
2327 Order == AtomicOrdering::SequentiallyConsistent) {
2328 // The wait below only needs to wait on the prior atomic.
2329 Changed |=
2330 CC->insertWait(MI, Scope: MOI.getScope(), AddrSpace: MOI.getInstrAddrSpace(),
2331 Op: SIMemOp::LOAD, IsCrossAddrSpaceOrdering: MOI.getIsCrossAddressSpaceOrdering(),
2332 Pos: Position::AFTER, Order, /*AtomicsOnly=*/true);
2333 if (!MOI.isAVNone()) {
2334 Changed |= CC->insertAcquire(
2335 MI, Scope: MOI.getScope(), AddrSpace: MOI.getOrderingAddrSpace(), Pos: Position::AFTER);
2336 }
2337 }
2338
2339 return Changed;
2340 }
2341
2342 // Atomic instructions already bypass caches to the scope specified by the
2343 // SyncScope operand. Only non-atomic volatile and nontemporal/last-use
2344 // instructions need additional treatment.
2345 Changed |= CC->enableVolatileAndOrNonTemporal(
2346 MI, AddrSpace: MOI.getInstrAddrSpace(), Op: SIMemOp::LOAD, IsVolatile: MOI.isVolatile(),
2347 IsNonTemporal: MOI.isNonTemporal(), IsLastUse: MOI.isLastUse());
2348
2349 Changed |= CC->finalizeLoad(MI);
2350 Changed |= CC->setCFS(MI, CFSBits: MOI.getCFS());
2351
2352 return Changed;
2353}
2354
2355bool SIMemoryLegalizer::expandStore(const SIMemOpInfo &MOI,
2356 MachineBasicBlock::iterator &MI) {
2357 assert(!MI->mayLoad() && MI->mayStore());
2358
2359 LLVM_DEBUG(dbgs() << "Expanding store: " << *MI);
2360
2361 bool Changed = false;
2362 // FIXME: Necessary hack because iterator can lose track of the store.
2363 MachineInstr &StoreMI = *MI;
2364
2365 if (MOI.isAtomic()) {
2366 LLVM_DEBUG(dbgs() << " Atomic: ordering=" << toIRString(MOI.getOrdering())
2367 << ", scope=" << toString(MOI.getScope())
2368 << ", ordering-AS=" << MOI.getOrderingAddrSpace()
2369 << ", instr-AS=" << MOI.getInstrAddrSpace() << "\n");
2370 if (MOI.getOrdering() == AtomicOrdering::Monotonic ||
2371 MOI.getOrdering() == AtomicOrdering::Release ||
2372 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
2373 Changed |= CC->enableStoreCacheBypass(MI, Scope: MOI.getScope(),
2374 AddrSpace: MOI.getOrderingAddrSpace());
2375 }
2376
2377 // Handle cooperative atomics after cache bypass step, as it may override
2378 // the scope of the instruction to a greater scope.
2379 if (MOI.isCooperative())
2380 Changed |= CC->handleCooperativeAtomic(MI&: *MI);
2381
2382 if (MOI.getOrdering() == AtomicOrdering::Release ||
2383 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
2384 Changed |=
2385 CC->insertRelease(MI, Scope: MOI.getScope(), AddrSpace: MOI.getOrderingAddrSpace(),
2386 IsCrossAddrSpaceOrdering: MOI.getIsCrossAddressSpaceOrdering(),
2387 Pos: Position::BEFORE, IsAVNone: MOI.isAVNone());
2388 }
2389
2390 Changed |= CC->finalizeStore(MI&: StoreMI, /*Atomic=*/true);
2391 Changed |= CC->setCFS(MI, CFSBits: MOI.getCFS());
2392 return Changed;
2393 }
2394
2395 // Atomic instructions already bypass caches to the scope specified by the
2396 // SyncScope operand. Only non-atomic volatile and nontemporal instructions
2397 // need additional treatment.
2398 Changed |= CC->enableVolatileAndOrNonTemporal(
2399 MI, AddrSpace: MOI.getInstrAddrSpace(), Op: SIMemOp::STORE, IsVolatile: MOI.isVolatile(),
2400 IsNonTemporal: MOI.isNonTemporal());
2401
2402 // GFX12 specific, scope(desired coherence domain in cache hierarchy) is
2403 // instruction field, do not confuse it with atomic scope.
2404 Changed |= CC->finalizeStore(MI&: StoreMI, /*Atomic=*/false);
2405
2406 Changed |= CC->setCFS(MI, CFSBits: MOI.getCFS());
2407 return Changed;
2408}
2409
2410bool SIMemoryLegalizer::expandAtomicFence(const SIMemOpInfo &MOI,
2411 MachineBasicBlock::iterator &MI) {
2412 assert(MI->getOpcode() == AMDGPU::ATOMIC_FENCE);
2413
2414 LLVM_DEBUG(dbgs() << "Expanding atomic fence: " << *MI);
2415
2416 AtomicPseudoMIs.push_back(x: MI);
2417 bool Changed = false;
2418
2419 const SIAtomicAddrSpace OrderingAddrSpace = MOI.getOrderingAddrSpace();
2420
2421 if (MOI.isAtomic()) {
2422 LLVM_DEBUG(dbgs() << " Atomic: ordering=" << toIRString(MOI.getOrdering())
2423 << ", scope=" << toString(MOI.getScope())
2424 << ", ordering-AS=" << OrderingAddrSpace << "\n");
2425 const AtomicOrdering Order = MOI.getOrdering();
2426 if (Order == AtomicOrdering::Acquire) {
2427 // Acquire fences only need to wait on the previous atomic they pair with.
2428 Changed |= CC->insertWait(MI, Scope: MOI.getScope(), AddrSpace: OrderingAddrSpace,
2429 Op: SIMemOp::LOAD | SIMemOp::STORE,
2430 IsCrossAddrSpaceOrdering: MOI.getIsCrossAddressSpaceOrdering(),
2431 Pos: Position::BEFORE, Order, /*AtomicsOnly=*/true);
2432 }
2433
2434 if (Order == AtomicOrdering::Release ||
2435 Order == AtomicOrdering::AcquireRelease ||
2436 Order == AtomicOrdering::SequentiallyConsistent) {
2437 /// TODO: This relies on a barrier always generating a waitcnt
2438 /// for LDS to ensure it is not reordered with the completion of
2439 /// the proceeding LDS operations. If barrier had a memory
2440 /// ordering and memory scope, then library does not need to
2441 /// generate a fence. Could add support in this file for
2442 /// barrier. SIInsertWaitcnt.cpp could then stop unconditionally
2443 /// adding S_WAITCNT before a S_BARRIER.
2444 Changed |= CC->insertRelease(MI, Scope: MOI.getScope(), AddrSpace: OrderingAddrSpace,
2445 IsCrossAddrSpaceOrdering: MOI.getIsCrossAddressSpaceOrdering(),
2446 Pos: Position::BEFORE, IsAVNone: MOI.isAVNone());
2447 }
2448
2449 // TODO: If both release and invalidate are happening they could be combined
2450 // to use the single "BUFFER_WBINV*" instruction. This could be done by
2451 // reorganizing this code or as part of optimizing SIInsertWaitcnt pass to
2452 // track cache invalidate and write back instructions.
2453
2454 if ((Order == AtomicOrdering::Acquire ||
2455 Order == AtomicOrdering::AcquireRelease ||
2456 Order == AtomicOrdering::SequentiallyConsistent) &&
2457 !MOI.isAVNone()) {
2458 Changed |= CC->insertAcquire(MI, Scope: MOI.getScope(), AddrSpace: OrderingAddrSpace,
2459 Pos: Position::BEFORE);
2460 }
2461
2462 return Changed;
2463 }
2464
2465 return Changed;
2466}
2467
2468bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(const SIMemOpInfo &MOI,
2469 MachineBasicBlock::iterator &MI) {
2470 assert(MI->mayLoad() && MI->mayStore());
2471
2472 LLVM_DEBUG(dbgs() << "Expanding atomic cmpxchg/rmw: " << *MI);
2473
2474 bool Changed = false;
2475 MachineInstr &RMWMI = *MI;
2476
2477 if (MOI.isAtomic()) {
2478 LLVM_DEBUG(dbgs() << " Atomic: ordering=" << toIRString(MOI.getOrdering())
2479 << ", failure-ordering="
2480 << toIRString(MOI.getFailureOrdering())
2481 << ", scope=" << toString(MOI.getScope())
2482 << ", ordering-AS=" << MOI.getOrderingAddrSpace()
2483 << ", instr-AS=" << MOI.getInstrAddrSpace() << "\n");
2484 const AtomicOrdering Order = MOI.getOrdering();
2485 if (Order == AtomicOrdering::Monotonic ||
2486 Order == AtomicOrdering::Acquire || Order == AtomicOrdering::Release ||
2487 Order == AtomicOrdering::AcquireRelease ||
2488 Order == AtomicOrdering::SequentiallyConsistent) {
2489 Changed |= CC->enableRMWCacheBypass(MI, Scope: MOI.getScope(),
2490 AddrSpace: MOI.getInstrAddrSpace());
2491 }
2492
2493 if (Order == AtomicOrdering::Release ||
2494 Order == AtomicOrdering::AcquireRelease ||
2495 Order == AtomicOrdering::SequentiallyConsistent ||
2496 MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) {
2497 Changed |=
2498 CC->insertRelease(MI, Scope: MOI.getScope(), AddrSpace: MOI.getOrderingAddrSpace(),
2499 IsCrossAddrSpaceOrdering: MOI.getIsCrossAddressSpaceOrdering(),
2500 Pos: Position::BEFORE, IsAVNone: MOI.isAVNone());
2501 }
2502
2503 if (Order == AtomicOrdering::Acquire ||
2504 Order == AtomicOrdering::AcquireRelease ||
2505 Order == AtomicOrdering::SequentiallyConsistent ||
2506 MOI.getFailureOrdering() == AtomicOrdering::Acquire ||
2507 MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) {
2508 // Only wait on the previous atomic.
2509 Changed |=
2510 CC->insertWait(MI, Scope: MOI.getScope(), AddrSpace: MOI.getInstrAddrSpace(),
2511 Op: isAtomicRet(MI: *MI) ? SIMemOp::LOAD : SIMemOp::STORE,
2512 IsCrossAddrSpaceOrdering: MOI.getIsCrossAddressSpaceOrdering(), Pos: Position::AFTER,
2513 Order, /*AtomicsOnly=*/true);
2514 if (!MOI.isAVNone()) {
2515 Changed |= CC->insertAcquire(
2516 MI, Scope: MOI.getScope(), AddrSpace: MOI.getOrderingAddrSpace(), Pos: Position::AFTER);
2517 }
2518 }
2519
2520 Changed |= CC->finalizeStore(MI&: RMWMI, /*Atomic=*/true);
2521 Changed |= CC->setCFS(MI, CFSBits: MOI.getCFS());
2522 return Changed;
2523 }
2524
2525 Changed |= CC->setCFS(MI, CFSBits: MOI.getCFS());
2526 return Changed;
2527}
2528
2529bool SIMemoryLegalizer::expandLDSDMA(const SIMemOpInfo &MOI,
2530 MachineBasicBlock::iterator &MI) {
2531 assert(MI->mayLoad() && MI->mayStore());
2532
2533 LLVM_DEBUG(dbgs() << "Expanding LDS DMA: " << *MI);
2534
2535 // The volatility or nontemporal-ness of the operation is a
2536 // function of the global memory, not the LDS.
2537 SIMemOp OpKind =
2538 SIInstrInfo::mayWriteLDSThroughDMA(MI: *MI) ? SIMemOp::LOAD : SIMemOp::STORE;
2539
2540 // Handle volatile and/or nontemporal markers on direct-to-LDS loads and
2541 // stores. The operation is treated as a volatile/nontemporal store
2542 // to its second argument.
2543 bool Changed = CC->enableVolatileAndOrNonTemporal(
2544 MI, AddrSpace: MOI.getInstrAddrSpace(), Op: OpKind, IsVolatile: MOI.isVolatile(),
2545 IsNonTemporal: MOI.isNonTemporal(), IsLastUse: MOI.isLastUse());
2546 Changed |= CC->setCFS(MI, CFSBits: MOI.getCFS());
2547 return Changed;
2548}
2549
2550bool SIMemoryLegalizerLegacy::runOnMachineFunction(MachineFunction &MF) {
2551 const MachineModuleInfo &MMI =
2552 getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
2553 return SIMemoryLegalizer(MMI).run(MF);
2554}
2555
2556PreservedAnalyses
2557SIMemoryLegalizerPass::run(MachineFunction &MF,
2558 MachineFunctionAnalysisManager &MFAM) {
2559 auto *MMI = MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
2560 .getCachedResult<MachineModuleAnalysis>(
2561 IR&: *MF.getFunction().getParent());
2562 assert(MMI && "MachineModuleAnalysis must be available");
2563 if (!SIMemoryLegalizer(MMI->getMMI()).run(MF))
2564 return PreservedAnalyses::all();
2565 return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
2566}
2567
2568bool SIMemoryLegalizer::run(MachineFunction &MF) {
2569 bool Changed = false;
2570
2571 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2572 const Function &F = MF.getFunction();
2573 SIMemOpAccess MOA(MMI.getObjFileInfo<AMDGPUMachineModuleInfo>(), ST, MF);
2574 bool TgSplit = ST.hasTgSplitSupport() && AMDGPU::isTgSplitEnabled(F);
2575 CC = SICacheControl::create(ST, TgSplit);
2576
2577 for (auto &MBB : MF) {
2578 for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) {
2579
2580 // Unbundle instructions after the post-RA scheduler.
2581 if (MI->isBundle() && MI->mayLoadOrStore()) {
2582 MachineBasicBlock::instr_iterator II(MI->getIterator());
2583 for (MachineBasicBlock::instr_iterator I = ++II, E = MBB.instr_end();
2584 I != E && I->isBundledWithPred(); ++I) {
2585 I->unbundleFromPred();
2586 for (MachineOperand &MO : I->operands())
2587 if (MO.isReg())
2588 MO.setIsInternalRead(false);
2589 }
2590
2591 MI = MI->eraseFromParent();
2592 }
2593
2594 if (SIInstrFlags::isMaybeAtomic(O: *MI)) {
2595 if (const auto &MOI = MOA.getLoadInfo(MI))
2596 Changed |= expandLoad(MOI: *MOI, MI);
2597 else if (const auto &MOI = MOA.getStoreInfo(MI))
2598 Changed |= expandStore(MOI: *MOI, MI);
2599 else if (const auto &MOI = MOA.getLDSDMAInfo(MI))
2600 Changed |= expandLDSDMA(MOI: *MOI, MI);
2601 else if (const auto &MOI = MOA.getAtomicFenceInfo(MI))
2602 Changed |= expandAtomicFence(MOI: *MOI, MI);
2603 else if (const auto &MOI = MOA.getAtomicCmpxchgOrRmwInfo(MI))
2604 Changed |= expandAtomicCmpxchgOrRmw(MOI: *MOI, MI);
2605 }
2606
2607 if (isNonVolatileMemoryAccess(MI: *MI))
2608 Changed |= CC->handleNonVolatile(MI&: *MI);
2609 }
2610 }
2611
2612 Changed |= removeAtomicPseudoMIs();
2613 return Changed;
2614}
2615
2616INITIALIZE_PASS(SIMemoryLegalizerLegacy, DEBUG_TYPE, PASS_NAME, false, false)
2617
2618char SIMemoryLegalizerLegacy::ID = 0;
2619char &llvm::SIMemoryLegalizerID = SIMemoryLegalizerLegacy::ID;
2620
2621FunctionPass *llvm::createSIMemoryLegalizerPass() {
2622 return new SIMemoryLegalizerLegacy();
2623}
2624