1//===- SIInsertHardClauses.cpp - Insert Hard Clauses ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert s_clause instructions to form hard clauses.
11///
12/// Clausing load instructions can give cache coherency benefits. Before gfx10,
13/// the hardware automatically detected "soft clauses", which were sequences of
14/// memory instructions of the same type. In gfx10 this detection was removed,
15/// and the s_clause instruction was introduced to explicitly mark "hard
16/// clauses".
17///
18/// It's the scheduler's job to form the clauses by putting similar memory
19/// instructions next to each other. Our job is just to insert an s_clause
20/// instruction to mark the start of each clause.
21///
22/// Note that hard clauses are very similar to, but logically distinct from, the
23/// groups of instructions that have to be restartable when XNACK is enabled.
24/// The rules are slightly different in each case. For example an s_nop
25/// instruction breaks a restartable group, but can appear in the middle of a
26/// hard clause. (Before gfx10 there wasn't a distinction, and both were called
27/// "soft clauses" or just "clauses".)
28///
29/// The SIFormMemoryClauses pass and GCNHazardRecognizer deal with restartable
30/// groups, not hard clauses.
31//
32//===----------------------------------------------------------------------===//
33
34#include "AMDGPU.h"
35#include "GCNSubtarget.h"
36#include "llvm/ADT/SmallVector.h"
37#include "llvm/CodeGen/MachineFunctionPass.h"
38#include "llvm/CodeGen/MachinePassManager.h"
39
40using namespace llvm;
41
42#define DEBUG_TYPE "si-insert-hard-clauses"
43
44namespace {
45
46enum HardClauseType {
47 // For targets where loads, stores and atomics share one clause type per
48 // memory class (see hasMixedLoadStoreClauses):
49
50 // Texture, buffer, global or scratch memory instructions.
51 HARDCLAUSE_VMEM,
52 // Flat (not global or scratch) memory instructions.
53 HARDCLAUSE_FLAT,
54
55 // For targets where each memory class is typed separately for loads,
56 // stores and atomics:
57
58 // Texture memory instructions.
59 HARDCLAUSE_MIMG_LOAD,
60 HARDCLAUSE_MIMG_STORE,
61 HARDCLAUSE_MIMG_ATOMIC,
62 HARDCLAUSE_MIMG_SAMPLE,
63 // Buffer, global or scratch memory instructions.
64 HARDCLAUSE_VMEM_LOAD,
65 HARDCLAUSE_VMEM_STORE,
66 HARDCLAUSE_VMEM_ATOMIC,
67 // Flat (not global or scratch) memory instructions.
68 HARDCLAUSE_FLAT_LOAD,
69 HARDCLAUSE_FLAT_STORE,
70 HARDCLAUSE_FLAT_ATOMIC,
71 // BVH instructions.
72 HARDCLAUSE_BVH,
73
74 // Common:
75
76 // Instructions that access LDS.
77 HARDCLAUSE_LDS,
78 // Scalar memory instructions.
79 HARDCLAUSE_SMEM,
80 // VALU instructions.
81 HARDCLAUSE_VALU,
82 LAST_REAL_HARDCLAUSE_TYPE = HARDCLAUSE_VALU,
83
84 // Internal instructions, which are allowed in the middle of a hard clause,
85 // except for s_waitcnt.
86 HARDCLAUSE_INTERNAL,
87 // Meta instructions that do not result in any ISA like KILL.
88 HARDCLAUSE_IGNORE,
89 // Instructions that are not allowed in a hard clause: SALU, export, branch,
90 // message, GDS, s_waitcnt and anything else not mentioned above.
91 HARDCLAUSE_ILLEGAL,
92};
93
94class SIInsertHardClauses {
95public:
96 const GCNSubtarget *ST = nullptr;
97
98 HardClauseType getHardClauseType(const MachineInstr &MI) {
99 // The replay expands to scalar EXEC moves surrounding its memory operation.
100 if (MI.getOpcode() == AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay)
101 return HARDCLAUSE_ILLEGAL;
102 if (MI.mayLoad() || (MI.mayStore() && ST->shouldClusterStores())) {
103 if (ST->hasMixedLoadStoreClauses()) {
104 if ((SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) ||
105 SIInstrInfo::isSegmentSpecificFLAT(MI)) {
106 if (ST->hasNSAClauseBug()) {
107 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: MI.getOpcode());
108 if (Info && Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA)
109 return HARDCLAUSE_ILLEGAL;
110 }
111 return HARDCLAUSE_VMEM;
112 }
113 if (SIInstrInfo::isFLAT(MI))
114 return HARDCLAUSE_FLAT;
115 } else {
116 if (SIInstrInfo::isMIMG(MI)) {
117 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc: MI.getOpcode());
118 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
119 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
120 if (BaseInfo->BVH)
121 return HARDCLAUSE_BVH;
122 if (BaseInfo->Sampler || BaseInfo->MSAA)
123 return HARDCLAUSE_MIMG_SAMPLE;
124 return MI.mayLoad() ? MI.mayStore() ? HARDCLAUSE_MIMG_ATOMIC
125 : HARDCLAUSE_MIMG_LOAD
126 : HARDCLAUSE_MIMG_STORE;
127 }
128 if ((SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) ||
129 SIInstrInfo::isSegmentSpecificFLAT(MI)) {
130 return MI.mayLoad() ? MI.mayStore() ? HARDCLAUSE_VMEM_ATOMIC
131 : HARDCLAUSE_VMEM_LOAD
132 : HARDCLAUSE_VMEM_STORE;
133 }
134 if (SIInstrInfo::isFLAT(MI)) {
135 return MI.mayLoad() ? MI.mayStore() ? HARDCLAUSE_FLAT_ATOMIC
136 : HARDCLAUSE_FLAT_LOAD
137 : HARDCLAUSE_FLAT_STORE;
138 }
139 }
140 // TODO: LDS
141 if (SIInstrInfo::isSMRD(MI))
142 return HARDCLAUSE_SMEM;
143 }
144
145 // Don't form VALU clauses. It's not clear what benefit they give, if any.
146
147 // In practice s_nop is the only internal instruction we're likely to see.
148 // It's safe to treat the rest as illegal.
149 if (MI.getOpcode() == AMDGPU::S_NOP)
150 return HARDCLAUSE_INTERNAL;
151 if (MI.isMetaInstruction())
152 return HARDCLAUSE_IGNORE;
153 return HARDCLAUSE_ILLEGAL;
154 }
155
156 // Track information about a clause as we discover it.
157 struct ClauseInfo {
158 // The type of all (non-internal) instructions in the clause.
159 HardClauseType Type = HARDCLAUSE_ILLEGAL;
160 // The first (necessarily non-internal) instruction in the clause.
161 MachineInstr *First = nullptr;
162 // The last non-internal instruction in the clause.
163 MachineInstr *Last = nullptr;
164 // The length of the clause including any internal instructions in the
165 // middle (but not at the end) of the clause.
166 unsigned Length = 0;
167 // Internal instructions at the and of a clause should not be included in
168 // the clause. Count them in TrailingInternalLength until a new memory
169 // instruction is added.
170 unsigned TrailingInternalLength = 0;
171 // The base operands of *Last.
172 SmallVector<const MachineOperand *, 4> BaseOps;
173 };
174
175 bool emitClause(const ClauseInfo &CI, const SIInstrInfo *SII) {
176 if (CI.First == CI.Last)
177 return false;
178 assert(CI.Length <= ST->maxHardClauseLength() &&
179 "Hard clause is too long!");
180
181 auto &MBB = *CI.First->getParent();
182 auto ClauseMI =
183 BuildMI(BB&: MBB, I&: *CI.First, MIMD: DebugLoc(), MCID: SII->get(Opcode: AMDGPU::S_CLAUSE))
184 .addImm(Val: CI.Length - 1);
185 finalizeBundle(MBB, FirstMI: ClauseMI->getIterator(),
186 LastMI: std::next(x: CI.Last->getIterator()));
187 return true;
188 }
189
190 // \return if scopes are different on gfx1250 and disallowed to be claused.
191 bool isIncompatibleScope(const MachineInstr &MI1, const MachineInstr &MI2,
192 const SIInstrInfo *SII) const {
193 assert(ST->getGeneration() == AMDGPUSubtarget::GFX12 &&
194 ST->hasGFX1250Insts());
195 int CPol1 = 0, CPol2 = 0;
196 if (const MachineOperand *Op =
197 SII->getNamedOperand(MI: MI1, OperandName: AMDGPU::OpName::cpol)) {
198 CPol1 = Op->getImm() & AMDGPU::CPol::SCOPE;
199 }
200 if (const MachineOperand *Op =
201 SII->getNamedOperand(MI: MI2, OperandName: AMDGPU::OpName::cpol)) {
202 CPol2 = Op->getImm() & AMDGPU::CPol::SCOPE;
203 }
204 return CPol1 != CPol2;
205 }
206
207 bool run(MachineFunction &MF) {
208 ST = &MF.getSubtarget<GCNSubtarget>();
209 if (!ST->hasHardClauses())
210 return false;
211
212 unsigned MaxClauseLength =
213 ST->getCLOpts().hard_clause_length_limit.value_or(
214 u: MF.getFunction().getFnAttributeAsParsedInteger(
215 Kind: "amdgpu-hard-clause-length-limit", Default: 255));
216 MaxClauseLength = std::min(a: MaxClauseLength, b: ST->maxHardClauseLength());
217 if (MaxClauseLength <= 1)
218 return false;
219
220 const SIInstrInfo *SII = ST->getInstrInfo();
221
222 bool Changed = false;
223 for (auto &MBB : MF) {
224 ClauseInfo CI;
225 unsigned ExistingClauseRemaining = 0;
226 for (auto &MI : MBB) {
227 HardClauseType Type;
228 if (ExistingClauseRemaining) {
229 if (!MI.isMetaInstruction())
230 ExistingClauseRemaining--;
231 Type = HARDCLAUSE_ILLEGAL;
232 } else if (MI.getOpcode() == AMDGPU::S_CLAUSE) {
233 // Respect existing explicit clauses. Re-clausing instructions that
234 // are already covered by an S_CLAUSE can create nested clauses.
235 ExistingClauseRemaining = (MI.getOperand(i: 0).getImm() & 63) + 1;
236 Type = HARDCLAUSE_ILLEGAL;
237 } else {
238 Type = getHardClauseType(MI);
239 }
240
241 int64_t Dummy1;
242 bool Dummy2;
243 LocationSize Dummy3 = LocationSize::precise(Value: 0);
244 SmallVector<const MachineOperand *, 4> BaseOps;
245 if (Type <= LAST_REAL_HARDCLAUSE_TYPE) {
246 if (!SII->getMemOperandsWithOffsetWidth(LdSt: MI, BaseOps, Offset&: Dummy1, OffsetIsScalable&: Dummy2,
247 Width&: Dummy3)) {
248 // We failed to get the base operands, so we'll never clause this
249 // instruction with any other, so pretend it's illegal.
250 Type = HARDCLAUSE_ILLEGAL;
251 }
252 }
253
254 if (CI.Length == MaxClauseLength ||
255 (CI.Length && Type != HARDCLAUSE_INTERNAL &&
256 Type != HARDCLAUSE_IGNORE &&
257 (Type != CI.Type ||
258 // shouldClusterMemOps() would also apply the pre-RA memory
259 // cluster budget, which must not limit clauses formed after
260 // register allocation.
261 !SII->memOpsHaveSameBase(BaseOps1: CI.BaseOps, BaseOps2: BaseOps))) ||
262 (CI.Length && ST->hasGFX1250_STRICT() &&
263 isIncompatibleScope(MI1: MI, MI2: *CI.Last, SII))) {
264 // Finish the current clause.
265 Changed |= emitClause(CI, SII);
266 CI = ClauseInfo();
267 }
268
269 if (CI.Length) {
270 // Extend the current clause.
271 if (Type != HARDCLAUSE_IGNORE) {
272 if (Type == HARDCLAUSE_INTERNAL) {
273 ++CI.TrailingInternalLength;
274 } else {
275 ++CI.Length;
276 CI.Length += CI.TrailingInternalLength;
277 CI.TrailingInternalLength = 0;
278 CI.Last = &MI;
279 CI.BaseOps = std::move(BaseOps);
280 }
281 }
282 } else if (Type <= LAST_REAL_HARDCLAUSE_TYPE) {
283 // Start a new clause.
284 CI = ClauseInfo{.Type: Type, .First: &MI, .Last: &MI, .Length: 1, .TrailingInternalLength: 0, .BaseOps: std::move(BaseOps)};
285 }
286 }
287
288 // Finish the last clause in the basic block if any.
289 if (CI.Length)
290 Changed |= emitClause(CI, SII);
291 }
292
293 return Changed;
294 }
295};
296
297class SIInsertHardClausesLegacy : public MachineFunctionPass {
298public:
299 static char ID;
300 SIInsertHardClausesLegacy() : MachineFunctionPass(ID) {}
301
302 bool runOnMachineFunction(MachineFunction &MF) override {
303 if (skipFunction(F: MF.getFunction()))
304 return false;
305
306 return SIInsertHardClauses().run(MF);
307 }
308
309 void getAnalysisUsage(AnalysisUsage &AU) const override {
310 AU.setPreservesCFG();
311 MachineFunctionPass::getAnalysisUsage(AU);
312 }
313};
314
315} // namespace
316
317PreservedAnalyses
318llvm::SIInsertHardClausesPass::run(MachineFunction &MF,
319 MachineFunctionAnalysisManager &MFAM) {
320 if (!SIInsertHardClauses().run(MF))
321 return PreservedAnalyses::all();
322
323 auto PA = getMachineFunctionPassPreservedAnalyses();
324 PA.preserveSet<CFGAnalyses>();
325 return PA;
326}
327
328char SIInsertHardClausesLegacy::ID = 0;
329
330char &llvm::SIInsertHardClausesID = SIInsertHardClausesLegacy::ID;
331
332INITIALIZE_PASS(SIInsertHardClausesLegacy, DEBUG_TYPE, "SI Insert Hard Clauses",
333 false, false)
334