1//===-- SIFormMemoryClauses.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass extends the live ranges of registers used as pointers in
10/// sequences of adjacent SMEM and VMEM instructions if XNACK is enabled. A
11/// load that would overwrite a pointer would require breaking the soft clause.
12/// Artificially extend the live ranges of the pointer operands by adding
13/// implicit-def early-clobber operands throughout the soft clause.
14///
15//===----------------------------------------------------------------------===//
16
17#include "SIFormMemoryClauses.h"
18#include "AMDGPU.h"
19#include "GCNRegPressure.h"
20#include "SIMachineFunctionInfo.h"
21#include "llvm/InitializePasses.h"
22
23using namespace llvm;
24
25#define DEBUG_TYPE "si-form-memory-clauses"
26
27// Clauses longer then 15 instructions would overflow one of the counters
28// and stall. They can stall even earlier if there are outstanding counters.
29static cl::opt<unsigned>
30MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(Val: 15),
31 cl::desc("Maximum length of a memory clause, instructions"));
32
33namespace {
34
35class SIFormMemoryClausesImpl {
36 using RegUse = DenseMap<unsigned, std::pair<RegState, LaneBitmask>>;
37
38 bool canBundle(const MachineInstr &MI, const RegUse &Defs,
39 const RegUse &Uses) const;
40 bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
41 void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
42 bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
43 GCNDownwardRPTracker &RPT);
44
45 const GCNSubtarget *ST;
46 const SIRegisterInfo *TRI;
47 const MachineRegisterInfo *MRI;
48 SIMachineFunctionInfo *MFI;
49 LiveIntervals *LIS;
50
51 unsigned LastRecordedOccupancy;
52 unsigned MaxVGPRs;
53 unsigned MaxSGPRs;
54
55public:
56 SIFormMemoryClausesImpl(LiveIntervals *LS) : LIS(LS) {}
57 bool run(MachineFunction &MF);
58};
59
60class SIFormMemoryClausesLegacy : public MachineFunctionPass {
61public:
62 static char ID;
63
64 SIFormMemoryClausesLegacy() : MachineFunctionPass(ID) {}
65
66 bool runOnMachineFunction(MachineFunction &MF) override;
67
68 StringRef getPassName() const override {
69 return "SI Form memory clauses";
70 }
71
72 void getAnalysisUsage(AnalysisUsage &AU) const override {
73 AU.addRequired<LiveIntervalsWrapperPass>();
74 AU.setPreservesAll();
75 MachineFunctionPass::getAnalysisUsage(AU);
76 }
77
78 MachineFunctionProperties getClearedProperties() const override {
79 return MachineFunctionProperties().setIsSSA();
80 }
81};
82
83} // End anonymous namespace.
84
85INITIALIZE_PASS_BEGIN(SIFormMemoryClausesLegacy, DEBUG_TYPE,
86 "SI Form memory clauses", false, false)
87INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
88INITIALIZE_PASS_END(SIFormMemoryClausesLegacy, DEBUG_TYPE,
89 "SI Form memory clauses", false, false)
90
91char SIFormMemoryClausesLegacy::ID = 0;
92
93char &llvm::SIFormMemoryClausesID = SIFormMemoryClausesLegacy::ID;
94
95FunctionPass *llvm::createSIFormMemoryClausesLegacyPass() {
96 return new SIFormMemoryClausesLegacy();
97}
98
99static bool isVMEMClauseInst(const MachineInstr &MI) {
100 return SIInstrInfo::isVMEM(MI);
101}
102
103static bool isSMEMClauseInst(const MachineInstr &MI) {
104 return SIInstrInfo::isSMRD(MI);
105}
106
107// There no sense to create store clauses, they do not define anything,
108// thus there is nothing to set early-clobber.
109static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
110 assert(!MI.isDebugInstr() && "debug instructions should not reach here");
111 if (MI.isBundled())
112 return false;
113 if (!MI.mayLoad() || MI.mayStore())
114 return false;
115 if (SIInstrInfo::isAtomic(MI))
116 return false;
117 if (IsVMEMClause && !isVMEMClauseInst(MI))
118 return false;
119 if (!IsVMEMClause && !isSMEMClauseInst(MI))
120 return false;
121 // If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it.
122 for (const MachineOperand &ResMO : MI.defs()) {
123 Register ResReg = ResMO.getReg();
124 for (const MachineOperand &MO : MI.all_uses()) {
125 if (MO.getReg() == ResReg)
126 return false;
127 }
128 break; // Only check the first def.
129 }
130 return true;
131}
132
133static RegState getMopState(const MachineOperand &MO) {
134 RegState S = {};
135 if (MO.isImplicit())
136 S |= RegState::Implicit;
137 if (MO.isDead())
138 S |= RegState::Dead;
139 if (MO.isUndef())
140 S |= RegState::Undef;
141 if (MO.isKill())
142 S |= RegState::Kill;
143 if (MO.isEarlyClobber())
144 S |= RegState::EarlyClobber;
145 if (MO.getReg().isPhysical() && MO.isRenamable())
146 S |= RegState::Renamable;
147 return S;
148}
149
150// Returns false if there is a use of a def already in the map.
151// In this case we must break the clause.
152bool SIFormMemoryClausesImpl::canBundle(const MachineInstr &MI,
153 const RegUse &Defs,
154 const RegUse &Uses) const {
155 // Check interference with defs.
156 for (const MachineOperand &MO : MI.operands()) {
157 // TODO: Prologue/Epilogue Insertion pass does not process bundled
158 // instructions.
159 if (MO.isFI())
160 return false;
161
162 if (!MO.isReg())
163 continue;
164
165 Register Reg = MO.getReg();
166
167 // If it is tied we will need to write same register as we read.
168 if (MO.isTied())
169 return false;
170
171 const RegUse &Map = MO.isDef() ? Uses : Defs;
172 auto Conflict = Map.find(Val: Reg);
173 if (Conflict == Map.end())
174 continue;
175
176 if (Reg.isPhysical())
177 return false;
178
179 LaneBitmask Mask = TRI->getSubRegIndexLaneMask(SubIdx: MO.getSubReg());
180 if ((Conflict->second.second & Mask).any())
181 return false;
182 }
183
184 return true;
185}
186
187// Since all defs in the clause are early clobber we can run out of registers.
188// Function returns false if pressure would hit the limit if instruction is
189// bundled into a memory clause.
190bool SIFormMemoryClausesImpl::checkPressure(const MachineInstr &MI,
191 GCNDownwardRPTracker &RPT) {
192 // NB: skip advanceBeforeNext() call. Since all defs will be marked
193 // early-clobber they will all stay alive at least to the end of the
194 // clause. Therefor we should not decrease pressure even if load
195 // pointer becomes dead and could otherwise be reused for destination.
196 RPT.advanceToNext();
197 GCNRegPressure MaxPressure = RPT.moveMaxPressure();
198 unsigned Occupancy = MaxPressure.getOccupancy(
199 ST: *ST,
200 DynamicVGPRBlockSize: MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
201
202 // Don't push over half the register budget. We don't want to introduce
203 // spilling just to form a soft clause.
204 //
205 // FIXME: This pressure check is fundamentally broken. First, this is checking
206 // the global pressure, not the pressure at this specific point in the
207 // program. Second, it's not accounting for the increased liveness of the use
208 // operands due to the early clobber we will introduce. Third, the pressure
209 // tracking does not account for the alignment requirements for SGPRs, or the
210 // fragmentation of registers the allocator will need to satisfy.
211 if (Occupancy >= MFI->getMinAllowedOccupancy() &&
212 MaxPressure.getVGPRNum(UnifiedVGPRFile: ST->hasGFX90AInsts()) <= MaxVGPRs / 2 &&
213 MaxPressure.getSGPRNum() <= MaxSGPRs / 2) {
214 LastRecordedOccupancy = Occupancy;
215 return true;
216 }
217 return false;
218}
219
220// Collect register defs and uses along with their lane masks and states.
221void SIFormMemoryClausesImpl::collectRegUses(const MachineInstr &MI,
222 RegUse &Defs, RegUse &Uses) const {
223 for (const MachineOperand &MO : MI.operands()) {
224 if (!MO.isReg())
225 continue;
226 Register Reg = MO.getReg();
227 if (!Reg)
228 continue;
229
230 LaneBitmask Mask = Reg.isVirtual()
231 ? TRI->getSubRegIndexLaneMask(SubIdx: MO.getSubReg())
232 : LaneBitmask::getAll();
233 RegUse &Map = MO.isDef() ? Defs : Uses;
234
235 RegState State = getMopState(MO);
236 auto [Loc, Inserted] = Map.try_emplace(Key: Reg, Args&: State, Args&: Mask);
237 if (!Inserted) {
238 Loc->second.first |= State;
239 Loc->second.second |= Mask;
240 }
241 }
242}
243
244// Check register def/use conflicts, occupancy limits and collect def/use maps.
245// Return true if instruction can be bundled with previous. If it cannot
246// def/use maps are not updated.
247bool SIFormMemoryClausesImpl::processRegUses(const MachineInstr &MI,
248 RegUse &Defs, RegUse &Uses,
249 GCNDownwardRPTracker &RPT) {
250 if (!canBundle(MI, Defs, Uses))
251 return false;
252
253 if (!checkPressure(MI, RPT))
254 return false;
255
256 collectRegUses(MI, Defs, Uses);
257 return true;
258}
259
260bool SIFormMemoryClausesImpl::run(MachineFunction &MF) {
261 ST = &MF.getSubtarget<GCNSubtarget>();
262 if (!ST->isXNACKEnabled())
263 return false;
264
265 const SIInstrInfo *TII = ST->getInstrInfo();
266 TRI = ST->getRegisterInfo();
267 MRI = &MF.getRegInfo();
268 MFI = MF.getInfo<SIMachineFunctionInfo>();
269 SlotIndexes *Ind = LIS->getSlotIndexes();
270 bool Changed = false;
271
272 MaxVGPRs = TRI->getAllocatableSet(MF, RC: &AMDGPU::VGPR_32RegClass).count();
273 MaxSGPRs = TRI->getAllocatableSet(MF, RC: &AMDGPU::SGPR_32RegClass).count();
274 unsigned FuncMaxClause = MF.getFunction().getFnAttributeAsParsedInteger(
275 Kind: "amdgpu-max-memory-clause", Default: MaxClause);
276
277 for (MachineBasicBlock &MBB : MF) {
278 GCNDownwardRPTracker RPT(*LIS);
279 MachineBasicBlock::instr_iterator Next;
280 for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
281 MachineInstr &MI = *I;
282 Next = std::next(x: I);
283
284 if (MI.isMetaInstruction())
285 continue;
286
287 bool IsVMEM = isVMEMClauseInst(MI);
288
289 if (!isValidClauseInst(MI, IsVMEMClause: IsVMEM))
290 continue;
291
292 if (!RPT.getNext().isValid())
293 RPT.reset(MI);
294 else { // Advance the state to the current MI.
295 RPT.advance(End: MachineBasicBlock::const_iterator(MI));
296 RPT.advanceBeforeNext();
297 }
298
299 const GCNRPTracker::LiveRegSet LiveRegsCopy(RPT.getLiveRegs());
300 RegUse Defs, Uses;
301 if (!processRegUses(MI, Defs, Uses, RPT)) {
302 RPT.reset(MI, LiveRegs: &LiveRegsCopy);
303 continue;
304 }
305
306 MachineBasicBlock::iterator LastClauseInst = Next;
307 unsigned Length = 1;
308 for ( ; Next != E && Length < FuncMaxClause; ++Next) {
309 // Debug instructions should not change the kill insertion.
310 if (Next->isMetaInstruction())
311 continue;
312
313 if (!isValidClauseInst(MI: *Next, IsVMEMClause: IsVMEM))
314 break;
315
316 // A load from pointer which was loaded inside the same bundle is an
317 // impossible clause because we will need to write and read the same
318 // register inside. In this case processRegUses will return false.
319 if (!processRegUses(MI: *Next, Defs, Uses, RPT))
320 break;
321
322 LastClauseInst = Next;
323 ++Length;
324 }
325 if (Length < 2) {
326 RPT.reset(MI, LiveRegs: &LiveRegsCopy);
327 continue;
328 }
329
330 Changed = true;
331 MFI->limitOccupancy(Limit: LastRecordedOccupancy);
332
333 assert(!LastClauseInst->isMetaInstruction());
334
335 SlotIndex ClauseLiveInIdx = LIS->getInstructionIndex(Instr: MI);
336 SlotIndex ClauseLiveOutIdx =
337 LIS->getInstructionIndex(Instr: *LastClauseInst).getNextIndex();
338
339 // Track the last inserted kill.
340 MachineInstrBuilder Kill;
341
342 // Insert one kill per register, with operands covering all necessary
343 // subregisters.
344 for (auto &&R : Uses) {
345 Register Reg = R.first;
346 if (Reg.isPhysical())
347 continue;
348
349 // Collect the register operands we should extend the live ranges of.
350 SmallVector<std::tuple<RegState, unsigned>> KillOps;
351 const LiveInterval &LI = LIS->getInterval(Reg: R.first);
352
353 if (!LI.hasSubRanges()) {
354 if (!LI.liveAt(index: ClauseLiveOutIdx)) {
355 KillOps.emplace_back(Args: R.second.first | RegState::Kill,
356 Args: AMDGPU::NoSubRegister);
357 }
358 } else {
359 LaneBitmask KilledMask;
360 for (const LiveInterval::SubRange &SR : LI.subranges()) {
361 if (SR.liveAt(index: ClauseLiveInIdx) && !SR.liveAt(index: ClauseLiveOutIdx))
362 KilledMask |= SR.LaneMask;
363 }
364
365 if (KilledMask.none())
366 continue;
367
368 SmallVector<unsigned> KilledIndexes;
369 bool Success = TRI->getCoveringSubRegIndexes(
370 RC: MRI->getRegClass(Reg), LaneMask: KilledMask, Indexes&: KilledIndexes);
371 (void)Success;
372 assert(Success && "Failed to find subregister mask to cover lanes");
373 for (unsigned SubReg : KilledIndexes) {
374 KillOps.emplace_back(Args: R.second.first | RegState::Kill, Args&: SubReg);
375 }
376 }
377
378 if (KillOps.empty())
379 continue;
380
381 // We only want to extend the live ranges of used registers. If they
382 // already have existing uses beyond the bundle, we don't need the kill.
383 //
384 // It's possible all of the use registers were already live past the
385 // bundle.
386 Kill = BuildMI(BB&: *MI.getParent(), I: std::next(x: LastClauseInst),
387 MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::KILL));
388 for (auto &Op : KillOps)
389 Kill.addUse(RegNo: Reg, Flags: std::get<0>(t&: Op), SubReg: std::get<1>(t&: Op));
390 Ind->insertMachineInstrInMaps(MI&: *Kill);
391 }
392
393 // Restore the state after processing the end of the bundle.
394 RPT.reset(MI, LiveRegs: &LiveRegsCopy);
395
396 if (!Kill)
397 continue;
398
399 for (auto &&R : Defs) {
400 Register Reg = R.first;
401 Uses.erase(Val: Reg);
402 if (Reg.isPhysical())
403 continue;
404 LIS->removeInterval(Reg);
405 LIS->createAndComputeVirtRegInterval(Reg);
406 }
407
408 for (auto &&R : Uses) {
409 Register Reg = R.first;
410 if (Reg.isPhysical())
411 continue;
412 LIS->removeInterval(Reg);
413 LIS->createAndComputeVirtRegInterval(Reg);
414 }
415 }
416 }
417
418 return Changed;
419}
420
421bool SIFormMemoryClausesLegacy::runOnMachineFunction(MachineFunction &MF) {
422 if (skipFunction(F: MF.getFunction()))
423 return false;
424
425 LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
426 return SIFormMemoryClausesImpl(LIS).run(MF);
427}
428
429PreservedAnalyses
430SIFormMemoryClausesPass::run(MachineFunction &MF,
431 MachineFunctionAnalysisManager &MFAM) {
432 LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(IR&: MF);
433 SIFormMemoryClausesImpl(&LIS).run(MF);
434 return PreservedAnalyses::all();
435}
436