1//===-- SIFormMemoryClauses.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass extends the live ranges of registers used as pointers in
10/// sequences of adjacent SMEM and VMEM instructions if XNACK is enabled. A
11/// load that would overwrite a pointer would require breaking the soft clause.
12/// Artificially extend the live ranges of the pointer operands by adding
13/// implicit-def early-clobber operands throughout the soft clause.
14///
15//===----------------------------------------------------------------------===//
16
17#include "SIFormMemoryClauses.h"
18#include "AMDGPU.h"
19#include "GCNRegPressure.h"
20#include "SIMachineFunctionInfo.h"
21#include "llvm/InitializePasses.h"
22
23using namespace llvm;
24
25#define DEBUG_TYPE "si-form-memory-clauses"
26
27namespace {
28
29class SIFormMemoryClausesImpl {
30 using RegUse = DenseMap<unsigned, std::pair<RegState, LaneBitmask>>;
31
32 bool canBundle(const MachineInstr &MI, const RegUse &Defs,
33 const RegUse &Uses) const;
34 bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
35 void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
36 bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
37 GCNDownwardRPTracker &RPT);
38
39 const GCNSubtarget *ST;
40 const SIRegisterInfo *TRI;
41 const MachineRegisterInfo *MRI;
42 SIMachineFunctionInfo *MFI;
43 LiveIntervals *LIS;
44
45 unsigned LastRecordedOccupancy;
46 unsigned MaxVGPRs;
47 unsigned MaxSGPRs;
48
49public:
50 SIFormMemoryClausesImpl(LiveIntervals *LS) : LIS(LS) {}
51 bool run(MachineFunction &MF);
52};
53
54class SIFormMemoryClausesLegacy : public MachineFunctionPass {
55public:
56 static char ID;
57
58 SIFormMemoryClausesLegacy() : MachineFunctionPass(ID) {}
59
60 bool runOnMachineFunction(MachineFunction &MF) override;
61
62 StringRef getPassName() const override {
63 return "SI Form memory clauses";
64 }
65
66 void getAnalysisUsage(AnalysisUsage &AU) const override {
67 AU.addRequired<LiveIntervalsWrapperPass>();
68 AU.setPreservesAll();
69 MachineFunctionPass::getAnalysisUsage(AU);
70 }
71
72 MachineFunctionProperties getClearedProperties() const override {
73 return MachineFunctionProperties().setIsSSA();
74 }
75};
76
77} // End anonymous namespace.
78
79INITIALIZE_PASS_BEGIN(SIFormMemoryClausesLegacy, DEBUG_TYPE,
80 "SI Form memory clauses", false, false)
81INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
82INITIALIZE_PASS_END(SIFormMemoryClausesLegacy, DEBUG_TYPE,
83 "SI Form memory clauses", false, false)
84
85char SIFormMemoryClausesLegacy::ID = 0;
86
87char &llvm::SIFormMemoryClausesID = SIFormMemoryClausesLegacy::ID;
88
89static bool isVMEMClauseInst(const MachineInstr &MI) {
90 return SIInstrInfo::isVMEM(MI);
91}
92
93static bool isSMEMClauseInst(const MachineInstr &MI) {
94 return SIInstrInfo::isSMRD(MI);
95}
96
97// There no sense to create store clauses, they do not define anything,
98// thus there is nothing to set early-clobber.
99static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
100 assert(!MI.isDebugInstr() && "debug instructions should not reach here");
101 if (MI.isBundled())
102 return false;
103 if (!MI.mayLoad() || MI.mayStore())
104 return false;
105 if (SIInstrInfo::isAtomic(MI))
106 return false;
107 if (IsVMEMClause && !isVMEMClauseInst(MI))
108 return false;
109 if (!IsVMEMClause && !isSMEMClauseInst(MI))
110 return false;
111 // If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it.
112 for (const MachineOperand &ResMO : MI.defs()) {
113 Register ResReg = ResMO.getReg();
114 for (const MachineOperand &MO : MI.all_uses()) {
115 if (MO.getReg() == ResReg)
116 return false;
117 }
118 break; // Only check the first def.
119 }
120 return true;
121}
122
123static RegState getMopState(const MachineOperand &MO) {
124 RegState S = {};
125 if (MO.isImplicit())
126 S |= RegState::Implicit;
127 if (MO.isDead())
128 S |= RegState::Dead;
129 if (MO.isUndef())
130 S |= RegState::Undef;
131 if (MO.isKill())
132 S |= RegState::Kill;
133 if (MO.isEarlyClobber())
134 S |= RegState::EarlyClobber;
135 if (MO.getReg().isPhysical() && MO.isRenamable())
136 S |= RegState::Renamable;
137 return S;
138}
139
140// Returns false if there is a use of a def already in the map.
141// In this case we must break the clause.
142bool SIFormMemoryClausesImpl::canBundle(const MachineInstr &MI,
143 const RegUse &Defs,
144 const RegUse &Uses) const {
145 // Check interference with defs.
146 for (const MachineOperand &MO : MI.operands()) {
147 // TODO: Prologue/Epilogue Insertion pass does not process bundled
148 // instructions.
149 if (MO.isFI())
150 return false;
151
152 if (!MO.isReg())
153 continue;
154
155 Register Reg = MO.getReg();
156
157 // If it is tied we will need to write same register as we read.
158 if (MO.isTied())
159 return false;
160
161 const RegUse &Map = MO.isDef() ? Uses : Defs;
162 auto Conflict = Map.find(Val: Reg);
163 if (Conflict == Map.end())
164 continue;
165
166 if (Reg.isPhysical())
167 return false;
168
169 LaneBitmask Mask = TRI->getSubRegIndexLaneMask(SubIdx: MO.getSubReg());
170 if ((Conflict->second.second & Mask).any())
171 return false;
172 }
173
174 return true;
175}
176
177// Since all defs in the clause are early clobber we can run out of registers.
178// Function returns false if pressure would hit the limit if instruction is
179// bundled into a memory clause.
180bool SIFormMemoryClausesImpl::checkPressure(const MachineInstr &MI,
181 GCNDownwardRPTracker &RPT) {
182 // NB: do not retire the uses of MI. Since all defs will be marked
183 // early-clobber they will all stay alive at least to the end of the clause.
184 // Therefore we should not decrease pressure even if load pointer becomes
185 // dead and could otherwise be reused for destination. The tracker is reset
186 // after the clause is formed, so its position does not need to be updated.
187 RPT.addDefs(MI);
188 GCNRegPressure MaxPressure = RPT.moveMaxPressure();
189 unsigned Occupancy = MaxPressure.getOccupancy(
190 ST: *ST,
191 DynamicVGPRBlockSize: MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
192
193 // Don't push over half the register budget. We don't want to introduce
194 // spilling just to form a soft clause.
195 //
196 // FIXME: This pressure check is fundamentally broken. First, this is checking
197 // the global pressure, not the pressure at this specific point in the
198 // program. Second, it's not accounting for the increased liveness of the use
199 // operands due to the early clobber we will introduce. Third, the pressure
200 // tracking does not account for the alignment requirements for SGPRs, or the
201 // fragmentation of registers the allocator will need to satisfy.
202 if (Occupancy >= MFI->getMinAllowedOccupancy() &&
203 MaxPressure.getVGPRNum(UnifiedVGPRFile: ST->hasGFX90AInsts()) <= MaxVGPRs / 2 &&
204 MaxPressure.getSGPRNum() <= MaxSGPRs / 2) {
205 LastRecordedOccupancy = Occupancy;
206 return true;
207 }
208 return false;
209}
210
211// Collect register defs and uses along with their lane masks and states.
212void SIFormMemoryClausesImpl::collectRegUses(const MachineInstr &MI,
213 RegUse &Defs, RegUse &Uses) const {
214 for (const MachineOperand &MO : MI.operands()) {
215 if (!MO.isReg())
216 continue;
217 Register Reg = MO.getReg();
218 if (!Reg)
219 continue;
220
221 LaneBitmask Mask = Reg.isVirtual()
222 ? TRI->getSubRegIndexLaneMask(SubIdx: MO.getSubReg())
223 : LaneBitmask::getAll();
224 RegUse &Map = MO.isDef() ? Defs : Uses;
225
226 RegState State = getMopState(MO);
227 auto [Loc, Inserted] = Map.try_emplace(Key: Reg, Args&: State, Args&: Mask);
228 if (!Inserted) {
229 Loc->second.first |= State;
230 Loc->second.second |= Mask;
231 }
232 }
233}
234
235// Check register def/use conflicts, occupancy limits and collect def/use maps.
236// Return true if instruction can be bundled with previous. If it cannot
237// def/use maps are not updated.
238bool SIFormMemoryClausesImpl::processRegUses(const MachineInstr &MI,
239 RegUse &Defs, RegUse &Uses,
240 GCNDownwardRPTracker &RPT) {
241 if (!canBundle(MI, Defs, Uses))
242 return false;
243
244 if (!checkPressure(MI, RPT))
245 return false;
246
247 collectRegUses(MI, Defs, Uses);
248 return true;
249}
250
251bool SIFormMemoryClausesImpl::run(MachineFunction &MF) {
252 ST = &MF.getSubtarget<GCNSubtarget>();
253 if (!ST->isXNACKEnabled())
254 return false;
255
256 const SIInstrInfo *TII = ST->getInstrInfo();
257 TRI = ST->getRegisterInfo();
258 MRI = &MF.getRegInfo();
259 MFI = MF.getInfo<SIMachineFunctionInfo>();
260 SlotIndexes *Ind = LIS->getSlotIndexes();
261 bool Changed = false;
262
263 MaxVGPRs = TRI->getAllocatableSet(MF, RC: &AMDGPU::VGPR_32RegClass).count();
264 MaxSGPRs = TRI->getAllocatableSet(MF, RC: &AMDGPU::SGPR_32RegClass).count();
265 unsigned FuncMaxClause = MF.getFunction().getFnAttributeAsParsedInteger(
266 Kind: "amdgpu-max-memory-clause", Default: ST->getCLOpts().max_memory_clause);
267
268 for (MachineBasicBlock &MBB : MF) {
269 GCNDownwardRPTracker RPT(*LIS);
270 MachineBasicBlock::instr_iterator Next;
271 for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
272 MachineInstr &MI = *I;
273 Next = std::next(x: I);
274
275 if (MI.isMetaInstruction())
276 continue;
277
278 bool IsVMEM = isVMEMClauseInst(MI);
279
280 if (!isValidClauseInst(MI, IsVMEMClause: IsVMEM))
281 continue;
282
283 if (!RPT.getNext().isValid())
284 RPT.reset(MI, End: MBB.end());
285 else { // Advance the state to the current MI.
286 RPT.advance(End: MachineBasicBlock::const_iterator(MI));
287 RPT.advanceBeforeNext();
288 }
289
290 const GCNRPTracker::LiveRegSet LiveRegsCopy(RPT.getLiveRegs());
291 RegUse Defs, Uses;
292 if (!processRegUses(MI, Defs, Uses, RPT)) {
293 RPT.reset(MI, End: MBB.end(), LiveRegsCopy: &LiveRegsCopy);
294 continue;
295 }
296
297 MachineBasicBlock::iterator LastClauseInst = Next;
298 unsigned Length = 1;
299 for ( ; Next != E && Length < FuncMaxClause; ++Next) {
300 // Debug instructions should not change the kill insertion.
301 if (Next->isMetaInstruction())
302 continue;
303
304 if (!isValidClauseInst(MI: *Next, IsVMEMClause: IsVMEM))
305 break;
306
307 // A load from pointer which was loaded inside the same bundle is an
308 // impossible clause because we will need to write and read the same
309 // register inside. In this case processRegUses will return false.
310 if (!processRegUses(MI: *Next, Defs, Uses, RPT))
311 break;
312
313 LastClauseInst = Next;
314 ++Length;
315 }
316 if (Length < 2) {
317 RPT.reset(MI, End: MBB.end(), LiveRegsCopy: &LiveRegsCopy);
318 continue;
319 }
320
321 Changed = true;
322 MFI->limitOccupancy(Limit: LastRecordedOccupancy);
323
324 assert(!LastClauseInst->isMetaInstruction());
325
326 SlotIndex ClauseLiveInIdx = LIS->getInstructionIndex(Instr: MI);
327 SlotIndex ClauseLiveOutIdx =
328 LIS->getInstructionIndex(Instr: *LastClauseInst).getNextIndex();
329
330 // Track the last inserted kill.
331 MachineInstrBuilder Kill;
332
333 // Insert one kill per register, with operands covering all necessary
334 // subregisters.
335 for (auto &&R : Uses) {
336 Register Reg = R.first;
337 if (Reg.isPhysical())
338 continue;
339
340 // Collect the register operands we should extend the live ranges of.
341 SmallVector<std::tuple<RegState, unsigned>> KillOps;
342 const LiveInterval &LI = LIS->getInterval(Reg: R.first);
343
344 if (!LI.hasSubRanges()) {
345 if (!LI.liveAt(index: ClauseLiveOutIdx)) {
346 KillOps.emplace_back(Args&: R.second.first, Args: AMDGPU::NoSubRegister);
347 }
348 } else {
349 LaneBitmask KilledMask;
350 for (const LiveInterval::SubRange &SR : LI.subranges()) {
351 if (SR.liveAt(index: ClauseLiveInIdx) && !SR.liveAt(index: ClauseLiveOutIdx))
352 KilledMask |= SR.LaneMask;
353 }
354
355 if (KilledMask.none())
356 continue;
357
358 SmallVector<unsigned> KilledIndexes;
359 bool Success = TRI->getCoveringSubRegIndexes(
360 RC: MRI->getRegClass(Reg), LaneMask: KilledMask, Indexes&: KilledIndexes);
361 (void)Success;
362 assert(Success && "Failed to find subregister mask to cover lanes");
363 for (unsigned SubReg : KilledIndexes) {
364 KillOps.emplace_back(Args&: R.second.first, Args&: SubReg);
365 }
366 }
367
368 if (KillOps.empty())
369 continue;
370
371 // We only want to extend the live ranges of used registers. If they
372 // already have existing uses beyond the bundle, we don't need the kill.
373 //
374 // It's possible all of the use registers were already live past the
375 // bundle.
376 Kill = BuildMI(BB&: *MI.getParent(), I: std::next(x: LastClauseInst),
377 MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::KILL));
378 for (auto [State, SubReg] : KillOps)
379 Kill.addUse(RegNo: Reg, Flags: State, SubReg);
380 Ind->insertMachineInstrInMaps(MI&: *Kill);
381 }
382
383 // Restore the state after processing the end of the bundle.
384 RPT.reset(MI, End: MBB.end(), LiveRegsCopy: &LiveRegsCopy);
385
386 if (!Kill)
387 continue;
388
389 for (auto &&R : Defs) {
390 Register Reg = R.first;
391 Uses.erase(Val: Reg);
392 if (Reg.isPhysical())
393 continue;
394 LIS->removeInterval(Reg);
395 LIS->createAndComputeVirtRegInterval(Reg);
396 }
397
398 for (auto &&R : Uses) {
399 Register Reg = R.first;
400 if (Reg.isPhysical())
401 continue;
402 LIS->removeInterval(Reg);
403 LIS->createAndComputeVirtRegInterval(Reg);
404 }
405 }
406 }
407
408 return Changed;
409}
410
411bool SIFormMemoryClausesLegacy::runOnMachineFunction(MachineFunction &MF) {
412 if (skipFunction(F: MF.getFunction()))
413 return false;
414
415 LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
416 return SIFormMemoryClausesImpl(LIS).run(MF);
417}
418
419PreservedAnalyses
420SIFormMemoryClausesPass::run(MachineFunction &MF,
421 MachineFunctionAnalysisManager &MFAM) {
422 LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(IR&: MF);
423 SIFormMemoryClausesImpl(&LIS).run(MF);
424 return PreservedAnalyses::all();
425}
426