1//===-- SIFormMemoryClauses.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass extends the live ranges of registers used as pointers in
10/// sequences of adjacent SMEM and VMEM instructions if XNACK is enabled. A
11/// load that would overwrite a pointer would require breaking the soft clause.
12/// Artificially extend the live ranges of the pointer operands by adding
13/// implicit-def early-clobber operands throughout the soft clause.
14///
15//===----------------------------------------------------------------------===//
16
17#include "SIFormMemoryClauses.h"
18#include "AMDGPU.h"
19#include "GCNRegPressure.h"
20#include "SIMachineFunctionInfo.h"
21#include "llvm/InitializePasses.h"
22
23using namespace llvm;
24
25#define DEBUG_TYPE "si-form-memory-clauses"
26
27// Clauses longer then 15 instructions would overflow one of the counters
28// and stall. They can stall even earlier if there are outstanding counters.
29static cl::opt<unsigned>
30MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(Val: 15),
31 cl::desc("Maximum length of a memory clause, instructions"));
32
33namespace {
34
35class SIFormMemoryClausesImpl {
36 using RegUse = DenseMap<unsigned, std::pair<RegState, LaneBitmask>>;
37
38 bool canBundle(const MachineInstr &MI, const RegUse &Defs,
39 const RegUse &Uses) const;
40 bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
41 void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
42 bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
43 GCNDownwardRPTracker &RPT);
44
45 const GCNSubtarget *ST;
46 const SIRegisterInfo *TRI;
47 const MachineRegisterInfo *MRI;
48 SIMachineFunctionInfo *MFI;
49 LiveIntervals *LIS;
50
51 unsigned LastRecordedOccupancy;
52 unsigned MaxVGPRs;
53 unsigned MaxSGPRs;
54
55public:
56 SIFormMemoryClausesImpl(LiveIntervals *LS) : LIS(LS) {}
57 bool run(MachineFunction &MF);
58};
59
60class SIFormMemoryClausesLegacy : public MachineFunctionPass {
61public:
62 static char ID;
63
64 SIFormMemoryClausesLegacy() : MachineFunctionPass(ID) {}
65
66 bool runOnMachineFunction(MachineFunction &MF) override;
67
68 StringRef getPassName() const override {
69 return "SI Form memory clauses";
70 }
71
72 void getAnalysisUsage(AnalysisUsage &AU) const override {
73 AU.addRequired<LiveIntervalsWrapperPass>();
74 AU.setPreservesAll();
75 MachineFunctionPass::getAnalysisUsage(AU);
76 }
77
78 MachineFunctionProperties getClearedProperties() const override {
79 return MachineFunctionProperties().setIsSSA();
80 }
81};
82
83} // End anonymous namespace.
84
85INITIALIZE_PASS_BEGIN(SIFormMemoryClausesLegacy, DEBUG_TYPE,
86 "SI Form memory clauses", false, false)
87INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
88INITIALIZE_PASS_END(SIFormMemoryClausesLegacy, DEBUG_TYPE,
89 "SI Form memory clauses", false, false)
90
91char SIFormMemoryClausesLegacy::ID = 0;
92
93char &llvm::SIFormMemoryClausesID = SIFormMemoryClausesLegacy::ID;
94
95static bool isVMEMClauseInst(const MachineInstr &MI) {
96 return SIInstrInfo::isVMEM(MI);
97}
98
99static bool isSMEMClauseInst(const MachineInstr &MI) {
100 return SIInstrInfo::isSMRD(MI);
101}
102
103// There no sense to create store clauses, they do not define anything,
104// thus there is nothing to set early-clobber.
105static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
106 assert(!MI.isDebugInstr() && "debug instructions should not reach here");
107 if (MI.isBundled())
108 return false;
109 if (!MI.mayLoad() || MI.mayStore())
110 return false;
111 if (SIInstrInfo::isAtomic(MI))
112 return false;
113 if (IsVMEMClause && !isVMEMClauseInst(MI))
114 return false;
115 if (!IsVMEMClause && !isSMEMClauseInst(MI))
116 return false;
117 // If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it.
118 for (const MachineOperand &ResMO : MI.defs()) {
119 Register ResReg = ResMO.getReg();
120 for (const MachineOperand &MO : MI.all_uses()) {
121 if (MO.getReg() == ResReg)
122 return false;
123 }
124 break; // Only check the first def.
125 }
126 return true;
127}
128
129static RegState getMopState(const MachineOperand &MO) {
130 RegState S = {};
131 if (MO.isImplicit())
132 S |= RegState::Implicit;
133 if (MO.isDead())
134 S |= RegState::Dead;
135 if (MO.isUndef())
136 S |= RegState::Undef;
137 if (MO.isKill())
138 S |= RegState::Kill;
139 if (MO.isEarlyClobber())
140 S |= RegState::EarlyClobber;
141 if (MO.getReg().isPhysical() && MO.isRenamable())
142 S |= RegState::Renamable;
143 return S;
144}
145
146// Returns false if there is a use of a def already in the map.
147// In this case we must break the clause.
148bool SIFormMemoryClausesImpl::canBundle(const MachineInstr &MI,
149 const RegUse &Defs,
150 const RegUse &Uses) const {
151 // Check interference with defs.
152 for (const MachineOperand &MO : MI.operands()) {
153 // TODO: Prologue/Epilogue Insertion pass does not process bundled
154 // instructions.
155 if (MO.isFI())
156 return false;
157
158 if (!MO.isReg())
159 continue;
160
161 Register Reg = MO.getReg();
162
163 // If it is tied we will need to write same register as we read.
164 if (MO.isTied())
165 return false;
166
167 const RegUse &Map = MO.isDef() ? Uses : Defs;
168 auto Conflict = Map.find(Val: Reg);
169 if (Conflict == Map.end())
170 continue;
171
172 if (Reg.isPhysical())
173 return false;
174
175 LaneBitmask Mask = TRI->getSubRegIndexLaneMask(SubIdx: MO.getSubReg());
176 if ((Conflict->second.second & Mask).any())
177 return false;
178 }
179
180 return true;
181}
182
183// Since all defs in the clause are early clobber we can run out of registers.
184// Function returns false if pressure would hit the limit if instruction is
185// bundled into a memory clause.
186bool SIFormMemoryClausesImpl::checkPressure(const MachineInstr &MI,
187 GCNDownwardRPTracker &RPT) {
188 // NB: skip advanceBeforeNext() call. Since all defs will be marked
189 // early-clobber they will all stay alive at least to the end of the
190 // clause. Therefor we should not decrease pressure even if load
191 // pointer becomes dead and could otherwise be reused for destination.
192 RPT.advanceToNext();
193 GCNRegPressure MaxPressure = RPT.moveMaxPressure();
194 unsigned Occupancy = MaxPressure.getOccupancy(
195 ST: *ST,
196 DynamicVGPRBlockSize: MI.getMF()->getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
197
198 // Don't push over half the register budget. We don't want to introduce
199 // spilling just to form a soft clause.
200 //
201 // FIXME: This pressure check is fundamentally broken. First, this is checking
202 // the global pressure, not the pressure at this specific point in the
203 // program. Second, it's not accounting for the increased liveness of the use
204 // operands due to the early clobber we will introduce. Third, the pressure
205 // tracking does not account for the alignment requirements for SGPRs, or the
206 // fragmentation of registers the allocator will need to satisfy.
207 if (Occupancy >= MFI->getMinAllowedOccupancy() &&
208 MaxPressure.getVGPRNum(UnifiedVGPRFile: ST->hasGFX90AInsts()) <= MaxVGPRs / 2 &&
209 MaxPressure.getSGPRNum() <= MaxSGPRs / 2) {
210 LastRecordedOccupancy = Occupancy;
211 return true;
212 }
213 return false;
214}
215
216// Collect register defs and uses along with their lane masks and states.
217void SIFormMemoryClausesImpl::collectRegUses(const MachineInstr &MI,
218 RegUse &Defs, RegUse &Uses) const {
219 for (const MachineOperand &MO : MI.operands()) {
220 if (!MO.isReg())
221 continue;
222 Register Reg = MO.getReg();
223 if (!Reg)
224 continue;
225
226 LaneBitmask Mask = Reg.isVirtual()
227 ? TRI->getSubRegIndexLaneMask(SubIdx: MO.getSubReg())
228 : LaneBitmask::getAll();
229 RegUse &Map = MO.isDef() ? Defs : Uses;
230
231 RegState State = getMopState(MO);
232 auto [Loc, Inserted] = Map.try_emplace(Key: Reg, Args&: State, Args&: Mask);
233 if (!Inserted) {
234 Loc->second.first |= State;
235 Loc->second.second |= Mask;
236 }
237 }
238}
239
240// Check register def/use conflicts, occupancy limits and collect def/use maps.
241// Return true if instruction can be bundled with previous. If it cannot
242// def/use maps are not updated.
243bool SIFormMemoryClausesImpl::processRegUses(const MachineInstr &MI,
244 RegUse &Defs, RegUse &Uses,
245 GCNDownwardRPTracker &RPT) {
246 if (!canBundle(MI, Defs, Uses))
247 return false;
248
249 if (!checkPressure(MI, RPT))
250 return false;
251
252 collectRegUses(MI, Defs, Uses);
253 return true;
254}
255
256bool SIFormMemoryClausesImpl::run(MachineFunction &MF) {
257 ST = &MF.getSubtarget<GCNSubtarget>();
258 if (!ST->isXNACKEnabled())
259 return false;
260
261 const SIInstrInfo *TII = ST->getInstrInfo();
262 TRI = ST->getRegisterInfo();
263 MRI = &MF.getRegInfo();
264 MFI = MF.getInfo<SIMachineFunctionInfo>();
265 SlotIndexes *Ind = LIS->getSlotIndexes();
266 bool Changed = false;
267
268 MaxVGPRs = TRI->getAllocatableSet(MF, RC: &AMDGPU::VGPR_32RegClass).count();
269 MaxSGPRs = TRI->getAllocatableSet(MF, RC: &AMDGPU::SGPR_32RegClass).count();
270 unsigned FuncMaxClause = MF.getFunction().getFnAttributeAsParsedInteger(
271 Kind: "amdgpu-max-memory-clause", Default: MaxClause);
272
273 for (MachineBasicBlock &MBB : MF) {
274 GCNDownwardRPTracker RPT(*LIS);
275 MachineBasicBlock::instr_iterator Next;
276 for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
277 MachineInstr &MI = *I;
278 Next = std::next(x: I);
279
280 if (MI.isMetaInstruction())
281 continue;
282
283 bool IsVMEM = isVMEMClauseInst(MI);
284
285 if (!isValidClauseInst(MI, IsVMEMClause: IsVMEM))
286 continue;
287
288 if (!RPT.getNext().isValid())
289 RPT.reset(MI, End: MBB.end());
290 else { // Advance the state to the current MI.
291 RPT.advance(End: MachineBasicBlock::const_iterator(MI));
292 RPT.advanceBeforeNext();
293 }
294
295 const GCNRPTracker::LiveRegSet LiveRegsCopy(RPT.getLiveRegs());
296 RegUse Defs, Uses;
297 if (!processRegUses(MI, Defs, Uses, RPT)) {
298 RPT.reset(MI, End: MBB.end(), LiveRegs: &LiveRegsCopy);
299 continue;
300 }
301
302 MachineBasicBlock::iterator LastClauseInst = Next;
303 unsigned Length = 1;
304 for ( ; Next != E && Length < FuncMaxClause; ++Next) {
305 // Debug instructions should not change the kill insertion.
306 if (Next->isMetaInstruction())
307 continue;
308
309 if (!isValidClauseInst(MI: *Next, IsVMEMClause: IsVMEM))
310 break;
311
312 // A load from pointer which was loaded inside the same bundle is an
313 // impossible clause because we will need to write and read the same
314 // register inside. In this case processRegUses will return false.
315 if (!processRegUses(MI: *Next, Defs, Uses, RPT))
316 break;
317
318 LastClauseInst = Next;
319 ++Length;
320 }
321 if (Length < 2) {
322 RPT.reset(MI, End: MBB.end(), LiveRegs: &LiveRegsCopy);
323 continue;
324 }
325
326 Changed = true;
327 MFI->limitOccupancy(Limit: LastRecordedOccupancy);
328
329 assert(!LastClauseInst->isMetaInstruction());
330
331 SlotIndex ClauseLiveInIdx = LIS->getInstructionIndex(Instr: MI);
332 SlotIndex ClauseLiveOutIdx =
333 LIS->getInstructionIndex(Instr: *LastClauseInst).getNextIndex();
334
335 // Track the last inserted kill.
336 MachineInstrBuilder Kill;
337
338 // Insert one kill per register, with operands covering all necessary
339 // subregisters.
340 for (auto &&R : Uses) {
341 Register Reg = R.first;
342 if (Reg.isPhysical())
343 continue;
344
345 // Collect the register operands we should extend the live ranges of.
346 SmallVector<std::tuple<RegState, unsigned>> KillOps;
347 const LiveInterval &LI = LIS->getInterval(Reg: R.first);
348
349 if (!LI.hasSubRanges()) {
350 if (!LI.liveAt(index: ClauseLiveOutIdx)) {
351 KillOps.emplace_back(Args: R.second.first | RegState::Kill,
352 Args: AMDGPU::NoSubRegister);
353 }
354 } else {
355 LaneBitmask KilledMask;
356 for (const LiveInterval::SubRange &SR : LI.subranges()) {
357 if (SR.liveAt(index: ClauseLiveInIdx) && !SR.liveAt(index: ClauseLiveOutIdx))
358 KilledMask |= SR.LaneMask;
359 }
360
361 if (KilledMask.none())
362 continue;
363
364 SmallVector<unsigned> KilledIndexes;
365 bool Success = TRI->getCoveringSubRegIndexes(
366 RC: MRI->getRegClass(Reg), LaneMask: KilledMask, Indexes&: KilledIndexes);
367 (void)Success;
368 assert(Success && "Failed to find subregister mask to cover lanes");
369 for (unsigned SubReg : KilledIndexes) {
370 KillOps.emplace_back(Args: R.second.first | RegState::Kill, Args&: SubReg);
371 }
372 }
373
374 if (KillOps.empty())
375 continue;
376
377 // We only want to extend the live ranges of used registers. If they
378 // already have existing uses beyond the bundle, we don't need the kill.
379 //
380 // It's possible all of the use registers were already live past the
381 // bundle.
382 Kill = BuildMI(BB&: *MI.getParent(), I: std::next(x: LastClauseInst),
383 MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::KILL));
384 for (auto &Op : KillOps)
385 Kill.addUse(RegNo: Reg, Flags: std::get<0>(t&: Op), SubReg: std::get<1>(t&: Op));
386 Ind->insertMachineInstrInMaps(MI&: *Kill);
387 }
388
389 // Restore the state after processing the end of the bundle.
390 RPT.reset(MI, End: MBB.end(), LiveRegs: &LiveRegsCopy);
391
392 if (!Kill)
393 continue;
394
395 for (auto &&R : Defs) {
396 Register Reg = R.first;
397 Uses.erase(Val: Reg);
398 if (Reg.isPhysical())
399 continue;
400 LIS->removeInterval(Reg);
401 LIS->createAndComputeVirtRegInterval(Reg);
402 }
403
404 for (auto &&R : Uses) {
405 Register Reg = R.first;
406 if (Reg.isPhysical())
407 continue;
408 LIS->removeInterval(Reg);
409 LIS->createAndComputeVirtRegInterval(Reg);
410 }
411 }
412 }
413
414 return Changed;
415}
416
417bool SIFormMemoryClausesLegacy::runOnMachineFunction(MachineFunction &MF) {
418 if (skipFunction(F: MF.getFunction()))
419 return false;
420
421 LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
422 return SIFormMemoryClausesImpl(LIS).run(MF);
423}
424
425PreservedAnalyses
426SIFormMemoryClausesPass::run(MachineFunction &MF,
427 MachineFunctionAnalysisManager &MFAM) {
428 LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(IR&: MF);
429 SIFormMemoryClausesImpl(&LIS).run(MF);
430 return PreservedAnalyses::all();
431}
432