1//===-- SIPostRABundler.cpp -----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass creates bundles of memory instructions to protect adjacent loads
11/// and stores from being rescheduled apart from each other post-RA.
12///
13//===----------------------------------------------------------------------===//
14
15#include "SIPostRABundler.h"
16#include "AMDGPU.h"
17#include "GCNSubtarget.h"
18#include "llvm/ADT/SmallSet.h"
19#include "llvm/CodeGen/MachineFunctionPass.h"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "si-post-ra-bundler"
24
25namespace {
26
27class SIPostRABundlerLegacy : public MachineFunctionPass {
28public:
29 static char ID;
30
31public:
32 SIPostRABundlerLegacy() : MachineFunctionPass(ID) {}
33
34 bool runOnMachineFunction(MachineFunction &MF) override;
35
36 StringRef getPassName() const override {
37 return "SI post-RA bundler";
38 }
39
40 void getAnalysisUsage(AnalysisUsage &AU) const override {
41 AU.setPreservesAll();
42 MachineFunctionPass::getAnalysisUsage(AU);
43 }
44};
45
46class SIPostRABundler {
47public:
48 bool run(MachineFunction &MF);
49
50private:
51 const SIRegisterInfo *TRI;
52
53 SmallSet<Register, 16> Defs;
54
55 void collectUsedRegUnits(const MachineInstr &MI,
56 BitVector &UsedRegUnits) const;
57
58 bool isBundleCandidate(const MachineInstr &MI) const;
59 bool isDependentLoad(const MachineInstr &MI) const;
60 bool canBundle(const MachineInstr &MI, const MachineInstr &NextMI) const;
61};
62
63} // End anonymous namespace.
64
65INITIALIZE_PASS(SIPostRABundlerLegacy, DEBUG_TYPE, "SI post-RA bundler", false,
66 false)
67
68char SIPostRABundlerLegacy::ID = 0;
69
70char &llvm::SIPostRABundlerLegacyID = SIPostRABundlerLegacy::ID;
71
72FunctionPass *llvm::createSIPostRABundlerPass() {
73 return new SIPostRABundlerLegacy();
74}
75
76bool SIPostRABundler::isDependentLoad(const MachineInstr &MI) const {
77 if (!MI.mayLoad())
78 return false;
79
80 for (const MachineOperand &Op : MI.explicit_operands()) {
81 if (!Op.isReg())
82 continue;
83 Register Reg = Op.getReg();
84 for (Register Def : Defs)
85 if (TRI->regsOverlap(RegA: Reg, RegB: Def))
86 return true;
87 }
88
89 return false;
90}
91
92void SIPostRABundler::collectUsedRegUnits(const MachineInstr &MI,
93 BitVector &UsedRegUnits) const {
94 if (MI.isDebugInstr())
95 return;
96
97 for (const MachineOperand &Op : MI.operands()) {
98 if (!Op.isReg() || !Op.readsReg())
99 continue;
100
101 Register Reg = Op.getReg();
102 assert(!Op.getSubReg() &&
103 "subregister indexes should not be present after RA");
104
105 for (MCRegUnit Unit : TRI->regunits(Reg))
106 UsedRegUnits.set(static_cast<unsigned>(Unit));
107 }
108}
109
110static bool isMemoryInst(const MachineInstr &MI) {
111 return SIInstrFlags::isMUBUF(O: MI) || SIInstrFlags::isMTBUF(O: MI) ||
112 SIInstrFlags::isSMRD(O: MI) || SIInstrFlags::isDS(O: MI) ||
113 SIInstrFlags::isFLAT(O: MI) || SIInstrFlags::isMIMG(O: MI) ||
114 SIInstrFlags::isVIMAGE(O: MI) || SIInstrFlags::isVSAMPLE(O: MI);
115}
116
117static bool hasSameMemFormat(const MachineInstr &A, const MachineInstr &B) {
118 return SIInstrFlags::isMUBUF(O: A) == SIInstrFlags::isMUBUF(O: B) &&
119 SIInstrFlags::isMTBUF(O: A) == SIInstrFlags::isMTBUF(O: B) &&
120 SIInstrFlags::isSMRD(O: A) == SIInstrFlags::isSMRD(O: B) &&
121 SIInstrFlags::isDS(O: A) == SIInstrFlags::isDS(O: B) &&
122 SIInstrFlags::isFLAT(O: A) == SIInstrFlags::isFLAT(O: B) &&
123 SIInstrFlags::isMIMG(O: A) == SIInstrFlags::isMIMG(O: B) &&
124 SIInstrFlags::isVIMAGE(O: A) == SIInstrFlags::isVIMAGE(O: B) &&
125 SIInstrFlags::isVSAMPLE(O: A) == SIInstrFlags::isVSAMPLE(O: B);
126}
127
128bool SIPostRABundler::isBundleCandidate(const MachineInstr &MI) const {
129 return isMemoryInst(MI) && MI.mayLoadOrStore() && !MI.isBundled();
130}
131
132bool SIPostRABundler::canBundle(const MachineInstr &MI,
133 const MachineInstr &NextMI) const {
134 return isMemoryInst(MI) && MI.mayLoadOrStore() && !NextMI.isBundled() &&
135 NextMI.mayLoad() == MI.mayLoad() &&
136 NextMI.mayStore() == MI.mayStore() && hasSameMemFormat(A: MI, B: NextMI) &&
137 !isDependentLoad(MI: NextMI);
138}
139
140bool SIPostRABundlerLegacy::runOnMachineFunction(MachineFunction &MF) {
141 if (skipFunction(F: MF.getFunction()))
142 return false;
143 return SIPostRABundler().run(MF);
144}
145
146PreservedAnalyses SIPostRABundlerPass::run(MachineFunction &MF,
147 MachineFunctionAnalysisManager &) {
148 SIPostRABundler().run(MF);
149 return PreservedAnalyses::all();
150}
151
152bool SIPostRABundler::run(MachineFunction &MF) {
153
154 TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
155 BitVector BundleUsedRegUnits(TRI->getNumRegUnits());
156 BitVector KillUsedRegUnits(TRI->getNumRegUnits());
157
158 bool Changed = false;
159 for (MachineBasicBlock &MBB : MF) {
160 bool HasIGLPInstrs = llvm::any_of(Range: MBB.instrs(), P: [](MachineInstr &MI) {
161 unsigned Opc = MI.getOpcode();
162 return Opc == AMDGPU::SCHED_GROUP_BARRIER || Opc == AMDGPU::IGLP_OPT;
163 });
164
165 // Don't cluster with IGLP instructions.
166 if (HasIGLPInstrs)
167 continue;
168
169 MachineBasicBlock::instr_iterator Next;
170 MachineBasicBlock::instr_iterator B = MBB.instr_begin();
171 MachineBasicBlock::instr_iterator E = MBB.instr_end();
172
173 for (auto I = B; I != E; I = Next) {
174 Next = std::next(x: I);
175 if (!isBundleCandidate(MI: *I))
176 continue;
177
178 assert(Defs.empty());
179
180 if (I->getNumExplicitDefs() != 0)
181 Defs.insert(V: I->defs().begin()->getReg());
182
183 MachineBasicBlock::instr_iterator BundleStart = I;
184 MachineBasicBlock::instr_iterator BundleEnd = I;
185 unsigned ClauseLength = 1;
186 for (I = Next; I != E; I = Next) {
187 Next = std::next(x: I);
188
189 assert(BundleEnd != I);
190 if (canBundle(MI: *BundleEnd, NextMI: *I)) {
191 BundleEnd = I;
192 if (I->getNumExplicitDefs() != 0)
193 Defs.insert(V: I->defs().begin()->getReg());
194 ++ClauseLength;
195 } else if (!I->isMetaInstruction() ||
196 I->getOpcode() == AMDGPU::SCHED_BARRIER) {
197 // SCHED_BARRIER is not bundled to be honored by scheduler later.
198 // Allow other meta instructions in between bundle candidates, but do
199 // not start or end a bundle on one.
200 //
201 // TODO: It may be better to move meta instructions like dbg_value
202 // after the bundle. We're relying on the memory legalizer to unbundle
203 // these.
204 break;
205 }
206 }
207
208 Next = std::next(x: BundleEnd);
209 if (ClauseLength > 1) {
210 Changed = true;
211
212 // Before register allocation, kills are inserted after potential soft
213 // clauses to hint register allocation. Look for kills that look like
214 // this, and erase them.
215 if (Next != E && Next->isKill()) {
216
217 // TODO: Should maybe back-propagate kill flags to the bundle.
218 for (const MachineInstr &BundleMI : make_range(x: BundleStart, y: Next))
219 collectUsedRegUnits(MI: BundleMI, UsedRegUnits&: BundleUsedRegUnits);
220
221 BundleUsedRegUnits.flip();
222
223 while (Next != E && Next->isKill()) {
224 MachineInstr &Kill = *Next;
225 collectUsedRegUnits(MI: Kill, UsedRegUnits&: KillUsedRegUnits);
226
227 KillUsedRegUnits &= BundleUsedRegUnits;
228
229 // Erase the kill if it's a subset of the used registers.
230 //
231 // TODO: Should we just remove all kills? Is there any real reason to
232 // keep them after RA?
233 if (KillUsedRegUnits.none()) {
234 ++Next;
235 Kill.eraseFromParent();
236 } else
237 break;
238
239 KillUsedRegUnits.reset();
240 }
241
242 BundleUsedRegUnits.reset();
243 }
244
245 finalizeBundle(MBB, FirstMI: BundleStart, LastMI: Next);
246 }
247
248 Defs.clear();
249 }
250 }
251
252 return Changed;
253}
254