1//===-- SIPostRABundler.cpp -----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass creates bundles of memory instructions to protect adjacent loads
11/// and stores from being rescheduled apart from each other post-RA.
12///
13//===----------------------------------------------------------------------===//
14
15#include "SIPostRABundler.h"
16#include "AMDGPU.h"
17#include "GCNSubtarget.h"
18#include "llvm/CodeGen/MachineFunctionPass.h"
19
20using namespace llvm;
21
22#define DEBUG_TYPE "si-post-ra-bundler"
23
24namespace {
25
26class SIPostRABundlerLegacy : public MachineFunctionPass {
27public:
28 static char ID;
29
30public:
31 SIPostRABundlerLegacy() : MachineFunctionPass(ID) {}
32
33 bool runOnMachineFunction(MachineFunction &MF) override;
34
35 StringRef getPassName() const override {
36 return "SI post-RA bundler";
37 }
38
39 void getAnalysisUsage(AnalysisUsage &AU) const override {
40 AU.setPreservesAll();
41 MachineFunctionPass::getAnalysisUsage(AU);
42 }
43};
44
45class SIPostRABundler {
46public:
47 bool run(MachineFunction &MF);
48
49private:
50 const SIRegisterInfo *TRI;
51
52 SmallSet<Register, 16> Defs;
53
54 void collectUsedRegUnits(const MachineInstr &MI,
55 BitVector &UsedRegUnits) const;
56
57 bool isBundleCandidate(const MachineInstr &MI) const;
58 bool isDependentLoad(const MachineInstr &MI) const;
59 bool canBundle(const MachineInstr &MI, const MachineInstr &NextMI) const;
60};
61
62} // End anonymous namespace.
63
64INITIALIZE_PASS(SIPostRABundlerLegacy, DEBUG_TYPE, "SI post-RA bundler", false,
65 false)
66
67char SIPostRABundlerLegacy::ID = 0;
68
69char &llvm::SIPostRABundlerLegacyID = SIPostRABundlerLegacy::ID;
70
71FunctionPass *llvm::createSIPostRABundlerPass() {
72 return new SIPostRABundlerLegacy();
73}
74
75bool SIPostRABundler::isDependentLoad(const MachineInstr &MI) const {
76 if (!MI.mayLoad())
77 return false;
78
79 for (const MachineOperand &Op : MI.explicit_operands()) {
80 if (!Op.isReg())
81 continue;
82 Register Reg = Op.getReg();
83 for (Register Def : Defs)
84 if (TRI->regsOverlap(RegA: Reg, RegB: Def))
85 return true;
86 }
87
88 return false;
89}
90
91void SIPostRABundler::collectUsedRegUnits(const MachineInstr &MI,
92 BitVector &UsedRegUnits) const {
93 if (MI.isDebugInstr())
94 return;
95
96 for (const MachineOperand &Op : MI.operands()) {
97 if (!Op.isReg() || !Op.readsReg())
98 continue;
99
100 Register Reg = Op.getReg();
101 assert(!Op.getSubReg() &&
102 "subregister indexes should not be present after RA");
103
104 for (MCRegUnit Unit : TRI->regunits(Reg))
105 UsedRegUnits.set(static_cast<unsigned>(Unit));
106 }
107}
108
109static bool isMemoryInst(const MachineInstr &MI) {
110 return SIInstrFlags::isMUBUF(O: MI) || SIInstrFlags::isMTBUF(O: MI) ||
111 SIInstrFlags::isSMRD(O: MI) || SIInstrFlags::isDS(O: MI) ||
112 SIInstrFlags::isFLAT(O: MI) || SIInstrFlags::isMIMG(O: MI) ||
113 SIInstrFlags::isVIMAGE(O: MI) || SIInstrFlags::isVSAMPLE(O: MI);
114}
115
116static bool hasSameMemFormat(const MachineInstr &A, const MachineInstr &B) {
117 return SIInstrFlags::isMUBUF(O: A) == SIInstrFlags::isMUBUF(O: B) &&
118 SIInstrFlags::isMTBUF(O: A) == SIInstrFlags::isMTBUF(O: B) &&
119 SIInstrFlags::isSMRD(O: A) == SIInstrFlags::isSMRD(O: B) &&
120 SIInstrFlags::isDS(O: A) == SIInstrFlags::isDS(O: B) &&
121 SIInstrFlags::isFLAT(O: A) == SIInstrFlags::isFLAT(O: B) &&
122 SIInstrFlags::isMIMG(O: A) == SIInstrFlags::isMIMG(O: B) &&
123 SIInstrFlags::isVIMAGE(O: A) == SIInstrFlags::isVIMAGE(O: B) &&
124 SIInstrFlags::isVSAMPLE(O: A) == SIInstrFlags::isVSAMPLE(O: B);
125}
126
127bool SIPostRABundler::isBundleCandidate(const MachineInstr &MI) const {
128 return isMemoryInst(MI) && MI.mayLoadOrStore() && !MI.isBundled();
129}
130
131bool SIPostRABundler::canBundle(const MachineInstr &MI,
132 const MachineInstr &NextMI) const {
133 return isMemoryInst(MI) && MI.mayLoadOrStore() && !NextMI.isBundled() &&
134 NextMI.mayLoad() == MI.mayLoad() &&
135 NextMI.mayStore() == MI.mayStore() && hasSameMemFormat(A: MI, B: NextMI) &&
136 !isDependentLoad(MI: NextMI);
137}
138
139bool SIPostRABundlerLegacy::runOnMachineFunction(MachineFunction &MF) {
140 if (skipFunction(F: MF.getFunction()))
141 return false;
142 return SIPostRABundler().run(MF);
143}
144
145PreservedAnalyses SIPostRABundlerPass::run(MachineFunction &MF,
146 MachineFunctionAnalysisManager &) {
147 SIPostRABundler().run(MF);
148 return PreservedAnalyses::all();
149}
150
151bool SIPostRABundler::run(MachineFunction &MF) {
152
153 TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
154 BitVector BundleUsedRegUnits(TRI->getNumRegUnits());
155 BitVector KillUsedRegUnits(TRI->getNumRegUnits());
156
157 bool Changed = false;
158 for (MachineBasicBlock &MBB : MF) {
159 bool HasIGLPInstrs = llvm::any_of(Range: MBB.instrs(), P: [](MachineInstr &MI) {
160 unsigned Opc = MI.getOpcode();
161 return Opc == AMDGPU::SCHED_GROUP_BARRIER || Opc == AMDGPU::IGLP_OPT;
162 });
163
164 // Don't cluster with IGLP instructions.
165 if (HasIGLPInstrs)
166 continue;
167
168 MachineBasicBlock::instr_iterator Next;
169 MachineBasicBlock::instr_iterator B = MBB.instr_begin();
170 MachineBasicBlock::instr_iterator E = MBB.instr_end();
171
172 for (auto I = B; I != E; I = Next) {
173 Next = std::next(x: I);
174 if (!isBundleCandidate(MI: *I))
175 continue;
176
177 assert(Defs.empty());
178
179 if (I->getNumExplicitDefs() != 0)
180 Defs.insert(V: I->defs().begin()->getReg());
181
182 MachineBasicBlock::instr_iterator BundleStart = I;
183 MachineBasicBlock::instr_iterator BundleEnd = I;
184 unsigned ClauseLength = 1;
185 for (I = Next; I != E; I = Next) {
186 Next = std::next(x: I);
187
188 assert(BundleEnd != I);
189 if (canBundle(MI: *BundleEnd, NextMI: *I)) {
190 BundleEnd = I;
191 if (I->getNumExplicitDefs() != 0)
192 Defs.insert(V: I->defs().begin()->getReg());
193 ++ClauseLength;
194 } else if (!I->isMetaInstruction() ||
195 I->getOpcode() == AMDGPU::SCHED_BARRIER) {
196 // SCHED_BARRIER is not bundled to be honored by scheduler later.
197 // Allow other meta instructions in between bundle candidates, but do
198 // not start or end a bundle on one.
199 //
200 // TODO: It may be better to move meta instructions like dbg_value
201 // after the bundle. We're relying on the memory legalizer to unbundle
202 // these.
203 break;
204 }
205 }
206
207 Next = std::next(x: BundleEnd);
208 if (ClauseLength > 1) {
209 Changed = true;
210
211 // Before register allocation, kills are inserted after potential soft
212 // clauses to hint register allocation. Look for kills that look like
213 // this, and erase them.
214 if (Next != E && Next->isKill()) {
215
216 // TODO: Should maybe back-propagate kill flags to the bundle.
217 for (const MachineInstr &BundleMI : make_range(x: BundleStart, y: Next))
218 collectUsedRegUnits(MI: BundleMI, UsedRegUnits&: BundleUsedRegUnits);
219
220 BundleUsedRegUnits.flip();
221
222 while (Next != E && Next->isKill()) {
223 MachineInstr &Kill = *Next;
224 collectUsedRegUnits(MI: Kill, UsedRegUnits&: KillUsedRegUnits);
225
226 KillUsedRegUnits &= BundleUsedRegUnits;
227
228 // Erase the kill if it's a subset of the used registers.
229 //
230 // TODO: Should we just remove all kills? Is there any real reason to
231 // keep them after RA?
232 if (KillUsedRegUnits.none()) {
233 ++Next;
234 Kill.eraseFromParent();
235 } else
236 break;
237
238 KillUsedRegUnits.reset();
239 }
240
241 BundleUsedRegUnits.reset();
242 }
243
244 finalizeBundle(MBB, FirstMI: BundleStart, LastMI: Next);
245 }
246
247 Defs.clear();
248 }
249 }
250
251 return Changed;
252}
253