1//===----- R600Packetizer.cpp - VLIW packetizer ---------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass implements instructions packetization for R600. It unsets isLast
11/// bit of instructions inside a bundle and substitutes src register with
12/// PreviousVector when applicable.
13//
14//===----------------------------------------------------------------------===//
15
16#include "MCTargetDesc/R600MCTargetDesc.h"
17#include "R600.h"
18#include "R600Subtarget.h"
19#include "llvm/CodeGen/DFAPacketizer.h"
20#include "llvm/CodeGen/MachineDominators.h"
21#include "llvm/CodeGen/MachineLoopInfo.h"
22#include "llvm/CodeGen/ScheduleDAG.h"
23
24using namespace llvm;
25
26#define DEBUG_TYPE "packets"
27
28namespace {
29
30class R600Packetizer : public MachineFunctionPass {
31
32public:
33 static char ID;
34 R600Packetizer() : MachineFunctionPass(ID) {}
35
36 void getAnalysisUsage(AnalysisUsage &AU) const override {
37 AU.setPreservesCFG();
38 AU.addRequired<MachineLoopInfoWrapperPass>();
39 MachineFunctionPass::getAnalysisUsage(AU);
40 }
41
42 StringRef getPassName() const override { return "R600 Packetizer"; }
43
44 bool runOnMachineFunction(MachineFunction &Fn) override;
45};
46
47class R600PacketizerList : public VLIWPacketizerList {
48private:
49 const R600InstrInfo *TII;
50 const R600RegisterInfo &TRI;
51 bool VLIW5;
52 bool ConsideredInstUsesAlreadyWrittenVectorElement;
53
54 unsigned getSlot(const MachineInstr &MI) const {
55 return TRI.getHWRegChan(reg: MI.getOperand(i: 0).getReg());
56 }
57
58 /// \returns register to PV chan mapping for bundle/single instructions that
59 /// immediately precedes I.
60 DenseMap<unsigned, unsigned> getPreviousVector(MachineBasicBlock::iterator I)
61 const {
62 DenseMap<unsigned, unsigned> Result;
63 I--;
64 if (!TII->isALUInstr(Opcode: I->getOpcode()) && !I->isBundle())
65 return Result;
66 MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
67 if (I->isBundle())
68 BI++;
69 int LastDstChan = -1;
70 do {
71 bool isTrans = false;
72 int BISlot = getSlot(MI: *BI);
73 if (LastDstChan >= BISlot)
74 isTrans = true;
75 LastDstChan = BISlot;
76 if (TII->isPredicated(MI: *BI))
77 continue;
78 int OperandIdx = TII->getOperandIdx(Opcode: BI->getOpcode(), Op: R600::OpName::write);
79 if (OperandIdx > -1 && BI->getOperand(i: OperandIdx).getImm() == 0)
80 continue;
81 int DstIdx = TII->getOperandIdx(Opcode: BI->getOpcode(), Op: R600::OpName::dst);
82 if (DstIdx == -1) {
83 continue;
84 }
85 Register Dst = BI->getOperand(i: DstIdx).getReg();
86 if (isTrans || TII->isTransOnly(MI: *BI)) {
87 Result[Dst] = R600::PS;
88 continue;
89 }
90 if (BI->getOpcode() == R600::DOT4_r600 ||
91 BI->getOpcode() == R600::DOT4_eg) {
92 Result[Dst] = R600::PV_X;
93 continue;
94 }
95 if (Dst == R600::OQAP) {
96 continue;
97 }
98 unsigned PVReg = 0;
99 switch (TRI.getHWRegChan(reg: Dst)) {
100 case 0:
101 PVReg = R600::PV_X;
102 break;
103 case 1:
104 PVReg = R600::PV_Y;
105 break;
106 case 2:
107 PVReg = R600::PV_Z;
108 break;
109 case 3:
110 PVReg = R600::PV_W;
111 break;
112 default:
113 llvm_unreachable("Invalid Chan");
114 }
115 Result[Dst] = PVReg;
116 } while ((++BI)->isBundledWithPred());
117 return Result;
118 }
119
120 void substitutePV(MachineInstr &MI, const DenseMap<unsigned, unsigned> &PVs)
121 const {
122 const R600::OpName Ops[] = {R600::OpName::src0, R600::OpName::src1,
123 R600::OpName::src2};
124 for (R600::OpName Op : Ops) {
125 int OperandIdx = TII->getOperandIdx(Opcode: MI.getOpcode(), Op);
126 if (OperandIdx < 0)
127 continue;
128 Register Src = MI.getOperand(i: OperandIdx).getReg();
129 const auto It = PVs.find(Val: Src);
130 if (It != PVs.end())
131 MI.getOperand(i: OperandIdx).setReg(It->second);
132 }
133 }
134public:
135 // Ctor.
136 R600PacketizerList(MachineFunction &MF, const R600Subtarget &ST,
137 MachineLoopInfo &MLI)
138 : VLIWPacketizerList(MF, MLI, nullptr),
139 TII(ST.getInstrInfo()),
140 TRI(TII->getRegisterInfo()) {
141 VLIW5 = !ST.hasCaymanISA();
142 }
143
144 // initPacketizerState - initialize some internal flags.
145 void initPacketizerState() override {
146 ConsideredInstUsesAlreadyWrittenVectorElement = false;
147 }
148
149 // ignorePseudoInstruction - Ignore bundling of pseudo instructions.
150 bool ignorePseudoInstruction(const MachineInstr &MI,
151 const MachineBasicBlock *MBB) override {
152 return false;
153 }
154
155 // isSoloInstruction - return true if instruction MI can not be packetized
156 // with any other instruction, which means that MI itself is a packet.
157 bool isSoloInstruction(const MachineInstr &MI) override {
158 if (TII->isVector(MI))
159 return true;
160 if (!TII->isALUInstr(Opcode: MI.getOpcode()))
161 return true;
162 if (MI.getOpcode() == R600::GROUP_BARRIER)
163 return true;
164 // XXX: This can be removed once the packetizer properly handles all the
165 // LDS instruction group restrictions.
166 return TII->isLDSInstr(Opcode: MI.getOpcode());
167 }
168
169 // isLegalToPacketizeTogether - Is it legal to packetize SUI and SUJ
170 // together.
171 bool isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) override {
172 MachineInstr *MII = SUI->getInstr(), *MIJ = SUJ->getInstr();
173 if (getSlot(MI: *MII) == getSlot(MI: *MIJ))
174 ConsideredInstUsesAlreadyWrittenVectorElement = true;
175 // Does MII and MIJ share the same pred_sel ?
176 int OpI = TII->getOperandIdx(Opcode: MII->getOpcode(), Op: R600::OpName::pred_sel),
177 OpJ = TII->getOperandIdx(Opcode: MIJ->getOpcode(), Op: R600::OpName::pred_sel);
178 Register PredI = (OpI > -1)?MII->getOperand(i: OpI).getReg() : Register(),
179 PredJ = (OpJ > -1)?MIJ->getOperand(i: OpJ).getReg() : Register();
180 if (PredI != PredJ)
181 return false;
182 if (SUJ->isSucc(N: SUI)) {
183 for (const SDep &Dep : SUJ->Succs) {
184 if (Dep.getSUnit() != SUI)
185 continue;
186 if (Dep.getKind() == SDep::Anti)
187 continue;
188 if (Dep.getKind() == SDep::Output)
189 if (MII->getOperand(i: 0).getReg() != MIJ->getOperand(i: 0).getReg())
190 continue;
191 return false;
192 }
193 }
194
195 bool ARDef =
196 TII->definesAddressRegister(MI&: *MII) || TII->definesAddressRegister(MI&: *MIJ);
197 bool ARUse =
198 TII->usesAddressRegister(MI&: *MII) || TII->usesAddressRegister(MI&: *MIJ);
199
200 return !ARDef || !ARUse;
201 }
202
203 // isLegalToPruneDependencies - Is it legal to prune dependency between SUI
204 // and SUJ.
205 bool isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) override {
206 return false;
207 }
208
209 void setIsLastBit(MachineInstr *MI, unsigned Bit) const {
210 unsigned LastOp = TII->getOperandIdx(Opcode: MI->getOpcode(), Op: R600::OpName::last);
211 MI->getOperand(i: LastOp).setImm(Bit);
212 }
213
214 bool isBundlableWithCurrentPMI(MachineInstr &MI,
215 const DenseMap<unsigned, unsigned> &PV,
216 std::vector<R600InstrInfo::BankSwizzle> &BS,
217 bool &isTransSlot) {
218 isTransSlot = TII->isTransOnly(MI);
219 assert (!isTransSlot || VLIW5);
220
221 // Is the dst reg sequence legal ?
222 if (!isTransSlot && !CurrentPacketMIs.empty()) {
223 if (getSlot(MI) <= getSlot(MI: *CurrentPacketMIs.back())) {
224 if (ConsideredInstUsesAlreadyWrittenVectorElement &&
225 !TII->isVectorOnly(MI) && VLIW5) {
226 isTransSlot = true;
227 LLVM_DEBUG({
228 dbgs() << "Considering as Trans Inst :";
229 MI.dump();
230 });
231 }
232 else
233 return false;
234 }
235 }
236
237 // Are the Constants limitations met ?
238 CurrentPacketMIs.push_back(x: &MI);
239 if (!TII->fitsConstReadLimitations(CurrentPacketMIs)) {
240 LLVM_DEBUG({
241 dbgs() << "Couldn't pack :\n";
242 MI.dump();
243 dbgs() << "with the following packets :\n";
244 for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) {
245 CurrentPacketMIs[i]->dump();
246 dbgs() << "\n";
247 }
248 dbgs() << "because of Consts read limitations\n";
249 });
250 CurrentPacketMIs.pop_back();
251 return false;
252 }
253
254 // Is there a BankSwizzle set that meet Read Port limitations ?
255 if (!TII->fitsReadPortLimitations(MIs: CurrentPacketMIs,
256 PV, BS, isLastAluTrans: isTransSlot)) {
257 LLVM_DEBUG({
258 dbgs() << "Couldn't pack :\n";
259 MI.dump();
260 dbgs() << "with the following packets :\n";
261 for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) {
262 CurrentPacketMIs[i]->dump();
263 dbgs() << "\n";
264 }
265 dbgs() << "because of Read port limitations\n";
266 });
267 CurrentPacketMIs.pop_back();
268 return false;
269 }
270
271 // We cannot read LDS source registers from the Trans slot.
272 if (isTransSlot && TII->readsLDSSrcReg(MI))
273 return false;
274
275 CurrentPacketMIs.pop_back();
276 return true;
277 }
278
279 MachineBasicBlock::iterator addToPacket(MachineInstr &MI) override {
280 MachineBasicBlock::iterator FirstInBundle =
281 CurrentPacketMIs.empty() ? &MI : CurrentPacketMIs.front();
282 const DenseMap<unsigned, unsigned> &PV =
283 getPreviousVector(I: FirstInBundle);
284 std::vector<R600InstrInfo::BankSwizzle> BS;
285 bool isTransSlot;
286
287 if (isBundlableWithCurrentPMI(MI, PV, BS, isTransSlot)) {
288 for (unsigned i = 0, e = CurrentPacketMIs.size(); i < e; i++) {
289 MachineInstr *MI = CurrentPacketMIs[i];
290 unsigned Op = TII->getOperandIdx(Opcode: MI->getOpcode(),
291 Op: R600::OpName::bank_swizzle);
292 MI->getOperand(i: Op).setImm(BS[i]);
293 }
294 unsigned Op =
295 TII->getOperandIdx(Opcode: MI.getOpcode(), Op: R600::OpName::bank_swizzle);
296 MI.getOperand(i: Op).setImm(BS.back());
297 if (!CurrentPacketMIs.empty())
298 setIsLastBit(MI: CurrentPacketMIs.back(), Bit: 0);
299 substitutePV(MI, PVs: PV);
300 MachineBasicBlock::iterator It = VLIWPacketizerList::addToPacket(MI);
301 if (isTransSlot) {
302 endPacket(MBB: std::next(x: It)->getParent(), MI: std::next(x: It));
303 }
304 return It;
305 }
306 endPacket(MBB: MI.getParent(), MI);
307 if (TII->isTransOnly(MI))
308 return MI;
309 return VLIWPacketizerList::addToPacket(MI);
310 }
311};
312
313bool R600Packetizer::runOnMachineFunction(MachineFunction &Fn) {
314 const R600Subtarget &ST = Fn.getSubtarget<R600Subtarget>();
315 const R600InstrInfo *TII = ST.getInstrInfo();
316
317 MachineLoopInfo &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
318
319 const InstrItineraryData *II = ST.getInstrItineraryData();
320 // If there is no itineraries information, abandon.
321 if (II->Itineraries == nullptr)
322 return false;
323
324 // Instantiate the packetizer.
325 R600PacketizerList Packetizer(Fn, ST, MLI);
326
327 // DFA state table should not be empty.
328 assert(Packetizer.getResourceTracker() && "Empty DFA table!");
329 assert(Packetizer.getResourceTracker()->getInstrItins());
330
331 if (Packetizer.getResourceTracker()->getInstrItins()->isEmpty())
332 return false;
333
334 //
335 // Loop over all basic blocks and remove KILL pseudo-instructions
336 // These instructions confuse the dependence analysis. Consider:
337 // D0 = ... (Insn 0)
338 // R0 = KILL R0, D0 (Insn 1)
339 // R0 = ... (Insn 2)
340 // Here, Insn 1 will result in the dependence graph not emitting an output
341 // dependence between Insn 0 and Insn 2. This can lead to incorrect
342 // packetization
343 //
344 for (MachineBasicBlock &MBB : Fn) {
345 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
346 if (MI.isKill() || MI.getOpcode() == R600::IMPLICIT_DEF ||
347 (MI.getOpcode() == R600::CF_ALU && !MI.getOperand(i: 8).getImm()))
348 MBB.erase(I: MI);
349 }
350 }
351
352 // Loop over all of the basic blocks.
353 for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
354 MBB != MBBe; ++MBB) {
355 // Find scheduling regions and schedule / packetize each region.
356 unsigned RemainingCount = MBB->size();
357 for(MachineBasicBlock::iterator RegionEnd = MBB->end();
358 RegionEnd != MBB->begin();) {
359 // The next region starts above the previous region. Look backward in the
360 // instruction stream until we find the nearest boundary.
361 MachineBasicBlock::iterator I = RegionEnd;
362 for(;I != MBB->begin(); --I, --RemainingCount) {
363 if (TII->isSchedulingBoundary(MI: *std::prev(x: I), MBB: &*MBB, MF: Fn))
364 break;
365 }
366 I = MBB->begin();
367
368 // Skip empty scheduling regions.
369 if (I == RegionEnd) {
370 RegionEnd = std::prev(x: RegionEnd);
371 --RemainingCount;
372 continue;
373 }
374 // Skip regions with one instruction.
375 if (I == std::prev(x: RegionEnd)) {
376 RegionEnd = std::prev(x: RegionEnd);
377 continue;
378 }
379
380 Packetizer.PacketizeMIs(MBB: &*MBB, BeginItr: &*I, EndItr: RegionEnd);
381 RegionEnd = I;
382 }
383 }
384
385 return true;
386
387}
388
389} // end anonymous namespace
390
391INITIALIZE_PASS_BEGIN(R600Packetizer, DEBUG_TYPE,
392 "R600 Packetizer", false, false)
393INITIALIZE_PASS_END(R600Packetizer, DEBUG_TYPE,
394 "R600 Packetizer", false, false)
395
396char R600Packetizer::ID = 0;
397
398char &llvm::R600PacketizerID = R600Packetizer::ID;
399
400llvm::FunctionPass *llvm::createR600Packetizer() {
401 return new R600Packetizer();
402}
403