1//===- R600MergeVectorRegisters.cpp ---------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass merges inputs of swizzeable instructions into vector sharing
11/// common data and/or have enough undef subreg using swizzle abilities.
12///
13/// For instance let's consider the following pseudo code :
14/// %5 = REG_SEQ %1, sub0, %2, sub1, %3, sub2, undef, sub3
15/// ...
16/// %7 = REG_SEQ %1, sub0, %3, sub1, undef, sub2, %4, sub3
17/// (swizzable Inst) %7, SwizzleMask : sub0, sub1, sub2, sub3
18///
19/// is turned into :
20/// %5 = REG_SEQ %1, sub0, %2, sub1, %3, sub2, undef, sub3
21/// ...
22/// %7 = INSERT_SUBREG %4, sub3
23/// (swizzable Inst) %7, SwizzleMask : sub0, sub2, sub1, sub3
24///
25/// This allow regalloc to reduce register pressure for vector registers and
26/// to reduce MOV count.
27//===----------------------------------------------------------------------===//
28
29#include "MCTargetDesc/R600MCTargetDesc.h"
30#include "R600.h"
31#include "R600Defines.h"
32#include "R600Subtarget.h"
33#include "llvm/CodeGen/MachineDominators.h"
34
35using namespace llvm;
36
37#define DEBUG_TYPE "vec-merger"
38
39static bool isImplicitlyDef(MachineRegisterInfo &MRI, Register Reg) {
40 if (Reg.isPhysical())
41 return false;
42 const MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
43 return MI && MI->isImplicitDef();
44}
45
46namespace {
47
48class RegSeqInfo {
49public:
50 MachineInstr *Instr;
51 DenseMap<Register, unsigned> RegToChan;
52 std::vector<Register> UndefReg;
53
54 RegSeqInfo(MachineRegisterInfo &MRI, MachineInstr *MI) : Instr(MI) {
55 assert(MI->getOpcode() == R600::REG_SEQUENCE);
56 for (unsigned i = 1, e = Instr->getNumOperands(); i < e; i+=2) {
57 MachineOperand &MO = Instr->getOperand(i);
58 unsigned Chan = Instr->getOperand(i: i + 1).getImm();
59 if (isImplicitlyDef(MRI, Reg: MO.getReg()))
60 UndefReg.emplace_back(args&: Chan);
61 else
62 RegToChan[MO.getReg()] = Chan;
63 }
64 }
65
66 RegSeqInfo() = default;
67
68 bool operator==(const RegSeqInfo &RSI) const {
69 return RSI.Instr == Instr;
70 }
71};
72
73class R600VectorRegMerger : public MachineFunctionPass {
74private:
75 using InstructionSetMap = DenseMap<unsigned, std::vector<MachineInstr *>>;
76
77 MachineRegisterInfo *MRI;
78 const R600InstrInfo *TII = nullptr;
79 DenseMap<MachineInstr *, RegSeqInfo> PreviousRegSeq;
80 InstructionSetMap PreviousRegSeqByReg;
81 InstructionSetMap PreviousRegSeqByUndefCount;
82
83 bool canSwizzle(const MachineInstr &MI) const;
84 bool areAllUsesSwizzeable(Register Reg) const;
85 void SwizzleInput(MachineInstr &,
86 const std::vector<std::pair<unsigned, unsigned>> &RemapChan) const;
87 bool tryMergeVector(const RegSeqInfo *Untouched, RegSeqInfo *ToMerge,
88 std::vector<std::pair<unsigned, unsigned>> &Remap) const;
89 bool tryMergeUsingCommonSlot(RegSeqInfo &RSI, RegSeqInfo &CompatibleRSI,
90 std::vector<std::pair<unsigned, unsigned>> &RemapChan);
91 bool tryMergeUsingFreeSlot(RegSeqInfo &RSI, RegSeqInfo &CompatibleRSI,
92 std::vector<std::pair<unsigned, unsigned>> &RemapChan);
93 MachineInstr *RebuildVector(RegSeqInfo *MI, const RegSeqInfo *BaseVec,
94 const std::vector<std::pair<unsigned, unsigned>> &RemapChan) const;
95 void RemoveMI(MachineInstr *);
96 void trackRSI(const RegSeqInfo &RSI);
97
98public:
99 static char ID;
100
101 R600VectorRegMerger() : MachineFunctionPass(ID) {}
102
103 void getAnalysisUsage(AnalysisUsage &AU) const override {
104 AU.setPreservesCFG();
105 MachineFunctionPass::getAnalysisUsage(AU);
106 }
107
108 MachineFunctionProperties getRequiredProperties() const override {
109 return MachineFunctionProperties().setIsSSA();
110 }
111
112 StringRef getPassName() const override {
113 return "R600 Vector Registers Merge Pass";
114 }
115
116 bool runOnMachineFunction(MachineFunction &Fn) override;
117};
118
119} // end anonymous namespace
120
121INITIALIZE_PASS_BEGIN(R600VectorRegMerger, DEBUG_TYPE,
122 "R600 Vector Reg Merger", false, false)
123INITIALIZE_PASS_END(R600VectorRegMerger, DEBUG_TYPE,
124 "R600 Vector Reg Merger", false, false)
125
126char R600VectorRegMerger::ID = 0;
127
128char &llvm::R600VectorRegMergerID = R600VectorRegMerger::ID;
129
130bool R600VectorRegMerger::canSwizzle(const MachineInstr &MI)
131 const {
132 if (TII->get(Opcode: MI.getOpcode()).TSFlags & R600_InstFlag::TEX_INST)
133 return true;
134 switch (MI.getOpcode()) {
135 case R600::R600_ExportSwz:
136 case R600::EG_ExportSwz:
137 return true;
138 default:
139 return false;
140 }
141}
142
143bool R600VectorRegMerger::tryMergeVector(const RegSeqInfo *Untouched,
144 RegSeqInfo *ToMerge, std::vector< std::pair<unsigned, unsigned>> &Remap)
145 const {
146 unsigned CurrentUndexIdx = 0;
147 for (auto &It : ToMerge->RegToChan) {
148 auto PosInUntouched = Untouched->RegToChan.find(Val: It.first);
149 if (PosInUntouched != Untouched->RegToChan.end()) {
150 Remap.emplace_back(args&: It.second, args: (*PosInUntouched).second);
151 continue;
152 }
153 if (CurrentUndexIdx >= Untouched->UndefReg.size())
154 return false;
155 Remap.emplace_back(args&: It.second, args: Untouched->UndefReg[CurrentUndexIdx++]);
156 }
157
158 return true;
159}
160
161static
162unsigned getReassignedChan(
163 const std::vector<std::pair<unsigned, unsigned>> &RemapChan,
164 unsigned Chan) {
165 for (const auto &J : RemapChan) {
166 if (J.first == Chan)
167 return J.second;
168 }
169 llvm_unreachable("Chan wasn't reassigned");
170}
171
172MachineInstr *R600VectorRegMerger::RebuildVector(
173 RegSeqInfo *RSI, const RegSeqInfo *BaseRSI,
174 const std::vector<std::pair<unsigned, unsigned>> &RemapChan) const {
175 Register Reg = RSI->Instr->getOperand(i: 0).getReg();
176 MachineBasicBlock::iterator Pos = RSI->Instr;
177 MachineBasicBlock &MBB = *Pos->getParent();
178 const DebugLoc &DL = Pos->getDebugLoc();
179
180 Register SrcVec = BaseRSI->Instr->getOperand(i: 0).getReg();
181 DenseMap<Register, unsigned> UpdatedRegToChan = BaseRSI->RegToChan;
182 std::vector<Register> UpdatedUndef = BaseRSI->UndefReg;
183 for (const auto &It : RSI->RegToChan) {
184 Register DstReg = MRI->createVirtualRegister(RegClass: &R600::R600_Reg128RegClass);
185 unsigned SubReg = It.first;
186 unsigned Swizzle = It.second;
187 unsigned Chan = getReassignedChan(RemapChan, Chan: Swizzle);
188
189 MachineInstr *Tmp = BuildMI(BB&: MBB, I: Pos, MIMD: DL, MCID: TII->get(Opcode: R600::INSERT_SUBREG),
190 DestReg: DstReg)
191 .addReg(RegNo: SrcVec)
192 .addReg(RegNo: SubReg)
193 .addImm(Val: Chan);
194 UpdatedRegToChan[SubReg] = Chan;
195 std::vector<Register>::iterator ChanPos = llvm::find(Range&: UpdatedUndef, Val: Chan);
196 if (ChanPos != UpdatedUndef.end())
197 UpdatedUndef.erase(position: ChanPos);
198 assert(!is_contained(UpdatedUndef, Chan) &&
199 "UpdatedUndef shouldn't contain Chan more than once!");
200 LLVM_DEBUG(dbgs() << " ->"; Tmp->dump(););
201 (void)Tmp;
202 SrcVec = DstReg;
203 }
204 MachineInstr *NewMI =
205 BuildMI(BB&: MBB, I: Pos, MIMD: DL, MCID: TII->get(Opcode: R600::COPY), DestReg: Reg).addReg(RegNo: SrcVec);
206 LLVM_DEBUG(dbgs() << " ->"; NewMI->dump(););
207
208 LLVM_DEBUG(dbgs() << " Updating Swizzle:\n");
209 for (MachineRegisterInfo::use_instr_iterator It = MRI->use_instr_begin(RegNo: Reg),
210 E = MRI->use_instr_end(); It != E; ++It) {
211 LLVM_DEBUG(dbgs() << " "; (*It).dump(); dbgs() << " ->");
212 SwizzleInput(*It, RemapChan);
213 LLVM_DEBUG((*It).dump());
214 }
215 RSI->Instr->eraseFromParent();
216
217 // Update RSI
218 RSI->Instr = NewMI;
219 RSI->RegToChan = std::move(UpdatedRegToChan);
220 RSI->UndefReg = std::move(UpdatedUndef);
221
222 return NewMI;
223}
224
225void R600VectorRegMerger::RemoveMI(MachineInstr *MI) {
226 for (auto &It : PreviousRegSeqByReg) {
227 std::vector<MachineInstr *> &MIs = It.second;
228 MIs.erase(first: llvm::find(Range&: MIs, Val: MI), last: MIs.end());
229 }
230 for (auto &It : PreviousRegSeqByUndefCount) {
231 std::vector<MachineInstr *> &MIs = It.second;
232 MIs.erase(first: llvm::find(Range&: MIs, Val: MI), last: MIs.end());
233 }
234}
235
236void R600VectorRegMerger::SwizzleInput(MachineInstr &MI,
237 const std::vector<std::pair<unsigned, unsigned>> &RemapChan) const {
238 unsigned Offset;
239 if (TII->get(Opcode: MI.getOpcode()).TSFlags & R600_InstFlag::TEX_INST)
240 Offset = 2;
241 else
242 Offset = 3;
243 for (unsigned i = 0; i < 4; i++) {
244 unsigned Swizzle = MI.getOperand(i: i + Offset).getImm() + 1;
245 for (const auto &J : RemapChan) {
246 if (J.first == Swizzle) {
247 MI.getOperand(i: i + Offset).setImm(J.second - 1);
248 break;
249 }
250 }
251 }
252}
253
254bool R600VectorRegMerger::areAllUsesSwizzeable(Register Reg) const {
255 return llvm::all_of(Range: MRI->use_instructions(Reg),
256 P: [&](const MachineInstr &MI) { return canSwizzle(MI); });
257}
258
259bool R600VectorRegMerger::tryMergeUsingCommonSlot(RegSeqInfo &RSI,
260 RegSeqInfo &CompatibleRSI,
261 std::vector<std::pair<unsigned, unsigned>> &RemapChan) {
262 for (MachineInstr::mop_iterator MOp = RSI.Instr->operands_begin(),
263 MOE = RSI.Instr->operands_end(); MOp != MOE; ++MOp) {
264 if (!MOp->isReg())
265 continue;
266 auto &Insts = PreviousRegSeqByReg[MOp->getReg()];
267 if (Insts.empty())
268 continue;
269 for (MachineInstr *MI : Insts) {
270 CompatibleRSI = PreviousRegSeq[MI];
271 if (RSI == CompatibleRSI)
272 continue;
273 if (tryMergeVector(Untouched: &CompatibleRSI, ToMerge: &RSI, Remap&: RemapChan))
274 return true;
275 }
276 }
277 return false;
278}
279
280bool R600VectorRegMerger::tryMergeUsingFreeSlot(RegSeqInfo &RSI,
281 RegSeqInfo &CompatibleRSI,
282 std::vector<std::pair<unsigned, unsigned>> &RemapChan) {
283 unsigned NeededUndefs = 4 - RSI.UndefReg.size();
284 std::vector<MachineInstr *> &MIs =
285 PreviousRegSeqByUndefCount[NeededUndefs];
286 if (MIs.empty())
287 return false;
288 CompatibleRSI = PreviousRegSeq[MIs.back()];
289 tryMergeVector(Untouched: &CompatibleRSI, ToMerge: &RSI, Remap&: RemapChan);
290 return true;
291}
292
293void R600VectorRegMerger::trackRSI(const RegSeqInfo &RSI) {
294 for (DenseMap<Register, unsigned>::const_iterator
295 It = RSI.RegToChan.begin(), E = RSI.RegToChan.end(); It != E; ++It) {
296 PreviousRegSeqByReg[(*It).first].push_back(x: RSI.Instr);
297 }
298 PreviousRegSeqByUndefCount[RSI.UndefReg.size()].push_back(x: RSI.Instr);
299 PreviousRegSeq[RSI.Instr] = RSI;
300}
301
302bool R600VectorRegMerger::runOnMachineFunction(MachineFunction &Fn) {
303 if (skipFunction(F: Fn.getFunction()))
304 return false;
305
306 const R600Subtarget &ST = Fn.getSubtarget<R600Subtarget>();
307 TII = ST.getInstrInfo();
308 MRI = &Fn.getRegInfo();
309
310 for (MachineBasicBlock &MB : Fn) {
311 PreviousRegSeq.clear();
312 PreviousRegSeqByReg.clear();
313 PreviousRegSeqByUndefCount.clear();
314
315 for (MachineBasicBlock::iterator MII = MB.begin(), MIIE = MB.end();
316 MII != MIIE; ++MII) {
317 MachineInstr &MI = *MII;
318 if (MI.getOpcode() != R600::REG_SEQUENCE) {
319 if (TII->get(Opcode: MI.getOpcode()).TSFlags & R600_InstFlag::TEX_INST) {
320 Register Reg = MI.getOperand(i: 1).getReg();
321 for (MachineInstr &DefMI : MRI->def_instructions(Reg))
322 RemoveMI(MI: &DefMI);
323 }
324 continue;
325 }
326
327 RegSeqInfo RSI(*MRI, &MI);
328
329 // All uses of MI are swizzeable ?
330 Register Reg = MI.getOperand(i: 0).getReg();
331 if (!areAllUsesSwizzeable(Reg))
332 continue;
333
334 LLVM_DEBUG({
335 dbgs() << "Trying to optimize ";
336 MI.dump();
337 });
338
339 RegSeqInfo CandidateRSI;
340 std::vector<std::pair<unsigned, unsigned>> RemapChan;
341 LLVM_DEBUG(dbgs() << "Using common slots...\n";);
342 if (tryMergeUsingCommonSlot(RSI, CompatibleRSI&: CandidateRSI, RemapChan)) {
343 // Remove CandidateRSI mapping
344 RemoveMI(MI: CandidateRSI.Instr);
345 MII = RebuildVector(RSI: &RSI, BaseRSI: &CandidateRSI, RemapChan);
346 trackRSI(RSI);
347 continue;
348 }
349 LLVM_DEBUG(dbgs() << "Using free slots...\n";);
350 RemapChan.clear();
351 if (tryMergeUsingFreeSlot(RSI, CompatibleRSI&: CandidateRSI, RemapChan)) {
352 RemoveMI(MI: CandidateRSI.Instr);
353 MII = RebuildVector(RSI: &RSI, BaseRSI: &CandidateRSI, RemapChan);
354 trackRSI(RSI);
355 continue;
356 }
357 //Failed to merge
358 trackRSI(RSI);
359 }
360 }
361 return false;
362}
363
364llvm::FunctionPass *llvm::createR600VectorRegMerger() {
365 return new R600VectorRegMerger();
366}
367