1//=======- GCNDPPCombine.cpp - optimization for DPP instructions ---==========//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// The pass combines V_MOV_B32_dpp instruction with its VALU uses as a DPP src0
9// operand. If any of the use instruction cannot be combined with the mov the
10// whole sequence is reverted.
11//
12// $old = ...
13// $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane,
14// dpp_controls..., $row_mask, $bank_mask, $bound_ctrl
15// $res = VALU $dpp_value [, src1]
16//
17// to
18//
19// $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, [src1,]
20// dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl
21//
22// Combining rules :
23//
24// if $row_mask and $bank_mask are fully enabled (0xF) and
25// $bound_ctrl==DPP_BOUND_ZERO or $old==0
26// -> $combined_old = undef,
27// $combined_bound_ctrl = DPP_BOUND_ZERO
28//
29// if the VALU op is binary and
30// $bound_ctrl==DPP_BOUND_OFF and
31// $old==identity value (immediate) for the VALU op
32// -> $combined_old = src1,
33// $combined_bound_ctrl = DPP_BOUND_OFF
34//
35// Otherwise cancel.
36//
37// The mov_dpp instruction should reside in the same BB as all its uses
38//===----------------------------------------------------------------------===//
39
40#include "GCNDPPCombine.h"
41#include "AMDGPU.h"
42#include "GCNSubtarget.h"
43#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
44#include "llvm/ADT/Statistic.h"
45#include "llvm/CodeGen/MachineFunctionPass.h"
46#include "llvm/CodeGen/MachineInstrBuilder.h"
47
48using namespace llvm;
49
50#define DEBUG_TYPE "gcn-dpp-combine"
51
52STATISTIC(NumDPPMovsCombined, "Number of DPP moves combined.");
53
54namespace {
55
56class GCNDPPCombine {
57 MachineRegisterInfo *MRI;
58 const SIInstrInfo *TII;
59 const GCNSubtarget *ST;
60
61 using RegSubRegPair = TargetInstrInfo::RegSubRegPair;
62
63 MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const;
64
65 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
66 RegSubRegPair CombOldVGPR,
67 MachineOperand *OldOpnd, bool CombBCZ,
68 bool IsShrinkable) const;
69
70 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
71 RegSubRegPair CombOldVGPR, bool CombBCZ,
72 bool IsShrinkable) const;
73
74 bool hasNoImmOrEqual(MachineInstr &MI, AMDGPU::OpName OpndName, int64_t Value,
75 int64_t Mask = -1) const;
76
77 bool combineDPPMov(MachineInstr &MI) const;
78
79 int getDPPOp(unsigned Op, bool IsShrinkable) const;
80 bool isShrinkable(MachineInstr &MI) const;
81
82public:
83 bool run(MachineFunction &MF);
84};
85
86class GCNDPPCombineLegacy : public MachineFunctionPass {
87public:
88 static char ID;
89
90 GCNDPPCombineLegacy() : MachineFunctionPass(ID) {}
91
92 bool runOnMachineFunction(MachineFunction &MF) override;
93
94 StringRef getPassName() const override { return "GCN DPP Combine"; }
95
96 void getAnalysisUsage(AnalysisUsage &AU) const override {
97 AU.setPreservesCFG();
98 MachineFunctionPass::getAnalysisUsage(AU);
99 }
100
101 MachineFunctionProperties getRequiredProperties() const override {
102 return MachineFunctionProperties().setIsSSA();
103 }
104};
105
106} // end anonymous namespace
107
108INITIALIZE_PASS(GCNDPPCombineLegacy, DEBUG_TYPE, "GCN DPP Combine", false,
109 false)
110
111char GCNDPPCombineLegacy::ID = 0;
112
113char &llvm::GCNDPPCombineLegacyID = GCNDPPCombineLegacy::ID;
114
115FunctionPass *llvm::createGCNDPPCombinePass() {
116 return new GCNDPPCombineLegacy();
117}
118
119bool GCNDPPCombine::isShrinkable(MachineInstr &MI) const {
120 unsigned Op = MI.getOpcode();
121 if (!TII->isVOP3(Opcode: Op)) {
122 return false;
123 }
124 if (!TII->hasVALU32BitEncoding(Opcode: Op)) {
125 LLVM_DEBUG(dbgs() << " Inst hasn't e32 equivalent\n");
126 return false;
127 }
128 // Do not shrink True16 instructions pre-RA to avoid the restriction in
129 // register allocation from only being able to use 128 VGPRs
130 if (AMDGPU::isTrue16Inst(Opc: Op))
131 return false;
132 if (const auto *SDst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst)) {
133 // Give up if there are any uses of the sdst in carry-out or VOPC.
134 // The shrunken form of the instruction would write it to vcc instead of to
135 // a virtual register. If we rewrote the uses the shrinking would be
136 // possible.
137 if (!MRI->use_nodbg_empty(RegNo: SDst->getReg()))
138 return false;
139 }
140 // check if other than abs|neg modifiers are set (opsel for example)
141 const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG);
142 if (!hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::src0_modifiers, Value: 0, Mask) ||
143 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::src1_modifiers, Value: 0, Mask) ||
144 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::clamp, Value: 0) ||
145 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::omod, Value: 0) ||
146 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::byte_sel, Value: 0)) {
147 LLVM_DEBUG(dbgs() << " Inst has non-default modifiers\n");
148 return false;
149 }
150 return true;
151}
152
153int GCNDPPCombine::getDPPOp(unsigned Op, bool IsShrinkable) const {
154 int DPP32 = AMDGPU::getDPPOp32(Opcode: Op);
155 if (IsShrinkable) {
156 assert(DPP32 == -1);
157 int E32 = AMDGPU::getVOPe32(Opcode: Op);
158 DPP32 = (E32 == -1) ? -1 : AMDGPU::getDPPOp32(Opcode: E32);
159 }
160 if (DPP32 != -1 && TII->pseudoToMCOpcode(Opcode: DPP32) != -1)
161 return DPP32;
162 int DPP64 = -1;
163 if (ST->hasVOP3DPP())
164 DPP64 = AMDGPU::getDPPOp64(Opcode: Op);
165 if (DPP64 != -1 && TII->pseudoToMCOpcode(Opcode: DPP64) != -1)
166 return DPP64;
167 return -1;
168}
169
170// tracks the register operand definition and returns:
171// 1. immediate operand used to initialize the register if found
172// 2. nullptr if the register operand is undef
173// 3. the operand itself otherwise
174MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const {
175 auto *Def = getVRegSubRegDef(P: getRegSubRegPair(O: OldOpnd), MRI: *MRI);
176 if (!Def)
177 return nullptr;
178
179 switch(Def->getOpcode()) {
180 default: break;
181 case AMDGPU::IMPLICIT_DEF:
182 return nullptr;
183 case AMDGPU::COPY:
184 case AMDGPU::V_MOV_B32_e32:
185 case AMDGPU::V_MOV_B64_PSEUDO:
186 case AMDGPU::V_MOV_B64_e32:
187 case AMDGPU::V_MOV_B64_e64: {
188 auto &Op1 = Def->getOperand(i: 1);
189 if (Op1.isImm())
190 return &Op1;
191 break;
192 }
193 }
194 return &OldOpnd;
195}
196
197MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
198 MachineInstr &MovMI,
199 RegSubRegPair CombOldVGPR,
200 bool CombBCZ,
201 bool IsShrinkable) const {
202 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
203 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
204 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
205
206 bool HasVOP3DPP = ST->hasVOP3DPP();
207 auto OrigOp = OrigMI.getOpcode();
208 if (ST->useRealTrue16Insts() && AMDGPU::isTrue16Inst(Opc: OrigOp)) {
209 LLVM_DEBUG(
210 dbgs() << " failed: Did not expect any 16-bit uses of dpp values\n");
211 return nullptr;
212 }
213 auto DPPOp = getDPPOp(Op: OrigOp, IsShrinkable);
214 if (DPPOp == -1) {
215 LLVM_DEBUG(dbgs() << " failed: no DPP opcode\n");
216 return nullptr;
217 }
218 int OrigOpE32 = AMDGPU::getVOPe32(Opcode: OrigOp);
219 // Prior checks cover Mask with VOPC condition, but not on purpose
220 auto *RowMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::row_mask);
221 assert(RowMaskOpnd && RowMaskOpnd->isImm());
222 auto *BankMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bank_mask);
223 assert(BankMaskOpnd && BankMaskOpnd->isImm());
224 const bool MaskAllLanes =
225 RowMaskOpnd->getImm() == 0xF && BankMaskOpnd->getImm() == 0xF;
226 (void)MaskAllLanes;
227 assert((MaskAllLanes ||
228 !(TII->isVOPC(DPPOp) || (TII->isVOP3(DPPOp) && OrigOpE32 != -1 &&
229 TII->isVOPC(OrigOpE32)))) &&
230 "VOPC cannot form DPP unless mask is full");
231
232 auto DPPInst = BuildMI(BB&: *OrigMI.getParent(), I&: OrigMI,
233 MIMD: OrigMI.getDebugLoc(), MCID: TII->get(Opcode: DPPOp))
234 .setMIFlags(OrigMI.getFlags());
235
236 bool Fail = false;
237 do {
238 int NumOperands = 0;
239 if (auto *Dst = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::vdst)) {
240 DPPInst.add(MO: *Dst);
241 ++NumOperands;
242 }
243 if (auto *SDst = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::sdst)) {
244 if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::sdst)) {
245 DPPInst.add(MO: *SDst);
246 ++NumOperands;
247 }
248 // If we shrunk a 64bit vop3b to 32bits, just ignore the sdst
249 }
250
251 const int OldIdx = AMDGPU::getNamedOperandIdx(Opcode: DPPOp, Name: AMDGPU::OpName::old);
252 if (OldIdx != -1) {
253 assert(OldIdx == NumOperands);
254 assert(isOfRegClass(
255 CombOldVGPR,
256 *MRI->getRegClass(
257 TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg()),
258 *MRI));
259 auto *Def = getVRegSubRegDef(P: CombOldVGPR, MRI: *MRI);
260 DPPInst.addReg(RegNo: CombOldVGPR.Reg, Flags: getUndefRegState(B: !Def),
261 SubReg: CombOldVGPR.SubReg);
262 ++NumOperands;
263 } else if (TII->isVOPC(Opcode: DPPOp) || (TII->isVOP3(Opcode: DPPOp) && OrigOpE32 != -1 &&
264 TII->isVOPC(Opcode: OrigOpE32))) {
265 // VOPC DPP and VOPC promoted to VOP3 DPP do not have an old operand
266 // because they write to SGPRs not VGPRs
267 } else {
268 // TODO: this discards MAC/FMA instructions for now, let's add it later
269 LLVM_DEBUG(dbgs() << " failed: no old operand in DPP instruction,"
270 " TBD\n");
271 Fail = true;
272 break;
273 }
274
275 auto *Mod0 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src0_modifiers);
276 if (Mod0) {
277 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
278 AMDGPU::OpName::src0_modifiers));
279 assert(HasVOP3DPP ||
280 (0LL == (Mod0->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
281 DPPInst.addImm(Val: Mod0->getImm());
282 ++NumOperands;
283 } else if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
284 DPPInst.addImm(Val: 0);
285 ++NumOperands;
286 }
287 auto *Src0 = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::src0);
288 assert(Src0);
289 [[maybe_unused]] int Src0Idx = NumOperands;
290
291 DPPInst.add(MO: *Src0);
292 DPPInst->getOperand(i: NumOperands).setIsKill(false);
293 ++NumOperands;
294
295 auto *Mod1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1_modifiers);
296 if (Mod1) {
297 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
298 AMDGPU::OpName::src1_modifiers));
299 assert(HasVOP3DPP ||
300 (0LL == (Mod1->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
301 DPPInst.addImm(Val: Mod1->getImm());
302 ++NumOperands;
303 } else if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::src1_modifiers)) {
304 DPPInst.addImm(Val: 0);
305 ++NumOperands;
306 }
307 auto *Src1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1);
308 if (Src1) {
309 assert(AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::src1) &&
310 "dpp version of instruction missing src1");
311
312 DPPInst.add(MO: *Src1);
313 ++NumOperands;
314 }
315
316 auto *Mod2 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src2_modifiers);
317 if (Mod2) {
318 assert(NumOperands ==
319 AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::src2_modifiers));
320 assert(HasVOP3DPP ||
321 (0LL == (Mod2->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
322 DPPInst.addImm(Val: Mod2->getImm());
323 ++NumOperands;
324 }
325 auto *Src2 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src2);
326 if (Src2) {
327 if (!AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::src2)) {
328 LLVM_DEBUG(dbgs() << " failed: dpp does not have src2\n");
329 Fail = true;
330 break;
331 }
332 DPPInst.add(MO: *Src2);
333 ++NumOperands;
334 }
335
336 if (HasVOP3DPP) {
337 auto *ClampOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::clamp);
338 if (ClampOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::clamp)) {
339 DPPInst.addImm(Val: ClampOpr->getImm());
340 }
341 auto *VdstInOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::vdst_in);
342 if (VdstInOpr &&
343 AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::vdst_in)) {
344 DPPInst.add(MO: *VdstInOpr);
345 }
346 auto *OmodOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::omod);
347 if (OmodOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::omod)) {
348 DPPInst.addImm(Val: OmodOpr->getImm());
349 }
350 // Validate OP_SEL has to be set to all 0 and OP_SEL_HI has to be set to
351 // all 1.
352 if (TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::op_sel)) {
353 int64_t OpSel = 0;
354 OpSel |= (Mod0 ? (!!(Mod0->getImm() & SISrcMods::OP_SEL_0) << 0) : 0);
355 OpSel |= (Mod1 ? (!!(Mod1->getImm() & SISrcMods::OP_SEL_0) << 1) : 0);
356 OpSel |= (Mod2 ? (!!(Mod2->getImm() & SISrcMods::OP_SEL_0) << 2) : 0);
357 if (Mod0 && TII->isVOP3(MI: OrigMI) && !TII->isVOP3P(MI: OrigMI))
358 OpSel |= !!(Mod0->getImm() & SISrcMods::DST_OP_SEL) << 3;
359
360 if (OpSel != 0) {
361 LLVM_DEBUG(dbgs() << " failed: op_sel must be zero\n");
362 Fail = true;
363 break;
364 }
365 if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::op_sel))
366 DPPInst.addImm(Val: OpSel);
367 }
368 if (TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::op_sel_hi)) {
369 int64_t OpSelHi = 0;
370 OpSelHi |= (Mod0 ? (!!(Mod0->getImm() & SISrcMods::OP_SEL_1) << 0) : 0);
371 OpSelHi |= (Mod1 ? (!!(Mod1->getImm() & SISrcMods::OP_SEL_1) << 1) : 0);
372 OpSelHi |= (Mod2 ? (!!(Mod2->getImm() & SISrcMods::OP_SEL_1) << 2) : 0);
373
374 // Only vop3p has op_sel_hi, and all vop3p have 3 operands, so check
375 // the bitmask for 3 op_sel_hi bits set
376 assert(Src2 && "Expected vop3p with 3 operands");
377 if (OpSelHi != 7) {
378 LLVM_DEBUG(dbgs() << " failed: op_sel_hi must be all set to one\n");
379 Fail = true;
380 break;
381 }
382 if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::op_sel_hi))
383 DPPInst.addImm(Val: OpSelHi);
384 }
385 auto *NegOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::neg_lo);
386 if (NegOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::neg_lo)) {
387 DPPInst.addImm(Val: NegOpr->getImm());
388 }
389 auto *NegHiOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::neg_hi);
390 if (NegHiOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::neg_hi)) {
391 DPPInst.addImm(Val: NegHiOpr->getImm());
392 }
393 auto *ByteSelOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::byte_sel);
394 if (ByteSelOpr &&
395 AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::byte_sel)) {
396 DPPInst.addImm(Val: ByteSelOpr->getImm());
397 }
398 if (MachineOperand *BitOp3 =
399 TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::bitop3)) {
400 assert(AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::bitop3));
401 DPPInst.add(MO: *BitOp3);
402 }
403 }
404 DPPInst.add(MO: *TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::dpp_ctrl));
405 DPPInst.add(MO: *TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::row_mask));
406 DPPInst.add(MO: *TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bank_mask));
407 DPPInst.addImm(Val: CombBCZ ? 1 : 0);
408
409 constexpr AMDGPU::OpName Srcs[] = {
410 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
411
412 // FIXME: isOperandLegal expects to operate on an completely built
413 // instruction. We should have better legality APIs to check if the
414 // candidate operands will be legal without building the instruction first.
415 for (auto [I, OpName] : enumerate(First: Srcs)) {
416 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: DPPOp, Name: OpName);
417 if (OpIdx == -1)
418 break;
419
420 if (!TII->isOperandLegal(MI: *DPPInst, OpIdx)) {
421 LLVM_DEBUG(dbgs() << " failed: src" << I << " operand is illegal\n");
422 Fail = true;
423 break;
424 }
425 }
426 } while (false);
427
428 if (Fail) {
429 DPPInst.getInstr()->eraseFromParent();
430 return nullptr;
431 }
432 LLVM_DEBUG(dbgs() << " combined: " << *DPPInst.getInstr());
433 return DPPInst.getInstr();
434}
435
436static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd) {
437 assert(OldOpnd->isImm());
438 switch (OrigMIOp) {
439 default: break;
440 case AMDGPU::V_ADD_U32_e32:
441 case AMDGPU::V_ADD_U32_e64:
442 case AMDGPU::V_ADD_CO_U32_e32:
443 case AMDGPU::V_ADD_CO_U32_e64:
444 case AMDGPU::V_OR_B32_e32:
445 case AMDGPU::V_OR_B32_e64:
446 case AMDGPU::V_SUBREV_U32_e32:
447 case AMDGPU::V_SUBREV_U32_e64:
448 case AMDGPU::V_SUBREV_CO_U32_e32:
449 case AMDGPU::V_SUBREV_CO_U32_e64:
450 case AMDGPU::V_MAX_U32_e32:
451 case AMDGPU::V_MAX_U32_e64:
452 case AMDGPU::V_XOR_B32_e32:
453 case AMDGPU::V_XOR_B32_e64:
454 if (OldOpnd->getImm() == 0)
455 return true;
456 break;
457 case AMDGPU::V_AND_B32_e32:
458 case AMDGPU::V_AND_B32_e64:
459 case AMDGPU::V_MIN_U32_e32:
460 case AMDGPU::V_MIN_U32_e64:
461 if (static_cast<uint32_t>(OldOpnd->getImm()) ==
462 std::numeric_limits<uint32_t>::max())
463 return true;
464 break;
465 case AMDGPU::V_MIN_I32_e32:
466 case AMDGPU::V_MIN_I32_e64:
467 if (static_cast<int32_t>(OldOpnd->getImm()) ==
468 std::numeric_limits<int32_t>::max())
469 return true;
470 break;
471 case AMDGPU::V_MAX_I32_e32:
472 case AMDGPU::V_MAX_I32_e64:
473 if (static_cast<int32_t>(OldOpnd->getImm()) ==
474 std::numeric_limits<int32_t>::min())
475 return true;
476 break;
477 case AMDGPU::V_MUL_I32_I24_e32:
478 case AMDGPU::V_MUL_I32_I24_e64:
479 case AMDGPU::V_MUL_U32_U24_e32:
480 case AMDGPU::V_MUL_U32_U24_e64:
481 if (OldOpnd->getImm() == 1)
482 return true;
483 break;
484 case AMDGPU::V_MIN_F32_e32:
485 case AMDGPU::V_MIN_F32_e64:
486 if (static_cast<uint32_t>(OldOpnd->getImm()) == /*+inf=*/0x7F800000)
487 return true;
488 break;
489 case AMDGPU::V_MAX_F32_e32:
490 case AMDGPU::V_MAX_F32_e64:
491 if (static_cast<uint32_t>(OldOpnd->getImm()) == /*-inf=*/0xFF800000)
492 return true;
493 break;
494 case AMDGPU::V_MIN_F64_e64:
495 case AMDGPU::V_MIN_NUM_F64_e64:
496 if (static_cast<uint64_t>(OldOpnd->getImm()) == /*+inf=*/0x7FF0000000000000)
497 return true;
498 break;
499 case AMDGPU::V_MAX_F64_e64:
500 case AMDGPU::V_MAX_NUM_F64_e64:
501 if (static_cast<uint64_t>(OldOpnd->getImm()) == /*-inf=*/0xFFF0000000000000)
502 return true;
503 break;
504 case AMDGPU::V_MIN_F16_e32:
505 case AMDGPU::V_MIN_F16_e64:
506 case AMDGPU::V_MIN_F16_t16_e32:
507 case AMDGPU::V_MIN_F16_t16_e64:
508 case AMDGPU::V_MIN_F16_fake16_e32:
509 case AMDGPU::V_MIN_F16_fake16_e64:
510 if (static_cast<uint16_t>(OldOpnd->getImm()) == /*+inf=*/0x7C00)
511 return true;
512 break;
513 case AMDGPU::V_MAX_F16_e32:
514 case AMDGPU::V_MAX_F16_e64:
515 case AMDGPU::V_MAX_F16_t16_e32:
516 case AMDGPU::V_MAX_F16_t16_e64:
517 case AMDGPU::V_MAX_F16_fake16_e32:
518 case AMDGPU::V_MAX_F16_fake16_e64:
519 if (static_cast<uint16_t>(OldOpnd->getImm()) == /*-inf=*/0xFC00)
520 return true;
521 break;
522 }
523 return false;
524}
525
526MachineInstr *GCNDPPCombine::createDPPInst(
527 MachineInstr &OrigMI, MachineInstr &MovMI, RegSubRegPair CombOldVGPR,
528 MachineOperand *OldOpndValue, bool CombBCZ, bool IsShrinkable) const {
529 assert(CombOldVGPR.Reg);
530 if (!CombBCZ && OldOpndValue && OldOpndValue->isImm()) {
531 auto *Src1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1);
532 if (!Src1 || !Src1->isReg()) {
533 LLVM_DEBUG(dbgs() << " failed: no src1 or it isn't a register\n");
534 return nullptr;
535 }
536 if (!isIdentityValue(OrigMIOp: OrigMI.getOpcode(), OldOpnd: OldOpndValue)) {
537 LLVM_DEBUG(dbgs() << " failed: old immediate isn't an identity\n");
538 return nullptr;
539 }
540 CombOldVGPR = getRegSubRegPair(O: *Src1);
541 auto *MovDst = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::vdst);
542 const TargetRegisterClass *RC = MRI->getRegClass(Reg: MovDst->getReg());
543 if (!isOfRegClass(P: CombOldVGPR, TRC: *RC, MRI&: *MRI)) {
544 LLVM_DEBUG(dbgs() << " failed: src1 has wrong register class\n");
545 return nullptr;
546 }
547 }
548 return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ, IsShrinkable);
549}
550
551// returns true if MI doesn't have OpndName immediate operand or the
552// operand has Value
553bool GCNDPPCombine::hasNoImmOrEqual(MachineInstr &MI, AMDGPU::OpName OpndName,
554 int64_t Value, int64_t Mask) const {
555 auto *Imm = TII->getNamedOperand(MI, OperandName: OpndName);
556 if (!Imm)
557 return true;
558
559 assert(Imm->isImm());
560 return (Imm->getImm() & Mask) == Value;
561}
562
563bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
564 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
565 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
566 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
567 LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);
568
569 auto *DstOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::vdst);
570 assert(DstOpnd && DstOpnd->isReg());
571 auto DPPMovReg = DstOpnd->getReg();
572 if (DPPMovReg.isPhysical()) {
573 LLVM_DEBUG(dbgs() << " failed: dpp move writes physreg\n");
574 return false;
575 }
576 if (execMayBeModifiedBeforeAnyUse(MRI: *MRI, VReg: DPPMovReg, DefMI: MovMI)) {
577 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same"
578 " for all uses\n");
579 return false;
580 }
581
582 auto *DppCtrl = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::dpp_ctrl);
583 assert(DppCtrl && DppCtrl->isImm());
584 unsigned DppCtrlVal = DppCtrl->getImm();
585 if ((MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
586 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp)) {
587 if (!ST->hasFeature(Feature: AMDGPU::FeatureDPALU_DPP)) {
588 LLVM_DEBUG(dbgs() << " failed: 64 bit dpp move is unsupported\n");
589 // Split it.
590 return false;
591 }
592 if (!AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: DppCtrlVal)) {
593 LLVM_DEBUG(dbgs() << " failed: 64 bit dpp move uses unsupported"
594 " control value\n");
595 // Let it split, then control may become legal.
596 return false;
597 }
598 }
599
600 auto *RowMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::row_mask);
601 assert(RowMaskOpnd && RowMaskOpnd->isImm());
602 auto *BankMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bank_mask);
603 assert(BankMaskOpnd && BankMaskOpnd->isImm());
604 const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF &&
605 BankMaskOpnd->getImm() == 0xF;
606
607 auto *BCZOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bound_ctrl);
608 assert(BCZOpnd && BCZOpnd->isImm());
609 bool BoundCtrlZero = BCZOpnd->getImm();
610
611 auto *OldOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::old);
612 auto *SrcOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::src0);
613 assert(OldOpnd && OldOpnd->isReg());
614 assert(SrcOpnd && SrcOpnd->isReg());
615 if (OldOpnd->getReg().isPhysical() || SrcOpnd->getReg().isPhysical()) {
616 LLVM_DEBUG(dbgs() << " failed: dpp move reads physreg\n");
617 return false;
618 }
619
620 auto * const OldOpndValue = getOldOpndValue(OldOpnd&: *OldOpnd);
621 // OldOpndValue is either undef (IMPLICIT_DEF) or immediate or something else
622 // We could use: assert(!OldOpndValue || OldOpndValue->isImm())
623 // but the third option is used to distinguish undef from non-immediate
624 // to reuse IMPLICIT_DEF instruction later
625 assert(!OldOpndValue || OldOpndValue->isImm() || OldOpndValue == OldOpnd);
626
627 bool CombBCZ = false;
628
629 if (MaskAllLanes && BoundCtrlZero) { // [1]
630 CombBCZ = true;
631 } else {
632 if (!OldOpndValue || !OldOpndValue->isImm()) {
633 LLVM_DEBUG(dbgs() << " failed: the DPP mov isn't combinable\n");
634 return false;
635 }
636
637 if (OldOpndValue->getImm() == 0) {
638 if (MaskAllLanes) {
639 assert(!BoundCtrlZero); // by check [1]
640 CombBCZ = true;
641 }
642 } else if (BoundCtrlZero) {
643 assert(!MaskAllLanes); // by check [1]
644 LLVM_DEBUG(dbgs() <<
645 " failed: old!=0 and bctrl:0 and not all lanes isn't combinable\n");
646 return false;
647 }
648 }
649
650 LLVM_DEBUG(dbgs() << " old=";
651 if (!OldOpndValue)
652 dbgs() << "undef";
653 else
654 dbgs() << *OldOpndValue;
655 dbgs() << ", bound_ctrl=" << CombBCZ << '\n');
656
657 SmallVector<MachineInstr*, 4> OrigMIs, DPPMIs;
658 DenseMap<MachineInstr*, SmallVector<unsigned, 4>> RegSeqWithOpNos;
659 auto CombOldVGPR = getRegSubRegPair(O: *OldOpnd);
660 // try to reuse previous old reg if its undefined (IMPLICIT_DEF)
661 if (CombBCZ && OldOpndValue) { // CombOldVGPR should be undef
662 const TargetRegisterClass *RC = MRI->getRegClass(Reg: DPPMovReg);
663 CombOldVGPR = RegSubRegPair(
664 MRI->createVirtualRegister(RegClass: RC));
665 auto UndefInst = BuildMI(BB&: *MovMI.getParent(), I&: MovMI, MIMD: MovMI.getDebugLoc(),
666 MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: CombOldVGPR.Reg);
667 DPPMIs.push_back(Elt: UndefInst.getInstr());
668 }
669
670 OrigMIs.push_back(Elt: &MovMI);
671 bool Rollback = true;
672 SmallVector<MachineOperand *, 16> Uses(
673 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: DPPMovReg)));
674
675 while (!Uses.empty()) {
676 MachineOperand *Use = Uses.pop_back_val();
677 Rollback = true;
678
679 auto &OrigMI = *Use->getParent();
680 LLVM_DEBUG(dbgs() << " try: " << OrigMI);
681
682 auto OrigOp = OrigMI.getOpcode();
683 assert((TII->get(OrigOp).getSize() != 4 || !AMDGPU::isTrue16Inst(OrigOp)) &&
684 "There should not be e32 True16 instructions pre-RA");
685 if (OrigOp == AMDGPU::REG_SEQUENCE) {
686 Register FwdReg = OrigMI.getOperand(i: 0).getReg();
687 unsigned FwdSubReg = 0;
688
689 if (execMayBeModifiedBeforeAnyUse(MRI: *MRI, VReg: FwdReg, DefMI: OrigMI)) {
690 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same"
691 " for all uses\n");
692 break;
693 }
694
695 unsigned OpNo, E = OrigMI.getNumOperands();
696 for (OpNo = 1; OpNo < E; OpNo += 2) {
697 if (OrigMI.getOperand(i: OpNo).getReg() == DPPMovReg) {
698 FwdSubReg = OrigMI.getOperand(i: OpNo + 1).getImm();
699 break;
700 }
701 }
702
703 if (!FwdSubReg)
704 break;
705
706 for (auto &Op : MRI->use_nodbg_operands(Reg: FwdReg)) {
707 if (Op.getSubReg() == FwdSubReg)
708 Uses.push_back(Elt: &Op);
709 }
710 RegSeqWithOpNos[&OrigMI].push_back(Elt: OpNo);
711 continue;
712 }
713
714 bool IsShrinkable = isShrinkable(MI&: OrigMI);
715 if (!(IsShrinkable ||
716 ((TII->isVOP3P(Opcode: OrigOp) || TII->isVOPC(Opcode: OrigOp) ||
717 TII->isVOP3(Opcode: OrigOp)) &&
718 ST->hasVOP3DPP()) ||
719 TII->isVOP1(Opcode: OrigOp) || TII->isVOP2(Opcode: OrigOp))) {
720 LLVM_DEBUG(dbgs() << " failed: not VOP1/2/3/3P/C\n");
721 break;
722 }
723 if (OrigMI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: ST->getRegisterInfo())) {
724 LLVM_DEBUG(dbgs() << " failed: can't combine v_cmpx\n");
725 break;
726 }
727
728 auto *Src0 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src0);
729 auto *Src1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1);
730 if (Use != Src0 && !(Use == Src1 && OrigMI.isCommutable())) { // [1]
731 LLVM_DEBUG(dbgs() << " failed: no suitable operands\n");
732 break;
733 }
734
735 auto *Src2 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src2);
736 assert(Src0 && "Src1 without Src0?");
737 if ((Use == Src0 && ((Src1 && Src1->isIdenticalTo(Other: *Src0)) ||
738 (Src2 && Src2->isIdenticalTo(Other: *Src0)))) ||
739 (Use == Src1 && (Src1->isIdenticalTo(Other: *Src0) ||
740 (Src2 && Src2->isIdenticalTo(Other: *Src1))))) {
741 LLVM_DEBUG(
742 dbgs()
743 << " " << OrigMI
744 << " failed: DPP register is used more than once per instruction\n");
745 break;
746 }
747
748 if (!ST->hasFeature(Feature: AMDGPU::FeatureDPALU_DPP) &&
749 AMDGPU::isDPALU_DPP32BitOpc(Opc: OrigOp)) {
750 LLVM_DEBUG(dbgs() << " " << OrigMI
751 << " failed: DPP ALU DPP is not supported\n");
752 break;
753 }
754
755 if (!AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: DppCtrlVal) &&
756 AMDGPU::isDPALU_DPP(OpDesc: TII->get(Opcode: OrigOp), MII: *TII, ST: *ST)) {
757 LLVM_DEBUG(dbgs() << " " << OrigMI
758 << " failed: not valid 64-bit DPP control value\n");
759 break;
760 }
761
762 LLVM_DEBUG(dbgs() << " combining: " << OrigMI);
763 if (Use == Src0) {
764 if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR,
765 OldOpndValue, CombBCZ, IsShrinkable)) {
766 DPPMIs.push_back(Elt: DPPInst);
767 Rollback = false;
768 }
769 } else {
770 assert(Use == Src1 && OrigMI.isCommutable()); // by check [1]
771 auto *BB = OrigMI.getParent();
772 auto *NewMI = BB->getParent()->CloneMachineInstr(Orig: &OrigMI);
773 BB->insert(I: OrigMI, MI: NewMI);
774 if (TII->commuteInstruction(MI&: *NewMI)) {
775 LLVM_DEBUG(dbgs() << " commuted: " << *NewMI);
776 if (auto *DPPInst =
777 createDPPInst(OrigMI&: *NewMI, MovMI, CombOldVGPR, OldOpndValue, CombBCZ,
778 IsShrinkable)) {
779 DPPMIs.push_back(Elt: DPPInst);
780 Rollback = false;
781 }
782 } else
783 LLVM_DEBUG(dbgs() << " failed: cannot be commuted\n");
784 NewMI->eraseFromParent();
785 }
786 if (Rollback)
787 break;
788 OrigMIs.push_back(Elt: &OrigMI);
789 }
790
791 Rollback |= !Uses.empty();
792
793 for (auto *MI : *(Rollback? &DPPMIs : &OrigMIs))
794 MI->eraseFromParent();
795
796 if (!Rollback) {
797 for (auto &S : RegSeqWithOpNos) {
798 if (MRI->use_nodbg_empty(RegNo: S.first->getOperand(i: 0).getReg())) {
799 S.first->eraseFromParent();
800 continue;
801 }
802 while (!S.second.empty())
803 S.first->getOperand(i: S.second.pop_back_val()).setIsUndef();
804 }
805 }
806
807 return !Rollback;
808}
809
810bool GCNDPPCombineLegacy::runOnMachineFunction(MachineFunction &MF) {
811 if (skipFunction(F: MF.getFunction()))
812 return false;
813
814 return GCNDPPCombine().run(MF);
815}
816
817bool GCNDPPCombine::run(MachineFunction &MF) {
818 ST = &MF.getSubtarget<GCNSubtarget>();
819 if (!ST->hasDPP())
820 return false;
821
822 MRI = &MF.getRegInfo();
823 TII = ST->getInstrInfo();
824
825 bool Changed = false;
826 for (auto &MBB : MF) {
827 for (MachineInstr &MI : llvm::make_early_inc_range(Range: llvm::reverse(C&: MBB))) {
828 if (MI.getOpcode() == AMDGPU::V_MOV_B32_dpp && combineDPPMov(MovMI&: MI)) {
829 Changed = true;
830 ++NumDPPMovsCombined;
831 } else if (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
832 MI.getOpcode() == AMDGPU::V_MOV_B64_dpp) {
833 if (ST->hasDPALU_DPP() && combineDPPMov(MovMI&: MI)) {
834 Changed = true;
835 ++NumDPPMovsCombined;
836 } else {
837 auto Split = TII->expandMovDPP64(MI);
838 for (auto *M : {Split.first, Split.second}) {
839 if (M && combineDPPMov(MovMI&: *M))
840 ++NumDPPMovsCombined;
841 }
842 Changed = true;
843 }
844 }
845 }
846 }
847 return Changed;
848}
849
850PreservedAnalyses GCNDPPCombinePass::run(MachineFunction &MF,
851 MachineFunctionAnalysisManager &) {
852 MFPropsModifier _(*this, MF);
853
854 if (MF.getFunction().hasOptNone())
855 return PreservedAnalyses::all();
856
857 bool Changed = GCNDPPCombine().run(MF);
858 if (!Changed)
859 return PreservedAnalyses::all();
860
861 auto PA = getMachineFunctionPassPreservedAnalyses();
862 PA.preserveSet<CFGAnalyses>();
863 return PA;
864}
865