1//=======- GCNDPPCombine.cpp - optimization for DPP instructions ---==========//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// The pass combines V_MOV_B32_dpp instruction with its VALU uses as a DPP src0
9// operand. If any of the use instruction cannot be combined with the mov the
10// whole sequence is reverted.
11//
12// $old = ...
13// $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane,
14// dpp_controls..., $row_mask, $bank_mask, $bound_ctrl
15// $res = VALU $dpp_value [, src1]
16//
17// to
18//
19// $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, [src1,]
20// dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl
21//
22// Combining rules :
23//
24// if $row_mask and $bank_mask are fully enabled (0xF) and
25// $bound_ctrl==DPP_BOUND_ZERO or $old==0
26// -> $combined_old = undef,
27// $combined_bound_ctrl = DPP_BOUND_ZERO
28//
29// if the VALU op is binary and
30// $bound_ctrl==DPP_BOUND_OFF and
31// $old==identity value (immediate) for the VALU op
32// -> $combined_old = src1,
33// $combined_bound_ctrl = DPP_BOUND_OFF
34//
35// Otherwise cancel.
36//
37// The mov_dpp instruction should reside in the same BB as all its uses
38//===----------------------------------------------------------------------===//
39
40#include "GCNDPPCombine.h"
41#include "AMDGPU.h"
42#include "GCNSubtarget.h"
43#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
44#include "llvm/ADT/Statistic.h"
45#include "llvm/CodeGen/MachineFunctionPass.h"
46#include "llvm/CodeGen/MachineInstrBuilder.h"
47
48using namespace llvm;
49
50#define DEBUG_TYPE "gcn-dpp-combine"
51
52STATISTIC(NumDPPMovsCombined, "Number of DPP moves combined.");
53
54namespace {
55
56class GCNDPPCombine {
57 MachineRegisterInfo *MRI;
58 const SIInstrInfo *TII;
59 const GCNSubtarget *ST;
60
61 using RegSubRegPair = TargetInstrInfo::RegSubRegPair;
62
63 MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const;
64
65 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
66 RegSubRegPair CombOldVGPR,
67 MachineOperand *OldOpnd, bool CombBCZ,
68 bool IsShrinkable) const;
69
70 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
71 RegSubRegPair CombOldVGPR, bool CombBCZ,
72 bool IsShrinkable) const;
73
74 bool hasNoImmOrEqual(MachineInstr &MI, AMDGPU::OpName OpndName, int64_t Value,
75 int64_t Mask = -1) const;
76
77 bool combineDPPMov(MachineInstr &MI) const;
78
79 int getDPPOp(unsigned Op, bool IsShrinkable) const;
80 bool isShrinkable(MachineInstr &MI) const;
81
82public:
83 bool run(MachineFunction &MF);
84};
85
86class GCNDPPCombineLegacy : public MachineFunctionPass {
87public:
88 static char ID;
89
90 GCNDPPCombineLegacy() : MachineFunctionPass(ID) {}
91
92 bool runOnMachineFunction(MachineFunction &MF) override;
93
94 StringRef getPassName() const override { return "GCN DPP Combine"; }
95
96 void getAnalysisUsage(AnalysisUsage &AU) const override {
97 AU.setPreservesCFG();
98 MachineFunctionPass::getAnalysisUsage(AU);
99 }
100
101 MachineFunctionProperties getRequiredProperties() const override {
102 return MachineFunctionProperties().setIsSSA();
103 }
104};
105
106} // end anonymous namespace
107
108INITIALIZE_PASS(GCNDPPCombineLegacy, DEBUG_TYPE, "GCN DPP Combine", false,
109 false)
110
111char GCNDPPCombineLegacy::ID = 0;
112
113char &llvm::GCNDPPCombineLegacyID = GCNDPPCombineLegacy::ID;
114
115bool GCNDPPCombine::isShrinkable(MachineInstr &MI) const {
116 unsigned Op = MI.getOpcode();
117 if (!TII->isVOP3(Opcode: Op)) {
118 return false;
119 }
120 if (!TII->hasVALU32BitEncoding(Opcode: Op)) {
121 LLVM_DEBUG(dbgs() << " Inst hasn't e32 equivalent\n");
122 return false;
123 }
124 // Do not shrink True16 instructions pre-RA to avoid the restriction in
125 // register allocation from only being able to use 128 VGPRs
126 if (AMDGPU::isTrue16Inst(Opc: Op))
127 return false;
128 if (const auto *SDst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst)) {
129 // Give up if there are any uses of the sdst in carry-out or VOPC.
130 // The shrunken form of the instruction would write it to vcc instead of to
131 // a virtual register. If we rewrote the uses the shrinking would be
132 // possible.
133 if (!MRI->use_nodbg_empty(RegNo: SDst->getReg()))
134 return false;
135 }
136 // check if other than abs|neg modifiers are set (opsel for example)
137 const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG);
138 if (!hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::src0_modifiers, Value: 0, Mask) ||
139 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::src1_modifiers, Value: 0, Mask) ||
140 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::clamp, Value: 0) ||
141 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::omod, Value: 0) ||
142 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::byte_sel, Value: 0)) {
143 LLVM_DEBUG(dbgs() << " Inst has non-default modifiers\n");
144 return false;
145 }
146 return true;
147}
148
149int GCNDPPCombine::getDPPOp(unsigned Op, bool IsShrinkable) const {
150 int DPP32 = AMDGPU::getDPPOp32(Opcode: Op);
151 if (IsShrinkable) {
152 assert(DPP32 == -1);
153 int E32 = AMDGPU::getVOPe32(Opcode: Op);
154 DPP32 = (E32 == -1) ? -1 : AMDGPU::getDPPOp32(Opcode: E32);
155 }
156 if (DPP32 != -1 && TII->pseudoToMCOpcode(Opcode: DPP32) != -1)
157 return DPP32;
158 int DPP64 = -1;
159 if (ST->hasVOP3DPP())
160 DPP64 = AMDGPU::getDPPOp64(Opcode: Op);
161 if (DPP64 != -1 && TII->pseudoToMCOpcode(Opcode: DPP64) != -1)
162 return DPP64;
163 return -1;
164}
165
166// tracks the register operand definition and returns:
167// 1. immediate operand used to initialize the register if found
168// 2. nullptr if the register operand is undef
169// 3. the operand itself otherwise
170MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const {
171 auto *Def = getVRegSubRegDef(P: getRegSubRegPair(O: OldOpnd), MRI: *MRI);
172 if (!Def)
173 return nullptr;
174
175 switch(Def->getOpcode()) {
176 default: break;
177 case AMDGPU::IMPLICIT_DEF:
178 return nullptr;
179 case AMDGPU::COPY:
180 case AMDGPU::V_MOV_B32_e32:
181 case AMDGPU::V_MOV_B64_PSEUDO:
182 case AMDGPU::V_MOV_B64_e32:
183 case AMDGPU::V_MOV_B64_e64: {
184 auto &Op1 = Def->getOperand(i: 1);
185 if (Op1.isImm())
186 return &Op1;
187 break;
188 }
189 }
190 return &OldOpnd;
191}
192
193MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
194 MachineInstr &MovMI,
195 RegSubRegPair CombOldVGPR,
196 bool CombBCZ,
197 bool IsShrinkable) const {
198 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
199 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
200 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
201
202 bool HasVOP3DPP = ST->hasVOP3DPP();
203 auto OrigOp = OrigMI.getOpcode();
204 if (ST->useRealTrue16Insts() && AMDGPU::isTrue16Inst(Opc: OrigOp)) {
205 LLVM_DEBUG(
206 dbgs() << " failed: Did not expect any 16-bit uses of dpp values\n");
207 return nullptr;
208 }
209 auto DPPOp = getDPPOp(Op: OrigOp, IsShrinkable);
210 if (DPPOp == -1) {
211 LLVM_DEBUG(dbgs() << " failed: no DPP opcode\n");
212 return nullptr;
213 }
214
215 const int OldIdx = AMDGPU::getNamedOperandIdx(Opcode: DPPOp, Name: AMDGPU::OpName::old);
216 const int MovDstIdx =
217 AMDGPU::getNamedOperandIdx(Opcode: MovMI.getOpcode(), Name: AMDGPU::OpName::vdst);
218 if (OldIdx != -1 &&
219 TII->getOpSize(Opcode: DPPOp, OpNo: OldIdx) != TII->getOpSize(MI: MovMI, OpNo: MovDstIdx)) {
220 LLVM_DEBUG(dbgs() << " failed: old operand size differs from dst\n");
221 return nullptr;
222 }
223
224 int OrigOpE32 = AMDGPU::getVOPe32(Opcode: OrigOp);
225 // Prior checks cover Mask with VOPC condition, but not on purpose
226 auto *RowMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::row_mask);
227 assert(RowMaskOpnd && RowMaskOpnd->isImm());
228 auto *BankMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bank_mask);
229 assert(BankMaskOpnd && BankMaskOpnd->isImm());
230 const bool MaskAllLanes =
231 RowMaskOpnd->getImm() == 0xF && BankMaskOpnd->getImm() == 0xF;
232 (void)MaskAllLanes;
233 assert((MaskAllLanes ||
234 !(TII->isVOPC(DPPOp) || (TII->isVOP3(DPPOp) && OrigOpE32 != -1 &&
235 TII->isVOPC(OrigOpE32)))) &&
236 "VOPC cannot form DPP unless mask is full");
237
238 auto DPPInst = BuildMI(BB&: *OrigMI.getParent(), I&: OrigMI,
239 MIMD: OrigMI.getDebugLoc(), MCID: TII->get(Opcode: DPPOp))
240 .setMIFlags(OrigMI.getFlags());
241
242 bool Fail = false;
243 do {
244 int NumOperands = 0;
245 if (auto *Dst = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::vdst)) {
246 DPPInst.add(MO: *Dst);
247 ++NumOperands;
248 }
249 if (auto *SDst = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::sdst)) {
250 if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::sdst)) {
251 DPPInst.add(MO: *SDst);
252 ++NumOperands;
253 }
254 // If we shrunk a 64bit vop3b to 32bits, just ignore the sdst
255 }
256
257 if (OldIdx != -1) {
258 assert(OldIdx == NumOperands);
259 assert(isOfRegClass(
260 CombOldVGPR,
261 *MRI->getRegClass(
262 TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg()),
263 *MRI));
264 auto *Def = getVRegSubRegDef(P: CombOldVGPR, MRI: *MRI);
265 DPPInst.addReg(RegNo: CombOldVGPR.Reg, Flags: getUndefRegState(B: !Def),
266 SubReg: CombOldVGPR.SubReg);
267 ++NumOperands;
268 } else if (TII->isVOPC(Opcode: DPPOp) || (TII->isVOP3(Opcode: DPPOp) && OrigOpE32 != -1 &&
269 TII->isVOPC(Opcode: OrigOpE32))) {
270 // VOPC DPP and VOPC promoted to VOP3 DPP do not have an old operand
271 // because they write to SGPRs not VGPRs
272 } else {
273 // TODO: this discards MAC/FMA instructions for now, let's add it later
274 LLVM_DEBUG(dbgs() << " failed: no old operand in DPP instruction,"
275 " TBD\n");
276 Fail = true;
277 break;
278 }
279
280 auto *Mod0 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src0_modifiers);
281 if (Mod0) {
282 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
283 AMDGPU::OpName::src0_modifiers));
284 assert(HasVOP3DPP ||
285 (0LL == (Mod0->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
286 DPPInst.addImm(Val: Mod0->getImm());
287 ++NumOperands;
288 } else if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
289 DPPInst.addImm(Val: 0);
290 ++NumOperands;
291 }
292 auto *Src0 = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::src0);
293 assert(Src0);
294 [[maybe_unused]] int Src0Idx = NumOperands;
295
296 DPPInst.add(MO: *Src0);
297 DPPInst->getOperand(i: NumOperands).setIsKill(false);
298 ++NumOperands;
299
300 auto *Mod1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1_modifiers);
301 if (Mod1) {
302 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
303 AMDGPU::OpName::src1_modifiers));
304 assert(HasVOP3DPP ||
305 (0LL == (Mod1->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
306 DPPInst.addImm(Val: Mod1->getImm());
307 ++NumOperands;
308 } else if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::src1_modifiers)) {
309 DPPInst.addImm(Val: 0);
310 ++NumOperands;
311 }
312 auto *Src1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1);
313 if (Src1) {
314 assert(AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::src1) &&
315 "dpp version of instruction missing src1");
316
317 DPPInst.add(MO: *Src1);
318 ++NumOperands;
319 }
320
321 auto *Mod2 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src2_modifiers);
322 if (Mod2) {
323 assert(NumOperands ==
324 AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::src2_modifiers));
325 assert(HasVOP3DPP ||
326 (0LL == (Mod2->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
327 DPPInst.addImm(Val: Mod2->getImm());
328 ++NumOperands;
329 }
330 auto *Src2 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src2);
331 if (Src2) {
332 if (!AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::src2)) {
333 LLVM_DEBUG(dbgs() << " failed: dpp does not have src2\n");
334 Fail = true;
335 break;
336 }
337 DPPInst.add(MO: *Src2);
338 ++NumOperands;
339 }
340
341 if (HasVOP3DPP) {
342 auto *ClampOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::clamp);
343 if (ClampOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::clamp)) {
344 DPPInst.addImm(Val: ClampOpr->getImm());
345 }
346 auto *VdstInOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::vdst_in);
347 if (VdstInOpr &&
348 AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::vdst_in)) {
349 DPPInst.add(MO: *VdstInOpr);
350 }
351 auto *OmodOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::omod);
352 if (OmodOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::omod)) {
353 DPPInst.addImm(Val: OmodOpr->getImm());
354 }
355 // Validate OP_SEL has to be set to all 0 and OP_SEL_HI has to be set to
356 // all 1.
357 if (TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::op_sel)) {
358 int64_t OpSel = 0;
359 OpSel |= (Mod0 ? (!!(Mod0->getImm() & SISrcMods::OP_SEL_0) << 0) : 0);
360 OpSel |= (Mod1 ? (!!(Mod1->getImm() & SISrcMods::OP_SEL_0) << 1) : 0);
361 OpSel |= (Mod2 ? (!!(Mod2->getImm() & SISrcMods::OP_SEL_0) << 2) : 0);
362 if (Mod0 && TII->isVOP3(MI: OrigMI) && !TII->isVOP3P(MI: OrigMI))
363 OpSel |= !!(Mod0->getImm() & SISrcMods::DST_OP_SEL) << 3;
364
365 if (OpSel != 0) {
366 LLVM_DEBUG(dbgs() << " failed: op_sel must be zero\n");
367 Fail = true;
368 break;
369 }
370 if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::op_sel))
371 DPPInst.addImm(Val: OpSel);
372 }
373 if (TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::op_sel_hi)) {
374 int64_t OpSelHi = 0;
375 OpSelHi |= (Mod0 ? (!!(Mod0->getImm() & SISrcMods::OP_SEL_1) << 0) : 0);
376 OpSelHi |= (Mod1 ? (!!(Mod1->getImm() & SISrcMods::OP_SEL_1) << 1) : 0);
377 OpSelHi |= (Mod2 ? (!!(Mod2->getImm() & SISrcMods::OP_SEL_1) << 2) : 0);
378
379 // Only vop3p has op_sel_hi, and all vop3p have 3 operands, so check
380 // the bitmask for 3 op_sel_hi bits set
381 assert(Src2 && "Expected vop3p with 3 operands");
382 if (OpSelHi != 7) {
383 LLVM_DEBUG(dbgs() << " failed: op_sel_hi must be all set to one\n");
384 Fail = true;
385 break;
386 }
387 if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::op_sel_hi))
388 DPPInst.addImm(Val: OpSelHi);
389 }
390 auto *NegOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::neg_lo);
391 if (NegOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::neg_lo)) {
392 DPPInst.addImm(Val: NegOpr->getImm());
393 }
394 auto *NegHiOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::neg_hi);
395 if (NegHiOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::neg_hi)) {
396 DPPInst.addImm(Val: NegHiOpr->getImm());
397 }
398 auto *ByteSelOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::byte_sel);
399 if (ByteSelOpr &&
400 AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::byte_sel)) {
401 DPPInst.addImm(Val: ByteSelOpr->getImm());
402 }
403 if (MachineOperand *BitOp3 =
404 TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::bitop3)) {
405 assert(AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::bitop3));
406 DPPInst.add(MO: *BitOp3);
407 }
408 }
409 DPPInst.add(MO: *TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::dpp_ctrl));
410 DPPInst.add(MO: *TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::row_mask));
411 DPPInst.add(MO: *TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bank_mask));
412 DPPInst.addImm(Val: CombBCZ ? 1 : 0);
413
414 constexpr AMDGPU::OpName Srcs[] = {
415 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
416
417 // FIXME: isOperandLegal expects to operate on an completely built
418 // instruction. We should have better legality APIs to check if the
419 // candidate operands will be legal without building the instruction first.
420 for (auto [I, OpName] : enumerate(First: Srcs)) {
421 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: DPPOp, Name: OpName);
422 if (OpIdx == -1)
423 break;
424
425 if (!TII->isOperandLegal(MI: *DPPInst, OpIdx)) {
426 LLVM_DEBUG(dbgs() << " failed: src" << I << " operand is illegal\n");
427 Fail = true;
428 break;
429 }
430 }
431 } while (false);
432
433 if (Fail) {
434 DPPInst.getInstr()->eraseFromParent();
435 return nullptr;
436 }
437 LLVM_DEBUG(dbgs() << " combined: " << *DPPInst.getInstr());
438 return DPPInst.getInstr();
439}
440
441static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd) {
442 assert(OldOpnd->isImm());
443 switch (OrigMIOp) {
444 default: break;
445 case AMDGPU::V_ADD_U32_e32:
446 case AMDGPU::V_ADD_U32_e64:
447 case AMDGPU::V_ADD_CO_U32_e32:
448 case AMDGPU::V_ADD_CO_U32_e64:
449 case AMDGPU::V_OR_B32_e32:
450 case AMDGPU::V_OR_B32_e64:
451 case AMDGPU::V_SUBREV_U32_e32:
452 case AMDGPU::V_SUBREV_U32_e64:
453 case AMDGPU::V_SUBREV_CO_U32_e32:
454 case AMDGPU::V_SUBREV_CO_U32_e64:
455 case AMDGPU::V_MAX_U32_e32:
456 case AMDGPU::V_MAX_U32_e64:
457 case AMDGPU::V_XOR_B32_e32:
458 case AMDGPU::V_XOR_B32_e64:
459 if (OldOpnd->getImm() == 0)
460 return true;
461 break;
462 case AMDGPU::V_AND_B32_e32:
463 case AMDGPU::V_AND_B32_e64:
464 case AMDGPU::V_MIN_U32_e32:
465 case AMDGPU::V_MIN_U32_e64:
466 if (static_cast<uint32_t>(OldOpnd->getImm()) ==
467 std::numeric_limits<uint32_t>::max())
468 return true;
469 break;
470 case AMDGPU::V_MIN_I32_e32:
471 case AMDGPU::V_MIN_I32_e64:
472 if (static_cast<int32_t>(OldOpnd->getImm()) ==
473 std::numeric_limits<int32_t>::max())
474 return true;
475 break;
476 case AMDGPU::V_MAX_I32_e32:
477 case AMDGPU::V_MAX_I32_e64:
478 if (static_cast<int32_t>(OldOpnd->getImm()) ==
479 std::numeric_limits<int32_t>::min())
480 return true;
481 break;
482 case AMDGPU::V_MUL_I32_I24_e32:
483 case AMDGPU::V_MUL_I32_I24_e64:
484 case AMDGPU::V_MUL_U32_U24_e32:
485 case AMDGPU::V_MUL_U32_U24_e64:
486 if (OldOpnd->getImm() == 1)
487 return true;
488 break;
489 case AMDGPU::V_MIN_F32_e32:
490 case AMDGPU::V_MIN_F32_e64:
491 if (static_cast<uint32_t>(OldOpnd->getImm()) == /*+inf=*/0x7F800000)
492 return true;
493 break;
494 case AMDGPU::V_MAX_F32_e32:
495 case AMDGPU::V_MAX_F32_e64:
496 if (static_cast<uint32_t>(OldOpnd->getImm()) == /*-inf=*/0xFF800000)
497 return true;
498 break;
499 case AMDGPU::V_MIN_F64_e64:
500 case AMDGPU::V_MIN_NUM_F64_e64:
501 if (static_cast<uint64_t>(OldOpnd->getImm()) == /*+inf=*/0x7FF0000000000000)
502 return true;
503 break;
504 case AMDGPU::V_MAX_F64_e64:
505 case AMDGPU::V_MAX_NUM_F64_e64:
506 if (static_cast<uint64_t>(OldOpnd->getImm()) == /*-inf=*/0xFFF0000000000000)
507 return true;
508 break;
509 case AMDGPU::V_MIN_F16_e32:
510 case AMDGPU::V_MIN_F16_e64:
511 case AMDGPU::V_MIN_F16_t16_e32:
512 case AMDGPU::V_MIN_F16_t16_e64:
513 case AMDGPU::V_MIN_F16_fake16_e32:
514 case AMDGPU::V_MIN_F16_fake16_e64:
515 if (static_cast<uint16_t>(OldOpnd->getImm()) == /*+inf=*/0x7C00)
516 return true;
517 break;
518 case AMDGPU::V_MAX_F16_e32:
519 case AMDGPU::V_MAX_F16_e64:
520 case AMDGPU::V_MAX_F16_t16_e32:
521 case AMDGPU::V_MAX_F16_t16_e64:
522 case AMDGPU::V_MAX_F16_fake16_e32:
523 case AMDGPU::V_MAX_F16_fake16_e64:
524 if (static_cast<uint16_t>(OldOpnd->getImm()) == /*-inf=*/0xFC00)
525 return true;
526 break;
527 }
528 return false;
529}
530
531MachineInstr *GCNDPPCombine::createDPPInst(
532 MachineInstr &OrigMI, MachineInstr &MovMI, RegSubRegPair CombOldVGPR,
533 MachineOperand *OldOpndValue, bool CombBCZ, bool IsShrinkable) const {
534 assert(CombOldVGPR.Reg);
535 if (!CombBCZ && OldOpndValue && OldOpndValue->isImm()) {
536 auto *Src1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1);
537 if (!Src1 || !Src1->isReg()) {
538 LLVM_DEBUG(dbgs() << " failed: no src1 or it isn't a register\n");
539 return nullptr;
540 }
541 if (!isIdentityValue(OrigMIOp: OrigMI.getOpcode(), OldOpnd: OldOpndValue)) {
542 LLVM_DEBUG(dbgs() << " failed: old immediate isn't an identity\n");
543 return nullptr;
544 }
545 CombOldVGPR = getRegSubRegPair(O: *Src1);
546 auto *MovDst = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::vdst);
547 const TargetRegisterClass *RC = MRI->getRegClass(Reg: MovDst->getReg());
548 if (!isOfRegClass(P: CombOldVGPR, TRC: *RC, MRI&: *MRI)) {
549 LLVM_DEBUG(dbgs() << " failed: src1 has wrong register class\n");
550 return nullptr;
551 }
552 }
553 return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ, IsShrinkable);
554}
555
556// returns true if MI doesn't have OpndName immediate operand or the
557// operand has Value
558bool GCNDPPCombine::hasNoImmOrEqual(MachineInstr &MI, AMDGPU::OpName OpndName,
559 int64_t Value, int64_t Mask) const {
560 auto *Imm = TII->getNamedOperand(MI, OperandName: OpndName);
561 if (!Imm)
562 return true;
563
564 assert(Imm->isImm());
565 return (Imm->getImm() & Mask) == Value;
566}
567
568bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
569 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
570 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
571 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
572 LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);
573
574 auto *DstOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::vdst);
575 assert(DstOpnd && DstOpnd->isReg());
576 auto DPPMovReg = DstOpnd->getReg();
577 if (DPPMovReg.isPhysical()) {
578 LLVM_DEBUG(dbgs() << " failed: dpp move writes physreg\n");
579 return false;
580 }
581 if (execMayBeModifiedBeforeAnyUse(MRI: *MRI, VReg: DPPMovReg, DefMI: MovMI)) {
582 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same"
583 " for all uses\n");
584 return false;
585 }
586
587 auto *DppCtrl = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::dpp_ctrl);
588 assert(DppCtrl && DppCtrl->isImm());
589 unsigned DppCtrlVal = DppCtrl->getImm();
590 if ((MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
591 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp)) {
592 if (!ST->hasFeature(Feature: AMDGPU::FeatureDPALU_DPP)) {
593 LLVM_DEBUG(dbgs() << " failed: 64 bit dpp move is unsupported\n");
594 // Split it.
595 return false;
596 }
597 if (!AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: DppCtrlVal)) {
598 LLVM_DEBUG(dbgs() << " failed: 64 bit dpp move uses unsupported"
599 " control value\n");
600 // Let it split, then control may become legal.
601 return false;
602 }
603 }
604
605 auto *RowMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::row_mask);
606 assert(RowMaskOpnd && RowMaskOpnd->isImm());
607 auto *BankMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bank_mask);
608 assert(BankMaskOpnd && BankMaskOpnd->isImm());
609 const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF &&
610 BankMaskOpnd->getImm() == 0xF;
611
612 auto *BCZOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bound_ctrl);
613 assert(BCZOpnd && BCZOpnd->isImm());
614 bool BoundCtrlZero = BCZOpnd->getImm();
615
616 auto *OldOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::old);
617 auto *SrcOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::src0);
618 assert(OldOpnd && OldOpnd->isReg());
619 assert(SrcOpnd && SrcOpnd->isReg());
620 if (OldOpnd->getReg().isPhysical() || SrcOpnd->getReg().isPhysical()) {
621 LLVM_DEBUG(dbgs() << " failed: dpp move reads physreg\n");
622 return false;
623 }
624
625 auto * const OldOpndValue = getOldOpndValue(OldOpnd&: *OldOpnd);
626 // OldOpndValue is either undef (IMPLICIT_DEF) or immediate or something else
627 // We could use: assert(!OldOpndValue || OldOpndValue->isImm())
628 // but the third option is used to distinguish undef from non-immediate
629 // to reuse IMPLICIT_DEF instruction later
630 assert(!OldOpndValue || OldOpndValue->isImm() || OldOpndValue == OldOpnd);
631
632 bool CombBCZ = false;
633
634 if (MaskAllLanes && BoundCtrlZero) { // [1]
635 CombBCZ = true;
636 } else {
637 if (!OldOpndValue || !OldOpndValue->isImm()) {
638 LLVM_DEBUG(dbgs() << " failed: the DPP mov isn't combinable\n");
639 return false;
640 }
641
642 if (OldOpndValue->getImm() == 0) {
643 if (MaskAllLanes) {
644 assert(!BoundCtrlZero); // by check [1]
645 CombBCZ = true;
646 }
647 } else if (BoundCtrlZero) {
648 assert(!MaskAllLanes); // by check [1]
649 LLVM_DEBUG(dbgs() <<
650 " failed: old!=0 and bctrl:0 and not all lanes isn't combinable\n");
651 return false;
652 }
653 }
654
655 LLVM_DEBUG(dbgs() << " old=";
656 if (!OldOpndValue)
657 dbgs() << "undef";
658 else
659 dbgs() << *OldOpndValue;
660 dbgs() << ", bound_ctrl=" << CombBCZ << '\n');
661
662 SmallVector<MachineInstr*, 4> OrigMIs, DPPMIs;
663 DenseMap<MachineInstr*, SmallVector<unsigned, 4>> RegSeqWithOpNos;
664 auto CombOldVGPR = getRegSubRegPair(O: *OldOpnd);
665 // try to reuse previous old reg if its undefined (IMPLICIT_DEF)
666 if (CombBCZ && OldOpndValue) { // CombOldVGPR should be undef
667 const TargetRegisterClass *RC = MRI->getRegClass(Reg: DPPMovReg);
668 CombOldVGPR = RegSubRegPair(
669 MRI->createVirtualRegister(RegClass: RC));
670 auto UndefInst = BuildMI(BB&: *MovMI.getParent(), I&: MovMI, MIMD: MovMI.getDebugLoc(),
671 MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: CombOldVGPR.Reg);
672 DPPMIs.push_back(Elt: UndefInst.getInstr());
673 }
674
675 OrigMIs.push_back(Elt: &MovMI);
676 bool Rollback = true;
677 SmallVector<MachineOperand *, 16> Uses(
678 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: DPPMovReg)));
679
680 while (!Uses.empty()) {
681 MachineOperand *Use = Uses.pop_back_val();
682 Rollback = true;
683
684 auto &OrigMI = *Use->getParent();
685 LLVM_DEBUG(dbgs() << " try: " << OrigMI);
686
687 auto OrigOp = OrigMI.getOpcode();
688 assert((TII->get(OrigOp).getSize() != 4 || !AMDGPU::isTrue16Inst(OrigOp)) &&
689 "There should not be e32 True16 instructions pre-RA");
690 if (OrigOp == AMDGPU::REG_SEQUENCE) {
691 Register FwdReg = OrigMI.getOperand(i: 0).getReg();
692 unsigned FwdSubReg = 0;
693
694 if (execMayBeModifiedBeforeAnyUse(MRI: *MRI, VReg: FwdReg, DefMI: OrigMI)) {
695 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same"
696 " for all uses\n");
697 break;
698 }
699
700 unsigned OpNo, E = OrigMI.getNumOperands();
701 for (OpNo = 1; OpNo < E; OpNo += 2) {
702 if (OrigMI.getOperand(i: OpNo).getReg() == DPPMovReg) {
703 FwdSubReg = OrigMI.getOperand(i: OpNo + 1).getImm();
704 break;
705 }
706 }
707
708 if (!FwdSubReg)
709 break;
710
711 for (auto &Op : MRI->use_nodbg_operands(Reg: FwdReg)) {
712 if (Op.getSubReg() == FwdSubReg)
713 Uses.push_back(Elt: &Op);
714 }
715 RegSeqWithOpNos[&OrigMI].push_back(Elt: OpNo);
716 continue;
717 }
718
719 bool IsShrinkable = isShrinkable(MI&: OrigMI);
720 if (!(IsShrinkable ||
721 ((TII->isVOP3P(Opcode: OrigOp) || TII->isVOPC(Opcode: OrigOp) ||
722 TII->isVOP3(Opcode: OrigOp)) &&
723 ST->hasVOP3DPP()) ||
724 TII->isVOP1(Opcode: OrigOp) || TII->isVOP2(Opcode: OrigOp))) {
725 LLVM_DEBUG(dbgs() << " failed: not VOP1/2/3/3P/C\n");
726 break;
727 }
728 if (OrigMI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: ST->getRegisterInfo())) {
729 LLVM_DEBUG(dbgs() << " failed: can't combine v_cmpx\n");
730 break;
731 }
732
733 auto *Src0 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src0);
734 auto *Src1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1);
735 if (Use != Src0 && !(Use == Src1 && OrigMI.isCommutable())) { // [1]
736 LLVM_DEBUG(dbgs() << " failed: no suitable operands\n");
737 break;
738 }
739
740 auto *Src2 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src2);
741 assert(Src0 && "Src1 without Src0?");
742 if ((Use == Src0 && ((Src1 && Src1->isIdenticalTo(Other: *Src0)) ||
743 (Src2 && Src2->isIdenticalTo(Other: *Src0)))) ||
744 (Use == Src1 && (Src1->isIdenticalTo(Other: *Src0) ||
745 (Src2 && Src2->isIdenticalTo(Other: *Src1))))) {
746 LLVM_DEBUG(
747 dbgs()
748 << " " << OrigMI
749 << " failed: DPP register is used more than once per instruction\n");
750 break;
751 }
752
753 // We have to be careful to prevent trying to fold into the first source
754 // operand of instructions that apply DPP to the second source operand.
755 // This could be directly, or when folding into an instruction that will
756 // get commuted into one.
757 int FoldedOp =
758 (Use == Src0) ? static_cast<int>(OrigOp) : TII->commuteOpcode(Opc: OrigOp);
759 if (FoldedOp < 0 || TII->isSrc1DPPRevOpcode(ST: *ST, Opcode: FoldedOp)) {
760 LLVM_DEBUG(
761 dbgs() << " failed: Use operand cannot have DPP applied to it\n");
762 break;
763 }
764
765 // Without DPALU DPP there are no 64-bit DPP encodings. The 64-bit move is
766 // rejected above, but a 32-bit move folded into a source of a 64-bit
767 // instruction reaches here, so the operands have to be checked too.
768 if (AMDGPU::isDPALU_DPP(OpDesc: TII->get(Opcode: OrigOp), MII: *TII, ST: *ST)) {
769 if (!ST->hasFeature(Feature: AMDGPU::FeatureDPALU_DPP)) {
770 LLVM_DEBUG(dbgs() << " " << OrigMI
771 << " failed: DPP ALU DPP is not supported\n");
772 break;
773 }
774
775 if (!AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: DppCtrlVal)) {
776 LLVM_DEBUG(dbgs() << " " << OrigMI
777 << " failed: not valid 64-bit DPP control value\n");
778 break;
779 }
780 }
781
782 LLVM_DEBUG(dbgs() << " combining: " << OrigMI);
783 if (Use == Src0) {
784 if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR,
785 OldOpndValue, CombBCZ, IsShrinkable)) {
786 DPPMIs.push_back(Elt: DPPInst);
787 Rollback = false;
788 }
789 } else {
790 assert(Use == Src1 && OrigMI.isCommutable()); // by check [1]
791 auto *BB = OrigMI.getParent();
792 auto *NewMI = BB->getParent()->CloneMachineInstr(Orig: &OrigMI);
793 BB->insert(I: OrigMI, MI: NewMI);
794 if (TII->commuteInstruction(MI&: *NewMI)) {
795 LLVM_DEBUG(dbgs() << " commuted: " << *NewMI);
796 if (auto *DPPInst =
797 createDPPInst(OrigMI&: *NewMI, MovMI, CombOldVGPR, OldOpndValue, CombBCZ,
798 IsShrinkable)) {
799 DPPMIs.push_back(Elt: DPPInst);
800 Rollback = false;
801 }
802 } else
803 LLVM_DEBUG(dbgs() << " failed: cannot be commuted\n");
804 NewMI->eraseFromParent();
805 }
806 if (Rollback)
807 break;
808 OrigMIs.push_back(Elt: &OrigMI);
809 }
810
811 Rollback |= !Uses.empty();
812
813 for (auto *MI : *(Rollback? &DPPMIs : &OrigMIs))
814 MI->eraseFromParent();
815
816 if (!Rollback) {
817 for (auto &S : RegSeqWithOpNos) {
818 if (MRI->use_nodbg_empty(RegNo: S.first->getOperand(i: 0).getReg())) {
819 S.first->eraseFromParent();
820 continue;
821 }
822 while (!S.second.empty())
823 S.first->getOperand(i: S.second.pop_back_val()).setIsUndef();
824 }
825 }
826
827 return !Rollback;
828}
829
830bool GCNDPPCombineLegacy::runOnMachineFunction(MachineFunction &MF) {
831 if (skipFunction(F: MF.getFunction()))
832 return false;
833
834 return GCNDPPCombine().run(MF);
835}
836
837bool GCNDPPCombine::run(MachineFunction &MF) {
838 ST = &MF.getSubtarget<GCNSubtarget>();
839 if (!ST->hasDPP())
840 return false;
841
842 MRI = &MF.getRegInfo();
843 TII = ST->getInstrInfo();
844
845 bool Changed = false;
846 for (auto &MBB : MF) {
847 for (MachineInstr &MI : llvm::make_early_inc_range(Range: llvm::reverse(C&: MBB))) {
848 if (MI.getOpcode() == AMDGPU::V_MOV_B32_dpp && combineDPPMov(MovMI&: MI)) {
849 Changed = true;
850 ++NumDPPMovsCombined;
851 } else if (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
852 MI.getOpcode() == AMDGPU::V_MOV_B64_dpp) {
853 if (ST->hasDPALU_DPP() && combineDPPMov(MovMI&: MI)) {
854 Changed = true;
855 ++NumDPPMovsCombined;
856 } else {
857 auto Split = TII->expandMovDPP64(MI);
858 for (auto *M : {Split.first, Split.second}) {
859 if (M && combineDPPMov(MovMI&: *M))
860 ++NumDPPMovsCombined;
861 }
862 Changed = true;
863 }
864 }
865 }
866 }
867 return Changed;
868}
869
870PreservedAnalyses GCNDPPCombinePass::run(MachineFunction &MF,
871 MachineFunctionAnalysisManager &) {
872 MFPropsModifier _(*this, MF);
873
874 if (MF.getFunction().hasOptNone())
875 return PreservedAnalyses::all();
876
877 bool Changed = GCNDPPCombine().run(MF);
878 if (!Changed)
879 return PreservedAnalyses::all();
880
881 auto PA = getMachineFunctionPassPreservedAnalyses();
882 PA.preserveSet<CFGAnalyses>();
883 return PA;
884}
885