1//=======- GCNDPPCombine.cpp - optimization for DPP instructions ---==========//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// The pass combines V_MOV_B32_dpp instruction with its VALU uses as a DPP src0
9// operand. If any of the use instruction cannot be combined with the mov the
10// whole sequence is reverted.
11//
12// $old = ...
13// $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane,
14// dpp_controls..., $row_mask, $bank_mask, $bound_ctrl
15// $res = VALU $dpp_value [, src1]
16//
17// to
18//
19// $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, [src1,]
20// dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl
21//
22// Combining rules :
23//
24// if $row_mask and $bank_mask are fully enabled (0xF) and
25// $bound_ctrl==DPP_BOUND_ZERO or $old==0
26// -> $combined_old = undef,
27// $combined_bound_ctrl = DPP_BOUND_ZERO
28//
29// if the VALU op is binary and
30// $bound_ctrl==DPP_BOUND_OFF and
31// $old==identity value (immediate) for the VALU op
32// -> $combined_old = src1,
33// $combined_bound_ctrl = DPP_BOUND_OFF
34//
35// Otherwise cancel.
36//
37// The mov_dpp instruction should reside in the same BB as all its uses
38//===----------------------------------------------------------------------===//
39
40#include "GCNDPPCombine.h"
41#include "AMDGPU.h"
42#include "GCNSubtarget.h"
43#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
44#include "llvm/ADT/Statistic.h"
45#include "llvm/CodeGen/MachineFunctionPass.h"
46#include "llvm/CodeGen/MachineInstrBuilder.h"
47#include "llvm/CodeGen/RegisterClassInfo.h"
48
49using namespace llvm;
50
51#define DEBUG_TYPE "gcn-dpp-combine"
52
53STATISTIC(NumDPPMovsCombined, "Number of DPP moves combined.");
54
55namespace {
56
57class GCNDPPCombine {
58 MachineRegisterInfo *MRI;
59 const SIInstrInfo *TII;
60 const GCNSubtarget *ST;
61
62 using RegSubRegPair = TargetInstrInfo::RegSubRegPair;
63
64 MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const;
65
66 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
67 RegSubRegPair CombOldVGPR,
68 MachineOperand *OldOpnd, bool CombBCZ,
69 bool IsShrinkable) const;
70
71 MachineInstr *createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI,
72 RegSubRegPair CombOldVGPR, bool CombBCZ,
73 bool IsShrinkable) const;
74
75 bool hasNoImmOrEqual(MachineInstr &MI, AMDGPU::OpName OpndName, int64_t Value,
76 int64_t Mask = -1) const;
77
78 bool combineDPPMov(MachineInstr &MI) const;
79
80 int getDPPOp(unsigned Op, bool IsShrinkable) const;
81 bool isShrinkable(MachineInstr &MI) const;
82
83public:
84 bool run(MachineFunction &MF);
85};
86
87class GCNDPPCombineLegacy : public MachineFunctionPass {
88public:
89 static char ID;
90
91 GCNDPPCombineLegacy() : MachineFunctionPass(ID) {}
92
93 bool runOnMachineFunction(MachineFunction &MF) override;
94
95 StringRef getPassName() const override { return "GCN DPP Combine"; }
96
97 void getAnalysisUsage(AnalysisUsage &AU) const override {
98 AU.setPreservesCFG();
99 MachineFunctionPass::getAnalysisUsage(AU);
100 }
101
102 MachineFunctionProperties getRequiredProperties() const override {
103 return MachineFunctionProperties().setIsSSA();
104 }
105};
106
107} // end anonymous namespace
108
109INITIALIZE_PASS(GCNDPPCombineLegacy, DEBUG_TYPE, "GCN DPP Combine", false,
110 false)
111
112char GCNDPPCombineLegacy::ID = 0;
113
114char &llvm::GCNDPPCombineLegacyID = GCNDPPCombineLegacy::ID;
115
116FunctionPass *llvm::createGCNDPPCombinePass() {
117 return new GCNDPPCombineLegacy();
118}
119
120bool GCNDPPCombine::isShrinkable(MachineInstr &MI) const {
121 unsigned Op = MI.getOpcode();
122 if (!TII->isVOP3(Opcode: Op)) {
123 return false;
124 }
125 if (!TII->hasVALU32BitEncoding(Opcode: Op)) {
126 LLVM_DEBUG(dbgs() << " Inst hasn't e32 equivalent\n");
127 return false;
128 }
129 // Do not shrink True16 instructions pre-RA to avoid the restriction in
130 // register allocation from only being able to use 128 VGPRs
131 if (AMDGPU::isTrue16Inst(Opc: Op))
132 return false;
133 if (const auto *SDst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst)) {
134 // Give up if there are any uses of the sdst in carry-out or VOPC.
135 // The shrunken form of the instruction would write it to vcc instead of to
136 // a virtual register. If we rewrote the uses the shrinking would be
137 // possible.
138 if (!MRI->use_nodbg_empty(RegNo: SDst->getReg()))
139 return false;
140 }
141 // check if other than abs|neg modifiers are set (opsel for example)
142 const int64_t Mask = ~(SISrcMods::ABS | SISrcMods::NEG);
143 if (!hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::src0_modifiers, Value: 0, Mask) ||
144 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::src1_modifiers, Value: 0, Mask) ||
145 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::clamp, Value: 0) ||
146 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::omod, Value: 0) ||
147 !hasNoImmOrEqual(MI, OpndName: AMDGPU::OpName::byte_sel, Value: 0)) {
148 LLVM_DEBUG(dbgs() << " Inst has non-default modifiers\n");
149 return false;
150 }
151 return true;
152}
153
154int GCNDPPCombine::getDPPOp(unsigned Op, bool IsShrinkable) const {
155 int DPP32 = AMDGPU::getDPPOp32(Opcode: Op);
156 if (IsShrinkable) {
157 assert(DPP32 == -1);
158 int E32 = AMDGPU::getVOPe32(Opcode: Op);
159 DPP32 = (E32 == -1) ? -1 : AMDGPU::getDPPOp32(Opcode: E32);
160 }
161 if (DPP32 != -1 && TII->pseudoToMCOpcode(Opcode: DPP32) != -1)
162 return DPP32;
163 int DPP64 = -1;
164 if (ST->hasVOP3DPP())
165 DPP64 = AMDGPU::getDPPOp64(Opcode: Op);
166 if (DPP64 != -1 && TII->pseudoToMCOpcode(Opcode: DPP64) != -1)
167 return DPP64;
168 return -1;
169}
170
171// tracks the register operand definition and returns:
172// 1. immediate operand used to initialize the register if found
173// 2. nullptr if the register operand is undef
174// 3. the operand itself otherwise
175MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const {
176 auto *Def = getVRegSubRegDef(P: getRegSubRegPair(O: OldOpnd), MRI: *MRI);
177 if (!Def)
178 return nullptr;
179
180 switch(Def->getOpcode()) {
181 default: break;
182 case AMDGPU::IMPLICIT_DEF:
183 return nullptr;
184 case AMDGPU::COPY:
185 case AMDGPU::V_MOV_B32_e32:
186 case AMDGPU::V_MOV_B64_PSEUDO:
187 case AMDGPU::V_MOV_B64_e32:
188 case AMDGPU::V_MOV_B64_e64: {
189 auto &Op1 = Def->getOperand(i: 1);
190 if (Op1.isImm())
191 return &Op1;
192 break;
193 }
194 }
195 return &OldOpnd;
196}
197
198MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
199 MachineInstr &MovMI,
200 RegSubRegPair CombOldVGPR,
201 bool CombBCZ,
202 bool IsShrinkable) const {
203 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
204 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
205 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
206
207 bool HasVOP3DPP = ST->hasVOP3DPP();
208 auto OrigOp = OrigMI.getOpcode();
209 if (ST->useRealTrue16Insts() && AMDGPU::isTrue16Inst(Opc: OrigOp)) {
210 LLVM_DEBUG(
211 dbgs() << " failed: Did not expect any 16-bit uses of dpp values\n");
212 return nullptr;
213 }
214 auto DPPOp = getDPPOp(Op: OrigOp, IsShrinkable);
215 if (DPPOp == -1) {
216 LLVM_DEBUG(dbgs() << " failed: no DPP opcode\n");
217 return nullptr;
218 }
219 int OrigOpE32 = AMDGPU::getVOPe32(Opcode: OrigOp);
220 // Prior checks cover Mask with VOPC condition, but not on purpose
221 auto *RowMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::row_mask);
222 assert(RowMaskOpnd && RowMaskOpnd->isImm());
223 auto *BankMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bank_mask);
224 assert(BankMaskOpnd && BankMaskOpnd->isImm());
225 const bool MaskAllLanes =
226 RowMaskOpnd->getImm() == 0xF && BankMaskOpnd->getImm() == 0xF;
227 (void)MaskAllLanes;
228 assert((MaskAllLanes ||
229 !(TII->isVOPC(DPPOp) || (TII->isVOP3(DPPOp) && OrigOpE32 != -1 &&
230 TII->isVOPC(OrigOpE32)))) &&
231 "VOPC cannot form DPP unless mask is full");
232
233 auto DPPInst = BuildMI(BB&: *OrigMI.getParent(), I&: OrigMI,
234 MIMD: OrigMI.getDebugLoc(), MCID: TII->get(Opcode: DPPOp))
235 .setMIFlags(OrigMI.getFlags());
236
237 bool Fail = false;
238 do {
239 int NumOperands = 0;
240 if (auto *Dst = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::vdst)) {
241 DPPInst.add(MO: *Dst);
242 ++NumOperands;
243 }
244 if (auto *SDst = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::sdst)) {
245 if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::sdst)) {
246 DPPInst.add(MO: *SDst);
247 ++NumOperands;
248 }
249 // If we shrunk a 64bit vop3b to 32bits, just ignore the sdst
250 }
251
252 const int OldIdx = AMDGPU::getNamedOperandIdx(Opcode: DPPOp, Name: AMDGPU::OpName::old);
253 if (OldIdx != -1) {
254 assert(OldIdx == NumOperands);
255 assert(isOfRegClass(
256 CombOldVGPR,
257 *MRI->getRegClass(
258 TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg()),
259 *MRI));
260 auto *Def = getVRegSubRegDef(P: CombOldVGPR, MRI: *MRI);
261 DPPInst.addReg(RegNo: CombOldVGPR.Reg, Flags: getUndefRegState(B: !Def),
262 SubReg: CombOldVGPR.SubReg);
263 ++NumOperands;
264 } else if (TII->isVOPC(Opcode: DPPOp) || (TII->isVOP3(Opcode: DPPOp) && OrigOpE32 != -1 &&
265 TII->isVOPC(Opcode: OrigOpE32))) {
266 // VOPC DPP and VOPC promoted to VOP3 DPP do not have an old operand
267 // because they write to SGPRs not VGPRs
268 } else {
269 // TODO: this discards MAC/FMA instructions for now, let's add it later
270 LLVM_DEBUG(dbgs() << " failed: no old operand in DPP instruction,"
271 " TBD\n");
272 Fail = true;
273 break;
274 }
275
276 auto *Mod0 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src0_modifiers);
277 if (Mod0) {
278 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
279 AMDGPU::OpName::src0_modifiers));
280 assert(HasVOP3DPP ||
281 (0LL == (Mod0->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
282 DPPInst.addImm(Val: Mod0->getImm());
283 ++NumOperands;
284 } else if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
285 DPPInst.addImm(Val: 0);
286 ++NumOperands;
287 }
288 auto *Src0 = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::src0);
289 assert(Src0);
290 [[maybe_unused]] int Src0Idx = NumOperands;
291
292 DPPInst.add(MO: *Src0);
293 DPPInst->getOperand(i: NumOperands).setIsKill(false);
294 ++NumOperands;
295
296 auto *Mod1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1_modifiers);
297 if (Mod1) {
298 assert(NumOperands == AMDGPU::getNamedOperandIdx(DPPOp,
299 AMDGPU::OpName::src1_modifiers));
300 assert(HasVOP3DPP ||
301 (0LL == (Mod1->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
302 DPPInst.addImm(Val: Mod1->getImm());
303 ++NumOperands;
304 } else if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::src1_modifiers)) {
305 DPPInst.addImm(Val: 0);
306 ++NumOperands;
307 }
308 auto *Src1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1);
309 if (Src1) {
310 assert(AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::src1) &&
311 "dpp version of instruction missing src1");
312
313 DPPInst.add(MO: *Src1);
314 ++NumOperands;
315 }
316
317 auto *Mod2 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src2_modifiers);
318 if (Mod2) {
319 assert(NumOperands ==
320 AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::src2_modifiers));
321 assert(HasVOP3DPP ||
322 (0LL == (Mod2->getImm() & ~(SISrcMods::ABS | SISrcMods::NEG))));
323 DPPInst.addImm(Val: Mod2->getImm());
324 ++NumOperands;
325 }
326 auto *Src2 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src2);
327 if (Src2) {
328 if (!AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::src2)) {
329 LLVM_DEBUG(dbgs() << " failed: dpp does not have src2\n");
330 Fail = true;
331 break;
332 }
333 DPPInst.add(MO: *Src2);
334 ++NumOperands;
335 }
336
337 if (HasVOP3DPP) {
338 auto *ClampOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::clamp);
339 if (ClampOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::clamp)) {
340 DPPInst.addImm(Val: ClampOpr->getImm());
341 }
342 auto *VdstInOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::vdst_in);
343 if (VdstInOpr &&
344 AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::vdst_in)) {
345 DPPInst.add(MO: *VdstInOpr);
346 }
347 auto *OmodOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::omod);
348 if (OmodOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::omod)) {
349 DPPInst.addImm(Val: OmodOpr->getImm());
350 }
351 // Validate OP_SEL has to be set to all 0 and OP_SEL_HI has to be set to
352 // all 1.
353 if (TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::op_sel)) {
354 int64_t OpSel = 0;
355 OpSel |= (Mod0 ? (!!(Mod0->getImm() & SISrcMods::OP_SEL_0) << 0) : 0);
356 OpSel |= (Mod1 ? (!!(Mod1->getImm() & SISrcMods::OP_SEL_0) << 1) : 0);
357 OpSel |= (Mod2 ? (!!(Mod2->getImm() & SISrcMods::OP_SEL_0) << 2) : 0);
358 if (Mod0 && TII->isVOP3(MI: OrigMI) && !TII->isVOP3P(MI: OrigMI))
359 OpSel |= !!(Mod0->getImm() & SISrcMods::DST_OP_SEL) << 3;
360
361 if (OpSel != 0) {
362 LLVM_DEBUG(dbgs() << " failed: op_sel must be zero\n");
363 Fail = true;
364 break;
365 }
366 if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::op_sel))
367 DPPInst.addImm(Val: OpSel);
368 }
369 if (TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::op_sel_hi)) {
370 int64_t OpSelHi = 0;
371 OpSelHi |= (Mod0 ? (!!(Mod0->getImm() & SISrcMods::OP_SEL_1) << 0) : 0);
372 OpSelHi |= (Mod1 ? (!!(Mod1->getImm() & SISrcMods::OP_SEL_1) << 1) : 0);
373 OpSelHi |= (Mod2 ? (!!(Mod2->getImm() & SISrcMods::OP_SEL_1) << 2) : 0);
374
375 // Only vop3p has op_sel_hi, and all vop3p have 3 operands, so check
376 // the bitmask for 3 op_sel_hi bits set
377 assert(Src2 && "Expected vop3p with 3 operands");
378 if (OpSelHi != 7) {
379 LLVM_DEBUG(dbgs() << " failed: op_sel_hi must be all set to one\n");
380 Fail = true;
381 break;
382 }
383 if (AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::op_sel_hi))
384 DPPInst.addImm(Val: OpSelHi);
385 }
386 auto *NegOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::neg_lo);
387 if (NegOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::neg_lo)) {
388 DPPInst.addImm(Val: NegOpr->getImm());
389 }
390 auto *NegHiOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::neg_hi);
391 if (NegHiOpr && AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::neg_hi)) {
392 DPPInst.addImm(Val: NegHiOpr->getImm());
393 }
394 auto *ByteSelOpr = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::byte_sel);
395 if (ByteSelOpr &&
396 AMDGPU::hasNamedOperand(Opcode: DPPOp, NamedIdx: AMDGPU::OpName::byte_sel)) {
397 DPPInst.addImm(Val: ByteSelOpr->getImm());
398 }
399 if (MachineOperand *BitOp3 =
400 TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::bitop3)) {
401 assert(AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::bitop3));
402 DPPInst.add(MO: *BitOp3);
403 }
404 }
405 DPPInst.add(MO: *TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::dpp_ctrl));
406 DPPInst.add(MO: *TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::row_mask));
407 DPPInst.add(MO: *TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bank_mask));
408 DPPInst.addImm(Val: CombBCZ ? 1 : 0);
409
410 constexpr AMDGPU::OpName Srcs[] = {
411 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
412
413 // FIXME: isOperandLegal expects to operate on an completely built
414 // instruction. We should have better legality APIs to check if the
415 // candidate operands will be legal without building the instruction first.
416 for (auto [I, OpName] : enumerate(First: Srcs)) {
417 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: DPPOp, Name: OpName);
418 if (OpIdx == -1)
419 break;
420
421 if (!TII->isOperandLegal(MI: *DPPInst, OpIdx)) {
422 LLVM_DEBUG(dbgs() << " failed: src" << I << " operand is illegal\n");
423 Fail = true;
424 break;
425 }
426 }
427 } while (false);
428
429 if (Fail) {
430 DPPInst.getInstr()->eraseFromParent();
431 return nullptr;
432 }
433 LLVM_DEBUG(dbgs() << " combined: " << *DPPInst.getInstr());
434 return DPPInst.getInstr();
435}
436
437static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd) {
438 assert(OldOpnd->isImm());
439 switch (OrigMIOp) {
440 default: break;
441 case AMDGPU::V_ADD_U32_e32:
442 case AMDGPU::V_ADD_U32_e64:
443 case AMDGPU::V_ADD_CO_U32_e32:
444 case AMDGPU::V_ADD_CO_U32_e64:
445 case AMDGPU::V_OR_B32_e32:
446 case AMDGPU::V_OR_B32_e64:
447 case AMDGPU::V_SUBREV_U32_e32:
448 case AMDGPU::V_SUBREV_U32_e64:
449 case AMDGPU::V_SUBREV_CO_U32_e32:
450 case AMDGPU::V_SUBREV_CO_U32_e64:
451 case AMDGPU::V_MAX_U32_e32:
452 case AMDGPU::V_MAX_U32_e64:
453 case AMDGPU::V_XOR_B32_e32:
454 case AMDGPU::V_XOR_B32_e64:
455 if (OldOpnd->getImm() == 0)
456 return true;
457 break;
458 case AMDGPU::V_AND_B32_e32:
459 case AMDGPU::V_AND_B32_e64:
460 case AMDGPU::V_MIN_U32_e32:
461 case AMDGPU::V_MIN_U32_e64:
462 if (static_cast<uint32_t>(OldOpnd->getImm()) ==
463 std::numeric_limits<uint32_t>::max())
464 return true;
465 break;
466 case AMDGPU::V_MIN_I32_e32:
467 case AMDGPU::V_MIN_I32_e64:
468 if (static_cast<int32_t>(OldOpnd->getImm()) ==
469 std::numeric_limits<int32_t>::max())
470 return true;
471 break;
472 case AMDGPU::V_MAX_I32_e32:
473 case AMDGPU::V_MAX_I32_e64:
474 if (static_cast<int32_t>(OldOpnd->getImm()) ==
475 std::numeric_limits<int32_t>::min())
476 return true;
477 break;
478 case AMDGPU::V_MUL_I32_I24_e32:
479 case AMDGPU::V_MUL_I32_I24_e64:
480 case AMDGPU::V_MUL_U32_U24_e32:
481 case AMDGPU::V_MUL_U32_U24_e64:
482 if (OldOpnd->getImm() == 1)
483 return true;
484 break;
485 case AMDGPU::V_MIN_F32_e32:
486 case AMDGPU::V_MIN_F32_e64:
487 if (static_cast<uint32_t>(OldOpnd->getImm()) == /*+inf=*/0x7F800000)
488 return true;
489 break;
490 case AMDGPU::V_MAX_F32_e32:
491 case AMDGPU::V_MAX_F32_e64:
492 if (static_cast<uint32_t>(OldOpnd->getImm()) == /*-inf=*/0xFF800000)
493 return true;
494 break;
495 case AMDGPU::V_MIN_F64_e64:
496 case AMDGPU::V_MIN_NUM_F64_e64:
497 if (static_cast<uint64_t>(OldOpnd->getImm()) == /*+inf=*/0x7FF0000000000000)
498 return true;
499 break;
500 case AMDGPU::V_MAX_F64_e64:
501 case AMDGPU::V_MAX_NUM_F64_e64:
502 if (static_cast<uint64_t>(OldOpnd->getImm()) == /*-inf=*/0xFFF0000000000000)
503 return true;
504 break;
505 case AMDGPU::V_MIN_F16_e32:
506 case AMDGPU::V_MIN_F16_e64:
507 case AMDGPU::V_MIN_F16_t16_e32:
508 case AMDGPU::V_MIN_F16_t16_e64:
509 case AMDGPU::V_MIN_F16_fake16_e32:
510 case AMDGPU::V_MIN_F16_fake16_e64:
511 if (static_cast<uint16_t>(OldOpnd->getImm()) == /*+inf=*/0x7C00)
512 return true;
513 break;
514 case AMDGPU::V_MAX_F16_e32:
515 case AMDGPU::V_MAX_F16_e64:
516 case AMDGPU::V_MAX_F16_t16_e32:
517 case AMDGPU::V_MAX_F16_t16_e64:
518 case AMDGPU::V_MAX_F16_fake16_e32:
519 case AMDGPU::V_MAX_F16_fake16_e64:
520 if (static_cast<uint16_t>(OldOpnd->getImm()) == /*-inf=*/0xFC00)
521 return true;
522 break;
523 }
524 return false;
525}
526
527MachineInstr *GCNDPPCombine::createDPPInst(
528 MachineInstr &OrigMI, MachineInstr &MovMI, RegSubRegPair CombOldVGPR,
529 MachineOperand *OldOpndValue, bool CombBCZ, bool IsShrinkable) const {
530 assert(CombOldVGPR.Reg);
531 if (!CombBCZ && OldOpndValue && OldOpndValue->isImm()) {
532 auto *Src1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1);
533 if (!Src1 || !Src1->isReg()) {
534 LLVM_DEBUG(dbgs() << " failed: no src1 or it isn't a register\n");
535 return nullptr;
536 }
537 if (!isIdentityValue(OrigMIOp: OrigMI.getOpcode(), OldOpnd: OldOpndValue)) {
538 LLVM_DEBUG(dbgs() << " failed: old immediate isn't an identity\n");
539 return nullptr;
540 }
541 CombOldVGPR = getRegSubRegPair(O: *Src1);
542 auto *MovDst = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::vdst);
543 const TargetRegisterClass *RC = MRI->getRegClass(Reg: MovDst->getReg());
544 if (!isOfRegClass(P: CombOldVGPR, TRC: *RC, MRI&: *MRI)) {
545 LLVM_DEBUG(dbgs() << " failed: src1 has wrong register class\n");
546 return nullptr;
547 }
548 }
549 return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ, IsShrinkable);
550}
551
552// returns true if MI doesn't have OpndName immediate operand or the
553// operand has Value
554bool GCNDPPCombine::hasNoImmOrEqual(MachineInstr &MI, AMDGPU::OpName OpndName,
555 int64_t Value, int64_t Mask) const {
556 auto *Imm = TII->getNamedOperand(MI, OperandName: OpndName);
557 if (!Imm)
558 return true;
559
560 assert(Imm->isImm());
561 return (Imm->getImm() & Mask) == Value;
562}
563
564bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
565 assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp ||
566 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp ||
567 MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
568 LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);
569
570 auto *DstOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::vdst);
571 assert(DstOpnd && DstOpnd->isReg());
572 auto DPPMovReg = DstOpnd->getReg();
573 if (DPPMovReg.isPhysical()) {
574 LLVM_DEBUG(dbgs() << " failed: dpp move writes physreg\n");
575 return false;
576 }
577 if (execMayBeModifiedBeforeAnyUse(MRI: *MRI, VReg: DPPMovReg, DefMI: MovMI)) {
578 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same"
579 " for all uses\n");
580 return false;
581 }
582
583 auto *DppCtrl = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::dpp_ctrl);
584 assert(DppCtrl && DppCtrl->isImm());
585 unsigned DppCtrlVal = DppCtrl->getImm();
586 if ((MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
587 MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp)) {
588 if (!ST->hasFeature(Feature: AMDGPU::FeatureDPALU_DPP)) {
589 LLVM_DEBUG(dbgs() << " failed: 64 bit dpp move is unsupported\n");
590 // Split it.
591 return false;
592 }
593 if (!AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: DppCtrlVal)) {
594 LLVM_DEBUG(dbgs() << " failed: 64 bit dpp move uses unsupported"
595 " control value\n");
596 // Let it split, then control may become legal.
597 return false;
598 }
599 }
600
601 auto *RowMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::row_mask);
602 assert(RowMaskOpnd && RowMaskOpnd->isImm());
603 auto *BankMaskOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bank_mask);
604 assert(BankMaskOpnd && BankMaskOpnd->isImm());
605 const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF &&
606 BankMaskOpnd->getImm() == 0xF;
607
608 auto *BCZOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::bound_ctrl);
609 assert(BCZOpnd && BCZOpnd->isImm());
610 bool BoundCtrlZero = BCZOpnd->getImm();
611
612 auto *OldOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::old);
613 auto *SrcOpnd = TII->getNamedOperand(MI&: MovMI, OperandName: AMDGPU::OpName::src0);
614 assert(OldOpnd && OldOpnd->isReg());
615 assert(SrcOpnd && SrcOpnd->isReg());
616 if (OldOpnd->getReg().isPhysical() || SrcOpnd->getReg().isPhysical()) {
617 LLVM_DEBUG(dbgs() << " failed: dpp move reads physreg\n");
618 return false;
619 }
620
621 auto * const OldOpndValue = getOldOpndValue(OldOpnd&: *OldOpnd);
622 // OldOpndValue is either undef (IMPLICIT_DEF) or immediate or something else
623 // We could use: assert(!OldOpndValue || OldOpndValue->isImm())
624 // but the third option is used to distinguish undef from non-immediate
625 // to reuse IMPLICIT_DEF instruction later
626 assert(!OldOpndValue || OldOpndValue->isImm() || OldOpndValue == OldOpnd);
627
628 bool CombBCZ = false;
629
630 if (MaskAllLanes && BoundCtrlZero) { // [1]
631 CombBCZ = true;
632 } else {
633 if (!OldOpndValue || !OldOpndValue->isImm()) {
634 LLVM_DEBUG(dbgs() << " failed: the DPP mov isn't combinable\n");
635 return false;
636 }
637
638 if (OldOpndValue->getImm() == 0) {
639 if (MaskAllLanes) {
640 assert(!BoundCtrlZero); // by check [1]
641 CombBCZ = true;
642 }
643 } else if (BoundCtrlZero) {
644 assert(!MaskAllLanes); // by check [1]
645 LLVM_DEBUG(dbgs() <<
646 " failed: old!=0 and bctrl:0 and not all lanes isn't combinable\n");
647 return false;
648 }
649 }
650
651 LLVM_DEBUG(dbgs() << " old=";
652 if (!OldOpndValue)
653 dbgs() << "undef";
654 else
655 dbgs() << *OldOpndValue;
656 dbgs() << ", bound_ctrl=" << CombBCZ << '\n');
657
658 SmallVector<MachineInstr*, 4> OrigMIs, DPPMIs;
659 DenseMap<MachineInstr*, SmallVector<unsigned, 4>> RegSeqWithOpNos;
660 auto CombOldVGPR = getRegSubRegPair(O: *OldOpnd);
661 // try to reuse previous old reg if its undefined (IMPLICIT_DEF)
662 if (CombBCZ && OldOpndValue) { // CombOldVGPR should be undef
663 const TargetRegisterClass *RC = MRI->getRegClass(Reg: DPPMovReg);
664 CombOldVGPR = RegSubRegPair(
665 MRI->createVirtualRegister(RegClass: RC));
666 auto UndefInst = BuildMI(BB&: *MovMI.getParent(), I&: MovMI, MIMD: MovMI.getDebugLoc(),
667 MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: CombOldVGPR.Reg);
668 DPPMIs.push_back(Elt: UndefInst.getInstr());
669 }
670
671 OrigMIs.push_back(Elt: &MovMI);
672 bool Rollback = true;
673 SmallVector<MachineOperand *, 16> Uses(
674 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: DPPMovReg)));
675
676 while (!Uses.empty()) {
677 MachineOperand *Use = Uses.pop_back_val();
678 Rollback = true;
679
680 auto &OrigMI = *Use->getParent();
681 LLVM_DEBUG(dbgs() << " try: " << OrigMI);
682
683 auto OrigOp = OrigMI.getOpcode();
684 assert((TII->get(OrigOp).getSize() != 4 || !AMDGPU::isTrue16Inst(OrigOp)) &&
685 "There should not be e32 True16 instructions pre-RA");
686 if (OrigOp == AMDGPU::REG_SEQUENCE) {
687 Register FwdReg = OrigMI.getOperand(i: 0).getReg();
688 unsigned FwdSubReg = 0;
689
690 if (execMayBeModifiedBeforeAnyUse(MRI: *MRI, VReg: FwdReg, DefMI: OrigMI)) {
691 LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same"
692 " for all uses\n");
693 break;
694 }
695
696 unsigned OpNo, E = OrigMI.getNumOperands();
697 for (OpNo = 1; OpNo < E; OpNo += 2) {
698 if (OrigMI.getOperand(i: OpNo).getReg() == DPPMovReg) {
699 FwdSubReg = OrigMI.getOperand(i: OpNo + 1).getImm();
700 break;
701 }
702 }
703
704 if (!FwdSubReg)
705 break;
706
707 for (auto &Op : MRI->use_nodbg_operands(Reg: FwdReg)) {
708 if (Op.getSubReg() == FwdSubReg)
709 Uses.push_back(Elt: &Op);
710 }
711 RegSeqWithOpNos[&OrigMI].push_back(Elt: OpNo);
712 continue;
713 }
714
715 bool IsShrinkable = isShrinkable(MI&: OrigMI);
716 if (!(IsShrinkable ||
717 ((TII->isVOP3P(Opcode: OrigOp) || TII->isVOPC(Opcode: OrigOp) ||
718 TII->isVOP3(Opcode: OrigOp)) &&
719 ST->hasVOP3DPP()) ||
720 TII->isVOP1(Opcode: OrigOp) || TII->isVOP2(Opcode: OrigOp))) {
721 LLVM_DEBUG(dbgs() << " failed: not VOP1/2/3/3P/C\n");
722 break;
723 }
724 if (OrigMI.modifiesRegister(Reg: AMDGPU::EXEC, TRI: ST->getRegisterInfo())) {
725 LLVM_DEBUG(dbgs() << " failed: can't combine v_cmpx\n");
726 break;
727 }
728
729 auto *Src0 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src0);
730 auto *Src1 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src1);
731 if (Use != Src0 && !(Use == Src1 && OrigMI.isCommutable())) { // [1]
732 LLVM_DEBUG(dbgs() << " failed: no suitable operands\n");
733 break;
734 }
735
736 auto *Src2 = TII->getNamedOperand(MI&: OrigMI, OperandName: AMDGPU::OpName::src2);
737 assert(Src0 && "Src1 without Src0?");
738 if ((Use == Src0 && ((Src1 && Src1->isIdenticalTo(Other: *Src0)) ||
739 (Src2 && Src2->isIdenticalTo(Other: *Src0)))) ||
740 (Use == Src1 && (Src1->isIdenticalTo(Other: *Src0) ||
741 (Src2 && Src2->isIdenticalTo(Other: *Src1))))) {
742 LLVM_DEBUG(
743 dbgs()
744 << " " << OrigMI
745 << " failed: DPP register is used more than once per instruction\n");
746 break;
747 }
748
749 if (!ST->hasFeature(Feature: AMDGPU::FeatureDPALU_DPP) &&
750 AMDGPU::isDPALU_DPP32BitOpc(Opc: OrigOp)) {
751 LLVM_DEBUG(dbgs() << " " << OrigMI
752 << " failed: DPP ALU DPP is not supported\n");
753 break;
754 }
755
756 if (!AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: DppCtrlVal) &&
757 AMDGPU::isDPALU_DPP(OpDesc: TII->get(Opcode: OrigOp), MII: *TII, ST: *ST)) {
758 LLVM_DEBUG(dbgs() << " " << OrigMI
759 << " failed: not valid 64-bit DPP control value\n");
760 break;
761 }
762
763 LLVM_DEBUG(dbgs() << " combining: " << OrigMI);
764 if (Use == Src0) {
765 if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR,
766 OldOpndValue, CombBCZ, IsShrinkable)) {
767 DPPMIs.push_back(Elt: DPPInst);
768 Rollback = false;
769 }
770 } else {
771 assert(Use == Src1 && OrigMI.isCommutable()); // by check [1]
772 auto *BB = OrigMI.getParent();
773 auto *NewMI = BB->getParent()->CloneMachineInstr(Orig: &OrigMI);
774 BB->insert(I: OrigMI, MI: NewMI);
775 if (TII->commuteInstruction(MI&: *NewMI)) {
776 LLVM_DEBUG(dbgs() << " commuted: " << *NewMI);
777 if (auto *DPPInst =
778 createDPPInst(OrigMI&: *NewMI, MovMI, CombOldVGPR, OldOpndValue, CombBCZ,
779 IsShrinkable)) {
780 DPPMIs.push_back(Elt: DPPInst);
781 Rollback = false;
782 }
783 } else
784 LLVM_DEBUG(dbgs() << " failed: cannot be commuted\n");
785 NewMI->eraseFromParent();
786 }
787 if (Rollback)
788 break;
789 OrigMIs.push_back(Elt: &OrigMI);
790 }
791
792 Rollback |= !Uses.empty();
793
794 for (auto *MI : *(Rollback? &DPPMIs : &OrigMIs))
795 MI->eraseFromParent();
796
797 if (!Rollback) {
798 for (auto &S : RegSeqWithOpNos) {
799 if (MRI->use_nodbg_empty(RegNo: S.first->getOperand(i: 0).getReg())) {
800 S.first->eraseFromParent();
801 continue;
802 }
803 while (!S.second.empty())
804 S.first->getOperand(i: S.second.pop_back_val()).setIsUndef();
805 }
806 }
807
808 return !Rollback;
809}
810
811bool GCNDPPCombineLegacy::runOnMachineFunction(MachineFunction &MF) {
812 if (skipFunction(F: MF.getFunction()))
813 return false;
814
815 return GCNDPPCombine().run(MF);
816}
817
818bool GCNDPPCombine::run(MachineFunction &MF) {
819 ST = &MF.getSubtarget<GCNSubtarget>();
820 if (!ST->hasDPP())
821 return false;
822
823 MRI = &MF.getRegInfo();
824 TII = ST->getInstrInfo();
825
826 bool Changed = false;
827 for (auto &MBB : MF) {
828 for (MachineInstr &MI : llvm::make_early_inc_range(Range: llvm::reverse(C&: MBB))) {
829 if (MI.getOpcode() == AMDGPU::V_MOV_B32_dpp && combineDPPMov(MovMI&: MI)) {
830 Changed = true;
831 ++NumDPPMovsCombined;
832 } else if (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
833 MI.getOpcode() == AMDGPU::V_MOV_B64_dpp) {
834 if (ST->hasDPALU_DPP() && combineDPPMov(MovMI&: MI)) {
835 Changed = true;
836 ++NumDPPMovsCombined;
837 } else {
838 auto Split = TII->expandMovDPP64(MI);
839 for (auto *M : {Split.first, Split.second}) {
840 if (M && combineDPPMov(MovMI&: *M))
841 ++NumDPPMovsCombined;
842 }
843 Changed = true;
844 }
845 }
846 }
847 }
848 return Changed;
849}
850
851PreservedAnalyses GCNDPPCombinePass::run(MachineFunction &MF,
852 MachineFunctionAnalysisManager &) {
853 MFPropsModifier _(*this, MF);
854
855 if (MF.getFunction().hasOptNone())
856 return PreservedAnalyses::all();
857
858 bool Changed = GCNDPPCombine().run(MF);
859 if (!Changed)
860 return PreservedAnalyses::all();
861
862 auto PA = getMachineFunctionPassPreservedAnalyses();
863 PA.preserveSet<CFGAnalyses>();
864 return PA;
865}
866