1//===- GCNVOPDUtils.cpp - GCN VOPD Utils ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file contains the AMDGPU DAG scheduling
10/// mutation to pair VOPD instructions back to back. It also contains
11// subroutines useful in the creation of VOPD instructions
12//
13//===----------------------------------------------------------------------===//
14
15#include "GCNVOPDUtils.h"
16#include "AMDGPUSubtarget.h"
17#include "GCNSubtarget.h"
18#include "SIInstrInfo.h"
19#include "Utils/AMDGPUBaseInfo.h"
20#include "llvm/ADT/SetVector.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/CodeGen/MachineBasicBlock.h"
23#include "llvm/CodeGen/MachineFunction.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineOperand.h"
26#include "llvm/CodeGen/MachineRegisterInfo.h"
27#include "llvm/CodeGen/MacroFusion.h"
28#include "llvm/CodeGen/ScheduleDAG.h"
29#include "llvm/CodeGen/ScheduleDAGMutation.h"
30#include "llvm/CodeGen/TargetInstrInfo.h"
31#include "llvm/MC/MCInst.h"
32#include "llvm/TargetParser/AMDGPUTargetParser.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "gcn-vopd-utils"
37
38// Return the register class of the VOPDOpc operand named
39// src/vsrc<SrcIdx><CompIdx>, which is the slot src<SrcIdx> of MI<CompIdx> maps
40// to.
41static const TargetRegisterClass *getVOPDSrcRegClass(const SIInstrInfo &TII,
42 int VOPDOpc,
43 unsigned CompIdx,
44 unsigned SrcIdx) {
45 using namespace AMDGPU;
46 int OpIdx = -1;
47 const bool IsX = CompIdx == VOPD::X;
48 switch (SrcIdx) {
49 case 0:
50 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::src0X : OpName::src0Y);
51 break;
52 case 1:
53 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::vsrc1X : OpName::vsrc1Y);
54 break;
55 case 2:
56 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::vsrc2X : OpName::vsrc2Y);
57 if (OpIdx == -1)
58 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::src2X : OpName::src2Y);
59 break;
60 default:
61 llvm_unreachable("unexpected VOPD source index");
62 }
63
64 assert(OpIdx != -1);
65 return TII.getRegClass(MCID: TII.get(Opcode: VOPDOpc), OpNum: OpIdx);
66}
67
68// Check if physical register from src<SrcIdx> operand of MI<CompIdx> matches
69// register class constraints in corresponding VOPDOpc operand with name
70// src/vsrc<SrcIdx><CompIdx>.
71static bool isValidVOPDSrc(const SIInstrInfo &TII, int VOPDOpc,
72 unsigned CompIdx, unsigned SrcIdx,
73 Register PhysSrcReg) {
74 return getVOPDSrcRegClass(TII, VOPDOpc, CompIdx, SrcIdx)
75 ->contains(Reg: PhysSrcReg);
76}
77
78static const MachineOperand &getNamedOp(const MachineInstr &MI,
79 AMDGPU::OpName Name) {
80 return MI.getOperand(i: getNamedOperandIdx(Opcode: MI.getOpcode(), Name));
81}
82
83// Check if MI is a VOP3P instruction with operands that satisfy the constraints
84// for mapping it to a VOP2/VOPD opcode: no modifiers, no clamp, src1 and src2
85// are registers (src0 can be register or literal), and src2 is same as dst.
86// An immediate in src1 is accepted if src0 is a register and the operands can
87// be commuted. \p Commute is set if that is needed.
88static bool canMapVOP3PToVOPD(const SIInstrInfo &TII, const MachineInstr &MI,
89 bool &Commute) {
90 Commute = false;
91 unsigned Opc = MI.getOpcode();
92 if (Opc != AMDGPU::V_DOT2_F32_F16 && Opc != AMDGPU::V_DOT2_F32_BF16)
93 return false;
94 // src0 can be register or literal
95 if (getNamedOp(MI, Name: AMDGPU::OpName::src0_modifiers).getImm() !=
96 SISrcMods::OP_SEL_1)
97 return false;
98 if (getNamedOp(MI, Name: AMDGPU::OpName::src1_modifiers).getImm() !=
99 SISrcMods::OP_SEL_1)
100 return false;
101 int Src0Idx = getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
102 int Src1Idx = getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
103 const MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
104 const MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
105 // VOPD needs a register in src1. If src1 is an immediate, the dot2 can still
106 // be used once src0 and src1 are swapped.
107 if (!Src1.isReg()) {
108 if (!Src0.isReg() || !Src1.isImm() ||
109 !TII.isLegalToSwap(MI, fromIdx: Src0Idx, toIdx: Src1Idx))
110 return false;
111 Commute = true;
112 }
113 if (getNamedOp(MI, Name: AMDGPU::OpName::src2_modifiers).getImm() !=
114 SISrcMods::OP_SEL_1)
115 return false;
116 if (!getNamedOp(MI, Name: AMDGPU::OpName::src2).isReg())
117 return false;
118 if (getNamedOp(MI, Name: AMDGPU::OpName::clamp).getImm() != 0)
119 return false;
120 return getNamedOp(MI, Name: AMDGPU::OpName::vdst).getReg() ==
121 getNamedOp(MI, Name: AMDGPU::OpName::src2).getReg();
122}
123
124// In a VOPD3 whose OPX is a 64-bit operation, an OPY VGPR source operand reads
125// back the wrong value if it is the last VGPR the wave owns.
126// A wave always owns a whole number of VGPR allocation granules, so only a
127// register just below a granule boundary can be the last one. The wave also
128// owns at least as many VGPRs as this function uses, so a source which has a
129// register above it in use here cannot be the last one either.
130// The number of VGPRs the wave is actually given is not available until the
131// assembler has seen the whole module, but that can only come out above this
132// function's own usage, so this is conservatively correct.
133static bool isVOPD3F64OPYSrcHazard(const SIInstrInfo &TII,
134 const MachineInstr &MIX,
135 const MachineInstr &MIY) {
136 const MachineFunction &MF = *MIX.getMF();
137 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
138 if (!ST.hasVOPD3F64OPYSrcHazard())
139 return false;
140
141 // Every 64-bit VOPD3 OPX opcode has a 64-bit vdst, and no 32-bit one has.
142 int VDstIdx =
143 AMDGPU::getNamedOperandIdx(Opcode: MIX.getOpcode(), Name: AMDGPU::OpName::vdst);
144 assert(VDstIdx != -1 && "VOPD3 OPX component must have a vdst");
145 if (TII.getOpSize(MI: MIX, OpNo: VDstIdx) != 8)
146 return false;
147
148 unsigned Granule =
149 AMDGPU::getVGPRAllocGranule(AK: ST.getTargetID().getGPUKind(), IsWave32: ST.isWave32());
150 const SIRegisterInfo *TRI = ST.getRegisterInfo();
151 unsigned NumVGPRs = TRI->getNumUsedPhysRegs(
152 MRI: MF.getRegInfo(), RC: AMDGPU::VGPR_32RegClass, /*IncludeCalls=*/false);
153 for (AMDGPU::OpName Name :
154 {AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2}) {
155 const MachineOperand *Src = TII.getNamedOperand(MI: MIY, OperandName: Name);
156 // Every OPY source which can be a VGPR is 32 bits wide.
157 if (!Src || !Src->isReg() ||
158 !AMDGPU::VGPR_32RegClass.contains(Reg: Src->getReg()))
159 continue;
160 unsigned Idx = TRI->getHWRegIndex(Reg: Src->getReg());
161 if ((Idx + 1) % Granule == 0 && Idx + 1 >= NumVGPRs)
162 return true;
163 }
164 return false;
165}
166
167static bool canMaterializeVOPDLiterals(const MachineFunction &MF) {
168 // A free register cannot be found without liveness. A move also makes the
169 // code longer, so a function which asked for small code keeps its literals.
170 return MF.getProperties().hasTracksLiveness() &&
171 !MF.getFunction().hasOptSize();
172}
173
174static bool checkVOPDRegConstraints(
175 const SIInstrInfo &TII, const MachineInstr &MIX, const MachineInstr &MIY,
176 bool IsVOPD3, bool AllowSameVGPR,
177 SmallVectorImpl<VOPDLiteralFixup> &LiteralFixups, bool (&CommuteXY)[2]) {
178 namespace VOPD = AMDGPU::VOPD;
179
180 const MachineFunction *MF = MIX.getMF();
181 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
182
183 if (IsVOPD3 && !ST.hasVOPD3())
184 return false;
185 if (IsVOPD3 && isVOPD3F64OPYSrcHazard(TII, MIX, MIY))
186 return false;
187 // Components which only map once src0 and src1 are commuted. The checks
188 // below look at their operands as they will be after the commute.
189 bool Commute[2] = {false, false};
190 if (!IsVOPD3 &&
191 ((TII.isVOP3(MI: MIX) && !canMapVOP3PToVOPD(TII, MI: MIX, Commute&: Commute[VOPD::X])) ||
192 (TII.isVOP3(MI: MIY) && !canMapVOP3PToVOPD(TII, MI: MIY, Commute&: Commute[VOPD::Y]))))
193 return false;
194 if (TII.isDPP(MI: MIX) || TII.isDPP(MI: MIY))
195 return false;
196
197 // Collected here and handed over only on success, so a failed check cannot
198 // leave anything behind.
199 SmallVector<VOPDLiteralFixup, 2> Fixups;
200
201 const SIRegisterInfo *TRI = ST.getRegisterInfo();
202 const MachineRegisterInfo &MRI = MF->getRegInfo();
203 // Literals also count against scalar bus limit
204 SmallVector<const MachineOperand *> UniqueLiterals;
205 auto AddLiteral = [&](const MachineOperand &Op) {
206 for (auto &Literal : UniqueLiterals) {
207 if (Literal->isIdenticalTo(Other: Op))
208 return;
209 }
210 UniqueLiterals.push_back(Elt: &Op);
211 };
212 // Immediates which the caller will move into a scalar register. Identical
213 // values share one register, so they count like one scalar operand each.
214 SmallSet<int32_t, 2> MaterializedLiterals;
215 SmallSet<Register, 4> UniqueScalarRegs;
216
217 unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST);
218 unsigned XOpc = AMDGPU::getVOPDOpcode(Opc: MIX.getOpcode(), VOPD3: IsVOPD3);
219 unsigned YOpc = AMDGPU::getVOPDOpcode(Opc: MIY.getOpcode(), VOPD3: IsVOPD3);
220 int VOPDOpc = AMDGPU::getVOPDFull(OpX: XOpc, OpY: YOpc, EncodingFamily, VOPD3: IsVOPD3);
221 assert(VOPDOpc != -1);
222
223 auto InstInfo = AMDGPU::getVOPDInstInfo(OpX: MIX.getDesc(), OpY: MIY.getDesc());
224
225 for (auto CompIdx : VOPD::COMPONENTS) {
226 const MachineInstr &MI = (CompIdx == VOPD::X) ? MIX : MIY;
227 const AMDGPU::OpName Src0Name =
228 Commute[CompIdx] ? AMDGPU::OpName::src1 : AMDGPU::OpName::src0;
229 const AMDGPU::OpName Src1Name =
230 Commute[CompIdx] ? AMDGPU::OpName::src0 : AMDGPU::OpName::src1;
231
232 const MachineOperand &Src0 = *TII.getNamedOperand(MI, OperandName: Src0Name);
233 if (Src0.isReg()) {
234 if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, SrcIdx: 0, PhysSrcReg: Src0.getReg()))
235 return false;
236 if (TII.regUsesConstantBus(Reg: Src0, MRI))
237 UniqueScalarRegs.insert(V: Src0.getReg());
238 } else if (!TII.isInlineConstant(MO: Src0)) {
239 if (!IsVOPD3) {
240 AddLiteral(Src0);
241 } else {
242 // A VOPD3 component cannot encode a literal, but src0 can read a
243 // scalar register. The pair is therefore still possible if the caller
244 // moves the value into one.
245 if (!canMaterializeVOPDLiterals(MF: *MF) || !Src0.isImm())
246 return false;
247 // Only a 32-bit slot is handled, because the caller produces the value
248 // with a single S_MOV_B32.
249 int OpIdx = getNamedOperandIdx(Opcode: MI.getOpcode(), Name: Src0Name);
250 if (TII.getOpSize(MI, OpNo: OpIdx) != 4)
251 return false;
252 const TargetRegisterClass *SlotRC =
253 TRI->getCommonSubClass(A: getVOPDSrcRegClass(TII, VOPDOpc, CompIdx, SrcIdx: 0),
254 B: &AMDGPU::SGPR_32RegClass);
255 if (!SlotRC)
256 return false;
257 // Only the low bits reach the register, so two operands which name
258 // the same value share one move whichever way they were written.
259 int32_t Imm = static_cast<int32_t>(Src0.getImm());
260 MaterializedLiterals.insert(V: Imm);
261 Fixups.push_back(Elt: {.CompIdx: CompIdx, .OpIdx: static_cast<unsigned>(OpIdx), .Imm: Imm, .SlotRC: SlotRC});
262 }
263 }
264
265 // V_FMAMK_F32 (src1) and V_FMAAK_F32 (src2) have a mandatory literal.
266 // VOPD3 instructions don't set MandatoryLiteralIdx.
267 if (InstInfo[CompIdx].hasMandatoryLiteral()) {
268 auto CompOprIdx = InstInfo[CompIdx].getMandatoryLiteralCompOperandIndex();
269 AddLiteral(MI.getOperand(i: CompOprIdx));
270 }
271
272 // VOPD only. Affects V_CNDMASK_B32_e32.
273 if (MI.getDesc().hasImplicitUseOfPhysReg(Reg: AMDGPU::VCC))
274 UniqueScalarRegs.insert(V: AMDGPU::VCC_LO);
275
276 if (const MachineOperand *Src1 = TII.getNamedOperand(MI, OperandName: Src1Name)) {
277 if (Src1->isReg()) {
278 if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, SrcIdx: 1, PhysSrcReg: Src1->getReg()))
279 return false;
280 assert(TRI->isVectorRegister(MRI, Src1->getReg()));
281 } else if (IsVOPD3) {
282 return false;
283 }
284 }
285
286 if (IsVOPD3) {
287 if (const MachineOperand *Src2 =
288 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)) {
289 if (AMDGPU::hasNamedOperand(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::bitop3)) {
290 // BITOP3 can be converted to DUAL_BITOP2 when src2 is zero.
291 if (!Src2->isImm() || Src2->getImm())
292 return false;
293 } else {
294 if (!Src2->isReg())
295 return false;
296 if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, SrcIdx: 2, PhysSrcReg: Src2->getReg()))
297 return false;
298 if (TII.regUsesConstantBus(Reg: *Src2, MRI)) {
299 assert(MI.getOpcode() == AMDGPU::V_CNDMASK_B32_e64);
300 UniqueScalarRegs.insert(V: Src2->getReg());
301 }
302 }
303 }
304 for (auto OpName : {AMDGPU::OpName::clamp, AMDGPU::OpName::omod,
305 AMDGPU::OpName::op_sel}) {
306 if (TII.hasModifiersSet(MI, OpName))
307 return false;
308 }
309
310 // Neg is allowed, other modifiers are not. NB: even though sext has the
311 // same value as neg, there are no combinable instructions with sext.
312 for (auto OpName :
313 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
314 AMDGPU::OpName::src2_modifiers}) {
315 const MachineOperand *Mods = TII.getNamedOperand(MI, OperandName: OpName);
316 if (Mods && (Mods->getImm() & ~SISrcMods::NEG))
317 return false;
318 }
319 }
320 }
321
322 if (UniqueLiterals.size() > 1)
323 return false;
324 // Keep materialization pair-local and do not increase instruction count or
325 // register pressure by producing two different values for one pair.
326 if (MaterializedLiterals.size() > 1)
327 return false;
328 if ((UniqueLiterals.size() + MaterializedLiterals.size() +
329 UniqueScalarRegs.size()) > 2)
330 return false;
331
332 auto GetVRegIdx = [&](unsigned OpcodeIdx, unsigned OperandIdx) {
333 const MachineInstr &MI = (OpcodeIdx == VOPD::X) ? MIX : MIY;
334 if (Commute[OpcodeIdx]) {
335 int Src0Idx = getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
336 int Src1Idx = getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
337 if ((int)OperandIdx == Src0Idx)
338 OperandIdx = Src1Idx;
339 else if ((int)OperandIdx == Src1Idx)
340 OperandIdx = Src0Idx;
341 }
342 const MachineOperand &Operand = MI.getOperand(i: OperandIdx);
343 if (Operand.isReg() && TRI->isVectorRegister(MRI, Reg: Operand.getReg()))
344 return Operand.getReg();
345 return Register();
346 };
347
348 // On GFX1170+ if both OpX and OpY are V_MOV_B32 then OPY uses SRC2
349 // source-cache.
350 bool SkipSrc = (ST.hasGFX11_7Insts() || ST.hasGFX12Insts()) &&
351 MIX.getOpcode() == AMDGPU::V_MOV_B32_e32 &&
352 MIY.getOpcode() == AMDGPU::V_MOV_B32_e32;
353
354 // Check VGPR bank constraints for operand registers across both instructions.
355 if (InstInfo.hasInvalidOperand(GetRegIdx: GetVRegIdx, MRI: *TRI, SkipSrc, AllowSameVGPR,
356 VOPD3: IsVOPD3, HasGFX11InterlockHazard: ST.hasVOPDInterlockHazard()))
357 return false;
358
359 LLVM_DEBUG(dbgs() << "VOPD Reg Constraints Passed\n\tX: " << MIX
360 << "\n\tY: " << MIY << "\n");
361 LiteralFixups.assign(RHS: Fixups);
362 CommuteXY[VOPD::X] = Commute[VOPD::X];
363 CommuteXY[VOPD::Y] = Commute[VOPD::Y];
364 return true;
365}
366
367/// Core pair-eligibility check for a single VOPD encoding variant (VOPD or
368/// VOPD3). Returns the X/Y assignment on success, or std::nullopt otherwise.
369static std::optional<VOPDMatchInfo>
370tryMatchVOPDPairVariant(const SIInstrInfo &TII, unsigned EncodingFamily,
371 MachineInstr &FirstMI, MachineInstr &SecondMI,
372 bool IsVOPD3) {
373 unsigned Opc = FirstMI.getOpcode();
374 unsigned Opc2 = SecondMI.getOpcode();
375 AMDGPU::CanBeVOPD FirstCanBeVOPD =
376 AMDGPU::getCanBeVOPD(Opc, EncodingFamily, VOPD3: IsVOPD3);
377 AMDGPU::CanBeVOPD SecondCanBeVOPD =
378 AMDGPU::getCanBeVOPD(Opc: Opc2, EncodingFamily, VOPD3: IsVOPD3);
379
380 if (!(FirstCanBeVOPD.X && SecondCanBeVOPD.Y) &&
381 !(FirstCanBeVOPD.Y && SecondCanBeVOPD.X))
382 return std::nullopt;
383
384 // If SecondMI depends on FirstMI they cannot execute at the same time.
385 if (TII.hasRAWDependency(FirstMI, SecondMI))
386 return std::nullopt;
387
388 const GCNSubtarget &ST = TII.getSubtarget();
389 bool AllowSameVGPR = ST.hasGFX12Insts();
390
391 // Only a VOPD3 component can need a fixup; a plain one may hold a literal.
392 // checkVOPDRegConstraints() only writes these when it succeeds.
393 SmallVector<VOPDLiteralFixup, 2> Fixups;
394 bool CommuteXY[2] = {false, false};
395
396 if (FirstCanBeVOPD.X && SecondCanBeVOPD.Y) {
397 if (checkVOPDRegConstraints(TII, MIX: FirstMI, MIY: SecondMI, IsVOPD3, AllowSameVGPR,
398 LiteralFixups&: Fixups, CommuteXY))
399 return VOPDMatchInfo{
400 .InOrder: {&FirstMI, &SecondMI},
401 .XIdx: 0,
402 .IsVOPD3: IsVOPD3,
403 .LiteralFixups: std::move(Fixups),
404 .Commute: {CommuteXY[AMDGPU::VOPD::X], CommuteXY[AMDGPU::VOPD::Y]}};
405 }
406
407 if (FirstCanBeVOPD.Y && SecondCanBeVOPD.X) {
408 // AllowSameVGPR relaxes the VGPR bank overlap check for source operands.
409 // Only enable it when there is no antidependency.
410 bool IsAntiDep = TII.hasRAWDependency(FirstMI: SecondMI, SecondMI: FirstMI);
411 AllowSameVGPR &= !IsAntiDep;
412 if (IsAntiDep && !TII.isVOPDAntidependencyAllowed(MI: SecondMI))
413 return std::nullopt;
414 if (checkVOPDRegConstraints(TII, MIX: SecondMI, MIY: FirstMI, IsVOPD3, AllowSameVGPR,
415 LiteralFixups&: Fixups, CommuteXY))
416 return VOPDMatchInfo{
417 .InOrder: {&FirstMI, &SecondMI},
418 .XIdx: 1,
419 .IsVOPD3: IsVOPD3,
420 .LiteralFixups: std::move(Fixups),
421 .Commute: {CommuteXY[AMDGPU::VOPD::Y], CommuteXY[AMDGPU::VOPD::X]}};
422 }
423
424 return std::nullopt;
425}
426
427std::optional<VOPDMatchInfo> llvm::tryMatchVOPDPair(const SIInstrInfo &TII,
428 MachineInstr &FirstMI,
429 MachineInstr &SecondMI) {
430 const GCNSubtarget &ST = TII.getSubtarget();
431 unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST);
432 if (auto Match = tryMatchVOPDPairVariant(TII, EncodingFamily, FirstMI,
433 SecondMI, /*IsVOPD3=*/false))
434 return Match;
435 if (ST.hasVOPD3())
436 return tryMatchVOPDPairVariant(TII, EncodingFamily, FirstMI, SecondMI,
437 /*IsVOPD3=*/true);
438 return std::nullopt;
439}
440
441/// Check if the instr pair, FirstMI and SecondMI, should be scheduled
442/// together. Given SecondMI, when FirstMI is unspecified, then check if
443/// SecondMI may be part of a fused pair at all.
444static bool shouldScheduleVOPDAdjacent(const TargetInstrInfo &TII,
445 const TargetSubtargetInfo &TSI,
446 const MachineInstr *FirstMI,
447 const MachineInstr &SecondMI,
448 const SDep *) {
449 const SIInstrInfo &STII = static_cast<const SIInstrInfo &>(TII);
450 const GCNSubtarget &ST = STII.getSubtarget();
451
452 // One instruction case: just check whether SecondMI is eligible at all.
453 if (!FirstMI) {
454 unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST);
455 unsigned Opc2 = SecondMI.getOpcode();
456 auto CheckCanBeVOPD = [&](bool VOPD3) {
457 AMDGPU::CanBeVOPD CanBeVOPD =
458 AMDGPU::getCanBeVOPD(Opc: Opc2, EncodingFamily, VOPD3);
459 return CanBeVOPD.Y || CanBeVOPD.X;
460 };
461 return CheckCanBeVOPD(false) || (ST.hasVOPD3() && CheckCanBeVOPD(true));
462 }
463
464#ifdef EXPENSIVE_CHECKS
465 assert([&]() -> bool {
466 for (auto MII = MachineBasicBlock::const_iterator(FirstMI);
467 MII != FirstMI->getParent()->instr_end(); ++MII) {
468 if (&*MII == &SecondMI)
469 return true;
470 }
471 return false;
472 }() && "Expected FirstMI to precede SecondMI");
473#endif
474
475 return tryMatchVOPDPair(TII: STII, FirstMI&: *const_cast<MachineInstr *>(FirstMI),
476 SecondMI&: const_cast<MachineInstr &>(SecondMI))
477 .has_value();
478}
479
480/// Collect all load (dependents if \p Forward else dependencies) that connect
481/// to the \p Head SU.
482/// \p Visited should allocate enough bits for the number of SUnits, but its
483/// value can otherwise be uninitialized.
484static void collectLoads(SmallSetVector<SUnit *, 8> &Loads, BitVector &Visited,
485 SUnit &Head, bool Forward, bool StopAtLoads) {
486 if (Head.isBoundaryNode())
487 return;
488
489 Visited.reset();
490
491 SmallVector<SUnit *> Stack;
492 Stack.push_back(Elt: &Head);
493 while (!Stack.empty()) {
494 SUnit *SU = Stack.pop_back_val();
495 const SmallVector<SDep, 4> &Deps = Forward ? SU->Succs : SU->Preds;
496 for (const SDep &Edge : Deps) {
497 if (StopAtLoads && Edge.getKind() != SDep::Data)
498 continue;
499 SUnit *Dep = Edge.getSUnit();
500 if (Dep->isBoundaryNode() || Visited.test(Idx: Dep->NodeNum))
501 continue;
502 Visited.set(Dep->NodeNum);
503
504 if (Dep->isInstr() && Dep->getInstr()->mayLoad()) {
505 Loads.insert(X: Dep);
506 if (StopAtLoads)
507 continue;
508 }
509 Stack.push_back(Elt: Dep);
510 }
511 }
512}
513
514/// Checks whether fusing SU \p I with SU \p J would force the loads preceding
515/// \p J to complete before loads depending on \p I.
516///
517/// \p ILoadSuccs should hold all first load successors of \p I (via
518/// collectLoads with StopAtLoads=true). For set bits in \p LoadPredsComputed,
519/// the corresponding set in \p LoadPredsCache should hold all transitive load
520/// dependencies (via collectLoads with StopAtLoads=false). The \p Scratch
521/// bitvector should allocate enough bits for the number of SUnits.
522static bool
523loadsMayOverlap([[maybe_unused]] SUnit &I,
524 const SmallSetVector<SUnit *, 8> &ILoadSuccs, SUnit &J,
525 BitVector &LoadPredsComputed,
526 SmallVector<SmallSetVector<SUnit *, 8>> &LoadPredsCache,
527 BitVector &Scratch) {
528
529 if (ILoadSuccs.empty())
530 return false;
531
532 SmallSetVector<SUnit *, 8> &JLoadPreds = LoadPredsCache[J.NodeNum];
533 if (!LoadPredsComputed.test(Idx: J.NodeNum)) {
534 collectLoads(Loads&: JLoadPreds, Visited&: Scratch, Head&: J, /*Forward=*/false,
535 /*StopAtLoads=*/true);
536 LoadPredsComputed.set(J.NodeNum);
537 }
538 if (JLoadPreds.empty())
539 return false;
540
541 for (SUnit *ILoad : ILoadSuccs) {
542 SmallSetVector<SUnit *, 8> &ILoadDeps = LoadPredsCache[ILoad->NodeNum];
543 if (!LoadPredsComputed.test(Idx: ILoad->NodeNum)) {
544 collectLoads(Loads&: ILoadDeps, Visited&: Scratch, Head&: *ILoad, /*Forward=*/false,
545 /*StopAtLoads=*/false);
546 LoadPredsComputed.set(ILoad->NodeNum);
547 }
548
549 for (SUnit *JLoad : JLoadPreds) {
550 if (ILoad == JLoad) {
551 LLVM_DEBUG(
552 dbgs() << "Will not pair " << I << " with " << J << "\n"
553 << " Fusion would introduce a cyclic dependency with "
554 << *ILoad << "\n");
555 return true;
556 }
557
558 if (!ILoadDeps.contains(key: JLoad)) {
559 LLVM_DEBUG(dbgs() << "Will not pair " << I << " with " << J << "\n"
560 << " Fusion may force " << *JLoad
561 << " to complete its load before dispatching "
562 << *ILoad << "\n");
563 return true;
564 }
565 }
566 }
567 return false;
568}
569
570namespace {
571/// Adapts design from MacroFusion
572/// Puts valid candidate instructions back-to-back so they can easily
573/// be turned into VOPD instructions
574/// Greedily pairs instruction candidates. O(n^2) algorithm.
575struct VOPDPairingMutation : ScheduleDAGMutation {
576 MacroFusionPredTy shouldScheduleAdjacent; // NOLINT: function pointer
577
578 VOPDPairingMutation(
579 MacroFusionPredTy shouldScheduleAdjacent) // NOLINT: function pointer
580 : shouldScheduleAdjacent(shouldScheduleAdjacent) {}
581
582 void apply(ScheduleDAGInstrs *DAG) override {
583 const TargetInstrInfo &TII = *DAG->TII;
584 const GCNSubtarget &ST = DAG->MF.getSubtarget<GCNSubtarget>();
585 if (!AMDGPU::hasVOPD(STI: ST) || !ST.isWave32()) {
586 LLVM_DEBUG(dbgs() << "Target does not support VOPDPairingMutation\n");
587 return;
588 }
589
590 BitVector VOPDCapable(DAG->SUnits.size());
591 unsigned IIdx = 0;
592 // Pre-compute whether each individual instruction can be VOPD
593 for (auto ISUI = DAG->SUnits.begin(), E = DAG->SUnits.end(); ISUI != E;
594 ++ISUI, ++IIdx) {
595 const MachineInstr *IMI = ISUI->getInstr();
596 if (shouldScheduleAdjacent(TII, ST, nullptr, *IMI, nullptr) &&
597 hasLessThanNumFused(SU: *ISUI, FuseLimit: 2))
598 VOPDCapable[IIdx] = true;
599 }
600
601 IIdx = 0;
602 SmallSetVector<SUnit *, 8> ILoadSuccs;
603
604 // Cache collected load predecessors.
605 // For VOPDCapable nodes, this caches collectLoads with StopAtLoads=true
606 // For loads, this caches collectLoads with StopAtLoads=false
607 BitVector LoadPredsComputed(DAG->SUnits.size());
608 SmallVector<SmallSetVector<SUnit *, 8>> LoadPredsCache(DAG->SUnits.size());
609
610 BitVector Scratch(DAG->SUnits.size());
611 for (auto ISUI = DAG->SUnits.begin(), E = DAG->SUnits.end(); ISUI != E;
612 ++ISUI, ++IIdx) {
613 if (!VOPDCapable[IIdx])
614 continue;
615 const MachineInstr *IMI = ISUI->getInstr();
616
617 ILoadSuccs.clear();
618 collectLoads(Loads&: ILoadSuccs, Visited&: Scratch, Head&: *ISUI, /*Forward=*/true,
619 /*StopAtLoads=*/true);
620
621 unsigned JIdx = IIdx + 1;
622 for (auto JSUI = ISUI + 1; JSUI != E; ++JSUI, ++JIdx) {
623 if (!VOPDCapable[JIdx] || JSUI->isBoundaryNode())
624 continue;
625 const MachineInstr *JMI = JSUI->getInstr();
626 if (!hasLessThanNumFused(SU: *JSUI, FuseLimit: 2) ||
627 !shouldScheduleAdjacent(TII, ST, IMI, *JMI, nullptr))
628 continue;
629
630 if (loadsMayOverlap(I&: *ISUI, ILoadSuccs, J&: *JSUI, LoadPredsComputed,
631 LoadPredsCache, Scratch))
632 continue;
633
634 if (fuseInstructionPair(DAG&: *DAG, FirstSU&: *ISUI, SecondSU&: *JSUI)) {
635 // Clear to prevent future checks/fusing
636 VOPDCapable[JIdx] = false;
637 break;
638 }
639 }
640 }
641 LLVM_DEBUG(dbgs() << "Completed VOPDPairingMutation\n");
642 }
643};
644} // namespace
645
646std::unique_ptr<ScheduleDAGMutation> llvm::createVOPDPairingMutation() {
647 return std::make_unique<VOPDPairingMutation>(args&: shouldScheduleVOPDAdjacent);
648}
649