1//===- GCNVOPDUtils.cpp - GCN VOPD Utils ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file contains the AMDGPU DAG scheduling
10/// mutation to pair VOPD instructions back to back. It also contains
11// subroutines useful in the creation of VOPD instructions
12//
13//===----------------------------------------------------------------------===//
14
15#include "GCNVOPDUtils.h"
16#include "AMDGPUSubtarget.h"
17#include "GCNSubtarget.h"
18#include "SIInstrInfo.h"
19#include "Utils/AMDGPUBaseInfo.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/CodeGen/MachineBasicBlock.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineInstr.h"
24#include "llvm/CodeGen/MachineOperand.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/MacroFusion.h"
27#include "llvm/CodeGen/ScheduleDAG.h"
28#include "llvm/CodeGen/ScheduleDAGMutation.h"
29#include "llvm/CodeGen/TargetInstrInfo.h"
30#include "llvm/MC/MCInst.h"
31
32using namespace llvm;
33
34#define DEBUG_TYPE "gcn-vopd-utils"
35
36// Return the register class of the VOPDOpc operand named
37// src/vsrc<SrcIdx><CompIdx>, which is the slot src<SrcIdx> of MI<CompIdx> maps
38// to.
39static const TargetRegisterClass *getVOPDSrcRegClass(const SIInstrInfo &TII,
40 int VOPDOpc,
41 unsigned CompIdx,
42 unsigned SrcIdx) {
43 using namespace AMDGPU;
44 int OpIdx = -1;
45 const bool IsX = CompIdx == VOPD::X;
46 switch (SrcIdx) {
47 case 0:
48 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::src0X : OpName::src0Y);
49 break;
50 case 1:
51 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::vsrc1X : OpName::vsrc1Y);
52 break;
53 case 2:
54 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::vsrc2X : OpName::vsrc2Y);
55 if (OpIdx == -1)
56 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::src2X : OpName::src2Y);
57 break;
58 default:
59 llvm_unreachable("unexpected VOPD source index");
60 }
61
62 assert(OpIdx != -1);
63 return TII.getRegClass(MCID: TII.get(Opcode: VOPDOpc), OpNum: OpIdx);
64}
65
66// Check if physical register from src<SrcIdx> operand of MI<CompIdx> matches
67// register class constraints in corresponding VOPDOpc operand with name
68// src/vsrc<SrcIdx><CompIdx>.
69static bool isValidVOPDSrc(const SIInstrInfo &TII, int VOPDOpc,
70 unsigned CompIdx, unsigned SrcIdx,
71 Register PhysSrcReg) {
72 return getVOPDSrcRegClass(TII, VOPDOpc, CompIdx, SrcIdx)
73 ->contains(Reg: PhysSrcReg);
74}
75
76static const MachineOperand &getNamedOp(const MachineInstr &MI,
77 AMDGPU::OpName Name) {
78 return MI.getOperand(i: getNamedOperandIdx(Opcode: MI.getOpcode(), Name));
79}
80
81// Check if MI is a VOP3P instruction with operands that satisfy the constraints
82// for mapping it to a VOP2/VOPD opcode: no modifiers, no clamp, src1 and src2
83// are registers (src0 can be register or literal), and src2 is same as dst.
84static bool canMapVOP3PToVOPD(const MachineInstr &MI) {
85 unsigned Opc = MI.getOpcode();
86 if (Opc != AMDGPU::V_DOT2_F32_F16 && Opc != AMDGPU::V_DOT2_F32_BF16)
87 return false;
88 // src0 can be register or literal
89 if (getNamedOp(MI, Name: AMDGPU::OpName::src0_modifiers).getImm() !=
90 SISrcMods::OP_SEL_1)
91 return false;
92 if (getNamedOp(MI, Name: AMDGPU::OpName::src1_modifiers).getImm() !=
93 SISrcMods::OP_SEL_1)
94 return false;
95 if (!getNamedOp(MI, Name: AMDGPU::OpName::src1).isReg())
96 return false;
97 if (getNamedOp(MI, Name: AMDGPU::OpName::src2_modifiers).getImm() !=
98 SISrcMods::OP_SEL_1)
99 return false;
100 if (!getNamedOp(MI, Name: AMDGPU::OpName::src2).isReg())
101 return false;
102 if (getNamedOp(MI, Name: AMDGPU::OpName::clamp).getImm() != 0)
103 return false;
104 return getNamedOp(MI, Name: AMDGPU::OpName::vdst).getReg() ==
105 getNamedOp(MI, Name: AMDGPU::OpName::src2).getReg();
106}
107
108static bool canMaterializeVOPDLiterals(const MachineFunction &MF) {
109 // A free register cannot be found without liveness. A move also makes the
110 // code longer, so a function which asked for small code keeps its literals.
111 return MF.getProperties().hasTracksLiveness() &&
112 !MF.getFunction().hasOptSize();
113}
114
115static bool
116checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX,
117 const MachineInstr &MIY, bool IsVOPD3,
118 bool AllowSameVGPR,
119 SmallVectorImpl<VOPDLiteralFixup> &LiteralFixups) {
120 namespace VOPD = AMDGPU::VOPD;
121
122 const MachineFunction *MF = MIX.getMF();
123 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
124
125 if (IsVOPD3 && !ST.hasVOPD3())
126 return false;
127 if (!IsVOPD3 && ((TII.isVOP3(MI: MIX) && !canMapVOP3PToVOPD(MI: MIX)) ||
128 (TII.isVOP3(MI: MIY) && !canMapVOP3PToVOPD(MI: MIY))))
129 return false;
130 if (TII.isDPP(MI: MIX) || TII.isDPP(MI: MIY))
131 return false;
132
133 // Collected here and handed over only on success, so a failed check cannot
134 // leave anything behind.
135 SmallVector<VOPDLiteralFixup, 2> Fixups;
136
137 const SIRegisterInfo *TRI = ST.getRegisterInfo();
138 const MachineRegisterInfo &MRI = MF->getRegInfo();
139 // Literals also count against scalar bus limit
140 SmallVector<const MachineOperand *> UniqueLiterals;
141 auto AddLiteral = [&](const MachineOperand &Op) {
142 for (auto &Literal : UniqueLiterals) {
143 if (Literal->isIdenticalTo(Other: Op))
144 return;
145 }
146 UniqueLiterals.push_back(Elt: &Op);
147 };
148 // Immediates which the caller will move into a scalar register. Identical
149 // values share one register, so they count like one scalar operand each.
150 SmallSet<int32_t, 2> MaterializedLiterals;
151 SmallSet<Register, 4> UniqueScalarRegs;
152
153 unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST);
154 unsigned XOpc = AMDGPU::getVOPDOpcode(Opc: MIX.getOpcode(), VOPD3: IsVOPD3);
155 unsigned YOpc = AMDGPU::getVOPDOpcode(Opc: MIY.getOpcode(), VOPD3: IsVOPD3);
156 int VOPDOpc = AMDGPU::getVOPDFull(OpX: XOpc, OpY: YOpc, EncodingFamily, VOPD3: IsVOPD3);
157 assert(VOPDOpc != -1);
158
159 auto InstInfo = AMDGPU::getVOPDInstInfo(OpX: MIX.getDesc(), OpY: MIY.getDesc());
160
161 for (auto CompIdx : VOPD::COMPONENTS) {
162 const MachineInstr &MI = (CompIdx == VOPD::X) ? MIX : MIY;
163
164 const MachineOperand &Src0 = *TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
165 if (Src0.isReg()) {
166 if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, SrcIdx: 0, PhysSrcReg: Src0.getReg()))
167 return false;
168 if (TII.regUsesConstantBus(Reg: Src0, MRI))
169 UniqueScalarRegs.insert(V: Src0.getReg());
170 } else if (!TII.isInlineConstant(MO: Src0)) {
171 if (!IsVOPD3) {
172 AddLiteral(Src0);
173 } else {
174 // A VOPD3 component cannot encode a literal, but src0 can read a
175 // scalar register. The pair is therefore still possible if the caller
176 // moves the value into one.
177 if (!canMaterializeVOPDLiterals(MF: *MF) || !Src0.isImm())
178 return false;
179 // Only a 32-bit slot is handled, because the caller produces the value
180 // with a single S_MOV_B32.
181 int OpIdx = getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
182 if (TII.getOpSize(MI, OpNo: OpIdx) != 4)
183 return false;
184 const TargetRegisterClass *SlotRC =
185 TRI->getCommonSubClass(A: getVOPDSrcRegClass(TII, VOPDOpc, CompIdx, SrcIdx: 0),
186 B: &AMDGPU::SGPR_32RegClass);
187 if (!SlotRC)
188 return false;
189 // Only the low bits reach the register, so two operands which name
190 // the same value share one move whichever way they were written.
191 int32_t Imm = static_cast<int32_t>(Src0.getImm());
192 MaterializedLiterals.insert(V: Imm);
193 Fixups.push_back(Elt: {.CompIdx: CompIdx, .OpIdx: static_cast<unsigned>(OpIdx), .Imm: Imm, .SlotRC: SlotRC});
194 }
195 }
196
197 // V_FMAMK_F32 (src1) and V_FMAAK_F32 (src2) have a mandatory literal.
198 // VOPD3 instructions don't set MandatoryLiteralIdx.
199 if (InstInfo[CompIdx].hasMandatoryLiteral()) {
200 auto CompOprIdx = InstInfo[CompIdx].getMandatoryLiteralCompOperandIndex();
201 AddLiteral(MI.getOperand(i: CompOprIdx));
202 }
203
204 // VOPD only. Affects V_CNDMASK_B32_e32.
205 if (MI.getDesc().hasImplicitUseOfPhysReg(Reg: AMDGPU::VCC))
206 UniqueScalarRegs.insert(V: AMDGPU::VCC_LO);
207
208 if (const MachineOperand *Src1 =
209 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)) {
210 if (Src1->isReg()) {
211 if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, SrcIdx: 1, PhysSrcReg: Src1->getReg()))
212 return false;
213 assert(TRI->isVectorRegister(MRI, Src1->getReg()));
214 } else if (IsVOPD3) {
215 return false;
216 }
217 }
218
219 if (IsVOPD3) {
220 if (const MachineOperand *Src2 =
221 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)) {
222 if (AMDGPU::hasNamedOperand(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::bitop3)) {
223 // BITOP3 can be converted to DUAL_BITOP2 when src2 is zero.
224 if (!Src2->isImm() || Src2->getImm())
225 return false;
226 } else {
227 if (!Src2->isReg())
228 return false;
229 if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, SrcIdx: 2, PhysSrcReg: Src2->getReg()))
230 return false;
231 if (TII.regUsesConstantBus(Reg: *Src2, MRI)) {
232 assert(MI.getOpcode() == AMDGPU::V_CNDMASK_B32_e64);
233 UniqueScalarRegs.insert(V: Src2->getReg());
234 }
235 }
236 }
237 for (auto OpName : {AMDGPU::OpName::clamp, AMDGPU::OpName::omod,
238 AMDGPU::OpName::op_sel}) {
239 if (TII.hasModifiersSet(MI, OpName))
240 return false;
241 }
242
243 // Neg is allowed, other modifiers are not. NB: even though sext has the
244 // same value as neg, there are no combinable instructions with sext.
245 for (auto OpName :
246 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
247 AMDGPU::OpName::src2_modifiers}) {
248 const MachineOperand *Mods = TII.getNamedOperand(MI, OperandName: OpName);
249 if (Mods && (Mods->getImm() & ~SISrcMods::NEG))
250 return false;
251 }
252 }
253 }
254
255 if (UniqueLiterals.size() > 1)
256 return false;
257 // Keep materialization pair-local and do not increase instruction count or
258 // register pressure by producing two different values for one pair.
259 if (MaterializedLiterals.size() > 1)
260 return false;
261 if ((UniqueLiterals.size() + MaterializedLiterals.size() +
262 UniqueScalarRegs.size()) > 2)
263 return false;
264
265 auto GetVRegIdx = [&](unsigned OpcodeIdx, unsigned OperandIdx) {
266 const MachineInstr &MI = (OpcodeIdx == VOPD::X) ? MIX : MIY;
267 const MachineOperand &Operand = MI.getOperand(i: OperandIdx);
268 if (Operand.isReg() && TRI->isVectorRegister(MRI, Reg: Operand.getReg()))
269 return Operand.getReg();
270 return Register();
271 };
272
273 // On GFX1170+ if both OpX and OpY are V_MOV_B32 then OPY uses SRC2
274 // source-cache.
275 bool SkipSrc = (ST.hasGFX11_7Insts() || ST.hasGFX12Insts()) &&
276 MIX.getOpcode() == AMDGPU::V_MOV_B32_e32 &&
277 MIY.getOpcode() == AMDGPU::V_MOV_B32_e32;
278
279 // Check VGPR bank constraints for operand registers across both instructions.
280 if (InstInfo.hasInvalidOperand(GetRegIdx: GetVRegIdx, MRI: *TRI, SkipSrc, AllowSameVGPR,
281 VOPD3: IsVOPD3, HasGFX11InterlockHazard: ST.hasGFX11VOPDInterlockHazard()))
282 return false;
283
284 LLVM_DEBUG(dbgs() << "VOPD Reg Constraints Passed\n\tX: " << MIX
285 << "\n\tY: " << MIY << "\n");
286 LiteralFixups.assign(RHS: Fixups);
287 return true;
288}
289
290/// Core pair-eligibility check for a single VOPD encoding variant (VOPD or
291/// VOPD3). Returns the X/Y assignment on success, or std::nullopt otherwise.
292static std::optional<VOPDMatchInfo>
293tryMatchVOPDPairVariant(const SIInstrInfo &TII, unsigned EncodingFamily,
294 MachineInstr &FirstMI, MachineInstr &SecondMI,
295 bool IsVOPD3) {
296 unsigned Opc = FirstMI.getOpcode();
297 unsigned Opc2 = SecondMI.getOpcode();
298 AMDGPU::CanBeVOPD FirstCanBeVOPD =
299 AMDGPU::getCanBeVOPD(Opc, EncodingFamily, VOPD3: IsVOPD3);
300 AMDGPU::CanBeVOPD SecondCanBeVOPD =
301 AMDGPU::getCanBeVOPD(Opc: Opc2, EncodingFamily, VOPD3: IsVOPD3);
302
303 if (!(FirstCanBeVOPD.X && SecondCanBeVOPD.Y) &&
304 !(FirstCanBeVOPD.Y && SecondCanBeVOPD.X))
305 return std::nullopt;
306
307 // If SecondMI depends on FirstMI they cannot execute at the same time.
308 if (TII.hasRAWDependency(FirstMI, SecondMI))
309 return std::nullopt;
310
311 const GCNSubtarget &ST = TII.getSubtarget();
312 bool AllowSameVGPR = ST.hasGFX12Insts();
313
314 // Only a VOPD3 component can need a fixup; a plain one may hold a literal.
315 // checkVOPDRegConstraints() only writes this when it succeeds.
316 SmallVector<VOPDLiteralFixup, 2> Fixups;
317
318 if (FirstCanBeVOPD.X && SecondCanBeVOPD.Y) {
319 if (checkVOPDRegConstraints(TII, MIX: FirstMI, MIY: SecondMI, IsVOPD3, AllowSameVGPR,
320 LiteralFixups&: Fixups))
321 return VOPDMatchInfo{
322 .InOrder: {&FirstMI, &SecondMI}, .XIdx: 0, .IsVOPD3: IsVOPD3, .LiteralFixups: std::move(Fixups)};
323 }
324
325 if (FirstCanBeVOPD.Y && SecondCanBeVOPD.X) {
326 // AllowSameVGPR relaxes the VGPR bank overlap check for source operands.
327 // Only enable it when there is no antidependency.
328 bool IsAntiDep = TII.hasRAWDependency(FirstMI: SecondMI, SecondMI: FirstMI);
329 AllowSameVGPR &= !IsAntiDep;
330 if (IsAntiDep && !TII.isVOPDAntidependencyAllowed(MI: SecondMI))
331 return std::nullopt;
332 if (checkVOPDRegConstraints(TII, MIX: SecondMI, MIY: FirstMI, IsVOPD3, AllowSameVGPR,
333 LiteralFixups&: Fixups))
334 return VOPDMatchInfo{
335 .InOrder: {&FirstMI, &SecondMI}, .XIdx: 1, .IsVOPD3: IsVOPD3, .LiteralFixups: std::move(Fixups)};
336 }
337
338 return std::nullopt;
339}
340
341std::optional<VOPDMatchInfo> llvm::tryMatchVOPDPair(const SIInstrInfo &TII,
342 MachineInstr &FirstMI,
343 MachineInstr &SecondMI) {
344 const GCNSubtarget &ST = TII.getSubtarget();
345 unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST);
346 if (auto Match = tryMatchVOPDPairVariant(TII, EncodingFamily, FirstMI,
347 SecondMI, /*IsVOPD3=*/false))
348 return Match;
349 if (ST.hasVOPD3())
350 return tryMatchVOPDPairVariant(TII, EncodingFamily, FirstMI, SecondMI,
351 /*IsVOPD3=*/true);
352 return std::nullopt;
353}
354
355/// Check if the instr pair, FirstMI and SecondMI, should be scheduled
356/// together. Given SecondMI, when FirstMI is unspecified, then check if
357/// SecondMI may be part of a fused pair at all.
358static bool shouldScheduleVOPDAdjacent(const TargetInstrInfo &TII,
359 const TargetSubtargetInfo &TSI,
360 const MachineInstr *FirstMI,
361 const MachineInstr &SecondMI,
362 const SDep *) {
363 const SIInstrInfo &STII = static_cast<const SIInstrInfo &>(TII);
364 const GCNSubtarget &ST = STII.getSubtarget();
365
366 // One instruction case: just check whether SecondMI is eligible at all.
367 if (!FirstMI) {
368 unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST);
369 unsigned Opc2 = SecondMI.getOpcode();
370 auto CheckCanBeVOPD = [&](bool VOPD3) {
371 AMDGPU::CanBeVOPD CanBeVOPD =
372 AMDGPU::getCanBeVOPD(Opc: Opc2, EncodingFamily, VOPD3);
373 return CanBeVOPD.Y || CanBeVOPD.X;
374 };
375 return CheckCanBeVOPD(false) || (ST.hasVOPD3() && CheckCanBeVOPD(true));
376 }
377
378#ifdef EXPENSIVE_CHECKS
379 assert([&]() -> bool {
380 for (auto MII = MachineBasicBlock::const_iterator(FirstMI);
381 MII != FirstMI->getParent()->instr_end(); ++MII) {
382 if (&*MII == &SecondMI)
383 return true;
384 }
385 return false;
386 }() && "Expected FirstMI to precede SecondMI");
387#endif
388
389 return tryMatchVOPDPair(TII: STII, FirstMI&: *const_cast<MachineInstr *>(FirstMI),
390 SecondMI&: const_cast<MachineInstr &>(SecondMI))
391 .has_value();
392}
393
394/// Collect all load (dependents if \p Forward else dependencies) that connect
395/// to the \p Head SU.
396/// \p Visited should allocate enough bits for the number of SUnits, but its
397/// value can otherwise be uninitialized.
398static void collectLoads(SmallPtrSet<SUnit *, 8> &Loads, BitVector &Visited,
399 SUnit &Head, bool Forward, bool StopAtLoads) {
400 if (Head.isBoundaryNode())
401 return;
402
403 Visited.reset();
404
405 SmallVector<SUnit *> Stack;
406 Stack.push_back(Elt: &Head);
407 while (!Stack.empty()) {
408 SUnit *SU = Stack.pop_back_val();
409 const SmallVector<SDep, 4> &Deps = Forward ? SU->Succs : SU->Preds;
410 for (const SDep &Edge : Deps) {
411 if (StopAtLoads && Edge.getKind() != SDep::Data)
412 continue;
413 SUnit *Dep = Edge.getSUnit();
414 if (Dep->isBoundaryNode() || Visited.test(Idx: Dep->NodeNum))
415 continue;
416 Visited.set(Dep->NodeNum);
417
418 if (Dep->isInstr() && Dep->getInstr()->mayLoad()) {
419 Loads.insert(Ptr: Dep);
420 if (StopAtLoads)
421 continue;
422 }
423 Stack.push_back(Elt: Dep);
424 }
425 }
426}
427
428/// Checks whether fusing SU \p I with SU \p J would force the loads preceding
429/// \p J to complete before loads depending on \p I.
430///
431/// \p ILoadSuccs should hold all first load successors of \p I (via
432/// collectLoads with StopAtLoads=true). For set bits in \p LoadPredsComputed,
433/// the corresponding set in \p LoadPredsCache should hold all transitive load
434/// dependencies (via collectLoads with StopAtLoads=false). The \p Scratch
435/// bitvector should allocate enough bits for the number of SUnits.
436static bool loadsMayOverlap(
437 [[maybe_unused]] SUnit &I, const SmallPtrSet<SUnit *, 8> &ILoadSuccs,
438 SUnit &J, BitVector &LoadPredsComputed,
439 SmallVector<SmallPtrSet<SUnit *, 8>> &LoadPredsCache, BitVector &Scratch) {
440
441 if (ILoadSuccs.empty())
442 return false;
443
444 SmallPtrSet<SUnit *, 8> &JLoadPreds = LoadPredsCache[J.NodeNum];
445 if (!LoadPredsComputed.test(Idx: J.NodeNum)) {
446 collectLoads(Loads&: JLoadPreds, Visited&: Scratch, Head&: J, /*Forward=*/false,
447 /*StopAtLoads=*/true);
448 LoadPredsComputed.set(J.NodeNum);
449 }
450 if (JLoadPreds.empty())
451 return false;
452
453 for (SUnit *ILoad : ILoadSuccs) {
454 SmallPtrSet<SUnit *, 8> &ILoadDeps = LoadPredsCache[ILoad->NodeNum];
455 if (!LoadPredsComputed.test(Idx: ILoad->NodeNum)) {
456 collectLoads(Loads&: ILoadDeps, Visited&: Scratch, Head&: *ILoad, /*Forward=*/false,
457 /*StopAtLoads=*/false);
458 LoadPredsComputed.set(ILoad->NodeNum);
459 }
460
461 for (SUnit *JLoad : JLoadPreds) {
462 if (ILoad == JLoad) {
463 LLVM_DEBUG(
464 dbgs() << "Will not pair SU(" << I.NodeNum << ") with SU("
465 << J.NodeNum << ")\n"
466 << " Fusion would introduce a cyclic dependency with SU("
467 << ILoad->NodeNum << ")\n");
468 return true;
469 }
470
471 if (!ILoadDeps.contains(Ptr: JLoad)) {
472 LLVM_DEBUG(dbgs() << "Will not pair SU(" << I.NodeNum << ") with SU("
473 << J.NodeNum << ")\n"
474 << " Fusion may force SU(" << JLoad->NodeNum
475 << ") to complete its load before dispatching SU("
476 << ILoad->NodeNum << ")\n");
477 return true;
478 }
479 }
480 }
481 return false;
482}
483
484namespace {
485/// Adapts design from MacroFusion
486/// Puts valid candidate instructions back-to-back so they can easily
487/// be turned into VOPD instructions
488/// Greedily pairs instruction candidates. O(n^2) algorithm.
489struct VOPDPairingMutation : ScheduleDAGMutation {
490 MacroFusionPredTy shouldScheduleAdjacent; // NOLINT: function pointer
491
492 VOPDPairingMutation(
493 MacroFusionPredTy shouldScheduleAdjacent) // NOLINT: function pointer
494 : shouldScheduleAdjacent(shouldScheduleAdjacent) {}
495
496 void apply(ScheduleDAGInstrs *DAG) override {
497 const TargetInstrInfo &TII = *DAG->TII;
498 const GCNSubtarget &ST = DAG->MF.getSubtarget<GCNSubtarget>();
499 if (!AMDGPU::hasVOPD(STI: ST) || !ST.isWave32()) {
500 LLVM_DEBUG(dbgs() << "Target does not support VOPDPairingMutation\n");
501 return;
502 }
503
504 BitVector VOPDCapable(DAG->SUnits.size());
505 unsigned IIdx = 0;
506 // Pre-compute whether each individual instruction can be VOPD
507 for (auto ISUI = DAG->SUnits.begin(), E = DAG->SUnits.end(); ISUI != E;
508 ++ISUI, ++IIdx) {
509 const MachineInstr *IMI = ISUI->getInstr();
510 if (shouldScheduleAdjacent(TII, ST, nullptr, *IMI, nullptr) &&
511 hasLessThanNumFused(SU: *ISUI, FuseLimit: 2))
512 VOPDCapable[IIdx] = true;
513 }
514
515 IIdx = 0;
516 SmallPtrSet<SUnit *, 8> ILoadSuccs;
517
518 // Cache collected load predecessors.
519 // For VOPDCapable nodes, this caches collectLoads with StopAtLoads=true
520 // For loads, this caches collectLoads with StopAtLoads=false
521 BitVector LoadPredsComputed(DAG->SUnits.size());
522 SmallVector<SmallPtrSet<SUnit *, 8>> LoadPredsCache(DAG->SUnits.size());
523
524 BitVector Scratch(DAG->SUnits.size());
525 for (auto ISUI = DAG->SUnits.begin(), E = DAG->SUnits.end(); ISUI != E;
526 ++ISUI, ++IIdx) {
527 if (!VOPDCapable[IIdx])
528 continue;
529 const MachineInstr *IMI = ISUI->getInstr();
530
531 ILoadSuccs.clear();
532 collectLoads(Loads&: ILoadSuccs, Visited&: Scratch, Head&: *ISUI, /*Forward=*/true,
533 /*StopAtLoads=*/true);
534
535 unsigned JIdx = IIdx + 1;
536 for (auto JSUI = ISUI + 1; JSUI != E; ++JSUI, ++JIdx) {
537 if (!VOPDCapable[JIdx] || JSUI->isBoundaryNode())
538 continue;
539 const MachineInstr *JMI = JSUI->getInstr();
540 if (!hasLessThanNumFused(SU: *JSUI, FuseLimit: 2) ||
541 !shouldScheduleAdjacent(TII, ST, IMI, *JMI, nullptr))
542 continue;
543
544 if (loadsMayOverlap(I&: *ISUI, ILoadSuccs, J&: *JSUI, LoadPredsComputed,
545 LoadPredsCache, Scratch))
546 continue;
547
548 if (fuseInstructionPair(DAG&: *DAG, FirstSU&: *ISUI, SecondSU&: *JSUI)) {
549 // Clear to prevent future checks/fusing
550 VOPDCapable[JIdx] = false;
551 break;
552 }
553 }
554 }
555 LLVM_DEBUG(dbgs() << "Completed VOPDPairingMutation\n");
556 }
557};
558} // namespace
559
560std::unique_ptr<ScheduleDAGMutation> llvm::createVOPDPairingMutation() {
561 return std::make_unique<VOPDPairingMutation>(args&: shouldScheduleVOPDAdjacent);
562}
563