1//===- GCNVOPDUtils.cpp - GCN VOPD Utils ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file contains the AMDGPU DAG scheduling
10/// mutation to pair VOPD instructions back to back. It also contains
11// subroutines useful in the creation of VOPD instructions
12//
13//===----------------------------------------------------------------------===//
14
15#include "GCNVOPDUtils.h"
16#include "AMDGPUSubtarget.h"
17#include "GCNSubtarget.h"
18#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
19#include "SIInstrInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/SmallVector.h"
23#include "llvm/CodeGen/MachineBasicBlock.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineOperand.h"
26#include "llvm/CodeGen/MachineRegisterInfo.h"
27#include "llvm/CodeGen/MacroFusion.h"
28#include "llvm/CodeGen/ScheduleDAG.h"
29#include "llvm/CodeGen/ScheduleDAGMutation.h"
30#include "llvm/CodeGen/TargetInstrInfo.h"
31#include "llvm/MC/MCInst.h"
32
33using namespace llvm;
34
35#define DEBUG_TYPE "gcn-vopd-utils"
36
37// Check if physical register from src<SrcIdx> operand of MI<CompIdx> matches
38// register class constraints in corresponding VOPDOpc operand with name
39// src/vsrc<SrcIdx><CompIdx>.
40static bool isValidVOPDSrc(const SIInstrInfo &TII, int VOPDOpc,
41 unsigned CompIdx, unsigned SrcIdx,
42 Register PhysSrcReg) {
43 using namespace AMDGPU;
44 int OpIdx = -1;
45 const bool IsX = CompIdx == VOPD::X;
46 switch (SrcIdx) {
47 case 0:
48 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::src0X : OpName::src0Y);
49 break;
50 case 1:
51 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::vsrc1X : OpName::vsrc1Y);
52 break;
53 case 2:
54 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::vsrc2X : OpName::vsrc2Y);
55 if (OpIdx == -1)
56 OpIdx = getNamedOperandIdx(Opcode: VOPDOpc, Name: IsX ? OpName::src2X : OpName::src2Y);
57 break;
58 default:
59 llvm_unreachable("unexpected VOPD source index");
60 }
61
62 assert(OpIdx != -1);
63 return TII.getRegClass(MCID: TII.get(Opcode: VOPDOpc), OpNum: OpIdx)->contains(Reg: PhysSrcReg);
64}
65
66static const MachineOperand &getNamedOp(const MachineInstr &MI,
67 AMDGPU::OpName Name) {
68 return MI.getOperand(i: getNamedOperandIdx(Opcode: MI.getOpcode(), Name));
69}
70
71// Check if MI is a VOP3P instruction with operands that satisfy the constraints
72// for mapping it to a VOP2/VOPD opcode: no modifiers, no clamp, src1 and src2
73// are registers (src0 can be register or literal), and src2 is same as dst.
74static bool canMapVOP3PToVOPD(const MachineInstr &MI) {
75 unsigned Opc = MI.getOpcode();
76 if (Opc != AMDGPU::V_DOT2_F32_F16 && Opc != AMDGPU::V_DOT2_F32_BF16)
77 return false;
78 // src0 can be register or literal
79 if (getNamedOp(MI, Name: AMDGPU::OpName::src0_modifiers).getImm() !=
80 SISrcMods::OP_SEL_1)
81 return false;
82 if (getNamedOp(MI, Name: AMDGPU::OpName::src1_modifiers).getImm() !=
83 SISrcMods::OP_SEL_1)
84 return false;
85 if (!getNamedOp(MI, Name: AMDGPU::OpName::src1).isReg())
86 return false;
87 if (getNamedOp(MI, Name: AMDGPU::OpName::src2_modifiers).getImm() !=
88 SISrcMods::OP_SEL_1)
89 return false;
90 if (!getNamedOp(MI, Name: AMDGPU::OpName::src2).isReg())
91 return false;
92 if (getNamedOp(MI, Name: AMDGPU::OpName::clamp).getImm() != 0)
93 return false;
94 return getNamedOp(MI, Name: AMDGPU::OpName::vdst).getReg() ==
95 getNamedOp(MI, Name: AMDGPU::OpName::src2).getReg();
96}
97
98bool llvm::checkVOPDRegConstraints(const SIInstrInfo &TII,
99 const MachineInstr &MIX,
100 const MachineInstr &MIY, bool IsVOPD3,
101 bool AllowSameVGPR) {
102 namespace VOPD = AMDGPU::VOPD;
103
104 const MachineFunction *MF = MIX.getMF();
105 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
106
107 if (IsVOPD3 && !ST.hasVOPD3())
108 return false;
109 if (!IsVOPD3 && ((TII.isVOP3(MI: MIX) && !canMapVOP3PToVOPD(MI: MIX)) ||
110 (TII.isVOP3(MI: MIY) && !canMapVOP3PToVOPD(MI: MIY))))
111 return false;
112 if (TII.isDPP(MI: MIX) || TII.isDPP(MI: MIY))
113 return false;
114
115 const SIRegisterInfo *TRI = ST.getRegisterInfo();
116 const MachineRegisterInfo &MRI = MF->getRegInfo();
117 // Literals also count against scalar bus limit
118 SmallVector<const MachineOperand *> UniqueLiterals;
119 auto addLiteral = [&](const MachineOperand &Op) {
120 for (auto &Literal : UniqueLiterals) {
121 if (Literal->isIdenticalTo(Other: Op))
122 return;
123 }
124 UniqueLiterals.push_back(Elt: &Op);
125 };
126 SmallSet<Register, 4> UniqueScalarRegs;
127
128 unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST);
129 unsigned XOpc = AMDGPU::getVOPDOpcode(Opc: MIX.getOpcode(), VOPD3: IsVOPD3);
130 unsigned YOpc = AMDGPU::getVOPDOpcode(Opc: MIY.getOpcode(), VOPD3: IsVOPD3);
131 int VOPDOpc = AMDGPU::getVOPDFull(OpX: XOpc, OpY: YOpc, EncodingFamily, VOPD3: IsVOPD3);
132 assert(VOPDOpc != -1);
133
134 auto InstInfo = AMDGPU::getVOPDInstInfo(OpX: MIX.getDesc(), OpY: MIY.getDesc());
135
136 for (auto CompIdx : VOPD::COMPONENTS) {
137 const MachineInstr &MI = (CompIdx == VOPD::X) ? MIX : MIY;
138
139 const MachineOperand &Src0 = *TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
140 if (Src0.isReg()) {
141 if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, SrcIdx: 0, PhysSrcReg: Src0.getReg()))
142 return false;
143 if (!TRI->isVectorRegister(MRI, Reg: Src0.getReg()))
144 UniqueScalarRegs.insert(V: Src0.getReg());
145 } else if (!TII.isInlineConstant(MO: Src0)) {
146 if (IsVOPD3)
147 return false;
148 addLiteral(Src0);
149 }
150
151 // V_FMAMK_F32 (src1) and V_FMAAK_F32 (src2) have a mandatory literal.
152 // VOPD3 instructions don't set MandatoryLiteralIdx.
153 if (InstInfo[CompIdx].hasMandatoryLiteral()) {
154 auto CompOprIdx = InstInfo[CompIdx].getMandatoryLiteralCompOperandIndex();
155 addLiteral(MI.getOperand(i: CompOprIdx));
156 }
157
158 // VOPD only. Affects V_CNDMASK_B32_e32.
159 if (MI.getDesc().hasImplicitUseOfPhysReg(Reg: AMDGPU::VCC))
160 UniqueScalarRegs.insert(V: AMDGPU::VCC_LO);
161
162 if (const MachineOperand *Src1 =
163 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)) {
164 if (Src1->isReg()) {
165 if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, SrcIdx: 1, PhysSrcReg: Src1->getReg()))
166 return false;
167 assert(TRI->isVectorRegister(MRI, Src1->getReg()));
168 } else if (IsVOPD3) {
169 return false;
170 }
171 }
172
173 if (IsVOPD3) {
174 if (const MachineOperand *Src2 =
175 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)) {
176 if (AMDGPU::hasNamedOperand(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::bitop3)) {
177 // BITOP3 can be converted to DUAL_BITOP2 when src2 is zero.
178 if (!Src2->isImm() || Src2->getImm())
179 return false;
180 } else {
181 if (!Src2->isReg())
182 return false;
183 if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, SrcIdx: 2, PhysSrcReg: Src2->getReg()))
184 return false;
185 if (!TRI->isVectorRegister(MRI, Reg: Src2->getReg())) {
186 assert(MI.getOpcode() == AMDGPU::V_CNDMASK_B32_e64);
187 UniqueScalarRegs.insert(V: Src2->getReg());
188 }
189 }
190 }
191 for (auto OpName : {AMDGPU::OpName::clamp, AMDGPU::OpName::omod,
192 AMDGPU::OpName::op_sel}) {
193 if (TII.hasModifiersSet(MI, OpName))
194 return false;
195 }
196
197 // Neg is allowed, other modifiers are not. NB: even though sext has the
198 // same value as neg, there are no combinable instructions with sext.
199 for (auto OpName :
200 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
201 AMDGPU::OpName::src2_modifiers}) {
202 const MachineOperand *Mods = TII.getNamedOperand(MI, OperandName: OpName);
203 if (Mods && (Mods->getImm() & ~SISrcMods::NEG))
204 return false;
205 }
206 }
207 }
208
209 if (UniqueLiterals.size() > 1)
210 return false;
211 if ((UniqueLiterals.size() + UniqueScalarRegs.size()) > 2)
212 return false;
213
214 auto getVRegIdx = [&](unsigned OpcodeIdx, unsigned OperandIdx) {
215 const MachineInstr &MI = (OpcodeIdx == VOPD::X) ? MIX : MIY;
216 const MachineOperand &Operand = MI.getOperand(i: OperandIdx);
217 if (Operand.isReg() && TRI->isVectorRegister(MRI, Reg: Operand.getReg()))
218 return Operand.getReg();
219 return Register();
220 };
221
222 // On GFX1170+ if both OpX and OpY are V_MOV_B32 then OPY uses SRC2
223 // source-cache.
224 bool SkipSrc = (ST.hasGFX11_7Insts() || ST.hasGFX12Insts()) &&
225 MIX.getOpcode() == AMDGPU::V_MOV_B32_e32 &&
226 MIY.getOpcode() == AMDGPU::V_MOV_B32_e32;
227
228 // Check VGPR bank constraints for operand registers across both instructions.
229 if (InstInfo.hasInvalidOperand(GetRegIdx: getVRegIdx, MRI: *TRI, SkipSrc, AllowSameVGPR,
230 VOPD3: IsVOPD3))
231 return false;
232
233 LLVM_DEBUG(dbgs() << "VOPD Reg Constraints Passed\n\tX: " << MIX
234 << "\n\tY: " << MIY << "\n");
235 return true;
236}
237
238/// Core pair-eligibility check for a single VOPD encoding variant (VOPD or
239/// VOPD3). Returns the X/Y assignment on success, or std::nullopt otherwise.
240static std::optional<VOPDMatchInfo>
241tryMatchVOPDPairVariant(const SIInstrInfo &TII, unsigned EncodingFamily,
242 MachineInstr &FirstMI, MachineInstr &SecondMI,
243 bool IsVOPD3) {
244 unsigned Opc = FirstMI.getOpcode();
245 unsigned Opc2 = SecondMI.getOpcode();
246 AMDGPU::CanBeVOPD FirstCanBeVOPD =
247 AMDGPU::getCanBeVOPD(Opc, EncodingFamily, VOPD3: IsVOPD3);
248 AMDGPU::CanBeVOPD SecondCanBeVOPD =
249 AMDGPU::getCanBeVOPD(Opc: Opc2, EncodingFamily, VOPD3: IsVOPD3);
250
251 if (!(FirstCanBeVOPD.X && SecondCanBeVOPD.Y) &&
252 !(FirstCanBeVOPD.Y && SecondCanBeVOPD.X))
253 return std::nullopt;
254
255 // If SecondMI depends on FirstMI they cannot execute at the same time.
256 if (TII.hasRAWDependency(FirstMI, SecondMI))
257 return std::nullopt;
258
259 const GCNSubtarget &ST = TII.getSubtarget();
260 bool AllowSameVGPR = ST.hasGFX12Insts();
261
262 if (FirstCanBeVOPD.X && SecondCanBeVOPD.Y) {
263 if (checkVOPDRegConstraints(TII, MIX: FirstMI, MIY: SecondMI, IsVOPD3, AllowSameVGPR))
264 return VOPDMatchInfo{.MIX: &FirstMI, .MIY: &SecondMI, .IsVOPD3: IsVOPD3};
265 }
266
267 if (FirstCanBeVOPD.Y && SecondCanBeVOPD.X) {
268 // AllowSameVGPR relaxes the VGPR bank overlap check for source operands.
269 // Only enable it when there is no antidependency.
270 bool IsAntiDep = TII.hasRAWDependency(FirstMI: SecondMI, SecondMI: FirstMI);
271 AllowSameVGPR &= !IsAntiDep;
272 if (IsAntiDep && !TII.isVOPDAntidependencyAllowed(MI: SecondMI))
273 return std::nullopt;
274 if (checkVOPDRegConstraints(TII, MIX: SecondMI, MIY: FirstMI, IsVOPD3, AllowSameVGPR))
275 return VOPDMatchInfo{.MIX: &SecondMI, .MIY: &FirstMI, .IsVOPD3: IsVOPD3};
276 }
277
278 return std::nullopt;
279}
280
281std::optional<VOPDMatchInfo> llvm::tryMatchVOPDPair(const SIInstrInfo &TII,
282 MachineInstr &FirstMI,
283 MachineInstr &SecondMI) {
284 const GCNSubtarget &ST = TII.getSubtarget();
285 unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST);
286 if (auto Match = tryMatchVOPDPairVariant(TII, EncodingFamily, FirstMI,
287 SecondMI, /*IsVOPD3=*/false))
288 return Match;
289 if (ST.hasVOPD3())
290 return tryMatchVOPDPairVariant(TII, EncodingFamily, FirstMI, SecondMI,
291 /*IsVOPD3=*/true);
292 return std::nullopt;
293}
294
295/// Check if the instr pair, FirstMI and SecondMI, should be scheduled
296/// together. Given SecondMI, when FirstMI is unspecified, then check if
297/// SecondMI may be part of a fused pair at all.
298static bool shouldScheduleVOPDAdjacent(const TargetInstrInfo &TII,
299 const TargetSubtargetInfo &TSI,
300 const MachineInstr *FirstMI,
301 const MachineInstr &SecondMI,
302 const SDep *) {
303 const SIInstrInfo &STII = static_cast<const SIInstrInfo &>(TII);
304 const GCNSubtarget &ST = STII.getSubtarget();
305
306 // One instruction case: just check whether SecondMI is eligible at all.
307 if (!FirstMI) {
308 unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST);
309 unsigned Opc2 = SecondMI.getOpcode();
310 auto checkCanBeVOPD = [&](bool VOPD3) {
311 AMDGPU::CanBeVOPD CanBeVOPD =
312 AMDGPU::getCanBeVOPD(Opc: Opc2, EncodingFamily, VOPD3);
313 return CanBeVOPD.Y || CanBeVOPD.X;
314 };
315 return checkCanBeVOPD(false) || (ST.hasVOPD3() && checkCanBeVOPD(true));
316 }
317
318#ifdef EXPENSIVE_CHECKS
319 assert([&]() -> bool {
320 for (auto MII = MachineBasicBlock::const_iterator(FirstMI);
321 MII != FirstMI->getParent()->instr_end(); ++MII) {
322 if (&*MII == &SecondMI)
323 return true;
324 }
325 return false;
326 }() && "Expected FirstMI to precede SecondMI");
327#endif
328
329 return tryMatchVOPDPair(TII: STII, FirstMI&: *const_cast<MachineInstr *>(FirstMI),
330 SecondMI&: const_cast<MachineInstr &>(SecondMI))
331 .has_value();
332}
333
334/// Collect all load (dependents if \p Forward else dependencies) that connect
335/// to the \p Head SU.
336/// \p Visited should allocate enough bits for the number of SUnits, but its
337/// value can otherwise be uninitialized.
338static void collectLoads(SmallPtrSet<SUnit *, 8> &Loads, BitVector &Visited,
339 SUnit &Head, bool Forward, bool StopAtLoads) {
340 if (Head.isBoundaryNode())
341 return;
342
343 Visited.reset();
344
345 SmallVector<SUnit *> Stack;
346 Stack.push_back(Elt: &Head);
347 while (!Stack.empty()) {
348 SUnit *SU = Stack.pop_back_val();
349 const SmallVector<SDep, 4> &Deps = Forward ? SU->Succs : SU->Preds;
350 for (const SDep &Edge : Deps) {
351 if (StopAtLoads && Edge.getKind() != SDep::Data)
352 continue;
353 SUnit *Dep = Edge.getSUnit();
354 if (Dep->isBoundaryNode() || Visited.test(Idx: Dep->NodeNum))
355 continue;
356 Visited.set(Dep->NodeNum);
357
358 if (Dep->isInstr() && Dep->getInstr()->mayLoad()) {
359 Loads.insert(Ptr: Dep);
360 if (StopAtLoads)
361 continue;
362 }
363 Stack.push_back(Elt: Dep);
364 }
365 }
366}
367
368/// Checks whether fusing SU \p I with SU \p J would force the loads preceding
369/// \p J to complete before loads depending on \p I.
370///
371/// \p ILoadSuccs should hold all first load successors of \p I (via
372/// collectLoads with StopAtLoads=true). For set bits in \p LoadPredsComputed,
373/// the corresponding set in \p LoadPredsCache should hold all transitive load
374/// dependencies (via collectLoads with StopAtLoads=false). The \p Scratch
375/// bitvector should allocate enough bits for the number of SUnits.
376static bool loadsMayOverlap(
377 [[maybe_unused]] SUnit &I, const SmallPtrSet<SUnit *, 8> &ILoadSuccs,
378 SUnit &J, BitVector &LoadPredsComputed,
379 SmallVector<SmallPtrSet<SUnit *, 8>> &LoadPredsCache, BitVector &Scratch) {
380
381 if (ILoadSuccs.empty())
382 return false;
383
384 SmallPtrSet<SUnit *, 8> &JLoadPreds = LoadPredsCache[J.NodeNum];
385 if (!LoadPredsComputed.test(Idx: J.NodeNum)) {
386 collectLoads(Loads&: JLoadPreds, Visited&: Scratch, Head&: J, /*Forward=*/false,
387 /*StopAtLoads=*/true);
388 LoadPredsComputed.set(J.NodeNum);
389 }
390 if (JLoadPreds.empty())
391 return false;
392
393 for (SUnit *ILoad : ILoadSuccs) {
394 SmallPtrSet<SUnit *, 8> &ILoadDeps = LoadPredsCache[ILoad->NodeNum];
395 if (!LoadPredsComputed.test(Idx: ILoad->NodeNum)) {
396 collectLoads(Loads&: ILoadDeps, Visited&: Scratch, Head&: *ILoad, /*Forward=*/false,
397 /*StopAtLoads=*/false);
398 LoadPredsComputed.set(ILoad->NodeNum);
399 }
400
401 for (SUnit *JLoad : JLoadPreds) {
402 if (ILoad == JLoad) {
403 LLVM_DEBUG(
404 dbgs() << "Will not pair SU(" << I.NodeNum << ") with SU("
405 << J.NodeNum << ")\n"
406 << " Fusion would introduce a cyclic dependency with SU("
407 << ILoad->NodeNum << ")\n");
408 return true;
409 }
410
411 if (!ILoadDeps.contains(Ptr: JLoad)) {
412 LLVM_DEBUG(dbgs() << "Will not pair SU(" << I.NodeNum << ") with SU("
413 << J.NodeNum << ")\n"
414 << " Fusion may force SU(" << JLoad->NodeNum
415 << ") to complete its load before dispatching SU("
416 << ILoad->NodeNum << ")\n");
417 return true;
418 }
419 }
420 }
421 return false;
422}
423
424namespace {
425/// Adapts design from MacroFusion
426/// Puts valid candidate instructions back-to-back so they can easily
427/// be turned into VOPD instructions
428/// Greedily pairs instruction candidates. O(n^2) algorithm.
429struct VOPDPairingMutation : ScheduleDAGMutation {
430 MacroFusionPredTy shouldScheduleAdjacent; // NOLINT: function pointer
431
432 VOPDPairingMutation(
433 MacroFusionPredTy shouldScheduleAdjacent) // NOLINT: function pointer
434 : shouldScheduleAdjacent(shouldScheduleAdjacent) {}
435
436 void apply(ScheduleDAGInstrs *DAG) override {
437 const TargetInstrInfo &TII = *DAG->TII;
438 const GCNSubtarget &ST = DAG->MF.getSubtarget<GCNSubtarget>();
439 if (!AMDGPU::hasVOPD(STI: ST) || !ST.isWave32()) {
440 LLVM_DEBUG(dbgs() << "Target does not support VOPDPairingMutation\n");
441 return;
442 }
443
444 BitVector VOPDCapable(DAG->SUnits.size());
445 unsigned IIdx = 0;
446 // Pre-compute whether each individual instruction can be VOPD
447 for (auto ISUI = DAG->SUnits.begin(), E = DAG->SUnits.end(); ISUI != E;
448 ++ISUI, ++IIdx) {
449 const MachineInstr *IMI = ISUI->getInstr();
450 if (shouldScheduleAdjacent(TII, ST, nullptr, *IMI, nullptr) &&
451 hasLessThanNumFused(SU: *ISUI, FuseLimit: 2))
452 VOPDCapable[IIdx] = true;
453 }
454
455 IIdx = 0;
456 SmallPtrSet<SUnit *, 8> ILoadSuccs;
457
458 // Cache collected load predecessors.
459 // For VOPDCapable nodes, this caches collectLoads with StopAtLoads=true
460 // For loads, this caches collectLoads with StopAtLoads=false
461 BitVector LoadPredsComputed(DAG->SUnits.size());
462 SmallVector<SmallPtrSet<SUnit *, 8>> LoadPredsCache(DAG->SUnits.size());
463
464 BitVector Scratch(DAG->SUnits.size());
465 for (auto ISUI = DAG->SUnits.begin(), E = DAG->SUnits.end(); ISUI != E;
466 ++ISUI, ++IIdx) {
467 if (!VOPDCapable[IIdx])
468 continue;
469 const MachineInstr *IMI = ISUI->getInstr();
470
471 ILoadSuccs.clear();
472 collectLoads(Loads&: ILoadSuccs, Visited&: Scratch, Head&: *ISUI, /*Forward=*/true,
473 /*StopAtLoads=*/true);
474
475 unsigned JIdx = IIdx + 1;
476 for (auto JSUI = ISUI + 1; JSUI != E; ++JSUI, ++JIdx) {
477 if (!VOPDCapable[JIdx] || JSUI->isBoundaryNode())
478 continue;
479 const MachineInstr *JMI = JSUI->getInstr();
480 if (!hasLessThanNumFused(SU: *JSUI, FuseLimit: 2) ||
481 !shouldScheduleAdjacent(TII, ST, IMI, *JMI, nullptr))
482 continue;
483
484 if (loadsMayOverlap(I&: *ISUI, ILoadSuccs, J&: *JSUI, LoadPredsComputed,
485 LoadPredsCache, Scratch))
486 continue;
487
488 if (fuseInstructionPair(DAG&: *DAG, FirstSU&: *ISUI, SecondSU&: *JSUI)) {
489 // Clear to prevent future checks/fusing
490 VOPDCapable[JIdx] = false;
491 break;
492 }
493 }
494 }
495 LLVM_DEBUG(dbgs() << "Completed VOPDPairingMutation\n");
496 }
497};
498} // namespace
499
500std::unique_ptr<ScheduleDAGMutation> llvm::createVOPDPairingMutation() {
501 return std::make_unique<VOPDPairingMutation>(args&: shouldScheduleVOPDAdjacent);
502}
503