1//===- SIPeepholeSDWA.cpp - Peephole optimization for SDWA instructions ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass tries to apply several peephole SDWA patterns.
10///
11/// E.g. original:
12/// V_LSHRREV_B32_e32 %0, 16, %1
13/// V_ADD_CO_U32_e32 %2, %0, %3
14/// V_LSHLREV_B32_e32 %4, 16, %2
15///
16/// Replace:
17/// V_ADD_CO_U32_sdwa %4, %1, %3
18/// dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
19///
20//===----------------------------------------------------------------------===//
21
22#include "SIPeepholeSDWA.h"
23#include "AMDGPU.h"
24#include "GCNSubtarget.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/CodeGen/MachineFunctionPass.h"
27#include <optional>
28
29using namespace llvm;
30
31#define DEBUG_TYPE "si-peephole-sdwa"
32
33STATISTIC(NumSDWAPatternsFound, "Number of SDWA patterns found.");
34STATISTIC(NumSDWAInstructionsPeepholed,
35 "Number of instruction converted to SDWA.");
36
37namespace {
38
39bool isConvertibleToSDWA(MachineInstr &MI, const GCNSubtarget &ST,
40 const SIInstrInfo *TII);
41class SDWAOperand;
42class SDWADstOperand;
43
44using SDWAOperandsVector = SmallVector<SDWAOperand *, 4>;
45using SDWAOperandsMap = MapVector<MachineInstr *, SDWAOperandsVector>;
46
47class SIPeepholeSDWA {
48private:
49 MachineRegisterInfo *MRI;
50 const SIRegisterInfo *TRI;
51 const SIInstrInfo *TII;
52
53 MapVector<MachineInstr *, std::unique_ptr<SDWAOperand>> SDWAOperands;
54 SDWAOperandsMap PotentialMatches;
55 SmallVector<MachineInstr *, 8> ConvertedInstructions;
56
57 std::optional<int64_t> foldToImm(const MachineOperand &Op) const;
58
59 // If MI is a v_and_b32 with a 0xffff or 0xff immediate, return the masked
60 // value operand and the matching SDWA selector (WORD_0 / BYTE_0).
61 std::optional<std::pair<MachineOperand *, AMDGPU::SDWA::SdwaSel>>
62 matchAndMask(MachineInstr &MI) const;
63
64 // VOPC SDWA instructions carry the SDWA TSFlag but have no dst_sel operand.
65 bool isSDWAWithDstSel(const MachineInstr &Inst) const;
66
67 void matchSDWAOperands(MachineBasicBlock &MBB);
68 std::unique_ptr<SDWAOperand> matchSDWAOperand(MachineInstr &MI);
69 void pseudoOpConvertToVOP2(MachineInstr &MI,
70 const GCNSubtarget &ST) const;
71 void convertVcndmaskToVOP2(MachineInstr &MI, const GCNSubtarget &ST) const;
72 MachineInstr *createSDWAVersion(MachineInstr &MI);
73 bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands);
74 void legalizeScalarOperands(MachineInstr &MI, const GCNSubtarget &ST) const;
75 bool splitLshlOrForSDWA(MachineBasicBlock &MBB);
76
77public:
78 bool run(MachineFunction &MF);
79};
80
81class SIPeepholeSDWALegacy : public MachineFunctionPass {
82public:
83 static char ID;
84
85 SIPeepholeSDWALegacy() : MachineFunctionPass(ID) {}
86
87 StringRef getPassName() const override { return "SI Peephole SDWA"; }
88
89 bool runOnMachineFunction(MachineFunction &MF) override;
90
91 void getAnalysisUsage(AnalysisUsage &AU) const override {
92 AU.setPreservesCFG();
93 MachineFunctionPass::getAnalysisUsage(AU);
94 }
95};
96
97using namespace AMDGPU::SDWA;
98
99class SDWAOperand {
100private:
101 MachineOperand *Target; // Operand that would be used in converted instruction
102 MachineOperand *Replaced; // Operand that would be replace by Target
103
104 /// Returns true iff the SDWA selection of this SDWAOperand can be combined
105 /// with the SDWA selections of its uses in \p MI.
106 virtual bool canCombineSelections(const MachineInstr &MI,
107 const SIInstrInfo *TII) = 0;
108
109public:
110 SDWAOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp)
111 : Target(TargetOp), Replaced(ReplacedOp) {
112 assert(Target->isReg());
113 assert(Replaced->isReg());
114 }
115
116 virtual ~SDWAOperand() = default;
117
118 virtual MachineInstr *potentialToConvert(const SIInstrInfo *TII,
119 const GCNSubtarget &ST,
120 SDWAOperandsMap *PotentialMatches = nullptr) = 0;
121 virtual bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) = 0;
122
123 MachineOperand *getTargetOperand() const { return Target; }
124 MachineOperand *getReplacedOperand() const { return Replaced; }
125 MachineInstr *getParentInst() const { return Target->getParent(); }
126
127 MachineRegisterInfo *getMRI() const {
128 return &getParentInst()->getMF()->getRegInfo();
129 }
130
131#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
132 virtual void print(raw_ostream& OS) const = 0;
133 void dump() const { print(dbgs()); }
134#endif
135};
136
137class SDWASrcOperand : public SDWAOperand {
138private:
139 SdwaSel SrcSel;
140 bool Abs;
141 bool Neg;
142 bool Sext;
143
144public:
145 SDWASrcOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
146 SdwaSel SrcSel_ = DWORD, bool Abs_ = false, bool Neg_ = false,
147 bool Sext_ = false)
148 : SDWAOperand(TargetOp, ReplacedOp), SrcSel(SrcSel_), Abs(Abs_),
149 Neg(Neg_), Sext(Sext_) {}
150
151 MachineInstr *potentialToConvert(const SIInstrInfo *TII,
152 const GCNSubtarget &ST,
153 SDWAOperandsMap *PotentialMatches = nullptr) override;
154 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
155 bool canCombineSelections(const MachineInstr &MI,
156 const SIInstrInfo *TII) override;
157
158 SdwaSel getSrcSel() const { return SrcSel; }
159 bool getAbs() const { return Abs; }
160 bool getNeg() const { return Neg; }
161 bool getSext() const { return Sext; }
162
163 uint64_t getSrcMods(uint64_t Mods, SdwaSel ExistingSel) const;
164
165#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
166 void print(raw_ostream& OS) const override;
167#endif
168};
169
170class SDWADstOperand : public SDWAOperand {
171private:
172 SdwaSel DstSel;
173 DstUnused DstUn;
174
175public:
176 SDWADstOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
177 SdwaSel DstSel_ = DWORD, DstUnused DstUn_ = UNUSED_PAD)
178 : SDWAOperand(TargetOp, ReplacedOp), DstSel(DstSel_), DstUn(DstUn_) {}
179
180 MachineInstr *potentialToConvert(const SIInstrInfo *TII,
181 const GCNSubtarget &ST,
182 SDWAOperandsMap *PotentialMatches = nullptr) override;
183 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
184 bool canCombineSelections(const MachineInstr &MI,
185 const SIInstrInfo *TII) override;
186
187 SdwaSel getDstSel() const { return DstSel; }
188 DstUnused getDstUnused() const { return DstUn; }
189
190#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
191 void print(raw_ostream& OS) const override;
192#endif
193};
194
195class SDWADstPreserveOperand : public SDWADstOperand {
196private:
197 MachineOperand *Preserve;
198
199public:
200 SDWADstPreserveOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
201 MachineOperand *PreserveOp, SdwaSel DstSel_ = DWORD)
202 : SDWADstOperand(TargetOp, ReplacedOp, DstSel_, UNUSED_PRESERVE),
203 Preserve(PreserveOp) {}
204
205 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
206 bool canCombineSelections(const MachineInstr &MI,
207 const SIInstrInfo *TII) override;
208
209 MachineOperand *getPreservedOperand() const { return Preserve; }
210
211#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
212 void print(raw_ostream& OS) const override;
213#endif
214};
215
216} // end anonymous namespace
217
218INITIALIZE_PASS(SIPeepholeSDWALegacy, DEBUG_TYPE, "SI Peephole SDWA", false,
219 false)
220
221char SIPeepholeSDWALegacy::ID = 0;
222
223char &llvm::SIPeepholeSDWALegacyID = SIPeepholeSDWALegacy::ID;
224
225#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
226static raw_ostream& operator<<(raw_ostream &OS, SdwaSel Sel) {
227 switch(Sel) {
228 case BYTE_0: OS << "BYTE_0"; break;
229 case BYTE_1: OS << "BYTE_1"; break;
230 case BYTE_2: OS << "BYTE_2"; break;
231 case BYTE_3: OS << "BYTE_3"; break;
232 case WORD_0: OS << "WORD_0"; break;
233 case WORD_1: OS << "WORD_1"; break;
234 case DWORD: OS << "DWORD"; break;
235 }
236 return OS;
237}
238
239static raw_ostream& operator<<(raw_ostream &OS, const DstUnused &Un) {
240 switch(Un) {
241 case UNUSED_PAD: OS << "UNUSED_PAD"; break;
242 case UNUSED_SEXT: OS << "UNUSED_SEXT"; break;
243 case UNUSED_PRESERVE: OS << "UNUSED_PRESERVE"; break;
244 }
245 return OS;
246}
247
248LLVM_DUMP_METHOD
249void SDWASrcOperand::print(raw_ostream& OS) const {
250 OS << "SDWA src: " << *getTargetOperand()
251 << " src_sel:" << getSrcSel()
252 << " abs:" << getAbs() << " neg:" << getNeg()
253 << " sext:" << getSext() << '\n';
254}
255
256LLVM_DUMP_METHOD
257void SDWADstOperand::print(raw_ostream& OS) const {
258 OS << "SDWA dst: " << *getTargetOperand()
259 << " dst_sel:" << getDstSel()
260 << " dst_unused:" << getDstUnused() << '\n';
261}
262
263LLVM_DUMP_METHOD
264void SDWADstPreserveOperand::print(raw_ostream& OS) const {
265 OS << "SDWA preserve dst: " << *getTargetOperand()
266 << " dst_sel:" << getDstSel()
267 << " preserve:" << *getPreservedOperand() << '\n';
268}
269
270#endif
271
272static void copyRegOperand(MachineOperand &To, const MachineOperand &From) {
273 assert(To.isReg() && From.isReg());
274 To.setReg(From.getReg());
275 To.setSubReg(From.getSubReg());
276 To.setIsUndef(From.isUndef());
277 if (To.isUse()) {
278 To.setIsKill(From.isKill());
279 } else {
280 To.setIsDead(From.isDead());
281 }
282}
283
284static bool isSameReg(const MachineOperand &LHS, const MachineOperand &RHS) {
285 return LHS.isReg() &&
286 RHS.isReg() &&
287 LHS.getReg() == RHS.getReg() &&
288 LHS.getSubReg() == RHS.getSubReg();
289}
290
291static MachineOperand *findSingleRegUse(const MachineOperand *Reg,
292 const MachineRegisterInfo *MRI) {
293 if (!Reg->isReg() || !Reg->isDef())
294 return nullptr;
295
296 return MRI->getOneNonDBGUse(RegNo: Reg->getReg());
297}
298
299static MachineOperand *findSingleRegDef(const MachineOperand *Reg,
300 const MachineRegisterInfo *MRI) {
301 if (!Reg->isReg())
302 return nullptr;
303
304 return MRI->getOneDef(Reg: Reg->getReg());
305}
306
307/// Combine an SDWA source instruction's existing source selection \p Sel
308/// with the SDWA selection \p OperandSel of its operand. If the selections
309/// are compatible, return the combined selection, otherwise return a
310/// nullopt.
311/// This applies to src_sel only. Extracting from an already extracted field
312/// is again a single field, but dst_sel writes a field and leaves the rest to
313/// dst_unused, so two of them do not fold into one.
314/// See SDWADstOperand::canCombineSelections.
315/// For example, if we have Sel = BYTE_0 Sel and OperandSel = WORD_1:
316/// BYTE_0 Sel (WORD_1 Sel (%X)) -> BYTE_2 Sel (%X)
317static std::optional<SdwaSel> combineSdwaSrcSel(SdwaSel Sel,
318 SdwaSel OperandSel) {
319 if (Sel == SdwaSel::DWORD)
320 return OperandSel;
321
322 if (OperandSel == SdwaSel::DWORD)
323 return Sel;
324
325 if (Sel == SdwaSel::WORD_1 || Sel == SdwaSel::BYTE_2 ||
326 Sel == SdwaSel::BYTE_3)
327 return {};
328
329 // WORD_0 wholly contains the BYTE_0, BYTE_1 and WORD_0 that Sel can be here.
330 if (OperandSel == SdwaSel::WORD_0)
331 return Sel;
332
333 // Reading BYTE_0 of a BYTE_0 field is that same byte.
334 if (OperandSel == SdwaSel::BYTE_0 && Sel == SdwaSel::BYTE_0)
335 return SdwaSel::BYTE_0;
336
337 if (OperandSel == SdwaSel::WORD_1) {
338 if (Sel == SdwaSel::BYTE_0)
339 return SdwaSel::BYTE_2;
340 if (Sel == SdwaSel::BYTE_1)
341 return SdwaSel::BYTE_3;
342 if (Sel == SdwaSel::WORD_0)
343 return SdwaSel::WORD_1;
344 }
345
346 return {};
347}
348
349uint64_t SDWASrcOperand::getSrcMods(uint64_t Mods, SdwaSel ExistingSel) const {
350 if (Abs || Neg) {
351 assert(!Sext &&
352 "Float and integer src modifiers can't be set simultaneously");
353 Mods |= Abs ? SISrcMods::ABS : 0u;
354 Mods ^= Neg ? SISrcMods::NEG : 0u;
355 } else if (ExistingSel == SdwaSel::DWORD) {
356 // SEXT is a no-op for a DWORD selection, so MI may carry one that means
357 // nothing. The combined selection is the operand's field, so use the
358 // operand's SEXT and drop whatever MI had. When ExistingSel narrows the
359 // field instead, the operand's SEXT bits fall outside the combined
360 // field, so MI's own SEXT is kept as is.
361 Mods &= ~uint64_t(SISrcMods::SEXT);
362 Mods |= Sext ? SISrcMods::SEXT : 0u;
363 }
364
365 return Mods;
366}
367
368MachineInstr *SDWASrcOperand::potentialToConvert(const SIInstrInfo *TII,
369 const GCNSubtarget &ST,
370 SDWAOperandsMap *PotentialMatches) {
371 if (PotentialMatches != nullptr) {
372 // Fill out the map for all uses if all can be converted
373 MachineOperand *Reg = getReplacedOperand();
374 if (!Reg->isReg() || !Reg->isDef())
375 return nullptr;
376
377 for (MachineInstr &UseMI : getMRI()->use_nodbg_instructions(Reg: Reg->getReg()))
378 // Check that all instructions that use Reg can be converted
379 if (!isConvertibleToSDWA(MI&: UseMI, ST, TII) ||
380 !canCombineSelections(MI: UseMI, TII))
381 return nullptr;
382
383 // Now that it's guaranteed all uses are legal, iterate over the uses again
384 // to add them for later conversion.
385 for (MachineOperand &UseMO : getMRI()->use_nodbg_operands(Reg: Reg->getReg())) {
386 // Should not get a subregister here
387 assert(isSameReg(UseMO, *Reg));
388
389 SDWAOperandsMap &potentialMatchesMap = *PotentialMatches;
390 MachineInstr *UseMI = UseMO.getParent();
391 potentialMatchesMap[UseMI].push_back(Elt: this);
392 }
393 return nullptr;
394 }
395
396 // For SDWA src operand potential instruction is one that use register
397 // defined by parent instruction
398 MachineOperand *PotentialMO = findSingleRegUse(Reg: getReplacedOperand(), MRI: getMRI());
399 if (!PotentialMO)
400 return nullptr;
401
402 MachineInstr *Parent = PotentialMO->getParent();
403
404 return canCombineSelections(MI: *Parent, TII) ? Parent : nullptr;
405}
406
407bool SDWASrcOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
408 assert((!Sext || !TII->getSubtarget().zeroesHigh16BitsOfDest(
409 getParentInst()->getOpcode())) &&
410 "Cannot use sign-extension with instruction that zeroes high bits");
411 switch (MI.getOpcode()) {
412 case AMDGPU::V_CVT_F32_FP8_sdwa:
413 case AMDGPU::V_CVT_F32_BF8_sdwa:
414 case AMDGPU::V_CVT_PK_F32_FP8_sdwa:
415 case AMDGPU::V_CVT_PK_F32_BF8_sdwa:
416 // Does not support input modifiers: noabs, noneg, nosext.
417 return false;
418 case AMDGPU::V_CNDMASK_B32_sdwa:
419 // SISrcMods uses the same bitmask for SEXT and NEG modifiers and
420 // hence the compiler can only support one type of modifier for
421 // each SDWA instruction. For V_CNDMASK_B32_sdwa, this is NEG
422 // since its operands get printed using
423 // AMDGPUInstPrinter::printOperandAndFPInputMods which produces
424 // the output intended for NEG if SEXT is set.
425 //
426 // The ISA does actually support both modifiers on most SDWA
427 // instructions.
428 //
429 // FIXME Accept SEXT here after fixing this issue.
430 if (Sext)
431 return false;
432 break;
433 }
434
435 // Find operand in instruction that matches source operand and replace it with
436 // target operand. Set corresponding src_sel
437 bool IsPreserveSrc = false;
438 MachineOperand *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
439 MachineOperand *SrcSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_sel);
440 MachineOperand *SrcMods =
441 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
442 assert(Src && (Src->isReg() || Src->isImm()));
443 if (!isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
444 // If this is not src0 then it could be src1
445 Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
446 SrcSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_sel);
447 SrcMods = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
448
449 if (!Src ||
450 !isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
451 // It's possible this Src is a tied operand for
452 // UNUSED_PRESERVE, in which case we can either
453 // abandon the peephole attempt, or if legal we can
454 // copy the target operand into the tied slot
455 // if the preserve operation will effectively cause the same
456 // result by overwriting the rest of the dst.
457 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
458 MachineOperand *DstUnused =
459 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
460
461 if (Dst &&
462 DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) {
463 // This will work if the tied src is accessing WORD_0, and the dst is
464 // writing WORD_1. Modifiers don't matter because all the bits that
465 // would be impacted are being overwritten by the dst.
466 // Any other case will not work.
467 SdwaSel DstSel = static_cast<SdwaSel>(
468 TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::dst_sel));
469 if (DstSel == AMDGPU::SDWA::SdwaSel::WORD_1 &&
470 getSrcSel() == AMDGPU::SDWA::SdwaSel::WORD_0) {
471 IsPreserveSrc = true;
472 auto DstIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
473 Name: AMDGPU::OpName::vdst);
474 auto TiedIdx = MI.findTiedOperandIdx(OpIdx: DstIdx);
475 Src = &MI.getOperand(i: TiedIdx);
476 SrcSel = nullptr;
477 SrcMods = nullptr;
478 } else {
479 // Not legal to convert this src
480 return false;
481 }
482 }
483 }
484 assert(Src && Src->isReg());
485
486 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa ||
487 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa ||
488 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
489 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
490 !isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
491 // In case of v_mac_f16/32_sdwa this pass can try to apply src operand to
492 // src2. This is not allowed.
493 return false;
494 }
495
496 assert(isSameReg(*Src, *getReplacedOperand()) &&
497 (IsPreserveSrc || (SrcSel && SrcMods)));
498 }
499 copyRegOperand(To&: *Src, From: *getTargetOperand());
500 if (!IsPreserveSrc) {
501 SdwaSel ExistingSel = static_cast<SdwaSel>(SrcSel->getImm());
502 SrcSel->setImm(*combineSdwaSrcSel(Sel: ExistingSel, OperandSel: getSrcSel()));
503 SrcMods->setImm(getSrcMods(Mods: SrcMods->getImm(), ExistingSel));
504 }
505 getTargetOperand()->setIsKill(false);
506 return true;
507}
508
509/// Verify that \p Op is the same register as the operand of the SDWA
510/// instruction \p MI named by \p SrcOpName and that the SDWA
511/// selection \p SrcSelOpName can be combined with the \p OpSel.
512static bool canCombineOpSel(const MachineInstr &MI, const SIInstrInfo *TII,
513 AMDGPU::OpName SrcOpName,
514 AMDGPU::OpName SrcSelOpName, MachineOperand *Op,
515 SdwaSel OpSel) {
516 assert(TII->isSDWA(MI.getOpcode()));
517
518 const MachineOperand *Src = TII->getNamedOperand(MI, OperandName: SrcOpName);
519 if (!Src || !isSameReg(LHS: *Src, RHS: *Op))
520 return true;
521
522 auto SrcSel = static_cast<SdwaSel>(TII->getNamedImmOperand(MI, OperandName: SrcSelOpName));
523 return combineSdwaSrcSel(Sel: SrcSel, OperandSel: OpSel).has_value();
524}
525
526bool SDWASrcOperand::canCombineSelections(const MachineInstr &MI,
527 const SIInstrInfo *TII) {
528 if (!TII->isSDWA(Opcode: MI.getOpcode()))
529 return true;
530
531 using namespace AMDGPU;
532
533 return canCombineOpSel(MI, TII, SrcOpName: OpName::src0, SrcSelOpName: OpName::src0_sel,
534 Op: getReplacedOperand(), OpSel: getSrcSel()) &&
535 canCombineOpSel(MI, TII, SrcOpName: OpName::src1, SrcSelOpName: OpName::src1_sel,
536 Op: getReplacedOperand(), OpSel: getSrcSel());
537}
538
539MachineInstr *SDWADstOperand::potentialToConvert(const SIInstrInfo *TII,
540 const GCNSubtarget &ST,
541 SDWAOperandsMap *PotentialMatches) {
542 // For SDWA dst operand potential instruction is one that defines register
543 // that this operand uses
544 MachineRegisterInfo *MRI = getMRI();
545 MachineInstr *ParentMI = getParentInst();
546
547 MachineOperand *PotentialMO = findSingleRegDef(Reg: getReplacedOperand(), MRI);
548 if (!PotentialMO)
549 return nullptr;
550
551 // Check that ParentMI is the only instruction that uses replaced register
552 for (MachineInstr &UseInst : MRI->use_nodbg_instructions(Reg: PotentialMO->getReg())) {
553 if (&UseInst != ParentMI)
554 return nullptr;
555 }
556
557 MachineInstr *Parent = PotentialMO->getParent();
558 return canCombineSelections(MI: *Parent, TII) ? Parent : nullptr;
559}
560
561bool SDWADstOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
562 // Replace vdst operand in MI with target operand. Set dst_sel and dst_unused
563
564 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa ||
565 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa ||
566 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
567 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
568 getDstSel() != AMDGPU::SDWA::DWORD) {
569 // v_mac_f16/32_sdwa allow dst_sel to be equal only to DWORD
570 return false;
571 }
572
573 MachineOperand *Operand = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
574 assert(Operand &&
575 Operand->isReg() &&
576 isSameReg(*Operand, *getReplacedOperand()));
577 copyRegOperand(To&: *Operand, From: *getTargetOperand());
578 MachineOperand *DstSel= TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_sel);
579 assert(DstSel);
580
581 DstSel->setImm(getDstSel());
582
583 MachineOperand *DstUnused= TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
584 assert(DstUnused);
585 DstUnused->setImm(getDstUnused());
586
587 // Remove original instruction because it would conflict with our new
588 // instruction by register definition
589 getParentInst()->eraseFromParent();
590 return true;
591}
592
593bool SDWADstOperand::canCombineSelections(const MachineInstr &MI,
594 const SIInstrInfo *TII) {
595 if (!TII->isSDWA(Opcode: MI.getOpcode()))
596 return true;
597
598 // Destination selections are never combined, so only fold into an MI that
599 // is not yet writing a sub-field of its destination.
600 return TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::dst_sel) == SdwaSel::DWORD;
601}
602
603bool SDWADstPreserveOperand::convertToSDWA(MachineInstr &MI,
604 const SIInstrInfo *TII) {
605 // MI should be moved right before v_or_b32.
606 // For this we should clear all kill flags on uses of MI src-operands or else
607 // we can encounter problem with use of killed operand.
608 for (MachineOperand &MO : MI.uses()) {
609 if (!MO.isReg())
610 continue;
611 getMRI()->clearKillFlags(Reg: MO.getReg());
612 }
613
614 // Move MI before v_or_b32
615 MI.getParent()->remove(I: &MI);
616 getParentInst()->getParent()->insert(I: getParentInst(), MI: &MI);
617
618 // Add Implicit use of preserved register
619 MachineInstrBuilder MIB(*MI.getMF(), MI);
620 MIB.addReg(RegNo: getPreservedOperand()->getReg(),
621 Flags: RegState::ImplicitKill,
622 SubReg: getPreservedOperand()->getSubReg());
623
624 // Tie dst to implicit use
625 MI.tieOperands(DefIdx: AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdst),
626 UseIdx: MI.getNumOperands() - 1);
627
628 // Convert MI as any other SDWADstOperand and remove v_or_b32
629 return SDWADstOperand::convertToSDWA(MI, TII);
630}
631
632bool SDWADstPreserveOperand::canCombineSelections(const MachineInstr &MI,
633 const SIInstrInfo *TII) {
634 // DstSel came from the dst_sel already on MI, only dst_unused changes here.
635 assert(TII->getNamedImmOperand(MI, AMDGPU::OpName::dst_sel) == getDstSel());
636 return true;
637}
638
639std::optional<int64_t>
640SIPeepholeSDWA::foldToImm(const MachineOperand &Op) const {
641 if (Op.isImm()) {
642 return Op.getImm();
643 }
644
645 // If this is not immediate then it can be copy of immediate value, e.g.:
646 // %1 = S_MOV_B32 255;
647 if (Op.isReg()) {
648 for (const MachineOperand &Def : MRI->def_operands(Reg: Op.getReg())) {
649 if (!isSameReg(LHS: Op, RHS: Def))
650 continue;
651
652 const MachineInstr *DefInst = Def.getParent();
653 if (!TII->isFoldableCopy(MI: *DefInst))
654 return std::nullopt;
655
656 const MachineOperand &Copied = DefInst->getOperand(i: 1);
657 if (!Copied.isImm())
658 return std::nullopt;
659
660 return Copied.getImm();
661 }
662 }
663
664 return std::nullopt;
665}
666
667std::optional<std::pair<MachineOperand *, SdwaSel>>
668SIPeepholeSDWA::matchAndMask(MachineInstr &MI) const {
669 if (MI.getOpcode() != AMDGPU::V_AND_B32_e32 &&
670 MI.getOpcode() != AMDGPU::V_AND_B32_e64)
671 return std::nullopt;
672
673 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
674 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
675 MachineOperand *ValSrc = Src1;
676 std::optional<int64_t> Imm = foldToImm(Op: *Src0);
677 if (!Imm) {
678 Imm = foldToImm(Op: *Src1);
679 ValSrc = Src0;
680 }
681 if (!Imm || (*Imm != 0x0000ffff && *Imm != 0x000000ff))
682 return std::nullopt;
683
684 return std::make_pair(x&: ValSrc, y: *Imm == 0x0000ffff ? WORD_0 : BYTE_0);
685}
686
687bool SIPeepholeSDWA::isSDWAWithDstSel(const MachineInstr &Inst) const {
688 return TII->isSDWA(MI: Inst) &&
689 AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::dst_sel);
690}
691
692std::unique_ptr<SDWAOperand>
693SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
694 unsigned Opcode = MI.getOpcode();
695 switch (Opcode) {
696 case AMDGPU::V_LSHRREV_B32_e32:
697 case AMDGPU::V_ASHRREV_I32_e32:
698 case AMDGPU::V_LSHLREV_B32_e32:
699 case AMDGPU::V_LSHRREV_B32_e64:
700 case AMDGPU::V_ASHRREV_I32_e64:
701 case AMDGPU::V_LSHLREV_B32_e64: {
702 // from: v_lshrrev_b32_e32 v1, 16/24, v0
703 // to SDWA src:v0 src_sel:WORD_1/BYTE_3
704
705 // from: v_ashrrev_i32_e32 v1, 16/24, v0
706 // to SDWA src:v0 src_sel:WORD_1/BYTE_3 sext:1
707
708 // from: v_lshlrev_b32_e32 v1, 16/24, v0
709 // to SDWA dst:v1 dst_sel:WORD_1/BYTE_3 dst_unused:UNUSED_PAD
710 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
711 auto Imm = foldToImm(Op: *Src0);
712 if (!Imm)
713 break;
714
715 if (*Imm != 16 && *Imm != 24)
716 break;
717
718 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
719 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
720 if (!Src1->isReg() || Src1->getReg().isPhysical() ||
721 Dst->getReg().isPhysical())
722 break;
723
724 if (Opcode == AMDGPU::V_LSHLREV_B32_e32 ||
725 Opcode == AMDGPU::V_LSHLREV_B32_e64) {
726 return std::make_unique<SDWADstOperand>(
727 args&: Dst, args&: Src1, args: *Imm == 16 ? WORD_1 : BYTE_3, args: UNUSED_PAD);
728 }
729 return std::make_unique<SDWASrcOperand>(
730 args&: Src1, args&: Dst, args: *Imm == 16 ? WORD_1 : BYTE_3, args: false, args: false,
731 args: Opcode != AMDGPU::V_LSHRREV_B32_e32 &&
732 Opcode != AMDGPU::V_LSHRREV_B32_e64);
733 break;
734 }
735
736 case AMDGPU::V_LSHRREV_B16_e32:
737 case AMDGPU::V_LSHLREV_B16_e32:
738 case AMDGPU::V_LSHRREV_B16_e64:
739 case AMDGPU::V_LSHRREV_B16_opsel_e64:
740 case AMDGPU::V_LSHLREV_B16_opsel_e64:
741 case AMDGPU::V_LSHLREV_B16_e64: {
742 // V_ASHRREV_I16_e32 and V_ASHRREV_I16_e64 are
743 // not included here because they zero-fill the high 16-bits.
744
745 // from: v_lshrrev_b16_e32 v1, 8, v0
746 // to SDWA src:v0 src_sel:BYTE_1
747
748 // from: v_lshlrev_b16_e32 v1, 8, v0
749 // to SDWA dst:v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD
750 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
751 auto Imm = foldToImm(Op: *Src0);
752 if (!Imm || *Imm != 8)
753 break;
754
755 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
756 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
757
758 if (!Src1->isReg() || Src1->getReg().isPhysical() ||
759 Dst->getReg().isPhysical())
760 break;
761
762 if (Opcode == AMDGPU::V_LSHLREV_B16_e32 ||
763 Opcode == AMDGPU::V_LSHLREV_B16_opsel_e64 ||
764 Opcode == AMDGPU::V_LSHLREV_B16_e64)
765 return std::make_unique<SDWADstOperand>(args&: Dst, args&: Src1, args: BYTE_1, args: UNUSED_PAD);
766 return std::make_unique<SDWASrcOperand>(args&: Src1, args&: Dst, args: BYTE_1, args: false, args: false,
767 args: false);
768 break;
769 }
770
771 case AMDGPU::V_BFE_I32_e64:
772 case AMDGPU::V_BFE_U32_e64: {
773 // e.g.:
774 // from: v_bfe_u32 v1, v0, 8, 8
775 // to SDWA src:v0 src_sel:BYTE_1
776
777 // offset | width | src_sel
778 // ------------------------
779 // 0 | 8 | BYTE_0
780 // 0 | 16 | WORD_0
781 // 0 | 32 | DWORD ?
782 // 8 | 8 | BYTE_1
783 // 16 | 8 | BYTE_2
784 // 16 | 16 | WORD_1
785 // 24 | 8 | BYTE_3
786
787 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
788 auto Offset = foldToImm(Op: *Src1);
789 if (!Offset)
790 break;
791
792 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
793 auto Width = foldToImm(Op: *Src2);
794 if (!Width)
795 break;
796
797 SdwaSel SrcSel = DWORD;
798
799 if (*Offset == 0 && *Width == 8)
800 SrcSel = BYTE_0;
801 else if (*Offset == 0 && *Width == 16)
802 SrcSel = WORD_0;
803 else if (*Offset == 0 && *Width == 32)
804 SrcSel = DWORD;
805 else if (*Offset == 8 && *Width == 8)
806 SrcSel = BYTE_1;
807 else if (*Offset == 16 && *Width == 8)
808 SrcSel = BYTE_2;
809 else if (*Offset == 16 && *Width == 16)
810 SrcSel = WORD_1;
811 else if (*Offset == 24 && *Width == 8)
812 SrcSel = BYTE_3;
813 else
814 break;
815
816 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
817 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
818
819 if (!Src0->isReg() || Src0->getReg().isPhysical() ||
820 Dst->getReg().isPhysical())
821 break;
822
823 return std::make_unique<SDWASrcOperand>(
824 args&: Src0, args&: Dst, args&: SrcSel, args: false, args: false, args: Opcode != AMDGPU::V_BFE_U32_e64);
825 }
826
827 case AMDGPU::V_AND_B32_e32:
828 case AMDGPU::V_AND_B32_e64: {
829 // e.g.:
830 // from: v_and_b32_e32 v1, 0x0000ffff/0x000000ff, v0
831 // to SDWA src:v0 src_sel:WORD_0/BYTE_0
832 auto Mask = matchAndMask(MI);
833 if (!Mask)
834 break;
835 MachineOperand *ValSrc = Mask->first;
836
837 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
838
839 if (!ValSrc->isReg() || ValSrc->getReg().isPhysical() ||
840 Dst->getReg().isPhysical())
841 break;
842
843 return std::make_unique<SDWASrcOperand>(args&: ValSrc, args&: Dst, args&: Mask->second);
844 }
845
846 case AMDGPU::V_OR_B32_e32:
847 case AMDGPU::V_OR_B32_e64: {
848 // Patterns for dst_unused:UNUSED_PRESERVE.
849 // e.g., from:
850 // v_add_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD
851 // src1_sel:WORD_1 src2_sel:WORD1
852 // v_add_f16_e32 v3, v1, v2
853 // v_or_b32_e32 v4, v0, v3
854 // to SDWA preserve dst:v4 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE preserve:v3
855
856 // Check if one of operands of v_or_b32 is SDWA instruction
857 using CheckRetType =
858 std::optional<std::pair<MachineOperand *, MachineOperand *>>;
859 auto CheckOROperandsForSDWA =
860 [&](const MachineOperand *Op1, const MachineOperand *Op2) -> CheckRetType {
861 if (!Op1 || !Op1->isReg() || !Op2 || !Op2->isReg())
862 return CheckRetType(std::nullopt);
863
864 MachineOperand *Op1Def = findSingleRegDef(Reg: Op1, MRI);
865 if (!Op1Def)
866 return CheckRetType(std::nullopt);
867
868 MachineInstr *Op1Inst = Op1Def->getParent();
869 if (!isSDWAWithDstSel(Inst: *Op1Inst))
870 return CheckRetType(std::nullopt);
871
872 MachineOperand *Op2Def = findSingleRegDef(Reg: Op2, MRI);
873 if (!Op2Def)
874 return CheckRetType(std::nullopt);
875
876 return CheckRetType(std::pair(Op1Def, Op2Def));
877 };
878
879 MachineOperand *OrSDWA = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
880 MachineOperand *OrOther = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
881 assert(OrSDWA && OrOther);
882 auto Res = CheckOROperandsForSDWA(OrSDWA, OrOther);
883 if (!Res) {
884 OrSDWA = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
885 OrOther = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
886 assert(OrSDWA && OrOther);
887 Res = CheckOROperandsForSDWA(OrSDWA, OrOther);
888 if (!Res)
889 break;
890 }
891
892 MachineOperand *OrSDWADef = Res->first;
893 MachineOperand *OrOtherDef = Res->second;
894 assert(OrSDWADef && OrOtherDef);
895
896 MachineInstr *SDWAInst = OrSDWADef->getParent();
897 MachineInstr *OtherInst = OrOtherDef->getParent();
898
899 // Check that OtherInstr is actually bitwise compatible with SDWAInst = their
900 // destination patterns don't overlap. Compatible instruction can be either
901 // regular instruction with compatible bitness or SDWA instruction with
902 // correct dst_sel
903 // SDWAInst | OtherInst bitness / OtherInst dst_sel
904 // -----------------------------------------------------
905 // DWORD | no / no
906 // WORD_0 | no / BYTE_2/3, WORD_1
907 // WORD_1 | 8/16-bit instructions / BYTE_0/1, WORD_0
908 // BYTE_0 | no / BYTE_1/2/3, WORD_1
909 // BYTE_1 | 8-bit / BYTE_0/2/3, WORD_1
910 // BYTE_2 | 8/16-bit / BYTE_0/1/3. WORD_0
911 // BYTE_3 | 8/16/24-bit / BYTE_0/1/2, WORD_0
912 // E.g. if SDWAInst is v_add_f16_sdwa dst_sel:WORD_1 then v_add_f16 is OK
913 // but v_add_f32 is not.
914
915 // TODO: add support for non-SDWA instructions as OtherInst.
916 // For now this only works with SDWA instructions. For regular instructions
917 // there is no way to determine if the instruction writes only 8/16/24-bit
918 // out of full register size and all registers are at min 32-bit wide.
919 if (!isSDWAWithDstSel(Inst: *OtherInst))
920 break;
921
922 SdwaSel DstSel = static_cast<SdwaSel>(
923 TII->getNamedImmOperand(MI: *SDWAInst, OperandName: AMDGPU::OpName::dst_sel));
924 SdwaSel OtherDstSel = static_cast<SdwaSel>(
925 TII->getNamedImmOperand(MI: *OtherInst, OperandName: AMDGPU::OpName::dst_sel));
926
927 bool DstSelAgree = false;
928 switch (DstSel) {
929 case WORD_0: DstSelAgree = ((OtherDstSel == BYTE_2) ||
930 (OtherDstSel == BYTE_3) ||
931 (OtherDstSel == WORD_1));
932 break;
933 case WORD_1: DstSelAgree = ((OtherDstSel == BYTE_0) ||
934 (OtherDstSel == BYTE_1) ||
935 (OtherDstSel == WORD_0));
936 break;
937 case BYTE_0: DstSelAgree = ((OtherDstSel == BYTE_1) ||
938 (OtherDstSel == BYTE_2) ||
939 (OtherDstSel == BYTE_3) ||
940 (OtherDstSel == WORD_1));
941 break;
942 case BYTE_1: DstSelAgree = ((OtherDstSel == BYTE_0) ||
943 (OtherDstSel == BYTE_2) ||
944 (OtherDstSel == BYTE_3) ||
945 (OtherDstSel == WORD_1));
946 break;
947 case BYTE_2: DstSelAgree = ((OtherDstSel == BYTE_0) ||
948 (OtherDstSel == BYTE_1) ||
949 (OtherDstSel == BYTE_3) ||
950 (OtherDstSel == WORD_0));
951 break;
952 case BYTE_3: DstSelAgree = ((OtherDstSel == BYTE_0) ||
953 (OtherDstSel == BYTE_1) ||
954 (OtherDstSel == BYTE_2) ||
955 (OtherDstSel == WORD_0));
956 break;
957 default: DstSelAgree = false;
958 }
959
960 if (!DstSelAgree)
961 break;
962
963 // Also OtherInst dst_unused should be UNUSED_PAD
964 DstUnused OtherDstUnused = static_cast<DstUnused>(
965 TII->getNamedImmOperand(MI: *OtherInst, OperandName: AMDGPU::OpName::dst_unused));
966 if (OtherDstUnused != DstUnused::UNUSED_PAD)
967 break;
968
969 // Create DstPreserveOperand
970 MachineOperand *OrDst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
971 assert(OrDst && OrDst->isReg());
972
973 return std::make_unique<SDWADstPreserveOperand>(
974 args&: OrDst, args&: OrSDWADef, args&: OrOtherDef, args&: DstSel);
975
976 }
977 }
978
979 return std::unique_ptr<SDWAOperand>(nullptr);
980}
981
982#if !defined(NDEBUG)
983static raw_ostream& operator<<(raw_ostream &OS, const SDWAOperand &Operand) {
984 Operand.print(OS);
985 return OS;
986}
987#endif
988
989void SIPeepholeSDWA::matchSDWAOperands(MachineBasicBlock &MBB) {
990 for (MachineInstr &MI : MBB) {
991 if (auto Operand = matchSDWAOperand(MI)) {
992 LLVM_DEBUG(dbgs() << "Match: " << MI << "To: " << *Operand << '\n');
993 SDWAOperands[&MI] = std::move(Operand);
994 ++NumSDWAPatternsFound;
995 }
996 }
997}
998
999// Convert the V_ADD_CO_U32_e64 into V_ADD_CO_U32_e32. This allows
1000// isConvertibleToSDWA to perform its transformation on V_ADD_CO_U32_e32 into
1001// V_ADD_CO_U32_sdwa.
1002//
1003// We are transforming from a VOP3 into a VOP2 form of the instruction.
1004// %19:vgpr_32 = V_AND_B32_e32 255,
1005// killed %16:vgpr_32, implicit $exec
1006// %47:vgpr_32, %49:sreg_64_xexec = V_ADD_CO_U32_e64
1007// %26.sub0:vreg_64, %19:vgpr_32, implicit $exec
1008// %48:vgpr_32, dead %50:sreg_64_xexec = V_ADDC_U32_e64
1009// %26.sub1:vreg_64, %54:vgpr_32, killed %49:sreg_64_xexec, implicit $exec
1010//
1011// becomes
1012// %47:vgpr_32 = V_ADD_CO_U32_sdwa
1013// 0, %26.sub0:vreg_64, 0, killed %16:vgpr_32, 0, 6, 0, 6, 0,
1014// implicit-def $vcc, implicit $exec
1015// %48:vgpr_32, dead %50:sreg_64_xexec = V_ADDC_U32_e64
1016// %26.sub1:vreg_64, %54:vgpr_32, killed $vcc, implicit $exec
1017void SIPeepholeSDWA::pseudoOpConvertToVOP2(MachineInstr &MI,
1018 const GCNSubtarget &ST) const {
1019 int Opc = MI.getOpcode();
1020 assert((Opc == AMDGPU::V_ADD_CO_U32_e64 || Opc == AMDGPU::V_SUB_CO_U32_e64) &&
1021 "Currently only handles V_ADD_CO_U32_e64 or V_SUB_CO_U32_e64");
1022
1023 // Can the candidate MI be shrunk?
1024 if (!TII->canShrink(MI, MRI: *MRI))
1025 return;
1026 Opc = AMDGPU::getVOPe32(Opcode: Opc);
1027 // Find the related ADD instruction.
1028 const MachineOperand *Sdst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
1029 if (!Sdst)
1030 return;
1031 MachineOperand *NextOp = findSingleRegUse(Reg: Sdst, MRI);
1032 if (!NextOp)
1033 return;
1034 MachineInstr &MISucc = *NextOp->getParent();
1035
1036 // Make sure the carry in/out are subsequently unused.
1037 MachineOperand *CarryIn = TII->getNamedOperand(MI&: MISucc, OperandName: AMDGPU::OpName::src2);
1038 if (!CarryIn)
1039 return;
1040 MachineOperand *CarryOut = TII->getNamedOperand(MI&: MISucc, OperandName: AMDGPU::OpName::sdst);
1041 if (!CarryOut)
1042 return;
1043 if (!MRI->hasOneNonDBGUse(RegNo: CarryIn->getReg()) ||
1044 !MRI->use_nodbg_empty(RegNo: CarryOut->getReg()))
1045 return;
1046 // Make sure VCC or its subregs are dead before MI.
1047 MachineBasicBlock &MBB = *MI.getParent();
1048 if (MISucc.getParent() != &MBB)
1049 return; // Loop depends on MI and MISucc in same MBB.
1050 MachineBasicBlock::LivenessQueryResult Liveness =
1051 MBB.computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: MI, Neighborhood: 25);
1052 if (Liveness != MachineBasicBlock::LQR_Dead)
1053 return;
1054 // Check if VCC is referenced in range of (MI,MISucc].
1055 for (auto I = std::next(x: MI.getIterator()), E = MISucc.getIterator();
1056 I != E; ++I) {
1057 if (I->modifiesRegister(Reg: AMDGPU::VCC, TRI))
1058 return;
1059 }
1060
1061 // Replace MI with V_{SUB|ADD}_I32_e32
1062 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc))
1063 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst))
1064 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
1065 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1))
1066 .setMIFlags(MI.getFlags());
1067
1068 MI.eraseFromParent();
1069
1070 // Since the carry output of MI is now VCC, update its use in MISucc.
1071
1072 MISucc.substituteRegister(FromReg: CarryIn->getReg(), ToReg: TRI->getVCC(), SubIdx: 0, RegInfo: *TRI);
1073}
1074
1075/// Try to convert an \p MI in VOP3 which takes an src2 carry-in
1076/// operand into the corresponding VOP2 form which expects the
1077/// argument in VCC. To this end, add an copy from the carry-in to
1078/// VCC. The conversion will only be applied if \p MI can be shrunk
1079/// to VOP2 and if VCC can be proven to be dead before \p MI.
1080void SIPeepholeSDWA::convertVcndmaskToVOP2(MachineInstr &MI,
1081 const GCNSubtarget &ST) const {
1082 assert(MI.getOpcode() == AMDGPU::V_CNDMASK_B32_e64);
1083
1084 LLVM_DEBUG(dbgs() << "Attempting VOP2 conversion: " << MI);
1085 if (!TII->canShrink(MI, MRI: *MRI)) {
1086 LLVM_DEBUG(dbgs() << "Cannot shrink instruction\n");
1087 return;
1088 }
1089
1090 const MachineOperand &CarryIn =
1091 *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1092 Register CarryReg = CarryIn.getReg();
1093 MachineInstr *CarryDef = MRI->getVRegDef(Reg: CarryReg);
1094 if (!CarryDef) {
1095 LLVM_DEBUG(dbgs() << "Missing carry-in operand definition\n");
1096 return;
1097 }
1098
1099 // Make sure VCC or its subregs are dead before MI.
1100 MCRegister Vcc = TRI->getVCC();
1101 MachineBasicBlock &MBB = *MI.getParent();
1102 MachineBasicBlock::LivenessQueryResult Liveness =
1103 MBB.computeRegisterLiveness(TRI, Reg: Vcc, Before: MI);
1104 if (Liveness != MachineBasicBlock::LQR_Dead) {
1105 LLVM_DEBUG(dbgs() << "VCC not known to be dead before instruction\n");
1106 return;
1107 }
1108
1109 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: Vcc).add(MO: CarryIn);
1110
1111 auto Converted = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(),
1112 MCID: TII->get(Opcode: AMDGPU::getVOPe32(Opcode: MI.getOpcode())))
1113 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst))
1114 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
1115 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1))
1116 .setMIFlags(MI.getFlags());
1117 TII->fixImplicitOperands(MI&: *Converted);
1118 LLVM_DEBUG(dbgs() << "Converted to VOP2: " << *Converted);
1119 (void)Converted;
1120 MI.eraseFromParent();
1121}
1122
1123namespace {
1124bool isConvertibleToSDWA(MachineInstr &MI,
1125 const GCNSubtarget &ST,
1126 const SIInstrInfo* TII) {
1127 // Check if this is already an SDWA instruction
1128 unsigned Opc = MI.getOpcode();
1129 if (TII->isSDWA(Opcode: Opc))
1130 return true;
1131
1132 // Can only be handled after ealier conversion to
1133 // AMDGPU::V_CNDMASK_B32_e32 which is not always possible.
1134 if (Opc == AMDGPU::V_CNDMASK_B32_e64)
1135 return false;
1136
1137 // Check if this instruction has opcode that supports SDWA
1138 if (AMDGPU::getSDWAOp(Opcode: Opc) == -1)
1139 Opc = AMDGPU::getVOPe32(Opcode: Opc);
1140
1141 if (AMDGPU::getSDWAOp(Opcode: Opc) == -1)
1142 return false;
1143
1144 if (!ST.hasSDWAOmod() && TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
1145 return false;
1146
1147 if (TII->isVOPC(Opcode: Opc)) {
1148 if (!ST.hasSDWASdst()) {
1149 const MachineOperand *SDst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
1150 if (SDst && (SDst->getReg() != AMDGPU::VCC &&
1151 SDst->getReg() != AMDGPU::VCC_LO))
1152 return false;
1153 }
1154
1155 if (!ST.hasSDWAOutModsVOPC() &&
1156 (TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) ||
1157 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod)))
1158 return false;
1159
1160 } else if (TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst) ||
1161 !TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)) {
1162 return false;
1163 }
1164
1165 if (!ST.hasSDWAMac() && (Opc == AMDGPU::V_FMAC_F16_e32 ||
1166 Opc == AMDGPU::V_FMAC_F32_e32 ||
1167 Opc == AMDGPU::V_MAC_F16_e32 ||
1168 Opc == AMDGPU::V_MAC_F32_e32))
1169 return false;
1170
1171 // Check if target supports this SDWA opcode
1172 if (TII->pseudoToMCOpcode(Opcode: Opc) == -1 ||
1173 TII->pseudoToMCOpcode(Opcode: AMDGPU::getSDWAOp(Opcode: Opc)) == -1)
1174 return false;
1175
1176 if (MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)) {
1177 if (!Src0->isReg() && !Src0->isImm())
1178 return false;
1179 }
1180
1181 if (MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)) {
1182 if (!Src1->isReg() && !Src1->isImm())
1183 return false;
1184 }
1185
1186 return true;
1187}
1188} // namespace
1189
1190MachineInstr *SIPeepholeSDWA::createSDWAVersion(MachineInstr &MI) {
1191 unsigned Opcode = MI.getOpcode();
1192 assert(!TII->isSDWA(Opcode));
1193
1194 int SDWAOpcode = AMDGPU::getSDWAOp(Opcode);
1195 if (SDWAOpcode == -1)
1196 SDWAOpcode = AMDGPU::getSDWAOp(Opcode: AMDGPU::getVOPe32(Opcode));
1197 assert(SDWAOpcode != -1);
1198
1199 const MCInstrDesc &SDWADesc = TII->get(Opcode: SDWAOpcode);
1200
1201 // Create SDWA version of instruction MI and initialize its operands
1202 MachineInstrBuilder SDWAInst =
1203 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: SDWADesc)
1204 .setMIFlags(MI.getFlags());
1205
1206 // Copy dst, if it is present in original then should also be present in SDWA
1207 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1208 if (Dst) {
1209 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::vdst));
1210 SDWAInst.add(MO: *Dst);
1211 } else if ((Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst))) {
1212 assert(Dst && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst));
1213 SDWAInst.add(MO: *Dst);
1214 } else {
1215 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst));
1216 SDWAInst.addReg(RegNo: TRI->getVCC(), Flags: RegState::Define);
1217 }
1218
1219 // Copy src0, initialize src0_modifiers. All sdwa instructions has src0 and
1220 // src0_modifiers (except for v_nop_sdwa, but it can't get here)
1221 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1222 assert(Src0 && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0) &&
1223 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_modifiers));
1224 if (auto *Mod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers))
1225 SDWAInst.addImm(Val: Mod->getImm());
1226 else
1227 SDWAInst.addImm(Val: 0);
1228 SDWAInst.add(MO: *Src0);
1229
1230 // Copy src1 if present, initialize src1_modifiers.
1231 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1232 if (Src1) {
1233 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1) &&
1234 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_modifiers));
1235 if (auto *Mod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers))
1236 SDWAInst.addImm(Val: Mod->getImm());
1237 else
1238 SDWAInst.addImm(Val: 0);
1239 SDWAInst.add(MO: *Src1);
1240 }
1241
1242 if (SDWAOpcode == AMDGPU::V_FMAC_F16_sdwa ||
1243 SDWAOpcode == AMDGPU::V_FMAC_F32_sdwa ||
1244 SDWAOpcode == AMDGPU::V_MAC_F16_sdwa ||
1245 SDWAOpcode == AMDGPU::V_MAC_F32_sdwa) {
1246 // v_mac_f16/32 has additional src2 operand tied to vdst
1247 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1248 assert(Src2);
1249 SDWAInst.add(MO: *Src2);
1250 }
1251
1252 // Copy clamp if present, initialize otherwise
1253 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::clamp));
1254 MachineOperand *Clamp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
1255 if (Clamp) {
1256 SDWAInst.add(MO: *Clamp);
1257 } else {
1258 SDWAInst.addImm(Val: 0);
1259 }
1260
1261 // Copy omod if present, initialize otherwise if needed
1262 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::omod)) {
1263 MachineOperand *OMod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
1264 if (OMod) {
1265 SDWAInst.add(MO: *OMod);
1266 } else {
1267 SDWAInst.addImm(Val: 0);
1268 }
1269 }
1270
1271 // Initialize SDWA specific operands
1272 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::dst_sel))
1273 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1274
1275 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::dst_unused))
1276 SDWAInst.addImm(Val: AMDGPU::SDWA::DstUnused::UNUSED_PAD);
1277
1278 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_sel));
1279 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1280
1281 if (Src1) {
1282 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_sel));
1283 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1284 }
1285
1286 // Check for a preserved register that needs to be copied.
1287 MachineInstr *Ret = SDWAInst.getInstr();
1288 TII->fixImplicitOperands(MI&: *Ret);
1289 return Ret;
1290}
1291
1292bool SIPeepholeSDWA::convertToSDWA(MachineInstr &MI,
1293 const SDWAOperandsVector &SDWAOperands) {
1294 LLVM_DEBUG(dbgs() << "Convert instruction:" << MI);
1295
1296 MachineInstr *SDWAInst;
1297 if (TII->isSDWA(Opcode: MI.getOpcode())) {
1298 // Clone the instruction to allow revoking changes
1299 // made to MI during the processing of the operands
1300 // if the conversion fails.
1301 SDWAInst = MI.getMF()->CloneMachineInstr(Orig: &MI);
1302 MI.getParent()->insert(I: MI.getIterator(), M: SDWAInst);
1303 } else {
1304 SDWAInst = createSDWAVersion(MI);
1305 }
1306
1307 // Apply all sdwa operand patterns.
1308 bool Converted = false;
1309 for (auto &Operand : SDWAOperands) {
1310 LLVM_DEBUG(dbgs() << *SDWAInst << "\nOperand: " << *Operand);
1311 // There should be no intersection between SDWA operands and potential MIs
1312 // e.g.:
1313 // v_and_b32 v0, 0xff, v1 -> src:v1 sel:BYTE_0
1314 // v_and_b32 v2, 0xff, v0 -> src:v0 sel:BYTE_0
1315 // v_add_u32 v3, v4, v2
1316 //
1317 // In that example it is possible that we would fold 2nd instruction into
1318 // 3rd (v_add_u32_sdwa) and then try to fold 1st instruction into 2nd (that
1319 // was already destroyed). So if SDWAOperand is also a potential MI then do
1320 // not apply it.
1321 if (PotentialMatches.count(Key: Operand->getParentInst()) == 0)
1322 Converted |= Operand->convertToSDWA(MI&: *SDWAInst, TII);
1323 }
1324
1325 if (!Converted) {
1326 SDWAInst->eraseFromParent();
1327 return false;
1328 }
1329
1330 ConvertedInstructions.push_back(Elt: SDWAInst);
1331 for (MachineOperand &MO : SDWAInst->uses()) {
1332 if (!MO.isReg())
1333 continue;
1334
1335 MRI->clearKillFlags(Reg: MO.getReg());
1336 }
1337 LLVM_DEBUG(dbgs() << "\nInto:" << *SDWAInst << '\n');
1338 ++NumSDWAInstructionsPeepholed;
1339
1340 MI.eraseFromParent();
1341 return true;
1342}
1343
1344// If an instruction was converted to SDWA it should not have immediates or SGPR
1345// operands (allowed one SGPR on GFX9). Copy its scalar operands into VGPRs.
1346void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI,
1347 const GCNSubtarget &ST) const {
1348 const MCInstrDesc &Desc = TII->get(Opcode: MI.getOpcode());
1349 unsigned ConstantBusCount = 0;
1350 for (MachineOperand &Op : MI.explicit_uses()) {
1351 if (Op.isReg()) {
1352 if (TRI->isVGPR(MRI: *MRI, Reg: Op.getReg()))
1353 continue;
1354
1355 if (ST.hasSDWAScalar() && ConstantBusCount == 0) {
1356 ++ConstantBusCount;
1357 continue;
1358 }
1359 } else if (!Op.isImm())
1360 continue;
1361
1362 unsigned I = Op.getOperandNo();
1363 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: Desc, OpNum: I);
1364 if (!OpRC || !TRI->isVSSuperClass(RC: OpRC))
1365 continue;
1366
1367 Register VGPR = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1368 auto Copy = BuildMI(BB&: *MI.getParent(), I: MI.getIterator(), MIMD: MI.getDebugLoc(),
1369 MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VGPR);
1370 if (Op.isImm())
1371 Copy.addImm(Val: Op.getImm());
1372 else if (Op.isReg())
1373 Copy.addReg(RegNo: Op.getReg(), Flags: getKillRegState(B: Op.isKill()), SubReg: Op.getSubReg());
1374 Op.ChangeToRegister(Reg: VGPR, isDef: false);
1375 }
1376}
1377
1378// Re-fold the masked high-half pack (hi << 16) | (z & 0xffff) into a single
1379// v_or_b32_sdwa src1_sel:WORD_0, which ISel's fused v_lshl_or_b32 blocks.
1380bool SIPeepholeSDWA::splitLshlOrForSDWA(MachineBasicBlock &MBB) {
1381 struct Candidate {
1382 MachineInstr *LshlOr;
1383 MachineInstr *AndMI;
1384 MachineOperand *Hi;
1385 MachineOperand *ValSrc;
1386 };
1387 SmallVector<Candidate, 4> Candidates;
1388
1389 for (MachineInstr &MI : MBB) {
1390 if (MI.getOpcode() != AMDGPU::V_LSHL_OR_B32_e64)
1391 continue;
1392
1393 MachineOperand *Shift = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1394 std::optional<int64_t> ShiftImm = foldToImm(Op: *Shift);
1395 if (!ShiftImm || *ShiftImm != 16)
1396 continue;
1397
1398 MachineOperand *Hi = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1399 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1400 // Src2 must be a virtual reg so getVRegDef below is valid.
1401 if (!Hi->isReg() || !Src2->isReg() || !Src2->getReg().isVirtual())
1402 continue;
1403
1404 // The 0xffff mask must come from a single-use v_and so it can be dropped.
1405 if (!MRI->hasOneNonDBGUse(RegNo: Src2->getReg()))
1406 continue;
1407 MachineInstr *AndMI = MRI->getVRegDef(Reg: Src2->getReg());
1408 if (!AndMI)
1409 continue;
1410 std::optional<std::pair<MachineOperand *, SdwaSel>> Mask =
1411 matchAndMask(MI&: *AndMI);
1412 if (!Mask || Mask->second != WORD_0)
1413 continue;
1414 MachineOperand *ValSrc = Mask->first;
1415 if (!ValSrc->isReg() || !TRI->isVGPR(MRI: *MRI, Reg: ValSrc->getReg()))
1416 continue;
1417
1418 Candidates.push_back(Elt: {.LshlOr: &MI, .AndMI: AndMI, .Hi: Hi, .ValSrc: ValSrc});
1419 }
1420
1421 for (const Candidate &C : Candidates) {
1422 MachineOperand *Dst = TII->getNamedOperand(MI&: *C.LshlOr, OperandName: AMDGPU::OpName::vdst);
1423
1424 Register ShiftReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1425 BuildMI(BB&: *C.LshlOr->getParent(), I&: *C.LshlOr, MIMD: C.LshlOr->getDebugLoc(),
1426 MCID: TII->get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftReg)
1427 .addImm(Val: 16)
1428 .add(MO: *C.Hi);
1429
1430 // vdst, src0_mods, src0, src1_mods, src1, clamp, dst_sel, dst_unused,
1431 // src0_sel, src1_sel.
1432 BuildMI(BB&: *C.LshlOr->getParent(), I&: *C.LshlOr, MIMD: C.LshlOr->getDebugLoc(),
1433 MCID: TII->get(Opcode: AMDGPU::V_OR_B32_sdwa))
1434 .add(MO: *Dst)
1435 .addImm(Val: 0)
1436 .addReg(RegNo: ShiftReg)
1437 .addImm(Val: 0)
1438 .add(MO: *C.ValSrc)
1439 .addImm(Val: 0)
1440 .addImm(Val: DWORD)
1441 .addImm(Val: UNUSED_PAD)
1442 .addImm(Val: DWORD)
1443 .addImm(Val: WORD_0);
1444
1445 MRI->clearKillFlags(Reg: C.ValSrc->getReg());
1446 C.LshlOr->eraseFromParent();
1447 C.AndMI->eraseFromParent();
1448 }
1449
1450 return !Candidates.empty();
1451}
1452
1453bool SIPeepholeSDWALegacy::runOnMachineFunction(MachineFunction &MF) {
1454 if (skipFunction(F: MF.getFunction()))
1455 return false;
1456
1457 return SIPeepholeSDWA().run(MF);
1458}
1459
1460bool SIPeepholeSDWA::run(MachineFunction &MF) {
1461 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1462
1463 if (!ST.hasSDWA())
1464 return false;
1465
1466 MRI = &MF.getRegInfo();
1467 TRI = ST.getRegisterInfo();
1468 TII = ST.getInstrInfo();
1469
1470 // Find all SDWA operands in MF.
1471 bool Ret = false;
1472 for (MachineBasicBlock &MBB : MF) {
1473 bool Changed = false;
1474 do {
1475 Ret |= splitLshlOrForSDWA(MBB);
1476
1477 // Preprocess the ADD/SUB pairs so they could be SDWA'ed.
1478 // Look for a possible ADD or SUB that resulted from a previously lowered
1479 // V_{ADD|SUB}_U64_PSEUDO. The function pseudoOpConvertToVOP2
1480 // lowers the pair of instructions into e32 form.
1481 matchSDWAOperands(MBB);
1482 for (const auto &OperandPair : SDWAOperands) {
1483 const auto &Operand = OperandPair.second;
1484 MachineInstr *PotentialMI = Operand->potentialToConvert(TII, ST);
1485 if (!PotentialMI)
1486 continue;
1487
1488 switch (PotentialMI->getOpcode()) {
1489 case AMDGPU::V_ADD_CO_U32_e64:
1490 case AMDGPU::V_SUB_CO_U32_e64:
1491 pseudoOpConvertToVOP2(MI&: *PotentialMI, ST);
1492 break;
1493 case AMDGPU::V_CNDMASK_B32_e64:
1494 convertVcndmaskToVOP2(MI&: *PotentialMI, ST);
1495 break;
1496 };
1497 }
1498 SDWAOperands.clear();
1499
1500 // Generate potential match list.
1501 matchSDWAOperands(MBB);
1502
1503 for (const auto &OperandPair : SDWAOperands) {
1504 const auto &Operand = OperandPair.second;
1505 MachineInstr *PotentialMI =
1506 Operand->potentialToConvert(TII, ST, PotentialMatches: &PotentialMatches);
1507
1508 if (PotentialMI && isConvertibleToSDWA(MI&: *PotentialMI, ST, TII))
1509 PotentialMatches[PotentialMI].push_back(Elt: Operand.get());
1510 }
1511
1512 for (auto &PotentialPair : PotentialMatches) {
1513 MachineInstr &PotentialMI = *PotentialPair.first;
1514 convertToSDWA(MI&: PotentialMI, SDWAOperands: PotentialPair.second);
1515 }
1516
1517 PotentialMatches.clear();
1518 SDWAOperands.clear();
1519
1520 Changed = !ConvertedInstructions.empty();
1521
1522 if (Changed)
1523 Ret = true;
1524 while (!ConvertedInstructions.empty())
1525 legalizeScalarOperands(MI&: *ConvertedInstructions.pop_back_val(), ST);
1526 } while (Changed);
1527 }
1528
1529 return Ret;
1530}
1531
1532PreservedAnalyses SIPeepholeSDWAPass::run(MachineFunction &MF,
1533 MachineFunctionAnalysisManager &) {
1534 if (MF.getFunction().hasOptNone() || !SIPeepholeSDWA().run(MF))
1535 return PreservedAnalyses::all();
1536
1537 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1538 PA.preserveSet<CFGAnalyses>();
1539 return PA;
1540}
1541