1//===- SIPeepholeSDWA.cpp - Peephole optimization for SDWA instructions ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass tries to apply several peephole SDWA patterns.
10///
11/// E.g. original:
12/// V_LSHRREV_B32_e32 %0, 16, %1
13/// V_ADD_CO_U32_e32 %2, %0, %3
14/// V_LSHLREV_B32_e32 %4, 16, %2
15///
16/// Replace:
17/// V_ADD_CO_U32_sdwa %4, %1, %3
18/// dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
19///
20//===----------------------------------------------------------------------===//
21
22#include "SIPeepholeSDWA.h"
23#include "AMDGPU.h"
24#include "GCNSubtarget.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/CodeGen/MachineFunctionPass.h"
27#include <optional>
28
29using namespace llvm;
30
31#define DEBUG_TYPE "si-peephole-sdwa"
32
33STATISTIC(NumSDWAPatternsFound, "Number of SDWA patterns found.");
34STATISTIC(NumSDWAInstructionsPeepholed,
35 "Number of instruction converted to SDWA.");
36
37namespace {
38
39bool isConvertibleToSDWA(MachineInstr &MI, const GCNSubtarget &ST,
40 const SIInstrInfo *TII);
41class SDWAOperand;
42class SDWADstOperand;
43
44using SDWAOperandsVector = SmallVector<SDWAOperand *, 4>;
45using SDWAOperandsMap = MapVector<MachineInstr *, SDWAOperandsVector>;
46
47class SIPeepholeSDWA {
48private:
49 MachineRegisterInfo *MRI;
50 const SIRegisterInfo *TRI;
51 const SIInstrInfo *TII;
52
53 MapVector<MachineInstr *, std::unique_ptr<SDWAOperand>> SDWAOperands;
54 SDWAOperandsMap PotentialMatches;
55 SmallVector<MachineInstr *, 8> ConvertedInstructions;
56
57 std::optional<int64_t> foldToImm(const MachineOperand &Op) const;
58
59 // If MI is a v_and_b32 with a 0xffff or 0xff immediate, return the masked
60 // value operand and the matching SDWA selector (WORD_0 / BYTE_0).
61 std::optional<std::pair<MachineOperand *, AMDGPU::SDWA::SdwaSel>>
62 matchAndMask(MachineInstr &MI) const;
63
64 // VOPC SDWA instructions carry the SDWA TSFlag but have no dst_sel operand.
65 bool isSDWAWithDstSel(const MachineInstr &Inst) const;
66
67 void matchSDWAOperands(MachineBasicBlock &MBB);
68 std::unique_ptr<SDWAOperand> matchSDWAOperand(MachineInstr &MI);
69 void pseudoOpConvertToVOP2(MachineInstr &MI,
70 const GCNSubtarget &ST) const;
71 void convertVcndmaskToVOP2(MachineInstr &MI, const GCNSubtarget &ST) const;
72 MachineInstr *createSDWAVersion(MachineInstr &MI);
73 bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands);
74 void legalizeScalarOperands(MachineInstr &MI, const GCNSubtarget &ST) const;
75 bool splitLshlOrForSDWA(MachineBasicBlock &MBB);
76
77public:
78 bool run(MachineFunction &MF);
79};
80
81class SIPeepholeSDWALegacy : public MachineFunctionPass {
82public:
83 static char ID;
84
85 SIPeepholeSDWALegacy() : MachineFunctionPass(ID) {}
86
87 StringRef getPassName() const override { return "SI Peephole SDWA"; }
88
89 bool runOnMachineFunction(MachineFunction &MF) override;
90
91 void getAnalysisUsage(AnalysisUsage &AU) const override {
92 AU.setPreservesCFG();
93 MachineFunctionPass::getAnalysisUsage(AU);
94 }
95};
96
97using namespace AMDGPU::SDWA;
98
99class SDWAOperand {
100private:
101 MachineOperand *Target; // Operand that would be used in converted instruction
102 MachineOperand *Replaced; // Operand that would be replace by Target
103
104 /// Returns true iff the SDWA selection of this SDWAOperand can be combined
105 /// with the SDWA selections of its uses in \p MI.
106 virtual bool canCombineSelections(const MachineInstr &MI,
107 const SIInstrInfo *TII) = 0;
108
109public:
110 SDWAOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp)
111 : Target(TargetOp), Replaced(ReplacedOp) {
112 assert(Target->isReg());
113 assert(Replaced->isReg());
114 }
115
116 virtual ~SDWAOperand() = default;
117
118 virtual MachineInstr *potentialToConvert(const SIInstrInfo *TII,
119 const GCNSubtarget &ST,
120 SDWAOperandsMap *PotentialMatches = nullptr) = 0;
121 virtual bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) = 0;
122
123 MachineOperand *getTargetOperand() const { return Target; }
124 MachineOperand *getReplacedOperand() const { return Replaced; }
125 MachineInstr *getParentInst() const { return Target->getParent(); }
126
127 MachineRegisterInfo *getMRI() const {
128 return &getParentInst()->getMF()->getRegInfo();
129 }
130
131#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
132 virtual void print(raw_ostream& OS) const = 0;
133 void dump() const { print(dbgs()); }
134#endif
135};
136
137class SDWASrcOperand : public SDWAOperand {
138private:
139 SdwaSel SrcSel;
140 bool Abs;
141 bool Neg;
142 bool Sext;
143
144public:
145 SDWASrcOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
146 SdwaSel SrcSel_ = DWORD, bool Abs_ = false, bool Neg_ = false,
147 bool Sext_ = false)
148 : SDWAOperand(TargetOp, ReplacedOp), SrcSel(SrcSel_), Abs(Abs_),
149 Neg(Neg_), Sext(Sext_) {}
150
151 MachineInstr *potentialToConvert(const SIInstrInfo *TII,
152 const GCNSubtarget &ST,
153 SDWAOperandsMap *PotentialMatches = nullptr) override;
154 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
155 bool canCombineSelections(const MachineInstr &MI,
156 const SIInstrInfo *TII) override;
157
158 SdwaSel getSrcSel() const { return SrcSel; }
159 bool getAbs() const { return Abs; }
160 bool getNeg() const { return Neg; }
161 bool getSext() const { return Sext; }
162
163 uint64_t getSrcMods(const SIInstrInfo *TII,
164 const MachineOperand *SrcOp) const;
165
166#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
167 void print(raw_ostream& OS) const override;
168#endif
169};
170
171class SDWADstOperand : public SDWAOperand {
172private:
173 SdwaSel DstSel;
174 DstUnused DstUn;
175
176public:
177 SDWADstOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
178 SdwaSel DstSel_ = DWORD, DstUnused DstUn_ = UNUSED_PAD)
179 : SDWAOperand(TargetOp, ReplacedOp), DstSel(DstSel_), DstUn(DstUn_) {}
180
181 MachineInstr *potentialToConvert(const SIInstrInfo *TII,
182 const GCNSubtarget &ST,
183 SDWAOperandsMap *PotentialMatches = nullptr) override;
184 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
185 bool canCombineSelections(const MachineInstr &MI,
186 const SIInstrInfo *TII) override;
187
188 SdwaSel getDstSel() const { return DstSel; }
189 DstUnused getDstUnused() const { return DstUn; }
190
191#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
192 void print(raw_ostream& OS) const override;
193#endif
194};
195
196class SDWADstPreserveOperand : public SDWADstOperand {
197private:
198 MachineOperand *Preserve;
199
200public:
201 SDWADstPreserveOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
202 MachineOperand *PreserveOp, SdwaSel DstSel_ = DWORD)
203 : SDWADstOperand(TargetOp, ReplacedOp, DstSel_, UNUSED_PRESERVE),
204 Preserve(PreserveOp) {}
205
206 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
207 bool canCombineSelections(const MachineInstr &MI,
208 const SIInstrInfo *TII) override;
209
210 MachineOperand *getPreservedOperand() const { return Preserve; }
211
212#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
213 void print(raw_ostream& OS) const override;
214#endif
215};
216
217} // end anonymous namespace
218
219INITIALIZE_PASS(SIPeepholeSDWALegacy, DEBUG_TYPE, "SI Peephole SDWA", false,
220 false)
221
222char SIPeepholeSDWALegacy::ID = 0;
223
224char &llvm::SIPeepholeSDWALegacyID = SIPeepholeSDWALegacy::ID;
225
226FunctionPass *llvm::createSIPeepholeSDWALegacyPass() {
227 return new SIPeepholeSDWALegacy();
228}
229
230#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
231static raw_ostream& operator<<(raw_ostream &OS, SdwaSel Sel) {
232 switch(Sel) {
233 case BYTE_0: OS << "BYTE_0"; break;
234 case BYTE_1: OS << "BYTE_1"; break;
235 case BYTE_2: OS << "BYTE_2"; break;
236 case BYTE_3: OS << "BYTE_3"; break;
237 case WORD_0: OS << "WORD_0"; break;
238 case WORD_1: OS << "WORD_1"; break;
239 case DWORD: OS << "DWORD"; break;
240 }
241 return OS;
242}
243
244static raw_ostream& operator<<(raw_ostream &OS, const DstUnused &Un) {
245 switch(Un) {
246 case UNUSED_PAD: OS << "UNUSED_PAD"; break;
247 case UNUSED_SEXT: OS << "UNUSED_SEXT"; break;
248 case UNUSED_PRESERVE: OS << "UNUSED_PRESERVE"; break;
249 }
250 return OS;
251}
252
253LLVM_DUMP_METHOD
254void SDWASrcOperand::print(raw_ostream& OS) const {
255 OS << "SDWA src: " << *getTargetOperand()
256 << " src_sel:" << getSrcSel()
257 << " abs:" << getAbs() << " neg:" << getNeg()
258 << " sext:" << getSext() << '\n';
259}
260
261LLVM_DUMP_METHOD
262void SDWADstOperand::print(raw_ostream& OS) const {
263 OS << "SDWA dst: " << *getTargetOperand()
264 << " dst_sel:" << getDstSel()
265 << " dst_unused:" << getDstUnused() << '\n';
266}
267
268LLVM_DUMP_METHOD
269void SDWADstPreserveOperand::print(raw_ostream& OS) const {
270 OS << "SDWA preserve dst: " << *getTargetOperand()
271 << " dst_sel:" << getDstSel()
272 << " preserve:" << *getPreservedOperand() << '\n';
273}
274
275#endif
276
277static void copyRegOperand(MachineOperand &To, const MachineOperand &From) {
278 assert(To.isReg() && From.isReg());
279 To.setReg(From.getReg());
280 To.setSubReg(From.getSubReg());
281 To.setIsUndef(From.isUndef());
282 if (To.isUse()) {
283 To.setIsKill(From.isKill());
284 } else {
285 To.setIsDead(From.isDead());
286 }
287}
288
289static bool isSameReg(const MachineOperand &LHS, const MachineOperand &RHS) {
290 return LHS.isReg() &&
291 RHS.isReg() &&
292 LHS.getReg() == RHS.getReg() &&
293 LHS.getSubReg() == RHS.getSubReg();
294}
295
296static MachineOperand *findSingleRegUse(const MachineOperand *Reg,
297 const MachineRegisterInfo *MRI) {
298 if (!Reg->isReg() || !Reg->isDef())
299 return nullptr;
300
301 return MRI->getOneNonDBGUse(RegNo: Reg->getReg());
302}
303
304static MachineOperand *findSingleRegDef(const MachineOperand *Reg,
305 const MachineRegisterInfo *MRI) {
306 if (!Reg->isReg())
307 return nullptr;
308
309 return MRI->getOneDef(Reg: Reg->getReg());
310}
311
312/// Combine an SDWA instruction's existing SDWA selection \p Sel with
313/// the SDWA selection \p OperandSel of its operand. If the selections
314/// are compatible, return the combined selection, otherwise return a
315/// nullopt.
316/// For example, if we have Sel = BYTE_0 Sel and OperandSel = WORD_1:
317/// BYTE_0 Sel (WORD_1 Sel (%X)) -> BYTE_2 Sel (%X)
318static std::optional<SdwaSel> combineSdwaSel(SdwaSel Sel, SdwaSel OperandSel) {
319 if (Sel == SdwaSel::DWORD)
320 return OperandSel;
321
322 if (Sel == OperandSel || OperandSel == SdwaSel::DWORD)
323 return Sel;
324
325 if (Sel == SdwaSel::WORD_1 || Sel == SdwaSel::BYTE_2 ||
326 Sel == SdwaSel::BYTE_3)
327 return {};
328
329 if (OperandSel == SdwaSel::WORD_0)
330 return Sel;
331
332 if (OperandSel == SdwaSel::WORD_1) {
333 if (Sel == SdwaSel::BYTE_0)
334 return SdwaSel::BYTE_2;
335 if (Sel == SdwaSel::BYTE_1)
336 return SdwaSel::BYTE_3;
337 if (Sel == SdwaSel::WORD_0)
338 return SdwaSel::WORD_1;
339 }
340
341 return {};
342}
343
344uint64_t SDWASrcOperand::getSrcMods(const SIInstrInfo *TII,
345 const MachineOperand *SrcOp) const {
346 uint64_t Mods = 0;
347 const auto *MI = SrcOp->getParent();
348 if (TII->getNamedOperand(MI: *MI, OperandName: AMDGPU::OpName::src0) == SrcOp) {
349 if (auto *Mod = TII->getNamedOperand(MI: *MI, OperandName: AMDGPU::OpName::src0_modifiers)) {
350 Mods = Mod->getImm();
351 }
352 } else if (TII->getNamedOperand(MI: *MI, OperandName: AMDGPU::OpName::src1) == SrcOp) {
353 if (auto *Mod = TII->getNamedOperand(MI: *MI, OperandName: AMDGPU::OpName::src1_modifiers)) {
354 Mods = Mod->getImm();
355 }
356 }
357 if (Abs || Neg) {
358 assert(!Sext &&
359 "Float and integer src modifiers can't be set simultaneously");
360 Mods |= Abs ? SISrcMods::ABS : 0u;
361 Mods ^= Neg ? SISrcMods::NEG : 0u;
362 } else if (Sext) {
363 Mods |= SISrcMods::SEXT;
364 }
365
366 return Mods;
367}
368
369MachineInstr *SDWASrcOperand::potentialToConvert(const SIInstrInfo *TII,
370 const GCNSubtarget &ST,
371 SDWAOperandsMap *PotentialMatches) {
372 if (PotentialMatches != nullptr) {
373 // Fill out the map for all uses if all can be converted
374 MachineOperand *Reg = getReplacedOperand();
375 if (!Reg->isReg() || !Reg->isDef())
376 return nullptr;
377
378 for (MachineInstr &UseMI : getMRI()->use_nodbg_instructions(Reg: Reg->getReg()))
379 // Check that all instructions that use Reg can be converted
380 if (!isConvertibleToSDWA(MI&: UseMI, ST, TII) ||
381 !canCombineSelections(MI: UseMI, TII))
382 return nullptr;
383
384 // Now that it's guaranteed all uses are legal, iterate over the uses again
385 // to add them for later conversion.
386 for (MachineOperand &UseMO : getMRI()->use_nodbg_operands(Reg: Reg->getReg())) {
387 // Should not get a subregister here
388 assert(isSameReg(UseMO, *Reg));
389
390 SDWAOperandsMap &potentialMatchesMap = *PotentialMatches;
391 MachineInstr *UseMI = UseMO.getParent();
392 potentialMatchesMap[UseMI].push_back(Elt: this);
393 }
394 return nullptr;
395 }
396
397 // For SDWA src operand potential instruction is one that use register
398 // defined by parent instruction
399 MachineOperand *PotentialMO = findSingleRegUse(Reg: getReplacedOperand(), MRI: getMRI());
400 if (!PotentialMO)
401 return nullptr;
402
403 MachineInstr *Parent = PotentialMO->getParent();
404
405 return canCombineSelections(MI: *Parent, TII) ? Parent : nullptr;
406}
407
408bool SDWASrcOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
409 assert((!Sext || !TII->getSubtarget().zeroesHigh16BitsOfDest(
410 getParentInst()->getOpcode())) &&
411 "Cannot use sign-extension with instruction that zeroes high bits");
412 switch (MI.getOpcode()) {
413 case AMDGPU::V_CVT_F32_FP8_sdwa:
414 case AMDGPU::V_CVT_F32_BF8_sdwa:
415 case AMDGPU::V_CVT_PK_F32_FP8_sdwa:
416 case AMDGPU::V_CVT_PK_F32_BF8_sdwa:
417 // Does not support input modifiers: noabs, noneg, nosext.
418 return false;
419 case AMDGPU::V_CNDMASK_B32_sdwa:
420 // SISrcMods uses the same bitmask for SEXT and NEG modifiers and
421 // hence the compiler can only support one type of modifier for
422 // each SDWA instruction. For V_CNDMASK_B32_sdwa, this is NEG
423 // since its operands get printed using
424 // AMDGPUInstPrinter::printOperandAndFPInputMods which produces
425 // the output intended for NEG if SEXT is set.
426 //
427 // The ISA does actually support both modifiers on most SDWA
428 // instructions.
429 //
430 // FIXME Accept SEXT here after fixing this issue.
431 if (Sext)
432 return false;
433 break;
434 }
435
436 // Find operand in instruction that matches source operand and replace it with
437 // target operand. Set corresponding src_sel
438 bool IsPreserveSrc = false;
439 MachineOperand *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
440 MachineOperand *SrcSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_sel);
441 MachineOperand *SrcMods =
442 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
443 assert(Src && (Src->isReg() || Src->isImm()));
444 if (!isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
445 // If this is not src0 then it could be src1
446 Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
447 SrcSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_sel);
448 SrcMods = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
449
450 if (!Src ||
451 !isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
452 // It's possible this Src is a tied operand for
453 // UNUSED_PRESERVE, in which case we can either
454 // abandon the peephole attempt, or if legal we can
455 // copy the target operand into the tied slot
456 // if the preserve operation will effectively cause the same
457 // result by overwriting the rest of the dst.
458 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
459 MachineOperand *DstUnused =
460 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
461
462 if (Dst &&
463 DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) {
464 // This will work if the tied src is accessing WORD_0, and the dst is
465 // writing WORD_1. Modifiers don't matter because all the bits that
466 // would be impacted are being overwritten by the dst.
467 // Any other case will not work.
468 SdwaSel DstSel = static_cast<SdwaSel>(
469 TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::dst_sel));
470 if (DstSel == AMDGPU::SDWA::SdwaSel::WORD_1 &&
471 getSrcSel() == AMDGPU::SDWA::SdwaSel::WORD_0) {
472 IsPreserveSrc = true;
473 auto DstIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
474 Name: AMDGPU::OpName::vdst);
475 auto TiedIdx = MI.findTiedOperandIdx(OpIdx: DstIdx);
476 Src = &MI.getOperand(i: TiedIdx);
477 SrcSel = nullptr;
478 SrcMods = nullptr;
479 } else {
480 // Not legal to convert this src
481 return false;
482 }
483 }
484 }
485 assert(Src && Src->isReg());
486
487 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa ||
488 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa ||
489 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
490 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
491 !isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
492 // In case of v_mac_f16/32_sdwa this pass can try to apply src operand to
493 // src2. This is not allowed.
494 return false;
495 }
496
497 assert(isSameReg(*Src, *getReplacedOperand()) &&
498 (IsPreserveSrc || (SrcSel && SrcMods)));
499 }
500 copyRegOperand(To&: *Src, From: *getTargetOperand());
501 if (!IsPreserveSrc) {
502 SdwaSel ExistingSel = static_cast<SdwaSel>(SrcSel->getImm());
503 SrcSel->setImm(*combineSdwaSel(Sel: ExistingSel, OperandSel: getSrcSel()));
504 SrcMods->setImm(getSrcMods(TII, SrcOp: Src));
505 }
506 getTargetOperand()->setIsKill(false);
507 return true;
508}
509
510/// Verify that the SDWA selection operand \p SrcSelOpName of the SDWA
511/// instruction \p MI can be combined with the selection \p OpSel.
512static bool canCombineOpSel(const MachineInstr &MI, const SIInstrInfo *TII,
513 AMDGPU::OpName SrcSelOpName, SdwaSel OpSel) {
514 assert(TII->isSDWA(MI.getOpcode()));
515
516 const MachineOperand *SrcSelOp = TII->getNamedOperand(MI, OperandName: SrcSelOpName);
517 SdwaSel SrcSel = static_cast<SdwaSel>(SrcSelOp->getImm());
518
519 return combineSdwaSel(Sel: SrcSel, OperandSel: OpSel).has_value();
520}
521
522/// Verify that \p Op is the same register as the operand of the SDWA
523/// instruction \p MI named by \p SrcOpName and that the SDWA
524/// selection \p SrcSelOpName can be combined with the \p OpSel.
525static bool canCombineOpSel(const MachineInstr &MI, const SIInstrInfo *TII,
526 AMDGPU::OpName SrcOpName,
527 AMDGPU::OpName SrcSelOpName, MachineOperand *Op,
528 SdwaSel OpSel) {
529 assert(TII->isSDWA(MI.getOpcode()));
530
531 const MachineOperand *Src = TII->getNamedOperand(MI, OperandName: SrcOpName);
532 if (!Src || !isSameReg(LHS: *Src, RHS: *Op))
533 return true;
534
535 return canCombineOpSel(MI, TII, SrcSelOpName, OpSel);
536}
537
538bool SDWASrcOperand::canCombineSelections(const MachineInstr &MI,
539 const SIInstrInfo *TII) {
540 if (!TII->isSDWA(Opcode: MI.getOpcode()))
541 return true;
542
543 using namespace AMDGPU;
544
545 return canCombineOpSel(MI, TII, SrcOpName: OpName::src0, SrcSelOpName: OpName::src0_sel,
546 Op: getReplacedOperand(), OpSel: getSrcSel()) &&
547 canCombineOpSel(MI, TII, SrcOpName: OpName::src1, SrcSelOpName: OpName::src1_sel,
548 Op: getReplacedOperand(), OpSel: getSrcSel());
549}
550
551MachineInstr *SDWADstOperand::potentialToConvert(const SIInstrInfo *TII,
552 const GCNSubtarget &ST,
553 SDWAOperandsMap *PotentialMatches) {
554 // For SDWA dst operand potential instruction is one that defines register
555 // that this operand uses
556 MachineRegisterInfo *MRI = getMRI();
557 MachineInstr *ParentMI = getParentInst();
558
559 MachineOperand *PotentialMO = findSingleRegDef(Reg: getReplacedOperand(), MRI);
560 if (!PotentialMO)
561 return nullptr;
562
563 // Check that ParentMI is the only instruction that uses replaced register
564 for (MachineInstr &UseInst : MRI->use_nodbg_instructions(Reg: PotentialMO->getReg())) {
565 if (&UseInst != ParentMI)
566 return nullptr;
567 }
568
569 MachineInstr *Parent = PotentialMO->getParent();
570 return canCombineSelections(MI: *Parent, TII) ? Parent : nullptr;
571}
572
573bool SDWADstOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
574 // Replace vdst operand in MI with target operand. Set dst_sel and dst_unused
575
576 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa ||
577 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa ||
578 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
579 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
580 getDstSel() != AMDGPU::SDWA::DWORD) {
581 // v_mac_f16/32_sdwa allow dst_sel to be equal only to DWORD
582 return false;
583 }
584
585 MachineOperand *Operand = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
586 assert(Operand &&
587 Operand->isReg() &&
588 isSameReg(*Operand, *getReplacedOperand()));
589 copyRegOperand(To&: *Operand, From: *getTargetOperand());
590 MachineOperand *DstSel= TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_sel);
591 assert(DstSel);
592
593 SdwaSel ExistingSel = static_cast<SdwaSel>(DstSel->getImm());
594 DstSel->setImm(combineSdwaSel(Sel: ExistingSel, OperandSel: getDstSel()).value());
595
596 MachineOperand *DstUnused= TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
597 assert(DstUnused);
598 DstUnused->setImm(getDstUnused());
599
600 // Remove original instruction because it would conflict with our new
601 // instruction by register definition
602 getParentInst()->eraseFromParent();
603 return true;
604}
605
606bool SDWADstOperand::canCombineSelections(const MachineInstr &MI,
607 const SIInstrInfo *TII) {
608 if (!TII->isSDWA(Opcode: MI.getOpcode()))
609 return true;
610
611 return canCombineOpSel(MI, TII, SrcSelOpName: AMDGPU::OpName::dst_sel, OpSel: getDstSel());
612}
613
614bool SDWADstPreserveOperand::convertToSDWA(MachineInstr &MI,
615 const SIInstrInfo *TII) {
616 // MI should be moved right before v_or_b32.
617 // For this we should clear all kill flags on uses of MI src-operands or else
618 // we can encounter problem with use of killed operand.
619 for (MachineOperand &MO : MI.uses()) {
620 if (!MO.isReg())
621 continue;
622 getMRI()->clearKillFlags(Reg: MO.getReg());
623 }
624
625 // Move MI before v_or_b32
626 MI.getParent()->remove(I: &MI);
627 getParentInst()->getParent()->insert(I: getParentInst(), MI: &MI);
628
629 // Add Implicit use of preserved register
630 MachineInstrBuilder MIB(*MI.getMF(), MI);
631 MIB.addReg(RegNo: getPreservedOperand()->getReg(),
632 Flags: RegState::ImplicitKill,
633 SubReg: getPreservedOperand()->getSubReg());
634
635 // Tie dst to implicit use
636 MI.tieOperands(DefIdx: AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdst),
637 UseIdx: MI.getNumOperands() - 1);
638
639 // Convert MI as any other SDWADstOperand and remove v_or_b32
640 return SDWADstOperand::convertToSDWA(MI, TII);
641}
642
643bool SDWADstPreserveOperand::canCombineSelections(const MachineInstr &MI,
644 const SIInstrInfo *TII) {
645 return SDWADstOperand::canCombineSelections(MI, TII);
646}
647
648std::optional<int64_t>
649SIPeepholeSDWA::foldToImm(const MachineOperand &Op) const {
650 if (Op.isImm()) {
651 return Op.getImm();
652 }
653
654 // If this is not immediate then it can be copy of immediate value, e.g.:
655 // %1 = S_MOV_B32 255;
656 if (Op.isReg()) {
657 for (const MachineOperand &Def : MRI->def_operands(Reg: Op.getReg())) {
658 if (!isSameReg(LHS: Op, RHS: Def))
659 continue;
660
661 const MachineInstr *DefInst = Def.getParent();
662 if (!TII->isFoldableCopy(MI: *DefInst))
663 return std::nullopt;
664
665 const MachineOperand &Copied = DefInst->getOperand(i: 1);
666 if (!Copied.isImm())
667 return std::nullopt;
668
669 return Copied.getImm();
670 }
671 }
672
673 return std::nullopt;
674}
675
676std::optional<std::pair<MachineOperand *, SdwaSel>>
677SIPeepholeSDWA::matchAndMask(MachineInstr &MI) const {
678 if (MI.getOpcode() != AMDGPU::V_AND_B32_e32 &&
679 MI.getOpcode() != AMDGPU::V_AND_B32_e64)
680 return std::nullopt;
681
682 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
683 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
684 MachineOperand *ValSrc = Src1;
685 std::optional<int64_t> Imm = foldToImm(Op: *Src0);
686 if (!Imm) {
687 Imm = foldToImm(Op: *Src1);
688 ValSrc = Src0;
689 }
690 if (!Imm || (*Imm != 0x0000ffff && *Imm != 0x000000ff))
691 return std::nullopt;
692
693 return std::make_pair(x&: ValSrc, y: *Imm == 0x0000ffff ? WORD_0 : BYTE_0);
694}
695
696bool SIPeepholeSDWA::isSDWAWithDstSel(const MachineInstr &Inst) const {
697 return TII->isSDWA(MI: Inst) &&
698 AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::dst_sel);
699}
700
701std::unique_ptr<SDWAOperand>
702SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
703 unsigned Opcode = MI.getOpcode();
704 switch (Opcode) {
705 case AMDGPU::V_LSHRREV_B32_e32:
706 case AMDGPU::V_ASHRREV_I32_e32:
707 case AMDGPU::V_LSHLREV_B32_e32:
708 case AMDGPU::V_LSHRREV_B32_e64:
709 case AMDGPU::V_ASHRREV_I32_e64:
710 case AMDGPU::V_LSHLREV_B32_e64: {
711 // from: v_lshrrev_b32_e32 v1, 16/24, v0
712 // to SDWA src:v0 src_sel:WORD_1/BYTE_3
713
714 // from: v_ashrrev_i32_e32 v1, 16/24, v0
715 // to SDWA src:v0 src_sel:WORD_1/BYTE_3 sext:1
716
717 // from: v_lshlrev_b32_e32 v1, 16/24, v0
718 // to SDWA dst:v1 dst_sel:WORD_1/BYTE_3 dst_unused:UNUSED_PAD
719 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
720 auto Imm = foldToImm(Op: *Src0);
721 if (!Imm)
722 break;
723
724 if (*Imm != 16 && *Imm != 24)
725 break;
726
727 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
728 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
729 if (!Src1->isReg() || Src1->getReg().isPhysical() ||
730 Dst->getReg().isPhysical())
731 break;
732
733 if (Opcode == AMDGPU::V_LSHLREV_B32_e32 ||
734 Opcode == AMDGPU::V_LSHLREV_B32_e64) {
735 return std::make_unique<SDWADstOperand>(
736 args&: Dst, args&: Src1, args: *Imm == 16 ? WORD_1 : BYTE_3, args: UNUSED_PAD);
737 }
738 return std::make_unique<SDWASrcOperand>(
739 args&: Src1, args&: Dst, args: *Imm == 16 ? WORD_1 : BYTE_3, args: false, args: false,
740 args: Opcode != AMDGPU::V_LSHRREV_B32_e32 &&
741 Opcode != AMDGPU::V_LSHRREV_B32_e64);
742 break;
743 }
744
745 case AMDGPU::V_LSHRREV_B16_e32:
746 case AMDGPU::V_LSHLREV_B16_e32:
747 case AMDGPU::V_LSHRREV_B16_e64:
748 case AMDGPU::V_LSHRREV_B16_opsel_e64:
749 case AMDGPU::V_LSHLREV_B16_opsel_e64:
750 case AMDGPU::V_LSHLREV_B16_e64: {
751 // V_ASHRREV_I16_e32 and V_ASHRREV_I16_e64 are
752 // not included here because they zero-fill the high 16-bits.
753
754 // from: v_lshrrev_b16_e32 v1, 8, v0
755 // to SDWA src:v0 src_sel:BYTE_1
756
757 // from: v_lshlrev_b16_e32 v1, 8, v0
758 // to SDWA dst:v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD
759 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
760 auto Imm = foldToImm(Op: *Src0);
761 if (!Imm || *Imm != 8)
762 break;
763
764 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
765 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
766
767 if (!Src1->isReg() || Src1->getReg().isPhysical() ||
768 Dst->getReg().isPhysical())
769 break;
770
771 if (Opcode == AMDGPU::V_LSHLREV_B16_e32 ||
772 Opcode == AMDGPU::V_LSHLREV_B16_opsel_e64 ||
773 Opcode == AMDGPU::V_LSHLREV_B16_e64)
774 return std::make_unique<SDWADstOperand>(args&: Dst, args&: Src1, args: BYTE_1, args: UNUSED_PAD);
775 return std::make_unique<SDWASrcOperand>(args&: Src1, args&: Dst, args: BYTE_1, args: false, args: false,
776 args: false);
777 break;
778 }
779
780 case AMDGPU::V_BFE_I32_e64:
781 case AMDGPU::V_BFE_U32_e64: {
782 // e.g.:
783 // from: v_bfe_u32 v1, v0, 8, 8
784 // to SDWA src:v0 src_sel:BYTE_1
785
786 // offset | width | src_sel
787 // ------------------------
788 // 0 | 8 | BYTE_0
789 // 0 | 16 | WORD_0
790 // 0 | 32 | DWORD ?
791 // 8 | 8 | BYTE_1
792 // 16 | 8 | BYTE_2
793 // 16 | 16 | WORD_1
794 // 24 | 8 | BYTE_3
795
796 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
797 auto Offset = foldToImm(Op: *Src1);
798 if (!Offset)
799 break;
800
801 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
802 auto Width = foldToImm(Op: *Src2);
803 if (!Width)
804 break;
805
806 SdwaSel SrcSel = DWORD;
807
808 if (*Offset == 0 && *Width == 8)
809 SrcSel = BYTE_0;
810 else if (*Offset == 0 && *Width == 16)
811 SrcSel = WORD_0;
812 else if (*Offset == 0 && *Width == 32)
813 SrcSel = DWORD;
814 else if (*Offset == 8 && *Width == 8)
815 SrcSel = BYTE_1;
816 else if (*Offset == 16 && *Width == 8)
817 SrcSel = BYTE_2;
818 else if (*Offset == 16 && *Width == 16)
819 SrcSel = WORD_1;
820 else if (*Offset == 24 && *Width == 8)
821 SrcSel = BYTE_3;
822 else
823 break;
824
825 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
826 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
827
828 if (!Src0->isReg() || Src0->getReg().isPhysical() ||
829 Dst->getReg().isPhysical())
830 break;
831
832 return std::make_unique<SDWASrcOperand>(
833 args&: Src0, args&: Dst, args&: SrcSel, args: false, args: false, args: Opcode != AMDGPU::V_BFE_U32_e64);
834 }
835
836 case AMDGPU::V_AND_B32_e32:
837 case AMDGPU::V_AND_B32_e64: {
838 // e.g.:
839 // from: v_and_b32_e32 v1, 0x0000ffff/0x000000ff, v0
840 // to SDWA src:v0 src_sel:WORD_0/BYTE_0
841 auto Mask = matchAndMask(MI);
842 if (!Mask)
843 break;
844 MachineOperand *ValSrc = Mask->first;
845
846 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
847
848 if (!ValSrc->isReg() || ValSrc->getReg().isPhysical() ||
849 Dst->getReg().isPhysical())
850 break;
851
852 return std::make_unique<SDWASrcOperand>(args&: ValSrc, args&: Dst, args&: Mask->second);
853 }
854
855 case AMDGPU::V_OR_B32_e32:
856 case AMDGPU::V_OR_B32_e64: {
857 // Patterns for dst_unused:UNUSED_PRESERVE.
858 // e.g., from:
859 // v_add_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD
860 // src1_sel:WORD_1 src2_sel:WORD1
861 // v_add_f16_e32 v3, v1, v2
862 // v_or_b32_e32 v4, v0, v3
863 // to SDWA preserve dst:v4 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE preserve:v3
864
865 // Check if one of operands of v_or_b32 is SDWA instruction
866 using CheckRetType =
867 std::optional<std::pair<MachineOperand *, MachineOperand *>>;
868 auto CheckOROperandsForSDWA =
869 [&](const MachineOperand *Op1, const MachineOperand *Op2) -> CheckRetType {
870 if (!Op1 || !Op1->isReg() || !Op2 || !Op2->isReg())
871 return CheckRetType(std::nullopt);
872
873 MachineOperand *Op1Def = findSingleRegDef(Reg: Op1, MRI);
874 if (!Op1Def)
875 return CheckRetType(std::nullopt);
876
877 MachineInstr *Op1Inst = Op1Def->getParent();
878 if (!isSDWAWithDstSel(Inst: *Op1Inst))
879 return CheckRetType(std::nullopt);
880
881 MachineOperand *Op2Def = findSingleRegDef(Reg: Op2, MRI);
882 if (!Op2Def)
883 return CheckRetType(std::nullopt);
884
885 return CheckRetType(std::pair(Op1Def, Op2Def));
886 };
887
888 MachineOperand *OrSDWA = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
889 MachineOperand *OrOther = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
890 assert(OrSDWA && OrOther);
891 auto Res = CheckOROperandsForSDWA(OrSDWA, OrOther);
892 if (!Res) {
893 OrSDWA = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
894 OrOther = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
895 assert(OrSDWA && OrOther);
896 Res = CheckOROperandsForSDWA(OrSDWA, OrOther);
897 if (!Res)
898 break;
899 }
900
901 MachineOperand *OrSDWADef = Res->first;
902 MachineOperand *OrOtherDef = Res->second;
903 assert(OrSDWADef && OrOtherDef);
904
905 MachineInstr *SDWAInst = OrSDWADef->getParent();
906 MachineInstr *OtherInst = OrOtherDef->getParent();
907
908 // Check that OtherInstr is actually bitwise compatible with SDWAInst = their
909 // destination patterns don't overlap. Compatible instruction can be either
910 // regular instruction with compatible bitness or SDWA instruction with
911 // correct dst_sel
912 // SDWAInst | OtherInst bitness / OtherInst dst_sel
913 // -----------------------------------------------------
914 // DWORD | no / no
915 // WORD_0 | no / BYTE_2/3, WORD_1
916 // WORD_1 | 8/16-bit instructions / BYTE_0/1, WORD_0
917 // BYTE_0 | no / BYTE_1/2/3, WORD_1
918 // BYTE_1 | 8-bit / BYTE_0/2/3, WORD_1
919 // BYTE_2 | 8/16-bit / BYTE_0/1/3. WORD_0
920 // BYTE_3 | 8/16/24-bit / BYTE_0/1/2, WORD_0
921 // E.g. if SDWAInst is v_add_f16_sdwa dst_sel:WORD_1 then v_add_f16 is OK
922 // but v_add_f32 is not.
923
924 // TODO: add support for non-SDWA instructions as OtherInst.
925 // For now this only works with SDWA instructions. For regular instructions
926 // there is no way to determine if the instruction writes only 8/16/24-bit
927 // out of full register size and all registers are at min 32-bit wide.
928 if (!isSDWAWithDstSel(Inst: *OtherInst))
929 break;
930
931 SdwaSel DstSel = static_cast<SdwaSel>(
932 TII->getNamedImmOperand(MI: *SDWAInst, OperandName: AMDGPU::OpName::dst_sel));
933 SdwaSel OtherDstSel = static_cast<SdwaSel>(
934 TII->getNamedImmOperand(MI: *OtherInst, OperandName: AMDGPU::OpName::dst_sel));
935
936 bool DstSelAgree = false;
937 switch (DstSel) {
938 case WORD_0: DstSelAgree = ((OtherDstSel == BYTE_2) ||
939 (OtherDstSel == BYTE_3) ||
940 (OtherDstSel == WORD_1));
941 break;
942 case WORD_1: DstSelAgree = ((OtherDstSel == BYTE_0) ||
943 (OtherDstSel == BYTE_1) ||
944 (OtherDstSel == WORD_0));
945 break;
946 case BYTE_0: DstSelAgree = ((OtherDstSel == BYTE_1) ||
947 (OtherDstSel == BYTE_2) ||
948 (OtherDstSel == BYTE_3) ||
949 (OtherDstSel == WORD_1));
950 break;
951 case BYTE_1: DstSelAgree = ((OtherDstSel == BYTE_0) ||
952 (OtherDstSel == BYTE_2) ||
953 (OtherDstSel == BYTE_3) ||
954 (OtherDstSel == WORD_1));
955 break;
956 case BYTE_2: DstSelAgree = ((OtherDstSel == BYTE_0) ||
957 (OtherDstSel == BYTE_1) ||
958 (OtherDstSel == BYTE_3) ||
959 (OtherDstSel == WORD_0));
960 break;
961 case BYTE_3: DstSelAgree = ((OtherDstSel == BYTE_0) ||
962 (OtherDstSel == BYTE_1) ||
963 (OtherDstSel == BYTE_2) ||
964 (OtherDstSel == WORD_0));
965 break;
966 default: DstSelAgree = false;
967 }
968
969 if (!DstSelAgree)
970 break;
971
972 // Also OtherInst dst_unused should be UNUSED_PAD
973 DstUnused OtherDstUnused = static_cast<DstUnused>(
974 TII->getNamedImmOperand(MI: *OtherInst, OperandName: AMDGPU::OpName::dst_unused));
975 if (OtherDstUnused != DstUnused::UNUSED_PAD)
976 break;
977
978 // Create DstPreserveOperand
979 MachineOperand *OrDst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
980 assert(OrDst && OrDst->isReg());
981
982 return std::make_unique<SDWADstPreserveOperand>(
983 args&: OrDst, args&: OrSDWADef, args&: OrOtherDef, args&: DstSel);
984
985 }
986 }
987
988 return std::unique_ptr<SDWAOperand>(nullptr);
989}
990
991#if !defined(NDEBUG)
992static raw_ostream& operator<<(raw_ostream &OS, const SDWAOperand &Operand) {
993 Operand.print(OS);
994 return OS;
995}
996#endif
997
998void SIPeepholeSDWA::matchSDWAOperands(MachineBasicBlock &MBB) {
999 for (MachineInstr &MI : MBB) {
1000 if (auto Operand = matchSDWAOperand(MI)) {
1001 LLVM_DEBUG(dbgs() << "Match: " << MI << "To: " << *Operand << '\n');
1002 SDWAOperands[&MI] = std::move(Operand);
1003 ++NumSDWAPatternsFound;
1004 }
1005 }
1006}
1007
1008// Convert the V_ADD_CO_U32_e64 into V_ADD_CO_U32_e32. This allows
1009// isConvertibleToSDWA to perform its transformation on V_ADD_CO_U32_e32 into
1010// V_ADD_CO_U32_sdwa.
1011//
1012// We are transforming from a VOP3 into a VOP2 form of the instruction.
1013// %19:vgpr_32 = V_AND_B32_e32 255,
1014// killed %16:vgpr_32, implicit $exec
1015// %47:vgpr_32, %49:sreg_64_xexec = V_ADD_CO_U32_e64
1016// %26.sub0:vreg_64, %19:vgpr_32, implicit $exec
1017// %48:vgpr_32, dead %50:sreg_64_xexec = V_ADDC_U32_e64
1018// %26.sub1:vreg_64, %54:vgpr_32, killed %49:sreg_64_xexec, implicit $exec
1019//
1020// becomes
1021// %47:vgpr_32 = V_ADD_CO_U32_sdwa
1022// 0, %26.sub0:vreg_64, 0, killed %16:vgpr_32, 0, 6, 0, 6, 0,
1023// implicit-def $vcc, implicit $exec
1024// %48:vgpr_32, dead %50:sreg_64_xexec = V_ADDC_U32_e64
1025// %26.sub1:vreg_64, %54:vgpr_32, killed $vcc, implicit $exec
1026void SIPeepholeSDWA::pseudoOpConvertToVOP2(MachineInstr &MI,
1027 const GCNSubtarget &ST) const {
1028 int Opc = MI.getOpcode();
1029 assert((Opc == AMDGPU::V_ADD_CO_U32_e64 || Opc == AMDGPU::V_SUB_CO_U32_e64) &&
1030 "Currently only handles V_ADD_CO_U32_e64 or V_SUB_CO_U32_e64");
1031
1032 // Can the candidate MI be shrunk?
1033 if (!TII->canShrink(MI, MRI: *MRI))
1034 return;
1035 Opc = AMDGPU::getVOPe32(Opcode: Opc);
1036 // Find the related ADD instruction.
1037 const MachineOperand *Sdst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
1038 if (!Sdst)
1039 return;
1040 MachineOperand *NextOp = findSingleRegUse(Reg: Sdst, MRI);
1041 if (!NextOp)
1042 return;
1043 MachineInstr &MISucc = *NextOp->getParent();
1044
1045 // Make sure the carry in/out are subsequently unused.
1046 MachineOperand *CarryIn = TII->getNamedOperand(MI&: MISucc, OperandName: AMDGPU::OpName::src2);
1047 if (!CarryIn)
1048 return;
1049 MachineOperand *CarryOut = TII->getNamedOperand(MI&: MISucc, OperandName: AMDGPU::OpName::sdst);
1050 if (!CarryOut)
1051 return;
1052 if (!MRI->hasOneNonDBGUse(RegNo: CarryIn->getReg()) ||
1053 !MRI->use_nodbg_empty(RegNo: CarryOut->getReg()))
1054 return;
1055 // Make sure VCC or its subregs are dead before MI.
1056 MachineBasicBlock &MBB = *MI.getParent();
1057 if (MISucc.getParent() != &MBB)
1058 return; // Loop depends on MI and MISucc in same MBB.
1059 MachineBasicBlock::LivenessQueryResult Liveness =
1060 MBB.computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: MI, Neighborhood: 25);
1061 if (Liveness != MachineBasicBlock::LQR_Dead)
1062 return;
1063 // Check if VCC is referenced in range of (MI,MISucc].
1064 for (auto I = std::next(x: MI.getIterator()), E = MISucc.getIterator();
1065 I != E; ++I) {
1066 if (I->modifiesRegister(Reg: AMDGPU::VCC, TRI))
1067 return;
1068 }
1069
1070 // Replace MI with V_{SUB|ADD}_I32_e32
1071 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc))
1072 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst))
1073 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
1074 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1))
1075 .setMIFlags(MI.getFlags());
1076
1077 MI.eraseFromParent();
1078
1079 // Since the carry output of MI is now VCC, update its use in MISucc.
1080
1081 MISucc.substituteRegister(FromReg: CarryIn->getReg(), ToReg: TRI->getVCC(), SubIdx: 0, RegInfo: *TRI);
1082}
1083
1084/// Try to convert an \p MI in VOP3 which takes an src2 carry-in
1085/// operand into the corresponding VOP2 form which expects the
1086/// argument in VCC. To this end, add an copy from the carry-in to
1087/// VCC. The conversion will only be applied if \p MI can be shrunk
1088/// to VOP2 and if VCC can be proven to be dead before \p MI.
1089void SIPeepholeSDWA::convertVcndmaskToVOP2(MachineInstr &MI,
1090 const GCNSubtarget &ST) const {
1091 assert(MI.getOpcode() == AMDGPU::V_CNDMASK_B32_e64);
1092
1093 LLVM_DEBUG(dbgs() << "Attempting VOP2 conversion: " << MI);
1094 if (!TII->canShrink(MI, MRI: *MRI)) {
1095 LLVM_DEBUG(dbgs() << "Cannot shrink instruction\n");
1096 return;
1097 }
1098
1099 const MachineOperand &CarryIn =
1100 *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1101 Register CarryReg = CarryIn.getReg();
1102 MachineInstr *CarryDef = MRI->getVRegDef(Reg: CarryReg);
1103 if (!CarryDef) {
1104 LLVM_DEBUG(dbgs() << "Missing carry-in operand definition\n");
1105 return;
1106 }
1107
1108 // Make sure VCC or its subregs are dead before MI.
1109 MCRegister Vcc = TRI->getVCC();
1110 MachineBasicBlock &MBB = *MI.getParent();
1111 MachineBasicBlock::LivenessQueryResult Liveness =
1112 MBB.computeRegisterLiveness(TRI, Reg: Vcc, Before: MI);
1113 if (Liveness != MachineBasicBlock::LQR_Dead) {
1114 LLVM_DEBUG(dbgs() << "VCC not known to be dead before instruction\n");
1115 return;
1116 }
1117
1118 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: Vcc).add(MO: CarryIn);
1119
1120 auto Converted = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(),
1121 MCID: TII->get(Opcode: AMDGPU::getVOPe32(Opcode: MI.getOpcode())))
1122 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst))
1123 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
1124 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1))
1125 .setMIFlags(MI.getFlags());
1126 TII->fixImplicitOperands(MI&: *Converted);
1127 LLVM_DEBUG(dbgs() << "Converted to VOP2: " << *Converted);
1128 (void)Converted;
1129 MI.eraseFromParent();
1130}
1131
1132namespace {
1133bool isConvertibleToSDWA(MachineInstr &MI,
1134 const GCNSubtarget &ST,
1135 const SIInstrInfo* TII) {
1136 // Check if this is already an SDWA instruction
1137 unsigned Opc = MI.getOpcode();
1138 if (TII->isSDWA(Opcode: Opc))
1139 return true;
1140
1141 // Can only be handled after ealier conversion to
1142 // AMDGPU::V_CNDMASK_B32_e32 which is not always possible.
1143 if (Opc == AMDGPU::V_CNDMASK_B32_e64)
1144 return false;
1145
1146 // Check if this instruction has opcode that supports SDWA
1147 if (AMDGPU::getSDWAOp(Opcode: Opc) == -1)
1148 Opc = AMDGPU::getVOPe32(Opcode: Opc);
1149
1150 if (AMDGPU::getSDWAOp(Opcode: Opc) == -1)
1151 return false;
1152
1153 if (!ST.hasSDWAOmod() && TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
1154 return false;
1155
1156 if (TII->isVOPC(Opcode: Opc)) {
1157 if (!ST.hasSDWASdst()) {
1158 const MachineOperand *SDst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
1159 if (SDst && (SDst->getReg() != AMDGPU::VCC &&
1160 SDst->getReg() != AMDGPU::VCC_LO))
1161 return false;
1162 }
1163
1164 if (!ST.hasSDWAOutModsVOPC() &&
1165 (TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) ||
1166 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod)))
1167 return false;
1168
1169 } else if (TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst) ||
1170 !TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)) {
1171 return false;
1172 }
1173
1174 if (!ST.hasSDWAMac() && (Opc == AMDGPU::V_FMAC_F16_e32 ||
1175 Opc == AMDGPU::V_FMAC_F32_e32 ||
1176 Opc == AMDGPU::V_MAC_F16_e32 ||
1177 Opc == AMDGPU::V_MAC_F32_e32))
1178 return false;
1179
1180 // Check if target supports this SDWA opcode
1181 if (TII->pseudoToMCOpcode(Opcode: Opc) == -1 ||
1182 TII->pseudoToMCOpcode(Opcode: AMDGPU::getSDWAOp(Opcode: Opc)) == -1)
1183 return false;
1184
1185 if (MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)) {
1186 if (!Src0->isReg() && !Src0->isImm())
1187 return false;
1188 }
1189
1190 if (MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)) {
1191 if (!Src1->isReg() && !Src1->isImm())
1192 return false;
1193 }
1194
1195 return true;
1196}
1197} // namespace
1198
1199MachineInstr *SIPeepholeSDWA::createSDWAVersion(MachineInstr &MI) {
1200 unsigned Opcode = MI.getOpcode();
1201 assert(!TII->isSDWA(Opcode));
1202
1203 int SDWAOpcode = AMDGPU::getSDWAOp(Opcode);
1204 if (SDWAOpcode == -1)
1205 SDWAOpcode = AMDGPU::getSDWAOp(Opcode: AMDGPU::getVOPe32(Opcode));
1206 assert(SDWAOpcode != -1);
1207
1208 const MCInstrDesc &SDWADesc = TII->get(Opcode: SDWAOpcode);
1209
1210 // Create SDWA version of instruction MI and initialize its operands
1211 MachineInstrBuilder SDWAInst =
1212 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: SDWADesc)
1213 .setMIFlags(MI.getFlags());
1214
1215 // Copy dst, if it is present in original then should also be present in SDWA
1216 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1217 if (Dst) {
1218 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::vdst));
1219 SDWAInst.add(MO: *Dst);
1220 } else if ((Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst))) {
1221 assert(Dst && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst));
1222 SDWAInst.add(MO: *Dst);
1223 } else {
1224 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst));
1225 SDWAInst.addReg(RegNo: TRI->getVCC(), Flags: RegState::Define);
1226 }
1227
1228 // Copy src0, initialize src0_modifiers. All sdwa instructions has src0 and
1229 // src0_modifiers (except for v_nop_sdwa, but it can't get here)
1230 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1231 assert(Src0 && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0) &&
1232 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_modifiers));
1233 if (auto *Mod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers))
1234 SDWAInst.addImm(Val: Mod->getImm());
1235 else
1236 SDWAInst.addImm(Val: 0);
1237 SDWAInst.add(MO: *Src0);
1238
1239 // Copy src1 if present, initialize src1_modifiers.
1240 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1241 if (Src1) {
1242 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1) &&
1243 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_modifiers));
1244 if (auto *Mod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers))
1245 SDWAInst.addImm(Val: Mod->getImm());
1246 else
1247 SDWAInst.addImm(Val: 0);
1248 SDWAInst.add(MO: *Src1);
1249 }
1250
1251 if (SDWAOpcode == AMDGPU::V_FMAC_F16_sdwa ||
1252 SDWAOpcode == AMDGPU::V_FMAC_F32_sdwa ||
1253 SDWAOpcode == AMDGPU::V_MAC_F16_sdwa ||
1254 SDWAOpcode == AMDGPU::V_MAC_F32_sdwa) {
1255 // v_mac_f16/32 has additional src2 operand tied to vdst
1256 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1257 assert(Src2);
1258 SDWAInst.add(MO: *Src2);
1259 }
1260
1261 // Copy clamp if present, initialize otherwise
1262 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::clamp));
1263 MachineOperand *Clamp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
1264 if (Clamp) {
1265 SDWAInst.add(MO: *Clamp);
1266 } else {
1267 SDWAInst.addImm(Val: 0);
1268 }
1269
1270 // Copy omod if present, initialize otherwise if needed
1271 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::omod)) {
1272 MachineOperand *OMod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
1273 if (OMod) {
1274 SDWAInst.add(MO: *OMod);
1275 } else {
1276 SDWAInst.addImm(Val: 0);
1277 }
1278 }
1279
1280 // Initialize SDWA specific operands
1281 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::dst_sel))
1282 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1283
1284 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::dst_unused))
1285 SDWAInst.addImm(Val: AMDGPU::SDWA::DstUnused::UNUSED_PAD);
1286
1287 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_sel));
1288 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1289
1290 if (Src1) {
1291 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_sel));
1292 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1293 }
1294
1295 // Check for a preserved register that needs to be copied.
1296 MachineInstr *Ret = SDWAInst.getInstr();
1297 TII->fixImplicitOperands(MI&: *Ret);
1298 return Ret;
1299}
1300
1301bool SIPeepholeSDWA::convertToSDWA(MachineInstr &MI,
1302 const SDWAOperandsVector &SDWAOperands) {
1303 LLVM_DEBUG(dbgs() << "Convert instruction:" << MI);
1304
1305 MachineInstr *SDWAInst;
1306 if (TII->isSDWA(Opcode: MI.getOpcode())) {
1307 // Clone the instruction to allow revoking changes
1308 // made to MI during the processing of the operands
1309 // if the conversion fails.
1310 SDWAInst = MI.getMF()->CloneMachineInstr(Orig: &MI);
1311 MI.getParent()->insert(I: MI.getIterator(), M: SDWAInst);
1312 } else {
1313 SDWAInst = createSDWAVersion(MI);
1314 }
1315
1316 // Apply all sdwa operand patterns.
1317 bool Converted = false;
1318 for (auto &Operand : SDWAOperands) {
1319 LLVM_DEBUG(dbgs() << *SDWAInst << "\nOperand: " << *Operand);
1320 // There should be no intersection between SDWA operands and potential MIs
1321 // e.g.:
1322 // v_and_b32 v0, 0xff, v1 -> src:v1 sel:BYTE_0
1323 // v_and_b32 v2, 0xff, v0 -> src:v0 sel:BYTE_0
1324 // v_add_u32 v3, v4, v2
1325 //
1326 // In that example it is possible that we would fold 2nd instruction into
1327 // 3rd (v_add_u32_sdwa) and then try to fold 1st instruction into 2nd (that
1328 // was already destroyed). So if SDWAOperand is also a potential MI then do
1329 // not apply it.
1330 if (PotentialMatches.count(Key: Operand->getParentInst()) == 0)
1331 Converted |= Operand->convertToSDWA(MI&: *SDWAInst, TII);
1332 }
1333
1334 if (!Converted) {
1335 SDWAInst->eraseFromParent();
1336 return false;
1337 }
1338
1339 ConvertedInstructions.push_back(Elt: SDWAInst);
1340 for (MachineOperand &MO : SDWAInst->uses()) {
1341 if (!MO.isReg())
1342 continue;
1343
1344 MRI->clearKillFlags(Reg: MO.getReg());
1345 }
1346 LLVM_DEBUG(dbgs() << "\nInto:" << *SDWAInst << '\n');
1347 ++NumSDWAInstructionsPeepholed;
1348
1349 MI.eraseFromParent();
1350 return true;
1351}
1352
1353// If an instruction was converted to SDWA it should not have immediates or SGPR
1354// operands (allowed one SGPR on GFX9). Copy its scalar operands into VGPRs.
1355void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI,
1356 const GCNSubtarget &ST) const {
1357 const MCInstrDesc &Desc = TII->get(Opcode: MI.getOpcode());
1358 unsigned ConstantBusCount = 0;
1359 for (MachineOperand &Op : MI.explicit_uses()) {
1360 if (Op.isReg()) {
1361 if (TRI->isVGPR(MRI: *MRI, Reg: Op.getReg()))
1362 continue;
1363
1364 if (ST.hasSDWAScalar() && ConstantBusCount == 0) {
1365 ++ConstantBusCount;
1366 continue;
1367 }
1368 } else if (!Op.isImm())
1369 continue;
1370
1371 unsigned I = Op.getOperandNo();
1372 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: Desc, OpNum: I);
1373 if (!OpRC || !TRI->isVSSuperClass(RC: OpRC))
1374 continue;
1375
1376 Register VGPR = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1377 auto Copy = BuildMI(BB&: *MI.getParent(), I: MI.getIterator(), MIMD: MI.getDebugLoc(),
1378 MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VGPR);
1379 if (Op.isImm())
1380 Copy.addImm(Val: Op.getImm());
1381 else if (Op.isReg())
1382 Copy.addReg(RegNo: Op.getReg(), Flags: getKillRegState(B: Op.isKill()), SubReg: Op.getSubReg());
1383 Op.ChangeToRegister(Reg: VGPR, isDef: false);
1384 }
1385}
1386
1387// Re-fold the masked high-half pack (hi << 16) | (z & 0xffff) into a single
1388// v_or_b32_sdwa src1_sel:WORD_0, which ISel's fused v_lshl_or_b32 blocks.
1389bool SIPeepholeSDWA::splitLshlOrForSDWA(MachineBasicBlock &MBB) {
1390 struct Candidate {
1391 MachineInstr *LshlOr;
1392 MachineInstr *AndMI;
1393 MachineOperand *Hi;
1394 MachineOperand *ValSrc;
1395 };
1396 SmallVector<Candidate, 4> Candidates;
1397
1398 for (MachineInstr &MI : MBB) {
1399 if (MI.getOpcode() != AMDGPU::V_LSHL_OR_B32_e64)
1400 continue;
1401
1402 MachineOperand *Shift = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1403 std::optional<int64_t> ShiftImm = foldToImm(Op: *Shift);
1404 if (!ShiftImm || *ShiftImm != 16)
1405 continue;
1406
1407 MachineOperand *Hi = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1408 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1409 // Src2 must be a virtual reg so getVRegDef below is valid.
1410 if (!Hi->isReg() || !Src2->isReg() || !Src2->getReg().isVirtual())
1411 continue;
1412
1413 // The 0xffff mask must come from a single-use v_and so it can be dropped.
1414 if (!MRI->hasOneNonDBGUse(RegNo: Src2->getReg()))
1415 continue;
1416 MachineInstr *AndMI = MRI->getVRegDef(Reg: Src2->getReg());
1417 if (!AndMI)
1418 continue;
1419 std::optional<std::pair<MachineOperand *, SdwaSel>> Mask =
1420 matchAndMask(MI&: *AndMI);
1421 if (!Mask || Mask->second != WORD_0)
1422 continue;
1423 MachineOperand *ValSrc = Mask->first;
1424 if (!ValSrc->isReg() || !TRI->isVGPR(MRI: *MRI, Reg: ValSrc->getReg()))
1425 continue;
1426
1427 Candidates.push_back(Elt: {.LshlOr: &MI, .AndMI: AndMI, .Hi: Hi, .ValSrc: ValSrc});
1428 }
1429
1430 for (const Candidate &C : Candidates) {
1431 MachineOperand *Dst = TII->getNamedOperand(MI&: *C.LshlOr, OperandName: AMDGPU::OpName::vdst);
1432
1433 Register ShiftReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1434 BuildMI(BB&: *C.LshlOr->getParent(), I&: *C.LshlOr, MIMD: C.LshlOr->getDebugLoc(),
1435 MCID: TII->get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftReg)
1436 .addImm(Val: 16)
1437 .add(MO: *C.Hi);
1438
1439 // vdst, src0_mods, src0, src1_mods, src1, clamp, dst_sel, dst_unused,
1440 // src0_sel, src1_sel.
1441 BuildMI(BB&: *C.LshlOr->getParent(), I&: *C.LshlOr, MIMD: C.LshlOr->getDebugLoc(),
1442 MCID: TII->get(Opcode: AMDGPU::V_OR_B32_sdwa))
1443 .add(MO: *Dst)
1444 .addImm(Val: 0)
1445 .addReg(RegNo: ShiftReg)
1446 .addImm(Val: 0)
1447 .add(MO: *C.ValSrc)
1448 .addImm(Val: 0)
1449 .addImm(Val: DWORD)
1450 .addImm(Val: UNUSED_PAD)
1451 .addImm(Val: DWORD)
1452 .addImm(Val: WORD_0);
1453
1454 MRI->clearKillFlags(Reg: C.ValSrc->getReg());
1455 C.LshlOr->eraseFromParent();
1456 C.AndMI->eraseFromParent();
1457 }
1458
1459 return !Candidates.empty();
1460}
1461
1462bool SIPeepholeSDWALegacy::runOnMachineFunction(MachineFunction &MF) {
1463 if (skipFunction(F: MF.getFunction()))
1464 return false;
1465
1466 return SIPeepholeSDWA().run(MF);
1467}
1468
1469bool SIPeepholeSDWA::run(MachineFunction &MF) {
1470 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1471
1472 if (!ST.hasSDWA())
1473 return false;
1474
1475 MRI = &MF.getRegInfo();
1476 TRI = ST.getRegisterInfo();
1477 TII = ST.getInstrInfo();
1478
1479 // Find all SDWA operands in MF.
1480 bool Ret = false;
1481 for (MachineBasicBlock &MBB : MF) {
1482 bool Changed = false;
1483 do {
1484 Ret |= splitLshlOrForSDWA(MBB);
1485
1486 // Preprocess the ADD/SUB pairs so they could be SDWA'ed.
1487 // Look for a possible ADD or SUB that resulted from a previously lowered
1488 // V_{ADD|SUB}_U64_PSEUDO. The function pseudoOpConvertToVOP2
1489 // lowers the pair of instructions into e32 form.
1490 matchSDWAOperands(MBB);
1491 for (const auto &OperandPair : SDWAOperands) {
1492 const auto &Operand = OperandPair.second;
1493 MachineInstr *PotentialMI = Operand->potentialToConvert(TII, ST);
1494 if (!PotentialMI)
1495 continue;
1496
1497 switch (PotentialMI->getOpcode()) {
1498 case AMDGPU::V_ADD_CO_U32_e64:
1499 case AMDGPU::V_SUB_CO_U32_e64:
1500 pseudoOpConvertToVOP2(MI&: *PotentialMI, ST);
1501 break;
1502 case AMDGPU::V_CNDMASK_B32_e64:
1503 convertVcndmaskToVOP2(MI&: *PotentialMI, ST);
1504 break;
1505 };
1506 }
1507 SDWAOperands.clear();
1508
1509 // Generate potential match list.
1510 matchSDWAOperands(MBB);
1511
1512 for (const auto &OperandPair : SDWAOperands) {
1513 const auto &Operand = OperandPair.second;
1514 MachineInstr *PotentialMI =
1515 Operand->potentialToConvert(TII, ST, PotentialMatches: &PotentialMatches);
1516
1517 if (PotentialMI && isConvertibleToSDWA(MI&: *PotentialMI, ST, TII))
1518 PotentialMatches[PotentialMI].push_back(Elt: Operand.get());
1519 }
1520
1521 for (auto &PotentialPair : PotentialMatches) {
1522 MachineInstr &PotentialMI = *PotentialPair.first;
1523 convertToSDWA(MI&: PotentialMI, SDWAOperands: PotentialPair.second);
1524 }
1525
1526 PotentialMatches.clear();
1527 SDWAOperands.clear();
1528
1529 Changed = !ConvertedInstructions.empty();
1530
1531 if (Changed)
1532 Ret = true;
1533 while (!ConvertedInstructions.empty())
1534 legalizeScalarOperands(MI&: *ConvertedInstructions.pop_back_val(), ST);
1535 } while (Changed);
1536 }
1537
1538 return Ret;
1539}
1540
1541PreservedAnalyses SIPeepholeSDWAPass::run(MachineFunction &MF,
1542 MachineFunctionAnalysisManager &) {
1543 if (MF.getFunction().hasOptNone() || !SIPeepholeSDWA().run(MF))
1544 return PreservedAnalyses::all();
1545
1546 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1547 PA.preserveSet<CFGAnalyses>();
1548 return PA;
1549}
1550