1//===- SIPeepholeSDWA.cpp - Peephole optimization for SDWA instructions ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass tries to apply several peephole SDWA patterns.
10///
11/// E.g. original:
12/// V_LSHRREV_B32_e32 %0, 16, %1
13/// V_ADD_CO_U32_e32 %2, %0, %3
14/// V_LSHLREV_B32_e32 %4, 16, %2
15///
16/// Replace:
17/// V_ADD_CO_U32_sdwa %4, %1, %3
18/// dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
19///
20//===----------------------------------------------------------------------===//
21
22#include "SIPeepholeSDWA.h"
23#include "AMDGPU.h"
24#include "GCNSubtarget.h"
25#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
26#include "llvm/ADT/MapVector.h"
27#include "llvm/ADT/Statistic.h"
28#include "llvm/CodeGen/MachineFunctionPass.h"
29#include "llvm/CodeGen/RegisterClassInfo.h"
30#include <optional>
31
32using namespace llvm;
33
34#define DEBUG_TYPE "si-peephole-sdwa"
35
36STATISTIC(NumSDWAPatternsFound, "Number of SDWA patterns found.");
37STATISTIC(NumSDWAInstructionsPeepholed,
38 "Number of instruction converted to SDWA.");
39
40namespace {
41
42bool isConvertibleToSDWA(MachineInstr &MI, const GCNSubtarget &ST,
43 const SIInstrInfo *TII);
44class SDWAOperand;
45class SDWADstOperand;
46
47using SDWAOperandsVector = SmallVector<SDWAOperand *, 4>;
48using SDWAOperandsMap = MapVector<MachineInstr *, SDWAOperandsVector>;
49
50class SIPeepholeSDWA {
51private:
52 MachineRegisterInfo *MRI;
53 const SIRegisterInfo *TRI;
54 const SIInstrInfo *TII;
55
56 MapVector<MachineInstr *, std::unique_ptr<SDWAOperand>> SDWAOperands;
57 SDWAOperandsMap PotentialMatches;
58 SmallVector<MachineInstr *, 8> ConvertedInstructions;
59
60 std::optional<int64_t> foldToImm(const MachineOperand &Op) const;
61
62 // If MI is a v_and_b32 with a 0xffff or 0xff immediate, return the masked
63 // value operand and the matching SDWA selector (WORD_0 / BYTE_0).
64 std::optional<std::pair<MachineOperand *, AMDGPU::SDWA::SdwaSel>>
65 matchAndMask(MachineInstr &MI) const;
66
67 // VOPC SDWA instructions carry the SDWA TSFlag but have no dst_sel operand.
68 bool isSDWAWithDstSel(const MachineInstr &Inst) const;
69
70 void matchSDWAOperands(MachineBasicBlock &MBB);
71 std::unique_ptr<SDWAOperand> matchSDWAOperand(MachineInstr &MI);
72 void pseudoOpConvertToVOP2(MachineInstr &MI,
73 const GCNSubtarget &ST) const;
74 void convertVcndmaskToVOP2(MachineInstr &MI, const GCNSubtarget &ST) const;
75 MachineInstr *createSDWAVersion(MachineInstr &MI);
76 bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands);
77 void legalizeScalarOperands(MachineInstr &MI, const GCNSubtarget &ST) const;
78 bool splitLshlOrForSDWA(MachineBasicBlock &MBB);
79
80public:
81 bool run(MachineFunction &MF);
82};
83
84class SIPeepholeSDWALegacy : public MachineFunctionPass {
85public:
86 static char ID;
87
88 SIPeepholeSDWALegacy() : MachineFunctionPass(ID) {}
89
90 StringRef getPassName() const override { return "SI Peephole SDWA"; }
91
92 bool runOnMachineFunction(MachineFunction &MF) override;
93
94 void getAnalysisUsage(AnalysisUsage &AU) const override {
95 AU.setPreservesCFG();
96 MachineFunctionPass::getAnalysisUsage(AU);
97 }
98};
99
100using namespace AMDGPU::SDWA;
101
102class SDWAOperand {
103private:
104 MachineOperand *Target; // Operand that would be used in converted instruction
105 MachineOperand *Replaced; // Operand that would be replace by Target
106
107 /// Returns true iff the SDWA selection of this SDWAOperand can be combined
108 /// with the SDWA selections of its uses in \p MI.
109 virtual bool canCombineSelections(const MachineInstr &MI,
110 const SIInstrInfo *TII) = 0;
111
112public:
113 SDWAOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp)
114 : Target(TargetOp), Replaced(ReplacedOp) {
115 assert(Target->isReg());
116 assert(Replaced->isReg());
117 }
118
119 virtual ~SDWAOperand() = default;
120
121 virtual MachineInstr *potentialToConvert(const SIInstrInfo *TII,
122 const GCNSubtarget &ST,
123 SDWAOperandsMap *PotentialMatches = nullptr) = 0;
124 virtual bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) = 0;
125
126 MachineOperand *getTargetOperand() const { return Target; }
127 MachineOperand *getReplacedOperand() const { return Replaced; }
128 MachineInstr *getParentInst() const { return Target->getParent(); }
129
130 MachineRegisterInfo *getMRI() const {
131 return &getParentInst()->getMF()->getRegInfo();
132 }
133
134#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
135 virtual void print(raw_ostream& OS) const = 0;
136 void dump() const { print(dbgs()); }
137#endif
138};
139
140class SDWASrcOperand : public SDWAOperand {
141private:
142 SdwaSel SrcSel;
143 bool Abs;
144 bool Neg;
145 bool Sext;
146
147public:
148 SDWASrcOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
149 SdwaSel SrcSel_ = DWORD, bool Abs_ = false, bool Neg_ = false,
150 bool Sext_ = false)
151 : SDWAOperand(TargetOp, ReplacedOp), SrcSel(SrcSel_), Abs(Abs_),
152 Neg(Neg_), Sext(Sext_) {}
153
154 MachineInstr *potentialToConvert(const SIInstrInfo *TII,
155 const GCNSubtarget &ST,
156 SDWAOperandsMap *PotentialMatches = nullptr) override;
157 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
158 bool canCombineSelections(const MachineInstr &MI,
159 const SIInstrInfo *TII) override;
160
161 SdwaSel getSrcSel() const { return SrcSel; }
162 bool getAbs() const { return Abs; }
163 bool getNeg() const { return Neg; }
164 bool getSext() const { return Sext; }
165
166 uint64_t getSrcMods(const SIInstrInfo *TII,
167 const MachineOperand *SrcOp) const;
168
169#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
170 void print(raw_ostream& OS) const override;
171#endif
172};
173
174class SDWADstOperand : public SDWAOperand {
175private:
176 SdwaSel DstSel;
177 DstUnused DstUn;
178
179public:
180 SDWADstOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
181 SdwaSel DstSel_ = DWORD, DstUnused DstUn_ = UNUSED_PAD)
182 : SDWAOperand(TargetOp, ReplacedOp), DstSel(DstSel_), DstUn(DstUn_) {}
183
184 MachineInstr *potentialToConvert(const SIInstrInfo *TII,
185 const GCNSubtarget &ST,
186 SDWAOperandsMap *PotentialMatches = nullptr) override;
187 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
188 bool canCombineSelections(const MachineInstr &MI,
189 const SIInstrInfo *TII) override;
190
191 SdwaSel getDstSel() const { return DstSel; }
192 DstUnused getDstUnused() const { return DstUn; }
193
194#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
195 void print(raw_ostream& OS) const override;
196#endif
197};
198
199class SDWADstPreserveOperand : public SDWADstOperand {
200private:
201 MachineOperand *Preserve;
202
203public:
204 SDWADstPreserveOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
205 MachineOperand *PreserveOp, SdwaSel DstSel_ = DWORD)
206 : SDWADstOperand(TargetOp, ReplacedOp, DstSel_, UNUSED_PRESERVE),
207 Preserve(PreserveOp) {}
208
209 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
210 bool canCombineSelections(const MachineInstr &MI,
211 const SIInstrInfo *TII) override;
212
213 MachineOperand *getPreservedOperand() const { return Preserve; }
214
215#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
216 void print(raw_ostream& OS) const override;
217#endif
218};
219
220} // end anonymous namespace
221
222INITIALIZE_PASS(SIPeepholeSDWALegacy, DEBUG_TYPE, "SI Peephole SDWA", false,
223 false)
224
225char SIPeepholeSDWALegacy::ID = 0;
226
227char &llvm::SIPeepholeSDWALegacyID = SIPeepholeSDWALegacy::ID;
228
229FunctionPass *llvm::createSIPeepholeSDWALegacyPass() {
230 return new SIPeepholeSDWALegacy();
231}
232
233#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
234static raw_ostream& operator<<(raw_ostream &OS, SdwaSel Sel) {
235 switch(Sel) {
236 case BYTE_0: OS << "BYTE_0"; break;
237 case BYTE_1: OS << "BYTE_1"; break;
238 case BYTE_2: OS << "BYTE_2"; break;
239 case BYTE_3: OS << "BYTE_3"; break;
240 case WORD_0: OS << "WORD_0"; break;
241 case WORD_1: OS << "WORD_1"; break;
242 case DWORD: OS << "DWORD"; break;
243 }
244 return OS;
245}
246
247static raw_ostream& operator<<(raw_ostream &OS, const DstUnused &Un) {
248 switch(Un) {
249 case UNUSED_PAD: OS << "UNUSED_PAD"; break;
250 case UNUSED_SEXT: OS << "UNUSED_SEXT"; break;
251 case UNUSED_PRESERVE: OS << "UNUSED_PRESERVE"; break;
252 }
253 return OS;
254}
255
256LLVM_DUMP_METHOD
257void SDWASrcOperand::print(raw_ostream& OS) const {
258 OS << "SDWA src: " << *getTargetOperand()
259 << " src_sel:" << getSrcSel()
260 << " abs:" << getAbs() << " neg:" << getNeg()
261 << " sext:" << getSext() << '\n';
262}
263
264LLVM_DUMP_METHOD
265void SDWADstOperand::print(raw_ostream& OS) const {
266 OS << "SDWA dst: " << *getTargetOperand()
267 << " dst_sel:" << getDstSel()
268 << " dst_unused:" << getDstUnused() << '\n';
269}
270
271LLVM_DUMP_METHOD
272void SDWADstPreserveOperand::print(raw_ostream& OS) const {
273 OS << "SDWA preserve dst: " << *getTargetOperand()
274 << " dst_sel:" << getDstSel()
275 << " preserve:" << *getPreservedOperand() << '\n';
276}
277
278#endif
279
280static void copyRegOperand(MachineOperand &To, const MachineOperand &From) {
281 assert(To.isReg() && From.isReg());
282 To.setReg(From.getReg());
283 To.setSubReg(From.getSubReg());
284 To.setIsUndef(From.isUndef());
285 if (To.isUse()) {
286 To.setIsKill(From.isKill());
287 } else {
288 To.setIsDead(From.isDead());
289 }
290}
291
292static bool isSameReg(const MachineOperand &LHS, const MachineOperand &RHS) {
293 return LHS.isReg() &&
294 RHS.isReg() &&
295 LHS.getReg() == RHS.getReg() &&
296 LHS.getSubReg() == RHS.getSubReg();
297}
298
299static MachineOperand *findSingleRegUse(const MachineOperand *Reg,
300 const MachineRegisterInfo *MRI) {
301 if (!Reg->isReg() || !Reg->isDef())
302 return nullptr;
303
304 return MRI->getOneNonDBGUse(RegNo: Reg->getReg());
305}
306
307static MachineOperand *findSingleRegDef(const MachineOperand *Reg,
308 const MachineRegisterInfo *MRI) {
309 if (!Reg->isReg())
310 return nullptr;
311
312 return MRI->getOneDef(Reg: Reg->getReg());
313}
314
315/// Combine an SDWA instruction's existing SDWA selection \p Sel with
316/// the SDWA selection \p OperandSel of its operand. If the selections
317/// are compatible, return the combined selection, otherwise return a
318/// nullopt.
319/// For example, if we have Sel = BYTE_0 Sel and OperandSel = WORD_1:
320/// BYTE_0 Sel (WORD_1 Sel (%X)) -> BYTE_2 Sel (%X)
321static std::optional<SdwaSel> combineSdwaSel(SdwaSel Sel, SdwaSel OperandSel) {
322 if (Sel == SdwaSel::DWORD)
323 return OperandSel;
324
325 if (Sel == OperandSel || OperandSel == SdwaSel::DWORD)
326 return Sel;
327
328 if (Sel == SdwaSel::WORD_1 || Sel == SdwaSel::BYTE_2 ||
329 Sel == SdwaSel::BYTE_3)
330 return {};
331
332 if (OperandSel == SdwaSel::WORD_0)
333 return Sel;
334
335 if (OperandSel == SdwaSel::WORD_1) {
336 if (Sel == SdwaSel::BYTE_0)
337 return SdwaSel::BYTE_2;
338 if (Sel == SdwaSel::BYTE_1)
339 return SdwaSel::BYTE_3;
340 if (Sel == SdwaSel::WORD_0)
341 return SdwaSel::WORD_1;
342 }
343
344 return {};
345}
346
347uint64_t SDWASrcOperand::getSrcMods(const SIInstrInfo *TII,
348 const MachineOperand *SrcOp) const {
349 uint64_t Mods = 0;
350 const auto *MI = SrcOp->getParent();
351 if (TII->getNamedOperand(MI: *MI, OperandName: AMDGPU::OpName::src0) == SrcOp) {
352 if (auto *Mod = TII->getNamedOperand(MI: *MI, OperandName: AMDGPU::OpName::src0_modifiers)) {
353 Mods = Mod->getImm();
354 }
355 } else if (TII->getNamedOperand(MI: *MI, OperandName: AMDGPU::OpName::src1) == SrcOp) {
356 if (auto *Mod = TII->getNamedOperand(MI: *MI, OperandName: AMDGPU::OpName::src1_modifiers)) {
357 Mods = Mod->getImm();
358 }
359 }
360 if (Abs || Neg) {
361 assert(!Sext &&
362 "Float and integer src modifiers can't be set simultaneously");
363 Mods |= Abs ? SISrcMods::ABS : 0u;
364 Mods ^= Neg ? SISrcMods::NEG : 0u;
365 } else if (Sext) {
366 Mods |= SISrcMods::SEXT;
367 }
368
369 return Mods;
370}
371
372MachineInstr *SDWASrcOperand::potentialToConvert(const SIInstrInfo *TII,
373 const GCNSubtarget &ST,
374 SDWAOperandsMap *PotentialMatches) {
375 if (PotentialMatches != nullptr) {
376 // Fill out the map for all uses if all can be converted
377 MachineOperand *Reg = getReplacedOperand();
378 if (!Reg->isReg() || !Reg->isDef())
379 return nullptr;
380
381 for (MachineInstr &UseMI : getMRI()->use_nodbg_instructions(Reg: Reg->getReg()))
382 // Check that all instructions that use Reg can be converted
383 if (!isConvertibleToSDWA(MI&: UseMI, ST, TII) ||
384 !canCombineSelections(MI: UseMI, TII))
385 return nullptr;
386
387 // Now that it's guaranteed all uses are legal, iterate over the uses again
388 // to add them for later conversion.
389 for (MachineOperand &UseMO : getMRI()->use_nodbg_operands(Reg: Reg->getReg())) {
390 // Should not get a subregister here
391 assert(isSameReg(UseMO, *Reg));
392
393 SDWAOperandsMap &potentialMatchesMap = *PotentialMatches;
394 MachineInstr *UseMI = UseMO.getParent();
395 potentialMatchesMap[UseMI].push_back(Elt: this);
396 }
397 return nullptr;
398 }
399
400 // For SDWA src operand potential instruction is one that use register
401 // defined by parent instruction
402 MachineOperand *PotentialMO = findSingleRegUse(Reg: getReplacedOperand(), MRI: getMRI());
403 if (!PotentialMO)
404 return nullptr;
405
406 MachineInstr *Parent = PotentialMO->getParent();
407
408 return canCombineSelections(MI: *Parent, TII) ? Parent : nullptr;
409}
410
411bool SDWASrcOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
412 assert((!Sext || !TII->getSubtarget().zeroesHigh16BitsOfDest(
413 getParentInst()->getOpcode())) &&
414 "Cannot use sign-extension with instruction that zeroes high bits");
415 switch (MI.getOpcode()) {
416 case AMDGPU::V_CVT_F32_FP8_sdwa:
417 case AMDGPU::V_CVT_F32_BF8_sdwa:
418 case AMDGPU::V_CVT_PK_F32_FP8_sdwa:
419 case AMDGPU::V_CVT_PK_F32_BF8_sdwa:
420 // Does not support input modifiers: noabs, noneg, nosext.
421 return false;
422 case AMDGPU::V_CNDMASK_B32_sdwa:
423 // SISrcMods uses the same bitmask for SEXT and NEG modifiers and
424 // hence the compiler can only support one type of modifier for
425 // each SDWA instruction. For V_CNDMASK_B32_sdwa, this is NEG
426 // since its operands get printed using
427 // AMDGPUInstPrinter::printOperandAndFPInputMods which produces
428 // the output intended for NEG if SEXT is set.
429 //
430 // The ISA does actually support both modifiers on most SDWA
431 // instructions.
432 //
433 // FIXME Accept SEXT here after fixing this issue.
434 if (Sext)
435 return false;
436 break;
437 }
438
439 // Find operand in instruction that matches source operand and replace it with
440 // target operand. Set corresponding src_sel
441 bool IsPreserveSrc = false;
442 MachineOperand *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
443 MachineOperand *SrcSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_sel);
444 MachineOperand *SrcMods =
445 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
446 assert(Src && (Src->isReg() || Src->isImm()));
447 if (!isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
448 // If this is not src0 then it could be src1
449 Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
450 SrcSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_sel);
451 SrcMods = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
452
453 if (!Src ||
454 !isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
455 // It's possible this Src is a tied operand for
456 // UNUSED_PRESERVE, in which case we can either
457 // abandon the peephole attempt, or if legal we can
458 // copy the target operand into the tied slot
459 // if the preserve operation will effectively cause the same
460 // result by overwriting the rest of the dst.
461 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
462 MachineOperand *DstUnused =
463 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
464
465 if (Dst &&
466 DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) {
467 // This will work if the tied src is accessing WORD_0, and the dst is
468 // writing WORD_1. Modifiers don't matter because all the bits that
469 // would be impacted are being overwritten by the dst.
470 // Any other case will not work.
471 SdwaSel DstSel = static_cast<SdwaSel>(
472 TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::dst_sel));
473 if (DstSel == AMDGPU::SDWA::SdwaSel::WORD_1 &&
474 getSrcSel() == AMDGPU::SDWA::SdwaSel::WORD_0) {
475 IsPreserveSrc = true;
476 auto DstIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
477 Name: AMDGPU::OpName::vdst);
478 auto TiedIdx = MI.findTiedOperandIdx(OpIdx: DstIdx);
479 Src = &MI.getOperand(i: TiedIdx);
480 SrcSel = nullptr;
481 SrcMods = nullptr;
482 } else {
483 // Not legal to convert this src
484 return false;
485 }
486 }
487 }
488 assert(Src && Src->isReg());
489
490 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa ||
491 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa ||
492 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
493 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
494 !isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
495 // In case of v_mac_f16/32_sdwa this pass can try to apply src operand to
496 // src2. This is not allowed.
497 return false;
498 }
499
500 assert(isSameReg(*Src, *getReplacedOperand()) &&
501 (IsPreserveSrc || (SrcSel && SrcMods)));
502 }
503 copyRegOperand(To&: *Src, From: *getTargetOperand());
504 if (!IsPreserveSrc) {
505 SdwaSel ExistingSel = static_cast<SdwaSel>(SrcSel->getImm());
506 SrcSel->setImm(*combineSdwaSel(Sel: ExistingSel, OperandSel: getSrcSel()));
507 SrcMods->setImm(getSrcMods(TII, SrcOp: Src));
508 }
509 getTargetOperand()->setIsKill(false);
510 return true;
511}
512
513/// Verify that the SDWA selection operand \p SrcSelOpName of the SDWA
514/// instruction \p MI can be combined with the selection \p OpSel.
515static bool canCombineOpSel(const MachineInstr &MI, const SIInstrInfo *TII,
516 AMDGPU::OpName SrcSelOpName, SdwaSel OpSel) {
517 assert(TII->isSDWA(MI.getOpcode()));
518
519 const MachineOperand *SrcSelOp = TII->getNamedOperand(MI, OperandName: SrcSelOpName);
520 SdwaSel SrcSel = static_cast<SdwaSel>(SrcSelOp->getImm());
521
522 return combineSdwaSel(Sel: SrcSel, OperandSel: OpSel).has_value();
523}
524
525/// Verify that \p Op is the same register as the operand of the SDWA
526/// instruction \p MI named by \p SrcOpName and that the SDWA
527/// selection \p SrcSelOpName can be combined with the \p OpSel.
528static bool canCombineOpSel(const MachineInstr &MI, const SIInstrInfo *TII,
529 AMDGPU::OpName SrcOpName,
530 AMDGPU::OpName SrcSelOpName, MachineOperand *Op,
531 SdwaSel OpSel) {
532 assert(TII->isSDWA(MI.getOpcode()));
533
534 const MachineOperand *Src = TII->getNamedOperand(MI, OperandName: SrcOpName);
535 if (!Src || !isSameReg(LHS: *Src, RHS: *Op))
536 return true;
537
538 return canCombineOpSel(MI, TII, SrcSelOpName, OpSel);
539}
540
541bool SDWASrcOperand::canCombineSelections(const MachineInstr &MI,
542 const SIInstrInfo *TII) {
543 if (!TII->isSDWA(Opcode: MI.getOpcode()))
544 return true;
545
546 using namespace AMDGPU;
547
548 return canCombineOpSel(MI, TII, SrcOpName: OpName::src0, SrcSelOpName: OpName::src0_sel,
549 Op: getReplacedOperand(), OpSel: getSrcSel()) &&
550 canCombineOpSel(MI, TII, SrcOpName: OpName::src1, SrcSelOpName: OpName::src1_sel,
551 Op: getReplacedOperand(), OpSel: getSrcSel());
552}
553
554MachineInstr *SDWADstOperand::potentialToConvert(const SIInstrInfo *TII,
555 const GCNSubtarget &ST,
556 SDWAOperandsMap *PotentialMatches) {
557 // For SDWA dst operand potential instruction is one that defines register
558 // that this operand uses
559 MachineRegisterInfo *MRI = getMRI();
560 MachineInstr *ParentMI = getParentInst();
561
562 MachineOperand *PotentialMO = findSingleRegDef(Reg: getReplacedOperand(), MRI);
563 if (!PotentialMO)
564 return nullptr;
565
566 // Check that ParentMI is the only instruction that uses replaced register
567 for (MachineInstr &UseInst : MRI->use_nodbg_instructions(Reg: PotentialMO->getReg())) {
568 if (&UseInst != ParentMI)
569 return nullptr;
570 }
571
572 MachineInstr *Parent = PotentialMO->getParent();
573 return canCombineSelections(MI: *Parent, TII) ? Parent : nullptr;
574}
575
576bool SDWADstOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
577 // Replace vdst operand in MI with target operand. Set dst_sel and dst_unused
578
579 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa ||
580 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa ||
581 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
582 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
583 getDstSel() != AMDGPU::SDWA::DWORD) {
584 // v_mac_f16/32_sdwa allow dst_sel to be equal only to DWORD
585 return false;
586 }
587
588 MachineOperand *Operand = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
589 assert(Operand &&
590 Operand->isReg() &&
591 isSameReg(*Operand, *getReplacedOperand()));
592 copyRegOperand(To&: *Operand, From: *getTargetOperand());
593 MachineOperand *DstSel= TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_sel);
594 assert(DstSel);
595
596 SdwaSel ExistingSel = static_cast<SdwaSel>(DstSel->getImm());
597 DstSel->setImm(combineSdwaSel(Sel: ExistingSel, OperandSel: getDstSel()).value());
598
599 MachineOperand *DstUnused= TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
600 assert(DstUnused);
601 DstUnused->setImm(getDstUnused());
602
603 // Remove original instruction because it would conflict with our new
604 // instruction by register definition
605 getParentInst()->eraseFromParent();
606 return true;
607}
608
609bool SDWADstOperand::canCombineSelections(const MachineInstr &MI,
610 const SIInstrInfo *TII) {
611 if (!TII->isSDWA(Opcode: MI.getOpcode()))
612 return true;
613
614 return canCombineOpSel(MI, TII, SrcSelOpName: AMDGPU::OpName::dst_sel, OpSel: getDstSel());
615}
616
617bool SDWADstPreserveOperand::convertToSDWA(MachineInstr &MI,
618 const SIInstrInfo *TII) {
619 // MI should be moved right before v_or_b32.
620 // For this we should clear all kill flags on uses of MI src-operands or else
621 // we can encounter problem with use of killed operand.
622 for (MachineOperand &MO : MI.uses()) {
623 if (!MO.isReg())
624 continue;
625 getMRI()->clearKillFlags(Reg: MO.getReg());
626 }
627
628 // Move MI before v_or_b32
629 MI.getParent()->remove(I: &MI);
630 getParentInst()->getParent()->insert(I: getParentInst(), MI: &MI);
631
632 // Add Implicit use of preserved register
633 MachineInstrBuilder MIB(*MI.getMF(), MI);
634 MIB.addReg(RegNo: getPreservedOperand()->getReg(),
635 Flags: RegState::ImplicitKill,
636 SubReg: getPreservedOperand()->getSubReg());
637
638 // Tie dst to implicit use
639 MI.tieOperands(DefIdx: AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdst),
640 UseIdx: MI.getNumOperands() - 1);
641
642 // Convert MI as any other SDWADstOperand and remove v_or_b32
643 return SDWADstOperand::convertToSDWA(MI, TII);
644}
645
646bool SDWADstPreserveOperand::canCombineSelections(const MachineInstr &MI,
647 const SIInstrInfo *TII) {
648 return SDWADstOperand::canCombineSelections(MI, TII);
649}
650
651std::optional<int64_t>
652SIPeepholeSDWA::foldToImm(const MachineOperand &Op) const {
653 if (Op.isImm()) {
654 return Op.getImm();
655 }
656
657 // If this is not immediate then it can be copy of immediate value, e.g.:
658 // %1 = S_MOV_B32 255;
659 if (Op.isReg()) {
660 for (const MachineOperand &Def : MRI->def_operands(Reg: Op.getReg())) {
661 if (!isSameReg(LHS: Op, RHS: Def))
662 continue;
663
664 const MachineInstr *DefInst = Def.getParent();
665 if (!TII->isFoldableCopy(MI: *DefInst))
666 return std::nullopt;
667
668 const MachineOperand &Copied = DefInst->getOperand(i: 1);
669 if (!Copied.isImm())
670 return std::nullopt;
671
672 return Copied.getImm();
673 }
674 }
675
676 return std::nullopt;
677}
678
679std::optional<std::pair<MachineOperand *, SdwaSel>>
680SIPeepholeSDWA::matchAndMask(MachineInstr &MI) const {
681 if (MI.getOpcode() != AMDGPU::V_AND_B32_e32 &&
682 MI.getOpcode() != AMDGPU::V_AND_B32_e64)
683 return std::nullopt;
684
685 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
686 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
687 MachineOperand *ValSrc = Src1;
688 std::optional<int64_t> Imm = foldToImm(Op: *Src0);
689 if (!Imm) {
690 Imm = foldToImm(Op: *Src1);
691 ValSrc = Src0;
692 }
693 if (!Imm || (*Imm != 0x0000ffff && *Imm != 0x000000ff))
694 return std::nullopt;
695
696 return std::make_pair(x&: ValSrc, y: *Imm == 0x0000ffff ? WORD_0 : BYTE_0);
697}
698
699bool SIPeepholeSDWA::isSDWAWithDstSel(const MachineInstr &Inst) const {
700 return TII->isSDWA(MI: Inst) &&
701 AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::dst_sel);
702}
703
704std::unique_ptr<SDWAOperand>
705SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
706 unsigned Opcode = MI.getOpcode();
707 switch (Opcode) {
708 case AMDGPU::V_LSHRREV_B32_e32:
709 case AMDGPU::V_ASHRREV_I32_e32:
710 case AMDGPU::V_LSHLREV_B32_e32:
711 case AMDGPU::V_LSHRREV_B32_e64:
712 case AMDGPU::V_ASHRREV_I32_e64:
713 case AMDGPU::V_LSHLREV_B32_e64: {
714 // from: v_lshrrev_b32_e32 v1, 16/24, v0
715 // to SDWA src:v0 src_sel:WORD_1/BYTE_3
716
717 // from: v_ashrrev_i32_e32 v1, 16/24, v0
718 // to SDWA src:v0 src_sel:WORD_1/BYTE_3 sext:1
719
720 // from: v_lshlrev_b32_e32 v1, 16/24, v0
721 // to SDWA dst:v1 dst_sel:WORD_1/BYTE_3 dst_unused:UNUSED_PAD
722 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
723 auto Imm = foldToImm(Op: *Src0);
724 if (!Imm)
725 break;
726
727 if (*Imm != 16 && *Imm != 24)
728 break;
729
730 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
731 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
732 if (!Src1->isReg() || Src1->getReg().isPhysical() ||
733 Dst->getReg().isPhysical())
734 break;
735
736 if (Opcode == AMDGPU::V_LSHLREV_B32_e32 ||
737 Opcode == AMDGPU::V_LSHLREV_B32_e64) {
738 return std::make_unique<SDWADstOperand>(
739 args&: Dst, args&: Src1, args: *Imm == 16 ? WORD_1 : BYTE_3, args: UNUSED_PAD);
740 }
741 return std::make_unique<SDWASrcOperand>(
742 args&: Src1, args&: Dst, args: *Imm == 16 ? WORD_1 : BYTE_3, args: false, args: false,
743 args: Opcode != AMDGPU::V_LSHRREV_B32_e32 &&
744 Opcode != AMDGPU::V_LSHRREV_B32_e64);
745 break;
746 }
747
748 case AMDGPU::V_LSHRREV_B16_e32:
749 case AMDGPU::V_LSHLREV_B16_e32:
750 case AMDGPU::V_LSHRREV_B16_e64:
751 case AMDGPU::V_LSHRREV_B16_opsel_e64:
752 case AMDGPU::V_LSHLREV_B16_opsel_e64:
753 case AMDGPU::V_LSHLREV_B16_e64: {
754 // V_ASHRREV_I16_e32 and V_ASHRREV_I16_e64 are
755 // not included here because they zero-fill the high 16-bits.
756
757 // from: v_lshrrev_b16_e32 v1, 8, v0
758 // to SDWA src:v0 src_sel:BYTE_1
759
760 // from: v_lshlrev_b16_e32 v1, 8, v0
761 // to SDWA dst:v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD
762 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
763 auto Imm = foldToImm(Op: *Src0);
764 if (!Imm || *Imm != 8)
765 break;
766
767 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
768 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
769
770 if (!Src1->isReg() || Src1->getReg().isPhysical() ||
771 Dst->getReg().isPhysical())
772 break;
773
774 if (Opcode == AMDGPU::V_LSHLREV_B16_e32 ||
775 Opcode == AMDGPU::V_LSHLREV_B16_opsel_e64 ||
776 Opcode == AMDGPU::V_LSHLREV_B16_e64)
777 return std::make_unique<SDWADstOperand>(args&: Dst, args&: Src1, args: BYTE_1, args: UNUSED_PAD);
778 return std::make_unique<SDWASrcOperand>(args&: Src1, args&: Dst, args: BYTE_1, args: false, args: false,
779 args: false);
780 break;
781 }
782
783 case AMDGPU::V_BFE_I32_e64:
784 case AMDGPU::V_BFE_U32_e64: {
785 // e.g.:
786 // from: v_bfe_u32 v1, v0, 8, 8
787 // to SDWA src:v0 src_sel:BYTE_1
788
789 // offset | width | src_sel
790 // ------------------------
791 // 0 | 8 | BYTE_0
792 // 0 | 16 | WORD_0
793 // 0 | 32 | DWORD ?
794 // 8 | 8 | BYTE_1
795 // 16 | 8 | BYTE_2
796 // 16 | 16 | WORD_1
797 // 24 | 8 | BYTE_3
798
799 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
800 auto Offset = foldToImm(Op: *Src1);
801 if (!Offset)
802 break;
803
804 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
805 auto Width = foldToImm(Op: *Src2);
806 if (!Width)
807 break;
808
809 SdwaSel SrcSel = DWORD;
810
811 if (*Offset == 0 && *Width == 8)
812 SrcSel = BYTE_0;
813 else if (*Offset == 0 && *Width == 16)
814 SrcSel = WORD_0;
815 else if (*Offset == 0 && *Width == 32)
816 SrcSel = DWORD;
817 else if (*Offset == 8 && *Width == 8)
818 SrcSel = BYTE_1;
819 else if (*Offset == 16 && *Width == 8)
820 SrcSel = BYTE_2;
821 else if (*Offset == 16 && *Width == 16)
822 SrcSel = WORD_1;
823 else if (*Offset == 24 && *Width == 8)
824 SrcSel = BYTE_3;
825 else
826 break;
827
828 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
829 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
830
831 if (!Src0->isReg() || Src0->getReg().isPhysical() ||
832 Dst->getReg().isPhysical())
833 break;
834
835 return std::make_unique<SDWASrcOperand>(
836 args&: Src0, args&: Dst, args&: SrcSel, args: false, args: false, args: Opcode != AMDGPU::V_BFE_U32_e64);
837 }
838
839 case AMDGPU::V_AND_B32_e32:
840 case AMDGPU::V_AND_B32_e64: {
841 // e.g.:
842 // from: v_and_b32_e32 v1, 0x0000ffff/0x000000ff, v0
843 // to SDWA src:v0 src_sel:WORD_0/BYTE_0
844 auto Mask = matchAndMask(MI);
845 if (!Mask)
846 break;
847 MachineOperand *ValSrc = Mask->first;
848
849 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
850
851 if (!ValSrc->isReg() || ValSrc->getReg().isPhysical() ||
852 Dst->getReg().isPhysical())
853 break;
854
855 return std::make_unique<SDWASrcOperand>(args&: ValSrc, args&: Dst, args&: Mask->second);
856 }
857
858 case AMDGPU::V_OR_B32_e32:
859 case AMDGPU::V_OR_B32_e64: {
860 // Patterns for dst_unused:UNUSED_PRESERVE.
861 // e.g., from:
862 // v_add_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD
863 // src1_sel:WORD_1 src2_sel:WORD1
864 // v_add_f16_e32 v3, v1, v2
865 // v_or_b32_e32 v4, v0, v3
866 // to SDWA preserve dst:v4 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE preserve:v3
867
868 // Check if one of operands of v_or_b32 is SDWA instruction
869 using CheckRetType =
870 std::optional<std::pair<MachineOperand *, MachineOperand *>>;
871 auto CheckOROperandsForSDWA =
872 [&](const MachineOperand *Op1, const MachineOperand *Op2) -> CheckRetType {
873 if (!Op1 || !Op1->isReg() || !Op2 || !Op2->isReg())
874 return CheckRetType(std::nullopt);
875
876 MachineOperand *Op1Def = findSingleRegDef(Reg: Op1, MRI);
877 if (!Op1Def)
878 return CheckRetType(std::nullopt);
879
880 MachineInstr *Op1Inst = Op1Def->getParent();
881 if (!isSDWAWithDstSel(Inst: *Op1Inst))
882 return CheckRetType(std::nullopt);
883
884 MachineOperand *Op2Def = findSingleRegDef(Reg: Op2, MRI);
885 if (!Op2Def)
886 return CheckRetType(std::nullopt);
887
888 return CheckRetType(std::pair(Op1Def, Op2Def));
889 };
890
891 MachineOperand *OrSDWA = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
892 MachineOperand *OrOther = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
893 assert(OrSDWA && OrOther);
894 auto Res = CheckOROperandsForSDWA(OrSDWA, OrOther);
895 if (!Res) {
896 OrSDWA = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
897 OrOther = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
898 assert(OrSDWA && OrOther);
899 Res = CheckOROperandsForSDWA(OrSDWA, OrOther);
900 if (!Res)
901 break;
902 }
903
904 MachineOperand *OrSDWADef = Res->first;
905 MachineOperand *OrOtherDef = Res->second;
906 assert(OrSDWADef && OrOtherDef);
907
908 MachineInstr *SDWAInst = OrSDWADef->getParent();
909 MachineInstr *OtherInst = OrOtherDef->getParent();
910
911 // Check that OtherInstr is actually bitwise compatible with SDWAInst = their
912 // destination patterns don't overlap. Compatible instruction can be either
913 // regular instruction with compatible bitness or SDWA instruction with
914 // correct dst_sel
915 // SDWAInst | OtherInst bitness / OtherInst dst_sel
916 // -----------------------------------------------------
917 // DWORD | no / no
918 // WORD_0 | no / BYTE_2/3, WORD_1
919 // WORD_1 | 8/16-bit instructions / BYTE_0/1, WORD_0
920 // BYTE_0 | no / BYTE_1/2/3, WORD_1
921 // BYTE_1 | 8-bit / BYTE_0/2/3, WORD_1
922 // BYTE_2 | 8/16-bit / BYTE_0/1/3. WORD_0
923 // BYTE_3 | 8/16/24-bit / BYTE_0/1/2, WORD_0
924 // E.g. if SDWAInst is v_add_f16_sdwa dst_sel:WORD_1 then v_add_f16 is OK
925 // but v_add_f32 is not.
926
927 // TODO: add support for non-SDWA instructions as OtherInst.
928 // For now this only works with SDWA instructions. For regular instructions
929 // there is no way to determine if the instruction writes only 8/16/24-bit
930 // out of full register size and all registers are at min 32-bit wide.
931 if (!isSDWAWithDstSel(Inst: *OtherInst))
932 break;
933
934 SdwaSel DstSel = static_cast<SdwaSel>(
935 TII->getNamedImmOperand(MI: *SDWAInst, OperandName: AMDGPU::OpName::dst_sel));
936 SdwaSel OtherDstSel = static_cast<SdwaSel>(
937 TII->getNamedImmOperand(MI: *OtherInst, OperandName: AMDGPU::OpName::dst_sel));
938
939 bool DstSelAgree = false;
940 switch (DstSel) {
941 case WORD_0: DstSelAgree = ((OtherDstSel == BYTE_2) ||
942 (OtherDstSel == BYTE_3) ||
943 (OtherDstSel == WORD_1));
944 break;
945 case WORD_1: DstSelAgree = ((OtherDstSel == BYTE_0) ||
946 (OtherDstSel == BYTE_1) ||
947 (OtherDstSel == WORD_0));
948 break;
949 case BYTE_0: DstSelAgree = ((OtherDstSel == BYTE_1) ||
950 (OtherDstSel == BYTE_2) ||
951 (OtherDstSel == BYTE_3) ||
952 (OtherDstSel == WORD_1));
953 break;
954 case BYTE_1: DstSelAgree = ((OtherDstSel == BYTE_0) ||
955 (OtherDstSel == BYTE_2) ||
956 (OtherDstSel == BYTE_3) ||
957 (OtherDstSel == WORD_1));
958 break;
959 case BYTE_2: DstSelAgree = ((OtherDstSel == BYTE_0) ||
960 (OtherDstSel == BYTE_1) ||
961 (OtherDstSel == BYTE_3) ||
962 (OtherDstSel == WORD_0));
963 break;
964 case BYTE_3: DstSelAgree = ((OtherDstSel == BYTE_0) ||
965 (OtherDstSel == BYTE_1) ||
966 (OtherDstSel == BYTE_2) ||
967 (OtherDstSel == WORD_0));
968 break;
969 default: DstSelAgree = false;
970 }
971
972 if (!DstSelAgree)
973 break;
974
975 // Also OtherInst dst_unused should be UNUSED_PAD
976 DstUnused OtherDstUnused = static_cast<DstUnused>(
977 TII->getNamedImmOperand(MI: *OtherInst, OperandName: AMDGPU::OpName::dst_unused));
978 if (OtherDstUnused != DstUnused::UNUSED_PAD)
979 break;
980
981 // Create DstPreserveOperand
982 MachineOperand *OrDst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
983 assert(OrDst && OrDst->isReg());
984
985 return std::make_unique<SDWADstPreserveOperand>(
986 args&: OrDst, args&: OrSDWADef, args&: OrOtherDef, args&: DstSel);
987
988 }
989 }
990
991 return std::unique_ptr<SDWAOperand>(nullptr);
992}
993
994#if !defined(NDEBUG)
995static raw_ostream& operator<<(raw_ostream &OS, const SDWAOperand &Operand) {
996 Operand.print(OS);
997 return OS;
998}
999#endif
1000
1001void SIPeepholeSDWA::matchSDWAOperands(MachineBasicBlock &MBB) {
1002 for (MachineInstr &MI : MBB) {
1003 if (auto Operand = matchSDWAOperand(MI)) {
1004 LLVM_DEBUG(dbgs() << "Match: " << MI << "To: " << *Operand << '\n');
1005 SDWAOperands[&MI] = std::move(Operand);
1006 ++NumSDWAPatternsFound;
1007 }
1008 }
1009}
1010
1011// Convert the V_ADD_CO_U32_e64 into V_ADD_CO_U32_e32. This allows
1012// isConvertibleToSDWA to perform its transformation on V_ADD_CO_U32_e32 into
1013// V_ADD_CO_U32_sdwa.
1014//
1015// We are transforming from a VOP3 into a VOP2 form of the instruction.
1016// %19:vgpr_32 = V_AND_B32_e32 255,
1017// killed %16:vgpr_32, implicit $exec
1018// %47:vgpr_32, %49:sreg_64_xexec = V_ADD_CO_U32_e64
1019// %26.sub0:vreg_64, %19:vgpr_32, implicit $exec
1020// %48:vgpr_32, dead %50:sreg_64_xexec = V_ADDC_U32_e64
1021// %26.sub1:vreg_64, %54:vgpr_32, killed %49:sreg_64_xexec, implicit $exec
1022//
1023// becomes
1024// %47:vgpr_32 = V_ADD_CO_U32_sdwa
1025// 0, %26.sub0:vreg_64, 0, killed %16:vgpr_32, 0, 6, 0, 6, 0,
1026// implicit-def $vcc, implicit $exec
1027// %48:vgpr_32, dead %50:sreg_64_xexec = V_ADDC_U32_e64
1028// %26.sub1:vreg_64, %54:vgpr_32, killed $vcc, implicit $exec
1029void SIPeepholeSDWA::pseudoOpConvertToVOP2(MachineInstr &MI,
1030 const GCNSubtarget &ST) const {
1031 int Opc = MI.getOpcode();
1032 assert((Opc == AMDGPU::V_ADD_CO_U32_e64 || Opc == AMDGPU::V_SUB_CO_U32_e64) &&
1033 "Currently only handles V_ADD_CO_U32_e64 or V_SUB_CO_U32_e64");
1034
1035 // Can the candidate MI be shrunk?
1036 if (!TII->canShrink(MI, MRI: *MRI))
1037 return;
1038 Opc = AMDGPU::getVOPe32(Opcode: Opc);
1039 // Find the related ADD instruction.
1040 const MachineOperand *Sdst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
1041 if (!Sdst)
1042 return;
1043 MachineOperand *NextOp = findSingleRegUse(Reg: Sdst, MRI);
1044 if (!NextOp)
1045 return;
1046 MachineInstr &MISucc = *NextOp->getParent();
1047
1048 // Make sure the carry in/out are subsequently unused.
1049 MachineOperand *CarryIn = TII->getNamedOperand(MI&: MISucc, OperandName: AMDGPU::OpName::src2);
1050 if (!CarryIn)
1051 return;
1052 MachineOperand *CarryOut = TII->getNamedOperand(MI&: MISucc, OperandName: AMDGPU::OpName::sdst);
1053 if (!CarryOut)
1054 return;
1055 if (!MRI->hasOneNonDBGUse(RegNo: CarryIn->getReg()) ||
1056 !MRI->use_nodbg_empty(RegNo: CarryOut->getReg()))
1057 return;
1058 // Make sure VCC or its subregs are dead before MI.
1059 MachineBasicBlock &MBB = *MI.getParent();
1060 MachineBasicBlock::LivenessQueryResult Liveness =
1061 MBB.computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: MI, Neighborhood: 25);
1062 if (Liveness != MachineBasicBlock::LQR_Dead)
1063 return;
1064 // Check if VCC is referenced in range of (MI,MISucc].
1065 for (auto I = std::next(x: MI.getIterator()), E = MISucc.getIterator();
1066 I != E; ++I) {
1067 if (I->modifiesRegister(Reg: AMDGPU::VCC, TRI))
1068 return;
1069 }
1070
1071 // Replace MI with V_{SUB|ADD}_I32_e32
1072 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc))
1073 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst))
1074 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
1075 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1))
1076 .setMIFlags(MI.getFlags());
1077
1078 MI.eraseFromParent();
1079
1080 // Since the carry output of MI is now VCC, update its use in MISucc.
1081
1082 MISucc.substituteRegister(FromReg: CarryIn->getReg(), ToReg: TRI->getVCC(), SubIdx: 0, RegInfo: *TRI);
1083}
1084
1085/// Try to convert an \p MI in VOP3 which takes an src2 carry-in
1086/// operand into the corresponding VOP2 form which expects the
1087/// argument in VCC. To this end, add an copy from the carry-in to
1088/// VCC. The conversion will only be applied if \p MI can be shrunk
1089/// to VOP2 and if VCC can be proven to be dead before \p MI.
1090void SIPeepholeSDWA::convertVcndmaskToVOP2(MachineInstr &MI,
1091 const GCNSubtarget &ST) const {
1092 assert(MI.getOpcode() == AMDGPU::V_CNDMASK_B32_e64);
1093
1094 LLVM_DEBUG(dbgs() << "Attempting VOP2 conversion: " << MI);
1095 if (!TII->canShrink(MI, MRI: *MRI)) {
1096 LLVM_DEBUG(dbgs() << "Cannot shrink instruction\n");
1097 return;
1098 }
1099
1100 const MachineOperand &CarryIn =
1101 *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1102 Register CarryReg = CarryIn.getReg();
1103 MachineInstr *CarryDef = MRI->getVRegDef(Reg: CarryReg);
1104 if (!CarryDef) {
1105 LLVM_DEBUG(dbgs() << "Missing carry-in operand definition\n");
1106 return;
1107 }
1108
1109 // Make sure VCC or its subregs are dead before MI.
1110 MCRegister Vcc = TRI->getVCC();
1111 MachineBasicBlock &MBB = *MI.getParent();
1112 MachineBasicBlock::LivenessQueryResult Liveness =
1113 MBB.computeRegisterLiveness(TRI, Reg: Vcc, Before: MI);
1114 if (Liveness != MachineBasicBlock::LQR_Dead) {
1115 LLVM_DEBUG(dbgs() << "VCC not known to be dead before instruction\n");
1116 return;
1117 }
1118
1119 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: Vcc).add(MO: CarryIn);
1120
1121 auto Converted = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(),
1122 MCID: TII->get(Opcode: AMDGPU::getVOPe32(Opcode: MI.getOpcode())))
1123 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst))
1124 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
1125 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1))
1126 .setMIFlags(MI.getFlags());
1127 TII->fixImplicitOperands(MI&: *Converted);
1128 LLVM_DEBUG(dbgs() << "Converted to VOP2: " << *Converted);
1129 (void)Converted;
1130 MI.eraseFromParent();
1131}
1132
1133namespace {
1134bool isConvertibleToSDWA(MachineInstr &MI,
1135 const GCNSubtarget &ST,
1136 const SIInstrInfo* TII) {
1137 // Check if this is already an SDWA instruction
1138 unsigned Opc = MI.getOpcode();
1139 if (TII->isSDWA(Opcode: Opc))
1140 return true;
1141
1142 // Can only be handled after ealier conversion to
1143 // AMDGPU::V_CNDMASK_B32_e32 which is not always possible.
1144 if (Opc == AMDGPU::V_CNDMASK_B32_e64)
1145 return false;
1146
1147 // Check if this instruction has opcode that supports SDWA
1148 if (AMDGPU::getSDWAOp(Opcode: Opc) == -1)
1149 Opc = AMDGPU::getVOPe32(Opcode: Opc);
1150
1151 if (AMDGPU::getSDWAOp(Opcode: Opc) == -1)
1152 return false;
1153
1154 if (!ST.hasSDWAOmod() && TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
1155 return false;
1156
1157 if (TII->isVOPC(Opcode: Opc)) {
1158 if (!ST.hasSDWASdst()) {
1159 const MachineOperand *SDst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
1160 if (SDst && (SDst->getReg() != AMDGPU::VCC &&
1161 SDst->getReg() != AMDGPU::VCC_LO))
1162 return false;
1163 }
1164
1165 if (!ST.hasSDWAOutModsVOPC() &&
1166 (TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) ||
1167 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod)))
1168 return false;
1169
1170 } else if (TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst) ||
1171 !TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)) {
1172 return false;
1173 }
1174
1175 if (!ST.hasSDWAMac() && (Opc == AMDGPU::V_FMAC_F16_e32 ||
1176 Opc == AMDGPU::V_FMAC_F32_e32 ||
1177 Opc == AMDGPU::V_MAC_F16_e32 ||
1178 Opc == AMDGPU::V_MAC_F32_e32))
1179 return false;
1180
1181 // Check if target supports this SDWA opcode
1182 if (TII->pseudoToMCOpcode(Opcode: Opc) == -1 ||
1183 TII->pseudoToMCOpcode(Opcode: AMDGPU::getSDWAOp(Opcode: Opc)) == -1)
1184 return false;
1185
1186 if (MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)) {
1187 if (!Src0->isReg() && !Src0->isImm())
1188 return false;
1189 }
1190
1191 if (MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)) {
1192 if (!Src1->isReg() && !Src1->isImm())
1193 return false;
1194 }
1195
1196 return true;
1197}
1198} // namespace
1199
1200MachineInstr *SIPeepholeSDWA::createSDWAVersion(MachineInstr &MI) {
1201 unsigned Opcode = MI.getOpcode();
1202 assert(!TII->isSDWA(Opcode));
1203
1204 int SDWAOpcode = AMDGPU::getSDWAOp(Opcode);
1205 if (SDWAOpcode == -1)
1206 SDWAOpcode = AMDGPU::getSDWAOp(Opcode: AMDGPU::getVOPe32(Opcode));
1207 assert(SDWAOpcode != -1);
1208
1209 const MCInstrDesc &SDWADesc = TII->get(Opcode: SDWAOpcode);
1210
1211 // Create SDWA version of instruction MI and initialize its operands
1212 MachineInstrBuilder SDWAInst =
1213 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: SDWADesc)
1214 .setMIFlags(MI.getFlags());
1215
1216 // Copy dst, if it is present in original then should also be present in SDWA
1217 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1218 if (Dst) {
1219 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::vdst));
1220 SDWAInst.add(MO: *Dst);
1221 } else if ((Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst))) {
1222 assert(Dst && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst));
1223 SDWAInst.add(MO: *Dst);
1224 } else {
1225 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst));
1226 SDWAInst.addReg(RegNo: TRI->getVCC(), Flags: RegState::Define);
1227 }
1228
1229 // Copy src0, initialize src0_modifiers. All sdwa instructions has src0 and
1230 // src0_modifiers (except for v_nop_sdwa, but it can't get here)
1231 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1232 assert(Src0 && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0) &&
1233 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_modifiers));
1234 if (auto *Mod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers))
1235 SDWAInst.addImm(Val: Mod->getImm());
1236 else
1237 SDWAInst.addImm(Val: 0);
1238 SDWAInst.add(MO: *Src0);
1239
1240 // Copy src1 if present, initialize src1_modifiers.
1241 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1242 if (Src1) {
1243 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1) &&
1244 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_modifiers));
1245 if (auto *Mod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers))
1246 SDWAInst.addImm(Val: Mod->getImm());
1247 else
1248 SDWAInst.addImm(Val: 0);
1249 SDWAInst.add(MO: *Src1);
1250 }
1251
1252 if (SDWAOpcode == AMDGPU::V_FMAC_F16_sdwa ||
1253 SDWAOpcode == AMDGPU::V_FMAC_F32_sdwa ||
1254 SDWAOpcode == AMDGPU::V_MAC_F16_sdwa ||
1255 SDWAOpcode == AMDGPU::V_MAC_F32_sdwa) {
1256 // v_mac_f16/32 has additional src2 operand tied to vdst
1257 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1258 assert(Src2);
1259 SDWAInst.add(MO: *Src2);
1260 }
1261
1262 // Copy clamp if present, initialize otherwise
1263 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::clamp));
1264 MachineOperand *Clamp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
1265 if (Clamp) {
1266 SDWAInst.add(MO: *Clamp);
1267 } else {
1268 SDWAInst.addImm(Val: 0);
1269 }
1270
1271 // Copy omod if present, initialize otherwise if needed
1272 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::omod)) {
1273 MachineOperand *OMod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
1274 if (OMod) {
1275 SDWAInst.add(MO: *OMod);
1276 } else {
1277 SDWAInst.addImm(Val: 0);
1278 }
1279 }
1280
1281 // Initialize SDWA specific operands
1282 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::dst_sel))
1283 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1284
1285 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::dst_unused))
1286 SDWAInst.addImm(Val: AMDGPU::SDWA::DstUnused::UNUSED_PAD);
1287
1288 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_sel));
1289 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1290
1291 if (Src1) {
1292 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_sel));
1293 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1294 }
1295
1296 // Check for a preserved register that needs to be copied.
1297 MachineInstr *Ret = SDWAInst.getInstr();
1298 TII->fixImplicitOperands(MI&: *Ret);
1299 return Ret;
1300}
1301
1302bool SIPeepholeSDWA::convertToSDWA(MachineInstr &MI,
1303 const SDWAOperandsVector &SDWAOperands) {
1304 LLVM_DEBUG(dbgs() << "Convert instruction:" << MI);
1305
1306 MachineInstr *SDWAInst;
1307 if (TII->isSDWA(Opcode: MI.getOpcode())) {
1308 // Clone the instruction to allow revoking changes
1309 // made to MI during the processing of the operands
1310 // if the conversion fails.
1311 SDWAInst = MI.getMF()->CloneMachineInstr(Orig: &MI);
1312 MI.getParent()->insert(I: MI.getIterator(), M: SDWAInst);
1313 } else {
1314 SDWAInst = createSDWAVersion(MI);
1315 }
1316
1317 // Apply all sdwa operand patterns.
1318 bool Converted = false;
1319 for (auto &Operand : SDWAOperands) {
1320 LLVM_DEBUG(dbgs() << *SDWAInst << "\nOperand: " << *Operand);
1321 // There should be no intersection between SDWA operands and potential MIs
1322 // e.g.:
1323 // v_and_b32 v0, 0xff, v1 -> src:v1 sel:BYTE_0
1324 // v_and_b32 v2, 0xff, v0 -> src:v0 sel:BYTE_0
1325 // v_add_u32 v3, v4, v2
1326 //
1327 // In that example it is possible that we would fold 2nd instruction into
1328 // 3rd (v_add_u32_sdwa) and then try to fold 1st instruction into 2nd (that
1329 // was already destroyed). So if SDWAOperand is also a potential MI then do
1330 // not apply it.
1331 if (PotentialMatches.count(Key: Operand->getParentInst()) == 0)
1332 Converted |= Operand->convertToSDWA(MI&: *SDWAInst, TII);
1333 }
1334
1335 if (!Converted) {
1336 SDWAInst->eraseFromParent();
1337 return false;
1338 }
1339
1340 ConvertedInstructions.push_back(Elt: SDWAInst);
1341 for (MachineOperand &MO : SDWAInst->uses()) {
1342 if (!MO.isReg())
1343 continue;
1344
1345 MRI->clearKillFlags(Reg: MO.getReg());
1346 }
1347 LLVM_DEBUG(dbgs() << "\nInto:" << *SDWAInst << '\n');
1348 ++NumSDWAInstructionsPeepholed;
1349
1350 MI.eraseFromParent();
1351 return true;
1352}
1353
1354// If an instruction was converted to SDWA it should not have immediates or SGPR
1355// operands (allowed one SGPR on GFX9). Copy its scalar operands into VGPRs.
1356void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI,
1357 const GCNSubtarget &ST) const {
1358 const MCInstrDesc &Desc = TII->get(Opcode: MI.getOpcode());
1359 unsigned ConstantBusCount = 0;
1360 for (MachineOperand &Op : MI.explicit_uses()) {
1361 if (Op.isReg()) {
1362 if (TRI->isVGPR(MRI: *MRI, Reg: Op.getReg()))
1363 continue;
1364
1365 if (ST.hasSDWAScalar() && ConstantBusCount == 0) {
1366 ++ConstantBusCount;
1367 continue;
1368 }
1369 } else if (!Op.isImm())
1370 continue;
1371
1372 unsigned I = Op.getOperandNo();
1373 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: Desc, OpNum: I);
1374 if (!OpRC || !TRI->isVSSuperClass(RC: OpRC))
1375 continue;
1376
1377 Register VGPR = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1378 auto Copy = BuildMI(BB&: *MI.getParent(), I: MI.getIterator(), MIMD: MI.getDebugLoc(),
1379 MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VGPR);
1380 if (Op.isImm())
1381 Copy.addImm(Val: Op.getImm());
1382 else if (Op.isReg())
1383 Copy.addReg(RegNo: Op.getReg(), Flags: getKillRegState(B: Op.isKill()), SubReg: Op.getSubReg());
1384 Op.ChangeToRegister(Reg: VGPR, isDef: false);
1385 }
1386}
1387
1388// Re-fold the masked high-half pack (hi << 16) | (z & 0xffff) into a single
1389// v_or_b32_sdwa src1_sel:WORD_0, which ISel's fused v_lshl_or_b32 blocks.
1390bool SIPeepholeSDWA::splitLshlOrForSDWA(MachineBasicBlock &MBB) {
1391 struct Candidate {
1392 MachineInstr *LshlOr;
1393 MachineInstr *AndMI;
1394 MachineOperand *Hi;
1395 MachineOperand *ValSrc;
1396 };
1397 SmallVector<Candidate, 4> Candidates;
1398
1399 for (MachineInstr &MI : MBB) {
1400 if (MI.getOpcode() != AMDGPU::V_LSHL_OR_B32_e64)
1401 continue;
1402
1403 MachineOperand *Shift = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1404 std::optional<int64_t> ShiftImm = foldToImm(Op: *Shift);
1405 if (!ShiftImm || *ShiftImm != 16)
1406 continue;
1407
1408 MachineOperand *Hi = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1409 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1410 // Src2 must be a virtual reg so getVRegDef below is valid.
1411 if (!Hi->isReg() || !Src2->isReg() || !Src2->getReg().isVirtual())
1412 continue;
1413
1414 // The 0xffff mask must come from a single-use v_and so it can be dropped.
1415 if (!MRI->hasOneNonDBGUse(RegNo: Src2->getReg()))
1416 continue;
1417 MachineInstr *AndMI = MRI->getVRegDef(Reg: Src2->getReg());
1418 if (!AndMI)
1419 continue;
1420 std::optional<std::pair<MachineOperand *, SdwaSel>> Mask =
1421 matchAndMask(MI&: *AndMI);
1422 if (!Mask || Mask->second != WORD_0)
1423 continue;
1424 MachineOperand *ValSrc = Mask->first;
1425 if (!ValSrc->isReg() || !TRI->isVGPR(MRI: *MRI, Reg: ValSrc->getReg()))
1426 continue;
1427
1428 Candidates.push_back(Elt: {.LshlOr: &MI, .AndMI: AndMI, .Hi: Hi, .ValSrc: ValSrc});
1429 }
1430
1431 for (const Candidate &C : Candidates) {
1432 MachineOperand *Dst = TII->getNamedOperand(MI&: *C.LshlOr, OperandName: AMDGPU::OpName::vdst);
1433
1434 Register ShiftReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1435 BuildMI(BB&: *C.LshlOr->getParent(), I&: *C.LshlOr, MIMD: C.LshlOr->getDebugLoc(),
1436 MCID: TII->get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftReg)
1437 .addImm(Val: 16)
1438 .add(MO: *C.Hi);
1439
1440 // vdst, src0_mods, src0, src1_mods, src1, clamp, dst_sel, dst_unused,
1441 // src0_sel, src1_sel.
1442 BuildMI(BB&: *C.LshlOr->getParent(), I&: *C.LshlOr, MIMD: C.LshlOr->getDebugLoc(),
1443 MCID: TII->get(Opcode: AMDGPU::V_OR_B32_sdwa))
1444 .add(MO: *Dst)
1445 .addImm(Val: 0)
1446 .addReg(RegNo: ShiftReg)
1447 .addImm(Val: 0)
1448 .add(MO: *C.ValSrc)
1449 .addImm(Val: 0)
1450 .addImm(Val: DWORD)
1451 .addImm(Val: UNUSED_PAD)
1452 .addImm(Val: DWORD)
1453 .addImm(Val: WORD_0);
1454
1455 MRI->clearKillFlags(Reg: C.ValSrc->getReg());
1456 C.LshlOr->eraseFromParent();
1457 C.AndMI->eraseFromParent();
1458 }
1459
1460 return !Candidates.empty();
1461}
1462
1463bool SIPeepholeSDWALegacy::runOnMachineFunction(MachineFunction &MF) {
1464 if (skipFunction(F: MF.getFunction()))
1465 return false;
1466
1467 return SIPeepholeSDWA().run(MF);
1468}
1469
1470bool SIPeepholeSDWA::run(MachineFunction &MF) {
1471 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1472
1473 if (!ST.hasSDWA())
1474 return false;
1475
1476 MRI = &MF.getRegInfo();
1477 TRI = ST.getRegisterInfo();
1478 TII = ST.getInstrInfo();
1479
1480 // Find all SDWA operands in MF.
1481 bool Ret = false;
1482 for (MachineBasicBlock &MBB : MF) {
1483 bool Changed = false;
1484 do {
1485 Ret |= splitLshlOrForSDWA(MBB);
1486
1487 // Preprocess the ADD/SUB pairs so they could be SDWA'ed.
1488 // Look for a possible ADD or SUB that resulted from a previously lowered
1489 // V_{ADD|SUB}_U64_PSEUDO. The function pseudoOpConvertToVOP2
1490 // lowers the pair of instructions into e32 form.
1491 matchSDWAOperands(MBB);
1492 for (const auto &OperandPair : SDWAOperands) {
1493 const auto &Operand = OperandPair.second;
1494 MachineInstr *PotentialMI = Operand->potentialToConvert(TII, ST);
1495 if (!PotentialMI)
1496 continue;
1497
1498 switch (PotentialMI->getOpcode()) {
1499 case AMDGPU::V_ADD_CO_U32_e64:
1500 case AMDGPU::V_SUB_CO_U32_e64:
1501 pseudoOpConvertToVOP2(MI&: *PotentialMI, ST);
1502 break;
1503 case AMDGPU::V_CNDMASK_B32_e64:
1504 convertVcndmaskToVOP2(MI&: *PotentialMI, ST);
1505 break;
1506 };
1507 }
1508 SDWAOperands.clear();
1509
1510 // Generate potential match list.
1511 matchSDWAOperands(MBB);
1512
1513 for (const auto &OperandPair : SDWAOperands) {
1514 const auto &Operand = OperandPair.second;
1515 MachineInstr *PotentialMI =
1516 Operand->potentialToConvert(TII, ST, PotentialMatches: &PotentialMatches);
1517
1518 if (PotentialMI && isConvertibleToSDWA(MI&: *PotentialMI, ST, TII))
1519 PotentialMatches[PotentialMI].push_back(Elt: Operand.get());
1520 }
1521
1522 for (auto &PotentialPair : PotentialMatches) {
1523 MachineInstr &PotentialMI = *PotentialPair.first;
1524 convertToSDWA(MI&: PotentialMI, SDWAOperands: PotentialPair.second);
1525 }
1526
1527 PotentialMatches.clear();
1528 SDWAOperands.clear();
1529
1530 Changed = !ConvertedInstructions.empty();
1531
1532 if (Changed)
1533 Ret = true;
1534 while (!ConvertedInstructions.empty())
1535 legalizeScalarOperands(MI&: *ConvertedInstructions.pop_back_val(), ST);
1536 } while (Changed);
1537 }
1538
1539 return Ret;
1540}
1541
1542PreservedAnalyses SIPeepholeSDWAPass::run(MachineFunction &MF,
1543 MachineFunctionAnalysisManager &) {
1544 if (MF.getFunction().hasOptNone() || !SIPeepholeSDWA().run(MF))
1545 return PreservedAnalyses::all();
1546
1547 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1548 PA.preserveSet<CFGAnalyses>();
1549 return PA;
1550}
1551