1//===- SIPeepholeSDWA.cpp - Peephole optimization for SDWA instructions ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass tries to apply several peephole SDWA patterns.
10///
11/// E.g. original:
12/// V_LSHRREV_B32_e32 %0, 16, %1
13/// V_ADD_CO_U32_e32 %2, %0, %3
14/// V_LSHLREV_B32_e32 %4, 16, %2
15///
16/// Replace:
17/// V_ADD_CO_U32_sdwa %4, %1, %3
18/// dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
19///
20//===----------------------------------------------------------------------===//
21
22#include "SIPeepholeSDWA.h"
23#include "AMDGPU.h"
24#include "GCNSubtarget.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/CodeGen/MachineFunctionPass.h"
27#include <optional>
28
29using namespace llvm;
30
31#define DEBUG_TYPE "si-peephole-sdwa"
32
33STATISTIC(NumSDWAPatternsFound, "Number of SDWA patterns found.");
34STATISTIC(NumSDWAInstructionsPeepholed,
35 "Number of instruction converted to SDWA.");
36
37namespace {
38
39bool isConvertibleToSDWA(MachineInstr &MI, const GCNSubtarget &ST,
40 const SIInstrInfo *TII);
41class SDWAOperand;
42class SDWADstOperand;
43
44using SDWAOperandsVector = SmallVector<SDWAOperand *, 4>;
45using SDWAOperandsMap = MapVector<MachineInstr *, SDWAOperandsVector>;
46
47class SIPeepholeSDWA {
48private:
49 MachineRegisterInfo *MRI;
50 const SIRegisterInfo *TRI;
51 const SIInstrInfo *TII;
52
53 MapVector<MachineInstr *, std::unique_ptr<SDWAOperand>> SDWAOperands;
54 SDWAOperandsMap PotentialMatches;
55 SmallVector<MachineInstr *, 8> ConvertedInstructions;
56
57 std::optional<int64_t> foldToImm(const MachineOperand &Op) const;
58
59 // If MI is a v_and_b32 with a 0xffff or 0xff immediate, return the masked
60 // value operand and the matching SDWA selector (WORD_0 / BYTE_0).
61 std::optional<std::pair<MachineOperand *, AMDGPU::SDWA::SdwaSel>>
62 matchAndMask(MachineInstr &MI) const;
63
64 // VOPC SDWA instructions carry the SDWA TSFlag but have no dst_sel operand.
65 bool isSDWAWithDstSel(const MachineInstr &Inst) const;
66
67 void matchSDWAOperands(MachineBasicBlock &MBB);
68 std::unique_ptr<SDWAOperand> matchSDWAOperand(MachineInstr &MI);
69 void pseudoOpConvertToVOP2(MachineInstr &MI,
70 const GCNSubtarget &ST) const;
71 void convertVcndmaskToVOP2(MachineInstr &MI, const GCNSubtarget &ST) const;
72 MachineInstr *createSDWAVersion(MachineInstr &MI);
73 bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands);
74 void legalizeScalarOperands(MachineInstr &MI, const GCNSubtarget &ST) const;
75 bool splitLshlOrForSDWA(MachineBasicBlock &MBB);
76
77public:
78 bool run(MachineFunction &MF);
79};
80
81class SIPeepholeSDWALegacy : public MachineFunctionPass {
82public:
83 static char ID;
84
85 SIPeepholeSDWALegacy() : MachineFunctionPass(ID) {}
86
87 StringRef getPassName() const override { return "SI Peephole SDWA"; }
88
89 bool runOnMachineFunction(MachineFunction &MF) override;
90
91 void getAnalysisUsage(AnalysisUsage &AU) const override {
92 AU.setPreservesCFG();
93 MachineFunctionPass::getAnalysisUsage(AU);
94 }
95};
96
97using namespace AMDGPU::SDWA;
98
99class SDWAOperand {
100private:
101 MachineOperand *Target; // Operand that would be used in converted instruction
102 MachineOperand *Replaced; // Operand that would be replace by Target
103
104 /// Returns true iff the SDWA selection of this SDWAOperand can be combined
105 /// with the SDWA selections of its uses in \p MI.
106 virtual bool canCombineSelections(const MachineInstr &MI,
107 const SIInstrInfo *TII) = 0;
108
109public:
110 SDWAOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp)
111 : Target(TargetOp), Replaced(ReplacedOp) {
112 assert(Target->isReg());
113 assert(Replaced->isReg());
114 }
115
116 virtual ~SDWAOperand() = default;
117
118 virtual MachineInstr *potentialToConvert(const SIInstrInfo *TII,
119 const GCNSubtarget &ST,
120 SDWAOperandsMap *PotentialMatches = nullptr) = 0;
121 virtual bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) = 0;
122
123 MachineOperand *getTargetOperand() const { return Target; }
124 MachineOperand *getReplacedOperand() const { return Replaced; }
125 MachineInstr *getParentInst() const { return Target->getParent(); }
126
127 MachineRegisterInfo *getMRI() const {
128 return &getParentInst()->getMF()->getRegInfo();
129 }
130
131#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
132 virtual void print(raw_ostream& OS) const = 0;
133 void dump() const { print(dbgs()); }
134#endif
135};
136
137class SDWASrcOperand : public SDWAOperand {
138private:
139 SdwaSel SrcSel;
140 bool Abs;
141 bool Neg;
142 bool Sext;
143
144public:
145 SDWASrcOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
146 SdwaSel SrcSel_ = DWORD, bool Abs_ = false, bool Neg_ = false,
147 bool Sext_ = false)
148 : SDWAOperand(TargetOp, ReplacedOp), SrcSel(SrcSel_), Abs(Abs_),
149 Neg(Neg_), Sext(Sext_) {}
150
151 MachineInstr *potentialToConvert(const SIInstrInfo *TII,
152 const GCNSubtarget &ST,
153 SDWAOperandsMap *PotentialMatches = nullptr) override;
154 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
155 bool canCombineSelections(const MachineInstr &MI,
156 const SIInstrInfo *TII) override;
157
158 SdwaSel getSrcSel() const { return SrcSel; }
159 bool getAbs() const { return Abs; }
160 bool getNeg() const { return Neg; }
161 bool getSext() const { return Sext; }
162
163 uint64_t getSrcMods(uint64_t Mods, SdwaSel ExistingSel) const;
164
165#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
166 void print(raw_ostream& OS) const override;
167#endif
168};
169
170class SDWADstOperand : public SDWAOperand {
171private:
172 SdwaSel DstSel;
173 DstUnused DstUn;
174
175public:
176 SDWADstOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
177 SdwaSel DstSel_ = DWORD, DstUnused DstUn_ = UNUSED_PAD)
178 : SDWAOperand(TargetOp, ReplacedOp), DstSel(DstSel_), DstUn(DstUn_) {}
179
180 MachineInstr *potentialToConvert(const SIInstrInfo *TII,
181 const GCNSubtarget &ST,
182 SDWAOperandsMap *PotentialMatches = nullptr) override;
183 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
184 bool canCombineSelections(const MachineInstr &MI,
185 const SIInstrInfo *TII) override;
186
187 SdwaSel getDstSel() const { return DstSel; }
188 DstUnused getDstUnused() const { return DstUn; }
189
190#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
191 void print(raw_ostream& OS) const override;
192#endif
193};
194
195class SDWADstPreserveOperand : public SDWADstOperand {
196private:
197 MachineOperand *Preserve;
198
199public:
200 SDWADstPreserveOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
201 MachineOperand *PreserveOp, SdwaSel DstSel_ = DWORD)
202 : SDWADstOperand(TargetOp, ReplacedOp, DstSel_, UNUSED_PRESERVE),
203 Preserve(PreserveOp) {}
204
205 bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
206 bool canCombineSelections(const MachineInstr &MI,
207 const SIInstrInfo *TII) override;
208
209 MachineOperand *getPreservedOperand() const { return Preserve; }
210
211#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
212 void print(raw_ostream& OS) const override;
213#endif
214};
215
216} // end anonymous namespace
217
218INITIALIZE_PASS(SIPeepholeSDWALegacy, DEBUG_TYPE, "SI Peephole SDWA", false,
219 false)
220
221char SIPeepholeSDWALegacy::ID = 0;
222
223char &llvm::SIPeepholeSDWALegacyID = SIPeepholeSDWALegacy::ID;
224
225#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
226static raw_ostream& operator<<(raw_ostream &OS, SdwaSel Sel) {
227 switch(Sel) {
228 case BYTE_0: OS << "BYTE_0"; break;
229 case BYTE_1: OS << "BYTE_1"; break;
230 case BYTE_2: OS << "BYTE_2"; break;
231 case BYTE_3: OS << "BYTE_3"; break;
232 case WORD_0: OS << "WORD_0"; break;
233 case WORD_1: OS << "WORD_1"; break;
234 case DWORD: OS << "DWORD"; break;
235 }
236 return OS;
237}
238
239static raw_ostream& operator<<(raw_ostream &OS, const DstUnused &Un) {
240 switch(Un) {
241 case UNUSED_PAD: OS << "UNUSED_PAD"; break;
242 case UNUSED_SEXT: OS << "UNUSED_SEXT"; break;
243 case UNUSED_PRESERVE: OS << "UNUSED_PRESERVE"; break;
244 }
245 return OS;
246}
247
248LLVM_DUMP_METHOD
249void SDWASrcOperand::print(raw_ostream& OS) const {
250 OS << "SDWA src: " << *getTargetOperand()
251 << " src_sel:" << getSrcSel()
252 << " abs:" << getAbs() << " neg:" << getNeg()
253 << " sext:" << getSext() << '\n';
254}
255
256LLVM_DUMP_METHOD
257void SDWADstOperand::print(raw_ostream& OS) const {
258 OS << "SDWA dst: " << *getTargetOperand()
259 << " dst_sel:" << getDstSel()
260 << " dst_unused:" << getDstUnused() << '\n';
261}
262
263LLVM_DUMP_METHOD
264void SDWADstPreserveOperand::print(raw_ostream& OS) const {
265 OS << "SDWA preserve dst: " << *getTargetOperand()
266 << " dst_sel:" << getDstSel()
267 << " preserve:" << *getPreservedOperand() << '\n';
268}
269
270#endif
271
272static void copyRegOperand(MachineOperand &To, const MachineOperand &From) {
273 assert(To.isReg() && From.isReg());
274 To.setReg(From.getReg());
275 To.setSubReg(From.getSubReg());
276 To.setIsUndef(From.isUndef());
277 if (To.isUse()) {
278 To.setIsKill(From.isKill());
279 } else {
280 To.setIsDead(From.isDead());
281 }
282}
283
284static bool isSameReg(const MachineOperand &LHS, const MachineOperand &RHS) {
285 return LHS.isReg() &&
286 RHS.isReg() &&
287 LHS.getReg() == RHS.getReg() &&
288 LHS.getSubReg() == RHS.getSubReg();
289}
290
291static MachineOperand *findSingleRegUse(const MachineOperand *Reg,
292 const MachineRegisterInfo *MRI) {
293 if (!Reg->isReg() || !Reg->isDef())
294 return nullptr;
295
296 return MRI->getOneNonDBGUse(RegNo: Reg->getReg());
297}
298
299static MachineOperand *findSingleRegDef(const MachineOperand *Reg,
300 const MachineRegisterInfo *MRI) {
301 if (!Reg->isReg())
302 return nullptr;
303
304 return MRI->getOneDef(Reg: Reg->getReg());
305}
306
307/// Combine an SDWA source instruction's existing source selection \p Sel
308/// with the SDWA selection \p OperandSel of its operand. If the selections
309/// are compatible, return the combined selection, otherwise return a
310/// nullopt.
311/// This applies to src_sel only. Extracting from an already extracted field
312/// is again a single field, but dst_sel writes a field and leaves the rest to
313/// dst_unused, so two of them do not fold into one.
314/// See SDWADstOperand::canCombineSelections.
315/// For example, if we have Sel = BYTE_0 Sel and OperandSel = WORD_1:
316/// BYTE_0 Sel (WORD_1 Sel (%X)) -> BYTE_2 Sel (%X)
317static std::optional<SdwaSel> combineSdwaSrcSel(SdwaSel Sel,
318 SdwaSel OperandSel) {
319 if (Sel == SdwaSel::DWORD)
320 return OperandSel;
321
322 if (OperandSel == SdwaSel::DWORD)
323 return Sel;
324
325 if (Sel == SdwaSel::WORD_1 || Sel == SdwaSel::BYTE_2 ||
326 Sel == SdwaSel::BYTE_3)
327 return {};
328
329 // WORD_0 wholly contains the BYTE_0, BYTE_1 and WORD_0 that Sel can be here.
330 if (OperandSel == SdwaSel::WORD_0)
331 return Sel;
332
333 // Reading BYTE_0 of a BYTE_0 field is that same byte.
334 if (OperandSel == SdwaSel::BYTE_0 && Sel == SdwaSel::BYTE_0)
335 return SdwaSel::BYTE_0;
336
337 if (OperandSel == SdwaSel::WORD_1) {
338 if (Sel == SdwaSel::BYTE_0)
339 return SdwaSel::BYTE_2;
340 if (Sel == SdwaSel::BYTE_1)
341 return SdwaSel::BYTE_3;
342 if (Sel == SdwaSel::WORD_0)
343 return SdwaSel::WORD_1;
344 }
345
346 return {};
347}
348
349uint64_t SDWASrcOperand::getSrcMods(uint64_t Mods, SdwaSel ExistingSel) const {
350 if (Abs || Neg) {
351 assert(!Sext &&
352 "Float and integer src modifiers can't be set simultaneously");
353 Mods |= Abs ? SISrcMods::ABS : 0u;
354 Mods ^= Neg ? SISrcMods::NEG : 0u;
355 } else if (ExistingSel == SdwaSel::DWORD) {
356 // SEXT is a no-op for a DWORD selection, so MI may carry one that means
357 // nothing. The combined selection is the operand's field, so use the
358 // operand's SEXT and drop whatever MI had. When ExistingSel narrows the
359 // field instead, the operand's SEXT bits fall outside the combined
360 // field, so MI's own SEXT is kept as is.
361 Mods &= ~uint64_t(SISrcMods::SEXT);
362 Mods |= Sext ? SISrcMods::SEXT : 0u;
363 }
364
365 return Mods;
366}
367
368MachineInstr *SDWASrcOperand::potentialToConvert(const SIInstrInfo *TII,
369 const GCNSubtarget &ST,
370 SDWAOperandsMap *PotentialMatches) {
371 if (PotentialMatches != nullptr) {
372 // Fill out the map for all uses if all can be converted
373 MachineOperand *Reg = getReplacedOperand();
374 if (!Reg->isReg() || !Reg->isDef())
375 return nullptr;
376
377 for (MachineInstr &UseMI : getMRI()->use_nodbg_instructions(Reg: Reg->getReg()))
378 // Check that all instructions that use Reg can be converted
379 if (!isConvertibleToSDWA(MI&: UseMI, ST, TII) ||
380 !canCombineSelections(MI: UseMI, TII))
381 return nullptr;
382
383 // Now that it's guaranteed all uses are legal, iterate over the uses again
384 // to add them for later conversion.
385 for (MachineOperand &UseMO : getMRI()->use_nodbg_operands(Reg: Reg->getReg())) {
386 // Should not get a subregister here
387 assert(isSameReg(UseMO, *Reg));
388
389 SDWAOperandsMap &potentialMatchesMap = *PotentialMatches;
390 MachineInstr *UseMI = UseMO.getParent();
391 potentialMatchesMap[UseMI].push_back(Elt: this);
392 }
393 return nullptr;
394 }
395
396 // For SDWA src operand potential instruction is one that use register
397 // defined by parent instruction
398 MachineOperand *PotentialMO = findSingleRegUse(Reg: getReplacedOperand(), MRI: getMRI());
399 if (!PotentialMO)
400 return nullptr;
401
402 MachineInstr *Parent = PotentialMO->getParent();
403
404 return canCombineSelections(MI: *Parent, TII) ? Parent : nullptr;
405}
406
407bool SDWASrcOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
408 assert((!Sext || !TII->getSubtarget().zeroesHigh16BitsOfDest(
409 getParentInst()->getOpcode())) &&
410 "Cannot use sign-extension with instruction that zeroes high bits");
411 switch (MI.getOpcode()) {
412 case AMDGPU::V_CVT_F32_FP8_sdwa:
413 case AMDGPU::V_CVT_F32_BF8_sdwa:
414 case AMDGPU::V_CVT_PK_F32_FP8_sdwa:
415 case AMDGPU::V_CVT_PK_F32_BF8_sdwa:
416 // Does not support input modifiers: noabs, noneg, nosext.
417 return false;
418 case AMDGPU::V_CNDMASK_B32_sdwa:
419 // SISrcMods uses the same bitmask for SEXT and NEG modifiers and
420 // hence the compiler can only support one type of modifier for
421 // each SDWA instruction. For V_CNDMASK_B32_sdwa, this is NEG
422 // since its operands get printed using
423 // AMDGPUInstPrinter::printOperandAndFPInputMods which produces
424 // the output intended for NEG if SEXT is set.
425 //
426 // The ISA does actually support both modifiers on most SDWA
427 // instructions.
428 //
429 // FIXME Accept SEXT here after fixing this issue.
430 if (Sext)
431 return false;
432 break;
433 }
434
435 // Find operand in instruction that matches source operand and replace it with
436 // target operand. Set corresponding src_sel
437 bool IsPreserveSrc = false;
438 MachineOperand *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
439 MachineOperand *SrcSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_sel);
440 MachineOperand *SrcMods =
441 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
442 assert(Src && (Src->isReg() || Src->isImm()));
443 if (!isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
444 // If this is not src0 then it could be src1
445 Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
446 SrcSel = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_sel);
447 SrcMods = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
448
449 if (!Src ||
450 !isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
451 // It's possible this Src is a tied operand for
452 // UNUSED_PRESERVE, in which case we can either
453 // abandon the peephole attempt, or if legal we can
454 // copy the target operand into the tied slot
455 // if the preserve operation will effectively cause the same
456 // result by overwriting the rest of the dst.
457 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
458 MachineOperand *DstUnused =
459 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
460
461 if (Dst &&
462 DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) {
463 // This will work if the tied src is accessing WORD_0, and the dst is
464 // writing WORD_1. Modifiers don't matter because all the bits that
465 // would be impacted are being overwritten by the dst.
466 // Any other case will not work.
467 SdwaSel DstSel = static_cast<SdwaSel>(
468 TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::dst_sel));
469 if (DstSel == AMDGPU::SDWA::SdwaSel::WORD_1 &&
470 getSrcSel() == AMDGPU::SDWA::SdwaSel::WORD_0) {
471 IsPreserveSrc = true;
472 auto DstIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
473 Name: AMDGPU::OpName::vdst);
474 auto TiedIdx = MI.findTiedOperandIdx(OpIdx: DstIdx);
475 Src = &MI.getOperand(i: TiedIdx);
476 SrcSel = nullptr;
477 SrcMods = nullptr;
478 } else {
479 // Not legal to convert this src
480 return false;
481 }
482 }
483 }
484 assert(Src && Src->isReg());
485
486 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa ||
487 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa ||
488 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
489 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
490 !isSameReg(LHS: *Src, RHS: *getReplacedOperand())) {
491 // In case of v_mac_f16/32_sdwa this pass can try to apply src operand to
492 // src2. This is not allowed.
493 return false;
494 }
495
496 assert(isSameReg(*Src, *getReplacedOperand()) &&
497 (IsPreserveSrc || (SrcSel && SrcMods)));
498 }
499 copyRegOperand(To&: *Src, From: *getTargetOperand());
500 if (!IsPreserveSrc) {
501 SdwaSel ExistingSel = static_cast<SdwaSel>(SrcSel->getImm());
502 SrcSel->setImm(*combineSdwaSrcSel(Sel: ExistingSel, OperandSel: getSrcSel()));
503 SrcMods->setImm(getSrcMods(Mods: SrcMods->getImm(), ExistingSel));
504 }
505 getTargetOperand()->setIsKill(false);
506 return true;
507}
508
509/// Verify that \p Op is the same register as the operand of the SDWA
510/// instruction \p MI named by \p SrcOpName and that the SDWA
511/// selection \p SrcSelOpName can be combined with the \p OpSel.
512static bool canCombineOpSel(const MachineInstr &MI, const SIInstrInfo *TII,
513 AMDGPU::OpName SrcOpName,
514 AMDGPU::OpName SrcSelOpName, MachineOperand *Op,
515 SdwaSel OpSel) {
516 assert(TII->isSDWA(MI.getOpcode()));
517
518 const MachineOperand *Src = TII->getNamedOperand(MI, OperandName: SrcOpName);
519 if (!Src || !isSameReg(LHS: *Src, RHS: *Op))
520 return true;
521
522 auto SrcSel = static_cast<SdwaSel>(TII->getNamedImmOperand(MI, OperandName: SrcSelOpName));
523 return combineSdwaSrcSel(Sel: SrcSel, OperandSel: OpSel).has_value();
524}
525
526bool SDWASrcOperand::canCombineSelections(const MachineInstr &MI,
527 const SIInstrInfo *TII) {
528 if (!TII->isSDWA(Opcode: MI.getOpcode()))
529 return true;
530
531 using namespace AMDGPU;
532
533 return canCombineOpSel(MI, TII, SrcOpName: OpName::src0, SrcSelOpName: OpName::src0_sel,
534 Op: getReplacedOperand(), OpSel: getSrcSel()) &&
535 canCombineOpSel(MI, TII, SrcOpName: OpName::src1, SrcSelOpName: OpName::src1_sel,
536 Op: getReplacedOperand(), OpSel: getSrcSel());
537}
538
539MachineInstr *SDWADstOperand::potentialToConvert(const SIInstrInfo *TII,
540 const GCNSubtarget &ST,
541 SDWAOperandsMap *PotentialMatches) {
542 // For SDWA dst operand potential instruction is one that defines register
543 // that this operand uses
544 MachineRegisterInfo *MRI = getMRI();
545 MachineInstr *ParentMI = getParentInst();
546
547 MachineOperand *PotentialMO = findSingleRegDef(Reg: getReplacedOperand(), MRI);
548 if (!PotentialMO)
549 return nullptr;
550
551 // Check that ParentMI is the only instruction that uses replaced register
552 for (MachineInstr &UseInst : MRI->use_nodbg_instructions(Reg: PotentialMO->getReg())) {
553 if (&UseInst != ParentMI)
554 return nullptr;
555 }
556
557 MachineInstr *Parent = PotentialMO->getParent();
558 return canCombineSelections(MI: *Parent, TII) ? Parent : nullptr;
559}
560
561bool SDWADstOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
562 // Replace vdst operand in MI with target operand. Set dst_sel and dst_unused
563
564 if ((MI.getOpcode() == AMDGPU::V_FMAC_F16_sdwa ||
565 MI.getOpcode() == AMDGPU::V_FMAC_F32_sdwa ||
566 MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
567 MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
568 getDstSel() != AMDGPU::SDWA::DWORD) {
569 // v_mac_f16/32_sdwa allow dst_sel to be equal only to DWORD
570 return false;
571 }
572
573 MachineOperand *Operand = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
574 assert(Operand &&
575 Operand->isReg() &&
576 isSameReg(*Operand, *getReplacedOperand()));
577 copyRegOperand(To&: *Operand, From: *getTargetOperand());
578 MachineOperand *DstSel= TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_sel);
579 assert(DstSel);
580
581 DstSel->setImm(getDstSel());
582
583 MachineOperand *DstUnused= TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dst_unused);
584 assert(DstUnused);
585 DstUnused->setImm(getDstUnused());
586
587 // Remove original instruction because it would conflict with our new
588 // instruction by register definition
589 getParentInst()->eraseFromParent();
590 return true;
591}
592
593bool SDWADstOperand::canCombineSelections(const MachineInstr &MI,
594 const SIInstrInfo *TII) {
595 if (!TII->isSDWA(Opcode: MI.getOpcode()))
596 return true;
597
598 // Destination selections are never combined, so only fold into an MI that
599 // is not yet writing a sub-field of its destination.
600 return TII->getNamedImmOperand(MI, OperandName: AMDGPU::OpName::dst_sel) == SdwaSel::DWORD;
601}
602
603bool SDWADstPreserveOperand::convertToSDWA(MachineInstr &MI,
604 const SIInstrInfo *TII) {
605 // Move MI before v_or_b32
606 MI.getParent()->remove(I: &MI);
607 getParentInst()->getParent()->insert(I: getParentInst(), MI: &MI);
608
609 // Add Implicit use of preserved register
610 MachineInstrBuilder MIB(*MI.getMF(), MI);
611 MIB.addReg(RegNo: getPreservedOperand()->getReg(), Flags: RegState::Implicit,
612 SubReg: getPreservedOperand()->getSubReg());
613
614 // Tie dst to implicit use
615 MI.tieOperands(DefIdx: AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdst),
616 UseIdx: MI.getNumOperands() - 1);
617
618 // Convert MI as any other SDWADstOperand and remove v_or_b32
619 return SDWADstOperand::convertToSDWA(MI, TII);
620}
621
622bool SDWADstPreserveOperand::canCombineSelections(const MachineInstr &MI,
623 const SIInstrInfo *TII) {
624 // DstSel came from the dst_sel already on MI, only dst_unused changes here.
625 assert(TII->getNamedImmOperand(MI, AMDGPU::OpName::dst_sel) == getDstSel());
626 return true;
627}
628
629std::optional<int64_t>
630SIPeepholeSDWA::foldToImm(const MachineOperand &Op) const {
631 if (Op.isImm()) {
632 return Op.getImm();
633 }
634
635 // If this is not immediate then it can be copy of immediate value, e.g.:
636 // %1 = S_MOV_B32 255;
637 if (Op.isReg()) {
638 for (const MachineOperand &Def : MRI->def_operands(Reg: Op.getReg())) {
639 if (!isSameReg(LHS: Op, RHS: Def))
640 continue;
641
642 const MachineInstr *DefInst = Def.getParent();
643 if (!TII->isFoldableCopy(MI: *DefInst))
644 return std::nullopt;
645
646 const MachineOperand &Copied = DefInst->getOperand(i: 1);
647 if (!Copied.isImm())
648 return std::nullopt;
649
650 return Copied.getImm();
651 }
652 }
653
654 return std::nullopt;
655}
656
657std::optional<std::pair<MachineOperand *, SdwaSel>>
658SIPeepholeSDWA::matchAndMask(MachineInstr &MI) const {
659 if (MI.getOpcode() != AMDGPU::V_AND_B32_e32 &&
660 MI.getOpcode() != AMDGPU::V_AND_B32_e64)
661 return std::nullopt;
662
663 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
664 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
665 MachineOperand *ValSrc = Src1;
666 std::optional<int64_t> Imm = foldToImm(Op: *Src0);
667 if (!Imm) {
668 Imm = foldToImm(Op: *Src1);
669 ValSrc = Src0;
670 }
671 if (!Imm || (*Imm != 0x0000ffff && *Imm != 0x000000ff))
672 return std::nullopt;
673
674 return std::make_pair(x&: ValSrc, y: *Imm == 0x0000ffff ? WORD_0 : BYTE_0);
675}
676
677bool SIPeepholeSDWA::isSDWAWithDstSel(const MachineInstr &Inst) const {
678 return TII->isSDWA(MI: Inst) &&
679 AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::dst_sel);
680}
681
682std::unique_ptr<SDWAOperand>
683SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
684 unsigned Opcode = MI.getOpcode();
685 switch (Opcode) {
686 case AMDGPU::V_LSHRREV_B32_e32:
687 case AMDGPU::V_ASHRREV_I32_e32:
688 case AMDGPU::V_LSHLREV_B32_e32:
689 case AMDGPU::V_LSHRREV_B32_e64:
690 case AMDGPU::V_ASHRREV_I32_e64:
691 case AMDGPU::V_LSHLREV_B32_e64: {
692 // from: v_lshrrev_b32_e32 v1, 16/24, v0
693 // to SDWA src:v0 src_sel:WORD_1/BYTE_3
694
695 // from: v_ashrrev_i32_e32 v1, 16/24, v0
696 // to SDWA src:v0 src_sel:WORD_1/BYTE_3 sext:1
697
698 // from: v_lshlrev_b32_e32 v1, 16/24, v0
699 // to SDWA dst:v1 dst_sel:WORD_1/BYTE_3 dst_unused:UNUSED_PAD
700 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
701 auto Imm = foldToImm(Op: *Src0);
702 if (!Imm)
703 break;
704
705 if (*Imm != 16 && *Imm != 24)
706 break;
707
708 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
709 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
710 if (!Src1->isReg() || Src1->getReg().isPhysical() ||
711 Dst->getReg().isPhysical())
712 break;
713
714 if (Opcode == AMDGPU::V_LSHLREV_B32_e32 ||
715 Opcode == AMDGPU::V_LSHLREV_B32_e64) {
716 return std::make_unique<SDWADstOperand>(
717 args&: Dst, args&: Src1, args: *Imm == 16 ? WORD_1 : BYTE_3, args: UNUSED_PAD);
718 }
719 return std::make_unique<SDWASrcOperand>(
720 args&: Src1, args&: Dst, args: *Imm == 16 ? WORD_1 : BYTE_3, args: false, args: false,
721 args: Opcode != AMDGPU::V_LSHRREV_B32_e32 &&
722 Opcode != AMDGPU::V_LSHRREV_B32_e64);
723 break;
724 }
725
726 case AMDGPU::V_LSHRREV_B16_e32:
727 case AMDGPU::V_LSHLREV_B16_e32:
728 case AMDGPU::V_LSHRREV_B16_e64:
729 case AMDGPU::V_LSHRREV_B16_opsel_e64:
730 case AMDGPU::V_LSHLREV_B16_opsel_e64:
731 case AMDGPU::V_LSHLREV_B16_e64: {
732 // V_ASHRREV_I16_e32 and V_ASHRREV_I16_e64 are
733 // not included here because they zero-fill the high 16-bits.
734
735 // from: v_lshrrev_b16_e32 v1, 8, v0
736 // to SDWA src:v0 src_sel:BYTE_1
737
738 // from: v_lshlrev_b16_e32 v1, 8, v0
739 // to SDWA dst:v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD
740 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
741 auto Imm = foldToImm(Op: *Src0);
742 if (!Imm || *Imm != 8)
743 break;
744
745 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
746 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
747
748 if (!Src1->isReg() || Src1->getReg().isPhysical() ||
749 Dst->getReg().isPhysical())
750 break;
751
752 if (Opcode == AMDGPU::V_LSHLREV_B16_e32 ||
753 Opcode == AMDGPU::V_LSHLREV_B16_opsel_e64 ||
754 Opcode == AMDGPU::V_LSHLREV_B16_e64)
755 return std::make_unique<SDWADstOperand>(args&: Dst, args&: Src1, args: BYTE_1, args: UNUSED_PAD);
756 return std::make_unique<SDWASrcOperand>(args&: Src1, args&: Dst, args: BYTE_1, args: false, args: false,
757 args: false);
758 break;
759 }
760
761 case AMDGPU::V_BFE_I32_e64:
762 case AMDGPU::V_BFE_U32_e64: {
763 // e.g.:
764 // from: v_bfe_u32 v1, v0, 8, 8
765 // to SDWA src:v0 src_sel:BYTE_1
766
767 // offset | width | src_sel
768 // ------------------------
769 // 0 | 8 | BYTE_0
770 // 0 | 16 | WORD_0
771 // 0 | 32 | DWORD ?
772 // 8 | 8 | BYTE_1
773 // 16 | 8 | BYTE_2
774 // 16 | 16 | WORD_1
775 // 24 | 8 | BYTE_3
776
777 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
778 auto Offset = foldToImm(Op: *Src1);
779 if (!Offset)
780 break;
781
782 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
783 auto Width = foldToImm(Op: *Src2);
784 if (!Width)
785 break;
786
787 SdwaSel SrcSel = DWORD;
788
789 if (*Offset == 0 && *Width == 8)
790 SrcSel = BYTE_0;
791 else if (*Offset == 0 && *Width == 16)
792 SrcSel = WORD_0;
793 else if (*Offset == 0 && *Width == 32)
794 SrcSel = DWORD;
795 else if (*Offset == 8 && *Width == 8)
796 SrcSel = BYTE_1;
797 else if (*Offset == 16 && *Width == 8)
798 SrcSel = BYTE_2;
799 else if (*Offset == 16 && *Width == 16)
800 SrcSel = WORD_1;
801 else if (*Offset == 24 && *Width == 8)
802 SrcSel = BYTE_3;
803 else
804 break;
805
806 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
807 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
808
809 if (!Src0->isReg() || Src0->getReg().isPhysical() ||
810 Dst->getReg().isPhysical())
811 break;
812
813 return std::make_unique<SDWASrcOperand>(
814 args&: Src0, args&: Dst, args&: SrcSel, args: false, args: false, args: Opcode != AMDGPU::V_BFE_U32_e64);
815 }
816
817 case AMDGPU::V_AND_B32_e32:
818 case AMDGPU::V_AND_B32_e64: {
819 // e.g.:
820 // from: v_and_b32_e32 v1, 0x0000ffff/0x000000ff, v0
821 // to SDWA src:v0 src_sel:WORD_0/BYTE_0
822 auto Mask = matchAndMask(MI);
823 if (!Mask)
824 break;
825 MachineOperand *ValSrc = Mask->first;
826
827 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
828
829 if (!ValSrc->isReg() || ValSrc->getReg().isPhysical() ||
830 Dst->getReg().isPhysical())
831 break;
832
833 return std::make_unique<SDWASrcOperand>(args&: ValSrc, args&: Dst, args&: Mask->second);
834 }
835
836 case AMDGPU::V_OR_B32_e32:
837 case AMDGPU::V_OR_B32_e64: {
838 // Patterns for dst_unused:UNUSED_PRESERVE.
839 // e.g., from:
840 // v_add_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD
841 // src1_sel:WORD_1 src2_sel:WORD1
842 // v_add_f16_e32 v3, v1, v2
843 // v_or_b32_e32 v4, v0, v3
844 // to SDWA preserve dst:v4 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE preserve:v3
845
846 // Check if one of operands of v_or_b32 is SDWA instruction
847 using CheckRetType =
848 std::optional<std::pair<MachineOperand *, MachineOperand *>>;
849 auto CheckOROperandsForSDWA =
850 [&](const MachineOperand *Op1, const MachineOperand *Op2) -> CheckRetType {
851 if (!Op1 || !Op1->isReg() || !Op2 || !Op2->isReg())
852 return CheckRetType(std::nullopt);
853
854 MachineOperand *Op1Def = findSingleRegDef(Reg: Op1, MRI);
855 if (!Op1Def)
856 return CheckRetType(std::nullopt);
857
858 MachineInstr *Op1Inst = Op1Def->getParent();
859 if (!isSDWAWithDstSel(Inst: *Op1Inst))
860 return CheckRetType(std::nullopt);
861
862 MachineOperand *Op2Def = findSingleRegDef(Reg: Op2, MRI);
863 if (!Op2Def)
864 return CheckRetType(std::nullopt);
865
866 return CheckRetType(std::pair(Op1Def, Op2Def));
867 };
868
869 MachineOperand *OrSDWA = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
870 MachineOperand *OrOther = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
871 assert(OrSDWA && OrOther);
872 auto Res = CheckOROperandsForSDWA(OrSDWA, OrOther);
873 if (!Res) {
874 OrSDWA = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
875 OrOther = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
876 assert(OrSDWA && OrOther);
877 Res = CheckOROperandsForSDWA(OrSDWA, OrOther);
878 if (!Res)
879 break;
880 }
881
882 MachineOperand *OrSDWADef = Res->first;
883 MachineOperand *OrOtherDef = Res->second;
884 assert(OrSDWADef && OrOtherDef);
885
886 MachineInstr *SDWAInst = OrSDWADef->getParent();
887 MachineInstr *OtherInst = OrOtherDef->getParent();
888
889 // Check that OtherInstr is actually bitwise compatible with SDWAInst = their
890 // destination patterns don't overlap. Compatible instruction can be either
891 // regular instruction with compatible bitness or SDWA instruction with
892 // correct dst_sel
893 // SDWAInst | OtherInst bitness / OtherInst dst_sel
894 // -----------------------------------------------------
895 // DWORD | no / no
896 // WORD_0 | no / BYTE_2/3, WORD_1
897 // WORD_1 | 8/16-bit instructions / BYTE_0/1, WORD_0
898 // BYTE_0 | no / BYTE_1/2/3, WORD_1
899 // BYTE_1 | 8-bit / BYTE_0/2/3, WORD_1
900 // BYTE_2 | 8/16-bit / BYTE_0/1/3. WORD_0
901 // BYTE_3 | 8/16/24-bit / BYTE_0/1/2, WORD_0
902 // E.g. if SDWAInst is v_add_f16_sdwa dst_sel:WORD_1 then v_add_f16 is OK
903 // but v_add_f32 is not.
904
905 // TODO: add support for non-SDWA instructions as OtherInst.
906 // For now this only works with SDWA instructions. For regular instructions
907 // there is no way to determine if the instruction writes only 8/16/24-bit
908 // out of full register size and all registers are at min 32-bit wide.
909 if (!isSDWAWithDstSel(Inst: *OtherInst))
910 break;
911
912 SdwaSel DstSel = static_cast<SdwaSel>(
913 TII->getNamedImmOperand(MI: *SDWAInst, OperandName: AMDGPU::OpName::dst_sel));
914 SdwaSel OtherDstSel = static_cast<SdwaSel>(
915 TII->getNamedImmOperand(MI: *OtherInst, OperandName: AMDGPU::OpName::dst_sel));
916
917 bool DstSelAgree = false;
918 switch (DstSel) {
919 case WORD_0: DstSelAgree = ((OtherDstSel == BYTE_2) ||
920 (OtherDstSel == BYTE_3) ||
921 (OtherDstSel == WORD_1));
922 break;
923 case WORD_1: DstSelAgree = ((OtherDstSel == BYTE_0) ||
924 (OtherDstSel == BYTE_1) ||
925 (OtherDstSel == WORD_0));
926 break;
927 case BYTE_0: DstSelAgree = ((OtherDstSel == BYTE_1) ||
928 (OtherDstSel == BYTE_2) ||
929 (OtherDstSel == BYTE_3) ||
930 (OtherDstSel == WORD_1));
931 break;
932 case BYTE_1: DstSelAgree = ((OtherDstSel == BYTE_0) ||
933 (OtherDstSel == BYTE_2) ||
934 (OtherDstSel == BYTE_3) ||
935 (OtherDstSel == WORD_1));
936 break;
937 case BYTE_2: DstSelAgree = ((OtherDstSel == BYTE_0) ||
938 (OtherDstSel == BYTE_1) ||
939 (OtherDstSel == BYTE_3) ||
940 (OtherDstSel == WORD_0));
941 break;
942 case BYTE_3: DstSelAgree = ((OtherDstSel == BYTE_0) ||
943 (OtherDstSel == BYTE_1) ||
944 (OtherDstSel == BYTE_2) ||
945 (OtherDstSel == WORD_0));
946 break;
947 default: DstSelAgree = false;
948 }
949
950 if (!DstSelAgree)
951 break;
952
953 // Also OtherInst dst_unused should be UNUSED_PAD
954 DstUnused OtherDstUnused = static_cast<DstUnused>(
955 TII->getNamedImmOperand(MI: *OtherInst, OperandName: AMDGPU::OpName::dst_unused));
956 if (OtherDstUnused != DstUnused::UNUSED_PAD)
957 break;
958
959 // Create DstPreserveOperand
960 MachineOperand *OrDst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
961 assert(OrDst && OrDst->isReg());
962
963 return std::make_unique<SDWADstPreserveOperand>(
964 args&: OrDst, args&: OrSDWADef, args&: OrOtherDef, args&: DstSel);
965
966 }
967 }
968
969 return std::unique_ptr<SDWAOperand>(nullptr);
970}
971
972#if !defined(NDEBUG)
973static raw_ostream& operator<<(raw_ostream &OS, const SDWAOperand &Operand) {
974 Operand.print(OS);
975 return OS;
976}
977#endif
978
979void SIPeepholeSDWA::matchSDWAOperands(MachineBasicBlock &MBB) {
980 for (MachineInstr &MI : MBB) {
981 if (auto Operand = matchSDWAOperand(MI)) {
982 LLVM_DEBUG(dbgs() << "Match: " << MI << "To: " << *Operand << '\n');
983 SDWAOperands[&MI] = std::move(Operand);
984 ++NumSDWAPatternsFound;
985 }
986 }
987}
988
989// Convert the V_ADD_CO_U32_e64 into V_ADD_CO_U32_e32. This allows
990// isConvertibleToSDWA to perform its transformation on V_ADD_CO_U32_e32 into
991// V_ADD_CO_U32_sdwa.
992//
993// We are transforming from a VOP3 into a VOP2 form of the instruction.
994// %19:vgpr_32 = V_AND_B32_e32 255,
995// killed %16:vgpr_32, implicit $exec
996// %47:vgpr_32, %49:sreg_64_xexec = V_ADD_CO_U32_e64
997// %26.sub0:vreg_64, %19:vgpr_32, implicit $exec
998// %48:vgpr_32, dead %50:sreg_64_xexec = V_ADDC_U32_e64
999// %26.sub1:vreg_64, %54:vgpr_32, killed %49:sreg_64_xexec, implicit $exec
1000//
1001// becomes
1002// %47:vgpr_32 = V_ADD_CO_U32_sdwa
1003// 0, %26.sub0:vreg_64, 0, killed %16:vgpr_32, 0, 6, 0, 6, 0,
1004// implicit-def $vcc, implicit $exec
1005// %48:vgpr_32, dead %50:sreg_64_xexec = V_ADDC_U32_e64
1006// %26.sub1:vreg_64, %54:vgpr_32, killed $vcc, implicit $exec
1007void SIPeepholeSDWA::pseudoOpConvertToVOP2(MachineInstr &MI,
1008 const GCNSubtarget &ST) const {
1009 int Opc = MI.getOpcode();
1010 assert((Opc == AMDGPU::V_ADD_CO_U32_e64 || Opc == AMDGPU::V_SUB_CO_U32_e64) &&
1011 "Currently only handles V_ADD_CO_U32_e64 or V_SUB_CO_U32_e64");
1012
1013 // Can the candidate MI be shrunk?
1014 if (!TII->canShrink(MI, MRI: *MRI))
1015 return;
1016 Opc = AMDGPU::getVOPe32(Opcode: Opc);
1017 // Find the related ADD instruction.
1018 const MachineOperand *Sdst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
1019 if (!Sdst)
1020 return;
1021 MachineOperand *NextOp = findSingleRegUse(Reg: Sdst, MRI);
1022 if (!NextOp)
1023 return;
1024 MachineInstr &MISucc = *NextOp->getParent();
1025
1026 // Make sure the carry in/out are subsequently unused.
1027 MachineOperand *CarryIn = TII->getNamedOperand(MI&: MISucc, OperandName: AMDGPU::OpName::src2);
1028 if (!CarryIn)
1029 return;
1030 MachineOperand *CarryOut = TII->getNamedOperand(MI&: MISucc, OperandName: AMDGPU::OpName::sdst);
1031 if (!CarryOut)
1032 return;
1033 if (!MRI->hasOneNonDBGUse(RegNo: CarryIn->getReg()) ||
1034 !MRI->use_nodbg_empty(RegNo: CarryOut->getReg()))
1035 return;
1036 // Make sure VCC or its subregs are dead before MI.
1037 MachineBasicBlock &MBB = *MI.getParent();
1038 if (MISucc.getParent() != &MBB)
1039 return; // Loop depends on MI and MISucc in same MBB.
1040 MachineBasicBlock::LivenessQueryResult Liveness =
1041 MBB.computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: MI, Neighborhood: 25);
1042 if (Liveness != MachineBasicBlock::LQR_Dead)
1043 return;
1044 // Check if VCC is referenced in range of (MI,MISucc].
1045 for (auto I = std::next(x: MI.getIterator()), E = MISucc.getIterator();
1046 I != E; ++I) {
1047 if (I->modifiesRegister(Reg: AMDGPU::VCC, TRI))
1048 return;
1049 }
1050
1051 // Replace MI with V_{SUB|ADD}_I32_e32
1052 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc))
1053 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst))
1054 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
1055 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1))
1056 .setMIFlags(MI.getFlags());
1057
1058 MI.eraseFromParent();
1059
1060 // Since the carry output of MI is now VCC, update its use in MISucc.
1061
1062 MISucc.substituteRegister(FromReg: CarryIn->getReg(), ToReg: TRI->getVCC(), SubIdx: 0, RegInfo: *TRI);
1063}
1064
1065/// Try to convert an \p MI in VOP3 which takes an src2 carry-in
1066/// operand into the corresponding VOP2 form which expects the
1067/// argument in VCC. To this end, add an copy from the carry-in to
1068/// VCC. The conversion will only be applied if \p MI can be shrunk
1069/// to VOP2 and if VCC can be proven to be dead before \p MI.
1070void SIPeepholeSDWA::convertVcndmaskToVOP2(MachineInstr &MI,
1071 const GCNSubtarget &ST) const {
1072 assert(MI.getOpcode() == AMDGPU::V_CNDMASK_B32_e64);
1073
1074 LLVM_DEBUG(dbgs() << "Attempting VOP2 conversion: " << MI);
1075 if (!TII->canShrink(MI, MRI: *MRI)) {
1076 LLVM_DEBUG(dbgs() << "Cannot shrink instruction\n");
1077 return;
1078 }
1079
1080 const MachineOperand &CarryIn =
1081 *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1082 Register CarryReg = CarryIn.getReg();
1083 MachineInstr *CarryDef = MRI->getVRegDef(Reg: CarryReg);
1084 if (!CarryDef) {
1085 LLVM_DEBUG(dbgs() << "Missing carry-in operand definition\n");
1086 return;
1087 }
1088
1089 // Make sure VCC or its subregs are dead before MI.
1090 MCRegister Vcc = TRI->getVCC();
1091 MachineBasicBlock &MBB = *MI.getParent();
1092 MachineBasicBlock::LivenessQueryResult Liveness =
1093 MBB.computeRegisterLiveness(TRI, Reg: Vcc, Before: MI);
1094 if (Liveness != MachineBasicBlock::LQR_Dead) {
1095 LLVM_DEBUG(dbgs() << "VCC not known to be dead before instruction\n");
1096 return;
1097 }
1098
1099 BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: Vcc).add(MO: CarryIn);
1100
1101 auto Converted = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(),
1102 MCID: TII->get(Opcode: AMDGPU::getVOPe32(Opcode: MI.getOpcode())))
1103 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst))
1104 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0))
1105 .add(MO: *TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1))
1106 .setMIFlags(MI.getFlags());
1107 TII->fixImplicitOperands(MI&: *Converted);
1108 LLVM_DEBUG(dbgs() << "Converted to VOP2: " << *Converted);
1109 (void)Converted;
1110 MI.eraseFromParent();
1111}
1112
1113namespace {
1114bool isConvertibleToSDWA(MachineInstr &MI,
1115 const GCNSubtarget &ST,
1116 const SIInstrInfo* TII) {
1117 // Check if this is already an SDWA instruction
1118 unsigned Opc = MI.getOpcode();
1119 if (TII->isSDWA(Opcode: Opc))
1120 return true;
1121
1122 // Can only be handled after ealier conversion to
1123 // AMDGPU::V_CNDMASK_B32_e32 which is not always possible.
1124 if (Opc == AMDGPU::V_CNDMASK_B32_e64)
1125 return false;
1126
1127 // Check if this instruction has opcode that supports SDWA
1128 if (AMDGPU::getSDWAOp(Opcode: Opc) == -1)
1129 Opc = AMDGPU::getVOPe32(Opcode: Opc);
1130
1131 if (AMDGPU::getSDWAOp(Opcode: Opc) == -1)
1132 return false;
1133
1134 if (!ST.hasSDWAOmod() && TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
1135 return false;
1136
1137 if (TII->isVOPC(Opcode: Opc)) {
1138 if (!ST.hasSDWASdst()) {
1139 const MachineOperand *SDst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst);
1140 if (SDst && (SDst->getReg() != AMDGPU::VCC &&
1141 SDst->getReg() != AMDGPU::VCC_LO))
1142 return false;
1143 }
1144
1145 if (!ST.hasSDWAOutModsVOPC() &&
1146 (TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) ||
1147 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod)))
1148 return false;
1149
1150 } else if (TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst) ||
1151 !TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst)) {
1152 return false;
1153 }
1154
1155 if (!ST.hasSDWAMac() && (Opc == AMDGPU::V_FMAC_F16_e32 ||
1156 Opc == AMDGPU::V_FMAC_F32_e32 ||
1157 Opc == AMDGPU::V_MAC_F16_e32 ||
1158 Opc == AMDGPU::V_MAC_F32_e32))
1159 return false;
1160
1161 // Check if target supports this SDWA opcode
1162 if (TII->pseudoToMCOpcode(Opcode: Opc) == -1 ||
1163 TII->pseudoToMCOpcode(Opcode: AMDGPU::getSDWAOp(Opcode: Opc)) == -1)
1164 return false;
1165
1166 if (MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)) {
1167 if (!Src0->isReg() && !Src0->isImm())
1168 return false;
1169 }
1170
1171 if (MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)) {
1172 if (!Src1->isReg() && !Src1->isImm())
1173 return false;
1174 }
1175
1176 return true;
1177}
1178} // namespace
1179
1180MachineInstr *SIPeepholeSDWA::createSDWAVersion(MachineInstr &MI) {
1181 unsigned Opcode = MI.getOpcode();
1182 assert(!TII->isSDWA(Opcode));
1183
1184 int SDWAOpcode = AMDGPU::getSDWAOp(Opcode);
1185 if (SDWAOpcode == -1)
1186 SDWAOpcode = AMDGPU::getSDWAOp(Opcode: AMDGPU::getVOPe32(Opcode));
1187 assert(SDWAOpcode != -1);
1188
1189 const MCInstrDesc &SDWADesc = TII->get(Opcode: SDWAOpcode);
1190
1191 // Create SDWA version of instruction MI and initialize its operands
1192 MachineInstrBuilder SDWAInst =
1193 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: SDWADesc)
1194 .setMIFlags(MI.getFlags());
1195
1196 // Copy dst, if it is present in original then should also be present in SDWA
1197 MachineOperand *Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdst);
1198 if (Dst) {
1199 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::vdst));
1200 SDWAInst.add(MO: *Dst);
1201 } else if ((Dst = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::sdst))) {
1202 assert(Dst && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst));
1203 SDWAInst.add(MO: *Dst);
1204 } else {
1205 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::sdst));
1206 SDWAInst.addReg(RegNo: TRI->getVCC(), Flags: RegState::Define);
1207 }
1208
1209 // Copy src0, initialize src0_modifiers. All sdwa instructions has src0 and
1210 // src0_modifiers (except for v_nop_sdwa, but it can't get here)
1211 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1212 assert(Src0 && AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0) &&
1213 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_modifiers));
1214 if (auto *Mod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers))
1215 SDWAInst.addImm(Val: Mod->getImm());
1216 else
1217 SDWAInst.addImm(Val: 0);
1218 SDWAInst.add(MO: *Src0);
1219
1220 // Copy src1 if present, initialize src1_modifiers.
1221 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1222 if (Src1) {
1223 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1) &&
1224 AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_modifiers));
1225 if (auto *Mod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers))
1226 SDWAInst.addImm(Val: Mod->getImm());
1227 else
1228 SDWAInst.addImm(Val: 0);
1229 SDWAInst.add(MO: *Src1);
1230 }
1231
1232 if (SDWAOpcode == AMDGPU::V_FMAC_F16_sdwa ||
1233 SDWAOpcode == AMDGPU::V_FMAC_F32_sdwa ||
1234 SDWAOpcode == AMDGPU::V_MAC_F16_sdwa ||
1235 SDWAOpcode == AMDGPU::V_MAC_F32_sdwa) {
1236 // v_mac_f16/32 has additional src2 operand tied to vdst
1237 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1238 assert(Src2);
1239 SDWAInst.add(MO: *Src2);
1240 }
1241
1242 // Copy clamp if present, initialize otherwise
1243 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::clamp));
1244 MachineOperand *Clamp = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp);
1245 if (Clamp) {
1246 SDWAInst.add(MO: *Clamp);
1247 } else {
1248 SDWAInst.addImm(Val: 0);
1249 }
1250
1251 // Copy omod if present, initialize otherwise if needed
1252 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::omod)) {
1253 MachineOperand *OMod = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::omod);
1254 if (OMod) {
1255 SDWAInst.add(MO: *OMod);
1256 } else {
1257 SDWAInst.addImm(Val: 0);
1258 }
1259 }
1260
1261 // Initialize SDWA specific operands
1262 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::dst_sel))
1263 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1264
1265 if (AMDGPU::hasNamedOperand(Opcode: SDWAOpcode, NamedIdx: AMDGPU::OpName::dst_unused))
1266 SDWAInst.addImm(Val: AMDGPU::SDWA::DstUnused::UNUSED_PAD);
1267
1268 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src0_sel));
1269 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1270
1271 if (Src1) {
1272 assert(AMDGPU::hasNamedOperand(SDWAOpcode, AMDGPU::OpName::src1_sel));
1273 SDWAInst.addImm(Val: AMDGPU::SDWA::SdwaSel::DWORD);
1274 }
1275
1276 // Check for a preserved register that needs to be copied.
1277 MachineInstr *Ret = SDWAInst.getInstr();
1278 TII->fixImplicitOperands(MI&: *Ret);
1279 return Ret;
1280}
1281
1282bool SIPeepholeSDWA::convertToSDWA(MachineInstr &MI,
1283 const SDWAOperandsVector &SDWAOperands) {
1284 LLVM_DEBUG(dbgs() << "Convert instruction:" << MI);
1285
1286 MachineInstr *SDWAInst;
1287 if (TII->isSDWA(Opcode: MI.getOpcode())) {
1288 // Clone the instruction to allow revoking changes
1289 // made to MI during the processing of the operands
1290 // if the conversion fails.
1291 SDWAInst = MI.getMF()->CloneMachineInstr(Orig: &MI);
1292 MI.getParent()->insert(I: MI.getIterator(), M: SDWAInst);
1293 } else {
1294 SDWAInst = createSDWAVersion(MI);
1295 }
1296
1297 // Apply all sdwa operand patterns.
1298 bool Converted = false;
1299 for (auto &Operand : SDWAOperands) {
1300 LLVM_DEBUG(dbgs() << *SDWAInst << "\nOperand: " << *Operand);
1301 // There should be no intersection between SDWA operands and potential MIs
1302 // e.g.:
1303 // v_and_b32 v0, 0xff, v1 -> src:v1 sel:BYTE_0
1304 // v_and_b32 v2, 0xff, v0 -> src:v0 sel:BYTE_0
1305 // v_add_u32 v3, v4, v2
1306 //
1307 // In that example it is possible that we would fold 2nd instruction into
1308 // 3rd (v_add_u32_sdwa) and then try to fold 1st instruction into 2nd (that
1309 // was already destroyed). So if SDWAOperand is also a potential MI then do
1310 // not apply it.
1311 if (PotentialMatches.count(Key: Operand->getParentInst()) == 0)
1312 Converted |= Operand->convertToSDWA(MI&: *SDWAInst, TII);
1313 }
1314
1315 if (!Converted) {
1316 SDWAInst->eraseFromParent();
1317 return false;
1318 }
1319
1320 ConvertedInstructions.push_back(Elt: SDWAInst);
1321 LLVM_DEBUG(dbgs() << "\nInto:" << *SDWAInst << '\n');
1322 ++NumSDWAInstructionsPeepholed;
1323
1324 MI.eraseFromParent();
1325 return true;
1326}
1327
1328// If an instruction was converted to SDWA it should not have immediates or SGPR
1329// operands (allowed one SGPR on GFX9). Copy its scalar operands into VGPRs.
1330void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI,
1331 const GCNSubtarget &ST) const {
1332 const MCInstrDesc &Desc = TII->get(Opcode: MI.getOpcode());
1333 unsigned ConstantBusCount = 0;
1334 for (MachineOperand &Op : MI.explicit_uses()) {
1335 if (Op.isReg()) {
1336 if (TRI->isVGPR(MRI: *MRI, Reg: Op.getReg()))
1337 continue;
1338
1339 if (ST.hasSDWAScalar() && ConstantBusCount == 0) {
1340 ++ConstantBusCount;
1341 continue;
1342 }
1343 } else if (!Op.isImm())
1344 continue;
1345
1346 unsigned I = Op.getOperandNo();
1347 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: Desc, OpNum: I);
1348 if (!OpRC || !TRI->isVSSuperClass(RC: OpRC))
1349 continue;
1350
1351 Register VGPR = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1352 auto Copy = BuildMI(BB&: *MI.getParent(), I: MI.getIterator(), MIMD: MI.getDebugLoc(),
1353 MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VGPR);
1354 if (Op.isImm())
1355 Copy.addImm(Val: Op.getImm());
1356 else if (Op.isReg())
1357 Copy.addReg(RegNo: Op.getReg(), Flags: getKillRegState(B: Op.isKill()), SubReg: Op.getSubReg());
1358 Op.ChangeToRegister(Reg: VGPR, isDef: false);
1359 }
1360}
1361
1362// Re-fold the masked high-half pack (hi << 16) | (z & 0xffff) into a single
1363// v_or_b32_sdwa src1_sel:WORD_0, which ISel's fused v_lshl_or_b32 blocks.
1364bool SIPeepholeSDWA::splitLshlOrForSDWA(MachineBasicBlock &MBB) {
1365 struct Candidate {
1366 MachineInstr *LshlOr;
1367 MachineInstr *AndMI;
1368 MachineOperand *Hi;
1369 MachineOperand *ValSrc;
1370 };
1371 SmallVector<Candidate, 4> Candidates;
1372
1373 for (MachineInstr &MI : MBB) {
1374 if (MI.getOpcode() != AMDGPU::V_LSHL_OR_B32_e64)
1375 continue;
1376
1377 MachineOperand *Shift = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1378 std::optional<int64_t> ShiftImm = foldToImm(Op: *Shift);
1379 if (!ShiftImm || *ShiftImm != 16)
1380 continue;
1381
1382 MachineOperand *Hi = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1383 MachineOperand *Src2 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src2);
1384 // Src2 must be a virtual reg so getVRegDef below is valid.
1385 if (!Hi->isReg() || !Src2->isReg() || !Src2->getReg().isVirtual())
1386 continue;
1387
1388 // The 0xffff mask must come from a single-use v_and so it can be dropped.
1389 if (!MRI->hasOneNonDBGUse(RegNo: Src2->getReg()))
1390 continue;
1391 MachineInstr *AndMI = MRI->getVRegDef(Reg: Src2->getReg());
1392 if (!AndMI)
1393 continue;
1394 std::optional<std::pair<MachineOperand *, SdwaSel>> Mask =
1395 matchAndMask(MI&: *AndMI);
1396 if (!Mask || Mask->second != WORD_0)
1397 continue;
1398 MachineOperand *ValSrc = Mask->first;
1399 if (!ValSrc->isReg() || !TRI->isVGPR(MRI: *MRI, Reg: ValSrc->getReg()))
1400 continue;
1401
1402 Candidates.push_back(Elt: {.LshlOr: &MI, .AndMI: AndMI, .Hi: Hi, .ValSrc: ValSrc});
1403 }
1404
1405 for (const Candidate &C : Candidates) {
1406 MachineOperand *Dst = TII->getNamedOperand(MI&: *C.LshlOr, OperandName: AMDGPU::OpName::vdst);
1407
1408 Register ShiftReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1409 BuildMI(BB&: *C.LshlOr->getParent(), I&: *C.LshlOr, MIMD: C.LshlOr->getDebugLoc(),
1410 MCID: TII->get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftReg)
1411 .addImm(Val: 16)
1412 .add(MO: *C.Hi);
1413
1414 // vdst, src0_mods, src0, src1_mods, src1, clamp, dst_sel, dst_unused,
1415 // src0_sel, src1_sel.
1416 BuildMI(BB&: *C.LshlOr->getParent(), I&: *C.LshlOr, MIMD: C.LshlOr->getDebugLoc(),
1417 MCID: TII->get(Opcode: AMDGPU::V_OR_B32_sdwa))
1418 .add(MO: *Dst)
1419 .addImm(Val: 0)
1420 .addReg(RegNo: ShiftReg)
1421 .addImm(Val: 0)
1422 .add(MO: *C.ValSrc)
1423 .addImm(Val: 0)
1424 .addImm(Val: DWORD)
1425 .addImm(Val: UNUSED_PAD)
1426 .addImm(Val: DWORD)
1427 .addImm(Val: WORD_0);
1428
1429 C.LshlOr->eraseFromParent();
1430 C.AndMI->eraseFromParent();
1431 }
1432
1433 return !Candidates.empty();
1434}
1435
1436bool SIPeepholeSDWALegacy::runOnMachineFunction(MachineFunction &MF) {
1437 if (skipFunction(F: MF.getFunction()))
1438 return false;
1439
1440 return SIPeepholeSDWA().run(MF);
1441}
1442
1443bool SIPeepholeSDWA::run(MachineFunction &MF) {
1444 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1445
1446 if (!ST.hasSDWA())
1447 return false;
1448
1449 MRI = &MF.getRegInfo();
1450 TRI = ST.getRegisterInfo();
1451 TII = ST.getInstrInfo();
1452
1453 // Find all SDWA operands in MF.
1454 bool Ret = false;
1455 for (MachineBasicBlock &MBB : MF) {
1456 bool Changed = false;
1457 do {
1458 Ret |= splitLshlOrForSDWA(MBB);
1459
1460 // Preprocess the ADD/SUB pairs so they could be SDWA'ed.
1461 // Look for a possible ADD or SUB that resulted from a previously lowered
1462 // V_{ADD|SUB}_U64_PSEUDO. The function pseudoOpConvertToVOP2
1463 // lowers the pair of instructions into e32 form.
1464 matchSDWAOperands(MBB);
1465 for (const auto &OperandPair : SDWAOperands) {
1466 const auto &Operand = OperandPair.second;
1467 MachineInstr *PotentialMI = Operand->potentialToConvert(TII, ST);
1468 if (!PotentialMI)
1469 continue;
1470
1471 switch (PotentialMI->getOpcode()) {
1472 case AMDGPU::V_ADD_CO_U32_e64:
1473 case AMDGPU::V_SUB_CO_U32_e64:
1474 pseudoOpConvertToVOP2(MI&: *PotentialMI, ST);
1475 break;
1476 case AMDGPU::V_CNDMASK_B32_e64:
1477 convertVcndmaskToVOP2(MI&: *PotentialMI, ST);
1478 break;
1479 };
1480 }
1481 SDWAOperands.clear();
1482
1483 // Generate potential match list.
1484 matchSDWAOperands(MBB);
1485
1486 for (const auto &OperandPair : SDWAOperands) {
1487 const auto &Operand = OperandPair.second;
1488 MachineInstr *PotentialMI =
1489 Operand->potentialToConvert(TII, ST, PotentialMatches: &PotentialMatches);
1490
1491 if (PotentialMI && isConvertibleToSDWA(MI&: *PotentialMI, ST, TII))
1492 PotentialMatches[PotentialMI].push_back(Elt: Operand.get());
1493 }
1494
1495 for (auto &PotentialPair : PotentialMatches) {
1496 MachineInstr &PotentialMI = *PotentialPair.first;
1497 convertToSDWA(MI&: PotentialMI, SDWAOperands: PotentialPair.second);
1498 }
1499
1500 PotentialMatches.clear();
1501 SDWAOperands.clear();
1502
1503 Changed = !ConvertedInstructions.empty();
1504
1505 if (Changed)
1506 Ret = true;
1507 while (!ConvertedInstructions.empty())
1508 legalizeScalarOperands(MI&: *ConvertedInstructions.pop_back_val(), ST);
1509 } while (Changed);
1510 }
1511
1512 return Ret;
1513}
1514
1515PreservedAnalyses SIPeepholeSDWAPass::run(MachineFunction &MF,
1516 MachineFunctionAnalysisManager &) {
1517 if (MF.getFunction().hasOptNone() || !SIPeepholeSDWA().run(MF))
1518 return PreservedAnalyses::all();
1519
1520 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1521 PA.preserveSet<CFGAnalyses>();
1522 return PA;
1523}
1524