1//===-- SIFoldOperands.cpp - Fold operands --- ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7/// \file
8//===----------------------------------------------------------------------===//
9//
10
11#include "SIFoldOperands.h"
12#include "AMDGPU.h"
13#include "AMDGPULaneMaskUtils.h"
14#include "GCNSubtarget.h"
15#include "SIInstrInfo.h"
16#include "SIMachineFunctionInfo.h"
17#include "SIRegisterInfo.h"
18#include "llvm/ADT/DepthFirstIterator.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineFunctionPass.h"
21#include "llvm/CodeGen/MachineLoopInfo.h"
22#include "llvm/CodeGen/MachineOperand.h"
23#include "llvm/InitializePasses.h"
24
25#define DEBUG_TYPE "si-fold-operands"
26using namespace llvm;
27
28namespace {
29
30/// Track a value we may want to fold into downstream users, applying
31/// subregister extracts along the way.
32struct FoldableDef {
33 union {
34 MachineOperand *OpToFold = nullptr;
35 uint64_t ImmToFold;
36 int FrameIndexToFold;
37 };
38
39 /// Register class of the originally defined value.
40 const TargetRegisterClass *DefRC = nullptr;
41
42 /// Track the original defining instruction for the value.
43 const MachineInstr *DefMI = nullptr;
44
45 /// Subregister to apply to the value at the use point.
46 unsigned DefSubReg = AMDGPU::NoSubRegister;
47
48 /// Kind of value stored in the union.
49 MachineOperand::MachineOperandType Kind;
50
51 FoldableDef() = delete;
52 FoldableDef(MachineOperand &FoldOp, const TargetRegisterClass *DefRC,
53 unsigned DefSubReg = AMDGPU::NoSubRegister)
54 : DefRC(DefRC), DefSubReg(DefSubReg), Kind(FoldOp.getType()) {
55
56 if (FoldOp.isImm()) {
57 ImmToFold = FoldOp.getImm();
58 } else if (FoldOp.isFI()) {
59 FrameIndexToFold = FoldOp.getIndex();
60 } else {
61 assert(FoldOp.isReg() || FoldOp.isGlobal());
62 OpToFold = &FoldOp;
63 }
64
65 DefMI = FoldOp.getParent();
66 }
67
68 FoldableDef(int64_t FoldImm, const TargetRegisterClass *DefRC,
69 unsigned DefSubReg = AMDGPU::NoSubRegister)
70 : ImmToFold(FoldImm), DefRC(DefRC), DefSubReg(DefSubReg),
71 Kind(MachineOperand::MO_Immediate) {}
72
73 /// Copy the current def and apply \p SubReg to the value.
74 FoldableDef getWithSubReg(const SIRegisterInfo &TRI, unsigned SubReg) const {
75 FoldableDef Copy(*this);
76 Copy.DefSubReg = TRI.composeSubRegIndices(a: DefSubReg, b: SubReg);
77 return Copy;
78 }
79
80 bool isReg() const { return Kind == MachineOperand::MO_Register; }
81
82 Register getReg() const {
83 assert(isReg());
84 return OpToFold->getReg();
85 }
86
87 unsigned getSubReg() const {
88 assert(isReg());
89 return OpToFold->getSubReg();
90 }
91
92 bool isImm() const { return Kind == MachineOperand::MO_Immediate; }
93
94 bool isFI() const {
95 return Kind == MachineOperand::MO_FrameIndex;
96 }
97
98 int getFI() const {
99 assert(isFI());
100 return FrameIndexToFold;
101 }
102
103 bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; }
104
105 /// Return the effective immediate value defined by this instruction, after
106 /// application of any subregister extracts which may exist between the use
107 /// and def instruction.
108 std::optional<int64_t> getEffectiveImmVal() const {
109 assert(isImm());
110 return SIInstrInfo::extractSubregFromImm(ImmVal: ImmToFold, SubRegIndex: DefSubReg);
111 }
112
113 /// Check if it is legal to fold this effective value into \p MI's \p OpNo
114 /// operand.
115 bool isOperandLegal(const SIInstrInfo &TII, const MachineInstr &MI,
116 unsigned OpIdx) const {
117 switch (Kind) {
118 case MachineOperand::MO_Immediate: {
119 std::optional<int64_t> ImmToFold = getEffectiveImmVal();
120 if (!ImmToFold)
121 return false;
122
123 // TODO: Should verify the subregister index is supported by the class
124 // TODO: Avoid the temporary MachineOperand
125 MachineOperand TmpOp = MachineOperand::CreateImm(Val: *ImmToFold);
126 return TII.isOperandLegal(MI, OpIdx, MO: &TmpOp);
127 }
128 case MachineOperand::MO_FrameIndex: {
129 if (DefSubReg != AMDGPU::NoSubRegister)
130 return false;
131 MachineOperand TmpOp = MachineOperand::CreateFI(Idx: FrameIndexToFold);
132 return TII.isOperandLegal(MI, OpIdx, MO: &TmpOp);
133 }
134 default:
135 // TODO: Try to apply DefSubReg, for global address we can extract
136 // low/high.
137 if (DefSubReg != AMDGPU::NoSubRegister)
138 return false;
139 return TII.isOperandLegal(MI, OpIdx, MO: OpToFold);
140 }
141
142 llvm_unreachable("covered MachineOperand kind switch");
143 }
144};
145
146struct FoldCandidate {
147 MachineInstr *UseMI;
148 FoldableDef Def;
149 int ShrinkOpcode;
150 unsigned UseOpNo;
151 bool Commuted;
152
153 FoldCandidate(MachineInstr *MI, unsigned OpNo, FoldableDef Def,
154 bool Commuted = false, int ShrinkOp = -1)
155 : UseMI(MI), Def(Def), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo),
156 Commuted(Commuted) {}
157
158 bool isFI() const { return Def.isFI(); }
159
160 int getFI() const {
161 assert(isFI());
162 return Def.FrameIndexToFold;
163 }
164
165 bool isImm() const { return Def.isImm(); }
166
167 bool isReg() const { return Def.isReg(); }
168
169 Register getReg() const { return Def.getReg(); }
170
171 bool isGlobal() const { return Def.isGlobal(); }
172
173 bool needsShrink() const { return ShrinkOpcode != -1; }
174};
175
176class SIFoldOperandsImpl {
177public:
178 MachineFunction *MF;
179 MachineRegisterInfo *MRI;
180 const SIInstrInfo *TII;
181 const SIRegisterInfo *TRI;
182 const GCNSubtarget *ST;
183 const SIMachineFunctionInfo *MFI;
184 const MachineLoopInfo *MLI;
185
186 bool frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
187 const FoldableDef &OpToFold) const;
188
189 // TODO: Just use TII::getVALUOp
190 unsigned convertToVALUOp(unsigned Opc, bool UseVOP3 = false) const {
191 switch (Opc) {
192 case AMDGPU::S_ADD_I32: {
193 if (ST->hasAddNoCarryInsts())
194 return UseVOP3 ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_U32_e32;
195 return UseVOP3 ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
196 }
197 case AMDGPU::S_OR_B32:
198 return UseVOP3 ? AMDGPU::V_OR_B32_e64 : AMDGPU::V_OR_B32_e32;
199 case AMDGPU::S_AND_B32:
200 return UseVOP3 ? AMDGPU::V_AND_B32_e64 : AMDGPU::V_AND_B32_e32;
201 case AMDGPU::S_MUL_I32:
202 return AMDGPU::V_MUL_LO_U32_e64;
203 default:
204 return AMDGPU::INSTRUCTION_LIST_END;
205 }
206 }
207
208 bool foldCopyToVGPROfScalarAddOfFrameIndex(Register DstReg, Register SrcReg,
209 MachineInstr &MI) const;
210
211 bool updateOperand(FoldCandidate &Fold) const;
212
213 bool canUseImmWithOpSel(const MachineInstr *MI, unsigned UseOpNo,
214 int64_t ImmVal) const;
215
216 /// Try to fold immediate \p ImmVal into \p MI's operand at index \p UseOpNo.
217 bool tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
218 int64_t ImmVal) const;
219
220 bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
221 MachineInstr *MI, unsigned OpNo,
222 const FoldableDef &OpToFold) const;
223 bool isUseSafeToFold(const MachineInstr &MI,
224 const MachineOperand &UseMO) const;
225 bool isTemporallyDivergentUse(const FoldableDef &OpToFold,
226 const MachineInstr &UseMI) const;
227
228 const TargetRegisterClass *getRegSeqInit(
229 MachineInstr &RegSeq,
230 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs) const;
231
232 const TargetRegisterClass *
233 getRegSeqInit(SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
234 Register UseReg) const;
235
236 std::pair<int64_t, const TargetRegisterClass *>
237 isRegSeqSplat(MachineInstr &RegSeg) const;
238
239 bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
240 int64_t SplatVal,
241 const TargetRegisterClass *SplatRC) const;
242
243 bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
244 unsigned UseOpIdx,
245 SmallVectorImpl<FoldCandidate> &FoldList) const;
246 bool foldOperand(FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
247 SmallVectorImpl<FoldCandidate> &FoldList,
248 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
249
250 struct ANDMaskResult {
251 int64_t Mask;
252 Register Reg;
253 };
254
255 std::optional<ANDMaskResult> getANDMaskRegOperand(MachineInstr &AndMI) const;
256
257 bool tryConstantFoldOp(MachineInstr *MI) const;
258 bool tryFoldCndMask(MachineInstr &MI) const;
259 bool tryFoldRedundantAND(MachineInstr &ChildMI) const;
260 bool tryFoldAndExec(MachineInstr &MI) const;
261 bool foldInstOperand(MachineInstr &MI, const FoldableDef &OpToFold) const;
262
263 bool foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const;
264 bool tryFoldFoldableCopy(MachineInstr &MI,
265 MachineOperand *&CurrentKnownM0Val) const;
266
267 const MachineOperand *isClamp(const MachineInstr &MI) const;
268 bool tryFoldClamp(MachineInstr &MI);
269
270 std::pair<const MachineOperand *, int> isOMod(const MachineInstr &MI) const;
271 bool tryFoldOMod(MachineInstr &MI);
272 bool tryFoldSGPRSplatRegSequence(MachineInstr &MI);
273 bool tryFoldRegSequence(MachineInstr &MI);
274 bool tryFoldPhiAGPR(MachineInstr &MI);
275 bool tryFoldLoad(MachineInstr &MI);
276
277 bool tryOptimizeAGPRPhis(MachineBasicBlock &MBB);
278
279public:
280 SIFoldOperandsImpl() = default;
281
282 bool run(MachineFunction &MF, const MachineLoopInfo *MLI);
283};
284
285class SIFoldOperandsLegacy : public MachineFunctionPass {
286public:
287 static char ID;
288
289 SIFoldOperandsLegacy() : MachineFunctionPass(ID) {}
290
291 bool runOnMachineFunction(MachineFunction &MF) override {
292 if (skipFunction(F: MF.getFunction()))
293 return false;
294 const MachineLoopInfo *MLI =
295 &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
296 return SIFoldOperandsImpl().run(MF, MLI);
297 }
298
299 StringRef getPassName() const override { return "SI Fold Operands"; }
300
301 void getAnalysisUsage(AnalysisUsage &AU) const override {
302 AU.setPreservesCFG();
303 AU.addRequired<MachineLoopInfoWrapperPass>();
304 AU.addPreserved<MachineLoopInfoWrapperPass>();
305 MachineFunctionPass::getAnalysisUsage(AU);
306 }
307
308 MachineFunctionProperties getRequiredProperties() const override {
309 return MachineFunctionProperties().setIsSSA();
310 }
311};
312
313} // End anonymous namespace.
314
315INITIALIZE_PASS_BEGIN(SIFoldOperandsLegacy, DEBUG_TYPE, "SI Fold Operands",
316 false, false)
317INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
318INITIALIZE_PASS_END(SIFoldOperandsLegacy, DEBUG_TYPE, "SI Fold Operands", false,
319 false)
320
321char SIFoldOperandsLegacy::ID = 0;
322
323char &llvm::SIFoldOperandsLegacyID = SIFoldOperandsLegacy::ID;
324
325static const TargetRegisterClass *getRegOpRC(const MachineRegisterInfo &MRI,
326 const TargetRegisterInfo &TRI,
327 const MachineOperand &MO) {
328 const TargetRegisterClass *RC = MRI.getRegClass(Reg: MO.getReg());
329 if (const TargetRegisterClass *SubRC =
330 TRI.getSubRegisterClass(SuperRC: RC, SubRegIdx: MO.getSubReg()))
331 RC = SubRC;
332 return RC;
333}
334
335// Map multiply-accumulate opcode to corresponding multiply-add opcode if any.
336static unsigned macToMad(unsigned Opc) {
337 switch (Opc) {
338 case AMDGPU::V_MAC_F32_e64:
339 return AMDGPU::V_MAD_F32_e64;
340 case AMDGPU::V_MAC_F16_e64:
341 return AMDGPU::V_MAD_F16_e64;
342 case AMDGPU::V_FMAC_F32_e64:
343 return AMDGPU::V_FMA_F32_e64;
344 case AMDGPU::V_FMAC_F16_e64:
345 return AMDGPU::V_FMA_F16_gfx9_e64;
346 case AMDGPU::V_FMAC_F16_t16_e64:
347 return AMDGPU::V_FMA_F16_gfx9_t16_e64;
348 case AMDGPU::V_FMAC_F16_fake16_e64:
349 return AMDGPU::V_FMA_F16_gfx9_fake16_e64;
350 case AMDGPU::V_FMAC_LEGACY_F32_e64:
351 return AMDGPU::V_FMA_LEGACY_F32_e64;
352 case AMDGPU::V_FMAC_F64_e64:
353 return AMDGPU::V_FMA_F64_e64;
354 }
355 return AMDGPU::INSTRUCTION_LIST_END;
356}
357
358// TODO: Add heuristic that the frame index might not fit in the addressing mode
359// immediate offset to avoid materializing in loops.
360bool SIFoldOperandsImpl::frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
361 const FoldableDef &OpToFold) const {
362 if (!OpToFold.isFI())
363 return false;
364
365 const unsigned Opc = UseMI.getOpcode();
366 switch (Opc) {
367 case AMDGPU::S_ADD_I32:
368 case AMDGPU::S_ADD_U32:
369 case AMDGPU::V_ADD_U32_e32:
370 case AMDGPU::V_ADD_CO_U32_e32:
371 // TODO: Possibly relax hasOneUse. It matters more for mubuf, since we have
372 // to insert the wave size shift at every point we use the index.
373 // TODO: Fix depending on visit order to fold immediates into the operand
374 return UseMI.getOperand(i: OpNo == 1 ? 2 : 1).isImm() &&
375 MRI->hasOneNonDBGUse(RegNo: UseMI.getOperand(i: OpNo).getReg());
376 case AMDGPU::V_ADD_U32_e64:
377 case AMDGPU::V_ADD_CO_U32_e64:
378 return UseMI.getOperand(i: OpNo == 2 ? 3 : 2).isImm() &&
379 MRI->hasOneNonDBGUse(RegNo: UseMI.getOperand(i: OpNo).getReg());
380 default:
381 break;
382 }
383
384 if (TII->isMUBUF(MI: UseMI))
385 return OpNo == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
386 if (!TII->isFLATScratch(MI: UseMI))
387 return false;
388
389 int SIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
390 if (OpNo == SIdx)
391 return true;
392
393 int VIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
394 return OpNo == VIdx && SIdx == -1;
395}
396
397/// Fold %vgpr = COPY (S_ADD_I32 x, frameindex)
398///
399/// => %vgpr = V_ADD_U32 x, frameindex
400bool SIFoldOperandsImpl::foldCopyToVGPROfScalarAddOfFrameIndex(
401 Register DstReg, Register SrcReg, MachineInstr &MI) const {
402 if (!SrcReg.isVirtual())
403 return false;
404
405 if (TRI->isVGPR(MRI: *MRI, Reg: DstReg) && TRI->isSGPRReg(MRI: *MRI, Reg: SrcReg) &&
406 MRI->hasOneNonDBGUse(RegNo: SrcReg)) {
407 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
408 if (!Def || Def->getNumOperands() != 4)
409 return false;
410
411 MachineOperand *Src0 = &Def->getOperand(i: 1);
412 MachineOperand *Src1 = &Def->getOperand(i: 2);
413
414 // TODO: This is profitable with more operand types, and for more
415 // opcodes. But ultimately this is working around poor / nonexistent
416 // regbankselect.
417 if (!Src0->isFI() && !Src1->isFI())
418 return false;
419
420 if (Src0->isFI())
421 std::swap(a&: Src0, b&: Src1);
422
423 const bool UseVOP3 = !Src0->isImm() || TII->isInlineConstant(MO: *Src0);
424 unsigned NewOp = convertToVALUOp(Opc: Def->getOpcode(), UseVOP3);
425 if (NewOp == AMDGPU::INSTRUCTION_LIST_END ||
426 !Def->getOperand(i: 3).isDead()) // Check if scc is dead
427 return false;
428
429 MachineBasicBlock *MBB = Def->getParent();
430 const DebugLoc &DL = Def->getDebugLoc();
431 if (NewOp != AMDGPU::V_ADD_CO_U32_e32) {
432 MachineInstrBuilder Add =
433 BuildMI(BB&: *MBB, I&: *Def, MIMD: DL, MCID: TII->get(Opcode: NewOp), DestReg: DstReg);
434
435 if (Add->getDesc().getNumDefs() == 2) {
436 Register CarryOutReg = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
437 Add.addDef(RegNo: CarryOutReg, Flags: RegState::Dead);
438 MRI->setRegAllocationHint(VReg: CarryOutReg, Type: 0, PrefReg: TRI->getVCC());
439 }
440
441 Add.add(MO: *Src0).add(MO: *Src1).setMIFlags(Def->getFlags());
442 if (AMDGPU::hasNamedOperand(Opcode: NewOp, NamedIdx: AMDGPU::OpName::clamp))
443 Add.addImm(Val: 0);
444
445 Def->eraseFromParent();
446 MI.eraseFromParent();
447 return true;
448 }
449
450 assert(NewOp == AMDGPU::V_ADD_CO_U32_e32);
451
452 MachineBasicBlock::LivenessQueryResult Liveness =
453 MBB->computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: *Def, Neighborhood: 16);
454 if (Liveness == MachineBasicBlock::LQR_Dead) {
455 // TODO: If src1 satisfies operand constraints, use vop3 version.
456 BuildMI(BB&: *MBB, I&: *Def, MIMD: DL, MCID: TII->get(Opcode: NewOp), DestReg: DstReg)
457 .add(MO: *Src0)
458 .add(MO: *Src1)
459 .setOperandDead(3) // implicit-def $vcc
460 .setMIFlags(Def->getFlags());
461 Def->eraseFromParent();
462 MI.eraseFromParent();
463 return true;
464 }
465 }
466
467 return false;
468}
469
470bool SIFoldOperandsImpl::canUseImmWithOpSel(const MachineInstr *MI,
471 unsigned UseOpNo,
472 int64_t ImmVal) const {
473 if (!SIInstrFlags::isPacked(O: *MI) || SIInstrFlags::isMAI(O: *MI) ||
474 SIInstrFlags::isWMMA(O: *MI) || SIInstrFlags::isSWMMAC(O: *MI) ||
475 (ST->hasDOTOpSelHazard() && SIInstrFlags::isDOT(O: *MI)))
476 return false;
477
478 const MachineOperand &Old = MI->getOperand(i: UseOpNo);
479 int OpNo = MI->getOperandNo(I: &Old);
480
481 unsigned Opcode = MI->getOpcode();
482 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
483 switch (OpType) {
484 default:
485 return false;
486 case AMDGPU::OPERAND_REG_IMM_V2FP16:
487 case AMDGPU::OPERAND_REG_IMM_V2BF16:
488 case AMDGPU::OPERAND_REG_IMM_V2INT16:
489 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
490 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
491 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
492 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
493 // VOP3 packed instructions ignore op_sel source modifiers, we cannot encode
494 // two different constants.
495 if (SIInstrFlags::isVOP3(O: *MI) && !SIInstrFlags::isVOP3P(O: *MI) &&
496 static_cast<uint16_t>(ImmVal) != static_cast<uint16_t>(ImmVal >> 16))
497 return false;
498 break;
499 }
500
501 return true;
502}
503
504bool SIFoldOperandsImpl::tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
505 int64_t ImmVal) const {
506 MachineOperand &Old = MI->getOperand(i: UseOpNo);
507 unsigned Opcode = MI->getOpcode();
508 int OpNo = MI->getOperandNo(I: &Old);
509 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
510
511 bool BF16FromUpperFP32 = ST->hasBF16InlineConstFromUpperFP32() &&
512 (OpType == AMDGPU::OPERAND_REG_IMM_V2BF16 ||
513 OpType == AMDGPU::OPERAND_REG_INLINE_C_V2BF16);
514
515 // If the literal can be inlined as-is, apply it and short-circuit the
516 // tests below. The main motivation for this is to avoid unintuitive
517 // uses of opsel.
518 if (!BF16FromUpperFP32 && AMDGPU::isInlinableLiteralV216(Literal: ImmVal, OpType)) {
519 Old.ChangeToImmediate(ImmVal);
520 return true;
521 }
522
523 // Refer to op_sel/op_sel_hi and check if we can change the immediate and
524 // op_sel in a way that allows an inline constant.
525 AMDGPU::OpName ModName = AMDGPU::OpName::NUM_OPERAND_NAMES;
526 unsigned SrcIdx = ~0;
527 if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0)) {
528 ModName = AMDGPU::OpName::src0_modifiers;
529 SrcIdx = 0;
530 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1)) {
531 ModName = AMDGPU::OpName::src1_modifiers;
532 SrcIdx = 1;
533 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src2)) {
534 ModName = AMDGPU::OpName::src2_modifiers;
535 SrcIdx = 2;
536 }
537 assert(ModName != AMDGPU::OpName::NUM_OPERAND_NAMES);
538 int ModIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: ModName);
539 MachineOperand &Mod = MI->getOperand(i: ModIdx);
540 unsigned ModVal = Mod.getImm();
541
542 uint16_t ImmLo =
543 static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_0 ? 16 : 0));
544 uint16_t ImmHi =
545 static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_1 ? 16 : 0));
546 uint32_t Imm = (static_cast<uint32_t>(ImmHi) << 16) | ImmLo;
547 unsigned NewModVal = ModVal & ~(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
548
549 // Helper function that attempts to inline the given value with a newly
550 // chosen opsel pattern.
551 auto tryFoldToInline = [&](uint32_t Imm) -> bool {
552 uint16_t Lo = static_cast<uint16_t>(Imm);
553 uint16_t Hi = static_cast<uint16_t>(Imm >> 16);
554
555 if (BF16FromUpperFP32) {
556 // Only <C, 0>, <0, C> and <C, C> are foldable, for an inlinable C.
557 uint16_t C = Lo ? Lo : Hi;
558 bool IsBF16 = true;
559 if ((Lo && Hi && Lo != Hi) ||
560 !AMDGPU::isInlinableLiteralBF16(Literal: C, HasInv2Pi: ST->hasInv2PiInlineImm(), IsBF16: &IsBF16))
561 return false;
562
563 // BF16 inline constants are materialized as fp32, so the 16-bit value
564 // sits in the high half; integer ones are sign-extended, so it sits in
565 // the low half. The two cases therefore need opposite opsel settings.
566 int64_t NewImm = IsBF16 ? C : SignExtend64<16>(x: C);
567 uint32_t Bits = IsBF16 ? static_cast<uint32_t>(C) << 16
568 : static_cast<uint32_t>(NewImm);
569 uint16_t ReadLo = static_cast<uint16_t>(Bits);
570 uint16_t ReadHi = static_cast<uint16_t>(Bits >> 16);
571
572 // Each half has to read one of the two, so a half that wants the sign
573 // extension of a negative integer cannot be served.
574 if ((Lo != ReadLo && Lo != ReadHi) || (Hi != ReadLo && Hi != ReadHi))
575 return false;
576
577 // Set opsel for a half that reads the upper 16 bits of the constant.
578 if (Lo != ReadLo)
579 NewModVal |= SISrcMods::OP_SEL_0;
580 if (Hi == ReadHi)
581 NewModVal |= SISrcMods::OP_SEL_1;
582 Mod.setImm(NewModVal);
583 Old.ChangeToImmediate(ImmVal: NewImm);
584 return true;
585 }
586
587 if (AMDGPU::isInlinableLiteralV216(Literal: Imm, OpType)) {
588 Mod.setImm(NewModVal | SISrcMods::OP_SEL_1);
589 Old.ChangeToImmediate(ImmVal: Imm);
590 return true;
591 }
592
593 // Try to shuffle the halves around and leverage opsel to get an inline
594 // constant.
595 if (Lo == Hi) {
596 if (AMDGPU::isInlinableLiteralV216(Literal: Lo, OpType)) {
597 Mod.setImm(NewModVal);
598 Old.ChangeToImmediate(ImmVal: Lo);
599 return true;
600 }
601
602 if (static_cast<int16_t>(Lo) < 0) {
603 int32_t SExt = static_cast<int16_t>(Lo);
604 if (AMDGPU::isInlinableLiteralV216(Literal: SExt, OpType)) {
605 Mod.setImm(NewModVal);
606 Old.ChangeToImmediate(ImmVal: SExt);
607 return true;
608 }
609 }
610
611 // This check is only useful for integer instructions
612 if (OpType == AMDGPU::OPERAND_REG_IMM_V2INT16) {
613 if (AMDGPU::isInlinableLiteralV216(Literal: Lo << 16, OpType)) {
614 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
615 Old.ChangeToImmediate(ImmVal: static_cast<uint32_t>(Lo) << 16);
616 return true;
617 }
618 }
619 } else {
620 uint32_t Swapped = (static_cast<uint32_t>(Lo) << 16) | Hi;
621 if (AMDGPU::isInlinableLiteralV216(Literal: Swapped, OpType)) {
622 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0);
623 Old.ChangeToImmediate(ImmVal: Swapped);
624 return true;
625 }
626 }
627
628 return false;
629 };
630
631 if (tryFoldToInline(Imm))
632 return true;
633
634 // Replace integer addition by subtraction and vice versa if it allows
635 // folding the immediate to an inline constant.
636 //
637 // We should only ever get here for SrcIdx == 1 due to canonicalization
638 // earlier in the pipeline, but we double-check here to be safe / fully
639 // general.
640 bool IsUAdd = Opcode == AMDGPU::V_PK_ADD_U16;
641 bool IsUSub = Opcode == AMDGPU::V_PK_SUB_U16;
642 if (SrcIdx == 1 && (IsUAdd || IsUSub)) {
643 unsigned ClampIdx =
644 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::clamp);
645 bool Clamp = MI->getOperand(i: ClampIdx).getImm() != 0;
646
647 if (!Clamp) {
648 uint16_t NegLo = -static_cast<uint16_t>(Imm);
649 uint16_t NegHi = -static_cast<uint16_t>(Imm >> 16);
650 uint32_t NegImm = (static_cast<uint32_t>(NegHi) << 16) | NegLo;
651
652 if (tryFoldToInline(NegImm)) {
653 unsigned NegOpcode =
654 IsUAdd ? AMDGPU::V_PK_SUB_U16 : AMDGPU::V_PK_ADD_U16;
655 MI->setDesc(TII->get(Opcode: NegOpcode));
656 return true;
657 }
658 }
659 }
660
661 return false;
662}
663
664bool SIFoldOperandsImpl::updateOperand(FoldCandidate &Fold) const {
665 MachineInstr *MI = Fold.UseMI;
666 MachineOperand &Old = MI->getOperand(i: Fold.UseOpNo);
667 assert(Old.isReg());
668
669 std::optional<int64_t> ImmVal;
670 if (Fold.isImm())
671 ImmVal = Fold.Def.getEffectiveImmVal();
672
673 if (ImmVal && canUseImmWithOpSel(MI: Fold.UseMI, UseOpNo: Fold.UseOpNo, ImmVal: *ImmVal)) {
674 if (tryFoldImmWithOpSel(MI: Fold.UseMI, UseOpNo: Fold.UseOpNo, ImmVal: *ImmVal))
675 return true;
676
677 // We can't represent the candidate as an inline constant. Try as a literal
678 // with the original opsel, checking constant bus limitations.
679 MachineOperand New = MachineOperand::CreateImm(Val: *ImmVal);
680 int OpNo = MI->getOperandNo(I: &Old);
681 if (!TII->isOperandLegal(MI: *MI, OpIdx: OpNo, MO: &New))
682 return false;
683
684 Old.ChangeToImmediate(ImmVal: *ImmVal);
685 return true;
686 }
687
688 if ((Fold.isImm() || Fold.isFI() || Fold.isGlobal()) && Fold.needsShrink()) {
689 MachineBasicBlock *MBB = MI->getParent();
690 auto Liveness = MBB->computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: MI, Neighborhood: 16);
691 if (Liveness != MachineBasicBlock::LQR_Dead) {
692 LLVM_DEBUG(dbgs() << "Not shrinking due to live vcc: " << *MI);
693 return false;
694 }
695
696 int Op32 = Fold.ShrinkOpcode;
697 MachineOperand &Dst0 = MI->getOperand(i: 0);
698 MachineOperand &Dst1 = MI->getOperand(i: 1);
699 assert(Dst0.isDef() && Dst1.isDef());
700
701 bool HaveNonDbgCarryUse = !MRI->use_nodbg_empty(RegNo: Dst1.getReg());
702
703 const TargetRegisterClass *Dst0RC = MRI->getRegClass(Reg: Dst0.getReg());
704 Register NewReg0 = MRI->createVirtualRegister(RegClass: Dst0RC);
705
706 MachineInstr *Inst32 = TII->buildShrunkInst(MI&: *MI, NewOpcode: Op32);
707
708 if (HaveNonDbgCarryUse) {
709 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
710 DestReg: Dst1.getReg())
711 .addReg(RegNo: AMDGPU::VCC, Flags: RegState::Kill);
712 } else {
713 // We only reach here when the carry-out vcc is dead so propagate the dead
714 // flag.
715 Inst32->getOperand(i: 3).setIsDead();
716 }
717
718 // Keep the old instruction around to avoid breaking iterators, but
719 // replace it with a dummy instruction to remove uses.
720 //
721 // FIXME: We should not invert how this pass looks at operands to avoid
722 // this. Should track set of foldable movs instead of looking for uses
723 // when looking at a use.
724 Dst0.setReg(NewReg0);
725 for (unsigned I = MI->getNumOperands() - 1; I > 0; --I)
726 MI->removeOperand(OpNo: I);
727 MI->setDesc(TII->get(Opcode: AMDGPU::IMPLICIT_DEF));
728
729 if (Fold.Commuted)
730 TII->commuteInstruction(MI&: *Inst32, NewMI: false);
731 return true;
732 }
733
734 assert(!Fold.needsShrink() && "not handled");
735
736 if (ImmVal) {
737 if (Old.isTied()) {
738 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: MI->getOpcode());
739 if (NewMFMAOpc == -1)
740 return false;
741 MI->setDesc(TII->get(Opcode: NewMFMAOpc));
742 MI->untieRegOperand(OpIdx: 0);
743 const MCInstrDesc &MCID = MI->getDesc();
744 for (unsigned I = 0; I < MI->getNumDefs(); ++I)
745 if (MCID.getOperandConstraint(OpNum: I, Constraint: MCOI::EARLY_CLOBBER) != -1)
746 MI->getOperand(i: I).setIsEarlyClobber(true);
747 }
748
749 // TODO: Should we try to avoid adding this to the candidate list?
750 MachineOperand New = MachineOperand::CreateImm(Val: *ImmVal);
751 int OpNo = MI->getOperandNo(I: &Old);
752 if (!TII->isOperandLegal(MI: *MI, OpIdx: OpNo, MO: &New))
753 return false;
754
755 if (ST->hasBF16InlineConstFromUpperFP32() &&
756 OpNo ==
757 AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(), Name: AMDGPU::OpName::src0)) {
758 unsigned Opcode = MI->getOpcode();
759 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
760 if ((OpType == AMDGPU::OPERAND_REG_IMM_BF16 ||
761 OpType == AMDGPU::OPERAND_REG_INLINE_C_BF16) &&
762 TII->isInlineConstant(ImmVal: *ImmVal, OperandType: OpType)) {
763 // We can fold it, but we need to set OPSEL
764 int Mod0 =
765 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0_modifiers);
766 if (Mod0 == -1)
767 return false;
768 MachineOperand &ModOp = MI->getOperand(i: Mod0);
769 if (ModOp.getImm())
770 return false;
771 ModOp.setImm(SISrcMods::OP_SEL_0);
772 }
773 }
774
775 Old.ChangeToImmediate(ImmVal: *ImmVal);
776 return true;
777 }
778
779 if (Fold.isGlobal()) {
780 Old.ChangeToGA(GV: Fold.Def.OpToFold->getGlobal(),
781 Offset: Fold.Def.OpToFold->getOffset(),
782 TargetFlags: Fold.Def.OpToFold->getTargetFlags());
783 return true;
784 }
785
786 if (Fold.isFI()) {
787 Old.ChangeToFrameIndex(Idx: Fold.getFI());
788 return true;
789 }
790
791 MachineOperand *New = Fold.Def.OpToFold;
792
793 // Verify the register is compatible with the operand.
794 if (const TargetRegisterClass *OpRC =
795 TII->getRegClass(MCID: MI->getDesc(), OpNum: Fold.UseOpNo)) {
796 const TargetRegisterClass *NewRC =
797 TRI->getRegClassForReg(MRI: *MRI, Reg: New->getReg());
798
799 const TargetRegisterClass *ConstrainRC = OpRC;
800 if (New->getSubReg()) {
801 ConstrainRC =
802 TRI->getMatchingSuperRegClass(A: NewRC, B: OpRC, Idx: New->getSubReg());
803
804 if (!ConstrainRC)
805 return false;
806 }
807
808 if (New->getReg().isVirtual() &&
809 !MRI->constrainRegClass(Reg: New->getReg(), RC: ConstrainRC)) {
810 LLVM_DEBUG(dbgs() << "Cannot constrain " << printReg(New->getReg(), TRI)
811 << TRI->getRegClassName(ConstrainRC) << '\n');
812 return false;
813 }
814 }
815
816 // Rework once the VS_16 register class is updated to include proper
817 // 16-bit SGPRs instead of 32-bit ones.
818 if (Old.getSubReg() == AMDGPU::lo16 && TRI->isSGPRReg(MRI: *MRI, Reg: New->getReg()))
819 Old.setSubReg(AMDGPU::NoSubRegister);
820 if (New->getReg().isPhysical()) {
821 Old.substPhysReg(Reg: New->getReg(), *TRI);
822 } else {
823 Register OldReg = Old.getReg();
824 Old.substVirtReg(Reg: New->getReg(), SubIdx: New->getSubReg(), *TRI);
825 Old.setIsUndef(New->isUndef());
826
827 // If MI is in a BUNDLE, also update header's matching implicit use.
828 if (MI->isBundledWithPred()) {
829 MachineInstr &Header = *getBundleStart(I: MI->getIterator());
830 for (MachineOperand &MO : Header.operands()) {
831 if (MO.getReg() == OldReg) {
832 MO.setReg(New->getReg());
833 MO.setSubReg(New->getSubReg());
834 }
835 }
836 }
837 }
838 return true;
839}
840
841static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
842 FoldCandidate &&Entry) {
843 // Skip additional folding on the same operand.
844 for (FoldCandidate &Fold : FoldList)
845 if (Fold.UseMI == Entry.UseMI && Fold.UseOpNo == Entry.UseOpNo)
846 return;
847 LLVM_DEBUG(dbgs() << "Append " << (Entry.Commuted ? "commuted" : "normal")
848 << " operand " << Entry.UseOpNo << "\n " << *Entry.UseMI);
849 FoldList.push_back(Elt: Entry);
850}
851
852static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
853 MachineInstr *MI, unsigned OpNo,
854 const FoldableDef &FoldOp,
855 bool Commuted = false, int ShrinkOp = -1) {
856 appendFoldCandidate(FoldList,
857 Entry: FoldCandidate(MI, OpNo, FoldOp, Commuted, ShrinkOp));
858}
859
860// Returns true if the instruction is a packed F32 instruction and the
861// corresponding scalar operand reads 32 bits and replicates the bits to both
862// channels.
863static bool isPKF32InstrReplicatesLower32BitsOfScalarOperand(
864 const GCNSubtarget *ST, MachineInstr *MI, unsigned OpNo) {
865 if (!ST->hasPKF32InstsReplicatingLower32BitsOfScalarInput())
866 return false;
867 const MCOperandInfo &OpDesc = MI->getDesc().operands()[OpNo];
868 return OpDesc.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32;
869}
870
871// Packed FP32 instructions only read 32 bits from a scalar operand (SGPR or
872// literal) and replicates the bits to both channels. Therefore, if the hi and
873// lo are not same, we can't fold it.
874static bool checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(
875 const FoldableDef &OpToFold) {
876 assert(OpToFold.isImm() && "Expected immediate operand");
877 uint64_t ImmVal = OpToFold.getEffectiveImmVal().value();
878 uint32_t Lo = Lo_32(Value: ImmVal);
879 uint32_t Hi = Hi_32(Value: ImmVal);
880 return Lo == Hi;
881}
882
883bool SIFoldOperandsImpl::tryAddToFoldList(
884 SmallVectorImpl<FoldCandidate> &FoldList, MachineInstr *MI, unsigned OpNo,
885 const FoldableDef &OpToFold) const {
886 const unsigned Opc = MI->getOpcode();
887
888 auto tryToFoldAsFMAAKorMK = [&]() {
889 if (!OpToFold.isImm())
890 return false;
891
892 const bool TryAK = OpNo == 3;
893 const unsigned NewOpc = TryAK ? AMDGPU::S_FMAAK_F32 : AMDGPU::S_FMAMK_F32;
894 MI->setDesc(TII->get(Opcode: NewOpc));
895
896 // We have to fold into operand which would be Imm not into OpNo.
897 bool FoldAsFMAAKorMK =
898 tryAddToFoldList(FoldList, MI, OpNo: TryAK ? 3 : 2, OpToFold);
899 if (FoldAsFMAAKorMK) {
900 // Untie Src2 of fmac.
901 MI->untieRegOperand(OpIdx: 3);
902 // For fmamk swap operands 1 and 2 if OpToFold was meant for operand 1.
903 if (OpNo == 1) {
904 MachineOperand &Op1 = MI->getOperand(i: 1);
905 MachineOperand &Op2 = MI->getOperand(i: 2);
906 Register OldReg = Op1.getReg();
907 // Operand 2 might be an inlinable constant
908 if (Op2.isImm()) {
909 Op1.ChangeToImmediate(ImmVal: Op2.getImm());
910 Op2.ChangeToRegister(Reg: OldReg, isDef: false);
911 } else {
912 Op1.setReg(Op2.getReg());
913 Op2.setReg(OldReg);
914 }
915 }
916 return true;
917 }
918 MI->setDesc(TII->get(Opcode: Opc));
919 return false;
920 };
921
922 bool IsLegal = OpToFold.isOperandLegal(TII: *TII, MI: *MI, OpIdx: OpNo);
923 if (!IsLegal && OpToFold.isImm()) {
924 if (std::optional<int64_t> ImmVal = OpToFold.getEffectiveImmVal())
925 IsLegal = canUseImmWithOpSel(MI, UseOpNo: OpNo, ImmVal: *ImmVal);
926 }
927
928 if (!IsLegal) {
929 // Special case for v_mac_{f16, f32}_e64 if we are trying to fold into src2
930 unsigned NewOpc = macToMad(Opc);
931 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) {
932 // Check if changing this to a v_mad_{f16, f32} instruction will allow us
933 // to fold the operand.
934 MI->setDesc(TII->get(Opcode: NewOpc));
935 bool AddOpSel = !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::op_sel) &&
936 AMDGPU::hasNamedOperand(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::op_sel);
937 if (AddOpSel)
938 MI->addOperand(Op: MachineOperand::CreateImm(Val: 0));
939 bool FoldAsMAD = tryAddToFoldList(FoldList, MI, OpNo, OpToFold);
940 if (FoldAsMAD) {
941 MI->untieRegOperand(OpIdx: OpNo);
942 return true;
943 }
944 if (AddOpSel)
945 MI->removeOperand(OpNo: MI->getNumExplicitOperands() - 1);
946 MI->setDesc(TII->get(Opcode: Opc));
947 }
948
949 // Special case for s_fmac_f32 if we are trying to fold into Src2.
950 // By transforming into fmaak we can untie Src2 and make folding legal.
951 if (Opc == AMDGPU::S_FMAC_F32 && OpNo == 3) {
952 if (tryToFoldAsFMAAKorMK())
953 return true;
954 }
955
956 // Inlineable constant might have been folded into Imm operand of fmaak or
957 // fmamk and we are trying to fold a non-inlinable constant.
958 if ((Opc == AMDGPU::S_FMAAK_F32 || Opc == AMDGPU::S_FMAMK_F32) &&
959 OpToFold.isImm()) {
960 std::optional<int64_t> ImmVal = OpToFold.getEffectiveImmVal();
961 if (ImmVal && !TII->isInlineConstant(MI: *MI, OpIdx: OpNo, ImmVal: *ImmVal)) {
962 unsigned ImmIdx = Opc == AMDGPU::S_FMAAK_F32 ? 3 : 2;
963 MachineOperand &OpImm = MI->getOperand(i: ImmIdx);
964 if (!OpImm.isReg() &&
965 TII->isInlineConstant(MI: *MI, UseMO: MI->getOperand(i: OpNo), DefMO: OpImm))
966 return tryToFoldAsFMAAKorMK();
967 }
968 }
969
970 // Special case for s_setreg_b32
971 if (OpToFold.isImm()) {
972 unsigned ImmOpc = 0;
973 if (Opc == AMDGPU::S_SETREG_B32)
974 ImmOpc = AMDGPU::S_SETREG_IMM32_B32;
975 else if (Opc == AMDGPU::S_SETREG_B32_mode)
976 ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode;
977 if (ImmOpc) {
978 MI->setDesc(TII->get(Opcode: ImmOpc));
979 appendFoldCandidate(FoldList, MI, OpNo, FoldOp: OpToFold);
980 return true;
981 }
982 }
983
984 // Operand is not legal, so try to commute the instruction to
985 // see if this makes it possible to fold.
986 unsigned CommuteOpNo = TargetInstrInfo::CommuteAnyOperandIndex;
987 bool CanCommute = TII->findCommutedOpIndices(MI: *MI, SrcOpIdx0&: OpNo, SrcOpIdx1&: CommuteOpNo);
988 if (!CanCommute)
989 return false;
990
991 MachineOperand &Op = MI->getOperand(i: OpNo);
992 MachineOperand &CommutedOp = MI->getOperand(i: CommuteOpNo);
993
994 // One of operands might be an Imm operand, and OpNo may refer to it after
995 // the call of commuteInstruction() below. Such situations are avoided
996 // here explicitly as OpNo must be a register operand to be a candidate
997 // for memory folding.
998 if (!Op.isReg() || !CommutedOp.isReg())
999 return false;
1000
1001 // The same situation with an immediate could reproduce if both inputs are
1002 // the same register.
1003 if (Op.isReg() && CommutedOp.isReg() &&
1004 (Op.getReg() == CommutedOp.getReg() &&
1005 Op.getSubReg() == CommutedOp.getSubReg()))
1006 return false;
1007
1008 if (!TII->commuteInstruction(MI&: *MI, NewMI: false, OpIdx1: OpNo, OpIdx2: CommuteOpNo))
1009 return false;
1010
1011 int Op32 = -1;
1012 if (!OpToFold.isOperandLegal(TII: *TII, MI: *MI, OpIdx: CommuteOpNo)) {
1013 if ((Opc != AMDGPU::V_ADD_CO_U32_e64 && Opc != AMDGPU::V_SUB_CO_U32_e64 &&
1014 Opc != AMDGPU::V_SUBREV_CO_U32_e64) || // FIXME
1015 (!OpToFold.isImm() && !OpToFold.isFI() && !OpToFold.isGlobal())) {
1016 TII->commuteInstruction(MI&: *MI, NewMI: false, OpIdx1: OpNo, OpIdx2: CommuteOpNo);
1017 return false;
1018 }
1019
1020 // Verify the other operand is a VGPR, otherwise we would violate the
1021 // constant bus restriction.
1022 MachineOperand &OtherOp = MI->getOperand(i: OpNo);
1023 if (!OtherOp.isReg() ||
1024 !TII->getRegisterInfo().isVGPR(MRI: *MRI, Reg: OtherOp.getReg()))
1025 return false;
1026
1027 assert(MI->getOperand(1).isDef());
1028
1029 // Make sure to get the 32-bit version of the commuted opcode.
1030 unsigned MaybeCommutedOpc = MI->getOpcode();
1031 Op32 = AMDGPU::getVOPe32(Opcode: MaybeCommutedOpc);
1032 }
1033
1034 appendFoldCandidate(FoldList, MI, OpNo: CommuteOpNo, FoldOp: OpToFold, /*Commuted=*/true,
1035 ShrinkOp: Op32);
1036 return true;
1037 }
1038
1039 // Special case for s_fmac_f32 if we are trying to fold into Src0 or Src1.
1040 // By changing into fmamk we can untie Src2.
1041 // If folding for Src0 happens first and it is identical operand to Src1 we
1042 // should avoid transforming into fmamk which requires commuting as it would
1043 // cause folding into Src1 to fail later on due to wrong OpNo used.
1044 if (Opc == AMDGPU::S_FMAC_F32 &&
1045 (OpNo != 1 || !MI->getOperand(i: 1).isIdenticalTo(Other: MI->getOperand(i: 2)))) {
1046 if (tryToFoldAsFMAAKorMK())
1047 return true;
1048 }
1049
1050 // Special case for PK_F32 instructions if we are trying to fold an imm to
1051 // src0 or src1.
1052 if (OpToFold.isImm() &&
1053 isPKF32InstrReplicatesLower32BitsOfScalarOperand(ST, MI, OpNo) &&
1054 !checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(OpToFold))
1055 return false;
1056
1057 appendFoldCandidate(FoldList, MI, OpNo, FoldOp: OpToFold);
1058 return true;
1059}
1060
1061bool SIFoldOperandsImpl::isUseSafeToFold(const MachineInstr &MI,
1062 const MachineOperand &UseMO) const {
1063 // Operands of SDWA instructions must be registers.
1064 return !TII->isSDWA(MI);
1065}
1066
1067// Returns true if any instruction in \p L modifies EXEC.
1068static bool loopModifiesExec(const MachineLoop &L, const SIRegisterInfo &TRI) {
1069 for (const MachineBasicBlock *MBB : L.getBlocks())
1070 for (const MachineInstr &MI : *MBB)
1071 if (MI.modifiesRegister(Reg: TRI.getExec(), TRI: &TRI))
1072 return true;
1073 return false;
1074}
1075
1076// An SGPR->VGPR copy inside a divergent loop latches each lane value as it
1077// exits. Folding its scalar source into a use after the loop would make every
1078// lane read the same reconverged value, so do not fold across the loop exit.
1079bool SIFoldOperandsImpl::isTemporallyDivergentUse(
1080 const FoldableDef &OpToFold, const MachineInstr &UseMI) const {
1081 if (!OpToFold.isReg())
1082 return false;
1083 const MachineInstr *DefMI = OpToFold.DefMI;
1084 if (!DefMI || !DefMI->isCopy() ||
1085 TRI->isSGPRReg(MRI: *MRI, Reg: DefMI->getOperand(i: 0).getReg()) ||
1086 !TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg()))
1087 return false;
1088 const MachineLoop *DefLoop = MLI->getLoopFor(BB: DefMI->getParent());
1089 return DefLoop && !DefLoop->contains(BB: UseMI.getParent()) &&
1090 loopModifiesExec(L: *DefLoop, TRI: *TRI);
1091}
1092
1093static MachineOperand *lookUpCopyChain(const SIInstrInfo &TII,
1094 const MachineRegisterInfo &MRI,
1095 Register SrcReg) {
1096 MachineOperand *Sub = nullptr;
1097 for (MachineInstr *SubDef = MRI.getVRegDef(Reg: SrcReg);
1098 SubDef && TII.isFoldableCopy(MI: *SubDef);
1099 SubDef = MRI.getVRegDef(Reg: Sub->getReg())) {
1100 unsigned SrcIdx = TII.getFoldableCopySrcIdx(MI: *SubDef);
1101 MachineOperand &SrcOp = SubDef->getOperand(i: SrcIdx);
1102
1103 if (SrcOp.isImm())
1104 return &SrcOp;
1105 if (!SrcOp.isReg() || SrcOp.getReg().isPhysical())
1106 break;
1107 Sub = &SrcOp;
1108 // TODO: Support compose
1109 if (SrcOp.getSubReg())
1110 break;
1111 }
1112
1113 return Sub;
1114}
1115
1116const TargetRegisterClass *SIFoldOperandsImpl::getRegSeqInit(
1117 MachineInstr &RegSeq,
1118 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs) const {
1119
1120 assert(RegSeq.isRegSequence());
1121
1122 const TargetRegisterClass *RC = nullptr;
1123
1124 for (unsigned I = 1, E = RegSeq.getNumExplicitOperands(); I != E; I += 2) {
1125 MachineOperand &SrcOp = RegSeq.getOperand(i: I);
1126 if (SrcOp.getReg().isPhysical())
1127 return nullptr;
1128 unsigned SubRegIdx = RegSeq.getOperand(i: I + 1).getImm();
1129
1130 // Only accept reg_sequence with uniform reg class inputs for simplicity.
1131 const TargetRegisterClass *OpRC = getRegOpRC(MRI: *MRI, TRI: *TRI, MO: SrcOp);
1132 if (!RC)
1133 RC = OpRC;
1134 else if (!TRI->getCommonSubClass(A: RC, B: OpRC))
1135 return nullptr;
1136
1137 if (SrcOp.getSubReg()) {
1138 // TODO: Handle subregister compose
1139 Defs.emplace_back(Args: &SrcOp, Args&: SubRegIdx);
1140 continue;
1141 }
1142
1143 MachineOperand *DefSrc = lookUpCopyChain(TII: *TII, MRI: *MRI, SrcReg: SrcOp.getReg());
1144 if (DefSrc && (DefSrc->isReg() || DefSrc->isImm())) {
1145 Defs.emplace_back(Args&: DefSrc, Args&: SubRegIdx);
1146 continue;
1147 }
1148
1149 Defs.emplace_back(Args: &SrcOp, Args&: SubRegIdx);
1150 }
1151
1152 return RC;
1153}
1154
1155// Find a def of the UseReg, check if it is a reg_sequence and find initializers
1156// for each subreg, tracking it to an immediate if possible. Returns the
1157// register class of the inputs on success.
1158const TargetRegisterClass *SIFoldOperandsImpl::getRegSeqInit(
1159 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
1160 Register UseReg) const {
1161 MachineInstr *Def = MRI->getVRegDef(Reg: UseReg);
1162 if (!Def || !Def->isRegSequence())
1163 return nullptr;
1164
1165 return getRegSeqInit(RegSeq&: *Def, Defs);
1166}
1167
1168std::pair<int64_t, const TargetRegisterClass *>
1169SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
1170 SmallVector<std::pair<MachineOperand *, unsigned>, 32> Defs;
1171 const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
1172 if (!SrcRC)
1173 return {};
1174
1175 bool TryToMatchSplat64 = false;
1176
1177 std::optional<int64_t> Imm;
1178 for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
1179 const MachineOperand *Op = Defs[I].first;
1180 if (!Op->isImm()) {
1181 if (Op->isReg()) {
1182 MachineInstr *Def = MRI->getVRegDef(Reg: Op->getReg());
1183 if (!Def || Def->isImplicitDef())
1184 continue;
1185 }
1186 return {};
1187 }
1188
1189 int64_t SubImm = Op->getImm();
1190 if (!Imm) {
1191 Imm = SubImm;
1192 continue;
1193 }
1194
1195 if (Imm != SubImm) {
1196 if (I == 1 && (E & 1) == 0) {
1197 // If we have an even number of inputs, there's a chance this is a
1198 // 64-bit element splat broken into 32-bit pieces.
1199 TryToMatchSplat64 = true;
1200 break;
1201 }
1202
1203 return {}; // Can only fold splat constants
1204 }
1205 }
1206
1207 if (!TryToMatchSplat64) {
1208 if (Imm)
1209 return {*Imm, SrcRC};
1210 return {};
1211 }
1212
1213 // Fallback to recognizing 64-bit splats broken into 32-bit pieces
1214 // (i.e. recognize every other other element is 0 for 64-bit immediates)
1215 int64_t SplatVal64;
1216 for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
1217 const MachineOperand *Op0 = Defs[I].first;
1218 const MachineOperand *Op1 = Defs[I + 1].first;
1219
1220 if (!Op0->isImm() || !Op1->isImm())
1221 return {};
1222
1223 unsigned SubReg0 = Defs[I].second;
1224 unsigned SubReg1 = Defs[I + 1].second;
1225
1226 // Assume we're going to generally encounter reg_sequences with sorted
1227 // subreg indexes, so reject any that aren't consecutive.
1228 if (TRI->getChannelFromSubReg(SubReg: SubReg0) + 1 !=
1229 TRI->getChannelFromSubReg(SubReg: SubReg1))
1230 return {};
1231
1232 if (TRI->getSubRegIdxSize(Idx: SubReg0) != 32)
1233 return {};
1234
1235 int64_t MergedVal = Make_64(High: Op1->getImm(), Low: Op0->getImm());
1236 if (I == 0)
1237 SplatVal64 = MergedVal;
1238 else if (SplatVal64 != MergedVal)
1239 return {};
1240 }
1241
1242 const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
1243 MRI->getRegClass(Reg: RegSeq.getOperand(i: 0).getReg()), AMDGPU::sub0_sub1);
1244
1245 return {SplatVal64, RC64};
1246}
1247
1248bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
1249 MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
1250 const TargetRegisterClass *SplatRC) const {
1251 const MCInstrDesc &Desc = UseMI->getDesc();
1252 if (UseOpIdx >= Desc.getNumOperands())
1253 return false;
1254
1255 // Filter out unhandled pseudos.
1256 if (!AMDGPU::isSISrcOperand(Desc, OpNo: UseOpIdx))
1257 return false;
1258
1259 int16_t RCID = TII->getOpRegClassID(OpInfo: Desc.operands()[UseOpIdx]);
1260 if (RCID == -1)
1261 return false;
1262
1263 const TargetRegisterClass *OpRC = TRI->getRegClass(i: RCID);
1264
1265 // Special case 0/-1, since when interpreted as a 64-bit element both halves
1266 // have the same bits. These are the only cases where a splat has the same
1267 // interpretation for 32-bit and 64-bit splats.
1268 if (SplatVal != 0 && SplatVal != -1) {
1269 // We need to figure out the scalar type read by the operand. e.g. the MFMA
1270 // operand will be AReg_128, and we want to check if it's compatible with an
1271 // AReg_32 constant.
1272 uint8_t OpTy = Desc.operands()[UseOpIdx].OperandType;
1273 switch (OpTy) {
1274 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1275 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1276 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1277 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1278 case AMDGPU::OPERAND_REG_IMM_V2FP32:
1279 OpRC = TRI->getSubRegisterClass(OpRC, AMDGPU::sub0);
1280 break;
1281 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
1282 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1283 case AMDGPU::OPERAND_REG_IMM_V2FP64:
1284 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1285 case AMDGPU::OPERAND_REG_IMM_V2INT64:
1286 OpRC = TRI->getSubRegisterClass(OpRC, AMDGPU::sub0_sub1);
1287 break;
1288 default:
1289 return false;
1290 }
1291
1292 if (!TRI->getCommonSubClass(A: OpRC, B: SplatRC))
1293 return false;
1294 }
1295
1296 MachineOperand TmpOp = MachineOperand::CreateImm(Val: SplatVal);
1297 if (!TII->isOperandLegal(MI: *UseMI, OpIdx: UseOpIdx, MO: &TmpOp))
1298 return false;
1299
1300 return true;
1301}
1302
1303bool SIFoldOperandsImpl::tryToFoldACImm(
1304 const FoldableDef &OpToFold, MachineInstr *UseMI, unsigned UseOpIdx,
1305 SmallVectorImpl<FoldCandidate> &FoldList) const {
1306 const MCInstrDesc &Desc = UseMI->getDesc();
1307 if (UseOpIdx >= Desc.getNumOperands())
1308 return false;
1309
1310 // Filter out unhandled pseudos.
1311 if (!AMDGPU::isSISrcOperand(Desc, OpNo: UseOpIdx))
1312 return false;
1313
1314 if (OpToFold.isImm() && OpToFold.isOperandLegal(TII: *TII, MI: *UseMI, OpIdx: UseOpIdx)) {
1315 if (isPKF32InstrReplicatesLower32BitsOfScalarOperand(ST, MI: UseMI, OpNo: UseOpIdx) &&
1316 !checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(OpToFold))
1317 return false;
1318 appendFoldCandidate(FoldList, MI: UseMI, OpNo: UseOpIdx, FoldOp: OpToFold);
1319 return true;
1320 }
1321
1322 return false;
1323}
1324
1325bool SIFoldOperandsImpl::foldOperand(
1326 FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
1327 SmallVectorImpl<FoldCandidate> &FoldList,
1328 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const {
1329 bool Changed = false;
1330 const MachineOperand *UseOp = &UseMI->getOperand(i: UseOpIdx);
1331
1332 if (!isUseSafeToFold(MI: *UseMI, UseMO: *UseOp))
1333 return Changed;
1334
1335 if (isTemporallyDivergentUse(OpToFold, UseMI: *UseMI))
1336 return Changed;
1337
1338 // FIXME: Fold operands with subregs.
1339 if (UseOp->isReg() && OpToFold.isReg()) {
1340 if (UseOp->isImplicit())
1341 return Changed;
1342 // Allow folding from SGPRs to 16-bit VGPRs.
1343 if (UseOp->getSubReg() != AMDGPU::NoSubRegister &&
1344 (UseOp->getSubReg() != AMDGPU::lo16 ||
1345 !TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg())))
1346 return Changed;
1347 }
1348
1349 // Special case for REG_SEQUENCE: We can't fold literals into
1350 // REG_SEQUENCE instructions, so we have to fold them into the
1351 // uses of REG_SEQUENCE.
1352 if (UseMI->isRegSequence()) {
1353 Register RegSeqDstReg = UseMI->getOperand(i: 0).getReg();
1354 unsigned RegSeqDstSubReg = UseMI->getOperand(i: UseOpIdx + 1).getImm();
1355
1356 int64_t SplatVal;
1357 const TargetRegisterClass *SplatRC;
1358 std::tie(args&: SplatVal, args&: SplatRC) = isRegSeqSplat(RegSeq&: *UseMI);
1359
1360 // Grab the use operands first
1361 SmallVector<MachineOperand *, 4> UsesToProcess(
1362 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: RegSeqDstReg)));
1363 for (unsigned I = 0; I != UsesToProcess.size(); ++I) {
1364 MachineOperand *RSUse = UsesToProcess[I];
1365 MachineInstr *RSUseMI = RSUse->getParent();
1366 unsigned OpNo = RSUseMI->getOperandNo(I: RSUse);
1367
1368 if (SplatRC) {
1369 if (RSUseMI->isCopy()) {
1370 Register DstReg = RSUseMI->getOperand(i: 0).getReg();
1371 append_range(C&: UsesToProcess,
1372 R: make_pointer_range(Range: MRI->use_nodbg_operands(Reg: DstReg)));
1373 continue;
1374 }
1375 if (tryFoldRegSeqSplat(UseMI: RSUseMI, UseOpIdx: OpNo, SplatVal, SplatRC)) {
1376 FoldableDef SplatDef(SplatVal, SplatRC);
1377 appendFoldCandidate(FoldList, MI: RSUseMI, OpNo, FoldOp: SplatDef);
1378 Changed = true;
1379 continue;
1380 }
1381 }
1382
1383 // TODO: Handle general compose
1384 if (RSUse->getSubReg() != RegSeqDstSubReg)
1385 continue;
1386
1387 // FIXME: We should avoid recursing here. There should be a cleaner split
1388 // between the in-place mutations and adding to the fold list.
1389 Changed |= foldOperand(OpToFold, UseMI: RSUseMI, UseOpIdx: RSUseMI->getOperandNo(I: RSUse),
1390 FoldList, CopiesToReplace);
1391 }
1392
1393 return Changed;
1394 }
1395
1396 if (tryToFoldACImm(OpToFold, UseMI, UseOpIdx, FoldList))
1397 return true;
1398
1399 if (frameIndexMayFold(UseMI: *UseMI, OpNo: UseOpIdx, OpToFold)) {
1400 // Verify that this is a stack access.
1401 // FIXME: Should probably use stack pseudos before frame lowering.
1402
1403 if (TII->isMUBUF(MI: *UseMI)) {
1404 if (TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::srsrc)->getReg() !=
1405 MFI->getScratchRSrcReg())
1406 return Changed;
1407
1408 // Ensure this is either relative to the current frame or the current
1409 // wave.
1410 MachineOperand &SOff =
1411 *TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::soffset);
1412 if (!SOff.isImm() || SOff.getImm() != 0)
1413 return Changed;
1414 }
1415
1416 const unsigned Opc = UseMI->getOpcode();
1417 if (TII->isFLATScratch(MI: *UseMI) &&
1418 AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr) &&
1419 !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::saddr)) {
1420 unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(Opcode: Opc);
1421 unsigned CPol =
1422 TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::cpol)->getImm();
1423 if ((CPol & AMDGPU::CPol::SCAL) &&
1424 !AMDGPU::supportsScaleOffset(MII: *TII, Opcode: NewOpc))
1425 return Changed;
1426
1427 UseMI->setDesc(TII->get(Opcode: NewOpc));
1428 }
1429
1430 // A frame index will resolve to a positive constant, so it should always be
1431 // safe to fold the addressing mode, even pre-GFX9.
1432 UseMI->getOperand(i: UseOpIdx).ChangeToFrameIndex(Idx: OpToFold.getFI());
1433
1434 return true;
1435 }
1436
1437 bool FoldingImmLike =
1438 OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
1439
1440 if (FoldingImmLike && UseMI->isCopy()) {
1441 Register DestReg = UseMI->getOperand(i: 0).getReg();
1442 Register SrcReg = UseMI->getOperand(i: 1).getReg();
1443 unsigned UseSubReg = UseMI->getOperand(i: 1).getSubReg();
1444 assert(SrcReg.isVirtual());
1445
1446 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: SrcReg);
1447
1448 // Don't fold into a copy to a physical register with the same class. Doing
1449 // so would interfere with the register coalescer's logic which would avoid
1450 // redundant initializations.
1451 if (DestReg.isPhysical() && SrcRC->contains(Reg: DestReg))
1452 return Changed;
1453
1454 const TargetRegisterClass *DestRC = TRI->getRegClassForReg(MRI: *MRI, Reg: DestReg);
1455 // In order to fold immediates into copies, we need to change the copy to a
1456 // MOV. Find a compatible mov instruction with the value.
1457 for (unsigned MovOp :
1458 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
1459 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_MOV_B16_t16_e64,
1460 AMDGPU::V_ACCVGPR_WRITE_B32_e64, AMDGPU::AV_MOV_B32_IMM_PSEUDO,
1461 AMDGPU::AV_MOV_B64_IMM_PSEUDO}) {
1462 const MCInstrDesc &MovDesc = TII->get(Opcode: MovOp);
1463 const TargetRegisterClass *MovDstRC =
1464 TRI->getRegClass(i: TII->getOpRegClassID(OpInfo: MovDesc.operands()[0]));
1465
1466 // Fold if the destination register class of the MOV instruction (ResRC)
1467 // is a superclass of (or equal to) the destination register class of the
1468 // COPY (DestRC). If this condition fails, folding would be illegal.
1469 if (!DestRC->hasSuperClassEq(RC: MovDstRC))
1470 continue;
1471
1472 const int SrcIdx = MovOp == AMDGPU::V_MOV_B16_t16_e64 ? 2 : 1;
1473
1474 int16_t RegClassID = TII->getOpRegClassID(OpInfo: MovDesc.operands()[SrcIdx]);
1475 if (RegClassID != -1) {
1476 const TargetRegisterClass *MovSrcRC = TRI->getRegClass(i: RegClassID);
1477
1478 if (UseSubReg)
1479 MovSrcRC = TRI->getMatchingSuperRegClass(A: SrcRC, B: MovSrcRC, Idx: UseSubReg);
1480
1481 // FIXME: We should be able to directly check immediate operand legality
1482 // for all cases, but gfx908 hacks break.
1483 if (MovOp == AMDGPU::AV_MOV_B32_IMM_PSEUDO &&
1484 (!OpToFold.isImm() ||
1485 !TII->isImmOperandLegal(InstDesc: MovDesc, OpNo: SrcIdx,
1486 ImmVal: *OpToFold.getEffectiveImmVal())))
1487 break;
1488
1489 if (!MRI->constrainRegClass(Reg: SrcReg, RC: MovSrcRC))
1490 break;
1491
1492 // FIXME: This is mutating the instruction only and deferring the actual
1493 // fold of the immediate
1494 } else {
1495 // For the _IMM_PSEUDO cases, there can be value restrictions on the
1496 // immediate to verify. Technically we should always verify this, but it
1497 // only matters for these concrete cases.
1498 // TODO: Handle non-imm case if it's useful.
1499 if (!OpToFold.isImm() ||
1500 !TII->isImmOperandLegal(InstDesc: MovDesc, OpNo: 1, ImmVal: *OpToFold.getEffectiveImmVal()))
1501 break;
1502 }
1503
1504 MachineInstr::mop_iterator ImpOpI = UseMI->implicit_operands().begin();
1505 MachineInstr::mop_iterator ImpOpE = UseMI->implicit_operands().end();
1506 while (ImpOpI != ImpOpE) {
1507 MachineInstr::mop_iterator Tmp = ImpOpI;
1508 ImpOpI++;
1509 UseMI->removeOperand(OpNo: UseMI->getOperandNo(I: Tmp));
1510 }
1511 UseMI->setDesc(MovDesc);
1512
1513 if (MovOp == AMDGPU::V_MOV_B16_t16_e64) {
1514 const auto &SrcOp = UseMI->getOperand(i: UseOpIdx);
1515 MachineOperand NewSrcOp(SrcOp);
1516 UseMI->removeOperand(OpNo: 1);
1517 UseMI->addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0)); // src0_modifiers
1518 UseMI->addOperand(Op: NewSrcOp); // src0
1519 UseMI->addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0)); // op_sel
1520 UseOpIdx = SrcIdx;
1521 UseOp = &UseMI->getOperand(i: UseOpIdx);
1522 }
1523 CopiesToReplace.push_back(Elt: UseMI);
1524 Changed = true;
1525 break;
1526 }
1527
1528 // We failed to replace the copy, so give up.
1529 if (UseMI->getOpcode() == AMDGPU::COPY)
1530 return Changed;
1531
1532 } else {
1533 if (UseMI->isCopy() && OpToFold.isReg() &&
1534 UseMI->getOperand(i: 0).getReg().isVirtual() &&
1535 !UseMI->getOperand(i: 1).getSubReg() &&
1536 OpToFold.DefMI->implicit_operands().empty()) {
1537 LLVM_DEBUG(dbgs() << "Folding " << *OpToFold.OpToFold << "\n into "
1538 << *UseMI);
1539 unsigned Size = TII->getOpSize(MI: *UseMI, OpNo: 1);
1540 Register UseReg = OpToFold.getReg();
1541 UseMI->getOperand(i: 1).setReg(UseReg);
1542 unsigned SubRegIdx = OpToFold.getSubReg();
1543 // Hack to allow 32-bit SGPRs to be folded into True16 instructions
1544 // Remove this if 16-bit SGPRs (i.e. SGPR_LO16) are added to the
1545 // VS_16RegClass
1546 if (Size == 2 && TRI->isVGPR(MRI: *MRI, Reg: UseMI->getOperand(i: 0).getReg()) &&
1547 TRI->isSGPRReg(MRI: *MRI, Reg: UseReg) && SubRegIdx != AMDGPU::NoSubRegister) {
1548 // SGPRs only have lo16 subregisters, so the value is in the low half
1549 // of a 32-bit SGPR. Use that whole 32-bit SGPR instead.
1550 unsigned Channel = TRI->getChannelFromSubReg(SubReg: SubRegIdx);
1551 const TargetRegisterClass *UseRC = TRI->getRegClassForReg(MRI: *MRI, Reg: UseReg);
1552 SubRegIdx = TRI->getRegSizeInBits(RC: *UseRC) == 32
1553 ? AMDGPU::NoSubRegister
1554 : SIRegisterInfo::getSubRegFromChannel(Channel);
1555 }
1556 UseMI->getOperand(i: 1).setSubReg(SubRegIdx);
1557 UseMI->getOperand(i: 1).setIsKill(false);
1558 CopiesToReplace.push_back(Elt: UseMI);
1559 OpToFold.OpToFold->setIsKill(false);
1560 Changed = true;
1561
1562 if (foldCopyToAGPRRegSequence(CopyMI: UseMI))
1563 return true;
1564 }
1565
1566 unsigned UseOpc = UseMI->getOpcode();
1567 if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 ||
1568 (UseOpc == AMDGPU::V_READLANE_B32 &&
1569 (int)UseOpIdx ==
1570 AMDGPU::getNamedOperandIdx(Opcode: UseOpc, Name: AMDGPU::OpName::src0))) {
1571 // %vgpr = V_MOV_B32 imm
1572 // %sgpr = V_READFIRSTLANE_B32 %vgpr
1573 // =>
1574 // %sgpr = S_MOV_B32 imm
1575 if (FoldingImmLike) {
1576 if (execMayBeModifiedBeforeUse(MRI: *MRI,
1577 VReg: UseMI->getOperand(i: UseOpIdx).getReg(),
1578 DefMI: *OpToFold.DefMI, UseMI: *UseMI))
1579 return Changed;
1580
1581 UseMI->setDesc(TII->get(Opcode: AMDGPU::S_MOV_B32));
1582 UseMI->clearFlag(Flag: MachineInstr::NoConvergent);
1583
1584 if (OpToFold.isImm()) {
1585 UseMI->getOperand(i: 1).ChangeToImmediate(
1586 ImmVal: *OpToFold.getEffectiveImmVal());
1587 } else if (OpToFold.isFI())
1588 UseMI->getOperand(i: 1).ChangeToFrameIndex(Idx: OpToFold.getFI());
1589 else {
1590 assert(OpToFold.isGlobal());
1591 UseMI->getOperand(i: 1).ChangeToGA(GV: OpToFold.OpToFold->getGlobal(),
1592 Offset: OpToFold.OpToFold->getOffset(),
1593 TargetFlags: OpToFold.OpToFold->getTargetFlags());
1594 }
1595 UseMI->removeOperand(OpNo: 2); // Remove exec read (or src1 for readlane)
1596 return true;
1597 }
1598
1599 if (OpToFold.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg())) {
1600 if (execMayBeModifiedBeforeUse(MRI: *MRI,
1601 VReg: UseMI->getOperand(i: UseOpIdx).getReg(),
1602 DefMI: *OpToFold.DefMI, UseMI: *UseMI))
1603 return Changed;
1604
1605 // %vgpr = COPY %sgpr0
1606 // %sgpr1 = V_READFIRSTLANE_B32 %vgpr
1607 // =>
1608 // %sgpr1 = COPY %sgpr0
1609 UseMI->setDesc(TII->get(Opcode: AMDGPU::COPY));
1610 UseMI->getOperand(i: 1).setReg(OpToFold.getReg());
1611 UseMI->getOperand(i: 1).setSubReg(OpToFold.getSubReg());
1612 UseMI->getOperand(i: 1).setIsKill(false);
1613 UseMI->removeOperand(OpNo: 2); // Remove exec read (or src1 for readlane)
1614 UseMI->clearFlag(Flag: MachineInstr::NoConvergent);
1615 return true;
1616 }
1617 }
1618
1619 const MCInstrDesc &UseDesc = UseMI->getDesc();
1620
1621 // Don't fold into target independent nodes. Target independent opcodes
1622 // don't have defined register classes.
1623 if (UseDesc.isVariadic() || UseOp->isImplicit() ||
1624 UseDesc.operands()[UseOpIdx].RegClass == -1)
1625 return Changed;
1626 }
1627
1628 // FIXME: We could try to change the instruction from 64-bit to 32-bit
1629 // to enable more folding opportunities. The shrink operands pass
1630 // already does this.
1631
1632 Changed |= tryAddToFoldList(FoldList, MI: UseMI, OpNo: UseOpIdx, OpToFold);
1633 return Changed;
1634}
1635
1636static bool evalBinaryInstruction(unsigned Opcode, int32_t &Result,
1637 uint32_t LHS, uint32_t RHS) {
1638 switch (Opcode) {
1639 case AMDGPU::S_ADD_I32:
1640 case AMDGPU::S_ADD_U32:
1641 Result = LHS + RHS;
1642 return true;
1643 case AMDGPU::S_SUB_I32:
1644 case AMDGPU::S_SUB_U32:
1645 Result = LHS - RHS;
1646 return true;
1647 case AMDGPU::V_AND_B32_e64:
1648 case AMDGPU::V_AND_B32_e32:
1649 case AMDGPU::S_AND_B32:
1650 Result = LHS & RHS;
1651 return true;
1652 case AMDGPU::V_OR_B32_e64:
1653 case AMDGPU::V_OR_B32_e32:
1654 case AMDGPU::S_OR_B32:
1655 Result = LHS | RHS;
1656 return true;
1657 case AMDGPU::V_XOR_B32_e64:
1658 case AMDGPU::V_XOR_B32_e32:
1659 case AMDGPU::S_XOR_B32:
1660 Result = LHS ^ RHS;
1661 return true;
1662 case AMDGPU::S_XNOR_B32:
1663 Result = ~(LHS ^ RHS);
1664 return true;
1665 case AMDGPU::S_NAND_B32:
1666 Result = ~(LHS & RHS);
1667 return true;
1668 case AMDGPU::S_NOR_B32:
1669 Result = ~(LHS | RHS);
1670 return true;
1671 case AMDGPU::S_ANDN2_B32:
1672 Result = LHS & ~RHS;
1673 return true;
1674 case AMDGPU::S_ORN2_B32:
1675 Result = LHS | ~RHS;
1676 return true;
1677 case AMDGPU::V_LSHL_B32_e64:
1678 case AMDGPU::V_LSHL_B32_e32:
1679 case AMDGPU::S_LSHL_B32:
1680 // The instruction ignores the high bits for out of bounds shifts.
1681 Result = LHS << (RHS & 31);
1682 return true;
1683 case AMDGPU::V_LSHLREV_B32_e64:
1684 case AMDGPU::V_LSHLREV_B32_e32:
1685 Result = RHS << (LHS & 31);
1686 return true;
1687 case AMDGPU::V_LSHR_B32_e64:
1688 case AMDGPU::V_LSHR_B32_e32:
1689 case AMDGPU::S_LSHR_B32:
1690 Result = LHS >> (RHS & 31);
1691 return true;
1692 case AMDGPU::V_LSHRREV_B32_e64:
1693 case AMDGPU::V_LSHRREV_B32_e32:
1694 Result = RHS >> (LHS & 31);
1695 return true;
1696 case AMDGPU::V_ASHR_I32_e64:
1697 case AMDGPU::V_ASHR_I32_e32:
1698 case AMDGPU::S_ASHR_I32:
1699 Result = static_cast<int32_t>(LHS) >> (RHS & 31);
1700 return true;
1701 case AMDGPU::V_ASHRREV_I32_e64:
1702 case AMDGPU::V_ASHRREV_I32_e32:
1703 Result = static_cast<int32_t>(RHS) >> (LHS & 31);
1704 return true;
1705 default:
1706 return false;
1707 }
1708}
1709
1710static unsigned getMovOpc(bool IsScalar) {
1711 return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1712}
1713
1714// Try to simplify operations with a constant that may appear after instruction
1715// selection.
1716// TODO: See if a frame index with a fixed offset can fold.
1717bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
1718 if (!MI->allImplicitDefsAreDead())
1719 return false;
1720
1721 unsigned Opc = MI->getOpcode();
1722
1723 // (src0 << src1) + 0 -> src0 << src1
1724 if (Opc == AMDGPU::V_LSHL_ADD_U32_e64) {
1725 int DataIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
1726 int AddIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
1727 std::optional<int64_t> Addend =
1728 TII->getImmOrMaterializedImm(MRI: *MRI, Op: MI->getOperand(i: AddIdx));
1729
1730 if (Addend && static_cast<uint32_t>(*Addend) == 0) {
1731 MachineOperand Data = MI->getOperand(i: DataIdx);
1732 MI->removeOperand(OpNo: AddIdx);
1733 MI->removeOperand(OpNo: DataIdx);
1734 MI->addOperand(MF&: *MF, Op: Data);
1735 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::V_LSHLREV_B32_e64));
1736 return true;
1737 }
1738 }
1739
1740 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
1741 if (Src0Idx == -1)
1742 return false;
1743
1744 MachineOperand *Src0 = &MI->getOperand(i: Src0Idx);
1745 std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src0);
1746
1747 if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 ||
1748 Opc == AMDGPU::S_NOT_B32) &&
1749 Src0Imm) {
1750 MI->getOperand(i: 1).ChangeToImmediate(ImmVal: ~*Src0Imm);
1751 TII->mutateAndCleanupImplicit(
1752 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_NOT_B32)));
1753 return true;
1754 }
1755
1756 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
1757 if (Src1Idx == -1)
1758 return false;
1759
1760 MachineOperand *Src1 = &MI->getOperand(i: Src1Idx);
1761 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
1762
1763 if (!Src0Imm && !Src1Imm)
1764 return false;
1765
1766 // and k0, k1 -> v_mov_b32 (k0 & k1)
1767 // or k0, k1 -> v_mov_b32 (k0 | k1)
1768 // xor k0, k1 -> v_mov_b32 (k0 ^ k1)
1769 if (Src0Imm && Src1Imm) {
1770 int32_t NewImm;
1771 if (!evalBinaryInstruction(Opcode: Opc, Result&: NewImm, LHS: *Src0Imm, RHS: *Src1Imm))
1772 return false;
1773
1774 bool IsSGPR = TRI->isSGPRReg(MRI: *MRI, Reg: MI->getOperand(i: 0).getReg());
1775
1776 // Be careful to change the right operand, src0 may belong to a different
1777 // instruction.
1778 MI->getOperand(i: Src0Idx).ChangeToImmediate(ImmVal: NewImm);
1779 MI->removeOperand(OpNo: Src1Idx);
1780 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: IsSGPR)));
1781 return true;
1782 }
1783
1784 // S_SUB_* is not commutable, so handle it before the commutability gate.
1785 // Only `x - 0 -> copy x` is valid; `0 - x` is a negation, not a copy.
1786 if (Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U32) {
1787 if (Src1Imm && static_cast<int32_t>(*Src1Imm) == 0) {
1788 // y = sub x, 0 => y = copy x
1789 MI->removeOperand(OpNo: Src1Idx);
1790 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1791 return true;
1792 }
1793 return false;
1794 }
1795
1796 if (!MI->isCommutable())
1797 return false;
1798
1799 if (Src0Imm && !Src1Imm) {
1800 std::swap(a&: Src0, b&: Src1);
1801 std::swap(a&: Src0Idx, b&: Src1Idx);
1802 std::swap(lhs&: Src0Imm, rhs&: Src1Imm);
1803 }
1804
1805 int32_t Src1Val = static_cast<int32_t>(*Src1Imm);
1806 if (Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_ADD_U32) {
1807 if (Src1Val == 0) {
1808 // y = add x, 0 => y = copy x
1809 MI->removeOperand(OpNo: Src1Idx);
1810 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1811 return true;
1812 }
1813 return false;
1814 }
1815
1816 if (Opc == AMDGPU::V_OR_B32_e64 ||
1817 Opc == AMDGPU::V_OR_B32_e32 ||
1818 Opc == AMDGPU::S_OR_B32) {
1819 if (Src1Val == 0) {
1820 // y = or x, 0 => y = copy x
1821 MI->removeOperand(OpNo: Src1Idx);
1822 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1823 } else if (Src1Val == -1) {
1824 // y = or x, -1 => y = v_mov_b32 -1
1825 MI->removeOperand(OpNo: Src0Idx);
1826 TII->mutateAndCleanupImplicit(
1827 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_OR_B32)));
1828 } else
1829 return false;
1830
1831 return true;
1832 }
1833
1834 if (Opc == AMDGPU::V_AND_B32_e64 || Opc == AMDGPU::V_AND_B32_e32 ||
1835 Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_AND_B64) {
1836 int64_t C = Opc == AMDGPU::S_AND_B64 ? *Src1Imm : Src1Val;
1837 if (C == 0) {
1838 // y = and x, 0 => y = mov 0
1839 MI->removeOperand(OpNo: Src0Idx);
1840 TII->mutateAndCleanupImplicit(
1841 MI&: *MI, NewDesc: TII->get(Opcode: TII->getMovOpcode(DstRC: TII->getOpRegClass(MI: *MI, OpNo: 0))));
1842 } else if (C == -1) {
1843 // y = and x, -1 => y = copy x
1844 MI->removeOperand(OpNo: Src1Idx);
1845 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1846 } else
1847 return false;
1848
1849 return true;
1850 }
1851
1852 if (Opc == AMDGPU::V_XOR_B32_e64 || Opc == AMDGPU::V_XOR_B32_e32 ||
1853 Opc == AMDGPU::S_XOR_B32) {
1854 if (Src1Val == 0) {
1855 // y = xor x, 0 => y = copy x
1856 MI->removeOperand(OpNo: Src1Idx);
1857 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1858 return true;
1859 }
1860 }
1861
1862 return false;
1863}
1864
1865// Fold a select whose two sources are the same value into a copy or move.
1866bool SIFoldOperandsImpl::tryFoldCndMask(MachineInstr &MI) const {
1867 unsigned Opc = MI.getOpcode();
1868 if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 &&
1869 Opc != AMDGPU::S_CSELECT_B32 && Opc != AMDGPU::S_CSELECT_B64)
1870 return false;
1871
1872 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1873 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1874 if (!Src1->isIdenticalTo(Other: *Src0)) {
1875 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
1876 if (!Src1Imm)
1877 return false;
1878
1879 std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src0);
1880 if (!Src0Imm || *Src0Imm != *Src1Imm)
1881 return false;
1882 }
1883
1884 int Src1ModIdx =
1885 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1_modifiers);
1886 int Src0ModIdx =
1887 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers);
1888 if ((Src1ModIdx != -1 && MI.getOperand(i: Src1ModIdx).getImm() != 0) ||
1889 (Src0ModIdx != -1 && MI.getOperand(i: Src0ModIdx).getImm() != 0))
1890 return false;
1891
1892 unsigned NewOpc;
1893 if (Src0->isReg()) {
1894 NewOpc = AMDGPU::COPY;
1895 } else {
1896 switch (Opc) {
1897 case AMDGPU::S_CSELECT_B32:
1898 NewOpc = AMDGPU::S_MOV_B32;
1899 break;
1900 case AMDGPU::S_CSELECT_B64:
1901 NewOpc = AMDGPU::S_MOV_B64;
1902 break;
1903 default:
1904 NewOpc = AMDGPU::V_MOV_B32_e32;
1905 break;
1906 }
1907 }
1908
1909 LLVM_DEBUG(dbgs() << "Folded " << MI << " into ");
1910 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
1911 if (Src2Idx != -1)
1912 MI.removeOperand(OpNo: Src2Idx);
1913 MI.removeOperand(OpNo: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1));
1914 if (Src1ModIdx != -1)
1915 MI.removeOperand(OpNo: Src1ModIdx);
1916 if (Src0ModIdx != -1)
1917 MI.removeOperand(OpNo: Src0ModIdx);
1918 TII->mutateAndCleanupImplicit(MI, NewDesc: TII->get(Opcode: NewOpc));
1919 LLVM_DEBUG(dbgs() << MI);
1920 return true;
1921}
1922
1923// Extract mask and register from an AND instruction.
1924// Immediate can be in operand 1 or 2.
1925std::optional<SIFoldOperandsImpl::ANDMaskResult>
1926SIFoldOperandsImpl::getANDMaskRegOperand(MachineInstr &AndMI) const {
1927 unsigned Opc = AndMI.getOpcode();
1928 if (Opc != AMDGPU::V_AND_B32_e64 && Opc != AMDGPU::V_AND_B32_e32 &&
1929 Opc != AMDGPU::S_AND_B32)
1930 return std::nullopt;
1931
1932 std::optional<int64_t> MaskImm =
1933 TII->getImmOrMaterializedImm(MRI: *MRI, Op: AndMI.getOperand(i: 1));
1934 if (MaskImm && AndMI.getOperand(i: 2).isReg())
1935 return ANDMaskResult{.Mask: *MaskImm, .Reg: AndMI.getOperand(i: 2).getReg()};
1936
1937 MaskImm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: AndMI.getOperand(i: 2));
1938 if (MaskImm && AndMI.getOperand(i: 1).isReg())
1939 return ANDMaskResult{.Mask: *MaskImm, .Reg: AndMI.getOperand(i: 1).getReg()};
1940
1941 return std::nullopt;
1942}
1943
1944// Operands of an op_sel instruction that are not present in the equivalent
1945// legacy instruction, in operand order.
1946static constexpr AMDGPU::OpName OpSelOnlyOpNames[] = {
1947 AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src1_modifiers,
1948 AMDGPU::OpName::src2_modifiers, AMDGPU::OpName::op_sel};
1949
1950// If MI is an op_sel instruction on GFX9 that does not use op_sel, return the
1951// opcode of the equivalent legacy instruction, which zeros the high 16 bits of
1952// the destination. Otherwise return 0.
1953static unsigned getZeroHi16LegacyOpcode(const GCNSubtarget &ST,
1954 const MachineInstr &MI) {
1955 if (ST.getGeneration() != AMDGPUSubtarget::GFX9)
1956 return 0;
1957
1958 unsigned LegacyOpc;
1959 switch (MI.getOpcode()) {
1960 case AMDGPU::V_MAD_U16_gfx9_e64:
1961 LegacyOpc = AMDGPU::V_MAD_U16_e64;
1962 break;
1963 case AMDGPU::V_MAD_I16_gfx9_e64:
1964 LegacyOpc = AMDGPU::V_MAD_I16_e64;
1965 break;
1966 default:
1967 return 0;
1968 }
1969
1970 for (AMDGPU::OpName OpName : OpSelOnlyOpNames) {
1971 if (MI.getOperand(i: AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: OpName))
1972 .getImm() != 0)
1973 return 0;
1974 }
1975
1976 return LegacyOpc;
1977}
1978
1979// Eliminate redundant 32-bit AND operations by detecting when ChildMI's mask
1980// contains ParentMI's mask.
1981//
1982// For example:
1983// ParentMI: %1 = AND %0, 0x7fff
1984// ChildMI: %2 = AND %1, 0xffff
1985//
1986// This also handles cases where ParentMI implicitly zeros high bits (e.g., f16
1987// operations that write 16-bit results into 32-bit registers), making a
1988// subsequent AND with 0xffff redundant. On GFX9, ParentMI may be switched to a
1989// legacy opcode that zeros the high bits for this purpose.
1990bool SIFoldOperandsImpl::tryFoldRedundantAND(MachineInstr &ChildMI) const {
1991 // Ensure implicit defs (e.g., $scc) are not live.
1992 if (!ChildMI.allImplicitDefsAreDead())
1993 return false;
1994
1995 std::optional<ANDMaskResult> ChildResult = getANDMaskRegOperand(AndMI&: ChildMI);
1996 if (!ChildResult)
1997 return false;
1998
1999 if (!ChildResult->Reg.isVirtual())
2000 return false;
2001
2002 MachineInstr *ParentMI = MRI->getVRegDef(Reg: ChildResult->Reg);
2003 if (!ParentMI)
2004 return false;
2005
2006 int64_t ParentMask = 0;
2007 unsigned LegacyOpc = 0;
2008 std::optional<ANDMaskResult> ParentResult = getANDMaskRegOperand(AndMI&: *ParentMI);
2009 if (ParentResult) {
2010 // Parent is an AND - extract its mask.
2011 ParentMask = ParentResult->Mask;
2012 } else if (ST->zeroesHigh16BitsOfDest(Opcode: ParentMI->getOpcode())) {
2013 // Parent instruction implicitly zeros high 16 bits.
2014 ParentMask = 0xffff;
2015 } else if ((LegacyOpc = getZeroHi16LegacyOpcode(ST: *ST, MI: *ParentMI))) {
2016 // Parent instruction can be switched to a legacy opcode that implicitly
2017 // zeros high 16 bits.
2018 ParentMask = 0xffff;
2019 } else {
2020 return false;
2021 }
2022
2023 // Check if ChildMI is not redundant.
2024 if ((ParentMask & ChildResult->Mask) != ParentMask)
2025 return false;
2026
2027 Register Dst = ChildMI.getOperand(i: 0).getReg();
2028 Register Src = ChildResult->Reg;
2029
2030 // Src must be legal in every use of Dst. An S_AND_B32 parent with a
2031 // V_AND_B32 child defines Src in the scalar bank, and a use that requires a
2032 // VGPR does not accept it.
2033 if (!Dst.isVirtual() || !MRI->constrainRegClass(Reg: Src, RC: MRI->getRegClass(Reg: Dst)))
2034 return false;
2035
2036 if (LegacyOpc) {
2037 unsigned Opc = ParentMI->getOpcode();
2038 for (AMDGPU::OpName OpName : reverse(C: OpSelOnlyOpNames))
2039 ParentMI->removeOperand(OpNo: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: OpName));
2040 TII->mutateAndCleanupImplicit(MI&: *ParentMI, NewDesc: TII->get(Opcode: LegacyOpc));
2041 }
2042
2043 MRI->replaceRegWith(FromReg: Dst, ToReg: Src);
2044
2045 ChildMI.eraseFromParent();
2046 return true;
2047}
2048
2049/// Remove S_AND of a lane mask with EXEC, when the lane mask is already known
2050/// to have 0 in the bits of all inactive lanes.
2051///
2052/// Instruction selection inserts these unconditionally because it has not
2053/// analysed what produced the lane mask.
2054bool SIFoldOperandsImpl::tryFoldAndExec(MachineInstr &MI) const {
2055 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST: *ST);
2056 if (MI.getOpcode() != LMC.AndOpc)
2057 return false;
2058
2059 // The AND is going to be removed, so nothing may use the SCC it defines.
2060 if (!MI.allImplicitDefsAreDead())
2061 return false;
2062
2063 // Find the EXEC operand, and the lane mask it is being ANDed with.
2064 unsigned ExecIdx = 0;
2065 for (unsigned I : {1u, 2u}) {
2066 const MachineOperand &MO = MI.getOperand(i: I);
2067 if (MO.isReg() && MO.getReg() == LMC.ExecReg)
2068 ExecIdx = I;
2069 }
2070 if (!ExecIdx)
2071 return false;
2072 MachineOperand &Src = MI.getOperand(i: 3 - ExecIdx);
2073 if (!Src.isReg() || !Src.getReg().isVirtual() || Src.getSubReg())
2074 return false;
2075
2076 Register SrcReg = Src.getReg();
2077 if (!TII->isMaskedByExec(Reg: SrcReg, Use: MI, MRI: *MRI))
2078 return false;
2079
2080 LLVM_DEBUG(dbgs() << "Folding redundant AND with EXEC: " << MI);
2081
2082 Register DstReg = MI.getOperand(i: 0).getReg();
2083 if (DstReg.isVirtual()) {
2084 if (!MRI->constrainRegClass(Reg: SrcReg, RC: MRI->getRegClass(Reg: DstReg)))
2085 return false;
2086 MRI->replaceRegWith(FromReg: DstReg, ToReg: SrcReg);
2087 } else {
2088 // A physical destination, e.g. the $vcc written by moveToVALU. Register
2089 // allocation will usually make this copy an identity copy.
2090 MachineBasicBlock *MBB = MI.getParent();
2091 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: DstReg)
2092 .addReg(RegNo: SrcReg);
2093 }
2094
2095 MI.eraseFromParent();
2096 return true;
2097}
2098
2099bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
2100 const FoldableDef &OpToFold) const {
2101 // We need mutate the operands of new mov instructions to add implicit
2102 // uses of EXEC, but adding them invalidates the use_iterator, so defer
2103 // this.
2104 SmallVector<MachineInstr *, 4> CopiesToReplace;
2105 SmallVector<FoldCandidate, 4> FoldList;
2106 MachineOperand &Dst = MI.getOperand(i: 0);
2107 bool Changed = false;
2108
2109 SmallVector<MachineOperand *, 4> UsesToProcess(
2110 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: Dst.getReg())));
2111 for (auto *U : UsesToProcess) {
2112 MachineInstr *UseMI = U->getParent();
2113
2114 FoldableDef SubOpToFold = OpToFold.getWithSubReg(TRI: *TRI, SubReg: U->getSubReg());
2115 Changed |= foldOperand(OpToFold: SubOpToFold, UseMI, UseOpIdx: UseMI->getOperandNo(I: U), FoldList,
2116 CopiesToReplace);
2117 }
2118
2119 if (CopiesToReplace.empty() && FoldList.empty())
2120 return Changed;
2121
2122 // Make sure we add EXEC uses to any new v_mov instructions created.
2123 for (MachineInstr *Copy : CopiesToReplace)
2124 Copy->addImplicitDefUseOperands(MF&: *MF);
2125
2126 SetVector<MachineInstr *> ConstantFoldCandidates;
2127 for (FoldCandidate &Fold : FoldList) {
2128 assert(!Fold.isReg() || Fold.Def.OpToFold);
2129 if (Fold.isReg() && Fold.getReg().isVirtual()) {
2130 Register Reg = Fold.getReg();
2131 const MachineInstr *DefMI = Fold.Def.DefMI;
2132 if (DefMI->readsRegister(Reg: AMDGPU::EXEC, TRI) &&
2133 execMayBeModifiedBeforeUse(MRI: *MRI, VReg: Reg, DefMI: *DefMI, UseMI: *Fold.UseMI))
2134 continue;
2135 }
2136 if (updateOperand(Fold)) {
2137 LLVM_DEBUG(dbgs() << "Folded source from " << MI << " into OpNo "
2138 << static_cast<int>(Fold.UseOpNo) << " of "
2139 << *Fold.UseMI);
2140
2141 if (Fold.isImm())
2142 ConstantFoldCandidates.insert(X: Fold.UseMI);
2143
2144 } else if (Fold.Commuted) {
2145 // Restoring instruction's original operand order if fold has failed.
2146 TII->commuteInstruction(MI&: *Fold.UseMI, NewMI: false);
2147 }
2148 }
2149
2150 for (MachineInstr *MI : ConstantFoldCandidates) {
2151 if (tryConstantFoldOp(MI)) {
2152 LLVM_DEBUG(dbgs() << "Constant folded " << *MI);
2153 Changed = true;
2154 }
2155 }
2156 return true;
2157}
2158
2159/// Fold %agpr = COPY (REG_SEQUENCE x_MOV_B32, ...) into REG_SEQUENCE
2160/// (V_ACCVGPR_WRITE_B32_e64) ... depending on the reg_sequence input values.
2161bool SIFoldOperandsImpl::foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const {
2162 // It is very tricky to store a value into an AGPR. v_accvgpr_write_b32 can
2163 // only accept VGPR or inline immediate. Recreate a reg_sequence with its
2164 // initializers right here, so we will rematerialize immediates and avoid
2165 // copies via different reg classes.
2166 const TargetRegisterClass *DefRC =
2167 MRI->getRegClass(Reg: CopyMI->getOperand(i: 0).getReg());
2168 if (!TRI->isAGPRClass(RC: DefRC))
2169 return false;
2170
2171 Register UseReg = CopyMI->getOperand(i: 1).getReg();
2172 MachineInstr *RegSeq = MRI->getVRegDef(Reg: UseReg);
2173 if (!RegSeq || !RegSeq->isRegSequence())
2174 return false;
2175
2176 const DebugLoc &DL = CopyMI->getDebugLoc();
2177 MachineBasicBlock &MBB = *CopyMI->getParent();
2178
2179 MachineInstrBuilder B(*MBB.getParent(), CopyMI);
2180 DenseMap<TargetInstrInfo::RegSubRegPair, Register> VGPRCopies;
2181
2182 const TargetRegisterClass *UseRC =
2183 MRI->getRegClass(Reg: CopyMI->getOperand(i: 1).getReg());
2184
2185 // Value, subregindex for new REG_SEQUENCE
2186 SmallVector<std::pair<MachineOperand *, unsigned>, 32> NewDefs;
2187
2188 unsigned NumRegSeqOperands = RegSeq->getNumOperands();
2189 unsigned NumFoldable = 0;
2190
2191 for (unsigned I = 1; I != NumRegSeqOperands; I += 2) {
2192 MachineOperand &RegOp = RegSeq->getOperand(i: I);
2193 unsigned SubRegIdx = RegSeq->getOperand(i: I + 1).getImm();
2194
2195 if (RegOp.getSubReg()) {
2196 // TODO: Handle subregister compose
2197 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2198 continue;
2199 }
2200
2201 MachineOperand *Lookup = lookUpCopyChain(TII: *TII, MRI: *MRI, SrcReg: RegOp.getReg());
2202 if (!Lookup)
2203 Lookup = &RegOp;
2204
2205 if (Lookup->isImm()) {
2206 // Check if this is an agpr_32 subregister.
2207 const TargetRegisterClass *DestSuperRC = TRI->getMatchingSuperRegClass(
2208 A: DefRC, B: &AMDGPU::AGPR_32RegClass, Idx: SubRegIdx);
2209 if (DestSuperRC &&
2210 TII->isInlineConstant(MO: *Lookup, OperandType: AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
2211 ++NumFoldable;
2212 NewDefs.emplace_back(Args&: Lookup, Args&: SubRegIdx);
2213 continue;
2214 }
2215 }
2216
2217 const TargetRegisterClass *InputRC =
2218 Lookup->isReg() ? MRI->getRegClass(Reg: Lookup->getReg())
2219 : MRI->getRegClass(Reg: RegOp.getReg());
2220
2221 // TODO: Account for Lookup->getSubReg()
2222
2223 // If we can't find a matching super class, this is an SGPR->AGPR or
2224 // VGPR->AGPR subreg copy (or something constant-like we have to materialize
2225 // in the AGPR). We can't directly copy from SGPR to AGPR on gfx908, so we
2226 // want to rewrite to copy to an intermediate VGPR class.
2227 const TargetRegisterClass *MatchRC =
2228 TRI->getMatchingSuperRegClass(A: DefRC, B: InputRC, Idx: SubRegIdx);
2229 if (!MatchRC) {
2230 ++NumFoldable;
2231 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2232 continue;
2233 }
2234
2235 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2236 }
2237
2238 // Do not clone a reg_sequence and merely change the result register class.
2239 if (NumFoldable == 0)
2240 return false;
2241
2242 CopyMI->setDesc(TII->get(Opcode: AMDGPU::REG_SEQUENCE));
2243 for (unsigned I = CopyMI->getNumOperands() - 1; I > 0; --I)
2244 CopyMI->removeOperand(OpNo: I);
2245
2246 for (auto [Def, DestSubIdx] : NewDefs) {
2247 if (!Def->isReg()) {
2248 // TODO: Should we use single write for each repeated value like in
2249 // register case?
2250 Register Tmp = MRI->createVirtualRegister(RegClass: &AMDGPU::AGPR_32RegClass);
2251 BuildMI(BB&: MBB, I: CopyMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: Tmp)
2252 .add(MO: *Def);
2253 B.addReg(RegNo: Tmp);
2254 } else {
2255 TargetInstrInfo::RegSubRegPair Src = getRegSubRegPair(O: *Def);
2256 Def->setIsKill(false);
2257
2258 Register &VGPRCopy = VGPRCopies[Src];
2259 if (!VGPRCopy) {
2260 const TargetRegisterClass *VGPRUseSubRC =
2261 TRI->getSubRegisterClass(UseRC, DestSubIdx);
2262
2263 // We cannot build a reg_sequence out of the same registers, they
2264 // must be copied. Better do it here before copyPhysReg() created
2265 // several reads to do the AGPR->VGPR->AGPR copy.
2266
2267 // Direct copy from SGPR to AGPR is not possible on gfx908. To avoid
2268 // creation of exploded copies SGPR->VGPR->AGPR in the copyPhysReg()
2269 // later, create a copy here and track if we already have such a copy.
2270 const TargetRegisterClass *SubRC =
2271 TRI->getSubRegisterClass(MRI->getRegClass(Reg: Src.Reg), Src.SubReg);
2272 if (!VGPRUseSubRC->hasSubClassEq(RC: SubRC)) {
2273 // TODO: Try to reconstrain class
2274 VGPRCopy = MRI->createVirtualRegister(RegClass: VGPRUseSubRC);
2275 BuildMI(BB&: MBB, I: CopyMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: VGPRCopy).add(MO: *Def);
2276 B.addReg(RegNo: VGPRCopy);
2277 } else {
2278 // If it is already a VGPR, do not copy the register.
2279 B.add(MO: *Def);
2280 }
2281 } else {
2282 B.addReg(RegNo: VGPRCopy);
2283 }
2284 }
2285
2286 B.addImm(Val: DestSubIdx);
2287 }
2288
2289 LLVM_DEBUG(dbgs() << "Folded " << *CopyMI);
2290 return true;
2291}
2292
2293bool SIFoldOperandsImpl::tryFoldFoldableCopy(
2294 MachineInstr &MI, MachineOperand *&CurrentKnownM0Val) const {
2295 Register DstReg = MI.getOperand(i: 0).getReg();
2296 // Specially track simple redefs of m0 to the same value in a block, so we
2297 // can erase the later ones.
2298 if (DstReg == AMDGPU::M0) {
2299 MachineOperand &NewM0Val = MI.getOperand(i: 1);
2300 if (CurrentKnownM0Val && CurrentKnownM0Val->isIdenticalTo(Other: NewM0Val)) {
2301 MI.eraseFromParent();
2302 return true;
2303 }
2304
2305 // We aren't tracking other physical registers
2306 CurrentKnownM0Val = (NewM0Val.isReg() && NewM0Val.getReg().isPhysical())
2307 ? nullptr
2308 : &NewM0Val;
2309 return false;
2310 }
2311
2312 MachineOperand *OpToFoldPtr;
2313 if (MI.getOpcode() == AMDGPU::V_MOV_B16_t16_e64) {
2314 // Folding when any src_modifiers are non-zero is unsupported
2315 if (TII->hasAnyModifiersSet(MI))
2316 return false;
2317 OpToFoldPtr = &MI.getOperand(i: 2);
2318 } else
2319 OpToFoldPtr = &MI.getOperand(i: 1);
2320 MachineOperand &OpToFold = *OpToFoldPtr;
2321 bool FoldingImm = OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
2322
2323 // FIXME: We could also be folding things like TargetIndexes.
2324 if (!FoldingImm && !OpToFold.isReg())
2325 return false;
2326
2327 // Fold virtual registers and constant physical registers.
2328 if (OpToFold.isReg() && OpToFold.getReg().isPhysical() &&
2329 !TRI->isConstantPhysReg(PhysReg: OpToFold.getReg()))
2330 return false;
2331
2332 // Prevent folding operands backwards in the function. For example,
2333 // the COPY opcode must not be replaced by 1 in this example:
2334 //
2335 // %3 = COPY %vgpr0; VGPR_32:%3
2336 // ...
2337 // %vgpr0 = V_MOV_B32_e32 1, implicit %exec
2338 if (!DstReg.isVirtual())
2339 return false;
2340
2341 const TargetRegisterClass *DstRC =
2342 MRI->getRegClass(Reg: MI.getOperand(i: 0).getReg());
2343
2344 // True16: Fix malformed 16-bit sgpr COPY produced by peephole-opt
2345 // Can remove this code if proper 16-bit SGPRs are implemented
2346 // Example: Pre-peephole-opt
2347 // %29:sgpr_lo16 = COPY %16.lo16:sreg_32
2348 // %32:sreg_32 = COPY %29:sgpr_lo16
2349 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2350 // Post-peephole-opt and DCE
2351 // %32:sreg_32 = COPY %16.lo16:sreg_32
2352 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2353 // After this transform
2354 // %32:sreg_32 = COPY %16:sreg_32
2355 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2356 // After the fold operands pass
2357 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %16:sreg_32
2358 if (MI.getOpcode() == AMDGPU::COPY && OpToFold.isReg() &&
2359 OpToFold.getSubReg()) {
2360 if (DstRC == &AMDGPU::SReg_32RegClass &&
2361 DstRC == MRI->getRegClass(Reg: OpToFold.getReg())) {
2362 if (!TRI->getMatchingSuperRegClass(A: DstRC, B: &AMDGPU::SGPR_LO16RegClass,
2363 Idx: OpToFold.getSubReg()))
2364 return false;
2365 OpToFold.setSubReg(0);
2366 }
2367 }
2368
2369 // Fold copy to AGPR through reg_sequence
2370 // TODO: Handle with subregister extract
2371 if (OpToFold.isReg() && MI.isCopy() && !MI.getOperand(i: 1).getSubReg()) {
2372 if (foldCopyToAGPRRegSequence(CopyMI: &MI))
2373 return true;
2374 }
2375
2376 FoldableDef Def(OpToFold, DstRC);
2377 bool Changed = foldInstOperand(MI, OpToFold: Def);
2378
2379 // If we managed to fold all uses of this copy then we might as well
2380 // delete it now.
2381 // The only reason we need to follow chains of copies here is that
2382 // tryFoldRegSequence looks forward through copies before folding a
2383 // REG_SEQUENCE into its eventual users.
2384 auto *InstToErase = &MI;
2385 while (MRI->use_nodbg_empty(RegNo: InstToErase->getOperand(i: 0).getReg())) {
2386 auto &SrcOp = InstToErase->getOperand(i: 1);
2387 auto SrcReg = SrcOp.isReg() ? SrcOp.getReg() : Register();
2388 InstToErase->eraseFromParent();
2389 Changed = true;
2390 InstToErase = nullptr;
2391 if (!SrcReg || SrcReg.isPhysical())
2392 break;
2393 InstToErase = MRI->getVRegDef(Reg: SrcReg);
2394 if (!InstToErase || !TII->isFoldableCopy(MI: *InstToErase))
2395 break;
2396 }
2397
2398 if (InstToErase && InstToErase->isRegSequence() &&
2399 MRI->use_nodbg_empty(RegNo: InstToErase->getOperand(i: 0).getReg())) {
2400 InstToErase->eraseFromParent();
2401 Changed = true;
2402 }
2403
2404 if (Changed)
2405 return true;
2406
2407 // Run this after foldInstOperand to avoid turning scalar additions into
2408 // vector additions when the result scalar result could just be folded into
2409 // the user(s).
2410 return OpToFold.isReg() &&
2411 foldCopyToVGPROfScalarAddOfFrameIndex(DstReg, SrcReg: OpToFold.getReg(), MI);
2412}
2413
2414// Clamp patterns are canonically selected to v_max_* instructions, so only
2415// handle them.
2416const MachineOperand *
2417SIFoldOperandsImpl::isClamp(const MachineInstr &MI) const {
2418 unsigned Op = MI.getOpcode();
2419 switch (Op) {
2420 case AMDGPU::V_MAX_F32_e64:
2421 case AMDGPU::V_MAX_F16_e64:
2422 case AMDGPU::V_MAX_F16_t16_e64:
2423 case AMDGPU::V_MAX_F16_fake16_e64:
2424 case AMDGPU::V_MAX_F64_e64:
2425 case AMDGPU::V_MAX_NUM_F64_e64:
2426 case AMDGPU::V_PK_MAX_F16:
2427 case AMDGPU::V_MAX_BF16_PSEUDO_e64:
2428 case AMDGPU::V_PK_MAX_NUM_BF16: {
2429 if (MI.mayRaiseFPException())
2430 return nullptr;
2431
2432 if (!TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp)->getImm())
2433 return nullptr;
2434
2435 // Make sure sources are identical.
2436 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2437 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2438 if (!Src0->isReg() || !Src1->isReg() ||
2439 Src0->getReg() != Src1->getReg() ||
2440 Src0->getSubReg() != Src1->getSubReg() ||
2441 Src0->getSubReg() != AMDGPU::NoSubRegister)
2442 return nullptr;
2443
2444 // Can't fold up if we have modifiers.
2445 if (TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
2446 return nullptr;
2447
2448 unsigned Src0Mods
2449 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm();
2450 unsigned Src1Mods
2451 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm();
2452
2453 // Having a 0 op_sel_hi would require swizzling the output in the source
2454 // instruction, which we can't do.
2455 unsigned UnsetMods =
2456 (Op == AMDGPU::V_PK_MAX_F16 || Op == AMDGPU::V_PK_MAX_NUM_BF16)
2457 ? SISrcMods::OP_SEL_1
2458 : 0u;
2459 if (Src0Mods != UnsetMods || Src1Mods != UnsetMods)
2460 return nullptr;
2461 return Src0;
2462 }
2463 default:
2464 return nullptr;
2465 }
2466}
2467
2468// FIXME: Clamp for v_mad_mixhi_f16 handled during isel.
2469bool SIFoldOperandsImpl::tryFoldClamp(MachineInstr &MI) {
2470 const MachineOperand *ClampSrc = isClamp(MI);
2471 if (!ClampSrc || !MRI->hasOneNonDBGUser(RegNo: ClampSrc->getReg()))
2472 return false;
2473
2474 if (!ClampSrc->getReg().isVirtual())
2475 return false;
2476
2477 // Look through COPY. COPY only observed with True16. Setting the clamp bit on
2478 // Def changes the value of every register in the chain, so each of them must
2479 // have no other users.
2480 MachineInstr *Def = MRI->getVRegDef(Reg: ClampSrc->getReg());
2481 while (Def && Def->isCopy()) {
2482 Register CopySrcReg = Def->getOperand(i: 1).getReg();
2483 if (!CopySrcReg.isVirtual() || !MRI->hasOneNonDBGUser(RegNo: CopySrcReg))
2484 return false;
2485 Def = MRI->getVRegDef(Reg: CopySrcReg);
2486 }
2487 if (!Def)
2488 return false;
2489
2490 // The type of clamp must be compatible.
2491 if (!SIInstrInfo::hasSameClamp(A: *Def, B: MI))
2492 return false;
2493
2494 if (Def->mayRaiseFPException())
2495 return false;
2496
2497 MachineOperand *DefClamp = TII->getNamedOperand(MI&: *Def, OperandName: AMDGPU::OpName::clamp);
2498 if (!DefClamp)
2499 return false;
2500
2501 LLVM_DEBUG(dbgs() << "Folding clamp " << *DefClamp << " into " << *Def);
2502
2503 // Clamp is applied after omod, so it is OK if omod is set.
2504 DefClamp->setImm(1);
2505
2506 Register DefReg = Def->getOperand(i: 0).getReg();
2507 Register MIDstReg = MI.getOperand(i: 0).getReg();
2508 if (TRI->isSGPRReg(MRI: *MRI, Reg: DefReg)) {
2509 // Pseudo scalar instructions have a SGPR for dst and clamp is a v_max*
2510 // instruction with a VGPR dst.
2511 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
2512 DestReg: MIDstReg)
2513 .addReg(RegNo: DefReg);
2514 } else {
2515 MRI->replaceRegWith(FromReg: MIDstReg, ToReg: DefReg);
2516 }
2517 MI.eraseFromParent();
2518
2519 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
2520 // instruction, so we might as well convert it to the more flexible VOP3-only
2521 // mad/fma form.
2522 if (TII->convertToThreeAddress(MI&: *Def, /*LIS=*/nullptr))
2523 Def->eraseFromParent();
2524
2525 return true;
2526}
2527
2528static int getOModValue(unsigned Opc, int64_t Val) {
2529 switch (Opc) {
2530 case AMDGPU::V_MUL_F64_e64:
2531 case AMDGPU::V_MUL_F64_pseudo_e64: {
2532 switch (Val) {
2533 case 0x3fe0000000000000: // 0.5
2534 return SIOutMods::DIV2;
2535 case 0x4000000000000000: // 2.0
2536 return SIOutMods::MUL2;
2537 case 0x4010000000000000: // 4.0
2538 return SIOutMods::MUL4;
2539 default:
2540 return SIOutMods::NONE;
2541 }
2542 }
2543 case AMDGPU::V_MUL_F32_e64: {
2544 switch (static_cast<uint32_t>(Val)) {
2545 case 0x3f000000: // 0.5
2546 return SIOutMods::DIV2;
2547 case 0x40000000: // 2.0
2548 return SIOutMods::MUL2;
2549 case 0x40800000: // 4.0
2550 return SIOutMods::MUL4;
2551 default:
2552 return SIOutMods::NONE;
2553 }
2554 }
2555 case AMDGPU::V_MUL_F16_e64:
2556 case AMDGPU::V_MUL_F16_t16_e64:
2557 case AMDGPU::V_MUL_F16_fake16_e64: {
2558 switch (static_cast<uint16_t>(Val)) {
2559 case 0x3800: // 0.5
2560 return SIOutMods::DIV2;
2561 case 0x4000: // 2.0
2562 return SIOutMods::MUL2;
2563 case 0x4400: // 4.0
2564 return SIOutMods::MUL4;
2565 default:
2566 return SIOutMods::NONE;
2567 }
2568 }
2569 case AMDGPU::V_PK_MUL_BF16: {
2570 switch (static_cast<uint16_t>(Val)) {
2571 case 0x3F00: // 0.5 in BF16
2572 return SIOutMods::DIV2;
2573 case 0x4000: // 2.0 in BF16
2574 return SIOutMods::MUL2;
2575 case 0x4080: // 4.0 in BF16
2576 return SIOutMods::MUL4;
2577 default:
2578 return SIOutMods::NONE;
2579 }
2580 }
2581 default:
2582 llvm_unreachable("invalid mul opcode");
2583 }
2584}
2585
2586// FIXME: Does this really not support denormals with f16?
2587// FIXME: Does this need to check IEEE mode bit? SNaNs are generally not
2588// handled, so will anything other than that break?
2589std::pair<const MachineOperand *, int>
2590SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
2591 unsigned Op = MI.getOpcode();
2592 switch (Op) {
2593 case AMDGPU::V_MUL_F64_e64:
2594 case AMDGPU::V_MUL_F64_pseudo_e64:
2595 case AMDGPU::V_MUL_F32_e64:
2596 case AMDGPU::V_MUL_F16_t16_e64:
2597 case AMDGPU::V_MUL_F16_fake16_e64:
2598 case AMDGPU::V_MUL_F16_e64: {
2599 // If output denormals are enabled, omod is ignored.
2600 if ((Op == AMDGPU::V_MUL_F32_e64 &&
2601 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
2602 ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F64_pseudo_e64 ||
2603 Op == AMDGPU::V_MUL_F16_e64 || Op == AMDGPU::V_MUL_F16_t16_e64 ||
2604 Op == AMDGPU::V_MUL_F16_fake16_e64) &&
2605 MFI->getMode().FP64FP16Denormals.Output !=
2606 DenormalMode::PreserveSign) ||
2607 MI.mayRaiseFPException())
2608 return {nullptr, SIOutMods::NONE};
2609
2610 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2611 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2612
2613 // If there is an immediate operand, it must be Src1
2614 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
2615 if (!Src1Imm)
2616 return {nullptr, SIOutMods::NONE};
2617
2618 int OMod = getOModValue(Opc: Op, Val: *Src1Imm);
2619 if (OMod == SIOutMods::NONE ||
2620 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers) ||
2621 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers) ||
2622 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2623 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2624 return {nullptr, SIOutMods::NONE};
2625
2626 return {Src0, OMod};
2627 }
2628 case AMDGPU::V_ADD_F64_e64:
2629 case AMDGPU::V_ADD_F64_pseudo_e64:
2630 case AMDGPU::V_ADD_F32_e64:
2631 case AMDGPU::V_ADD_F16_e64:
2632 case AMDGPU::V_ADD_F16_t16_e64:
2633 case AMDGPU::V_ADD_F16_fake16_e64: {
2634 // If output denormals are enabled, omod is ignored.
2635 if ((Op == AMDGPU::V_ADD_F32_e64 &&
2636 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
2637 ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F64_pseudo_e64 ||
2638 Op == AMDGPU::V_ADD_F16_e64 || Op == AMDGPU::V_ADD_F16_t16_e64 ||
2639 Op == AMDGPU::V_ADD_F16_fake16_e64) &&
2640 MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign))
2641 return {nullptr, SIOutMods::NONE};
2642
2643 // Look through the DAGCombiner canonicalization fmul x, 2 -> fadd x, x
2644 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2645 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2646
2647 if (Src0->isReg() && Src1->isReg() && Src0->getReg() == Src1->getReg() &&
2648 Src0->getSubReg() == Src1->getSubReg() &&
2649 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers) &&
2650 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers) &&
2651 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) &&
2652 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
2653 return {Src0, SIOutMods::MUL2};
2654
2655 return {nullptr, SIOutMods::NONE};
2656 }
2657 case AMDGPU::V_PK_MUL_BF16: {
2658 // OMOD folding for BF16 packed multiply. bf16 has no denormal mode of its
2659 // own; it follows the default ("denormal-fp-math") mode, which is the same
2660 // field as f64/f16.
2661 if (MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign ||
2662 MI.mayRaiseFPException())
2663 return {nullptr, SIOutMods::NONE};
2664
2665 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2666 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2667
2668 // If there is an immediate operand, it must be Src1
2669 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
2670 if (!Src1Imm)
2671 return {nullptr, SIOutMods::NONE};
2672
2673 int OMod = getOModValue(Opc: AMDGPU::V_PK_MUL_BF16, Val: *Src1Imm);
2674 if (OMod == SIOutMods::NONE)
2675 return {nullptr, SIOutMods::NONE};
2676
2677 // Modifiers other than op_sel_hi block OMOD folding. Per getOModValue
2678 // above, Src1 is an inline constant (0.5/2.0/4.0), which may carry
2679 // op_sel_lo to read it from the upper FP32 half, so allow that on src1.
2680 const MachineOperand *Src0Mods =
2681 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2682 const MachineOperand *Src1Mods =
2683 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2684 if ((Src0Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2685 (Src1Mods->getImm() & ~(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)) ||
2686 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2687 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2688 return {nullptr, SIOutMods::NONE};
2689
2690 return {Src0, OMod};
2691 }
2692 case AMDGPU::V_PK_ADD_BF16: {
2693 // OMOD folding for BF16 packed add: x + x -> x * 2. See the bf16 denormal
2694 // mode note in the V_PK_MUL_BF16 case above.
2695 if (MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign)
2696 return {nullptr, SIOutMods::NONE};
2697
2698 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2699 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2700
2701 if (!Src0->isReg() || !Src1->isReg() || Src0->getReg() != Src1->getReg() ||
2702 Src0->getSubReg() != Src1->getSubReg())
2703 return {nullptr, SIOutMods::NONE};
2704
2705 // Modifiers other than op_sel_hi block OMOD folding
2706 const MachineOperand *Src0Mods =
2707 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2708 const MachineOperand *Src1Mods =
2709 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2710 if ((Src0Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2711 (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2712 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2713 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2714 return {nullptr, SIOutMods::NONE};
2715
2716 return {Src0, SIOutMods::MUL2};
2717 }
2718 default:
2719 return {nullptr, SIOutMods::NONE};
2720 }
2721}
2722
2723// FIXME: Does this need to check IEEE bit on function?
2724bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
2725 const MachineOperand *RegOp;
2726 int OMod;
2727 std::tie(args&: RegOp, args&: OMod) = isOMod(MI);
2728 if (OMod == SIOutMods::NONE || !RegOp->isReg() ||
2729 RegOp->getSubReg() != AMDGPU::NoSubRegister ||
2730 !MRI->hasOneNonDBGUser(RegNo: RegOp->getReg()))
2731 return false;
2732
2733 MachineInstr *Def = MRI->getVRegDef(Reg: RegOp->getReg());
2734 Register OModSrcReg = Def->getOperand(i: 0).getReg();
2735
2736 // In real-true16 mode, vgpr_16 results are packed into vgpr_32 via
2737 // REG_SEQUENCE. Look through it to find the actual instruction.
2738 if (Def->isRegSequence() && Def->getNumOperands() == 5 &&
2739 Def->getOperand(i: 2).getImm() == AMDGPU::lo16) {
2740 // Only look through if the high 16 bits are undefined
2741 bool CanLookThrough = true;
2742 MachineInstr *Hi16Def = MRI->getVRegDef(Reg: Def->getOperand(i: 3).getReg());
2743 if (!Hi16Def || !Hi16Def->isImplicitDef())
2744 CanLookThrough = false;
2745
2746 if (CanLookThrough) {
2747 Register SrcReg = Def->getOperand(i: 1).getReg();
2748 if (!MRI->hasOneNonDBGUse(RegNo: SrcReg))
2749 return false;
2750
2751 Def = MRI->getVRegDef(Reg: SrcReg);
2752 if (!Def)
2753 return false;
2754 }
2755 }
2756
2757 MachineOperand *DefOMod = TII->getNamedOperand(MI&: *Def, OperandName: AMDGPU::OpName::omod);
2758 if (!DefOMod || DefOMod->getImm() != SIOutMods::NONE)
2759 return false;
2760
2761 if (Def->mayRaiseFPException())
2762 return false;
2763
2764 // Clamp is applied after omod. If the source already has clamp set, don't
2765 // fold it.
2766 if (TII->hasModifiersSet(MI: *Def, OpName: AMDGPU::OpName::clamp))
2767 return false;
2768
2769 LLVM_DEBUG(dbgs() << "Folding omod " << MI << " into " << *Def);
2770
2771 DefOMod->setImm(OMod);
2772 MRI->replaceRegWith(FromReg: MI.getOperand(i: 0).getReg(), ToReg: OModSrcReg);
2773 MI.eraseFromParent();
2774
2775 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
2776 // instruction, so we might as well convert it to the more flexible VOP3-only
2777 // mad/fma form.
2778 if (TII->convertToThreeAddress(MI&: *Def, /*LIS=*/nullptr))
2779 Def->eraseFromParent();
2780
2781 return true;
2782}
2783
2784// Try to optimize SGPR reg sequences that are splat <s, s> or <s, s, s, s>
2785// where all uses are PackedSingleSGPR64BitInst, replacing with <s, undef, ...>
2786bool SIFoldOperandsImpl::tryFoldSGPRSplatRegSequence(MachineInstr &MI) {
2787 assert(MI.isRegSequence());
2788
2789 if (!ST->hasPackedFP64SingleSGPROps() && !ST->hasPackedU64SingleSGPROps())
2790 return false;
2791
2792 Register Reg = MI.getOperand(i: 0).getReg();
2793
2794 // Only optimize 128-bit SGPR register sequences
2795 const TargetRegisterClass *RegClass = MRI->getRegClass(Reg);
2796 if (!TRI->isSGPRClass(RC: RegClass) || TRI->getRegSizeInBits(RC: *RegClass) != 128)
2797 return false;
2798
2799 SmallVector<std::pair<MachineOperand *, unsigned>, 32> Defs;
2800 if (!getRegSeqInit(Defs, UseReg: Reg))
2801 return false;
2802
2803 // Check if this is a splat pattern
2804 if (Defs.size() <= 1)
2805 return false;
2806
2807 const auto &[FirstOp, _] = Defs.front();
2808 if (!FirstOp->isReg())
2809 return false;
2810
2811 Register FirstReg = FirstOp->getReg();
2812 unsigned FirstSubReg = FirstOp->getSubReg();
2813
2814 const TargetRegisterClass *FirstRegClass = MRI->getRegClass(Reg: FirstReg);
2815 if (!TRI->isSGPRClass(RC: FirstRegClass))
2816 return false;
2817
2818 // Check remaining elements match first
2819 if (!llvm::all_of(Range: llvm::drop_begin(RangeOrContainer&: Defs), P: [&](const auto &Def) {
2820 const auto &[Op, _] = Def;
2821 return Op->isReg() && Op->getReg() == FirstReg &&
2822 Op->getSubReg() == FirstSubReg;
2823 }))
2824 return false;
2825
2826 // Check if all uses are isSingleSGPRReadInst
2827 for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
2828 if (!AMDGPU::isPackedSingleSGPR64BitInst(Opc: UseMI.getOpcode()))
2829 return false;
2830 }
2831
2832 // Create new reg sequence with <s, undef, undef, ...>
2833 Register NewDst = MRI->createVirtualRegister(RegClass);
2834 MachineInstrBuilder RS = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2835 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDst);
2836
2837 // Add the first operand
2838 FirstOp->setIsKill(false);
2839 RS.add(MO: *FirstOp);
2840 RS.addImm(Val: Defs[0].second);
2841
2842 // Add undef for remaining lanes
2843 // Create an undef virtual register for the same register class
2844 Register UndefReg = MRI->createVirtualRegister(RegClass: FirstRegClass);
2845 for (unsigned i = 1; i < Defs.size(); ++i) {
2846 RS.addReg(RegNo: UndefReg, Flags: RegState::Undef);
2847 RS.addImm(Val: Defs[i].second);
2848 }
2849
2850 // Replace all uses
2851 MRI->replaceRegWith(FromReg: Reg, ToReg: NewDst);
2852
2853 LLVM_DEBUG(dbgs() << "Folded splat SGPR reg_sequence: " << MI << " into "
2854 << *RS);
2855
2856 MI.eraseFromParent();
2857 return true;
2858}
2859
2860// Try to fold a reg_sequence with vgpr output and agpr inputs into an
2861// instruction which can take an agpr. So far that means a store.
2862bool SIFoldOperandsImpl::tryFoldRegSequence(MachineInstr &MI) {
2863 assert(MI.isRegSequence());
2864
2865 // Try to optimize SGPR splat sequences first
2866 if (tryFoldSGPRSplatRegSequence(MI))
2867 return true;
2868
2869 auto Reg = MI.getOperand(i: 0).getReg();
2870
2871 if (!ST->hasGFX90AInsts() || !TRI->isVGPR(MRI: *MRI, Reg) ||
2872 !MRI->hasOneNonDBGUse(RegNo: Reg))
2873 return false;
2874
2875 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
2876 if (!getRegSeqInit(Defs, UseReg: Reg))
2877 return false;
2878
2879 for (auto &[Op, SubIdx] : Defs) {
2880 if (!Op->isReg())
2881 return false;
2882 if (TRI->isAGPR(MRI: *MRI, Reg: Op->getReg()))
2883 continue;
2884 // Maybe this is a COPY from AREG
2885 const MachineInstr *SubDef = MRI->getVRegDef(Reg: Op->getReg());
2886 if (!SubDef || !SubDef->isCopy() || SubDef->getOperand(i: 1).getSubReg())
2887 return false;
2888 if (!TRI->isAGPR(MRI: *MRI, Reg: SubDef->getOperand(i: 1).getReg()))
2889 return false;
2890 }
2891
2892 MachineOperand *Op = &*MRI->use_nodbg_begin(RegNo: Reg);
2893 MachineInstr *UseMI = Op->getParent();
2894 while (UseMI->isCopy() && !Op->getSubReg()) {
2895 Reg = UseMI->getOperand(i: 0).getReg();
2896 if (!TRI->isVGPR(MRI: *MRI, Reg) || !MRI->hasOneNonDBGUse(RegNo: Reg))
2897 return false;
2898 Op = &*MRI->use_nodbg_begin(RegNo: Reg);
2899 UseMI = Op->getParent();
2900 }
2901
2902 if (Op->getSubReg())
2903 return false;
2904
2905 unsigned OpIdx = Op - &UseMI->getOperand(i: 0);
2906 const MCInstrDesc &InstDesc = UseMI->getDesc();
2907 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: InstDesc, OpNum: OpIdx);
2908 if (!OpRC || !TRI->isVectorSuperClass(RC: OpRC))
2909 return false;
2910
2911 const auto *NewDstRC = TRI->getEquivalentAGPRClass(SRC: MRI->getRegClass(Reg));
2912 auto Dst = MRI->createVirtualRegister(RegClass: NewDstRC);
2913 auto RS = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2914 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst);
2915
2916 for (auto &[Def, SubIdx] : Defs) {
2917 Def->setIsKill(false);
2918 if (TRI->isAGPR(MRI: *MRI, Reg: Def->getReg())) {
2919 RS.add(MO: *Def);
2920 } else { // This is a copy
2921 MachineInstr *SubDef = MRI->getVRegDef(Reg: Def->getReg());
2922 SubDef->getOperand(i: 1).setIsKill(false);
2923 RS.addReg(RegNo: SubDef->getOperand(i: 1).getReg(), Flags: {}, SubReg: Def->getSubReg());
2924 }
2925 RS.addImm(Val: SubIdx);
2926 }
2927
2928 Op->setReg(Dst);
2929 if (!TII->isOperandLegal(MI: *UseMI, OpIdx, MO: Op)) {
2930 Op->setReg(Reg);
2931 RS->eraseFromParent();
2932 return false;
2933 }
2934
2935 LLVM_DEBUG(dbgs() << "Folded " << *RS << " into " << *UseMI);
2936
2937 // Erase the REG_SEQUENCE eagerly, unless we followed a chain of COPY users,
2938 // in which case we can erase them all later in runOnMachineFunction.
2939 if (MRI->use_nodbg_empty(RegNo: MI.getOperand(i: 0).getReg()))
2940 MI.eraseFromParent();
2941 return true;
2942}
2943
2944/// Checks whether \p Copy is a AGPR -> VGPR copy. Returns `true` on success and
2945/// stores the AGPR register in \p OutReg and the subreg in \p OutSubReg
2946static bool isAGPRCopy(const SIRegisterInfo &TRI,
2947 const MachineRegisterInfo &MRI, const MachineInstr &Copy,
2948 Register &OutReg, unsigned &OutSubReg) {
2949 assert(Copy.isCopy());
2950
2951 const MachineOperand &CopySrc = Copy.getOperand(i: 1);
2952 Register CopySrcReg = CopySrc.getReg();
2953 if (!CopySrcReg.isVirtual())
2954 return false;
2955
2956 // Common case: copy from AGPR directly, e.g.
2957 // %1:vgpr_32 = COPY %0:agpr_32
2958 if (TRI.isAGPR(MRI, Reg: CopySrcReg)) {
2959 OutReg = CopySrcReg;
2960 OutSubReg = CopySrc.getSubReg();
2961 return true;
2962 }
2963
2964 // Sometimes it can also involve two copies, e.g.
2965 // %1:vgpr_256 = COPY %0:agpr_256
2966 // %2:vgpr_32 = COPY %1:vgpr_256.sub0
2967 const MachineInstr *CopySrcDef = MRI.getVRegDef(Reg: CopySrcReg);
2968 if (!CopySrcDef || !CopySrcDef->isCopy())
2969 return false;
2970
2971 const MachineOperand &OtherCopySrc = CopySrcDef->getOperand(i: 1);
2972 Register OtherCopySrcReg = OtherCopySrc.getReg();
2973 if (!OtherCopySrcReg.isVirtual() ||
2974 CopySrcDef->getOperand(i: 0).getSubReg() != AMDGPU::NoSubRegister ||
2975 OtherCopySrc.getSubReg() != AMDGPU::NoSubRegister ||
2976 !TRI.isAGPR(MRI, Reg: OtherCopySrcReg))
2977 return false;
2978
2979 OutReg = OtherCopySrcReg;
2980 OutSubReg = CopySrc.getSubReg();
2981 return true;
2982}
2983
2984// Try to hoist an AGPR to VGPR copy across a PHI.
2985// This should allow folding of an AGPR into a consumer which may support it.
2986//
2987// Example 1: LCSSA PHI
2988// loop:
2989// %1:vreg = COPY %0:areg
2990// exit:
2991// %2:vreg = PHI %1:vreg, %loop
2992// =>
2993// loop:
2994// exit:
2995// %1:areg = PHI %0:areg, %loop
2996// %2:vreg = COPY %1:areg
2997//
2998// Example 2: PHI with multiple incoming values:
2999// entry:
3000// %1:vreg = GLOBAL_LOAD(..)
3001// loop:
3002// %2:vreg = PHI %1:vreg, %entry, %5:vreg, %loop
3003// %3:areg = COPY %2:vreg
3004// %4:areg = (instr using %3:areg)
3005// %5:vreg = COPY %4:areg
3006// =>
3007// entry:
3008// %1:vreg = GLOBAL_LOAD(..)
3009// %2:areg = COPY %1:vreg
3010// loop:
3011// %3:areg = PHI %2:areg, %entry, %X:areg,
3012// %4:areg = (instr using %3:areg)
3013bool SIFoldOperandsImpl::tryFoldPhiAGPR(MachineInstr &PHI) {
3014 assert(PHI.isPHI());
3015
3016 Register PhiOut = PHI.getOperand(i: 0).getReg();
3017 if (!TRI->isVGPR(MRI: *MRI, Reg: PhiOut))
3018 return false;
3019
3020 // Iterate once over all incoming values of the PHI to check if this PHI is
3021 // eligible, and determine the exact AGPR RC we'll target.
3022 const TargetRegisterClass *ARC = nullptr;
3023 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
3024 MachineOperand &MO = PHI.getOperand(i: K);
3025 MachineInstr *Copy = MRI->getVRegDef(Reg: MO.getReg());
3026 if (!Copy || !Copy->isCopy())
3027 continue;
3028
3029 Register AGPRSrc;
3030 unsigned AGPRRegMask = AMDGPU::NoSubRegister;
3031 if (!isAGPRCopy(TRI: *TRI, MRI: *MRI, Copy: *Copy, OutReg&: AGPRSrc, OutSubReg&: AGPRRegMask))
3032 continue;
3033
3034 const TargetRegisterClass *CopyInRC = MRI->getRegClass(Reg: AGPRSrc);
3035 if (const auto *SubRC = TRI->getSubRegisterClass(CopyInRC, AGPRRegMask))
3036 CopyInRC = SubRC;
3037
3038 if (ARC && !ARC->hasSubClassEq(RC: CopyInRC))
3039 return false;
3040 ARC = CopyInRC;
3041 }
3042
3043 if (!ARC)
3044 return false;
3045
3046 bool IsAGPR32 = (ARC == &AMDGPU::AGPR_32RegClass);
3047
3048 // Rewrite the PHI's incoming values to ARC.
3049 LLVM_DEBUG(dbgs() << "Folding AGPR copies into: " << PHI);
3050 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
3051 MachineOperand &MO = PHI.getOperand(i: K);
3052 Register Reg = MO.getReg();
3053
3054 MachineBasicBlock::iterator InsertPt;
3055 MachineBasicBlock *InsertMBB = nullptr;
3056
3057 // Look at the def of Reg, ignoring all copies.
3058 unsigned CopyOpc = AMDGPU::COPY;
3059 if (MachineInstr *Def = MRI->getVRegDef(Reg)) {
3060
3061 // Look at pre-existing COPY instructions from ARC: Steal the operand. If
3062 // the copy was single-use, it will be removed by DCE later.
3063 if (Def->isCopy()) {
3064 Register AGPRSrc;
3065 unsigned AGPRSubReg = AMDGPU::NoSubRegister;
3066 if (isAGPRCopy(TRI: *TRI, MRI: *MRI, Copy: *Def, OutReg&: AGPRSrc, OutSubReg&: AGPRSubReg)) {
3067 MO.setReg(AGPRSrc);
3068 MO.setSubReg(AGPRSubReg);
3069 continue;
3070 }
3071
3072 // If this is a multi-use SGPR -> VGPR copy, use V_ACCVGPR_WRITE on
3073 // GFX908 directly instead of a COPY. Otherwise, SIFoldOperand may try
3074 // to fold the sgpr -> vgpr -> agpr copy into a sgpr -> agpr copy which
3075 // is unlikely to be profitable.
3076 //
3077 // Note that V_ACCVGPR_WRITE is only used for AGPR_32.
3078 MachineOperand &CopyIn = Def->getOperand(i: 1);
3079 if (IsAGPR32 && !ST->hasGFX90AInsts() && !MRI->hasOneNonDBGUse(RegNo: Reg) &&
3080 TRI->isSGPRReg(MRI: *MRI, Reg: CopyIn.getReg()))
3081 CopyOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
3082 }
3083
3084 InsertMBB = Def->getParent();
3085 InsertPt = InsertMBB->SkipPHIsLabelsAndDebug(I: ++Def->getIterator());
3086 } else {
3087 InsertMBB = PHI.getOperand(i: MO.getOperandNo() + 1).getMBB();
3088 InsertPt = InsertMBB->getFirstTerminator();
3089 }
3090
3091 Register NewReg = MRI->createVirtualRegister(RegClass: ARC);
3092 MachineInstr *MI = BuildMI(BB&: *InsertMBB, I: InsertPt, MIMD: PHI.getDebugLoc(),
3093 MCID: TII->get(Opcode: CopyOpc), DestReg: NewReg)
3094 .addReg(RegNo: Reg);
3095 MO.setReg(NewReg);
3096
3097 (void)MI;
3098 LLVM_DEBUG(dbgs() << " Created COPY: " << *MI);
3099 }
3100
3101 // Replace the PHI's result with a new register.
3102 Register NewReg = MRI->createVirtualRegister(RegClass: ARC);
3103 PHI.getOperand(i: 0).setReg(NewReg);
3104
3105 // COPY that new register back to the original PhiOut register. This COPY will
3106 // usually be folded out later.
3107 MachineBasicBlock *MBB = PHI.getParent();
3108 BuildMI(BB&: *MBB, I: MBB->getFirstNonPHI(), MIMD: PHI.getDebugLoc(),
3109 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: PhiOut)
3110 .addReg(RegNo: NewReg);
3111
3112 LLVM_DEBUG(dbgs() << " Done: Folded " << PHI);
3113 return true;
3114}
3115
3116// Attempt to convert VGPR load to an AGPR load.
3117bool SIFoldOperandsImpl::tryFoldLoad(MachineInstr &MI) {
3118 assert(MI.mayLoad());
3119 if (!ST->hasGFX90AInsts() || MI.getNumExplicitDefs() != 1)
3120 return false;
3121
3122 MachineOperand &Def = MI.getOperand(i: 0);
3123 if (!Def.isDef())
3124 return false;
3125
3126 Register DefReg = Def.getReg();
3127
3128 if (DefReg.isPhysical() || !TRI->isVGPR(MRI: *MRI, Reg: DefReg))
3129 return false;
3130
3131 SmallVector<const MachineInstr *, 8> Users(
3132 llvm::make_pointer_range(Range: MRI->use_nodbg_instructions(Reg: DefReg)));
3133 SmallVector<Register, 8> MoveRegs;
3134
3135 if (Users.empty())
3136 return false;
3137
3138 // Check that all uses a copy to an agpr or a reg_sequence producing an agpr.
3139 while (!Users.empty()) {
3140 const MachineInstr *I = Users.pop_back_val();
3141 if (!I->isCopy() && !I->isRegSequence())
3142 return false;
3143 Register DstReg = I->getOperand(i: 0).getReg();
3144 // Physical registers may have more than one instruction definitions
3145 if (DstReg.isPhysical())
3146 return false;
3147 if (TRI->isAGPR(MRI: *MRI, Reg: DstReg))
3148 continue;
3149 MoveRegs.push_back(Elt: DstReg);
3150 for (const MachineInstr &U : MRI->use_nodbg_instructions(Reg: DstReg))
3151 Users.push_back(Elt: &U);
3152 }
3153
3154 const TargetRegisterClass *RC = MRI->getRegClass(Reg: DefReg);
3155 MRI->setRegClass(Reg: DefReg, RC: TRI->getEquivalentAGPRClass(SRC: RC));
3156 if (!TII->isOperandLegal(MI, OpIdx: 0, MO: &Def)) {
3157 MRI->setRegClass(Reg: DefReg, RC);
3158 return false;
3159 }
3160
3161 while (!MoveRegs.empty()) {
3162 Register Reg = MoveRegs.pop_back_val();
3163 MRI->setRegClass(Reg, RC: TRI->getEquivalentAGPRClass(SRC: MRI->getRegClass(Reg)));
3164 }
3165
3166 LLVM_DEBUG(dbgs() << "Folded " << MI);
3167
3168 return true;
3169}
3170
3171// tryFoldPhiAGPR will aggressively try to create AGPR PHIs.
3172// For GFX90A and later, this is pretty much always a good thing, but for GFX908
3173// there's cases where it can create a lot more AGPR-AGPR copies, which are
3174// expensive on this architecture due to the lack of V_ACCVGPR_MOV.
3175//
3176// This function looks at all AGPR PHIs in a basic block and collects their
3177// operands. Then, it checks for register that are used more than once across
3178// all PHIs and caches them in a VGPR. This prevents ExpandPostRAPseudo from
3179// having to create one VGPR temporary per use, which can get very messy if
3180// these PHIs come from a broken-up large PHI (e.g. 32 AGPR phis, one per vector
3181// element).
3182//
3183// Example
3184// a:
3185// %in:agpr_256 = COPY %foo:vgpr_256
3186// c:
3187// %x:agpr_32 = ..
3188// b:
3189// %0:areg = PHI %in.sub0:agpr_32, %a, %x, %c
3190// %1:areg = PHI %in.sub0:agpr_32, %a, %y, %c
3191// %2:areg = PHI %in.sub0:agpr_32, %a, %z, %c
3192// =>
3193// a:
3194// %in:agpr_256 = COPY %foo:vgpr_256
3195// %tmp:vgpr_32 = V_ACCVGPR_READ_B32_e64 %in.sub0:agpr_32
3196// %tmp_agpr:agpr_32 = COPY %tmp
3197// c:
3198// %x:agpr_32 = ..
3199// b:
3200// %0:areg = PHI %tmp_agpr, %a, %x, %c
3201// %1:areg = PHI %tmp_agpr, %a, %y, %c
3202// %2:areg = PHI %tmp_agpr, %a, %z, %c
3203bool SIFoldOperandsImpl::tryOptimizeAGPRPhis(MachineBasicBlock &MBB) {
3204 // This is only really needed on GFX908 where AGPR-AGPR copies are
3205 // unreasonably difficult.
3206 if (ST->hasGFX90AInsts())
3207 return false;
3208
3209 // Look at all AGPR Phis and collect the register + subregister used.
3210 DenseMap<std::pair<Register, unsigned>, std::vector<MachineOperand *>>
3211 RegToMO;
3212
3213 for (auto &MI : MBB) {
3214 if (!MI.isPHI())
3215 break;
3216
3217 if (!TRI->isAGPR(MRI: *MRI, Reg: MI.getOperand(i: 0).getReg()))
3218 continue;
3219
3220 for (unsigned K = 1; K < MI.getNumOperands(); K += 2) {
3221 MachineOperand &PhiMO = MI.getOperand(i: K);
3222 if (!PhiMO.getSubReg())
3223 continue;
3224 RegToMO[{PhiMO.getReg(), PhiMO.getSubReg()}].push_back(x: &PhiMO);
3225 }
3226 }
3227
3228 // For all (Reg, SubReg) pair that are used more than once, cache the value in
3229 // a VGPR.
3230 bool Changed = false;
3231 for (const auto &[Entry, MOs] : RegToMO) {
3232 if (MOs.size() == 1)
3233 continue;
3234
3235 const auto [Reg, SubReg] = Entry;
3236 MachineInstr *Def = MRI->getVRegDef(Reg);
3237 MachineBasicBlock *DefMBB = Def->getParent();
3238
3239 // Create a copy in a VGPR using V_ACCVGPR_READ_B32_e64 so it's not folded
3240 // out.
3241 const TargetRegisterClass *ARC = getRegOpRC(MRI: *MRI, TRI: *TRI, MO: *MOs.front());
3242 Register TempVGPR =
3243 MRI->createVirtualRegister(RegClass: TRI->getEquivalentVGPRClass(SRC: ARC));
3244 MachineInstr *VGPRCopy =
3245 BuildMI(BB&: *DefMBB, I: ++Def->getIterator(), MIMD: Def->getDebugLoc(),
3246 MCID: TII->get(Opcode: AMDGPU::V_ACCVGPR_READ_B32_e64), DestReg: TempVGPR)
3247 .addReg(RegNo: Reg, /* flags */ Flags: {}, SubReg);
3248
3249 // Copy back to an AGPR and use that instead of the AGPR subreg in all MOs.
3250 Register TempAGPR = MRI->createVirtualRegister(RegClass: ARC);
3251 BuildMI(BB&: *DefMBB, I: ++VGPRCopy->getIterator(), MIMD: Def->getDebugLoc(),
3252 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: TempAGPR)
3253 .addReg(RegNo: TempVGPR);
3254
3255 LLVM_DEBUG(dbgs() << "Caching AGPR into VGPR: " << *VGPRCopy);
3256 for (MachineOperand *MO : MOs) {
3257 MO->setReg(TempAGPR);
3258 MO->setSubReg(AMDGPU::NoSubRegister);
3259 LLVM_DEBUG(dbgs() << " Changed PHI Operand: " << *MO << "\n");
3260 }
3261
3262 Changed = true;
3263 }
3264
3265 return Changed;
3266}
3267
3268bool SIFoldOperandsImpl::run(MachineFunction &MF, const MachineLoopInfo *MLI) {
3269 this->MF = &MF;
3270 MRI = &MF.getRegInfo();
3271 ST = &MF.getSubtarget<GCNSubtarget>();
3272 TII = ST->getInstrInfo();
3273 TRI = &TII->getRegisterInfo();
3274 MFI = MF.getInfo<SIMachineFunctionInfo>();
3275 this->MLI = MLI;
3276
3277 // omod is ignored by hardware if IEEE bit is enabled. omod also does not
3278 // correctly handle signed zeros.
3279 //
3280 // FIXME: Also need to check strictfp
3281 bool IsIEEEMode = MFI->getMode().IEEE;
3282
3283 bool Changed = false;
3284 for (MachineBasicBlock *MBB : depth_first(G: &MF)) {
3285 MachineOperand *CurrentKnownM0Val = nullptr;
3286 for (auto &MI : make_early_inc_range(Range&: *MBB)) {
3287 Changed |= tryFoldCndMask(MI);
3288
3289 // PeepholeOptimizer may have folded an inline immediate directly onto an
3290 // instruction operand without materializing it into a register first.
3291 // Such an instruction is never reached through a def->use edge in
3292 // foldInstOperand, so try to constant fold it here.
3293 if (tryConstantFoldOp(MI: &MI)) {
3294 Changed = true;
3295 continue;
3296 }
3297
3298 if (tryFoldRedundantAND(ChildMI&: MI)) {
3299 Changed = true;
3300 continue;
3301 }
3302
3303 if (tryFoldAndExec(MI)) {
3304 Changed = true;
3305 continue;
3306 }
3307
3308 if (MI.isRegSequence() && tryFoldRegSequence(MI)) {
3309 Changed = true;
3310 continue;
3311 }
3312
3313 if (MI.isPHI() && tryFoldPhiAGPR(PHI&: MI)) {
3314 Changed = true;
3315 continue;
3316 }
3317
3318 if (MI.mayLoad() && tryFoldLoad(MI)) {
3319 Changed = true;
3320 continue;
3321 }
3322
3323 if (TII->isFoldableCopy(MI)) {
3324 Changed |= tryFoldFoldableCopy(MI, CurrentKnownM0Val);
3325 continue;
3326 }
3327
3328 // Saw an unknown clobber of m0, so we no longer know what it is.
3329 if (CurrentKnownM0Val && MI.modifiesRegister(Reg: AMDGPU::M0, TRI))
3330 CurrentKnownM0Val = nullptr;
3331
3332 // TODO: Omod might be OK if there is NSZ only on the source
3333 // instruction, and not the omod multiply.
3334 if (IsIEEEMode || !MI.getFlag(Flag: MachineInstr::FmNsz) || !tryFoldOMod(MI))
3335 Changed |= tryFoldClamp(MI);
3336 }
3337
3338 Changed |= tryOptimizeAGPRPhis(MBB&: *MBB);
3339 }
3340
3341 return Changed;
3342}
3343
3344PreservedAnalyses
3345SIFoldOperandsPass::run(MachineFunction &MF,
3346 MachineFunctionAnalysisManager &MFAM) {
3347 MFPropsModifier _(*this, MF);
3348
3349 const MachineLoopInfo *MLI = &MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
3350 bool Changed = SIFoldOperandsImpl().run(MF, MLI);
3351 if (!Changed) {
3352 return PreservedAnalyses::all();
3353 }
3354 auto PA = getMachineFunctionPassPreservedAnalyses();
3355 PA.preserveSet<CFGAnalyses>();
3356 PA.preserve<MachineLoopAnalysis>();
3357 return PA;
3358}
3359