1//===-- SIFoldOperands.cpp - Fold operands --- ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7/// \file
8//===----------------------------------------------------------------------===//
9//
10
11#include "SIFoldOperands.h"
12#include "AMDGPU.h"
13#include "AMDGPULaneMaskUtils.h"
14#include "GCNSubtarget.h"
15#include "SIInstrInfo.h"
16#include "SIMachineFunctionInfo.h"
17#include "SIRegisterInfo.h"
18#include "llvm/ADT/DepthFirstIterator.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineFunctionPass.h"
21#include "llvm/CodeGen/MachineLoopInfo.h"
22#include "llvm/CodeGen/MachineOperand.h"
23#include "llvm/InitializePasses.h"
24
25#define DEBUG_TYPE "si-fold-operands"
26using namespace llvm;
27
28namespace {
29
30/// Track a value we may want to fold into downstream users, applying
31/// subregister extracts along the way.
32struct FoldableDef {
33 union {
34 MachineOperand *OpToFold = nullptr;
35 uint64_t ImmToFold;
36 int FrameIndexToFold;
37 };
38
39 /// Register class of the originally defined value.
40 const TargetRegisterClass *DefRC = nullptr;
41
42 /// Track the original defining instruction for the value.
43 const MachineInstr *DefMI = nullptr;
44
45 /// Subregister to apply to the value at the use point.
46 unsigned DefSubReg = AMDGPU::NoSubRegister;
47
48 /// Kind of value stored in the union.
49 MachineOperand::MachineOperandType Kind;
50
51 FoldableDef() = delete;
52 FoldableDef(MachineOperand &FoldOp, const TargetRegisterClass *DefRC,
53 unsigned DefSubReg = AMDGPU::NoSubRegister)
54 : DefRC(DefRC), DefSubReg(DefSubReg), Kind(FoldOp.getType()) {
55
56 if (FoldOp.isImm()) {
57 ImmToFold = FoldOp.getImm();
58 } else if (FoldOp.isFI()) {
59 FrameIndexToFold = FoldOp.getIndex();
60 } else {
61 assert(FoldOp.isReg() || FoldOp.isGlobal());
62 OpToFold = &FoldOp;
63 }
64
65 DefMI = FoldOp.getParent();
66 }
67
68 FoldableDef(int64_t FoldImm, const TargetRegisterClass *DefRC,
69 unsigned DefSubReg = AMDGPU::NoSubRegister)
70 : ImmToFold(FoldImm), DefRC(DefRC), DefSubReg(DefSubReg),
71 Kind(MachineOperand::MO_Immediate) {}
72
73 /// Copy the current def and apply \p SubReg to the value.
74 FoldableDef getWithSubReg(const SIRegisterInfo &TRI, unsigned SubReg) const {
75 FoldableDef Copy(*this);
76 Copy.DefSubReg = TRI.composeSubRegIndices(a: DefSubReg, b: SubReg);
77 return Copy;
78 }
79
80 bool isReg() const { return Kind == MachineOperand::MO_Register; }
81
82 Register getReg() const {
83 assert(isReg());
84 return OpToFold->getReg();
85 }
86
87 unsigned getSubReg() const {
88 assert(isReg());
89 return OpToFold->getSubReg();
90 }
91
92 bool isImm() const { return Kind == MachineOperand::MO_Immediate; }
93
94 bool isFI() const {
95 return Kind == MachineOperand::MO_FrameIndex;
96 }
97
98 int getFI() const {
99 assert(isFI());
100 return FrameIndexToFold;
101 }
102
103 bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; }
104
105 /// Return the effective immediate value defined by this instruction, after
106 /// application of any subregister extracts which may exist between the use
107 /// and def instruction.
108 std::optional<int64_t> getEffectiveImmVal() const {
109 assert(isImm());
110 return SIInstrInfo::extractSubregFromImm(ImmVal: ImmToFold, SubRegIndex: DefSubReg);
111 }
112
113 /// Check if it is legal to fold this effective value into \p MI's \p OpNo
114 /// operand.
115 bool isOperandLegal(const SIInstrInfo &TII, const MachineInstr &MI,
116 unsigned OpIdx) const {
117 switch (Kind) {
118 case MachineOperand::MO_Immediate: {
119 std::optional<int64_t> ImmToFold = getEffectiveImmVal();
120 if (!ImmToFold)
121 return false;
122
123 // TODO: Should verify the subregister index is supported by the class
124 // TODO: Avoid the temporary MachineOperand
125 MachineOperand TmpOp = MachineOperand::CreateImm(Val: *ImmToFold);
126 return TII.isOperandLegal(MI, OpIdx, MO: &TmpOp);
127 }
128 case MachineOperand::MO_FrameIndex: {
129 if (DefSubReg != AMDGPU::NoSubRegister)
130 return false;
131 MachineOperand TmpOp = MachineOperand::CreateFI(Idx: FrameIndexToFold);
132 return TII.isOperandLegal(MI, OpIdx, MO: &TmpOp);
133 }
134 default:
135 // TODO: Try to apply DefSubReg, for global address we can extract
136 // low/high.
137 if (DefSubReg != AMDGPU::NoSubRegister)
138 return false;
139 return TII.isOperandLegal(MI, OpIdx, MO: OpToFold);
140 }
141
142 llvm_unreachable("covered MachineOperand kind switch");
143 }
144};
145
146struct FoldCandidate {
147 MachineInstr *UseMI;
148 FoldableDef Def;
149 int ShrinkOpcode;
150 unsigned UseOpNo;
151 bool Commuted;
152
153 FoldCandidate(MachineInstr *MI, unsigned OpNo, FoldableDef Def,
154 bool Commuted = false, int ShrinkOp = -1)
155 : UseMI(MI), Def(Def), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo),
156 Commuted(Commuted) {}
157
158 bool isFI() const { return Def.isFI(); }
159
160 int getFI() const {
161 assert(isFI());
162 return Def.FrameIndexToFold;
163 }
164
165 bool isImm() const { return Def.isImm(); }
166
167 bool isReg() const { return Def.isReg(); }
168
169 Register getReg() const { return Def.getReg(); }
170
171 bool isGlobal() const { return Def.isGlobal(); }
172
173 bool needsShrink() const { return ShrinkOpcode != -1; }
174};
175
176class SIFoldOperandsImpl {
177public:
178 MachineFunction *MF;
179 MachineRegisterInfo *MRI;
180 const SIInstrInfo *TII;
181 const SIRegisterInfo *TRI;
182 const GCNSubtarget *ST;
183 const SIMachineFunctionInfo *MFI;
184 const MachineLoopInfo *MLI;
185
186 bool frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
187 const FoldableDef &OpToFold) const;
188
189 // TODO: Just use TII::getVALUOp
190 unsigned convertToVALUOp(unsigned Opc, bool UseVOP3 = false) const {
191 switch (Opc) {
192 case AMDGPU::S_ADD_I32: {
193 if (ST->hasAddNoCarryInsts())
194 return UseVOP3 ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_U32_e32;
195 return UseVOP3 ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
196 }
197 case AMDGPU::S_OR_B32:
198 return UseVOP3 ? AMDGPU::V_OR_B32_e64 : AMDGPU::V_OR_B32_e32;
199 case AMDGPU::S_AND_B32:
200 return UseVOP3 ? AMDGPU::V_AND_B32_e64 : AMDGPU::V_AND_B32_e32;
201 case AMDGPU::S_MUL_I32:
202 return AMDGPU::V_MUL_LO_U32_e64;
203 default:
204 return AMDGPU::INSTRUCTION_LIST_END;
205 }
206 }
207
208 bool foldCopyToVGPROfScalarAddOfFrameIndex(Register DstReg, Register SrcReg,
209 MachineInstr &MI) const;
210
211 bool updateOperand(FoldCandidate &Fold) const;
212
213 bool canUseImmWithOpSel(const MachineInstr *MI, unsigned UseOpNo,
214 int64_t ImmVal) const;
215
216 /// Try to fold immediate \p ImmVal into \p MI's operand at index \p UseOpNo.
217 bool tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
218 int64_t ImmVal) const;
219
220 bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
221 MachineInstr *MI, unsigned OpNo,
222 const FoldableDef &OpToFold) const;
223 bool isUseSafeToFold(const MachineInstr &MI,
224 const MachineOperand &UseMO) const;
225 bool isTemporallyDivergentUse(const FoldableDef &OpToFold,
226 const MachineInstr &UseMI) const;
227
228 const TargetRegisterClass *getRegSeqInit(
229 MachineInstr &RegSeq,
230 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs) const;
231
232 const TargetRegisterClass *
233 getRegSeqInit(SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
234 Register UseReg) const;
235
236 std::pair<int64_t, const TargetRegisterClass *>
237 isRegSeqSplat(MachineInstr &RegSeg) const;
238
239 bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
240 int64_t SplatVal,
241 const TargetRegisterClass *SplatRC) const;
242
243 bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
244 unsigned UseOpIdx,
245 SmallVectorImpl<FoldCandidate> &FoldList) const;
246 bool foldOperand(FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
247 SmallVectorImpl<FoldCandidate> &FoldList,
248 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
249
250 struct ANDMaskResult {
251 int64_t Mask;
252 Register Reg;
253 unsigned RegIdx;
254 };
255
256 std::optional<ANDMaskResult> getANDMaskRegOperand(MachineInstr &AndMI) const;
257
258 bool tryConstantFoldOp(MachineInstr *MI) const;
259 bool tryFoldCndMask(MachineInstr &MI) const;
260 bool tryFoldRedundantAND(MachineInstr &ChildMI) const;
261 bool tryFoldAndExec(MachineInstr &MI) const;
262 bool foldInstOperand(MachineInstr &MI, const FoldableDef &OpToFold) const;
263
264 bool foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const;
265 bool tryFoldFoldableCopy(MachineInstr &MI,
266 MachineOperand *&CurrentKnownM0Val) const;
267
268 const MachineOperand *isClamp(const MachineInstr &MI) const;
269 bool tryFoldClamp(MachineInstr &MI);
270
271 std::pair<const MachineOperand *, int> isOMod(const MachineInstr &MI) const;
272 bool tryFoldOMod(MachineInstr &MI);
273 bool tryFoldSGPRSplatRegSequence(MachineInstr &MI);
274 bool tryFoldRegSequence(MachineInstr &MI);
275 bool tryFoldPhiAGPR(MachineInstr &MI);
276 bool tryFoldLoad(MachineInstr &MI);
277
278 bool tryOptimizeAGPRPhis(MachineBasicBlock &MBB);
279
280public:
281 SIFoldOperandsImpl() = default;
282
283 bool run(MachineFunction &MF, const MachineLoopInfo *MLI);
284};
285
286class SIFoldOperandsLegacy : public MachineFunctionPass {
287public:
288 static char ID;
289
290 SIFoldOperandsLegacy() : MachineFunctionPass(ID) {}
291
292 bool runOnMachineFunction(MachineFunction &MF) override {
293 if (skipFunction(F: MF.getFunction()))
294 return false;
295 const MachineLoopInfo *MLI =
296 &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
297 return SIFoldOperandsImpl().run(MF, MLI);
298 }
299
300 StringRef getPassName() const override { return "SI Fold Operands"; }
301
302 void getAnalysisUsage(AnalysisUsage &AU) const override {
303 AU.setPreservesCFG();
304 AU.addRequired<MachineLoopInfoWrapperPass>();
305 AU.addPreserved<MachineLoopInfoWrapperPass>();
306 MachineFunctionPass::getAnalysisUsage(AU);
307 }
308
309 MachineFunctionProperties getRequiredProperties() const override {
310 return MachineFunctionProperties().setIsSSA();
311 }
312};
313
314} // End anonymous namespace.
315
316INITIALIZE_PASS_BEGIN(SIFoldOperandsLegacy, DEBUG_TYPE, "SI Fold Operands",
317 false, false)
318INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
319INITIALIZE_PASS_END(SIFoldOperandsLegacy, DEBUG_TYPE, "SI Fold Operands", false,
320 false)
321
322char SIFoldOperandsLegacy::ID = 0;
323
324char &llvm::SIFoldOperandsLegacyID = SIFoldOperandsLegacy::ID;
325
326static const TargetRegisterClass *getRegOpRC(const MachineRegisterInfo &MRI,
327 const TargetRegisterInfo &TRI,
328 const MachineOperand &MO) {
329 const TargetRegisterClass *RC = MRI.getRegClass(Reg: MO.getReg());
330 if (const TargetRegisterClass *SubRC =
331 TRI.getSubRegisterClass(SuperRC: RC, SubRegIdx: MO.getSubReg()))
332 RC = SubRC;
333 return RC;
334}
335
336// Map multiply-accumulate opcode to corresponding multiply-add opcode if any.
337static unsigned macToMad(unsigned Opc) {
338 switch (Opc) {
339 case AMDGPU::V_MAC_F32_e64:
340 return AMDGPU::V_MAD_F32_e64;
341 case AMDGPU::V_MAC_F16_e64:
342 return AMDGPU::V_MAD_F16_e64;
343 case AMDGPU::V_FMAC_F32_e64:
344 return AMDGPU::V_FMA_F32_e64;
345 case AMDGPU::V_FMAC_F16_e64:
346 return AMDGPU::V_FMA_F16_gfx9_e64;
347 case AMDGPU::V_FMAC_F16_t16_e64:
348 return AMDGPU::V_FMA_F16_gfx9_t16_e64;
349 case AMDGPU::V_FMAC_F16_fake16_e64:
350 return AMDGPU::V_FMA_F16_gfx9_fake16_e64;
351 case AMDGPU::V_FMAC_LEGACY_F32_e64:
352 return AMDGPU::V_FMA_LEGACY_F32_e64;
353 case AMDGPU::V_FMAC_F64_e64:
354 return AMDGPU::V_FMA_F64_e64;
355 }
356 return AMDGPU::INSTRUCTION_LIST_END;
357}
358
359// TODO: Add heuristic that the frame index might not fit in the addressing mode
360// immediate offset to avoid materializing in loops.
361bool SIFoldOperandsImpl::frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
362 const FoldableDef &OpToFold) const {
363 if (!OpToFold.isFI())
364 return false;
365
366 const unsigned Opc = UseMI.getOpcode();
367 switch (Opc) {
368 case AMDGPU::S_ADD_I32:
369 case AMDGPU::S_ADD_U32:
370 case AMDGPU::V_ADD_U32_e32:
371 case AMDGPU::V_ADD_CO_U32_e32:
372 // TODO: Possibly relax hasOneUse. It matters more for mubuf, since we have
373 // to insert the wave size shift at every point we use the index.
374 // TODO: Fix depending on visit order to fold immediates into the operand
375 return UseMI.getOperand(i: OpNo == 1 ? 2 : 1).isImm() &&
376 MRI->hasOneNonDBGUse(RegNo: UseMI.getOperand(i: OpNo).getReg());
377 case AMDGPU::V_ADD_U32_e64:
378 case AMDGPU::V_ADD_CO_U32_e64:
379 return UseMI.getOperand(i: OpNo == 2 ? 3 : 2).isImm() &&
380 MRI->hasOneNonDBGUse(RegNo: UseMI.getOperand(i: OpNo).getReg());
381 default:
382 break;
383 }
384
385 if (TII->isMUBUF(MI: UseMI))
386 return OpNo == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
387 if (!TII->isFLATScratch(MI: UseMI))
388 return false;
389
390 int SIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
391 if (OpNo == SIdx)
392 return true;
393
394 int VIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
395 return OpNo == VIdx && SIdx == -1;
396}
397
398/// Fold %vgpr = COPY (S_ADD_I32 x, frameindex)
399///
400/// => %vgpr = V_ADD_U32 x, frameindex
401bool SIFoldOperandsImpl::foldCopyToVGPROfScalarAddOfFrameIndex(
402 Register DstReg, Register SrcReg, MachineInstr &MI) const {
403 if (!SrcReg.isVirtual())
404 return false;
405
406 if (TRI->isVGPR(MRI: *MRI, Reg: DstReg) && TRI->isSGPRReg(MRI: *MRI, Reg: SrcReg) &&
407 MRI->hasOneNonDBGUse(RegNo: SrcReg)) {
408 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
409 if (!Def || Def->getNumOperands() != 4)
410 return false;
411
412 MachineOperand *Src0 = &Def->getOperand(i: 1);
413 MachineOperand *Src1 = &Def->getOperand(i: 2);
414
415 // TODO: This is profitable with more operand types, and for more
416 // opcodes. But ultimately this is working around poor / nonexistent
417 // regbankselect.
418 if (!Src0->isFI() && !Src1->isFI())
419 return false;
420
421 if (Src0->isFI())
422 std::swap(a&: Src0, b&: Src1);
423
424 const bool UseVOP3 = !Src0->isImm() || TII->isInlineConstant(MO: *Src0);
425 unsigned NewOp = convertToVALUOp(Opc: Def->getOpcode(), UseVOP3);
426 if (NewOp == AMDGPU::INSTRUCTION_LIST_END ||
427 !Def->getOperand(i: 3).isDead()) // Check if scc is dead
428 return false;
429
430 MachineBasicBlock *MBB = Def->getParent();
431 const DebugLoc &DL = Def->getDebugLoc();
432 if (NewOp != AMDGPU::V_ADD_CO_U32_e32) {
433 MachineInstrBuilder Add =
434 BuildMI(BB&: *MBB, I&: *Def, MIMD: DL, MCID: TII->get(Opcode: NewOp), DestReg: DstReg);
435
436 if (Add->getDesc().getNumDefs() == 2) {
437 Register CarryOutReg = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
438 Add.addDef(RegNo: CarryOutReg, Flags: RegState::Dead);
439 MRI->setRegAllocationHint(VReg: CarryOutReg, Type: 0, PrefReg: TRI->getVCC());
440 }
441
442 Add.add(MO: *Src0).add(MO: *Src1).setMIFlags(Def->getFlags());
443 if (AMDGPU::hasNamedOperand(Opcode: NewOp, NamedIdx: AMDGPU::OpName::clamp))
444 Add.addImm(Val: 0);
445
446 Def->eraseFromParent();
447 MI.eraseFromParent();
448 return true;
449 }
450
451 assert(NewOp == AMDGPU::V_ADD_CO_U32_e32);
452
453 MachineBasicBlock::LivenessQueryResult Liveness =
454 MBB->computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: *Def, Neighborhood: 16);
455 if (Liveness == MachineBasicBlock::LQR_Dead) {
456 // TODO: If src1 satisfies operand constraints, use vop3 version.
457 BuildMI(BB&: *MBB, I&: *Def, MIMD: DL, MCID: TII->get(Opcode: NewOp), DestReg: DstReg)
458 .add(MO: *Src0)
459 .add(MO: *Src1)
460 .setOperandDead(3) // implicit-def $vcc
461 .setMIFlags(Def->getFlags());
462 Def->eraseFromParent();
463 MI.eraseFromParent();
464 return true;
465 }
466 }
467
468 return false;
469}
470
471bool SIFoldOperandsImpl::canUseImmWithOpSel(const MachineInstr *MI,
472 unsigned UseOpNo,
473 int64_t ImmVal) const {
474 if (!SIInstrFlags::isPacked(O: *MI) || SIInstrFlags::isMAI(O: *MI) ||
475 SIInstrFlags::isWMMA(O: *MI) || SIInstrFlags::isSWMMAC(O: *MI) ||
476 (ST->hasDOTOpSelHazard() && SIInstrFlags::isDOT(O: *MI)))
477 return false;
478
479 const MachineOperand &Old = MI->getOperand(i: UseOpNo);
480 int OpNo = MI->getOperandNo(I: &Old);
481
482 unsigned Opcode = MI->getOpcode();
483 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
484 switch (OpType) {
485 default:
486 return false;
487 case AMDGPU::OPERAND_REG_IMM_V2FP16:
488 case AMDGPU::OPERAND_REG_IMM_V2BF16:
489 case AMDGPU::OPERAND_REG_IMM_V2INT16:
490 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
491 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
492 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
493 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
494 // VOP3 packed instructions ignore op_sel source modifiers, we cannot encode
495 // two different constants.
496 if (SIInstrFlags::isVOP3(O: *MI) && !SIInstrFlags::isVOP3P(O: *MI) &&
497 static_cast<uint16_t>(ImmVal) != static_cast<uint16_t>(ImmVal >> 16))
498 return false;
499 break;
500 }
501
502 return true;
503}
504
505bool SIFoldOperandsImpl::tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
506 int64_t ImmVal) const {
507 MachineOperand &Old = MI->getOperand(i: UseOpNo);
508 unsigned Opcode = MI->getOpcode();
509 int OpNo = MI->getOperandNo(I: &Old);
510 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
511
512 bool BF16FromUpperFP32 = ST->hasBF16InlineConstFromUpperFP32() &&
513 (OpType == AMDGPU::OPERAND_REG_IMM_V2BF16 ||
514 OpType == AMDGPU::OPERAND_REG_INLINE_C_V2BF16);
515
516 // If the literal can be inlined as-is, apply it and short-circuit the
517 // tests below. The main motivation for this is to avoid unintuitive
518 // uses of opsel.
519 if (!BF16FromUpperFP32 && AMDGPU::isInlinableLiteralV216(Literal: ImmVal, OpType)) {
520 Old.ChangeToImmediate(ImmVal);
521 return true;
522 }
523
524 // Refer to op_sel/op_sel_hi and check if we can change the immediate and
525 // op_sel in a way that allows an inline constant.
526 AMDGPU::OpName ModName = AMDGPU::OpName::NUM_OPERAND_NAMES;
527 unsigned SrcIdx = ~0;
528 if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0)) {
529 ModName = AMDGPU::OpName::src0_modifiers;
530 SrcIdx = 0;
531 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1)) {
532 ModName = AMDGPU::OpName::src1_modifiers;
533 SrcIdx = 1;
534 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src2)) {
535 ModName = AMDGPU::OpName::src2_modifiers;
536 SrcIdx = 2;
537 }
538 assert(ModName != AMDGPU::OpName::NUM_OPERAND_NAMES);
539 int ModIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: ModName);
540 MachineOperand &Mod = MI->getOperand(i: ModIdx);
541 unsigned ModVal = Mod.getImm();
542
543 uint16_t ImmLo =
544 static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_0 ? 16 : 0));
545 uint16_t ImmHi =
546 static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_1 ? 16 : 0));
547 uint32_t Imm = (static_cast<uint32_t>(ImmHi) << 16) | ImmLo;
548 unsigned NewModVal = ModVal & ~(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
549
550 // Helper function that attempts to inline the given value with a newly
551 // chosen opsel pattern.
552 auto tryFoldToInline = [&](uint32_t Imm) -> bool {
553 if (!BF16FromUpperFP32 && AMDGPU::isInlinableLiteralV216(Literal: Imm, OpType)) {
554 Mod.setImm(NewModVal | SISrcMods::OP_SEL_1);
555 Old.ChangeToImmediate(ImmVal: Imm);
556 return true;
557 }
558
559 // Try to shuffle the halves around and leverage opsel to get an inline
560 // constant.
561 uint16_t Lo = static_cast<uint16_t>(Imm);
562 uint16_t Hi = static_cast<uint16_t>(Imm >> 16);
563 if (Lo == Hi) {
564 if (AMDGPU::isInlinableLiteralV216(Literal: Lo, OpType)) {
565 // If the target has feature 'BF16InlineConstFromUpperFP32', packed BF16
566 // instructions using inline constant must use OPSEL to select the upper
567 // 16-bits from FP32.
568 if (BF16FromUpperFP32)
569 NewModVal |= (SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
570 Mod.setImm(NewModVal);
571 Old.ChangeToImmediate(ImmVal: Lo);
572 return true;
573 }
574
575 if (!BF16FromUpperFP32 && static_cast<int16_t>(Lo) < 0) {
576 int32_t SExt = static_cast<int16_t>(Lo);
577 if (AMDGPU::isInlinableLiteralV216(Literal: SExt, OpType)) {
578 Mod.setImm(NewModVal);
579 Old.ChangeToImmediate(ImmVal: SExt);
580 return true;
581 }
582 }
583
584 // This check is only useful for integer instructions
585 if (OpType == AMDGPU::OPERAND_REG_IMM_V2INT16) {
586 if (AMDGPU::isInlinableLiteralV216(Literal: Lo << 16, OpType)) {
587 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
588 Old.ChangeToImmediate(ImmVal: static_cast<uint32_t>(Lo) << 16);
589 return true;
590 }
591 }
592 } else {
593 uint32_t Swapped = (static_cast<uint32_t>(Lo) << 16) | Hi;
594 if (!BF16FromUpperFP32 &&
595 AMDGPU::isInlinableLiteralV216(Literal: Swapped, OpType)) {
596 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0);
597 Old.ChangeToImmediate(ImmVal: Swapped);
598 return true;
599 }
600 }
601
602 return false;
603 };
604
605 if (tryFoldToInline(Imm))
606 return true;
607
608 // Replace integer addition by subtraction and vice versa if it allows
609 // folding the immediate to an inline constant.
610 //
611 // We should only ever get here for SrcIdx == 1 due to canonicalization
612 // earlier in the pipeline, but we double-check here to be safe / fully
613 // general.
614 bool IsUAdd = Opcode == AMDGPU::V_PK_ADD_U16;
615 bool IsUSub = Opcode == AMDGPU::V_PK_SUB_U16;
616 if (SrcIdx == 1 && (IsUAdd || IsUSub)) {
617 unsigned ClampIdx =
618 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::clamp);
619 bool Clamp = MI->getOperand(i: ClampIdx).getImm() != 0;
620
621 if (!Clamp) {
622 uint16_t NegLo = -static_cast<uint16_t>(Imm);
623 uint16_t NegHi = -static_cast<uint16_t>(Imm >> 16);
624 uint32_t NegImm = (static_cast<uint32_t>(NegHi) << 16) | NegLo;
625
626 if (tryFoldToInline(NegImm)) {
627 unsigned NegOpcode =
628 IsUAdd ? AMDGPU::V_PK_SUB_U16 : AMDGPU::V_PK_ADD_U16;
629 MI->setDesc(TII->get(Opcode: NegOpcode));
630 return true;
631 }
632 }
633 }
634
635 return false;
636}
637
638bool SIFoldOperandsImpl::updateOperand(FoldCandidate &Fold) const {
639 MachineInstr *MI = Fold.UseMI;
640 MachineOperand &Old = MI->getOperand(i: Fold.UseOpNo);
641 assert(Old.isReg());
642
643 std::optional<int64_t> ImmVal;
644 if (Fold.isImm())
645 ImmVal = Fold.Def.getEffectiveImmVal();
646
647 if (ImmVal && canUseImmWithOpSel(MI: Fold.UseMI, UseOpNo: Fold.UseOpNo, ImmVal: *ImmVal)) {
648 if (tryFoldImmWithOpSel(MI: Fold.UseMI, UseOpNo: Fold.UseOpNo, ImmVal: *ImmVal))
649 return true;
650
651 // We can't represent the candidate as an inline constant. Try as a literal
652 // with the original opsel, checking constant bus limitations.
653 MachineOperand New = MachineOperand::CreateImm(Val: *ImmVal);
654 int OpNo = MI->getOperandNo(I: &Old);
655 if (!TII->isOperandLegal(MI: *MI, OpIdx: OpNo, MO: &New))
656 return false;
657
658 Old.ChangeToImmediate(ImmVal: *ImmVal);
659 return true;
660 }
661
662 if ((Fold.isImm() || Fold.isFI() || Fold.isGlobal()) && Fold.needsShrink()) {
663 MachineBasicBlock *MBB = MI->getParent();
664 auto Liveness = MBB->computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: MI, Neighborhood: 16);
665 if (Liveness != MachineBasicBlock::LQR_Dead) {
666 LLVM_DEBUG(dbgs() << "Not shrinking due to live vcc: " << *MI);
667 return false;
668 }
669
670 int Op32 = Fold.ShrinkOpcode;
671 MachineOperand &Dst0 = MI->getOperand(i: 0);
672 MachineOperand &Dst1 = MI->getOperand(i: 1);
673 assert(Dst0.isDef() && Dst1.isDef());
674
675 bool HaveNonDbgCarryUse = !MRI->use_nodbg_empty(RegNo: Dst1.getReg());
676
677 const TargetRegisterClass *Dst0RC = MRI->getRegClass(Reg: Dst0.getReg());
678 Register NewReg0 = MRI->createVirtualRegister(RegClass: Dst0RC);
679
680 MachineInstr *Inst32 = TII->buildShrunkInst(MI&: *MI, NewOpcode: Op32);
681
682 if (HaveNonDbgCarryUse) {
683 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
684 DestReg: Dst1.getReg())
685 .addReg(RegNo: AMDGPU::VCC, Flags: RegState::Kill);
686 } else {
687 // We only reach here when the carry-out vcc is dead so propagate the dead
688 // flag.
689 Inst32->getOperand(i: 3).setIsDead();
690 }
691
692 // Keep the old instruction around to avoid breaking iterators, but
693 // replace it with a dummy instruction to remove uses.
694 //
695 // FIXME: We should not invert how this pass looks at operands to avoid
696 // this. Should track set of foldable movs instead of looking for uses
697 // when looking at a use.
698 Dst0.setReg(NewReg0);
699 for (unsigned I = MI->getNumOperands() - 1; I > 0; --I)
700 MI->removeOperand(OpNo: I);
701 MI->setDesc(TII->get(Opcode: AMDGPU::IMPLICIT_DEF));
702
703 if (Fold.Commuted)
704 TII->commuteInstruction(MI&: *Inst32, NewMI: false);
705 return true;
706 }
707
708 assert(!Fold.needsShrink() && "not handled");
709
710 if (ImmVal) {
711 if (Old.isTied()) {
712 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: MI->getOpcode());
713 if (NewMFMAOpc == -1)
714 return false;
715 MI->setDesc(TII->get(Opcode: NewMFMAOpc));
716 MI->untieRegOperand(OpIdx: 0);
717 const MCInstrDesc &MCID = MI->getDesc();
718 for (unsigned I = 0; I < MI->getNumDefs(); ++I)
719 if (MCID.getOperandConstraint(OpNum: I, Constraint: MCOI::EARLY_CLOBBER) != -1)
720 MI->getOperand(i: I).setIsEarlyClobber(true);
721 }
722
723 // TODO: Should we try to avoid adding this to the candidate list?
724 MachineOperand New = MachineOperand::CreateImm(Val: *ImmVal);
725 int OpNo = MI->getOperandNo(I: &Old);
726 if (!TII->isOperandLegal(MI: *MI, OpIdx: OpNo, MO: &New))
727 return false;
728
729 if (ST->hasBF16InlineConstFromUpperFP32() &&
730 OpNo ==
731 AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(), Name: AMDGPU::OpName::src0)) {
732 unsigned Opcode = MI->getOpcode();
733 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
734 if ((OpType == AMDGPU::OPERAND_REG_IMM_BF16 ||
735 OpType == AMDGPU::OPERAND_REG_INLINE_C_BF16) &&
736 TII->isInlineConstant(ImmVal: *ImmVal, OperandType: OpType)) {
737 // We can fold it, but we need to set OPSEL
738 int Mod0 =
739 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0_modifiers);
740 if (Mod0 == -1)
741 return false;
742 MachineOperand &ModOp = MI->getOperand(i: Mod0);
743 if (ModOp.getImm())
744 return false;
745 ModOp.setImm(SISrcMods::OP_SEL_0);
746 }
747 }
748
749 Old.ChangeToImmediate(ImmVal: *ImmVal);
750 return true;
751 }
752
753 if (Fold.isGlobal()) {
754 Old.ChangeToGA(GV: Fold.Def.OpToFold->getGlobal(),
755 Offset: Fold.Def.OpToFold->getOffset(),
756 TargetFlags: Fold.Def.OpToFold->getTargetFlags());
757 return true;
758 }
759
760 if (Fold.isFI()) {
761 Old.ChangeToFrameIndex(Idx: Fold.getFI());
762 return true;
763 }
764
765 MachineOperand *New = Fold.Def.OpToFold;
766
767 // Verify the register is compatible with the operand.
768 if (const TargetRegisterClass *OpRC =
769 TII->getRegClass(MCID: MI->getDesc(), OpNum: Fold.UseOpNo)) {
770 const TargetRegisterClass *NewRC =
771 TRI->getRegClassForReg(MRI: *MRI, Reg: New->getReg());
772
773 const TargetRegisterClass *ConstrainRC = OpRC;
774 if (New->getSubReg()) {
775 ConstrainRC =
776 TRI->getMatchingSuperRegClass(A: NewRC, B: OpRC, Idx: New->getSubReg());
777
778 if (!ConstrainRC)
779 return false;
780 }
781
782 if (New->getReg().isVirtual() &&
783 !MRI->constrainRegClass(Reg: New->getReg(), RC: ConstrainRC)) {
784 LLVM_DEBUG(dbgs() << "Cannot constrain " << printReg(New->getReg(), TRI)
785 << TRI->getRegClassName(ConstrainRC) << '\n');
786 return false;
787 }
788 }
789
790 // Rework once the VS_16 register class is updated to include proper
791 // 16-bit SGPRs instead of 32-bit ones.
792 if (Old.getSubReg() == AMDGPU::lo16 && TRI->isSGPRReg(MRI: *MRI, Reg: New->getReg()))
793 Old.setSubReg(AMDGPU::NoSubRegister);
794 if (New->getReg().isPhysical()) {
795 Old.substPhysReg(Reg: New->getReg(), *TRI);
796 } else {
797 Register OldReg = Old.getReg();
798 Old.substVirtReg(Reg: New->getReg(), SubIdx: New->getSubReg(), *TRI);
799 Old.setIsUndef(New->isUndef());
800
801 // If MI is in a BUNDLE, also update header's matching implicit use.
802 if (MI->isBundledWithPred()) {
803 MachineInstr &Header = *getBundleStart(I: MI->getIterator());
804 for (MachineOperand &MO : Header.operands()) {
805 if (MO.getReg() == OldReg) {
806 MO.setReg(New->getReg());
807 MO.setSubReg(New->getSubReg());
808 }
809 }
810 }
811 }
812 return true;
813}
814
815static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
816 FoldCandidate &&Entry) {
817 // Skip additional folding on the same operand.
818 for (FoldCandidate &Fold : FoldList)
819 if (Fold.UseMI == Entry.UseMI && Fold.UseOpNo == Entry.UseOpNo)
820 return;
821 LLVM_DEBUG(dbgs() << "Append " << (Entry.Commuted ? "commuted" : "normal")
822 << " operand " << Entry.UseOpNo << "\n " << *Entry.UseMI);
823 FoldList.push_back(Elt: Entry);
824}
825
826static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
827 MachineInstr *MI, unsigned OpNo,
828 const FoldableDef &FoldOp,
829 bool Commuted = false, int ShrinkOp = -1) {
830 appendFoldCandidate(FoldList,
831 Entry: FoldCandidate(MI, OpNo, FoldOp, Commuted, ShrinkOp));
832}
833
834// Returns true if the instruction is a packed F32 instruction and the
835// corresponding scalar operand reads 32 bits and replicates the bits to both
836// channels.
837static bool isPKF32InstrReplicatesLower32BitsOfScalarOperand(
838 const GCNSubtarget *ST, MachineInstr *MI, unsigned OpNo) {
839 if (!ST->hasPKF32InstsReplicatingLower32BitsOfScalarInput())
840 return false;
841 const MCOperandInfo &OpDesc = MI->getDesc().operands()[OpNo];
842 return OpDesc.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32;
843}
844
845// Packed FP32 instructions only read 32 bits from a scalar operand (SGPR or
846// literal) and replicates the bits to both channels. Therefore, if the hi and
847// lo are not same, we can't fold it.
848static bool checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(
849 const FoldableDef &OpToFold) {
850 assert(OpToFold.isImm() && "Expected immediate operand");
851 uint64_t ImmVal = OpToFold.getEffectiveImmVal().value();
852 uint32_t Lo = Lo_32(Value: ImmVal);
853 uint32_t Hi = Hi_32(Value: ImmVal);
854 return Lo == Hi;
855}
856
857bool SIFoldOperandsImpl::tryAddToFoldList(
858 SmallVectorImpl<FoldCandidate> &FoldList, MachineInstr *MI, unsigned OpNo,
859 const FoldableDef &OpToFold) const {
860 const unsigned Opc = MI->getOpcode();
861
862 auto tryToFoldAsFMAAKorMK = [&]() {
863 if (!OpToFold.isImm())
864 return false;
865
866 const bool TryAK = OpNo == 3;
867 const unsigned NewOpc = TryAK ? AMDGPU::S_FMAAK_F32 : AMDGPU::S_FMAMK_F32;
868 MI->setDesc(TII->get(Opcode: NewOpc));
869
870 // We have to fold into operand which would be Imm not into OpNo.
871 bool FoldAsFMAAKorMK =
872 tryAddToFoldList(FoldList, MI, OpNo: TryAK ? 3 : 2, OpToFold);
873 if (FoldAsFMAAKorMK) {
874 // Untie Src2 of fmac.
875 MI->untieRegOperand(OpIdx: 3);
876 // For fmamk swap operands 1 and 2 if OpToFold was meant for operand 1.
877 if (OpNo == 1) {
878 MachineOperand &Op1 = MI->getOperand(i: 1);
879 MachineOperand &Op2 = MI->getOperand(i: 2);
880 Register OldReg = Op1.getReg();
881 // Operand 2 might be an inlinable constant
882 if (Op2.isImm()) {
883 Op1.ChangeToImmediate(ImmVal: Op2.getImm());
884 Op2.ChangeToRegister(Reg: OldReg, isDef: false);
885 } else {
886 Op1.setReg(Op2.getReg());
887 Op2.setReg(OldReg);
888 }
889 }
890 return true;
891 }
892 MI->setDesc(TII->get(Opcode: Opc));
893 return false;
894 };
895
896 bool IsLegal = OpToFold.isOperandLegal(TII: *TII, MI: *MI, OpIdx: OpNo);
897 if (!IsLegal && OpToFold.isImm()) {
898 if (std::optional<int64_t> ImmVal = OpToFold.getEffectiveImmVal())
899 IsLegal = canUseImmWithOpSel(MI, UseOpNo: OpNo, ImmVal: *ImmVal);
900 }
901
902 if (!IsLegal) {
903 // Special case for v_mac_{f16, f32}_e64 if we are trying to fold into src2
904 unsigned NewOpc = macToMad(Opc);
905 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) {
906 // Check if changing this to a v_mad_{f16, f32} instruction will allow us
907 // to fold the operand.
908 MI->setDesc(TII->get(Opcode: NewOpc));
909 bool AddOpSel = !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::op_sel) &&
910 AMDGPU::hasNamedOperand(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::op_sel);
911 if (AddOpSel)
912 MI->addOperand(Op: MachineOperand::CreateImm(Val: 0));
913 bool FoldAsMAD = tryAddToFoldList(FoldList, MI, OpNo, OpToFold);
914 if (FoldAsMAD) {
915 MI->untieRegOperand(OpIdx: OpNo);
916 return true;
917 }
918 if (AddOpSel)
919 MI->removeOperand(OpNo: MI->getNumExplicitOperands() - 1);
920 MI->setDesc(TII->get(Opcode: Opc));
921 }
922
923 // Special case for s_fmac_f32 if we are trying to fold into Src2.
924 // By transforming into fmaak we can untie Src2 and make folding legal.
925 if (Opc == AMDGPU::S_FMAC_F32 && OpNo == 3) {
926 if (tryToFoldAsFMAAKorMK())
927 return true;
928 }
929
930 // Inlineable constant might have been folded into Imm operand of fmaak or
931 // fmamk and we are trying to fold a non-inlinable constant.
932 if ((Opc == AMDGPU::S_FMAAK_F32 || Opc == AMDGPU::S_FMAMK_F32) &&
933 OpToFold.isImm()) {
934 std::optional<int64_t> ImmVal = OpToFold.getEffectiveImmVal();
935 if (ImmVal && !TII->isInlineConstant(MI: *MI, OpIdx: OpNo, ImmVal: *ImmVal)) {
936 unsigned ImmIdx = Opc == AMDGPU::S_FMAAK_F32 ? 3 : 2;
937 MachineOperand &OpImm = MI->getOperand(i: ImmIdx);
938 if (!OpImm.isReg() &&
939 TII->isInlineConstant(MI: *MI, UseMO: MI->getOperand(i: OpNo), DefMO: OpImm))
940 return tryToFoldAsFMAAKorMK();
941 }
942 }
943
944 // Special case for s_setreg_b32
945 if (OpToFold.isImm()) {
946 unsigned ImmOpc = 0;
947 if (Opc == AMDGPU::S_SETREG_B32)
948 ImmOpc = AMDGPU::S_SETREG_IMM32_B32;
949 else if (Opc == AMDGPU::S_SETREG_B32_mode)
950 ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode;
951 if (ImmOpc) {
952 MI->setDesc(TII->get(Opcode: ImmOpc));
953 appendFoldCandidate(FoldList, MI, OpNo, FoldOp: OpToFold);
954 return true;
955 }
956 }
957
958 // Operand is not legal, so try to commute the instruction to
959 // see if this makes it possible to fold.
960 unsigned CommuteOpNo = TargetInstrInfo::CommuteAnyOperandIndex;
961 bool CanCommute = TII->findCommutedOpIndices(MI: *MI, SrcOpIdx0&: OpNo, SrcOpIdx1&: CommuteOpNo);
962 if (!CanCommute)
963 return false;
964
965 MachineOperand &Op = MI->getOperand(i: OpNo);
966 MachineOperand &CommutedOp = MI->getOperand(i: CommuteOpNo);
967
968 // One of operands might be an Imm operand, and OpNo may refer to it after
969 // the call of commuteInstruction() below. Such situations are avoided
970 // here explicitly as OpNo must be a register operand to be a candidate
971 // for memory folding.
972 if (!Op.isReg() || !CommutedOp.isReg())
973 return false;
974
975 // The same situation with an immediate could reproduce if both inputs are
976 // the same register.
977 if (Op.isReg() && CommutedOp.isReg() &&
978 (Op.getReg() == CommutedOp.getReg() &&
979 Op.getSubReg() == CommutedOp.getSubReg()))
980 return false;
981
982 if (!TII->commuteInstruction(MI&: *MI, NewMI: false, OpIdx1: OpNo, OpIdx2: CommuteOpNo))
983 return false;
984
985 int Op32 = -1;
986 if (!OpToFold.isOperandLegal(TII: *TII, MI: *MI, OpIdx: CommuteOpNo)) {
987 if ((Opc != AMDGPU::V_ADD_CO_U32_e64 && Opc != AMDGPU::V_SUB_CO_U32_e64 &&
988 Opc != AMDGPU::V_SUBREV_CO_U32_e64) || // FIXME
989 (!OpToFold.isImm() && !OpToFold.isFI() && !OpToFold.isGlobal())) {
990 TII->commuteInstruction(MI&: *MI, NewMI: false, OpIdx1: OpNo, OpIdx2: CommuteOpNo);
991 return false;
992 }
993
994 // Verify the other operand is a VGPR, otherwise we would violate the
995 // constant bus restriction.
996 MachineOperand &OtherOp = MI->getOperand(i: OpNo);
997 if (!OtherOp.isReg() ||
998 !TII->getRegisterInfo().isVGPR(MRI: *MRI, Reg: OtherOp.getReg()))
999 return false;
1000
1001 assert(MI->getOperand(1).isDef());
1002
1003 // Make sure to get the 32-bit version of the commuted opcode.
1004 unsigned MaybeCommutedOpc = MI->getOpcode();
1005 Op32 = AMDGPU::getVOPe32(Opcode: MaybeCommutedOpc);
1006 }
1007
1008 appendFoldCandidate(FoldList, MI, OpNo: CommuteOpNo, FoldOp: OpToFold, /*Commuted=*/true,
1009 ShrinkOp: Op32);
1010 return true;
1011 }
1012
1013 // Special case for s_fmac_f32 if we are trying to fold into Src0 or Src1.
1014 // By changing into fmamk we can untie Src2.
1015 // If folding for Src0 happens first and it is identical operand to Src1 we
1016 // should avoid transforming into fmamk which requires commuting as it would
1017 // cause folding into Src1 to fail later on due to wrong OpNo used.
1018 if (Opc == AMDGPU::S_FMAC_F32 &&
1019 (OpNo != 1 || !MI->getOperand(i: 1).isIdenticalTo(Other: MI->getOperand(i: 2)))) {
1020 if (tryToFoldAsFMAAKorMK())
1021 return true;
1022 }
1023
1024 // Special case for PK_F32 instructions if we are trying to fold an imm to
1025 // src0 or src1.
1026 if (OpToFold.isImm() &&
1027 isPKF32InstrReplicatesLower32BitsOfScalarOperand(ST, MI, OpNo) &&
1028 !checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(OpToFold))
1029 return false;
1030
1031 appendFoldCandidate(FoldList, MI, OpNo, FoldOp: OpToFold);
1032 return true;
1033}
1034
1035bool SIFoldOperandsImpl::isUseSafeToFold(const MachineInstr &MI,
1036 const MachineOperand &UseMO) const {
1037 // Operands of SDWA instructions must be registers.
1038 return !TII->isSDWA(MI);
1039}
1040
1041// Returns true if any instruction in \p L modifies EXEC.
1042static bool loopModifiesExec(const MachineLoop &L, const SIRegisterInfo &TRI) {
1043 for (const MachineBasicBlock *MBB : L.getBlocks())
1044 for (const MachineInstr &MI : *MBB)
1045 if (MI.modifiesRegister(Reg: TRI.getExec(), TRI: &TRI))
1046 return true;
1047 return false;
1048}
1049
1050// An SGPR->VGPR copy inside a divergent loop latches each lane value as it
1051// exits. Folding its scalar source into a use after the loop would make every
1052// lane read the same reconverged value, so do not fold across the loop exit.
1053bool SIFoldOperandsImpl::isTemporallyDivergentUse(
1054 const FoldableDef &OpToFold, const MachineInstr &UseMI) const {
1055 if (!OpToFold.isReg())
1056 return false;
1057 const MachineInstr *DefMI = OpToFold.DefMI;
1058 if (!DefMI || !DefMI->isCopy() ||
1059 TRI->isSGPRReg(MRI: *MRI, Reg: DefMI->getOperand(i: 0).getReg()) ||
1060 !TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg()))
1061 return false;
1062 const MachineLoop *DefLoop = MLI->getLoopFor(BB: DefMI->getParent());
1063 return DefLoop && !DefLoop->contains(BB: UseMI.getParent()) &&
1064 loopModifiesExec(L: *DefLoop, TRI: *TRI);
1065}
1066
1067static MachineOperand *lookUpCopyChain(const SIInstrInfo &TII,
1068 const MachineRegisterInfo &MRI,
1069 Register SrcReg) {
1070 MachineOperand *Sub = nullptr;
1071 for (MachineInstr *SubDef = MRI.getVRegDef(Reg: SrcReg);
1072 SubDef && TII.isFoldableCopy(MI: *SubDef);
1073 SubDef = MRI.getVRegDef(Reg: Sub->getReg())) {
1074 unsigned SrcIdx = TII.getFoldableCopySrcIdx(MI: *SubDef);
1075 MachineOperand &SrcOp = SubDef->getOperand(i: SrcIdx);
1076
1077 if (SrcOp.isImm())
1078 return &SrcOp;
1079 if (!SrcOp.isReg() || SrcOp.getReg().isPhysical())
1080 break;
1081 Sub = &SrcOp;
1082 // TODO: Support compose
1083 if (SrcOp.getSubReg())
1084 break;
1085 }
1086
1087 return Sub;
1088}
1089
1090const TargetRegisterClass *SIFoldOperandsImpl::getRegSeqInit(
1091 MachineInstr &RegSeq,
1092 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs) const {
1093
1094 assert(RegSeq.isRegSequence());
1095
1096 const TargetRegisterClass *RC = nullptr;
1097
1098 for (unsigned I = 1, E = RegSeq.getNumExplicitOperands(); I != E; I += 2) {
1099 MachineOperand &SrcOp = RegSeq.getOperand(i: I);
1100 if (SrcOp.getReg().isPhysical())
1101 return nullptr;
1102 unsigned SubRegIdx = RegSeq.getOperand(i: I + 1).getImm();
1103
1104 // Only accept reg_sequence with uniform reg class inputs for simplicity.
1105 const TargetRegisterClass *OpRC = getRegOpRC(MRI: *MRI, TRI: *TRI, MO: SrcOp);
1106 if (!RC)
1107 RC = OpRC;
1108 else if (!TRI->getCommonSubClass(A: RC, B: OpRC))
1109 return nullptr;
1110
1111 if (SrcOp.getSubReg()) {
1112 // TODO: Handle subregister compose
1113 Defs.emplace_back(Args: &SrcOp, Args&: SubRegIdx);
1114 continue;
1115 }
1116
1117 MachineOperand *DefSrc = lookUpCopyChain(TII: *TII, MRI: *MRI, SrcReg: SrcOp.getReg());
1118 if (DefSrc && (DefSrc->isReg() || DefSrc->isImm())) {
1119 Defs.emplace_back(Args&: DefSrc, Args&: SubRegIdx);
1120 continue;
1121 }
1122
1123 Defs.emplace_back(Args: &SrcOp, Args&: SubRegIdx);
1124 }
1125
1126 return RC;
1127}
1128
1129// Find a def of the UseReg, check if it is a reg_sequence and find initializers
1130// for each subreg, tracking it to an immediate if possible. Returns the
1131// register class of the inputs on success.
1132const TargetRegisterClass *SIFoldOperandsImpl::getRegSeqInit(
1133 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
1134 Register UseReg) const {
1135 MachineInstr *Def = MRI->getVRegDef(Reg: UseReg);
1136 if (!Def || !Def->isRegSequence())
1137 return nullptr;
1138
1139 return getRegSeqInit(RegSeq&: *Def, Defs);
1140}
1141
1142std::pair<int64_t, const TargetRegisterClass *>
1143SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
1144 SmallVector<std::pair<MachineOperand *, unsigned>, 32> Defs;
1145 const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
1146 if (!SrcRC)
1147 return {};
1148
1149 bool TryToMatchSplat64 = false;
1150
1151 std::optional<int64_t> Imm;
1152 for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
1153 const MachineOperand *Op = Defs[I].first;
1154 if (!Op->isImm()) {
1155 if (Op->isReg()) {
1156 MachineInstr *Def = MRI->getVRegDef(Reg: Op->getReg());
1157 if (!Def || Def->isImplicitDef())
1158 continue;
1159 }
1160 return {};
1161 }
1162
1163 int64_t SubImm = Op->getImm();
1164 if (!Imm) {
1165 Imm = SubImm;
1166 continue;
1167 }
1168
1169 if (Imm != SubImm) {
1170 if (I == 1 && (E & 1) == 0) {
1171 // If we have an even number of inputs, there's a chance this is a
1172 // 64-bit element splat broken into 32-bit pieces.
1173 TryToMatchSplat64 = true;
1174 break;
1175 }
1176
1177 return {}; // Can only fold splat constants
1178 }
1179 }
1180
1181 if (!TryToMatchSplat64) {
1182 if (Imm)
1183 return {*Imm, SrcRC};
1184 return {};
1185 }
1186
1187 // Fallback to recognizing 64-bit splats broken into 32-bit pieces
1188 // (i.e. recognize every other other element is 0 for 64-bit immediates)
1189 int64_t SplatVal64;
1190 for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
1191 const MachineOperand *Op0 = Defs[I].first;
1192 const MachineOperand *Op1 = Defs[I + 1].first;
1193
1194 if (!Op0->isImm() || !Op1->isImm())
1195 return {};
1196
1197 unsigned SubReg0 = Defs[I].second;
1198 unsigned SubReg1 = Defs[I + 1].second;
1199
1200 // Assume we're going to generally encounter reg_sequences with sorted
1201 // subreg indexes, so reject any that aren't consecutive.
1202 if (TRI->getChannelFromSubReg(SubReg: SubReg0) + 1 !=
1203 TRI->getChannelFromSubReg(SubReg: SubReg1))
1204 return {};
1205
1206 if (TRI->getSubRegIdxSize(Idx: SubReg0) != 32)
1207 return {};
1208
1209 int64_t MergedVal = Make_64(High: Op1->getImm(), Low: Op0->getImm());
1210 if (I == 0)
1211 SplatVal64 = MergedVal;
1212 else if (SplatVal64 != MergedVal)
1213 return {};
1214 }
1215
1216 const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
1217 MRI->getRegClass(Reg: RegSeq.getOperand(i: 0).getReg()), AMDGPU::sub0_sub1);
1218
1219 return {SplatVal64, RC64};
1220}
1221
1222bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
1223 MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
1224 const TargetRegisterClass *SplatRC) const {
1225 const MCInstrDesc &Desc = UseMI->getDesc();
1226 if (UseOpIdx >= Desc.getNumOperands())
1227 return false;
1228
1229 // Filter out unhandled pseudos.
1230 if (!AMDGPU::isSISrcOperand(Desc, OpNo: UseOpIdx))
1231 return false;
1232
1233 int16_t RCID = TII->getOpRegClassID(OpInfo: Desc.operands()[UseOpIdx]);
1234 if (RCID == -1)
1235 return false;
1236
1237 const TargetRegisterClass *OpRC = TRI->getRegClass(i: RCID);
1238
1239 // Special case 0/-1, since when interpreted as a 64-bit element both halves
1240 // have the same bits. These are the only cases where a splat has the same
1241 // interpretation for 32-bit and 64-bit splats.
1242 if (SplatVal != 0 && SplatVal != -1) {
1243 // We need to figure out the scalar type read by the operand. e.g. the MFMA
1244 // operand will be AReg_128, and we want to check if it's compatible with an
1245 // AReg_32 constant.
1246 uint8_t OpTy = Desc.operands()[UseOpIdx].OperandType;
1247 switch (OpTy) {
1248 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1249 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1250 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1251 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1252 case AMDGPU::OPERAND_REG_IMM_V2FP32:
1253 OpRC = TRI->getSubRegisterClass(OpRC, AMDGPU::sub0);
1254 break;
1255 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
1256 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1257 case AMDGPU::OPERAND_REG_IMM_V2FP64:
1258 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1259 case AMDGPU::OPERAND_REG_IMM_V2INT64:
1260 OpRC = TRI->getSubRegisterClass(OpRC, AMDGPU::sub0_sub1);
1261 break;
1262 default:
1263 return false;
1264 }
1265
1266 if (!TRI->getCommonSubClass(A: OpRC, B: SplatRC))
1267 return false;
1268 }
1269
1270 MachineOperand TmpOp = MachineOperand::CreateImm(Val: SplatVal);
1271 if (!TII->isOperandLegal(MI: *UseMI, OpIdx: UseOpIdx, MO: &TmpOp))
1272 return false;
1273
1274 return true;
1275}
1276
1277bool SIFoldOperandsImpl::tryToFoldACImm(
1278 const FoldableDef &OpToFold, MachineInstr *UseMI, unsigned UseOpIdx,
1279 SmallVectorImpl<FoldCandidate> &FoldList) const {
1280 const MCInstrDesc &Desc = UseMI->getDesc();
1281 if (UseOpIdx >= Desc.getNumOperands())
1282 return false;
1283
1284 // Filter out unhandled pseudos.
1285 if (!AMDGPU::isSISrcOperand(Desc, OpNo: UseOpIdx))
1286 return false;
1287
1288 if (OpToFold.isImm() && OpToFold.isOperandLegal(TII: *TII, MI: *UseMI, OpIdx: UseOpIdx)) {
1289 if (isPKF32InstrReplicatesLower32BitsOfScalarOperand(ST, MI: UseMI, OpNo: UseOpIdx) &&
1290 !checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(OpToFold))
1291 return false;
1292 appendFoldCandidate(FoldList, MI: UseMI, OpNo: UseOpIdx, FoldOp: OpToFold);
1293 return true;
1294 }
1295
1296 return false;
1297}
1298
1299bool SIFoldOperandsImpl::foldOperand(
1300 FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
1301 SmallVectorImpl<FoldCandidate> &FoldList,
1302 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const {
1303 bool Changed = false;
1304 const MachineOperand *UseOp = &UseMI->getOperand(i: UseOpIdx);
1305
1306 if (!isUseSafeToFold(MI: *UseMI, UseMO: *UseOp))
1307 return Changed;
1308
1309 if (isTemporallyDivergentUse(OpToFold, UseMI: *UseMI))
1310 return Changed;
1311
1312 // FIXME: Fold operands with subregs.
1313 if (UseOp->isReg() && OpToFold.isReg()) {
1314 if (UseOp->isImplicit())
1315 return Changed;
1316 // Allow folding from SGPRs to 16-bit VGPRs.
1317 if (UseOp->getSubReg() != AMDGPU::NoSubRegister &&
1318 (UseOp->getSubReg() != AMDGPU::lo16 ||
1319 !TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg())))
1320 return Changed;
1321 }
1322
1323 // Special case for REG_SEQUENCE: We can't fold literals into
1324 // REG_SEQUENCE instructions, so we have to fold them into the
1325 // uses of REG_SEQUENCE.
1326 if (UseMI->isRegSequence()) {
1327 Register RegSeqDstReg = UseMI->getOperand(i: 0).getReg();
1328 unsigned RegSeqDstSubReg = UseMI->getOperand(i: UseOpIdx + 1).getImm();
1329
1330 int64_t SplatVal;
1331 const TargetRegisterClass *SplatRC;
1332 std::tie(args&: SplatVal, args&: SplatRC) = isRegSeqSplat(RegSeq&: *UseMI);
1333
1334 // Grab the use operands first
1335 SmallVector<MachineOperand *, 4> UsesToProcess(
1336 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: RegSeqDstReg)));
1337 for (unsigned I = 0; I != UsesToProcess.size(); ++I) {
1338 MachineOperand *RSUse = UsesToProcess[I];
1339 MachineInstr *RSUseMI = RSUse->getParent();
1340 unsigned OpNo = RSUseMI->getOperandNo(I: RSUse);
1341
1342 if (SplatRC) {
1343 if (RSUseMI->isCopy()) {
1344 Register DstReg = RSUseMI->getOperand(i: 0).getReg();
1345 append_range(C&: UsesToProcess,
1346 R: make_pointer_range(Range: MRI->use_nodbg_operands(Reg: DstReg)));
1347 continue;
1348 }
1349 if (tryFoldRegSeqSplat(UseMI: RSUseMI, UseOpIdx: OpNo, SplatVal, SplatRC)) {
1350 FoldableDef SplatDef(SplatVal, SplatRC);
1351 appendFoldCandidate(FoldList, MI: RSUseMI, OpNo, FoldOp: SplatDef);
1352 Changed = true;
1353 continue;
1354 }
1355 }
1356
1357 // TODO: Handle general compose
1358 if (RSUse->getSubReg() != RegSeqDstSubReg)
1359 continue;
1360
1361 // FIXME: We should avoid recursing here. There should be a cleaner split
1362 // between the in-place mutations and adding to the fold list.
1363 Changed |= foldOperand(OpToFold, UseMI: RSUseMI, UseOpIdx: RSUseMI->getOperandNo(I: RSUse),
1364 FoldList, CopiesToReplace);
1365 }
1366
1367 return Changed;
1368 }
1369
1370 if (tryToFoldACImm(OpToFold, UseMI, UseOpIdx, FoldList))
1371 return true;
1372
1373 if (frameIndexMayFold(UseMI: *UseMI, OpNo: UseOpIdx, OpToFold)) {
1374 // Verify that this is a stack access.
1375 // FIXME: Should probably use stack pseudos before frame lowering.
1376
1377 if (TII->isMUBUF(MI: *UseMI)) {
1378 if (TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::srsrc)->getReg() !=
1379 MFI->getScratchRSrcReg())
1380 return Changed;
1381
1382 // Ensure this is either relative to the current frame or the current
1383 // wave.
1384 MachineOperand &SOff =
1385 *TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::soffset);
1386 if (!SOff.isImm() || SOff.getImm() != 0)
1387 return Changed;
1388 }
1389
1390 const unsigned Opc = UseMI->getOpcode();
1391 if (TII->isFLATScratch(MI: *UseMI) &&
1392 AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr) &&
1393 !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::saddr)) {
1394 unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(Opcode: Opc);
1395 unsigned CPol =
1396 TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::cpol)->getImm();
1397 if ((CPol & AMDGPU::CPol::SCAL) &&
1398 !AMDGPU::supportsScaleOffset(MII: *TII, Opcode: NewOpc))
1399 return Changed;
1400
1401 UseMI->setDesc(TII->get(Opcode: NewOpc));
1402 }
1403
1404 // A frame index will resolve to a positive constant, so it should always be
1405 // safe to fold the addressing mode, even pre-GFX9.
1406 UseMI->getOperand(i: UseOpIdx).ChangeToFrameIndex(Idx: OpToFold.getFI());
1407
1408 return true;
1409 }
1410
1411 bool FoldingImmLike =
1412 OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
1413
1414 if (FoldingImmLike && UseMI->isCopy()) {
1415 Register DestReg = UseMI->getOperand(i: 0).getReg();
1416 Register SrcReg = UseMI->getOperand(i: 1).getReg();
1417 unsigned UseSubReg = UseMI->getOperand(i: 1).getSubReg();
1418 assert(SrcReg.isVirtual());
1419
1420 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: SrcReg);
1421
1422 // Don't fold into a copy to a physical register with the same class. Doing
1423 // so would interfere with the register coalescer's logic which would avoid
1424 // redundant initializations.
1425 if (DestReg.isPhysical() && SrcRC->contains(Reg: DestReg))
1426 return Changed;
1427
1428 const TargetRegisterClass *DestRC = TRI->getRegClassForReg(MRI: *MRI, Reg: DestReg);
1429 // In order to fold immediates into copies, we need to change the copy to a
1430 // MOV. Find a compatible mov instruction with the value.
1431 for (unsigned MovOp :
1432 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
1433 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_MOV_B16_t16_e64,
1434 AMDGPU::V_ACCVGPR_WRITE_B32_e64, AMDGPU::AV_MOV_B32_IMM_PSEUDO,
1435 AMDGPU::AV_MOV_B64_IMM_PSEUDO}) {
1436 const MCInstrDesc &MovDesc = TII->get(Opcode: MovOp);
1437 const TargetRegisterClass *MovDstRC =
1438 TRI->getRegClass(i: TII->getOpRegClassID(OpInfo: MovDesc.operands()[0]));
1439
1440 // Fold if the destination register class of the MOV instruction (ResRC)
1441 // is a superclass of (or equal to) the destination register class of the
1442 // COPY (DestRC). If this condition fails, folding would be illegal.
1443 if (!DestRC->hasSuperClassEq(RC: MovDstRC))
1444 continue;
1445
1446 const int SrcIdx = MovOp == AMDGPU::V_MOV_B16_t16_e64 ? 2 : 1;
1447
1448 int16_t RegClassID = TII->getOpRegClassID(OpInfo: MovDesc.operands()[SrcIdx]);
1449 if (RegClassID != -1) {
1450 const TargetRegisterClass *MovSrcRC = TRI->getRegClass(i: RegClassID);
1451
1452 if (UseSubReg)
1453 MovSrcRC = TRI->getMatchingSuperRegClass(A: SrcRC, B: MovSrcRC, Idx: UseSubReg);
1454
1455 // FIXME: We should be able to directly check immediate operand legality
1456 // for all cases, but gfx908 hacks break.
1457 if (MovOp == AMDGPU::AV_MOV_B32_IMM_PSEUDO &&
1458 (!OpToFold.isImm() ||
1459 !TII->isImmOperandLegal(InstDesc: MovDesc, OpNo: SrcIdx,
1460 ImmVal: *OpToFold.getEffectiveImmVal())))
1461 break;
1462
1463 if (!MRI->constrainRegClass(Reg: SrcReg, RC: MovSrcRC))
1464 break;
1465
1466 // FIXME: This is mutating the instruction only and deferring the actual
1467 // fold of the immediate
1468 } else {
1469 // For the _IMM_PSEUDO cases, there can be value restrictions on the
1470 // immediate to verify. Technically we should always verify this, but it
1471 // only matters for these concrete cases.
1472 // TODO: Handle non-imm case if it's useful.
1473 if (!OpToFold.isImm() ||
1474 !TII->isImmOperandLegal(InstDesc: MovDesc, OpNo: 1, ImmVal: *OpToFold.getEffectiveImmVal()))
1475 break;
1476 }
1477
1478 MachineInstr::mop_iterator ImpOpI = UseMI->implicit_operands().begin();
1479 MachineInstr::mop_iterator ImpOpE = UseMI->implicit_operands().end();
1480 while (ImpOpI != ImpOpE) {
1481 MachineInstr::mop_iterator Tmp = ImpOpI;
1482 ImpOpI++;
1483 UseMI->removeOperand(OpNo: UseMI->getOperandNo(I: Tmp));
1484 }
1485 UseMI->setDesc(MovDesc);
1486
1487 if (MovOp == AMDGPU::V_MOV_B16_t16_e64) {
1488 const auto &SrcOp = UseMI->getOperand(i: UseOpIdx);
1489 MachineOperand NewSrcOp(SrcOp);
1490 UseMI->removeOperand(OpNo: 1);
1491 UseMI->addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0)); // src0_modifiers
1492 UseMI->addOperand(Op: NewSrcOp); // src0
1493 UseMI->addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0)); // op_sel
1494 UseOpIdx = SrcIdx;
1495 UseOp = &UseMI->getOperand(i: UseOpIdx);
1496 }
1497 CopiesToReplace.push_back(Elt: UseMI);
1498 Changed = true;
1499 break;
1500 }
1501
1502 // We failed to replace the copy, so give up.
1503 if (UseMI->getOpcode() == AMDGPU::COPY)
1504 return Changed;
1505
1506 } else {
1507 if (UseMI->isCopy() && OpToFold.isReg() &&
1508 UseMI->getOperand(i: 0).getReg().isVirtual() &&
1509 !UseMI->getOperand(i: 1).getSubReg() &&
1510 OpToFold.DefMI->implicit_operands().empty()) {
1511 LLVM_DEBUG(dbgs() << "Folding " << *OpToFold.OpToFold << "\n into "
1512 << *UseMI);
1513 unsigned Size = TII->getOpSize(MI: *UseMI, OpNo: 1);
1514 Register UseReg = OpToFold.getReg();
1515 UseMI->getOperand(i: 1).setReg(UseReg);
1516 unsigned SubRegIdx = OpToFold.getSubReg();
1517 // Hack to allow 32-bit SGPRs to be folded into True16 instructions
1518 // Remove this if 16-bit SGPRs (i.e. SGPR_LO16) are added to the
1519 // VS_16RegClass
1520 if (Size == 2 && TRI->isVGPR(MRI: *MRI, Reg: UseMI->getOperand(i: 0).getReg()) &&
1521 TRI->isSGPRReg(MRI: *MRI, Reg: UseReg) && SubRegIdx != AMDGPU::NoSubRegister) {
1522 // SGPRs only have lo16 subregisters, so the value is in the low half
1523 // of a 32-bit SGPR. Use that whole 32-bit SGPR instead.
1524 unsigned Channel = TRI->getChannelFromSubReg(SubReg: SubRegIdx);
1525 const TargetRegisterClass *UseRC = TRI->getRegClassForReg(MRI: *MRI, Reg: UseReg);
1526 SubRegIdx = TRI->getRegSizeInBits(RC: *UseRC) == 32
1527 ? AMDGPU::NoSubRegister
1528 : SIRegisterInfo::getSubRegFromChannel(Channel);
1529 }
1530 UseMI->getOperand(i: 1).setSubReg(SubRegIdx);
1531 UseMI->getOperand(i: 1).setIsKill(false);
1532 CopiesToReplace.push_back(Elt: UseMI);
1533 OpToFold.OpToFold->setIsKill(false);
1534 Changed = true;
1535
1536 // Remove kill flags as kills may now be out of order with uses.
1537 MRI->clearKillFlags(Reg: UseReg);
1538 if (foldCopyToAGPRRegSequence(CopyMI: UseMI))
1539 return true;
1540 }
1541
1542 unsigned UseOpc = UseMI->getOpcode();
1543 if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 ||
1544 (UseOpc == AMDGPU::V_READLANE_B32 &&
1545 (int)UseOpIdx ==
1546 AMDGPU::getNamedOperandIdx(Opcode: UseOpc, Name: AMDGPU::OpName::src0))) {
1547 // %vgpr = V_MOV_B32 imm
1548 // %sgpr = V_READFIRSTLANE_B32 %vgpr
1549 // =>
1550 // %sgpr = S_MOV_B32 imm
1551 if (FoldingImmLike) {
1552 if (execMayBeModifiedBeforeUse(MRI: *MRI,
1553 VReg: UseMI->getOperand(i: UseOpIdx).getReg(),
1554 DefMI: *OpToFold.DefMI, UseMI: *UseMI))
1555 return Changed;
1556
1557 UseMI->setDesc(TII->get(Opcode: AMDGPU::S_MOV_B32));
1558 UseMI->clearFlag(Flag: MachineInstr::NoConvergent);
1559
1560 if (OpToFold.isImm()) {
1561 UseMI->getOperand(i: 1).ChangeToImmediate(
1562 ImmVal: *OpToFold.getEffectiveImmVal());
1563 } else if (OpToFold.isFI())
1564 UseMI->getOperand(i: 1).ChangeToFrameIndex(Idx: OpToFold.getFI());
1565 else {
1566 assert(OpToFold.isGlobal());
1567 UseMI->getOperand(i: 1).ChangeToGA(GV: OpToFold.OpToFold->getGlobal(),
1568 Offset: OpToFold.OpToFold->getOffset(),
1569 TargetFlags: OpToFold.OpToFold->getTargetFlags());
1570 }
1571 UseMI->removeOperand(OpNo: 2); // Remove exec read (or src1 for readlane)
1572 return true;
1573 }
1574
1575 if (OpToFold.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg())) {
1576 if (execMayBeModifiedBeforeUse(MRI: *MRI,
1577 VReg: UseMI->getOperand(i: UseOpIdx).getReg(),
1578 DefMI: *OpToFold.DefMI, UseMI: *UseMI))
1579 return Changed;
1580
1581 // %vgpr = COPY %sgpr0
1582 // %sgpr1 = V_READFIRSTLANE_B32 %vgpr
1583 // =>
1584 // %sgpr1 = COPY %sgpr0
1585 UseMI->setDesc(TII->get(Opcode: AMDGPU::COPY));
1586 UseMI->getOperand(i: 1).setReg(OpToFold.getReg());
1587 UseMI->getOperand(i: 1).setSubReg(OpToFold.getSubReg());
1588 UseMI->getOperand(i: 1).setIsKill(false);
1589 UseMI->removeOperand(OpNo: 2); // Remove exec read (or src1 for readlane)
1590 UseMI->clearFlag(Flag: MachineInstr::NoConvergent);
1591 return true;
1592 }
1593 }
1594
1595 const MCInstrDesc &UseDesc = UseMI->getDesc();
1596
1597 // Don't fold into target independent nodes. Target independent opcodes
1598 // don't have defined register classes.
1599 if (UseDesc.isVariadic() || UseOp->isImplicit() ||
1600 UseDesc.operands()[UseOpIdx].RegClass == -1)
1601 return Changed;
1602 }
1603
1604 // FIXME: We could try to change the instruction from 64-bit to 32-bit
1605 // to enable more folding opportunities. The shrink operands pass
1606 // already does this.
1607
1608 Changed |= tryAddToFoldList(FoldList, MI: UseMI, OpNo: UseOpIdx, OpToFold);
1609 return Changed;
1610}
1611
1612static bool evalBinaryInstruction(unsigned Opcode, int32_t &Result,
1613 uint32_t LHS, uint32_t RHS) {
1614 switch (Opcode) {
1615 case AMDGPU::S_ADD_I32:
1616 case AMDGPU::S_ADD_U32:
1617 Result = LHS + RHS;
1618 return true;
1619 case AMDGPU::S_SUB_I32:
1620 case AMDGPU::S_SUB_U32:
1621 Result = LHS - RHS;
1622 return true;
1623 case AMDGPU::V_AND_B32_e64:
1624 case AMDGPU::V_AND_B32_e32:
1625 case AMDGPU::S_AND_B32:
1626 Result = LHS & RHS;
1627 return true;
1628 case AMDGPU::V_OR_B32_e64:
1629 case AMDGPU::V_OR_B32_e32:
1630 case AMDGPU::S_OR_B32:
1631 Result = LHS | RHS;
1632 return true;
1633 case AMDGPU::V_XOR_B32_e64:
1634 case AMDGPU::V_XOR_B32_e32:
1635 case AMDGPU::S_XOR_B32:
1636 Result = LHS ^ RHS;
1637 return true;
1638 case AMDGPU::S_XNOR_B32:
1639 Result = ~(LHS ^ RHS);
1640 return true;
1641 case AMDGPU::S_NAND_B32:
1642 Result = ~(LHS & RHS);
1643 return true;
1644 case AMDGPU::S_NOR_B32:
1645 Result = ~(LHS | RHS);
1646 return true;
1647 case AMDGPU::S_ANDN2_B32:
1648 Result = LHS & ~RHS;
1649 return true;
1650 case AMDGPU::S_ORN2_B32:
1651 Result = LHS | ~RHS;
1652 return true;
1653 case AMDGPU::V_LSHL_B32_e64:
1654 case AMDGPU::V_LSHL_B32_e32:
1655 case AMDGPU::S_LSHL_B32:
1656 // The instruction ignores the high bits for out of bounds shifts.
1657 Result = LHS << (RHS & 31);
1658 return true;
1659 case AMDGPU::V_LSHLREV_B32_e64:
1660 case AMDGPU::V_LSHLREV_B32_e32:
1661 Result = RHS << (LHS & 31);
1662 return true;
1663 case AMDGPU::V_LSHR_B32_e64:
1664 case AMDGPU::V_LSHR_B32_e32:
1665 case AMDGPU::S_LSHR_B32:
1666 Result = LHS >> (RHS & 31);
1667 return true;
1668 case AMDGPU::V_LSHRREV_B32_e64:
1669 case AMDGPU::V_LSHRREV_B32_e32:
1670 Result = RHS >> (LHS & 31);
1671 return true;
1672 case AMDGPU::V_ASHR_I32_e64:
1673 case AMDGPU::V_ASHR_I32_e32:
1674 case AMDGPU::S_ASHR_I32:
1675 Result = static_cast<int32_t>(LHS) >> (RHS & 31);
1676 return true;
1677 case AMDGPU::V_ASHRREV_I32_e64:
1678 case AMDGPU::V_ASHRREV_I32_e32:
1679 Result = static_cast<int32_t>(RHS) >> (LHS & 31);
1680 return true;
1681 default:
1682 return false;
1683 }
1684}
1685
1686static unsigned getMovOpc(bool IsScalar) {
1687 return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1688}
1689
1690// Try to simplify operations with a constant that may appear after instruction
1691// selection.
1692// TODO: See if a frame index with a fixed offset can fold.
1693bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
1694 if (!MI->allImplicitDefsAreDead())
1695 return false;
1696
1697 unsigned Opc = MI->getOpcode();
1698
1699 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
1700 if (Src0Idx == -1)
1701 return false;
1702
1703 MachineOperand *Src0 = &MI->getOperand(i: Src0Idx);
1704 std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src0);
1705
1706 if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 ||
1707 Opc == AMDGPU::S_NOT_B32) &&
1708 Src0Imm) {
1709 MI->getOperand(i: 1).ChangeToImmediate(ImmVal: ~*Src0Imm);
1710 TII->mutateAndCleanupImplicit(
1711 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_NOT_B32)));
1712 return true;
1713 }
1714
1715 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
1716 if (Src1Idx == -1)
1717 return false;
1718
1719 MachineOperand *Src1 = &MI->getOperand(i: Src1Idx);
1720 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
1721
1722 if (!Src0Imm && !Src1Imm)
1723 return false;
1724
1725 // and k0, k1 -> v_mov_b32 (k0 & k1)
1726 // or k0, k1 -> v_mov_b32 (k0 | k1)
1727 // xor k0, k1 -> v_mov_b32 (k0 ^ k1)
1728 if (Src0Imm && Src1Imm) {
1729 int32_t NewImm;
1730 if (!evalBinaryInstruction(Opcode: Opc, Result&: NewImm, LHS: *Src0Imm, RHS: *Src1Imm))
1731 return false;
1732
1733 bool IsSGPR = TRI->isSGPRReg(MRI: *MRI, Reg: MI->getOperand(i: 0).getReg());
1734
1735 // Be careful to change the right operand, src0 may belong to a different
1736 // instruction.
1737 MI->getOperand(i: Src0Idx).ChangeToImmediate(ImmVal: NewImm);
1738 MI->removeOperand(OpNo: Src1Idx);
1739 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: IsSGPR)));
1740 return true;
1741 }
1742
1743 // S_SUB_* is not commutable, so handle it before the commutability gate.
1744 // Only `x - 0 -> copy x` is valid; `0 - x` is a negation, not a copy.
1745 if (Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U32) {
1746 if (Src1Imm && static_cast<int32_t>(*Src1Imm) == 0) {
1747 // y = sub x, 0 => y = copy x
1748 MI->removeOperand(OpNo: Src1Idx);
1749 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1750 return true;
1751 }
1752 return false;
1753 }
1754
1755 if (!MI->isCommutable())
1756 return false;
1757
1758 if (Src0Imm && !Src1Imm) {
1759 std::swap(a&: Src0, b&: Src1);
1760 std::swap(a&: Src0Idx, b&: Src1Idx);
1761 std::swap(lhs&: Src0Imm, rhs&: Src1Imm);
1762 }
1763
1764 int32_t Src1Val = static_cast<int32_t>(*Src1Imm);
1765 if (Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_ADD_U32) {
1766 if (Src1Val == 0) {
1767 // y = add x, 0 => y = copy x
1768 MI->removeOperand(OpNo: Src1Idx);
1769 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1770 return true;
1771 }
1772 return false;
1773 }
1774
1775 if (Opc == AMDGPU::V_OR_B32_e64 ||
1776 Opc == AMDGPU::V_OR_B32_e32 ||
1777 Opc == AMDGPU::S_OR_B32) {
1778 if (Src1Val == 0) {
1779 // y = or x, 0 => y = copy x
1780 MI->removeOperand(OpNo: Src1Idx);
1781 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1782 } else if (Src1Val == -1) {
1783 // y = or x, -1 => y = v_mov_b32 -1
1784 MI->removeOperand(OpNo: Src0Idx);
1785 TII->mutateAndCleanupImplicit(
1786 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_OR_B32)));
1787 } else
1788 return false;
1789
1790 return true;
1791 }
1792
1793 if (Opc == AMDGPU::V_AND_B32_e64 || Opc == AMDGPU::V_AND_B32_e32 ||
1794 Opc == AMDGPU::S_AND_B32) {
1795 if (Src1Val == 0) {
1796 // y = and x, 0 => y = v_mov_b32 0
1797 MI->removeOperand(OpNo: Src0Idx);
1798 TII->mutateAndCleanupImplicit(
1799 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_AND_B32)));
1800 } else if (Src1Val == -1) {
1801 // y = and x, -1 => y = copy x
1802 MI->removeOperand(OpNo: Src1Idx);
1803 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1804 } else
1805 return false;
1806
1807 return true;
1808 }
1809
1810 if (Opc == AMDGPU::V_XOR_B32_e64 || Opc == AMDGPU::V_XOR_B32_e32 ||
1811 Opc == AMDGPU::S_XOR_B32) {
1812 if (Src1Val == 0) {
1813 // y = xor x, 0 => y = copy x
1814 MI->removeOperand(OpNo: Src1Idx);
1815 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1816 return true;
1817 }
1818 }
1819
1820 return false;
1821}
1822
1823// Try to fold an instruction into a simpler one
1824bool SIFoldOperandsImpl::tryFoldCndMask(MachineInstr &MI) const {
1825 unsigned Opc = MI.getOpcode();
1826 if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 &&
1827 Opc != AMDGPU::V_CNDMASK_B64_PSEUDO)
1828 return false;
1829
1830 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1831 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1832 if (!Src1->isIdenticalTo(Other: *Src0)) {
1833 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
1834 if (!Src1Imm)
1835 return false;
1836
1837 std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src0);
1838 if (!Src0Imm || *Src0Imm != *Src1Imm)
1839 return false;
1840 }
1841
1842 int Src1ModIdx =
1843 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1_modifiers);
1844 int Src0ModIdx =
1845 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers);
1846 if ((Src1ModIdx != -1 && MI.getOperand(i: Src1ModIdx).getImm() != 0) ||
1847 (Src0ModIdx != -1 && MI.getOperand(i: Src0ModIdx).getImm() != 0))
1848 return false;
1849
1850 LLVM_DEBUG(dbgs() << "Folded " << MI << " into ");
1851 auto &NewDesc =
1852 TII->get(Opcode: Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(IsScalar: false));
1853 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
1854 if (Src2Idx != -1)
1855 MI.removeOperand(OpNo: Src2Idx);
1856 MI.removeOperand(OpNo: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1));
1857 if (Src1ModIdx != -1)
1858 MI.removeOperand(OpNo: Src1ModIdx);
1859 if (Src0ModIdx != -1)
1860 MI.removeOperand(OpNo: Src0ModIdx);
1861 TII->mutateAndCleanupImplicit(MI, NewDesc);
1862 LLVM_DEBUG(dbgs() << MI);
1863 return true;
1864}
1865
1866// Extract mask, register, and register operand index from an AND instruction.
1867// Immediate can be in operand 1 or 2.
1868std::optional<SIFoldOperandsImpl::ANDMaskResult>
1869SIFoldOperandsImpl::getANDMaskRegOperand(MachineInstr &AndMI) const {
1870 unsigned Opc = AndMI.getOpcode();
1871 if (Opc != AMDGPU::V_AND_B32_e64 && Opc != AMDGPU::V_AND_B32_e32 &&
1872 Opc != AMDGPU::S_AND_B32)
1873 return std::nullopt;
1874
1875 std::optional<int64_t> MaskImm =
1876 TII->getImmOrMaterializedImm(MRI: *MRI, Op: AndMI.getOperand(i: 1));
1877 if (MaskImm && AndMI.getOperand(i: 2).isReg())
1878 return ANDMaskResult{.Mask: *MaskImm, .Reg: AndMI.getOperand(i: 2).getReg(), .RegIdx: 2};
1879
1880 MaskImm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: AndMI.getOperand(i: 2));
1881 if (MaskImm && AndMI.getOperand(i: 1).isReg())
1882 return ANDMaskResult{.Mask: *MaskImm, .Reg: AndMI.getOperand(i: 1).getReg(), .RegIdx: 1};
1883
1884 return std::nullopt;
1885}
1886
1887// Eliminate redundant 32-bit AND operations by detecting when ChildMI's mask
1888// contains ParentMI's mask.
1889//
1890// For example:
1891// ParentMI: %1 = AND %0, 0x7fff
1892// ChildMI: %2 = AND %1, 0xffff
1893//
1894// This also handles cases where ParentMI implicitly zeros high bits (e.g., f16
1895// operations that write 16-bit results into 32-bit registers), making a
1896// subsequent AND with 0xffff redundant.
1897bool SIFoldOperandsImpl::tryFoldRedundantAND(MachineInstr &ChildMI) const {
1898 // Ensure implicit defs (e.g., $scc) are not live.
1899 if (!ChildMI.allImplicitDefsAreDead())
1900 return false;
1901
1902 std::optional<ANDMaskResult> ChildResult = getANDMaskRegOperand(AndMI&: ChildMI);
1903 if (!ChildResult)
1904 return false;
1905
1906 if (!ChildResult->Reg.isVirtual())
1907 return false;
1908
1909 MachineInstr *ParentMI = MRI->getVRegDef(Reg: ChildResult->Reg);
1910 if (!ParentMI)
1911 return false;
1912
1913 int64_t ParentMask = 0;
1914 std::optional<ANDMaskResult> ParentResult = getANDMaskRegOperand(AndMI&: *ParentMI);
1915 if (ParentResult) {
1916 // Parent is an AND - extract its mask.
1917 ParentMask = ParentResult->Mask;
1918 } else if (ST->zeroesHigh16BitsOfDest(Opcode: ParentMI->getOpcode())) {
1919 // Parent instruction implicitly zeros high 16 bits.
1920 ParentMask = 0xffff;
1921 } else {
1922 return false;
1923 }
1924
1925 // Check if ChildMI is not redundant.
1926 if ((ParentMask & ChildResult->Mask) != ParentMask)
1927 return false;
1928
1929 Register Dst = ChildMI.getOperand(i: 0).getReg();
1930 Register Src = ChildResult->Reg;
1931
1932 // Src must be legal in every use of Dst. An S_AND_B32 parent with a
1933 // V_AND_B32 child defines Src in the scalar bank, and a use that requires a
1934 // VGPR does not accept it.
1935 if (!Dst.isVirtual() || !MRI->constrainRegClass(Reg: Src, RC: MRI->getRegClass(Reg: Dst)))
1936 return false;
1937
1938 MRI->replaceRegWith(FromReg: Dst, ToReg: Src);
1939
1940 // Clear kill flags if the register operand is not marked as kill.
1941 if (!ChildMI.getOperand(i: ChildResult->RegIdx).isKill())
1942 MRI->clearKillFlags(Reg: Src);
1943
1944 ChildMI.eraseFromParent();
1945 return true;
1946}
1947
1948/// Remove S_AND of a lane mask with EXEC, when the lane mask is already known
1949/// to have 0 in the bits of all inactive lanes.
1950///
1951/// Instruction selection inserts these unconditionally because it has not
1952/// analysed what produced the lane mask.
1953bool SIFoldOperandsImpl::tryFoldAndExec(MachineInstr &MI) const {
1954 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST: *ST);
1955 if (MI.getOpcode() != LMC.AndOpc)
1956 return false;
1957
1958 // The AND is going to be removed, so nothing may use the SCC it defines.
1959 if (!MI.allImplicitDefsAreDead())
1960 return false;
1961
1962 // Find the EXEC operand, and the lane mask it is being ANDed with.
1963 unsigned ExecIdx = 0;
1964 for (unsigned I : {1u, 2u}) {
1965 const MachineOperand &MO = MI.getOperand(i: I);
1966 if (MO.isReg() && MO.getReg() == LMC.ExecReg)
1967 ExecIdx = I;
1968 }
1969 if (!ExecIdx)
1970 return false;
1971 MachineOperand &Src = MI.getOperand(i: 3 - ExecIdx);
1972 if (!Src.isReg() || !Src.getReg().isVirtual() || Src.getSubReg())
1973 return false;
1974
1975 Register SrcReg = Src.getReg();
1976 if (!TII->isMaskedByExec(Reg: SrcReg, Use: MI, MRI: *MRI))
1977 return false;
1978
1979 LLVM_DEBUG(dbgs() << "Folding redundant AND with EXEC: " << MI);
1980
1981 Register DstReg = MI.getOperand(i: 0).getReg();
1982 if (DstReg.isVirtual()) {
1983 if (!MRI->constrainRegClass(Reg: SrcReg, RC: MRI->getRegClass(Reg: DstReg)))
1984 return false;
1985 MRI->replaceRegWith(FromReg: DstReg, ToReg: SrcReg);
1986 } else {
1987 // A physical destination, e.g. the $vcc written by moveToVALU. Register
1988 // allocation will usually make this copy an identity copy.
1989 MachineBasicBlock *MBB = MI.getParent();
1990 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: DstReg)
1991 .addReg(RegNo: SrcReg);
1992 }
1993
1994 if (!Src.isKill())
1995 MRI->clearKillFlags(Reg: SrcReg);
1996 MI.eraseFromParent();
1997 return true;
1998}
1999
2000bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
2001 const FoldableDef &OpToFold) const {
2002 // We need mutate the operands of new mov instructions to add implicit
2003 // uses of EXEC, but adding them invalidates the use_iterator, so defer
2004 // this.
2005 SmallVector<MachineInstr *, 4> CopiesToReplace;
2006 SmallVector<FoldCandidate, 4> FoldList;
2007 MachineOperand &Dst = MI.getOperand(i: 0);
2008 bool Changed = false;
2009
2010 SmallVector<MachineOperand *, 4> UsesToProcess(
2011 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: Dst.getReg())));
2012 for (auto *U : UsesToProcess) {
2013 MachineInstr *UseMI = U->getParent();
2014
2015 FoldableDef SubOpToFold = OpToFold.getWithSubReg(TRI: *TRI, SubReg: U->getSubReg());
2016 Changed |= foldOperand(OpToFold: SubOpToFold, UseMI, UseOpIdx: UseMI->getOperandNo(I: U), FoldList,
2017 CopiesToReplace);
2018 }
2019
2020 if (CopiesToReplace.empty() && FoldList.empty())
2021 return Changed;
2022
2023 // Make sure we add EXEC uses to any new v_mov instructions created.
2024 for (MachineInstr *Copy : CopiesToReplace)
2025 Copy->addImplicitDefUseOperands(MF&: *MF);
2026
2027 SetVector<MachineInstr *> ConstantFoldCandidates;
2028 for (FoldCandidate &Fold : FoldList) {
2029 assert(!Fold.isReg() || Fold.Def.OpToFold);
2030 if (Fold.isReg() && Fold.getReg().isVirtual()) {
2031 Register Reg = Fold.getReg();
2032 const MachineInstr *DefMI = Fold.Def.DefMI;
2033 if (DefMI->readsRegister(Reg: AMDGPU::EXEC, TRI) &&
2034 execMayBeModifiedBeforeUse(MRI: *MRI, VReg: Reg, DefMI: *DefMI, UseMI: *Fold.UseMI))
2035 continue;
2036 }
2037 if (updateOperand(Fold)) {
2038 // Clear kill flags.
2039 if (Fold.isReg()) {
2040 assert(Fold.Def.OpToFold && Fold.isReg());
2041 // FIXME: Probably shouldn't bother trying to fold if not an
2042 // SGPR. PeepholeOptimizer can eliminate redundant VGPR->VGPR
2043 // copies.
2044 MRI->clearKillFlags(Reg: Fold.getReg());
2045 }
2046 LLVM_DEBUG(dbgs() << "Folded source from " << MI << " into OpNo "
2047 << static_cast<int>(Fold.UseOpNo) << " of "
2048 << *Fold.UseMI);
2049
2050 if (Fold.isImm())
2051 ConstantFoldCandidates.insert(X: Fold.UseMI);
2052
2053 } else if (Fold.Commuted) {
2054 // Restoring instruction's original operand order if fold has failed.
2055 TII->commuteInstruction(MI&: *Fold.UseMI, NewMI: false);
2056 }
2057 }
2058
2059 for (MachineInstr *MI : ConstantFoldCandidates) {
2060 if (tryConstantFoldOp(MI)) {
2061 LLVM_DEBUG(dbgs() << "Constant folded " << *MI);
2062 Changed = true;
2063 }
2064 }
2065 return true;
2066}
2067
2068/// Fold %agpr = COPY (REG_SEQUENCE x_MOV_B32, ...) into REG_SEQUENCE
2069/// (V_ACCVGPR_WRITE_B32_e64) ... depending on the reg_sequence input values.
2070bool SIFoldOperandsImpl::foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const {
2071 // It is very tricky to store a value into an AGPR. v_accvgpr_write_b32 can
2072 // only accept VGPR or inline immediate. Recreate a reg_sequence with its
2073 // initializers right here, so we will rematerialize immediates and avoid
2074 // copies via different reg classes.
2075 const TargetRegisterClass *DefRC =
2076 MRI->getRegClass(Reg: CopyMI->getOperand(i: 0).getReg());
2077 if (!TRI->isAGPRClass(RC: DefRC))
2078 return false;
2079
2080 Register UseReg = CopyMI->getOperand(i: 1).getReg();
2081 MachineInstr *RegSeq = MRI->getVRegDef(Reg: UseReg);
2082 if (!RegSeq || !RegSeq->isRegSequence())
2083 return false;
2084
2085 const DebugLoc &DL = CopyMI->getDebugLoc();
2086 MachineBasicBlock &MBB = *CopyMI->getParent();
2087
2088 MachineInstrBuilder B(*MBB.getParent(), CopyMI);
2089 DenseMap<TargetInstrInfo::RegSubRegPair, Register> VGPRCopies;
2090
2091 const TargetRegisterClass *UseRC =
2092 MRI->getRegClass(Reg: CopyMI->getOperand(i: 1).getReg());
2093
2094 // Value, subregindex for new REG_SEQUENCE
2095 SmallVector<std::pair<MachineOperand *, unsigned>, 32> NewDefs;
2096
2097 unsigned NumRegSeqOperands = RegSeq->getNumOperands();
2098 unsigned NumFoldable = 0;
2099
2100 for (unsigned I = 1; I != NumRegSeqOperands; I += 2) {
2101 MachineOperand &RegOp = RegSeq->getOperand(i: I);
2102 unsigned SubRegIdx = RegSeq->getOperand(i: I + 1).getImm();
2103
2104 if (RegOp.getSubReg()) {
2105 // TODO: Handle subregister compose
2106 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2107 continue;
2108 }
2109
2110 MachineOperand *Lookup = lookUpCopyChain(TII: *TII, MRI: *MRI, SrcReg: RegOp.getReg());
2111 if (!Lookup)
2112 Lookup = &RegOp;
2113
2114 if (Lookup->isImm()) {
2115 // Check if this is an agpr_32 subregister.
2116 const TargetRegisterClass *DestSuperRC = TRI->getMatchingSuperRegClass(
2117 A: DefRC, B: &AMDGPU::AGPR_32RegClass, Idx: SubRegIdx);
2118 if (DestSuperRC &&
2119 TII->isInlineConstant(MO: *Lookup, OperandType: AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
2120 ++NumFoldable;
2121 NewDefs.emplace_back(Args&: Lookup, Args&: SubRegIdx);
2122 continue;
2123 }
2124 }
2125
2126 const TargetRegisterClass *InputRC =
2127 Lookup->isReg() ? MRI->getRegClass(Reg: Lookup->getReg())
2128 : MRI->getRegClass(Reg: RegOp.getReg());
2129
2130 // TODO: Account for Lookup->getSubReg()
2131
2132 // If we can't find a matching super class, this is an SGPR->AGPR or
2133 // VGPR->AGPR subreg copy (or something constant-like we have to materialize
2134 // in the AGPR). We can't directly copy from SGPR to AGPR on gfx908, so we
2135 // want to rewrite to copy to an intermediate VGPR class.
2136 const TargetRegisterClass *MatchRC =
2137 TRI->getMatchingSuperRegClass(A: DefRC, B: InputRC, Idx: SubRegIdx);
2138 if (!MatchRC) {
2139 ++NumFoldable;
2140 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2141 continue;
2142 }
2143
2144 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2145 }
2146
2147 // Do not clone a reg_sequence and merely change the result register class.
2148 if (NumFoldable == 0)
2149 return false;
2150
2151 CopyMI->setDesc(TII->get(Opcode: AMDGPU::REG_SEQUENCE));
2152 for (unsigned I = CopyMI->getNumOperands() - 1; I > 0; --I)
2153 CopyMI->removeOperand(OpNo: I);
2154
2155 for (auto [Def, DestSubIdx] : NewDefs) {
2156 if (!Def->isReg()) {
2157 // TODO: Should we use single write for each repeated value like in
2158 // register case?
2159 Register Tmp = MRI->createVirtualRegister(RegClass: &AMDGPU::AGPR_32RegClass);
2160 BuildMI(BB&: MBB, I: CopyMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: Tmp)
2161 .add(MO: *Def);
2162 B.addReg(RegNo: Tmp);
2163 } else {
2164 TargetInstrInfo::RegSubRegPair Src = getRegSubRegPair(O: *Def);
2165 Def->setIsKill(false);
2166
2167 Register &VGPRCopy = VGPRCopies[Src];
2168 if (!VGPRCopy) {
2169 const TargetRegisterClass *VGPRUseSubRC =
2170 TRI->getSubRegisterClass(UseRC, DestSubIdx);
2171
2172 // We cannot build a reg_sequence out of the same registers, they
2173 // must be copied. Better do it here before copyPhysReg() created
2174 // several reads to do the AGPR->VGPR->AGPR copy.
2175
2176 // Direct copy from SGPR to AGPR is not possible on gfx908. To avoid
2177 // creation of exploded copies SGPR->VGPR->AGPR in the copyPhysReg()
2178 // later, create a copy here and track if we already have such a copy.
2179 const TargetRegisterClass *SubRC =
2180 TRI->getSubRegisterClass(MRI->getRegClass(Reg: Src.Reg), Src.SubReg);
2181 if (!VGPRUseSubRC->hasSubClassEq(RC: SubRC)) {
2182 // TODO: Try to reconstrain class
2183 VGPRCopy = MRI->createVirtualRegister(RegClass: VGPRUseSubRC);
2184 BuildMI(BB&: MBB, I: CopyMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: VGPRCopy).add(MO: *Def);
2185 B.addReg(RegNo: VGPRCopy);
2186 } else {
2187 // If it is already a VGPR, do not copy the register.
2188 B.add(MO: *Def);
2189 }
2190 } else {
2191 B.addReg(RegNo: VGPRCopy);
2192 }
2193 }
2194
2195 B.addImm(Val: DestSubIdx);
2196 }
2197
2198 LLVM_DEBUG(dbgs() << "Folded " << *CopyMI);
2199 return true;
2200}
2201
2202bool SIFoldOperandsImpl::tryFoldFoldableCopy(
2203 MachineInstr &MI, MachineOperand *&CurrentKnownM0Val) const {
2204 Register DstReg = MI.getOperand(i: 0).getReg();
2205 // Specially track simple redefs of m0 to the same value in a block, so we
2206 // can erase the later ones.
2207 if (DstReg == AMDGPU::M0) {
2208 MachineOperand &NewM0Val = MI.getOperand(i: 1);
2209 if (CurrentKnownM0Val && CurrentKnownM0Val->isIdenticalTo(Other: NewM0Val)) {
2210 MI.eraseFromParent();
2211 return true;
2212 }
2213
2214 // We aren't tracking other physical registers
2215 CurrentKnownM0Val = (NewM0Val.isReg() && NewM0Val.getReg().isPhysical())
2216 ? nullptr
2217 : &NewM0Val;
2218 return false;
2219 }
2220
2221 MachineOperand *OpToFoldPtr;
2222 if (MI.getOpcode() == AMDGPU::V_MOV_B16_t16_e64) {
2223 // Folding when any src_modifiers are non-zero is unsupported
2224 if (TII->hasAnyModifiersSet(MI))
2225 return false;
2226 OpToFoldPtr = &MI.getOperand(i: 2);
2227 } else
2228 OpToFoldPtr = &MI.getOperand(i: 1);
2229 MachineOperand &OpToFold = *OpToFoldPtr;
2230 bool FoldingImm = OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
2231
2232 // FIXME: We could also be folding things like TargetIndexes.
2233 if (!FoldingImm && !OpToFold.isReg())
2234 return false;
2235
2236 // Fold virtual registers and constant physical registers.
2237 if (OpToFold.isReg() && OpToFold.getReg().isPhysical() &&
2238 !TRI->isConstantPhysReg(PhysReg: OpToFold.getReg()))
2239 return false;
2240
2241 // Prevent folding operands backwards in the function. For example,
2242 // the COPY opcode must not be replaced by 1 in this example:
2243 //
2244 // %3 = COPY %vgpr0; VGPR_32:%3
2245 // ...
2246 // %vgpr0 = V_MOV_B32_e32 1, implicit %exec
2247 if (!DstReg.isVirtual())
2248 return false;
2249
2250 const TargetRegisterClass *DstRC =
2251 MRI->getRegClass(Reg: MI.getOperand(i: 0).getReg());
2252
2253 // True16: Fix malformed 16-bit sgpr COPY produced by peephole-opt
2254 // Can remove this code if proper 16-bit SGPRs are implemented
2255 // Example: Pre-peephole-opt
2256 // %29:sgpr_lo16 = COPY %16.lo16:sreg_32
2257 // %32:sreg_32 = COPY %29:sgpr_lo16
2258 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2259 // Post-peephole-opt and DCE
2260 // %32:sreg_32 = COPY %16.lo16:sreg_32
2261 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2262 // After this transform
2263 // %32:sreg_32 = COPY %16:sreg_32
2264 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2265 // After the fold operands pass
2266 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %16:sreg_32
2267 if (MI.getOpcode() == AMDGPU::COPY && OpToFold.isReg() &&
2268 OpToFold.getSubReg()) {
2269 if (DstRC == &AMDGPU::SReg_32RegClass &&
2270 DstRC == MRI->getRegClass(Reg: OpToFold.getReg())) {
2271 if (!TRI->getMatchingSuperRegClass(A: DstRC, B: &AMDGPU::SGPR_LO16RegClass,
2272 Idx: OpToFold.getSubReg()))
2273 return false;
2274 OpToFold.setSubReg(0);
2275 }
2276 }
2277
2278 // Fold copy to AGPR through reg_sequence
2279 // TODO: Handle with subregister extract
2280 if (OpToFold.isReg() && MI.isCopy() && !MI.getOperand(i: 1).getSubReg()) {
2281 if (foldCopyToAGPRRegSequence(CopyMI: &MI))
2282 return true;
2283 }
2284
2285 FoldableDef Def(OpToFold, DstRC);
2286 bool Changed = foldInstOperand(MI, OpToFold: Def);
2287
2288 // If we managed to fold all uses of this copy then we might as well
2289 // delete it now.
2290 // The only reason we need to follow chains of copies here is that
2291 // tryFoldRegSequence looks forward through copies before folding a
2292 // REG_SEQUENCE into its eventual users.
2293 auto *InstToErase = &MI;
2294 while (MRI->use_nodbg_empty(RegNo: InstToErase->getOperand(i: 0).getReg())) {
2295 auto &SrcOp = InstToErase->getOperand(i: 1);
2296 auto SrcReg = SrcOp.isReg() ? SrcOp.getReg() : Register();
2297 InstToErase->eraseFromParent();
2298 Changed = true;
2299 InstToErase = nullptr;
2300 if (!SrcReg || SrcReg.isPhysical())
2301 break;
2302 InstToErase = MRI->getVRegDef(Reg: SrcReg);
2303 if (!InstToErase || !TII->isFoldableCopy(MI: *InstToErase))
2304 break;
2305 }
2306
2307 if (InstToErase && InstToErase->isRegSequence() &&
2308 MRI->use_nodbg_empty(RegNo: InstToErase->getOperand(i: 0).getReg())) {
2309 InstToErase->eraseFromParent();
2310 Changed = true;
2311 }
2312
2313 if (Changed)
2314 return true;
2315
2316 // Run this after foldInstOperand to avoid turning scalar additions into
2317 // vector additions when the result scalar result could just be folded into
2318 // the user(s).
2319 return OpToFold.isReg() &&
2320 foldCopyToVGPROfScalarAddOfFrameIndex(DstReg, SrcReg: OpToFold.getReg(), MI);
2321}
2322
2323// Clamp patterns are canonically selected to v_max_* instructions, so only
2324// handle them.
2325const MachineOperand *
2326SIFoldOperandsImpl::isClamp(const MachineInstr &MI) const {
2327 unsigned Op = MI.getOpcode();
2328 switch (Op) {
2329 case AMDGPU::V_MAX_F32_e64:
2330 case AMDGPU::V_MAX_F16_e64:
2331 case AMDGPU::V_MAX_F16_t16_e64:
2332 case AMDGPU::V_MAX_F16_fake16_e64:
2333 case AMDGPU::V_MAX_F64_e64:
2334 case AMDGPU::V_MAX_NUM_F64_e64:
2335 case AMDGPU::V_PK_MAX_F16:
2336 case AMDGPU::V_MAX_BF16_PSEUDO_e64:
2337 case AMDGPU::V_PK_MAX_NUM_BF16: {
2338 if (MI.mayRaiseFPException())
2339 return nullptr;
2340
2341 if (!TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp)->getImm())
2342 return nullptr;
2343
2344 // Make sure sources are identical.
2345 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2346 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2347 if (!Src0->isReg() || !Src1->isReg() ||
2348 Src0->getReg() != Src1->getReg() ||
2349 Src0->getSubReg() != Src1->getSubReg() ||
2350 Src0->getSubReg() != AMDGPU::NoSubRegister)
2351 return nullptr;
2352
2353 // Can't fold up if we have modifiers.
2354 if (TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
2355 return nullptr;
2356
2357 unsigned Src0Mods
2358 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm();
2359 unsigned Src1Mods
2360 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm();
2361
2362 // Having a 0 op_sel_hi would require swizzling the output in the source
2363 // instruction, which we can't do.
2364 unsigned UnsetMods =
2365 (Op == AMDGPU::V_PK_MAX_F16 || Op == AMDGPU::V_PK_MAX_NUM_BF16)
2366 ? SISrcMods::OP_SEL_1
2367 : 0u;
2368 if (Src0Mods != UnsetMods || Src1Mods != UnsetMods)
2369 return nullptr;
2370 return Src0;
2371 }
2372 default:
2373 return nullptr;
2374 }
2375}
2376
2377// FIXME: Clamp for v_mad_mixhi_f16 handled during isel.
2378bool SIFoldOperandsImpl::tryFoldClamp(MachineInstr &MI) {
2379 const MachineOperand *ClampSrc = isClamp(MI);
2380 if (!ClampSrc || !MRI->hasOneNonDBGUser(RegNo: ClampSrc->getReg()))
2381 return false;
2382
2383 if (!ClampSrc->getReg().isVirtual())
2384 return false;
2385
2386 // Look through COPY. COPY only observed with True16.
2387 Register DefSrcReg = TRI->lookThruCopyLike(SrcReg: ClampSrc->getReg(), MRI);
2388 MachineInstr *Def =
2389 MRI->getVRegDef(Reg: DefSrcReg.isVirtual() ? DefSrcReg : ClampSrc->getReg());
2390
2391 // The type of clamp must be compatible.
2392 if (!SIInstrInfo::hasSameClamp(A: *Def, B: MI))
2393 return false;
2394
2395 if (Def->mayRaiseFPException())
2396 return false;
2397
2398 MachineOperand *DefClamp = TII->getNamedOperand(MI&: *Def, OperandName: AMDGPU::OpName::clamp);
2399 if (!DefClamp)
2400 return false;
2401
2402 LLVM_DEBUG(dbgs() << "Folding clamp " << *DefClamp << " into " << *Def);
2403
2404 // Clamp is applied after omod, so it is OK if omod is set.
2405 DefClamp->setImm(1);
2406
2407 Register DefReg = Def->getOperand(i: 0).getReg();
2408 Register MIDstReg = MI.getOperand(i: 0).getReg();
2409 if (TRI->isSGPRReg(MRI: *MRI, Reg: DefReg)) {
2410 // Pseudo scalar instructions have a SGPR for dst and clamp is a v_max*
2411 // instruction with a VGPR dst.
2412 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
2413 DestReg: MIDstReg)
2414 .addReg(RegNo: DefReg);
2415 } else {
2416 MRI->replaceRegWith(FromReg: MIDstReg, ToReg: DefReg);
2417 }
2418 MI.eraseFromParent();
2419
2420 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
2421 // instruction, so we might as well convert it to the more flexible VOP3-only
2422 // mad/fma form.
2423 if (TII->convertToThreeAddress(MI&: *Def, /*LIS=*/nullptr))
2424 Def->eraseFromParent();
2425
2426 return true;
2427}
2428
2429static int getOModValue(unsigned Opc, int64_t Val) {
2430 switch (Opc) {
2431 case AMDGPU::V_MUL_F64_e64:
2432 case AMDGPU::V_MUL_F64_pseudo_e64: {
2433 switch (Val) {
2434 case 0x3fe0000000000000: // 0.5
2435 return SIOutMods::DIV2;
2436 case 0x4000000000000000: // 2.0
2437 return SIOutMods::MUL2;
2438 case 0x4010000000000000: // 4.0
2439 return SIOutMods::MUL4;
2440 default:
2441 return SIOutMods::NONE;
2442 }
2443 }
2444 case AMDGPU::V_MUL_F32_e64: {
2445 switch (static_cast<uint32_t>(Val)) {
2446 case 0x3f000000: // 0.5
2447 return SIOutMods::DIV2;
2448 case 0x40000000: // 2.0
2449 return SIOutMods::MUL2;
2450 case 0x40800000: // 4.0
2451 return SIOutMods::MUL4;
2452 default:
2453 return SIOutMods::NONE;
2454 }
2455 }
2456 case AMDGPU::V_MUL_F16_e64:
2457 case AMDGPU::V_MUL_F16_t16_e64:
2458 case AMDGPU::V_MUL_F16_fake16_e64: {
2459 switch (static_cast<uint16_t>(Val)) {
2460 case 0x3800: // 0.5
2461 return SIOutMods::DIV2;
2462 case 0x4000: // 2.0
2463 return SIOutMods::MUL2;
2464 case 0x4400: // 4.0
2465 return SIOutMods::MUL4;
2466 default:
2467 return SIOutMods::NONE;
2468 }
2469 }
2470 case AMDGPU::V_PK_MUL_BF16: {
2471 switch (static_cast<uint16_t>(Val)) {
2472 case 0x3F00: // 0.5 in BF16
2473 return SIOutMods::DIV2;
2474 case 0x4000: // 2.0 in BF16
2475 return SIOutMods::MUL2;
2476 case 0x4080: // 4.0 in BF16
2477 return SIOutMods::MUL4;
2478 default:
2479 return SIOutMods::NONE;
2480 }
2481 }
2482 default:
2483 llvm_unreachable("invalid mul opcode");
2484 }
2485}
2486
2487// FIXME: Does this really not support denormals with f16?
2488// FIXME: Does this need to check IEEE mode bit? SNaNs are generally not
2489// handled, so will anything other than that break?
2490std::pair<const MachineOperand *, int>
2491SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
2492 unsigned Op = MI.getOpcode();
2493 switch (Op) {
2494 case AMDGPU::V_MUL_F64_e64:
2495 case AMDGPU::V_MUL_F64_pseudo_e64:
2496 case AMDGPU::V_MUL_F32_e64:
2497 case AMDGPU::V_MUL_F16_t16_e64:
2498 case AMDGPU::V_MUL_F16_fake16_e64:
2499 case AMDGPU::V_MUL_F16_e64: {
2500 // If output denormals are enabled, omod is ignored.
2501 if ((Op == AMDGPU::V_MUL_F32_e64 &&
2502 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
2503 ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F64_pseudo_e64 ||
2504 Op == AMDGPU::V_MUL_F16_e64 || Op == AMDGPU::V_MUL_F16_t16_e64 ||
2505 Op == AMDGPU::V_MUL_F16_fake16_e64) &&
2506 MFI->getMode().FP64FP16Denormals.Output !=
2507 DenormalMode::PreserveSign) ||
2508 MI.mayRaiseFPException())
2509 return {nullptr, SIOutMods::NONE};
2510
2511 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2512 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2513
2514 // If there is an immediate operand, it must be Src1
2515 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
2516 if (!Src1Imm)
2517 return {nullptr, SIOutMods::NONE};
2518
2519 int OMod = getOModValue(Opc: Op, Val: *Src1Imm);
2520 if (OMod == SIOutMods::NONE ||
2521 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers) ||
2522 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers) ||
2523 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2524 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2525 return {nullptr, SIOutMods::NONE};
2526
2527 return {Src0, OMod};
2528 }
2529 case AMDGPU::V_ADD_F64_e64:
2530 case AMDGPU::V_ADD_F64_pseudo_e64:
2531 case AMDGPU::V_ADD_F32_e64:
2532 case AMDGPU::V_ADD_F16_e64:
2533 case AMDGPU::V_ADD_F16_t16_e64:
2534 case AMDGPU::V_ADD_F16_fake16_e64: {
2535 // If output denormals are enabled, omod is ignored.
2536 if ((Op == AMDGPU::V_ADD_F32_e64 &&
2537 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
2538 ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F64_pseudo_e64 ||
2539 Op == AMDGPU::V_ADD_F16_e64 || Op == AMDGPU::V_ADD_F16_t16_e64 ||
2540 Op == AMDGPU::V_ADD_F16_fake16_e64) &&
2541 MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign))
2542 return {nullptr, SIOutMods::NONE};
2543
2544 // Look through the DAGCombiner canonicalization fmul x, 2 -> fadd x, x
2545 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2546 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2547
2548 if (Src0->isReg() && Src1->isReg() && Src0->getReg() == Src1->getReg() &&
2549 Src0->getSubReg() == Src1->getSubReg() &&
2550 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers) &&
2551 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers) &&
2552 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) &&
2553 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
2554 return {Src0, SIOutMods::MUL2};
2555
2556 return {nullptr, SIOutMods::NONE};
2557 }
2558 case AMDGPU::V_PK_MUL_BF16: {
2559 // OMOD folding for BF16 packed multiply. bf16 has no denormal mode of its
2560 // own; it follows the default ("denormal-fp-math") mode, which is the same
2561 // field as f64/f16.
2562 if (MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign ||
2563 MI.mayRaiseFPException())
2564 return {nullptr, SIOutMods::NONE};
2565
2566 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2567 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2568
2569 // If there is an immediate operand, it must be Src1
2570 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
2571 if (!Src1Imm)
2572 return {nullptr, SIOutMods::NONE};
2573
2574 int OMod = getOModValue(Opc: AMDGPU::V_PK_MUL_BF16, Val: *Src1Imm);
2575 if (OMod == SIOutMods::NONE)
2576 return {nullptr, SIOutMods::NONE};
2577
2578 // Modifiers other than op_sel_hi block OMOD folding. Per getOModValue
2579 // above, Src1 is an inline constant (0.5/2.0/4.0), which may carry
2580 // op_sel_lo to read it from the upper FP32 half, so allow that on src1.
2581 const MachineOperand *Src0Mods =
2582 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2583 const MachineOperand *Src1Mods =
2584 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2585 if ((Src0Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2586 (Src1Mods->getImm() & ~(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)) ||
2587 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2588 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2589 return {nullptr, SIOutMods::NONE};
2590
2591 return {Src0, OMod};
2592 }
2593 case AMDGPU::V_PK_ADD_BF16: {
2594 // OMOD folding for BF16 packed add: x + x -> x * 2. See the bf16 denormal
2595 // mode note in the V_PK_MUL_BF16 case above.
2596 if (MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign)
2597 return {nullptr, SIOutMods::NONE};
2598
2599 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2600 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2601
2602 if (!Src0->isReg() || !Src1->isReg() || Src0->getReg() != Src1->getReg() ||
2603 Src0->getSubReg() != Src1->getSubReg())
2604 return {nullptr, SIOutMods::NONE};
2605
2606 // Modifiers other than op_sel_hi block OMOD folding
2607 const MachineOperand *Src0Mods =
2608 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2609 const MachineOperand *Src1Mods =
2610 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2611 if ((Src0Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2612 (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2613 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2614 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2615 return {nullptr, SIOutMods::NONE};
2616
2617 return {Src0, SIOutMods::MUL2};
2618 }
2619 default:
2620 return {nullptr, SIOutMods::NONE};
2621 }
2622}
2623
2624// FIXME: Does this need to check IEEE bit on function?
2625bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
2626 const MachineOperand *RegOp;
2627 int OMod;
2628 std::tie(args&: RegOp, args&: OMod) = isOMod(MI);
2629 if (OMod == SIOutMods::NONE || !RegOp->isReg() ||
2630 RegOp->getSubReg() != AMDGPU::NoSubRegister ||
2631 !MRI->hasOneNonDBGUser(RegNo: RegOp->getReg()))
2632 return false;
2633
2634 MachineInstr *Def = MRI->getVRegDef(Reg: RegOp->getReg());
2635 Register OModSrcReg = Def->getOperand(i: 0).getReg();
2636
2637 // In real-true16 mode, vgpr_16 results are packed into vgpr_32 via
2638 // REG_SEQUENCE. Look through it to find the actual instruction.
2639 if (Def->isRegSequence() && Def->getNumOperands() == 5 &&
2640 Def->getOperand(i: 2).getImm() == AMDGPU::lo16) {
2641 // Only look through if the high 16 bits are undefined
2642 bool CanLookThrough = true;
2643 MachineInstr *Hi16Def = MRI->getVRegDef(Reg: Def->getOperand(i: 3).getReg());
2644 if (!Hi16Def || !Hi16Def->isImplicitDef())
2645 CanLookThrough = false;
2646
2647 if (CanLookThrough) {
2648 Register SrcReg = Def->getOperand(i: 1).getReg();
2649 if (!MRI->hasOneNonDBGUse(RegNo: SrcReg))
2650 return false;
2651
2652 Def = MRI->getVRegDef(Reg: SrcReg);
2653 if (!Def)
2654 return false;
2655 }
2656 }
2657
2658 MachineOperand *DefOMod = TII->getNamedOperand(MI&: *Def, OperandName: AMDGPU::OpName::omod);
2659 if (!DefOMod || DefOMod->getImm() != SIOutMods::NONE)
2660 return false;
2661
2662 if (Def->mayRaiseFPException())
2663 return false;
2664
2665 // Clamp is applied after omod. If the source already has clamp set, don't
2666 // fold it.
2667 if (TII->hasModifiersSet(MI: *Def, OpName: AMDGPU::OpName::clamp))
2668 return false;
2669
2670 LLVM_DEBUG(dbgs() << "Folding omod " << MI << " into " << *Def);
2671
2672 DefOMod->setImm(OMod);
2673 MRI->replaceRegWith(FromReg: MI.getOperand(i: 0).getReg(), ToReg: OModSrcReg);
2674 // Kill flags can be wrong if we replaced a def inside a loop with a def
2675 // outside the loop.
2676 MRI->clearKillFlags(Reg: OModSrcReg);
2677 MI.eraseFromParent();
2678
2679 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
2680 // instruction, so we might as well convert it to the more flexible VOP3-only
2681 // mad/fma form.
2682 if (TII->convertToThreeAddress(MI&: *Def, /*LIS=*/nullptr))
2683 Def->eraseFromParent();
2684
2685 return true;
2686}
2687
2688// Try to optimize SGPR reg sequences that are splat <s, s> or <s, s, s, s>
2689// where all uses are PackedSingleSGPR64BitInst, replacing with <s, undef, ...>
2690bool SIFoldOperandsImpl::tryFoldSGPRSplatRegSequence(MachineInstr &MI) {
2691 assert(MI.isRegSequence());
2692
2693 if (!ST->hasPackedFP64SingleSGPROps() && !ST->hasPackedU64SingleSGPROps())
2694 return false;
2695
2696 Register Reg = MI.getOperand(i: 0).getReg();
2697
2698 // Only optimize 128-bit SGPR register sequences
2699 const TargetRegisterClass *RegClass = MRI->getRegClass(Reg);
2700 if (!TRI->isSGPRClass(RC: RegClass) || TRI->getRegSizeInBits(RC: *RegClass) != 128)
2701 return false;
2702
2703 SmallVector<std::pair<MachineOperand *, unsigned>, 32> Defs;
2704 if (!getRegSeqInit(Defs, UseReg: Reg))
2705 return false;
2706
2707 // Check if this is a splat pattern
2708 if (Defs.size() <= 1)
2709 return false;
2710
2711 const auto &[FirstOp, _] = Defs.front();
2712 if (!FirstOp->isReg())
2713 return false;
2714
2715 Register FirstReg = FirstOp->getReg();
2716 unsigned FirstSubReg = FirstOp->getSubReg();
2717
2718 const TargetRegisterClass *FirstRegClass = MRI->getRegClass(Reg: FirstReg);
2719 if (!TRI->isSGPRClass(RC: FirstRegClass))
2720 return false;
2721
2722 // Check remaining elements match first
2723 if (!llvm::all_of(Range: llvm::drop_begin(RangeOrContainer&: Defs), P: [&](const auto &Def) {
2724 const auto &[Op, _] = Def;
2725 return Op->isReg() && Op->getReg() == FirstReg &&
2726 Op->getSubReg() == FirstSubReg;
2727 }))
2728 return false;
2729
2730 // Check if all uses are isSingleSGPRReadInst
2731 for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
2732 if (!AMDGPU::isPackedSingleSGPR64BitInst(Opc: UseMI.getOpcode()))
2733 return false;
2734 }
2735
2736 // Create new reg sequence with <s, undef, undef, ...>
2737 Register NewDst = MRI->createVirtualRegister(RegClass);
2738 MachineInstrBuilder RS = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2739 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDst);
2740
2741 // Add the first operand
2742 FirstOp->setIsKill(false);
2743 RS.add(MO: *FirstOp);
2744 RS.addImm(Val: Defs[0].second);
2745
2746 // Add undef for remaining lanes
2747 // Create an undef virtual register for the same register class
2748 Register UndefReg = MRI->createVirtualRegister(RegClass: FirstRegClass);
2749 for (unsigned i = 1; i < Defs.size(); ++i) {
2750 RS.addReg(RegNo: UndefReg, Flags: RegState::Undef);
2751 RS.addImm(Val: Defs[i].second);
2752 }
2753
2754 // Replace all uses
2755 MRI->replaceRegWith(FromReg: Reg, ToReg: NewDst);
2756
2757 LLVM_DEBUG(dbgs() << "Folded splat SGPR reg_sequence: " << MI << " into "
2758 << *RS);
2759
2760 MI.eraseFromParent();
2761 return true;
2762}
2763
2764// Try to fold a reg_sequence with vgpr output and agpr inputs into an
2765// instruction which can take an agpr. So far that means a store.
2766bool SIFoldOperandsImpl::tryFoldRegSequence(MachineInstr &MI) {
2767 assert(MI.isRegSequence());
2768
2769 // Try to optimize SGPR splat sequences first
2770 if (tryFoldSGPRSplatRegSequence(MI))
2771 return true;
2772
2773 auto Reg = MI.getOperand(i: 0).getReg();
2774
2775 if (!ST->hasGFX90AInsts() || !TRI->isVGPR(MRI: *MRI, Reg) ||
2776 !MRI->hasOneNonDBGUse(RegNo: Reg))
2777 return false;
2778
2779 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
2780 if (!getRegSeqInit(Defs, UseReg: Reg))
2781 return false;
2782
2783 for (auto &[Op, SubIdx] : Defs) {
2784 if (!Op->isReg())
2785 return false;
2786 if (TRI->isAGPR(MRI: *MRI, Reg: Op->getReg()))
2787 continue;
2788 // Maybe this is a COPY from AREG
2789 const MachineInstr *SubDef = MRI->getVRegDef(Reg: Op->getReg());
2790 if (!SubDef || !SubDef->isCopy() || SubDef->getOperand(i: 1).getSubReg())
2791 return false;
2792 if (!TRI->isAGPR(MRI: *MRI, Reg: SubDef->getOperand(i: 1).getReg()))
2793 return false;
2794 }
2795
2796 MachineOperand *Op = &*MRI->use_nodbg_begin(RegNo: Reg);
2797 MachineInstr *UseMI = Op->getParent();
2798 while (UseMI->isCopy() && !Op->getSubReg()) {
2799 Reg = UseMI->getOperand(i: 0).getReg();
2800 if (!TRI->isVGPR(MRI: *MRI, Reg) || !MRI->hasOneNonDBGUse(RegNo: Reg))
2801 return false;
2802 Op = &*MRI->use_nodbg_begin(RegNo: Reg);
2803 UseMI = Op->getParent();
2804 }
2805
2806 if (Op->getSubReg())
2807 return false;
2808
2809 unsigned OpIdx = Op - &UseMI->getOperand(i: 0);
2810 const MCInstrDesc &InstDesc = UseMI->getDesc();
2811 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: InstDesc, OpNum: OpIdx);
2812 if (!OpRC || !TRI->isVectorSuperClass(RC: OpRC))
2813 return false;
2814
2815 const auto *NewDstRC = TRI->getEquivalentAGPRClass(SRC: MRI->getRegClass(Reg));
2816 auto Dst = MRI->createVirtualRegister(RegClass: NewDstRC);
2817 auto RS = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2818 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst);
2819
2820 for (auto &[Def, SubIdx] : Defs) {
2821 Def->setIsKill(false);
2822 if (TRI->isAGPR(MRI: *MRI, Reg: Def->getReg())) {
2823 RS.add(MO: *Def);
2824 } else { // This is a copy
2825 MachineInstr *SubDef = MRI->getVRegDef(Reg: Def->getReg());
2826 SubDef->getOperand(i: 1).setIsKill(false);
2827 RS.addReg(RegNo: SubDef->getOperand(i: 1).getReg(), Flags: {}, SubReg: Def->getSubReg());
2828 }
2829 RS.addImm(Val: SubIdx);
2830 }
2831
2832 Op->setReg(Dst);
2833 if (!TII->isOperandLegal(MI: *UseMI, OpIdx, MO: Op)) {
2834 Op->setReg(Reg);
2835 RS->eraseFromParent();
2836 return false;
2837 }
2838
2839 LLVM_DEBUG(dbgs() << "Folded " << *RS << " into " << *UseMI);
2840
2841 // Erase the REG_SEQUENCE eagerly, unless we followed a chain of COPY users,
2842 // in which case we can erase them all later in runOnMachineFunction.
2843 if (MRI->use_nodbg_empty(RegNo: MI.getOperand(i: 0).getReg()))
2844 MI.eraseFromParent();
2845 return true;
2846}
2847
2848/// Checks whether \p Copy is a AGPR -> VGPR copy. Returns `true` on success and
2849/// stores the AGPR register in \p OutReg and the subreg in \p OutSubReg
2850static bool isAGPRCopy(const SIRegisterInfo &TRI,
2851 const MachineRegisterInfo &MRI, const MachineInstr &Copy,
2852 Register &OutReg, unsigned &OutSubReg) {
2853 assert(Copy.isCopy());
2854
2855 const MachineOperand &CopySrc = Copy.getOperand(i: 1);
2856 Register CopySrcReg = CopySrc.getReg();
2857 if (!CopySrcReg.isVirtual())
2858 return false;
2859
2860 // Common case: copy from AGPR directly, e.g.
2861 // %1:vgpr_32 = COPY %0:agpr_32
2862 if (TRI.isAGPR(MRI, Reg: CopySrcReg)) {
2863 OutReg = CopySrcReg;
2864 OutSubReg = CopySrc.getSubReg();
2865 return true;
2866 }
2867
2868 // Sometimes it can also involve two copies, e.g.
2869 // %1:vgpr_256 = COPY %0:agpr_256
2870 // %2:vgpr_32 = COPY %1:vgpr_256.sub0
2871 const MachineInstr *CopySrcDef = MRI.getVRegDef(Reg: CopySrcReg);
2872 if (!CopySrcDef || !CopySrcDef->isCopy())
2873 return false;
2874
2875 const MachineOperand &OtherCopySrc = CopySrcDef->getOperand(i: 1);
2876 Register OtherCopySrcReg = OtherCopySrc.getReg();
2877 if (!OtherCopySrcReg.isVirtual() ||
2878 CopySrcDef->getOperand(i: 0).getSubReg() != AMDGPU::NoSubRegister ||
2879 OtherCopySrc.getSubReg() != AMDGPU::NoSubRegister ||
2880 !TRI.isAGPR(MRI, Reg: OtherCopySrcReg))
2881 return false;
2882
2883 OutReg = OtherCopySrcReg;
2884 OutSubReg = CopySrc.getSubReg();
2885 return true;
2886}
2887
2888// Try to hoist an AGPR to VGPR copy across a PHI.
2889// This should allow folding of an AGPR into a consumer which may support it.
2890//
2891// Example 1: LCSSA PHI
2892// loop:
2893// %1:vreg = COPY %0:areg
2894// exit:
2895// %2:vreg = PHI %1:vreg, %loop
2896// =>
2897// loop:
2898// exit:
2899// %1:areg = PHI %0:areg, %loop
2900// %2:vreg = COPY %1:areg
2901//
2902// Example 2: PHI with multiple incoming values:
2903// entry:
2904// %1:vreg = GLOBAL_LOAD(..)
2905// loop:
2906// %2:vreg = PHI %1:vreg, %entry, %5:vreg, %loop
2907// %3:areg = COPY %2:vreg
2908// %4:areg = (instr using %3:areg)
2909// %5:vreg = COPY %4:areg
2910// =>
2911// entry:
2912// %1:vreg = GLOBAL_LOAD(..)
2913// %2:areg = COPY %1:vreg
2914// loop:
2915// %3:areg = PHI %2:areg, %entry, %X:areg,
2916// %4:areg = (instr using %3:areg)
2917bool SIFoldOperandsImpl::tryFoldPhiAGPR(MachineInstr &PHI) {
2918 assert(PHI.isPHI());
2919
2920 Register PhiOut = PHI.getOperand(i: 0).getReg();
2921 if (!TRI->isVGPR(MRI: *MRI, Reg: PhiOut))
2922 return false;
2923
2924 // Iterate once over all incoming values of the PHI to check if this PHI is
2925 // eligible, and determine the exact AGPR RC we'll target.
2926 const TargetRegisterClass *ARC = nullptr;
2927 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
2928 MachineOperand &MO = PHI.getOperand(i: K);
2929 MachineInstr *Copy = MRI->getVRegDef(Reg: MO.getReg());
2930 if (!Copy || !Copy->isCopy())
2931 continue;
2932
2933 Register AGPRSrc;
2934 unsigned AGPRRegMask = AMDGPU::NoSubRegister;
2935 if (!isAGPRCopy(TRI: *TRI, MRI: *MRI, Copy: *Copy, OutReg&: AGPRSrc, OutSubReg&: AGPRRegMask))
2936 continue;
2937
2938 const TargetRegisterClass *CopyInRC = MRI->getRegClass(Reg: AGPRSrc);
2939 if (const auto *SubRC = TRI->getSubRegisterClass(CopyInRC, AGPRRegMask))
2940 CopyInRC = SubRC;
2941
2942 if (ARC && !ARC->hasSubClassEq(RC: CopyInRC))
2943 return false;
2944 ARC = CopyInRC;
2945 }
2946
2947 if (!ARC)
2948 return false;
2949
2950 bool IsAGPR32 = (ARC == &AMDGPU::AGPR_32RegClass);
2951
2952 // Rewrite the PHI's incoming values to ARC.
2953 LLVM_DEBUG(dbgs() << "Folding AGPR copies into: " << PHI);
2954 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
2955 MachineOperand &MO = PHI.getOperand(i: K);
2956 Register Reg = MO.getReg();
2957
2958 MachineBasicBlock::iterator InsertPt;
2959 MachineBasicBlock *InsertMBB = nullptr;
2960
2961 // Look at the def of Reg, ignoring all copies.
2962 unsigned CopyOpc = AMDGPU::COPY;
2963 if (MachineInstr *Def = MRI->getVRegDef(Reg)) {
2964
2965 // Look at pre-existing COPY instructions from ARC: Steal the operand. If
2966 // the copy was single-use, it will be removed by DCE later.
2967 if (Def->isCopy()) {
2968 Register AGPRSrc;
2969 unsigned AGPRSubReg = AMDGPU::NoSubRegister;
2970 if (isAGPRCopy(TRI: *TRI, MRI: *MRI, Copy: *Def, OutReg&: AGPRSrc, OutSubReg&: AGPRSubReg)) {
2971 MO.setReg(AGPRSrc);
2972 MO.setSubReg(AGPRSubReg);
2973 continue;
2974 }
2975
2976 // If this is a multi-use SGPR -> VGPR copy, use V_ACCVGPR_WRITE on
2977 // GFX908 directly instead of a COPY. Otherwise, SIFoldOperand may try
2978 // to fold the sgpr -> vgpr -> agpr copy into a sgpr -> agpr copy which
2979 // is unlikely to be profitable.
2980 //
2981 // Note that V_ACCVGPR_WRITE is only used for AGPR_32.
2982 MachineOperand &CopyIn = Def->getOperand(i: 1);
2983 if (IsAGPR32 && !ST->hasGFX90AInsts() && !MRI->hasOneNonDBGUse(RegNo: Reg) &&
2984 TRI->isSGPRReg(MRI: *MRI, Reg: CopyIn.getReg()))
2985 CopyOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
2986 }
2987
2988 InsertMBB = Def->getParent();
2989 InsertPt = InsertMBB->SkipPHIsLabelsAndDebug(I: ++Def->getIterator());
2990 } else {
2991 InsertMBB = PHI.getOperand(i: MO.getOperandNo() + 1).getMBB();
2992 InsertPt = InsertMBB->getFirstTerminator();
2993 }
2994
2995 Register NewReg = MRI->createVirtualRegister(RegClass: ARC);
2996 MachineInstr *MI = BuildMI(BB&: *InsertMBB, I: InsertPt, MIMD: PHI.getDebugLoc(),
2997 MCID: TII->get(Opcode: CopyOpc), DestReg: NewReg)
2998 .addReg(RegNo: Reg);
2999 MO.setReg(NewReg);
3000
3001 (void)MI;
3002 LLVM_DEBUG(dbgs() << " Created COPY: " << *MI);
3003 }
3004
3005 // Replace the PHI's result with a new register.
3006 Register NewReg = MRI->createVirtualRegister(RegClass: ARC);
3007 PHI.getOperand(i: 0).setReg(NewReg);
3008
3009 // COPY that new register back to the original PhiOut register. This COPY will
3010 // usually be folded out later.
3011 MachineBasicBlock *MBB = PHI.getParent();
3012 BuildMI(BB&: *MBB, I: MBB->getFirstNonPHI(), MIMD: PHI.getDebugLoc(),
3013 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: PhiOut)
3014 .addReg(RegNo: NewReg);
3015
3016 LLVM_DEBUG(dbgs() << " Done: Folded " << PHI);
3017 return true;
3018}
3019
3020// Attempt to convert VGPR load to an AGPR load.
3021bool SIFoldOperandsImpl::tryFoldLoad(MachineInstr &MI) {
3022 assert(MI.mayLoad());
3023 if (!ST->hasGFX90AInsts() || MI.getNumExplicitDefs() != 1)
3024 return false;
3025
3026 MachineOperand &Def = MI.getOperand(i: 0);
3027 if (!Def.isDef())
3028 return false;
3029
3030 Register DefReg = Def.getReg();
3031
3032 if (DefReg.isPhysical() || !TRI->isVGPR(MRI: *MRI, Reg: DefReg))
3033 return false;
3034
3035 SmallVector<const MachineInstr *, 8> Users(
3036 llvm::make_pointer_range(Range: MRI->use_nodbg_instructions(Reg: DefReg)));
3037 SmallVector<Register, 8> MoveRegs;
3038
3039 if (Users.empty())
3040 return false;
3041
3042 // Check that all uses a copy to an agpr or a reg_sequence producing an agpr.
3043 while (!Users.empty()) {
3044 const MachineInstr *I = Users.pop_back_val();
3045 if (!I->isCopy() && !I->isRegSequence())
3046 return false;
3047 Register DstReg = I->getOperand(i: 0).getReg();
3048 // Physical registers may have more than one instruction definitions
3049 if (DstReg.isPhysical())
3050 return false;
3051 if (TRI->isAGPR(MRI: *MRI, Reg: DstReg))
3052 continue;
3053 MoveRegs.push_back(Elt: DstReg);
3054 for (const MachineInstr &U : MRI->use_nodbg_instructions(Reg: DstReg))
3055 Users.push_back(Elt: &U);
3056 }
3057
3058 const TargetRegisterClass *RC = MRI->getRegClass(Reg: DefReg);
3059 MRI->setRegClass(Reg: DefReg, RC: TRI->getEquivalentAGPRClass(SRC: RC));
3060 if (!TII->isOperandLegal(MI, OpIdx: 0, MO: &Def)) {
3061 MRI->setRegClass(Reg: DefReg, RC);
3062 return false;
3063 }
3064
3065 while (!MoveRegs.empty()) {
3066 Register Reg = MoveRegs.pop_back_val();
3067 MRI->setRegClass(Reg, RC: TRI->getEquivalentAGPRClass(SRC: MRI->getRegClass(Reg)));
3068 }
3069
3070 LLVM_DEBUG(dbgs() << "Folded " << MI);
3071
3072 return true;
3073}
3074
3075// tryFoldPhiAGPR will aggressively try to create AGPR PHIs.
3076// For GFX90A and later, this is pretty much always a good thing, but for GFX908
3077// there's cases where it can create a lot more AGPR-AGPR copies, which are
3078// expensive on this architecture due to the lack of V_ACCVGPR_MOV.
3079//
3080// This function looks at all AGPR PHIs in a basic block and collects their
3081// operands. Then, it checks for register that are used more than once across
3082// all PHIs and caches them in a VGPR. This prevents ExpandPostRAPseudo from
3083// having to create one VGPR temporary per use, which can get very messy if
3084// these PHIs come from a broken-up large PHI (e.g. 32 AGPR phis, one per vector
3085// element).
3086//
3087// Example
3088// a:
3089// %in:agpr_256 = COPY %foo:vgpr_256
3090// c:
3091// %x:agpr_32 = ..
3092// b:
3093// %0:areg = PHI %in.sub0:agpr_32, %a, %x, %c
3094// %1:areg = PHI %in.sub0:agpr_32, %a, %y, %c
3095// %2:areg = PHI %in.sub0:agpr_32, %a, %z, %c
3096// =>
3097// a:
3098// %in:agpr_256 = COPY %foo:vgpr_256
3099// %tmp:vgpr_32 = V_ACCVGPR_READ_B32_e64 %in.sub0:agpr_32
3100// %tmp_agpr:agpr_32 = COPY %tmp
3101// c:
3102// %x:agpr_32 = ..
3103// b:
3104// %0:areg = PHI %tmp_agpr, %a, %x, %c
3105// %1:areg = PHI %tmp_agpr, %a, %y, %c
3106// %2:areg = PHI %tmp_agpr, %a, %z, %c
3107bool SIFoldOperandsImpl::tryOptimizeAGPRPhis(MachineBasicBlock &MBB) {
3108 // This is only really needed on GFX908 where AGPR-AGPR copies are
3109 // unreasonably difficult.
3110 if (ST->hasGFX90AInsts())
3111 return false;
3112
3113 // Look at all AGPR Phis and collect the register + subregister used.
3114 DenseMap<std::pair<Register, unsigned>, std::vector<MachineOperand *>>
3115 RegToMO;
3116
3117 for (auto &MI : MBB) {
3118 if (!MI.isPHI())
3119 break;
3120
3121 if (!TRI->isAGPR(MRI: *MRI, Reg: MI.getOperand(i: 0).getReg()))
3122 continue;
3123
3124 for (unsigned K = 1; K < MI.getNumOperands(); K += 2) {
3125 MachineOperand &PhiMO = MI.getOperand(i: K);
3126 if (!PhiMO.getSubReg())
3127 continue;
3128 RegToMO[{PhiMO.getReg(), PhiMO.getSubReg()}].push_back(x: &PhiMO);
3129 }
3130 }
3131
3132 // For all (Reg, SubReg) pair that are used more than once, cache the value in
3133 // a VGPR.
3134 bool Changed = false;
3135 for (const auto &[Entry, MOs] : RegToMO) {
3136 if (MOs.size() == 1)
3137 continue;
3138
3139 const auto [Reg, SubReg] = Entry;
3140 MachineInstr *Def = MRI->getVRegDef(Reg);
3141 MachineBasicBlock *DefMBB = Def->getParent();
3142
3143 // Create a copy in a VGPR using V_ACCVGPR_READ_B32_e64 so it's not folded
3144 // out.
3145 const TargetRegisterClass *ARC = getRegOpRC(MRI: *MRI, TRI: *TRI, MO: *MOs.front());
3146 Register TempVGPR =
3147 MRI->createVirtualRegister(RegClass: TRI->getEquivalentVGPRClass(SRC: ARC));
3148 MachineInstr *VGPRCopy =
3149 BuildMI(BB&: *DefMBB, I: ++Def->getIterator(), MIMD: Def->getDebugLoc(),
3150 MCID: TII->get(Opcode: AMDGPU::V_ACCVGPR_READ_B32_e64), DestReg: TempVGPR)
3151 .addReg(RegNo: Reg, /* flags */ Flags: {}, SubReg);
3152
3153 // Copy back to an AGPR and use that instead of the AGPR subreg in all MOs.
3154 Register TempAGPR = MRI->createVirtualRegister(RegClass: ARC);
3155 BuildMI(BB&: *DefMBB, I: ++VGPRCopy->getIterator(), MIMD: Def->getDebugLoc(),
3156 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: TempAGPR)
3157 .addReg(RegNo: TempVGPR);
3158
3159 LLVM_DEBUG(dbgs() << "Caching AGPR into VGPR: " << *VGPRCopy);
3160 for (MachineOperand *MO : MOs) {
3161 MO->setReg(TempAGPR);
3162 MO->setSubReg(AMDGPU::NoSubRegister);
3163 LLVM_DEBUG(dbgs() << " Changed PHI Operand: " << *MO << "\n");
3164 }
3165
3166 Changed = true;
3167 }
3168
3169 return Changed;
3170}
3171
3172bool SIFoldOperandsImpl::run(MachineFunction &MF, const MachineLoopInfo *MLI) {
3173 this->MF = &MF;
3174 MRI = &MF.getRegInfo();
3175 ST = &MF.getSubtarget<GCNSubtarget>();
3176 TII = ST->getInstrInfo();
3177 TRI = &TII->getRegisterInfo();
3178 MFI = MF.getInfo<SIMachineFunctionInfo>();
3179 this->MLI = MLI;
3180
3181 // omod is ignored by hardware if IEEE bit is enabled. omod also does not
3182 // correctly handle signed zeros.
3183 //
3184 // FIXME: Also need to check strictfp
3185 bool IsIEEEMode = MFI->getMode().IEEE;
3186
3187 bool Changed = false;
3188 for (MachineBasicBlock *MBB : depth_first(G: &MF)) {
3189 MachineOperand *CurrentKnownM0Val = nullptr;
3190 for (auto &MI : make_early_inc_range(Range&: *MBB)) {
3191 Changed |= tryFoldCndMask(MI);
3192
3193 // PeepholeOptimizer may have folded an inline immediate directly onto an
3194 // instruction operand without materializing it into a register first.
3195 // Such an instruction is never reached through a def->use edge in
3196 // foldInstOperand, so try to constant fold it here.
3197 if (tryConstantFoldOp(MI: &MI)) {
3198 Changed = true;
3199 continue;
3200 }
3201
3202 if (tryFoldRedundantAND(ChildMI&: MI)) {
3203 Changed = true;
3204 continue;
3205 }
3206
3207 if (tryFoldAndExec(MI)) {
3208 Changed = true;
3209 continue;
3210 }
3211
3212 if (MI.isRegSequence() && tryFoldRegSequence(MI)) {
3213 Changed = true;
3214 continue;
3215 }
3216
3217 if (MI.isPHI() && tryFoldPhiAGPR(PHI&: MI)) {
3218 Changed = true;
3219 continue;
3220 }
3221
3222 if (MI.mayLoad() && tryFoldLoad(MI)) {
3223 Changed = true;
3224 continue;
3225 }
3226
3227 if (TII->isFoldableCopy(MI)) {
3228 Changed |= tryFoldFoldableCopy(MI, CurrentKnownM0Val);
3229 continue;
3230 }
3231
3232 // Saw an unknown clobber of m0, so we no longer know what it is.
3233 if (CurrentKnownM0Val && MI.modifiesRegister(Reg: AMDGPU::M0, TRI))
3234 CurrentKnownM0Val = nullptr;
3235
3236 // TODO: Omod might be OK if there is NSZ only on the source
3237 // instruction, and not the omod multiply.
3238 if (IsIEEEMode || !MI.getFlag(Flag: MachineInstr::FmNsz) || !tryFoldOMod(MI))
3239 Changed |= tryFoldClamp(MI);
3240 }
3241
3242 Changed |= tryOptimizeAGPRPhis(MBB&: *MBB);
3243 }
3244
3245 return Changed;
3246}
3247
3248PreservedAnalyses
3249SIFoldOperandsPass::run(MachineFunction &MF,
3250 MachineFunctionAnalysisManager &MFAM) {
3251 MFPropsModifier _(*this, MF);
3252
3253 const MachineLoopInfo *MLI = &MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
3254 bool Changed = SIFoldOperandsImpl().run(MF, MLI);
3255 if (!Changed) {
3256 return PreservedAnalyses::all();
3257 }
3258 auto PA = getMachineFunctionPassPreservedAnalyses();
3259 PA.preserveSet<CFGAnalyses>();
3260 PA.preserve<MachineLoopAnalysis>();
3261 return PA;
3262}
3263