1//===-- SIFoldOperands.cpp - Fold operands --- ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7/// \file
8//===----------------------------------------------------------------------===//
9//
10
11#include "SIFoldOperands.h"
12#include "AMDGPU.h"
13#include "GCNSubtarget.h"
14#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
15#include "SIInstrInfo.h"
16#include "SIMachineFunctionInfo.h"
17#include "SIRegisterInfo.h"
18#include "llvm/ADT/DepthFirstIterator.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineFunctionPass.h"
21#include "llvm/CodeGen/MachineLoopInfo.h"
22#include "llvm/CodeGen/MachineOperand.h"
23#include "llvm/CodeGen/RegisterClassInfo.h"
24#include "llvm/InitializePasses.h"
25
26#define DEBUG_TYPE "si-fold-operands"
27using namespace llvm;
28
29namespace {
30
31/// Track a value we may want to fold into downstream users, applying
32/// subregister extracts along the way.
33struct FoldableDef {
34 union {
35 MachineOperand *OpToFold = nullptr;
36 uint64_t ImmToFold;
37 int FrameIndexToFold;
38 };
39
40 /// Register class of the originally defined value.
41 const TargetRegisterClass *DefRC = nullptr;
42
43 /// Track the original defining instruction for the value.
44 const MachineInstr *DefMI = nullptr;
45
46 /// Subregister to apply to the value at the use point.
47 unsigned DefSubReg = AMDGPU::NoSubRegister;
48
49 /// Kind of value stored in the union.
50 MachineOperand::MachineOperandType Kind;
51
52 FoldableDef() = delete;
53 FoldableDef(MachineOperand &FoldOp, const TargetRegisterClass *DefRC,
54 unsigned DefSubReg = AMDGPU::NoSubRegister)
55 : DefRC(DefRC), DefSubReg(DefSubReg), Kind(FoldOp.getType()) {
56
57 if (FoldOp.isImm()) {
58 ImmToFold = FoldOp.getImm();
59 } else if (FoldOp.isFI()) {
60 FrameIndexToFold = FoldOp.getIndex();
61 } else {
62 assert(FoldOp.isReg() || FoldOp.isGlobal());
63 OpToFold = &FoldOp;
64 }
65
66 DefMI = FoldOp.getParent();
67 }
68
69 FoldableDef(int64_t FoldImm, const TargetRegisterClass *DefRC,
70 unsigned DefSubReg = AMDGPU::NoSubRegister)
71 : ImmToFold(FoldImm), DefRC(DefRC), DefSubReg(DefSubReg),
72 Kind(MachineOperand::MO_Immediate) {}
73
74 /// Copy the current def and apply \p SubReg to the value.
75 FoldableDef getWithSubReg(const SIRegisterInfo &TRI, unsigned SubReg) const {
76 FoldableDef Copy(*this);
77 Copy.DefSubReg = TRI.composeSubRegIndices(a: DefSubReg, b: SubReg);
78 return Copy;
79 }
80
81 bool isReg() const { return Kind == MachineOperand::MO_Register; }
82
83 Register getReg() const {
84 assert(isReg());
85 return OpToFold->getReg();
86 }
87
88 unsigned getSubReg() const {
89 assert(isReg());
90 return OpToFold->getSubReg();
91 }
92
93 bool isImm() const { return Kind == MachineOperand::MO_Immediate; }
94
95 bool isFI() const {
96 return Kind == MachineOperand::MO_FrameIndex;
97 }
98
99 int getFI() const {
100 assert(isFI());
101 return FrameIndexToFold;
102 }
103
104 bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; }
105
106 /// Return the effective immediate value defined by this instruction, after
107 /// application of any subregister extracts which may exist between the use
108 /// and def instruction.
109 std::optional<int64_t> getEffectiveImmVal() const {
110 assert(isImm());
111 return SIInstrInfo::extractSubregFromImm(ImmVal: ImmToFold, SubRegIndex: DefSubReg);
112 }
113
114 /// Check if it is legal to fold this effective value into \p MI's \p OpNo
115 /// operand.
116 bool isOperandLegal(const SIInstrInfo &TII, const MachineInstr &MI,
117 unsigned OpIdx) const {
118 switch (Kind) {
119 case MachineOperand::MO_Immediate: {
120 std::optional<int64_t> ImmToFold = getEffectiveImmVal();
121 if (!ImmToFold)
122 return false;
123
124 // TODO: Should verify the subregister index is supported by the class
125 // TODO: Avoid the temporary MachineOperand
126 MachineOperand TmpOp = MachineOperand::CreateImm(Val: *ImmToFold);
127 return TII.isOperandLegal(MI, OpIdx, MO: &TmpOp);
128 }
129 case MachineOperand::MO_FrameIndex: {
130 if (DefSubReg != AMDGPU::NoSubRegister)
131 return false;
132 MachineOperand TmpOp = MachineOperand::CreateFI(Idx: FrameIndexToFold);
133 return TII.isOperandLegal(MI, OpIdx, MO: &TmpOp);
134 }
135 default:
136 // TODO: Try to apply DefSubReg, for global address we can extract
137 // low/high.
138 if (DefSubReg != AMDGPU::NoSubRegister)
139 return false;
140 return TII.isOperandLegal(MI, OpIdx, MO: OpToFold);
141 }
142
143 llvm_unreachable("covered MachineOperand kind switch");
144 }
145};
146
147struct FoldCandidate {
148 MachineInstr *UseMI;
149 FoldableDef Def;
150 int ShrinkOpcode;
151 unsigned UseOpNo;
152 bool Commuted;
153
154 FoldCandidate(MachineInstr *MI, unsigned OpNo, FoldableDef Def,
155 bool Commuted = false, int ShrinkOp = -1)
156 : UseMI(MI), Def(Def), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo),
157 Commuted(Commuted) {}
158
159 bool isFI() const { return Def.isFI(); }
160
161 int getFI() const {
162 assert(isFI());
163 return Def.FrameIndexToFold;
164 }
165
166 bool isImm() const { return Def.isImm(); }
167
168 bool isReg() const { return Def.isReg(); }
169
170 Register getReg() const { return Def.getReg(); }
171
172 bool isGlobal() const { return Def.isGlobal(); }
173
174 bool needsShrink() const { return ShrinkOpcode != -1; }
175};
176
177class SIFoldOperandsImpl {
178public:
179 MachineFunction *MF;
180 MachineRegisterInfo *MRI;
181 const SIInstrInfo *TII;
182 const SIRegisterInfo *TRI;
183 const GCNSubtarget *ST;
184 const SIMachineFunctionInfo *MFI;
185 const MachineLoopInfo *MLI;
186
187 bool frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
188 const FoldableDef &OpToFold) const;
189
190 // TODO: Just use TII::getVALUOp
191 unsigned convertToVALUOp(unsigned Opc, bool UseVOP3 = false) const {
192 switch (Opc) {
193 case AMDGPU::S_ADD_I32: {
194 if (ST->hasAddNoCarryInsts())
195 return UseVOP3 ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_U32_e32;
196 return UseVOP3 ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
197 }
198 case AMDGPU::S_OR_B32:
199 return UseVOP3 ? AMDGPU::V_OR_B32_e64 : AMDGPU::V_OR_B32_e32;
200 case AMDGPU::S_AND_B32:
201 return UseVOP3 ? AMDGPU::V_AND_B32_e64 : AMDGPU::V_AND_B32_e32;
202 case AMDGPU::S_MUL_I32:
203 return AMDGPU::V_MUL_LO_U32_e64;
204 default:
205 return AMDGPU::INSTRUCTION_LIST_END;
206 }
207 }
208
209 bool foldCopyToVGPROfScalarAddOfFrameIndex(Register DstReg, Register SrcReg,
210 MachineInstr &MI) const;
211
212 bool updateOperand(FoldCandidate &Fold) const;
213
214 bool canUseImmWithOpSel(const MachineInstr *MI, unsigned UseOpNo,
215 int64_t ImmVal) const;
216
217 /// Try to fold immediate \p ImmVal into \p MI's operand at index \p UseOpNo.
218 bool tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
219 int64_t ImmVal) const;
220
221 bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
222 MachineInstr *MI, unsigned OpNo,
223 const FoldableDef &OpToFold) const;
224 bool isUseSafeToFold(const MachineInstr &MI,
225 const MachineOperand &UseMO) const;
226 bool isTemporallyDivergentUse(const FoldableDef &OpToFold,
227 const MachineInstr &UseMI) const;
228
229 const TargetRegisterClass *getRegSeqInit(
230 MachineInstr &RegSeq,
231 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs) const;
232
233 const TargetRegisterClass *
234 getRegSeqInit(SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
235 Register UseReg) const;
236
237 std::pair<int64_t, const TargetRegisterClass *>
238 isRegSeqSplat(MachineInstr &RegSeg) const;
239
240 bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
241 int64_t SplatVal,
242 const TargetRegisterClass *SplatRC) const;
243
244 bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
245 unsigned UseOpIdx,
246 SmallVectorImpl<FoldCandidate> &FoldList) const;
247 bool foldOperand(FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
248 SmallVectorImpl<FoldCandidate> &FoldList,
249 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
250
251 struct ANDMaskResult {
252 int64_t Mask;
253 Register Reg;
254 unsigned RegIdx;
255 };
256
257 std::optional<ANDMaskResult> getANDMaskRegOperand(MachineInstr &AndMI) const;
258
259 bool tryConstantFoldOp(MachineInstr *MI) const;
260 bool tryFoldCndMask(MachineInstr &MI) const;
261 bool tryFoldRedundantAND(MachineInstr &ChildMI) const;
262 bool foldInstOperand(MachineInstr &MI, const FoldableDef &OpToFold) const;
263
264 bool foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const;
265 bool tryFoldFoldableCopy(MachineInstr &MI,
266 MachineOperand *&CurrentKnownM0Val) const;
267
268 const MachineOperand *isClamp(const MachineInstr &MI) const;
269 bool tryFoldClamp(MachineInstr &MI);
270
271 std::pair<const MachineOperand *, int> isOMod(const MachineInstr &MI) const;
272 bool tryFoldOMod(MachineInstr &MI);
273 bool tryFoldSGPRSplatRegSequence(MachineInstr &MI);
274 bool tryFoldRegSequence(MachineInstr &MI);
275 bool tryFoldPhiAGPR(MachineInstr &MI);
276 bool tryFoldLoad(MachineInstr &MI);
277
278 bool tryOptimizeAGPRPhis(MachineBasicBlock &MBB);
279
280public:
281 SIFoldOperandsImpl() = default;
282
283 bool run(MachineFunction &MF, const MachineLoopInfo *MLI);
284};
285
286class SIFoldOperandsLegacy : public MachineFunctionPass {
287public:
288 static char ID;
289
290 SIFoldOperandsLegacy() : MachineFunctionPass(ID) {}
291
292 bool runOnMachineFunction(MachineFunction &MF) override {
293 if (skipFunction(F: MF.getFunction()))
294 return false;
295 const MachineLoopInfo *MLI =
296 &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
297 return SIFoldOperandsImpl().run(MF, MLI);
298 }
299
300 StringRef getPassName() const override { return "SI Fold Operands"; }
301
302 void getAnalysisUsage(AnalysisUsage &AU) const override {
303 AU.setPreservesCFG();
304 AU.addRequired<MachineLoopInfoWrapperPass>();
305 AU.addPreserved<MachineLoopInfoWrapperPass>();
306 MachineFunctionPass::getAnalysisUsage(AU);
307 }
308
309 MachineFunctionProperties getRequiredProperties() const override {
310 return MachineFunctionProperties().setIsSSA();
311 }
312};
313
314} // End anonymous namespace.
315
316INITIALIZE_PASS_BEGIN(SIFoldOperandsLegacy, DEBUG_TYPE, "SI Fold Operands",
317 false, false)
318INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
319INITIALIZE_PASS_END(SIFoldOperandsLegacy, DEBUG_TYPE, "SI Fold Operands", false,
320 false)
321
322char SIFoldOperandsLegacy::ID = 0;
323
324char &llvm::SIFoldOperandsLegacyID = SIFoldOperandsLegacy::ID;
325
326static const TargetRegisterClass *getRegOpRC(const MachineRegisterInfo &MRI,
327 const TargetRegisterInfo &TRI,
328 const MachineOperand &MO) {
329 const TargetRegisterClass *RC = MRI.getRegClass(Reg: MO.getReg());
330 if (const TargetRegisterClass *SubRC =
331 TRI.getSubRegisterClass(SuperRC: RC, SubRegIdx: MO.getSubReg()))
332 RC = SubRC;
333 return RC;
334}
335
336// Map multiply-accumulate opcode to corresponding multiply-add opcode if any.
337static unsigned macToMad(unsigned Opc) {
338 switch (Opc) {
339 case AMDGPU::V_MAC_F32_e64:
340 return AMDGPU::V_MAD_F32_e64;
341 case AMDGPU::V_MAC_F16_e64:
342 return AMDGPU::V_MAD_F16_e64;
343 case AMDGPU::V_FMAC_F32_e64:
344 return AMDGPU::V_FMA_F32_e64;
345 case AMDGPU::V_FMAC_F16_e64:
346 return AMDGPU::V_FMA_F16_gfx9_e64;
347 case AMDGPU::V_FMAC_F16_t16_e64:
348 return AMDGPU::V_FMA_F16_gfx9_t16_e64;
349 case AMDGPU::V_FMAC_F16_fake16_e64:
350 return AMDGPU::V_FMA_F16_gfx9_fake16_e64;
351 case AMDGPU::V_FMAC_LEGACY_F32_e64:
352 return AMDGPU::V_FMA_LEGACY_F32_e64;
353 case AMDGPU::V_FMAC_F64_e64:
354 return AMDGPU::V_FMA_F64_e64;
355 }
356 return AMDGPU::INSTRUCTION_LIST_END;
357}
358
359// TODO: Add heuristic that the frame index might not fit in the addressing mode
360// immediate offset to avoid materializing in loops.
361bool SIFoldOperandsImpl::frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
362 const FoldableDef &OpToFold) const {
363 if (!OpToFold.isFI())
364 return false;
365
366 const unsigned Opc = UseMI.getOpcode();
367 switch (Opc) {
368 case AMDGPU::S_ADD_I32:
369 case AMDGPU::S_ADD_U32:
370 case AMDGPU::V_ADD_U32_e32:
371 case AMDGPU::V_ADD_CO_U32_e32:
372 // TODO: Possibly relax hasOneUse. It matters more for mubuf, since we have
373 // to insert the wave size shift at every point we use the index.
374 // TODO: Fix depending on visit order to fold immediates into the operand
375 return UseMI.getOperand(i: OpNo == 1 ? 2 : 1).isImm() &&
376 MRI->hasOneNonDBGUse(RegNo: UseMI.getOperand(i: OpNo).getReg());
377 case AMDGPU::V_ADD_U32_e64:
378 case AMDGPU::V_ADD_CO_U32_e64:
379 return UseMI.getOperand(i: OpNo == 2 ? 3 : 2).isImm() &&
380 MRI->hasOneNonDBGUse(RegNo: UseMI.getOperand(i: OpNo).getReg());
381 default:
382 break;
383 }
384
385 if (TII->isMUBUF(MI: UseMI))
386 return OpNo == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
387 if (!TII->isFLATScratch(MI: UseMI))
388 return false;
389
390 int SIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
391 if (OpNo == SIdx)
392 return true;
393
394 int VIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
395 return OpNo == VIdx && SIdx == -1;
396}
397
398/// Fold %vgpr = COPY (S_ADD_I32 x, frameindex)
399///
400/// => %vgpr = V_ADD_U32 x, frameindex
401bool SIFoldOperandsImpl::foldCopyToVGPROfScalarAddOfFrameIndex(
402 Register DstReg, Register SrcReg, MachineInstr &MI) const {
403 if (!SrcReg.isVirtual())
404 return false;
405
406 if (TRI->isVGPR(MRI: *MRI, Reg: DstReg) && TRI->isSGPRReg(MRI: *MRI, Reg: SrcReg) &&
407 MRI->hasOneNonDBGUse(RegNo: SrcReg)) {
408 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
409 if (!Def || Def->getNumOperands() != 4)
410 return false;
411
412 MachineOperand *Src0 = &Def->getOperand(i: 1);
413 MachineOperand *Src1 = &Def->getOperand(i: 2);
414
415 // TODO: This is profitable with more operand types, and for more
416 // opcodes. But ultimately this is working around poor / nonexistent
417 // regbankselect.
418 if (!Src0->isFI() && !Src1->isFI())
419 return false;
420
421 if (Src0->isFI())
422 std::swap(a&: Src0, b&: Src1);
423
424 const bool UseVOP3 = !Src0->isImm() || TII->isInlineConstant(MO: *Src0);
425 unsigned NewOp = convertToVALUOp(Opc: Def->getOpcode(), UseVOP3);
426 if (NewOp == AMDGPU::INSTRUCTION_LIST_END ||
427 !Def->getOperand(i: 3).isDead()) // Check if scc is dead
428 return false;
429
430 MachineBasicBlock *MBB = Def->getParent();
431 const DebugLoc &DL = Def->getDebugLoc();
432 if (NewOp != AMDGPU::V_ADD_CO_U32_e32) {
433 MachineInstrBuilder Add =
434 BuildMI(BB&: *MBB, I&: *Def, MIMD: DL, MCID: TII->get(Opcode: NewOp), DestReg: DstReg);
435
436 if (Add->getDesc().getNumDefs() == 2) {
437 Register CarryOutReg = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
438 Add.addDef(RegNo: CarryOutReg, Flags: RegState::Dead);
439 MRI->setRegAllocationHint(VReg: CarryOutReg, Type: 0, PrefReg: TRI->getVCC());
440 }
441
442 Add.add(MO: *Src0).add(MO: *Src1).setMIFlags(Def->getFlags());
443 if (AMDGPU::hasNamedOperand(Opcode: NewOp, NamedIdx: AMDGPU::OpName::clamp))
444 Add.addImm(Val: 0);
445
446 Def->eraseFromParent();
447 MI.eraseFromParent();
448 return true;
449 }
450
451 assert(NewOp == AMDGPU::V_ADD_CO_U32_e32);
452
453 MachineBasicBlock::LivenessQueryResult Liveness =
454 MBB->computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: *Def, Neighborhood: 16);
455 if (Liveness == MachineBasicBlock::LQR_Dead) {
456 // TODO: If src1 satisfies operand constraints, use vop3 version.
457 BuildMI(BB&: *MBB, I&: *Def, MIMD: DL, MCID: TII->get(Opcode: NewOp), DestReg: DstReg)
458 .add(MO: *Src0)
459 .add(MO: *Src1)
460 .setOperandDead(3) // implicit-def $vcc
461 .setMIFlags(Def->getFlags());
462 Def->eraseFromParent();
463 MI.eraseFromParent();
464 return true;
465 }
466 }
467
468 return false;
469}
470
471FunctionPass *llvm::createSIFoldOperandsLegacyPass() {
472 return new SIFoldOperandsLegacy();
473}
474
475bool SIFoldOperandsImpl::canUseImmWithOpSel(const MachineInstr *MI,
476 unsigned UseOpNo,
477 int64_t ImmVal) const {
478 if (!SIInstrFlags::isPacked(O: *MI) || SIInstrFlags::isMAI(O: *MI) ||
479 SIInstrFlags::isWMMA(O: *MI) || SIInstrFlags::isSWMMAC(O: *MI) ||
480 (ST->hasDOTOpSelHazard() && SIInstrFlags::isDOT(O: *MI)))
481 return false;
482
483 const MachineOperand &Old = MI->getOperand(i: UseOpNo);
484 int OpNo = MI->getOperandNo(I: &Old);
485
486 unsigned Opcode = MI->getOpcode();
487 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
488 switch (OpType) {
489 default:
490 return false;
491 case AMDGPU::OPERAND_REG_IMM_V2FP16:
492 case AMDGPU::OPERAND_REG_IMM_V2BF16:
493 case AMDGPU::OPERAND_REG_IMM_V2INT16:
494 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
495 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
496 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
497 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
498 // VOP3 packed instructions ignore op_sel source modifiers, we cannot encode
499 // two different constants.
500 if (SIInstrFlags::isVOP3(O: *MI) && !SIInstrFlags::isVOP3P(O: *MI) &&
501 static_cast<uint16_t>(ImmVal) != static_cast<uint16_t>(ImmVal >> 16))
502 return false;
503 break;
504 }
505
506 return true;
507}
508
509bool SIFoldOperandsImpl::tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
510 int64_t ImmVal) const {
511 MachineOperand &Old = MI->getOperand(i: UseOpNo);
512 unsigned Opcode = MI->getOpcode();
513 int OpNo = MI->getOperandNo(I: &Old);
514 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
515
516 // If the literal can be inlined as-is, apply it and short-circuit the
517 // tests below. The main motivation for this is to avoid unintuitive
518 // uses of opsel.
519 if (AMDGPU::isInlinableLiteralV216(Literal: ImmVal, OpType)) {
520 Old.ChangeToImmediate(ImmVal);
521 return true;
522 }
523
524 // Refer to op_sel/op_sel_hi and check if we can change the immediate and
525 // op_sel in a way that allows an inline constant.
526 AMDGPU::OpName ModName = AMDGPU::OpName::NUM_OPERAND_NAMES;
527 unsigned SrcIdx = ~0;
528 if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0)) {
529 ModName = AMDGPU::OpName::src0_modifiers;
530 SrcIdx = 0;
531 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1)) {
532 ModName = AMDGPU::OpName::src1_modifiers;
533 SrcIdx = 1;
534 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src2)) {
535 ModName = AMDGPU::OpName::src2_modifiers;
536 SrcIdx = 2;
537 }
538 assert(ModName != AMDGPU::OpName::NUM_OPERAND_NAMES);
539 int ModIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: ModName);
540 MachineOperand &Mod = MI->getOperand(i: ModIdx);
541 unsigned ModVal = Mod.getImm();
542
543 uint16_t ImmLo =
544 static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_0 ? 16 : 0));
545 uint16_t ImmHi =
546 static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_1 ? 16 : 0));
547 uint32_t Imm = (static_cast<uint32_t>(ImmHi) << 16) | ImmLo;
548 unsigned NewModVal = ModVal & ~(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
549
550 // Helper function that attempts to inline the given value with a newly
551 // chosen opsel pattern.
552 auto tryFoldToInline = [&](uint32_t Imm) -> bool {
553 if (AMDGPU::isInlinableLiteralV216(Literal: Imm, OpType)) {
554 Mod.setImm(NewModVal | SISrcMods::OP_SEL_1);
555 Old.ChangeToImmediate(ImmVal: Imm);
556 return true;
557 }
558
559 // Try to shuffle the halves around and leverage opsel to get an inline
560 // constant.
561 uint16_t Lo = static_cast<uint16_t>(Imm);
562 uint16_t Hi = static_cast<uint16_t>(Imm >> 16);
563 if (Lo == Hi) {
564 if (AMDGPU::isInlinableLiteralV216(Literal: Lo, OpType)) {
565 // If the target has feature 'BF16InlineConstFromUpperFP32', packed BF16
566 // instructions using inline constant must use OPSEL to select the upper
567 // 16-bits from FP32.
568 if (ST->hasBF16InlineConstFromUpperFP32() &&
569 (OpType == AMDGPU::OPERAND_REG_INLINE_C_V2BF16 ||
570 OpType == AMDGPU::OPERAND_REG_IMM_V2BF16))
571 NewModVal |= (SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
572 Mod.setImm(NewModVal);
573 Old.ChangeToImmediate(ImmVal: Lo);
574 return true;
575 }
576
577 if (static_cast<int16_t>(Lo) < 0) {
578 int32_t SExt = static_cast<int16_t>(Lo);
579 if (AMDGPU::isInlinableLiteralV216(Literal: SExt, OpType)) {
580 Mod.setImm(NewModVal);
581 Old.ChangeToImmediate(ImmVal: SExt);
582 return true;
583 }
584 }
585
586 // This check is only useful for integer instructions
587 if (OpType == AMDGPU::OPERAND_REG_IMM_V2INT16) {
588 if (AMDGPU::isInlinableLiteralV216(Literal: Lo << 16, OpType)) {
589 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
590 Old.ChangeToImmediate(ImmVal: static_cast<uint32_t>(Lo) << 16);
591 return true;
592 }
593 }
594 } else {
595 uint32_t Swapped = (static_cast<uint32_t>(Lo) << 16) | Hi;
596 if (AMDGPU::isInlinableLiteralV216(Literal: Swapped, OpType)) {
597 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0);
598 Old.ChangeToImmediate(ImmVal: Swapped);
599 return true;
600 }
601 }
602
603 return false;
604 };
605
606 if (tryFoldToInline(Imm))
607 return true;
608
609 // Replace integer addition by subtraction and vice versa if it allows
610 // folding the immediate to an inline constant.
611 //
612 // We should only ever get here for SrcIdx == 1 due to canonicalization
613 // earlier in the pipeline, but we double-check here to be safe / fully
614 // general.
615 bool IsUAdd = Opcode == AMDGPU::V_PK_ADD_U16;
616 bool IsUSub = Opcode == AMDGPU::V_PK_SUB_U16;
617 if (SrcIdx == 1 && (IsUAdd || IsUSub)) {
618 unsigned ClampIdx =
619 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::clamp);
620 bool Clamp = MI->getOperand(i: ClampIdx).getImm() != 0;
621
622 if (!Clamp) {
623 uint16_t NegLo = -static_cast<uint16_t>(Imm);
624 uint16_t NegHi = -static_cast<uint16_t>(Imm >> 16);
625 uint32_t NegImm = (static_cast<uint32_t>(NegHi) << 16) | NegLo;
626
627 if (tryFoldToInline(NegImm)) {
628 unsigned NegOpcode =
629 IsUAdd ? AMDGPU::V_PK_SUB_U16 : AMDGPU::V_PK_ADD_U16;
630 MI->setDesc(TII->get(Opcode: NegOpcode));
631 return true;
632 }
633 }
634 }
635
636 return false;
637}
638
639bool SIFoldOperandsImpl::updateOperand(FoldCandidate &Fold) const {
640 MachineInstr *MI = Fold.UseMI;
641 MachineOperand &Old = MI->getOperand(i: Fold.UseOpNo);
642 assert(Old.isReg());
643
644 std::optional<int64_t> ImmVal;
645 if (Fold.isImm())
646 ImmVal = Fold.Def.getEffectiveImmVal();
647
648 if (ImmVal && canUseImmWithOpSel(MI: Fold.UseMI, UseOpNo: Fold.UseOpNo, ImmVal: *ImmVal)) {
649 if (tryFoldImmWithOpSel(MI: Fold.UseMI, UseOpNo: Fold.UseOpNo, ImmVal: *ImmVal))
650 return true;
651
652 // We can't represent the candidate as an inline constant. Try as a literal
653 // with the original opsel, checking constant bus limitations.
654 MachineOperand New = MachineOperand::CreateImm(Val: *ImmVal);
655 int OpNo = MI->getOperandNo(I: &Old);
656 if (!TII->isOperandLegal(MI: *MI, OpIdx: OpNo, MO: &New))
657 return false;
658 Old.ChangeToImmediate(ImmVal: *ImmVal);
659 return true;
660 }
661
662 if ((Fold.isImm() || Fold.isFI() || Fold.isGlobal()) && Fold.needsShrink()) {
663 MachineBasicBlock *MBB = MI->getParent();
664 auto Liveness = MBB->computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: MI, Neighborhood: 16);
665 if (Liveness != MachineBasicBlock::LQR_Dead) {
666 LLVM_DEBUG(dbgs() << "Not shrinking due to live vcc: " << *MI);
667 return false;
668 }
669
670 int Op32 = Fold.ShrinkOpcode;
671 MachineOperand &Dst0 = MI->getOperand(i: 0);
672 MachineOperand &Dst1 = MI->getOperand(i: 1);
673 assert(Dst0.isDef() && Dst1.isDef());
674
675 bool HaveNonDbgCarryUse = !MRI->use_nodbg_empty(RegNo: Dst1.getReg());
676
677 const TargetRegisterClass *Dst0RC = MRI->getRegClass(Reg: Dst0.getReg());
678 Register NewReg0 = MRI->createVirtualRegister(RegClass: Dst0RC);
679
680 MachineInstr *Inst32 = TII->buildShrunkInst(MI&: *MI, NewOpcode: Op32);
681
682 if (HaveNonDbgCarryUse) {
683 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
684 DestReg: Dst1.getReg())
685 .addReg(RegNo: AMDGPU::VCC, Flags: RegState::Kill);
686 }
687
688 // Keep the old instruction around to avoid breaking iterators, but
689 // replace it with a dummy instruction to remove uses.
690 //
691 // FIXME: We should not invert how this pass looks at operands to avoid
692 // this. Should track set of foldable movs instead of looking for uses
693 // when looking at a use.
694 Dst0.setReg(NewReg0);
695 for (unsigned I = MI->getNumOperands() - 1; I > 0; --I)
696 MI->removeOperand(OpNo: I);
697 MI->setDesc(TII->get(Opcode: AMDGPU::IMPLICIT_DEF));
698
699 if (Fold.Commuted)
700 TII->commuteInstruction(MI&: *Inst32, NewMI: false);
701 return true;
702 }
703
704 assert(!Fold.needsShrink() && "not handled");
705
706 if (ImmVal) {
707 if (Old.isTied()) {
708 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: MI->getOpcode());
709 if (NewMFMAOpc == -1)
710 return false;
711 MI->setDesc(TII->get(Opcode: NewMFMAOpc));
712 MI->untieRegOperand(OpIdx: 0);
713 const MCInstrDesc &MCID = MI->getDesc();
714 for (unsigned I = 0; I < MI->getNumDefs(); ++I)
715 if (MCID.getOperandConstraint(OpNum: I, Constraint: MCOI::EARLY_CLOBBER) != -1)
716 MI->getOperand(i: I).setIsEarlyClobber(true);
717 }
718
719 // TODO: Should we try to avoid adding this to the candidate list?
720 MachineOperand New = MachineOperand::CreateImm(Val: *ImmVal);
721 int OpNo = MI->getOperandNo(I: &Old);
722 if (!TII->isOperandLegal(MI: *MI, OpIdx: OpNo, MO: &New))
723 return false;
724
725 if (ST->hasBF16InlineConstFromUpperFP32() &&
726 OpNo ==
727 AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(), Name: AMDGPU::OpName::src0)) {
728 unsigned Opcode = MI->getOpcode();
729 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
730 if ((OpType == AMDGPU::OPERAND_REG_IMM_BF16 ||
731 OpType == AMDGPU::OPERAND_REG_INLINE_C_BF16) &&
732 TII->isInlineConstant(ImmVal: *ImmVal, OperandType: OpType)) {
733 // We can fold it, but we need to set OPSEL
734 int Mod0 =
735 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0_modifiers);
736 if (Mod0 == -1)
737 return false;
738 MachineOperand &ModOp = MI->getOperand(i: Mod0);
739 if (ModOp.getImm())
740 return false;
741 ModOp.setImm(SISrcMods::OP_SEL_0);
742 }
743 }
744
745 Old.ChangeToImmediate(ImmVal: *ImmVal);
746 return true;
747 }
748
749 if (Fold.isGlobal()) {
750 Old.ChangeToGA(GV: Fold.Def.OpToFold->getGlobal(),
751 Offset: Fold.Def.OpToFold->getOffset(),
752 TargetFlags: Fold.Def.OpToFold->getTargetFlags());
753 return true;
754 }
755
756 if (Fold.isFI()) {
757 Old.ChangeToFrameIndex(Idx: Fold.getFI());
758 return true;
759 }
760
761 MachineOperand *New = Fold.Def.OpToFold;
762
763 // Verify the register is compatible with the operand.
764 if (const TargetRegisterClass *OpRC =
765 TII->getRegClass(MCID: MI->getDesc(), OpNum: Fold.UseOpNo)) {
766 const TargetRegisterClass *NewRC =
767 TRI->getRegClassForReg(MRI: *MRI, Reg: New->getReg());
768
769 const TargetRegisterClass *ConstrainRC = OpRC;
770 if (New->getSubReg()) {
771 ConstrainRC =
772 TRI->getMatchingSuperRegClass(A: NewRC, B: OpRC, Idx: New->getSubReg());
773
774 if (!ConstrainRC)
775 return false;
776 }
777
778 if (New->getReg().isVirtual() &&
779 !MRI->constrainRegClass(Reg: New->getReg(), RC: ConstrainRC)) {
780 LLVM_DEBUG(dbgs() << "Cannot constrain " << printReg(New->getReg(), TRI)
781 << TRI->getRegClassName(ConstrainRC) << '\n');
782 return false;
783 }
784 }
785
786 // Rework once the VS_16 register class is updated to include proper
787 // 16-bit SGPRs instead of 32-bit ones.
788 if (Old.getSubReg() == AMDGPU::lo16 && TRI->isSGPRReg(MRI: *MRI, Reg: New->getReg()))
789 Old.setSubReg(AMDGPU::NoSubRegister);
790 if (New->getReg().isPhysical()) {
791 Old.substPhysReg(Reg: New->getReg(), *TRI);
792 } else {
793 Register OldReg = Old.getReg();
794 Old.substVirtReg(Reg: New->getReg(), SubIdx: New->getSubReg(), *TRI);
795 Old.setIsUndef(New->isUndef());
796
797 // If MI is in a BUNDLE, also update header's matching implicit use.
798 if (MI->isBundledWithPred()) {
799 MachineInstr &Header = *getBundleStart(I: MI->getIterator());
800 for (MachineOperand &MO : Header.operands()) {
801 if (MO.getReg() == OldReg) {
802 MO.setReg(New->getReg());
803 MO.setSubReg(New->getSubReg());
804 }
805 }
806 }
807 }
808 return true;
809}
810
811static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
812 FoldCandidate &&Entry) {
813 // Skip additional folding on the same operand.
814 for (FoldCandidate &Fold : FoldList)
815 if (Fold.UseMI == Entry.UseMI && Fold.UseOpNo == Entry.UseOpNo)
816 return;
817 LLVM_DEBUG(dbgs() << "Append " << (Entry.Commuted ? "commuted" : "normal")
818 << " operand " << Entry.UseOpNo << "\n " << *Entry.UseMI);
819 FoldList.push_back(Elt: Entry);
820}
821
822static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
823 MachineInstr *MI, unsigned OpNo,
824 const FoldableDef &FoldOp,
825 bool Commuted = false, int ShrinkOp = -1) {
826 appendFoldCandidate(FoldList,
827 Entry: FoldCandidate(MI, OpNo, FoldOp, Commuted, ShrinkOp));
828}
829
830// Returns true if the instruction is a packed F32 instruction and the
831// corresponding scalar operand reads 32 bits and replicates the bits to both
832// channels.
833static bool isPKF32InstrReplicatesLower32BitsOfScalarOperand(
834 const GCNSubtarget *ST, MachineInstr *MI, unsigned OpNo) {
835 if (!ST->hasPKF32InstsReplicatingLower32BitsOfScalarInput())
836 return false;
837 const MCOperandInfo &OpDesc = MI->getDesc().operands()[OpNo];
838 return OpDesc.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32;
839}
840
841// Packed FP32 instructions only read 32 bits from a scalar operand (SGPR or
842// literal) and replicates the bits to both channels. Therefore, if the hi and
843// lo are not same, we can't fold it.
844static bool checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(
845 const FoldableDef &OpToFold) {
846 assert(OpToFold.isImm() && "Expected immediate operand");
847 uint64_t ImmVal = OpToFold.getEffectiveImmVal().value();
848 uint32_t Lo = Lo_32(Value: ImmVal);
849 uint32_t Hi = Hi_32(Value: ImmVal);
850 return Lo == Hi;
851}
852
853bool SIFoldOperandsImpl::tryAddToFoldList(
854 SmallVectorImpl<FoldCandidate> &FoldList, MachineInstr *MI, unsigned OpNo,
855 const FoldableDef &OpToFold) const {
856 const unsigned Opc = MI->getOpcode();
857
858 auto tryToFoldAsFMAAKorMK = [&]() {
859 if (!OpToFold.isImm())
860 return false;
861
862 const bool TryAK = OpNo == 3;
863 const unsigned NewOpc = TryAK ? AMDGPU::S_FMAAK_F32 : AMDGPU::S_FMAMK_F32;
864 MI->setDesc(TII->get(Opcode: NewOpc));
865
866 // We have to fold into operand which would be Imm not into OpNo.
867 bool FoldAsFMAAKorMK =
868 tryAddToFoldList(FoldList, MI, OpNo: TryAK ? 3 : 2, OpToFold);
869 if (FoldAsFMAAKorMK) {
870 // Untie Src2 of fmac.
871 MI->untieRegOperand(OpIdx: 3);
872 // For fmamk swap operands 1 and 2 if OpToFold was meant for operand 1.
873 if (OpNo == 1) {
874 MachineOperand &Op1 = MI->getOperand(i: 1);
875 MachineOperand &Op2 = MI->getOperand(i: 2);
876 Register OldReg = Op1.getReg();
877 // Operand 2 might be an inlinable constant
878 if (Op2.isImm()) {
879 Op1.ChangeToImmediate(ImmVal: Op2.getImm());
880 Op2.ChangeToRegister(Reg: OldReg, isDef: false);
881 } else {
882 Op1.setReg(Op2.getReg());
883 Op2.setReg(OldReg);
884 }
885 }
886 return true;
887 }
888 MI->setDesc(TII->get(Opcode: Opc));
889 return false;
890 };
891
892 bool IsLegal = OpToFold.isOperandLegal(TII: *TII, MI: *MI, OpIdx: OpNo);
893 if (!IsLegal && OpToFold.isImm()) {
894 if (std::optional<int64_t> ImmVal = OpToFold.getEffectiveImmVal())
895 IsLegal = canUseImmWithOpSel(MI, UseOpNo: OpNo, ImmVal: *ImmVal);
896 }
897
898 if (!IsLegal) {
899 // Special case for v_mac_{f16, f32}_e64 if we are trying to fold into src2
900 unsigned NewOpc = macToMad(Opc);
901 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) {
902 // Check if changing this to a v_mad_{f16, f32} instruction will allow us
903 // to fold the operand.
904 MI->setDesc(TII->get(Opcode: NewOpc));
905 bool AddOpSel = !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::op_sel) &&
906 AMDGPU::hasNamedOperand(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::op_sel);
907 if (AddOpSel)
908 MI->addOperand(Op: MachineOperand::CreateImm(Val: 0));
909 bool FoldAsMAD = tryAddToFoldList(FoldList, MI, OpNo, OpToFold);
910 if (FoldAsMAD) {
911 MI->untieRegOperand(OpIdx: OpNo);
912 return true;
913 }
914 if (AddOpSel)
915 MI->removeOperand(OpNo: MI->getNumExplicitOperands() - 1);
916 MI->setDesc(TII->get(Opcode: Opc));
917 }
918
919 // Special case for s_fmac_f32 if we are trying to fold into Src2.
920 // By transforming into fmaak we can untie Src2 and make folding legal.
921 if (Opc == AMDGPU::S_FMAC_F32 && OpNo == 3) {
922 if (tryToFoldAsFMAAKorMK())
923 return true;
924 }
925
926 // Special case for s_setreg_b32
927 if (OpToFold.isImm()) {
928 unsigned ImmOpc = 0;
929 if (Opc == AMDGPU::S_SETREG_B32)
930 ImmOpc = AMDGPU::S_SETREG_IMM32_B32;
931 else if (Opc == AMDGPU::S_SETREG_B32_mode)
932 ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode;
933 if (ImmOpc) {
934 MI->setDesc(TII->get(Opcode: ImmOpc));
935 appendFoldCandidate(FoldList, MI, OpNo, FoldOp: OpToFold);
936 return true;
937 }
938 }
939
940 // Operand is not legal, so try to commute the instruction to
941 // see if this makes it possible to fold.
942 unsigned CommuteOpNo = TargetInstrInfo::CommuteAnyOperandIndex;
943 bool CanCommute = TII->findCommutedOpIndices(MI: *MI, SrcOpIdx0&: OpNo, SrcOpIdx1&: CommuteOpNo);
944 if (!CanCommute)
945 return false;
946
947 MachineOperand &Op = MI->getOperand(i: OpNo);
948 MachineOperand &CommutedOp = MI->getOperand(i: CommuteOpNo);
949
950 // One of operands might be an Imm operand, and OpNo may refer to it after
951 // the call of commuteInstruction() below. Such situations are avoided
952 // here explicitly as OpNo must be a register operand to be a candidate
953 // for memory folding.
954 if (!Op.isReg() || !CommutedOp.isReg())
955 return false;
956
957 // The same situation with an immediate could reproduce if both inputs are
958 // the same register.
959 if (Op.isReg() && CommutedOp.isReg() &&
960 (Op.getReg() == CommutedOp.getReg() &&
961 Op.getSubReg() == CommutedOp.getSubReg()))
962 return false;
963
964 if (!TII->commuteInstruction(MI&: *MI, NewMI: false, OpIdx1: OpNo, OpIdx2: CommuteOpNo))
965 return false;
966
967 int Op32 = -1;
968 if (!OpToFold.isOperandLegal(TII: *TII, MI: *MI, OpIdx: CommuteOpNo)) {
969 if ((Opc != AMDGPU::V_ADD_CO_U32_e64 && Opc != AMDGPU::V_SUB_CO_U32_e64 &&
970 Opc != AMDGPU::V_SUBREV_CO_U32_e64) || // FIXME
971 (!OpToFold.isImm() && !OpToFold.isFI() && !OpToFold.isGlobal())) {
972 TII->commuteInstruction(MI&: *MI, NewMI: false, OpIdx1: OpNo, OpIdx2: CommuteOpNo);
973 return false;
974 }
975
976 // Verify the other operand is a VGPR, otherwise we would violate the
977 // constant bus restriction.
978 MachineOperand &OtherOp = MI->getOperand(i: OpNo);
979 if (!OtherOp.isReg() ||
980 !TII->getRegisterInfo().isVGPR(MRI: *MRI, Reg: OtherOp.getReg()))
981 return false;
982
983 assert(MI->getOperand(1).isDef());
984
985 // Make sure to get the 32-bit version of the commuted opcode.
986 unsigned MaybeCommutedOpc = MI->getOpcode();
987 Op32 = AMDGPU::getVOPe32(Opcode: MaybeCommutedOpc);
988 }
989
990 appendFoldCandidate(FoldList, MI, OpNo: CommuteOpNo, FoldOp: OpToFold, /*Commuted=*/true,
991 ShrinkOp: Op32);
992 return true;
993 }
994
995 // Special case for s_fmac_f32 if we are trying to fold into Src0 or Src1.
996 // By changing into fmamk we can untie Src2.
997 // If folding for Src0 happens first and it is identical operand to Src1 we
998 // should avoid transforming into fmamk which requires commuting as it would
999 // cause folding into Src1 to fail later on due to wrong OpNo used.
1000 if (Opc == AMDGPU::S_FMAC_F32 &&
1001 (OpNo != 1 || !MI->getOperand(i: 1).isIdenticalTo(Other: MI->getOperand(i: 2)))) {
1002 if (tryToFoldAsFMAAKorMK())
1003 return true;
1004 }
1005
1006 // Special case for PK_F32 instructions if we are trying to fold an imm to
1007 // src0 or src1.
1008 if (OpToFold.isImm() &&
1009 isPKF32InstrReplicatesLower32BitsOfScalarOperand(ST, MI, OpNo) &&
1010 !checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(OpToFold))
1011 return false;
1012
1013 appendFoldCandidate(FoldList, MI, OpNo, FoldOp: OpToFold);
1014 return true;
1015}
1016
1017bool SIFoldOperandsImpl::isUseSafeToFold(const MachineInstr &MI,
1018 const MachineOperand &UseMO) const {
1019 // Operands of SDWA instructions must be registers.
1020 return !TII->isSDWA(MI);
1021}
1022
1023// Returns true if any instruction in \p L modifies EXEC.
1024static bool loopModifiesExec(const MachineLoop &L, const SIRegisterInfo &TRI) {
1025 for (const MachineBasicBlock *MBB : L.getBlocks())
1026 for (const MachineInstr &MI : *MBB)
1027 if (MI.modifiesRegister(Reg: TRI.getExec(), TRI: &TRI))
1028 return true;
1029 return false;
1030}
1031
1032// An SGPR->VGPR copy inside a divergent loop latches each lane value as it
1033// exits. Folding its scalar source into a use after the loop would make every
1034// lane read the same reconverged value, so do not fold across the loop exit.
1035bool SIFoldOperandsImpl::isTemporallyDivergentUse(
1036 const FoldableDef &OpToFold, const MachineInstr &UseMI) const {
1037 if (!OpToFold.isReg())
1038 return false;
1039 const MachineInstr *DefMI = OpToFold.DefMI;
1040 if (!DefMI || !DefMI->isCopy() ||
1041 TRI->isSGPRReg(MRI: *MRI, Reg: DefMI->getOperand(i: 0).getReg()) ||
1042 !TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg()))
1043 return false;
1044 const MachineLoop *DefLoop = MLI->getLoopFor(BB: DefMI->getParent());
1045 return DefLoop && !DefLoop->contains(BB: UseMI.getParent()) &&
1046 loopModifiesExec(L: *DefLoop, TRI: *TRI);
1047}
1048
1049static MachineOperand *lookUpCopyChain(const SIInstrInfo &TII,
1050 const MachineRegisterInfo &MRI,
1051 Register SrcReg) {
1052 MachineOperand *Sub = nullptr;
1053 for (MachineInstr *SubDef = MRI.getVRegDef(Reg: SrcReg);
1054 SubDef && TII.isFoldableCopy(MI: *SubDef);
1055 SubDef = MRI.getVRegDef(Reg: Sub->getReg())) {
1056 unsigned SrcIdx = TII.getFoldableCopySrcIdx(MI: *SubDef);
1057 MachineOperand &SrcOp = SubDef->getOperand(i: SrcIdx);
1058
1059 if (SrcOp.isImm())
1060 return &SrcOp;
1061 if (!SrcOp.isReg() || SrcOp.getReg().isPhysical())
1062 break;
1063 Sub = &SrcOp;
1064 // TODO: Support compose
1065 if (SrcOp.getSubReg())
1066 break;
1067 }
1068
1069 return Sub;
1070}
1071
1072const TargetRegisterClass *SIFoldOperandsImpl::getRegSeqInit(
1073 MachineInstr &RegSeq,
1074 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs) const {
1075
1076 assert(RegSeq.isRegSequence());
1077
1078 const TargetRegisterClass *RC = nullptr;
1079
1080 for (unsigned I = 1, E = RegSeq.getNumExplicitOperands(); I != E; I += 2) {
1081 MachineOperand &SrcOp = RegSeq.getOperand(i: I);
1082 if (SrcOp.getReg().isPhysical())
1083 return nullptr;
1084 unsigned SubRegIdx = RegSeq.getOperand(i: I + 1).getImm();
1085
1086 // Only accept reg_sequence with uniform reg class inputs for simplicity.
1087 const TargetRegisterClass *OpRC = getRegOpRC(MRI: *MRI, TRI: *TRI, MO: SrcOp);
1088 if (!RC)
1089 RC = OpRC;
1090 else if (!TRI->getCommonSubClass(A: RC, B: OpRC))
1091 return nullptr;
1092
1093 if (SrcOp.getSubReg()) {
1094 // TODO: Handle subregister compose
1095 Defs.emplace_back(Args: &SrcOp, Args&: SubRegIdx);
1096 continue;
1097 }
1098
1099 MachineOperand *DefSrc = lookUpCopyChain(TII: *TII, MRI: *MRI, SrcReg: SrcOp.getReg());
1100 if (DefSrc && (DefSrc->isReg() || DefSrc->isImm())) {
1101 Defs.emplace_back(Args&: DefSrc, Args&: SubRegIdx);
1102 continue;
1103 }
1104
1105 Defs.emplace_back(Args: &SrcOp, Args&: SubRegIdx);
1106 }
1107
1108 return RC;
1109}
1110
1111// Find a def of the UseReg, check if it is a reg_sequence and find initializers
1112// for each subreg, tracking it to an immediate if possible. Returns the
1113// register class of the inputs on success.
1114const TargetRegisterClass *SIFoldOperandsImpl::getRegSeqInit(
1115 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
1116 Register UseReg) const {
1117 MachineInstr *Def = MRI->getVRegDef(Reg: UseReg);
1118 if (!Def || !Def->isRegSequence())
1119 return nullptr;
1120
1121 return getRegSeqInit(RegSeq&: *Def, Defs);
1122}
1123
1124std::pair<int64_t, const TargetRegisterClass *>
1125SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
1126 SmallVector<std::pair<MachineOperand *, unsigned>, 32> Defs;
1127 const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
1128 if (!SrcRC)
1129 return {};
1130
1131 bool TryToMatchSplat64 = false;
1132
1133 std::optional<int64_t> Imm;
1134 for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
1135 const MachineOperand *Op = Defs[I].first;
1136 if (!Op->isImm()) {
1137 if (Op->isReg()) {
1138 MachineInstr *Def = MRI->getVRegDef(Reg: Op->getReg());
1139 if (!Def || Def->isImplicitDef())
1140 continue;
1141 }
1142 return {};
1143 }
1144
1145 int64_t SubImm = Op->getImm();
1146 if (!Imm) {
1147 Imm = SubImm;
1148 continue;
1149 }
1150
1151 if (Imm != SubImm) {
1152 if (I == 1 && (E & 1) == 0) {
1153 // If we have an even number of inputs, there's a chance this is a
1154 // 64-bit element splat broken into 32-bit pieces.
1155 TryToMatchSplat64 = true;
1156 break;
1157 }
1158
1159 return {}; // Can only fold splat constants
1160 }
1161 }
1162
1163 if (!TryToMatchSplat64) {
1164 if (Imm)
1165 return {*Imm, SrcRC};
1166 return {};
1167 }
1168
1169 // Fallback to recognizing 64-bit splats broken into 32-bit pieces
1170 // (i.e. recognize every other other element is 0 for 64-bit immediates)
1171 int64_t SplatVal64;
1172 for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
1173 const MachineOperand *Op0 = Defs[I].first;
1174 const MachineOperand *Op1 = Defs[I + 1].first;
1175
1176 if (!Op0->isImm() || !Op1->isImm())
1177 return {};
1178
1179 unsigned SubReg0 = Defs[I].second;
1180 unsigned SubReg1 = Defs[I + 1].second;
1181
1182 // Assume we're going to generally encounter reg_sequences with sorted
1183 // subreg indexes, so reject any that aren't consecutive.
1184 if (TRI->getChannelFromSubReg(SubReg: SubReg0) + 1 !=
1185 TRI->getChannelFromSubReg(SubReg: SubReg1))
1186 return {};
1187
1188 if (TRI->getSubRegIdxSize(Idx: SubReg0) != 32)
1189 return {};
1190
1191 int64_t MergedVal = Make_64(High: Op1->getImm(), Low: Op0->getImm());
1192 if (I == 0)
1193 SplatVal64 = MergedVal;
1194 else if (SplatVal64 != MergedVal)
1195 return {};
1196 }
1197
1198 const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
1199 MRI->getRegClass(Reg: RegSeq.getOperand(i: 0).getReg()), AMDGPU::sub0_sub1);
1200
1201 return {SplatVal64, RC64};
1202}
1203
1204bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
1205 MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
1206 const TargetRegisterClass *SplatRC) const {
1207 const MCInstrDesc &Desc = UseMI->getDesc();
1208 if (UseOpIdx >= Desc.getNumOperands())
1209 return false;
1210
1211 // Filter out unhandled pseudos.
1212 if (!AMDGPU::isSISrcOperand(Desc, OpNo: UseOpIdx))
1213 return false;
1214
1215 int16_t RCID = TII->getOpRegClassID(OpInfo: Desc.operands()[UseOpIdx]);
1216 if (RCID == -1)
1217 return false;
1218
1219 const TargetRegisterClass *OpRC = TRI->getRegClass(i: RCID);
1220
1221 // Special case 0/-1, since when interpreted as a 64-bit element both halves
1222 // have the same bits. These are the only cases where a splat has the same
1223 // interpretation for 32-bit and 64-bit splats.
1224 if (SplatVal != 0 && SplatVal != -1) {
1225 // We need to figure out the scalar type read by the operand. e.g. the MFMA
1226 // operand will be AReg_128, and we want to check if it's compatible with an
1227 // AReg_32 constant.
1228 uint8_t OpTy = Desc.operands()[UseOpIdx].OperandType;
1229 switch (OpTy) {
1230 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1231 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1232 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1233 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1234 case AMDGPU::OPERAND_REG_IMM_V2FP32:
1235 OpRC = TRI->getSubRegisterClass(OpRC, AMDGPU::sub0);
1236 break;
1237 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
1238 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1239 case AMDGPU::OPERAND_REG_IMM_V2FP64:
1240 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1241 case AMDGPU::OPERAND_REG_IMM_V2INT64:
1242 OpRC = TRI->getSubRegisterClass(OpRC, AMDGPU::sub0_sub1);
1243 break;
1244 default:
1245 return false;
1246 }
1247
1248 if (!TRI->getCommonSubClass(A: OpRC, B: SplatRC))
1249 return false;
1250 }
1251
1252 MachineOperand TmpOp = MachineOperand::CreateImm(Val: SplatVal);
1253 if (!TII->isOperandLegal(MI: *UseMI, OpIdx: UseOpIdx, MO: &TmpOp))
1254 return false;
1255
1256 return true;
1257}
1258
1259bool SIFoldOperandsImpl::tryToFoldACImm(
1260 const FoldableDef &OpToFold, MachineInstr *UseMI, unsigned UseOpIdx,
1261 SmallVectorImpl<FoldCandidate> &FoldList) const {
1262 const MCInstrDesc &Desc = UseMI->getDesc();
1263 if (UseOpIdx >= Desc.getNumOperands())
1264 return false;
1265
1266 // Filter out unhandled pseudos.
1267 if (!AMDGPU::isSISrcOperand(Desc, OpNo: UseOpIdx))
1268 return false;
1269
1270 if (OpToFold.isImm() && OpToFold.isOperandLegal(TII: *TII, MI: *UseMI, OpIdx: UseOpIdx)) {
1271 if (isPKF32InstrReplicatesLower32BitsOfScalarOperand(ST, MI: UseMI, OpNo: UseOpIdx) &&
1272 !checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(OpToFold))
1273 return false;
1274 appendFoldCandidate(FoldList, MI: UseMI, OpNo: UseOpIdx, FoldOp: OpToFold);
1275 return true;
1276 }
1277
1278 return false;
1279}
1280
1281bool SIFoldOperandsImpl::foldOperand(
1282 FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
1283 SmallVectorImpl<FoldCandidate> &FoldList,
1284 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const {
1285 bool Changed = false;
1286 const MachineOperand *UseOp = &UseMI->getOperand(i: UseOpIdx);
1287
1288 if (!isUseSafeToFold(MI: *UseMI, UseMO: *UseOp))
1289 return Changed;
1290
1291 if (isTemporallyDivergentUse(OpToFold, UseMI: *UseMI))
1292 return Changed;
1293
1294 // FIXME: Fold operands with subregs.
1295 if (UseOp->isReg() && OpToFold.isReg()) {
1296 if (UseOp->isImplicit())
1297 return Changed;
1298 // Allow folding from SGPRs to 16-bit VGPRs.
1299 if (UseOp->getSubReg() != AMDGPU::NoSubRegister &&
1300 (UseOp->getSubReg() != AMDGPU::lo16 ||
1301 !TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg())))
1302 return Changed;
1303 }
1304
1305 // Special case for REG_SEQUENCE: We can't fold literals into
1306 // REG_SEQUENCE instructions, so we have to fold them into the
1307 // uses of REG_SEQUENCE.
1308 if (UseMI->isRegSequence()) {
1309 Register RegSeqDstReg = UseMI->getOperand(i: 0).getReg();
1310 unsigned RegSeqDstSubReg = UseMI->getOperand(i: UseOpIdx + 1).getImm();
1311
1312 int64_t SplatVal;
1313 const TargetRegisterClass *SplatRC;
1314 std::tie(args&: SplatVal, args&: SplatRC) = isRegSeqSplat(RegSeq&: *UseMI);
1315
1316 // Grab the use operands first
1317 SmallVector<MachineOperand *, 4> UsesToProcess(
1318 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: RegSeqDstReg)));
1319 for (unsigned I = 0; I != UsesToProcess.size(); ++I) {
1320 MachineOperand *RSUse = UsesToProcess[I];
1321 MachineInstr *RSUseMI = RSUse->getParent();
1322 unsigned OpNo = RSUseMI->getOperandNo(I: RSUse);
1323
1324 if (SplatRC) {
1325 if (RSUseMI->isCopy()) {
1326 Register DstReg = RSUseMI->getOperand(i: 0).getReg();
1327 append_range(C&: UsesToProcess,
1328 R: make_pointer_range(Range: MRI->use_nodbg_operands(Reg: DstReg)));
1329 continue;
1330 }
1331 if (tryFoldRegSeqSplat(UseMI: RSUseMI, UseOpIdx: OpNo, SplatVal, SplatRC)) {
1332 FoldableDef SplatDef(SplatVal, SplatRC);
1333 appendFoldCandidate(FoldList, MI: RSUseMI, OpNo, FoldOp: SplatDef);
1334 Changed = true;
1335 continue;
1336 }
1337 }
1338
1339 // TODO: Handle general compose
1340 if (RSUse->getSubReg() != RegSeqDstSubReg)
1341 continue;
1342
1343 // FIXME: We should avoid recursing here. There should be a cleaner split
1344 // between the in-place mutations and adding to the fold list.
1345 Changed |= foldOperand(OpToFold, UseMI: RSUseMI, UseOpIdx: RSUseMI->getOperandNo(I: RSUse),
1346 FoldList, CopiesToReplace);
1347 }
1348
1349 return Changed;
1350 }
1351
1352 if (tryToFoldACImm(OpToFold, UseMI, UseOpIdx, FoldList))
1353 return true;
1354
1355 if (frameIndexMayFold(UseMI: *UseMI, OpNo: UseOpIdx, OpToFold)) {
1356 // Verify that this is a stack access.
1357 // FIXME: Should probably use stack pseudos before frame lowering.
1358
1359 if (TII->isMUBUF(MI: *UseMI)) {
1360 if (TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::srsrc)->getReg() !=
1361 MFI->getScratchRSrcReg())
1362 return Changed;
1363
1364 // Ensure this is either relative to the current frame or the current
1365 // wave.
1366 MachineOperand &SOff =
1367 *TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::soffset);
1368 if (!SOff.isImm() || SOff.getImm() != 0)
1369 return Changed;
1370 }
1371
1372 const unsigned Opc = UseMI->getOpcode();
1373 if (TII->isFLATScratch(MI: *UseMI) &&
1374 AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr) &&
1375 !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::saddr)) {
1376 unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(Opcode: Opc);
1377 unsigned CPol =
1378 TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::cpol)->getImm();
1379 if ((CPol & AMDGPU::CPol::SCAL) &&
1380 !AMDGPU::supportsScaleOffset(MII: *TII, Opcode: NewOpc))
1381 return Changed;
1382
1383 UseMI->setDesc(TII->get(Opcode: NewOpc));
1384 }
1385
1386 // A frame index will resolve to a positive constant, so it should always be
1387 // safe to fold the addressing mode, even pre-GFX9.
1388 UseMI->getOperand(i: UseOpIdx).ChangeToFrameIndex(Idx: OpToFold.getFI());
1389
1390 return true;
1391 }
1392
1393 bool FoldingImmLike =
1394 OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
1395
1396 if (FoldingImmLike && UseMI->isCopy()) {
1397 Register DestReg = UseMI->getOperand(i: 0).getReg();
1398 Register SrcReg = UseMI->getOperand(i: 1).getReg();
1399 unsigned UseSubReg = UseMI->getOperand(i: 1).getSubReg();
1400 assert(SrcReg.isVirtual());
1401
1402 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: SrcReg);
1403
1404 // Don't fold into a copy to a physical register with the same class. Doing
1405 // so would interfere with the register coalescer's logic which would avoid
1406 // redundant initializations.
1407 if (DestReg.isPhysical() && SrcRC->contains(Reg: DestReg))
1408 return Changed;
1409
1410 const TargetRegisterClass *DestRC = TRI->getRegClassForReg(MRI: *MRI, Reg: DestReg);
1411 // In order to fold immediates into copies, we need to change the copy to a
1412 // MOV. Find a compatible mov instruction with the value.
1413 for (unsigned MovOp :
1414 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
1415 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_MOV_B16_t16_e64,
1416 AMDGPU::V_ACCVGPR_WRITE_B32_e64, AMDGPU::AV_MOV_B32_IMM_PSEUDO,
1417 AMDGPU::AV_MOV_B64_IMM_PSEUDO}) {
1418 const MCInstrDesc &MovDesc = TII->get(Opcode: MovOp);
1419 const TargetRegisterClass *MovDstRC =
1420 TRI->getRegClass(i: TII->getOpRegClassID(OpInfo: MovDesc.operands()[0]));
1421
1422 // Fold if the destination register class of the MOV instruction (ResRC)
1423 // is a superclass of (or equal to) the destination register class of the
1424 // COPY (DestRC). If this condition fails, folding would be illegal.
1425 if (!DestRC->hasSuperClassEq(RC: MovDstRC))
1426 continue;
1427
1428 const int SrcIdx = MovOp == AMDGPU::V_MOV_B16_t16_e64 ? 2 : 1;
1429
1430 int16_t RegClassID = TII->getOpRegClassID(OpInfo: MovDesc.operands()[SrcIdx]);
1431 if (RegClassID != -1) {
1432 const TargetRegisterClass *MovSrcRC = TRI->getRegClass(i: RegClassID);
1433
1434 if (UseSubReg)
1435 MovSrcRC = TRI->getMatchingSuperRegClass(A: SrcRC, B: MovSrcRC, Idx: UseSubReg);
1436
1437 // FIXME: We should be able to directly check immediate operand legality
1438 // for all cases, but gfx908 hacks break.
1439 if (MovOp == AMDGPU::AV_MOV_B32_IMM_PSEUDO &&
1440 (!OpToFold.isImm() ||
1441 !TII->isImmOperandLegal(InstDesc: MovDesc, OpNo: SrcIdx,
1442 ImmVal: *OpToFold.getEffectiveImmVal())))
1443 break;
1444
1445 if (!MRI->constrainRegClass(Reg: SrcReg, RC: MovSrcRC))
1446 break;
1447
1448 // FIXME: This is mutating the instruction only and deferring the actual
1449 // fold of the immediate
1450 } else {
1451 // For the _IMM_PSEUDO cases, there can be value restrictions on the
1452 // immediate to verify. Technically we should always verify this, but it
1453 // only matters for these concrete cases.
1454 // TODO: Handle non-imm case if it's useful.
1455 if (!OpToFold.isImm() ||
1456 !TII->isImmOperandLegal(InstDesc: MovDesc, OpNo: 1, ImmVal: *OpToFold.getEffectiveImmVal()))
1457 break;
1458 }
1459
1460 MachineInstr::mop_iterator ImpOpI = UseMI->implicit_operands().begin();
1461 MachineInstr::mop_iterator ImpOpE = UseMI->implicit_operands().end();
1462 while (ImpOpI != ImpOpE) {
1463 MachineInstr::mop_iterator Tmp = ImpOpI;
1464 ImpOpI++;
1465 UseMI->removeOperand(OpNo: UseMI->getOperandNo(I: Tmp));
1466 }
1467 UseMI->setDesc(MovDesc);
1468
1469 if (MovOp == AMDGPU::V_MOV_B16_t16_e64) {
1470 const auto &SrcOp = UseMI->getOperand(i: UseOpIdx);
1471 MachineOperand NewSrcOp(SrcOp);
1472 UseMI->removeOperand(OpNo: 1);
1473 UseMI->addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0)); // src0_modifiers
1474 UseMI->addOperand(Op: NewSrcOp); // src0
1475 UseMI->addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0)); // op_sel
1476 UseOpIdx = SrcIdx;
1477 UseOp = &UseMI->getOperand(i: UseOpIdx);
1478 }
1479 CopiesToReplace.push_back(Elt: UseMI);
1480 Changed = true;
1481 break;
1482 }
1483
1484 // We failed to replace the copy, so give up.
1485 if (UseMI->getOpcode() == AMDGPU::COPY)
1486 return Changed;
1487
1488 } else {
1489 if (UseMI->isCopy() && OpToFold.isReg() &&
1490 UseMI->getOperand(i: 0).getReg().isVirtual() &&
1491 !UseMI->getOperand(i: 1).getSubReg() &&
1492 OpToFold.DefMI->implicit_operands().empty()) {
1493 LLVM_DEBUG(dbgs() << "Folding " << *OpToFold.OpToFold << "\n into "
1494 << *UseMI);
1495 unsigned Size = TII->getOpSize(MI: *UseMI, OpNo: 1);
1496 Register UseReg = OpToFold.getReg();
1497 UseMI->getOperand(i: 1).setReg(UseReg);
1498 unsigned SubRegIdx = OpToFold.getSubReg();
1499 // Hack to allow 32-bit SGPRs to be folded into True16 instructions
1500 // Remove this if 16-bit SGPRs (i.e. SGPR_LO16) are added to the
1501 // VS_16RegClass
1502 //
1503 // Excerpt from AMDGPUGenRegisterInfoEnums.inc
1504 // NoSubRegister, //0
1505 // hi16, // 1
1506 // lo16, // 2
1507 // sub0, // 3
1508 // ...
1509 // sub1, // 11
1510 // sub1_hi16, // 12
1511 // sub1_lo16, // 13
1512 static_assert(AMDGPU::sub1_hi16 == 12, "Subregister layout has changed");
1513 if (Size == 2 && TRI->isVGPR(MRI: *MRI, Reg: UseMI->getOperand(i: 0).getReg()) &&
1514 TRI->isSGPRReg(MRI: *MRI, Reg: UseReg)) {
1515 // Produce the 32 bit subregister index to which the 16-bit subregister
1516 // is aligned.
1517 if (SubRegIdx > AMDGPU::sub1) {
1518 LaneBitmask M = TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx);
1519 M |= M.getLane(Lane: M.getHighestLane() - 1);
1520 SmallVector<unsigned, 4> Indexes;
1521 TRI->getCoveringSubRegIndexes(RC: TRI->getRegClassForReg(MRI: *MRI, Reg: UseReg), LaneMask: M,
1522 Indexes);
1523 assert(Indexes.size() == 1 && "Expected one 32-bit subreg to cover");
1524 SubRegIdx = Indexes[0];
1525 // 32-bit registers do not have a sub0 index
1526 } else if (TII->getOpSize(MI: *UseMI, OpNo: 1) == 4)
1527 SubRegIdx = 0;
1528 else
1529 SubRegIdx = AMDGPU::sub0;
1530 }
1531 UseMI->getOperand(i: 1).setSubReg(SubRegIdx);
1532 UseMI->getOperand(i: 1).setIsKill(false);
1533 CopiesToReplace.push_back(Elt: UseMI);
1534 OpToFold.OpToFold->setIsKill(false);
1535 Changed = true;
1536
1537 // Remove kill flags as kills may now be out of order with uses.
1538 MRI->clearKillFlags(Reg: UseReg);
1539 if (foldCopyToAGPRRegSequence(CopyMI: UseMI))
1540 return true;
1541 }
1542
1543 unsigned UseOpc = UseMI->getOpcode();
1544 if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 ||
1545 (UseOpc == AMDGPU::V_READLANE_B32 &&
1546 (int)UseOpIdx ==
1547 AMDGPU::getNamedOperandIdx(Opcode: UseOpc, Name: AMDGPU::OpName::src0))) {
1548 // %vgpr = V_MOV_B32 imm
1549 // %sgpr = V_READFIRSTLANE_B32 %vgpr
1550 // =>
1551 // %sgpr = S_MOV_B32 imm
1552 if (FoldingImmLike) {
1553 if (execMayBeModifiedBeforeUse(MRI: *MRI,
1554 VReg: UseMI->getOperand(i: UseOpIdx).getReg(),
1555 DefMI: *OpToFold.DefMI, UseMI: *UseMI))
1556 return Changed;
1557
1558 UseMI->setDesc(TII->get(Opcode: AMDGPU::S_MOV_B32));
1559 UseMI->clearFlag(Flag: MachineInstr::NoConvergent);
1560
1561 if (OpToFold.isImm()) {
1562 UseMI->getOperand(i: 1).ChangeToImmediate(
1563 ImmVal: *OpToFold.getEffectiveImmVal());
1564 } else if (OpToFold.isFI())
1565 UseMI->getOperand(i: 1).ChangeToFrameIndex(Idx: OpToFold.getFI());
1566 else {
1567 assert(OpToFold.isGlobal());
1568 UseMI->getOperand(i: 1).ChangeToGA(GV: OpToFold.OpToFold->getGlobal(),
1569 Offset: OpToFold.OpToFold->getOffset(),
1570 TargetFlags: OpToFold.OpToFold->getTargetFlags());
1571 }
1572 UseMI->removeOperand(OpNo: 2); // Remove exec read (or src1 for readlane)
1573 return true;
1574 }
1575
1576 if (OpToFold.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg())) {
1577 if (execMayBeModifiedBeforeUse(MRI: *MRI,
1578 VReg: UseMI->getOperand(i: UseOpIdx).getReg(),
1579 DefMI: *OpToFold.DefMI, UseMI: *UseMI))
1580 return Changed;
1581
1582 // %vgpr = COPY %sgpr0
1583 // %sgpr1 = V_READFIRSTLANE_B32 %vgpr
1584 // =>
1585 // %sgpr1 = COPY %sgpr0
1586 UseMI->setDesc(TII->get(Opcode: AMDGPU::COPY));
1587 UseMI->getOperand(i: 1).setReg(OpToFold.getReg());
1588 UseMI->getOperand(i: 1).setSubReg(OpToFold.getSubReg());
1589 UseMI->getOperand(i: 1).setIsKill(false);
1590 UseMI->removeOperand(OpNo: 2); // Remove exec read (or src1 for readlane)
1591 UseMI->clearFlag(Flag: MachineInstr::NoConvergent);
1592 return true;
1593 }
1594 }
1595
1596 const MCInstrDesc &UseDesc = UseMI->getDesc();
1597
1598 // Don't fold into target independent nodes. Target independent opcodes
1599 // don't have defined register classes.
1600 if (UseDesc.isVariadic() || UseOp->isImplicit() ||
1601 UseDesc.operands()[UseOpIdx].RegClass == -1)
1602 return Changed;
1603 }
1604
1605 // FIXME: We could try to change the instruction from 64-bit to 32-bit
1606 // to enable more folding opportunities. The shrink operands pass
1607 // already does this.
1608
1609 Changed |= tryAddToFoldList(FoldList, MI: UseMI, OpNo: UseOpIdx, OpToFold);
1610 return Changed;
1611}
1612
1613static bool evalBinaryInstruction(unsigned Opcode, int32_t &Result,
1614 uint32_t LHS, uint32_t RHS) {
1615 switch (Opcode) {
1616 case AMDGPU::S_ADD_I32:
1617 case AMDGPU::S_ADD_U32:
1618 Result = LHS + RHS;
1619 return true;
1620 case AMDGPU::S_SUB_I32:
1621 case AMDGPU::S_SUB_U32:
1622 Result = LHS - RHS;
1623 return true;
1624 case AMDGPU::V_AND_B32_e64:
1625 case AMDGPU::V_AND_B32_e32:
1626 case AMDGPU::S_AND_B32:
1627 Result = LHS & RHS;
1628 return true;
1629 case AMDGPU::V_OR_B32_e64:
1630 case AMDGPU::V_OR_B32_e32:
1631 case AMDGPU::S_OR_B32:
1632 Result = LHS | RHS;
1633 return true;
1634 case AMDGPU::V_XOR_B32_e64:
1635 case AMDGPU::V_XOR_B32_e32:
1636 case AMDGPU::S_XOR_B32:
1637 Result = LHS ^ RHS;
1638 return true;
1639 case AMDGPU::S_XNOR_B32:
1640 Result = ~(LHS ^ RHS);
1641 return true;
1642 case AMDGPU::S_NAND_B32:
1643 Result = ~(LHS & RHS);
1644 return true;
1645 case AMDGPU::S_NOR_B32:
1646 Result = ~(LHS | RHS);
1647 return true;
1648 case AMDGPU::S_ANDN2_B32:
1649 Result = LHS & ~RHS;
1650 return true;
1651 case AMDGPU::S_ORN2_B32:
1652 Result = LHS | ~RHS;
1653 return true;
1654 case AMDGPU::V_LSHL_B32_e64:
1655 case AMDGPU::V_LSHL_B32_e32:
1656 case AMDGPU::S_LSHL_B32:
1657 // The instruction ignores the high bits for out of bounds shifts.
1658 Result = LHS << (RHS & 31);
1659 return true;
1660 case AMDGPU::V_LSHLREV_B32_e64:
1661 case AMDGPU::V_LSHLREV_B32_e32:
1662 Result = RHS << (LHS & 31);
1663 return true;
1664 case AMDGPU::V_LSHR_B32_e64:
1665 case AMDGPU::V_LSHR_B32_e32:
1666 case AMDGPU::S_LSHR_B32:
1667 Result = LHS >> (RHS & 31);
1668 return true;
1669 case AMDGPU::V_LSHRREV_B32_e64:
1670 case AMDGPU::V_LSHRREV_B32_e32:
1671 Result = RHS >> (LHS & 31);
1672 return true;
1673 case AMDGPU::V_ASHR_I32_e64:
1674 case AMDGPU::V_ASHR_I32_e32:
1675 case AMDGPU::S_ASHR_I32:
1676 Result = static_cast<int32_t>(LHS) >> (RHS & 31);
1677 return true;
1678 case AMDGPU::V_ASHRREV_I32_e64:
1679 case AMDGPU::V_ASHRREV_I32_e32:
1680 Result = static_cast<int32_t>(RHS) >> (LHS & 31);
1681 return true;
1682 default:
1683 return false;
1684 }
1685}
1686
1687static unsigned getMovOpc(bool IsScalar) {
1688 return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1689}
1690
1691// Try to simplify operations with a constant that may appear after instruction
1692// selection.
1693// TODO: See if a frame index with a fixed offset can fold.
1694bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
1695 if (!MI->allImplicitDefsAreDead())
1696 return false;
1697
1698 unsigned Opc = MI->getOpcode();
1699
1700 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
1701 if (Src0Idx == -1)
1702 return false;
1703
1704 MachineOperand *Src0 = &MI->getOperand(i: Src0Idx);
1705 std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src0);
1706
1707 if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 ||
1708 Opc == AMDGPU::S_NOT_B32) &&
1709 Src0Imm) {
1710 MI->getOperand(i: 1).ChangeToImmediate(ImmVal: ~*Src0Imm);
1711 TII->mutateAndCleanupImplicit(
1712 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_NOT_B32)));
1713 return true;
1714 }
1715
1716 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
1717 if (Src1Idx == -1)
1718 return false;
1719
1720 MachineOperand *Src1 = &MI->getOperand(i: Src1Idx);
1721 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
1722
1723 if (!Src0Imm && !Src1Imm)
1724 return false;
1725
1726 // and k0, k1 -> v_mov_b32 (k0 & k1)
1727 // or k0, k1 -> v_mov_b32 (k0 | k1)
1728 // xor k0, k1 -> v_mov_b32 (k0 ^ k1)
1729 if (Src0Imm && Src1Imm) {
1730 int32_t NewImm;
1731 if (!evalBinaryInstruction(Opcode: Opc, Result&: NewImm, LHS: *Src0Imm, RHS: *Src1Imm))
1732 return false;
1733
1734 bool IsSGPR = TRI->isSGPRReg(MRI: *MRI, Reg: MI->getOperand(i: 0).getReg());
1735
1736 // Be careful to change the right operand, src0 may belong to a different
1737 // instruction.
1738 MI->getOperand(i: Src0Idx).ChangeToImmediate(ImmVal: NewImm);
1739 MI->removeOperand(OpNo: Src1Idx);
1740 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: IsSGPR)));
1741 return true;
1742 }
1743
1744 // S_SUB_* is not commutable, so handle it before the commutability gate.
1745 // Only `x - 0 -> copy x` is valid; `0 - x` is a negation, not a copy.
1746 if (Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U32) {
1747 if (Src1Imm && static_cast<int32_t>(*Src1Imm) == 0) {
1748 // y = sub x, 0 => y = copy x
1749 MI->removeOperand(OpNo: Src1Idx);
1750 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1751 return true;
1752 }
1753 return false;
1754 }
1755
1756 if (!MI->isCommutable())
1757 return false;
1758
1759 if (Src0Imm && !Src1Imm) {
1760 std::swap(a&: Src0, b&: Src1);
1761 std::swap(a&: Src0Idx, b&: Src1Idx);
1762 std::swap(lhs&: Src0Imm, rhs&: Src1Imm);
1763 }
1764
1765 int32_t Src1Val = static_cast<int32_t>(*Src1Imm);
1766 if (Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_ADD_U32) {
1767 if (Src1Val == 0) {
1768 // y = add x, 0 => y = copy x
1769 MI->removeOperand(OpNo: Src1Idx);
1770 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1771 return true;
1772 }
1773 return false;
1774 }
1775
1776 if (Opc == AMDGPU::V_OR_B32_e64 ||
1777 Opc == AMDGPU::V_OR_B32_e32 ||
1778 Opc == AMDGPU::S_OR_B32) {
1779 if (Src1Val == 0) {
1780 // y = or x, 0 => y = copy x
1781 MI->removeOperand(OpNo: Src1Idx);
1782 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1783 } else if (Src1Val == -1) {
1784 // y = or x, -1 => y = v_mov_b32 -1
1785 MI->removeOperand(OpNo: Src0Idx);
1786 TII->mutateAndCleanupImplicit(
1787 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_OR_B32)));
1788 } else
1789 return false;
1790
1791 return true;
1792 }
1793
1794 if (Opc == AMDGPU::V_AND_B32_e64 || Opc == AMDGPU::V_AND_B32_e32 ||
1795 Opc == AMDGPU::S_AND_B32) {
1796 if (Src1Val == 0) {
1797 // y = and x, 0 => y = v_mov_b32 0
1798 MI->removeOperand(OpNo: Src0Idx);
1799 TII->mutateAndCleanupImplicit(
1800 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_AND_B32)));
1801 } else if (Src1Val == -1) {
1802 // y = and x, -1 => y = copy x
1803 MI->removeOperand(OpNo: Src1Idx);
1804 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1805 } else
1806 return false;
1807
1808 return true;
1809 }
1810
1811 if (Opc == AMDGPU::V_XOR_B32_e64 || Opc == AMDGPU::V_XOR_B32_e32 ||
1812 Opc == AMDGPU::S_XOR_B32) {
1813 if (Src1Val == 0) {
1814 // y = xor x, 0 => y = copy x
1815 MI->removeOperand(OpNo: Src1Idx);
1816 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1817 return true;
1818 }
1819 }
1820
1821 return false;
1822}
1823
1824// Try to fold an instruction into a simpler one
1825bool SIFoldOperandsImpl::tryFoldCndMask(MachineInstr &MI) const {
1826 unsigned Opc = MI.getOpcode();
1827 if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 &&
1828 Opc != AMDGPU::V_CNDMASK_B64_PSEUDO)
1829 return false;
1830
1831 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1832 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1833 if (!Src1->isIdenticalTo(Other: *Src0)) {
1834 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
1835 if (!Src1Imm)
1836 return false;
1837
1838 std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src0);
1839 if (!Src0Imm || *Src0Imm != *Src1Imm)
1840 return false;
1841 }
1842
1843 int Src1ModIdx =
1844 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1_modifiers);
1845 int Src0ModIdx =
1846 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers);
1847 if ((Src1ModIdx != -1 && MI.getOperand(i: Src1ModIdx).getImm() != 0) ||
1848 (Src0ModIdx != -1 && MI.getOperand(i: Src0ModIdx).getImm() != 0))
1849 return false;
1850
1851 LLVM_DEBUG(dbgs() << "Folded " << MI << " into ");
1852 auto &NewDesc =
1853 TII->get(Opcode: Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(IsScalar: false));
1854 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
1855 if (Src2Idx != -1)
1856 MI.removeOperand(OpNo: Src2Idx);
1857 MI.removeOperand(OpNo: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1));
1858 if (Src1ModIdx != -1)
1859 MI.removeOperand(OpNo: Src1ModIdx);
1860 if (Src0ModIdx != -1)
1861 MI.removeOperand(OpNo: Src0ModIdx);
1862 TII->mutateAndCleanupImplicit(MI, NewDesc);
1863 LLVM_DEBUG(dbgs() << MI);
1864 return true;
1865}
1866
1867// Extract mask, register, and register operand index from an AND instruction.
1868// Immediate can be in operand 1 or 2.
1869std::optional<SIFoldOperandsImpl::ANDMaskResult>
1870SIFoldOperandsImpl::getANDMaskRegOperand(MachineInstr &AndMI) const {
1871 unsigned Opc = AndMI.getOpcode();
1872 if (Opc != AMDGPU::V_AND_B32_e64 && Opc != AMDGPU::V_AND_B32_e32 &&
1873 Opc != AMDGPU::S_AND_B32)
1874 return std::nullopt;
1875
1876 std::optional<int64_t> MaskImm =
1877 TII->getImmOrMaterializedImm(MRI: *MRI, Op: AndMI.getOperand(i: 1));
1878 if (MaskImm && AndMI.getOperand(i: 2).isReg())
1879 return ANDMaskResult{.Mask: *MaskImm, .Reg: AndMI.getOperand(i: 2).getReg(), .RegIdx: 2};
1880
1881 MaskImm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: AndMI.getOperand(i: 2));
1882 if (MaskImm && AndMI.getOperand(i: 1).isReg())
1883 return ANDMaskResult{.Mask: *MaskImm, .Reg: AndMI.getOperand(i: 1).getReg(), .RegIdx: 1};
1884
1885 return std::nullopt;
1886}
1887
1888// Eliminate redundant 32-bit AND operations by detecting when ChildMI's mask
1889// contains ParentMI's mask.
1890//
1891// For example:
1892// ParentMI: %1 = AND %0, 0x7fff
1893// ChildMI: %2 = AND %1, 0xffff
1894//
1895// This also handles cases where ParentMI implicitly zeros high bits (e.g., f16
1896// operations that write 16-bit results into 32-bit registers), making a
1897// subsequent AND with 0xffff redundant.
1898bool SIFoldOperandsImpl::tryFoldRedundantAND(MachineInstr &ChildMI) const {
1899 // Ensure implicit defs (e.g., $scc) are not live.
1900 if (!ChildMI.allImplicitDefsAreDead())
1901 return false;
1902
1903 std::optional<ANDMaskResult> ChildResult = getANDMaskRegOperand(AndMI&: ChildMI);
1904 if (!ChildResult)
1905 return false;
1906
1907 if (!ChildResult->Reg.isVirtual())
1908 return false;
1909
1910 MachineInstr *ParentMI = MRI->getVRegDef(Reg: ChildResult->Reg);
1911 if (!ParentMI)
1912 return false;
1913
1914 int64_t ParentMask = 0;
1915 std::optional<ANDMaskResult> ParentResult = getANDMaskRegOperand(AndMI&: *ParentMI);
1916 if (ParentResult) {
1917 // Parent is an AND - extract its mask.
1918 ParentMask = ParentResult->Mask;
1919 } else if (ST->zeroesHigh16BitsOfDest(Opcode: ParentMI->getOpcode())) {
1920 // Parent instruction implicitly zeros high 16 bits.
1921 ParentMask = 0xffff;
1922 } else {
1923 return false;
1924 }
1925
1926 // Check if ChildMI is not redundant.
1927 if ((ParentMask & ChildResult->Mask) != ParentMask)
1928 return false;
1929
1930 Register Dst = ChildMI.getOperand(i: 0).getReg();
1931 Register Src = ChildResult->Reg;
1932
1933 // Src must be legal in every use of Dst. An S_AND_B32 parent with a
1934 // V_AND_B32 child defines Src in the scalar bank, and a use that requires a
1935 // VGPR does not accept it.
1936 if (!Dst.isVirtual() || !MRI->constrainRegClass(Reg: Src, RC: MRI->getRegClass(Reg: Dst)))
1937 return false;
1938
1939 MRI->replaceRegWith(FromReg: Dst, ToReg: Src);
1940
1941 // Clear kill flags if the register operand is not marked as kill.
1942 if (!ChildMI.getOperand(i: ChildResult->RegIdx).isKill())
1943 MRI->clearKillFlags(Reg: Src);
1944
1945 ChildMI.eraseFromParent();
1946 return true;
1947}
1948
1949bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
1950 const FoldableDef &OpToFold) const {
1951 // We need mutate the operands of new mov instructions to add implicit
1952 // uses of EXEC, but adding them invalidates the use_iterator, so defer
1953 // this.
1954 SmallVector<MachineInstr *, 4> CopiesToReplace;
1955 SmallVector<FoldCandidate, 4> FoldList;
1956 MachineOperand &Dst = MI.getOperand(i: 0);
1957 bool Changed = false;
1958
1959 SmallVector<MachineOperand *, 4> UsesToProcess(
1960 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: Dst.getReg())));
1961 for (auto *U : UsesToProcess) {
1962 MachineInstr *UseMI = U->getParent();
1963
1964 FoldableDef SubOpToFold = OpToFold.getWithSubReg(TRI: *TRI, SubReg: U->getSubReg());
1965 Changed |= foldOperand(OpToFold: SubOpToFold, UseMI, UseOpIdx: UseMI->getOperandNo(I: U), FoldList,
1966 CopiesToReplace);
1967 }
1968
1969 if (CopiesToReplace.empty() && FoldList.empty())
1970 return Changed;
1971
1972 // Make sure we add EXEC uses to any new v_mov instructions created.
1973 for (MachineInstr *Copy : CopiesToReplace)
1974 Copy->addImplicitDefUseOperands(MF&: *MF);
1975
1976 SetVector<MachineInstr *> ConstantFoldCandidates;
1977 for (FoldCandidate &Fold : FoldList) {
1978 assert(!Fold.isReg() || Fold.Def.OpToFold);
1979 if (Fold.isReg() && Fold.getReg().isVirtual()) {
1980 Register Reg = Fold.getReg();
1981 const MachineInstr *DefMI = Fold.Def.DefMI;
1982 if (DefMI->readsRegister(Reg: AMDGPU::EXEC, TRI) &&
1983 execMayBeModifiedBeforeUse(MRI: *MRI, VReg: Reg, DefMI: *DefMI, UseMI: *Fold.UseMI))
1984 continue;
1985 }
1986 if (updateOperand(Fold)) {
1987 // Clear kill flags.
1988 if (Fold.isReg()) {
1989 assert(Fold.Def.OpToFold && Fold.isReg());
1990 // FIXME: Probably shouldn't bother trying to fold if not an
1991 // SGPR. PeepholeOptimizer can eliminate redundant VGPR->VGPR
1992 // copies.
1993 MRI->clearKillFlags(Reg: Fold.getReg());
1994 }
1995 LLVM_DEBUG(dbgs() << "Folded source from " << MI << " into OpNo "
1996 << static_cast<int>(Fold.UseOpNo) << " of "
1997 << *Fold.UseMI);
1998
1999 if (Fold.isImm())
2000 ConstantFoldCandidates.insert(X: Fold.UseMI);
2001
2002 } else if (Fold.Commuted) {
2003 // Restoring instruction's original operand order if fold has failed.
2004 TII->commuteInstruction(MI&: *Fold.UseMI, NewMI: false);
2005 }
2006 }
2007
2008 for (MachineInstr *MI : ConstantFoldCandidates) {
2009 if (tryConstantFoldOp(MI)) {
2010 LLVM_DEBUG(dbgs() << "Constant folded " << *MI);
2011 Changed = true;
2012 }
2013 }
2014 return true;
2015}
2016
2017/// Fold %agpr = COPY (REG_SEQUENCE x_MOV_B32, ...) into REG_SEQUENCE
2018/// (V_ACCVGPR_WRITE_B32_e64) ... depending on the reg_sequence input values.
2019bool SIFoldOperandsImpl::foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const {
2020 // It is very tricky to store a value into an AGPR. v_accvgpr_write_b32 can
2021 // only accept VGPR or inline immediate. Recreate a reg_sequence with its
2022 // initializers right here, so we will rematerialize immediates and avoid
2023 // copies via different reg classes.
2024 const TargetRegisterClass *DefRC =
2025 MRI->getRegClass(Reg: CopyMI->getOperand(i: 0).getReg());
2026 if (!TRI->isAGPRClass(RC: DefRC))
2027 return false;
2028
2029 Register UseReg = CopyMI->getOperand(i: 1).getReg();
2030 MachineInstr *RegSeq = MRI->getVRegDef(Reg: UseReg);
2031 if (!RegSeq || !RegSeq->isRegSequence())
2032 return false;
2033
2034 const DebugLoc &DL = CopyMI->getDebugLoc();
2035 MachineBasicBlock &MBB = *CopyMI->getParent();
2036
2037 MachineInstrBuilder B(*MBB.getParent(), CopyMI);
2038 DenseMap<TargetInstrInfo::RegSubRegPair, Register> VGPRCopies;
2039
2040 const TargetRegisterClass *UseRC =
2041 MRI->getRegClass(Reg: CopyMI->getOperand(i: 1).getReg());
2042
2043 // Value, subregindex for new REG_SEQUENCE
2044 SmallVector<std::pair<MachineOperand *, unsigned>, 32> NewDefs;
2045
2046 unsigned NumRegSeqOperands = RegSeq->getNumOperands();
2047 unsigned NumFoldable = 0;
2048
2049 for (unsigned I = 1; I != NumRegSeqOperands; I += 2) {
2050 MachineOperand &RegOp = RegSeq->getOperand(i: I);
2051 unsigned SubRegIdx = RegSeq->getOperand(i: I + 1).getImm();
2052
2053 if (RegOp.getSubReg()) {
2054 // TODO: Handle subregister compose
2055 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2056 continue;
2057 }
2058
2059 MachineOperand *Lookup = lookUpCopyChain(TII: *TII, MRI: *MRI, SrcReg: RegOp.getReg());
2060 if (!Lookup)
2061 Lookup = &RegOp;
2062
2063 if (Lookup->isImm()) {
2064 // Check if this is an agpr_32 subregister.
2065 const TargetRegisterClass *DestSuperRC = TRI->getMatchingSuperRegClass(
2066 A: DefRC, B: &AMDGPU::AGPR_32RegClass, Idx: SubRegIdx);
2067 if (DestSuperRC &&
2068 TII->isInlineConstant(MO: *Lookup, OperandType: AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
2069 ++NumFoldable;
2070 NewDefs.emplace_back(Args&: Lookup, Args&: SubRegIdx);
2071 continue;
2072 }
2073 }
2074
2075 const TargetRegisterClass *InputRC =
2076 Lookup->isReg() ? MRI->getRegClass(Reg: Lookup->getReg())
2077 : MRI->getRegClass(Reg: RegOp.getReg());
2078
2079 // TODO: Account for Lookup->getSubReg()
2080
2081 // If we can't find a matching super class, this is an SGPR->AGPR or
2082 // VGPR->AGPR subreg copy (or something constant-like we have to materialize
2083 // in the AGPR). We can't directly copy from SGPR to AGPR on gfx908, so we
2084 // want to rewrite to copy to an intermediate VGPR class.
2085 const TargetRegisterClass *MatchRC =
2086 TRI->getMatchingSuperRegClass(A: DefRC, B: InputRC, Idx: SubRegIdx);
2087 if (!MatchRC) {
2088 ++NumFoldable;
2089 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2090 continue;
2091 }
2092
2093 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2094 }
2095
2096 // Do not clone a reg_sequence and merely change the result register class.
2097 if (NumFoldable == 0)
2098 return false;
2099
2100 CopyMI->setDesc(TII->get(Opcode: AMDGPU::REG_SEQUENCE));
2101 for (unsigned I = CopyMI->getNumOperands() - 1; I > 0; --I)
2102 CopyMI->removeOperand(OpNo: I);
2103
2104 for (auto [Def, DestSubIdx] : NewDefs) {
2105 if (!Def->isReg()) {
2106 // TODO: Should we use single write for each repeated value like in
2107 // register case?
2108 Register Tmp = MRI->createVirtualRegister(RegClass: &AMDGPU::AGPR_32RegClass);
2109 BuildMI(BB&: MBB, I: CopyMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: Tmp)
2110 .add(MO: *Def);
2111 B.addReg(RegNo: Tmp);
2112 } else {
2113 TargetInstrInfo::RegSubRegPair Src = getRegSubRegPair(O: *Def);
2114 Def->setIsKill(false);
2115
2116 Register &VGPRCopy = VGPRCopies[Src];
2117 if (!VGPRCopy) {
2118 const TargetRegisterClass *VGPRUseSubRC =
2119 TRI->getSubRegisterClass(UseRC, DestSubIdx);
2120
2121 // We cannot build a reg_sequence out of the same registers, they
2122 // must be copied. Better do it here before copyPhysReg() created
2123 // several reads to do the AGPR->VGPR->AGPR copy.
2124
2125 // Direct copy from SGPR to AGPR is not possible on gfx908. To avoid
2126 // creation of exploded copies SGPR->VGPR->AGPR in the copyPhysReg()
2127 // later, create a copy here and track if we already have such a copy.
2128 const TargetRegisterClass *SubRC =
2129 TRI->getSubRegisterClass(MRI->getRegClass(Reg: Src.Reg), Src.SubReg);
2130 if (!VGPRUseSubRC->hasSubClassEq(RC: SubRC)) {
2131 // TODO: Try to reconstrain class
2132 VGPRCopy = MRI->createVirtualRegister(RegClass: VGPRUseSubRC);
2133 BuildMI(BB&: MBB, I: CopyMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: VGPRCopy).add(MO: *Def);
2134 B.addReg(RegNo: VGPRCopy);
2135 } else {
2136 // If it is already a VGPR, do not copy the register.
2137 B.add(MO: *Def);
2138 }
2139 } else {
2140 B.addReg(RegNo: VGPRCopy);
2141 }
2142 }
2143
2144 B.addImm(Val: DestSubIdx);
2145 }
2146
2147 LLVM_DEBUG(dbgs() << "Folded " << *CopyMI);
2148 return true;
2149}
2150
2151bool SIFoldOperandsImpl::tryFoldFoldableCopy(
2152 MachineInstr &MI, MachineOperand *&CurrentKnownM0Val) const {
2153 Register DstReg = MI.getOperand(i: 0).getReg();
2154 // Specially track simple redefs of m0 to the same value in a block, so we
2155 // can erase the later ones.
2156 if (DstReg == AMDGPU::M0) {
2157 MachineOperand &NewM0Val = MI.getOperand(i: 1);
2158 if (CurrentKnownM0Val && CurrentKnownM0Val->isIdenticalTo(Other: NewM0Val)) {
2159 MI.eraseFromParent();
2160 return true;
2161 }
2162
2163 // We aren't tracking other physical registers
2164 CurrentKnownM0Val = (NewM0Val.isReg() && NewM0Val.getReg().isPhysical())
2165 ? nullptr
2166 : &NewM0Val;
2167 return false;
2168 }
2169
2170 MachineOperand *OpToFoldPtr;
2171 if (MI.getOpcode() == AMDGPU::V_MOV_B16_t16_e64) {
2172 // Folding when any src_modifiers are non-zero is unsupported
2173 if (TII->hasAnyModifiersSet(MI))
2174 return false;
2175 OpToFoldPtr = &MI.getOperand(i: 2);
2176 } else
2177 OpToFoldPtr = &MI.getOperand(i: 1);
2178 MachineOperand &OpToFold = *OpToFoldPtr;
2179 bool FoldingImm = OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
2180
2181 // FIXME: We could also be folding things like TargetIndexes.
2182 if (!FoldingImm && !OpToFold.isReg())
2183 return false;
2184
2185 // Fold virtual registers and constant physical registers.
2186 if (OpToFold.isReg() && OpToFold.getReg().isPhysical() &&
2187 !TRI->isConstantPhysReg(PhysReg: OpToFold.getReg()))
2188 return false;
2189
2190 // Prevent folding operands backwards in the function. For example,
2191 // the COPY opcode must not be replaced by 1 in this example:
2192 //
2193 // %3 = COPY %vgpr0; VGPR_32:%3
2194 // ...
2195 // %vgpr0 = V_MOV_B32_e32 1, implicit %exec
2196 if (!DstReg.isVirtual())
2197 return false;
2198
2199 const TargetRegisterClass *DstRC =
2200 MRI->getRegClass(Reg: MI.getOperand(i: 0).getReg());
2201
2202 // True16: Fix malformed 16-bit sgpr COPY produced by peephole-opt
2203 // Can remove this code if proper 16-bit SGPRs are implemented
2204 // Example: Pre-peephole-opt
2205 // %29:sgpr_lo16 = COPY %16.lo16:sreg_32
2206 // %32:sreg_32 = COPY %29:sgpr_lo16
2207 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2208 // Post-peephole-opt and DCE
2209 // %32:sreg_32 = COPY %16.lo16:sreg_32
2210 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2211 // After this transform
2212 // %32:sreg_32 = COPY %16:sreg_32
2213 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2214 // After the fold operands pass
2215 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %16:sreg_32
2216 if (MI.getOpcode() == AMDGPU::COPY && OpToFold.isReg() &&
2217 OpToFold.getSubReg()) {
2218 if (DstRC == &AMDGPU::SReg_32RegClass &&
2219 DstRC == MRI->getRegClass(Reg: OpToFold.getReg())) {
2220 assert(OpToFold.getSubReg() == AMDGPU::lo16);
2221 OpToFold.setSubReg(0);
2222 }
2223 }
2224
2225 // Fold copy to AGPR through reg_sequence
2226 // TODO: Handle with subregister extract
2227 if (OpToFold.isReg() && MI.isCopy() && !MI.getOperand(i: 1).getSubReg()) {
2228 if (foldCopyToAGPRRegSequence(CopyMI: &MI))
2229 return true;
2230 }
2231
2232 FoldableDef Def(OpToFold, DstRC);
2233 bool Changed = foldInstOperand(MI, OpToFold: Def);
2234
2235 // If we managed to fold all uses of this copy then we might as well
2236 // delete it now.
2237 // The only reason we need to follow chains of copies here is that
2238 // tryFoldRegSequence looks forward through copies before folding a
2239 // REG_SEQUENCE into its eventual users.
2240 auto *InstToErase = &MI;
2241 while (MRI->use_nodbg_empty(RegNo: InstToErase->getOperand(i: 0).getReg())) {
2242 auto &SrcOp = InstToErase->getOperand(i: 1);
2243 auto SrcReg = SrcOp.isReg() ? SrcOp.getReg() : Register();
2244 InstToErase->eraseFromParent();
2245 Changed = true;
2246 InstToErase = nullptr;
2247 if (!SrcReg || SrcReg.isPhysical())
2248 break;
2249 InstToErase = MRI->getVRegDef(Reg: SrcReg);
2250 if (!InstToErase || !TII->isFoldableCopy(MI: *InstToErase))
2251 break;
2252 }
2253
2254 if (InstToErase && InstToErase->isRegSequence() &&
2255 MRI->use_nodbg_empty(RegNo: InstToErase->getOperand(i: 0).getReg())) {
2256 InstToErase->eraseFromParent();
2257 Changed = true;
2258 }
2259
2260 if (Changed)
2261 return true;
2262
2263 // Run this after foldInstOperand to avoid turning scalar additions into
2264 // vector additions when the result scalar result could just be folded into
2265 // the user(s).
2266 return OpToFold.isReg() &&
2267 foldCopyToVGPROfScalarAddOfFrameIndex(DstReg, SrcReg: OpToFold.getReg(), MI);
2268}
2269
2270// Clamp patterns are canonically selected to v_max_* instructions, so only
2271// handle them.
2272const MachineOperand *
2273SIFoldOperandsImpl::isClamp(const MachineInstr &MI) const {
2274 unsigned Op = MI.getOpcode();
2275 switch (Op) {
2276 case AMDGPU::V_MAX_F32_e64:
2277 case AMDGPU::V_MAX_F16_e64:
2278 case AMDGPU::V_MAX_F16_t16_e64:
2279 case AMDGPU::V_MAX_F16_fake16_e64:
2280 case AMDGPU::V_MAX_F64_e64:
2281 case AMDGPU::V_MAX_NUM_F64_e64:
2282 case AMDGPU::V_PK_MAX_F16:
2283 case AMDGPU::V_MAX_BF16_PSEUDO_e64:
2284 case AMDGPU::V_PK_MAX_NUM_BF16: {
2285 if (MI.mayRaiseFPException())
2286 return nullptr;
2287
2288 if (!TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp)->getImm())
2289 return nullptr;
2290
2291 // Make sure sources are identical.
2292 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2293 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2294 if (!Src0->isReg() || !Src1->isReg() ||
2295 Src0->getReg() != Src1->getReg() ||
2296 Src0->getSubReg() != Src1->getSubReg() ||
2297 Src0->getSubReg() != AMDGPU::NoSubRegister)
2298 return nullptr;
2299
2300 // Can't fold up if we have modifiers.
2301 if (TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
2302 return nullptr;
2303
2304 unsigned Src0Mods
2305 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm();
2306 unsigned Src1Mods
2307 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm();
2308
2309 // Having a 0 op_sel_hi would require swizzling the output in the source
2310 // instruction, which we can't do.
2311 unsigned UnsetMods =
2312 (Op == AMDGPU::V_PK_MAX_F16 || Op == AMDGPU::V_PK_MAX_NUM_BF16)
2313 ? SISrcMods::OP_SEL_1
2314 : 0u;
2315 if (Src0Mods != UnsetMods && Src1Mods != UnsetMods)
2316 return nullptr;
2317 return Src0;
2318 }
2319 default:
2320 return nullptr;
2321 }
2322}
2323
2324// FIXME: Clamp for v_mad_mixhi_f16 handled during isel.
2325bool SIFoldOperandsImpl::tryFoldClamp(MachineInstr &MI) {
2326 const MachineOperand *ClampSrc = isClamp(MI);
2327 if (!ClampSrc || !MRI->hasOneNonDBGUser(RegNo: ClampSrc->getReg()))
2328 return false;
2329
2330 if (!ClampSrc->getReg().isVirtual())
2331 return false;
2332
2333 // Look through COPY. COPY only observed with True16.
2334 Register DefSrcReg = TRI->lookThruCopyLike(SrcReg: ClampSrc->getReg(), MRI);
2335 MachineInstr *Def =
2336 MRI->getVRegDef(Reg: DefSrcReg.isVirtual() ? DefSrcReg : ClampSrc->getReg());
2337
2338 // The type of clamp must be compatible.
2339 if (!SIInstrInfo::hasSameClamp(A: *Def, B: MI))
2340 return false;
2341
2342 if (Def->mayRaiseFPException())
2343 return false;
2344
2345 MachineOperand *DefClamp = TII->getNamedOperand(MI&: *Def, OperandName: AMDGPU::OpName::clamp);
2346 if (!DefClamp)
2347 return false;
2348
2349 LLVM_DEBUG(dbgs() << "Folding clamp " << *DefClamp << " into " << *Def);
2350
2351 // Clamp is applied after omod, so it is OK if omod is set.
2352 DefClamp->setImm(1);
2353
2354 Register DefReg = Def->getOperand(i: 0).getReg();
2355 Register MIDstReg = MI.getOperand(i: 0).getReg();
2356 if (TRI->isSGPRReg(MRI: *MRI, Reg: DefReg)) {
2357 // Pseudo scalar instructions have a SGPR for dst and clamp is a v_max*
2358 // instruction with a VGPR dst.
2359 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
2360 DestReg: MIDstReg)
2361 .addReg(RegNo: DefReg);
2362 } else {
2363 MRI->replaceRegWith(FromReg: MIDstReg, ToReg: DefReg);
2364 }
2365 MI.eraseFromParent();
2366
2367 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
2368 // instruction, so we might as well convert it to the more flexible VOP3-only
2369 // mad/fma form.
2370 if (TII->convertToThreeAddress(MI&: *Def, LV: nullptr, LIS: nullptr))
2371 Def->eraseFromParent();
2372
2373 return true;
2374}
2375
2376static int getOModValue(unsigned Opc, int64_t Val) {
2377 switch (Opc) {
2378 case AMDGPU::V_MUL_F64_e64:
2379 case AMDGPU::V_MUL_F64_pseudo_e64: {
2380 switch (Val) {
2381 case 0x3fe0000000000000: // 0.5
2382 return SIOutMods::DIV2;
2383 case 0x4000000000000000: // 2.0
2384 return SIOutMods::MUL2;
2385 case 0x4010000000000000: // 4.0
2386 return SIOutMods::MUL4;
2387 default:
2388 return SIOutMods::NONE;
2389 }
2390 }
2391 case AMDGPU::V_MUL_F32_e64: {
2392 switch (static_cast<uint32_t>(Val)) {
2393 case 0x3f000000: // 0.5
2394 return SIOutMods::DIV2;
2395 case 0x40000000: // 2.0
2396 return SIOutMods::MUL2;
2397 case 0x40800000: // 4.0
2398 return SIOutMods::MUL4;
2399 default:
2400 return SIOutMods::NONE;
2401 }
2402 }
2403 case AMDGPU::V_MUL_F16_e64:
2404 case AMDGPU::V_MUL_F16_t16_e64:
2405 case AMDGPU::V_MUL_F16_fake16_e64: {
2406 switch (static_cast<uint16_t>(Val)) {
2407 case 0x3800: // 0.5
2408 return SIOutMods::DIV2;
2409 case 0x4000: // 2.0
2410 return SIOutMods::MUL2;
2411 case 0x4400: // 4.0
2412 return SIOutMods::MUL4;
2413 default:
2414 return SIOutMods::NONE;
2415 }
2416 }
2417 case AMDGPU::V_PK_MUL_BF16: {
2418 switch (static_cast<uint16_t>(Val)) {
2419 case 0x3F00: // 0.5 in BF16
2420 return SIOutMods::DIV2;
2421 case 0x4000: // 2.0 in BF16
2422 return SIOutMods::MUL2;
2423 case 0x4080: // 4.0 in BF16
2424 return SIOutMods::MUL4;
2425 default:
2426 return SIOutMods::NONE;
2427 }
2428 }
2429 default:
2430 llvm_unreachable("invalid mul opcode");
2431 }
2432}
2433
2434// FIXME: Does this really not support denormals with f16?
2435// FIXME: Does this need to check IEEE mode bit? SNaNs are generally not
2436// handled, so will anything other than that break?
2437std::pair<const MachineOperand *, int>
2438SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
2439 unsigned Op = MI.getOpcode();
2440 switch (Op) {
2441 case AMDGPU::V_MUL_F64_e64:
2442 case AMDGPU::V_MUL_F64_pseudo_e64:
2443 case AMDGPU::V_MUL_F32_e64:
2444 case AMDGPU::V_MUL_F16_t16_e64:
2445 case AMDGPU::V_MUL_F16_fake16_e64:
2446 case AMDGPU::V_MUL_F16_e64: {
2447 // If output denormals are enabled, omod is ignored.
2448 if ((Op == AMDGPU::V_MUL_F32_e64 &&
2449 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
2450 ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F64_pseudo_e64 ||
2451 Op == AMDGPU::V_MUL_F16_e64 || Op == AMDGPU::V_MUL_F16_t16_e64 ||
2452 Op == AMDGPU::V_MUL_F16_fake16_e64) &&
2453 MFI->getMode().FP64FP16Denormals.Output !=
2454 DenormalMode::PreserveSign) ||
2455 MI.mayRaiseFPException())
2456 return {nullptr, SIOutMods::NONE};
2457
2458 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2459 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2460
2461 // If there is an immediate operand, it must be Src1
2462 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
2463 if (!Src1Imm)
2464 return {nullptr, SIOutMods::NONE};
2465
2466 int OMod = getOModValue(Opc: Op, Val: *Src1Imm);
2467 if (OMod == SIOutMods::NONE ||
2468 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers) ||
2469 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers) ||
2470 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2471 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2472 return {nullptr, SIOutMods::NONE};
2473
2474 return {Src0, OMod};
2475 }
2476 case AMDGPU::V_ADD_F64_e64:
2477 case AMDGPU::V_ADD_F64_pseudo_e64:
2478 case AMDGPU::V_ADD_F32_e64:
2479 case AMDGPU::V_ADD_F16_e64:
2480 case AMDGPU::V_ADD_F16_t16_e64:
2481 case AMDGPU::V_ADD_F16_fake16_e64: {
2482 // If output denormals are enabled, omod is ignored.
2483 if ((Op == AMDGPU::V_ADD_F32_e64 &&
2484 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
2485 ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F64_pseudo_e64 ||
2486 Op == AMDGPU::V_ADD_F16_e64 || Op == AMDGPU::V_ADD_F16_t16_e64 ||
2487 Op == AMDGPU::V_ADD_F16_fake16_e64) &&
2488 MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign))
2489 return {nullptr, SIOutMods::NONE};
2490
2491 // Look through the DAGCombiner canonicalization fmul x, 2 -> fadd x, x
2492 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2493 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2494
2495 if (Src0->isReg() && Src1->isReg() && Src0->getReg() == Src1->getReg() &&
2496 Src0->getSubReg() == Src1->getSubReg() &&
2497 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers) &&
2498 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers) &&
2499 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) &&
2500 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
2501 return {Src0, SIOutMods::MUL2};
2502
2503 return {nullptr, SIOutMods::NONE};
2504 }
2505 case AMDGPU::V_PK_MUL_BF16: {
2506 // OMOD folding for BF16 packed multiply. bf16 has no denormal mode of its
2507 // own; it follows the default ("denormal-fp-math") mode, which is the same
2508 // field as f64/f16.
2509 if (MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign ||
2510 MI.mayRaiseFPException())
2511 return {nullptr, SIOutMods::NONE};
2512
2513 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2514 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2515
2516 // If there is an immediate operand, it must be Src1
2517 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
2518 if (!Src1Imm)
2519 return {nullptr, SIOutMods::NONE};
2520
2521 int OMod = getOModValue(Opc: AMDGPU::V_PK_MUL_BF16, Val: *Src1Imm);
2522 if (OMod == SIOutMods::NONE)
2523 return {nullptr, SIOutMods::NONE};
2524
2525 // Modifiers other than op_sel_hi block OMOD folding
2526 const MachineOperand *Src0Mods =
2527 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2528 const MachineOperand *Src1Mods =
2529 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2530 if ((Src0Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2531 (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2532 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2533 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2534 return {nullptr, SIOutMods::NONE};
2535
2536 return {Src0, OMod};
2537 }
2538 case AMDGPU::V_PK_ADD_BF16: {
2539 // OMOD folding for BF16 packed add: x + x -> x * 2. See the bf16 denormal
2540 // mode note in the V_PK_MUL_BF16 case above.
2541 if (MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign)
2542 return {nullptr, SIOutMods::NONE};
2543
2544 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2545 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2546
2547 if (!Src0->isReg() || !Src1->isReg() || Src0->getReg() != Src1->getReg() ||
2548 Src0->getSubReg() != Src1->getSubReg())
2549 return {nullptr, SIOutMods::NONE};
2550
2551 // Modifiers other than op_sel_hi block OMOD folding
2552 const MachineOperand *Src0Mods =
2553 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2554 const MachineOperand *Src1Mods =
2555 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2556 if ((Src0Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2557 (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2558 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2559 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2560 return {nullptr, SIOutMods::NONE};
2561
2562 return {Src0, SIOutMods::MUL2};
2563 }
2564 default:
2565 return {nullptr, SIOutMods::NONE};
2566 }
2567}
2568
2569// FIXME: Does this need to check IEEE bit on function?
2570bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
2571 const MachineOperand *RegOp;
2572 int OMod;
2573 std::tie(args&: RegOp, args&: OMod) = isOMod(MI);
2574 if (OMod == SIOutMods::NONE || !RegOp->isReg() ||
2575 RegOp->getSubReg() != AMDGPU::NoSubRegister ||
2576 !MRI->hasOneNonDBGUser(RegNo: RegOp->getReg()))
2577 return false;
2578
2579 MachineInstr *Def = MRI->getVRegDef(Reg: RegOp->getReg());
2580 Register OModSrcReg = Def->getOperand(i: 0).getReg();
2581
2582 // In real-true16 mode, vgpr_16 results are packed into vgpr_32 via
2583 // REG_SEQUENCE. Look through it to find the actual instruction.
2584 if (Def->isRegSequence() && Def->getNumOperands() == 5 &&
2585 Def->getOperand(i: 2).getImm() == AMDGPU::lo16) {
2586 // Only look through if the high 16 bits are undefined
2587 bool CanLookThrough = true;
2588 MachineInstr *Hi16Def = MRI->getVRegDef(Reg: Def->getOperand(i: 3).getReg());
2589 if (!Hi16Def || !Hi16Def->isImplicitDef())
2590 CanLookThrough = false;
2591
2592 if (CanLookThrough) {
2593 Register SrcReg = Def->getOperand(i: 1).getReg();
2594 if (!MRI->hasOneNonDBGUse(RegNo: SrcReg))
2595 return false;
2596
2597 Def = MRI->getVRegDef(Reg: SrcReg);
2598 if (!Def)
2599 return false;
2600 }
2601 }
2602
2603 MachineOperand *DefOMod = TII->getNamedOperand(MI&: *Def, OperandName: AMDGPU::OpName::omod);
2604 if (!DefOMod || DefOMod->getImm() != SIOutMods::NONE)
2605 return false;
2606
2607 if (Def->mayRaiseFPException())
2608 return false;
2609
2610 // Clamp is applied after omod. If the source already has clamp set, don't
2611 // fold it.
2612 if (TII->hasModifiersSet(MI: *Def, OpName: AMDGPU::OpName::clamp))
2613 return false;
2614
2615 LLVM_DEBUG(dbgs() << "Folding omod " << MI << " into " << *Def);
2616
2617 DefOMod->setImm(OMod);
2618 MRI->replaceRegWith(FromReg: MI.getOperand(i: 0).getReg(), ToReg: OModSrcReg);
2619 // Kill flags can be wrong if we replaced a def inside a loop with a def
2620 // outside the loop.
2621 MRI->clearKillFlags(Reg: OModSrcReg);
2622 MI.eraseFromParent();
2623
2624 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
2625 // instruction, so we might as well convert it to the more flexible VOP3-only
2626 // mad/fma form.
2627 if (TII->convertToThreeAddress(MI&: *Def, LV: nullptr, LIS: nullptr))
2628 Def->eraseFromParent();
2629
2630 return true;
2631}
2632
2633// Try to optimize SGPR reg sequences that are splat <s, s> or <s, s, s, s>
2634// where all uses are PackedSingleSGPR64BitInst, replacing with <s, undef, ...>
2635bool SIFoldOperandsImpl::tryFoldSGPRSplatRegSequence(MachineInstr &MI) {
2636 assert(MI.isRegSequence());
2637
2638 if (!ST->hasPackedFP64SingleSGPROps() && !ST->hasPackedU64SingleSGPROps())
2639 return false;
2640
2641 Register Reg = MI.getOperand(i: 0).getReg();
2642
2643 // Only optimize 128-bit SGPR register sequences
2644 const TargetRegisterClass *RegClass = MRI->getRegClass(Reg);
2645 if (!TRI->isSGPRClass(RC: RegClass) || TRI->getRegSizeInBits(RC: *RegClass) != 128)
2646 return false;
2647
2648 SmallVector<std::pair<MachineOperand *, unsigned>, 32> Defs;
2649 if (!getRegSeqInit(Defs, UseReg: Reg))
2650 return false;
2651
2652 // Check if this is a splat pattern
2653 if (Defs.size() <= 1)
2654 return false;
2655
2656 const auto &[FirstOp, _] = Defs.front();
2657 if (!FirstOp->isReg())
2658 return false;
2659
2660 Register FirstReg = FirstOp->getReg();
2661 unsigned FirstSubReg = FirstOp->getSubReg();
2662
2663 const TargetRegisterClass *FirstRegClass = MRI->getRegClass(Reg: FirstReg);
2664 if (!TRI->isSGPRClass(RC: FirstRegClass))
2665 return false;
2666
2667 // Check remaining elements match first
2668 if (!llvm::all_of(Range: llvm::drop_begin(RangeOrContainer&: Defs), P: [&](const auto &Def) {
2669 const auto &[Op, _] = Def;
2670 return Op->isReg() && Op->getReg() == FirstReg &&
2671 Op->getSubReg() == FirstSubReg;
2672 }))
2673 return false;
2674
2675 // Check if all uses are isSingleSGPRReadInst
2676 for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
2677 if (!AMDGPU::isPackedSingleSGPR64BitInst(Opc: UseMI.getOpcode()))
2678 return false;
2679 }
2680
2681 // Create new reg sequence with <s, undef, undef, ...>
2682 Register NewDst = MRI->createVirtualRegister(RegClass);
2683 MachineInstrBuilder RS = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2684 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDst);
2685
2686 // Add the first operand
2687 FirstOp->setIsKill(false);
2688 RS.add(MO: *FirstOp);
2689 RS.addImm(Val: Defs[0].second);
2690
2691 // Add undef for remaining lanes
2692 // Create an undef virtual register for the same register class
2693 Register UndefReg = MRI->createVirtualRegister(RegClass: FirstRegClass);
2694 for (unsigned i = 1; i < Defs.size(); ++i) {
2695 RS.addReg(RegNo: UndefReg, Flags: RegState::Undef);
2696 RS.addImm(Val: Defs[i].second);
2697 }
2698
2699 // Replace all uses
2700 MRI->replaceRegWith(FromReg: Reg, ToReg: NewDst);
2701
2702 LLVM_DEBUG(dbgs() << "Folded splat SGPR reg_sequence: " << MI << " into "
2703 << *RS);
2704
2705 MI.eraseFromParent();
2706 return true;
2707}
2708
2709// Try to fold a reg_sequence with vgpr output and agpr inputs into an
2710// instruction which can take an agpr. So far that means a store.
2711bool SIFoldOperandsImpl::tryFoldRegSequence(MachineInstr &MI) {
2712 assert(MI.isRegSequence());
2713
2714 // Try to optimize SGPR splat sequences first
2715 if (tryFoldSGPRSplatRegSequence(MI))
2716 return true;
2717
2718 auto Reg = MI.getOperand(i: 0).getReg();
2719
2720 if (!ST->hasGFX90AInsts() || !TRI->isVGPR(MRI: *MRI, Reg) ||
2721 !MRI->hasOneNonDBGUse(RegNo: Reg))
2722 return false;
2723
2724 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
2725 if (!getRegSeqInit(Defs, UseReg: Reg))
2726 return false;
2727
2728 for (auto &[Op, SubIdx] : Defs) {
2729 if (!Op->isReg())
2730 return false;
2731 if (TRI->isAGPR(MRI: *MRI, Reg: Op->getReg()))
2732 continue;
2733 // Maybe this is a COPY from AREG
2734 const MachineInstr *SubDef = MRI->getVRegDef(Reg: Op->getReg());
2735 if (!SubDef || !SubDef->isCopy() || SubDef->getOperand(i: 1).getSubReg())
2736 return false;
2737 if (!TRI->isAGPR(MRI: *MRI, Reg: SubDef->getOperand(i: 1).getReg()))
2738 return false;
2739 }
2740
2741 MachineOperand *Op = &*MRI->use_nodbg_begin(RegNo: Reg);
2742 MachineInstr *UseMI = Op->getParent();
2743 while (UseMI->isCopy() && !Op->getSubReg()) {
2744 Reg = UseMI->getOperand(i: 0).getReg();
2745 if (!TRI->isVGPR(MRI: *MRI, Reg) || !MRI->hasOneNonDBGUse(RegNo: Reg))
2746 return false;
2747 Op = &*MRI->use_nodbg_begin(RegNo: Reg);
2748 UseMI = Op->getParent();
2749 }
2750
2751 if (Op->getSubReg())
2752 return false;
2753
2754 unsigned OpIdx = Op - &UseMI->getOperand(i: 0);
2755 const MCInstrDesc &InstDesc = UseMI->getDesc();
2756 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: InstDesc, OpNum: OpIdx);
2757 if (!OpRC || !TRI->isVectorSuperClass(RC: OpRC))
2758 return false;
2759
2760 const auto *NewDstRC = TRI->getEquivalentAGPRClass(SRC: MRI->getRegClass(Reg));
2761 auto Dst = MRI->createVirtualRegister(RegClass: NewDstRC);
2762 auto RS = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2763 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst);
2764
2765 for (auto &[Def, SubIdx] : Defs) {
2766 Def->setIsKill(false);
2767 if (TRI->isAGPR(MRI: *MRI, Reg: Def->getReg())) {
2768 RS.add(MO: *Def);
2769 } else { // This is a copy
2770 MachineInstr *SubDef = MRI->getVRegDef(Reg: Def->getReg());
2771 SubDef->getOperand(i: 1).setIsKill(false);
2772 RS.addReg(RegNo: SubDef->getOperand(i: 1).getReg(), Flags: {}, SubReg: Def->getSubReg());
2773 }
2774 RS.addImm(Val: SubIdx);
2775 }
2776
2777 Op->setReg(Dst);
2778 if (!TII->isOperandLegal(MI: *UseMI, OpIdx, MO: Op)) {
2779 Op->setReg(Reg);
2780 RS->eraseFromParent();
2781 return false;
2782 }
2783
2784 LLVM_DEBUG(dbgs() << "Folded " << *RS << " into " << *UseMI);
2785
2786 // Erase the REG_SEQUENCE eagerly, unless we followed a chain of COPY users,
2787 // in which case we can erase them all later in runOnMachineFunction.
2788 if (MRI->use_nodbg_empty(RegNo: MI.getOperand(i: 0).getReg()))
2789 MI.eraseFromParent();
2790 return true;
2791}
2792
2793/// Checks whether \p Copy is a AGPR -> VGPR copy. Returns `true` on success and
2794/// stores the AGPR register in \p OutReg and the subreg in \p OutSubReg
2795static bool isAGPRCopy(const SIRegisterInfo &TRI,
2796 const MachineRegisterInfo &MRI, const MachineInstr &Copy,
2797 Register &OutReg, unsigned &OutSubReg) {
2798 assert(Copy.isCopy());
2799
2800 const MachineOperand &CopySrc = Copy.getOperand(i: 1);
2801 Register CopySrcReg = CopySrc.getReg();
2802 if (!CopySrcReg.isVirtual())
2803 return false;
2804
2805 // Common case: copy from AGPR directly, e.g.
2806 // %1:vgpr_32 = COPY %0:agpr_32
2807 if (TRI.isAGPR(MRI, Reg: CopySrcReg)) {
2808 OutReg = CopySrcReg;
2809 OutSubReg = CopySrc.getSubReg();
2810 return true;
2811 }
2812
2813 // Sometimes it can also involve two copies, e.g.
2814 // %1:vgpr_256 = COPY %0:agpr_256
2815 // %2:vgpr_32 = COPY %1:vgpr_256.sub0
2816 const MachineInstr *CopySrcDef = MRI.getVRegDef(Reg: CopySrcReg);
2817 if (!CopySrcDef || !CopySrcDef->isCopy())
2818 return false;
2819
2820 const MachineOperand &OtherCopySrc = CopySrcDef->getOperand(i: 1);
2821 Register OtherCopySrcReg = OtherCopySrc.getReg();
2822 if (!OtherCopySrcReg.isVirtual() ||
2823 CopySrcDef->getOperand(i: 0).getSubReg() != AMDGPU::NoSubRegister ||
2824 OtherCopySrc.getSubReg() != AMDGPU::NoSubRegister ||
2825 !TRI.isAGPR(MRI, Reg: OtherCopySrcReg))
2826 return false;
2827
2828 OutReg = OtherCopySrcReg;
2829 OutSubReg = CopySrc.getSubReg();
2830 return true;
2831}
2832
2833// Try to hoist an AGPR to VGPR copy across a PHI.
2834// This should allow folding of an AGPR into a consumer which may support it.
2835//
2836// Example 1: LCSSA PHI
2837// loop:
2838// %1:vreg = COPY %0:areg
2839// exit:
2840// %2:vreg = PHI %1:vreg, %loop
2841// =>
2842// loop:
2843// exit:
2844// %1:areg = PHI %0:areg, %loop
2845// %2:vreg = COPY %1:areg
2846//
2847// Example 2: PHI with multiple incoming values:
2848// entry:
2849// %1:vreg = GLOBAL_LOAD(..)
2850// loop:
2851// %2:vreg = PHI %1:vreg, %entry, %5:vreg, %loop
2852// %3:areg = COPY %2:vreg
2853// %4:areg = (instr using %3:areg)
2854// %5:vreg = COPY %4:areg
2855// =>
2856// entry:
2857// %1:vreg = GLOBAL_LOAD(..)
2858// %2:areg = COPY %1:vreg
2859// loop:
2860// %3:areg = PHI %2:areg, %entry, %X:areg,
2861// %4:areg = (instr using %3:areg)
2862bool SIFoldOperandsImpl::tryFoldPhiAGPR(MachineInstr &PHI) {
2863 assert(PHI.isPHI());
2864
2865 Register PhiOut = PHI.getOperand(i: 0).getReg();
2866 if (!TRI->isVGPR(MRI: *MRI, Reg: PhiOut))
2867 return false;
2868
2869 // Iterate once over all incoming values of the PHI to check if this PHI is
2870 // eligible, and determine the exact AGPR RC we'll target.
2871 const TargetRegisterClass *ARC = nullptr;
2872 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
2873 MachineOperand &MO = PHI.getOperand(i: K);
2874 MachineInstr *Copy = MRI->getVRegDef(Reg: MO.getReg());
2875 if (!Copy || !Copy->isCopy())
2876 continue;
2877
2878 Register AGPRSrc;
2879 unsigned AGPRRegMask = AMDGPU::NoSubRegister;
2880 if (!isAGPRCopy(TRI: *TRI, MRI: *MRI, Copy: *Copy, OutReg&: AGPRSrc, OutSubReg&: AGPRRegMask))
2881 continue;
2882
2883 const TargetRegisterClass *CopyInRC = MRI->getRegClass(Reg: AGPRSrc);
2884 if (const auto *SubRC = TRI->getSubRegisterClass(CopyInRC, AGPRRegMask))
2885 CopyInRC = SubRC;
2886
2887 if (ARC && !ARC->hasSubClassEq(RC: CopyInRC))
2888 return false;
2889 ARC = CopyInRC;
2890 }
2891
2892 if (!ARC)
2893 return false;
2894
2895 bool IsAGPR32 = (ARC == &AMDGPU::AGPR_32RegClass);
2896
2897 // Rewrite the PHI's incoming values to ARC.
2898 LLVM_DEBUG(dbgs() << "Folding AGPR copies into: " << PHI);
2899 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
2900 MachineOperand &MO = PHI.getOperand(i: K);
2901 Register Reg = MO.getReg();
2902
2903 MachineBasicBlock::iterator InsertPt;
2904 MachineBasicBlock *InsertMBB = nullptr;
2905
2906 // Look at the def of Reg, ignoring all copies.
2907 unsigned CopyOpc = AMDGPU::COPY;
2908 if (MachineInstr *Def = MRI->getVRegDef(Reg)) {
2909
2910 // Look at pre-existing COPY instructions from ARC: Steal the operand. If
2911 // the copy was single-use, it will be removed by DCE later.
2912 if (Def->isCopy()) {
2913 Register AGPRSrc;
2914 unsigned AGPRSubReg = AMDGPU::NoSubRegister;
2915 if (isAGPRCopy(TRI: *TRI, MRI: *MRI, Copy: *Def, OutReg&: AGPRSrc, OutSubReg&: AGPRSubReg)) {
2916 MO.setReg(AGPRSrc);
2917 MO.setSubReg(AGPRSubReg);
2918 continue;
2919 }
2920
2921 // If this is a multi-use SGPR -> VGPR copy, use V_ACCVGPR_WRITE on
2922 // GFX908 directly instead of a COPY. Otherwise, SIFoldOperand may try
2923 // to fold the sgpr -> vgpr -> agpr copy into a sgpr -> agpr copy which
2924 // is unlikely to be profitable.
2925 //
2926 // Note that V_ACCVGPR_WRITE is only used for AGPR_32.
2927 MachineOperand &CopyIn = Def->getOperand(i: 1);
2928 if (IsAGPR32 && !ST->hasGFX90AInsts() && !MRI->hasOneNonDBGUse(RegNo: Reg) &&
2929 TRI->isSGPRReg(MRI: *MRI, Reg: CopyIn.getReg()))
2930 CopyOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
2931 }
2932
2933 InsertMBB = Def->getParent();
2934 InsertPt = InsertMBB->SkipPHIsLabelsAndDebug(I: ++Def->getIterator());
2935 } else {
2936 InsertMBB = PHI.getOperand(i: MO.getOperandNo() + 1).getMBB();
2937 InsertPt = InsertMBB->getFirstTerminator();
2938 }
2939
2940 Register NewReg = MRI->createVirtualRegister(RegClass: ARC);
2941 MachineInstr *MI = BuildMI(BB&: *InsertMBB, I: InsertPt, MIMD: PHI.getDebugLoc(),
2942 MCID: TII->get(Opcode: CopyOpc), DestReg: NewReg)
2943 .addReg(RegNo: Reg);
2944 MO.setReg(NewReg);
2945
2946 (void)MI;
2947 LLVM_DEBUG(dbgs() << " Created COPY: " << *MI);
2948 }
2949
2950 // Replace the PHI's result with a new register.
2951 Register NewReg = MRI->createVirtualRegister(RegClass: ARC);
2952 PHI.getOperand(i: 0).setReg(NewReg);
2953
2954 // COPY that new register back to the original PhiOut register. This COPY will
2955 // usually be folded out later.
2956 MachineBasicBlock *MBB = PHI.getParent();
2957 BuildMI(BB&: *MBB, I: MBB->getFirstNonPHI(), MIMD: PHI.getDebugLoc(),
2958 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: PhiOut)
2959 .addReg(RegNo: NewReg);
2960
2961 LLVM_DEBUG(dbgs() << " Done: Folded " << PHI);
2962 return true;
2963}
2964
2965// Attempt to convert VGPR load to an AGPR load.
2966bool SIFoldOperandsImpl::tryFoldLoad(MachineInstr &MI) {
2967 assert(MI.mayLoad());
2968 if (!ST->hasGFX90AInsts() || MI.getNumExplicitDefs() != 1)
2969 return false;
2970
2971 MachineOperand &Def = MI.getOperand(i: 0);
2972 if (!Def.isDef())
2973 return false;
2974
2975 Register DefReg = Def.getReg();
2976
2977 if (DefReg.isPhysical() || !TRI->isVGPR(MRI: *MRI, Reg: DefReg))
2978 return false;
2979
2980 SmallVector<const MachineInstr *, 8> Users(
2981 llvm::make_pointer_range(Range: MRI->use_nodbg_instructions(Reg: DefReg)));
2982 SmallVector<Register, 8> MoveRegs;
2983
2984 if (Users.empty())
2985 return false;
2986
2987 // Check that all uses a copy to an agpr or a reg_sequence producing an agpr.
2988 while (!Users.empty()) {
2989 const MachineInstr *I = Users.pop_back_val();
2990 if (!I->isCopy() && !I->isRegSequence())
2991 return false;
2992 Register DstReg = I->getOperand(i: 0).getReg();
2993 // Physical registers may have more than one instruction definitions
2994 if (DstReg.isPhysical())
2995 return false;
2996 if (TRI->isAGPR(MRI: *MRI, Reg: DstReg))
2997 continue;
2998 MoveRegs.push_back(Elt: DstReg);
2999 for (const MachineInstr &U : MRI->use_nodbg_instructions(Reg: DstReg))
3000 Users.push_back(Elt: &U);
3001 }
3002
3003 const TargetRegisterClass *RC = MRI->getRegClass(Reg: DefReg);
3004 MRI->setRegClass(Reg: DefReg, RC: TRI->getEquivalentAGPRClass(SRC: RC));
3005 if (!TII->isOperandLegal(MI, OpIdx: 0, MO: &Def)) {
3006 MRI->setRegClass(Reg: DefReg, RC);
3007 return false;
3008 }
3009
3010 while (!MoveRegs.empty()) {
3011 Register Reg = MoveRegs.pop_back_val();
3012 MRI->setRegClass(Reg, RC: TRI->getEquivalentAGPRClass(SRC: MRI->getRegClass(Reg)));
3013 }
3014
3015 LLVM_DEBUG(dbgs() << "Folded " << MI);
3016
3017 return true;
3018}
3019
3020// tryFoldPhiAGPR will aggressively try to create AGPR PHIs.
3021// For GFX90A and later, this is pretty much always a good thing, but for GFX908
3022// there's cases where it can create a lot more AGPR-AGPR copies, which are
3023// expensive on this architecture due to the lack of V_ACCVGPR_MOV.
3024//
3025// This function looks at all AGPR PHIs in a basic block and collects their
3026// operands. Then, it checks for register that are used more than once across
3027// all PHIs and caches them in a VGPR. This prevents ExpandPostRAPseudo from
3028// having to create one VGPR temporary per use, which can get very messy if
3029// these PHIs come from a broken-up large PHI (e.g. 32 AGPR phis, one per vector
3030// element).
3031//
3032// Example
3033// a:
3034// %in:agpr_256 = COPY %foo:vgpr_256
3035// c:
3036// %x:agpr_32 = ..
3037// b:
3038// %0:areg = PHI %in.sub0:agpr_32, %a, %x, %c
3039// %1:areg = PHI %in.sub0:agpr_32, %a, %y, %c
3040// %2:areg = PHI %in.sub0:agpr_32, %a, %z, %c
3041// =>
3042// a:
3043// %in:agpr_256 = COPY %foo:vgpr_256
3044// %tmp:vgpr_32 = V_ACCVGPR_READ_B32_e64 %in.sub0:agpr_32
3045// %tmp_agpr:agpr_32 = COPY %tmp
3046// c:
3047// %x:agpr_32 = ..
3048// b:
3049// %0:areg = PHI %tmp_agpr, %a, %x, %c
3050// %1:areg = PHI %tmp_agpr, %a, %y, %c
3051// %2:areg = PHI %tmp_agpr, %a, %z, %c
3052bool SIFoldOperandsImpl::tryOptimizeAGPRPhis(MachineBasicBlock &MBB) {
3053 // This is only really needed on GFX908 where AGPR-AGPR copies are
3054 // unreasonably difficult.
3055 if (ST->hasGFX90AInsts())
3056 return false;
3057
3058 // Look at all AGPR Phis and collect the register + subregister used.
3059 DenseMap<std::pair<Register, unsigned>, std::vector<MachineOperand *>>
3060 RegToMO;
3061
3062 for (auto &MI : MBB) {
3063 if (!MI.isPHI())
3064 break;
3065
3066 if (!TRI->isAGPR(MRI: *MRI, Reg: MI.getOperand(i: 0).getReg()))
3067 continue;
3068
3069 for (unsigned K = 1; K < MI.getNumOperands(); K += 2) {
3070 MachineOperand &PhiMO = MI.getOperand(i: K);
3071 if (!PhiMO.getSubReg())
3072 continue;
3073 RegToMO[{PhiMO.getReg(), PhiMO.getSubReg()}].push_back(x: &PhiMO);
3074 }
3075 }
3076
3077 // For all (Reg, SubReg) pair that are used more than once, cache the value in
3078 // a VGPR.
3079 bool Changed = false;
3080 for (const auto &[Entry, MOs] : RegToMO) {
3081 if (MOs.size() == 1)
3082 continue;
3083
3084 const auto [Reg, SubReg] = Entry;
3085 MachineInstr *Def = MRI->getVRegDef(Reg);
3086 MachineBasicBlock *DefMBB = Def->getParent();
3087
3088 // Create a copy in a VGPR using V_ACCVGPR_READ_B32_e64 so it's not folded
3089 // out.
3090 const TargetRegisterClass *ARC = getRegOpRC(MRI: *MRI, TRI: *TRI, MO: *MOs.front());
3091 Register TempVGPR =
3092 MRI->createVirtualRegister(RegClass: TRI->getEquivalentVGPRClass(SRC: ARC));
3093 MachineInstr *VGPRCopy =
3094 BuildMI(BB&: *DefMBB, I: ++Def->getIterator(), MIMD: Def->getDebugLoc(),
3095 MCID: TII->get(Opcode: AMDGPU::V_ACCVGPR_READ_B32_e64), DestReg: TempVGPR)
3096 .addReg(RegNo: Reg, /* flags */ Flags: {}, SubReg);
3097
3098 // Copy back to an AGPR and use that instead of the AGPR subreg in all MOs.
3099 Register TempAGPR = MRI->createVirtualRegister(RegClass: ARC);
3100 BuildMI(BB&: *DefMBB, I: ++VGPRCopy->getIterator(), MIMD: Def->getDebugLoc(),
3101 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: TempAGPR)
3102 .addReg(RegNo: TempVGPR);
3103
3104 LLVM_DEBUG(dbgs() << "Caching AGPR into VGPR: " << *VGPRCopy);
3105 for (MachineOperand *MO : MOs) {
3106 MO->setReg(TempAGPR);
3107 MO->setSubReg(AMDGPU::NoSubRegister);
3108 LLVM_DEBUG(dbgs() << " Changed PHI Operand: " << *MO << "\n");
3109 }
3110
3111 Changed = true;
3112 }
3113
3114 return Changed;
3115}
3116
3117bool SIFoldOperandsImpl::run(MachineFunction &MF, const MachineLoopInfo *MLI) {
3118 this->MF = &MF;
3119 MRI = &MF.getRegInfo();
3120 ST = &MF.getSubtarget<GCNSubtarget>();
3121 TII = ST->getInstrInfo();
3122 TRI = &TII->getRegisterInfo();
3123 MFI = MF.getInfo<SIMachineFunctionInfo>();
3124 this->MLI = MLI;
3125
3126 // omod is ignored by hardware if IEEE bit is enabled. omod also does not
3127 // correctly handle signed zeros.
3128 //
3129 // FIXME: Also need to check strictfp
3130 bool IsIEEEMode = MFI->getMode().IEEE;
3131
3132 bool Changed = false;
3133 for (MachineBasicBlock *MBB : depth_first(G: &MF)) {
3134 MachineOperand *CurrentKnownM0Val = nullptr;
3135 for (auto &MI : make_early_inc_range(Range&: *MBB)) {
3136 Changed |= tryFoldCndMask(MI);
3137
3138 // PeepholeOptimizer may have folded an inline immediate directly onto an
3139 // instruction operand without materializing it into a register first.
3140 // Such an instruction is never reached through a def->use edge in
3141 // foldInstOperand, so try to constant fold it here.
3142 if (tryConstantFoldOp(MI: &MI)) {
3143 Changed = true;
3144 continue;
3145 }
3146
3147 if (tryFoldRedundantAND(ChildMI&: MI)) {
3148 Changed = true;
3149 continue;
3150 }
3151
3152 if (MI.isRegSequence() && tryFoldRegSequence(MI)) {
3153 Changed = true;
3154 continue;
3155 }
3156
3157 if (MI.isPHI() && tryFoldPhiAGPR(PHI&: MI)) {
3158 Changed = true;
3159 continue;
3160 }
3161
3162 if (MI.mayLoad() && tryFoldLoad(MI)) {
3163 Changed = true;
3164 continue;
3165 }
3166
3167 if (TII->isFoldableCopy(MI)) {
3168 Changed |= tryFoldFoldableCopy(MI, CurrentKnownM0Val);
3169 continue;
3170 }
3171
3172 // Saw an unknown clobber of m0, so we no longer know what it is.
3173 if (CurrentKnownM0Val && MI.modifiesRegister(Reg: AMDGPU::M0, TRI))
3174 CurrentKnownM0Val = nullptr;
3175
3176 // TODO: Omod might be OK if there is NSZ only on the source
3177 // instruction, and not the omod multiply.
3178 if (IsIEEEMode || !MI.getFlag(Flag: MachineInstr::FmNsz) || !tryFoldOMod(MI))
3179 Changed |= tryFoldClamp(MI);
3180 }
3181
3182 Changed |= tryOptimizeAGPRPhis(MBB&: *MBB);
3183 }
3184
3185 return Changed;
3186}
3187
3188PreservedAnalyses
3189SIFoldOperandsPass::run(MachineFunction &MF,
3190 MachineFunctionAnalysisManager &MFAM) {
3191 MFPropsModifier _(*this, MF);
3192
3193 const MachineLoopInfo *MLI = &MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
3194 bool Changed = SIFoldOperandsImpl().run(MF, MLI);
3195 if (!Changed) {
3196 return PreservedAnalyses::all();
3197 }
3198 auto PA = getMachineFunctionPassPreservedAnalyses();
3199 PA.preserveSet<CFGAnalyses>();
3200 PA.preserve<MachineLoopAnalysis>();
3201 return PA;
3202}
3203