1//===-- SIFoldOperands.cpp - Fold operands --- ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7/// \file
8//===----------------------------------------------------------------------===//
9//
10
11#include "SIFoldOperands.h"
12#include "AMDGPU.h"
13#include "GCNSubtarget.h"
14#include "SIInstrInfo.h"
15#include "SIMachineFunctionInfo.h"
16#include "SIRegisterInfo.h"
17#include "llvm/ADT/DepthFirstIterator.h"
18#include "llvm/CodeGen/MachineFunction.h"
19#include "llvm/CodeGen/MachineFunctionPass.h"
20#include "llvm/CodeGen/MachineLoopInfo.h"
21#include "llvm/CodeGen/MachineOperand.h"
22#include "llvm/InitializePasses.h"
23
24#define DEBUG_TYPE "si-fold-operands"
25using namespace llvm;
26
27namespace {
28
29/// Track a value we may want to fold into downstream users, applying
30/// subregister extracts along the way.
31struct FoldableDef {
32 union {
33 MachineOperand *OpToFold = nullptr;
34 uint64_t ImmToFold;
35 int FrameIndexToFold;
36 };
37
38 /// Register class of the originally defined value.
39 const TargetRegisterClass *DefRC = nullptr;
40
41 /// Track the original defining instruction for the value.
42 const MachineInstr *DefMI = nullptr;
43
44 /// Subregister to apply to the value at the use point.
45 unsigned DefSubReg = AMDGPU::NoSubRegister;
46
47 /// Kind of value stored in the union.
48 MachineOperand::MachineOperandType Kind;
49
50 FoldableDef() = delete;
51 FoldableDef(MachineOperand &FoldOp, const TargetRegisterClass *DefRC,
52 unsigned DefSubReg = AMDGPU::NoSubRegister)
53 : DefRC(DefRC), DefSubReg(DefSubReg), Kind(FoldOp.getType()) {
54
55 if (FoldOp.isImm()) {
56 ImmToFold = FoldOp.getImm();
57 } else if (FoldOp.isFI()) {
58 FrameIndexToFold = FoldOp.getIndex();
59 } else {
60 assert(FoldOp.isReg() || FoldOp.isGlobal());
61 OpToFold = &FoldOp;
62 }
63
64 DefMI = FoldOp.getParent();
65 }
66
67 FoldableDef(int64_t FoldImm, const TargetRegisterClass *DefRC,
68 unsigned DefSubReg = AMDGPU::NoSubRegister)
69 : ImmToFold(FoldImm), DefRC(DefRC), DefSubReg(DefSubReg),
70 Kind(MachineOperand::MO_Immediate) {}
71
72 /// Copy the current def and apply \p SubReg to the value.
73 FoldableDef getWithSubReg(const SIRegisterInfo &TRI, unsigned SubReg) const {
74 FoldableDef Copy(*this);
75 Copy.DefSubReg = TRI.composeSubRegIndices(a: DefSubReg, b: SubReg);
76 return Copy;
77 }
78
79 bool isReg() const { return Kind == MachineOperand::MO_Register; }
80
81 Register getReg() const {
82 assert(isReg());
83 return OpToFold->getReg();
84 }
85
86 unsigned getSubReg() const {
87 assert(isReg());
88 return OpToFold->getSubReg();
89 }
90
91 bool isImm() const { return Kind == MachineOperand::MO_Immediate; }
92
93 bool isFI() const {
94 return Kind == MachineOperand::MO_FrameIndex;
95 }
96
97 int getFI() const {
98 assert(isFI());
99 return FrameIndexToFold;
100 }
101
102 bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; }
103
104 /// Return the effective immediate value defined by this instruction, after
105 /// application of any subregister extracts which may exist between the use
106 /// and def instruction.
107 std::optional<int64_t> getEffectiveImmVal() const {
108 assert(isImm());
109 return SIInstrInfo::extractSubregFromImm(ImmVal: ImmToFold, SubRegIndex: DefSubReg);
110 }
111
112 /// Check if it is legal to fold this effective value into \p MI's \p OpNo
113 /// operand.
114 bool isOperandLegal(const SIInstrInfo &TII, const MachineInstr &MI,
115 unsigned OpIdx) const {
116 switch (Kind) {
117 case MachineOperand::MO_Immediate: {
118 std::optional<int64_t> ImmToFold = getEffectiveImmVal();
119 if (!ImmToFold)
120 return false;
121
122 // TODO: Should verify the subregister index is supported by the class
123 // TODO: Avoid the temporary MachineOperand
124 MachineOperand TmpOp = MachineOperand::CreateImm(Val: *ImmToFold);
125 return TII.isOperandLegal(MI, OpIdx, MO: &TmpOp);
126 }
127 case MachineOperand::MO_FrameIndex: {
128 if (DefSubReg != AMDGPU::NoSubRegister)
129 return false;
130 MachineOperand TmpOp = MachineOperand::CreateFI(Idx: FrameIndexToFold);
131 return TII.isOperandLegal(MI, OpIdx, MO: &TmpOp);
132 }
133 default:
134 // TODO: Try to apply DefSubReg, for global address we can extract
135 // low/high.
136 if (DefSubReg != AMDGPU::NoSubRegister)
137 return false;
138 return TII.isOperandLegal(MI, OpIdx, MO: OpToFold);
139 }
140
141 llvm_unreachable("covered MachineOperand kind switch");
142 }
143};
144
145struct FoldCandidate {
146 MachineInstr *UseMI;
147 FoldableDef Def;
148 int ShrinkOpcode;
149 unsigned UseOpNo;
150 bool Commuted;
151
152 FoldCandidate(MachineInstr *MI, unsigned OpNo, FoldableDef Def,
153 bool Commuted = false, int ShrinkOp = -1)
154 : UseMI(MI), Def(Def), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo),
155 Commuted(Commuted) {}
156
157 bool isFI() const { return Def.isFI(); }
158
159 int getFI() const {
160 assert(isFI());
161 return Def.FrameIndexToFold;
162 }
163
164 bool isImm() const { return Def.isImm(); }
165
166 bool isReg() const { return Def.isReg(); }
167
168 Register getReg() const { return Def.getReg(); }
169
170 bool isGlobal() const { return Def.isGlobal(); }
171
172 bool needsShrink() const { return ShrinkOpcode != -1; }
173};
174
175class SIFoldOperandsImpl {
176public:
177 MachineFunction *MF;
178 MachineRegisterInfo *MRI;
179 const SIInstrInfo *TII;
180 const SIRegisterInfo *TRI;
181 const GCNSubtarget *ST;
182 const SIMachineFunctionInfo *MFI;
183 const MachineLoopInfo *MLI;
184
185 bool frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
186 const FoldableDef &OpToFold) const;
187
188 // TODO: Just use TII::getVALUOp
189 unsigned convertToVALUOp(unsigned Opc, bool UseVOP3 = false) const {
190 switch (Opc) {
191 case AMDGPU::S_ADD_I32: {
192 if (ST->hasAddNoCarryInsts())
193 return UseVOP3 ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_U32_e32;
194 return UseVOP3 ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
195 }
196 case AMDGPU::S_OR_B32:
197 return UseVOP3 ? AMDGPU::V_OR_B32_e64 : AMDGPU::V_OR_B32_e32;
198 case AMDGPU::S_AND_B32:
199 return UseVOP3 ? AMDGPU::V_AND_B32_e64 : AMDGPU::V_AND_B32_e32;
200 case AMDGPU::S_MUL_I32:
201 return AMDGPU::V_MUL_LO_U32_e64;
202 default:
203 return AMDGPU::INSTRUCTION_LIST_END;
204 }
205 }
206
207 bool foldCopyToVGPROfScalarAddOfFrameIndex(Register DstReg, Register SrcReg,
208 MachineInstr &MI) const;
209
210 bool updateOperand(FoldCandidate &Fold) const;
211
212 bool canUseImmWithOpSel(const MachineInstr *MI, unsigned UseOpNo,
213 int64_t ImmVal) const;
214
215 /// Try to fold immediate \p ImmVal into \p MI's operand at index \p UseOpNo.
216 bool tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
217 int64_t ImmVal) const;
218
219 bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
220 MachineInstr *MI, unsigned OpNo,
221 const FoldableDef &OpToFold) const;
222 bool isUseSafeToFold(const MachineInstr &MI,
223 const MachineOperand &UseMO) const;
224 bool isTemporallyDivergentUse(const FoldableDef &OpToFold,
225 const MachineInstr &UseMI) const;
226
227 const TargetRegisterClass *getRegSeqInit(
228 MachineInstr &RegSeq,
229 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs) const;
230
231 const TargetRegisterClass *
232 getRegSeqInit(SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
233 Register UseReg) const;
234
235 std::pair<int64_t, const TargetRegisterClass *>
236 isRegSeqSplat(MachineInstr &RegSeg) const;
237
238 bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
239 int64_t SplatVal,
240 const TargetRegisterClass *SplatRC) const;
241
242 bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
243 unsigned UseOpIdx,
244 SmallVectorImpl<FoldCandidate> &FoldList) const;
245 bool foldOperand(FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
246 SmallVectorImpl<FoldCandidate> &FoldList,
247 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
248
249 struct ANDMaskResult {
250 int64_t Mask;
251 Register Reg;
252 unsigned RegIdx;
253 };
254
255 std::optional<ANDMaskResult> getANDMaskRegOperand(MachineInstr &AndMI) const;
256
257 bool tryConstantFoldOp(MachineInstr *MI) const;
258 bool tryFoldCndMask(MachineInstr &MI) const;
259 bool tryFoldRedundantAND(MachineInstr &ChildMI) const;
260 bool foldInstOperand(MachineInstr &MI, const FoldableDef &OpToFold) const;
261
262 bool foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const;
263 bool tryFoldFoldableCopy(MachineInstr &MI,
264 MachineOperand *&CurrentKnownM0Val) const;
265
266 const MachineOperand *isClamp(const MachineInstr &MI) const;
267 bool tryFoldClamp(MachineInstr &MI);
268
269 std::pair<const MachineOperand *, int> isOMod(const MachineInstr &MI) const;
270 bool tryFoldOMod(MachineInstr &MI);
271 bool tryFoldSGPRSplatRegSequence(MachineInstr &MI);
272 bool tryFoldRegSequence(MachineInstr &MI);
273 bool tryFoldPhiAGPR(MachineInstr &MI);
274 bool tryFoldLoad(MachineInstr &MI);
275
276 bool tryOptimizeAGPRPhis(MachineBasicBlock &MBB);
277
278public:
279 SIFoldOperandsImpl() = default;
280
281 bool run(MachineFunction &MF, const MachineLoopInfo *MLI);
282};
283
284class SIFoldOperandsLegacy : public MachineFunctionPass {
285public:
286 static char ID;
287
288 SIFoldOperandsLegacy() : MachineFunctionPass(ID) {}
289
290 bool runOnMachineFunction(MachineFunction &MF) override {
291 if (skipFunction(F: MF.getFunction()))
292 return false;
293 const MachineLoopInfo *MLI =
294 &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
295 return SIFoldOperandsImpl().run(MF, MLI);
296 }
297
298 StringRef getPassName() const override { return "SI Fold Operands"; }
299
300 void getAnalysisUsage(AnalysisUsage &AU) const override {
301 AU.setPreservesCFG();
302 AU.addRequired<MachineLoopInfoWrapperPass>();
303 AU.addPreserved<MachineLoopInfoWrapperPass>();
304 MachineFunctionPass::getAnalysisUsage(AU);
305 }
306
307 MachineFunctionProperties getRequiredProperties() const override {
308 return MachineFunctionProperties().setIsSSA();
309 }
310};
311
312} // End anonymous namespace.
313
314INITIALIZE_PASS_BEGIN(SIFoldOperandsLegacy, DEBUG_TYPE, "SI Fold Operands",
315 false, false)
316INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
317INITIALIZE_PASS_END(SIFoldOperandsLegacy, DEBUG_TYPE, "SI Fold Operands", false,
318 false)
319
320char SIFoldOperandsLegacy::ID = 0;
321
322char &llvm::SIFoldOperandsLegacyID = SIFoldOperandsLegacy::ID;
323
324static const TargetRegisterClass *getRegOpRC(const MachineRegisterInfo &MRI,
325 const TargetRegisterInfo &TRI,
326 const MachineOperand &MO) {
327 const TargetRegisterClass *RC = MRI.getRegClass(Reg: MO.getReg());
328 if (const TargetRegisterClass *SubRC =
329 TRI.getSubRegisterClass(SuperRC: RC, SubRegIdx: MO.getSubReg()))
330 RC = SubRC;
331 return RC;
332}
333
334// Map multiply-accumulate opcode to corresponding multiply-add opcode if any.
335static unsigned macToMad(unsigned Opc) {
336 switch (Opc) {
337 case AMDGPU::V_MAC_F32_e64:
338 return AMDGPU::V_MAD_F32_e64;
339 case AMDGPU::V_MAC_F16_e64:
340 return AMDGPU::V_MAD_F16_e64;
341 case AMDGPU::V_FMAC_F32_e64:
342 return AMDGPU::V_FMA_F32_e64;
343 case AMDGPU::V_FMAC_F16_e64:
344 return AMDGPU::V_FMA_F16_gfx9_e64;
345 case AMDGPU::V_FMAC_F16_t16_e64:
346 return AMDGPU::V_FMA_F16_gfx9_t16_e64;
347 case AMDGPU::V_FMAC_F16_fake16_e64:
348 return AMDGPU::V_FMA_F16_gfx9_fake16_e64;
349 case AMDGPU::V_FMAC_LEGACY_F32_e64:
350 return AMDGPU::V_FMA_LEGACY_F32_e64;
351 case AMDGPU::V_FMAC_F64_e64:
352 return AMDGPU::V_FMA_F64_e64;
353 }
354 return AMDGPU::INSTRUCTION_LIST_END;
355}
356
357// TODO: Add heuristic that the frame index might not fit in the addressing mode
358// immediate offset to avoid materializing in loops.
359bool SIFoldOperandsImpl::frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
360 const FoldableDef &OpToFold) const {
361 if (!OpToFold.isFI())
362 return false;
363
364 const unsigned Opc = UseMI.getOpcode();
365 switch (Opc) {
366 case AMDGPU::S_ADD_I32:
367 case AMDGPU::S_ADD_U32:
368 case AMDGPU::V_ADD_U32_e32:
369 case AMDGPU::V_ADD_CO_U32_e32:
370 // TODO: Possibly relax hasOneUse. It matters more for mubuf, since we have
371 // to insert the wave size shift at every point we use the index.
372 // TODO: Fix depending on visit order to fold immediates into the operand
373 return UseMI.getOperand(i: OpNo == 1 ? 2 : 1).isImm() &&
374 MRI->hasOneNonDBGUse(RegNo: UseMI.getOperand(i: OpNo).getReg());
375 case AMDGPU::V_ADD_U32_e64:
376 case AMDGPU::V_ADD_CO_U32_e64:
377 return UseMI.getOperand(i: OpNo == 2 ? 3 : 2).isImm() &&
378 MRI->hasOneNonDBGUse(RegNo: UseMI.getOperand(i: OpNo).getReg());
379 default:
380 break;
381 }
382
383 if (TII->isMUBUF(MI: UseMI))
384 return OpNo == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
385 if (!TII->isFLATScratch(MI: UseMI))
386 return false;
387
388 int SIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
389 if (OpNo == SIdx)
390 return true;
391
392 int VIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
393 return OpNo == VIdx && SIdx == -1;
394}
395
396/// Fold %vgpr = COPY (S_ADD_I32 x, frameindex)
397///
398/// => %vgpr = V_ADD_U32 x, frameindex
399bool SIFoldOperandsImpl::foldCopyToVGPROfScalarAddOfFrameIndex(
400 Register DstReg, Register SrcReg, MachineInstr &MI) const {
401 if (!SrcReg.isVirtual())
402 return false;
403
404 if (TRI->isVGPR(MRI: *MRI, Reg: DstReg) && TRI->isSGPRReg(MRI: *MRI, Reg: SrcReg) &&
405 MRI->hasOneNonDBGUse(RegNo: SrcReg)) {
406 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
407 if (!Def || Def->getNumOperands() != 4)
408 return false;
409
410 MachineOperand *Src0 = &Def->getOperand(i: 1);
411 MachineOperand *Src1 = &Def->getOperand(i: 2);
412
413 // TODO: This is profitable with more operand types, and for more
414 // opcodes. But ultimately this is working around poor / nonexistent
415 // regbankselect.
416 if (!Src0->isFI() && !Src1->isFI())
417 return false;
418
419 if (Src0->isFI())
420 std::swap(a&: Src0, b&: Src1);
421
422 const bool UseVOP3 = !Src0->isImm() || TII->isInlineConstant(MO: *Src0);
423 unsigned NewOp = convertToVALUOp(Opc: Def->getOpcode(), UseVOP3);
424 if (NewOp == AMDGPU::INSTRUCTION_LIST_END ||
425 !Def->getOperand(i: 3).isDead()) // Check if scc is dead
426 return false;
427
428 MachineBasicBlock *MBB = Def->getParent();
429 const DebugLoc &DL = Def->getDebugLoc();
430 if (NewOp != AMDGPU::V_ADD_CO_U32_e32) {
431 MachineInstrBuilder Add =
432 BuildMI(BB&: *MBB, I&: *Def, MIMD: DL, MCID: TII->get(Opcode: NewOp), DestReg: DstReg);
433
434 if (Add->getDesc().getNumDefs() == 2) {
435 Register CarryOutReg = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
436 Add.addDef(RegNo: CarryOutReg, Flags: RegState::Dead);
437 MRI->setRegAllocationHint(VReg: CarryOutReg, Type: 0, PrefReg: TRI->getVCC());
438 }
439
440 Add.add(MO: *Src0).add(MO: *Src1).setMIFlags(Def->getFlags());
441 if (AMDGPU::hasNamedOperand(Opcode: NewOp, NamedIdx: AMDGPU::OpName::clamp))
442 Add.addImm(Val: 0);
443
444 Def->eraseFromParent();
445 MI.eraseFromParent();
446 return true;
447 }
448
449 assert(NewOp == AMDGPU::V_ADD_CO_U32_e32);
450
451 MachineBasicBlock::LivenessQueryResult Liveness =
452 MBB->computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: *Def, Neighborhood: 16);
453 if (Liveness == MachineBasicBlock::LQR_Dead) {
454 // TODO: If src1 satisfies operand constraints, use vop3 version.
455 BuildMI(BB&: *MBB, I&: *Def, MIMD: DL, MCID: TII->get(Opcode: NewOp), DestReg: DstReg)
456 .add(MO: *Src0)
457 .add(MO: *Src1)
458 .setOperandDead(3) // implicit-def $vcc
459 .setMIFlags(Def->getFlags());
460 Def->eraseFromParent();
461 MI.eraseFromParent();
462 return true;
463 }
464 }
465
466 return false;
467}
468
469FunctionPass *llvm::createSIFoldOperandsLegacyPass() {
470 return new SIFoldOperandsLegacy();
471}
472
473bool SIFoldOperandsImpl::canUseImmWithOpSel(const MachineInstr *MI,
474 unsigned UseOpNo,
475 int64_t ImmVal) const {
476 if (!SIInstrFlags::isPacked(O: *MI) || SIInstrFlags::isMAI(O: *MI) ||
477 SIInstrFlags::isWMMA(O: *MI) || SIInstrFlags::isSWMMAC(O: *MI) ||
478 (ST->hasDOTOpSelHazard() && SIInstrFlags::isDOT(O: *MI)))
479 return false;
480
481 const MachineOperand &Old = MI->getOperand(i: UseOpNo);
482 int OpNo = MI->getOperandNo(I: &Old);
483
484 unsigned Opcode = MI->getOpcode();
485 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
486 switch (OpType) {
487 default:
488 return false;
489 case AMDGPU::OPERAND_REG_IMM_V2FP16:
490 case AMDGPU::OPERAND_REG_IMM_V2BF16:
491 case AMDGPU::OPERAND_REG_IMM_V2INT16:
492 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
493 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
494 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
495 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
496 // VOP3 packed instructions ignore op_sel source modifiers, we cannot encode
497 // two different constants.
498 if (SIInstrFlags::isVOP3(O: *MI) && !SIInstrFlags::isVOP3P(O: *MI) &&
499 static_cast<uint16_t>(ImmVal) != static_cast<uint16_t>(ImmVal >> 16))
500 return false;
501 break;
502 }
503
504 return true;
505}
506
507bool SIFoldOperandsImpl::tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
508 int64_t ImmVal) const {
509 MachineOperand &Old = MI->getOperand(i: UseOpNo);
510 unsigned Opcode = MI->getOpcode();
511 int OpNo = MI->getOperandNo(I: &Old);
512 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
513
514 // If the literal can be inlined as-is, apply it and short-circuit the
515 // tests below. The main motivation for this is to avoid unintuitive
516 // uses of opsel.
517 if (AMDGPU::isInlinableLiteralV216(Literal: ImmVal, OpType)) {
518 Old.ChangeToImmediate(ImmVal);
519 return true;
520 }
521
522 // Refer to op_sel/op_sel_hi and check if we can change the immediate and
523 // op_sel in a way that allows an inline constant.
524 AMDGPU::OpName ModName = AMDGPU::OpName::NUM_OPERAND_NAMES;
525 unsigned SrcIdx = ~0;
526 if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0)) {
527 ModName = AMDGPU::OpName::src0_modifiers;
528 SrcIdx = 0;
529 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1)) {
530 ModName = AMDGPU::OpName::src1_modifiers;
531 SrcIdx = 1;
532 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src2)) {
533 ModName = AMDGPU::OpName::src2_modifiers;
534 SrcIdx = 2;
535 }
536 assert(ModName != AMDGPU::OpName::NUM_OPERAND_NAMES);
537 int ModIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: ModName);
538 MachineOperand &Mod = MI->getOperand(i: ModIdx);
539 unsigned ModVal = Mod.getImm();
540
541 uint16_t ImmLo =
542 static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_0 ? 16 : 0));
543 uint16_t ImmHi =
544 static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_1 ? 16 : 0));
545 uint32_t Imm = (static_cast<uint32_t>(ImmHi) << 16) | ImmLo;
546 unsigned NewModVal = ModVal & ~(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
547
548 // Helper function that attempts to inline the given value with a newly
549 // chosen opsel pattern.
550 auto tryFoldToInline = [&](uint32_t Imm) -> bool {
551 if (AMDGPU::isInlinableLiteralV216(Literal: Imm, OpType)) {
552 Mod.setImm(NewModVal | SISrcMods::OP_SEL_1);
553 Old.ChangeToImmediate(ImmVal: Imm);
554 return true;
555 }
556
557 // Try to shuffle the halves around and leverage opsel to get an inline
558 // constant.
559 uint16_t Lo = static_cast<uint16_t>(Imm);
560 uint16_t Hi = static_cast<uint16_t>(Imm >> 16);
561 if (Lo == Hi) {
562 if (AMDGPU::isInlinableLiteralV216(Literal: Lo, OpType)) {
563 // If the target has feature 'BF16InlineConstFromUpperFP32', packed BF16
564 // instructions using inline constant must use OPSEL to select the upper
565 // 16-bits from FP32.
566 if (ST->hasBF16InlineConstFromUpperFP32() &&
567 (OpType == AMDGPU::OPERAND_REG_INLINE_C_V2BF16 ||
568 OpType == AMDGPU::OPERAND_REG_IMM_V2BF16))
569 NewModVal |= (SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
570 Mod.setImm(NewModVal);
571 Old.ChangeToImmediate(ImmVal: Lo);
572 return true;
573 }
574
575 if (static_cast<int16_t>(Lo) < 0) {
576 int32_t SExt = static_cast<int16_t>(Lo);
577 if (AMDGPU::isInlinableLiteralV216(Literal: SExt, OpType)) {
578 Mod.setImm(NewModVal);
579 Old.ChangeToImmediate(ImmVal: SExt);
580 return true;
581 }
582 }
583
584 // This check is only useful for integer instructions
585 if (OpType == AMDGPU::OPERAND_REG_IMM_V2INT16) {
586 if (AMDGPU::isInlinableLiteralV216(Literal: Lo << 16, OpType)) {
587 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
588 Old.ChangeToImmediate(ImmVal: static_cast<uint32_t>(Lo) << 16);
589 return true;
590 }
591 }
592 } else {
593 uint32_t Swapped = (static_cast<uint32_t>(Lo) << 16) | Hi;
594 if (AMDGPU::isInlinableLiteralV216(Literal: Swapped, OpType)) {
595 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0);
596 Old.ChangeToImmediate(ImmVal: Swapped);
597 return true;
598 }
599 }
600
601 return false;
602 };
603
604 if (tryFoldToInline(Imm))
605 return true;
606
607 // Replace integer addition by subtraction and vice versa if it allows
608 // folding the immediate to an inline constant.
609 //
610 // We should only ever get here for SrcIdx == 1 due to canonicalization
611 // earlier in the pipeline, but we double-check here to be safe / fully
612 // general.
613 bool IsUAdd = Opcode == AMDGPU::V_PK_ADD_U16;
614 bool IsUSub = Opcode == AMDGPU::V_PK_SUB_U16;
615 if (SrcIdx == 1 && (IsUAdd || IsUSub)) {
616 unsigned ClampIdx =
617 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::clamp);
618 bool Clamp = MI->getOperand(i: ClampIdx).getImm() != 0;
619
620 if (!Clamp) {
621 uint16_t NegLo = -static_cast<uint16_t>(Imm);
622 uint16_t NegHi = -static_cast<uint16_t>(Imm >> 16);
623 uint32_t NegImm = (static_cast<uint32_t>(NegHi) << 16) | NegLo;
624
625 if (tryFoldToInline(NegImm)) {
626 unsigned NegOpcode =
627 IsUAdd ? AMDGPU::V_PK_SUB_U16 : AMDGPU::V_PK_ADD_U16;
628 MI->setDesc(TII->get(Opcode: NegOpcode));
629 return true;
630 }
631 }
632 }
633
634 return false;
635}
636
637bool SIFoldOperandsImpl::updateOperand(FoldCandidate &Fold) const {
638 MachineInstr *MI = Fold.UseMI;
639 MachineOperand &Old = MI->getOperand(i: Fold.UseOpNo);
640 assert(Old.isReg());
641
642 std::optional<int64_t> ImmVal;
643 if (Fold.isImm())
644 ImmVal = Fold.Def.getEffectiveImmVal();
645
646 if (ImmVal && canUseImmWithOpSel(MI: Fold.UseMI, UseOpNo: Fold.UseOpNo, ImmVal: *ImmVal)) {
647 if (tryFoldImmWithOpSel(MI: Fold.UseMI, UseOpNo: Fold.UseOpNo, ImmVal: *ImmVal))
648 return true;
649
650 // We can't represent the candidate as an inline constant. Try as a literal
651 // with the original opsel, checking constant bus limitations.
652 MachineOperand New = MachineOperand::CreateImm(Val: *ImmVal);
653 int OpNo = MI->getOperandNo(I: &Old);
654 if (!TII->isOperandLegal(MI: *MI, OpIdx: OpNo, MO: &New))
655 return false;
656 Old.ChangeToImmediate(ImmVal: *ImmVal);
657 return true;
658 }
659
660 if ((Fold.isImm() || Fold.isFI() || Fold.isGlobal()) && Fold.needsShrink()) {
661 MachineBasicBlock *MBB = MI->getParent();
662 auto Liveness = MBB->computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: MI, Neighborhood: 16);
663 if (Liveness != MachineBasicBlock::LQR_Dead) {
664 LLVM_DEBUG(dbgs() << "Not shrinking due to live vcc: " << *MI);
665 return false;
666 }
667
668 int Op32 = Fold.ShrinkOpcode;
669 MachineOperand &Dst0 = MI->getOperand(i: 0);
670 MachineOperand &Dst1 = MI->getOperand(i: 1);
671 assert(Dst0.isDef() && Dst1.isDef());
672
673 bool HaveNonDbgCarryUse = !MRI->use_nodbg_empty(RegNo: Dst1.getReg());
674
675 const TargetRegisterClass *Dst0RC = MRI->getRegClass(Reg: Dst0.getReg());
676 Register NewReg0 = MRI->createVirtualRegister(RegClass: Dst0RC);
677
678 MachineInstr *Inst32 = TII->buildShrunkInst(MI&: *MI, NewOpcode: Op32);
679
680 if (HaveNonDbgCarryUse) {
681 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
682 DestReg: Dst1.getReg())
683 .addReg(RegNo: AMDGPU::VCC, Flags: RegState::Kill);
684 }
685
686 // Keep the old instruction around to avoid breaking iterators, but
687 // replace it with a dummy instruction to remove uses.
688 //
689 // FIXME: We should not invert how this pass looks at operands to avoid
690 // this. Should track set of foldable movs instead of looking for uses
691 // when looking at a use.
692 Dst0.setReg(NewReg0);
693 for (unsigned I = MI->getNumOperands() - 1; I > 0; --I)
694 MI->removeOperand(OpNo: I);
695 MI->setDesc(TII->get(Opcode: AMDGPU::IMPLICIT_DEF));
696
697 if (Fold.Commuted)
698 TII->commuteInstruction(MI&: *Inst32, NewMI: false);
699 return true;
700 }
701
702 assert(!Fold.needsShrink() && "not handled");
703
704 if (ImmVal) {
705 if (Old.isTied()) {
706 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: MI->getOpcode());
707 if (NewMFMAOpc == -1)
708 return false;
709 MI->setDesc(TII->get(Opcode: NewMFMAOpc));
710 MI->untieRegOperand(OpIdx: 0);
711 const MCInstrDesc &MCID = MI->getDesc();
712 for (unsigned I = 0; I < MI->getNumDefs(); ++I)
713 if (MCID.getOperandConstraint(OpNum: I, Constraint: MCOI::EARLY_CLOBBER) != -1)
714 MI->getOperand(i: I).setIsEarlyClobber(true);
715 }
716
717 // TODO: Should we try to avoid adding this to the candidate list?
718 MachineOperand New = MachineOperand::CreateImm(Val: *ImmVal);
719 int OpNo = MI->getOperandNo(I: &Old);
720 if (!TII->isOperandLegal(MI: *MI, OpIdx: OpNo, MO: &New))
721 return false;
722
723 if (ST->hasBF16InlineConstFromUpperFP32() &&
724 OpNo ==
725 AMDGPU::getNamedOperandIdx(Opcode: MI->getOpcode(), Name: AMDGPU::OpName::src0)) {
726 unsigned Opcode = MI->getOpcode();
727 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
728 if ((OpType == AMDGPU::OPERAND_REG_IMM_BF16 ||
729 OpType == AMDGPU::OPERAND_REG_INLINE_C_BF16) &&
730 TII->isInlineConstant(ImmVal: *ImmVal, OperandType: OpType)) {
731 // We can fold it, but we need to set OPSEL
732 int Mod0 =
733 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0_modifiers);
734 if (Mod0 == -1)
735 return false;
736 MachineOperand &ModOp = MI->getOperand(i: Mod0);
737 if (ModOp.getImm())
738 return false;
739 ModOp.setImm(SISrcMods::OP_SEL_0);
740 }
741 }
742
743 Old.ChangeToImmediate(ImmVal: *ImmVal);
744 return true;
745 }
746
747 if (Fold.isGlobal()) {
748 Old.ChangeToGA(GV: Fold.Def.OpToFold->getGlobal(),
749 Offset: Fold.Def.OpToFold->getOffset(),
750 TargetFlags: Fold.Def.OpToFold->getTargetFlags());
751 return true;
752 }
753
754 if (Fold.isFI()) {
755 Old.ChangeToFrameIndex(Idx: Fold.getFI());
756 return true;
757 }
758
759 MachineOperand *New = Fold.Def.OpToFold;
760
761 // Verify the register is compatible with the operand.
762 if (const TargetRegisterClass *OpRC =
763 TII->getRegClass(MCID: MI->getDesc(), OpNum: Fold.UseOpNo)) {
764 const TargetRegisterClass *NewRC =
765 TRI->getRegClassForReg(MRI: *MRI, Reg: New->getReg());
766
767 const TargetRegisterClass *ConstrainRC = OpRC;
768 if (New->getSubReg()) {
769 ConstrainRC =
770 TRI->getMatchingSuperRegClass(A: NewRC, B: OpRC, Idx: New->getSubReg());
771
772 if (!ConstrainRC)
773 return false;
774 }
775
776 if (New->getReg().isVirtual() &&
777 !MRI->constrainRegClass(Reg: New->getReg(), RC: ConstrainRC)) {
778 LLVM_DEBUG(dbgs() << "Cannot constrain " << printReg(New->getReg(), TRI)
779 << TRI->getRegClassName(ConstrainRC) << '\n');
780 return false;
781 }
782 }
783
784 // Rework once the VS_16 register class is updated to include proper
785 // 16-bit SGPRs instead of 32-bit ones.
786 if (Old.getSubReg() == AMDGPU::lo16 && TRI->isSGPRReg(MRI: *MRI, Reg: New->getReg()))
787 Old.setSubReg(AMDGPU::NoSubRegister);
788 if (New->getReg().isPhysical()) {
789 Old.substPhysReg(Reg: New->getReg(), *TRI);
790 } else {
791 Register OldReg = Old.getReg();
792 Old.substVirtReg(Reg: New->getReg(), SubIdx: New->getSubReg(), *TRI);
793 Old.setIsUndef(New->isUndef());
794
795 // If MI is in a BUNDLE, also update header's matching implicit use.
796 if (MI->isBundledWithPred()) {
797 MachineInstr &Header = *getBundleStart(I: MI->getIterator());
798 for (MachineOperand &MO : Header.operands()) {
799 if (MO.getReg() == OldReg) {
800 MO.setReg(New->getReg());
801 MO.setSubReg(New->getSubReg());
802 }
803 }
804 }
805 }
806 return true;
807}
808
809static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
810 FoldCandidate &&Entry) {
811 // Skip additional folding on the same operand.
812 for (FoldCandidate &Fold : FoldList)
813 if (Fold.UseMI == Entry.UseMI && Fold.UseOpNo == Entry.UseOpNo)
814 return;
815 LLVM_DEBUG(dbgs() << "Append " << (Entry.Commuted ? "commuted" : "normal")
816 << " operand " << Entry.UseOpNo << "\n " << *Entry.UseMI);
817 FoldList.push_back(Elt: Entry);
818}
819
820static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
821 MachineInstr *MI, unsigned OpNo,
822 const FoldableDef &FoldOp,
823 bool Commuted = false, int ShrinkOp = -1) {
824 appendFoldCandidate(FoldList,
825 Entry: FoldCandidate(MI, OpNo, FoldOp, Commuted, ShrinkOp));
826}
827
828// Returns true if the instruction is a packed F32 instruction and the
829// corresponding scalar operand reads 32 bits and replicates the bits to both
830// channels.
831static bool isPKF32InstrReplicatesLower32BitsOfScalarOperand(
832 const GCNSubtarget *ST, MachineInstr *MI, unsigned OpNo) {
833 if (!ST->hasPKF32InstsReplicatingLower32BitsOfScalarInput())
834 return false;
835 const MCOperandInfo &OpDesc = MI->getDesc().operands()[OpNo];
836 return OpDesc.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32;
837}
838
839// Packed FP32 instructions only read 32 bits from a scalar operand (SGPR or
840// literal) and replicates the bits to both channels. Therefore, if the hi and
841// lo are not same, we can't fold it.
842static bool checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(
843 const FoldableDef &OpToFold) {
844 assert(OpToFold.isImm() && "Expected immediate operand");
845 uint64_t ImmVal = OpToFold.getEffectiveImmVal().value();
846 uint32_t Lo = Lo_32(Value: ImmVal);
847 uint32_t Hi = Hi_32(Value: ImmVal);
848 return Lo == Hi;
849}
850
851bool SIFoldOperandsImpl::tryAddToFoldList(
852 SmallVectorImpl<FoldCandidate> &FoldList, MachineInstr *MI, unsigned OpNo,
853 const FoldableDef &OpToFold) const {
854 const unsigned Opc = MI->getOpcode();
855
856 auto tryToFoldAsFMAAKorMK = [&]() {
857 if (!OpToFold.isImm())
858 return false;
859
860 const bool TryAK = OpNo == 3;
861 const unsigned NewOpc = TryAK ? AMDGPU::S_FMAAK_F32 : AMDGPU::S_FMAMK_F32;
862 MI->setDesc(TII->get(Opcode: NewOpc));
863
864 // We have to fold into operand which would be Imm not into OpNo.
865 bool FoldAsFMAAKorMK =
866 tryAddToFoldList(FoldList, MI, OpNo: TryAK ? 3 : 2, OpToFold);
867 if (FoldAsFMAAKorMK) {
868 // Untie Src2 of fmac.
869 MI->untieRegOperand(OpIdx: 3);
870 // For fmamk swap operands 1 and 2 if OpToFold was meant for operand 1.
871 if (OpNo == 1) {
872 MachineOperand &Op1 = MI->getOperand(i: 1);
873 MachineOperand &Op2 = MI->getOperand(i: 2);
874 Register OldReg = Op1.getReg();
875 // Operand 2 might be an inlinable constant
876 if (Op2.isImm()) {
877 Op1.ChangeToImmediate(ImmVal: Op2.getImm());
878 Op2.ChangeToRegister(Reg: OldReg, isDef: false);
879 } else {
880 Op1.setReg(Op2.getReg());
881 Op2.setReg(OldReg);
882 }
883 }
884 return true;
885 }
886 MI->setDesc(TII->get(Opcode: Opc));
887 return false;
888 };
889
890 bool IsLegal = OpToFold.isOperandLegal(TII: *TII, MI: *MI, OpIdx: OpNo);
891 if (!IsLegal && OpToFold.isImm()) {
892 if (std::optional<int64_t> ImmVal = OpToFold.getEffectiveImmVal())
893 IsLegal = canUseImmWithOpSel(MI, UseOpNo: OpNo, ImmVal: *ImmVal);
894 }
895
896 if (!IsLegal) {
897 // Special case for v_mac_{f16, f32}_e64 if we are trying to fold into src2
898 unsigned NewOpc = macToMad(Opc);
899 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) {
900 // Check if changing this to a v_mad_{f16, f32} instruction will allow us
901 // to fold the operand.
902 MI->setDesc(TII->get(Opcode: NewOpc));
903 bool AddOpSel = !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::op_sel) &&
904 AMDGPU::hasNamedOperand(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::op_sel);
905 if (AddOpSel)
906 MI->addOperand(Op: MachineOperand::CreateImm(Val: 0));
907 bool FoldAsMAD = tryAddToFoldList(FoldList, MI, OpNo, OpToFold);
908 if (FoldAsMAD) {
909 MI->untieRegOperand(OpIdx: OpNo);
910 return true;
911 }
912 if (AddOpSel)
913 MI->removeOperand(OpNo: MI->getNumExplicitOperands() - 1);
914 MI->setDesc(TII->get(Opcode: Opc));
915 }
916
917 // Special case for s_fmac_f32 if we are trying to fold into Src2.
918 // By transforming into fmaak we can untie Src2 and make folding legal.
919 if (Opc == AMDGPU::S_FMAC_F32 && OpNo == 3) {
920 if (tryToFoldAsFMAAKorMK())
921 return true;
922 }
923
924 // Inlineable constant might have been folded into Imm operand of fmaak or
925 // fmamk and we are trying to fold a non-inlinable constant.
926 if ((Opc == AMDGPU::S_FMAAK_F32 || Opc == AMDGPU::S_FMAMK_F32) &&
927 OpToFold.isImm()) {
928 std::optional<int64_t> ImmVal = OpToFold.getEffectiveImmVal();
929 if (ImmVal && !TII->isInlineConstant(MI: *MI, OpIdx: OpNo, ImmVal: *ImmVal)) {
930 unsigned ImmIdx = Opc == AMDGPU::S_FMAAK_F32 ? 3 : 2;
931 MachineOperand &OpImm = MI->getOperand(i: ImmIdx);
932 if (!OpImm.isReg() &&
933 TII->isInlineConstant(MI: *MI, UseMO: MI->getOperand(i: OpNo), DefMO: OpImm))
934 return tryToFoldAsFMAAKorMK();
935 }
936 }
937
938 // Special case for s_setreg_b32
939 if (OpToFold.isImm()) {
940 unsigned ImmOpc = 0;
941 if (Opc == AMDGPU::S_SETREG_B32)
942 ImmOpc = AMDGPU::S_SETREG_IMM32_B32;
943 else if (Opc == AMDGPU::S_SETREG_B32_mode)
944 ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode;
945 if (ImmOpc) {
946 MI->setDesc(TII->get(Opcode: ImmOpc));
947 appendFoldCandidate(FoldList, MI, OpNo, FoldOp: OpToFold);
948 return true;
949 }
950 }
951
952 // Operand is not legal, so try to commute the instruction to
953 // see if this makes it possible to fold.
954 unsigned CommuteOpNo = TargetInstrInfo::CommuteAnyOperandIndex;
955 bool CanCommute = TII->findCommutedOpIndices(MI: *MI, SrcOpIdx0&: OpNo, SrcOpIdx1&: CommuteOpNo);
956 if (!CanCommute)
957 return false;
958
959 MachineOperand &Op = MI->getOperand(i: OpNo);
960 MachineOperand &CommutedOp = MI->getOperand(i: CommuteOpNo);
961
962 // One of operands might be an Imm operand, and OpNo may refer to it after
963 // the call of commuteInstruction() below. Such situations are avoided
964 // here explicitly as OpNo must be a register operand to be a candidate
965 // for memory folding.
966 if (!Op.isReg() || !CommutedOp.isReg())
967 return false;
968
969 // The same situation with an immediate could reproduce if both inputs are
970 // the same register.
971 if (Op.isReg() && CommutedOp.isReg() &&
972 (Op.getReg() == CommutedOp.getReg() &&
973 Op.getSubReg() == CommutedOp.getSubReg()))
974 return false;
975
976 if (!TII->commuteInstruction(MI&: *MI, NewMI: false, OpIdx1: OpNo, OpIdx2: CommuteOpNo))
977 return false;
978
979 int Op32 = -1;
980 if (!OpToFold.isOperandLegal(TII: *TII, MI: *MI, OpIdx: CommuteOpNo)) {
981 if ((Opc != AMDGPU::V_ADD_CO_U32_e64 && Opc != AMDGPU::V_SUB_CO_U32_e64 &&
982 Opc != AMDGPU::V_SUBREV_CO_U32_e64) || // FIXME
983 (!OpToFold.isImm() && !OpToFold.isFI() && !OpToFold.isGlobal())) {
984 TII->commuteInstruction(MI&: *MI, NewMI: false, OpIdx1: OpNo, OpIdx2: CommuteOpNo);
985 return false;
986 }
987
988 // Verify the other operand is a VGPR, otherwise we would violate the
989 // constant bus restriction.
990 MachineOperand &OtherOp = MI->getOperand(i: OpNo);
991 if (!OtherOp.isReg() ||
992 !TII->getRegisterInfo().isVGPR(MRI: *MRI, Reg: OtherOp.getReg()))
993 return false;
994
995 assert(MI->getOperand(1).isDef());
996
997 // Make sure to get the 32-bit version of the commuted opcode.
998 unsigned MaybeCommutedOpc = MI->getOpcode();
999 Op32 = AMDGPU::getVOPe32(Opcode: MaybeCommutedOpc);
1000 }
1001
1002 appendFoldCandidate(FoldList, MI, OpNo: CommuteOpNo, FoldOp: OpToFold, /*Commuted=*/true,
1003 ShrinkOp: Op32);
1004 return true;
1005 }
1006
1007 // Special case for s_fmac_f32 if we are trying to fold into Src0 or Src1.
1008 // By changing into fmamk we can untie Src2.
1009 // If folding for Src0 happens first and it is identical operand to Src1 we
1010 // should avoid transforming into fmamk which requires commuting as it would
1011 // cause folding into Src1 to fail later on due to wrong OpNo used.
1012 if (Opc == AMDGPU::S_FMAC_F32 &&
1013 (OpNo != 1 || !MI->getOperand(i: 1).isIdenticalTo(Other: MI->getOperand(i: 2)))) {
1014 if (tryToFoldAsFMAAKorMK())
1015 return true;
1016 }
1017
1018 // Special case for PK_F32 instructions if we are trying to fold an imm to
1019 // src0 or src1.
1020 if (OpToFold.isImm() &&
1021 isPKF32InstrReplicatesLower32BitsOfScalarOperand(ST, MI, OpNo) &&
1022 !checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(OpToFold))
1023 return false;
1024
1025 appendFoldCandidate(FoldList, MI, OpNo, FoldOp: OpToFold);
1026 return true;
1027}
1028
1029bool SIFoldOperandsImpl::isUseSafeToFold(const MachineInstr &MI,
1030 const MachineOperand &UseMO) const {
1031 // Operands of SDWA instructions must be registers.
1032 return !TII->isSDWA(MI);
1033}
1034
1035// Returns true if any instruction in \p L modifies EXEC.
1036static bool loopModifiesExec(const MachineLoop &L, const SIRegisterInfo &TRI) {
1037 for (const MachineBasicBlock *MBB : L.getBlocks())
1038 for (const MachineInstr &MI : *MBB)
1039 if (MI.modifiesRegister(Reg: TRI.getExec(), TRI: &TRI))
1040 return true;
1041 return false;
1042}
1043
1044// An SGPR->VGPR copy inside a divergent loop latches each lane value as it
1045// exits. Folding its scalar source into a use after the loop would make every
1046// lane read the same reconverged value, so do not fold across the loop exit.
1047bool SIFoldOperandsImpl::isTemporallyDivergentUse(
1048 const FoldableDef &OpToFold, const MachineInstr &UseMI) const {
1049 if (!OpToFold.isReg())
1050 return false;
1051 const MachineInstr *DefMI = OpToFold.DefMI;
1052 if (!DefMI || !DefMI->isCopy() ||
1053 TRI->isSGPRReg(MRI: *MRI, Reg: DefMI->getOperand(i: 0).getReg()) ||
1054 !TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg()))
1055 return false;
1056 const MachineLoop *DefLoop = MLI->getLoopFor(BB: DefMI->getParent());
1057 return DefLoop && !DefLoop->contains(BB: UseMI.getParent()) &&
1058 loopModifiesExec(L: *DefLoop, TRI: *TRI);
1059}
1060
1061static MachineOperand *lookUpCopyChain(const SIInstrInfo &TII,
1062 const MachineRegisterInfo &MRI,
1063 Register SrcReg) {
1064 MachineOperand *Sub = nullptr;
1065 for (MachineInstr *SubDef = MRI.getVRegDef(Reg: SrcReg);
1066 SubDef && TII.isFoldableCopy(MI: *SubDef);
1067 SubDef = MRI.getVRegDef(Reg: Sub->getReg())) {
1068 unsigned SrcIdx = TII.getFoldableCopySrcIdx(MI: *SubDef);
1069 MachineOperand &SrcOp = SubDef->getOperand(i: SrcIdx);
1070
1071 if (SrcOp.isImm())
1072 return &SrcOp;
1073 if (!SrcOp.isReg() || SrcOp.getReg().isPhysical())
1074 break;
1075 Sub = &SrcOp;
1076 // TODO: Support compose
1077 if (SrcOp.getSubReg())
1078 break;
1079 }
1080
1081 return Sub;
1082}
1083
1084const TargetRegisterClass *SIFoldOperandsImpl::getRegSeqInit(
1085 MachineInstr &RegSeq,
1086 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs) const {
1087
1088 assert(RegSeq.isRegSequence());
1089
1090 const TargetRegisterClass *RC = nullptr;
1091
1092 for (unsigned I = 1, E = RegSeq.getNumExplicitOperands(); I != E; I += 2) {
1093 MachineOperand &SrcOp = RegSeq.getOperand(i: I);
1094 if (SrcOp.getReg().isPhysical())
1095 return nullptr;
1096 unsigned SubRegIdx = RegSeq.getOperand(i: I + 1).getImm();
1097
1098 // Only accept reg_sequence with uniform reg class inputs for simplicity.
1099 const TargetRegisterClass *OpRC = getRegOpRC(MRI: *MRI, TRI: *TRI, MO: SrcOp);
1100 if (!RC)
1101 RC = OpRC;
1102 else if (!TRI->getCommonSubClass(A: RC, B: OpRC))
1103 return nullptr;
1104
1105 if (SrcOp.getSubReg()) {
1106 // TODO: Handle subregister compose
1107 Defs.emplace_back(Args: &SrcOp, Args&: SubRegIdx);
1108 continue;
1109 }
1110
1111 MachineOperand *DefSrc = lookUpCopyChain(TII: *TII, MRI: *MRI, SrcReg: SrcOp.getReg());
1112 if (DefSrc && (DefSrc->isReg() || DefSrc->isImm())) {
1113 Defs.emplace_back(Args&: DefSrc, Args&: SubRegIdx);
1114 continue;
1115 }
1116
1117 Defs.emplace_back(Args: &SrcOp, Args&: SubRegIdx);
1118 }
1119
1120 return RC;
1121}
1122
1123// Find a def of the UseReg, check if it is a reg_sequence and find initializers
1124// for each subreg, tracking it to an immediate if possible. Returns the
1125// register class of the inputs on success.
1126const TargetRegisterClass *SIFoldOperandsImpl::getRegSeqInit(
1127 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
1128 Register UseReg) const {
1129 MachineInstr *Def = MRI->getVRegDef(Reg: UseReg);
1130 if (!Def || !Def->isRegSequence())
1131 return nullptr;
1132
1133 return getRegSeqInit(RegSeq&: *Def, Defs);
1134}
1135
1136std::pair<int64_t, const TargetRegisterClass *>
1137SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
1138 SmallVector<std::pair<MachineOperand *, unsigned>, 32> Defs;
1139 const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
1140 if (!SrcRC)
1141 return {};
1142
1143 bool TryToMatchSplat64 = false;
1144
1145 std::optional<int64_t> Imm;
1146 for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
1147 const MachineOperand *Op = Defs[I].first;
1148 if (!Op->isImm()) {
1149 if (Op->isReg()) {
1150 MachineInstr *Def = MRI->getVRegDef(Reg: Op->getReg());
1151 if (!Def || Def->isImplicitDef())
1152 continue;
1153 }
1154 return {};
1155 }
1156
1157 int64_t SubImm = Op->getImm();
1158 if (!Imm) {
1159 Imm = SubImm;
1160 continue;
1161 }
1162
1163 if (Imm != SubImm) {
1164 if (I == 1 && (E & 1) == 0) {
1165 // If we have an even number of inputs, there's a chance this is a
1166 // 64-bit element splat broken into 32-bit pieces.
1167 TryToMatchSplat64 = true;
1168 break;
1169 }
1170
1171 return {}; // Can only fold splat constants
1172 }
1173 }
1174
1175 if (!TryToMatchSplat64) {
1176 if (Imm)
1177 return {*Imm, SrcRC};
1178 return {};
1179 }
1180
1181 // Fallback to recognizing 64-bit splats broken into 32-bit pieces
1182 // (i.e. recognize every other other element is 0 for 64-bit immediates)
1183 int64_t SplatVal64;
1184 for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
1185 const MachineOperand *Op0 = Defs[I].first;
1186 const MachineOperand *Op1 = Defs[I + 1].first;
1187
1188 if (!Op0->isImm() || !Op1->isImm())
1189 return {};
1190
1191 unsigned SubReg0 = Defs[I].second;
1192 unsigned SubReg1 = Defs[I + 1].second;
1193
1194 // Assume we're going to generally encounter reg_sequences with sorted
1195 // subreg indexes, so reject any that aren't consecutive.
1196 if (TRI->getChannelFromSubReg(SubReg: SubReg0) + 1 !=
1197 TRI->getChannelFromSubReg(SubReg: SubReg1))
1198 return {};
1199
1200 if (TRI->getSubRegIdxSize(Idx: SubReg0) != 32)
1201 return {};
1202
1203 int64_t MergedVal = Make_64(High: Op1->getImm(), Low: Op0->getImm());
1204 if (I == 0)
1205 SplatVal64 = MergedVal;
1206 else if (SplatVal64 != MergedVal)
1207 return {};
1208 }
1209
1210 const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
1211 MRI->getRegClass(Reg: RegSeq.getOperand(i: 0).getReg()), AMDGPU::sub0_sub1);
1212
1213 return {SplatVal64, RC64};
1214}
1215
1216bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
1217 MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
1218 const TargetRegisterClass *SplatRC) const {
1219 const MCInstrDesc &Desc = UseMI->getDesc();
1220 if (UseOpIdx >= Desc.getNumOperands())
1221 return false;
1222
1223 // Filter out unhandled pseudos.
1224 if (!AMDGPU::isSISrcOperand(Desc, OpNo: UseOpIdx))
1225 return false;
1226
1227 int16_t RCID = TII->getOpRegClassID(OpInfo: Desc.operands()[UseOpIdx]);
1228 if (RCID == -1)
1229 return false;
1230
1231 const TargetRegisterClass *OpRC = TRI->getRegClass(i: RCID);
1232
1233 // Special case 0/-1, since when interpreted as a 64-bit element both halves
1234 // have the same bits. These are the only cases where a splat has the same
1235 // interpretation for 32-bit and 64-bit splats.
1236 if (SplatVal != 0 && SplatVal != -1) {
1237 // We need to figure out the scalar type read by the operand. e.g. the MFMA
1238 // operand will be AReg_128, and we want to check if it's compatible with an
1239 // AReg_32 constant.
1240 uint8_t OpTy = Desc.operands()[UseOpIdx].OperandType;
1241 switch (OpTy) {
1242 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1243 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1244 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1245 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1246 case AMDGPU::OPERAND_REG_IMM_V2FP32:
1247 OpRC = TRI->getSubRegisterClass(OpRC, AMDGPU::sub0);
1248 break;
1249 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
1250 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1251 case AMDGPU::OPERAND_REG_IMM_V2FP64:
1252 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1253 case AMDGPU::OPERAND_REG_IMM_V2INT64:
1254 OpRC = TRI->getSubRegisterClass(OpRC, AMDGPU::sub0_sub1);
1255 break;
1256 default:
1257 return false;
1258 }
1259
1260 if (!TRI->getCommonSubClass(A: OpRC, B: SplatRC))
1261 return false;
1262 }
1263
1264 MachineOperand TmpOp = MachineOperand::CreateImm(Val: SplatVal);
1265 if (!TII->isOperandLegal(MI: *UseMI, OpIdx: UseOpIdx, MO: &TmpOp))
1266 return false;
1267
1268 return true;
1269}
1270
1271bool SIFoldOperandsImpl::tryToFoldACImm(
1272 const FoldableDef &OpToFold, MachineInstr *UseMI, unsigned UseOpIdx,
1273 SmallVectorImpl<FoldCandidate> &FoldList) const {
1274 const MCInstrDesc &Desc = UseMI->getDesc();
1275 if (UseOpIdx >= Desc.getNumOperands())
1276 return false;
1277
1278 // Filter out unhandled pseudos.
1279 if (!AMDGPU::isSISrcOperand(Desc, OpNo: UseOpIdx))
1280 return false;
1281
1282 if (OpToFold.isImm() && OpToFold.isOperandLegal(TII: *TII, MI: *UseMI, OpIdx: UseOpIdx)) {
1283 if (isPKF32InstrReplicatesLower32BitsOfScalarOperand(ST, MI: UseMI, OpNo: UseOpIdx) &&
1284 !checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(OpToFold))
1285 return false;
1286 appendFoldCandidate(FoldList, MI: UseMI, OpNo: UseOpIdx, FoldOp: OpToFold);
1287 return true;
1288 }
1289
1290 return false;
1291}
1292
1293bool SIFoldOperandsImpl::foldOperand(
1294 FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
1295 SmallVectorImpl<FoldCandidate> &FoldList,
1296 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const {
1297 bool Changed = false;
1298 const MachineOperand *UseOp = &UseMI->getOperand(i: UseOpIdx);
1299
1300 if (!isUseSafeToFold(MI: *UseMI, UseMO: *UseOp))
1301 return Changed;
1302
1303 if (isTemporallyDivergentUse(OpToFold, UseMI: *UseMI))
1304 return Changed;
1305
1306 // FIXME: Fold operands with subregs.
1307 if (UseOp->isReg() && OpToFold.isReg()) {
1308 if (UseOp->isImplicit())
1309 return Changed;
1310 // Allow folding from SGPRs to 16-bit VGPRs.
1311 if (UseOp->getSubReg() != AMDGPU::NoSubRegister &&
1312 (UseOp->getSubReg() != AMDGPU::lo16 ||
1313 !TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg())))
1314 return Changed;
1315 }
1316
1317 // Special case for REG_SEQUENCE: We can't fold literals into
1318 // REG_SEQUENCE instructions, so we have to fold them into the
1319 // uses of REG_SEQUENCE.
1320 if (UseMI->isRegSequence()) {
1321 Register RegSeqDstReg = UseMI->getOperand(i: 0).getReg();
1322 unsigned RegSeqDstSubReg = UseMI->getOperand(i: UseOpIdx + 1).getImm();
1323
1324 int64_t SplatVal;
1325 const TargetRegisterClass *SplatRC;
1326 std::tie(args&: SplatVal, args&: SplatRC) = isRegSeqSplat(RegSeq&: *UseMI);
1327
1328 // Grab the use operands first
1329 SmallVector<MachineOperand *, 4> UsesToProcess(
1330 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: RegSeqDstReg)));
1331 for (unsigned I = 0; I != UsesToProcess.size(); ++I) {
1332 MachineOperand *RSUse = UsesToProcess[I];
1333 MachineInstr *RSUseMI = RSUse->getParent();
1334 unsigned OpNo = RSUseMI->getOperandNo(I: RSUse);
1335
1336 if (SplatRC) {
1337 if (RSUseMI->isCopy()) {
1338 Register DstReg = RSUseMI->getOperand(i: 0).getReg();
1339 append_range(C&: UsesToProcess,
1340 R: make_pointer_range(Range: MRI->use_nodbg_operands(Reg: DstReg)));
1341 continue;
1342 }
1343 if (tryFoldRegSeqSplat(UseMI: RSUseMI, UseOpIdx: OpNo, SplatVal, SplatRC)) {
1344 FoldableDef SplatDef(SplatVal, SplatRC);
1345 appendFoldCandidate(FoldList, MI: RSUseMI, OpNo, FoldOp: SplatDef);
1346 Changed = true;
1347 continue;
1348 }
1349 }
1350
1351 // TODO: Handle general compose
1352 if (RSUse->getSubReg() != RegSeqDstSubReg)
1353 continue;
1354
1355 // FIXME: We should avoid recursing here. There should be a cleaner split
1356 // between the in-place mutations and adding to the fold list.
1357 Changed |= foldOperand(OpToFold, UseMI: RSUseMI, UseOpIdx: RSUseMI->getOperandNo(I: RSUse),
1358 FoldList, CopiesToReplace);
1359 }
1360
1361 return Changed;
1362 }
1363
1364 if (tryToFoldACImm(OpToFold, UseMI, UseOpIdx, FoldList))
1365 return true;
1366
1367 if (frameIndexMayFold(UseMI: *UseMI, OpNo: UseOpIdx, OpToFold)) {
1368 // Verify that this is a stack access.
1369 // FIXME: Should probably use stack pseudos before frame lowering.
1370
1371 if (TII->isMUBUF(MI: *UseMI)) {
1372 if (TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::srsrc)->getReg() !=
1373 MFI->getScratchRSrcReg())
1374 return Changed;
1375
1376 // Ensure this is either relative to the current frame or the current
1377 // wave.
1378 MachineOperand &SOff =
1379 *TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::soffset);
1380 if (!SOff.isImm() || SOff.getImm() != 0)
1381 return Changed;
1382 }
1383
1384 const unsigned Opc = UseMI->getOpcode();
1385 if (TII->isFLATScratch(MI: *UseMI) &&
1386 AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr) &&
1387 !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::saddr)) {
1388 unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(Opcode: Opc);
1389 unsigned CPol =
1390 TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::cpol)->getImm();
1391 if ((CPol & AMDGPU::CPol::SCAL) &&
1392 !AMDGPU::supportsScaleOffset(MII: *TII, Opcode: NewOpc))
1393 return Changed;
1394
1395 UseMI->setDesc(TII->get(Opcode: NewOpc));
1396 }
1397
1398 // A frame index will resolve to a positive constant, so it should always be
1399 // safe to fold the addressing mode, even pre-GFX9.
1400 UseMI->getOperand(i: UseOpIdx).ChangeToFrameIndex(Idx: OpToFold.getFI());
1401
1402 return true;
1403 }
1404
1405 bool FoldingImmLike =
1406 OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
1407
1408 if (FoldingImmLike && UseMI->isCopy()) {
1409 Register DestReg = UseMI->getOperand(i: 0).getReg();
1410 Register SrcReg = UseMI->getOperand(i: 1).getReg();
1411 unsigned UseSubReg = UseMI->getOperand(i: 1).getSubReg();
1412 assert(SrcReg.isVirtual());
1413
1414 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: SrcReg);
1415
1416 // Don't fold into a copy to a physical register with the same class. Doing
1417 // so would interfere with the register coalescer's logic which would avoid
1418 // redundant initializations.
1419 if (DestReg.isPhysical() && SrcRC->contains(Reg: DestReg))
1420 return Changed;
1421
1422 const TargetRegisterClass *DestRC = TRI->getRegClassForReg(MRI: *MRI, Reg: DestReg);
1423 // In order to fold immediates into copies, we need to change the copy to a
1424 // MOV. Find a compatible mov instruction with the value.
1425 for (unsigned MovOp :
1426 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
1427 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_MOV_B16_t16_e64,
1428 AMDGPU::V_ACCVGPR_WRITE_B32_e64, AMDGPU::AV_MOV_B32_IMM_PSEUDO,
1429 AMDGPU::AV_MOV_B64_IMM_PSEUDO}) {
1430 const MCInstrDesc &MovDesc = TII->get(Opcode: MovOp);
1431 const TargetRegisterClass *MovDstRC =
1432 TRI->getRegClass(i: TII->getOpRegClassID(OpInfo: MovDesc.operands()[0]));
1433
1434 // Fold if the destination register class of the MOV instruction (ResRC)
1435 // is a superclass of (or equal to) the destination register class of the
1436 // COPY (DestRC). If this condition fails, folding would be illegal.
1437 if (!DestRC->hasSuperClassEq(RC: MovDstRC))
1438 continue;
1439
1440 const int SrcIdx = MovOp == AMDGPU::V_MOV_B16_t16_e64 ? 2 : 1;
1441
1442 int16_t RegClassID = TII->getOpRegClassID(OpInfo: MovDesc.operands()[SrcIdx]);
1443 if (RegClassID != -1) {
1444 const TargetRegisterClass *MovSrcRC = TRI->getRegClass(i: RegClassID);
1445
1446 if (UseSubReg)
1447 MovSrcRC = TRI->getMatchingSuperRegClass(A: SrcRC, B: MovSrcRC, Idx: UseSubReg);
1448
1449 // FIXME: We should be able to directly check immediate operand legality
1450 // for all cases, but gfx908 hacks break.
1451 if (MovOp == AMDGPU::AV_MOV_B32_IMM_PSEUDO &&
1452 (!OpToFold.isImm() ||
1453 !TII->isImmOperandLegal(InstDesc: MovDesc, OpNo: SrcIdx,
1454 ImmVal: *OpToFold.getEffectiveImmVal())))
1455 break;
1456
1457 if (!MRI->constrainRegClass(Reg: SrcReg, RC: MovSrcRC))
1458 break;
1459
1460 // FIXME: This is mutating the instruction only and deferring the actual
1461 // fold of the immediate
1462 } else {
1463 // For the _IMM_PSEUDO cases, there can be value restrictions on the
1464 // immediate to verify. Technically we should always verify this, but it
1465 // only matters for these concrete cases.
1466 // TODO: Handle non-imm case if it's useful.
1467 if (!OpToFold.isImm() ||
1468 !TII->isImmOperandLegal(InstDesc: MovDesc, OpNo: 1, ImmVal: *OpToFold.getEffectiveImmVal()))
1469 break;
1470 }
1471
1472 MachineInstr::mop_iterator ImpOpI = UseMI->implicit_operands().begin();
1473 MachineInstr::mop_iterator ImpOpE = UseMI->implicit_operands().end();
1474 while (ImpOpI != ImpOpE) {
1475 MachineInstr::mop_iterator Tmp = ImpOpI;
1476 ImpOpI++;
1477 UseMI->removeOperand(OpNo: UseMI->getOperandNo(I: Tmp));
1478 }
1479 UseMI->setDesc(MovDesc);
1480
1481 if (MovOp == AMDGPU::V_MOV_B16_t16_e64) {
1482 const auto &SrcOp = UseMI->getOperand(i: UseOpIdx);
1483 MachineOperand NewSrcOp(SrcOp);
1484 UseMI->removeOperand(OpNo: 1);
1485 UseMI->addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0)); // src0_modifiers
1486 UseMI->addOperand(Op: NewSrcOp); // src0
1487 UseMI->addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0)); // op_sel
1488 UseOpIdx = SrcIdx;
1489 UseOp = &UseMI->getOperand(i: UseOpIdx);
1490 }
1491 CopiesToReplace.push_back(Elt: UseMI);
1492 Changed = true;
1493 break;
1494 }
1495
1496 // We failed to replace the copy, so give up.
1497 if (UseMI->getOpcode() == AMDGPU::COPY)
1498 return Changed;
1499
1500 } else {
1501 if (UseMI->isCopy() && OpToFold.isReg() &&
1502 UseMI->getOperand(i: 0).getReg().isVirtual() &&
1503 !UseMI->getOperand(i: 1).getSubReg() &&
1504 OpToFold.DefMI->implicit_operands().empty()) {
1505 LLVM_DEBUG(dbgs() << "Folding " << *OpToFold.OpToFold << "\n into "
1506 << *UseMI);
1507 unsigned Size = TII->getOpSize(MI: *UseMI, OpNo: 1);
1508 Register UseReg = OpToFold.getReg();
1509 UseMI->getOperand(i: 1).setReg(UseReg);
1510 unsigned SubRegIdx = OpToFold.getSubReg();
1511 // Hack to allow 32-bit SGPRs to be folded into True16 instructions
1512 // Remove this if 16-bit SGPRs (i.e. SGPR_LO16) are added to the
1513 // VS_16RegClass
1514 //
1515 // Excerpt from AMDGPUGenRegisterInfoEnums.inc
1516 // NoSubRegister, //0
1517 // hi16, // 1
1518 // lo16, // 2
1519 // sub0, // 3
1520 // ...
1521 // sub1, // 11
1522 // sub1_hi16, // 12
1523 // sub1_lo16, // 13
1524 static_assert(AMDGPU::sub1_hi16 == 12, "Subregister layout has changed");
1525 if (Size == 2 && TRI->isVGPR(MRI: *MRI, Reg: UseMI->getOperand(i: 0).getReg()) &&
1526 TRI->isSGPRReg(MRI: *MRI, Reg: UseReg)) {
1527 // Produce the 32 bit subregister index to which the 16-bit subregister
1528 // is aligned.
1529 if (SubRegIdx > AMDGPU::sub1) {
1530 LaneBitmask M = TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx);
1531 M |= M.getLane(Lane: M.getHighestLane() - 1);
1532 SmallVector<unsigned, 4> Indexes;
1533 TRI->getCoveringSubRegIndexes(RC: TRI->getRegClassForReg(MRI: *MRI, Reg: UseReg), LaneMask: M,
1534 Indexes);
1535 assert(Indexes.size() == 1 && "Expected one 32-bit subreg to cover");
1536 SubRegIdx = Indexes[0];
1537 // 32-bit registers do not have a sub0 index
1538 } else if (TII->getOpSize(MI: *UseMI, OpNo: 1) == 4)
1539 SubRegIdx = 0;
1540 else
1541 SubRegIdx = AMDGPU::sub0;
1542 }
1543 UseMI->getOperand(i: 1).setSubReg(SubRegIdx);
1544 UseMI->getOperand(i: 1).setIsKill(false);
1545 CopiesToReplace.push_back(Elt: UseMI);
1546 OpToFold.OpToFold->setIsKill(false);
1547 Changed = true;
1548
1549 // Remove kill flags as kills may now be out of order with uses.
1550 MRI->clearKillFlags(Reg: UseReg);
1551 if (foldCopyToAGPRRegSequence(CopyMI: UseMI))
1552 return true;
1553 }
1554
1555 unsigned UseOpc = UseMI->getOpcode();
1556 if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 ||
1557 (UseOpc == AMDGPU::V_READLANE_B32 &&
1558 (int)UseOpIdx ==
1559 AMDGPU::getNamedOperandIdx(Opcode: UseOpc, Name: AMDGPU::OpName::src0))) {
1560 // %vgpr = V_MOV_B32 imm
1561 // %sgpr = V_READFIRSTLANE_B32 %vgpr
1562 // =>
1563 // %sgpr = S_MOV_B32 imm
1564 if (FoldingImmLike) {
1565 if (execMayBeModifiedBeforeUse(MRI: *MRI,
1566 VReg: UseMI->getOperand(i: UseOpIdx).getReg(),
1567 DefMI: *OpToFold.DefMI, UseMI: *UseMI))
1568 return Changed;
1569
1570 UseMI->setDesc(TII->get(Opcode: AMDGPU::S_MOV_B32));
1571 UseMI->clearFlag(Flag: MachineInstr::NoConvergent);
1572
1573 if (OpToFold.isImm()) {
1574 UseMI->getOperand(i: 1).ChangeToImmediate(
1575 ImmVal: *OpToFold.getEffectiveImmVal());
1576 } else if (OpToFold.isFI())
1577 UseMI->getOperand(i: 1).ChangeToFrameIndex(Idx: OpToFold.getFI());
1578 else {
1579 assert(OpToFold.isGlobal());
1580 UseMI->getOperand(i: 1).ChangeToGA(GV: OpToFold.OpToFold->getGlobal(),
1581 Offset: OpToFold.OpToFold->getOffset(),
1582 TargetFlags: OpToFold.OpToFold->getTargetFlags());
1583 }
1584 UseMI->removeOperand(OpNo: 2); // Remove exec read (or src1 for readlane)
1585 return true;
1586 }
1587
1588 if (OpToFold.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg())) {
1589 if (execMayBeModifiedBeforeUse(MRI: *MRI,
1590 VReg: UseMI->getOperand(i: UseOpIdx).getReg(),
1591 DefMI: *OpToFold.DefMI, UseMI: *UseMI))
1592 return Changed;
1593
1594 // %vgpr = COPY %sgpr0
1595 // %sgpr1 = V_READFIRSTLANE_B32 %vgpr
1596 // =>
1597 // %sgpr1 = COPY %sgpr0
1598 UseMI->setDesc(TII->get(Opcode: AMDGPU::COPY));
1599 UseMI->getOperand(i: 1).setReg(OpToFold.getReg());
1600 UseMI->getOperand(i: 1).setSubReg(OpToFold.getSubReg());
1601 UseMI->getOperand(i: 1).setIsKill(false);
1602 UseMI->removeOperand(OpNo: 2); // Remove exec read (or src1 for readlane)
1603 UseMI->clearFlag(Flag: MachineInstr::NoConvergent);
1604 return true;
1605 }
1606 }
1607
1608 const MCInstrDesc &UseDesc = UseMI->getDesc();
1609
1610 // Don't fold into target independent nodes. Target independent opcodes
1611 // don't have defined register classes.
1612 if (UseDesc.isVariadic() || UseOp->isImplicit() ||
1613 UseDesc.operands()[UseOpIdx].RegClass == -1)
1614 return Changed;
1615 }
1616
1617 // FIXME: We could try to change the instruction from 64-bit to 32-bit
1618 // to enable more folding opportunities. The shrink operands pass
1619 // already does this.
1620
1621 Changed |= tryAddToFoldList(FoldList, MI: UseMI, OpNo: UseOpIdx, OpToFold);
1622 return Changed;
1623}
1624
1625static bool evalBinaryInstruction(unsigned Opcode, int32_t &Result,
1626 uint32_t LHS, uint32_t RHS) {
1627 switch (Opcode) {
1628 case AMDGPU::S_ADD_I32:
1629 case AMDGPU::S_ADD_U32:
1630 Result = LHS + RHS;
1631 return true;
1632 case AMDGPU::S_SUB_I32:
1633 case AMDGPU::S_SUB_U32:
1634 Result = LHS - RHS;
1635 return true;
1636 case AMDGPU::V_AND_B32_e64:
1637 case AMDGPU::V_AND_B32_e32:
1638 case AMDGPU::S_AND_B32:
1639 Result = LHS & RHS;
1640 return true;
1641 case AMDGPU::V_OR_B32_e64:
1642 case AMDGPU::V_OR_B32_e32:
1643 case AMDGPU::S_OR_B32:
1644 Result = LHS | RHS;
1645 return true;
1646 case AMDGPU::V_XOR_B32_e64:
1647 case AMDGPU::V_XOR_B32_e32:
1648 case AMDGPU::S_XOR_B32:
1649 Result = LHS ^ RHS;
1650 return true;
1651 case AMDGPU::S_XNOR_B32:
1652 Result = ~(LHS ^ RHS);
1653 return true;
1654 case AMDGPU::S_NAND_B32:
1655 Result = ~(LHS & RHS);
1656 return true;
1657 case AMDGPU::S_NOR_B32:
1658 Result = ~(LHS | RHS);
1659 return true;
1660 case AMDGPU::S_ANDN2_B32:
1661 Result = LHS & ~RHS;
1662 return true;
1663 case AMDGPU::S_ORN2_B32:
1664 Result = LHS | ~RHS;
1665 return true;
1666 case AMDGPU::V_LSHL_B32_e64:
1667 case AMDGPU::V_LSHL_B32_e32:
1668 case AMDGPU::S_LSHL_B32:
1669 // The instruction ignores the high bits for out of bounds shifts.
1670 Result = LHS << (RHS & 31);
1671 return true;
1672 case AMDGPU::V_LSHLREV_B32_e64:
1673 case AMDGPU::V_LSHLREV_B32_e32:
1674 Result = RHS << (LHS & 31);
1675 return true;
1676 case AMDGPU::V_LSHR_B32_e64:
1677 case AMDGPU::V_LSHR_B32_e32:
1678 case AMDGPU::S_LSHR_B32:
1679 Result = LHS >> (RHS & 31);
1680 return true;
1681 case AMDGPU::V_LSHRREV_B32_e64:
1682 case AMDGPU::V_LSHRREV_B32_e32:
1683 Result = RHS >> (LHS & 31);
1684 return true;
1685 case AMDGPU::V_ASHR_I32_e64:
1686 case AMDGPU::V_ASHR_I32_e32:
1687 case AMDGPU::S_ASHR_I32:
1688 Result = static_cast<int32_t>(LHS) >> (RHS & 31);
1689 return true;
1690 case AMDGPU::V_ASHRREV_I32_e64:
1691 case AMDGPU::V_ASHRREV_I32_e32:
1692 Result = static_cast<int32_t>(RHS) >> (LHS & 31);
1693 return true;
1694 default:
1695 return false;
1696 }
1697}
1698
1699static unsigned getMovOpc(bool IsScalar) {
1700 return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1701}
1702
1703// Try to simplify operations with a constant that may appear after instruction
1704// selection.
1705// TODO: See if a frame index with a fixed offset can fold.
1706bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
1707 if (!MI->allImplicitDefsAreDead())
1708 return false;
1709
1710 unsigned Opc = MI->getOpcode();
1711
1712 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
1713 if (Src0Idx == -1)
1714 return false;
1715
1716 MachineOperand *Src0 = &MI->getOperand(i: Src0Idx);
1717 std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src0);
1718
1719 if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 ||
1720 Opc == AMDGPU::S_NOT_B32) &&
1721 Src0Imm) {
1722 MI->getOperand(i: 1).ChangeToImmediate(ImmVal: ~*Src0Imm);
1723 TII->mutateAndCleanupImplicit(
1724 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_NOT_B32)));
1725 return true;
1726 }
1727
1728 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
1729 if (Src1Idx == -1)
1730 return false;
1731
1732 MachineOperand *Src1 = &MI->getOperand(i: Src1Idx);
1733 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
1734
1735 if (!Src0Imm && !Src1Imm)
1736 return false;
1737
1738 // and k0, k1 -> v_mov_b32 (k0 & k1)
1739 // or k0, k1 -> v_mov_b32 (k0 | k1)
1740 // xor k0, k1 -> v_mov_b32 (k0 ^ k1)
1741 if (Src0Imm && Src1Imm) {
1742 int32_t NewImm;
1743 if (!evalBinaryInstruction(Opcode: Opc, Result&: NewImm, LHS: *Src0Imm, RHS: *Src1Imm))
1744 return false;
1745
1746 bool IsSGPR = TRI->isSGPRReg(MRI: *MRI, Reg: MI->getOperand(i: 0).getReg());
1747
1748 // Be careful to change the right operand, src0 may belong to a different
1749 // instruction.
1750 MI->getOperand(i: Src0Idx).ChangeToImmediate(ImmVal: NewImm);
1751 MI->removeOperand(OpNo: Src1Idx);
1752 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: IsSGPR)));
1753 return true;
1754 }
1755
1756 // S_SUB_* is not commutable, so handle it before the commutability gate.
1757 // Only `x - 0 -> copy x` is valid; `0 - x` is a negation, not a copy.
1758 if (Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U32) {
1759 if (Src1Imm && static_cast<int32_t>(*Src1Imm) == 0) {
1760 // y = sub x, 0 => y = copy x
1761 MI->removeOperand(OpNo: Src1Idx);
1762 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1763 return true;
1764 }
1765 return false;
1766 }
1767
1768 if (!MI->isCommutable())
1769 return false;
1770
1771 if (Src0Imm && !Src1Imm) {
1772 std::swap(a&: Src0, b&: Src1);
1773 std::swap(a&: Src0Idx, b&: Src1Idx);
1774 std::swap(lhs&: Src0Imm, rhs&: Src1Imm);
1775 }
1776
1777 int32_t Src1Val = static_cast<int32_t>(*Src1Imm);
1778 if (Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_ADD_U32) {
1779 if (Src1Val == 0) {
1780 // y = add x, 0 => y = copy x
1781 MI->removeOperand(OpNo: Src1Idx);
1782 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1783 return true;
1784 }
1785 return false;
1786 }
1787
1788 if (Opc == AMDGPU::V_OR_B32_e64 ||
1789 Opc == AMDGPU::V_OR_B32_e32 ||
1790 Opc == AMDGPU::S_OR_B32) {
1791 if (Src1Val == 0) {
1792 // y = or x, 0 => y = copy x
1793 MI->removeOperand(OpNo: Src1Idx);
1794 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1795 } else if (Src1Val == -1) {
1796 // y = or x, -1 => y = v_mov_b32 -1
1797 MI->removeOperand(OpNo: Src0Idx);
1798 TII->mutateAndCleanupImplicit(
1799 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_OR_B32)));
1800 } else
1801 return false;
1802
1803 return true;
1804 }
1805
1806 if (Opc == AMDGPU::V_AND_B32_e64 || Opc == AMDGPU::V_AND_B32_e32 ||
1807 Opc == AMDGPU::S_AND_B32) {
1808 if (Src1Val == 0) {
1809 // y = and x, 0 => y = v_mov_b32 0
1810 MI->removeOperand(OpNo: Src0Idx);
1811 TII->mutateAndCleanupImplicit(
1812 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_AND_B32)));
1813 } else if (Src1Val == -1) {
1814 // y = and x, -1 => y = copy x
1815 MI->removeOperand(OpNo: Src1Idx);
1816 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1817 } else
1818 return false;
1819
1820 return true;
1821 }
1822
1823 if (Opc == AMDGPU::V_XOR_B32_e64 || Opc == AMDGPU::V_XOR_B32_e32 ||
1824 Opc == AMDGPU::S_XOR_B32) {
1825 if (Src1Val == 0) {
1826 // y = xor x, 0 => y = copy x
1827 MI->removeOperand(OpNo: Src1Idx);
1828 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1829 return true;
1830 }
1831 }
1832
1833 return false;
1834}
1835
1836// Try to fold an instruction into a simpler one
1837bool SIFoldOperandsImpl::tryFoldCndMask(MachineInstr &MI) const {
1838 unsigned Opc = MI.getOpcode();
1839 if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 &&
1840 Opc != AMDGPU::V_CNDMASK_B64_PSEUDO)
1841 return false;
1842
1843 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1844 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1845 if (!Src1->isIdenticalTo(Other: *Src0)) {
1846 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
1847 if (!Src1Imm)
1848 return false;
1849
1850 std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src0);
1851 if (!Src0Imm || *Src0Imm != *Src1Imm)
1852 return false;
1853 }
1854
1855 int Src1ModIdx =
1856 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1_modifiers);
1857 int Src0ModIdx =
1858 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers);
1859 if ((Src1ModIdx != -1 && MI.getOperand(i: Src1ModIdx).getImm() != 0) ||
1860 (Src0ModIdx != -1 && MI.getOperand(i: Src0ModIdx).getImm() != 0))
1861 return false;
1862
1863 LLVM_DEBUG(dbgs() << "Folded " << MI << " into ");
1864 auto &NewDesc =
1865 TII->get(Opcode: Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(IsScalar: false));
1866 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
1867 if (Src2Idx != -1)
1868 MI.removeOperand(OpNo: Src2Idx);
1869 MI.removeOperand(OpNo: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1));
1870 if (Src1ModIdx != -1)
1871 MI.removeOperand(OpNo: Src1ModIdx);
1872 if (Src0ModIdx != -1)
1873 MI.removeOperand(OpNo: Src0ModIdx);
1874 TII->mutateAndCleanupImplicit(MI, NewDesc);
1875 LLVM_DEBUG(dbgs() << MI);
1876 return true;
1877}
1878
1879// Extract mask, register, and register operand index from an AND instruction.
1880// Immediate can be in operand 1 or 2.
1881std::optional<SIFoldOperandsImpl::ANDMaskResult>
1882SIFoldOperandsImpl::getANDMaskRegOperand(MachineInstr &AndMI) const {
1883 unsigned Opc = AndMI.getOpcode();
1884 if (Opc != AMDGPU::V_AND_B32_e64 && Opc != AMDGPU::V_AND_B32_e32 &&
1885 Opc != AMDGPU::S_AND_B32)
1886 return std::nullopt;
1887
1888 std::optional<int64_t> MaskImm =
1889 TII->getImmOrMaterializedImm(MRI: *MRI, Op: AndMI.getOperand(i: 1));
1890 if (MaskImm && AndMI.getOperand(i: 2).isReg())
1891 return ANDMaskResult{.Mask: *MaskImm, .Reg: AndMI.getOperand(i: 2).getReg(), .RegIdx: 2};
1892
1893 MaskImm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: AndMI.getOperand(i: 2));
1894 if (MaskImm && AndMI.getOperand(i: 1).isReg())
1895 return ANDMaskResult{.Mask: *MaskImm, .Reg: AndMI.getOperand(i: 1).getReg(), .RegIdx: 1};
1896
1897 return std::nullopt;
1898}
1899
1900// Eliminate redundant 32-bit AND operations by detecting when ChildMI's mask
1901// contains ParentMI's mask.
1902//
1903// For example:
1904// ParentMI: %1 = AND %0, 0x7fff
1905// ChildMI: %2 = AND %1, 0xffff
1906//
1907// This also handles cases where ParentMI implicitly zeros high bits (e.g., f16
1908// operations that write 16-bit results into 32-bit registers), making a
1909// subsequent AND with 0xffff redundant.
1910bool SIFoldOperandsImpl::tryFoldRedundantAND(MachineInstr &ChildMI) const {
1911 // Ensure implicit defs (e.g., $scc) are not live.
1912 if (!ChildMI.allImplicitDefsAreDead())
1913 return false;
1914
1915 std::optional<ANDMaskResult> ChildResult = getANDMaskRegOperand(AndMI&: ChildMI);
1916 if (!ChildResult)
1917 return false;
1918
1919 if (!ChildResult->Reg.isVirtual())
1920 return false;
1921
1922 MachineInstr *ParentMI = MRI->getVRegDef(Reg: ChildResult->Reg);
1923 if (!ParentMI)
1924 return false;
1925
1926 int64_t ParentMask = 0;
1927 std::optional<ANDMaskResult> ParentResult = getANDMaskRegOperand(AndMI&: *ParentMI);
1928 if (ParentResult) {
1929 // Parent is an AND - extract its mask.
1930 ParentMask = ParentResult->Mask;
1931 } else if (ST->zeroesHigh16BitsOfDest(Opcode: ParentMI->getOpcode())) {
1932 // Parent instruction implicitly zeros high 16 bits.
1933 ParentMask = 0xffff;
1934 } else {
1935 return false;
1936 }
1937
1938 // Check if ChildMI is not redundant.
1939 if ((ParentMask & ChildResult->Mask) != ParentMask)
1940 return false;
1941
1942 Register Dst = ChildMI.getOperand(i: 0).getReg();
1943 Register Src = ChildResult->Reg;
1944
1945 // Src must be legal in every use of Dst. An S_AND_B32 parent with a
1946 // V_AND_B32 child defines Src in the scalar bank, and a use that requires a
1947 // VGPR does not accept it.
1948 if (!Dst.isVirtual() || !MRI->constrainRegClass(Reg: Src, RC: MRI->getRegClass(Reg: Dst)))
1949 return false;
1950
1951 MRI->replaceRegWith(FromReg: Dst, ToReg: Src);
1952
1953 // Clear kill flags if the register operand is not marked as kill.
1954 if (!ChildMI.getOperand(i: ChildResult->RegIdx).isKill())
1955 MRI->clearKillFlags(Reg: Src);
1956
1957 ChildMI.eraseFromParent();
1958 return true;
1959}
1960
1961bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
1962 const FoldableDef &OpToFold) const {
1963 // We need mutate the operands of new mov instructions to add implicit
1964 // uses of EXEC, but adding them invalidates the use_iterator, so defer
1965 // this.
1966 SmallVector<MachineInstr *, 4> CopiesToReplace;
1967 SmallVector<FoldCandidate, 4> FoldList;
1968 MachineOperand &Dst = MI.getOperand(i: 0);
1969 bool Changed = false;
1970
1971 SmallVector<MachineOperand *, 4> UsesToProcess(
1972 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: Dst.getReg())));
1973 for (auto *U : UsesToProcess) {
1974 MachineInstr *UseMI = U->getParent();
1975
1976 FoldableDef SubOpToFold = OpToFold.getWithSubReg(TRI: *TRI, SubReg: U->getSubReg());
1977 Changed |= foldOperand(OpToFold: SubOpToFold, UseMI, UseOpIdx: UseMI->getOperandNo(I: U), FoldList,
1978 CopiesToReplace);
1979 }
1980
1981 if (CopiesToReplace.empty() && FoldList.empty())
1982 return Changed;
1983
1984 // Make sure we add EXEC uses to any new v_mov instructions created.
1985 for (MachineInstr *Copy : CopiesToReplace)
1986 Copy->addImplicitDefUseOperands(MF&: *MF);
1987
1988 SetVector<MachineInstr *> ConstantFoldCandidates;
1989 for (FoldCandidate &Fold : FoldList) {
1990 assert(!Fold.isReg() || Fold.Def.OpToFold);
1991 if (Fold.isReg() && Fold.getReg().isVirtual()) {
1992 Register Reg = Fold.getReg();
1993 const MachineInstr *DefMI = Fold.Def.DefMI;
1994 if (DefMI->readsRegister(Reg: AMDGPU::EXEC, TRI) &&
1995 execMayBeModifiedBeforeUse(MRI: *MRI, VReg: Reg, DefMI: *DefMI, UseMI: *Fold.UseMI))
1996 continue;
1997 }
1998 if (updateOperand(Fold)) {
1999 // Clear kill flags.
2000 if (Fold.isReg()) {
2001 assert(Fold.Def.OpToFold && Fold.isReg());
2002 // FIXME: Probably shouldn't bother trying to fold if not an
2003 // SGPR. PeepholeOptimizer can eliminate redundant VGPR->VGPR
2004 // copies.
2005 MRI->clearKillFlags(Reg: Fold.getReg());
2006 }
2007 LLVM_DEBUG(dbgs() << "Folded source from " << MI << " into OpNo "
2008 << static_cast<int>(Fold.UseOpNo) << " of "
2009 << *Fold.UseMI);
2010
2011 if (Fold.isImm())
2012 ConstantFoldCandidates.insert(X: Fold.UseMI);
2013
2014 } else if (Fold.Commuted) {
2015 // Restoring instruction's original operand order if fold has failed.
2016 TII->commuteInstruction(MI&: *Fold.UseMI, NewMI: false);
2017 }
2018 }
2019
2020 for (MachineInstr *MI : ConstantFoldCandidates) {
2021 if (tryConstantFoldOp(MI)) {
2022 LLVM_DEBUG(dbgs() << "Constant folded " << *MI);
2023 Changed = true;
2024 }
2025 }
2026 return true;
2027}
2028
2029/// Fold %agpr = COPY (REG_SEQUENCE x_MOV_B32, ...) into REG_SEQUENCE
2030/// (V_ACCVGPR_WRITE_B32_e64) ... depending on the reg_sequence input values.
2031bool SIFoldOperandsImpl::foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const {
2032 // It is very tricky to store a value into an AGPR. v_accvgpr_write_b32 can
2033 // only accept VGPR or inline immediate. Recreate a reg_sequence with its
2034 // initializers right here, so we will rematerialize immediates and avoid
2035 // copies via different reg classes.
2036 const TargetRegisterClass *DefRC =
2037 MRI->getRegClass(Reg: CopyMI->getOperand(i: 0).getReg());
2038 if (!TRI->isAGPRClass(RC: DefRC))
2039 return false;
2040
2041 Register UseReg = CopyMI->getOperand(i: 1).getReg();
2042 MachineInstr *RegSeq = MRI->getVRegDef(Reg: UseReg);
2043 if (!RegSeq || !RegSeq->isRegSequence())
2044 return false;
2045
2046 const DebugLoc &DL = CopyMI->getDebugLoc();
2047 MachineBasicBlock &MBB = *CopyMI->getParent();
2048
2049 MachineInstrBuilder B(*MBB.getParent(), CopyMI);
2050 DenseMap<TargetInstrInfo::RegSubRegPair, Register> VGPRCopies;
2051
2052 const TargetRegisterClass *UseRC =
2053 MRI->getRegClass(Reg: CopyMI->getOperand(i: 1).getReg());
2054
2055 // Value, subregindex for new REG_SEQUENCE
2056 SmallVector<std::pair<MachineOperand *, unsigned>, 32> NewDefs;
2057
2058 unsigned NumRegSeqOperands = RegSeq->getNumOperands();
2059 unsigned NumFoldable = 0;
2060
2061 for (unsigned I = 1; I != NumRegSeqOperands; I += 2) {
2062 MachineOperand &RegOp = RegSeq->getOperand(i: I);
2063 unsigned SubRegIdx = RegSeq->getOperand(i: I + 1).getImm();
2064
2065 if (RegOp.getSubReg()) {
2066 // TODO: Handle subregister compose
2067 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2068 continue;
2069 }
2070
2071 MachineOperand *Lookup = lookUpCopyChain(TII: *TII, MRI: *MRI, SrcReg: RegOp.getReg());
2072 if (!Lookup)
2073 Lookup = &RegOp;
2074
2075 if (Lookup->isImm()) {
2076 // Check if this is an agpr_32 subregister.
2077 const TargetRegisterClass *DestSuperRC = TRI->getMatchingSuperRegClass(
2078 A: DefRC, B: &AMDGPU::AGPR_32RegClass, Idx: SubRegIdx);
2079 if (DestSuperRC &&
2080 TII->isInlineConstant(MO: *Lookup, OperandType: AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
2081 ++NumFoldable;
2082 NewDefs.emplace_back(Args&: Lookup, Args&: SubRegIdx);
2083 continue;
2084 }
2085 }
2086
2087 const TargetRegisterClass *InputRC =
2088 Lookup->isReg() ? MRI->getRegClass(Reg: Lookup->getReg())
2089 : MRI->getRegClass(Reg: RegOp.getReg());
2090
2091 // TODO: Account for Lookup->getSubReg()
2092
2093 // If we can't find a matching super class, this is an SGPR->AGPR or
2094 // VGPR->AGPR subreg copy (or something constant-like we have to materialize
2095 // in the AGPR). We can't directly copy from SGPR to AGPR on gfx908, so we
2096 // want to rewrite to copy to an intermediate VGPR class.
2097 const TargetRegisterClass *MatchRC =
2098 TRI->getMatchingSuperRegClass(A: DefRC, B: InputRC, Idx: SubRegIdx);
2099 if (!MatchRC) {
2100 ++NumFoldable;
2101 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2102 continue;
2103 }
2104
2105 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
2106 }
2107
2108 // Do not clone a reg_sequence and merely change the result register class.
2109 if (NumFoldable == 0)
2110 return false;
2111
2112 CopyMI->setDesc(TII->get(Opcode: AMDGPU::REG_SEQUENCE));
2113 for (unsigned I = CopyMI->getNumOperands() - 1; I > 0; --I)
2114 CopyMI->removeOperand(OpNo: I);
2115
2116 for (auto [Def, DestSubIdx] : NewDefs) {
2117 if (!Def->isReg()) {
2118 // TODO: Should we use single write for each repeated value like in
2119 // register case?
2120 Register Tmp = MRI->createVirtualRegister(RegClass: &AMDGPU::AGPR_32RegClass);
2121 BuildMI(BB&: MBB, I: CopyMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: Tmp)
2122 .add(MO: *Def);
2123 B.addReg(RegNo: Tmp);
2124 } else {
2125 TargetInstrInfo::RegSubRegPair Src = getRegSubRegPair(O: *Def);
2126 Def->setIsKill(false);
2127
2128 Register &VGPRCopy = VGPRCopies[Src];
2129 if (!VGPRCopy) {
2130 const TargetRegisterClass *VGPRUseSubRC =
2131 TRI->getSubRegisterClass(UseRC, DestSubIdx);
2132
2133 // We cannot build a reg_sequence out of the same registers, they
2134 // must be copied. Better do it here before copyPhysReg() created
2135 // several reads to do the AGPR->VGPR->AGPR copy.
2136
2137 // Direct copy from SGPR to AGPR is not possible on gfx908. To avoid
2138 // creation of exploded copies SGPR->VGPR->AGPR in the copyPhysReg()
2139 // later, create a copy here and track if we already have such a copy.
2140 const TargetRegisterClass *SubRC =
2141 TRI->getSubRegisterClass(MRI->getRegClass(Reg: Src.Reg), Src.SubReg);
2142 if (!VGPRUseSubRC->hasSubClassEq(RC: SubRC)) {
2143 // TODO: Try to reconstrain class
2144 VGPRCopy = MRI->createVirtualRegister(RegClass: VGPRUseSubRC);
2145 BuildMI(BB&: MBB, I: CopyMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: VGPRCopy).add(MO: *Def);
2146 B.addReg(RegNo: VGPRCopy);
2147 } else {
2148 // If it is already a VGPR, do not copy the register.
2149 B.add(MO: *Def);
2150 }
2151 } else {
2152 B.addReg(RegNo: VGPRCopy);
2153 }
2154 }
2155
2156 B.addImm(Val: DestSubIdx);
2157 }
2158
2159 LLVM_DEBUG(dbgs() << "Folded " << *CopyMI);
2160 return true;
2161}
2162
2163bool SIFoldOperandsImpl::tryFoldFoldableCopy(
2164 MachineInstr &MI, MachineOperand *&CurrentKnownM0Val) const {
2165 Register DstReg = MI.getOperand(i: 0).getReg();
2166 // Specially track simple redefs of m0 to the same value in a block, so we
2167 // can erase the later ones.
2168 if (DstReg == AMDGPU::M0) {
2169 MachineOperand &NewM0Val = MI.getOperand(i: 1);
2170 if (CurrentKnownM0Val && CurrentKnownM0Val->isIdenticalTo(Other: NewM0Val)) {
2171 MI.eraseFromParent();
2172 return true;
2173 }
2174
2175 // We aren't tracking other physical registers
2176 CurrentKnownM0Val = (NewM0Val.isReg() && NewM0Val.getReg().isPhysical())
2177 ? nullptr
2178 : &NewM0Val;
2179 return false;
2180 }
2181
2182 MachineOperand *OpToFoldPtr;
2183 if (MI.getOpcode() == AMDGPU::V_MOV_B16_t16_e64) {
2184 // Folding when any src_modifiers are non-zero is unsupported
2185 if (TII->hasAnyModifiersSet(MI))
2186 return false;
2187 OpToFoldPtr = &MI.getOperand(i: 2);
2188 } else
2189 OpToFoldPtr = &MI.getOperand(i: 1);
2190 MachineOperand &OpToFold = *OpToFoldPtr;
2191 bool FoldingImm = OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
2192
2193 // FIXME: We could also be folding things like TargetIndexes.
2194 if (!FoldingImm && !OpToFold.isReg())
2195 return false;
2196
2197 // Fold virtual registers and constant physical registers.
2198 if (OpToFold.isReg() && OpToFold.getReg().isPhysical() &&
2199 !TRI->isConstantPhysReg(PhysReg: OpToFold.getReg()))
2200 return false;
2201
2202 // Prevent folding operands backwards in the function. For example,
2203 // the COPY opcode must not be replaced by 1 in this example:
2204 //
2205 // %3 = COPY %vgpr0; VGPR_32:%3
2206 // ...
2207 // %vgpr0 = V_MOV_B32_e32 1, implicit %exec
2208 if (!DstReg.isVirtual())
2209 return false;
2210
2211 const TargetRegisterClass *DstRC =
2212 MRI->getRegClass(Reg: MI.getOperand(i: 0).getReg());
2213
2214 // True16: Fix malformed 16-bit sgpr COPY produced by peephole-opt
2215 // Can remove this code if proper 16-bit SGPRs are implemented
2216 // Example: Pre-peephole-opt
2217 // %29:sgpr_lo16 = COPY %16.lo16:sreg_32
2218 // %32:sreg_32 = COPY %29:sgpr_lo16
2219 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2220 // Post-peephole-opt and DCE
2221 // %32:sreg_32 = COPY %16.lo16:sreg_32
2222 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2223 // After this transform
2224 // %32:sreg_32 = COPY %16:sreg_32
2225 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2226 // After the fold operands pass
2227 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %16:sreg_32
2228 if (MI.getOpcode() == AMDGPU::COPY && OpToFold.isReg() &&
2229 OpToFold.getSubReg()) {
2230 if (DstRC == &AMDGPU::SReg_32RegClass &&
2231 DstRC == MRI->getRegClass(Reg: OpToFold.getReg())) {
2232 if (!TRI->getMatchingSuperRegClass(A: DstRC, B: &AMDGPU::SGPR_LO16RegClass,
2233 Idx: OpToFold.getSubReg()))
2234 return false;
2235 OpToFold.setSubReg(0);
2236 }
2237 }
2238
2239 // Fold copy to AGPR through reg_sequence
2240 // TODO: Handle with subregister extract
2241 if (OpToFold.isReg() && MI.isCopy() && !MI.getOperand(i: 1).getSubReg()) {
2242 if (foldCopyToAGPRRegSequence(CopyMI: &MI))
2243 return true;
2244 }
2245
2246 FoldableDef Def(OpToFold, DstRC);
2247 bool Changed = foldInstOperand(MI, OpToFold: Def);
2248
2249 // If we managed to fold all uses of this copy then we might as well
2250 // delete it now.
2251 // The only reason we need to follow chains of copies here is that
2252 // tryFoldRegSequence looks forward through copies before folding a
2253 // REG_SEQUENCE into its eventual users.
2254 auto *InstToErase = &MI;
2255 while (MRI->use_nodbg_empty(RegNo: InstToErase->getOperand(i: 0).getReg())) {
2256 auto &SrcOp = InstToErase->getOperand(i: 1);
2257 auto SrcReg = SrcOp.isReg() ? SrcOp.getReg() : Register();
2258 InstToErase->eraseFromParent();
2259 Changed = true;
2260 InstToErase = nullptr;
2261 if (!SrcReg || SrcReg.isPhysical())
2262 break;
2263 InstToErase = MRI->getVRegDef(Reg: SrcReg);
2264 if (!InstToErase || !TII->isFoldableCopy(MI: *InstToErase))
2265 break;
2266 }
2267
2268 if (InstToErase && InstToErase->isRegSequence() &&
2269 MRI->use_nodbg_empty(RegNo: InstToErase->getOperand(i: 0).getReg())) {
2270 InstToErase->eraseFromParent();
2271 Changed = true;
2272 }
2273
2274 if (Changed)
2275 return true;
2276
2277 // Run this after foldInstOperand to avoid turning scalar additions into
2278 // vector additions when the result scalar result could just be folded into
2279 // the user(s).
2280 return OpToFold.isReg() &&
2281 foldCopyToVGPROfScalarAddOfFrameIndex(DstReg, SrcReg: OpToFold.getReg(), MI);
2282}
2283
2284// Clamp patterns are canonically selected to v_max_* instructions, so only
2285// handle them.
2286const MachineOperand *
2287SIFoldOperandsImpl::isClamp(const MachineInstr &MI) const {
2288 unsigned Op = MI.getOpcode();
2289 switch (Op) {
2290 case AMDGPU::V_MAX_F32_e64:
2291 case AMDGPU::V_MAX_F16_e64:
2292 case AMDGPU::V_MAX_F16_t16_e64:
2293 case AMDGPU::V_MAX_F16_fake16_e64:
2294 case AMDGPU::V_MAX_F64_e64:
2295 case AMDGPU::V_MAX_NUM_F64_e64:
2296 case AMDGPU::V_PK_MAX_F16:
2297 case AMDGPU::V_MAX_BF16_PSEUDO_e64:
2298 case AMDGPU::V_PK_MAX_NUM_BF16: {
2299 if (MI.mayRaiseFPException())
2300 return nullptr;
2301
2302 if (!TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp)->getImm())
2303 return nullptr;
2304
2305 // Make sure sources are identical.
2306 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2307 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2308 if (!Src0->isReg() || !Src1->isReg() ||
2309 Src0->getReg() != Src1->getReg() ||
2310 Src0->getSubReg() != Src1->getSubReg() ||
2311 Src0->getSubReg() != AMDGPU::NoSubRegister)
2312 return nullptr;
2313
2314 // Can't fold up if we have modifiers.
2315 if (TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
2316 return nullptr;
2317
2318 unsigned Src0Mods
2319 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm();
2320 unsigned Src1Mods
2321 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm();
2322
2323 // Having a 0 op_sel_hi would require swizzling the output in the source
2324 // instruction, which we can't do.
2325 unsigned UnsetMods =
2326 (Op == AMDGPU::V_PK_MAX_F16 || Op == AMDGPU::V_PK_MAX_NUM_BF16)
2327 ? SISrcMods::OP_SEL_1
2328 : 0u;
2329 if (Src0Mods != UnsetMods && Src1Mods != UnsetMods)
2330 return nullptr;
2331 return Src0;
2332 }
2333 default:
2334 return nullptr;
2335 }
2336}
2337
2338// FIXME: Clamp for v_mad_mixhi_f16 handled during isel.
2339bool SIFoldOperandsImpl::tryFoldClamp(MachineInstr &MI) {
2340 const MachineOperand *ClampSrc = isClamp(MI);
2341 if (!ClampSrc || !MRI->hasOneNonDBGUser(RegNo: ClampSrc->getReg()))
2342 return false;
2343
2344 if (!ClampSrc->getReg().isVirtual())
2345 return false;
2346
2347 // Look through COPY. COPY only observed with True16.
2348 Register DefSrcReg = TRI->lookThruCopyLike(SrcReg: ClampSrc->getReg(), MRI);
2349 MachineInstr *Def =
2350 MRI->getVRegDef(Reg: DefSrcReg.isVirtual() ? DefSrcReg : ClampSrc->getReg());
2351
2352 // The type of clamp must be compatible.
2353 if (!SIInstrInfo::hasSameClamp(A: *Def, B: MI))
2354 return false;
2355
2356 if (Def->mayRaiseFPException())
2357 return false;
2358
2359 MachineOperand *DefClamp = TII->getNamedOperand(MI&: *Def, OperandName: AMDGPU::OpName::clamp);
2360 if (!DefClamp)
2361 return false;
2362
2363 LLVM_DEBUG(dbgs() << "Folding clamp " << *DefClamp << " into " << *Def);
2364
2365 // Clamp is applied after omod, so it is OK if omod is set.
2366 DefClamp->setImm(1);
2367
2368 Register DefReg = Def->getOperand(i: 0).getReg();
2369 Register MIDstReg = MI.getOperand(i: 0).getReg();
2370 if (TRI->isSGPRReg(MRI: *MRI, Reg: DefReg)) {
2371 // Pseudo scalar instructions have a SGPR for dst and clamp is a v_max*
2372 // instruction with a VGPR dst.
2373 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
2374 DestReg: MIDstReg)
2375 .addReg(RegNo: DefReg);
2376 } else {
2377 MRI->replaceRegWith(FromReg: MIDstReg, ToReg: DefReg);
2378 }
2379 MI.eraseFromParent();
2380
2381 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
2382 // instruction, so we might as well convert it to the more flexible VOP3-only
2383 // mad/fma form.
2384 if (TII->convertToThreeAddress(MI&: *Def, LV: nullptr, LIS: nullptr))
2385 Def->eraseFromParent();
2386
2387 return true;
2388}
2389
2390static int getOModValue(unsigned Opc, int64_t Val) {
2391 switch (Opc) {
2392 case AMDGPU::V_MUL_F64_e64:
2393 case AMDGPU::V_MUL_F64_pseudo_e64: {
2394 switch (Val) {
2395 case 0x3fe0000000000000: // 0.5
2396 return SIOutMods::DIV2;
2397 case 0x4000000000000000: // 2.0
2398 return SIOutMods::MUL2;
2399 case 0x4010000000000000: // 4.0
2400 return SIOutMods::MUL4;
2401 default:
2402 return SIOutMods::NONE;
2403 }
2404 }
2405 case AMDGPU::V_MUL_F32_e64: {
2406 switch (static_cast<uint32_t>(Val)) {
2407 case 0x3f000000: // 0.5
2408 return SIOutMods::DIV2;
2409 case 0x40000000: // 2.0
2410 return SIOutMods::MUL2;
2411 case 0x40800000: // 4.0
2412 return SIOutMods::MUL4;
2413 default:
2414 return SIOutMods::NONE;
2415 }
2416 }
2417 case AMDGPU::V_MUL_F16_e64:
2418 case AMDGPU::V_MUL_F16_t16_e64:
2419 case AMDGPU::V_MUL_F16_fake16_e64: {
2420 switch (static_cast<uint16_t>(Val)) {
2421 case 0x3800: // 0.5
2422 return SIOutMods::DIV2;
2423 case 0x4000: // 2.0
2424 return SIOutMods::MUL2;
2425 case 0x4400: // 4.0
2426 return SIOutMods::MUL4;
2427 default:
2428 return SIOutMods::NONE;
2429 }
2430 }
2431 case AMDGPU::V_PK_MUL_BF16: {
2432 switch (static_cast<uint16_t>(Val)) {
2433 case 0x3F00: // 0.5 in BF16
2434 return SIOutMods::DIV2;
2435 case 0x4000: // 2.0 in BF16
2436 return SIOutMods::MUL2;
2437 case 0x4080: // 4.0 in BF16
2438 return SIOutMods::MUL4;
2439 default:
2440 return SIOutMods::NONE;
2441 }
2442 }
2443 default:
2444 llvm_unreachable("invalid mul opcode");
2445 }
2446}
2447
2448// FIXME: Does this really not support denormals with f16?
2449// FIXME: Does this need to check IEEE mode bit? SNaNs are generally not
2450// handled, so will anything other than that break?
2451std::pair<const MachineOperand *, int>
2452SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
2453 unsigned Op = MI.getOpcode();
2454 switch (Op) {
2455 case AMDGPU::V_MUL_F64_e64:
2456 case AMDGPU::V_MUL_F64_pseudo_e64:
2457 case AMDGPU::V_MUL_F32_e64:
2458 case AMDGPU::V_MUL_F16_t16_e64:
2459 case AMDGPU::V_MUL_F16_fake16_e64:
2460 case AMDGPU::V_MUL_F16_e64: {
2461 // If output denormals are enabled, omod is ignored.
2462 if ((Op == AMDGPU::V_MUL_F32_e64 &&
2463 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
2464 ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F64_pseudo_e64 ||
2465 Op == AMDGPU::V_MUL_F16_e64 || Op == AMDGPU::V_MUL_F16_t16_e64 ||
2466 Op == AMDGPU::V_MUL_F16_fake16_e64) &&
2467 MFI->getMode().FP64FP16Denormals.Output !=
2468 DenormalMode::PreserveSign) ||
2469 MI.mayRaiseFPException())
2470 return {nullptr, SIOutMods::NONE};
2471
2472 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2473 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2474
2475 // If there is an immediate operand, it must be Src1
2476 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
2477 if (!Src1Imm)
2478 return {nullptr, SIOutMods::NONE};
2479
2480 int OMod = getOModValue(Opc: Op, Val: *Src1Imm);
2481 if (OMod == SIOutMods::NONE ||
2482 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers) ||
2483 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers) ||
2484 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2485 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2486 return {nullptr, SIOutMods::NONE};
2487
2488 return {Src0, OMod};
2489 }
2490 case AMDGPU::V_ADD_F64_e64:
2491 case AMDGPU::V_ADD_F64_pseudo_e64:
2492 case AMDGPU::V_ADD_F32_e64:
2493 case AMDGPU::V_ADD_F16_e64:
2494 case AMDGPU::V_ADD_F16_t16_e64:
2495 case AMDGPU::V_ADD_F16_fake16_e64: {
2496 // If output denormals are enabled, omod is ignored.
2497 if ((Op == AMDGPU::V_ADD_F32_e64 &&
2498 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
2499 ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F64_pseudo_e64 ||
2500 Op == AMDGPU::V_ADD_F16_e64 || Op == AMDGPU::V_ADD_F16_t16_e64 ||
2501 Op == AMDGPU::V_ADD_F16_fake16_e64) &&
2502 MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign))
2503 return {nullptr, SIOutMods::NONE};
2504
2505 // Look through the DAGCombiner canonicalization fmul x, 2 -> fadd x, x
2506 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2507 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2508
2509 if (Src0->isReg() && Src1->isReg() && Src0->getReg() == Src1->getReg() &&
2510 Src0->getSubReg() == Src1->getSubReg() &&
2511 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers) &&
2512 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers) &&
2513 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) &&
2514 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
2515 return {Src0, SIOutMods::MUL2};
2516
2517 return {nullptr, SIOutMods::NONE};
2518 }
2519 case AMDGPU::V_PK_MUL_BF16: {
2520 // OMOD folding for BF16 packed multiply. bf16 has no denormal mode of its
2521 // own; it follows the default ("denormal-fp-math") mode, which is the same
2522 // field as f64/f16.
2523 if (MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign ||
2524 MI.mayRaiseFPException())
2525 return {nullptr, SIOutMods::NONE};
2526
2527 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2528 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2529
2530 // If there is an immediate operand, it must be Src1
2531 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(MRI: *MRI, Op: *Src1);
2532 if (!Src1Imm)
2533 return {nullptr, SIOutMods::NONE};
2534
2535 int OMod = getOModValue(Opc: AMDGPU::V_PK_MUL_BF16, Val: *Src1Imm);
2536 if (OMod == SIOutMods::NONE)
2537 return {nullptr, SIOutMods::NONE};
2538
2539 // Modifiers other than op_sel_hi block OMOD folding
2540 const MachineOperand *Src0Mods =
2541 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2542 const MachineOperand *Src1Mods =
2543 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2544 if ((Src0Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2545 (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2546 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2547 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2548 return {nullptr, SIOutMods::NONE};
2549
2550 return {Src0, OMod};
2551 }
2552 case AMDGPU::V_PK_ADD_BF16: {
2553 // OMOD folding for BF16 packed add: x + x -> x * 2. See the bf16 denormal
2554 // mode note in the V_PK_MUL_BF16 case above.
2555 if (MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign)
2556 return {nullptr, SIOutMods::NONE};
2557
2558 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2559 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2560
2561 if (!Src0->isReg() || !Src1->isReg() || Src0->getReg() != Src1->getReg() ||
2562 Src0->getSubReg() != Src1->getSubReg())
2563 return {nullptr, SIOutMods::NONE};
2564
2565 // Modifiers other than op_sel_hi block OMOD folding
2566 const MachineOperand *Src0Mods =
2567 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers);
2568 const MachineOperand *Src1Mods =
2569 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers);
2570 if ((Src0Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2571 (Src1Mods->getImm() & ~SISrcMods::OP_SEL_1) ||
2572 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2573 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2574 return {nullptr, SIOutMods::NONE};
2575
2576 return {Src0, SIOutMods::MUL2};
2577 }
2578 default:
2579 return {nullptr, SIOutMods::NONE};
2580 }
2581}
2582
2583// FIXME: Does this need to check IEEE bit on function?
2584bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
2585 const MachineOperand *RegOp;
2586 int OMod;
2587 std::tie(args&: RegOp, args&: OMod) = isOMod(MI);
2588 if (OMod == SIOutMods::NONE || !RegOp->isReg() ||
2589 RegOp->getSubReg() != AMDGPU::NoSubRegister ||
2590 !MRI->hasOneNonDBGUser(RegNo: RegOp->getReg()))
2591 return false;
2592
2593 MachineInstr *Def = MRI->getVRegDef(Reg: RegOp->getReg());
2594 Register OModSrcReg = Def->getOperand(i: 0).getReg();
2595
2596 // In real-true16 mode, vgpr_16 results are packed into vgpr_32 via
2597 // REG_SEQUENCE. Look through it to find the actual instruction.
2598 if (Def->isRegSequence() && Def->getNumOperands() == 5 &&
2599 Def->getOperand(i: 2).getImm() == AMDGPU::lo16) {
2600 // Only look through if the high 16 bits are undefined
2601 bool CanLookThrough = true;
2602 MachineInstr *Hi16Def = MRI->getVRegDef(Reg: Def->getOperand(i: 3).getReg());
2603 if (!Hi16Def || !Hi16Def->isImplicitDef())
2604 CanLookThrough = false;
2605
2606 if (CanLookThrough) {
2607 Register SrcReg = Def->getOperand(i: 1).getReg();
2608 if (!MRI->hasOneNonDBGUse(RegNo: SrcReg))
2609 return false;
2610
2611 Def = MRI->getVRegDef(Reg: SrcReg);
2612 if (!Def)
2613 return false;
2614 }
2615 }
2616
2617 MachineOperand *DefOMod = TII->getNamedOperand(MI&: *Def, OperandName: AMDGPU::OpName::omod);
2618 if (!DefOMod || DefOMod->getImm() != SIOutMods::NONE)
2619 return false;
2620
2621 if (Def->mayRaiseFPException())
2622 return false;
2623
2624 // Clamp is applied after omod. If the source already has clamp set, don't
2625 // fold it.
2626 if (TII->hasModifiersSet(MI: *Def, OpName: AMDGPU::OpName::clamp))
2627 return false;
2628
2629 LLVM_DEBUG(dbgs() << "Folding omod " << MI << " into " << *Def);
2630
2631 DefOMod->setImm(OMod);
2632 MRI->replaceRegWith(FromReg: MI.getOperand(i: 0).getReg(), ToReg: OModSrcReg);
2633 // Kill flags can be wrong if we replaced a def inside a loop with a def
2634 // outside the loop.
2635 MRI->clearKillFlags(Reg: OModSrcReg);
2636 MI.eraseFromParent();
2637
2638 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
2639 // instruction, so we might as well convert it to the more flexible VOP3-only
2640 // mad/fma form.
2641 if (TII->convertToThreeAddress(MI&: *Def, LV: nullptr, LIS: nullptr))
2642 Def->eraseFromParent();
2643
2644 return true;
2645}
2646
2647// Try to optimize SGPR reg sequences that are splat <s, s> or <s, s, s, s>
2648// where all uses are PackedSingleSGPR64BitInst, replacing with <s, undef, ...>
2649bool SIFoldOperandsImpl::tryFoldSGPRSplatRegSequence(MachineInstr &MI) {
2650 assert(MI.isRegSequence());
2651
2652 if (!ST->hasPackedFP64SingleSGPROps() && !ST->hasPackedU64SingleSGPROps())
2653 return false;
2654
2655 Register Reg = MI.getOperand(i: 0).getReg();
2656
2657 // Only optimize 128-bit SGPR register sequences
2658 const TargetRegisterClass *RegClass = MRI->getRegClass(Reg);
2659 if (!TRI->isSGPRClass(RC: RegClass) || TRI->getRegSizeInBits(RC: *RegClass) != 128)
2660 return false;
2661
2662 SmallVector<std::pair<MachineOperand *, unsigned>, 32> Defs;
2663 if (!getRegSeqInit(Defs, UseReg: Reg))
2664 return false;
2665
2666 // Check if this is a splat pattern
2667 if (Defs.size() <= 1)
2668 return false;
2669
2670 const auto &[FirstOp, _] = Defs.front();
2671 if (!FirstOp->isReg())
2672 return false;
2673
2674 Register FirstReg = FirstOp->getReg();
2675 unsigned FirstSubReg = FirstOp->getSubReg();
2676
2677 const TargetRegisterClass *FirstRegClass = MRI->getRegClass(Reg: FirstReg);
2678 if (!TRI->isSGPRClass(RC: FirstRegClass))
2679 return false;
2680
2681 // Check remaining elements match first
2682 if (!llvm::all_of(Range: llvm::drop_begin(RangeOrContainer&: Defs), P: [&](const auto &Def) {
2683 const auto &[Op, _] = Def;
2684 return Op->isReg() && Op->getReg() == FirstReg &&
2685 Op->getSubReg() == FirstSubReg;
2686 }))
2687 return false;
2688
2689 // Check if all uses are isSingleSGPRReadInst
2690 for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) {
2691 if (!AMDGPU::isPackedSingleSGPR64BitInst(Opc: UseMI.getOpcode()))
2692 return false;
2693 }
2694
2695 // Create new reg sequence with <s, undef, undef, ...>
2696 Register NewDst = MRI->createVirtualRegister(RegClass);
2697 MachineInstrBuilder RS = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2698 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: NewDst);
2699
2700 // Add the first operand
2701 FirstOp->setIsKill(false);
2702 RS.add(MO: *FirstOp);
2703 RS.addImm(Val: Defs[0].second);
2704
2705 // Add undef for remaining lanes
2706 // Create an undef virtual register for the same register class
2707 Register UndefReg = MRI->createVirtualRegister(RegClass: FirstRegClass);
2708 for (unsigned i = 1; i < Defs.size(); ++i) {
2709 RS.addReg(RegNo: UndefReg, Flags: RegState::Undef);
2710 RS.addImm(Val: Defs[i].second);
2711 }
2712
2713 // Replace all uses
2714 MRI->replaceRegWith(FromReg: Reg, ToReg: NewDst);
2715
2716 LLVM_DEBUG(dbgs() << "Folded splat SGPR reg_sequence: " << MI << " into "
2717 << *RS);
2718
2719 MI.eraseFromParent();
2720 return true;
2721}
2722
2723// Try to fold a reg_sequence with vgpr output and agpr inputs into an
2724// instruction which can take an agpr. So far that means a store.
2725bool SIFoldOperandsImpl::tryFoldRegSequence(MachineInstr &MI) {
2726 assert(MI.isRegSequence());
2727
2728 // Try to optimize SGPR splat sequences first
2729 if (tryFoldSGPRSplatRegSequence(MI))
2730 return true;
2731
2732 auto Reg = MI.getOperand(i: 0).getReg();
2733
2734 if (!ST->hasGFX90AInsts() || !TRI->isVGPR(MRI: *MRI, Reg) ||
2735 !MRI->hasOneNonDBGUse(RegNo: Reg))
2736 return false;
2737
2738 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
2739 if (!getRegSeqInit(Defs, UseReg: Reg))
2740 return false;
2741
2742 for (auto &[Op, SubIdx] : Defs) {
2743 if (!Op->isReg())
2744 return false;
2745 if (TRI->isAGPR(MRI: *MRI, Reg: Op->getReg()))
2746 continue;
2747 // Maybe this is a COPY from AREG
2748 const MachineInstr *SubDef = MRI->getVRegDef(Reg: Op->getReg());
2749 if (!SubDef || !SubDef->isCopy() || SubDef->getOperand(i: 1).getSubReg())
2750 return false;
2751 if (!TRI->isAGPR(MRI: *MRI, Reg: SubDef->getOperand(i: 1).getReg()))
2752 return false;
2753 }
2754
2755 MachineOperand *Op = &*MRI->use_nodbg_begin(RegNo: Reg);
2756 MachineInstr *UseMI = Op->getParent();
2757 while (UseMI->isCopy() && !Op->getSubReg()) {
2758 Reg = UseMI->getOperand(i: 0).getReg();
2759 if (!TRI->isVGPR(MRI: *MRI, Reg) || !MRI->hasOneNonDBGUse(RegNo: Reg))
2760 return false;
2761 Op = &*MRI->use_nodbg_begin(RegNo: Reg);
2762 UseMI = Op->getParent();
2763 }
2764
2765 if (Op->getSubReg())
2766 return false;
2767
2768 unsigned OpIdx = Op - &UseMI->getOperand(i: 0);
2769 const MCInstrDesc &InstDesc = UseMI->getDesc();
2770 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: InstDesc, OpNum: OpIdx);
2771 if (!OpRC || !TRI->isVectorSuperClass(RC: OpRC))
2772 return false;
2773
2774 const auto *NewDstRC = TRI->getEquivalentAGPRClass(SRC: MRI->getRegClass(Reg));
2775 auto Dst = MRI->createVirtualRegister(RegClass: NewDstRC);
2776 auto RS = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2777 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst);
2778
2779 for (auto &[Def, SubIdx] : Defs) {
2780 Def->setIsKill(false);
2781 if (TRI->isAGPR(MRI: *MRI, Reg: Def->getReg())) {
2782 RS.add(MO: *Def);
2783 } else { // This is a copy
2784 MachineInstr *SubDef = MRI->getVRegDef(Reg: Def->getReg());
2785 SubDef->getOperand(i: 1).setIsKill(false);
2786 RS.addReg(RegNo: SubDef->getOperand(i: 1).getReg(), Flags: {}, SubReg: Def->getSubReg());
2787 }
2788 RS.addImm(Val: SubIdx);
2789 }
2790
2791 Op->setReg(Dst);
2792 if (!TII->isOperandLegal(MI: *UseMI, OpIdx, MO: Op)) {
2793 Op->setReg(Reg);
2794 RS->eraseFromParent();
2795 return false;
2796 }
2797
2798 LLVM_DEBUG(dbgs() << "Folded " << *RS << " into " << *UseMI);
2799
2800 // Erase the REG_SEQUENCE eagerly, unless we followed a chain of COPY users,
2801 // in which case we can erase them all later in runOnMachineFunction.
2802 if (MRI->use_nodbg_empty(RegNo: MI.getOperand(i: 0).getReg()))
2803 MI.eraseFromParent();
2804 return true;
2805}
2806
2807/// Checks whether \p Copy is a AGPR -> VGPR copy. Returns `true` on success and
2808/// stores the AGPR register in \p OutReg and the subreg in \p OutSubReg
2809static bool isAGPRCopy(const SIRegisterInfo &TRI,
2810 const MachineRegisterInfo &MRI, const MachineInstr &Copy,
2811 Register &OutReg, unsigned &OutSubReg) {
2812 assert(Copy.isCopy());
2813
2814 const MachineOperand &CopySrc = Copy.getOperand(i: 1);
2815 Register CopySrcReg = CopySrc.getReg();
2816 if (!CopySrcReg.isVirtual())
2817 return false;
2818
2819 // Common case: copy from AGPR directly, e.g.
2820 // %1:vgpr_32 = COPY %0:agpr_32
2821 if (TRI.isAGPR(MRI, Reg: CopySrcReg)) {
2822 OutReg = CopySrcReg;
2823 OutSubReg = CopySrc.getSubReg();
2824 return true;
2825 }
2826
2827 // Sometimes it can also involve two copies, e.g.
2828 // %1:vgpr_256 = COPY %0:agpr_256
2829 // %2:vgpr_32 = COPY %1:vgpr_256.sub0
2830 const MachineInstr *CopySrcDef = MRI.getVRegDef(Reg: CopySrcReg);
2831 if (!CopySrcDef || !CopySrcDef->isCopy())
2832 return false;
2833
2834 const MachineOperand &OtherCopySrc = CopySrcDef->getOperand(i: 1);
2835 Register OtherCopySrcReg = OtherCopySrc.getReg();
2836 if (!OtherCopySrcReg.isVirtual() ||
2837 CopySrcDef->getOperand(i: 0).getSubReg() != AMDGPU::NoSubRegister ||
2838 OtherCopySrc.getSubReg() != AMDGPU::NoSubRegister ||
2839 !TRI.isAGPR(MRI, Reg: OtherCopySrcReg))
2840 return false;
2841
2842 OutReg = OtherCopySrcReg;
2843 OutSubReg = CopySrc.getSubReg();
2844 return true;
2845}
2846
2847// Try to hoist an AGPR to VGPR copy across a PHI.
2848// This should allow folding of an AGPR into a consumer which may support it.
2849//
2850// Example 1: LCSSA PHI
2851// loop:
2852// %1:vreg = COPY %0:areg
2853// exit:
2854// %2:vreg = PHI %1:vreg, %loop
2855// =>
2856// loop:
2857// exit:
2858// %1:areg = PHI %0:areg, %loop
2859// %2:vreg = COPY %1:areg
2860//
2861// Example 2: PHI with multiple incoming values:
2862// entry:
2863// %1:vreg = GLOBAL_LOAD(..)
2864// loop:
2865// %2:vreg = PHI %1:vreg, %entry, %5:vreg, %loop
2866// %3:areg = COPY %2:vreg
2867// %4:areg = (instr using %3:areg)
2868// %5:vreg = COPY %4:areg
2869// =>
2870// entry:
2871// %1:vreg = GLOBAL_LOAD(..)
2872// %2:areg = COPY %1:vreg
2873// loop:
2874// %3:areg = PHI %2:areg, %entry, %X:areg,
2875// %4:areg = (instr using %3:areg)
2876bool SIFoldOperandsImpl::tryFoldPhiAGPR(MachineInstr &PHI) {
2877 assert(PHI.isPHI());
2878
2879 Register PhiOut = PHI.getOperand(i: 0).getReg();
2880 if (!TRI->isVGPR(MRI: *MRI, Reg: PhiOut))
2881 return false;
2882
2883 // Iterate once over all incoming values of the PHI to check if this PHI is
2884 // eligible, and determine the exact AGPR RC we'll target.
2885 const TargetRegisterClass *ARC = nullptr;
2886 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
2887 MachineOperand &MO = PHI.getOperand(i: K);
2888 MachineInstr *Copy = MRI->getVRegDef(Reg: MO.getReg());
2889 if (!Copy || !Copy->isCopy())
2890 continue;
2891
2892 Register AGPRSrc;
2893 unsigned AGPRRegMask = AMDGPU::NoSubRegister;
2894 if (!isAGPRCopy(TRI: *TRI, MRI: *MRI, Copy: *Copy, OutReg&: AGPRSrc, OutSubReg&: AGPRRegMask))
2895 continue;
2896
2897 const TargetRegisterClass *CopyInRC = MRI->getRegClass(Reg: AGPRSrc);
2898 if (const auto *SubRC = TRI->getSubRegisterClass(CopyInRC, AGPRRegMask))
2899 CopyInRC = SubRC;
2900
2901 if (ARC && !ARC->hasSubClassEq(RC: CopyInRC))
2902 return false;
2903 ARC = CopyInRC;
2904 }
2905
2906 if (!ARC)
2907 return false;
2908
2909 bool IsAGPR32 = (ARC == &AMDGPU::AGPR_32RegClass);
2910
2911 // Rewrite the PHI's incoming values to ARC.
2912 LLVM_DEBUG(dbgs() << "Folding AGPR copies into: " << PHI);
2913 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
2914 MachineOperand &MO = PHI.getOperand(i: K);
2915 Register Reg = MO.getReg();
2916
2917 MachineBasicBlock::iterator InsertPt;
2918 MachineBasicBlock *InsertMBB = nullptr;
2919
2920 // Look at the def of Reg, ignoring all copies.
2921 unsigned CopyOpc = AMDGPU::COPY;
2922 if (MachineInstr *Def = MRI->getVRegDef(Reg)) {
2923
2924 // Look at pre-existing COPY instructions from ARC: Steal the operand. If
2925 // the copy was single-use, it will be removed by DCE later.
2926 if (Def->isCopy()) {
2927 Register AGPRSrc;
2928 unsigned AGPRSubReg = AMDGPU::NoSubRegister;
2929 if (isAGPRCopy(TRI: *TRI, MRI: *MRI, Copy: *Def, OutReg&: AGPRSrc, OutSubReg&: AGPRSubReg)) {
2930 MO.setReg(AGPRSrc);
2931 MO.setSubReg(AGPRSubReg);
2932 continue;
2933 }
2934
2935 // If this is a multi-use SGPR -> VGPR copy, use V_ACCVGPR_WRITE on
2936 // GFX908 directly instead of a COPY. Otherwise, SIFoldOperand may try
2937 // to fold the sgpr -> vgpr -> agpr copy into a sgpr -> agpr copy which
2938 // is unlikely to be profitable.
2939 //
2940 // Note that V_ACCVGPR_WRITE is only used for AGPR_32.
2941 MachineOperand &CopyIn = Def->getOperand(i: 1);
2942 if (IsAGPR32 && !ST->hasGFX90AInsts() && !MRI->hasOneNonDBGUse(RegNo: Reg) &&
2943 TRI->isSGPRReg(MRI: *MRI, Reg: CopyIn.getReg()))
2944 CopyOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
2945 }
2946
2947 InsertMBB = Def->getParent();
2948 InsertPt = InsertMBB->SkipPHIsLabelsAndDebug(I: ++Def->getIterator());
2949 } else {
2950 InsertMBB = PHI.getOperand(i: MO.getOperandNo() + 1).getMBB();
2951 InsertPt = InsertMBB->getFirstTerminator();
2952 }
2953
2954 Register NewReg = MRI->createVirtualRegister(RegClass: ARC);
2955 MachineInstr *MI = BuildMI(BB&: *InsertMBB, I: InsertPt, MIMD: PHI.getDebugLoc(),
2956 MCID: TII->get(Opcode: CopyOpc), DestReg: NewReg)
2957 .addReg(RegNo: Reg);
2958 MO.setReg(NewReg);
2959
2960 (void)MI;
2961 LLVM_DEBUG(dbgs() << " Created COPY: " << *MI);
2962 }
2963
2964 // Replace the PHI's result with a new register.
2965 Register NewReg = MRI->createVirtualRegister(RegClass: ARC);
2966 PHI.getOperand(i: 0).setReg(NewReg);
2967
2968 // COPY that new register back to the original PhiOut register. This COPY will
2969 // usually be folded out later.
2970 MachineBasicBlock *MBB = PHI.getParent();
2971 BuildMI(BB&: *MBB, I: MBB->getFirstNonPHI(), MIMD: PHI.getDebugLoc(),
2972 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: PhiOut)
2973 .addReg(RegNo: NewReg);
2974
2975 LLVM_DEBUG(dbgs() << " Done: Folded " << PHI);
2976 return true;
2977}
2978
2979// Attempt to convert VGPR load to an AGPR load.
2980bool SIFoldOperandsImpl::tryFoldLoad(MachineInstr &MI) {
2981 assert(MI.mayLoad());
2982 if (!ST->hasGFX90AInsts() || MI.getNumExplicitDefs() != 1)
2983 return false;
2984
2985 MachineOperand &Def = MI.getOperand(i: 0);
2986 if (!Def.isDef())
2987 return false;
2988
2989 Register DefReg = Def.getReg();
2990
2991 if (DefReg.isPhysical() || !TRI->isVGPR(MRI: *MRI, Reg: DefReg))
2992 return false;
2993
2994 SmallVector<const MachineInstr *, 8> Users(
2995 llvm::make_pointer_range(Range: MRI->use_nodbg_instructions(Reg: DefReg)));
2996 SmallVector<Register, 8> MoveRegs;
2997
2998 if (Users.empty())
2999 return false;
3000
3001 // Check that all uses a copy to an agpr or a reg_sequence producing an agpr.
3002 while (!Users.empty()) {
3003 const MachineInstr *I = Users.pop_back_val();
3004 if (!I->isCopy() && !I->isRegSequence())
3005 return false;
3006 Register DstReg = I->getOperand(i: 0).getReg();
3007 // Physical registers may have more than one instruction definitions
3008 if (DstReg.isPhysical())
3009 return false;
3010 if (TRI->isAGPR(MRI: *MRI, Reg: DstReg))
3011 continue;
3012 MoveRegs.push_back(Elt: DstReg);
3013 for (const MachineInstr &U : MRI->use_nodbg_instructions(Reg: DstReg))
3014 Users.push_back(Elt: &U);
3015 }
3016
3017 const TargetRegisterClass *RC = MRI->getRegClass(Reg: DefReg);
3018 MRI->setRegClass(Reg: DefReg, RC: TRI->getEquivalentAGPRClass(SRC: RC));
3019 if (!TII->isOperandLegal(MI, OpIdx: 0, MO: &Def)) {
3020 MRI->setRegClass(Reg: DefReg, RC);
3021 return false;
3022 }
3023
3024 while (!MoveRegs.empty()) {
3025 Register Reg = MoveRegs.pop_back_val();
3026 MRI->setRegClass(Reg, RC: TRI->getEquivalentAGPRClass(SRC: MRI->getRegClass(Reg)));
3027 }
3028
3029 LLVM_DEBUG(dbgs() << "Folded " << MI);
3030
3031 return true;
3032}
3033
3034// tryFoldPhiAGPR will aggressively try to create AGPR PHIs.
3035// For GFX90A and later, this is pretty much always a good thing, but for GFX908
3036// there's cases where it can create a lot more AGPR-AGPR copies, which are
3037// expensive on this architecture due to the lack of V_ACCVGPR_MOV.
3038//
3039// This function looks at all AGPR PHIs in a basic block and collects their
3040// operands. Then, it checks for register that are used more than once across
3041// all PHIs and caches them in a VGPR. This prevents ExpandPostRAPseudo from
3042// having to create one VGPR temporary per use, which can get very messy if
3043// these PHIs come from a broken-up large PHI (e.g. 32 AGPR phis, one per vector
3044// element).
3045//
3046// Example
3047// a:
3048// %in:agpr_256 = COPY %foo:vgpr_256
3049// c:
3050// %x:agpr_32 = ..
3051// b:
3052// %0:areg = PHI %in.sub0:agpr_32, %a, %x, %c
3053// %1:areg = PHI %in.sub0:agpr_32, %a, %y, %c
3054// %2:areg = PHI %in.sub0:agpr_32, %a, %z, %c
3055// =>
3056// a:
3057// %in:agpr_256 = COPY %foo:vgpr_256
3058// %tmp:vgpr_32 = V_ACCVGPR_READ_B32_e64 %in.sub0:agpr_32
3059// %tmp_agpr:agpr_32 = COPY %tmp
3060// c:
3061// %x:agpr_32 = ..
3062// b:
3063// %0:areg = PHI %tmp_agpr, %a, %x, %c
3064// %1:areg = PHI %tmp_agpr, %a, %y, %c
3065// %2:areg = PHI %tmp_agpr, %a, %z, %c
3066bool SIFoldOperandsImpl::tryOptimizeAGPRPhis(MachineBasicBlock &MBB) {
3067 // This is only really needed on GFX908 where AGPR-AGPR copies are
3068 // unreasonably difficult.
3069 if (ST->hasGFX90AInsts())
3070 return false;
3071
3072 // Look at all AGPR Phis and collect the register + subregister used.
3073 DenseMap<std::pair<Register, unsigned>, std::vector<MachineOperand *>>
3074 RegToMO;
3075
3076 for (auto &MI : MBB) {
3077 if (!MI.isPHI())
3078 break;
3079
3080 if (!TRI->isAGPR(MRI: *MRI, Reg: MI.getOperand(i: 0).getReg()))
3081 continue;
3082
3083 for (unsigned K = 1; K < MI.getNumOperands(); K += 2) {
3084 MachineOperand &PhiMO = MI.getOperand(i: K);
3085 if (!PhiMO.getSubReg())
3086 continue;
3087 RegToMO[{PhiMO.getReg(), PhiMO.getSubReg()}].push_back(x: &PhiMO);
3088 }
3089 }
3090
3091 // For all (Reg, SubReg) pair that are used more than once, cache the value in
3092 // a VGPR.
3093 bool Changed = false;
3094 for (const auto &[Entry, MOs] : RegToMO) {
3095 if (MOs.size() == 1)
3096 continue;
3097
3098 const auto [Reg, SubReg] = Entry;
3099 MachineInstr *Def = MRI->getVRegDef(Reg);
3100 MachineBasicBlock *DefMBB = Def->getParent();
3101
3102 // Create a copy in a VGPR using V_ACCVGPR_READ_B32_e64 so it's not folded
3103 // out.
3104 const TargetRegisterClass *ARC = getRegOpRC(MRI: *MRI, TRI: *TRI, MO: *MOs.front());
3105 Register TempVGPR =
3106 MRI->createVirtualRegister(RegClass: TRI->getEquivalentVGPRClass(SRC: ARC));
3107 MachineInstr *VGPRCopy =
3108 BuildMI(BB&: *DefMBB, I: ++Def->getIterator(), MIMD: Def->getDebugLoc(),
3109 MCID: TII->get(Opcode: AMDGPU::V_ACCVGPR_READ_B32_e64), DestReg: TempVGPR)
3110 .addReg(RegNo: Reg, /* flags */ Flags: {}, SubReg);
3111
3112 // Copy back to an AGPR and use that instead of the AGPR subreg in all MOs.
3113 Register TempAGPR = MRI->createVirtualRegister(RegClass: ARC);
3114 BuildMI(BB&: *DefMBB, I: ++VGPRCopy->getIterator(), MIMD: Def->getDebugLoc(),
3115 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: TempAGPR)
3116 .addReg(RegNo: TempVGPR);
3117
3118 LLVM_DEBUG(dbgs() << "Caching AGPR into VGPR: " << *VGPRCopy);
3119 for (MachineOperand *MO : MOs) {
3120 MO->setReg(TempAGPR);
3121 MO->setSubReg(AMDGPU::NoSubRegister);
3122 LLVM_DEBUG(dbgs() << " Changed PHI Operand: " << *MO << "\n");
3123 }
3124
3125 Changed = true;
3126 }
3127
3128 return Changed;
3129}
3130
3131bool SIFoldOperandsImpl::run(MachineFunction &MF, const MachineLoopInfo *MLI) {
3132 this->MF = &MF;
3133 MRI = &MF.getRegInfo();
3134 ST = &MF.getSubtarget<GCNSubtarget>();
3135 TII = ST->getInstrInfo();
3136 TRI = &TII->getRegisterInfo();
3137 MFI = MF.getInfo<SIMachineFunctionInfo>();
3138 this->MLI = MLI;
3139
3140 // omod is ignored by hardware if IEEE bit is enabled. omod also does not
3141 // correctly handle signed zeros.
3142 //
3143 // FIXME: Also need to check strictfp
3144 bool IsIEEEMode = MFI->getMode().IEEE;
3145
3146 bool Changed = false;
3147 for (MachineBasicBlock *MBB : depth_first(G: &MF)) {
3148 MachineOperand *CurrentKnownM0Val = nullptr;
3149 for (auto &MI : make_early_inc_range(Range&: *MBB)) {
3150 Changed |= tryFoldCndMask(MI);
3151
3152 // PeepholeOptimizer may have folded an inline immediate directly onto an
3153 // instruction operand without materializing it into a register first.
3154 // Such an instruction is never reached through a def->use edge in
3155 // foldInstOperand, so try to constant fold it here.
3156 if (tryConstantFoldOp(MI: &MI)) {
3157 Changed = true;
3158 continue;
3159 }
3160
3161 if (tryFoldRedundantAND(ChildMI&: MI)) {
3162 Changed = true;
3163 continue;
3164 }
3165
3166 if (MI.isRegSequence() && tryFoldRegSequence(MI)) {
3167 Changed = true;
3168 continue;
3169 }
3170
3171 if (MI.isPHI() && tryFoldPhiAGPR(PHI&: MI)) {
3172 Changed = true;
3173 continue;
3174 }
3175
3176 if (MI.mayLoad() && tryFoldLoad(MI)) {
3177 Changed = true;
3178 continue;
3179 }
3180
3181 if (TII->isFoldableCopy(MI)) {
3182 Changed |= tryFoldFoldableCopy(MI, CurrentKnownM0Val);
3183 continue;
3184 }
3185
3186 // Saw an unknown clobber of m0, so we no longer know what it is.
3187 if (CurrentKnownM0Val && MI.modifiesRegister(Reg: AMDGPU::M0, TRI))
3188 CurrentKnownM0Val = nullptr;
3189
3190 // TODO: Omod might be OK if there is NSZ only on the source
3191 // instruction, and not the omod multiply.
3192 if (IsIEEEMode || !MI.getFlag(Flag: MachineInstr::FmNsz) || !tryFoldOMod(MI))
3193 Changed |= tryFoldClamp(MI);
3194 }
3195
3196 Changed |= tryOptimizeAGPRPhis(MBB&: *MBB);
3197 }
3198
3199 return Changed;
3200}
3201
3202PreservedAnalyses
3203SIFoldOperandsPass::run(MachineFunction &MF,
3204 MachineFunctionAnalysisManager &MFAM) {
3205 MFPropsModifier _(*this, MF);
3206
3207 const MachineLoopInfo *MLI = &MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
3208 bool Changed = SIFoldOperandsImpl().run(MF, MLI);
3209 if (!Changed) {
3210 return PreservedAnalyses::all();
3211 }
3212 auto PA = getMachineFunctionPassPreservedAnalyses();
3213 PA.preserveSet<CFGAnalyses>();
3214 PA.preserve<MachineLoopAnalysis>();
3215 return PA;
3216}
3217