1//===-- SIFoldOperands.cpp - Fold operands --- ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7/// \file
8//===----------------------------------------------------------------------===//
9//
10
11#include "SIFoldOperands.h"
12#include "AMDGPU.h"
13#include "GCNSubtarget.h"
14#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
15#include "SIInstrInfo.h"
16#include "SIMachineFunctionInfo.h"
17#include "SIRegisterInfo.h"
18#include "llvm/ADT/DepthFirstIterator.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineFunctionPass.h"
21#include "llvm/CodeGen/MachineOperand.h"
22
23#define DEBUG_TYPE "si-fold-operands"
24using namespace llvm;
25
26namespace {
27
28/// Track a value we may want to fold into downstream users, applying
29/// subregister extracts along the way.
30struct FoldableDef {
31 union {
32 MachineOperand *OpToFold = nullptr;
33 uint64_t ImmToFold;
34 int FrameIndexToFold;
35 };
36
37 /// Register class of the originally defined value.
38 const TargetRegisterClass *DefRC = nullptr;
39
40 /// Track the original defining instruction for the value.
41 const MachineInstr *DefMI = nullptr;
42
43 /// Subregister to apply to the value at the use point.
44 unsigned DefSubReg = AMDGPU::NoSubRegister;
45
46 /// Kind of value stored in the union.
47 MachineOperand::MachineOperandType Kind;
48
49 FoldableDef() = delete;
50 FoldableDef(MachineOperand &FoldOp, const TargetRegisterClass *DefRC,
51 unsigned DefSubReg = AMDGPU::NoSubRegister)
52 : DefRC(DefRC), DefSubReg(DefSubReg), Kind(FoldOp.getType()) {
53
54 if (FoldOp.isImm()) {
55 ImmToFold = FoldOp.getImm();
56 } else if (FoldOp.isFI()) {
57 FrameIndexToFold = FoldOp.getIndex();
58 } else {
59 assert(FoldOp.isReg() || FoldOp.isGlobal());
60 OpToFold = &FoldOp;
61 }
62
63 DefMI = FoldOp.getParent();
64 }
65
66 FoldableDef(int64_t FoldImm, const TargetRegisterClass *DefRC,
67 unsigned DefSubReg = AMDGPU::NoSubRegister)
68 : ImmToFold(FoldImm), DefRC(DefRC), DefSubReg(DefSubReg),
69 Kind(MachineOperand::MO_Immediate) {}
70
71 /// Copy the current def and apply \p SubReg to the value.
72 FoldableDef getWithSubReg(const SIRegisterInfo &TRI, unsigned SubReg) const {
73 FoldableDef Copy(*this);
74 Copy.DefSubReg = TRI.composeSubRegIndices(a: DefSubReg, b: SubReg);
75 return Copy;
76 }
77
78 bool isReg() const { return Kind == MachineOperand::MO_Register; }
79
80 Register getReg() const {
81 assert(isReg());
82 return OpToFold->getReg();
83 }
84
85 unsigned getSubReg() const {
86 assert(isReg());
87 return OpToFold->getSubReg();
88 }
89
90 bool isImm() const { return Kind == MachineOperand::MO_Immediate; }
91
92 bool isFI() const {
93 return Kind == MachineOperand::MO_FrameIndex;
94 }
95
96 int getFI() const {
97 assert(isFI());
98 return FrameIndexToFold;
99 }
100
101 bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; }
102
103 /// Return the effective immediate value defined by this instruction, after
104 /// application of any subregister extracts which may exist between the use
105 /// and def instruction.
106 std::optional<int64_t> getEffectiveImmVal() const {
107 assert(isImm());
108 return SIInstrInfo::extractSubregFromImm(ImmVal: ImmToFold, SubRegIndex: DefSubReg);
109 }
110
111 /// Check if it is legal to fold this effective value into \p MI's \p OpNo
112 /// operand.
113 bool isOperandLegal(const SIInstrInfo &TII, const MachineInstr &MI,
114 unsigned OpIdx) const {
115 switch (Kind) {
116 case MachineOperand::MO_Immediate: {
117 std::optional<int64_t> ImmToFold = getEffectiveImmVal();
118 if (!ImmToFold)
119 return false;
120
121 // TODO: Should verify the subregister index is supported by the class
122 // TODO: Avoid the temporary MachineOperand
123 MachineOperand TmpOp = MachineOperand::CreateImm(Val: *ImmToFold);
124 return TII.isOperandLegal(MI, OpIdx, MO: &TmpOp);
125 }
126 case MachineOperand::MO_FrameIndex: {
127 if (DefSubReg != AMDGPU::NoSubRegister)
128 return false;
129 MachineOperand TmpOp = MachineOperand::CreateFI(Idx: FrameIndexToFold);
130 return TII.isOperandLegal(MI, OpIdx, MO: &TmpOp);
131 }
132 default:
133 // TODO: Try to apply DefSubReg, for global address we can extract
134 // low/high.
135 if (DefSubReg != AMDGPU::NoSubRegister)
136 return false;
137 return TII.isOperandLegal(MI, OpIdx, MO: OpToFold);
138 }
139
140 llvm_unreachable("covered MachineOperand kind switch");
141 }
142};
143
144struct FoldCandidate {
145 MachineInstr *UseMI;
146 FoldableDef Def;
147 int ShrinkOpcode;
148 unsigned UseOpNo;
149 bool Commuted;
150
151 FoldCandidate(MachineInstr *MI, unsigned OpNo, FoldableDef Def,
152 bool Commuted = false, int ShrinkOp = -1)
153 : UseMI(MI), Def(Def), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo),
154 Commuted(Commuted) {}
155
156 bool isFI() const { return Def.isFI(); }
157
158 int getFI() const {
159 assert(isFI());
160 return Def.FrameIndexToFold;
161 }
162
163 bool isImm() const { return Def.isImm(); }
164
165 bool isReg() const { return Def.isReg(); }
166
167 Register getReg() const { return Def.getReg(); }
168
169 bool isGlobal() const { return Def.isGlobal(); }
170
171 bool needsShrink() const { return ShrinkOpcode != -1; }
172};
173
174class SIFoldOperandsImpl {
175public:
176 MachineFunction *MF;
177 MachineRegisterInfo *MRI;
178 const SIInstrInfo *TII;
179 const SIRegisterInfo *TRI;
180 const GCNSubtarget *ST;
181 const SIMachineFunctionInfo *MFI;
182
183 bool frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
184 const FoldableDef &OpToFold) const;
185
186 // TODO: Just use TII::getVALUOp
187 unsigned convertToVALUOp(unsigned Opc, bool UseVOP3 = false) const {
188 switch (Opc) {
189 case AMDGPU::S_ADD_I32: {
190 if (ST->hasAddNoCarryInsts())
191 return UseVOP3 ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_U32_e32;
192 return UseVOP3 ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
193 }
194 case AMDGPU::S_OR_B32:
195 return UseVOP3 ? AMDGPU::V_OR_B32_e64 : AMDGPU::V_OR_B32_e32;
196 case AMDGPU::S_AND_B32:
197 return UseVOP3 ? AMDGPU::V_AND_B32_e64 : AMDGPU::V_AND_B32_e32;
198 case AMDGPU::S_MUL_I32:
199 return AMDGPU::V_MUL_LO_U32_e64;
200 default:
201 return AMDGPU::INSTRUCTION_LIST_END;
202 }
203 }
204
205 bool foldCopyToVGPROfScalarAddOfFrameIndex(Register DstReg, Register SrcReg,
206 MachineInstr &MI) const;
207
208 bool updateOperand(FoldCandidate &Fold) const;
209
210 bool canUseImmWithOpSel(const MachineInstr *MI, unsigned UseOpNo,
211 int64_t ImmVal) const;
212
213 /// Try to fold immediate \p ImmVal into \p MI's operand at index \p UseOpNo.
214 bool tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
215 int64_t ImmVal) const;
216
217 bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
218 MachineInstr *MI, unsigned OpNo,
219 const FoldableDef &OpToFold) const;
220 bool isUseSafeToFold(const MachineInstr &MI,
221 const MachineOperand &UseMO) const;
222
223 const TargetRegisterClass *getRegSeqInit(
224 MachineInstr &RegSeq,
225 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs) const;
226
227 const TargetRegisterClass *
228 getRegSeqInit(SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
229 Register UseReg) const;
230
231 std::pair<int64_t, const TargetRegisterClass *>
232 isRegSeqSplat(MachineInstr &RegSeg) const;
233
234 bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
235 int64_t SplatVal,
236 const TargetRegisterClass *SplatRC) const;
237
238 bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
239 unsigned UseOpIdx,
240 SmallVectorImpl<FoldCandidate> &FoldList) const;
241 bool foldOperand(FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
242 SmallVectorImpl<FoldCandidate> &FoldList,
243 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const;
244
245 bool tryConstantFoldOp(MachineInstr *MI) const;
246 bool tryFoldCndMask(MachineInstr &MI) const;
247 bool tryFoldZeroHighBits(MachineInstr &MI) const;
248 bool foldInstOperand(MachineInstr &MI, const FoldableDef &OpToFold) const;
249
250 bool foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const;
251 bool tryFoldFoldableCopy(MachineInstr &MI,
252 MachineOperand *&CurrentKnownM0Val) const;
253
254 const MachineOperand *isClamp(const MachineInstr &MI) const;
255 bool tryFoldClamp(MachineInstr &MI);
256
257 std::pair<const MachineOperand *, int> isOMod(const MachineInstr &MI) const;
258 bool tryFoldOMod(MachineInstr &MI);
259 bool tryFoldRegSequence(MachineInstr &MI);
260 bool tryFoldPhiAGPR(MachineInstr &MI);
261 bool tryFoldLoad(MachineInstr &MI);
262
263 bool tryOptimizeAGPRPhis(MachineBasicBlock &MBB);
264
265public:
266 SIFoldOperandsImpl() = default;
267
268 bool run(MachineFunction &MF);
269};
270
271class SIFoldOperandsLegacy : public MachineFunctionPass {
272public:
273 static char ID;
274
275 SIFoldOperandsLegacy() : MachineFunctionPass(ID) {}
276
277 bool runOnMachineFunction(MachineFunction &MF) override {
278 if (skipFunction(F: MF.getFunction()))
279 return false;
280 return SIFoldOperandsImpl().run(MF);
281 }
282
283 StringRef getPassName() const override { return "SI Fold Operands"; }
284
285 void getAnalysisUsage(AnalysisUsage &AU) const override {
286 AU.setPreservesCFG();
287 MachineFunctionPass::getAnalysisUsage(AU);
288 }
289
290 MachineFunctionProperties getRequiredProperties() const override {
291 return MachineFunctionProperties().setIsSSA();
292 }
293};
294
295} // End anonymous namespace.
296
297INITIALIZE_PASS(SIFoldOperandsLegacy, DEBUG_TYPE, "SI Fold Operands", false,
298 false)
299
300char SIFoldOperandsLegacy::ID = 0;
301
302char &llvm::SIFoldOperandsLegacyID = SIFoldOperandsLegacy::ID;
303
304static const TargetRegisterClass *getRegOpRC(const MachineRegisterInfo &MRI,
305 const TargetRegisterInfo &TRI,
306 const MachineOperand &MO) {
307 const TargetRegisterClass *RC = MRI.getRegClass(Reg: MO.getReg());
308 if (const TargetRegisterClass *SubRC =
309 TRI.getSubRegisterClass(SuperRC: RC, SubRegIdx: MO.getSubReg()))
310 RC = SubRC;
311 return RC;
312}
313
314// Map multiply-accumulate opcode to corresponding multiply-add opcode if any.
315static unsigned macToMad(unsigned Opc) {
316 switch (Opc) {
317 case AMDGPU::V_MAC_F32_e64:
318 return AMDGPU::V_MAD_F32_e64;
319 case AMDGPU::V_MAC_F16_e64:
320 return AMDGPU::V_MAD_F16_e64;
321 case AMDGPU::V_FMAC_F32_e64:
322 return AMDGPU::V_FMA_F32_e64;
323 case AMDGPU::V_FMAC_F16_e64:
324 return AMDGPU::V_FMA_F16_gfx9_e64;
325 case AMDGPU::V_FMAC_F16_t16_e64:
326 return AMDGPU::V_FMA_F16_gfx9_t16_e64;
327 case AMDGPU::V_FMAC_F16_fake16_e64:
328 return AMDGPU::V_FMA_F16_gfx9_fake16_e64;
329 case AMDGPU::V_FMAC_LEGACY_F32_e64:
330 return AMDGPU::V_FMA_LEGACY_F32_e64;
331 case AMDGPU::V_FMAC_F64_e64:
332 return AMDGPU::V_FMA_F64_e64;
333 }
334 return AMDGPU::INSTRUCTION_LIST_END;
335}
336
337// TODO: Add heuristic that the frame index might not fit in the addressing mode
338// immediate offset to avoid materializing in loops.
339bool SIFoldOperandsImpl::frameIndexMayFold(const MachineInstr &UseMI, int OpNo,
340 const FoldableDef &OpToFold) const {
341 if (!OpToFold.isFI())
342 return false;
343
344 const unsigned Opc = UseMI.getOpcode();
345 switch (Opc) {
346 case AMDGPU::S_ADD_I32:
347 case AMDGPU::S_ADD_U32:
348 case AMDGPU::V_ADD_U32_e32:
349 case AMDGPU::V_ADD_CO_U32_e32:
350 // TODO: Possibly relax hasOneUse. It matters more for mubuf, since we have
351 // to insert the wave size shift at every point we use the index.
352 // TODO: Fix depending on visit order to fold immediates into the operand
353 return UseMI.getOperand(i: OpNo == 1 ? 2 : 1).isImm() &&
354 MRI->hasOneNonDBGUse(RegNo: UseMI.getOperand(i: OpNo).getReg());
355 case AMDGPU::V_ADD_U32_e64:
356 case AMDGPU::V_ADD_CO_U32_e64:
357 return UseMI.getOperand(i: OpNo == 2 ? 3 : 2).isImm() &&
358 MRI->hasOneNonDBGUse(RegNo: UseMI.getOperand(i: OpNo).getReg());
359 default:
360 break;
361 }
362
363 if (TII->isMUBUF(MI: UseMI))
364 return OpNo == AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
365 if (!TII->isFLATScratch(MI: UseMI))
366 return false;
367
368 int SIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::saddr);
369 if (OpNo == SIdx)
370 return true;
371
372 int VIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
373 return OpNo == VIdx && SIdx == -1;
374}
375
376/// Fold %vgpr = COPY (S_ADD_I32 x, frameindex)
377///
378/// => %vgpr = V_ADD_U32 x, frameindex
379bool SIFoldOperandsImpl::foldCopyToVGPROfScalarAddOfFrameIndex(
380 Register DstReg, Register SrcReg, MachineInstr &MI) const {
381 if (TRI->isVGPR(MRI: *MRI, Reg: DstReg) && TRI->isSGPRReg(MRI: *MRI, Reg: SrcReg) &&
382 MRI->hasOneNonDBGUse(RegNo: SrcReg)) {
383 MachineInstr *Def = MRI->getVRegDef(Reg: SrcReg);
384 if (!Def || Def->getNumOperands() != 4)
385 return false;
386
387 MachineOperand *Src0 = &Def->getOperand(i: 1);
388 MachineOperand *Src1 = &Def->getOperand(i: 2);
389
390 // TODO: This is profitable with more operand types, and for more
391 // opcodes. But ultimately this is working around poor / nonexistent
392 // regbankselect.
393 if (!Src0->isFI() && !Src1->isFI())
394 return false;
395
396 if (Src0->isFI())
397 std::swap(a&: Src0, b&: Src1);
398
399 const bool UseVOP3 = !Src0->isImm() || TII->isInlineConstant(MO: *Src0);
400 unsigned NewOp = convertToVALUOp(Opc: Def->getOpcode(), UseVOP3);
401 if (NewOp == AMDGPU::INSTRUCTION_LIST_END ||
402 !Def->getOperand(i: 3).isDead()) // Check if scc is dead
403 return false;
404
405 MachineBasicBlock *MBB = Def->getParent();
406 const DebugLoc &DL = Def->getDebugLoc();
407 if (NewOp != AMDGPU::V_ADD_CO_U32_e32) {
408 MachineInstrBuilder Add =
409 BuildMI(BB&: *MBB, I&: *Def, MIMD: DL, MCID: TII->get(Opcode: NewOp), DestReg: DstReg);
410
411 if (Add->getDesc().getNumDefs() == 2) {
412 Register CarryOutReg = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
413 Add.addDef(RegNo: CarryOutReg, Flags: RegState::Dead);
414 MRI->setRegAllocationHint(VReg: CarryOutReg, Type: 0, PrefReg: TRI->getVCC());
415 }
416
417 Add.add(MO: *Src0).add(MO: *Src1).setMIFlags(Def->getFlags());
418 if (AMDGPU::hasNamedOperand(Opcode: NewOp, NamedIdx: AMDGPU::OpName::clamp))
419 Add.addImm(Val: 0);
420
421 Def->eraseFromParent();
422 MI.eraseFromParent();
423 return true;
424 }
425
426 assert(NewOp == AMDGPU::V_ADD_CO_U32_e32);
427
428 MachineBasicBlock::LivenessQueryResult Liveness =
429 MBB->computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: *Def, Neighborhood: 16);
430 if (Liveness == MachineBasicBlock::LQR_Dead) {
431 // TODO: If src1 satisfies operand constraints, use vop3 version.
432 BuildMI(BB&: *MBB, I&: *Def, MIMD: DL, MCID: TII->get(Opcode: NewOp), DestReg: DstReg)
433 .add(MO: *Src0)
434 .add(MO: *Src1)
435 .setOperandDead(3) // implicit-def $vcc
436 .setMIFlags(Def->getFlags());
437 Def->eraseFromParent();
438 MI.eraseFromParent();
439 return true;
440 }
441 }
442
443 return false;
444}
445
446FunctionPass *llvm::createSIFoldOperandsLegacyPass() {
447 return new SIFoldOperandsLegacy();
448}
449
450bool SIFoldOperandsImpl::canUseImmWithOpSel(const MachineInstr *MI,
451 unsigned UseOpNo,
452 int64_t ImmVal) const {
453 if (!SIInstrFlags::isPacked(O: *MI) || SIInstrFlags::isMAI(O: *MI) ||
454 SIInstrFlags::isWMMA(O: *MI) || SIInstrFlags::isSWMMAC(O: *MI) ||
455 (ST->hasDOTOpSelHazard() && SIInstrFlags::isDOT(O: *MI)))
456 return false;
457
458 const MachineOperand &Old = MI->getOperand(i: UseOpNo);
459 int OpNo = MI->getOperandNo(I: &Old);
460
461 unsigned Opcode = MI->getOpcode();
462 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
463 switch (OpType) {
464 default:
465 return false;
466 case AMDGPU::OPERAND_REG_IMM_V2FP16:
467 case AMDGPU::OPERAND_REG_IMM_V2BF16:
468 case AMDGPU::OPERAND_REG_IMM_V2INT16:
469 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
470 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
471 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
472 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
473 // VOP3 packed instructions ignore op_sel source modifiers, we cannot encode
474 // two different constants.
475 if (SIInstrFlags::isVOP3(O: *MI) && !SIInstrFlags::isVOP3P(O: *MI) &&
476 static_cast<uint16_t>(ImmVal) != static_cast<uint16_t>(ImmVal >> 16))
477 return false;
478 break;
479 }
480
481 return true;
482}
483
484bool SIFoldOperandsImpl::tryFoldImmWithOpSel(MachineInstr *MI, unsigned UseOpNo,
485 int64_t ImmVal) const {
486 MachineOperand &Old = MI->getOperand(i: UseOpNo);
487 unsigned Opcode = MI->getOpcode();
488 int OpNo = MI->getOperandNo(I: &Old);
489 uint8_t OpType = TII->get(Opcode).operands()[OpNo].OperandType;
490
491 // If the literal can be inlined as-is, apply it and short-circuit the
492 // tests below. The main motivation for this is to avoid unintuitive
493 // uses of opsel.
494 if (AMDGPU::isInlinableLiteralV216(Literal: ImmVal, OpType)) {
495 Old.ChangeToImmediate(ImmVal);
496 return true;
497 }
498
499 // Refer to op_sel/op_sel_hi and check if we can change the immediate and
500 // op_sel in a way that allows an inline constant.
501 AMDGPU::OpName ModName = AMDGPU::OpName::NUM_OPERAND_NAMES;
502 unsigned SrcIdx = ~0;
503 if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0)) {
504 ModName = AMDGPU::OpName::src0_modifiers;
505 SrcIdx = 0;
506 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1)) {
507 ModName = AMDGPU::OpName::src1_modifiers;
508 SrcIdx = 1;
509 } else if (OpNo == AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src2)) {
510 ModName = AMDGPU::OpName::src2_modifiers;
511 SrcIdx = 2;
512 }
513 assert(ModName != AMDGPU::OpName::NUM_OPERAND_NAMES);
514 int ModIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: ModName);
515 MachineOperand &Mod = MI->getOperand(i: ModIdx);
516 unsigned ModVal = Mod.getImm();
517
518 uint16_t ImmLo =
519 static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_0 ? 16 : 0));
520 uint16_t ImmHi =
521 static_cast<uint16_t>(ImmVal >> (ModVal & SISrcMods::OP_SEL_1 ? 16 : 0));
522 uint32_t Imm = (static_cast<uint32_t>(ImmHi) << 16) | ImmLo;
523 unsigned NewModVal = ModVal & ~(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
524
525 // Helper function that attempts to inline the given value with a newly
526 // chosen opsel pattern.
527 auto tryFoldToInline = [&](uint32_t Imm) -> bool {
528 if (AMDGPU::isInlinableLiteralV216(Literal: Imm, OpType)) {
529 Mod.setImm(NewModVal | SISrcMods::OP_SEL_1);
530 Old.ChangeToImmediate(ImmVal: Imm);
531 return true;
532 }
533
534 // Try to shuffle the halves around and leverage opsel to get an inline
535 // constant.
536 uint16_t Lo = static_cast<uint16_t>(Imm);
537 uint16_t Hi = static_cast<uint16_t>(Imm >> 16);
538 if (Lo == Hi) {
539 if (AMDGPU::isInlinableLiteralV216(Literal: Lo, OpType)) {
540 // If the target has feature 'BF16InlineConstFromUpperFP32', packed BF16
541 // instructions using inline constant must use OPSEL to select the upper
542 // 16-bits from FP32.
543 if (ST->hasBF16InlineConstFromUpperFP32() &&
544 (OpType == AMDGPU::OPERAND_REG_INLINE_C_V2BF16 ||
545 OpType == AMDGPU::OPERAND_REG_IMM_V2BF16))
546 NewModVal |= (SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
547 Mod.setImm(NewModVal);
548 Old.ChangeToImmediate(ImmVal: Lo);
549 return true;
550 }
551
552 if (static_cast<int16_t>(Lo) < 0) {
553 int32_t SExt = static_cast<int16_t>(Lo);
554 if (AMDGPU::isInlinableLiteralV216(Literal: SExt, OpType)) {
555 Mod.setImm(NewModVal);
556 Old.ChangeToImmediate(ImmVal: SExt);
557 return true;
558 }
559 }
560
561 // This check is only useful for integer instructions
562 if (OpType == AMDGPU::OPERAND_REG_IMM_V2INT16) {
563 if (AMDGPU::isInlinableLiteralV216(Literal: Lo << 16, OpType)) {
564 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1);
565 Old.ChangeToImmediate(ImmVal: static_cast<uint32_t>(Lo) << 16);
566 return true;
567 }
568 }
569 } else {
570 uint32_t Swapped = (static_cast<uint32_t>(Lo) << 16) | Hi;
571 if (AMDGPU::isInlinableLiteralV216(Literal: Swapped, OpType)) {
572 Mod.setImm(NewModVal | SISrcMods::OP_SEL_0);
573 Old.ChangeToImmediate(ImmVal: Swapped);
574 return true;
575 }
576 }
577
578 return false;
579 };
580
581 if (tryFoldToInline(Imm))
582 return true;
583
584 // Replace integer addition by subtraction and vice versa if it allows
585 // folding the immediate to an inline constant.
586 //
587 // We should only ever get here for SrcIdx == 1 due to canonicalization
588 // earlier in the pipeline, but we double-check here to be safe / fully
589 // general.
590 bool IsUAdd = Opcode == AMDGPU::V_PK_ADD_U16;
591 bool IsUSub = Opcode == AMDGPU::V_PK_SUB_U16;
592 if (SrcIdx == 1 && (IsUAdd || IsUSub)) {
593 unsigned ClampIdx =
594 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::clamp);
595 bool Clamp = MI->getOperand(i: ClampIdx).getImm() != 0;
596
597 if (!Clamp) {
598 uint16_t NegLo = -static_cast<uint16_t>(Imm);
599 uint16_t NegHi = -static_cast<uint16_t>(Imm >> 16);
600 uint32_t NegImm = (static_cast<uint32_t>(NegHi) << 16) | NegLo;
601
602 if (tryFoldToInline(NegImm)) {
603 unsigned NegOpcode =
604 IsUAdd ? AMDGPU::V_PK_SUB_U16 : AMDGPU::V_PK_ADD_U16;
605 MI->setDesc(TII->get(Opcode: NegOpcode));
606 return true;
607 }
608 }
609 }
610
611 return false;
612}
613
614bool SIFoldOperandsImpl::updateOperand(FoldCandidate &Fold) const {
615 MachineInstr *MI = Fold.UseMI;
616 MachineOperand &Old = MI->getOperand(i: Fold.UseOpNo);
617 assert(Old.isReg());
618
619 std::optional<int64_t> ImmVal;
620 if (Fold.isImm())
621 ImmVal = Fold.Def.getEffectiveImmVal();
622
623 if (ImmVal && canUseImmWithOpSel(MI: Fold.UseMI, UseOpNo: Fold.UseOpNo, ImmVal: *ImmVal)) {
624 if (tryFoldImmWithOpSel(MI: Fold.UseMI, UseOpNo: Fold.UseOpNo, ImmVal: *ImmVal))
625 return true;
626
627 // We can't represent the candidate as an inline constant. Try as a literal
628 // with the original opsel, checking constant bus limitations.
629 MachineOperand New = MachineOperand::CreateImm(Val: *ImmVal);
630 int OpNo = MI->getOperandNo(I: &Old);
631 if (!TII->isOperandLegal(MI: *MI, OpIdx: OpNo, MO: &New))
632 return false;
633 Old.ChangeToImmediate(ImmVal: *ImmVal);
634 return true;
635 }
636
637 if ((Fold.isImm() || Fold.isFI() || Fold.isGlobal()) && Fold.needsShrink()) {
638 MachineBasicBlock *MBB = MI->getParent();
639 auto Liveness = MBB->computeRegisterLiveness(TRI, Reg: AMDGPU::VCC, Before: MI, Neighborhood: 16);
640 if (Liveness != MachineBasicBlock::LQR_Dead) {
641 LLVM_DEBUG(dbgs() << "Not shrinking due to live vcc: " << *MI);
642 return false;
643 }
644
645 int Op32 = Fold.ShrinkOpcode;
646 MachineOperand &Dst0 = MI->getOperand(i: 0);
647 MachineOperand &Dst1 = MI->getOperand(i: 1);
648 assert(Dst0.isDef() && Dst1.isDef());
649
650 bool HaveNonDbgCarryUse = !MRI->use_nodbg_empty(RegNo: Dst1.getReg());
651
652 const TargetRegisterClass *Dst0RC = MRI->getRegClass(Reg: Dst0.getReg());
653 Register NewReg0 = MRI->createVirtualRegister(RegClass: Dst0RC);
654
655 MachineInstr *Inst32 = TII->buildShrunkInst(MI&: *MI, NewOpcode: Op32);
656
657 if (HaveNonDbgCarryUse) {
658 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
659 DestReg: Dst1.getReg())
660 .addReg(RegNo: AMDGPU::VCC, Flags: RegState::Kill);
661 }
662
663 // Keep the old instruction around to avoid breaking iterators, but
664 // replace it with a dummy instruction to remove uses.
665 //
666 // FIXME: We should not invert how this pass looks at operands to avoid
667 // this. Should track set of foldable movs instead of looking for uses
668 // when looking at a use.
669 Dst0.setReg(NewReg0);
670 for (unsigned I = MI->getNumOperands() - 1; I > 0; --I)
671 MI->removeOperand(OpNo: I);
672 MI->setDesc(TII->get(Opcode: AMDGPU::IMPLICIT_DEF));
673
674 if (Fold.Commuted)
675 TII->commuteInstruction(MI&: *Inst32, NewMI: false);
676 return true;
677 }
678
679 assert(!Fold.needsShrink() && "not handled");
680
681 if (ImmVal) {
682 if (Old.isTied()) {
683 int NewMFMAOpc = AMDGPU::getMFMAEarlyClobberOp(Opcode: MI->getOpcode());
684 if (NewMFMAOpc == -1)
685 return false;
686 MI->setDesc(TII->get(Opcode: NewMFMAOpc));
687 MI->untieRegOperand(OpIdx: 0);
688 const MCInstrDesc &MCID = MI->getDesc();
689 for (unsigned I = 0; I < MI->getNumDefs(); ++I)
690 if (MCID.getOperandConstraint(OpNum: I, Constraint: MCOI::EARLY_CLOBBER) != -1)
691 MI->getOperand(i: I).setIsEarlyClobber(true);
692 }
693
694 // TODO: Should we try to avoid adding this to the candidate list?
695 MachineOperand New = MachineOperand::CreateImm(Val: *ImmVal);
696 int OpNo = MI->getOperandNo(I: &Old);
697 if (!TII->isOperandLegal(MI: *MI, OpIdx: OpNo, MO: &New))
698 return false;
699
700 Old.ChangeToImmediate(ImmVal: *ImmVal);
701 return true;
702 }
703
704 if (Fold.isGlobal()) {
705 Old.ChangeToGA(GV: Fold.Def.OpToFold->getGlobal(),
706 Offset: Fold.Def.OpToFold->getOffset(),
707 TargetFlags: Fold.Def.OpToFold->getTargetFlags());
708 return true;
709 }
710
711 if (Fold.isFI()) {
712 Old.ChangeToFrameIndex(Idx: Fold.getFI());
713 return true;
714 }
715
716 MachineOperand *New = Fold.Def.OpToFold;
717
718 // Verify the register is compatible with the operand.
719 if (const TargetRegisterClass *OpRC =
720 TII->getRegClass(MCID: MI->getDesc(), OpNum: Fold.UseOpNo)) {
721 const TargetRegisterClass *NewRC =
722 TRI->getRegClassForReg(MRI: *MRI, Reg: New->getReg());
723
724 const TargetRegisterClass *ConstrainRC = OpRC;
725 if (New->getSubReg()) {
726 ConstrainRC =
727 TRI->getMatchingSuperRegClass(A: NewRC, B: OpRC, Idx: New->getSubReg());
728
729 if (!ConstrainRC)
730 return false;
731 }
732
733 if (New->getReg().isVirtual() &&
734 !MRI->constrainRegClass(Reg: New->getReg(), RC: ConstrainRC)) {
735 LLVM_DEBUG(dbgs() << "Cannot constrain " << printReg(New->getReg(), TRI)
736 << TRI->getRegClassName(ConstrainRC) << '\n');
737 return false;
738 }
739 }
740
741 // Rework once the VS_16 register class is updated to include proper
742 // 16-bit SGPRs instead of 32-bit ones.
743 if (Old.getSubReg() == AMDGPU::lo16 && TRI->isSGPRReg(MRI: *MRI, Reg: New->getReg()))
744 Old.setSubReg(AMDGPU::NoSubRegister);
745 if (New->getReg().isPhysical()) {
746 Old.substPhysReg(Reg: New->getReg(), *TRI);
747 } else {
748 Register OldReg = Old.getReg();
749 Old.substVirtReg(Reg: New->getReg(), SubIdx: New->getSubReg(), *TRI);
750 Old.setIsUndef(New->isUndef());
751
752 // If MI is in a BUNDLE, also update header's matching implicit use.
753 if (MI->isBundledWithPred()) {
754 MachineInstr &Header = *getBundleStart(I: MI->getIterator());
755 for (MachineOperand &MO : Header.operands()) {
756 if (MO.getReg() == OldReg) {
757 MO.setReg(New->getReg());
758 MO.setSubReg(New->getSubReg());
759 }
760 }
761 }
762 }
763 return true;
764}
765
766static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
767 FoldCandidate &&Entry) {
768 // Skip additional folding on the same operand.
769 for (FoldCandidate &Fold : FoldList)
770 if (Fold.UseMI == Entry.UseMI && Fold.UseOpNo == Entry.UseOpNo)
771 return;
772 LLVM_DEBUG(dbgs() << "Append " << (Entry.Commuted ? "commuted" : "normal")
773 << " operand " << Entry.UseOpNo << "\n " << *Entry.UseMI);
774 FoldList.push_back(Elt: Entry);
775}
776
777static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList,
778 MachineInstr *MI, unsigned OpNo,
779 const FoldableDef &FoldOp,
780 bool Commuted = false, int ShrinkOp = -1) {
781 appendFoldCandidate(FoldList,
782 Entry: FoldCandidate(MI, OpNo, FoldOp, Commuted, ShrinkOp));
783}
784
785// Returns true if the instruction is a packed F32 instruction and the
786// corresponding scalar operand reads 32 bits and replicates the bits to both
787// channels.
788static bool isPKF32InstrReplicatesLower32BitsOfScalarOperand(
789 const GCNSubtarget *ST, MachineInstr *MI, unsigned OpNo) {
790 if (!ST->hasPKF32InstsReplicatingLower32BitsOfScalarInput())
791 return false;
792 const MCOperandInfo &OpDesc = MI->getDesc().operands()[OpNo];
793 return OpDesc.OperandType == AMDGPU::OPERAND_REG_IMM_V2FP32;
794}
795
796// Packed FP32 instructions only read 32 bits from a scalar operand (SGPR or
797// literal) and replicates the bits to both channels. Therefore, if the hi and
798// lo are not same, we can't fold it.
799static bool checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(
800 const FoldableDef &OpToFold) {
801 assert(OpToFold.isImm() && "Expected immediate operand");
802 uint64_t ImmVal = OpToFold.getEffectiveImmVal().value();
803 uint32_t Lo = Lo_32(Value: ImmVal);
804 uint32_t Hi = Hi_32(Value: ImmVal);
805 return Lo == Hi;
806}
807
808bool SIFoldOperandsImpl::tryAddToFoldList(
809 SmallVectorImpl<FoldCandidate> &FoldList, MachineInstr *MI, unsigned OpNo,
810 const FoldableDef &OpToFold) const {
811 const unsigned Opc = MI->getOpcode();
812
813 auto tryToFoldAsFMAAKorMK = [&]() {
814 if (!OpToFold.isImm())
815 return false;
816
817 const bool TryAK = OpNo == 3;
818 const unsigned NewOpc = TryAK ? AMDGPU::S_FMAAK_F32 : AMDGPU::S_FMAMK_F32;
819 MI->setDesc(TII->get(Opcode: NewOpc));
820
821 // We have to fold into operand which would be Imm not into OpNo.
822 bool FoldAsFMAAKorMK =
823 tryAddToFoldList(FoldList, MI, OpNo: TryAK ? 3 : 2, OpToFold);
824 if (FoldAsFMAAKorMK) {
825 // Untie Src2 of fmac.
826 MI->untieRegOperand(OpIdx: 3);
827 // For fmamk swap operands 1 and 2 if OpToFold was meant for operand 1.
828 if (OpNo == 1) {
829 MachineOperand &Op1 = MI->getOperand(i: 1);
830 MachineOperand &Op2 = MI->getOperand(i: 2);
831 Register OldReg = Op1.getReg();
832 // Operand 2 might be an inlinable constant
833 if (Op2.isImm()) {
834 Op1.ChangeToImmediate(ImmVal: Op2.getImm());
835 Op2.ChangeToRegister(Reg: OldReg, isDef: false);
836 } else {
837 Op1.setReg(Op2.getReg());
838 Op2.setReg(OldReg);
839 }
840 }
841 return true;
842 }
843 MI->setDesc(TII->get(Opcode: Opc));
844 return false;
845 };
846
847 bool IsLegal = OpToFold.isOperandLegal(TII: *TII, MI: *MI, OpIdx: OpNo);
848 if (!IsLegal && OpToFold.isImm()) {
849 if (std::optional<int64_t> ImmVal = OpToFold.getEffectiveImmVal())
850 IsLegal = canUseImmWithOpSel(MI, UseOpNo: OpNo, ImmVal: *ImmVal);
851 }
852
853 if (!IsLegal) {
854 // Special case for v_mac_{f16, f32}_e64 if we are trying to fold into src2
855 unsigned NewOpc = macToMad(Opc);
856 if (NewOpc != AMDGPU::INSTRUCTION_LIST_END) {
857 // Check if changing this to a v_mad_{f16, f32} instruction will allow us
858 // to fold the operand.
859 MI->setDesc(TII->get(Opcode: NewOpc));
860 bool AddOpSel = !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::op_sel) &&
861 AMDGPU::hasNamedOperand(Opcode: NewOpc, NamedIdx: AMDGPU::OpName::op_sel);
862 if (AddOpSel)
863 MI->addOperand(Op: MachineOperand::CreateImm(Val: 0));
864 bool FoldAsMAD = tryAddToFoldList(FoldList, MI, OpNo, OpToFold);
865 if (FoldAsMAD) {
866 MI->untieRegOperand(OpIdx: OpNo);
867 return true;
868 }
869 if (AddOpSel)
870 MI->removeOperand(OpNo: MI->getNumExplicitOperands() - 1);
871 MI->setDesc(TII->get(Opcode: Opc));
872 }
873
874 // Special case for s_fmac_f32 if we are trying to fold into Src2.
875 // By transforming into fmaak we can untie Src2 and make folding legal.
876 if (Opc == AMDGPU::S_FMAC_F32 && OpNo == 3) {
877 if (tryToFoldAsFMAAKorMK())
878 return true;
879 }
880
881 // Special case for s_setreg_b32
882 if (OpToFold.isImm()) {
883 unsigned ImmOpc = 0;
884 if (Opc == AMDGPU::S_SETREG_B32)
885 ImmOpc = AMDGPU::S_SETREG_IMM32_B32;
886 else if (Opc == AMDGPU::S_SETREG_B32_mode)
887 ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode;
888 if (ImmOpc) {
889 MI->setDesc(TII->get(Opcode: ImmOpc));
890 appendFoldCandidate(FoldList, MI, OpNo, FoldOp: OpToFold);
891 return true;
892 }
893 }
894
895 // Operand is not legal, so try to commute the instruction to
896 // see if this makes it possible to fold.
897 unsigned CommuteOpNo = TargetInstrInfo::CommuteAnyOperandIndex;
898 bool CanCommute = TII->findCommutedOpIndices(MI: *MI, SrcOpIdx0&: OpNo, SrcOpIdx1&: CommuteOpNo);
899 if (!CanCommute)
900 return false;
901
902 MachineOperand &Op = MI->getOperand(i: OpNo);
903 MachineOperand &CommutedOp = MI->getOperand(i: CommuteOpNo);
904
905 // One of operands might be an Imm operand, and OpNo may refer to it after
906 // the call of commuteInstruction() below. Such situations are avoided
907 // here explicitly as OpNo must be a register operand to be a candidate
908 // for memory folding.
909 if (!Op.isReg() || !CommutedOp.isReg())
910 return false;
911
912 // The same situation with an immediate could reproduce if both inputs are
913 // the same register.
914 if (Op.isReg() && CommutedOp.isReg() &&
915 (Op.getReg() == CommutedOp.getReg() &&
916 Op.getSubReg() == CommutedOp.getSubReg()))
917 return false;
918
919 if (!TII->commuteInstruction(MI&: *MI, NewMI: false, OpIdx1: OpNo, OpIdx2: CommuteOpNo))
920 return false;
921
922 int Op32 = -1;
923 if (!OpToFold.isOperandLegal(TII: *TII, MI: *MI, OpIdx: CommuteOpNo)) {
924 if ((Opc != AMDGPU::V_ADD_CO_U32_e64 && Opc != AMDGPU::V_SUB_CO_U32_e64 &&
925 Opc != AMDGPU::V_SUBREV_CO_U32_e64) || // FIXME
926 (!OpToFold.isImm() && !OpToFold.isFI() && !OpToFold.isGlobal())) {
927 TII->commuteInstruction(MI&: *MI, NewMI: false, OpIdx1: OpNo, OpIdx2: CommuteOpNo);
928 return false;
929 }
930
931 // Verify the other operand is a VGPR, otherwise we would violate the
932 // constant bus restriction.
933 MachineOperand &OtherOp = MI->getOperand(i: OpNo);
934 if (!OtherOp.isReg() ||
935 !TII->getRegisterInfo().isVGPR(MRI: *MRI, Reg: OtherOp.getReg()))
936 return false;
937
938 assert(MI->getOperand(1).isDef());
939
940 // Make sure to get the 32-bit version of the commuted opcode.
941 unsigned MaybeCommutedOpc = MI->getOpcode();
942 Op32 = AMDGPU::getVOPe32(Opcode: MaybeCommutedOpc);
943 }
944
945 appendFoldCandidate(FoldList, MI, OpNo: CommuteOpNo, FoldOp: OpToFold, /*Commuted=*/true,
946 ShrinkOp: Op32);
947 return true;
948 }
949
950 // Special case for s_fmac_f32 if we are trying to fold into Src0 or Src1.
951 // By changing into fmamk we can untie Src2.
952 // If folding for Src0 happens first and it is identical operand to Src1 we
953 // should avoid transforming into fmamk which requires commuting as it would
954 // cause folding into Src1 to fail later on due to wrong OpNo used.
955 if (Opc == AMDGPU::S_FMAC_F32 &&
956 (OpNo != 1 || !MI->getOperand(i: 1).isIdenticalTo(Other: MI->getOperand(i: 2)))) {
957 if (tryToFoldAsFMAAKorMK())
958 return true;
959 }
960
961 // Special case for PK_F32 instructions if we are trying to fold an imm to
962 // src0 or src1.
963 if (OpToFold.isImm() &&
964 isPKF32InstrReplicatesLower32BitsOfScalarOperand(ST, MI, OpNo) &&
965 !checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(OpToFold))
966 return false;
967
968 appendFoldCandidate(FoldList, MI, OpNo, FoldOp: OpToFold);
969 return true;
970}
971
972bool SIFoldOperandsImpl::isUseSafeToFold(const MachineInstr &MI,
973 const MachineOperand &UseMO) const {
974 // Operands of SDWA instructions must be registers.
975 return !TII->isSDWA(MI);
976}
977
978static MachineOperand *lookUpCopyChain(const SIInstrInfo &TII,
979 const MachineRegisterInfo &MRI,
980 Register SrcReg) {
981 MachineOperand *Sub = nullptr;
982 for (MachineInstr *SubDef = MRI.getVRegDef(Reg: SrcReg);
983 SubDef && TII.isFoldableCopy(MI: *SubDef);
984 SubDef = MRI.getVRegDef(Reg: Sub->getReg())) {
985 unsigned SrcIdx = TII.getFoldableCopySrcIdx(MI: *SubDef);
986 MachineOperand &SrcOp = SubDef->getOperand(i: SrcIdx);
987
988 if (SrcOp.isImm())
989 return &SrcOp;
990 if (!SrcOp.isReg() || SrcOp.getReg().isPhysical())
991 break;
992 Sub = &SrcOp;
993 // TODO: Support compose
994 if (SrcOp.getSubReg())
995 break;
996 }
997
998 return Sub;
999}
1000
1001const TargetRegisterClass *SIFoldOperandsImpl::getRegSeqInit(
1002 MachineInstr &RegSeq,
1003 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs) const {
1004
1005 assert(RegSeq.isRegSequence());
1006
1007 const TargetRegisterClass *RC = nullptr;
1008
1009 for (unsigned I = 1, E = RegSeq.getNumExplicitOperands(); I != E; I += 2) {
1010 MachineOperand &SrcOp = RegSeq.getOperand(i: I);
1011 if (SrcOp.getReg().isPhysical())
1012 return nullptr;
1013 unsigned SubRegIdx = RegSeq.getOperand(i: I + 1).getImm();
1014
1015 // Only accept reg_sequence with uniform reg class inputs for simplicity.
1016 const TargetRegisterClass *OpRC = getRegOpRC(MRI: *MRI, TRI: *TRI, MO: SrcOp);
1017 if (!RC)
1018 RC = OpRC;
1019 else if (!TRI->getCommonSubClass(A: RC, B: OpRC))
1020 return nullptr;
1021
1022 if (SrcOp.getSubReg()) {
1023 // TODO: Handle subregister compose
1024 Defs.emplace_back(Args: &SrcOp, Args&: SubRegIdx);
1025 continue;
1026 }
1027
1028 MachineOperand *DefSrc = lookUpCopyChain(TII: *TII, MRI: *MRI, SrcReg: SrcOp.getReg());
1029 if (DefSrc && (DefSrc->isReg() || DefSrc->isImm())) {
1030 Defs.emplace_back(Args&: DefSrc, Args&: SubRegIdx);
1031 continue;
1032 }
1033
1034 Defs.emplace_back(Args: &SrcOp, Args&: SubRegIdx);
1035 }
1036
1037 return RC;
1038}
1039
1040// Find a def of the UseReg, check if it is a reg_sequence and find initializers
1041// for each subreg, tracking it to an immediate if possible. Returns the
1042// register class of the inputs on success.
1043const TargetRegisterClass *SIFoldOperandsImpl::getRegSeqInit(
1044 SmallVectorImpl<std::pair<MachineOperand *, unsigned>> &Defs,
1045 Register UseReg) const {
1046 MachineInstr *Def = MRI->getVRegDef(Reg: UseReg);
1047 if (!Def || !Def->isRegSequence())
1048 return nullptr;
1049
1050 return getRegSeqInit(RegSeq&: *Def, Defs);
1051}
1052
1053std::pair<int64_t, const TargetRegisterClass *>
1054SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
1055 SmallVector<std::pair<MachineOperand *, unsigned>, 32> Defs;
1056 const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
1057 if (!SrcRC)
1058 return {};
1059
1060 bool TryToMatchSplat64 = false;
1061
1062 int64_t Imm;
1063 for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
1064 const MachineOperand *Op = Defs[I].first;
1065 if (!Op->isImm())
1066 return {};
1067
1068 int64_t SubImm = Op->getImm();
1069 if (!I) {
1070 Imm = SubImm;
1071 continue;
1072 }
1073
1074 if (Imm != SubImm) {
1075 if (I == 1 && (E & 1) == 0) {
1076 // If we have an even number of inputs, there's a chance this is a
1077 // 64-bit element splat broken into 32-bit pieces.
1078 TryToMatchSplat64 = true;
1079 break;
1080 }
1081
1082 return {}; // Can only fold splat constants
1083 }
1084 }
1085
1086 if (!TryToMatchSplat64)
1087 return {Defs[0].first->getImm(), SrcRC};
1088
1089 // Fallback to recognizing 64-bit splats broken into 32-bit pieces
1090 // (i.e. recognize every other other element is 0 for 64-bit immediates)
1091 int64_t SplatVal64;
1092 for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
1093 const MachineOperand *Op0 = Defs[I].first;
1094 const MachineOperand *Op1 = Defs[I + 1].first;
1095
1096 if (!Op0->isImm() || !Op1->isImm())
1097 return {};
1098
1099 unsigned SubReg0 = Defs[I].second;
1100 unsigned SubReg1 = Defs[I + 1].second;
1101
1102 // Assume we're going to generally encounter reg_sequences with sorted
1103 // subreg indexes, so reject any that aren't consecutive.
1104 if (TRI->getChannelFromSubReg(SubReg: SubReg0) + 1 !=
1105 TRI->getChannelFromSubReg(SubReg: SubReg1))
1106 return {};
1107
1108 if (TRI->getSubRegIdxSize(Idx: SubReg0) != 32)
1109 return {};
1110
1111 int64_t MergedVal = Make_64(High: Op1->getImm(), Low: Op0->getImm());
1112 if (I == 0)
1113 SplatVal64 = MergedVal;
1114 else if (SplatVal64 != MergedVal)
1115 return {};
1116 }
1117
1118 const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
1119 MRI->getRegClass(Reg: RegSeq.getOperand(i: 0).getReg()), AMDGPU::sub0_sub1);
1120
1121 return {SplatVal64, RC64};
1122}
1123
1124bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
1125 MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
1126 const TargetRegisterClass *SplatRC) const {
1127 const MCInstrDesc &Desc = UseMI->getDesc();
1128 if (UseOpIdx >= Desc.getNumOperands())
1129 return false;
1130
1131 // Filter out unhandled pseudos.
1132 if (!AMDGPU::isSISrcOperand(Desc, OpNo: UseOpIdx))
1133 return false;
1134
1135 int16_t RCID = TII->getOpRegClassID(OpInfo: Desc.operands()[UseOpIdx]);
1136 if (RCID == -1)
1137 return false;
1138
1139 const TargetRegisterClass *OpRC = TRI->getRegClass(i: RCID);
1140
1141 // Special case 0/-1, since when interpreted as a 64-bit element both halves
1142 // have the same bits. These are the only cases where a splat has the same
1143 // interpretation for 32-bit and 64-bit splats.
1144 if (SplatVal != 0 && SplatVal != -1) {
1145 // We need to figure out the scalar type read by the operand. e.g. the MFMA
1146 // operand will be AReg_128, and we want to check if it's compatible with an
1147 // AReg_32 constant.
1148 uint8_t OpTy = Desc.operands()[UseOpIdx].OperandType;
1149 switch (OpTy) {
1150 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1151 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1152 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1153 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1154 OpRC = TRI->getSubRegisterClass(OpRC, AMDGPU::sub0);
1155 break;
1156 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
1157 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1158 case AMDGPU::OPERAND_REG_IMM_V2FP64:
1159 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1160 case AMDGPU::OPERAND_REG_IMM_V2INT64:
1161 OpRC = TRI->getSubRegisterClass(OpRC, AMDGPU::sub0_sub1);
1162 break;
1163 default:
1164 return false;
1165 }
1166
1167 if (!TRI->getCommonSubClass(A: OpRC, B: SplatRC))
1168 return false;
1169 }
1170
1171 MachineOperand TmpOp = MachineOperand::CreateImm(Val: SplatVal);
1172 if (!TII->isOperandLegal(MI: *UseMI, OpIdx: UseOpIdx, MO: &TmpOp))
1173 return false;
1174
1175 return true;
1176}
1177
1178bool SIFoldOperandsImpl::tryToFoldACImm(
1179 const FoldableDef &OpToFold, MachineInstr *UseMI, unsigned UseOpIdx,
1180 SmallVectorImpl<FoldCandidate> &FoldList) const {
1181 const MCInstrDesc &Desc = UseMI->getDesc();
1182 if (UseOpIdx >= Desc.getNumOperands())
1183 return false;
1184
1185 // Filter out unhandled pseudos.
1186 if (!AMDGPU::isSISrcOperand(Desc, OpNo: UseOpIdx))
1187 return false;
1188
1189 if (OpToFold.isImm() && OpToFold.isOperandLegal(TII: *TII, MI: *UseMI, OpIdx: UseOpIdx)) {
1190 if (isPKF32InstrReplicatesLower32BitsOfScalarOperand(ST, MI: UseMI, OpNo: UseOpIdx) &&
1191 !checkImmOpForPKF32InstrReplicatesLower32BitsOfScalarOperand(OpToFold))
1192 return false;
1193 appendFoldCandidate(FoldList, MI: UseMI, OpNo: UseOpIdx, FoldOp: OpToFold);
1194 return true;
1195 }
1196
1197 return false;
1198}
1199
1200bool SIFoldOperandsImpl::foldOperand(
1201 FoldableDef OpToFold, MachineInstr *UseMI, int UseOpIdx,
1202 SmallVectorImpl<FoldCandidate> &FoldList,
1203 SmallVectorImpl<MachineInstr *> &CopiesToReplace) const {
1204 bool Changed = false;
1205 const MachineOperand *UseOp = &UseMI->getOperand(i: UseOpIdx);
1206
1207 if (!isUseSafeToFold(MI: *UseMI, UseMO: *UseOp))
1208 return Changed;
1209
1210 // FIXME: Fold operands with subregs.
1211 if (UseOp->isReg() && OpToFold.isReg()) {
1212 if (UseOp->isImplicit())
1213 return Changed;
1214 // Allow folding from SGPRs to 16-bit VGPRs.
1215 if (UseOp->getSubReg() != AMDGPU::NoSubRegister &&
1216 (UseOp->getSubReg() != AMDGPU::lo16 ||
1217 !TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg())))
1218 return Changed;
1219 }
1220
1221 // Special case for REG_SEQUENCE: We can't fold literals into
1222 // REG_SEQUENCE instructions, so we have to fold them into the
1223 // uses of REG_SEQUENCE.
1224 if (UseMI->isRegSequence()) {
1225 Register RegSeqDstReg = UseMI->getOperand(i: 0).getReg();
1226 unsigned RegSeqDstSubReg = UseMI->getOperand(i: UseOpIdx + 1).getImm();
1227
1228 int64_t SplatVal;
1229 const TargetRegisterClass *SplatRC;
1230 std::tie(args&: SplatVal, args&: SplatRC) = isRegSeqSplat(RegSeq&: *UseMI);
1231
1232 // Grab the use operands first
1233 SmallVector<MachineOperand *, 4> UsesToProcess(
1234 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: RegSeqDstReg)));
1235 for (unsigned I = 0; I != UsesToProcess.size(); ++I) {
1236 MachineOperand *RSUse = UsesToProcess[I];
1237 MachineInstr *RSUseMI = RSUse->getParent();
1238 unsigned OpNo = RSUseMI->getOperandNo(I: RSUse);
1239
1240 if (SplatRC) {
1241 if (RSUseMI->isCopy()) {
1242 Register DstReg = RSUseMI->getOperand(i: 0).getReg();
1243 append_range(C&: UsesToProcess,
1244 R: make_pointer_range(Range: MRI->use_nodbg_operands(Reg: DstReg)));
1245 continue;
1246 }
1247 if (tryFoldRegSeqSplat(UseMI: RSUseMI, UseOpIdx: OpNo, SplatVal, SplatRC)) {
1248 FoldableDef SplatDef(SplatVal, SplatRC);
1249 appendFoldCandidate(FoldList, MI: RSUseMI, OpNo, FoldOp: SplatDef);
1250 Changed = true;
1251 continue;
1252 }
1253 }
1254
1255 // TODO: Handle general compose
1256 if (RSUse->getSubReg() != RegSeqDstSubReg)
1257 continue;
1258
1259 // FIXME: We should avoid recursing here. There should be a cleaner split
1260 // between the in-place mutations and adding to the fold list.
1261 Changed |= foldOperand(OpToFold, UseMI: RSUseMI, UseOpIdx: RSUseMI->getOperandNo(I: RSUse),
1262 FoldList, CopiesToReplace);
1263 }
1264
1265 return Changed;
1266 }
1267
1268 if (tryToFoldACImm(OpToFold, UseMI, UseOpIdx, FoldList))
1269 return true;
1270
1271 if (frameIndexMayFold(UseMI: *UseMI, OpNo: UseOpIdx, OpToFold)) {
1272 // Verify that this is a stack access.
1273 // FIXME: Should probably use stack pseudos before frame lowering.
1274
1275 if (TII->isMUBUF(MI: *UseMI)) {
1276 if (TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::srsrc)->getReg() !=
1277 MFI->getScratchRSrcReg())
1278 return Changed;
1279
1280 // Ensure this is either relative to the current frame or the current
1281 // wave.
1282 MachineOperand &SOff =
1283 *TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::soffset);
1284 if (!SOff.isImm() || SOff.getImm() != 0)
1285 return Changed;
1286 }
1287
1288 const unsigned Opc = UseMI->getOpcode();
1289 if (TII->isFLATScratch(MI: *UseMI) &&
1290 AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vaddr) &&
1291 !AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::saddr)) {
1292 unsigned NewOpc = AMDGPU::getFlatScratchInstSSfromSV(Opcode: Opc);
1293 unsigned CPol =
1294 TII->getNamedOperand(MI&: *UseMI, OperandName: AMDGPU::OpName::cpol)->getImm();
1295 if ((CPol & AMDGPU::CPol::SCAL) &&
1296 !AMDGPU::supportsScaleOffset(MII: *TII, Opcode: NewOpc))
1297 return Changed;
1298
1299 UseMI->setDesc(TII->get(Opcode: NewOpc));
1300 }
1301
1302 // A frame index will resolve to a positive constant, so it should always be
1303 // safe to fold the addressing mode, even pre-GFX9.
1304 UseMI->getOperand(i: UseOpIdx).ChangeToFrameIndex(Idx: OpToFold.getFI());
1305
1306 return true;
1307 }
1308
1309 bool FoldingImmLike =
1310 OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
1311
1312 if (FoldingImmLike && UseMI->isCopy()) {
1313 Register DestReg = UseMI->getOperand(i: 0).getReg();
1314 Register SrcReg = UseMI->getOperand(i: 1).getReg();
1315 unsigned UseSubReg = UseMI->getOperand(i: 1).getSubReg();
1316 assert(SrcReg.isVirtual());
1317
1318 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: SrcReg);
1319
1320 // Don't fold into a copy to a physical register with the same class. Doing
1321 // so would interfere with the register coalescer's logic which would avoid
1322 // redundant initializations.
1323 if (DestReg.isPhysical() && SrcRC->contains(Reg: DestReg))
1324 return Changed;
1325
1326 const TargetRegisterClass *DestRC = TRI->getRegClassForReg(MRI: *MRI, Reg: DestReg);
1327 // In order to fold immediates into copies, we need to change the copy to a
1328 // MOV. Find a compatible mov instruction with the value.
1329 for (unsigned MovOp :
1330 {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
1331 AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_MOV_B16_t16_e64,
1332 AMDGPU::V_ACCVGPR_WRITE_B32_e64, AMDGPU::AV_MOV_B32_IMM_PSEUDO,
1333 AMDGPU::AV_MOV_B64_IMM_PSEUDO}) {
1334 const MCInstrDesc &MovDesc = TII->get(Opcode: MovOp);
1335 const TargetRegisterClass *MovDstRC =
1336 TRI->getRegClass(i: TII->getOpRegClassID(OpInfo: MovDesc.operands()[0]));
1337
1338 // Fold if the destination register class of the MOV instruction (ResRC)
1339 // is a superclass of (or equal to) the destination register class of the
1340 // COPY (DestRC). If this condition fails, folding would be illegal.
1341 if (!DestRC->hasSuperClassEq(RC: MovDstRC))
1342 continue;
1343
1344 const int SrcIdx = MovOp == AMDGPU::V_MOV_B16_t16_e64 ? 2 : 1;
1345
1346 int16_t RegClassID = TII->getOpRegClassID(OpInfo: MovDesc.operands()[SrcIdx]);
1347 if (RegClassID != -1) {
1348 const TargetRegisterClass *MovSrcRC = TRI->getRegClass(i: RegClassID);
1349
1350 if (UseSubReg)
1351 MovSrcRC = TRI->getMatchingSuperRegClass(A: SrcRC, B: MovSrcRC, Idx: UseSubReg);
1352
1353 // FIXME: We should be able to directly check immediate operand legality
1354 // for all cases, but gfx908 hacks break.
1355 if (MovOp == AMDGPU::AV_MOV_B32_IMM_PSEUDO &&
1356 (!OpToFold.isImm() ||
1357 !TII->isImmOperandLegal(InstDesc: MovDesc, OpNo: SrcIdx,
1358 ImmVal: *OpToFold.getEffectiveImmVal())))
1359 break;
1360
1361 if (!MRI->constrainRegClass(Reg: SrcReg, RC: MovSrcRC))
1362 break;
1363
1364 // FIXME: This is mutating the instruction only and deferring the actual
1365 // fold of the immediate
1366 } else {
1367 // For the _IMM_PSEUDO cases, there can be value restrictions on the
1368 // immediate to verify. Technically we should always verify this, but it
1369 // only matters for these concrete cases.
1370 // TODO: Handle non-imm case if it's useful.
1371 if (!OpToFold.isImm() ||
1372 !TII->isImmOperandLegal(InstDesc: MovDesc, OpNo: 1, ImmVal: *OpToFold.getEffectiveImmVal()))
1373 break;
1374 }
1375
1376 MachineInstr::mop_iterator ImpOpI = UseMI->implicit_operands().begin();
1377 MachineInstr::mop_iterator ImpOpE = UseMI->implicit_operands().end();
1378 while (ImpOpI != ImpOpE) {
1379 MachineInstr::mop_iterator Tmp = ImpOpI;
1380 ImpOpI++;
1381 UseMI->removeOperand(OpNo: UseMI->getOperandNo(I: Tmp));
1382 }
1383 UseMI->setDesc(MovDesc);
1384
1385 if (MovOp == AMDGPU::V_MOV_B16_t16_e64) {
1386 const auto &SrcOp = UseMI->getOperand(i: UseOpIdx);
1387 MachineOperand NewSrcOp(SrcOp);
1388 UseMI->removeOperand(OpNo: 1);
1389 UseMI->addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0)); // src0_modifiers
1390 UseMI->addOperand(Op: NewSrcOp); // src0
1391 UseMI->addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0)); // op_sel
1392 UseOpIdx = SrcIdx;
1393 UseOp = &UseMI->getOperand(i: UseOpIdx);
1394 }
1395 CopiesToReplace.push_back(Elt: UseMI);
1396 Changed = true;
1397 break;
1398 }
1399
1400 // We failed to replace the copy, so give up.
1401 if (UseMI->getOpcode() == AMDGPU::COPY)
1402 return Changed;
1403
1404 } else {
1405 if (UseMI->isCopy() && OpToFold.isReg() &&
1406 UseMI->getOperand(i: 0).getReg().isVirtual() &&
1407 !UseMI->getOperand(i: 1).getSubReg() &&
1408 OpToFold.DefMI->implicit_operands().empty()) {
1409 LLVM_DEBUG(dbgs() << "Folding " << *OpToFold.OpToFold << "\n into "
1410 << *UseMI);
1411 unsigned Size = TII->getOpSize(MI: *UseMI, OpNo: 1);
1412 Register UseReg = OpToFold.getReg();
1413 UseMI->getOperand(i: 1).setReg(UseReg);
1414 unsigned SubRegIdx = OpToFold.getSubReg();
1415 // Hack to allow 32-bit SGPRs to be folded into True16 instructions
1416 // Remove this if 16-bit SGPRs (i.e. SGPR_LO16) are added to the
1417 // VS_16RegClass
1418 //
1419 // Excerpt from AMDGPUGenRegisterInfoEnums.inc
1420 // NoSubRegister, //0
1421 // hi16, // 1
1422 // lo16, // 2
1423 // sub0, // 3
1424 // ...
1425 // sub1, // 11
1426 // sub1_hi16, // 12
1427 // sub1_lo16, // 13
1428 static_assert(AMDGPU::sub1_hi16 == 12, "Subregister layout has changed");
1429 if (Size == 2 && TRI->isVGPR(MRI: *MRI, Reg: UseMI->getOperand(i: 0).getReg()) &&
1430 TRI->isSGPRReg(MRI: *MRI, Reg: UseReg)) {
1431 // Produce the 32 bit subregister index to which the 16-bit subregister
1432 // is aligned.
1433 if (SubRegIdx > AMDGPU::sub1) {
1434 LaneBitmask M = TRI->getSubRegIndexLaneMask(SubIdx: SubRegIdx);
1435 M |= M.getLane(Lane: M.getHighestLane() - 1);
1436 SmallVector<unsigned, 4> Indexes;
1437 TRI->getCoveringSubRegIndexes(RC: TRI->getRegClassForReg(MRI: *MRI, Reg: UseReg), LaneMask: M,
1438 Indexes);
1439 assert(Indexes.size() == 1 && "Expected one 32-bit subreg to cover");
1440 SubRegIdx = Indexes[0];
1441 // 32-bit registers do not have a sub0 index
1442 } else if (TII->getOpSize(MI: *UseMI, OpNo: 1) == 4)
1443 SubRegIdx = 0;
1444 else
1445 SubRegIdx = AMDGPU::sub0;
1446 }
1447 UseMI->getOperand(i: 1).setSubReg(SubRegIdx);
1448 UseMI->getOperand(i: 1).setIsKill(false);
1449 CopiesToReplace.push_back(Elt: UseMI);
1450 OpToFold.OpToFold->setIsKill(false);
1451 Changed = true;
1452
1453 // Remove kill flags as kills may now be out of order with uses.
1454 MRI->clearKillFlags(Reg: UseReg);
1455 if (foldCopyToAGPRRegSequence(CopyMI: UseMI))
1456 return true;
1457 }
1458
1459 unsigned UseOpc = UseMI->getOpcode();
1460 if (UseOpc == AMDGPU::V_READFIRSTLANE_B32 ||
1461 (UseOpc == AMDGPU::V_READLANE_B32 &&
1462 (int)UseOpIdx ==
1463 AMDGPU::getNamedOperandIdx(Opcode: UseOpc, Name: AMDGPU::OpName::src0))) {
1464 // %vgpr = V_MOV_B32 imm
1465 // %sgpr = V_READFIRSTLANE_B32 %vgpr
1466 // =>
1467 // %sgpr = S_MOV_B32 imm
1468 if (FoldingImmLike) {
1469 if (execMayBeModifiedBeforeUse(MRI: *MRI,
1470 VReg: UseMI->getOperand(i: UseOpIdx).getReg(),
1471 DefMI: *OpToFold.DefMI, UseMI: *UseMI))
1472 return Changed;
1473
1474 UseMI->setDesc(TII->get(Opcode: AMDGPU::S_MOV_B32));
1475 UseMI->clearFlag(Flag: MachineInstr::NoConvergent);
1476
1477 if (OpToFold.isImm()) {
1478 UseMI->getOperand(i: 1).ChangeToImmediate(
1479 ImmVal: *OpToFold.getEffectiveImmVal());
1480 } else if (OpToFold.isFI())
1481 UseMI->getOperand(i: 1).ChangeToFrameIndex(Idx: OpToFold.getFI());
1482 else {
1483 assert(OpToFold.isGlobal());
1484 UseMI->getOperand(i: 1).ChangeToGA(GV: OpToFold.OpToFold->getGlobal(),
1485 Offset: OpToFold.OpToFold->getOffset(),
1486 TargetFlags: OpToFold.OpToFold->getTargetFlags());
1487 }
1488 UseMI->removeOperand(OpNo: 2); // Remove exec read (or src1 for readlane)
1489 return true;
1490 }
1491
1492 if (OpToFold.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: OpToFold.getReg())) {
1493 if (execMayBeModifiedBeforeUse(MRI: *MRI,
1494 VReg: UseMI->getOperand(i: UseOpIdx).getReg(),
1495 DefMI: *OpToFold.DefMI, UseMI: *UseMI))
1496 return Changed;
1497
1498 // %vgpr = COPY %sgpr0
1499 // %sgpr1 = V_READFIRSTLANE_B32 %vgpr
1500 // =>
1501 // %sgpr1 = COPY %sgpr0
1502 UseMI->setDesc(TII->get(Opcode: AMDGPU::COPY));
1503 UseMI->getOperand(i: 1).setReg(OpToFold.getReg());
1504 UseMI->getOperand(i: 1).setSubReg(OpToFold.getSubReg());
1505 UseMI->getOperand(i: 1).setIsKill(false);
1506 UseMI->removeOperand(OpNo: 2); // Remove exec read (or src1 for readlane)
1507 UseMI->clearFlag(Flag: MachineInstr::NoConvergent);
1508 return true;
1509 }
1510 }
1511
1512 const MCInstrDesc &UseDesc = UseMI->getDesc();
1513
1514 // Don't fold into target independent nodes. Target independent opcodes
1515 // don't have defined register classes.
1516 if (UseDesc.isVariadic() || UseOp->isImplicit() ||
1517 UseDesc.operands()[UseOpIdx].RegClass == -1)
1518 return Changed;
1519 }
1520
1521 // FIXME: We could try to change the instruction from 64-bit to 32-bit
1522 // to enable more folding opportunities. The shrink operands pass
1523 // already does this.
1524
1525 Changed |= tryAddToFoldList(FoldList, MI: UseMI, OpNo: UseOpIdx, OpToFold);
1526 return Changed;
1527}
1528
1529static bool evalBinaryInstruction(unsigned Opcode, int32_t &Result,
1530 uint32_t LHS, uint32_t RHS) {
1531 switch (Opcode) {
1532 case AMDGPU::S_ADD_I32:
1533 case AMDGPU::S_ADD_U32:
1534 Result = LHS + RHS;
1535 return true;
1536 case AMDGPU::S_SUB_I32:
1537 case AMDGPU::S_SUB_U32:
1538 Result = LHS - RHS;
1539 return true;
1540 case AMDGPU::V_AND_B32_e64:
1541 case AMDGPU::V_AND_B32_e32:
1542 case AMDGPU::S_AND_B32:
1543 Result = LHS & RHS;
1544 return true;
1545 case AMDGPU::V_OR_B32_e64:
1546 case AMDGPU::V_OR_B32_e32:
1547 case AMDGPU::S_OR_B32:
1548 Result = LHS | RHS;
1549 return true;
1550 case AMDGPU::V_XOR_B32_e64:
1551 case AMDGPU::V_XOR_B32_e32:
1552 case AMDGPU::S_XOR_B32:
1553 Result = LHS ^ RHS;
1554 return true;
1555 case AMDGPU::S_XNOR_B32:
1556 Result = ~(LHS ^ RHS);
1557 return true;
1558 case AMDGPU::S_NAND_B32:
1559 Result = ~(LHS & RHS);
1560 return true;
1561 case AMDGPU::S_NOR_B32:
1562 Result = ~(LHS | RHS);
1563 return true;
1564 case AMDGPU::S_ANDN2_B32:
1565 Result = LHS & ~RHS;
1566 return true;
1567 case AMDGPU::S_ORN2_B32:
1568 Result = LHS | ~RHS;
1569 return true;
1570 case AMDGPU::V_LSHL_B32_e64:
1571 case AMDGPU::V_LSHL_B32_e32:
1572 case AMDGPU::S_LSHL_B32:
1573 // The instruction ignores the high bits for out of bounds shifts.
1574 Result = LHS << (RHS & 31);
1575 return true;
1576 case AMDGPU::V_LSHLREV_B32_e64:
1577 case AMDGPU::V_LSHLREV_B32_e32:
1578 Result = RHS << (LHS & 31);
1579 return true;
1580 case AMDGPU::V_LSHR_B32_e64:
1581 case AMDGPU::V_LSHR_B32_e32:
1582 case AMDGPU::S_LSHR_B32:
1583 Result = LHS >> (RHS & 31);
1584 return true;
1585 case AMDGPU::V_LSHRREV_B32_e64:
1586 case AMDGPU::V_LSHRREV_B32_e32:
1587 Result = RHS >> (LHS & 31);
1588 return true;
1589 case AMDGPU::V_ASHR_I32_e64:
1590 case AMDGPU::V_ASHR_I32_e32:
1591 case AMDGPU::S_ASHR_I32:
1592 Result = static_cast<int32_t>(LHS) >> (RHS & 31);
1593 return true;
1594 case AMDGPU::V_ASHRREV_I32_e64:
1595 case AMDGPU::V_ASHRREV_I32_e32:
1596 Result = static_cast<int32_t>(RHS) >> (LHS & 31);
1597 return true;
1598 default:
1599 return false;
1600 }
1601}
1602
1603static unsigned getMovOpc(bool IsScalar) {
1604 return IsScalar ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1605}
1606
1607// Try to simplify operations with a constant that may appear after instruction
1608// selection.
1609// TODO: See if a frame index with a fixed offset can fold.
1610bool SIFoldOperandsImpl::tryConstantFoldOp(MachineInstr *MI) const {
1611 if (!MI->allImplicitDefsAreDead())
1612 return false;
1613
1614 unsigned Opc = MI->getOpcode();
1615
1616 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
1617 if (Src0Idx == -1)
1618 return false;
1619
1620 MachineOperand *Src0 = &MI->getOperand(i: Src0Idx);
1621 std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(Op&: *Src0);
1622
1623 if ((Opc == AMDGPU::V_NOT_B32_e64 || Opc == AMDGPU::V_NOT_B32_e32 ||
1624 Opc == AMDGPU::S_NOT_B32) &&
1625 Src0Imm) {
1626 MI->getOperand(i: 1).ChangeToImmediate(ImmVal: ~*Src0Imm);
1627 TII->mutateAndCleanupImplicit(
1628 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_NOT_B32)));
1629 return true;
1630 }
1631
1632 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
1633 if (Src1Idx == -1)
1634 return false;
1635
1636 MachineOperand *Src1 = &MI->getOperand(i: Src1Idx);
1637 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(Op&: *Src1);
1638
1639 if (!Src0Imm && !Src1Imm)
1640 return false;
1641
1642 // and k0, k1 -> v_mov_b32 (k0 & k1)
1643 // or k0, k1 -> v_mov_b32 (k0 | k1)
1644 // xor k0, k1 -> v_mov_b32 (k0 ^ k1)
1645 if (Src0Imm && Src1Imm) {
1646 int32_t NewImm;
1647 if (!evalBinaryInstruction(Opcode: Opc, Result&: NewImm, LHS: *Src0Imm, RHS: *Src1Imm))
1648 return false;
1649
1650 bool IsSGPR = TRI->isSGPRReg(MRI: *MRI, Reg: MI->getOperand(i: 0).getReg());
1651
1652 // Be careful to change the right operand, src0 may belong to a different
1653 // instruction.
1654 MI->getOperand(i: Src0Idx).ChangeToImmediate(ImmVal: NewImm);
1655 MI->removeOperand(OpNo: Src1Idx);
1656 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: IsSGPR)));
1657 return true;
1658 }
1659
1660 // S_SUB_* is not commutable, so handle it before the commutability gate.
1661 // Only `x - 0 -> copy x` is valid; `0 - x` is a negation, not a copy.
1662 if (Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U32) {
1663 if (Src1Imm && static_cast<int32_t>(*Src1Imm) == 0) {
1664 // y = sub x, 0 => y = copy x
1665 MI->removeOperand(OpNo: Src1Idx);
1666 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1667 return true;
1668 }
1669 return false;
1670 }
1671
1672 if (!MI->isCommutable())
1673 return false;
1674
1675 if (Src0Imm && !Src1Imm) {
1676 std::swap(a&: Src0, b&: Src1);
1677 std::swap(a&: Src0Idx, b&: Src1Idx);
1678 std::swap(lhs&: Src0Imm, rhs&: Src1Imm);
1679 }
1680
1681 int32_t Src1Val = static_cast<int32_t>(*Src1Imm);
1682 if (Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_ADD_U32) {
1683 if (Src1Val == 0) {
1684 // y = add x, 0 => y = copy x
1685 MI->removeOperand(OpNo: Src1Idx);
1686 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1687 return true;
1688 }
1689 return false;
1690 }
1691
1692 if (Opc == AMDGPU::V_OR_B32_e64 ||
1693 Opc == AMDGPU::V_OR_B32_e32 ||
1694 Opc == AMDGPU::S_OR_B32) {
1695 if (Src1Val == 0) {
1696 // y = or x, 0 => y = copy x
1697 MI->removeOperand(OpNo: Src1Idx);
1698 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1699 } else if (Src1Val == -1) {
1700 // y = or x, -1 => y = v_mov_b32 -1
1701 MI->removeOperand(OpNo: Src0Idx);
1702 TII->mutateAndCleanupImplicit(
1703 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_OR_B32)));
1704 } else
1705 return false;
1706
1707 return true;
1708 }
1709
1710 if (Opc == AMDGPU::V_AND_B32_e64 || Opc == AMDGPU::V_AND_B32_e32 ||
1711 Opc == AMDGPU::S_AND_B32) {
1712 if (Src1Val == 0) {
1713 // y = and x, 0 => y = v_mov_b32 0
1714 MI->removeOperand(OpNo: Src0Idx);
1715 TII->mutateAndCleanupImplicit(
1716 MI&: *MI, NewDesc: TII->get(Opcode: getMovOpc(IsScalar: Opc == AMDGPU::S_AND_B32)));
1717 } else if (Src1Val == -1) {
1718 // y = and x, -1 => y = copy x
1719 MI->removeOperand(OpNo: Src1Idx);
1720 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1721 } else
1722 return false;
1723
1724 return true;
1725 }
1726
1727 if (Opc == AMDGPU::V_XOR_B32_e64 || Opc == AMDGPU::V_XOR_B32_e32 ||
1728 Opc == AMDGPU::S_XOR_B32) {
1729 if (Src1Val == 0) {
1730 // y = xor x, 0 => y = copy x
1731 MI->removeOperand(OpNo: Src1Idx);
1732 TII->mutateAndCleanupImplicit(MI&: *MI, NewDesc: TII->get(Opcode: AMDGPU::COPY));
1733 return true;
1734 }
1735 }
1736
1737 return false;
1738}
1739
1740// Try to fold an instruction into a simpler one
1741bool SIFoldOperandsImpl::tryFoldCndMask(MachineInstr &MI) const {
1742 unsigned Opc = MI.getOpcode();
1743 if (Opc != AMDGPU::V_CNDMASK_B32_e32 && Opc != AMDGPU::V_CNDMASK_B32_e64 &&
1744 Opc != AMDGPU::V_CNDMASK_B64_PSEUDO)
1745 return false;
1746
1747 MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
1748 MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
1749 if (!Src1->isIdenticalTo(Other: *Src0)) {
1750 std::optional<int64_t> Src1Imm = TII->getImmOrMaterializedImm(Op&: *Src1);
1751 if (!Src1Imm)
1752 return false;
1753
1754 std::optional<int64_t> Src0Imm = TII->getImmOrMaterializedImm(Op&: *Src0);
1755 if (!Src0Imm || *Src0Imm != *Src1Imm)
1756 return false;
1757 }
1758
1759 int Src1ModIdx =
1760 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1_modifiers);
1761 int Src0ModIdx =
1762 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers);
1763 if ((Src1ModIdx != -1 && MI.getOperand(i: Src1ModIdx).getImm() != 0) ||
1764 (Src0ModIdx != -1 && MI.getOperand(i: Src0ModIdx).getImm() != 0))
1765 return false;
1766
1767 LLVM_DEBUG(dbgs() << "Folded " << MI << " into ");
1768 auto &NewDesc =
1769 TII->get(Opcode: Src0->isReg() ? (unsigned)AMDGPU::COPY : getMovOpc(IsScalar: false));
1770 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
1771 if (Src2Idx != -1)
1772 MI.removeOperand(OpNo: Src2Idx);
1773 MI.removeOperand(OpNo: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1));
1774 if (Src1ModIdx != -1)
1775 MI.removeOperand(OpNo: Src1ModIdx);
1776 if (Src0ModIdx != -1)
1777 MI.removeOperand(OpNo: Src0ModIdx);
1778 TII->mutateAndCleanupImplicit(MI, NewDesc);
1779 LLVM_DEBUG(dbgs() << MI);
1780 return true;
1781}
1782
1783bool SIFoldOperandsImpl::tryFoldZeroHighBits(MachineInstr &MI) const {
1784 if (MI.getOpcode() != AMDGPU::V_AND_B32_e64 &&
1785 MI.getOpcode() != AMDGPU::V_AND_B32_e32)
1786 return false;
1787
1788 std::optional<int64_t> Src0Imm =
1789 TII->getImmOrMaterializedImm(Op&: MI.getOperand(i: 1));
1790 if (!Src0Imm || *Src0Imm != 0xffff || !MI.getOperand(i: 2).isReg())
1791 return false;
1792
1793 Register Src1 = MI.getOperand(i: 2).getReg();
1794 MachineInstr *SrcDef = MRI->getVRegDef(Reg: Src1);
1795 if (!ST->zeroesHigh16BitsOfDest(Opcode: SrcDef->getOpcode()))
1796 return false;
1797
1798 Register Dst = MI.getOperand(i: 0).getReg();
1799 MRI->replaceRegWith(FromReg: Dst, ToReg: Src1);
1800 if (!MI.getOperand(i: 2).isKill())
1801 MRI->clearKillFlags(Reg: Src1);
1802 MI.eraseFromParent();
1803 return true;
1804}
1805
1806bool SIFoldOperandsImpl::foldInstOperand(MachineInstr &MI,
1807 const FoldableDef &OpToFold) const {
1808 // We need mutate the operands of new mov instructions to add implicit
1809 // uses of EXEC, but adding them invalidates the use_iterator, so defer
1810 // this.
1811 SmallVector<MachineInstr *, 4> CopiesToReplace;
1812 SmallVector<FoldCandidate, 4> FoldList;
1813 MachineOperand &Dst = MI.getOperand(i: 0);
1814 bool Changed = false;
1815
1816 if (OpToFold.isImm()) {
1817 for (auto &UseMI :
1818 make_early_inc_range(Range: MRI->use_nodbg_instructions(Reg: Dst.getReg()))) {
1819 // Folding the immediate may reveal operations that can be constant
1820 // folded or replaced with a copy. This can happen for example after
1821 // frame indices are lowered to constants or from splitting 64-bit
1822 // constants.
1823 //
1824 // We may also encounter cases where one or both operands are
1825 // immediates materialized into a register, which would ordinarily not
1826 // be folded due to multiple uses or operand constraints.
1827 if (tryConstantFoldOp(MI: &UseMI)) {
1828 LLVM_DEBUG(dbgs() << "Constant folded " << UseMI);
1829 Changed = true;
1830 }
1831 }
1832 }
1833
1834 SmallVector<MachineOperand *, 4> UsesToProcess(
1835 llvm::make_pointer_range(Range: MRI->use_nodbg_operands(Reg: Dst.getReg())));
1836 for (auto *U : UsesToProcess) {
1837 MachineInstr *UseMI = U->getParent();
1838
1839 FoldableDef SubOpToFold = OpToFold.getWithSubReg(TRI: *TRI, SubReg: U->getSubReg());
1840 Changed |= foldOperand(OpToFold: SubOpToFold, UseMI, UseOpIdx: UseMI->getOperandNo(I: U), FoldList,
1841 CopiesToReplace);
1842 }
1843
1844 if (CopiesToReplace.empty() && FoldList.empty())
1845 return Changed;
1846
1847 // Make sure we add EXEC uses to any new v_mov instructions created.
1848 for (MachineInstr *Copy : CopiesToReplace)
1849 Copy->addImplicitDefUseOperands(MF&: *MF);
1850
1851 SetVector<MachineInstr *> ConstantFoldCandidates;
1852 for (FoldCandidate &Fold : FoldList) {
1853 assert(!Fold.isReg() || Fold.Def.OpToFold);
1854 if (Fold.isReg() && Fold.getReg().isVirtual()) {
1855 Register Reg = Fold.getReg();
1856 const MachineInstr *DefMI = Fold.Def.DefMI;
1857 if (DefMI->readsRegister(Reg: AMDGPU::EXEC, TRI) &&
1858 execMayBeModifiedBeforeUse(MRI: *MRI, VReg: Reg, DefMI: *DefMI, UseMI: *Fold.UseMI))
1859 continue;
1860 }
1861 if (updateOperand(Fold)) {
1862 // Clear kill flags.
1863 if (Fold.isReg()) {
1864 assert(Fold.Def.OpToFold && Fold.isReg());
1865 // FIXME: Probably shouldn't bother trying to fold if not an
1866 // SGPR. PeepholeOptimizer can eliminate redundant VGPR->VGPR
1867 // copies.
1868 MRI->clearKillFlags(Reg: Fold.getReg());
1869 }
1870 LLVM_DEBUG(dbgs() << "Folded source from " << MI << " into OpNo "
1871 << static_cast<int>(Fold.UseOpNo) << " of "
1872 << *Fold.UseMI);
1873
1874 if (Fold.isImm())
1875 ConstantFoldCandidates.insert(X: Fold.UseMI);
1876
1877 } else if (Fold.Commuted) {
1878 // Restoring instruction's original operand order if fold has failed.
1879 TII->commuteInstruction(MI&: *Fold.UseMI, NewMI: false);
1880 }
1881 }
1882
1883 for (MachineInstr *MI : ConstantFoldCandidates) {
1884 if (tryConstantFoldOp(MI)) {
1885 LLVM_DEBUG(dbgs() << "Constant folded " << *MI);
1886 Changed = true;
1887 }
1888 }
1889 return true;
1890}
1891
1892/// Fold %agpr = COPY (REG_SEQUENCE x_MOV_B32, ...) into REG_SEQUENCE
1893/// (V_ACCVGPR_WRITE_B32_e64) ... depending on the reg_sequence input values.
1894bool SIFoldOperandsImpl::foldCopyToAGPRRegSequence(MachineInstr *CopyMI) const {
1895 // It is very tricky to store a value into an AGPR. v_accvgpr_write_b32 can
1896 // only accept VGPR or inline immediate. Recreate a reg_sequence with its
1897 // initializers right here, so we will rematerialize immediates and avoid
1898 // copies via different reg classes.
1899 const TargetRegisterClass *DefRC =
1900 MRI->getRegClass(Reg: CopyMI->getOperand(i: 0).getReg());
1901 if (!TRI->isAGPRClass(RC: DefRC))
1902 return false;
1903
1904 Register UseReg = CopyMI->getOperand(i: 1).getReg();
1905 MachineInstr *RegSeq = MRI->getVRegDef(Reg: UseReg);
1906 if (!RegSeq || !RegSeq->isRegSequence())
1907 return false;
1908
1909 const DebugLoc &DL = CopyMI->getDebugLoc();
1910 MachineBasicBlock &MBB = *CopyMI->getParent();
1911
1912 MachineInstrBuilder B(*MBB.getParent(), CopyMI);
1913 DenseMap<TargetInstrInfo::RegSubRegPair, Register> VGPRCopies;
1914
1915 const TargetRegisterClass *UseRC =
1916 MRI->getRegClass(Reg: CopyMI->getOperand(i: 1).getReg());
1917
1918 // Value, subregindex for new REG_SEQUENCE
1919 SmallVector<std::pair<MachineOperand *, unsigned>, 32> NewDefs;
1920
1921 unsigned NumRegSeqOperands = RegSeq->getNumOperands();
1922 unsigned NumFoldable = 0;
1923
1924 for (unsigned I = 1; I != NumRegSeqOperands; I += 2) {
1925 MachineOperand &RegOp = RegSeq->getOperand(i: I);
1926 unsigned SubRegIdx = RegSeq->getOperand(i: I + 1).getImm();
1927
1928 if (RegOp.getSubReg()) {
1929 // TODO: Handle subregister compose
1930 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
1931 continue;
1932 }
1933
1934 MachineOperand *Lookup = lookUpCopyChain(TII: *TII, MRI: *MRI, SrcReg: RegOp.getReg());
1935 if (!Lookup)
1936 Lookup = &RegOp;
1937
1938 if (Lookup->isImm()) {
1939 // Check if this is an agpr_32 subregister.
1940 const TargetRegisterClass *DestSuperRC = TRI->getMatchingSuperRegClass(
1941 A: DefRC, B: &AMDGPU::AGPR_32RegClass, Idx: SubRegIdx);
1942 if (DestSuperRC &&
1943 TII->isInlineConstant(MO: *Lookup, OperandType: AMDGPU::OPERAND_REG_INLINE_C_INT32)) {
1944 ++NumFoldable;
1945 NewDefs.emplace_back(Args&: Lookup, Args&: SubRegIdx);
1946 continue;
1947 }
1948 }
1949
1950 const TargetRegisterClass *InputRC =
1951 Lookup->isReg() ? MRI->getRegClass(Reg: Lookup->getReg())
1952 : MRI->getRegClass(Reg: RegOp.getReg());
1953
1954 // TODO: Account for Lookup->getSubReg()
1955
1956 // If we can't find a matching super class, this is an SGPR->AGPR or
1957 // VGPR->AGPR subreg copy (or something constant-like we have to materialize
1958 // in the AGPR). We can't directly copy from SGPR to AGPR on gfx908, so we
1959 // want to rewrite to copy to an intermediate VGPR class.
1960 const TargetRegisterClass *MatchRC =
1961 TRI->getMatchingSuperRegClass(A: DefRC, B: InputRC, Idx: SubRegIdx);
1962 if (!MatchRC) {
1963 ++NumFoldable;
1964 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
1965 continue;
1966 }
1967
1968 NewDefs.emplace_back(Args: &RegOp, Args&: SubRegIdx);
1969 }
1970
1971 // Do not clone a reg_sequence and merely change the result register class.
1972 if (NumFoldable == 0)
1973 return false;
1974
1975 CopyMI->setDesc(TII->get(Opcode: AMDGPU::REG_SEQUENCE));
1976 for (unsigned I = CopyMI->getNumOperands() - 1; I > 0; --I)
1977 CopyMI->removeOperand(OpNo: I);
1978
1979 for (auto [Def, DestSubIdx] : NewDefs) {
1980 if (!Def->isReg()) {
1981 // TODO: Should we use single write for each repeated value like in
1982 // register case?
1983 Register Tmp = MRI->createVirtualRegister(RegClass: &AMDGPU::AGPR_32RegClass);
1984 BuildMI(BB&: MBB, I: CopyMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg: Tmp)
1985 .add(MO: *Def);
1986 B.addReg(RegNo: Tmp);
1987 } else {
1988 TargetInstrInfo::RegSubRegPair Src = getRegSubRegPair(O: *Def);
1989 Def->setIsKill(false);
1990
1991 Register &VGPRCopy = VGPRCopies[Src];
1992 if (!VGPRCopy) {
1993 const TargetRegisterClass *VGPRUseSubRC =
1994 TRI->getSubRegisterClass(UseRC, DestSubIdx);
1995
1996 // We cannot build a reg_sequence out of the same registers, they
1997 // must be copied. Better do it here before copyPhysReg() created
1998 // several reads to do the AGPR->VGPR->AGPR copy.
1999
2000 // Direct copy from SGPR to AGPR is not possible on gfx908. To avoid
2001 // creation of exploded copies SGPR->VGPR->AGPR in the copyPhysReg()
2002 // later, create a copy here and track if we already have such a copy.
2003 const TargetRegisterClass *SubRC =
2004 TRI->getSubRegisterClass(MRI->getRegClass(Reg: Src.Reg), Src.SubReg);
2005 if (!VGPRUseSubRC->hasSubClassEq(RC: SubRC)) {
2006 // TODO: Try to reconstrain class
2007 VGPRCopy = MRI->createVirtualRegister(RegClass: VGPRUseSubRC);
2008 BuildMI(BB&: MBB, I: CopyMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: VGPRCopy).add(MO: *Def);
2009 B.addReg(RegNo: VGPRCopy);
2010 } else {
2011 // If it is already a VGPR, do not copy the register.
2012 B.add(MO: *Def);
2013 }
2014 } else {
2015 B.addReg(RegNo: VGPRCopy);
2016 }
2017 }
2018
2019 B.addImm(Val: DestSubIdx);
2020 }
2021
2022 LLVM_DEBUG(dbgs() << "Folded " << *CopyMI);
2023 return true;
2024}
2025
2026bool SIFoldOperandsImpl::tryFoldFoldableCopy(
2027 MachineInstr &MI, MachineOperand *&CurrentKnownM0Val) const {
2028 Register DstReg = MI.getOperand(i: 0).getReg();
2029 // Specially track simple redefs of m0 to the same value in a block, so we
2030 // can erase the later ones.
2031 if (DstReg == AMDGPU::M0) {
2032 MachineOperand &NewM0Val = MI.getOperand(i: 1);
2033 if (CurrentKnownM0Val && CurrentKnownM0Val->isIdenticalTo(Other: NewM0Val)) {
2034 MI.eraseFromParent();
2035 return true;
2036 }
2037
2038 // We aren't tracking other physical registers
2039 CurrentKnownM0Val = (NewM0Val.isReg() && NewM0Val.getReg().isPhysical())
2040 ? nullptr
2041 : &NewM0Val;
2042 return false;
2043 }
2044
2045 MachineOperand *OpToFoldPtr;
2046 if (MI.getOpcode() == AMDGPU::V_MOV_B16_t16_e64) {
2047 // Folding when any src_modifiers are non-zero is unsupported
2048 if (TII->hasAnyModifiersSet(MI))
2049 return false;
2050 OpToFoldPtr = &MI.getOperand(i: 2);
2051 } else
2052 OpToFoldPtr = &MI.getOperand(i: 1);
2053 MachineOperand &OpToFold = *OpToFoldPtr;
2054 bool FoldingImm = OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal();
2055
2056 // FIXME: We could also be folding things like TargetIndexes.
2057 if (!FoldingImm && !OpToFold.isReg())
2058 return false;
2059
2060 // Fold virtual registers and constant physical registers.
2061 if (OpToFold.isReg() && OpToFold.getReg().isPhysical() &&
2062 !TRI->isConstantPhysReg(PhysReg: OpToFold.getReg()))
2063 return false;
2064
2065 // Prevent folding operands backwards in the function. For example,
2066 // the COPY opcode must not be replaced by 1 in this example:
2067 //
2068 // %3 = COPY %vgpr0; VGPR_32:%3
2069 // ...
2070 // %vgpr0 = V_MOV_B32_e32 1, implicit %exec
2071 if (!DstReg.isVirtual())
2072 return false;
2073
2074 const TargetRegisterClass *DstRC =
2075 MRI->getRegClass(Reg: MI.getOperand(i: 0).getReg());
2076
2077 // True16: Fix malformed 16-bit sgpr COPY produced by peephole-opt
2078 // Can remove this code if proper 16-bit SGPRs are implemented
2079 // Example: Pre-peephole-opt
2080 // %29:sgpr_lo16 = COPY %16.lo16:sreg_32
2081 // %32:sreg_32 = COPY %29:sgpr_lo16
2082 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2083 // Post-peephole-opt and DCE
2084 // %32:sreg_32 = COPY %16.lo16:sreg_32
2085 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2086 // After this transform
2087 // %32:sreg_32 = COPY %16:sreg_32
2088 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %32:sreg_32
2089 // After the fold operands pass
2090 // %30:sreg_32 = S_PACK_LL_B32_B16 killed %31:sreg_32, killed %16:sreg_32
2091 if (MI.getOpcode() == AMDGPU::COPY && OpToFold.isReg() &&
2092 OpToFold.getSubReg()) {
2093 if (DstRC == &AMDGPU::SReg_32RegClass &&
2094 DstRC == MRI->getRegClass(Reg: OpToFold.getReg())) {
2095 assert(OpToFold.getSubReg() == AMDGPU::lo16);
2096 OpToFold.setSubReg(0);
2097 }
2098 }
2099
2100 // Fold copy to AGPR through reg_sequence
2101 // TODO: Handle with subregister extract
2102 if (OpToFold.isReg() && MI.isCopy() && !MI.getOperand(i: 1).getSubReg()) {
2103 if (foldCopyToAGPRRegSequence(CopyMI: &MI))
2104 return true;
2105 }
2106
2107 FoldableDef Def(OpToFold, DstRC);
2108 bool Changed = foldInstOperand(MI, OpToFold: Def);
2109
2110 // If we managed to fold all uses of this copy then we might as well
2111 // delete it now.
2112 // The only reason we need to follow chains of copies here is that
2113 // tryFoldRegSequence looks forward through copies before folding a
2114 // REG_SEQUENCE into its eventual users.
2115 auto *InstToErase = &MI;
2116 while (MRI->use_nodbg_empty(RegNo: InstToErase->getOperand(i: 0).getReg())) {
2117 auto &SrcOp = InstToErase->getOperand(i: 1);
2118 auto SrcReg = SrcOp.isReg() ? SrcOp.getReg() : Register();
2119 InstToErase->eraseFromParent();
2120 Changed = true;
2121 InstToErase = nullptr;
2122 if (!SrcReg || SrcReg.isPhysical())
2123 break;
2124 InstToErase = MRI->getVRegDef(Reg: SrcReg);
2125 if (!InstToErase || !TII->isFoldableCopy(MI: *InstToErase))
2126 break;
2127 }
2128
2129 if (InstToErase && InstToErase->isRegSequence() &&
2130 MRI->use_nodbg_empty(RegNo: InstToErase->getOperand(i: 0).getReg())) {
2131 InstToErase->eraseFromParent();
2132 Changed = true;
2133 }
2134
2135 if (Changed)
2136 return true;
2137
2138 // Run this after foldInstOperand to avoid turning scalar additions into
2139 // vector additions when the result scalar result could just be folded into
2140 // the user(s).
2141 return OpToFold.isReg() &&
2142 foldCopyToVGPROfScalarAddOfFrameIndex(DstReg, SrcReg: OpToFold.getReg(), MI);
2143}
2144
2145// Clamp patterns are canonically selected to v_max_* instructions, so only
2146// handle them.
2147const MachineOperand *
2148SIFoldOperandsImpl::isClamp(const MachineInstr &MI) const {
2149 unsigned Op = MI.getOpcode();
2150 switch (Op) {
2151 case AMDGPU::V_MAX_F32_e64:
2152 case AMDGPU::V_MAX_F16_e64:
2153 case AMDGPU::V_MAX_F16_t16_e64:
2154 case AMDGPU::V_MAX_F16_fake16_e64:
2155 case AMDGPU::V_MAX_F64_e64:
2156 case AMDGPU::V_MAX_NUM_F64_e64:
2157 case AMDGPU::V_PK_MAX_F16:
2158 case AMDGPU::V_MAX_BF16_PSEUDO_e64:
2159 case AMDGPU::V_PK_MAX_NUM_BF16: {
2160 if (MI.mayRaiseFPException())
2161 return nullptr;
2162
2163 if (!TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::clamp)->getImm())
2164 return nullptr;
2165
2166 // Make sure sources are identical.
2167 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2168 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2169 if (!Src0->isReg() || !Src1->isReg() ||
2170 Src0->getReg() != Src1->getReg() ||
2171 Src0->getSubReg() != Src1->getSubReg() ||
2172 Src0->getSubReg() != AMDGPU::NoSubRegister)
2173 return nullptr;
2174
2175 // Can't fold up if we have modifiers.
2176 if (TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
2177 return nullptr;
2178
2179 unsigned Src0Mods
2180 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0_modifiers)->getImm();
2181 unsigned Src1Mods
2182 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1_modifiers)->getImm();
2183
2184 // Having a 0 op_sel_hi would require swizzling the output in the source
2185 // instruction, which we can't do.
2186 unsigned UnsetMods =
2187 (Op == AMDGPU::V_PK_MAX_F16 || Op == AMDGPU::V_PK_MAX_NUM_BF16)
2188 ? SISrcMods::OP_SEL_1
2189 : 0u;
2190 if (Src0Mods != UnsetMods && Src1Mods != UnsetMods)
2191 return nullptr;
2192 return Src0;
2193 }
2194 default:
2195 return nullptr;
2196 }
2197}
2198
2199// FIXME: Clamp for v_mad_mixhi_f16 handled during isel.
2200bool SIFoldOperandsImpl::tryFoldClamp(MachineInstr &MI) {
2201 const MachineOperand *ClampSrc = isClamp(MI);
2202 if (!ClampSrc || !MRI->hasOneNonDBGUser(RegNo: ClampSrc->getReg()))
2203 return false;
2204
2205 if (!ClampSrc->getReg().isVirtual())
2206 return false;
2207
2208 // Look through COPY. COPY only observed with True16.
2209 Register DefSrcReg = TRI->lookThruCopyLike(SrcReg: ClampSrc->getReg(), MRI);
2210 MachineInstr *Def =
2211 MRI->getVRegDef(Reg: DefSrcReg.isVirtual() ? DefSrcReg : ClampSrc->getReg());
2212
2213 // The type of clamp must be compatible.
2214 if (!SIInstrInfo::hasSameClamp(A: *Def, B: MI))
2215 return false;
2216
2217 if (Def->mayRaiseFPException())
2218 return false;
2219
2220 MachineOperand *DefClamp = TII->getNamedOperand(MI&: *Def, OperandName: AMDGPU::OpName::clamp);
2221 if (!DefClamp)
2222 return false;
2223
2224 LLVM_DEBUG(dbgs() << "Folding clamp " << *DefClamp << " into " << *Def);
2225
2226 // Clamp is applied after omod, so it is OK if omod is set.
2227 DefClamp->setImm(1);
2228
2229 Register DefReg = Def->getOperand(i: 0).getReg();
2230 Register MIDstReg = MI.getOperand(i: 0).getReg();
2231 if (TRI->isSGPRReg(MRI: *MRI, Reg: DefReg)) {
2232 // Pseudo scalar instructions have a SGPR for dst and clamp is a v_max*
2233 // instruction with a VGPR dst.
2234 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
2235 DestReg: MIDstReg)
2236 .addReg(RegNo: DefReg);
2237 } else {
2238 MRI->replaceRegWith(FromReg: MIDstReg, ToReg: DefReg);
2239 }
2240 MI.eraseFromParent();
2241
2242 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
2243 // instruction, so we might as well convert it to the more flexible VOP3-only
2244 // mad/fma form.
2245 if (TII->convertToThreeAddress(MI&: *Def, LV: nullptr, LIS: nullptr))
2246 Def->eraseFromParent();
2247
2248 return true;
2249}
2250
2251static int getOModValue(unsigned Opc, int64_t Val) {
2252 switch (Opc) {
2253 case AMDGPU::V_MUL_F64_e64:
2254 case AMDGPU::V_MUL_F64_pseudo_e64: {
2255 switch (Val) {
2256 case 0x3fe0000000000000: // 0.5
2257 return SIOutMods::DIV2;
2258 case 0x4000000000000000: // 2.0
2259 return SIOutMods::MUL2;
2260 case 0x4010000000000000: // 4.0
2261 return SIOutMods::MUL4;
2262 default:
2263 return SIOutMods::NONE;
2264 }
2265 }
2266 case AMDGPU::V_MUL_F32_e64: {
2267 switch (static_cast<uint32_t>(Val)) {
2268 case 0x3f000000: // 0.5
2269 return SIOutMods::DIV2;
2270 case 0x40000000: // 2.0
2271 return SIOutMods::MUL2;
2272 case 0x40800000: // 4.0
2273 return SIOutMods::MUL4;
2274 default:
2275 return SIOutMods::NONE;
2276 }
2277 }
2278 case AMDGPU::V_MUL_F16_e64:
2279 case AMDGPU::V_MUL_F16_t16_e64:
2280 case AMDGPU::V_MUL_F16_fake16_e64: {
2281 switch (static_cast<uint16_t>(Val)) {
2282 case 0x3800: // 0.5
2283 return SIOutMods::DIV2;
2284 case 0x4000: // 2.0
2285 return SIOutMods::MUL2;
2286 case 0x4400: // 4.0
2287 return SIOutMods::MUL4;
2288 default:
2289 return SIOutMods::NONE;
2290 }
2291 }
2292 default:
2293 llvm_unreachable("invalid mul opcode");
2294 }
2295}
2296
2297// FIXME: Does this really not support denormals with f16?
2298// FIXME: Does this need to check IEEE mode bit? SNaNs are generally not
2299// handled, so will anything other than that break?
2300std::pair<const MachineOperand *, int>
2301SIFoldOperandsImpl::isOMod(const MachineInstr &MI) const {
2302 unsigned Op = MI.getOpcode();
2303 switch (Op) {
2304 case AMDGPU::V_MUL_F64_e64:
2305 case AMDGPU::V_MUL_F64_pseudo_e64:
2306 case AMDGPU::V_MUL_F32_e64:
2307 case AMDGPU::V_MUL_F16_t16_e64:
2308 case AMDGPU::V_MUL_F16_fake16_e64:
2309 case AMDGPU::V_MUL_F16_e64: {
2310 // If output denormals are enabled, omod is ignored.
2311 if ((Op == AMDGPU::V_MUL_F32_e64 &&
2312 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
2313 ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F64_pseudo_e64 ||
2314 Op == AMDGPU::V_MUL_F16_e64 || Op == AMDGPU::V_MUL_F16_t16_e64 ||
2315 Op == AMDGPU::V_MUL_F16_fake16_e64) &&
2316 MFI->getMode().FP64FP16Denormals.Output !=
2317 DenormalMode::PreserveSign) ||
2318 MI.mayRaiseFPException())
2319 return std::pair(nullptr, SIOutMods::NONE);
2320
2321 const MachineOperand *RegOp = nullptr;
2322 const MachineOperand *ImmOp = nullptr;
2323 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2324 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2325 if (Src0->isImm()) {
2326 ImmOp = Src0;
2327 RegOp = Src1;
2328 } else if (Src1->isImm()) {
2329 ImmOp = Src1;
2330 RegOp = Src0;
2331 } else
2332 return std::pair(nullptr, SIOutMods::NONE);
2333
2334 int OMod = getOModValue(Opc: Op, Val: ImmOp->getImm());
2335 if (OMod == SIOutMods::NONE ||
2336 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers) ||
2337 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers) ||
2338 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod) ||
2339 TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp))
2340 return std::pair(nullptr, SIOutMods::NONE);
2341
2342 return std::pair(RegOp, OMod);
2343 }
2344 case AMDGPU::V_ADD_F64_e64:
2345 case AMDGPU::V_ADD_F64_pseudo_e64:
2346 case AMDGPU::V_ADD_F32_e64:
2347 case AMDGPU::V_ADD_F16_e64:
2348 case AMDGPU::V_ADD_F16_t16_e64:
2349 case AMDGPU::V_ADD_F16_fake16_e64: {
2350 // If output denormals are enabled, omod is ignored.
2351 if ((Op == AMDGPU::V_ADD_F32_e64 &&
2352 MFI->getMode().FP32Denormals.Output != DenormalMode::PreserveSign) ||
2353 ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F64_pseudo_e64 ||
2354 Op == AMDGPU::V_ADD_F16_e64 || Op == AMDGPU::V_ADD_F16_t16_e64 ||
2355 Op == AMDGPU::V_ADD_F16_fake16_e64) &&
2356 MFI->getMode().FP64FP16Denormals.Output != DenormalMode::PreserveSign))
2357 return std::pair(nullptr, SIOutMods::NONE);
2358
2359 // Look through the DAGCombiner canonicalization fmul x, 2 -> fadd x, x
2360 const MachineOperand *Src0 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src0);
2361 const MachineOperand *Src1 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src1);
2362
2363 if (Src0->isReg() && Src1->isReg() && Src0->getReg() == Src1->getReg() &&
2364 Src0->getSubReg() == Src1->getSubReg() &&
2365 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src0_modifiers) &&
2366 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::src1_modifiers) &&
2367 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::clamp) &&
2368 !TII->hasModifiersSet(MI, OpName: AMDGPU::OpName::omod))
2369 return std::pair(Src0, SIOutMods::MUL2);
2370
2371 return std::pair(nullptr, SIOutMods::NONE);
2372 }
2373 default:
2374 return std::pair(nullptr, SIOutMods::NONE);
2375 }
2376}
2377
2378// FIXME: Does this need to check IEEE bit on function?
2379bool SIFoldOperandsImpl::tryFoldOMod(MachineInstr &MI) {
2380 const MachineOperand *RegOp;
2381 int OMod;
2382 std::tie(args&: RegOp, args&: OMod) = isOMod(MI);
2383 if (OMod == SIOutMods::NONE || !RegOp->isReg() ||
2384 RegOp->getSubReg() != AMDGPU::NoSubRegister ||
2385 !MRI->hasOneNonDBGUser(RegNo: RegOp->getReg()))
2386 return false;
2387
2388 MachineInstr *Def = MRI->getVRegDef(Reg: RegOp->getReg());
2389 MachineOperand *DefOMod = TII->getNamedOperand(MI&: *Def, OperandName: AMDGPU::OpName::omod);
2390 if (!DefOMod || DefOMod->getImm() != SIOutMods::NONE)
2391 return false;
2392
2393 if (Def->mayRaiseFPException())
2394 return false;
2395
2396 // Clamp is applied after omod. If the source already has clamp set, don't
2397 // fold it.
2398 if (TII->hasModifiersSet(MI: *Def, OpName: AMDGPU::OpName::clamp))
2399 return false;
2400
2401 LLVM_DEBUG(dbgs() << "Folding omod " << MI << " into " << *Def);
2402
2403 DefOMod->setImm(OMod);
2404 MRI->replaceRegWith(FromReg: MI.getOperand(i: 0).getReg(), ToReg: Def->getOperand(i: 0).getReg());
2405 // Kill flags can be wrong if we replaced a def inside a loop with a def
2406 // outside the loop.
2407 MRI->clearKillFlags(Reg: Def->getOperand(i: 0).getReg());
2408 MI.eraseFromParent();
2409
2410 // Use of output modifiers forces VOP3 encoding for a VOP2 mac/fmac
2411 // instruction, so we might as well convert it to the more flexible VOP3-only
2412 // mad/fma form.
2413 if (TII->convertToThreeAddress(MI&: *Def, LV: nullptr, LIS: nullptr))
2414 Def->eraseFromParent();
2415
2416 return true;
2417}
2418
2419// Try to fold a reg_sequence with vgpr output and agpr inputs into an
2420// instruction which can take an agpr. So far that means a store.
2421bool SIFoldOperandsImpl::tryFoldRegSequence(MachineInstr &MI) {
2422 assert(MI.isRegSequence());
2423 auto Reg = MI.getOperand(i: 0).getReg();
2424
2425 if (!ST->hasGFX90AInsts() || !TRI->isVGPR(MRI: *MRI, Reg) ||
2426 !MRI->hasOneNonDBGUse(RegNo: Reg))
2427 return false;
2428
2429 SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs;
2430 if (!getRegSeqInit(Defs, UseReg: Reg))
2431 return false;
2432
2433 for (auto &[Op, SubIdx] : Defs) {
2434 if (!Op->isReg())
2435 return false;
2436 if (TRI->isAGPR(MRI: *MRI, Reg: Op->getReg()))
2437 continue;
2438 // Maybe this is a COPY from AREG
2439 const MachineInstr *SubDef = MRI->getVRegDef(Reg: Op->getReg());
2440 if (!SubDef || !SubDef->isCopy() || SubDef->getOperand(i: 1).getSubReg())
2441 return false;
2442 if (!TRI->isAGPR(MRI: *MRI, Reg: SubDef->getOperand(i: 1).getReg()))
2443 return false;
2444 }
2445
2446 MachineOperand *Op = &*MRI->use_nodbg_begin(RegNo: Reg);
2447 MachineInstr *UseMI = Op->getParent();
2448 while (UseMI->isCopy() && !Op->getSubReg()) {
2449 Reg = UseMI->getOperand(i: 0).getReg();
2450 if (!TRI->isVGPR(MRI: *MRI, Reg) || !MRI->hasOneNonDBGUse(RegNo: Reg))
2451 return false;
2452 Op = &*MRI->use_nodbg_begin(RegNo: Reg);
2453 UseMI = Op->getParent();
2454 }
2455
2456 if (Op->getSubReg())
2457 return false;
2458
2459 unsigned OpIdx = Op - &UseMI->getOperand(i: 0);
2460 const MCInstrDesc &InstDesc = UseMI->getDesc();
2461 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: InstDesc, OpNum: OpIdx);
2462 if (!OpRC || !TRI->isVectorSuperClass(RC: OpRC))
2463 return false;
2464
2465 const auto *NewDstRC = TRI->getEquivalentAGPRClass(SRC: MRI->getRegClass(Reg));
2466 auto Dst = MRI->createVirtualRegister(RegClass: NewDstRC);
2467 auto RS = BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
2468 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst);
2469
2470 for (auto &[Def, SubIdx] : Defs) {
2471 Def->setIsKill(false);
2472 if (TRI->isAGPR(MRI: *MRI, Reg: Def->getReg())) {
2473 RS.add(MO: *Def);
2474 } else { // This is a copy
2475 MachineInstr *SubDef = MRI->getVRegDef(Reg: Def->getReg());
2476 SubDef->getOperand(i: 1).setIsKill(false);
2477 RS.addReg(RegNo: SubDef->getOperand(i: 1).getReg(), Flags: {}, SubReg: Def->getSubReg());
2478 }
2479 RS.addImm(Val: SubIdx);
2480 }
2481
2482 Op->setReg(Dst);
2483 if (!TII->isOperandLegal(MI: *UseMI, OpIdx, MO: Op)) {
2484 Op->setReg(Reg);
2485 RS->eraseFromParent();
2486 return false;
2487 }
2488
2489 LLVM_DEBUG(dbgs() << "Folded " << *RS << " into " << *UseMI);
2490
2491 // Erase the REG_SEQUENCE eagerly, unless we followed a chain of COPY users,
2492 // in which case we can erase them all later in runOnMachineFunction.
2493 if (MRI->use_nodbg_empty(RegNo: MI.getOperand(i: 0).getReg()))
2494 MI.eraseFromParent();
2495 return true;
2496}
2497
2498/// Checks whether \p Copy is a AGPR -> VGPR copy. Returns `true` on success and
2499/// stores the AGPR register in \p OutReg and the subreg in \p OutSubReg
2500static bool isAGPRCopy(const SIRegisterInfo &TRI,
2501 const MachineRegisterInfo &MRI, const MachineInstr &Copy,
2502 Register &OutReg, unsigned &OutSubReg) {
2503 assert(Copy.isCopy());
2504
2505 const MachineOperand &CopySrc = Copy.getOperand(i: 1);
2506 Register CopySrcReg = CopySrc.getReg();
2507 if (!CopySrcReg.isVirtual())
2508 return false;
2509
2510 // Common case: copy from AGPR directly, e.g.
2511 // %1:vgpr_32 = COPY %0:agpr_32
2512 if (TRI.isAGPR(MRI, Reg: CopySrcReg)) {
2513 OutReg = CopySrcReg;
2514 OutSubReg = CopySrc.getSubReg();
2515 return true;
2516 }
2517
2518 // Sometimes it can also involve two copies, e.g.
2519 // %1:vgpr_256 = COPY %0:agpr_256
2520 // %2:vgpr_32 = COPY %1:vgpr_256.sub0
2521 const MachineInstr *CopySrcDef = MRI.getVRegDef(Reg: CopySrcReg);
2522 if (!CopySrcDef || !CopySrcDef->isCopy())
2523 return false;
2524
2525 const MachineOperand &OtherCopySrc = CopySrcDef->getOperand(i: 1);
2526 Register OtherCopySrcReg = OtherCopySrc.getReg();
2527 if (!OtherCopySrcReg.isVirtual() ||
2528 CopySrcDef->getOperand(i: 0).getSubReg() != AMDGPU::NoSubRegister ||
2529 OtherCopySrc.getSubReg() != AMDGPU::NoSubRegister ||
2530 !TRI.isAGPR(MRI, Reg: OtherCopySrcReg))
2531 return false;
2532
2533 OutReg = OtherCopySrcReg;
2534 OutSubReg = CopySrc.getSubReg();
2535 return true;
2536}
2537
2538// Try to hoist an AGPR to VGPR copy across a PHI.
2539// This should allow folding of an AGPR into a consumer which may support it.
2540//
2541// Example 1: LCSSA PHI
2542// loop:
2543// %1:vreg = COPY %0:areg
2544// exit:
2545// %2:vreg = PHI %1:vreg, %loop
2546// =>
2547// loop:
2548// exit:
2549// %1:areg = PHI %0:areg, %loop
2550// %2:vreg = COPY %1:areg
2551//
2552// Example 2: PHI with multiple incoming values:
2553// entry:
2554// %1:vreg = GLOBAL_LOAD(..)
2555// loop:
2556// %2:vreg = PHI %1:vreg, %entry, %5:vreg, %loop
2557// %3:areg = COPY %2:vreg
2558// %4:areg = (instr using %3:areg)
2559// %5:vreg = COPY %4:areg
2560// =>
2561// entry:
2562// %1:vreg = GLOBAL_LOAD(..)
2563// %2:areg = COPY %1:vreg
2564// loop:
2565// %3:areg = PHI %2:areg, %entry, %X:areg,
2566// %4:areg = (instr using %3:areg)
2567bool SIFoldOperandsImpl::tryFoldPhiAGPR(MachineInstr &PHI) {
2568 assert(PHI.isPHI());
2569
2570 Register PhiOut = PHI.getOperand(i: 0).getReg();
2571 if (!TRI->isVGPR(MRI: *MRI, Reg: PhiOut))
2572 return false;
2573
2574 // Iterate once over all incoming values of the PHI to check if this PHI is
2575 // eligible, and determine the exact AGPR RC we'll target.
2576 const TargetRegisterClass *ARC = nullptr;
2577 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
2578 MachineOperand &MO = PHI.getOperand(i: K);
2579 MachineInstr *Copy = MRI->getVRegDef(Reg: MO.getReg());
2580 if (!Copy || !Copy->isCopy())
2581 continue;
2582
2583 Register AGPRSrc;
2584 unsigned AGPRRegMask = AMDGPU::NoSubRegister;
2585 if (!isAGPRCopy(TRI: *TRI, MRI: *MRI, Copy: *Copy, OutReg&: AGPRSrc, OutSubReg&: AGPRRegMask))
2586 continue;
2587
2588 const TargetRegisterClass *CopyInRC = MRI->getRegClass(Reg: AGPRSrc);
2589 if (const auto *SubRC = TRI->getSubRegisterClass(CopyInRC, AGPRRegMask))
2590 CopyInRC = SubRC;
2591
2592 if (ARC && !ARC->hasSubClassEq(RC: CopyInRC))
2593 return false;
2594 ARC = CopyInRC;
2595 }
2596
2597 if (!ARC)
2598 return false;
2599
2600 bool IsAGPR32 = (ARC == &AMDGPU::AGPR_32RegClass);
2601
2602 // Rewrite the PHI's incoming values to ARC.
2603 LLVM_DEBUG(dbgs() << "Folding AGPR copies into: " << PHI);
2604 for (unsigned K = 1; K < PHI.getNumExplicitOperands(); K += 2) {
2605 MachineOperand &MO = PHI.getOperand(i: K);
2606 Register Reg = MO.getReg();
2607
2608 MachineBasicBlock::iterator InsertPt;
2609 MachineBasicBlock *InsertMBB = nullptr;
2610
2611 // Look at the def of Reg, ignoring all copies.
2612 unsigned CopyOpc = AMDGPU::COPY;
2613 if (MachineInstr *Def = MRI->getVRegDef(Reg)) {
2614
2615 // Look at pre-existing COPY instructions from ARC: Steal the operand. If
2616 // the copy was single-use, it will be removed by DCE later.
2617 if (Def->isCopy()) {
2618 Register AGPRSrc;
2619 unsigned AGPRSubReg = AMDGPU::NoSubRegister;
2620 if (isAGPRCopy(TRI: *TRI, MRI: *MRI, Copy: *Def, OutReg&: AGPRSrc, OutSubReg&: AGPRSubReg)) {
2621 MO.setReg(AGPRSrc);
2622 MO.setSubReg(AGPRSubReg);
2623 continue;
2624 }
2625
2626 // If this is a multi-use SGPR -> VGPR copy, use V_ACCVGPR_WRITE on
2627 // GFX908 directly instead of a COPY. Otherwise, SIFoldOperand may try
2628 // to fold the sgpr -> vgpr -> agpr copy into a sgpr -> agpr copy which
2629 // is unlikely to be profitable.
2630 //
2631 // Note that V_ACCVGPR_WRITE is only used for AGPR_32.
2632 MachineOperand &CopyIn = Def->getOperand(i: 1);
2633 if (IsAGPR32 && !ST->hasGFX90AInsts() && !MRI->hasOneNonDBGUse(RegNo: Reg) &&
2634 TRI->isSGPRReg(MRI: *MRI, Reg: CopyIn.getReg()))
2635 CopyOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
2636 }
2637
2638 InsertMBB = Def->getParent();
2639 InsertPt = InsertMBB->SkipPHIsLabelsAndDebug(I: ++Def->getIterator());
2640 } else {
2641 InsertMBB = PHI.getOperand(i: MO.getOperandNo() + 1).getMBB();
2642 InsertPt = InsertMBB->getFirstTerminator();
2643 }
2644
2645 Register NewReg = MRI->createVirtualRegister(RegClass: ARC);
2646 MachineInstr *MI = BuildMI(BB&: *InsertMBB, I: InsertPt, MIMD: PHI.getDebugLoc(),
2647 MCID: TII->get(Opcode: CopyOpc), DestReg: NewReg)
2648 .addReg(RegNo: Reg);
2649 MO.setReg(NewReg);
2650
2651 (void)MI;
2652 LLVM_DEBUG(dbgs() << " Created COPY: " << *MI);
2653 }
2654
2655 // Replace the PHI's result with a new register.
2656 Register NewReg = MRI->createVirtualRegister(RegClass: ARC);
2657 PHI.getOperand(i: 0).setReg(NewReg);
2658
2659 // COPY that new register back to the original PhiOut register. This COPY will
2660 // usually be folded out later.
2661 MachineBasicBlock *MBB = PHI.getParent();
2662 BuildMI(BB&: *MBB, I: MBB->getFirstNonPHI(), MIMD: PHI.getDebugLoc(),
2663 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: PhiOut)
2664 .addReg(RegNo: NewReg);
2665
2666 LLVM_DEBUG(dbgs() << " Done: Folded " << PHI);
2667 return true;
2668}
2669
2670// Attempt to convert VGPR load to an AGPR load.
2671bool SIFoldOperandsImpl::tryFoldLoad(MachineInstr &MI) {
2672 assert(MI.mayLoad());
2673 if (!ST->hasGFX90AInsts() || MI.getNumExplicitDefs() != 1)
2674 return false;
2675
2676 MachineOperand &Def = MI.getOperand(i: 0);
2677 if (!Def.isDef())
2678 return false;
2679
2680 Register DefReg = Def.getReg();
2681
2682 if (DefReg.isPhysical() || !TRI->isVGPR(MRI: *MRI, Reg: DefReg))
2683 return false;
2684
2685 SmallVector<const MachineInstr *, 8> Users(
2686 llvm::make_pointer_range(Range: MRI->use_nodbg_instructions(Reg: DefReg)));
2687 SmallVector<Register, 8> MoveRegs;
2688
2689 if (Users.empty())
2690 return false;
2691
2692 // Check that all uses a copy to an agpr or a reg_sequence producing an agpr.
2693 while (!Users.empty()) {
2694 const MachineInstr *I = Users.pop_back_val();
2695 if (!I->isCopy() && !I->isRegSequence())
2696 return false;
2697 Register DstReg = I->getOperand(i: 0).getReg();
2698 // Physical registers may have more than one instruction definitions
2699 if (DstReg.isPhysical())
2700 return false;
2701 if (TRI->isAGPR(MRI: *MRI, Reg: DstReg))
2702 continue;
2703 MoveRegs.push_back(Elt: DstReg);
2704 for (const MachineInstr &U : MRI->use_nodbg_instructions(Reg: DstReg))
2705 Users.push_back(Elt: &U);
2706 }
2707
2708 const TargetRegisterClass *RC = MRI->getRegClass(Reg: DefReg);
2709 MRI->setRegClass(Reg: DefReg, RC: TRI->getEquivalentAGPRClass(SRC: RC));
2710 if (!TII->isOperandLegal(MI, OpIdx: 0, MO: &Def)) {
2711 MRI->setRegClass(Reg: DefReg, RC);
2712 return false;
2713 }
2714
2715 while (!MoveRegs.empty()) {
2716 Register Reg = MoveRegs.pop_back_val();
2717 MRI->setRegClass(Reg, RC: TRI->getEquivalentAGPRClass(SRC: MRI->getRegClass(Reg)));
2718 }
2719
2720 LLVM_DEBUG(dbgs() << "Folded " << MI);
2721
2722 return true;
2723}
2724
2725// tryFoldPhiAGPR will aggressively try to create AGPR PHIs.
2726// For GFX90A and later, this is pretty much always a good thing, but for GFX908
2727// there's cases where it can create a lot more AGPR-AGPR copies, which are
2728// expensive on this architecture due to the lack of V_ACCVGPR_MOV.
2729//
2730// This function looks at all AGPR PHIs in a basic block and collects their
2731// operands. Then, it checks for register that are used more than once across
2732// all PHIs and caches them in a VGPR. This prevents ExpandPostRAPseudo from
2733// having to create one VGPR temporary per use, which can get very messy if
2734// these PHIs come from a broken-up large PHI (e.g. 32 AGPR phis, one per vector
2735// element).
2736//
2737// Example
2738// a:
2739// %in:agpr_256 = COPY %foo:vgpr_256
2740// c:
2741// %x:agpr_32 = ..
2742// b:
2743// %0:areg = PHI %in.sub0:agpr_32, %a, %x, %c
2744// %1:areg = PHI %in.sub0:agpr_32, %a, %y, %c
2745// %2:areg = PHI %in.sub0:agpr_32, %a, %z, %c
2746// =>
2747// a:
2748// %in:agpr_256 = COPY %foo:vgpr_256
2749// %tmp:vgpr_32 = V_ACCVGPR_READ_B32_e64 %in.sub0:agpr_32
2750// %tmp_agpr:agpr_32 = COPY %tmp
2751// c:
2752// %x:agpr_32 = ..
2753// b:
2754// %0:areg = PHI %tmp_agpr, %a, %x, %c
2755// %1:areg = PHI %tmp_agpr, %a, %y, %c
2756// %2:areg = PHI %tmp_agpr, %a, %z, %c
2757bool SIFoldOperandsImpl::tryOptimizeAGPRPhis(MachineBasicBlock &MBB) {
2758 // This is only really needed on GFX908 where AGPR-AGPR copies are
2759 // unreasonably difficult.
2760 if (ST->hasGFX90AInsts())
2761 return false;
2762
2763 // Look at all AGPR Phis and collect the register + subregister used.
2764 DenseMap<std::pair<Register, unsigned>, std::vector<MachineOperand *>>
2765 RegToMO;
2766
2767 for (auto &MI : MBB) {
2768 if (!MI.isPHI())
2769 break;
2770
2771 if (!TRI->isAGPR(MRI: *MRI, Reg: MI.getOperand(i: 0).getReg()))
2772 continue;
2773
2774 for (unsigned K = 1; K < MI.getNumOperands(); K += 2) {
2775 MachineOperand &PhiMO = MI.getOperand(i: K);
2776 if (!PhiMO.getSubReg())
2777 continue;
2778 RegToMO[{PhiMO.getReg(), PhiMO.getSubReg()}].push_back(x: &PhiMO);
2779 }
2780 }
2781
2782 // For all (Reg, SubReg) pair that are used more than once, cache the value in
2783 // a VGPR.
2784 bool Changed = false;
2785 for (const auto &[Entry, MOs] : RegToMO) {
2786 if (MOs.size() == 1)
2787 continue;
2788
2789 const auto [Reg, SubReg] = Entry;
2790 MachineInstr *Def = MRI->getVRegDef(Reg);
2791 MachineBasicBlock *DefMBB = Def->getParent();
2792
2793 // Create a copy in a VGPR using V_ACCVGPR_READ_B32_e64 so it's not folded
2794 // out.
2795 const TargetRegisterClass *ARC = getRegOpRC(MRI: *MRI, TRI: *TRI, MO: *MOs.front());
2796 Register TempVGPR =
2797 MRI->createVirtualRegister(RegClass: TRI->getEquivalentVGPRClass(SRC: ARC));
2798 MachineInstr *VGPRCopy =
2799 BuildMI(BB&: *DefMBB, I: ++Def->getIterator(), MIMD: Def->getDebugLoc(),
2800 MCID: TII->get(Opcode: AMDGPU::V_ACCVGPR_READ_B32_e64), DestReg: TempVGPR)
2801 .addReg(RegNo: Reg, /* flags */ Flags: {}, SubReg);
2802
2803 // Copy back to an AGPR and use that instead of the AGPR subreg in all MOs.
2804 Register TempAGPR = MRI->createVirtualRegister(RegClass: ARC);
2805 BuildMI(BB&: *DefMBB, I: ++VGPRCopy->getIterator(), MIMD: Def->getDebugLoc(),
2806 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: TempAGPR)
2807 .addReg(RegNo: TempVGPR);
2808
2809 LLVM_DEBUG(dbgs() << "Caching AGPR into VGPR: " << *VGPRCopy);
2810 for (MachineOperand *MO : MOs) {
2811 MO->setReg(TempAGPR);
2812 MO->setSubReg(AMDGPU::NoSubRegister);
2813 LLVM_DEBUG(dbgs() << " Changed PHI Operand: " << *MO << "\n");
2814 }
2815
2816 Changed = true;
2817 }
2818
2819 return Changed;
2820}
2821
2822bool SIFoldOperandsImpl::run(MachineFunction &MF) {
2823 this->MF = &MF;
2824 MRI = &MF.getRegInfo();
2825 ST = &MF.getSubtarget<GCNSubtarget>();
2826 TII = ST->getInstrInfo();
2827 TRI = &TII->getRegisterInfo();
2828 MFI = MF.getInfo<SIMachineFunctionInfo>();
2829
2830 // omod is ignored by hardware if IEEE bit is enabled. omod also does not
2831 // correctly handle signed zeros.
2832 //
2833 // FIXME: Also need to check strictfp
2834 bool IsIEEEMode = MFI->getMode().IEEE;
2835
2836 bool Changed = false;
2837 for (MachineBasicBlock *MBB : depth_first(G: &MF)) {
2838 MachineOperand *CurrentKnownM0Val = nullptr;
2839 for (auto &MI : make_early_inc_range(Range&: *MBB)) {
2840 Changed |= tryFoldCndMask(MI);
2841
2842 if (tryFoldZeroHighBits(MI)) {
2843 Changed = true;
2844 continue;
2845 }
2846
2847 if (MI.isRegSequence() && tryFoldRegSequence(MI)) {
2848 Changed = true;
2849 continue;
2850 }
2851
2852 if (MI.isPHI() && tryFoldPhiAGPR(PHI&: MI)) {
2853 Changed = true;
2854 continue;
2855 }
2856
2857 if (MI.mayLoad() && tryFoldLoad(MI)) {
2858 Changed = true;
2859 continue;
2860 }
2861
2862 if (TII->isFoldableCopy(MI)) {
2863 Changed |= tryFoldFoldableCopy(MI, CurrentKnownM0Val);
2864 continue;
2865 }
2866
2867 // Saw an unknown clobber of m0, so we no longer know what it is.
2868 if (CurrentKnownM0Val && MI.modifiesRegister(Reg: AMDGPU::M0, TRI))
2869 CurrentKnownM0Val = nullptr;
2870
2871 // TODO: Omod might be OK if there is NSZ only on the source
2872 // instruction, and not the omod multiply.
2873 if (IsIEEEMode || !MI.getFlag(Flag: MachineInstr::FmNsz) || !tryFoldOMod(MI))
2874 Changed |= tryFoldClamp(MI);
2875 }
2876
2877 Changed |= tryOptimizeAGPRPhis(MBB&: *MBB);
2878 }
2879
2880 return Changed;
2881}
2882
2883PreservedAnalyses SIFoldOperandsPass::run(MachineFunction &MF,
2884 MachineFunctionAnalysisManager &) {
2885 MFPropsModifier _(*this, MF);
2886
2887 bool Changed = SIFoldOperandsImpl().run(MF);
2888 if (!Changed) {
2889 return PreservedAnalyses::all();
2890 }
2891 auto PA = getMachineFunctionPassPreservedAnalyses();
2892 PA.preserveSet<CFGAnalyses>();
2893 return PA;
2894}
2895