1//===- AMDGPUInstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUInstructionSelector.h"
15#include "AMDGPU.h"
16#include "AMDGPUGlobalISelUtils.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPURegisterBankInfo.h"
19#include "AMDGPUTargetMachine.h"
20#include "SIMachineFunctionInfo.h"
21#include "Utils/AMDGPUBaseInfo.h"
22#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
25#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
26#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/IntrinsicsAMDGPU.h"
30#include <optional>
31
32#define DEBUG_TYPE "amdgpu-isel"
33
34using namespace llvm;
35using namespace MIPatternMatch;
36
37#define GET_GLOBALISEL_IMPL
38#define AMDGPUSubtarget GCNSubtarget
39#include "AMDGPUGenGlobalISel.inc"
40#undef GET_GLOBALISEL_IMPL
41#undef AMDGPUSubtarget
42
43AMDGPUInstructionSelector::AMDGPUInstructionSelector(
44 const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI)
45 : TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI), STI(STI),
46#define GET_GLOBALISEL_PREDICATES_INIT
47#include "AMDGPUGenGlobalISel.inc"
48#undef GET_GLOBALISEL_PREDICATES_INIT
49#define GET_GLOBALISEL_TEMPORARIES_INIT
50#include "AMDGPUGenGlobalISel.inc"
51#undef GET_GLOBALISEL_TEMPORARIES_INIT
52{
53}
54
55const char *AMDGPUInstructionSelector::getName() { return DEBUG_TYPE; }
56
57void AMDGPUInstructionSelector::setupMF(MachineFunction &MF,
58 GISelValueTracking *VT,
59 CodeGenCoverage *CoverageInfo,
60 ProfileSummaryInfo *PSI,
61 BlockFrequencyInfo *BFI) {
62 MRI = &MF.getRegInfo();
63 Subtarget = &MF.getSubtarget<GCNSubtarget>();
64 Subtarget->checkSubtargetFeatures(F: MF.getFunction());
65 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
66}
67
68// Return the wave level SGPR base address if this is a wave address.
69static Register getWaveAddress(const MachineInstr *Def) {
70 return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS
71 ? Def->getOperand(i: 1).getReg()
72 : Register();
73}
74
75bool AMDGPUInstructionSelector::isVCC(Register Reg,
76 const MachineRegisterInfo &MRI) const {
77 // The verifier is oblivious to s1 being a valid value for wavesize registers.
78 if (Reg.isPhysical())
79 return false;
80
81 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
82 const TargetRegisterClass *RC =
83 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
84 if (RC) {
85 const LLT Ty = MRI.getType(Reg);
86 if (!Ty.isValid() || Ty.getSizeInBits() != 1)
87 return false;
88 // G_TRUNC s1 result is never vcc.
89 return MRI.getVRegDef(Reg)->getOpcode() != AMDGPU::G_TRUNC &&
90 RC->hasSuperClassEq(RC: TRI.getBoolRC());
91 }
92
93 const RegisterBank *RB = cast<const RegisterBank *>(Val: RegClassOrBank);
94 return RB->getID() == AMDGPU::VCCRegBankID;
95}
96
97bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(MachineInstr &MI,
98 unsigned NewOpc) const {
99 MI.setDesc(TII.get(Opcode: NewOpc));
100 MI.removeOperand(OpNo: 1); // Remove intrinsic ID.
101 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
102
103 MachineOperand &Dst = MI.getOperand(i: 0);
104 MachineOperand &Src = MI.getOperand(i: 1);
105
106 // TODO: This should be legalized to s32 if needed
107 if (MRI->getType(Reg: Dst.getReg()) == LLT::scalar(SizeInBits: 1))
108 return false;
109
110 const TargetRegisterClass *DstRC
111 = TRI.getConstrainedRegClassForOperand(MO: Dst, MRI: *MRI);
112 const TargetRegisterClass *SrcRC
113 = TRI.getConstrainedRegClassForOperand(MO: Src, MRI: *MRI);
114 if (!DstRC || DstRC != SrcRC)
115 return false;
116
117 if (!RBI.constrainGenericRegister(Reg: Dst.getReg(), RC: *DstRC, MRI&: *MRI) ||
118 !RBI.constrainGenericRegister(Reg: Src.getReg(), RC: *SrcRC, MRI&: *MRI))
119 return false;
120 const MCInstrDesc &MCID = MI.getDesc();
121 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
122 MI.getOperand(i: 0).setIsEarlyClobber(true);
123 }
124 return true;
125}
126
127bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
128 const DebugLoc &DL = I.getDebugLoc();
129 MachineBasicBlock *BB = I.getParent();
130 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
131
132 const MachineOperand &Src = I.getOperand(i: 1);
133 MachineOperand &Dst = I.getOperand(i: 0);
134 Register DstReg = Dst.getReg();
135 Register SrcReg = Src.getReg();
136
137 if (isVCC(Reg: DstReg, MRI: *MRI)) {
138 if (SrcReg == AMDGPU::SCC) {
139 const TargetRegisterClass *RC
140 = TRI.getConstrainedRegClassForOperand(MO: Dst, MRI: *MRI);
141 if (!RC)
142 return true;
143 return RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI);
144 }
145
146 if (!isVCC(Reg: SrcReg, MRI: *MRI)) {
147 // TODO: Should probably leave the copy and let copyPhysReg expand it.
148 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI))
149 return false;
150
151 const TargetRegisterClass *SrcRC
152 = TRI.getConstrainedRegClassForOperand(MO: Src, MRI: *MRI);
153
154 std::optional<ValueAndVReg> ConstVal =
155 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI, LookThroughInstrs: true);
156 if (ConstVal) {
157 unsigned MovOpc =
158 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
159 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: DstReg)
160 .addImm(Val: ConstVal->Value.getBoolValue() ? -1 : 0);
161 } else {
162 Register MaskedReg = MRI->createVirtualRegister(RegClass: SrcRC);
163
164 // We can't trust the high bits at this point, so clear them.
165
166 // TODO: Skip masking high bits if def is known boolean.
167
168 if (AMDGPU::getRegBitWidth(RCID: SrcRC->getID()) == 16) {
169 assert(Subtarget->useRealTrue16Insts());
170 const int64_t NoMods = 0;
171 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B16_t16_e64), DestReg: MaskedReg)
172 .addImm(Val: NoMods)
173 .addImm(Val: 1)
174 .addImm(Val: NoMods)
175 .addReg(RegNo: SrcReg)
176 .addImm(Val: NoMods);
177 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U16_t16_e64), DestReg: DstReg)
178 .addImm(Val: NoMods)
179 .addImm(Val: 0)
180 .addImm(Val: NoMods)
181 .addReg(RegNo: MaskedReg)
182 .addImm(Val: NoMods);
183 } else {
184 bool IsSGPR = TRI.isSGPRClass(RC: SrcRC);
185 unsigned AndOpc = IsSGPR ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32;
186 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: MaskedReg)
187 .addImm(Val: 1)
188 .addReg(RegNo: SrcReg);
189 if (IsSGPR)
190 And.setOperandDead(3); // Dead scc
191
192 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U32_e64), DestReg: DstReg)
193 .addImm(Val: 0)
194 .addReg(RegNo: MaskedReg);
195 }
196 }
197
198 if (!MRI->getRegClassOrNull(Reg: SrcReg))
199 MRI->setRegClass(Reg: SrcReg, RC: SrcRC);
200 I.eraseFromParent();
201 return true;
202 }
203
204 const TargetRegisterClass *RC =
205 TRI.getConstrainedRegClassForOperand(MO: Dst, MRI: *MRI);
206 if (RC && !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
207 return false;
208
209 return true;
210 }
211
212 for (const MachineOperand &MO : I.operands()) {
213 if (MO.getReg().isPhysical())
214 continue;
215
216 const TargetRegisterClass *RC =
217 TRI.getConstrainedRegClassForOperand(MO, MRI: *MRI);
218 if (!RC)
219 continue;
220 RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI);
221 }
222 return true;
223}
224
225bool AMDGPUInstructionSelector::selectCOPY_SCC_VCC(MachineInstr &I) const {
226 const DebugLoc &DL = I.getDebugLoc();
227 MachineBasicBlock *BB = I.getParent();
228 Register VCCReg = I.getOperand(i: 1).getReg();
229 MachineInstr *Cmp;
230
231 // Set SCC as a side effect with S_CMP or S_OR.
232 if (STI.hasScalarCompareEq64()) {
233 unsigned CmpOpc =
234 STI.isWave64() ? AMDGPU::S_CMP_LG_U64 : AMDGPU::S_CMP_LG_U32;
235 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: CmpOpc)).addReg(RegNo: VCCReg).addImm(Val: 0);
236 } else {
237 Register DeadDst = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
238 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B64), DestReg: DeadDst)
239 .addReg(RegNo: VCCReg)
240 .addReg(RegNo: VCCReg);
241 }
242
243 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
244
245 Register DstReg = I.getOperand(i: 0).getReg();
246 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: AMDGPU::SCC);
247
248 I.eraseFromParent();
249 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
250}
251
252bool AMDGPUInstructionSelector::selectCOPY_VCC_SCC(MachineInstr &I) const {
253 const DebugLoc &DL = I.getDebugLoc();
254 MachineBasicBlock *BB = I.getParent();
255
256 Register DstReg = I.getOperand(i: 0).getReg();
257 Register SrcReg = I.getOperand(i: 1).getReg();
258 std::optional<ValueAndVReg> Arg =
259 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 1).getReg(), MRI: *MRI);
260
261 if (Arg) {
262 const int64_t Value = Arg->Value.getZExtValue();
263 if (Value == 0) {
264 unsigned Opcode = STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
265 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DstReg).addImm(Val: 0);
266 } else {
267 assert(Value == 1);
268 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: TRI.getExec());
269 }
270 I.eraseFromParent();
271 return RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI);
272 }
273
274 // RegBankLegalize ensures that SrcReg is bool in reg (high bits are 0).
275 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC).addReg(RegNo: SrcReg);
276
277 unsigned SelectOpcode =
278 STI.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
279 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
280 .addReg(RegNo: TRI.getExec())
281 .addImm(Val: 0);
282
283 I.eraseFromParent();
284 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
285 return true;
286}
287
288bool AMDGPUInstructionSelector::selectReadAnyLane(MachineInstr &I) const {
289 Register DstReg = I.getOperand(i: 0).getReg();
290 Register SrcReg = I.getOperand(i: 1).getReg();
291
292 const DebugLoc &DL = I.getDebugLoc();
293 MachineBasicBlock *BB = I.getParent();
294
295 auto RFL = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
296 .addReg(RegNo: SrcReg);
297
298 I.eraseFromParent();
299 constrainSelectedInstRegOperands(I&: *RFL, TII, TRI, RBI);
300 return true;
301}
302
303bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
304 const Register DefReg = I.getOperand(i: 0).getReg();
305 const LLT DefTy = MRI->getType(Reg: DefReg);
306
307 // S1 G_PHIs should not be selected in instruction-select, instead:
308 // - divergent S1 G_PHI should go through lane mask merging algorithm
309 // and be fully inst-selected in AMDGPUGlobalISelDivergenceLowering
310 // - uniform S1 G_PHI should be lowered into S32 G_PHI in AMDGPURegBankSelect
311 if (DefTy == LLT::scalar(SizeInBits: 1))
312 return false;
313
314 // TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy)
315
316 const RegClassOrRegBank &RegClassOrBank =
317 MRI->getRegClassOrRegBank(Reg: DefReg);
318
319 const TargetRegisterClass *DefRC =
320 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
321 if (!DefRC) {
322 if (!DefTy.isValid()) {
323 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
324 return false;
325 }
326
327 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
328 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, Bank: RB);
329 if (!DefRC) {
330 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
331 return false;
332 }
333 }
334
335 // If inputs have register bank, assign corresponding reg class.
336 // Note: registers don't need to have the same reg bank.
337 for (unsigned i = 1; i != I.getNumOperands(); i += 2) {
338 const Register SrcReg = I.getOperand(i).getReg();
339
340 const RegisterBank *RB = MRI->getRegBankOrNull(Reg: SrcReg);
341 if (RB) {
342 const LLT SrcTy = MRI->getType(Reg: SrcReg);
343 const TargetRegisterClass *SrcRC =
344 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *RB);
345 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
346 return false;
347 }
348 }
349
350 I.setDesc(TII.get(Opcode: TargetOpcode::PHI));
351 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
352}
353
354MachineOperand
355AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
356 const TargetRegisterClass &SubRC,
357 unsigned SubIdx) const {
358
359 MachineInstr *MI = MO.getParent();
360 MachineBasicBlock *BB = MO.getParent()->getParent();
361 Register DstReg = MRI->createVirtualRegister(RegClass: &SubRC);
362
363 if (MO.isReg()) {
364 unsigned ComposedSubIdx = TRI.composeSubRegIndices(a: MO.getSubReg(), b: SubIdx);
365 Register Reg = MO.getReg();
366 BuildMI(BB&: *BB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
367 .addReg(RegNo: Reg, Flags: {}, SubReg: ComposedSubIdx);
368
369 return MachineOperand::CreateReg(Reg: DstReg, isDef: MO.isDef(), isImp: MO.isImplicit(),
370 isKill: MO.isKill(), isDead: MO.isDead(), isUndef: MO.isUndef(),
371 isEarlyClobber: MO.isEarlyClobber(), SubReg: 0, isDebug: MO.isDebug(),
372 isInternalRead: MO.isInternalRead());
373 }
374
375 assert(MO.isImm());
376
377 APInt Imm(64, MO.getImm());
378
379 switch (SubIdx) {
380 default:
381 llvm_unreachable("do not know to split immediate with this sub index.");
382 case AMDGPU::sub0:
383 return MachineOperand::CreateImm(Val: Imm.getLoBits(numBits: 32).getSExtValue());
384 case AMDGPU::sub1:
385 return MachineOperand::CreateImm(Val: Imm.getHiBits(numBits: 32).getSExtValue());
386 }
387}
388
389static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
390 switch (Opc) {
391 case AMDGPU::G_AND:
392 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
393 case AMDGPU::G_OR:
394 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32;
395 case AMDGPU::G_XOR:
396 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32;
397 default:
398 llvm_unreachable("not a bit op");
399 }
400}
401
402bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
403 Register DstReg = I.getOperand(i: 0).getReg();
404 unsigned Size = RBI.getSizeInBits(Reg: DstReg, MRI: *MRI, TRI);
405
406 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
407 if (DstRB->getID() != AMDGPU::SGPRRegBankID &&
408 DstRB->getID() != AMDGPU::VCCRegBankID)
409 return false;
410
411 bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID &&
412 STI.isWave64());
413 I.setDesc(TII.get(Opcode: getLogicalBitOpcode(Opc: I.getOpcode(), Is64)));
414
415 // Dead implicit-def of scc
416 I.addOperand(Op: MachineOperand::CreateReg(Reg: AMDGPU::SCC, isDef: true, // isDef
417 isImp: true, // isImp
418 isKill: false, // isKill
419 isDead: true)); // isDead
420 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
421 return true;
422}
423
424bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
425 MachineBasicBlock *BB = I.getParent();
426 MachineFunction *MF = BB->getParent();
427 Register DstReg = I.getOperand(i: 0).getReg();
428 const DebugLoc &DL = I.getDebugLoc();
429 LLT Ty = MRI->getType(Reg: DstReg);
430 if (Ty.isVector())
431 return false;
432
433 unsigned Size = Ty.getSizeInBits();
434 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
435 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID;
436 const bool Sub = I.getOpcode() == TargetOpcode::G_SUB;
437
438 if (Size == 32) {
439 if (IsSALU) {
440 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
441 MachineInstr *Add =
442 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
443 .add(MO: I.getOperand(i: 1))
444 .add(MO: I.getOperand(i: 2))
445 .setOperandDead(3); // Dead scc
446 I.eraseFromParent();
447 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
448 return true;
449 }
450
451 if (STI.hasAddNoCarryInsts()) {
452 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64;
453 I.setDesc(TII.get(Opcode: Opc));
454 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
455 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
456 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
457 return true;
458 }
459
460 const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64;
461
462 Register UnusedCarry = MRI->createVirtualRegister(RegClass: TRI.getWaveMaskRegClass());
463 MachineInstr *Add
464 = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
465 .addDef(RegNo: UnusedCarry, Flags: RegState::Dead)
466 .add(MO: I.getOperand(i: 1))
467 .add(MO: I.getOperand(i: 2))
468 .addImm(Val: 0);
469 I.eraseFromParent();
470 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
471 return true;
472 }
473
474 assert(!Sub && "illegal sub should not reach here");
475
476 const TargetRegisterClass &RC
477 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass;
478 const TargetRegisterClass &HalfRC
479 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass;
480
481 MachineOperand Lo1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
482 MachineOperand Lo2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
483 MachineOperand Hi1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
484 MachineOperand Hi2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
485
486 Register DstLo = MRI->createVirtualRegister(RegClass: &HalfRC);
487 Register DstHi = MRI->createVirtualRegister(RegClass: &HalfRC);
488
489 if (IsSALU) {
490 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_U32), DestReg: DstLo)
491 .add(MO: Lo1)
492 .add(MO: Lo2);
493 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADDC_U32), DestReg: DstHi)
494 .add(MO: Hi1)
495 .add(MO: Hi2)
496 .setOperandDead(3); // Dead scc
497 } else {
498 const TargetRegisterClass *CarryRC = TRI.getWaveMaskRegClass();
499 Register CarryReg = MRI->createVirtualRegister(RegClass: CarryRC);
500 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: DstLo)
501 .addDef(RegNo: CarryReg)
502 .add(MO: Lo1)
503 .add(MO: Lo2)
504 .addImm(Val: 0);
505 MachineInstr *Addc = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: DstHi)
506 .addDef(RegNo: MRI->createVirtualRegister(RegClass: CarryRC), Flags: RegState::Dead)
507 .add(MO: Hi1)
508 .add(MO: Hi2)
509 .addReg(RegNo: CarryReg, Flags: RegState::Kill)
510 .addImm(Val: 0);
511
512 constrainSelectedInstRegOperands(I&: *Addc, TII, TRI, RBI);
513 }
514
515 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
516 .addReg(RegNo: DstLo)
517 .addImm(Val: AMDGPU::sub0)
518 .addReg(RegNo: DstHi)
519 .addImm(Val: AMDGPU::sub1);
520
521
522 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
523 return false;
524
525 I.eraseFromParent();
526 return true;
527}
528
529bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE(
530 MachineInstr &I) const {
531 MachineBasicBlock *BB = I.getParent();
532 MachineFunction *MF = BB->getParent();
533 const DebugLoc &DL = I.getDebugLoc();
534 Register Dst0Reg = I.getOperand(i: 0).getReg();
535 Register Dst1Reg = I.getOperand(i: 1).getReg();
536 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO ||
537 I.getOpcode() == AMDGPU::G_UADDE;
538 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE ||
539 I.getOpcode() == AMDGPU::G_USUBE;
540
541 if (isVCC(Reg: Dst1Reg, MRI: *MRI)) {
542 unsigned NoCarryOpc =
543 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
544 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
545 I.setDesc(TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc));
546 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
547 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
548 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
549 return true;
550 }
551
552 Register Src0Reg = I.getOperand(i: 2).getReg();
553 Register Src1Reg = I.getOperand(i: 3).getReg();
554
555 if (HasCarryIn) {
556 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
557 .addReg(RegNo: I.getOperand(i: 4).getReg());
558 }
559
560 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
561 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
562
563 auto CarryInst = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc), DestReg: Dst0Reg)
564 .add(MO: I.getOperand(i: 2))
565 .add(MO: I.getOperand(i: 3));
566
567 if (MRI->use_nodbg_empty(RegNo: Dst1Reg)) {
568 CarryInst.setOperandDead(3); // Dead scc
569 } else {
570 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst1Reg)
571 .addReg(RegNo: AMDGPU::SCC);
572 if (!MRI->getRegClassOrNull(Reg: Dst1Reg))
573 MRI->setRegClass(Reg: Dst1Reg, RC: &AMDGPU::SReg_32RegClass);
574 }
575
576 if (!RBI.constrainGenericRegister(Reg: Dst0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
577 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
578 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
579 return false;
580
581 if (HasCarryIn &&
582 !RBI.constrainGenericRegister(Reg: I.getOperand(i: 4).getReg(),
583 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
584 return false;
585
586 I.eraseFromParent();
587 return true;
588}
589
590bool AMDGPUInstructionSelector::selectG_AMDGPU_MAD_64_32(
591 MachineInstr &I) const {
592 MachineBasicBlock *BB = I.getParent();
593 MachineFunction *MF = BB->getParent();
594 const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
595 bool UseNoCarry = Subtarget->hasMadNC64_32Insts() &&
596 MRI->use_nodbg_empty(RegNo: I.getOperand(i: 1).getReg());
597
598 unsigned Opc;
599 if (Subtarget->hasMADIntraFwdBug())
600 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64
601 : AMDGPU::V_MAD_I64_I32_gfx11_e64;
602 else if (UseNoCarry)
603 Opc = IsUnsigned ? AMDGPU::V_MAD_NC_U64_U32_e64
604 : AMDGPU::V_MAD_NC_I64_I32_e64;
605 else
606 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64;
607
608 if (UseNoCarry)
609 I.removeOperand(OpNo: 1);
610
611 I.setDesc(TII.get(Opcode: Opc));
612 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
613 I.addImplicitDefUseOperands(MF&: *MF);
614 I.getOperand(i: 0).setIsEarlyClobber(true);
615 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
616 return true;
617}
618
619// TODO: We should probably legalize these to only using 32-bit results.
620bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
621 MachineBasicBlock *BB = I.getParent();
622 Register DstReg = I.getOperand(i: 0).getReg();
623 Register SrcReg = I.getOperand(i: 1).getReg();
624 LLT DstTy = MRI->getType(Reg: DstReg);
625 LLT SrcTy = MRI->getType(Reg: SrcReg);
626 const unsigned SrcSize = SrcTy.getSizeInBits();
627 unsigned DstSize = DstTy.getSizeInBits();
628
629 // TODO: Should handle any multiple of 32 offset.
630 unsigned Offset = I.getOperand(i: 2).getImm();
631 if (Offset % 32 != 0 || DstSize > 128)
632 return false;
633
634 // 16-bit operations really use 32-bit registers.
635 // FIXME: Probably should not allow 16-bit G_EXTRACT results.
636 if (DstSize == 16)
637 DstSize = 32;
638
639 const TargetRegisterClass *DstRC =
640 TRI.getConstrainedRegClassForOperand(MO: I.getOperand(i: 0), MRI: *MRI);
641 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
642 return false;
643
644 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
645 const TargetRegisterClass *SrcRC =
646 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
647 if (!SrcRC)
648 return false;
649 unsigned SubReg = SIRegisterInfo::getSubRegFromChannel(Channel: Offset / 32,
650 NumRegs: DstSize / 32);
651 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubReg);
652 if (!SrcRC)
653 return false;
654
655 SrcReg = constrainOperandRegClass(MF: *MF, TRI, MRI&: *MRI, TII, RBI, InsertPt&: I,
656 RegClass: *SrcRC, RegMO&: I.getOperand(i: 1));
657 const DebugLoc &DL = I.getDebugLoc();
658 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
659 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
660
661 I.eraseFromParent();
662 return true;
663}
664
665bool AMDGPUInstructionSelector::selectS16MergeToS32(MachineInstr &MI) const {
666 Register Dst = MI.getOperand(i: 0).getReg();
667 Register Src0 = MI.getOperand(i: 1).getReg();
668 Register Src1 = MI.getOperand(i: 2).getReg();
669
670 LLT Src0Ty = MRI->getType(Reg: Src0);
671 LLT Src1Ty = MRI->getType(Reg: Src1);
672
673 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
674 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0, MRI: *MRI, TRI);
675 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1, MRI: *MRI, TRI);
676 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
677
678 Register ShiftSrc0;
679 Register ShiftSrc1;
680
681 const DebugLoc &DL = MI.getDebugLoc();
682 MachineBasicBlock *BB = MI.getParent();
683
684 // VGPR case
685 if (IsVector) {
686 // If source are both VGPR16, use REG_SEQUENCE with lo16/hi16 subregisters
687 if (Src0Bank->getID() == AMDGPU::VGPRRegBankID &&
688 Src1Bank->getID() == AMDGPU::VGPRRegBankID &&
689 Src0Ty == LLT::scalar(SizeInBits: 16) && Src1Ty == LLT::scalar(SizeInBits: 16)) {
690 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
691 .addReg(RegNo: Src0)
692 .addImm(Val: AMDGPU::lo16)
693 .addReg(RegNo: Src1)
694 .addImm(Val: AMDGPU::hi16);
695
696 if (!RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
697 return false;
698
699 MI.eraseFromParent();
700 return true;
701 }
702
703 // Otherwise, use V_LSHL_OR_B32_e64
704 Register TmpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
705 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: TmpReg)
706 .addImm(Val: 0xFFFF)
707 .addReg(RegNo: Src0);
708 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
709
710 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: Dst)
711 .addReg(RegNo: Src1)
712 .addImm(Val: 16)
713 .addReg(RegNo: TmpReg);
714 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
715
716 MI.eraseFromParent();
717 return true;
718 }
719
720 // SGPR case -> S_PACK_*_B32_B16
721 // With multiple uses of the shift, this will duplicate the shift and
722 // increase register pressure.
723 //
724 // (merge (lshr_oneuse $src0, 16), (lshr_oneuse $src1, 16)
725 // => (S_PACK_HH_B32_B16 $src0, $src1)
726 // (merge (lshr_oneuse SReg_32:$src0, 16), $src1)
727 // => (S_PACK_HL_B32_B16 $src0, $src1)
728 // (merge $src0, (lshr_oneuse SReg_32:$src1, 16))
729 // => (S_PACK_LH_B32_B16 $src0, $src1)
730 // (merge $src0, $src1)
731 // => (S_PACK_LL_B32_B16 $src0, $src1)
732
733 bool Shift0 = mi_match(
734 R: Src0, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc0), R: m_SpecificICst(RequestedValue: 16))));
735
736 bool Shift1 = mi_match(
737 R: Src1, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc1), R: m_SpecificICst(RequestedValue: 16))));
738
739 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16;
740 if (Shift0 && Shift1) {
741 Opc = AMDGPU::S_PACK_HH_B32_B16;
742 MI.getOperand(i: 1).setReg(ShiftSrc0);
743 MI.getOperand(i: 2).setReg(ShiftSrc1);
744 } else if (Shift1) {
745 Opc = AMDGPU::S_PACK_LH_B32_B16;
746 MI.getOperand(i: 2).setReg(ShiftSrc1);
747 } else if (Shift0) {
748 auto ConstSrc1 =
749 getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
750 if (ConstSrc1 && ConstSrc1->Value == 0) {
751 // build_vector_trunc (lshr $src0, 16), 0 -> s_lshr_b32 $src0, 16
752 auto MIB = BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: Dst)
753 .addReg(RegNo: ShiftSrc0)
754 .addImm(Val: 16)
755 .setOperandDead(3); // Dead scc
756
757 MI.eraseFromParent();
758 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
759 return true;
760 }
761 if (STI.hasSPackHL()) {
762 Opc = AMDGPU::S_PACK_HL_B32_B16;
763 MI.getOperand(i: 1).setReg(ShiftSrc0);
764 }
765 }
766
767 MI.setDesc(TII.get(Opcode: Opc));
768 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
769 return true;
770}
771
772// Pack each pair of s16 into an s32 with S_PACK_LL_B32_B16, then combine the
773// s32 pieces into the destination with a REG_SEQUENCE.
774bool AMDGPUInstructionSelector::selectS16MergeToWide(MachineInstr &MI) const {
775 MachineBasicBlock *BB = MI.getParent();
776 const DebugLoc &DL = MI.getDebugLoc();
777 Register DstReg = MI.getOperand(i: 0).getReg();
778 const unsigned DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
779 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
780 const unsigned NumSrc = MI.getNumOperands() - 1;
781
782 // Pack each pair of s16 sources into an s32.
783 SmallVector<Register, 8> S32Regs;
784 for (unsigned I = 0; I != NumSrc; I += 2) {
785 Register S32 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
786 auto Pack = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_PACK_LL_B32_B16), DestReg: S32)
787 .addReg(RegNo: MI.getOperand(i: I + 1).getReg())
788 .addReg(RegNo: MI.getOperand(i: I + 2).getReg());
789 constrainSelectedInstRegOperands(I&: *Pack, TII, TRI, RBI);
790 S32Regs.push_back(Elt: S32);
791 }
792
793 // Combine the s32 pieces into the destination with a REG_SEQUENCE.
794 const TargetRegisterClass *DstRC =
795 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
796 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
797 return false;
798 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, /*EltSize=*/4);
799 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
800 for (unsigned I = 0, E = S32Regs.size(); I != E; ++I)
801 MIB.addReg(RegNo: S32Regs[I]).addImm(Val: SubRegs[I]);
802
803 MI.eraseFromParent();
804 return true;
805}
806
807bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
808 MachineBasicBlock *BB = MI.getParent();
809 Register DstReg = MI.getOperand(i: 0).getReg();
810 LLT DstTy = MRI->getType(Reg: DstReg);
811 LLT SrcTy = MRI->getType(Reg: MI.getOperand(i: 1).getReg());
812
813 const unsigned SrcSize = SrcTy.getSizeInBits();
814 if (SrcSize < 32) {
815 // Handle s32 <- G_MERGE_VALUES s16, s16
816 if (SrcSize == 16 && DstTy.getSizeInBits() == 32 &&
817 MI.getNumOperands() == 3) {
818 return selectS16MergeToS32(MI);
819 }
820 // With true16 a scalar s16 is a register type, so a scalar wider than 32
821 // bits can be built from s16 pieces.
822 bool IsWideS16Merge = SrcSize == 16 && DstTy.getSizeInBits() > 32 &&
823 DstTy.getSizeInBits() % 32 == 0;
824
825 // SGPRs have no 16-bit subregisters, so pack pairs of s16 with S_PACK.
826 if (IsWideS16Merge &&
827 RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
828 return selectS16MergeToWide(MI);
829
830 // A VGPR wide s16 merge falls through to the generic path below.
831 if (!IsWideS16Merge)
832 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
833 }
834
835 const DebugLoc &DL = MI.getDebugLoc();
836 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
837 const unsigned DstSize = DstTy.getSizeInBits();
838 const TargetRegisterClass *DstRC =
839 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
840 if (!DstRC)
841 return false;
842
843 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, EltSize: SrcSize / 8);
844 MachineInstrBuilder MIB =
845 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
846 for (int I = 0, E = MI.getNumOperands() - 1; I != E; ++I) {
847 MachineOperand &Src = MI.getOperand(i: I + 1);
848 MIB.addReg(RegNo: Src.getReg(), Flags: getUndefRegState(B: Src.isUndef()));
849 MIB.addImm(Val: SubRegs[I]);
850
851 const TargetRegisterClass *SrcRC
852 = TRI.getConstrainedRegClassForOperand(MO: Src, MRI: *MRI);
853 if (SrcRC && !RBI.constrainGenericRegister(Reg: Src.getReg(), RC: *SrcRC, MRI&: *MRI))
854 return false;
855 }
856
857 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
858 return false;
859
860 MI.eraseFromParent();
861 return true;
862}
863
864bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
865 MachineBasicBlock *BB = MI.getParent();
866 const int NumDst = MI.getNumOperands() - 1;
867
868 MachineOperand &Src = MI.getOperand(i: NumDst);
869
870 Register SrcReg = Src.getReg();
871 Register DstReg0 = MI.getOperand(i: 0).getReg();
872 LLT DstTy = MRI->getType(Reg: DstReg0);
873 LLT SrcTy = MRI->getType(Reg: SrcReg);
874
875 const unsigned DstSize = DstTy.getSizeInBits();
876 const unsigned SrcSize = SrcTy.getSizeInBits();
877 const DebugLoc &DL = MI.getDebugLoc();
878 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
879
880 const TargetRegisterClass *SrcRC =
881 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
882 if (!SrcRC || !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
883 return false;
884
885 // Note we could have mixed SGPR and VGPR destination banks for an SGPR
886 // source, and this relies on the fact that the same subregister indices are
887 // used for both.
888 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SrcRC, EltSize: DstSize / 8);
889 for (int I = 0, E = NumDst; I != E; ++I) {
890 MachineOperand &Dst = MI.getOperand(i: I);
891 // hi16:sreg_32 is not allowed so explicitly shift upper 16-bits.
892 if (SrcBank->getID() == AMDGPU::SGPRRegBankID &&
893 SubRegs[I] == AMDGPU::hi16) {
894 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: Dst.getReg())
895 .addReg(RegNo: SrcReg)
896 .addImm(Val: 16);
897 } else {
898 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: Dst.getReg())
899 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegs[I]);
900 }
901
902 // Make sure the subregister index is valid for the source register.
903 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[I]);
904 if (!SrcRC || !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
905 return false;
906
907 const TargetRegisterClass *DstRC =
908 TRI.getConstrainedRegClassForOperand(MO: Dst, MRI: *MRI);
909 if (DstRC && !RBI.constrainGenericRegister(Reg: Dst.getReg(), RC: *DstRC, MRI&: *MRI))
910 return false;
911 }
912
913 MI.eraseFromParent();
914 return true;
915}
916
917bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR(MachineInstr &MI) const {
918 assert(MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC ||
919 MI.getOpcode() == AMDGPU::G_BUILD_VECTOR);
920
921 Register Src0 = MI.getOperand(i: 1).getReg();
922 Register Src1 = MI.getOperand(i: 2).getReg();
923 LLT SrcTy = MRI->getType(Reg: Src0);
924 const unsigned SrcSize = SrcTy.getSizeInBits();
925
926 // BUILD_VECTOR with >=32 bits source is handled by MERGE_VALUE.
927 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR && SrcSize >= 32) {
928 return selectG_MERGE_VALUES(MI);
929 }
930
931 // Selection logic below is for V2S16 only.
932 // For G_BUILD_VECTOR_TRUNC, additionally check that the operands are s32.
933 Register Dst = MI.getOperand(i: 0).getReg();
934 if (MRI->getType(Reg: Dst) != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
935 (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC &&
936 SrcTy != LLT::scalar(SizeInBits: 32)))
937 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
938
939 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
940 if (DstBank->getID() == AMDGPU::AGPRRegBankID)
941 return false;
942
943 assert(DstBank->getID() == AMDGPU::SGPRRegBankID ||
944 DstBank->getID() == AMDGPU::VGPRRegBankID);
945 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
946
947 const DebugLoc &DL = MI.getDebugLoc();
948 MachineBasicBlock *BB = MI.getParent();
949
950 // First, before trying TableGen patterns, check if both sources are
951 // constants. In those cases, we can trivially compute the final constant
952 // and emit a simple move.
953 auto ConstSrc1 = getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
954 if (ConstSrc1) {
955 auto ConstSrc0 =
956 getAnyConstantVRegValWithLookThrough(VReg: Src0, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
957 if (ConstSrc0) {
958 const int64_t K0 = ConstSrc0->Value.getSExtValue();
959 const int64_t K1 = ConstSrc1->Value.getSExtValue();
960 uint32_t Lo16 = static_cast<uint32_t>(K0) & 0xffff;
961 uint32_t Hi16 = static_cast<uint32_t>(K1) & 0xffff;
962 uint32_t Imm = Lo16 | (Hi16 << 16);
963
964 // VALU
965 if (IsVector) {
966 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: Dst).addImm(Val: Imm);
967 MI.eraseFromParent();
968 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI);
969 }
970
971 // SALU
972 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: Dst).addImm(Val: Imm);
973 MI.eraseFromParent();
974 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
975 }
976 }
977
978 // Now try TableGen patterns.
979 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
980 return true;
981
982 // TODO: This should probably be a combine somewhere
983 // (build_vector $src0, undef) -> copy $src0
984 MachineInstr *Src1Def = getDefIgnoringCopies(Reg: Src1, MRI: *MRI);
985 if (Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) {
986 MI.setDesc(TII.get(Opcode: AMDGPU::COPY));
987 MI.removeOperand(OpNo: 2);
988 const auto &RC =
989 IsVector ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
990 return RBI.constrainGenericRegister(Reg: Dst, RC, MRI&: *MRI) &&
991 RBI.constrainGenericRegister(Reg: Src0, RC, MRI&: *MRI);
992 }
993
994 return selectS16MergeToS32(MI);
995}
996
997bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const {
998 const MachineOperand &MO = I.getOperand(i: 0);
999
1000 // FIXME: Interface for getConstrainedRegClassForOperand needs work. The
1001 // regbank check here is to know why getConstrainedRegClassForOperand failed.
1002 const TargetRegisterClass *RC = TRI.getConstrainedRegClassForOperand(MO, MRI: *MRI);
1003 if ((!RC && !MRI->getRegBankOrNull(Reg: MO.getReg())) ||
1004 (RC && RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI))) {
1005 I.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1006 return true;
1007 }
1008
1009 return false;
1010}
1011
1012bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
1013 MachineBasicBlock *BB = I.getParent();
1014
1015 Register DstReg = I.getOperand(i: 0).getReg();
1016 Register Src0Reg = I.getOperand(i: 1).getReg();
1017 Register Src1Reg = I.getOperand(i: 2).getReg();
1018 LLT Src1Ty = MRI->getType(Reg: Src1Reg);
1019
1020 unsigned DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
1021 unsigned InsSize = Src1Ty.getSizeInBits();
1022
1023 int64_t Offset = I.getOperand(i: 3).getImm();
1024
1025 // FIXME: These cases should have been illegal and unnecessary to check here.
1026 if (Offset % 32 != 0 || InsSize % 32 != 0)
1027 return false;
1028
1029 // Currently not handled by getSubRegFromChannel.
1030 if (InsSize > 128)
1031 return false;
1032
1033 unsigned SubReg = TRI.getSubRegFromChannel(Channel: Offset / 32, NumRegs: InsSize / 32);
1034 if (SubReg == AMDGPU::NoSubRegister)
1035 return false;
1036
1037 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1038 const TargetRegisterClass *DstRC =
1039 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
1040 if (!DstRC)
1041 return false;
1042
1043 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0Reg, MRI: *MRI, TRI);
1044 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1Reg, MRI: *MRI, TRI);
1045 const TargetRegisterClass *Src0RC =
1046 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *Src0Bank);
1047 const TargetRegisterClass *Src1RC =
1048 TRI.getRegClassForSizeOnBank(Size: InsSize, Bank: *Src1Bank);
1049
1050 // Deal with weird cases where the class only partially supports the subreg
1051 // index.
1052 Src0RC = TRI.getSubClassWithSubReg(Src0RC, SubReg);
1053 if (!Src0RC || !Src1RC)
1054 return false;
1055
1056 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
1057 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: *Src0RC, MRI&: *MRI) ||
1058 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: *Src1RC, MRI&: *MRI))
1059 return false;
1060
1061 const DebugLoc &DL = I.getDebugLoc();
1062 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1063 .addReg(RegNo: Src0Reg)
1064 .addReg(RegNo: Src1Reg)
1065 .addImm(Val: SubReg);
1066
1067 I.eraseFromParent();
1068 return true;
1069}
1070
1071bool AMDGPUInstructionSelector::selectG_SBFX_UBFX(MachineInstr &MI) const {
1072 Register DstReg = MI.getOperand(i: 0).getReg();
1073 Register SrcReg = MI.getOperand(i: 1).getReg();
1074 Register OffsetReg = MI.getOperand(i: 2).getReg();
1075 Register WidthReg = MI.getOperand(i: 3).getReg();
1076
1077 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID &&
1078 "scalar BFX instructions are expanded in regbankselect");
1079 assert(MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 &&
1080 "64-bit vector BFX instructions are expanded in regbankselect");
1081
1082 const DebugLoc &DL = MI.getDebugLoc();
1083 MachineBasicBlock *MBB = MI.getParent();
1084
1085 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SBFX;
1086 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
1087 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
1088 .addReg(RegNo: SrcReg)
1089 .addReg(RegNo: OffsetReg)
1090 .addReg(RegNo: WidthReg);
1091 MI.eraseFromParent();
1092 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1093 return true;
1094}
1095
1096bool AMDGPUInstructionSelector::selectInterpP1F16(MachineInstr &MI) const {
1097 if (STI.getLDSBankCount() != 16)
1098 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1099
1100 Register Dst = MI.getOperand(i: 0).getReg();
1101 Register Src0 = MI.getOperand(i: 2).getReg();
1102 Register M0Val = MI.getOperand(i: 6).getReg();
1103 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
1104 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI) ||
1105 !RBI.constrainGenericRegister(Reg: Src0, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
1106 return false;
1107
1108 // This requires 2 instructions. It is possible to write a pattern to support
1109 // this, but the generated isel emitter doesn't correctly deal with multiple
1110 // output instructions using the same physical register input. The copy to m0
1111 // is incorrectly placed before the second instruction.
1112 //
1113 // TODO: Match source modifiers.
1114
1115 Register InterpMov = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1116 const DebugLoc &DL = MI.getDebugLoc();
1117 MachineBasicBlock *MBB = MI.getParent();
1118
1119 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1120 .addReg(RegNo: M0Val);
1121 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_MOV_F32), DestReg: InterpMov)
1122 .addImm(Val: 2)
1123 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1124 .addImm(Val: MI.getOperand(i: 3).getImm()); // $attrchan
1125
1126 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_P1LV_F16), DestReg: Dst)
1127 .addImm(Val: 0) // $src0_modifiers
1128 .addReg(RegNo: Src0) // $src0
1129 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1130 .addImm(Val: MI.getOperand(i: 3).getImm()) // $attrchan
1131 .addImm(Val: 0) // $src2_modifiers
1132 .addReg(RegNo: InterpMov) // $src2 - 2 f16 values selected by high
1133 .addImm(Val: MI.getOperand(i: 5).getImm()) // $high
1134 .addImm(Val: 0) // $clamp
1135 .addImm(Val: 0); // $omod
1136
1137 MI.eraseFromParent();
1138 return true;
1139}
1140
1141// Writelane is special in that it can use SGPR and M0 (which would normally
1142// count as using the constant bus twice - but in this case it is allowed since
1143// the lane selector doesn't count as a use of the constant bus). However, it is
1144// still required to abide by the 1 SGPR rule. Fix this up if we might have
1145// multiple SGPRs.
1146bool AMDGPUInstructionSelector::selectWritelane(MachineInstr &MI) const {
1147 // With a constant bus limit of at least 2, there's no issue.
1148 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_WRITELANE_B32) > 1)
1149 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1150
1151 MachineBasicBlock *MBB = MI.getParent();
1152 const DebugLoc &DL = MI.getDebugLoc();
1153 Register VDst = MI.getOperand(i: 0).getReg();
1154 Register Val = MI.getOperand(i: 2).getReg();
1155 Register LaneSelect = MI.getOperand(i: 3).getReg();
1156 Register VDstIn = MI.getOperand(i: 4).getReg();
1157
1158 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_WRITELANE_B32), DestReg: VDst);
1159
1160 std::optional<ValueAndVReg> ConstSelect =
1161 getIConstantVRegValWithLookThrough(VReg: LaneSelect, MRI: *MRI);
1162 if (ConstSelect) {
1163 // The selector has to be an inline immediate, so we can use whatever for
1164 // the other operands.
1165 MIB.addReg(RegNo: Val);
1166 MIB.addImm(Val: ConstSelect->Value.getSExtValue() &
1167 maskTrailingOnes<uint64_t>(N: STI.getWavefrontSizeLog2()));
1168 } else {
1169 std::optional<ValueAndVReg> ConstVal =
1170 getIConstantVRegValWithLookThrough(VReg: Val, MRI: *MRI);
1171
1172 // If the value written is an inline immediate, we can get away without a
1173 // copy to m0.
1174 if (ConstVal && AMDGPU::isInlinableLiteral32(Literal: ConstVal->Value.getSExtValue(),
1175 HasInv2Pi: STI.hasInv2PiInlineImm())) {
1176 MIB.addImm(Val: ConstVal->Value.getSExtValue());
1177 MIB.addReg(RegNo: LaneSelect);
1178 } else {
1179 MIB.addReg(RegNo: Val);
1180
1181 // If the lane selector was originally in a VGPR and copied with
1182 // readfirstlane, there's a hazard to read the same SGPR from the
1183 // VALU. Constrain to a different SGPR to help avoid needing a nop later.
1184 RBI.constrainGenericRegister(Reg: LaneSelect, RC: AMDGPU::SReg_32_XM0RegClass, MRI&: *MRI);
1185
1186 BuildMI(BB&: *MBB, I&: *MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1187 .addReg(RegNo: LaneSelect);
1188 MIB.addReg(RegNo: AMDGPU::M0);
1189 }
1190 }
1191
1192 MIB.addReg(RegNo: VDstIn);
1193
1194 MI.eraseFromParent();
1195 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1196 return true;
1197}
1198
1199// We need to handle this here because tablegen doesn't support matching
1200// instructions with multiple outputs.
1201bool AMDGPUInstructionSelector::selectDivScale(MachineInstr &MI) const {
1202 Register Dst0 = MI.getOperand(i: 0).getReg();
1203 Register Dst1 = MI.getOperand(i: 1).getReg();
1204
1205 LLT Ty = MRI->getType(Reg: Dst0);
1206 unsigned Opc;
1207 if (Ty == LLT::scalar(SizeInBits: 32))
1208 Opc = AMDGPU::V_DIV_SCALE_F32_e64;
1209 else if (Ty == LLT::scalar(SizeInBits: 64))
1210 Opc = AMDGPU::V_DIV_SCALE_F64_e64;
1211 else
1212 return false;
1213
1214 // TODO: Match source modifiers.
1215
1216 const DebugLoc &DL = MI.getDebugLoc();
1217 MachineBasicBlock *MBB = MI.getParent();
1218
1219 Register Numer = MI.getOperand(i: 3).getReg();
1220 Register Denom = MI.getOperand(i: 4).getReg();
1221 unsigned ChooseDenom = MI.getOperand(i: 5).getImm();
1222
1223 Register Src0 = ChooseDenom != 0 ? Numer : Denom;
1224
1225 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
1226 .addDef(RegNo: Dst1)
1227 .addImm(Val: 0) // $src0_modifiers
1228 .addUse(RegNo: Src0) // $src0
1229 .addImm(Val: 0) // $src1_modifiers
1230 .addUse(RegNo: Denom) // $src1
1231 .addImm(Val: 0) // $src2_modifiers
1232 .addUse(RegNo: Numer) // $src2
1233 .addImm(Val: 0) // $clamp
1234 .addImm(Val: 0); // $omod
1235
1236 MI.eraseFromParent();
1237 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1238 return true;
1239}
1240
1241bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
1242 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
1243 switch (IntrinsicID) {
1244 case Intrinsic::amdgcn_if_break: {
1245 MachineBasicBlock *BB = I.getParent();
1246
1247 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1248 // SelectionDAG uses for wave32 vs wave64.
1249 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_IF_BREAK))
1250 .add(MO: I.getOperand(i: 0))
1251 .add(MO: I.getOperand(i: 2))
1252 .add(MO: I.getOperand(i: 3));
1253
1254 Register DstReg = I.getOperand(i: 0).getReg();
1255 Register Src0Reg = I.getOperand(i: 2).getReg();
1256 Register Src1Reg = I.getOperand(i: 3).getReg();
1257
1258 I.eraseFromParent();
1259
1260 for (Register Reg : { DstReg, Src0Reg, Src1Reg })
1261 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1262
1263 return true;
1264 }
1265 case Intrinsic::amdgcn_interp_p1_f16:
1266 return selectInterpP1F16(MI&: I);
1267 case Intrinsic::amdgcn_wqm:
1268 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::WQM);
1269 case Intrinsic::amdgcn_softwqm:
1270 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::SOFT_WQM);
1271 case Intrinsic::amdgcn_strict_wwm:
1272 case Intrinsic::amdgcn_wwm:
1273 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WWM);
1274 case Intrinsic::amdgcn_strict_wqm:
1275 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WQM);
1276 case Intrinsic::amdgcn_writelane:
1277 return selectWritelane(MI&: I);
1278 case Intrinsic::amdgcn_div_scale:
1279 return selectDivScale(MI&: I);
1280 case Intrinsic::amdgcn_icmp:
1281 case Intrinsic::amdgcn_fcmp:
1282 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
1283 return true;
1284 return selectIntrinsicCmp(MI&: I);
1285 case Intrinsic::amdgcn_ballot:
1286 return selectBallot(I);
1287 case Intrinsic::amdgcn_reloc_constant:
1288 return selectRelocConstant(I);
1289 case Intrinsic::amdgcn_groupstaticsize:
1290 return selectGroupStaticSize(I);
1291 case Intrinsic::returnaddress:
1292 return selectReturnAddress(I);
1293 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
1294 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
1295 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
1296 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
1297 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
1298 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
1299 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
1300 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
1301 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
1302 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
1303 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
1304 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
1305 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
1306 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
1307 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
1308 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
1309 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
1310 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
1311 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
1312 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
1313 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
1314 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
1315 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
1316 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
1317 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
1318 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
1319 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
1320 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
1321 return selectSMFMACIntrin(I);
1322 case Intrinsic::amdgcn_permlane16_swap:
1323 case Intrinsic::amdgcn_permlane32_swap:
1324 return selectPermlaneSwapIntrin(I, IntrID: IntrinsicID);
1325 case Intrinsic::amdgcn_wave_shuffle:
1326 return selectWaveShuffleIntrin(I);
1327 default:
1328 return selectImpl(I, CoverageInfo&: *CoverageInfo);
1329 }
1330}
1331
1332static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size,
1333 const GCNSubtarget &ST) {
1334 if (Size != 16 && Size != 32 && Size != 64)
1335 return -1;
1336
1337 if (Size == 16 && !ST.has16BitInsts())
1338 return -1;
1339
1340 const auto Select = [&](unsigned S16Opc, unsigned TrueS16Opc,
1341 unsigned FakeS16Opc, unsigned S32Opc,
1342 unsigned S64Opc) {
1343 if (Size == 16)
1344 return ST.hasTrue16BitInsts()
1345 ? ST.useRealTrue16Insts() ? TrueS16Opc : FakeS16Opc
1346 : S16Opc;
1347 if (Size == 32)
1348 return S32Opc;
1349 return S64Opc;
1350 };
1351
1352 switch (P) {
1353 default:
1354 llvm_unreachable("Unknown condition code!");
1355 case CmpInst::ICMP_NE:
1356 return Select(AMDGPU::V_CMP_NE_U16_e64, AMDGPU::V_CMP_NE_U16_t16_e64,
1357 AMDGPU::V_CMP_NE_U16_fake16_e64, AMDGPU::V_CMP_NE_U32_e64,
1358 AMDGPU::V_CMP_NE_U64_e64);
1359 case CmpInst::ICMP_EQ:
1360 return Select(AMDGPU::V_CMP_EQ_U16_e64, AMDGPU::V_CMP_EQ_U16_t16_e64,
1361 AMDGPU::V_CMP_EQ_U16_fake16_e64, AMDGPU::V_CMP_EQ_U32_e64,
1362 AMDGPU::V_CMP_EQ_U64_e64);
1363 case CmpInst::ICMP_SGT:
1364 return Select(AMDGPU::V_CMP_GT_I16_e64, AMDGPU::V_CMP_GT_I16_t16_e64,
1365 AMDGPU::V_CMP_GT_I16_fake16_e64, AMDGPU::V_CMP_GT_I32_e64,
1366 AMDGPU::V_CMP_GT_I64_e64);
1367 case CmpInst::ICMP_SGE:
1368 return Select(AMDGPU::V_CMP_GE_I16_e64, AMDGPU::V_CMP_GE_I16_t16_e64,
1369 AMDGPU::V_CMP_GE_I16_fake16_e64, AMDGPU::V_CMP_GE_I32_e64,
1370 AMDGPU::V_CMP_GE_I64_e64);
1371 case CmpInst::ICMP_SLT:
1372 return Select(AMDGPU::V_CMP_LT_I16_e64, AMDGPU::V_CMP_LT_I16_t16_e64,
1373 AMDGPU::V_CMP_LT_I16_fake16_e64, AMDGPU::V_CMP_LT_I32_e64,
1374 AMDGPU::V_CMP_LT_I64_e64);
1375 case CmpInst::ICMP_SLE:
1376 return Select(AMDGPU::V_CMP_LE_I16_e64, AMDGPU::V_CMP_LE_I16_t16_e64,
1377 AMDGPU::V_CMP_LE_I16_fake16_e64, AMDGPU::V_CMP_LE_I32_e64,
1378 AMDGPU::V_CMP_LE_I64_e64);
1379 case CmpInst::ICMP_UGT:
1380 return Select(AMDGPU::V_CMP_GT_U16_e64, AMDGPU::V_CMP_GT_U16_t16_e64,
1381 AMDGPU::V_CMP_GT_U16_fake16_e64, AMDGPU::V_CMP_GT_U32_e64,
1382 AMDGPU::V_CMP_GT_U64_e64);
1383 case CmpInst::ICMP_UGE:
1384 return Select(AMDGPU::V_CMP_GE_U16_e64, AMDGPU::V_CMP_GE_U16_t16_e64,
1385 AMDGPU::V_CMP_GE_U16_fake16_e64, AMDGPU::V_CMP_GE_U32_e64,
1386 AMDGPU::V_CMP_GE_U64_e64);
1387 case CmpInst::ICMP_ULT:
1388 return Select(AMDGPU::V_CMP_LT_U16_e64, AMDGPU::V_CMP_LT_U16_t16_e64,
1389 AMDGPU::V_CMP_LT_U16_fake16_e64, AMDGPU::V_CMP_LT_U32_e64,
1390 AMDGPU::V_CMP_LT_U64_e64);
1391 case CmpInst::ICMP_ULE:
1392 return Select(AMDGPU::V_CMP_LE_U16_e64, AMDGPU::V_CMP_LE_U16_t16_e64,
1393 AMDGPU::V_CMP_LE_U16_fake16_e64, AMDGPU::V_CMP_LE_U32_e64,
1394 AMDGPU::V_CMP_LE_U64_e64);
1395
1396 case CmpInst::FCMP_OEQ:
1397 return Select(AMDGPU::V_CMP_EQ_F16_e64, AMDGPU::V_CMP_EQ_F16_t16_e64,
1398 AMDGPU::V_CMP_EQ_F16_fake16_e64, AMDGPU::V_CMP_EQ_F32_e64,
1399 AMDGPU::V_CMP_EQ_F64_e64);
1400 case CmpInst::FCMP_OGT:
1401 return Select(AMDGPU::V_CMP_GT_F16_e64, AMDGPU::V_CMP_GT_F16_t16_e64,
1402 AMDGPU::V_CMP_GT_F16_fake16_e64, AMDGPU::V_CMP_GT_F32_e64,
1403 AMDGPU::V_CMP_GT_F64_e64);
1404 case CmpInst::FCMP_OGE:
1405 return Select(AMDGPU::V_CMP_GE_F16_e64, AMDGPU::V_CMP_GE_F16_t16_e64,
1406 AMDGPU::V_CMP_GE_F16_fake16_e64, AMDGPU::V_CMP_GE_F32_e64,
1407 AMDGPU::V_CMP_GE_F64_e64);
1408 case CmpInst::FCMP_OLT:
1409 return Select(AMDGPU::V_CMP_LT_F16_e64, AMDGPU::V_CMP_LT_F16_t16_e64,
1410 AMDGPU::V_CMP_LT_F16_fake16_e64, AMDGPU::V_CMP_LT_F32_e64,
1411 AMDGPU::V_CMP_LT_F64_e64);
1412 case CmpInst::FCMP_OLE:
1413 return Select(AMDGPU::V_CMP_LE_F16_e64, AMDGPU::V_CMP_LE_F16_t16_e64,
1414 AMDGPU::V_CMP_LE_F16_fake16_e64, AMDGPU::V_CMP_LE_F32_e64,
1415 AMDGPU::V_CMP_LE_F64_e64);
1416 case CmpInst::FCMP_ONE:
1417 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1418 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1419 AMDGPU::V_CMP_NEQ_F64_e64);
1420 case CmpInst::FCMP_ORD:
1421 return Select(AMDGPU::V_CMP_O_F16_e64, AMDGPU::V_CMP_O_F16_t16_e64,
1422 AMDGPU::V_CMP_O_F16_fake16_e64, AMDGPU::V_CMP_O_F32_e64,
1423 AMDGPU::V_CMP_O_F64_e64);
1424 case CmpInst::FCMP_UNO:
1425 return Select(AMDGPU::V_CMP_U_F16_e64, AMDGPU::V_CMP_U_F16_t16_e64,
1426 AMDGPU::V_CMP_U_F16_fake16_e64, AMDGPU::V_CMP_U_F32_e64,
1427 AMDGPU::V_CMP_U_F64_e64);
1428 case CmpInst::FCMP_UEQ:
1429 return Select(AMDGPU::V_CMP_NLG_F16_e64, AMDGPU::V_CMP_NLG_F16_t16_e64,
1430 AMDGPU::V_CMP_NLG_F16_fake16_e64, AMDGPU::V_CMP_NLG_F32_e64,
1431 AMDGPU::V_CMP_NLG_F64_e64);
1432 case CmpInst::FCMP_UGT:
1433 return Select(AMDGPU::V_CMP_NLE_F16_e64, AMDGPU::V_CMP_NLE_F16_t16_e64,
1434 AMDGPU::V_CMP_NLE_F16_fake16_e64, AMDGPU::V_CMP_NLE_F32_e64,
1435 AMDGPU::V_CMP_NLE_F64_e64);
1436 case CmpInst::FCMP_UGE:
1437 return Select(AMDGPU::V_CMP_NLT_F16_e64, AMDGPU::V_CMP_NLT_F16_t16_e64,
1438 AMDGPU::V_CMP_NLT_F16_fake16_e64, AMDGPU::V_CMP_NLT_F32_e64,
1439 AMDGPU::V_CMP_NLT_F64_e64);
1440 case CmpInst::FCMP_ULT:
1441 return Select(AMDGPU::V_CMP_NGE_F16_e64, AMDGPU::V_CMP_NGE_F16_t16_e64,
1442 AMDGPU::V_CMP_NGE_F16_fake16_e64, AMDGPU::V_CMP_NGE_F32_e64,
1443 AMDGPU::V_CMP_NGE_F64_e64);
1444 case CmpInst::FCMP_ULE:
1445 return Select(AMDGPU::V_CMP_NGT_F16_e64, AMDGPU::V_CMP_NGT_F16_t16_e64,
1446 AMDGPU::V_CMP_NGT_F16_fake16_e64, AMDGPU::V_CMP_NGT_F32_e64,
1447 AMDGPU::V_CMP_NGT_F64_e64);
1448 case CmpInst::FCMP_UNE:
1449 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1450 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1451 AMDGPU::V_CMP_NEQ_F64_e64);
1452 case CmpInst::FCMP_TRUE:
1453 return Select(AMDGPU::V_CMP_TRU_F16_e64, AMDGPU::V_CMP_TRU_F16_t16_e64,
1454 AMDGPU::V_CMP_TRU_F16_fake16_e64, AMDGPU::V_CMP_TRU_F32_e64,
1455 AMDGPU::V_CMP_TRU_F64_e64);
1456 case CmpInst::FCMP_FALSE:
1457 return Select(AMDGPU::V_CMP_F_F16_e64, AMDGPU::V_CMP_F_F16_t16_e64,
1458 AMDGPU::V_CMP_F_F16_fake16_e64, AMDGPU::V_CMP_F_F32_e64,
1459 AMDGPU::V_CMP_F_F64_e64);
1460 }
1461}
1462
1463int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
1464 unsigned Size) const {
1465 if (Size == 64) {
1466 if (!STI.hasScalarCompareEq64())
1467 return -1;
1468
1469 switch (P) {
1470 case CmpInst::ICMP_NE:
1471 return AMDGPU::S_CMP_LG_U64;
1472 case CmpInst::ICMP_EQ:
1473 return AMDGPU::S_CMP_EQ_U64;
1474 default:
1475 return -1;
1476 }
1477 }
1478
1479 if (Size == 32) {
1480 switch (P) {
1481 case CmpInst::ICMP_NE:
1482 return AMDGPU::S_CMP_LG_U32;
1483 case CmpInst::ICMP_EQ:
1484 return AMDGPU::S_CMP_EQ_U32;
1485 case CmpInst::ICMP_SGT:
1486 return AMDGPU::S_CMP_GT_I32;
1487 case CmpInst::ICMP_SGE:
1488 return AMDGPU::S_CMP_GE_I32;
1489 case CmpInst::ICMP_SLT:
1490 return AMDGPU::S_CMP_LT_I32;
1491 case CmpInst::ICMP_SLE:
1492 return AMDGPU::S_CMP_LE_I32;
1493 case CmpInst::ICMP_UGT:
1494 return AMDGPU::S_CMP_GT_U32;
1495 case CmpInst::ICMP_UGE:
1496 return AMDGPU::S_CMP_GE_U32;
1497 case CmpInst::ICMP_ULT:
1498 return AMDGPU::S_CMP_LT_U32;
1499 case CmpInst::ICMP_ULE:
1500 return AMDGPU::S_CMP_LE_U32;
1501 case CmpInst::FCMP_OEQ:
1502 return AMDGPU::S_CMP_EQ_F32;
1503 case CmpInst::FCMP_OGT:
1504 return AMDGPU::S_CMP_GT_F32;
1505 case CmpInst::FCMP_OGE:
1506 return AMDGPU::S_CMP_GE_F32;
1507 case CmpInst::FCMP_OLT:
1508 return AMDGPU::S_CMP_LT_F32;
1509 case CmpInst::FCMP_OLE:
1510 return AMDGPU::S_CMP_LE_F32;
1511 case CmpInst::FCMP_ONE:
1512 return AMDGPU::S_CMP_LG_F32;
1513 case CmpInst::FCMP_ORD:
1514 return AMDGPU::S_CMP_O_F32;
1515 case CmpInst::FCMP_UNO:
1516 return AMDGPU::S_CMP_U_F32;
1517 case CmpInst::FCMP_UEQ:
1518 return AMDGPU::S_CMP_NLG_F32;
1519 case CmpInst::FCMP_UGT:
1520 return AMDGPU::S_CMP_NLE_F32;
1521 case CmpInst::FCMP_UGE:
1522 return AMDGPU::S_CMP_NLT_F32;
1523 case CmpInst::FCMP_ULT:
1524 return AMDGPU::S_CMP_NGE_F32;
1525 case CmpInst::FCMP_ULE:
1526 return AMDGPU::S_CMP_NGT_F32;
1527 case CmpInst::FCMP_UNE:
1528 return AMDGPU::S_CMP_NEQ_F32;
1529 default:
1530 llvm_unreachable("Unknown condition code!");
1531 }
1532 }
1533
1534 if (Size == 16) {
1535 if (!STI.hasSALUFloatInsts())
1536 return -1;
1537
1538 switch (P) {
1539 case CmpInst::FCMP_OEQ:
1540 return AMDGPU::S_CMP_EQ_F16;
1541 case CmpInst::FCMP_OGT:
1542 return AMDGPU::S_CMP_GT_F16;
1543 case CmpInst::FCMP_OGE:
1544 return AMDGPU::S_CMP_GE_F16;
1545 case CmpInst::FCMP_OLT:
1546 return AMDGPU::S_CMP_LT_F16;
1547 case CmpInst::FCMP_OLE:
1548 return AMDGPU::S_CMP_LE_F16;
1549 case CmpInst::FCMP_ONE:
1550 return AMDGPU::S_CMP_LG_F16;
1551 case CmpInst::FCMP_ORD:
1552 return AMDGPU::S_CMP_O_F16;
1553 case CmpInst::FCMP_UNO:
1554 return AMDGPU::S_CMP_U_F16;
1555 case CmpInst::FCMP_UEQ:
1556 return AMDGPU::S_CMP_NLG_F16;
1557 case CmpInst::FCMP_UGT:
1558 return AMDGPU::S_CMP_NLE_F16;
1559 case CmpInst::FCMP_UGE:
1560 return AMDGPU::S_CMP_NLT_F16;
1561 case CmpInst::FCMP_ULT:
1562 return AMDGPU::S_CMP_NGE_F16;
1563 case CmpInst::FCMP_ULE:
1564 return AMDGPU::S_CMP_NGT_F16;
1565 case CmpInst::FCMP_UNE:
1566 return AMDGPU::S_CMP_NEQ_F16;
1567 default:
1568 llvm_unreachable("Unknown condition code!");
1569 }
1570 }
1571
1572 return -1;
1573}
1574
1575bool AMDGPUInstructionSelector::selectG_ICMP_or_FCMP(MachineInstr &I) const {
1576
1577 MachineBasicBlock *BB = I.getParent();
1578 const DebugLoc &DL = I.getDebugLoc();
1579
1580 Register SrcReg = I.getOperand(i: 2).getReg();
1581 unsigned Size = RBI.getSizeInBits(Reg: SrcReg, MRI: *MRI, TRI);
1582
1583 auto Pred = (CmpInst::Predicate)I.getOperand(i: 1).getPredicate();
1584
1585 Register CCReg = I.getOperand(i: 0).getReg();
1586 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
1587 int Opcode = getS_CMPOpcode(P: Pred, Size);
1588 if (Opcode == -1)
1589 return false;
1590 MachineInstr *ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode))
1591 .add(MO: I.getOperand(i: 2))
1592 .add(MO: I.getOperand(i: 3));
1593 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg)
1594 .addReg(RegNo: AMDGPU::SCC);
1595 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1596 bool Ret =
1597 RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
1598 I.eraseFromParent();
1599 return Ret;
1600 }
1601
1602 if (I.getOpcode() == AMDGPU::G_FCMP)
1603 return false;
1604
1605 int Opcode = getV_CMPOpcode(P: Pred, Size, ST: *Subtarget);
1606 if (Opcode == -1)
1607 return false;
1608
1609 MachineInstrBuilder ICmp;
1610 // t16 instructions
1611 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
1612 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1613 .addImm(Val: 0)
1614 .add(MO: I.getOperand(i: 2))
1615 .addImm(Val: 0)
1616 .add(MO: I.getOperand(i: 3))
1617 .addImm(Val: 0); // op_sel
1618 } else {
1619 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1620 .add(MO: I.getOperand(i: 2))
1621 .add(MO: I.getOperand(i: 3));
1622 }
1623
1624 RBI.constrainGenericRegister(Reg: ICmp->getOperand(i: 0).getReg(),
1625 RC: *TRI.getBoolRC(), MRI&: *MRI);
1626 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1627 I.eraseFromParent();
1628 return true;
1629}
1630
1631bool AMDGPUInstructionSelector::selectIntrinsicCmp(MachineInstr &I) const {
1632 Register Dst = I.getOperand(i: 0).getReg();
1633 if (isVCC(Reg: Dst, MRI: *MRI))
1634 return false;
1635
1636 LLT DstTy = MRI->getType(Reg: Dst);
1637 if (DstTy.getSizeInBits() != STI.getWavefrontSize())
1638 return false;
1639
1640 MachineBasicBlock *BB = I.getParent();
1641 const DebugLoc &DL = I.getDebugLoc();
1642 Register SrcReg = I.getOperand(i: 2).getReg();
1643 unsigned Size = RBI.getSizeInBits(Reg: SrcReg, MRI: *MRI, TRI);
1644
1645 // i1 inputs are not supported in GlobalISel.
1646 if (Size == 1)
1647 return false;
1648
1649 auto Pred = static_cast<CmpInst::Predicate>(I.getOperand(i: 4).getImm());
1650 if (!CmpInst::isIntPredicate(P: Pred) && !CmpInst::isFPPredicate(P: Pred)) {
1651 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Dst);
1652 I.eraseFromParent();
1653 return RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI);
1654 }
1655
1656 const int Opcode = getV_CMPOpcode(P: Pred, Size, ST: *Subtarget);
1657 if (Opcode == -1)
1658 return false;
1659
1660 MachineInstrBuilder SelectedMI;
1661 MachineOperand &LHS = I.getOperand(i: 2);
1662 MachineOperand &RHS = I.getOperand(i: 3);
1663 auto [Src0, Src0Mods] = selectVOP3ModsImpl(Src: LHS.getReg());
1664 auto [Src1, Src1Mods] = selectVOP3ModsImpl(Src: RHS.getReg());
1665 Register Src0Reg =
1666 copyToVGPRIfSrcFolded(Src: Src0, Mods: Src0Mods, Root: LHS, InsertPt: &I, /*ForceVGPR*/ true);
1667 Register Src1Reg =
1668 copyToVGPRIfSrcFolded(Src: Src1, Mods: Src1Mods, Root: RHS, InsertPt: &I, /*ForceVGPR*/ true);
1669 SelectedMI = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: Dst);
1670 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers))
1671 SelectedMI.addImm(Val: Src0Mods);
1672 SelectedMI.addReg(RegNo: Src0Reg);
1673 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src1_modifiers))
1674 SelectedMI.addImm(Val: Src1Mods);
1675 SelectedMI.addReg(RegNo: Src1Reg);
1676 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::clamp))
1677 SelectedMI.addImm(Val: 0); // clamp
1678 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::op_sel))
1679 SelectedMI.addImm(Val: 0); // op_sel
1680
1681 RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI);
1682 constrainSelectedInstRegOperands(I&: *SelectedMI, TII, TRI, RBI);
1683
1684 I.eraseFromParent();
1685 return true;
1686}
1687
1688// Ballot has to zero bits in input lane-mask that are zero in current exec,
1689// Done as AND with exec. For inputs that are results of instruction that
1690// implicitly use same exec, for example compares in same basic block or SCC to
1691// VCC copy, use copy.
1692static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI,
1693 MachineBasicBlock *MBB) {
1694 MachineInstr *MI = MRI.getVRegDef(Reg);
1695 if (MI->getParent() != MBB)
1696 return false;
1697
1698 // Lane mask generated by SCC to VCC copy.
1699 if (MI->getOpcode() == AMDGPU::COPY) {
1700 auto DstRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 0).getReg());
1701 auto SrcRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 1).getReg());
1702 if (DstRB && SrcRB && DstRB->getID() == AMDGPU::VCCRegBankID &&
1703 SrcRB->getID() == AMDGPU::SGPRRegBankID)
1704 return true;
1705 }
1706
1707 // Lane mask generated by SCC to VCC copy
1708 if (MI->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
1709 return true;
1710
1711 // Lane mask generated using compare with same exec.
1712 if (isa<GAnyCmp>(Val: MI))
1713 return true;
1714
1715 Register LHS, RHS;
1716 // Look through AND.
1717 if (mi_match(R: Reg, MRI, P: m_GAnd(L: m_Reg(R&: LHS), R: m_Reg(R&: RHS))))
1718 return isLaneMaskFromSameBlock(Reg: LHS, MRI, MBB) ||
1719 isLaneMaskFromSameBlock(Reg: RHS, MRI, MBB);
1720
1721 return false;
1722}
1723
1724bool AMDGPUInstructionSelector::selectBallot(MachineInstr &I) const {
1725 MachineBasicBlock *BB = I.getParent();
1726 const DebugLoc &DL = I.getDebugLoc();
1727 Register DstReg = I.getOperand(i: 0).getReg();
1728 Register SrcReg = I.getOperand(i: 2).getReg();
1729 const unsigned BallotSize = MRI->getType(Reg: DstReg).getSizeInBits();
1730 const unsigned WaveSize = STI.getWavefrontSize();
1731
1732 // In the common case, the return type matches the wave size.
1733 // However we also support emitting i64 ballots in wave32 mode.
1734 if (BallotSize != WaveSize && (BallotSize != 64 || WaveSize != 32))
1735 return false;
1736
1737 std::optional<ValueAndVReg> Arg =
1738 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI);
1739
1740 Register Dst = DstReg;
1741 // i64 ballot on Wave32: new Dst(i32) for WaveSize ballot.
1742 if (BallotSize != WaveSize) {
1743 Dst = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
1744 }
1745
1746 if (Arg) {
1747 const int64_t Value = Arg->Value.getZExtValue();
1748 if (Value == 0) {
1749 // Dst = S_MOV 0
1750 unsigned Opcode = WaveSize == 64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
1751 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: Dst).addImm(Val: 0);
1752 } else {
1753 // Dst = COPY EXEC
1754 assert(Value == 1);
1755 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: TRI.getExec());
1756 }
1757 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1758 return false;
1759 } else {
1760 if (isLaneMaskFromSameBlock(Reg: SrcReg, MRI&: *MRI, MBB: BB)) {
1761 // Dst = COPY SrcReg
1762 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: SrcReg);
1763 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1764 return false;
1765 } else {
1766 // Dst = S_AND SrcReg, EXEC
1767 unsigned AndOpc = WaveSize == 64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
1768 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: Dst)
1769 .addReg(RegNo: SrcReg)
1770 .addReg(RegNo: TRI.getExec())
1771 .setOperandDead(3); // Dead scc
1772 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
1773 }
1774 }
1775
1776 // i64 ballot on Wave32: zero-extend i32 ballot to i64.
1777 if (BallotSize != WaveSize) {
1778 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1779 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg).addImm(Val: 0);
1780 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
1781 .addReg(RegNo: Dst)
1782 .addImm(Val: AMDGPU::sub0)
1783 .addReg(RegNo: HiReg)
1784 .addImm(Val: AMDGPU::sub1);
1785 }
1786
1787 I.eraseFromParent();
1788 return true;
1789}
1790
1791bool AMDGPUInstructionSelector::selectRelocConstant(MachineInstr &I) const {
1792 Register DstReg = I.getOperand(i: 0).getReg();
1793 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1794 const TargetRegisterClass *DstRC = TRI.getRegClassForSizeOnBank(Size: 32, Bank: *DstBank);
1795 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1796 return false;
1797
1798 const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID;
1799
1800 Module *M = MF->getFunction().getParent();
1801 const MDNode *Metadata = I.getOperand(i: 2).getMetadata();
1802 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
1803 auto *RelocSymbol = cast<GlobalVariable>(
1804 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
1805
1806 MachineBasicBlock *BB = I.getParent();
1807 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(),
1808 MCID: TII.get(Opcode: IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DestReg: DstReg)
1809 .addGlobalAddress(GV: RelocSymbol, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1810
1811 I.eraseFromParent();
1812 return true;
1813}
1814
1815bool AMDGPUInstructionSelector::selectGroupStaticSize(MachineInstr &I) const {
1816 Triple::OSType OS = MF->getTarget().getTargetTriple().getOS();
1817
1818 Register DstReg = I.getOperand(i: 0).getReg();
1819 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1820 unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ?
1821 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1822
1823 MachineBasicBlock *MBB = I.getParent();
1824 const DebugLoc &DL = I.getDebugLoc();
1825
1826 auto MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Mov), DestReg: DstReg);
1827
1828 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
1829 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1830 MIB.addImm(Val: MFI->getLDSSize());
1831 } else {
1832 Module *M = MF->getFunction().getParent();
1833 const GlobalValue *GV =
1834 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::amdgcn_groupstaticsize);
1835 MIB.addGlobalAddress(GV, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1836 }
1837
1838 I.eraseFromParent();
1839 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1840 return true;
1841}
1842
1843bool AMDGPUInstructionSelector::selectReturnAddress(MachineInstr &I) const {
1844 MachineBasicBlock *MBB = I.getParent();
1845 MachineFunction &MF = *MBB->getParent();
1846 const DebugLoc &DL = I.getDebugLoc();
1847
1848 MachineOperand &Dst = I.getOperand(i: 0);
1849 Register DstReg = Dst.getReg();
1850 unsigned Depth = I.getOperand(i: 2).getImm();
1851
1852 const TargetRegisterClass *RC
1853 = TRI.getConstrainedRegClassForOperand(MO: Dst, MRI: *MRI);
1854 if (!RC->hasSubClassEq(RC: &AMDGPU::SGPR_64RegClass) ||
1855 !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
1856 return false;
1857
1858 // Check for kernel and shader functions
1859 if (Depth != 0 ||
1860 MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
1861 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg)
1862 .addImm(Val: 0);
1863 I.eraseFromParent();
1864 return true;
1865 }
1866
1867 MachineFrameInfo &MFI = MF.getFrameInfo();
1868 // There is a call to @llvm.returnaddress in this function
1869 MFI.setReturnAddressIsTaken(true);
1870
1871 // Get the return address reg and mark it as an implicit live-in
1872 Register ReturnAddrReg = TRI.getReturnAddressReg(MF);
1873 Register LiveIn = getFunctionLiveInPhysReg(MF, TII, PhysReg: ReturnAddrReg,
1874 RC: AMDGPU::SReg_64RegClass, DL);
1875 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
1876 .addReg(RegNo: LiveIn);
1877 I.eraseFromParent();
1878 return true;
1879}
1880
1881bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const {
1882 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1883 // SelectionDAG uses for wave32 vs wave64.
1884 MachineBasicBlock *BB = MI.getParent();
1885 BuildMI(BB&: *BB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_END_CF))
1886 .add(MO: MI.getOperand(i: 1));
1887
1888 Register Reg = MI.getOperand(i: 1).getReg();
1889 MI.eraseFromParent();
1890
1891 if (!MRI->getRegClassOrNull(Reg))
1892 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1893 return true;
1894}
1895
1896bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic(
1897 MachineInstr &MI, Intrinsic::ID IntrID) const {
1898 MachineBasicBlock *MBB = MI.getParent();
1899 MachineFunction *MF = MBB->getParent();
1900 const DebugLoc &DL = MI.getDebugLoc();
1901
1902 unsigned IndexOperand = MI.getOperand(i: 7).getImm();
1903 bool WaveRelease = MI.getOperand(i: 8).getImm() != 0;
1904 bool WaveDone = MI.getOperand(i: 9).getImm() != 0;
1905
1906 if (WaveDone && !WaveRelease) {
1907 // TODO: Move this to IR verifier
1908 const Function &Fn = MF->getFunction();
1909 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1910 Fn, "ds_ordered_count: wave_done requires wave_release", DL));
1911 }
1912
1913 unsigned OrderedCountIndex = IndexOperand & 0x3f;
1914 IndexOperand &= ~0x3f;
1915 unsigned CountDw = 0;
1916
1917 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) {
1918 CountDw = (IndexOperand >> 24) & 0xf;
1919 IndexOperand &= ~(0xf << 24);
1920
1921 if (CountDw < 1 || CountDw > 4) {
1922 const Function &Fn = MF->getFunction();
1923 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1924 Fn, "ds_ordered_count: dword count must be between 1 and 4", DL));
1925 CountDw = 1;
1926 }
1927 }
1928
1929 if (IndexOperand) {
1930 const Function &Fn = MF->getFunction();
1931 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1932 Fn, "ds_ordered_count: bad index operand", DL));
1933 }
1934
1935 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
1936 unsigned ShaderType = SIInstrInfo::getDSShaderTypeValue(MF: *MF);
1937
1938 unsigned Offset0 = OrderedCountIndex << 2;
1939 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
1940
1941 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10)
1942 Offset1 |= (CountDw - 1) << 6;
1943
1944 if (STI.getGeneration() < AMDGPUSubtarget::GFX11)
1945 Offset1 |= ShaderType << 2;
1946
1947 unsigned Offset = Offset0 | (Offset1 << 8);
1948
1949 Register M0Val = MI.getOperand(i: 2).getReg();
1950 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1951 .addReg(RegNo: M0Val);
1952
1953 Register DstReg = MI.getOperand(i: 0).getReg();
1954 Register ValReg = MI.getOperand(i: 3).getReg();
1955 MachineInstrBuilder DS =
1956 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_ORDERED_COUNT), DestReg: DstReg)
1957 .addReg(RegNo: ValReg)
1958 .addImm(Val: Offset)
1959 .cloneMemRefs(OtherMI: MI);
1960
1961 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
1962 return false;
1963
1964 constrainSelectedInstRegOperands(I&: *DS, TII, TRI, RBI);
1965 MI.eraseFromParent();
1966 return true;
1967}
1968
1969static unsigned gwsIntrinToOpcode(unsigned IntrID) {
1970 switch (IntrID) {
1971 case Intrinsic::amdgcn_ds_gws_init:
1972 return AMDGPU::DS_GWS_INIT;
1973 case Intrinsic::amdgcn_ds_gws_barrier:
1974 return AMDGPU::DS_GWS_BARRIER;
1975 case Intrinsic::amdgcn_ds_gws_sema_v:
1976 return AMDGPU::DS_GWS_SEMA_V;
1977 case Intrinsic::amdgcn_ds_gws_sema_br:
1978 return AMDGPU::DS_GWS_SEMA_BR;
1979 case Intrinsic::amdgcn_ds_gws_sema_p:
1980 return AMDGPU::DS_GWS_SEMA_P;
1981 case Intrinsic::amdgcn_ds_gws_sema_release_all:
1982 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
1983 default:
1984 llvm_unreachable("not a gws intrinsic");
1985 }
1986}
1987
1988bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
1989 Intrinsic::ID IID) const {
1990 if (!STI.hasGWS() || (IID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
1991 !STI.hasGWSSemaReleaseAll()))
1992 return false;
1993
1994 // intrinsic ID, vsrc, offset
1995 const bool HasVSrc = MI.getNumOperands() == 3;
1996 assert(HasVSrc || MI.getNumOperands() == 2);
1997
1998 Register BaseOffset = MI.getOperand(i: HasVSrc ? 2 : 1).getReg();
1999 const RegisterBank *OffsetRB = RBI.getRegBank(Reg: BaseOffset, MRI: *MRI, TRI);
2000 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID)
2001 return false;
2002
2003 MachineInstr *OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
2004 unsigned ImmOffset;
2005
2006 MachineBasicBlock *MBB = MI.getParent();
2007 const DebugLoc &DL = MI.getDebugLoc();
2008
2009 MachineInstr *Readfirstlane = nullptr;
2010
2011 // If we legalized the VGPR input, strip out the readfirstlane to analyze the
2012 // incoming offset, in case there's an add of a constant. We'll have to put it
2013 // back later.
2014 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) {
2015 Readfirstlane = OffsetDef;
2016 BaseOffset = OffsetDef->getOperand(i: 1).getReg();
2017 OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
2018 }
2019
2020 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) {
2021 // If we have a constant offset, try to use the 0 in m0 as the base.
2022 // TODO: Look into changing the default m0 initialization value. If the
2023 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
2024 // the immediate offset.
2025
2026 ImmOffset = OffsetDef->getOperand(i: 1).getCImm()->getZExtValue();
2027 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
2028 .addImm(Val: 0);
2029 } else {
2030 std::tie(args&: BaseOffset, args&: ImmOffset) =
2031 AMDGPU::getBaseWithConstantOffset(MRI&: *MRI, Reg: BaseOffset, ValueTracking: VT);
2032
2033 if (Readfirstlane) {
2034 // We have the constant offset now, so put the readfirstlane back on the
2035 // variable component.
2036 if (!RBI.constrainGenericRegister(Reg: BaseOffset, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
2037 return false;
2038
2039 Readfirstlane->getOperand(i: 1).setReg(BaseOffset);
2040 BaseOffset = Readfirstlane->getOperand(i: 0).getReg();
2041 } else {
2042 if (!RBI.constrainGenericRegister(Reg: BaseOffset,
2043 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
2044 return false;
2045 }
2046
2047 Register M0Base = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2048 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: M0Base)
2049 .addReg(RegNo: BaseOffset)
2050 .addImm(Val: 16)
2051 .setOperandDead(3); // Dead scc
2052
2053 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
2054 .addReg(RegNo: M0Base);
2055 }
2056
2057 // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
2058 // offset field) % 64. Some versions of the programming guide omit the m0
2059 // part, or claim it's from offset 0.
2060
2061 unsigned Opc = gwsIntrinToOpcode(IntrID: IID);
2062 const MCInstrDesc &InstrDesc = TII.get(Opcode: Opc);
2063
2064 if (HasVSrc) {
2065 Register VSrc = MI.getOperand(i: 1).getReg();
2066
2067 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
2068 const TargetRegisterClass *DataRC = TII.getRegClass(MCID: InstrDesc, OpNum: Data0Idx);
2069 const TargetRegisterClass *SubRC =
2070 TRI.getSubRegisterClass(DataRC, AMDGPU::sub0);
2071
2072 if (!SubRC) {
2073 // 32-bit normal case.
2074 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *DataRC, MRI&: *MRI))
2075 return false;
2076
2077 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2078 .addReg(RegNo: VSrc)
2079 .addImm(Val: ImmOffset)
2080 .cloneMemRefs(OtherMI: MI);
2081 } else {
2082 // Requires even register alignment, so create 64-bit value and pad the
2083 // top half with undef.
2084 Register DataReg = MRI->createVirtualRegister(RegClass: DataRC);
2085 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *SubRC, MRI&: *MRI))
2086 return false;
2087
2088 Register UndefReg = MRI->createVirtualRegister(RegClass: SubRC);
2089 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2090 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DataReg)
2091 .addReg(RegNo: VSrc)
2092 .addImm(Val: AMDGPU::sub0)
2093 .addReg(RegNo: UndefReg)
2094 .addImm(Val: AMDGPU::sub1);
2095
2096 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2097 .addReg(RegNo: DataReg)
2098 .addImm(Val: ImmOffset)
2099 .cloneMemRefs(OtherMI: MI);
2100 }
2101 } else {
2102 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2103 .addImm(Val: ImmOffset)
2104 .cloneMemRefs(OtherMI: MI);
2105 }
2106
2107 MI.eraseFromParent();
2108 return true;
2109}
2110
2111bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
2112 bool IsAppend) const {
2113 Register PtrBase = MI.getOperand(i: 2).getReg();
2114 LLT PtrTy = MRI->getType(Reg: PtrBase);
2115 bool IsGDS = PtrTy.getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2116
2117 unsigned Offset;
2118 std::tie(args&: PtrBase, args&: Offset) = selectDS1Addr1OffsetImpl(Root&: MI.getOperand(i: 2));
2119
2120 // TODO: Should this try to look through readfirstlane like GWS?
2121 if (!isDSOffsetLegal(Base: PtrBase, Offset)) {
2122 PtrBase = MI.getOperand(i: 2).getReg();
2123 Offset = 0;
2124 }
2125
2126 MachineBasicBlock *MBB = MI.getParent();
2127 const DebugLoc &DL = MI.getDebugLoc();
2128 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2129
2130 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
2131 .addReg(RegNo: PtrBase);
2132 if (!RBI.constrainGenericRegister(Reg: PtrBase, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
2133 return false;
2134
2135 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: MI.getOperand(i: 0).getReg())
2136 .addImm(Val: Offset)
2137 .addImm(Val: IsGDS ? -1 : 0)
2138 .cloneMemRefs(OtherMI: MI);
2139 MI.eraseFromParent();
2140 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2141 return true;
2142}
2143
2144bool AMDGPUInstructionSelector::selectInitWholeWave(MachineInstr &MI) const {
2145 MachineFunction *MF = MI.getMF();
2146 SIMachineFunctionInfo *MFInfo = MF->getInfo<SIMachineFunctionInfo>();
2147
2148 MFInfo->setInitWholeWave();
2149 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
2150}
2151
2152static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE,
2153 bool &IsTexFail) {
2154 if (TexFailCtrl)
2155 IsTexFail = true;
2156
2157 TFE = TexFailCtrl & 0x1;
2158 TexFailCtrl &= ~(uint64_t)0x1;
2159 LWE = TexFailCtrl & 0x2;
2160 TexFailCtrl &= ~(uint64_t)0x2;
2161
2162 return TexFailCtrl == 0;
2163}
2164
2165bool AMDGPUInstructionSelector::selectImageIntrinsic(
2166 MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
2167 MachineBasicBlock *MBB = MI.getParent();
2168 const DebugLoc &DL = MI.getDebugLoc();
2169 unsigned IntrOpcode = Intr->BaseOpcode;
2170
2171 // For image atomic: use no-return opcode if result is unused.
2172 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode) {
2173 Register ResultDef = MI.getOperand(i: 0).getReg();
2174 if (MRI->use_nodbg_empty(RegNo: ResultDef))
2175 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
2176 }
2177
2178 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
2179 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
2180
2181 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
2182 const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI);
2183 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI);
2184 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
2185 const bool IsGFX13Plus = AMDGPU::isGFX13Plus(STI);
2186
2187 const unsigned ArgOffset = MI.getNumExplicitDefs() + 1;
2188
2189 Register VDataIn = AMDGPU::NoRegister;
2190 Register VDataOut = AMDGPU::NoRegister;
2191 LLT VDataTy;
2192 int NumVDataDwords = -1;
2193 bool IsD16 = MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 ||
2194 MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16;
2195
2196 bool Unorm;
2197 if (!BaseOpcode->Sampler)
2198 Unorm = true;
2199 else
2200 Unorm = MI.getOperand(i: ArgOffset + Intr->UnormIndex).getImm() != 0;
2201
2202 bool TFE;
2203 bool LWE;
2204 bool IsTexFail = false;
2205 if (!parseTexFail(TexFailCtrl: MI.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex).getImm(),
2206 TFE, LWE, IsTexFail))
2207 return false;
2208
2209 const int Flags = MI.getOperand(i: ArgOffset + Intr->NumArgs).getImm();
2210 const bool IsA16 = (Flags & 1) != 0;
2211 const bool IsG16 = (Flags & 2) != 0;
2212
2213 // A16 implies 16 bit gradients if subtarget doesn't support G16
2214 if (IsA16 && !STI.hasG16() && !IsG16)
2215 return false;
2216
2217 unsigned DMask = 0;
2218 unsigned DMaskLanes = 0;
2219
2220 if (BaseOpcode->Atomic) {
2221 if (!BaseOpcode->NoReturn)
2222 VDataOut = MI.getOperand(i: 0).getReg();
2223 VDataIn = MI.getOperand(i: 2).getReg();
2224 LLT Ty = MRI->getType(Reg: VDataIn);
2225
2226 // Be careful to allow atomic swap on 16-bit element vectors.
2227 const bool Is64Bit = BaseOpcode->AtomicX2 ?
2228 Ty.getSizeInBits() == 128 :
2229 Ty.getSizeInBits() == 64;
2230
2231 if (BaseOpcode->AtomicX2) {
2232 assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister);
2233
2234 DMask = Is64Bit ? 0xf : 0x3;
2235 NumVDataDwords = Is64Bit ? 4 : 2;
2236 } else {
2237 DMask = Is64Bit ? 0x3 : 0x1;
2238 NumVDataDwords = Is64Bit ? 2 : 1;
2239 }
2240 } else {
2241 DMask = MI.getOperand(i: ArgOffset + Intr->DMaskIndex).getImm();
2242 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
2243
2244 if (BaseOpcode->Store) {
2245 VDataIn = MI.getOperand(i: 1).getReg();
2246 VDataTy = MRI->getType(Reg: VDataIn);
2247 NumVDataDwords = (VDataTy.getSizeInBits() + 31) / 32;
2248 } else if (BaseOpcode->NoReturn) {
2249 NumVDataDwords = 0;
2250 } else {
2251 VDataOut = MI.getOperand(i: 0).getReg();
2252 VDataTy = MRI->getType(Reg: VDataOut);
2253 NumVDataDwords = DMaskLanes;
2254
2255 if (IsD16 && !STI.hasUnpackedD16VMem())
2256 NumVDataDwords = (DMaskLanes + 1) / 2;
2257 }
2258 }
2259
2260 // Set G16 opcode
2261 if (Subtarget->hasG16() && IsG16) {
2262 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
2263 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
2264 assert(G16MappingInfo);
2265 IntrOpcode = G16MappingInfo->G16; // set opcode to variant with _g16
2266 }
2267
2268 // TODO: Check this in verifier.
2269 assert((!IsTexFail || DMaskLanes >= 1) && "should have legalized this");
2270
2271 unsigned CPol = MI.getOperand(i: ArgOffset + Intr->CachePolicyIndex).getImm();
2272 // Keep GLC only when the atomic's result is actually used.
2273 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
2274 CPol |= AMDGPU::CPol::GLC;
2275 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
2276 AMDGPU::CPol::VOLATILE))
2277 return false;
2278
2279 int NumVAddrRegs = 0;
2280 int NumVAddrDwords = 0;
2281 for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
2282 // Skip the $noregs and 0s inserted during legalization.
2283 MachineOperand &AddrOp = MI.getOperand(i: ArgOffset + I);
2284 if (!AddrOp.isReg())
2285 continue; // XXX - Break?
2286
2287 Register Addr = AddrOp.getReg();
2288 if (!Addr)
2289 break;
2290
2291 ++NumVAddrRegs;
2292 NumVAddrDwords += (MRI->getType(Reg: Addr).getSizeInBits() + 31) / 32;
2293 }
2294
2295 // The legalizer preprocessed the intrinsic arguments. If we aren't using
2296 // NSA, these should have been packed into a single value in the first
2297 // address register
2298 const bool UseNSA =
2299 NumVAddrRegs != 1 &&
2300 (STI.hasPartialNSAEncoding() ? NumVAddrDwords >= NumVAddrRegs
2301 : NumVAddrDwords == NumVAddrRegs);
2302 if (UseNSA && !STI.hasFeature(Feature: AMDGPU::FeatureNSAEncoding)) {
2303 LLVM_DEBUG(dbgs() << "Trying to use NSA on non-NSA target\n");
2304 return false;
2305 }
2306
2307 if (IsTexFail)
2308 ++NumVDataDwords;
2309
2310 int Opcode = -1;
2311 if (IsGFX13Plus) {
2312 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx13,
2313 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2314 } else if (IsGFX12Plus) {
2315 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
2316 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2317 } else if (IsGFX11Plus) {
2318 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2319 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
2320 : AMDGPU::MIMGEncGfx11Default,
2321 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2322 } else if (IsGFX10Plus) {
2323 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2324 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
2325 : AMDGPU::MIMGEncGfx10Default,
2326 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2327 } else {
2328 if (Subtarget->hasGFX90AInsts()) {
2329 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
2330 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2331 if (Opcode == -1) {
2332 LLVM_DEBUG(
2333 dbgs()
2334 << "requested image instruction is not supported on this GPU\n");
2335 return false;
2336 }
2337 }
2338 if (Opcode == -1 &&
2339 STI.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
2340 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
2341 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2342 if (Opcode == -1)
2343 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
2344 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2345 }
2346 if (Opcode == -1)
2347 return false;
2348
2349 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode))
2350 .cloneMemRefs(OtherMI: MI);
2351
2352 if (VDataOut) {
2353 if (BaseOpcode->AtomicX2) {
2354 const bool Is64 = MRI->getType(Reg: VDataOut).getSizeInBits() == 64;
2355
2356 Register TmpReg = MRI->createVirtualRegister(
2357 RegClass: Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass);
2358 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0;
2359
2360 MIB.addDef(RegNo: TmpReg);
2361 if (!MRI->use_empty(RegNo: VDataOut)) {
2362 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VDataOut)
2363 .addReg(RegNo: TmpReg, Flags: RegState::Kill, SubReg);
2364 }
2365
2366 } else {
2367 MIB.addDef(RegNo: VDataOut); // vdata output
2368 }
2369 }
2370
2371 if (VDataIn)
2372 MIB.addReg(RegNo: VDataIn); // vdata input
2373
2374 for (int I = 0; I != NumVAddrRegs; ++I) {
2375 MachineOperand &SrcOp = MI.getOperand(i: ArgOffset + Intr->VAddrStart + I);
2376 if (SrcOp.isReg()) {
2377 assert(SrcOp.getReg() != 0);
2378 MIB.addReg(RegNo: SrcOp.getReg());
2379 }
2380 }
2381
2382 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->RsrcIndex).getReg());
2383 if (BaseOpcode->Sampler)
2384 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->SampIndex).getReg());
2385
2386 MIB.addImm(Val: DMask); // dmask
2387
2388 if (IsGFX10Plus)
2389 MIB.addImm(Val: DimInfo->Encoding);
2390 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::unorm))
2391 MIB.addImm(Val: Unorm);
2392
2393 MIB.addImm(Val: CPol);
2394 MIB.addImm(Val: IsA16 && // a16 or r128
2395 STI.hasFeature(Feature: AMDGPU::FeatureR128A16) ? -1 : 0);
2396 if (IsGFX10Plus)
2397 MIB.addImm(Val: IsA16 ? -1 : 0);
2398
2399 if (!Subtarget->hasGFX90AInsts()) {
2400 MIB.addImm(Val: TFE); // tfe
2401 } else if (TFE) {
2402 LLVM_DEBUG(dbgs() << "TFE is not supported on this GPU\n");
2403 return false;
2404 }
2405
2406 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::lwe))
2407 MIB.addImm(Val: LWE); // lwe
2408 if (!IsGFX10Plus)
2409 MIB.addImm(Val: DimInfo->DA ? -1 : 0);
2410 if (BaseOpcode->HasD16)
2411 MIB.addImm(Val: IsD16 ? -1 : 0);
2412
2413 MI.eraseFromParent();
2414 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2415 TII.enforceOperandRCAlignment(MI&: *MIB, OpName: AMDGPU::OpName::vaddr);
2416 return true;
2417}
2418
2419// We need to handle this here because tablegen doesn't support matching
2420// instructions with multiple outputs.
2421bool AMDGPUInstructionSelector::selectDSBvhStackIntrinsic(
2422 MachineInstr &MI) const {
2423 Register Dst0 = MI.getOperand(i: 0).getReg();
2424 Register Dst1 = MI.getOperand(i: 1).getReg();
2425
2426 const DebugLoc &DL = MI.getDebugLoc();
2427 MachineBasicBlock *MBB = MI.getParent();
2428
2429 Register Addr = MI.getOperand(i: 3).getReg();
2430 Register Data0 = MI.getOperand(i: 4).getReg();
2431 Register Data1 = MI.getOperand(i: 5).getReg();
2432 unsigned Offset = MI.getOperand(i: 6).getImm();
2433
2434 unsigned Opc;
2435 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
2436 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2437 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2438 Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
2439 break;
2440 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2441 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32;
2442 break;
2443 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2444 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64;
2445 break;
2446 }
2447
2448 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
2449 .addDef(RegNo: Dst1)
2450 .addUse(RegNo: Addr)
2451 .addUse(RegNo: Data0)
2452 .addUse(RegNo: Data1)
2453 .addImm(Val: Offset)
2454 .cloneMemRefs(OtherMI: MI);
2455
2456 MI.eraseFromParent();
2457 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2458 return true;
2459}
2460
2461bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
2462 MachineInstr &I) const {
2463 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
2464 switch (IntrinsicID) {
2465 case Intrinsic::amdgcn_end_cf:
2466 return selectEndCfIntrinsic(MI&: I);
2467 case Intrinsic::amdgcn_ds_ordered_add:
2468 case Intrinsic::amdgcn_ds_ordered_swap:
2469 return selectDSOrderedIntrinsic(MI&: I, IntrID: IntrinsicID);
2470 case Intrinsic::amdgcn_ds_gws_init:
2471 case Intrinsic::amdgcn_ds_gws_barrier:
2472 case Intrinsic::amdgcn_ds_gws_sema_v:
2473 case Intrinsic::amdgcn_ds_gws_sema_br:
2474 case Intrinsic::amdgcn_ds_gws_sema_p:
2475 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2476 return selectDSGWSIntrinsic(MI&: I, IID: IntrinsicID);
2477 case Intrinsic::amdgcn_ds_append:
2478 return selectDSAppendConsume(MI&: I, IsAppend: true);
2479 case Intrinsic::amdgcn_ds_consume:
2480 return selectDSAppendConsume(MI&: I, IsAppend: false);
2481 case Intrinsic::amdgcn_init_whole_wave:
2482 return selectInitWholeWave(MI&: I);
2483 case Intrinsic::amdgcn_raw_buffer_load_lds:
2484 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
2485 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
2486 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
2487 case Intrinsic::amdgcn_struct_buffer_load_lds:
2488 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
2489 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
2490 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
2491 return selectBufferLoadLds(MI&: I);
2492 // Until we can store both the address space of the global and the LDS
2493 // arguments by having tto MachineMemOperands on an intrinsic, we just trust
2494 // that the argument is a global pointer (buffer pointers have been handled by
2495 // a LLVM IR-level lowering).
2496 case Intrinsic::amdgcn_load_to_lds:
2497 case Intrinsic::amdgcn_load_async_to_lds:
2498 case Intrinsic::amdgcn_global_load_lds:
2499 case Intrinsic::amdgcn_global_load_async_lds:
2500 return selectGlobalLoadLds(MI&: I);
2501 case Intrinsic::amdgcn_tensor_load_to_lds:
2502 case Intrinsic::amdgcn_tensor_store_from_lds:
2503 return selectTensorLoadStore(MI&: I, IID: IntrinsicID);
2504 case Intrinsic::amdgcn_asyncmark:
2505 case Intrinsic::amdgcn_wait_asyncmark:
2506 if (!Subtarget->hasAsyncMark())
2507 return false;
2508 break;
2509 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2510 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2511 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2512 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2513 return selectDSBvhStackIntrinsic(MI&: I);
2514 case Intrinsic::amdgcn_s_alloc_vgpr: {
2515 // S_ALLOC_VGPR doesn't have a destination register, it just implicitly sets
2516 // SCC. We then need to COPY it into the result vreg.
2517 MachineBasicBlock *MBB = I.getParent();
2518 const DebugLoc &DL = I.getDebugLoc();
2519
2520 Register ResReg = I.getOperand(i: 0).getReg();
2521
2522 MachineInstr *AllocMI = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ALLOC_VGPR))
2523 .add(MO: I.getOperand(i: 2));
2524 (void)BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: ResReg)
2525 .addReg(RegNo: AMDGPU::SCC);
2526 I.eraseFromParent();
2527 constrainSelectedInstRegOperands(I&: *AllocMI, TII, TRI, RBI);
2528 return RBI.constrainGenericRegister(Reg: ResReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2529 }
2530 case Intrinsic::amdgcn_s_barrier_init:
2531 case Intrinsic::amdgcn_s_barrier_signal_var:
2532 return selectNamedBarrierInit(I, IID: IntrinsicID);
2533 case Intrinsic::amdgcn_s_wakeup_barrier:
2534 case Intrinsic::amdgcn_s_barrier_join:
2535 case Intrinsic::amdgcn_s_get_named_barrier_state:
2536 return selectNamedBarrierInst(I, IID: IntrinsicID);
2537 case Intrinsic::amdgcn_s_get_barrier_state:
2538 return selectSGetBarrierState(I, IID: IntrinsicID);
2539 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
2540 return selectSBarrierSignalIsfirst(I, IID: IntrinsicID);
2541 }
2542 return selectImpl(I, CoverageInfo&: *CoverageInfo);
2543}
2544
2545bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
2546 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
2547 return true;
2548
2549 MachineBasicBlock *BB = I.getParent();
2550 const DebugLoc &DL = I.getDebugLoc();
2551
2552 Register DstReg = I.getOperand(i: 0).getReg();
2553 unsigned Size = RBI.getSizeInBits(Reg: DstReg, MRI: *MRI, TRI);
2554 assert(Size <= 32 || Size == 64);
2555 const MachineOperand &CCOp = I.getOperand(i: 1);
2556 Register CCReg = CCOp.getReg();
2557 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
2558 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 :
2559 AMDGPU::S_CSELECT_B32;
2560 MachineInstr *CopySCC = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
2561 .addReg(RegNo: CCReg);
2562
2563 // The generic constrainSelectedInstRegOperands doesn't work for the scc register
2564 // bank, because it does not cover the register class that we used to represent
2565 // for it. So we need to manually set the register class here.
2566 if (!MRI->getRegClassOrNull(Reg: CCReg))
2567 MRI->setRegClass(Reg: CCReg, RC: TRI.getConstrainedRegClassForOperand(MO: CCOp, MRI: *MRI));
2568 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
2569 .add(MO: I.getOperand(i: 2))
2570 .add(MO: I.getOperand(i: 3));
2571
2572 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2573 constrainSelectedInstRegOperands(I&: *CopySCC, TII, TRI, RBI);
2574 I.eraseFromParent();
2575 return true;
2576 }
2577
2578 // Wide VGPR select should have been split in RegBankSelect.
2579 if (Size > 32)
2580 return false;
2581
2582 MachineInstr *Select =
2583 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
2584 .addImm(Val: 0)
2585 .add(MO: I.getOperand(i: 3))
2586 .addImm(Val: 0)
2587 .add(MO: I.getOperand(i: 2))
2588 .add(MO: I.getOperand(i: 1));
2589
2590 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2591 I.eraseFromParent();
2592 return true;
2593}
2594
2595bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
2596 Register DstReg = I.getOperand(i: 0).getReg();
2597 Register SrcReg = I.getOperand(i: 1).getReg();
2598 const LLT DstTy = MRI->getType(Reg: DstReg);
2599 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2600 const LLT S1 = LLT::scalar(SizeInBits: 1);
2601
2602 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2603 const RegisterBank *DstRB;
2604 if (DstTy == S1) {
2605 // This is a special case. We don't treat s1 for legalization artifacts as
2606 // vcc booleans.
2607 DstRB = SrcRB;
2608 } else {
2609 DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2610 if (SrcRB != DstRB)
2611 return false;
2612 }
2613
2614 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
2615
2616 unsigned DstSize = DstTy.getSizeInBits();
2617 unsigned SrcSize = SrcTy.getSizeInBits();
2618
2619 const TargetRegisterClass *SrcRC =
2620 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcRB);
2621 const TargetRegisterClass *DstRC =
2622 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
2623 if (!SrcRC || !DstRC)
2624 return false;
2625
2626 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
2627 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
2628 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
2629 return false;
2630 }
2631
2632 if (DstRC == &AMDGPU::VGPR_16RegClass && SrcSize == 32) {
2633 assert(STI.useRealTrue16Insts());
2634 const DebugLoc &DL = I.getDebugLoc();
2635 MachineBasicBlock *MBB = I.getParent();
2636 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
2637 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::lo16);
2638 I.eraseFromParent();
2639 return true;
2640 }
2641
2642 if (DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) && SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
2643 MachineBasicBlock *MBB = I.getParent();
2644 const DebugLoc &DL = I.getDebugLoc();
2645
2646 Register LoReg = MRI->createVirtualRegister(RegClass: DstRC);
2647 Register HiReg = MRI->createVirtualRegister(RegClass: DstRC);
2648 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
2649 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
2650 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
2651 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
2652
2653 if (IsVALU && STI.hasSDWA()) {
2654 // Write the low 16-bits of the high element into the high 16-bits of the
2655 // low element.
2656 MachineInstr *MovSDWA =
2657 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: DstReg)
2658 .addImm(Val: 0) // $src0_modifiers
2659 .addReg(RegNo: HiReg) // $src0
2660 .addImm(Val: 0) // $clamp
2661 .addImm(Val: AMDGPU::SDWA::WORD_1) // $dst_sel
2662 .addImm(Val: AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused
2663 .addImm(Val: AMDGPU::SDWA::WORD_0) // $src0_sel
2664 .addReg(RegNo: LoReg, Flags: RegState::Implicit);
2665 MovSDWA->tieOperands(DefIdx: 0, UseIdx: MovSDWA->getNumOperands() - 1);
2666 } else {
2667 Register TmpReg0 = MRI->createVirtualRegister(RegClass: DstRC);
2668 Register TmpReg1 = MRI->createVirtualRegister(RegClass: DstRC);
2669 Register ImmReg = MRI->createVirtualRegister(RegClass: DstRC);
2670 if (IsVALU) {
2671 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: TmpReg0)
2672 .addImm(Val: 16)
2673 .addReg(RegNo: HiReg);
2674 } else {
2675 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg0)
2676 .addReg(RegNo: HiReg)
2677 .addImm(Val: 16)
2678 .setOperandDead(3); // Dead scc
2679 }
2680
2681 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2682 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
2683 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32;
2684
2685 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: ImmReg)
2686 .addImm(Val: 0xffff);
2687 auto And = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: TmpReg1)
2688 .addReg(RegNo: LoReg)
2689 .addReg(RegNo: ImmReg);
2690 auto Or = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: OrOpc), DestReg: DstReg)
2691 .addReg(RegNo: TmpReg0)
2692 .addReg(RegNo: TmpReg1);
2693
2694 if (!IsVALU) {
2695 And.setOperandDead(3); // Dead scc
2696 Or.setOperandDead(3); // Dead scc
2697 }
2698 }
2699
2700 I.eraseFromParent();
2701 return true;
2702 }
2703
2704 if (!DstTy.isScalar())
2705 return false;
2706
2707 if (SrcSize > 32) {
2708 unsigned SubRegIdx = DstSize < 32
2709 ? static_cast<unsigned>(AMDGPU::sub0)
2710 : TRI.getSubRegFromChannel(Channel: 0, NumRegs: DstSize / 32);
2711 if (SubRegIdx == AMDGPU::NoSubRegister)
2712 return false;
2713
2714 // Deal with weird cases where the class only partially supports the subreg
2715 // index.
2716 const TargetRegisterClass *SrcWithSubRC
2717 = TRI.getSubClassWithSubReg(SrcRC, SubRegIdx);
2718 if (!SrcWithSubRC)
2719 return false;
2720
2721 if (SrcWithSubRC != SrcRC) {
2722 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcWithSubRC, MRI&: *MRI))
2723 return false;
2724 }
2725
2726 I.getOperand(i: 1).setSubReg(SubRegIdx);
2727 }
2728
2729 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
2730 return true;
2731}
2732
2733/// \returns true if a bitmask for \p Size bits will be an inline immediate.
2734static bool shouldUseAndMask(unsigned Size, unsigned &Mask) {
2735 Mask = maskTrailingOnes<unsigned>(N: Size);
2736 int SignedMask = static_cast<int>(Mask);
2737 return SignedMask >= -16 && SignedMask <= 64;
2738}
2739
2740// Like RegisterBankInfo::getRegBank, but don't assume vcc for s1.
2741const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank(
2742 Register Reg, const MachineRegisterInfo &MRI,
2743 const TargetRegisterInfo &TRI) const {
2744 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
2745 if (auto *RB = dyn_cast<const RegisterBank *>(Val: RegClassOrBank))
2746 return RB;
2747
2748 // Ignore the type, since we don't use vcc in artifacts.
2749 if (auto *RC = dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank))
2750 return &RBI.getRegBankFromRegClass(RC: *RC, LLT());
2751 return nullptr;
2752}
2753
2754bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
2755 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG;
2756 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg;
2757 const DebugLoc &DL = I.getDebugLoc();
2758 MachineBasicBlock &MBB = *I.getParent();
2759 const Register DstReg = I.getOperand(i: 0).getReg();
2760 const Register SrcReg = I.getOperand(i: 1).getReg();
2761
2762 const LLT DstTy = MRI->getType(Reg: DstReg);
2763 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2764 const unsigned SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG ?
2765 I.getOperand(i: 2).getImm() : SrcTy.getSizeInBits();
2766 const unsigned DstSize = DstTy.getSizeInBits();
2767 if (!DstTy.isScalar())
2768 return false;
2769
2770 // Artifact casts should never use vcc.
2771 const RegisterBank *SrcBank = getArtifactRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2772
2773 // FIXME: This should probably be illegal and split earlier.
2774 if (I.getOpcode() == AMDGPU::G_ANYEXT) {
2775 if (DstSize <= 32)
2776 return selectCOPY(I);
2777
2778 const TargetRegisterClass *SrcRC =
2779 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcBank);
2780 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2781 const TargetRegisterClass *DstRC =
2782 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
2783
2784 Register UndefReg = MRI->createVirtualRegister(RegClass: SrcRC);
2785 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2786 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2787 .addReg(RegNo: SrcReg)
2788 .addImm(Val: AMDGPU::sub0)
2789 .addReg(RegNo: UndefReg)
2790 .addImm(Val: AMDGPU::sub1);
2791 I.eraseFromParent();
2792
2793 return RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) &&
2794 RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI);
2795 }
2796
2797 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) {
2798 // 64-bit should have been split up in RegBankSelect
2799
2800 // Try to use an and with a mask if it will save code size.
2801 unsigned Mask;
2802 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2803 MachineInstr *ExtI =
2804 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: DstReg)
2805 .addImm(Val: Mask)
2806 .addReg(RegNo: SrcReg);
2807 I.eraseFromParent();
2808 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2809 return true;
2810 }
2811
2812 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2813 MachineInstr *ExtI =
2814 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE), DestReg: DstReg)
2815 .addReg(RegNo: SrcReg)
2816 .addImm(Val: 0) // Offset
2817 .addImm(Val: SrcSize); // Width
2818 I.eraseFromParent();
2819 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2820 return true;
2821 }
2822
2823 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
2824 const TargetRegisterClass &SrcRC = InReg && DstSize > 32 ?
2825 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass;
2826 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: SrcRC, MRI&: *MRI))
2827 return false;
2828
2829 if (Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) {
2830 const unsigned SextOpc = SrcSize == 8 ?
2831 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16;
2832 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: SextOpc), DestReg: DstReg)
2833 .addReg(RegNo: SrcReg);
2834 I.eraseFromParent();
2835 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2836 }
2837
2838 // Using a single 32-bit SALU to calculate the high half is smaller than
2839 // S_BFE with a literal constant operand.
2840 if (DstSize > 32 && SrcSize == 32) {
2841 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2842 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2843 if (Signed) {
2844 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ASHR_I32), DestReg: HiReg)
2845 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2846 .addImm(Val: 31)
2847 .setOperandDead(3); // Dead scc
2848 } else {
2849 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg)
2850 .addImm(Val: 0);
2851 }
2852 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2853 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2854 .addImm(Val: AMDGPU::sub0)
2855 .addReg(RegNo: HiReg)
2856 .addImm(Val: AMDGPU::sub1);
2857 I.eraseFromParent();
2858 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass,
2859 MRI&: *MRI);
2860 }
2861
2862 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
2863 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2864
2865 // Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width.
2866 if (DstSize > 32 && (SrcSize <= 32 || InReg)) {
2867 // We need a 64-bit register source, but the high bits don't matter.
2868 Register ExtReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
2869 Register UndefReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2870 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2871
2872 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2873 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: ExtReg)
2874 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2875 .addImm(Val: AMDGPU::sub0)
2876 .addReg(RegNo: UndefReg)
2877 .addImm(Val: AMDGPU::sub1);
2878
2879 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE64), DestReg: DstReg)
2880 .addReg(RegNo: ExtReg)
2881 .addImm(Val: SrcSize << 16);
2882
2883 I.eraseFromParent();
2884 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI);
2885 }
2886
2887 unsigned Mask;
2888 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2889 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: DstReg)
2890 .addReg(RegNo: SrcReg)
2891 .addImm(Val: Mask)
2892 .setOperandDead(3); // Dead scc
2893 } else {
2894 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE32), DestReg: DstReg)
2895 .addReg(RegNo: SrcReg)
2896 .addImm(Val: SrcSize << 16);
2897 }
2898
2899 I.eraseFromParent();
2900 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2901 }
2902
2903 return false;
2904}
2905
2906static Register stripCopy(Register Reg, MachineRegisterInfo &MRI) {
2907 return getDefSrcRegIgnoringCopies(Reg, MRI)->Reg;
2908}
2909
2910static Register stripBitCast(Register Reg, MachineRegisterInfo &MRI) {
2911 Register BitcastSrc;
2912 if (mi_match(R: Reg, MRI, P: m_GBitcast(Src: m_Reg(R&: BitcastSrc))))
2913 Reg = BitcastSrc;
2914 return Reg;
2915}
2916
2917static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In,
2918 Register &Out) {
2919 // When unmerging a register that is composed of 2 x 16-bit values allow to
2920 // use an extract hi instruction for the upper 16 bits. We only need to check
2921 // the size of `In` as all defs are guaranteed to be the same type for
2922 // GUnmerge.
2923 if (auto *Unmerge = dyn_cast<GUnmerge>(Val: MRI.getVRegDef(Reg: In))) {
2924 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(i: 1).getReg() == In &&
2925 MRI.getType(Reg: In).getSizeInBits() == 16) {
2926 Out = Unmerge->getSourceReg();
2927 return true;
2928 }
2929 }
2930
2931 Register Trunc;
2932 if (!mi_match(R: In, MRI, P: m_GTrunc(Src: m_Reg(R&: Trunc))))
2933 return false;
2934
2935 Register LShlSrc;
2936 Register Cst;
2937 if (mi_match(R: Trunc, MRI, P: m_GLShr(L: m_Reg(R&: LShlSrc), R: m_Reg(R&: Cst)))) {
2938 Cst = stripCopy(Reg: Cst, MRI);
2939 if (mi_match(R: Cst, MRI, P: m_SpecificICst(RequestedValue: 16))) {
2940 Out = stripBitCast(Reg: LShlSrc, MRI);
2941 return true;
2942 }
2943 }
2944
2945 MachineInstr *Shuffle = MRI.getVRegDef(Reg: Trunc);
2946 if (Shuffle->getOpcode() != AMDGPU::G_SHUFFLE_VECTOR)
2947 return false;
2948
2949 assert(MRI.getType(Shuffle->getOperand(0).getReg()) ==
2950 LLT::fixed_vector(2, 16));
2951
2952 ArrayRef<int> Mask = Shuffle->getOperand(i: 3).getShuffleMask();
2953 assert(Mask.size() == 2);
2954
2955 if (Mask[0] == 1 && Mask[1] <= 1) {
2956 Out = Shuffle->getOperand(i: 0).getReg();
2957 return true;
2958 }
2959
2960 return false;
2961}
2962
2963bool AMDGPUInstructionSelector::selectG_FPEXT(MachineInstr &I) const {
2964 if (!Subtarget->hasSALUFloatInsts())
2965 return false;
2966
2967 Register Dst = I.getOperand(i: 0).getReg();
2968 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2969 if (DstRB->getID() != AMDGPU::SGPRRegBankID)
2970 return false;
2971
2972 Register Src = I.getOperand(i: 1).getReg();
2973
2974 if (MRI->getType(Reg: Dst) == LLT::scalar(SizeInBits: 32) &&
2975 MRI->getType(Reg: Src) == LLT::scalar(SizeInBits: 16)) {
2976 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
2977 MachineBasicBlock *BB = I.getParent();
2978 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_CVT_HI_F32_F16), DestReg: Dst)
2979 .addUse(RegNo: Src);
2980 I.eraseFromParent();
2981 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2982 }
2983 }
2984
2985 return false;
2986}
2987
2988bool AMDGPUInstructionSelector::selectG_FNEG(MachineInstr &MI) const {
2989 // Only manually handle the f64 SGPR case.
2990 //
2991 // FIXME: This is a workaround for 2.5 different tablegen problems. Because
2992 // the bit ops theoretically have a second result due to the implicit def of
2993 // SCC, the GlobalISelEmitter is overly conservative and rejects it. Fixing
2994 // that is easy by disabling the check. The result works, but uses a
2995 // nonsensical sreg32orlds_and_sreg_1 regclass.
2996 //
2997 // The DAG emitter is more problematic, and incorrectly adds both S_XOR_B32 to
2998 // the variadic REG_SEQUENCE operands.
2999
3000 Register Dst = MI.getOperand(i: 0).getReg();
3001 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
3002 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
3003 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
3004 return false;
3005
3006 Register Src = MI.getOperand(i: 1).getReg();
3007 MachineInstr *Fabs = getOpcodeDef(Opcode: TargetOpcode::G_FABS, Reg: Src, MRI: *MRI);
3008 if (Fabs)
3009 Src = Fabs->getOperand(i: 1).getReg();
3010
3011 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
3012 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
3013 return false;
3014
3015 MachineBasicBlock *BB = MI.getParent();
3016 const DebugLoc &DL = MI.getDebugLoc();
3017 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3018 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3019 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3020 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3021
3022 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3023 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
3024 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3025 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
3026 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
3027 .addImm(Val: 0x80000000);
3028
3029 // Set or toggle sign bit.
3030 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32;
3031 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: OpReg)
3032 .addReg(RegNo: HiReg)
3033 .addReg(RegNo: ConstReg)
3034 .setOperandDead(3); // Dead scc
3035 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
3036 .addReg(RegNo: LoReg)
3037 .addImm(Val: AMDGPU::sub0)
3038 .addReg(RegNo: OpReg)
3039 .addImm(Val: AMDGPU::sub1);
3040 MI.eraseFromParent();
3041 return true;
3042}
3043
3044// FIXME: This is a workaround for the same tablegen problems as G_FNEG
3045bool AMDGPUInstructionSelector::selectG_FABS(MachineInstr &MI) const {
3046 Register Dst = MI.getOperand(i: 0).getReg();
3047 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
3048 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
3049 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
3050 return false;
3051
3052 Register Src = MI.getOperand(i: 1).getReg();
3053 MachineBasicBlock *BB = MI.getParent();
3054 const DebugLoc &DL = MI.getDebugLoc();
3055 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3056 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3057 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3058 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3059
3060 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
3061 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
3062 return false;
3063
3064 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3065 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
3066 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3067 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
3068 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
3069 .addImm(Val: 0x7fffffff);
3070
3071 // Clear sign bit.
3072 // TODO: Should this used S_BITSET0_*?
3073 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: OpReg)
3074 .addReg(RegNo: HiReg)
3075 .addReg(RegNo: ConstReg)
3076 .setOperandDead(3); // Dead scc
3077 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
3078 .addReg(RegNo: LoReg)
3079 .addImm(Val: AMDGPU::sub0)
3080 .addReg(RegNo: OpReg)
3081 .addImm(Val: AMDGPU::sub1);
3082
3083 MI.eraseFromParent();
3084 return true;
3085}
3086
3087static bool isConstant(const MachineInstr &MI) {
3088 return MI.getOpcode() == TargetOpcode::G_CONSTANT;
3089}
3090
3091void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
3092 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const {
3093
3094 unsigned OpNo = Load.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1;
3095 const MachineInstr *PtrMI =
3096 MRI.getUniqueVRegDef(Reg: Load.getOperand(i: OpNo).getReg());
3097
3098 assert(PtrMI);
3099
3100 if (PtrMI->getOpcode() != TargetOpcode::G_PTR_ADD)
3101 return;
3102
3103 GEPInfo GEPInfo;
3104
3105 for (unsigned i = 1; i != 3; ++i) {
3106 const MachineOperand &GEPOp = PtrMI->getOperand(i);
3107 const MachineInstr *OpDef = MRI.getUniqueVRegDef(Reg: GEPOp.getReg());
3108 assert(OpDef);
3109 if (i == 2 && isConstant(MI: *OpDef)) {
3110 // TODO: Could handle constant base + variable offset, but a combine
3111 // probably should have commuted it.
3112 assert(GEPInfo.Imm == 0);
3113 GEPInfo.Imm = OpDef->getOperand(i: 1).getCImm()->getSExtValue();
3114 continue;
3115 }
3116 const RegisterBank *OpBank = RBI.getRegBank(Reg: GEPOp.getReg(), MRI, TRI);
3117 if (OpBank->getID() == AMDGPU::SGPRRegBankID)
3118 GEPInfo.SgprParts.push_back(Elt: GEPOp.getReg());
3119 else
3120 GEPInfo.VgprParts.push_back(Elt: GEPOp.getReg());
3121 }
3122
3123 AddrInfo.push_back(Elt: GEPInfo);
3124 getAddrModeInfo(Load: *PtrMI, MRI, AddrInfo);
3125}
3126
3127bool AMDGPUInstructionSelector::isSGPR(Register Reg) const {
3128 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID;
3129}
3130
3131bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const {
3132 if (!MI.hasOneMemOperand())
3133 return false;
3134
3135 const MachineMemOperand *MMO = *MI.memoperands_begin();
3136 const Value *Ptr = MMO->getValue();
3137
3138 // UndefValue means this is a load of a kernel input. These are uniform.
3139 // Sometimes LDS instructions have constant pointers.
3140 // If Ptr is null, then that means this mem operand contains a
3141 // PseudoSourceValue like GOT.
3142 if (!Ptr || isa<UndefValue, Argument, Constant, GlobalValue>(Val: Ptr))
3143 return true;
3144
3145 if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
3146 return true;
3147
3148 if (MI.getOpcode() == AMDGPU::G_PREFETCH)
3149 return RBI.getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI: *MRI, TRI)->getID() ==
3150 AMDGPU::SGPRRegBankID;
3151
3152 const Instruction *I = dyn_cast<Instruction>(Val: Ptr);
3153 return I && I->getMetadata(Kind: "amdgpu.uniform");
3154}
3155
3156bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const {
3157 for (const GEPInfo &GEPInfo : AddrInfo) {
3158 if (!GEPInfo.VgprParts.empty())
3159 return true;
3160 }
3161 return false;
3162}
3163
3164void AMDGPUInstructionSelector::initM0(MachineInstr &I) const {
3165 const LLT PtrTy = MRI->getType(Reg: I.getOperand(i: 1).getReg());
3166 unsigned AS = PtrTy.getAddressSpace();
3167 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) &&
3168 STI.ldsRequiresM0Init()) {
3169 MachineBasicBlock *BB = I.getParent();
3170
3171 // If DS instructions require M0 initialization, insert it before selecting.
3172 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
3173 .addImm(Val: -1);
3174 }
3175}
3176
3177bool AMDGPUInstructionSelector::selectG_LOAD_STORE_ATOMICRMW(
3178 MachineInstr &I) const {
3179 initM0(I);
3180 return selectImpl(I, CoverageInfo&: *CoverageInfo);
3181}
3182
3183static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI) {
3184 if (Reg.isPhysical())
3185 return false;
3186
3187 MachineInstr &MI = *MRI.getUniqueVRegDef(Reg);
3188 const unsigned Opcode = MI.getOpcode();
3189
3190 if (Opcode == AMDGPU::COPY)
3191 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI);
3192
3193 if (Opcode == AMDGPU::G_AND || Opcode == AMDGPU::G_OR ||
3194 Opcode == AMDGPU::G_XOR)
3195 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI) &&
3196 isVCmpResult(Reg: MI.getOperand(i: 2).getReg(), MRI);
3197
3198 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI))
3199 return GI->is(ID: Intrinsic::amdgcn_class);
3200
3201 return Opcode == AMDGPU::G_ICMP || Opcode == AMDGPU::G_FCMP;
3202}
3203
3204bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
3205 MachineBasicBlock *BB = I.getParent();
3206 MachineOperand &CondOp = I.getOperand(i: 0);
3207 Register CondReg = CondOp.getReg();
3208 const DebugLoc &DL = I.getDebugLoc();
3209
3210 unsigned BrOpcode;
3211 Register CondPhysReg;
3212 const TargetRegisterClass *ConstrainRC;
3213
3214 // In SelectionDAG, we inspect the IR block for uniformity metadata to decide
3215 // whether the branch is uniform when selecting the instruction. In
3216 // GlobalISel, we should push that decision into RegBankSelect. Assume for now
3217 // RegBankSelect knows what it's doing if the branch condition is scc, even
3218 // though it currently does not.
3219 if (!isVCC(Reg: CondReg, MRI: *MRI)) {
3220 if (MRI->getType(Reg: CondReg) != LLT::scalar(SizeInBits: 32))
3221 return false;
3222
3223 CondPhysReg = AMDGPU::SCC;
3224 BrOpcode = AMDGPU::S_CBRANCH_SCC1;
3225 ConstrainRC = &AMDGPU::SReg_32RegClass;
3226 } else {
3227 // FIXME: Should scc->vcc copies and with exec?
3228
3229 // Unless the value of CondReg is a result of a V_CMP* instruction then we
3230 // need to insert an and with exec.
3231 if (!isVCmpResult(Reg: CondReg, MRI&: *MRI)) {
3232 const bool Is64 = STI.isWave64();
3233 const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
3234 const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO;
3235
3236 Register TmpReg = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
3237 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: TmpReg)
3238 .addReg(RegNo: CondReg)
3239 .addReg(RegNo: Exec)
3240 .setOperandDead(3); // Dead scc
3241 CondReg = TmpReg;
3242 }
3243
3244 CondPhysReg = TRI.getVCC();
3245 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ;
3246 ConstrainRC = TRI.getBoolRC();
3247 }
3248
3249 if (!MRI->getRegClassOrNull(Reg: CondReg))
3250 MRI->setRegClass(Reg: CondReg, RC: ConstrainRC);
3251
3252 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CondPhysReg)
3253 .addReg(RegNo: CondReg);
3254 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: BrOpcode))
3255 .addMBB(MBB: I.getOperand(i: 1).getMBB());
3256
3257 I.eraseFromParent();
3258 return true;
3259}
3260
3261bool AMDGPUInstructionSelector::selectG_GLOBAL_VALUE(
3262 MachineInstr &I) const {
3263 Register DstReg = I.getOperand(i: 0).getReg();
3264 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3265 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3266 I.setDesc(TII.get(Opcode: IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32));
3267 if (IsVGPR)
3268 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
3269
3270 return RBI.constrainGenericRegister(
3271 Reg: DstReg, RC: IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, MRI&: *MRI);
3272}
3273
3274bool AMDGPUInstructionSelector::selectG_PTRMASK(MachineInstr &I) const {
3275 Register DstReg = I.getOperand(i: 0).getReg();
3276 Register SrcReg = I.getOperand(i: 1).getReg();
3277 Register MaskReg = I.getOperand(i: 2).getReg();
3278 LLT Ty = MRI->getType(Reg: DstReg);
3279 LLT MaskTy = MRI->getType(Reg: MaskReg);
3280 MachineBasicBlock *BB = I.getParent();
3281 const DebugLoc &DL = I.getDebugLoc();
3282
3283 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3284 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3285 const RegisterBank *MaskRB = RBI.getRegBank(Reg: MaskReg, MRI: *MRI, TRI);
3286 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3287 if (DstRB != SrcRB) // Should only happen for hand written MIR.
3288 return false;
3289
3290 // Try to avoid emitting a bit operation when we only need to touch half of
3291 // the 64-bit pointer.
3292 APInt MaskOnes = VT->getKnownOnes(R: MaskReg).zext(width: 64);
3293 const APInt MaskHi32 = APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32);
3294 const APInt MaskLo32 = APInt::getLowBitsSet(numBits: 64, loBitsSet: 32);
3295
3296 const bool CanCopyLow32 = (MaskOnes & MaskLo32) == MaskLo32;
3297 const bool CanCopyHi32 = (MaskOnes & MaskHi32) == MaskHi32;
3298
3299 if (!IsVGPR && Ty.getSizeInBits() == 64 &&
3300 !CanCopyLow32 && !CanCopyHi32) {
3301 auto MIB = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B64), DestReg: DstReg)
3302 .addReg(RegNo: SrcReg)
3303 .addReg(RegNo: MaskReg)
3304 .setOperandDead(3); // Dead scc
3305 I.eraseFromParent();
3306 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3307 return true;
3308 }
3309
3310 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
3311 const TargetRegisterClass &RegRC
3312 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
3313
3314 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *DstRB);
3315 const TargetRegisterClass *SrcRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *SrcRB);
3316 const TargetRegisterClass *MaskRC =
3317 TRI.getRegClassForTypeOnBank(Ty: MaskTy, Bank: *MaskRB);
3318
3319 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3320 !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3321 !RBI.constrainGenericRegister(Reg: MaskReg, RC: *MaskRC, MRI&: *MRI))
3322 return false;
3323
3324 if (Ty.getSizeInBits() == 32) {
3325 assert(MaskTy.getSizeInBits() == 32 &&
3326 "ptrmask should have been narrowed during legalize");
3327
3328 auto NewOp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: DstReg)
3329 .addReg(RegNo: SrcReg)
3330 .addReg(RegNo: MaskReg);
3331
3332 if (!IsVGPR)
3333 NewOp.setOperandDead(3); // Dead scc
3334 I.eraseFromParent();
3335 return true;
3336 }
3337
3338 Register HiReg = MRI->createVirtualRegister(RegClass: &RegRC);
3339 Register LoReg = MRI->createVirtualRegister(RegClass: &RegRC);
3340
3341 // Extract the subregisters from the source pointer.
3342 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3343 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
3344 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3345 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
3346
3347 Register MaskedLo, MaskedHi;
3348
3349 if (CanCopyLow32) {
3350 // If all the bits in the low half are 1, we only need a copy for it.
3351 MaskedLo = LoReg;
3352 } else {
3353 // Extract the mask subregister and apply the and.
3354 Register MaskLo = MRI->createVirtualRegister(RegClass: &RegRC);
3355 MaskedLo = MRI->createVirtualRegister(RegClass: &RegRC);
3356
3357 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskLo)
3358 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub0);
3359 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedLo)
3360 .addReg(RegNo: LoReg)
3361 .addReg(RegNo: MaskLo);
3362 }
3363
3364 if (CanCopyHi32) {
3365 // If all the bits in the high half are 1, we only need a copy for it.
3366 MaskedHi = HiReg;
3367 } else {
3368 Register MaskHi = MRI->createVirtualRegister(RegClass: &RegRC);
3369 MaskedHi = MRI->createVirtualRegister(RegClass: &RegRC);
3370
3371 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskHi)
3372 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub1);
3373 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedHi)
3374 .addReg(RegNo: HiReg)
3375 .addReg(RegNo: MaskHi);
3376 }
3377
3378 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
3379 .addReg(RegNo: MaskedLo)
3380 .addImm(Val: AMDGPU::sub0)
3381 .addReg(RegNo: MaskedHi)
3382 .addImm(Val: AMDGPU::sub1);
3383 I.eraseFromParent();
3384 return true;
3385}
3386
3387/// Return the register to use for the index value, and the subregister to use
3388/// for the indirectly accessed register.
3389static std::pair<Register, unsigned>
3390computeIndirectRegIndex(MachineRegisterInfo &MRI, const SIRegisterInfo &TRI,
3391 const TargetRegisterClass *SuperRC, Register IdxReg,
3392 unsigned EltSize, GISelValueTracking &ValueTracking) {
3393 Register IdxBaseReg;
3394 int Offset;
3395
3396 std::tie(args&: IdxBaseReg, args&: Offset) =
3397 AMDGPU::getBaseWithConstantOffset(MRI, Reg: IdxReg, ValueTracking: &ValueTracking);
3398 if (IdxBaseReg == AMDGPU::NoRegister) {
3399 // This will happen if the index is a known constant. This should ordinarily
3400 // be legalized out, but handle it as a register just in case.
3401 assert(Offset == 0);
3402 IdxBaseReg = IdxReg;
3403 }
3404
3405 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SuperRC, EltSize);
3406
3407 // Skip out of bounds offsets, or else we would end up using an undefined
3408 // register.
3409 if (static_cast<unsigned>(Offset) >= SubRegs.size())
3410 return std::pair(IdxReg, SubRegs[0]);
3411 return std::pair(IdxBaseReg, SubRegs[Offset]);
3412}
3413
3414bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
3415 MachineInstr &MI) const {
3416 Register DstReg = MI.getOperand(i: 0).getReg();
3417 Register SrcReg = MI.getOperand(i: 1).getReg();
3418 Register IdxReg = MI.getOperand(i: 2).getReg();
3419
3420 LLT DstTy = MRI->getType(Reg: DstReg);
3421 LLT SrcTy = MRI->getType(Reg: SrcReg);
3422
3423 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3424 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3425 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3426
3427 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3428 // into a waterfall loop.
3429 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3430 return false;
3431
3432 const TargetRegisterClass *SrcRC =
3433 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcRB);
3434 const TargetRegisterClass *DstRC =
3435 TRI.getRegClassForTypeOnBank(Ty: DstTy, Bank: *DstRB);
3436 if (!SrcRC || !DstRC)
3437 return false;
3438 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3439 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3440 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3441 return false;
3442
3443 MachineBasicBlock *BB = MI.getParent();
3444 const DebugLoc &DL = MI.getDebugLoc();
3445 const bool Is64 = DstTy.getSizeInBits() == 64;
3446
3447 unsigned SubReg;
3448 std::tie(args&: IdxReg, args&: SubReg) = computeIndirectRegIndex(
3449 MRI&: *MRI, TRI, SuperRC: SrcRC, IdxReg, EltSize: DstTy.getSizeInBits() / 8, ValueTracking&: *VT);
3450
3451 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) {
3452 if (DstTy.getSizeInBits() != 32 && !Is64)
3453 return false;
3454
3455 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3456 .addReg(RegNo: IdxReg);
3457
3458 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32;
3459 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
3460 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3461 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3462 MI.eraseFromParent();
3463 return true;
3464 }
3465
3466 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32)
3467 return false;
3468
3469 if (!STI.useVGPRIndexMode()) {
3470 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3471 .addReg(RegNo: IdxReg);
3472 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: DstReg)
3473 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3474 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3475 MI.eraseFromParent();
3476 return true;
3477 }
3478
3479 const MCInstrDesc &GPRIDXDesc =
3480 TII.getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *SrcRC), IsIndirectSrc: true);
3481 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3482 .addReg(RegNo: SrcReg)
3483 .addReg(RegNo: IdxReg)
3484 .addImm(Val: SubReg);
3485
3486 MI.eraseFromParent();
3487 return true;
3488}
3489
3490// TODO: Fold insert_vector_elt (extract_vector_elt) into movrelsd
3491bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT(
3492 MachineInstr &MI) const {
3493 Register DstReg = MI.getOperand(i: 0).getReg();
3494 Register VecReg = MI.getOperand(i: 1).getReg();
3495 Register ValReg = MI.getOperand(i: 2).getReg();
3496 Register IdxReg = MI.getOperand(i: 3).getReg();
3497
3498 LLT VecTy = MRI->getType(Reg: DstReg);
3499 LLT ValTy = MRI->getType(Reg: ValReg);
3500 unsigned VecSize = VecTy.getSizeInBits();
3501 unsigned ValSize = ValTy.getSizeInBits();
3502
3503 const RegisterBank *VecRB = RBI.getRegBank(Reg: VecReg, MRI: *MRI, TRI);
3504 const RegisterBank *ValRB = RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
3505 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3506
3507 assert(VecTy.getElementType() == ValTy);
3508
3509 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3510 // into a waterfall loop.
3511 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3512 return false;
3513
3514 const TargetRegisterClass *VecRC =
3515 TRI.getRegClassForTypeOnBank(Ty: VecTy, Bank: *VecRB);
3516 const TargetRegisterClass *ValRC =
3517 TRI.getRegClassForTypeOnBank(Ty: ValTy, Bank: *ValRB);
3518
3519 if (!RBI.constrainGenericRegister(Reg: VecReg, RC: *VecRC, MRI&: *MRI) ||
3520 !RBI.constrainGenericRegister(Reg: DstReg, RC: *VecRC, MRI&: *MRI) ||
3521 !RBI.constrainGenericRegister(Reg: ValReg, RC: *ValRC, MRI&: *MRI) ||
3522 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3523 return false;
3524
3525 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32)
3526 return false;
3527
3528 unsigned SubReg;
3529 std::tie(args&: IdxReg, args&: SubReg) =
3530 computeIndirectRegIndex(MRI&: *MRI, TRI, SuperRC: VecRC, IdxReg, EltSize: ValSize / 8, ValueTracking&: *VT);
3531
3532 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID &&
3533 STI.useVGPRIndexMode();
3534
3535 MachineBasicBlock *BB = MI.getParent();
3536 const DebugLoc &DL = MI.getDebugLoc();
3537
3538 if (!IndexMode) {
3539 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3540 .addReg(RegNo: IdxReg);
3541
3542 const MCInstrDesc &RegWriteOp = TII.getIndirectRegWriteMovRelPseudo(
3543 VecSize, EltSize: ValSize, IsSGPR: VecRB->getID() == AMDGPU::SGPRRegBankID);
3544 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: RegWriteOp, DestReg: DstReg)
3545 .addReg(RegNo: VecReg)
3546 .addReg(RegNo: ValReg)
3547 .addImm(Val: SubReg);
3548 MI.eraseFromParent();
3549 return true;
3550 }
3551
3552 const MCInstrDesc &GPRIDXDesc =
3553 TII.getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
3554 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3555 .addReg(RegNo: VecReg)
3556 .addReg(RegNo: ValReg)
3557 .addReg(RegNo: IdxReg)
3558 .addImm(Val: SubReg);
3559
3560 MI.eraseFromParent();
3561 return true;
3562}
3563
3564static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
3565 switch (Intr) {
3566 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
3567 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
3568 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
3569 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
3570 case Intrinsic::amdgcn_load_async_to_lds:
3571 case Intrinsic::amdgcn_global_load_async_lds:
3572 return true;
3573 }
3574 return false;
3575}
3576
3577bool AMDGPUInstructionSelector::selectBufferLoadLds(MachineInstr &MI) const {
3578 if (!Subtarget->hasVMemToLDSLoad())
3579 return false;
3580 unsigned Opc;
3581 unsigned Size = MI.getOperand(i: 3).getImm();
3582 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3583
3584 // The struct intrinsic variants add one additional operand over raw.
3585 const bool HasVIndex = MI.getNumOperands() == 9;
3586 Register VIndex;
3587 int OpOffset = 0;
3588 if (HasVIndex) {
3589 VIndex = MI.getOperand(i: 4).getReg();
3590 OpOffset = 1;
3591 }
3592
3593 Register VOffset = MI.getOperand(i: 4 + OpOffset).getReg();
3594 std::optional<ValueAndVReg> MaybeVOffset =
3595 getIConstantVRegValWithLookThrough(VReg: VOffset, MRI: *MRI);
3596 const bool HasVOffset = !MaybeVOffset || MaybeVOffset->Value.getZExtValue();
3597
3598 switch (Size) {
3599 default:
3600 return false;
3601 case 1:
3602 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
3603 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
3604 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
3605 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
3606 break;
3607 case 2:
3608 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
3609 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
3610 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
3611 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
3612 break;
3613 case 4:
3614 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
3615 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
3616 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
3617 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
3618 break;
3619 case 12:
3620 if (!Subtarget->hasLDSLoadB96_B128())
3621 return false;
3622
3623 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
3624 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
3625 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
3626 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
3627 break;
3628 case 16:
3629 if (!Subtarget->hasLDSLoadB96_B128())
3630 return false;
3631
3632 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
3633 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
3634 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
3635 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
3636 break;
3637 }
3638
3639 MachineBasicBlock *MBB = MI.getParent();
3640 const DebugLoc &DL = MI.getDebugLoc();
3641 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3642 .add(MO: MI.getOperand(i: 2));
3643
3644 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc));
3645
3646 if (HasVIndex && HasVOffset) {
3647 Register IdxReg = MRI->createVirtualRegister(RegClass: TRI.getVGPR64Class());
3648 BuildMI(BB&: *MBB, I: &*MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: IdxReg)
3649 .addReg(RegNo: VIndex)
3650 .addImm(Val: AMDGPU::sub0)
3651 .addReg(RegNo: VOffset)
3652 .addImm(Val: AMDGPU::sub1);
3653
3654 MIB.addReg(RegNo: IdxReg);
3655 } else if (HasVIndex) {
3656 MIB.addReg(RegNo: VIndex);
3657 } else if (HasVOffset) {
3658 MIB.addReg(RegNo: VOffset);
3659 }
3660
3661 MIB.add(MO: MI.getOperand(i: 1)); // rsrc
3662 MIB.add(MO: MI.getOperand(i: 5 + OpOffset)); // soffset
3663 MIB.add(MO: MI.getOperand(i: 6 + OpOffset)); // imm offset
3664 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
3665 unsigned Aux = MI.getOperand(i: 7 + OpOffset).getImm();
3666 MIB.addImm(Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL
3667 : AMDGPU::CPol::ALL_pregfx12)); // cpol
3668 MIB.addImm(
3669 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
3670 ? 1
3671 : 0); // swz
3672 MIB.addImm(Val: isAsyncLDSDMA(Intr: IntrinsicID));
3673
3674 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3675 // Don't set the offset value here because the pointer points to the base of
3676 // the buffer.
3677 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3678
3679 MachinePointerInfo StorePtrI = LoadPtrI;
3680 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3681 AddressSpace: AMDGPUAS::BUFFER_RESOURCE));
3682 LoadPtrI.AddrSpace = AMDGPUAS::BUFFER_RESOURCE;
3683 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3684
3685 auto F = LoadMMO->getFlags() &
3686 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3687 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3688 Size, BaseAlignment: LoadMMO->getBaseAlign());
3689
3690 MachineMemOperand *StoreMMO =
3691 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3692 Size: sizeof(int32_t), BaseAlignment: LoadMMO->getBaseAlign());
3693
3694 MIB.setMemRefs({LoadMMO, StoreMMO});
3695
3696 MI.eraseFromParent();
3697 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3698 return true;
3699}
3700
3701/// Match a zero extend from a 32-bit value to 64-bits.
3702Register AMDGPUInstructionSelector::matchZeroExtendFromS32(Register Reg) const {
3703 Register ZExtSrc;
3704 if (mi_match(R: Reg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrc))))
3705 return MRI->getType(Reg: ZExtSrc) == LLT::scalar(SizeInBits: 32) ? ZExtSrc : Register();
3706
3707 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 0)
3708 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3709 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3710 return Register();
3711
3712 assert(Def->getNumOperands() == 3 &&
3713 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3714 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_ZeroInt())) {
3715 return Def->getOperand(i: 1).getReg();
3716 }
3717
3718 return Register();
3719}
3720
3721/// Match a sign extend from a 32-bit value to 64-bits.
3722Register AMDGPUInstructionSelector::matchSignExtendFromS32(Register Reg) const {
3723 Register SExtSrc;
3724 if (mi_match(R: Reg, MRI: *MRI, P: m_GSExt(Src: m_Reg(R&: SExtSrc))))
3725 return MRI->getType(Reg: SExtSrc) == LLT::scalar(SizeInBits: 32) ? SExtSrc : Register();
3726
3727 // Match legalized form %sext = G_MERGE_VALUES (s32 %x), G_ASHR((S32 %x, 31))
3728 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3729 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3730 return Register();
3731
3732 assert(Def->getNumOperands() == 3 &&
3733 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3734 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI,
3735 P: m_GAShr(L: m_SpecificReg(RequestedReg: Def->getOperand(i: 1).getReg()),
3736 R: m_SpecificICst(RequestedValue: 31))))
3737 return Def->getOperand(i: 1).getReg();
3738
3739 if (VT->signBitIsZero(Op: Reg))
3740 return matchZeroExtendFromS32(Reg);
3741
3742 return Register();
3743}
3744
3745/// Match a zero extend from a 32-bit value to 64-bits, or \p Reg itself if it
3746/// is 32-bit.
3747Register
3748AMDGPUInstructionSelector::matchZeroExtendFromS32OrS32(Register Reg) const {
3749 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3750 : matchZeroExtendFromS32(Reg);
3751}
3752
3753/// Match a sign extend from a 32-bit value to 64-bits, or \p Reg itself if it
3754/// is 32-bit.
3755Register
3756AMDGPUInstructionSelector::matchSignExtendFromS32OrS32(Register Reg) const {
3757 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3758 : matchSignExtendFromS32(Reg);
3759}
3760
3761Register
3762AMDGPUInstructionSelector::matchExtendFromS32OrS32(Register Reg,
3763 bool IsSigned) const {
3764 if (IsSigned)
3765 return matchSignExtendFromS32OrS32(Reg);
3766
3767 return matchZeroExtendFromS32OrS32(Reg);
3768}
3769
3770Register AMDGPUInstructionSelector::matchAnyExtendFromS32(Register Reg) const {
3771 Register AnyExtSrc;
3772 if (mi_match(R: Reg, MRI: *MRI, P: m_GAnyExt(Src: m_Reg(R&: AnyExtSrc))))
3773 return MRI->getType(Reg: AnyExtSrc) == LLT::scalar(SizeInBits: 32) ? AnyExtSrc : Register();
3774
3775 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 G_IMPLICIT_DEF)
3776 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3777 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3778 return Register();
3779
3780 assert(Def->getNumOperands() == 3 &&
3781 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3782
3783 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_GImplicitDef()))
3784 return Def->getOperand(i: 1).getReg();
3785
3786 return Register();
3787}
3788
3789bool AMDGPUInstructionSelector::selectGlobalLoadLds(MachineInstr &MI) const{
3790 if (!Subtarget->hasVMemToLDSLoad())
3791 return false;
3792
3793 unsigned Opc;
3794 unsigned Size = MI.getOperand(i: 3).getImm();
3795 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3796
3797 switch (Size) {
3798 default:
3799 return false;
3800 case 1:
3801 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
3802 break;
3803 case 2:
3804 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
3805 break;
3806 case 4:
3807 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
3808 break;
3809 case 12:
3810 if (!Subtarget->hasLDSLoadB96_B128())
3811 return false;
3812 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
3813 break;
3814 case 16:
3815 if (!Subtarget->hasLDSLoadB96_B128())
3816 return false;
3817 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
3818 break;
3819 }
3820
3821 MachineBasicBlock *MBB = MI.getParent();
3822 const DebugLoc &DL = MI.getDebugLoc();
3823 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3824 .add(MO: MI.getOperand(i: 2));
3825
3826 Register Addr = MI.getOperand(i: 1).getReg();
3827 Register VOffset;
3828 // Try to split SAddr and VOffset. Global and LDS pointers share the same
3829 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
3830 if (!isSGPR(Reg: Addr)) {
3831 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
3832 if (isSGPR(Reg: AddrDef->Reg)) {
3833 Addr = AddrDef->Reg;
3834 } else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
3835 Register SAddr =
3836 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
3837 if (isSGPR(Reg: SAddr)) {
3838 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
3839 if (Register Off = matchZeroExtendFromS32(Reg: PtrBaseOffset)) {
3840 Addr = SAddr;
3841 VOffset = Off;
3842 }
3843 }
3844 }
3845 }
3846
3847 if (isSGPR(Reg: Addr)) {
3848 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
3849 if (!VOffset) {
3850 VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
3851 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
3852 .addImm(Val: 0);
3853 }
3854 }
3855
3856 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc))
3857 .addReg(RegNo: Addr);
3858
3859 if (isSGPR(Reg: Addr))
3860 MIB.addReg(RegNo: VOffset);
3861
3862 MIB.add(MO: MI.getOperand(i: 4)); // offset
3863
3864 unsigned Aux = MI.getOperand(i: 5).getImm();
3865 MIB.addImm(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS); // cpol
3866 MIB.addImm(Val: isAsyncLDSDMA(Intr: IntrinsicID));
3867
3868 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3869 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3870 LoadPtrI.Offset = MI.getOperand(i: 4).getImm();
3871 MachinePointerInfo StorePtrI = LoadPtrI;
3872 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3873 AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
3874 LoadPtrI.AddrSpace = AMDGPUAS::GLOBAL_ADDRESS;
3875 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3876 auto F = LoadMMO->getFlags() &
3877 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3878 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3879 Size, BaseAlignment: LoadMMO->getBaseAlign());
3880 MachineMemOperand *StoreMMO =
3881 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3882 Size: sizeof(int32_t), BaseAlignment: Align(4));
3883
3884 MIB.setMemRefs({LoadMMO, StoreMMO});
3885
3886 MI.eraseFromParent();
3887 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3888 return true;
3889}
3890
3891bool AMDGPUInstructionSelector::selectTensorLoadStore(MachineInstr &MI,
3892 Intrinsic::ID IID) const {
3893 bool IsLoad = IID == Intrinsic::amdgcn_tensor_load_to_lds;
3894 unsigned Opc =
3895 IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d4 : AMDGPU::TENSOR_STORE_FROM_LDS_d4;
3896 int NumGroups = 4;
3897
3898 // A lamda function to check whether an operand is a vector of all 0s.
3899 const auto isAllZeros = [&](MachineOperand &Opnd) {
3900 const MachineInstr *DefMI = MRI->getVRegDef(Reg: Opnd.getReg());
3901 if (!DefMI)
3902 return false;
3903 return llvm::isBuildVectorAllZeros(MI: *DefMI, MRI: *MRI, AllowUndef: true);
3904 };
3905
3906 // Use _D2 version if both group 2 and 3 are zero-initialized.
3907 if (isAllZeros(MI.getOperand(i: 3)) && isAllZeros(MI.getOperand(i: 4))) {
3908 NumGroups = 2;
3909 Opc = IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d2
3910 : AMDGPU::TENSOR_STORE_FROM_LDS_d2;
3911 }
3912
3913 // TODO: Handle the fifth group: MI.getOpetand(5), which is silently ignored
3914 // for now because all existing targets only support up to 4 groups.
3915 MachineBasicBlock *MBB = MI.getParent();
3916 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
3917 .add(MO: MI.getOperand(i: 1)) // D# group 0
3918 .add(MO: MI.getOperand(i: 2)); // D# group 1
3919
3920 if (NumGroups >= 4) { // Has at least 4 groups
3921 MIB.add(MO: MI.getOperand(i: 3)) // D# group 2
3922 .add(MO: MI.getOperand(i: 4)); // D# group 3
3923 }
3924
3925 MIB.addImm(Val: 0) // r128
3926 .add(MO: MI.getOperand(i: 6)); // cpol
3927
3928 MI.eraseFromParent();
3929 return true;
3930}
3931
3932bool AMDGPUInstructionSelector::selectBVHIntersectRayIntrinsic(
3933 MachineInstr &MI) const {
3934 unsigned OpcodeOpIdx =
3935 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY ? 1 : 3;
3936 MI.setDesc(TII.get(Opcode: MI.getOperand(i: OpcodeOpIdx).getImm()));
3937 MI.removeOperand(OpNo: OpcodeOpIdx);
3938 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
3939 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
3940 return true;
3941}
3942
3943// FIXME: This should be removed and let the patterns select. We just need the
3944// AGPR/VGPR combination versions.
3945bool AMDGPUInstructionSelector::selectSMFMACIntrin(MachineInstr &MI) const {
3946 unsigned Opc;
3947 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
3948 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
3949 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64;
3950 break;
3951 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
3952 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64;
3953 break;
3954 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
3955 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64;
3956 break;
3957 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
3958 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64;
3959 break;
3960 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
3961 Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64;
3962 break;
3963 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
3964 Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64;
3965 break;
3966 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
3967 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64;
3968 break;
3969 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
3970 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64;
3971 break;
3972 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
3973 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64;
3974 break;
3975 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
3976 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64;
3977 break;
3978 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
3979 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64;
3980 break;
3981 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
3982 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64;
3983 break;
3984 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
3985 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64;
3986 break;
3987 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
3988 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64;
3989 break;
3990 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
3991 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_F16_e64;
3992 break;
3993 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
3994 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_F16_e64;
3995 break;
3996 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
3997 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF16_e64;
3998 break;
3999 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
4000 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF16_e64;
4001 break;
4002 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
4003 Opc = AMDGPU::V_SMFMAC_I32_16X16X128_I8_e64;
4004 break;
4005 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
4006 Opc = AMDGPU::V_SMFMAC_I32_32X32X64_I8_e64;
4007 break;
4008 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
4009 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_BF8_e64;
4010 break;
4011 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
4012 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_FP8_e64;
4013 break;
4014 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
4015 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_BF8_e64;
4016 break;
4017 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
4018 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_FP8_e64;
4019 break;
4020 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
4021 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_BF8_e64;
4022 break;
4023 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
4024 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_FP8_e64;
4025 break;
4026 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
4027 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_BF8_e64;
4028 break;
4029 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
4030 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_FP8_e64;
4031 break;
4032 default:
4033 llvm_unreachable("unhandled smfmac intrinsic");
4034 }
4035
4036 auto VDst_In = MI.getOperand(i: 4);
4037
4038 MI.setDesc(TII.get(Opcode: Opc));
4039 MI.removeOperand(OpNo: 4); // VDst_In
4040 MI.removeOperand(OpNo: 1); // Intrinsic ID
4041 MI.addOperand(Op: VDst_In); // Readd VDst_In to the end
4042 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
4043 const MCInstrDesc &MCID = MI.getDesc();
4044 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
4045 MI.getOperand(i: 0).setIsEarlyClobber(true);
4046 }
4047 return true;
4048}
4049
4050bool AMDGPUInstructionSelector::selectPermlaneSwapIntrin(
4051 MachineInstr &MI, Intrinsic::ID IntrID) const {
4052 if (IntrID == Intrinsic::amdgcn_permlane16_swap &&
4053 !Subtarget->hasPermlane16Swap())
4054 return false;
4055 if (IntrID == Intrinsic::amdgcn_permlane32_swap &&
4056 !Subtarget->hasPermlane32Swap())
4057 return false;
4058
4059 unsigned Opcode = IntrID == Intrinsic::amdgcn_permlane16_swap
4060 ? AMDGPU::V_PERMLANE16_SWAP_B32_e64
4061 : AMDGPU::V_PERMLANE32_SWAP_B32_e64;
4062
4063 MI.removeOperand(OpNo: 2);
4064 MI.setDesc(TII.get(Opcode));
4065 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
4066
4067 MachineOperand &FI = MI.getOperand(i: 4);
4068 FI.setImm(FI.getImm() ? AMDGPU::DPP::DPP_FI_1 : AMDGPU::DPP::DPP_FI_0);
4069
4070 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
4071 return true;
4072}
4073
4074bool AMDGPUInstructionSelector::selectWaveAddress(MachineInstr &MI) const {
4075 Register DstReg = MI.getOperand(i: 0).getReg();
4076 Register SrcReg = MI.getOperand(i: 1).getReg();
4077 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4078 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4079 MachineBasicBlock *MBB = MI.getParent();
4080 const DebugLoc &DL = MI.getDebugLoc();
4081
4082 if (IsVALU) {
4083 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: DstReg)
4084 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4085 .addReg(RegNo: SrcReg);
4086 } else {
4087 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: DstReg)
4088 .addReg(RegNo: SrcReg)
4089 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4090 .setOperandDead(3); // Dead scc
4091 }
4092
4093 const TargetRegisterClass &RC =
4094 IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
4095 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
4096 return false;
4097
4098 MI.eraseFromParent();
4099 return true;
4100}
4101
4102bool AMDGPUInstructionSelector::selectWaveShuffleIntrin(
4103 MachineInstr &MI) const {
4104 assert(MI.getNumOperands() == 4);
4105 MachineBasicBlock *MBB = MI.getParent();
4106 const DebugLoc &DL = MI.getDebugLoc();
4107
4108 Register DstReg = MI.getOperand(i: 0).getReg();
4109 Register ValReg = MI.getOperand(i: 2).getReg();
4110 Register IdxReg = MI.getOperand(i: 3).getReg();
4111
4112 const LLT DstTy = MRI->getType(Reg: DstReg);
4113 unsigned DstSize = DstTy.getSizeInBits();
4114 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4115 const TargetRegisterClass *DstRC =
4116 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
4117
4118 if (DstTy != LLT::scalar(SizeInBits: 32))
4119 return false;
4120
4121 if (!Subtarget->supportsBPermute())
4122 return false;
4123
4124 // If we can bpermute across the whole wave, then just do that
4125 if (Subtarget->supportsWaveWideBPermute()) {
4126 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4127 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4128 .addImm(Val: 2)
4129 .addReg(RegNo: IdxReg);
4130
4131 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: DstReg)
4132 .addReg(RegNo: ShiftIdxReg)
4133 .addReg(RegNo: ValReg)
4134 .addImm(Val: 0);
4135 } else {
4136 // Otherwise, we need to make use of whole wave mode
4137 assert(Subtarget->isWave64());
4138
4139 // Set inactive lanes to poison
4140 Register UndefValReg =
4141 MRI->createVirtualRegister(RegClass: TRI.getRegClass(i: AMDGPU::SReg_32RegClassID));
4142 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefValReg);
4143
4144 Register UndefExecReg = MRI->createVirtualRegister(
4145 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4146 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExecReg);
4147
4148 Register PoisonValReg = MRI->createVirtualRegister(RegClass: DstRC);
4149 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonValReg)
4150 .addImm(Val: 0)
4151 .addReg(RegNo: ValReg)
4152 .addImm(Val: 0)
4153 .addReg(RegNo: UndefValReg)
4154 .addReg(RegNo: UndefExecReg);
4155
4156 // ds_bpermute requires index to be multiplied by 4
4157 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4158 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4159 .addImm(Val: 2)
4160 .addReg(RegNo: IdxReg);
4161
4162 Register PoisonIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4163 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonIdxReg)
4164 .addImm(Val: 0)
4165 .addReg(RegNo: ShiftIdxReg)
4166 .addImm(Val: 0)
4167 .addReg(RegNo: UndefValReg)
4168 .addReg(RegNo: UndefExecReg);
4169
4170 Register PoisonUnshiftedIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4171 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32),
4172 DestReg: PoisonUnshiftedIdxReg)
4173 .addImm(Val: 0)
4174 .addReg(RegNo: IdxReg)
4175 .addImm(Val: 0)
4176 .addReg(RegNo: UndefValReg)
4177 .addReg(RegNo: UndefExecReg);
4178
4179 // Get permutation of each half, then we'll select which one to use
4180 Register SameSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4181 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: SameSidePermReg)
4182 .addReg(RegNo: PoisonIdxReg)
4183 .addReg(RegNo: PoisonValReg)
4184 .addImm(Val: 0);
4185
4186 Register SwappedValReg = MRI->createVirtualRegister(RegClass: DstRC);
4187 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_PERMLANE64_B32), DestReg: SwappedValReg)
4188 .addReg(RegNo: PoisonValReg);
4189
4190 Register OppSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4191 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: OppSidePermReg)
4192 .addReg(RegNo: PoisonIdxReg)
4193 .addReg(RegNo: SwappedValReg)
4194 .addImm(Val: 0);
4195
4196 Register WWMSwapPermReg = MRI->createVirtualRegister(RegClass: DstRC);
4197 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::STRICT_WWM), DestReg: WWMSwapPermReg)
4198 .addReg(RegNo: OppSidePermReg);
4199
4200 // Select which side to take the permute from
4201 // We can get away with only using mbcnt_lo here since we're only
4202 // trying to detect which side of 32 each lane is on, and mbcnt_lo
4203 // returns 32 for lanes 32-63.
4204 Register ThreadIDReg = MRI->createVirtualRegister(RegClass: DstRC);
4205 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64), DestReg: ThreadIDReg)
4206 .addImm(Val: -1)
4207 .addImm(Val: 0);
4208
4209 Register XORReg = MRI->createVirtualRegister(RegClass: DstRC);
4210 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_XOR_B32_e64), DestReg: XORReg)
4211 .addReg(RegNo: ThreadIDReg)
4212 .addReg(RegNo: PoisonUnshiftedIdxReg);
4213
4214 Register ANDReg = MRI->createVirtualRegister(RegClass: DstRC);
4215 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: ANDReg)
4216 .addReg(RegNo: XORReg)
4217 .addImm(Val: 32);
4218
4219 Register CompareReg = MRI->createVirtualRegister(
4220 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4221 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CompareReg)
4222 .addReg(RegNo: ANDReg)
4223 .addImm(Val: 0);
4224
4225 // Finally do the selection
4226 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
4227 .addImm(Val: 0)
4228 .addReg(RegNo: WWMSwapPermReg)
4229 .addImm(Val: 0)
4230 .addReg(RegNo: SameSidePermReg)
4231 .addReg(RegNo: CompareReg);
4232 }
4233
4234 MI.eraseFromParent();
4235 return true;
4236}
4237
4238// Match BITOP3 operation and return a number of matched instructions plus
4239// truth table.
4240static std::pair<unsigned, uint8_t> BitOp3_Op(Register R,
4241 SmallVectorImpl<Register> &Src,
4242 const MachineRegisterInfo &MRI) {
4243 unsigned NumOpcodes = 0;
4244 uint8_t LHSBits, RHSBits;
4245
4246 auto getOperandBits = [&Src, R, &MRI](Register Op, uint8_t &Bits) -> bool {
4247 // Define truth table given Src0, Src1, Src2 bits permutations:
4248 // 0 0 0
4249 // 0 0 1
4250 // 0 1 0
4251 // 0 1 1
4252 // 1 0 0
4253 // 1 0 1
4254 // 1 1 0
4255 // 1 1 1
4256 const uint8_t SrcBits[3] = { 0xf0, 0xcc, 0xaa };
4257
4258 if (mi_match(R: Op, MRI, P: m_AllOnesInt())) {
4259 Bits = 0xff;
4260 return true;
4261 }
4262 if (mi_match(R: Op, MRI, P: m_ZeroInt())) {
4263 Bits = 0;
4264 return true;
4265 }
4266
4267 for (unsigned I = 0; I < Src.size(); ++I) {
4268 // Try to find existing reused operand
4269 if (Src[I] == Op) {
4270 Bits = SrcBits[I];
4271 return true;
4272 }
4273 // Try to replace parent operator
4274 if (Src[I] == R) {
4275 Bits = SrcBits[I];
4276 Src[I] = Op;
4277 return true;
4278 }
4279 }
4280
4281 if (Src.size() == 3) {
4282 // No room left for operands. Try one last time, there can be a 'not' of
4283 // one of our source operands. In this case we can compute the bits
4284 // without growing Src vector.
4285 Register LHS;
4286 if (mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: LHS)))) {
4287 LHS = getSrcRegIgnoringCopies(Reg: LHS, MRI);
4288 for (unsigned I = 0; I < Src.size(); ++I) {
4289 if (Src[I] == LHS) {
4290 Bits = ~SrcBits[I];
4291 return true;
4292 }
4293 }
4294 }
4295
4296 return false;
4297 }
4298
4299 Bits = SrcBits[Src.size()];
4300 Src.push_back(Elt: Op);
4301 return true;
4302 };
4303
4304 MachineInstr *MI = MRI.getVRegDef(Reg: R);
4305 switch (MI->getOpcode()) {
4306 case TargetOpcode::G_AND:
4307 case TargetOpcode::G_OR:
4308 case TargetOpcode::G_XOR: {
4309 Register LHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 1).getReg(), MRI);
4310 Register RHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 2).getReg(), MRI);
4311
4312 SmallVector<Register, 3> Backup(Src.begin(), Src.end());
4313 if (!getOperandBits(LHS, LHSBits) ||
4314 !getOperandBits(RHS, RHSBits)) {
4315 Src = std::move(Backup);
4316 return std::make_pair(x: 0, y: 0);
4317 }
4318
4319 // Recursion is naturally limited by the size of the operand vector.
4320 //
4321 // When LHS and RHS share a common sub-expression, one side's recursion
4322 // may decompose that sub-expression and replace the Src slot the other
4323 // side occupies with sub-operands via the "replace parent" path in
4324 // getOperandBits. The other side's cached bit-pattern then refers to a
4325 // slot whose contents changed, producing a wrong truth table.
4326 //
4327 // We detect this in three ways:
4328 // (A) If LHS recursed, its truth table is valid against the Src state
4329 // when LHS recursion completed (SrcAfterLHS). If RHS recursion
4330 // then mutates a Src slot that LHSBits depends on, LHSBits is
4331 // stale.
4332 // (B) If RHS did not recurse, RHSBits came from getOperandBits and
4333 // refers to a specific Src slot. If that slot's contents changed
4334 // (by either recursion), RHSBits is stale.
4335 // (C) Symmetrically for LHS if it did not recurse.
4336 SmallVector<Register, 3> SrcBeforeRecurse(Src.begin(), Src.end());
4337 uint8_t LHSBitsOrig = LHSBits;
4338 uint8_t RHSBitsOrig = RHSBits;
4339
4340 auto LHSOp = BitOp3_Op(R: LHS, Src, MRI);
4341 if (LHSOp.first) {
4342 NumOpcodes += LHSOp.first;
4343 LHSBits = LHSOp.second;
4344 }
4345
4346 SmallVector<Register, 3> SrcAfterLHS(Src.begin(), Src.end());
4347
4348 auto RHSOp = BitOp3_Op(R: RHS, Src, MRI);
4349 if (RHSOp.first) {
4350 NumOpcodes += RHSOp.first;
4351 RHSBits = RHSOp.second;
4352 }
4353
4354 // dependsOnSlot: true iff the truth table TT varies with slot Slot.
4355 auto dependsOnSlot = [](uint8_t TT, int Slot) -> bool {
4356 if (Slot < 0 || Slot > 2)
4357 return false;
4358 const uint8_t Masks[3] = {0x0f, 0x33, 0x55};
4359 const int Shifts[3] = {4, 2, 1};
4360 return ((TT ^ (TT >> Shifts[Slot])) & Masks[Slot]) != 0;
4361 };
4362
4363 // findSlot: locate the Src slot a getOperandBits result depends on,
4364 // including negated (NOT) patterns that getOperandBits resolves via
4365 // the ~SrcBits[I] shortcut.
4366 const uint8_t SrcBitsConst[3] = {0xf0, 0xcc, 0xaa};
4367 auto findSlot = [&](uint8_t Bits, Register Op,
4368 const SmallVectorImpl<Register> &S) -> int {
4369 Register NegatedInner;
4370 bool IsNegationOp = mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: NegatedInner)));
4371 if (IsNegationOp)
4372 NegatedInner = getSrcRegIgnoringCopies(Reg: NegatedInner, MRI);
4373 for (int I = 0; I < (int)S.size(); I++) {
4374 if (Bits == SrcBitsConst[I] && S[I] == Op)
4375 return I;
4376 if (IsNegationOp && Bits == (uint8_t)~SrcBitsConst[I] &&
4377 S[I] == NegatedInner)
4378 return I;
4379 }
4380 return -1;
4381 };
4382
4383 bool Stale = false;
4384
4385 // (A) LHS recursed: its truth table is against SrcAfterLHS.
4386 // Check if RHS recursion mutated a slot that LHSBits uses.
4387 if (LHSOp.first) {
4388 for (int I = 0; I < (int)SrcAfterLHS.size() && I < 3; I++) {
4389 if (I < (int)Src.size() && Src[I] != SrcAfterLHS[I] &&
4390 dependsOnSlot(LHSBits, I)) {
4391 Stale = true;
4392 break;
4393 }
4394 }
4395 }
4396
4397 // (B) RHS did not recurse: RHSBits from getOperandBits is against
4398 // SrcBeforeRecurse. Check if that slot was mutated since then.
4399 if (!Stale && !RHSOp.first) {
4400 int Slot = findSlot(RHSBitsOrig, RHS, SrcBeforeRecurse);
4401 if (Slot >= 0 &&
4402 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4403 Stale = true;
4404 }
4405
4406 // (C) LHS did not recurse: LHSBits from getOperandBits is against
4407 // SrcBeforeRecurse. Check if that slot was mutated since then.
4408 if (!Stale && !LHSOp.first) {
4409 int Slot = findSlot(LHSBitsOrig, LHS, SrcBeforeRecurse);
4410 if (Slot >= 0 &&
4411 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4412 Stale = true;
4413 }
4414
4415 if (Stale) {
4416 Src = std::move(SrcBeforeRecurse);
4417 LHSBits = LHSBitsOrig;
4418 RHSBits = RHSBitsOrig;
4419 NumOpcodes = 0;
4420 }
4421 break;
4422 }
4423 default:
4424 return std::make_pair(x: 0, y: 0);
4425 }
4426
4427 uint8_t TTbl;
4428 switch (MI->getOpcode()) {
4429 case TargetOpcode::G_AND:
4430 TTbl = LHSBits & RHSBits;
4431 break;
4432 case TargetOpcode::G_OR:
4433 TTbl = LHSBits | RHSBits;
4434 break;
4435 case TargetOpcode::G_XOR:
4436 TTbl = LHSBits ^ RHSBits;
4437 break;
4438 default:
4439 break;
4440 }
4441
4442 return std::make_pair(x: NumOpcodes + 1, y&: TTbl);
4443}
4444
4445bool AMDGPUInstructionSelector::selectBITOP3(MachineInstr &MI) const {
4446 if (!Subtarget->hasBitOp3Insts())
4447 return false;
4448
4449 Register DstReg = MI.getOperand(i: 0).getReg();
4450 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4451 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4452 if (!IsVALU)
4453 return false;
4454
4455 SmallVector<Register, 3> Src;
4456 uint8_t TTbl;
4457 unsigned NumOpcodes;
4458
4459 std::tie(args&: NumOpcodes, args&: TTbl) = BitOp3_Op(R: DstReg, Src, MRI: *MRI);
4460
4461 // Src.empty() case can happen if all operands are all zero or all ones.
4462 // Normally it shall be optimized out before reaching this.
4463 if (NumOpcodes < 2 || Src.empty())
4464 return false;
4465
4466 const bool IsB32 = MRI->getType(Reg: DstReg) == LLT::scalar(SizeInBits: 32);
4467 if (NumOpcodes == 2 && IsB32) {
4468 // Avoid using BITOP3 for OR3, XOR3, AND_OR. This is not faster but makes
4469 // asm more readable. This cannot be modeled with AddedComplexity because
4470 // selector does not know how many operations did we match.
4471 if (mi_match(MI, MRI: *MRI, P: m_GXor(L: m_GXor(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4472 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GOr(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4473 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GAnd(L: m_Reg(), R: m_Reg()), R: m_Reg())))
4474 return false;
4475 } else if (NumOpcodes < 4) {
4476 // For a uniform case threshold should be higher to account for moves
4477 // between VGPRs and SGPRs. It needs one operand in a VGPR, rest two can be
4478 // in SGPRs and a readtfirstlane after.
4479 return false;
4480 }
4481
4482 unsigned Opc = IsB32 ? AMDGPU::V_BITOP3_B32_e64 : AMDGPU::V_BITOP3_B16_e64;
4483 if (!IsB32 && STI.hasTrue16BitInsts())
4484 Opc = STI.useRealTrue16Insts() ? AMDGPU::V_BITOP3_B16_gfx1250_t16_e64
4485 : AMDGPU::V_BITOP3_B16_gfx1250_fake16_e64;
4486 unsigned CBL = STI.getConstantBusLimit(Opcode: Opc);
4487 MachineBasicBlock *MBB = MI.getParent();
4488 const DebugLoc &DL = MI.getDebugLoc();
4489
4490 for (unsigned I = 0; I < Src.size(); ++I) {
4491 const RegisterBank *RB = RBI.getRegBank(Reg: Src[I], MRI: *MRI, TRI);
4492 if (RB->getID() != AMDGPU::SGPRRegBankID)
4493 continue;
4494 if (CBL > 0) {
4495 --CBL;
4496 continue;
4497 }
4498 Register NewReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
4499 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: NewReg)
4500 .addReg(RegNo: Src[I]);
4501 Src[I] = NewReg;
4502 }
4503
4504 // Last operand can be ignored, turning a ternary operation into a binary.
4505 // For example: (~a & b & c) | (~a & b & ~c) -> (~a & b). We can replace
4506 // 'c' with 'a' here without changing the answer. In some pathological
4507 // cases it should be possible to get an operation with a single operand
4508 // too if optimizer would not catch it.
4509 while (Src.size() < 3)
4510 Src.push_back(Elt: Src[0]);
4511
4512 auto MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg);
4513 if (!IsB32)
4514 MIB.addImm(Val: 0); // src_mod0
4515 MIB.addReg(RegNo: Src[0]);
4516 if (!IsB32)
4517 MIB.addImm(Val: 0); // src_mod1
4518 MIB.addReg(RegNo: Src[1]);
4519 if (!IsB32)
4520 MIB.addImm(Val: 0); // src_mod2
4521 MIB.addReg(RegNo: Src[2])
4522 .addImm(Val: TTbl);
4523 if (!IsB32)
4524 MIB.addImm(Val: 0); // op_sel
4525
4526 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
4527 MI.eraseFromParent();
4528
4529 return true;
4530}
4531
4532bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const {
4533 Register SrcReg = MI.getOperand(i: 0).getReg();
4534 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
4535 return false;
4536
4537 MachineInstr *DefMI = MRI->getVRegDef(Reg: SrcReg);
4538 Register SP =
4539 Subtarget->getTargetLowering()->getStackPointerRegisterToSaveRestore();
4540 Register WaveAddr = getWaveAddress(Def: DefMI);
4541 MachineBasicBlock *MBB = MI.getParent();
4542 const DebugLoc &DL = MI.getDebugLoc();
4543
4544 if (!WaveAddr) {
4545 WaveAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
4546 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: WaveAddr)
4547 .addReg(RegNo: SrcReg)
4548 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4549 .setOperandDead(3); // Dead scc
4550 }
4551
4552 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: SP)
4553 .addReg(RegNo: WaveAddr);
4554
4555 MI.eraseFromParent();
4556 return true;
4557}
4558
4559bool AMDGPUInstructionSelector::select(MachineInstr &I) {
4560
4561 if (!I.isPreISelOpcode()) {
4562 if (I.isCopy())
4563 return selectCOPY(I);
4564 return true;
4565 }
4566
4567 switch (I.getOpcode()) {
4568 case TargetOpcode::G_AND:
4569 case TargetOpcode::G_OR:
4570 case TargetOpcode::G_XOR:
4571 if (selectBITOP3(MI&: I))
4572 return true;
4573 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4574 return true;
4575 return selectG_AND_OR_XOR(I);
4576 case TargetOpcode::G_ADD:
4577 case TargetOpcode::G_SUB:
4578 case TargetOpcode::G_PTR_ADD:
4579 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4580 return true;
4581 return selectG_ADD_SUB(I);
4582 case TargetOpcode::G_UADDO:
4583 case TargetOpcode::G_USUBO:
4584 case TargetOpcode::G_UADDE:
4585 case TargetOpcode::G_USUBE:
4586 return selectG_UADDO_USUBO_UADDE_USUBE(I);
4587 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4588 case AMDGPU::G_AMDGPU_MAD_I64_I32:
4589 return selectG_AMDGPU_MAD_64_32(I);
4590 case TargetOpcode::G_INTTOPTR:
4591 case TargetOpcode::G_BITCAST:
4592 case TargetOpcode::G_PTRTOINT:
4593 case TargetOpcode::G_FREEZE:
4594 return selectCOPY(I);
4595 case TargetOpcode::G_FNEG:
4596 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4597 return true;
4598 return selectG_FNEG(MI&: I);
4599 case TargetOpcode::G_FABS:
4600 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4601 return true;
4602 return selectG_FABS(MI&: I);
4603 case TargetOpcode::G_EXTRACT:
4604 return selectG_EXTRACT(I);
4605 case TargetOpcode::G_MERGE_VALUES:
4606 case TargetOpcode::G_CONCAT_VECTORS:
4607 return selectG_MERGE_VALUES(MI&: I);
4608 case TargetOpcode::G_UNMERGE_VALUES:
4609 return selectG_UNMERGE_VALUES(MI&: I);
4610 case TargetOpcode::G_BUILD_VECTOR:
4611 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
4612 return selectG_BUILD_VECTOR(MI&: I);
4613 case TargetOpcode::G_IMPLICIT_DEF:
4614 return selectG_IMPLICIT_DEF(I);
4615 case TargetOpcode::G_INSERT:
4616 return selectG_INSERT(I);
4617 case TargetOpcode::G_INTRINSIC:
4618 case TargetOpcode::G_INTRINSIC_CONVERGENT:
4619 return selectG_INTRINSIC(I);
4620 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
4621 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
4622 return selectG_INTRINSIC_W_SIDE_EFFECTS(I);
4623 case TargetOpcode::G_ICMP:
4624 case TargetOpcode::G_FCMP:
4625 if (selectG_ICMP_or_FCMP(I))
4626 return true;
4627 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4628 case TargetOpcode::G_LOAD:
4629 case TargetOpcode::G_ZEXTLOAD:
4630 case TargetOpcode::G_SEXTLOAD:
4631 case TargetOpcode::G_STORE:
4632 case TargetOpcode::G_ATOMIC_CMPXCHG:
4633 case TargetOpcode::G_ATOMICRMW_XCHG:
4634 case TargetOpcode::G_ATOMICRMW_ADD:
4635 case TargetOpcode::G_ATOMICRMW_SUB:
4636 case TargetOpcode::G_ATOMICRMW_AND:
4637 case TargetOpcode::G_ATOMICRMW_OR:
4638 case TargetOpcode::G_ATOMICRMW_XOR:
4639 case TargetOpcode::G_ATOMICRMW_MIN:
4640 case TargetOpcode::G_ATOMICRMW_MAX:
4641 case TargetOpcode::G_ATOMICRMW_UMIN:
4642 case TargetOpcode::G_ATOMICRMW_UMAX:
4643 case TargetOpcode::G_ATOMICRMW_UINC_WRAP:
4644 case TargetOpcode::G_ATOMICRMW_UDEC_WRAP:
4645 case TargetOpcode::G_ATOMICRMW_USUB_COND:
4646 case TargetOpcode::G_ATOMICRMW_USUB_SAT:
4647 case TargetOpcode::G_ATOMICRMW_FADD:
4648 case TargetOpcode::G_ATOMICRMW_FMIN:
4649 case TargetOpcode::G_ATOMICRMW_FMAX:
4650 return selectG_LOAD_STORE_ATOMICRMW(I);
4651 case TargetOpcode::G_SELECT:
4652 return selectG_SELECT(I);
4653 case TargetOpcode::G_TRUNC:
4654 return selectG_TRUNC(I);
4655 case TargetOpcode::G_SEXT:
4656 case TargetOpcode::G_ZEXT:
4657 case TargetOpcode::G_ANYEXT:
4658 case TargetOpcode::G_SEXT_INREG:
4659 // This is a workaround. For extension from type i1, `selectImpl()` uses
4660 // patterns from TD file and generates an illegal VGPR to SGPR COPY as type
4661 // i1 can only be hold in a SGPR class.
4662 if (MRI->getType(Reg: I.getOperand(i: 1).getReg()) != LLT::scalar(SizeInBits: 1) &&
4663 selectImpl(I, CoverageInfo&: *CoverageInfo))
4664 return true;
4665 return selectG_SZA_EXT(I);
4666 case TargetOpcode::G_FPEXT:
4667 if (selectG_FPEXT(I))
4668 return true;
4669 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4670 case TargetOpcode::G_BRCOND:
4671 return selectG_BRCOND(I);
4672 case TargetOpcode::G_GLOBAL_VALUE:
4673 return selectG_GLOBAL_VALUE(I);
4674 case TargetOpcode::G_PTRMASK:
4675 return selectG_PTRMASK(I);
4676 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4677 return selectG_EXTRACT_VECTOR_ELT(MI&: I);
4678 case TargetOpcode::G_INSERT_VECTOR_ELT:
4679 return selectG_INSERT_VECTOR_ELT(MI&: I);
4680 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
4681 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
4682 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
4683 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
4684 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
4685 const AMDGPU::ImageDimIntrinsicInfo *Intr =
4686 AMDGPU::getImageDimIntrinsicInfo(Intr: AMDGPU::getIntrinsicID(I));
4687 assert(Intr && "not an image intrinsic with image pseudo");
4688 return selectImageIntrinsic(MI&: I, Intr);
4689 }
4690 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY:
4691 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
4692 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
4693 return selectBVHIntersectRayIntrinsic(MI&: I);
4694 case AMDGPU::G_SBFX:
4695 case AMDGPU::G_UBFX:
4696 return selectG_SBFX_UBFX(MI&: I);
4697 case AMDGPU::G_SI_CALL:
4698 I.setDesc(TII.get(Opcode: AMDGPU::SI_CALL));
4699 return true;
4700 case AMDGPU::G_AMDGPU_WAVE_ADDRESS:
4701 return selectWaveAddress(MI&: I);
4702 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN: {
4703 I.setDesc(TII.get(Opcode: AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN));
4704 return true;
4705 }
4706 case AMDGPU::G_STACKRESTORE:
4707 return selectStackRestore(MI&: I);
4708 case AMDGPU::G_PHI:
4709 return selectPHI(I);
4710 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
4711 return selectCOPY_SCC_VCC(I);
4712 case AMDGPU::G_AMDGPU_COPY_VCC_SCC:
4713 return selectCOPY_VCC_SCC(I);
4714 case AMDGPU::G_AMDGPU_READANYLANE:
4715 return selectReadAnyLane(I);
4716 case TargetOpcode::G_CONSTANT:
4717 case TargetOpcode::G_FCONSTANT:
4718 default:
4719 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4720 }
4721 return false;
4722}
4723
4724InstructionSelector::ComplexRendererFns
4725AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const {
4726 return {{
4727 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4728 }};
4729
4730}
4731
4732std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3ModsImpl(
4733 Register Src, bool IsCanonicalizing, bool AllowAbs, bool OpSel) const {
4734 unsigned Mods = 0;
4735 MachineInstr *MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4736
4737 if (MI->getOpcode() == AMDGPU::G_FNEG) {
4738 Src = MI->getOperand(i: 1).getReg();
4739 Mods |= SISrcMods::NEG;
4740 MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4741 } else if (MI->getOpcode() == AMDGPU::G_FSUB && IsCanonicalizing) {
4742 // Fold fsub [+-]0 into fneg. This may not have folded depending on the
4743 // denormal mode, but we're implicitly canonicalizing in a source operand.
4744 const ConstantFP *LHS =
4745 getConstantFPVRegVal(VReg: MI->getOperand(i: 1).getReg(), MRI: *MRI);
4746 if (LHS && LHS->isZero()) {
4747 Mods |= SISrcMods::NEG;
4748 Src = MI->getOperand(i: 2).getReg();
4749 }
4750 }
4751
4752 if (AllowAbs && MI->getOpcode() == AMDGPU::G_FABS) {
4753 Src = MI->getOperand(i: 1).getReg();
4754 Mods |= SISrcMods::ABS;
4755 }
4756
4757 if (OpSel)
4758 Mods |= SISrcMods::OP_SEL_0;
4759
4760 return std::pair(Src, Mods);
4761}
4762
4763std::pair<Register, unsigned>
4764AMDGPUInstructionSelector::selectVOP3PModsF32Impl(Register Src) const {
4765 unsigned Mods;
4766 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src);
4767 Mods |= SISrcMods::OP_SEL_1;
4768 return std::pair(Src, Mods);
4769}
4770
4771Register AMDGPUInstructionSelector::copyToVGPRIfSrcFolded(
4772 Register Src, unsigned Mods, MachineOperand Root, MachineInstr *InsertPt,
4773 bool ForceVGPR) const {
4774 if ((Mods != 0 || ForceVGPR) &&
4775 RBI.getRegBank(Reg: Src, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) {
4776
4777 // If we looked through copies to find source modifiers on an SGPR operand,
4778 // we now have an SGPR register source. To avoid potentially violating the
4779 // constant bus restriction, we need to insert a copy to a VGPR.
4780 Register VGPRSrc = MRI->cloneVirtualRegister(VReg: Root.getReg());
4781 BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
4782 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VGPRSrc)
4783 .addReg(RegNo: Src);
4784 Src = VGPRSrc;
4785 }
4786
4787 return Src;
4788}
4789
4790///
4791/// This will select either an SGPR or VGPR operand and will save us from
4792/// having to write an extra tablegen pattern.
4793InstructionSelector::ComplexRendererFns
4794AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const {
4795 return {{
4796 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4797 }};
4798}
4799
4800InstructionSelector::ComplexRendererFns
4801AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const {
4802 Register Src;
4803 unsigned Mods;
4804 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4805
4806 return {{
4807 [=](MachineInstrBuilder &MIB) {
4808 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4809 },
4810 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4811 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4812 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4813 }};
4814}
4815
4816InstructionSelector::ComplexRendererFns
4817AMDGPUInstructionSelector::selectVOP3BMods0(MachineOperand &Root) const {
4818 Register Src;
4819 unsigned Mods;
4820 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
4821 /*IsCanonicalizing=*/true,
4822 /*AllowAbs=*/false);
4823
4824 return {{
4825 [=](MachineInstrBuilder &MIB) {
4826 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4827 },
4828 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4829 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4830 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4831 }};
4832}
4833
4834InstructionSelector::ComplexRendererFns
4835AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const {
4836 return {{
4837 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); },
4838 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4839 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4840 }};
4841}
4842
4843InstructionSelector::ComplexRendererFns
4844AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const {
4845 Register Src;
4846 unsigned Mods;
4847 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4848
4849 return {{
4850 [=](MachineInstrBuilder &MIB) {
4851 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4852 },
4853 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4854 }};
4855}
4856
4857InstructionSelector::ComplexRendererFns
4858AMDGPUInstructionSelector::selectVOP3ModsNonCanonicalizing(
4859 MachineOperand &Root) const {
4860 Register Src;
4861 unsigned Mods;
4862 std::tie(args&: Src, args&: Mods) =
4863 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/false);
4864
4865 return {{
4866 [=](MachineInstrBuilder &MIB) {
4867 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4868 },
4869 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4870 }};
4871}
4872
4873InstructionSelector::ComplexRendererFns
4874AMDGPUInstructionSelector::selectVOP3BMods(MachineOperand &Root) const {
4875 Register Src;
4876 unsigned Mods;
4877 std::tie(args&: Src, args&: Mods) =
4878 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/true,
4879 /*AllowAbs=*/false);
4880
4881 return {{
4882 [=](MachineInstrBuilder &MIB) {
4883 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4884 },
4885 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4886 }};
4887}
4888
4889InstructionSelector::ComplexRendererFns
4890AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
4891 Register Reg = Root.getReg();
4892 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
4893 if (Def->getOpcode() == AMDGPU::G_FNEG || Def->getOpcode() == AMDGPU::G_FABS)
4894 return {};
4895 return {{
4896 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
4897 }};
4898}
4899
4900enum class SrcStatus {
4901 IS_SAME,
4902 IS_UPPER_HALF,
4903 IS_LOWER_HALF,
4904 IS_UPPER_HALF_NEG,
4905 // This means current op = [op_upper, op_lower] and src = -op_lower.
4906 IS_LOWER_HALF_NEG,
4907 IS_HI_NEG,
4908 // This means current op = [op_upper, op_lower] and src = [op_upper,
4909 // -op_lower].
4910 IS_LO_NEG,
4911 IS_BOTH_NEG,
4912 INVALID,
4913 NEG_START = IS_UPPER_HALF_NEG,
4914 NEG_END = IS_BOTH_NEG,
4915 HALF_START = IS_UPPER_HALF,
4916 HALF_END = IS_LOWER_HALF_NEG
4917};
4918/// Test if the MI is truncating to half, such as `%reg0:n = G_TRUNC %reg1:2n`
4919static bool isTruncHalf(const MachineInstr *MI,
4920 const MachineRegisterInfo &MRI) {
4921 if (MI->getOpcode() != AMDGPU::G_TRUNC)
4922 return false;
4923
4924 unsigned DstSize = MRI.getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
4925 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4926 return DstSize * 2 == SrcSize;
4927}
4928
4929/// Test if the MI is logic shift right with half bits,
4930/// such as `%reg0:2n =G_LSHR %reg1:2n, CONST(n)`
4931static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
4932 if (MI->getOpcode() != AMDGPU::G_LSHR)
4933 return false;
4934
4935 Register ShiftSrc;
4936 std::optional<ValueAndVReg> ShiftAmt;
4937 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
4938 P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
4939 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4940 unsigned Shift = ShiftAmt->Value.getZExtValue();
4941 return Shift * 2 == SrcSize;
4942 }
4943 return false;
4944}
4945
4946/// Test if the MI is shift left with half bits,
4947/// such as `%reg0:2n =G_SHL %reg1:2n, CONST(n)`
4948static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
4949 if (MI->getOpcode() != AMDGPU::G_SHL)
4950 return false;
4951
4952 Register ShiftSrc;
4953 std::optional<ValueAndVReg> ShiftAmt;
4954 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
4955 P: m_GShl(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
4956 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4957 unsigned Shift = ShiftAmt->Value.getZExtValue();
4958 return Shift * 2 == SrcSize;
4959 }
4960 return false;
4961}
4962
4963/// Test function, if the MI is `%reg0:n, %reg1:n = G_UNMERGE_VALUES %reg2:2n`
4964static bool isUnmergeHalf(const MachineInstr *MI,
4965 const MachineRegisterInfo &MRI) {
4966 if (MI->getOpcode() != AMDGPU::G_UNMERGE_VALUES)
4967 return false;
4968 return MI->getNumOperands() == 3 && MI->getOperand(i: 0).isDef() &&
4969 MI->getOperand(i: 1).isDef() && !MI->getOperand(i: 2).isDef();
4970}
4971
4972enum class TypeClass { VECTOR_OF_TWO, SCALAR, NONE_OF_LISTED };
4973
4974static TypeClass isVectorOfTwoOrScalar(Register Reg,
4975 const MachineRegisterInfo &MRI) {
4976 LLT OpTy = MRI.getType(Reg);
4977 if (OpTy.isScalar())
4978 return TypeClass::SCALAR;
4979 if (OpTy.isVector() && OpTy.getNumElements() == 2)
4980 return TypeClass::VECTOR_OF_TWO;
4981 return TypeClass::NONE_OF_LISTED;
4982}
4983
4984static SrcStatus getNegStatus(Register Reg, SrcStatus S,
4985 const MachineRegisterInfo &MRI) {
4986 TypeClass NegType = isVectorOfTwoOrScalar(Reg, MRI);
4987 if (NegType != TypeClass::VECTOR_OF_TWO && NegType != TypeClass::SCALAR)
4988 return SrcStatus::INVALID;
4989
4990 switch (S) {
4991 case SrcStatus::IS_SAME:
4992 if (NegType == TypeClass::VECTOR_OF_TWO) {
4993 // Vector of 2:
4994 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
4995 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
4996 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
4997 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
4998 return SrcStatus::IS_BOTH_NEG;
4999 }
5000 if (NegType == TypeClass::SCALAR) {
5001 // Scalar:
5002 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
5003 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
5004 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5005 // [SrcHi, SrcLo] = [-OpHi, OpLo]
5006 return SrcStatus::IS_HI_NEG;
5007 }
5008 break;
5009 case SrcStatus::IS_HI_NEG:
5010 if (NegType == TypeClass::VECTOR_OF_TWO) {
5011 // Vector of 2:
5012 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5013 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
5014 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5015 // [SrcHi, SrcLo] = [-(-OpHi), -OpLo] = [OpHi, -OpLo]
5016 return SrcStatus::IS_LO_NEG;
5017 }
5018 if (NegType == TypeClass::SCALAR) {
5019 // Scalar:
5020 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5021 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
5022 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5023 // [SrcHi, SrcLo] = [-(-OpHi), OpLo] = [OpHi, OpLo]
5024 return SrcStatus::IS_SAME;
5025 }
5026 break;
5027 case SrcStatus::IS_LO_NEG:
5028 if (NegType == TypeClass::VECTOR_OF_TWO) {
5029 // Vector of 2:
5030 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5031 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5032 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5033 // [SrcHi, SrcLo] = [-OpHi, -(-OpLo)] = [-OpHi, OpLo]
5034 return SrcStatus::IS_HI_NEG;
5035 }
5036 if (NegType == TypeClass::SCALAR) {
5037 // Scalar:
5038 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5039 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5040 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5041 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
5042 return SrcStatus::IS_BOTH_NEG;
5043 }
5044 break;
5045 case SrcStatus::IS_BOTH_NEG:
5046 if (NegType == TypeClass::VECTOR_OF_TWO) {
5047 // Vector of 2:
5048 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5049 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5050 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5051 // [SrcHi, SrcLo] = [OpHi, OpLo]
5052 return SrcStatus::IS_SAME;
5053 }
5054 if (NegType == TypeClass::SCALAR) {
5055 // Scalar:
5056 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5057 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5058 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5059 // [SrcHi, SrcLo] = [OpHi, -OpLo]
5060 return SrcStatus::IS_LO_NEG;
5061 }
5062 break;
5063 case SrcStatus::IS_UPPER_HALF:
5064 // Vector of 2:
5065 // Src = CurrUpper
5066 // Curr = [CurrUpper, CurrLower]
5067 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5068 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5069 // Src = -OpUpper
5070 //
5071 // Scalar:
5072 // Src = CurrUpper
5073 // Curr = [CurrUpper, CurrLower]
5074 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5075 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5076 // Src = -OpUpper
5077 return SrcStatus::IS_UPPER_HALF_NEG;
5078 case SrcStatus::IS_LOWER_HALF:
5079 if (NegType == TypeClass::VECTOR_OF_TWO) {
5080 // Vector of 2:
5081 // Src = CurrLower
5082 // Curr = [CurrUpper, CurrLower]
5083 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5084 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5085 // Src = -OpLower
5086 return SrcStatus::IS_LOWER_HALF_NEG;
5087 }
5088 if (NegType == TypeClass::SCALAR) {
5089 // Scalar:
5090 // Src = CurrLower
5091 // Curr = [CurrUpper, CurrLower]
5092 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5093 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5094 // Src = OpLower
5095 return SrcStatus::IS_LOWER_HALF;
5096 }
5097 break;
5098 case SrcStatus::IS_UPPER_HALF_NEG:
5099 // Vector of 2:
5100 // Src = -CurrUpper
5101 // Curr = [CurrUpper, CurrLower]
5102 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5103 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5104 // Src = -(-OpUpper) = OpUpper
5105 //
5106 // Scalar:
5107 // Src = -CurrUpper
5108 // Curr = [CurrUpper, CurrLower]
5109 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5110 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5111 // Src = -(-OpUpper) = OpUpper
5112 return SrcStatus::IS_UPPER_HALF;
5113 case SrcStatus::IS_LOWER_HALF_NEG:
5114 if (NegType == TypeClass::VECTOR_OF_TWO) {
5115 // Vector of 2:
5116 // Src = -CurrLower
5117 // Curr = [CurrUpper, CurrLower]
5118 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5119 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5120 // Src = -(-OpLower) = OpLower
5121 return SrcStatus::IS_LOWER_HALF;
5122 }
5123 if (NegType == TypeClass::SCALAR) {
5124 // Scalar:
5125 // Src = -CurrLower
5126 // Curr = [CurrUpper, CurrLower]
5127 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5128 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5129 // Src = -OpLower
5130 return SrcStatus::IS_LOWER_HALF_NEG;
5131 }
5132 break;
5133 default:
5134 break;
5135 }
5136 llvm_unreachable("unexpected SrcStatus & NegType combination");
5137}
5138
5139static std::optional<std::pair<Register, SrcStatus>>
5140calcNextStatus(std::pair<Register, SrcStatus> Curr,
5141 const MachineRegisterInfo &MRI) {
5142 const MachineInstr *MI = MRI.getVRegDef(Reg: Curr.first);
5143
5144 unsigned Opc = MI->getOpcode();
5145
5146 // Handle general Opc cases.
5147 switch (Opc) {
5148 case AMDGPU::G_BITCAST:
5149 return std::optional<std::pair<Register, SrcStatus>>(
5150 {MI->getOperand(i: 1).getReg(), Curr.second});
5151 case AMDGPU::COPY:
5152 if (MI->getOperand(i: 1).getReg().isPhysical())
5153 return std::nullopt;
5154 return std::optional<std::pair<Register, SrcStatus>>(
5155 {MI->getOperand(i: 1).getReg(), Curr.second});
5156 case AMDGPU::G_FNEG: {
5157 SrcStatus Stat = getNegStatus(Reg: Curr.first, S: Curr.second, MRI);
5158 if (Stat == SrcStatus::INVALID)
5159 return std::nullopt;
5160 return std::optional<std::pair<Register, SrcStatus>>(
5161 {MI->getOperand(i: 1).getReg(), Stat});
5162 }
5163 default:
5164 break;
5165 }
5166
5167 // Calc next Stat from current Stat.
5168 switch (Curr.second) {
5169 case SrcStatus::IS_SAME:
5170 if (isTruncHalf(MI, MRI))
5171 return std::optional<std::pair<Register, SrcStatus>>(
5172 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5173 else if (isUnmergeHalf(MI, MRI)) {
5174 if (Curr.first == MI->getOperand(i: 0).getReg())
5175 return std::optional<std::pair<Register, SrcStatus>>(
5176 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF});
5177 return std::optional<std::pair<Register, SrcStatus>>(
5178 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF});
5179 }
5180 break;
5181 case SrcStatus::IS_HI_NEG:
5182 if (isTruncHalf(MI, MRI)) {
5183 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5184 // [CurrHi, CurrLo] = trunc [OpUpper, OpLower] = OpLower
5185 // = [OpLowerHi, OpLowerLo]
5186 // Src = [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5187 // = [-OpLowerHi, OpLowerLo]
5188 // = -OpLower
5189 return std::optional<std::pair<Register, SrcStatus>>(
5190 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5191 }
5192 if (isUnmergeHalf(MI, MRI)) {
5193 if (Curr.first == MI->getOperand(i: 0).getReg())
5194 return std::optional<std::pair<Register, SrcStatus>>(
5195 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5196 return std::optional<std::pair<Register, SrcStatus>>(
5197 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5198 }
5199 break;
5200 case SrcStatus::IS_UPPER_HALF:
5201 if (isShlHalf(MI, MRI))
5202 return std::optional<std::pair<Register, SrcStatus>>(
5203 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5204 break;
5205 case SrcStatus::IS_LOWER_HALF:
5206 if (isLshrHalf(MI, MRI))
5207 return std::optional<std::pair<Register, SrcStatus>>(
5208 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF});
5209 break;
5210 case SrcStatus::IS_UPPER_HALF_NEG:
5211 if (isShlHalf(MI, MRI))
5212 return std::optional<std::pair<Register, SrcStatus>>(
5213 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5214 break;
5215 case SrcStatus::IS_LOWER_HALF_NEG:
5216 if (isLshrHalf(MI, MRI))
5217 return std::optional<std::pair<Register, SrcStatus>>(
5218 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5219 break;
5220 default:
5221 break;
5222 }
5223 return std::nullopt;
5224}
5225
5226/// This is used to control valid status that current MI supports. For example,
5227/// non floating point intrinsic such as @llvm.amdgcn.sdot2 does not support NEG
5228/// bit on VOP3P.
5229/// The class can be further extended to recognize support on SEL, NEG, ABS bit
5230/// for different MI on different arch
5231class SearchOptions {
5232private:
5233 bool HasNeg = false;
5234 // Assume all complex pattern of VOP3P have opsel.
5235 bool HasOpsel = true;
5236
5237public:
5238 SearchOptions(Register Reg, const MachineRegisterInfo &MRI) {
5239 const MachineInstr *MI = MRI.getVRegDef(Reg);
5240 unsigned Opc = MI->getOpcode();
5241
5242 if (Opc == TargetOpcode::G_INTRINSIC) {
5243 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: *MI).getIntrinsicID();
5244 // Only float point intrinsic has neg & neg_hi bits.
5245 if (IntrinsicID == Intrinsic::amdgcn_fdot2)
5246 HasNeg = true;
5247 } else if (TargetInstrInfo::isGenericOpcode(Opc)) {
5248 // Keep same for generic op.
5249 HasNeg = true;
5250 }
5251 }
5252 bool checkOptions(SrcStatus Stat) const {
5253 if (!HasNeg &&
5254 (Stat >= SrcStatus::NEG_START && Stat <= SrcStatus::NEG_END)) {
5255 return false;
5256 }
5257 if (!HasOpsel &&
5258 (Stat >= SrcStatus::HALF_START && Stat <= SrcStatus::HALF_END)) {
5259 return false;
5260 }
5261 return true;
5262 }
5263};
5264
5265static SmallVector<std::pair<Register, SrcStatus>>
5266getSrcStats(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5267 int MaxDepth = 3) {
5268 int Depth = 0;
5269 auto Curr = calcNextStatus(Curr: {Reg, SrcStatus::IS_SAME}, MRI);
5270 SmallVector<std::pair<Register, SrcStatus>> Statlist;
5271
5272 while (Depth <= MaxDepth && Curr.has_value()) {
5273 Depth++;
5274 if (SO.checkOptions(Stat: Curr.value().second))
5275 Statlist.push_back(Elt: Curr.value());
5276 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5277 }
5278
5279 return Statlist;
5280}
5281
5282static std::pair<Register, SrcStatus>
5283getLastSameOrNeg(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5284 int MaxDepth = 3) {
5285 int Depth = 0;
5286 std::pair<Register, SrcStatus> LastSameOrNeg = {Reg, SrcStatus::IS_SAME};
5287 auto Curr = calcNextStatus(Curr: LastSameOrNeg, MRI);
5288
5289 while (Depth <= MaxDepth && Curr.has_value()) {
5290 Depth++;
5291 SrcStatus Stat = Curr.value().second;
5292 if (SO.checkOptions(Stat)) {
5293 if (Stat == SrcStatus::IS_SAME || Stat == SrcStatus::IS_HI_NEG ||
5294 Stat == SrcStatus::IS_LO_NEG || Stat == SrcStatus::IS_BOTH_NEG)
5295 LastSameOrNeg = Curr.value();
5296 }
5297 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5298 }
5299
5300 return LastSameOrNeg;
5301}
5302
5303static bool isSameBitWidth(Register Reg1, Register Reg2,
5304 const MachineRegisterInfo &MRI) {
5305 unsigned Width1 = MRI.getType(Reg: Reg1).getSizeInBits();
5306 unsigned Width2 = MRI.getType(Reg: Reg2).getSizeInBits();
5307 return Width1 == Width2;
5308}
5309
5310static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods) {
5311 // SrcStatus::IS_LOWER_HALF remain 0.
5312 if (HiStat == SrcStatus::IS_UPPER_HALF_NEG) {
5313 Mods ^= SISrcMods::NEG_HI;
5314 Mods |= SISrcMods::OP_SEL_1;
5315 } else if (HiStat == SrcStatus::IS_UPPER_HALF)
5316 Mods |= SISrcMods::OP_SEL_1;
5317 else if (HiStat == SrcStatus::IS_LOWER_HALF_NEG)
5318 Mods ^= SISrcMods::NEG_HI;
5319 else if (HiStat == SrcStatus::IS_HI_NEG)
5320 Mods ^= SISrcMods::NEG_HI;
5321
5322 if (LoStat == SrcStatus::IS_UPPER_HALF_NEG) {
5323 Mods ^= SISrcMods::NEG;
5324 Mods |= SISrcMods::OP_SEL_0;
5325 } else if (LoStat == SrcStatus::IS_UPPER_HALF)
5326 Mods |= SISrcMods::OP_SEL_0;
5327 else if (LoStat == SrcStatus::IS_LOWER_HALF_NEG)
5328 Mods |= SISrcMods::NEG;
5329 else if (LoStat == SrcStatus::IS_HI_NEG)
5330 Mods ^= SISrcMods::NEG;
5331
5332 return Mods;
5333}
5334
5335static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg,
5336 Register RootReg, const SIInstrInfo &TII,
5337 const MachineRegisterInfo &MRI) {
5338 auto IsHalfState = [](SrcStatus S) {
5339 return S == SrcStatus::IS_UPPER_HALF || S == SrcStatus::IS_UPPER_HALF_NEG ||
5340 S == SrcStatus::IS_LOWER_HALF || S == SrcStatus::IS_LOWER_HALF_NEG;
5341 };
5342 return isSameBitWidth(Reg1: NewReg, Reg2: RootReg, MRI) && IsHalfState(LoStat) &&
5343 IsHalfState(HiStat);
5344}
5345
5346std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
5347 Register RootReg, const MachineRegisterInfo &MRI, bool IsDOT) const {
5348 unsigned Mods = 0;
5349 // No modification if Root type is not form of <2 x Type>.
5350 if (isVectorOfTwoOrScalar(Reg: RootReg, MRI) != TypeClass::VECTOR_OF_TWO) {
5351 Mods |= SISrcMods::OP_SEL_1;
5352 return {RootReg, Mods};
5353 }
5354
5355 SearchOptions SO(RootReg, MRI);
5356
5357 std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(Reg: RootReg, MRI, SO);
5358
5359 if (Stat.second == SrcStatus::IS_BOTH_NEG)
5360 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
5361 else if (Stat.second == SrcStatus::IS_HI_NEG)
5362 Mods ^= SISrcMods::NEG_HI;
5363 else if (Stat.second == SrcStatus::IS_LO_NEG)
5364 Mods ^= SISrcMods::NEG;
5365
5366 // 64-bit VOP3P instructions do not have OPSEL or ABS. Bail on v2f64 or v2i64.
5367 // TODO: Select NEG_LO and NEG_HI modifiers from BUILD_VECTOR.
5368 if (MRI.getType(Reg: RootReg).getSizeInBits() == 128) {
5369 Mods |= SISrcMods::OP_SEL_1; // Just the default, OPSEL unsupported.
5370 return {Stat.first, Mods};
5371 }
5372
5373 MachineInstr *MI = MRI.getVRegDef(Reg: Stat.first);
5374
5375 if (MI->getOpcode() != AMDGPU::G_BUILD_VECTOR || MI->getNumOperands() != 3 ||
5376 (IsDOT && Subtarget->hasDOTOpSelHazard())) {
5377 Mods |= SISrcMods::OP_SEL_1;
5378 return {Stat.first, Mods};
5379 }
5380
5381 SmallVector<std::pair<Register, SrcStatus>> StatlistHi =
5382 getSrcStats(Reg: MI->getOperand(i: 2).getReg(), MRI, SO);
5383
5384 if (StatlistHi.empty()) {
5385 Mods |= SISrcMods::OP_SEL_1;
5386 return {Stat.first, Mods};
5387 }
5388
5389 SmallVector<std::pair<Register, SrcStatus>> StatlistLo =
5390 getSrcStats(Reg: MI->getOperand(i: 1).getReg(), MRI, SO);
5391
5392 if (StatlistLo.empty()) {
5393 Mods |= SISrcMods::OP_SEL_1;
5394 return {Stat.first, Mods};
5395 }
5396
5397 for (int I = StatlistHi.size() - 1; I >= 0; I--) {
5398 for (int J = StatlistLo.size() - 1; J >= 0; J--) {
5399 if (StatlistHi[I].first == StatlistLo[J].first &&
5400 isValidToPack(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second,
5401 NewReg: StatlistHi[I].first, RootReg, TII, MRI))
5402 return {StatlistHi[I].first,
5403 updateMods(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second, Mods)};
5404 }
5405 }
5406 // Packed instructions do not have abs modifiers.
5407 Mods |= SISrcMods::OP_SEL_1;
5408
5409 return {Stat.first, Mods};
5410}
5411
5412// Removed unused function `getAllKindImm` to eliminate dead code.
5413
5414static bool checkRB(Register Reg, unsigned int RBNo,
5415 const AMDGPURegisterBankInfo &RBI,
5416 const MachineRegisterInfo &MRI,
5417 const TargetRegisterInfo &TRI) {
5418 const RegisterBank *RB = RBI.getRegBank(Reg, MRI, TRI);
5419 return RB->getID() == RBNo;
5420}
5421
5422// This function is used to get the correct register bank for returned reg.
5423// Assume:
5424// 1. VOP3P is always legal for VGPR.
5425// 2. RootOp's regbank is legal.
5426// Thus
5427// 1. If RootOp is SGPR, then NewOp can be SGPR or VGPR.
5428// 2. If RootOp is VGPR, then NewOp must be VGPR.
5429static Register getLegalRegBank(Register NewReg, Register RootReg,
5430 const AMDGPURegisterBankInfo &RBI,
5431 MachineRegisterInfo &MRI,
5432 const TargetRegisterInfo &TRI,
5433 const SIInstrInfo &TII) {
5434 // RootOp can only be VGPR or SGPR (some hand written cases such as.
5435 // inst-select-ashr.v2s16.mir::ashr_v2s16_vs).
5436 if (checkRB(Reg: RootReg, RBNo: AMDGPU::SGPRRegBankID, RBI, MRI, TRI) ||
5437 checkRB(Reg: NewReg, RBNo: AMDGPU::VGPRRegBankID, RBI, MRI, TRI))
5438 return NewReg;
5439
5440 MachineInstr *MI = MRI.getVRegDef(Reg: RootReg);
5441 if (MI->getOpcode() == AMDGPU::COPY && NewReg == MI->getOperand(i: 1).getReg()) {
5442 // RootOp is VGPR, NewOp is not VGPR, but RootOp = COPY NewOp.
5443 return RootReg;
5444 }
5445
5446 MachineBasicBlock *BB = MI->getParent();
5447 Register DstReg = MRI.cloneVirtualRegister(VReg: RootReg);
5448
5449 MachineInstrBuilder MIB =
5450 BuildMI(BB&: *BB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
5451 .addReg(RegNo: NewReg);
5452
5453 // Only accept VGPR.
5454 return MIB->getOperand(i: 0).getReg();
5455}
5456
5457InstructionSelector::ComplexRendererFns
5458AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
5459 bool IsDOT) const {
5460 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5461 Register Reg;
5462 unsigned Mods;
5463 std::tie(args&: Reg, args&: Mods) = selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT);
5464
5465 Reg = getLegalRegBank(NewReg: Reg, RootReg: Root.getReg(), RBI, MRI, TRI, TII);
5466 return {{
5467 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
5468 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5469 }};
5470}
5471
5472InstructionSelector::ComplexRendererFns
5473AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
5474
5475 return selectVOP3PRetHelper(Root);
5476}
5477
5478InstructionSelector::ComplexRendererFns
5479AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
5480
5481 return selectVOP3PRetHelper(Root, IsDOT: true);
5482}
5483
5484InstructionSelector::ComplexRendererFns
5485AMDGPUInstructionSelector::selectVOP3PNoModsDOT(MachineOperand &Root) const {
5486 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5487 Register Src;
5488 unsigned Mods;
5489 std::tie(args&: Src, args&: Mods) = selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT: true /*IsDOT*/);
5490 if (Mods != SISrcMods::OP_SEL_1)
5491 return {};
5492
5493 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5494}
5495
5496InstructionSelector::ComplexRendererFns
5497AMDGPUInstructionSelector::selectVOP3PModsF32(MachineOperand &Root) const {
5498 Register Src;
5499 unsigned Mods;
5500 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5501
5502 return {{
5503 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5504 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5505 }};
5506}
5507
5508InstructionSelector::ComplexRendererFns
5509AMDGPUInstructionSelector::selectVOP3PNoModsF32(MachineOperand &Root) const {
5510 Register Src;
5511 unsigned Mods;
5512 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5513 if (Mods != SISrcMods::OP_SEL_1)
5514 return {};
5515
5516 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5517}
5518
5519InstructionSelector::ComplexRendererFns
5520AMDGPUInstructionSelector::selectWMMAOpSelVOP3PMods(
5521 MachineOperand &Root) const {
5522 assert((Root.isImm() && (Root.getImm() == -1 || Root.getImm() == 0)) &&
5523 "expected i1 value");
5524 unsigned Mods = SISrcMods::OP_SEL_1;
5525 if (Root.getImm() != 0)
5526 Mods |= SISrcMods::OP_SEL_0;
5527
5528 return {{
5529 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5530 }};
5531}
5532
5533static Register buildRegSequence(SmallVectorImpl<Register> &Elts,
5534 MachineInstr *InsertPt,
5535 MachineRegisterInfo &MRI) {
5536 const TargetRegisterClass *DstRegClass;
5537 switch (Elts.size()) {
5538 case 8:
5539 DstRegClass = &AMDGPU::VReg_256RegClass;
5540 break;
5541 case 4:
5542 DstRegClass = &AMDGPU::VReg_128RegClass;
5543 break;
5544 case 2:
5545 DstRegClass = &AMDGPU::VReg_64RegClass;
5546 break;
5547 default:
5548 llvm_unreachable("unhandled Reg sequence size");
5549 }
5550
5551 MachineIRBuilder B(*InsertPt);
5552 auto MIB = B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
5553 .addDef(RegNo: MRI.createVirtualRegister(RegClass: DstRegClass));
5554 for (unsigned i = 0; i < Elts.size(); ++i) {
5555 MIB.addReg(RegNo: Elts[i]);
5556 MIB.addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: i));
5557 }
5558 return MIB->getOperand(i: 0).getReg();
5559}
5560
5561static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods,
5562 SmallVectorImpl<Register> &Elts, Register &Src,
5563 MachineInstr *InsertPt,
5564 MachineRegisterInfo &MRI) {
5565 if (ModOpcode == TargetOpcode::G_FNEG) {
5566 Mods |= SISrcMods::NEG;
5567 // Check if all elements also have abs modifier
5568 SmallVector<Register, 8> NegAbsElts;
5569 for (auto El : Elts) {
5570 Register FabsSrc;
5571 if (!mi_match(R: El, MRI, P: m_GFabs(Src: m_Reg(R&: FabsSrc))))
5572 break;
5573 NegAbsElts.push_back(Elt: FabsSrc);
5574 }
5575 if (Elts.size() != NegAbsElts.size()) {
5576 // Neg
5577 Src = buildRegSequence(Elts, InsertPt, MRI);
5578 } else {
5579 // Neg and Abs
5580 Mods |= SISrcMods::NEG_HI;
5581 Src = buildRegSequence(Elts&: NegAbsElts, InsertPt, MRI);
5582 }
5583 } else {
5584 assert(ModOpcode == TargetOpcode::G_FABS);
5585 // Abs
5586 Mods |= SISrcMods::NEG_HI;
5587 Src = buildRegSequence(Elts, InsertPt, MRI);
5588 }
5589}
5590
5591InstructionSelector::ComplexRendererFns
5592AMDGPUInstructionSelector::selectWMMAModsF32NegAbs(MachineOperand &Root) const {
5593 Register Src = Root.getReg();
5594 unsigned Mods = SISrcMods::OP_SEL_1;
5595 SmallVector<Register, 8> EltsF32;
5596
5597 if (GBuildVector *BV = dyn_cast<GBuildVector>(Val: MRI->getVRegDef(Reg: Src))) {
5598 assert(BV->getNumSources() > 0);
5599 // Based on first element decide which mod we match, neg or abs
5600 MachineInstr *ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: 0));
5601 unsigned ModOpcode = (ElF32->getOpcode() == AMDGPU::G_FNEG)
5602 ? AMDGPU::G_FNEG
5603 : AMDGPU::G_FABS;
5604 for (unsigned i = 0; i < BV->getNumSources(); ++i) {
5605 ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: i));
5606 if (ElF32->getOpcode() != ModOpcode)
5607 break;
5608 EltsF32.push_back(Elt: ElF32->getOperand(i: 1).getReg());
5609 }
5610
5611 // All elements had ModOpcode modifier
5612 if (BV->getNumSources() == EltsF32.size()) {
5613 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsF32, Src, InsertPt: Root.getParent(),
5614 MRI&: *MRI);
5615 }
5616 }
5617
5618 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5619 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5620}
5621
5622InstructionSelector::ComplexRendererFns
5623AMDGPUInstructionSelector::selectWMMAModsF16Neg(MachineOperand &Root) const {
5624 Register Src = Root.getReg();
5625 unsigned Mods = SISrcMods::OP_SEL_1;
5626 SmallVector<Register, 8> EltsV2F16;
5627
5628 if (GConcatVectors *CV = dyn_cast<GConcatVectors>(Val: MRI->getVRegDef(Reg: Src))) {
5629 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5630 Register FNegSrc;
5631 if (!mi_match(R: CV->getSourceReg(I: i), MRI: *MRI, P: m_GFNeg(Src: m_Reg(R&: FNegSrc))))
5632 break;
5633 EltsV2F16.push_back(Elt: FNegSrc);
5634 }
5635
5636 // All elements had ModOpcode modifier
5637 if (CV->getNumSources() == EltsV2F16.size()) {
5638 Mods |= SISrcMods::NEG;
5639 Mods |= SISrcMods::NEG_HI;
5640 Src = buildRegSequence(Elts&: EltsV2F16, InsertPt: Root.getParent(), MRI&: *MRI);
5641 }
5642 }
5643
5644 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5645 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5646}
5647
5648InstructionSelector::ComplexRendererFns
5649AMDGPUInstructionSelector::selectWMMAModsF16NegAbs(MachineOperand &Root) const {
5650 Register Src = Root.getReg();
5651 unsigned Mods = SISrcMods::OP_SEL_1;
5652 SmallVector<Register, 8> EltsV2F16;
5653
5654 if (GConcatVectors *CV = dyn_cast<GConcatVectors>(Val: MRI->getVRegDef(Reg: Src))) {
5655 assert(CV->getNumSources() > 0);
5656 MachineInstr *ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: 0));
5657 // Based on first element decide which mod we match, neg or abs
5658 unsigned ModOpcode = (ElV2F16->getOpcode() == AMDGPU::G_FNEG)
5659 ? AMDGPU::G_FNEG
5660 : AMDGPU::G_FABS;
5661
5662 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5663 ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: i));
5664 if (ElV2F16->getOpcode() != ModOpcode)
5665 break;
5666 EltsV2F16.push_back(Elt: ElV2F16->getOperand(i: 1).getReg());
5667 }
5668
5669 // All elements had ModOpcode modifier
5670 if (CV->getNumSources() == EltsV2F16.size()) {
5671 MachineIRBuilder B(*Root.getParent());
5672 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsV2F16, Src, InsertPt: Root.getParent(),
5673 MRI&: *MRI);
5674 }
5675 }
5676
5677 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5678 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5679}
5680
5681InstructionSelector::ComplexRendererFns
5682AMDGPUInstructionSelector::selectWMMAVISrc(MachineOperand &Root) const {
5683 std::optional<FPValueAndVReg> FPValReg;
5684 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GFCstOrSplat(FPValReg))) {
5685 if (TII.isInlineConstant(Imm: FPValReg->Value)) {
5686 return {{[=](MachineInstrBuilder &MIB) {
5687 MIB.addImm(Val: FPValReg->Value.bitcastToAPInt().getSExtValue());
5688 }}};
5689 }
5690 // Non-inlineable splat floats should not fall-through for integer immediate
5691 // checks.
5692 return {};
5693 }
5694
5695 APInt ICst;
5696 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICstOrSplat(Cst&: ICst))) {
5697 if (TII.isInlineConstant(Imm: ICst)) {
5698 return {
5699 {[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ICst.getSExtValue()); }}};
5700 }
5701 }
5702
5703 return {};
5704}
5705
5706InstructionSelector::ComplexRendererFns
5707AMDGPUInstructionSelector::selectSWMMACIndex8(MachineOperand &Root) const {
5708 Register Src =
5709 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5710 unsigned Key = 0;
5711
5712 Register ShiftSrc;
5713 std::optional<ValueAndVReg> ShiftAmt;
5714 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5715 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5716 ShiftAmt->Value.getZExtValue() % 8 == 0) {
5717 Key = ShiftAmt->Value.getZExtValue() / 8;
5718 Src = ShiftSrc;
5719 }
5720
5721 return {{
5722 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5723 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5724 }};
5725}
5726
5727InstructionSelector::ComplexRendererFns
5728AMDGPUInstructionSelector::selectSWMMACIndex16(MachineOperand &Root) const {
5729
5730 Register Src =
5731 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5732 unsigned Key = 0;
5733
5734 Register ShiftSrc;
5735 std::optional<ValueAndVReg> ShiftAmt;
5736 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5737 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5738 ShiftAmt->Value.getZExtValue() == 16) {
5739 Src = ShiftSrc;
5740 Key = 1;
5741 }
5742
5743 return {{
5744 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5745 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5746 }};
5747}
5748
5749InstructionSelector::ComplexRendererFns
5750AMDGPUInstructionSelector::selectSWMMACIndex32(MachineOperand &Root) const {
5751 Register Src =
5752 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5753 unsigned Key = 0;
5754
5755 Register S32 = matchZeroExtendFromS32(Reg: Src);
5756 if (!S32)
5757 S32 = matchAnyExtendFromS32(Reg: Src);
5758
5759 if (S32) {
5760 const MachineInstr *Def = getDefIgnoringCopies(Reg: S32, MRI: *MRI);
5761 if (Def->getOpcode() == TargetOpcode::G_UNMERGE_VALUES) {
5762 assert(Def->getNumOperands() == 3);
5763 Register DstReg1 = Def->getOperand(i: 1).getReg();
5764 if (mi_match(R: S32, MRI: *MRI,
5765 P: m_any_of(preds: m_SpecificReg(RequestedReg: DstReg1), preds: m_Copy(Src: m_Reg(R&: DstReg1))))) {
5766 Src = Def->getOperand(i: 2).getReg();
5767 Key = 1;
5768 }
5769 }
5770 }
5771
5772 return {{
5773 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5774 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5775 }};
5776}
5777
5778InstructionSelector::ComplexRendererFns
5779AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const {
5780 Register Src;
5781 unsigned Mods;
5782 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
5783
5784 // FIXME: Handle op_sel
5785 return {{
5786 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5787 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5788 }};
5789}
5790
5791// FIXME-TRUE16 remove when fake16 is removed
5792InstructionSelector::ComplexRendererFns
5793AMDGPUInstructionSelector::selectVINTERPMods(MachineOperand &Root) const {
5794 Register Src;
5795 unsigned Mods;
5796 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5797 /*IsCanonicalizing=*/true,
5798 /*AllowAbs=*/false,
5799 /*OpSel=*/false);
5800
5801 return {{
5802 [=](MachineInstrBuilder &MIB) {
5803 MIB.addReg(
5804 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5805 },
5806 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5807 }};
5808}
5809
5810InstructionSelector::ComplexRendererFns
5811AMDGPUInstructionSelector::selectVINTERPModsHi(MachineOperand &Root) const {
5812 Register Src;
5813 unsigned Mods;
5814 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5815 /*IsCanonicalizing=*/true,
5816 /*AllowAbs=*/false,
5817 /*OpSel=*/true);
5818
5819 return {{
5820 [=](MachineInstrBuilder &MIB) {
5821 MIB.addReg(
5822 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5823 },
5824 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5825 }};
5826}
5827
5828// Given \p Offset and load specified by the \p Root operand check if \p Offset
5829// is a multiple of the load byte size. If it is update \p Offset to a
5830// pre-scaled value and return true.
5831bool AMDGPUInstructionSelector::selectScaleOffset(MachineOperand &Root,
5832 Register &Offset,
5833 bool IsSigned) const {
5834 if (!Subtarget->hasScaleOffset())
5835 return false;
5836
5837 const MachineInstr &MI = *Root.getParent();
5838 MachineMemOperand *MMO = *MI.memoperands_begin();
5839
5840 if (!MMO->getSize().hasValue())
5841 return false;
5842
5843 uint64_t Size = MMO->getSize().getValue();
5844
5845 Register OffsetReg = matchExtendFromS32OrS32(Reg: Offset, IsSigned);
5846 if (!OffsetReg)
5847 OffsetReg = Offset;
5848
5849 if (auto Def = getDefSrcRegIgnoringCopies(Reg: OffsetReg, MRI: *MRI))
5850 OffsetReg = Def->Reg;
5851
5852 Register Op0;
5853 MachineInstr *Mul;
5854 bool ScaleOffset =
5855 (isPowerOf2_64(Value: Size) &&
5856 mi_match(R: OffsetReg, MRI: *MRI,
5857 P: m_GShl(L: m_Reg(R&: Op0),
5858 R: m_any_of(preds: m_SpecificICst(RequestedValue: Log2_64(Value: Size)),
5859 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Log2_64(Value: Size))))))) ||
5860 mi_match(R: OffsetReg, MRI: *MRI,
5861 P: m_GMul(L: m_Reg(R&: Op0), R: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5862 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) ||
5863 mi_match(
5864 R: OffsetReg, MRI: *MRI,
5865 P: m_BinOp(Opcode: IsSigned ? AMDGPU::S_MUL_I64_I32_PSEUDO : AMDGPU::S_MUL_U64,
5866 L: m_Reg(R&: Op0), R: m_SpecificICst(RequestedValue: Size))) ||
5867 // Match G_AMDGPU_MAD_U64_U32 offset, c, 0
5868 (mi_match(R: OffsetReg, MRI: *MRI, P: m_MInstr(MI&: Mul)) &&
5869 (Mul->getOpcode() == (IsSigned ? AMDGPU::G_AMDGPU_MAD_I64_I32
5870 : AMDGPU::G_AMDGPU_MAD_U64_U32) ||
5871 (IsSigned && Mul->getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32 &&
5872 VT->signBitIsZero(Op: Mul->getOperand(i: 2).getReg()))) &&
5873 mi_match(R: Mul->getOperand(i: 4).getReg(), MRI: *MRI, P: m_ZeroInt()) &&
5874 mi_match(R: Mul->getOperand(i: 3).getReg(), MRI: *MRI,
5875 P: m_GTrunc(Src: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5876 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) &&
5877 mi_match(R: Mul->getOperand(i: 2).getReg(), MRI: *MRI, P: m_Reg(R&: Op0)));
5878
5879 if (ScaleOffset)
5880 Offset = Op0;
5881
5882 return ScaleOffset;
5883}
5884
5885bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root,
5886 Register &Base,
5887 Register *SOffset,
5888 int64_t *Offset,
5889 bool *ScaleOffset) const {
5890 MachineInstr *MI = Root.getParent();
5891 MachineBasicBlock *MBB = MI->getParent();
5892
5893 // FIXME: We should shrink the GEP if the offset is known to be <= 32-bits,
5894 // then we can select all ptr + 32-bit offsets.
5895 SmallVector<GEPInfo, 4> AddrInfo;
5896 getAddrModeInfo(Load: *MI, MRI: *MRI, AddrInfo);
5897
5898 if (AddrInfo.empty())
5899 return false;
5900
5901 const GEPInfo &GEPI = AddrInfo[0];
5902 std::optional<int64_t> EncodedImm;
5903
5904 if (ScaleOffset)
5905 *ScaleOffset = false;
5906
5907 if (SOffset && Offset) {
5908 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
5909 /*HasSOffset=*/true);
5910 if (GEPI.SgprParts.size() == 1 && GEPI.Imm != 0 && EncodedImm &&
5911 AddrInfo.size() > 1) {
5912 const GEPInfo &GEPI2 = AddrInfo[1];
5913 if (GEPI2.SgprParts.size() == 2 && GEPI2.Imm == 0) {
5914 Register OffsetReg = GEPI2.SgprParts[1];
5915 if (ScaleOffset)
5916 *ScaleOffset =
5917 selectScaleOffset(Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */);
5918 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
5919 if (OffsetReg) {
5920 Base = GEPI2.SgprParts[0];
5921 *SOffset = OffsetReg;
5922 *Offset = *EncodedImm;
5923 if (*Offset >= 0 || !AMDGPU::hasSMRDSignedImmOffset(ST: STI))
5924 return true;
5925
5926 // For unbuffered smem loads, it is illegal for the Immediate Offset
5927 // to be negative if the resulting (Offset + (M0 or SOffset or zero)
5928 // is negative. Handle the case where the Immediate Offset + SOffset
5929 // is negative.
5930 auto SKnown = VT->getKnownBits(R: *SOffset);
5931 if (*Offset + SKnown.getMinValue().getSExtValue() < 0)
5932 return false;
5933
5934 return true;
5935 }
5936 }
5937 }
5938 return false;
5939 }
5940
5941 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
5942 /*HasSOffset=*/false);
5943 if (Offset && GEPI.SgprParts.size() == 1 && EncodedImm) {
5944 Base = GEPI.SgprParts[0];
5945 *Offset = *EncodedImm;
5946 return true;
5947 }
5948
5949 // SGPR offset is unsigned.
5950 if (SOffset && GEPI.SgprParts.size() == 1 && isUInt<32>(x: GEPI.Imm) &&
5951 GEPI.Imm != 0) {
5952 // If we make it this far we have a load with an 32-bit immediate offset.
5953 // It is OK to select this using a sgpr offset, because we have already
5954 // failed trying to select this load into one of the _IMM variants since
5955 // the _IMM Patterns are considered before the _SGPR patterns.
5956 Base = GEPI.SgprParts[0];
5957 *SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5958 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: *SOffset)
5959 .addImm(Val: GEPI.Imm);
5960 return true;
5961 }
5962
5963 if (SOffset && GEPI.SgprParts.size() && GEPI.Imm == 0) {
5964 Register OffsetReg = GEPI.SgprParts[1];
5965 if (ScaleOffset)
5966 *ScaleOffset = selectScaleOffset(Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */);
5967 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
5968 if (OffsetReg) {
5969 Base = GEPI.SgprParts[0];
5970 *SOffset = OffsetReg;
5971 return true;
5972 }
5973 }
5974
5975 return false;
5976}
5977
5978InstructionSelector::ComplexRendererFns
5979AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
5980 Register Base;
5981 int64_t Offset;
5982 if (!selectSmrdOffset(Root, Base, /* SOffset= */ nullptr, Offset: &Offset,
5983 /* ScaleOffset */ nullptr))
5984 return std::nullopt;
5985
5986 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
5987 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }}};
5988}
5989
5990InstructionSelector::ComplexRendererFns
5991AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const {
5992 SmallVector<GEPInfo, 4> AddrInfo;
5993 getAddrModeInfo(Load: *Root.getParent(), MRI: *MRI, AddrInfo);
5994
5995 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
5996 return std::nullopt;
5997
5998 const GEPInfo &GEPInfo = AddrInfo[0];
5999 Register PtrReg = GEPInfo.SgprParts[0];
6000 std::optional<int64_t> EncodedImm =
6001 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: GEPInfo.Imm);
6002 if (!EncodedImm)
6003 return std::nullopt;
6004
6005 return {{
6006 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrReg); },
6007 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); }
6008 }};
6009}
6010
6011InstructionSelector::ComplexRendererFns
6012AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
6013 Register Base, SOffset;
6014 bool ScaleOffset;
6015 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, /* Offset= */ nullptr,
6016 ScaleOffset: &ScaleOffset))
6017 return std::nullopt;
6018
6019 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6020 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6021 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
6022 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
6023}
6024
6025InstructionSelector::ComplexRendererFns
6026AMDGPUInstructionSelector::selectSmrdSgprImm(MachineOperand &Root) const {
6027 Register Base, SOffset;
6028 int64_t Offset;
6029 bool ScaleOffset;
6030 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, Offset: &Offset, ScaleOffset: &ScaleOffset))
6031 return std::nullopt;
6032
6033 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6034 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6035 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
6036 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
6037 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
6038}
6039
6040std::pair<Register, int> AMDGPUInstructionSelector::selectFlatOffsetImpl(
6041 MachineOperand &Root, AMDGPU::FlatAddrSpace FlatVariant) const {
6042 MachineInstr *MI = Root.getParent();
6043
6044 auto Default = std::pair(Root.getReg(), 0);
6045
6046 if (!STI.hasFlatInstOffsets())
6047 return Default;
6048
6049 Register PtrBase;
6050 int64_t ConstOffset;
6051 bool IsInBounds;
6052 std::tie(args&: PtrBase, args&: ConstOffset, args&: IsInBounds) =
6053 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6054
6055 // Adding the offset to the base address with an immediate in a FLAT
6056 // instruction must not change the memory aperture in which the address falls.
6057 // Therefore we can only fold offsets from inbounds GEPs into FLAT
6058 // instructions.
6059 if (ConstOffset == 0 ||
6060 (FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch &&
6061 !isFlatScratchBaseLegal(Addr: Root.getReg())) ||
6062 (FlatVariant == AMDGPU::FlatAddrSpace::FLAT && !IsInBounds))
6063 return Default;
6064
6065 unsigned AddrSpace = (*MI->memoperands_begin())->getAddrSpace();
6066 if (!TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace, FlatVariant))
6067 return Default;
6068
6069 return std::pair(PtrBase, ConstOffset);
6070}
6071
6072InstructionSelector::ComplexRendererFns
6073AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const {
6074 auto PtrWithOffset = selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FLAT);
6075
6076 return {{
6077 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6078 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6079 }};
6080}
6081
6082InstructionSelector::ComplexRendererFns
6083AMDGPUInstructionSelector::selectGlobalOffset(MachineOperand &Root) const {
6084 auto PtrWithOffset =
6085 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6086
6087 return {{
6088 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6089 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6090 }};
6091}
6092
6093InstructionSelector::ComplexRendererFns
6094AMDGPUInstructionSelector::selectScratchOffset(MachineOperand &Root) const {
6095 auto PtrWithOffset =
6096 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch);
6097
6098 return {{
6099 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6100 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6101 }};
6102}
6103
6104// Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
6105InstructionSelector::ComplexRendererFns
6106AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root,
6107 unsigned CPolBits,
6108 bool NeedIOffset) const {
6109 Register Addr = Root.getReg();
6110 Register PtrBase;
6111 int64_t ConstOffset;
6112 int64_t ImmOffset = 0;
6113
6114 // Match the immediate offset first, which canonically is moved as low as
6115 // possible.
6116 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6117 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6118
6119 if (ConstOffset != 0) {
6120 if (NeedIOffset &&
6121 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6122 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal)) {
6123 Addr = PtrBase;
6124 ImmOffset = ConstOffset;
6125 } else {
6126 auto PtrBaseDef = getDefSrcRegIgnoringCopies(Reg: PtrBase, MRI: *MRI);
6127 if (isSGPR(Reg: PtrBaseDef->Reg)) {
6128 if (ConstOffset > 0) {
6129 // Offset is too large.
6130 //
6131 // saddr + large_offset -> saddr +
6132 // (voffset = large_offset & ~MaxOffset) +
6133 // (large_offset & MaxOffset);
6134 int64_t SplitImmOffset = 0, RemainderOffset = ConstOffset;
6135 if (NeedIOffset) {
6136 std::tie(args&: SplitImmOffset, args&: RemainderOffset) =
6137 TII.splitFlatOffset(COffsetVal: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6138 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6139 }
6140
6141 if (Subtarget->hasSignedGVSOffset() ? isInt<32>(x: RemainderOffset)
6142 : isUInt<32>(x: RemainderOffset)) {
6143 MachineInstr *MI = Root.getParent();
6144 MachineBasicBlock *MBB = MI->getParent();
6145 Register HighBits =
6146 MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6147
6148 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6149 DestReg: HighBits)
6150 .addImm(Val: RemainderOffset);
6151
6152 if (NeedIOffset)
6153 return {{
6154 [=](MachineInstrBuilder &MIB) {
6155 MIB.addReg(RegNo: PtrBase);
6156 }, // saddr
6157 [=](MachineInstrBuilder &MIB) {
6158 MIB.addReg(RegNo: HighBits);
6159 }, // voffset
6160 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: SplitImmOffset); },
6161 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6162 }};
6163 return {{
6164 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrBase); }, // saddr
6165 [=](MachineInstrBuilder &MIB) {
6166 MIB.addReg(RegNo: HighBits);
6167 }, // voffset
6168 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6169 }};
6170 }
6171 }
6172
6173 // We are adding a 64 bit SGPR and a constant. If constant bus limit
6174 // is 1 we would need to perform 1 or 2 extra moves for each half of
6175 // the constant and it is better to do a scalar add and then issue a
6176 // single VALU instruction to materialize zero. Otherwise it is less
6177 // instructions to perform VALU adds with immediates or inline literals.
6178 unsigned NumLiterals =
6179 !TII.isInlineConstant(Imm: APInt(32, Lo_32(Value: ConstOffset))) +
6180 !TII.isInlineConstant(Imm: APInt(32, Hi_32(Value: ConstOffset)));
6181 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_ADD_U32_e64) > NumLiterals)
6182 return std::nullopt;
6183 }
6184 }
6185 }
6186
6187 // Match the variable offset.
6188 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6189 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6190 // Look through the SGPR->VGPR copy.
6191 Register SAddr =
6192 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
6193
6194 if (isSGPR(Reg: SAddr)) {
6195 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
6196
6197 // It's possible voffset is an SGPR here, but the copy to VGPR will be
6198 // inserted later.
6199 bool ScaleOffset = selectScaleOffset(Root, Offset&: PtrBaseOffset,
6200 IsSigned: Subtarget->hasSignedGVSOffset());
6201 if (Register VOffset = matchExtendFromS32OrS32(
6202 Reg: PtrBaseOffset, IsSigned: Subtarget->hasSignedGVSOffset())) {
6203 if (NeedIOffset)
6204 return {{[=](MachineInstrBuilder &MIB) { // saddr
6205 MIB.addReg(RegNo: SAddr);
6206 },
6207 [=](MachineInstrBuilder &MIB) { // voffset
6208 MIB.addReg(RegNo: VOffset);
6209 },
6210 [=](MachineInstrBuilder &MIB) { // offset
6211 MIB.addImm(Val: ImmOffset);
6212 },
6213 [=](MachineInstrBuilder &MIB) { // cpol
6214 MIB.addImm(Val: CPolBits |
6215 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6216 }}};
6217 return {{[=](MachineInstrBuilder &MIB) { // saddr
6218 MIB.addReg(RegNo: SAddr);
6219 },
6220 [=](MachineInstrBuilder &MIB) { // voffset
6221 MIB.addReg(RegNo: VOffset);
6222 },
6223 [=](MachineInstrBuilder &MIB) { // cpol
6224 MIB.addImm(Val: CPolBits |
6225 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6226 }}};
6227 }
6228 }
6229 }
6230
6231 // FIXME: We should probably have folded COPY (G_IMPLICIT_DEF) earlier, and
6232 // drop this.
6233 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF ||
6234 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(Reg: AddrDef->Reg))
6235 return std::nullopt;
6236
6237 // It's cheaper to materialize a single 32-bit zero for vaddr than the two
6238 // moves required to copy a 64-bit SGPR to VGPR.
6239 MachineInstr *MI = Root.getParent();
6240 MachineBasicBlock *MBB = MI->getParent();
6241 Register VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6242
6243 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
6244 .addImm(Val: 0);
6245
6246 if (NeedIOffset)
6247 return {{
6248 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6249 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6250 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6251 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6252 }};
6253 return {{
6254 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6255 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6256 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6257 }};
6258}
6259
6260InstructionSelector::ComplexRendererFns
6261AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root) const {
6262 return selectGlobalSAddr(Root, CPolBits: 0);
6263}
6264
6265InstructionSelector::ComplexRendererFns
6266AMDGPUInstructionSelector::selectGlobalSAddrCPol(MachineOperand &Root) const {
6267 const MachineInstr &I = *Root.getParent();
6268
6269 // We are assuming CPol is always the last operand of the intrinsic.
6270 auto PassedCPol =
6271 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6272 return selectGlobalSAddr(Root, CPolBits: PassedCPol);
6273}
6274
6275InstructionSelector::ComplexRendererFns
6276AMDGPUInstructionSelector::selectGlobalSAddrCPolM0(MachineOperand &Root) const {
6277 const MachineInstr &I = *Root.getParent();
6278
6279 // We are assuming CPol is second from last operand of the intrinsic.
6280 auto PassedCPol =
6281 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6282 return selectGlobalSAddr(Root, CPolBits: PassedCPol);
6283}
6284
6285InstructionSelector::ComplexRendererFns
6286AMDGPUInstructionSelector::selectGlobalSAddrGLC(MachineOperand &Root) const {
6287 return selectGlobalSAddr(Root, CPolBits: AMDGPU::CPol::GLC);
6288}
6289
6290InstructionSelector::ComplexRendererFns
6291AMDGPUInstructionSelector::selectGlobalSAddrNoIOffset(
6292 MachineOperand &Root) const {
6293 const MachineInstr &I = *Root.getParent();
6294
6295 // We are assuming CPol is always the last operand of the intrinsic.
6296 auto PassedCPol =
6297 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6298 return selectGlobalSAddr(Root, CPolBits: PassedCPol, NeedIOffset: false);
6299}
6300
6301InstructionSelector::ComplexRendererFns
6302AMDGPUInstructionSelector::selectGlobalSAddrNoIOffsetM0(
6303 MachineOperand &Root) const {
6304 const MachineInstr &I = *Root.getParent();
6305
6306 // We are assuming CPol is second from last operand of the intrinsic.
6307 auto PassedCPol =
6308 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6309 return selectGlobalSAddr(Root, CPolBits: PassedCPol, NeedIOffset: false);
6310}
6311
6312InstructionSelector::ComplexRendererFns
6313AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const {
6314 Register Addr = Root.getReg();
6315 Register PtrBase;
6316 int64_t ConstOffset;
6317 int64_t ImmOffset = 0;
6318
6319 // Match the immediate offset first, which canonically is moved as low as
6320 // possible.
6321 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6322 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6323
6324 if (ConstOffset != 0 && isFlatScratchBaseLegal(Addr) &&
6325 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6326 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6327 Addr = PtrBase;
6328 ImmOffset = ConstOffset;
6329 }
6330
6331 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6332 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6333 int FI = AddrDef->MI->getOperand(i: 1).getIndex();
6334 return {{
6335 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6336 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6337 }};
6338 }
6339
6340 Register SAddr = AddrDef->Reg;
6341
6342 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6343 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6344 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6345 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6346 auto RHSDef = getDefSrcRegIgnoringCopies(Reg: RHS, MRI: *MRI);
6347
6348 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX &&
6349 isSGPR(Reg: RHSDef->Reg)) {
6350 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6351 MachineInstr &I = *Root.getParent();
6352 MachineBasicBlock *BB = I.getParent();
6353 const DebugLoc &DL = I.getDebugLoc();
6354 SAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6355
6356 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_I32), DestReg: SAddr)
6357 .addFrameIndex(Idx: FI)
6358 .addReg(RegNo: RHSDef->Reg)
6359 .setOperandDead(3); // Dead scc
6360 }
6361 }
6362
6363 if (!isSGPR(Reg: SAddr))
6364 return std::nullopt;
6365
6366 return {{
6367 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SAddr); }, // saddr
6368 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6369 }};
6370}
6371
6372// Check whether the flat scratch SVS swizzle bug affects this access.
6373bool AMDGPUInstructionSelector::checkFlatScratchSVSSwizzleBug(
6374 Register VAddr, Register SAddr, uint64_t ImmOffset) const {
6375 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
6376 return false;
6377
6378 // The bug affects the swizzling of SVS accesses if there is any carry out
6379 // from the two low order bits (i.e. from bit 1 into bit 2) when adding
6380 // voffset to (soffset + inst_offset).
6381 auto VKnown = VT->getKnownBits(R: VAddr);
6382 auto SKnown = KnownBits::add(LHS: VT->getKnownBits(R: SAddr),
6383 RHS: KnownBits::makeConstant(C: APInt(32, ImmOffset)));
6384 uint64_t VMax = VKnown.getMaxValue().getZExtValue();
6385 uint64_t SMax = SKnown.getMaxValue().getZExtValue();
6386 return (VMax & 3) + (SMax & 3) >= 4;
6387}
6388
6389InstructionSelector::ComplexRendererFns
6390AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const {
6391 Register Addr = Root.getReg();
6392 Register PtrBase;
6393 int64_t ConstOffset;
6394 int64_t ImmOffset = 0;
6395
6396 // Match the immediate offset first, which canonically is moved as low as
6397 // possible.
6398 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6399 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6400
6401 Register OrigAddr = Addr;
6402 if (ConstOffset != 0 &&
6403 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6404 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6405 Addr = PtrBase;
6406 ImmOffset = ConstOffset;
6407 }
6408
6409 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6410 if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD)
6411 return std::nullopt;
6412
6413 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6414 if (RBI.getRegBank(Reg: RHS, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
6415 return std::nullopt;
6416
6417 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6418 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6419
6420 if (OrigAddr != Addr) {
6421 if (!isFlatScratchBaseLegalSVImm(Addr: OrigAddr))
6422 return std::nullopt;
6423 } else {
6424 if (!isFlatScratchBaseLegalSV(Addr: OrigAddr))
6425 return std::nullopt;
6426 }
6427
6428 if (checkFlatScratchSVSSwizzleBug(VAddr: RHS, SAddr: LHS, ImmOffset))
6429 return std::nullopt;
6430
6431 unsigned CPol = selectScaleOffset(Root, Offset&: RHS, IsSigned: true /* IsSigned */)
6432 ? AMDGPU::CPol::SCAL
6433 : 0;
6434
6435 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6436 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6437 return {{
6438 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6439 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6440 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6441 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6442 }};
6443 }
6444
6445 if (!isSGPR(Reg: LHS))
6446 if (auto Def = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI))
6447 LHS = Def->Reg;
6448
6449 if (!isSGPR(Reg: LHS))
6450 return std::nullopt;
6451
6452 return {{
6453 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6454 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: LHS); }, // saddr
6455 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6456 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6457 }};
6458}
6459
6460InstructionSelector::ComplexRendererFns
6461AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
6462 MachineInstr *MI = Root.getParent();
6463 MachineBasicBlock *MBB = MI->getParent();
6464 MachineFunction *MF = MBB->getParent();
6465 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6466
6467 int64_t Offset = 0;
6468 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) &&
6469 Offset != AMDGPU::getNullPointerValue(AS: AMDGPUAS::PRIVATE_ADDRESS)) {
6470 Register HighBits = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6471
6472 // TODO: Should this be inside the render function? The iterator seems to
6473 // move.
6474 const int64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
6475 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6476 DestReg: HighBits)
6477 .addImm(Val: Offset & ~MaxOffset);
6478
6479 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6480 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6481 },
6482 [=](MachineInstrBuilder &MIB) { // vaddr
6483 MIB.addReg(RegNo: HighBits);
6484 },
6485 [=](MachineInstrBuilder &MIB) { // soffset
6486 // Use constant zero for soffset and rely on eliminateFrameIndex
6487 // to choose the appropriate frame register if need be.
6488 MIB.addImm(Val: 0);
6489 },
6490 [=](MachineInstrBuilder &MIB) { // offset
6491 MIB.addImm(Val: Offset & MaxOffset);
6492 }}};
6493 }
6494
6495 assert(Offset == 0 || Offset == -1);
6496
6497 // Try to fold a frame index directly into the MUBUF vaddr field, and any
6498 // offsets.
6499 std::optional<int> FI;
6500 Register VAddr = Root.getReg();
6501
6502 const MachineInstr *RootDef = MRI->getVRegDef(Reg: Root.getReg());
6503 Register PtrBase;
6504 int64_t ConstOffset;
6505 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6506 getPtrBaseWithConstantOffset(Root: VAddr, MRI: *MRI);
6507 if (ConstOffset != 0) {
6508 if (TII.isLegalMUBUFImmOffset(Imm: ConstOffset) &&
6509 (!STI.privateMemoryResourceIsRangeChecked() ||
6510 VT->signBitIsZero(Op: PtrBase))) {
6511 const MachineInstr *PtrBaseDef = MRI->getVRegDef(Reg: PtrBase);
6512 if (PtrBaseDef->getOpcode() == AMDGPU::G_FRAME_INDEX)
6513 FI = PtrBaseDef->getOperand(i: 1).getIndex();
6514 else
6515 VAddr = PtrBase;
6516 Offset = ConstOffset;
6517 }
6518 } else if (RootDef->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6519 FI = RootDef->getOperand(i: 1).getIndex();
6520 }
6521
6522 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6523 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6524 },
6525 [=](MachineInstrBuilder &MIB) { // vaddr
6526 if (FI)
6527 MIB.addFrameIndex(Idx: *FI);
6528 else
6529 MIB.addReg(RegNo: VAddr);
6530 },
6531 [=](MachineInstrBuilder &MIB) { // soffset
6532 // Use constant zero for soffset and rely on eliminateFrameIndex
6533 // to choose the appropriate frame register if need be.
6534 MIB.addImm(Val: 0);
6535 },
6536 [=](MachineInstrBuilder &MIB) { // offset
6537 MIB.addImm(Val: Offset);
6538 }}};
6539}
6540
6541bool AMDGPUInstructionSelector::isDSOffsetLegal(Register Base,
6542 int64_t Offset) const {
6543 if (!isUInt<16>(x: Offset))
6544 return false;
6545
6546 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6547 return true;
6548
6549 // On Southern Islands instruction with a negative base value and an offset
6550 // don't seem to work.
6551 return VT->signBitIsZero(Op: Base);
6552}
6553
6554bool AMDGPUInstructionSelector::isDSOffset2Legal(Register Base, int64_t Offset0,
6555 int64_t Offset1,
6556 unsigned Size) const {
6557 if (Offset0 % Size != 0 || Offset1 % Size != 0)
6558 return false;
6559 if (!isUInt<8>(x: Offset0 / Size) || !isUInt<8>(x: Offset1 / Size))
6560 return false;
6561
6562 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6563 return true;
6564
6565 // On Southern Islands instruction with a negative base value and an offset
6566 // don't seem to work.
6567 return VT->signBitIsZero(Op: Base);
6568}
6569
6570// Return whether the operation has NoUnsignedWrap property.
6571static bool isNoUnsignedWrap(MachineInstr *Addr) {
6572 return Addr->getOpcode() == TargetOpcode::G_OR ||
6573 (Addr->getOpcode() == TargetOpcode::G_PTR_ADD &&
6574 Addr->getFlag(Flag: MachineInstr::NoUWrap));
6575}
6576
6577// Check that the base address of flat scratch load/store in the form of `base +
6578// offset` is legal to be put in SGPR/VGPR (i.e. unsigned per hardware
6579// requirement). We always treat the first operand as the base address here.
6580bool AMDGPUInstructionSelector::isFlatScratchBaseLegal(Register Addr) const {
6581 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6582
6583 if (isNoUnsignedWrap(Addr: AddrMI))
6584 return true;
6585
6586 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6587 // values.
6588 if (STI.hasSignedScratchOffsets())
6589 return true;
6590
6591 Register LHS = AddrMI->getOperand(i: 1).getReg();
6592 Register RHS = AddrMI->getOperand(i: 2).getReg();
6593
6594 if (AddrMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
6595 std::optional<ValueAndVReg> RhsValReg =
6596 getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
6597 // If the immediate offset is negative and within certain range, the base
6598 // address cannot also be negative. If the base is also negative, the sum
6599 // would be either negative or much larger than the valid range of scratch
6600 // memory a thread can access.
6601 if (RhsValReg && RhsValReg->Value.getSExtValue() < 0 &&
6602 RhsValReg->Value.getSExtValue() > -0x40000000)
6603 return true;
6604 }
6605
6606 return VT->signBitIsZero(Op: LHS);
6607}
6608
6609// Check address value in SGPR/VGPR are legal for flat scratch in the form
6610// of: SGPR + VGPR.
6611bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSV(Register Addr) const {
6612 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6613
6614 if (isNoUnsignedWrap(Addr: AddrMI))
6615 return true;
6616
6617 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6618 // values.
6619 if (STI.hasSignedScratchOffsets())
6620 return true;
6621
6622 Register LHS = AddrMI->getOperand(i: 1).getReg();
6623 Register RHS = AddrMI->getOperand(i: 2).getReg();
6624 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6625}
6626
6627// Check address value in SGPR/VGPR are legal for flat scratch in the form
6628// of: SGPR + VGPR + Imm.
6629bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSVImm(
6630 Register Addr) const {
6631 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6632 // values.
6633 if (STI.hasSignedScratchOffsets())
6634 return true;
6635
6636 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6637 Register Base = AddrMI->getOperand(i: 1).getReg();
6638 std::optional<DefinitionAndSourceRegister> BaseDef =
6639 getDefSrcRegIgnoringCopies(Reg: Base, MRI: *MRI);
6640 std::optional<ValueAndVReg> RHSOffset =
6641 getIConstantVRegValWithLookThrough(VReg: AddrMI->getOperand(i: 2).getReg(), MRI: *MRI);
6642 assert(RHSOffset);
6643
6644 // If the immediate offset is negative and within certain range, the base
6645 // address cannot also be negative. If the base is also negative, the sum
6646 // would be either negative or much larger than the valid range of scratch
6647 // memory a thread can access.
6648 if (isNoUnsignedWrap(Addr: BaseDef->MI) &&
6649 (isNoUnsignedWrap(Addr: AddrMI) ||
6650 (RHSOffset->Value.getSExtValue() < 0 &&
6651 RHSOffset->Value.getSExtValue() > -0x40000000)))
6652 return true;
6653
6654 Register LHS = BaseDef->MI->getOperand(i: 1).getReg();
6655 Register RHS = BaseDef->MI->getOperand(i: 2).getReg();
6656 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6657}
6658
6659bool AMDGPUInstructionSelector::isUnneededShiftMask(const MachineInstr &MI,
6660 unsigned ShAmtBits) const {
6661 assert(MI.getOpcode() == TargetOpcode::G_AND);
6662
6663 std::optional<APInt> RHS =
6664 getIConstantVRegVal(VReg: MI.getOperand(i: 2).getReg(), MRI: *MRI);
6665 if (!RHS)
6666 return false;
6667
6668 if (RHS->countr_one() >= ShAmtBits)
6669 return true;
6670
6671 const APInt &LHSKnownZeros = VT->getKnownZeroes(R: MI.getOperand(i: 1).getReg());
6672 return (LHSKnownZeros | *RHS).countr_one() >= ShAmtBits;
6673}
6674
6675InstructionSelector::ComplexRendererFns
6676AMDGPUInstructionSelector::selectMUBUFScratchOffset(
6677 MachineOperand &Root) const {
6678 Register Reg = Root.getReg();
6679 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6680
6681 std::optional<DefinitionAndSourceRegister> Def =
6682 getDefSrcRegIgnoringCopies(Reg, MRI: *MRI);
6683 assert(Def && "this shouldn't be an optional result");
6684 Reg = Def->Reg;
6685
6686 if (Register WaveBase = getWaveAddress(Def: Def->MI)) {
6687 return {{
6688 [=](MachineInstrBuilder &MIB) { // rsrc
6689 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6690 },
6691 [=](MachineInstrBuilder &MIB) { // soffset
6692 MIB.addReg(RegNo: WaveBase);
6693 },
6694 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // offset
6695 }};
6696 }
6697
6698 int64_t Offset = 0;
6699
6700 // FIXME: Copy check is a hack
6701 Register BasePtr;
6702 if (mi_match(R: Reg, MRI: *MRI,
6703 P: m_GPtrAdd(L: m_Reg(R&: BasePtr),
6704 R: m_any_of(preds: m_ICst(Cst&: Offset), preds: m_Copy(Src: m_ICst(Cst&: Offset)))))) {
6705 if (!TII.isLegalMUBUFImmOffset(Imm: Offset))
6706 return {};
6707 MachineInstr *BasePtrDef = getDefIgnoringCopies(Reg: BasePtr, MRI: *MRI);
6708 Register WaveBase = getWaveAddress(Def: BasePtrDef);
6709 if (!WaveBase)
6710 return {};
6711
6712 return {{
6713 [=](MachineInstrBuilder &MIB) { // rsrc
6714 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6715 },
6716 [=](MachineInstrBuilder &MIB) { // soffset
6717 MIB.addReg(RegNo: WaveBase);
6718 },
6719 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6720 }};
6721 }
6722
6723 if (!mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) ||
6724 !TII.isLegalMUBUFImmOffset(Imm: Offset))
6725 return {};
6726
6727 return {{
6728 [=](MachineInstrBuilder &MIB) { // rsrc
6729 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6730 },
6731 [=](MachineInstrBuilder &MIB) { // soffset
6732 MIB.addImm(Val: 0);
6733 },
6734 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6735 }};
6736}
6737
6738std::pair<Register, unsigned>
6739AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(MachineOperand &Root) const {
6740 const MachineInstr *RootDef = MRI->getVRegDef(Reg: Root.getReg());
6741 int64_t ConstAddr = 0;
6742
6743 Register PtrBase;
6744 int64_t Offset;
6745 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6746 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6747
6748 if (Offset) {
6749 if (isDSOffsetLegal(Base: PtrBase, Offset)) {
6750 // (add n0, c0)
6751 return std::pair(PtrBase, Offset);
6752 }
6753 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) {
6754 // TODO
6755
6756
6757 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6758 // TODO
6759
6760 }
6761
6762 return std::pair(Root.getReg(), 0);
6763}
6764
6765InstructionSelector::ComplexRendererFns
6766AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
6767 Register Reg;
6768 unsigned Offset;
6769 std::tie(args&: Reg, args&: Offset) = selectDS1Addr1OffsetImpl(Root);
6770 return {{
6771 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6772 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }
6773 }};
6774}
6775
6776InstructionSelector::ComplexRendererFns
6777AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const {
6778 return selectDSReadWrite2(Root, size: 4);
6779}
6780
6781InstructionSelector::ComplexRendererFns
6782AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(MachineOperand &Root) const {
6783 return selectDSReadWrite2(Root, size: 8);
6784}
6785
6786InstructionSelector::ComplexRendererFns
6787AMDGPUInstructionSelector::selectDSReadWrite2(MachineOperand &Root,
6788 unsigned Size) const {
6789 Register Reg;
6790 unsigned Offset;
6791 std::tie(args&: Reg, args&: Offset) = selectDSReadWrite2Impl(Root, size: Size);
6792 return {{
6793 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6794 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
6795 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset+1); }
6796 }};
6797}
6798
6799std::pair<Register, unsigned>
6800AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root,
6801 unsigned Size) const {
6802 const MachineInstr *RootDef = MRI->getVRegDef(Reg: Root.getReg());
6803 int64_t ConstAddr = 0;
6804
6805 Register PtrBase;
6806 int64_t Offset;
6807 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6808 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6809
6810 if (Offset) {
6811 int64_t OffsetValue0 = Offset;
6812 int64_t OffsetValue1 = Offset + Size;
6813 if (isDSOffset2Legal(Base: PtrBase, Offset0: OffsetValue0, Offset1: OffsetValue1, Size)) {
6814 // (add n0, c0)
6815 return std::pair(PtrBase, OffsetValue0 / Size);
6816 }
6817 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) {
6818 // TODO
6819
6820 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6821 // TODO
6822
6823 }
6824
6825 return std::pair(Root.getReg(), 0);
6826}
6827
6828/// If \p Root is a G_PTR_ADD with a G_CONSTANT on the right hand side, return
6829/// the base value with the constant offset, and if the offset computation is
6830/// known to be inbounds. There may be intervening copies between \p Root and
6831/// the identified constant. Returns \p Root, 0, false if this does not match
6832/// the pattern.
6833std::tuple<Register, int64_t, bool>
6834AMDGPUInstructionSelector::getPtrBaseWithConstantOffset(
6835 Register Root, const MachineRegisterInfo &MRI) const {
6836 MachineInstr *RootI = getDefIgnoringCopies(Reg: Root, MRI);
6837 if (RootI->getOpcode() != TargetOpcode::G_PTR_ADD)
6838 return {Root, 0, false};
6839
6840 MachineOperand &RHS = RootI->getOperand(i: 2);
6841 std::optional<ValueAndVReg> MaybeOffset =
6842 getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI);
6843 if (!MaybeOffset)
6844 return {Root, 0, false};
6845 bool IsInBounds = RootI->getFlag(Flag: MachineInstr::MIFlag::InBounds);
6846 return {RootI->getOperand(i: 1).getReg(), MaybeOffset->Value.getSExtValue(),
6847 IsInBounds};
6848}
6849
6850static void addZeroImm(MachineInstrBuilder &MIB) {
6851 MIB.addImm(Val: 0);
6852}
6853
6854/// Return a resource descriptor for use with an arbitrary 64-bit pointer. If \p
6855/// BasePtr is not valid, a null base pointer will be used.
6856static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6857 uint32_t FormatLo, uint32_t FormatHi,
6858 Register BasePtr) {
6859 Register RSrc2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6860 Register RSrc3 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6861 Register RSrcHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6862 Register RSrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
6863
6864 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6865 .addDef(RegNo: RSrc2)
6866 .addImm(Val: FormatLo);
6867 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6868 .addDef(RegNo: RSrc3)
6869 .addImm(Val: FormatHi);
6870
6871 // Build the half of the subregister with the constants before building the
6872 // full 128-bit register. If we are building multiple resource descriptors,
6873 // this will allow CSEing of the 2-component register.
6874 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6875 .addDef(RegNo: RSrcHi)
6876 .addReg(RegNo: RSrc2)
6877 .addImm(Val: AMDGPU::sub0)
6878 .addReg(RegNo: RSrc3)
6879 .addImm(Val: AMDGPU::sub1);
6880
6881 Register RSrcLo = BasePtr;
6882 if (!BasePtr) {
6883 RSrcLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6884 B.buildInstr(Opcode: AMDGPU::S_MOV_B64)
6885 .addDef(RegNo: RSrcLo)
6886 .addImm(Val: 0);
6887 }
6888
6889 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6890 .addDef(RegNo: RSrc)
6891 .addReg(RegNo: RSrcLo)
6892 .addImm(Val: AMDGPU::sub0_sub1)
6893 .addReg(RegNo: RSrcHi)
6894 .addImm(Val: AMDGPU::sub2_sub3);
6895
6896 return RSrc;
6897}
6898
6899static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6900 const SIInstrInfo &TII, Register BasePtr) {
6901 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
6902
6903 // FIXME: Why are half the "default" bits ignored based on the addressing
6904 // mode?
6905 return buildRSRC(B, MRI, FormatLo: 0, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
6906}
6907
6908static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6909 const SIInstrInfo &TII, Register BasePtr) {
6910 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
6911
6912 // FIXME: Why are half the "default" bits ignored based on the addressing
6913 // mode?
6914 return buildRSRC(B, MRI, FormatLo: -1, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
6915}
6916
6917AMDGPUInstructionSelector::MUBUFAddressData
6918AMDGPUInstructionSelector::parseMUBUFAddress(Register Src) const {
6919 MUBUFAddressData Data;
6920 Data.N0 = Src;
6921
6922 Register PtrBase;
6923 int64_t Offset;
6924
6925 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6926 getPtrBaseWithConstantOffset(Root: Src, MRI: *MRI);
6927 if (isUInt<32>(x: Offset)) {
6928 Data.N0 = PtrBase;
6929 Data.Offset = Offset;
6930 }
6931
6932 if (MachineInstr *InputAdd
6933 = getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Data.N0, MRI: *MRI)) {
6934 Data.N2 = InputAdd->getOperand(i: 1).getReg();
6935 Data.N3 = InputAdd->getOperand(i: 2).getReg();
6936
6937 // FIXME: Need to fix extra SGPR->VGPRcopies inserted
6938 // FIXME: Don't know this was defined by operand 0
6939 //
6940 // TODO: Remove this when we have copy folding optimizations after
6941 // RegBankSelect.
6942 Data.N2 = getDefIgnoringCopies(Reg: Data.N2, MRI: *MRI)->getOperand(i: 0).getReg();
6943 Data.N3 = getDefIgnoringCopies(Reg: Data.N3, MRI: *MRI)->getOperand(i: 0).getReg();
6944 }
6945
6946 return Data;
6947}
6948
6949/// Return if the addr64 mubuf mode should be used for the given address.
6950bool AMDGPUInstructionSelector::shouldUseAddr64(MUBUFAddressData Addr) const {
6951 // (ptr_add N2, N3) -> addr64, or
6952 // (ptr_add (ptr_add N2, N3), C1) -> addr64
6953 if (Addr.N2)
6954 return true;
6955
6956 const RegisterBank *N0Bank = RBI.getRegBank(Reg: Addr.N0, MRI: *MRI, TRI);
6957 return N0Bank->getID() == AMDGPU::VGPRRegBankID;
6958}
6959
6960/// Split an immediate offset \p ImmOffset depending on whether it fits in the
6961/// immediate field. Modifies \p ImmOffset and sets \p SOffset to the variable
6962/// component.
6963void AMDGPUInstructionSelector::splitIllegalMUBUFOffset(
6964 MachineIRBuilder &B, Register &SOffset, int64_t &ImmOffset) const {
6965 if (TII.isLegalMUBUFImmOffset(Imm: ImmOffset))
6966 return;
6967
6968 // Illegal offset, store it in soffset.
6969 SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6970 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6971 .addDef(RegNo: SOffset)
6972 .addImm(Val: ImmOffset);
6973 ImmOffset = 0;
6974}
6975
6976bool AMDGPUInstructionSelector::selectMUBUFAddr64Impl(
6977 MachineOperand &Root, Register &VAddr, Register &RSrcReg,
6978 Register &SOffset, int64_t &Offset) const {
6979 // FIXME: Predicates should stop this from reaching here.
6980 // addr64 bit was removed for volcanic islands.
6981 if (!STI.hasAddr64() || STI.useFlatForGlobal())
6982 return false;
6983
6984 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
6985 if (!shouldUseAddr64(Addr: AddrData))
6986 return false;
6987
6988 Register N0 = AddrData.N0;
6989 Register N2 = AddrData.N2;
6990 Register N3 = AddrData.N3;
6991 Offset = AddrData.Offset;
6992
6993 // Base pointer for the SRD.
6994 Register SRDPtr;
6995
6996 if (N2) {
6997 if (RBI.getRegBank(Reg: N2, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
6998 assert(N3);
6999 if (RBI.getRegBank(Reg: N3, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7000 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
7001 // addr64, and construct the default resource from a 0 address.
7002 VAddr = N0;
7003 } else {
7004 SRDPtr = N3;
7005 VAddr = N2;
7006 }
7007 } else {
7008 // N2 is not divergent.
7009 SRDPtr = N2;
7010 VAddr = N3;
7011 }
7012 } else if (RBI.getRegBank(Reg: N0, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7013 // Use the default null pointer in the resource
7014 VAddr = N0;
7015 } else {
7016 // N0 -> offset, or
7017 // (N0 + C1) -> offset
7018 SRDPtr = N0;
7019 }
7020
7021 MachineIRBuilder B(*Root.getParent());
7022 RSrcReg = buildAddr64RSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
7023 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
7024 return true;
7025}
7026
7027bool AMDGPUInstructionSelector::selectMUBUFOffsetImpl(
7028 MachineOperand &Root, Register &RSrcReg, Register &SOffset,
7029 int64_t &Offset) const {
7030
7031 // FIXME: Pattern should not reach here.
7032 if (STI.useFlatForGlobal())
7033 return false;
7034
7035 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
7036 if (shouldUseAddr64(Addr: AddrData))
7037 return false;
7038
7039 // N0 -> offset, or
7040 // (N0 + C1) -> offset
7041 Register SRDPtr = AddrData.N0;
7042 Offset = AddrData.Offset;
7043
7044 // TODO: Look through extensions for 32-bit soffset.
7045 MachineIRBuilder B(*Root.getParent());
7046
7047 RSrcReg = buildOffsetSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
7048 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
7049 return true;
7050}
7051
7052InstructionSelector::ComplexRendererFns
7053AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const {
7054 Register VAddr;
7055 Register RSrcReg;
7056 Register SOffset;
7057 int64_t Offset = 0;
7058
7059 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset))
7060 return {};
7061
7062 // FIXME: Use defaulted operands for trailing 0s and remove from the complex
7063 // pattern.
7064 return {{
7065 [=](MachineInstrBuilder &MIB) { // rsrc
7066 MIB.addReg(RegNo: RSrcReg);
7067 },
7068 [=](MachineInstrBuilder &MIB) { // vaddr
7069 MIB.addReg(RegNo: VAddr);
7070 },
7071 [=](MachineInstrBuilder &MIB) { // soffset
7072 if (SOffset)
7073 MIB.addReg(RegNo: SOffset);
7074 else if (STI.hasRestrictedSOffset())
7075 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7076 else
7077 MIB.addImm(Val: 0);
7078 },
7079 [=](MachineInstrBuilder &MIB) { // offset
7080 MIB.addImm(Val: Offset);
7081 },
7082 addZeroImm, // cpol
7083 addZeroImm, // tfe
7084 addZeroImm // swz
7085 }};
7086}
7087
7088InstructionSelector::ComplexRendererFns
7089AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const {
7090 Register RSrcReg;
7091 Register SOffset;
7092 int64_t Offset = 0;
7093
7094 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset))
7095 return {};
7096
7097 return {{
7098 [=](MachineInstrBuilder &MIB) { // rsrc
7099 MIB.addReg(RegNo: RSrcReg);
7100 },
7101 [=](MachineInstrBuilder &MIB) { // soffset
7102 if (SOffset)
7103 MIB.addReg(RegNo: SOffset);
7104 else if (STI.hasRestrictedSOffset())
7105 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7106 else
7107 MIB.addImm(Val: 0);
7108 },
7109 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }, // offset
7110 addZeroImm, // cpol
7111 addZeroImm, // tfe
7112 addZeroImm, // swz
7113 }};
7114}
7115
7116InstructionSelector::ComplexRendererFns
7117AMDGPUInstructionSelector::selectBUFSOffset(MachineOperand &Root) const {
7118
7119 Register SOffset = Root.getReg();
7120
7121 if (STI.hasRestrictedSOffset() && mi_match(R: SOffset, MRI: *MRI, P: m_ZeroInt()))
7122 SOffset = AMDGPU::SGPR_NULL;
7123
7124 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); }}};
7125}
7126
7127/// Get an immediate that must be 32-bits, and treated as zero extended.
7128static std::optional<uint64_t>
7129getConstantZext32Val(Register Reg, const MachineRegisterInfo &MRI) {
7130 // getIConstantVRegVal sexts any values, so see if that matters.
7131 std::optional<int64_t> OffsetVal = getIConstantVRegSExtVal(VReg: Reg, MRI);
7132 if (!OffsetVal || !isInt<32>(x: *OffsetVal))
7133 return std::nullopt;
7134 return Lo_32(Value: *OffsetVal);
7135}
7136
7137InstructionSelector::ComplexRendererFns
7138AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const {
7139 std::optional<uint64_t> OffsetVal =
7140 Root.isImm() ? Root.getImm() : getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7141 if (!OffsetVal)
7142 return {};
7143
7144 std::optional<int64_t> EncodedImm =
7145 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: *OffsetVal, IsBuffer: true);
7146 if (!EncodedImm)
7147 return {};
7148
7149 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7150}
7151
7152InstructionSelector::ComplexRendererFns
7153AMDGPUInstructionSelector::selectSMRDBufferImm32(MachineOperand &Root) const {
7154 assert(STI.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
7155
7156 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7157 if (!OffsetVal)
7158 return {};
7159
7160 std::optional<int64_t> EncodedImm =
7161 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: *OffsetVal);
7162 if (!EncodedImm)
7163 return {};
7164
7165 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7166}
7167
7168InstructionSelector::ComplexRendererFns
7169AMDGPUInstructionSelector::selectSMRDBufferSgprImm(MachineOperand &Root) const {
7170 // Match the (soffset + offset) pair as a 32-bit register base and
7171 // an immediate offset.
7172 Register SOffset;
7173 unsigned Offset;
7174 std::tie(args&: SOffset, args&: Offset) = AMDGPU::getBaseWithConstantOffset(
7175 MRI&: *MRI, Reg: Root.getReg(), ValueTracking: VT, /*CheckNUW*/ true);
7176 if (!SOffset)
7177 return std::nullopt;
7178
7179 std::optional<int64_t> EncodedOffset =
7180 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: Offset, /* IsBuffer */ true);
7181 if (!EncodedOffset)
7182 return std::nullopt;
7183
7184 assert(MRI->getType(SOffset) == LLT::scalar(32));
7185 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
7186 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedOffset); }}};
7187}
7188
7189std::pair<Register, unsigned>
7190AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(MachineOperand &Root,
7191 bool &Matched) const {
7192 Matched = false;
7193
7194 Register Src;
7195 unsigned Mods;
7196 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
7197
7198 if (mi_match(R: Src, MRI: *MRI, P: m_GFPExt(Src: m_Reg(R&: Src)))) {
7199 assert(MRI->getType(Src) == LLT::scalar(16));
7200
7201 // Only change Src if src modifier could be gained. In such cases new Src
7202 // could be sgpr but this does not violate constant bus restriction for
7203 // instruction that is being selected.
7204 Src = stripBitCast(Reg: Src, MRI&: *MRI);
7205
7206 const auto CheckAbsNeg = [&]() {
7207 // Be careful about folding modifiers if we already have an abs. fneg is
7208 // applied last, so we don't want to apply an earlier fneg.
7209 if ((Mods & SISrcMods::ABS) == 0) {
7210 unsigned ModsTmp;
7211 std::tie(args&: Src, args&: ModsTmp) = selectVOP3ModsImpl(Src);
7212
7213 if ((ModsTmp & SISrcMods::NEG) != 0)
7214 Mods ^= SISrcMods::NEG;
7215
7216 if ((ModsTmp & SISrcMods::ABS) != 0)
7217 Mods |= SISrcMods::ABS;
7218 }
7219 };
7220
7221 CheckAbsNeg();
7222
7223 // op_sel/op_sel_hi decide the source type and source.
7224 // If the source's op_sel_hi is set, it indicates to do a conversion from
7225 // fp16. If the sources's op_sel is set, it picks the high half of the
7226 // source register.
7227
7228 Mods |= SISrcMods::OP_SEL_1;
7229
7230 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
7231 Mods |= SISrcMods::OP_SEL_0;
7232 CheckAbsNeg();
7233 }
7234
7235 Matched = true;
7236 }
7237
7238 return {Src, Mods};
7239}
7240
7241InstructionSelector::ComplexRendererFns
7242AMDGPUInstructionSelector::selectVOP3PMadMixModsExt(
7243 MachineOperand &Root) const {
7244 Register Src;
7245 unsigned Mods;
7246 bool Matched;
7247 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7248 if (!Matched)
7249 return {};
7250
7251 return {{
7252 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
7253 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7254 }};
7255}
7256
7257InstructionSelector::ComplexRendererFns
7258AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const {
7259 Register Src;
7260 unsigned Mods;
7261 bool Matched;
7262 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7263
7264 return {{
7265 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
7266 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7267 }};
7268}
7269
7270bool AMDGPUInstructionSelector::selectSBarrierSignalIsfirst(
7271 MachineInstr &I, Intrinsic::ID IntrID) const {
7272 MachineBasicBlock *MBB = I.getParent();
7273 const DebugLoc &DL = I.getDebugLoc();
7274 Register CCReg = I.getOperand(i: 0).getReg();
7275
7276 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7277 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_EQ_U32)).addImm(Val: 0).addImm(Val: 0);
7278
7279 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM))
7280 .addImm(Val: I.getOperand(i: 2).getImm());
7281
7282 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg).addReg(RegNo: AMDGPU::SCC);
7283
7284 I.eraseFromParent();
7285 return RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32_XM0_XEXECRegClass,
7286 MRI&: *MRI);
7287}
7288
7289bool AMDGPUInstructionSelector::selectSGetBarrierState(
7290 MachineInstr &I, Intrinsic::ID IntrID) const {
7291 MachineBasicBlock *MBB = I.getParent();
7292 const DebugLoc &DL = I.getDebugLoc();
7293 const MachineOperand &BarOp = I.getOperand(i: 2);
7294 std::optional<int64_t> BarValImm =
7295 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7296
7297 if (!BarValImm) {
7298 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7299 .addReg(RegNo: BarOp.getReg());
7300 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7301 }
7302 MachineInstrBuilder MIB;
7303 unsigned Opc = BarValImm ? AMDGPU::S_GET_BARRIER_STATE_IMM
7304 : AMDGPU::S_GET_BARRIER_STATE_M0;
7305 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7306
7307 auto DstReg = I.getOperand(i: 0).getReg();
7308 const TargetRegisterClass *DstRC =
7309 TRI.getConstrainedRegClassForOperand(MO: I.getOperand(i: 0), MRI: *MRI);
7310 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7311 return false;
7312 MIB.addDef(RegNo: DstReg);
7313 if (BarValImm) {
7314 MIB.addImm(Val: *BarValImm);
7315 }
7316 I.eraseFromParent();
7317 return true;
7318}
7319
7320unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID) {
7321 if (HasInlineConst) {
7322 switch (IntrID) {
7323 default:
7324 llvm_unreachable("not a named barrier op");
7325 case Intrinsic::amdgcn_s_barrier_join:
7326 return AMDGPU::S_BARRIER_JOIN_IMM;
7327 case Intrinsic::amdgcn_s_wakeup_barrier:
7328 return AMDGPU::S_WAKEUP_BARRIER_IMM;
7329 case Intrinsic::amdgcn_s_get_named_barrier_state:
7330 return AMDGPU::S_GET_BARRIER_STATE_IMM;
7331 };
7332 } else {
7333 switch (IntrID) {
7334 default:
7335 llvm_unreachable("not a named barrier op");
7336 case Intrinsic::amdgcn_s_barrier_join:
7337 return AMDGPU::S_BARRIER_JOIN_M0;
7338 case Intrinsic::amdgcn_s_wakeup_barrier:
7339 return AMDGPU::S_WAKEUP_BARRIER_M0;
7340 case Intrinsic::amdgcn_s_get_named_barrier_state:
7341 return AMDGPU::S_GET_BARRIER_STATE_M0;
7342 };
7343 }
7344}
7345
7346bool AMDGPUInstructionSelector::selectNamedBarrierInit(
7347 MachineInstr &I, Intrinsic::ID IntrID) const {
7348 MachineBasicBlock *MBB = I.getParent();
7349 const DebugLoc &DL = I.getDebugLoc();
7350 const MachineOperand &BarOp = I.getOperand(i: 1);
7351 const MachineOperand &CntOp = I.getOperand(i: 2);
7352
7353 // A member count of 0 means "keep existing member count". That plus a known
7354 // constant value for the barrier ID lets us use the immarg form.
7355 if (IntrID == Intrinsic::amdgcn_s_barrier_signal_var) {
7356 std::optional<int64_t> CntImm =
7357 getIConstantVRegSExtVal(VReg: CntOp.getReg(), MRI: *MRI);
7358 if (CntImm && *CntImm == 0) {
7359 std::optional<int64_t> BarValImm =
7360 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7361 if (BarValImm) {
7362 auto BarID = ((*BarValImm) >> 4) & 0x3F;
7363 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_IMM))
7364 .addImm(Val: BarID);
7365 I.eraseFromParent();
7366 return true;
7367 }
7368 }
7369 }
7370
7371 // BarID = (BarOp >> 4) & 0x3F
7372 Register TmpReg0 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7373 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: TmpReg0)
7374 .add(MO: BarOp)
7375 .addImm(Val: 4u)
7376 .setOperandDead(3); // Dead scc
7377
7378 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7379 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7380 .addReg(RegNo: TmpReg0)
7381 .addImm(Val: 0x3F)
7382 .setOperandDead(3); // Dead scc
7383
7384 // MO = ((CntOp & 0x3F) << shAmt) | BarID
7385 Register TmpReg2 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7386 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg2)
7387 .add(MO: CntOp)
7388 .addImm(Val: 0x3F)
7389 .setOperandDead(3); // Dead scc
7390
7391 Register TmpReg3 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7392 constexpr unsigned ShAmt = 16;
7393 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg3)
7394 .addReg(RegNo: TmpReg2)
7395 .addImm(Val: ShAmt)
7396 .setOperandDead(3); // Dead scc
7397
7398 Register TmpReg4 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7399 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B32), DestReg: TmpReg4)
7400 .addReg(RegNo: TmpReg1)
7401 .addReg(RegNo: TmpReg3)
7402 .setOperandDead(3); // Dead scc;
7403
7404 auto CopyMIB =
7405 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0).addReg(RegNo: TmpReg4);
7406 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7407
7408 unsigned Opc = IntrID == Intrinsic::amdgcn_s_barrier_init
7409 ? AMDGPU::S_BARRIER_INIT_M0
7410 : AMDGPU::S_BARRIER_SIGNAL_M0;
7411 MachineInstrBuilder MIB;
7412 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7413
7414 I.eraseFromParent();
7415 return true;
7416}
7417
7418bool AMDGPUInstructionSelector::selectNamedBarrierInst(
7419 MachineInstr &I, Intrinsic::ID IntrID) const {
7420 MachineBasicBlock *MBB = I.getParent();
7421 const DebugLoc &DL = I.getDebugLoc();
7422 MachineOperand BarOp = IntrID == Intrinsic::amdgcn_s_get_named_barrier_state
7423 ? I.getOperand(i: 2)
7424 : I.getOperand(i: 1);
7425 std::optional<int64_t> BarValImm =
7426 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7427
7428 if (!BarValImm) {
7429 // BarID = (BarOp >> 4) & 0x3F
7430 Register TmpReg0 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7431 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: TmpReg0)
7432 .addReg(RegNo: BarOp.getReg())
7433 .addImm(Val: 4u)
7434 .setOperandDead(3); // Dead scc;
7435
7436 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7437 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7438 .addReg(RegNo: TmpReg0)
7439 .addImm(Val: 0x3F)
7440 .setOperandDead(3); // Dead scc;
7441
7442 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7443 .addReg(RegNo: TmpReg1);
7444 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7445 }
7446
7447 MachineInstrBuilder MIB;
7448 unsigned Opc = getNamedBarrierOp(HasInlineConst: BarValImm.has_value(), IntrID);
7449 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7450
7451 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
7452 auto DstReg = I.getOperand(i: 0).getReg();
7453 const TargetRegisterClass *DstRC =
7454 TRI.getConstrainedRegClassForOperand(MO: I.getOperand(i: 0), MRI: *MRI);
7455 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7456 return false;
7457 MIB.addDef(RegNo: DstReg);
7458 }
7459
7460 if (BarValImm) {
7461 auto BarId = ((*BarValImm) >> 4) & 0x3F;
7462 MIB.addImm(Val: BarId);
7463 }
7464
7465 I.eraseFromParent();
7466 return true;
7467}
7468
7469void AMDGPUInstructionSelector::renderTruncImm32(MachineInstrBuilder &MIB,
7470 const MachineInstr &MI,
7471 int OpIdx) const {
7472 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7473 "Expected G_CONSTANT");
7474 MIB.addImm(Val: MI.getOperand(i: 1).getCImm()->getSExtValue());
7475}
7476
7477void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB,
7478 const MachineInstr &MI,
7479 int OpIdx) const {
7480 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7481 "Expected G_CONSTANT");
7482 MIB.addImm(Val: -MI.getOperand(i: 1).getCImm()->getSExtValue());
7483}
7484
7485void AMDGPUInstructionSelector::renderBitcastFPImm(MachineInstrBuilder &MIB,
7486 const MachineInstr &MI,
7487 int OpIdx) const {
7488 const MachineOperand &Op = MI.getOperand(i: 1);
7489 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1);
7490 MIB.addImm(Val: Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
7491}
7492
7493void AMDGPUInstructionSelector::renderCountTrailingOnesImm(
7494 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7495 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7496 "Expected G_CONSTANT");
7497 MIB.addImm(Val: MI.getOperand(i: 1).getCImm()->getValue().countTrailingOnes());
7498}
7499
7500/// This only really exists to satisfy DAG type checking machinery, so is a
7501/// no-op here.
7502void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB,
7503 const MachineInstr &MI,
7504 int OpIdx) const {
7505 const MachineOperand &Op = MI.getOperand(i: OpIdx);
7506 int64_t Imm;
7507 if (Op.isReg() && mi_match(R: Op.getReg(), MRI: *MRI, P: m_ICst(Cst&: Imm)))
7508 MIB.addImm(Val: Imm);
7509 else
7510 MIB.addImm(Val: Op.getImm());
7511}
7512
7513void AMDGPUInstructionSelector::renderZextBoolTImm(MachineInstrBuilder &MIB,
7514 const MachineInstr &MI,
7515 int OpIdx) const {
7516 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() != 0);
7517}
7518
7519void AMDGPUInstructionSelector::renderOpSelTImm(MachineInstrBuilder &MIB,
7520 const MachineInstr &MI,
7521 int OpIdx) const {
7522 assert(OpIdx >= 0 && "expected to match an immediate operand");
7523 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7524}
7525
7526void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_0(
7527 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7528 assert(OpIdx >= 0 && "expected to match an immediate operand");
7529 MIB.addImm(
7530 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7531}
7532
7533void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_1(
7534 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7535 assert(OpIdx >= 0 && "expected to match an immediate operand");
7536 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x1)
7537 ? (int64_t)(SISrcMods::OP_SEL_0 | SISrcMods::DST_OP_SEL)
7538 : (int64_t)SISrcMods::DST_OP_SEL);
7539}
7540
7541void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_0(
7542 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7543 assert(OpIdx >= 0 && "expected to match an immediate operand");
7544 MIB.addImm(
7545 Val: (MI.getOperand(i: OpIdx).getImm() & 0x2) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7546}
7547
7548void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_1(
7549 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7550 assert(OpIdx >= 0 && "expected to match an immediate operand");
7551 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7552 ? (int64_t)(SISrcMods::OP_SEL_0)
7553 : 0);
7554}
7555
7556void AMDGPUInstructionSelector::renderDstSelToOpSelXForm(
7557 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7558 assert(OpIdx >= 0 && "expected to match an immediate operand");
7559 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::DST_OP_SEL)
7560 : 0);
7561}
7562
7563void AMDGPUInstructionSelector::renderSrcSelToOpSelXForm(
7564 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7565 assert(OpIdx >= 0 && "expected to match an immediate operand");
7566 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::OP_SEL_0)
7567 : 0);
7568}
7569
7570void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_2_0(
7571 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7572 assert(OpIdx >= 0 && "expected to match an immediate operand");
7573 MIB.addImm(
7574 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7575}
7576
7577void AMDGPUInstructionSelector::renderDstSelToOpSel3XFormXForm(
7578 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7579 assert(OpIdx >= 0 && "expected to match an immediate operand");
7580 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7581 ? (int64_t)SISrcMods::DST_OP_SEL
7582 : 0);
7583}
7584
7585void AMDGPUInstructionSelector::renderExtractCPol(MachineInstrBuilder &MIB,
7586 const MachineInstr &MI,
7587 int OpIdx) const {
7588 assert(OpIdx >= 0 && "expected to match an immediate operand");
7589 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() &
7590 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7591 : AMDGPU::CPol::ALL_pregfx12));
7592}
7593
7594void AMDGPUInstructionSelector::renderExtractSWZ(MachineInstrBuilder &MIB,
7595 const MachineInstr &MI,
7596 int OpIdx) const {
7597 assert(OpIdx >= 0 && "expected to match an immediate operand");
7598 const bool Swizzle = MI.getOperand(i: OpIdx).getImm() &
7599 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::SWZ
7600 : AMDGPU::CPol::SWZ_pregfx12);
7601 MIB.addImm(Val: Swizzle);
7602}
7603
7604void AMDGPUInstructionSelector::renderExtractCpolSetGLC(
7605 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7606 assert(OpIdx >= 0 && "expected to match an immediate operand");
7607 const uint32_t Cpol = MI.getOperand(i: OpIdx).getImm() &
7608 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7609 : AMDGPU::CPol::ALL_pregfx12);
7610 MIB.addImm(Val: Cpol | AMDGPU::CPol::GLC);
7611}
7612
7613void AMDGPUInstructionSelector::renderFrameIndex(MachineInstrBuilder &MIB,
7614 const MachineInstr &MI,
7615 int OpIdx) const {
7616 MIB.addFrameIndex(Idx: MI.getOperand(i: 1).getIndex());
7617}
7618
7619void AMDGPUInstructionSelector::renderFPPow2ToExponent(MachineInstrBuilder &MIB,
7620 const MachineInstr &MI,
7621 int OpIdx) const {
7622 const APFloat &APF = MI.getOperand(i: 1).getFPImm()->getValueAPF();
7623 int ExpVal = APF.getExactLog2Abs();
7624 assert(ExpVal != INT_MIN);
7625 MIB.addImm(Val: ExpVal);
7626}
7627
7628void AMDGPUInstructionSelector::renderRoundMode(MachineInstrBuilder &MIB,
7629 const MachineInstr &MI,
7630 int OpIdx) const {
7631 // "round.towardzero" -> TowardZero 0 -> FP_ROUND_ROUND_TO_ZERO 3
7632 // "round.tonearest" -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
7633 // "round.upward" -> TowardPositive 2 -> FP_ROUND_ROUND_TO_INF 1
7634 // "round.downward -> TowardNegative 3 -> FP_ROUND_ROUND_TO_NEGINF 2
7635 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() + 3) % 4);
7636}
7637
7638void AMDGPUInstructionSelector::renderVOP3PModsNeg(MachineInstrBuilder &MIB,
7639 const MachineInstr &MI,
7640 int OpIdx) const {
7641 unsigned Mods = SISrcMods::OP_SEL_1;
7642 if (MI.getOperand(i: OpIdx).getImm())
7643 Mods ^= SISrcMods::NEG;
7644 MIB.addImm(Val: (int64_t)Mods);
7645}
7646
7647void AMDGPUInstructionSelector::renderVOP3PModsNegs(MachineInstrBuilder &MIB,
7648 const MachineInstr &MI,
7649 int OpIdx) const {
7650 unsigned Mods = SISrcMods::OP_SEL_1;
7651 if (MI.getOperand(i: OpIdx).getImm())
7652 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
7653 MIB.addImm(Val: (int64_t)Mods);
7654}
7655
7656void AMDGPUInstructionSelector::renderVOP3PModsNegAbs(MachineInstrBuilder &MIB,
7657 const MachineInstr &MI,
7658 int OpIdx) const {
7659 unsigned Val = MI.getOperand(i: OpIdx).getImm();
7660 unsigned Mods = SISrcMods::OP_SEL_1; // default: none
7661 if (Val == 1) // neg
7662 Mods ^= SISrcMods::NEG;
7663 if (Val == 2) // abs
7664 Mods ^= SISrcMods::ABS;
7665 if (Val == 3) // neg and abs
7666 Mods ^= (SISrcMods::NEG | SISrcMods::ABS);
7667 MIB.addImm(Val: (int64_t)Mods);
7668}
7669
7670void AMDGPUInstructionSelector::renderPrefetchLoc(MachineInstrBuilder &MIB,
7671 const MachineInstr &MI,
7672 int OpIdx) const {
7673 uint32_t V = MI.getOperand(i: 2).getImm();
7674 V = (AMDGPU::CPol::SCOPE_MASK - (V & AMDGPU::CPol::SCOPE_MASK))
7675 << AMDGPU::CPol::SCOPE_SHIFT;
7676 if (!Subtarget->hasSafeCUPrefetch())
7677 V = std::max(a: V, b: (uint32_t)AMDGPU::CPol::SCOPE_SE); // CU scope is unsafe
7678 MIB.addImm(Val: V);
7679}
7680
7681/// Convert from 2-bit value to enum values used for op_sel* source modifiers.
7682void AMDGPUInstructionSelector::renderScaledMAIIntrinsicOperand(
7683 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7684 unsigned Val = MI.getOperand(i: OpIdx).getImm();
7685 unsigned New = 0;
7686 if (Val & 0x1)
7687 New |= SISrcMods::OP_SEL_0;
7688 if (Val & 0x2)
7689 New |= SISrcMods::OP_SEL_1;
7690 MIB.addImm(Val: New);
7691}
7692
7693bool AMDGPUInstructionSelector::isInlineImmediate(const APInt &Imm) const {
7694 return TII.isInlineConstant(Imm);
7695}
7696
7697bool AMDGPUInstructionSelector::isInlineImmediate(const APFloat &Imm) const {
7698 return TII.isInlineConstant(Imm);
7699}
7700