1//===- AMDGPUInstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUInstructionSelector.h"
15#include "AMDGPU.h"
16#include "AMDGPUGlobalISelUtils.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPURegisterBankInfo.h"
19#include "SIMachineFunctionInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
22#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/IntrinsicsAMDGPU.h"
29#include <optional>
30
31#define DEBUG_TYPE "amdgpu-isel"
32
33using namespace llvm;
34using namespace MIPatternMatch;
35
36#define GET_GLOBALISEL_IMPL
37#define AMDGPUSubtarget GCNSubtarget
38#include "AMDGPUGenGlobalISel.inc"
39#undef GET_GLOBALISEL_IMPL
40#undef AMDGPUSubtarget
41
42AMDGPUInstructionSelector::AMDGPUInstructionSelector(
43 const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI)
44 : TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI), STI(STI),
45#define GET_GLOBALISEL_PREDICATES_INIT
46#include "AMDGPUGenGlobalISel.inc"
47#undef GET_GLOBALISEL_PREDICATES_INIT
48#define GET_GLOBALISEL_TEMPORARIES_INIT
49#include "AMDGPUGenGlobalISel.inc"
50#undef GET_GLOBALISEL_TEMPORARIES_INIT
51{
52}
53
54const char *AMDGPUInstructionSelector::getName() { return DEBUG_TYPE; }
55
56void AMDGPUInstructionSelector::setupMF(MachineFunction &MF,
57 GISelValueTracking *VT,
58 CodeGenCoverage *CoverageInfo,
59 ProfileSummaryInfo *PSI,
60 BlockFrequencyInfo *BFI) {
61 MRI = &MF.getRegInfo();
62 Subtarget = &MF.getSubtarget<GCNSubtarget>();
63 Subtarget->checkSubtargetFeatures(F: MF.getFunction());
64 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
65}
66
67// Return the wave level SGPR base address if this is a wave address.
68static Register getWaveAddress(const MachineInstr *Def) {
69 return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS
70 ? Def->getOperand(i: 1).getReg()
71 : Register();
72}
73
74bool AMDGPUInstructionSelector::isVCC(Register Reg,
75 const MachineRegisterInfo &MRI) const {
76 // The verifier is oblivious to s1 being a valid value for wavesize registers.
77 if (Reg.isPhysical())
78 return false;
79
80 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
81 const TargetRegisterClass *RC =
82 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
83 if (RC) {
84 const LLT Ty = MRI.getType(Reg);
85 if (!Ty.isValid() || Ty.getSizeInBits() != 1)
86 return false;
87 // G_TRUNC s1 result is never vcc.
88 return !mi_match(R: Reg, MRI, P: m_GTrunc(Src: m_Reg())) &&
89 RC->hasSuperClassEq(RC: TRI.getBoolRC());
90 }
91
92 const RegisterBank *RB = cast<const RegisterBank *>(Val: RegClassOrBank);
93 return RB->getID() == AMDGPU::VCCRegBankID;
94}
95
96bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(MachineInstr &MI,
97 unsigned NewOpc) const {
98 MI.setDesc(TII.get(Opcode: NewOpc));
99 MI.removeOperand(OpNo: 1); // Remove intrinsic ID.
100 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
101
102 Register DstReg = MI.getOperand(i: 0).getReg();
103 Register SrcReg = MI.getOperand(i: 1).getReg();
104
105 // TODO: This should be legalized to s32 if needed
106 if (MRI->getType(Reg: DstReg) == LLT::scalar(SizeInBits: 1))
107 return false;
108
109 const TargetRegisterClass *DstRC =
110 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
111 const TargetRegisterClass *SrcRC =
112 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
113 if (!DstRC || DstRC != SrcRC)
114 return false;
115
116 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
117 !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
118 return false;
119 const MCInstrDesc &MCID = MI.getDesc();
120 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
121 MI.getOperand(i: 0).setIsEarlyClobber(true);
122 }
123 return true;
124}
125
126bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
127 const DebugLoc &DL = I.getDebugLoc();
128 MachineBasicBlock *BB = I.getParent();
129 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
130
131 Register DstReg = I.getOperand(i: 0).getReg();
132 Register SrcReg = I.getOperand(i: 1).getReg();
133
134 if (isVCC(Reg: DstReg, MRI: *MRI)) {
135 if (SrcReg == AMDGPU::SCC) {
136 const TargetRegisterClass *RC =
137 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
138 if (!RC)
139 return true;
140 return RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI);
141 }
142
143 if (!isVCC(Reg: SrcReg, MRI: *MRI)) {
144 // TODO: Should probably leave the copy and let copyPhysReg expand it.
145 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI))
146 return false;
147
148 const TargetRegisterClass *SrcRC =
149 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
150
151 std::optional<ValueAndVReg> ConstVal =
152 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI, LookThroughInstrs: true);
153 if (ConstVal) {
154 unsigned MovOpc =
155 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
156 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: DstReg)
157 .addImm(Val: ConstVal->Value.getBoolValue() ? -1 : 0);
158 } else {
159 Register MaskedReg = MRI->createVirtualRegister(RegClass: SrcRC);
160
161 // We can't trust the high bits at this point, so clear them.
162
163 // TODO: Skip masking high bits if def is known boolean.
164
165 if (AMDGPU::getRegBitWidth(RCID: SrcRC->getID()) == 16) {
166 assert(Subtarget->useRealTrue16Insts());
167 const int64_t NoMods = 0;
168 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B16_t16_e64), DestReg: MaskedReg)
169 .addImm(Val: NoMods)
170 .addImm(Val: 1)
171 .addImm(Val: NoMods)
172 .addReg(RegNo: SrcReg)
173 .addImm(Val: NoMods);
174 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U16_t16_e64), DestReg: DstReg)
175 .addImm(Val: NoMods)
176 .addImm(Val: 0)
177 .addImm(Val: NoMods)
178 .addReg(RegNo: MaskedReg)
179 .addImm(Val: NoMods);
180 } else {
181 bool IsSGPR = TRI.isSGPRClass(RC: SrcRC);
182 unsigned AndOpc = IsSGPR ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32;
183 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: MaskedReg)
184 .addImm(Val: 1)
185 .addReg(RegNo: SrcReg);
186 if (IsSGPR)
187 And.setOperandDead(3); // Dead scc
188
189 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U32_e64), DestReg: DstReg)
190 .addImm(Val: 0)
191 .addReg(RegNo: MaskedReg);
192 }
193 }
194
195 if (!MRI->getRegClassOrNull(Reg: SrcReg))
196 MRI->setRegClass(Reg: SrcReg, RC: SrcRC);
197 I.eraseFromParent();
198 return true;
199 }
200
201 const TargetRegisterClass *RC =
202 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
203 if (RC && !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
204 return false;
205
206 return true;
207 }
208
209 for (const MachineOperand &MO : I.operands()) {
210 if (MO.getReg().isPhysical())
211 continue;
212
213 const TargetRegisterClass *RC =
214 TRI.getConstrainedRegClassForReg(Reg: MO.getReg(), MRI: *MRI);
215 if (!RC)
216 continue;
217 RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI);
218 }
219 return true;
220}
221
222bool AMDGPUInstructionSelector::selectCOPY_SCC_VCC(MachineInstr &I) const {
223 const DebugLoc &DL = I.getDebugLoc();
224 MachineBasicBlock *BB = I.getParent();
225 Register VCCReg = I.getOperand(i: 1).getReg();
226 MachineInstr *Cmp;
227
228 // Set SCC as a side effect with S_CMP or S_OR.
229 if (STI.hasScalarCompareEq64()) {
230 unsigned CmpOpc =
231 STI.isWave64() ? AMDGPU::S_CMP_LG_U64 : AMDGPU::S_CMP_LG_U32;
232 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: CmpOpc)).addReg(RegNo: VCCReg).addImm(Val: 0);
233 } else {
234 Register DeadDst = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
235 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B64), DestReg: DeadDst)
236 .addReg(RegNo: VCCReg)
237 .addReg(RegNo: VCCReg);
238 }
239
240 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
241
242 Register DstReg = I.getOperand(i: 0).getReg();
243 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: AMDGPU::SCC);
244
245 I.eraseFromParent();
246 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
247}
248
249bool AMDGPUInstructionSelector::selectCOPY_VCC_SCC(MachineInstr &I) const {
250 const DebugLoc &DL = I.getDebugLoc();
251 MachineBasicBlock *BB = I.getParent();
252
253 Register DstReg = I.getOperand(i: 0).getReg();
254 Register SrcReg = I.getOperand(i: 1).getReg();
255 std::optional<ValueAndVReg> Arg =
256 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 1).getReg(), MRI: *MRI);
257
258 if (Arg) {
259 const int64_t Value = Arg->Value.getZExtValue();
260 if (Value == 0) {
261 unsigned Opcode = STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
262 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DstReg).addImm(Val: 0);
263 } else {
264 assert(Value == 1);
265 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: TRI.getExec());
266 }
267 I.eraseFromParent();
268 return RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI);
269 }
270
271 // RegBankLegalize ensures that SrcReg is bool in reg (high bits are 0).
272 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC).addReg(RegNo: SrcReg);
273
274 unsigned SelectOpcode =
275 STI.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
276 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
277 .addReg(RegNo: TRI.getExec())
278 .addImm(Val: 0);
279
280 I.eraseFromParent();
281 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
282 return true;
283}
284
285bool AMDGPUInstructionSelector::selectReadAnyLane(MachineInstr &I) const {
286 Register DstReg = I.getOperand(i: 0).getReg();
287 Register SrcReg = I.getOperand(i: 1).getReg();
288
289 const DebugLoc &DL = I.getDebugLoc();
290 MachineBasicBlock *BB = I.getParent();
291
292 auto RFL = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
293 .addReg(RegNo: SrcReg);
294
295 I.eraseFromParent();
296 constrainSelectedInstRegOperands(I&: *RFL, TII, TRI, RBI);
297 return true;
298}
299
300bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
301 const Register DefReg = I.getOperand(i: 0).getReg();
302 const LLT DefTy = MRI->getType(Reg: DefReg);
303
304 // S1 G_PHIs should not be selected in instruction-select, instead:
305 // - divergent S1 G_PHI should go through lane mask merging algorithm
306 // and be fully inst-selected in AMDGPUGlobalISelDivergenceLowering
307 // - uniform S1 G_PHI should be lowered into S32 G_PHI in AMDGPURegBankSelect
308 if (DefTy == LLT::scalar(SizeInBits: 1))
309 return false;
310
311 // TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy)
312
313 const RegClassOrRegBank &RegClassOrBank =
314 MRI->getRegClassOrRegBank(Reg: DefReg);
315
316 const TargetRegisterClass *DefRC =
317 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
318 if (!DefRC) {
319 if (!DefTy.isValid()) {
320 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
321 return false;
322 }
323
324 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
325 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, Bank: RB);
326 if (!DefRC) {
327 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
328 return false;
329 }
330 }
331
332 // If inputs have register bank, assign corresponding reg class.
333 // Note: registers don't need to have the same reg bank.
334 for (unsigned i = 1; i != I.getNumOperands(); i += 2) {
335 const Register SrcReg = I.getOperand(i).getReg();
336
337 const RegisterBank *RB = MRI->getRegBankOrNull(Reg: SrcReg);
338 if (RB) {
339 const LLT SrcTy = MRI->getType(Reg: SrcReg);
340 const TargetRegisterClass *SrcRC =
341 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *RB);
342 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
343 return false;
344 }
345 }
346
347 I.setDesc(TII.get(Opcode: TargetOpcode::PHI));
348 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
349}
350
351MachineOperand
352AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
353 const TargetRegisterClass &SubRC,
354 unsigned SubIdx) const {
355
356 MachineInstr *MI = MO.getParent();
357 MachineBasicBlock *BB = MO.getParent()->getParent();
358 Register DstReg = MRI->createVirtualRegister(RegClass: &SubRC);
359
360 if (MO.isReg()) {
361 unsigned ComposedSubIdx = TRI.composeSubRegIndices(a: MO.getSubReg(), b: SubIdx);
362 Register Reg = MO.getReg();
363 BuildMI(BB&: *BB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
364 .addReg(RegNo: Reg, Flags: {}, SubReg: ComposedSubIdx);
365
366 return MachineOperand::CreateReg(Reg: DstReg, isDef: MO.isDef(), isImp: MO.isImplicit(),
367 isKill: MO.isKill(), isDead: MO.isDead(), isUndef: MO.isUndef(),
368 isEarlyClobber: MO.isEarlyClobber(), SubReg: 0, isDebug: MO.isDebug(),
369 isInternalRead: MO.isInternalRead());
370 }
371
372 assert(MO.isImm());
373
374 APInt Imm(64, MO.getImm());
375
376 switch (SubIdx) {
377 default:
378 llvm_unreachable("do not know to split immediate with this sub index.");
379 case AMDGPU::sub0:
380 return MachineOperand::CreateImm(Val: Imm.getLoBits(numBits: 32).getSExtValue());
381 case AMDGPU::sub1:
382 return MachineOperand::CreateImm(Val: Imm.getHiBits(numBits: 32).getSExtValue());
383 }
384}
385
386static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
387 switch (Opc) {
388 case AMDGPU::G_AND:
389 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
390 case AMDGPU::G_OR:
391 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32;
392 case AMDGPU::G_XOR:
393 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32;
394 default:
395 llvm_unreachable("not a bit op");
396 }
397}
398
399bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
400 Register DstReg = I.getOperand(i: 0).getReg();
401 unsigned Size = RBI.getSizeInBits(Reg: DstReg, MRI: *MRI, TRI);
402
403 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
404 if (DstRB->getID() != AMDGPU::SGPRRegBankID &&
405 DstRB->getID() != AMDGPU::VCCRegBankID)
406 return false;
407
408 bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID &&
409 STI.isWave64());
410 I.setDesc(TII.get(Opcode: getLogicalBitOpcode(Opc: I.getOpcode(), Is64)));
411
412 // Dead implicit-def of scc
413 I.addOperand(Op: MachineOperand::CreateReg(Reg: AMDGPU::SCC, isDef: true, // isDef
414 isImp: true, // isImp
415 isKill: false, // isKill
416 isDead: true)); // isDead
417 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
418 return true;
419}
420
421bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
422 MachineBasicBlock *BB = I.getParent();
423 MachineFunction *MF = BB->getParent();
424 Register DstReg = I.getOperand(i: 0).getReg();
425 const DebugLoc &DL = I.getDebugLoc();
426 LLT Ty = MRI->getType(Reg: DstReg);
427 if (Ty.isVector())
428 return false;
429
430 unsigned Size = Ty.getSizeInBits();
431 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
432 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID;
433 const bool Sub = I.getOpcode() == TargetOpcode::G_SUB;
434
435 if (Size == 32) {
436 if (IsSALU) {
437 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
438 MachineInstr *Add =
439 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
440 .add(MO: I.getOperand(i: 1))
441 .add(MO: I.getOperand(i: 2))
442 .setOperandDead(3); // Dead scc
443 I.eraseFromParent();
444 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
445 return true;
446 }
447
448 if (STI.hasAddNoCarryInsts()) {
449 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64;
450 I.setDesc(TII.get(Opcode: Opc));
451 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
452 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
453 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
454 return true;
455 }
456
457 const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64;
458
459 Register UnusedCarry = MRI->createVirtualRegister(RegClass: TRI.getWaveMaskRegClass());
460 MachineInstr *Add
461 = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
462 .addDef(RegNo: UnusedCarry, Flags: RegState::Dead)
463 .add(MO: I.getOperand(i: 1))
464 .add(MO: I.getOperand(i: 2))
465 .addImm(Val: 0);
466 I.eraseFromParent();
467 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
468 return true;
469 }
470
471 assert(!Sub && "illegal sub should not reach here");
472
473 const TargetRegisterClass &RC
474 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass;
475 const TargetRegisterClass &HalfRC
476 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass;
477
478 MachineOperand Lo1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
479 MachineOperand Lo2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
480 MachineOperand Hi1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
481 MachineOperand Hi2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
482
483 Register DstLo = MRI->createVirtualRegister(RegClass: &HalfRC);
484 Register DstHi = MRI->createVirtualRegister(RegClass: &HalfRC);
485
486 if (IsSALU) {
487 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_U32), DestReg: DstLo)
488 .add(MO: Lo1)
489 .add(MO: Lo2);
490 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADDC_U32), DestReg: DstHi)
491 .add(MO: Hi1)
492 .add(MO: Hi2)
493 .setOperandDead(3); // Dead scc
494 } else {
495 const TargetRegisterClass *CarryRC = TRI.getWaveMaskRegClass();
496 Register CarryReg = MRI->createVirtualRegister(RegClass: CarryRC);
497 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: DstLo)
498 .addDef(RegNo: CarryReg)
499 .add(MO: Lo1)
500 .add(MO: Lo2)
501 .addImm(Val: 0);
502 MachineInstr *Addc = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: DstHi)
503 .addDef(RegNo: MRI->createVirtualRegister(RegClass: CarryRC), Flags: RegState::Dead)
504 .add(MO: Hi1)
505 .add(MO: Hi2)
506 .addReg(RegNo: CarryReg, Flags: RegState::Kill)
507 .addImm(Val: 0);
508
509 constrainSelectedInstRegOperands(I&: *Addc, TII, TRI, RBI);
510 }
511
512 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
513 .addReg(RegNo: DstLo)
514 .addImm(Val: AMDGPU::sub0)
515 .addReg(RegNo: DstHi)
516 .addImm(Val: AMDGPU::sub1);
517
518
519 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
520 return false;
521
522 I.eraseFromParent();
523 return true;
524}
525
526bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE(
527 MachineInstr &I) const {
528 MachineBasicBlock *BB = I.getParent();
529 MachineFunction *MF = BB->getParent();
530 const DebugLoc &DL = I.getDebugLoc();
531 Register Dst0Reg = I.getOperand(i: 0).getReg();
532 Register Dst1Reg = I.getOperand(i: 1).getReg();
533 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO ||
534 I.getOpcode() == AMDGPU::G_UADDE;
535 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE ||
536 I.getOpcode() == AMDGPU::G_USUBE;
537
538 if (isVCC(Reg: Dst1Reg, MRI: *MRI)) {
539 unsigned NoCarryOpc =
540 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
541 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
542 I.setDesc(TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc));
543 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
544 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
545 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
546 return true;
547 }
548
549 Register Src0Reg = I.getOperand(i: 2).getReg();
550 Register Src1Reg = I.getOperand(i: 3).getReg();
551
552 if (HasCarryIn) {
553 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
554 .addReg(RegNo: I.getOperand(i: 4).getReg());
555 }
556
557 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
558 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
559
560 auto CarryInst = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc), DestReg: Dst0Reg)
561 .add(MO: I.getOperand(i: 2))
562 .add(MO: I.getOperand(i: 3));
563
564 if (MRI->use_nodbg_empty(RegNo: Dst1Reg)) {
565 CarryInst.setOperandDead(3); // Dead scc
566 } else {
567 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst1Reg)
568 .addReg(RegNo: AMDGPU::SCC);
569 if (!MRI->getRegClassOrNull(Reg: Dst1Reg))
570 MRI->setRegClass(Reg: Dst1Reg, RC: &AMDGPU::SReg_32RegClass);
571 }
572
573 if (!RBI.constrainGenericRegister(Reg: Dst0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
574 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
575 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
576 return false;
577
578 if (HasCarryIn &&
579 !RBI.constrainGenericRegister(Reg: I.getOperand(i: 4).getReg(),
580 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
581 return false;
582
583 I.eraseFromParent();
584 return true;
585}
586
587bool AMDGPUInstructionSelector::selectG_AMDGPU_MAD_64_32(
588 MachineInstr &I) const {
589 MachineBasicBlock *BB = I.getParent();
590 MachineFunction *MF = BB->getParent();
591 const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
592 bool UseNoCarry = Subtarget->hasMadNC64_32Insts() &&
593 MRI->use_nodbg_empty(RegNo: I.getOperand(i: 1).getReg());
594
595 unsigned Opc;
596 if (Subtarget->hasMADIntraFwdBug())
597 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64
598 : AMDGPU::V_MAD_I64_I32_gfx11_e64;
599 else if (UseNoCarry)
600 Opc = IsUnsigned ? AMDGPU::V_MAD_NC_U64_U32_e64
601 : AMDGPU::V_MAD_NC_I64_I32_e64;
602 else
603 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64;
604
605 if (UseNoCarry)
606 I.removeOperand(OpNo: 1);
607
608 I.setDesc(TII.get(Opcode: Opc));
609 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
610 I.addImplicitDefUseOperands(MF&: *MF);
611 I.getOperand(i: 0).setIsEarlyClobber(true);
612 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
613 return true;
614}
615
616// TODO: We should probably legalize these to only using 32-bit results.
617bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
618 MachineBasicBlock *BB = I.getParent();
619 Register DstReg = I.getOperand(i: 0).getReg();
620 Register SrcReg = I.getOperand(i: 1).getReg();
621 LLT DstTy = MRI->getType(Reg: DstReg);
622 LLT SrcTy = MRI->getType(Reg: SrcReg);
623 const unsigned SrcSize = SrcTy.getSizeInBits();
624 unsigned DstSize = DstTy.getSizeInBits();
625
626 // TODO: Should handle any multiple of 32 offset.
627 unsigned Offset = I.getOperand(i: 2).getImm();
628 if (Offset % 32 != 0 || DstSize > 128)
629 return false;
630
631 // 16-bit operations really use 32-bit registers.
632 // FIXME: Probably should not allow 16-bit G_EXTRACT results.
633 if (DstSize == 16)
634 DstSize = 32;
635
636 const TargetRegisterClass *DstRC =
637 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
638 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
639 return false;
640
641 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
642 const TargetRegisterClass *SrcRC =
643 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
644 if (!SrcRC)
645 return false;
646 unsigned SubReg = SIRegisterInfo::getSubRegFromChannel(Channel: Offset / 32,
647 NumRegs: DstSize / 32);
648 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubReg);
649 if (!SrcRC)
650 return false;
651
652 SrcReg = constrainOperandRegClass(MF: *MF, TRI, MRI&: *MRI, TII, RBI, InsertPt&: I,
653 RegClass: *SrcRC, RegMO&: I.getOperand(i: 1));
654 const DebugLoc &DL = I.getDebugLoc();
655 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
656 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
657
658 I.eraseFromParent();
659 return true;
660}
661
662bool AMDGPUInstructionSelector::selectS16MergeToS32(MachineInstr &MI) const {
663 Register Dst = MI.getOperand(i: 0).getReg();
664 Register Src0 = MI.getOperand(i: 1).getReg();
665 Register Src1 = MI.getOperand(i: 2).getReg();
666
667 LLT Src0Ty = MRI->getType(Reg: Src0);
668 LLT Src1Ty = MRI->getType(Reg: Src1);
669
670 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
671 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0, MRI: *MRI, TRI);
672 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1, MRI: *MRI, TRI);
673 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
674
675 Register ShiftSrc0;
676 Register ShiftSrc1;
677
678 const DebugLoc &DL = MI.getDebugLoc();
679 MachineBasicBlock *BB = MI.getParent();
680
681 // VGPR case
682 if (IsVector) {
683 // If source are both VGPR16, use REG_SEQUENCE with lo16/hi16 subregisters
684 if (Src0Bank->getID() == AMDGPU::VGPRRegBankID &&
685 Src1Bank->getID() == AMDGPU::VGPRRegBankID &&
686 Src0Ty == LLT::scalar(SizeInBits: 16) && Src1Ty == LLT::scalar(SizeInBits: 16)) {
687 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
688 .addReg(RegNo: Src0)
689 .addImm(Val: AMDGPU::lo16)
690 .addReg(RegNo: Src1)
691 .addImm(Val: AMDGPU::hi16);
692
693 if (!RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
694 return false;
695
696 MI.eraseFromParent();
697 return true;
698 }
699
700 // Otherwise, use V_LSHL_OR_B32_e64
701 Register TmpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
702 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: TmpReg)
703 .addImm(Val: 0xFFFF)
704 .addReg(RegNo: Src0);
705 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
706
707 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: Dst)
708 .addReg(RegNo: Src1)
709 .addImm(Val: 16)
710 .addReg(RegNo: TmpReg);
711 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
712
713 MI.eraseFromParent();
714 return true;
715 }
716
717 // SGPR case -> S_PACK_*_B32_B16
718 // With multiple uses of the shift, this will duplicate the shift and
719 // increase register pressure.
720 //
721 // (merge (lshr_oneuse $src0, 16), (lshr_oneuse $src1, 16)
722 // => (S_PACK_HH_B32_B16 $src0, $src1)
723 // (merge (lshr_oneuse SReg_32:$src0, 16), $src1)
724 // => (S_PACK_HL_B32_B16 $src0, $src1)
725 // (merge $src0, (lshr_oneuse SReg_32:$src1, 16))
726 // => (S_PACK_LH_B32_B16 $src0, $src1)
727 // (merge $src0, $src1)
728 // => (S_PACK_LL_B32_B16 $src0, $src1)
729
730 bool Shift0 = mi_match(
731 R: Src0, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc0), R: m_SpecificICst(RequestedValue: 16))));
732
733 bool Shift1 = mi_match(
734 R: Src1, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc1), R: m_SpecificICst(RequestedValue: 16))));
735
736 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16;
737 if (Shift0 && Shift1) {
738 Opc = AMDGPU::S_PACK_HH_B32_B16;
739 MI.getOperand(i: 1).setReg(ShiftSrc0);
740 MI.getOperand(i: 2).setReg(ShiftSrc1);
741 } else if (Shift1) {
742 Opc = AMDGPU::S_PACK_LH_B32_B16;
743 MI.getOperand(i: 2).setReg(ShiftSrc1);
744 } else if (Shift0) {
745 auto ConstSrc1 =
746 getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
747 if (ConstSrc1 && ConstSrc1->Value == 0) {
748 // build_vector_trunc (lshr $src0, 16), 0 -> s_lshr_b32 $src0, 16
749 auto MIB = BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: Dst)
750 .addReg(RegNo: ShiftSrc0)
751 .addImm(Val: 16)
752 .setOperandDead(3); // Dead scc
753
754 MI.eraseFromParent();
755 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
756 return true;
757 }
758 if (STI.hasSPackHL()) {
759 Opc = AMDGPU::S_PACK_HL_B32_B16;
760 MI.getOperand(i: 1).setReg(ShiftSrc0);
761 }
762 }
763
764 MI.setDesc(TII.get(Opcode: Opc));
765 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
766 return true;
767}
768
769// Pack each pair of s16 into an s32 with S_PACK_LL_B32_B16, then combine the
770// s32 pieces into the destination with a REG_SEQUENCE.
771bool AMDGPUInstructionSelector::selectS16MergeToWide(MachineInstr &MI) const {
772 MachineBasicBlock *BB = MI.getParent();
773 const DebugLoc &DL = MI.getDebugLoc();
774 Register DstReg = MI.getOperand(i: 0).getReg();
775 const unsigned DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
776 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
777 const unsigned NumSrc = MI.getNumOperands() - 1;
778
779 // Pack each pair of s16 sources into an s32.
780 SmallVector<Register, 8> S32Regs;
781 for (unsigned I = 0; I != NumSrc; I += 2) {
782 Register S32 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
783 auto Pack = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_PACK_LL_B32_B16), DestReg: S32)
784 .addReg(RegNo: MI.getOperand(i: I + 1).getReg())
785 .addReg(RegNo: MI.getOperand(i: I + 2).getReg());
786 constrainSelectedInstRegOperands(I&: *Pack, TII, TRI, RBI);
787 S32Regs.push_back(Elt: S32);
788 }
789
790 // Combine the s32 pieces into the destination with a REG_SEQUENCE.
791 const TargetRegisterClass *DstRC =
792 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
793 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
794 return false;
795 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, /*EltSize=*/4);
796 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
797 for (unsigned I = 0, E = S32Regs.size(); I != E; ++I)
798 MIB.addReg(RegNo: S32Regs[I]).addImm(Val: SubRegs[I]);
799
800 MI.eraseFromParent();
801 return true;
802}
803
804bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
805 MachineBasicBlock *BB = MI.getParent();
806 Register DstReg = MI.getOperand(i: 0).getReg();
807 LLT DstTy = MRI->getType(Reg: DstReg);
808 LLT SrcTy = MRI->getType(Reg: MI.getOperand(i: 1).getReg());
809
810 const unsigned SrcSize = SrcTy.getSizeInBits();
811 if (SrcSize < 32) {
812 // Handle s32 <- G_MERGE_VALUES s16, s16
813 if (SrcSize == 16 && DstTy.getSizeInBits() == 32 &&
814 MI.getNumOperands() == 3) {
815 return selectS16MergeToS32(MI);
816 }
817 // With true16 a scalar s16 is a register type, so a scalar wider than 32
818 // bits can be built from s16 pieces.
819 bool IsWideS16Merge = SrcSize == 16 && DstTy.getSizeInBits() > 32 &&
820 DstTy.getSizeInBits() % 32 == 0;
821
822 // SGPRs have no 16-bit subregisters, so pack pairs of s16 with S_PACK.
823 if (IsWideS16Merge &&
824 RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
825 return selectS16MergeToWide(MI);
826
827 // A VGPR wide s16 merge falls through to the generic path below.
828 if (!IsWideS16Merge)
829 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
830 }
831
832 const DebugLoc &DL = MI.getDebugLoc();
833 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
834 const unsigned DstSize = DstTy.getSizeInBits();
835 const TargetRegisterClass *DstRC =
836 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
837 if (!DstRC)
838 return false;
839
840 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, EltSize: SrcSize / 8);
841 MachineInstrBuilder MIB =
842 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
843 for (int I = 0, E = MI.getNumOperands() - 1; I != E; ++I) {
844 MachineOperand &Src = MI.getOperand(i: I + 1);
845 Register SrcReg = Src.getReg();
846 MIB.addReg(RegNo: SrcReg, Flags: getUndefRegState(B: Src.isUndef()));
847 MIB.addImm(Val: SubRegs[I]);
848
849 const TargetRegisterClass *SrcRC =
850 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
851 if (SrcRC && !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
852 return false;
853 }
854
855 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
856 return false;
857
858 MI.eraseFromParent();
859 return true;
860}
861
862bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
863 MachineBasicBlock *BB = MI.getParent();
864 const int NumDst = MI.getNumOperands() - 1;
865
866 MachineOperand &Src = MI.getOperand(i: NumDst);
867
868 Register SrcReg = Src.getReg();
869 Register DstReg0 = MI.getOperand(i: 0).getReg();
870 LLT DstTy = MRI->getType(Reg: DstReg0);
871 LLT SrcTy = MRI->getType(Reg: SrcReg);
872
873 const unsigned DstSize = DstTy.getSizeInBits();
874 const unsigned SrcSize = SrcTy.getSizeInBits();
875 const DebugLoc &DL = MI.getDebugLoc();
876 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
877
878 const TargetRegisterClass *SrcRC =
879 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
880
881 // Note we could have mixed SGPR and VGPR destination banks for an SGPR
882 // source, and this relies on the fact that the same subregister indices are
883 // used for both.
884 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SrcRC, EltSize: DstSize / 8);
885 for (int I = 0, E = NumDst; I != E; ++I) {
886 Register DstReg = MI.getOperand(i: I).getReg();
887 // hi16:sreg_32 is not allowed so explicitly shift upper 16-bits.
888 if (SrcBank->getID() == AMDGPU::SGPRRegBankID &&
889 SubRegs[I] == AMDGPU::hi16) {
890 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: DstReg)
891 .addReg(RegNo: SrcReg)
892 .addImm(Val: 16);
893 } else {
894 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
895 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegs[I]);
896
897 // Make sure the subregister index is valid for the source register.
898 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[I]);
899 }
900
901 if (!SrcRC || !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
902 return false;
903
904 const TargetRegisterClass *DstRC =
905 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
906 if (DstRC && !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
907 return false;
908 }
909
910 MI.eraseFromParent();
911 return true;
912}
913
914bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR(MachineInstr &MI) const {
915 assert(MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC ||
916 MI.getOpcode() == AMDGPU::G_BUILD_VECTOR);
917
918 Register Src0 = MI.getOperand(i: 1).getReg();
919 Register Src1 = MI.getOperand(i: 2).getReg();
920 LLT SrcTy = MRI->getType(Reg: Src0);
921 const unsigned SrcSize = SrcTy.getSizeInBits();
922
923 // BUILD_VECTOR with >=32 bits source is handled by MERGE_VALUE.
924 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR && SrcSize >= 32) {
925 return selectG_MERGE_VALUES(MI);
926 }
927
928 // Selection logic below is for V2S16 only.
929 // For G_BUILD_VECTOR_TRUNC, additionally check that the operands are s32.
930 Register Dst = MI.getOperand(i: 0).getReg();
931 if (MRI->getType(Reg: Dst) != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
932 (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC &&
933 SrcTy != LLT::scalar(SizeInBits: 32)))
934 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
935
936 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
937 if (DstBank->getID() == AMDGPU::AGPRRegBankID)
938 return false;
939
940 assert(DstBank->getID() == AMDGPU::SGPRRegBankID ||
941 DstBank->getID() == AMDGPU::VGPRRegBankID);
942 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
943
944 const DebugLoc &DL = MI.getDebugLoc();
945 MachineBasicBlock *BB = MI.getParent();
946
947 // First, before trying TableGen patterns, check if both sources are
948 // constants. In those cases, we can trivially compute the final constant
949 // and emit a simple move.
950 auto ConstSrc1 = getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
951 if (ConstSrc1) {
952 auto ConstSrc0 =
953 getAnyConstantVRegValWithLookThrough(VReg: Src0, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
954 if (ConstSrc0) {
955 const int64_t K0 = ConstSrc0->Value.getSExtValue();
956 const int64_t K1 = ConstSrc1->Value.getSExtValue();
957 uint32_t Lo16 = static_cast<uint32_t>(K0) & 0xffff;
958 uint32_t Hi16 = static_cast<uint32_t>(K1) & 0xffff;
959 uint32_t Imm = Lo16 | (Hi16 << 16);
960
961 // VALU
962 if (IsVector) {
963 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: Dst).addImm(Val: Imm);
964 MI.eraseFromParent();
965 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI);
966 }
967
968 // SALU
969 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: Dst).addImm(Val: Imm);
970 MI.eraseFromParent();
971 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
972 }
973 }
974
975 // Now try TableGen patterns.
976 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
977 return true;
978
979 // TODO: This should probably be a combine somewhere
980 // (build_vector $src0, undef) -> copy $src0
981 MachineInstr *Src1Def = getDefIgnoringCopies(Reg: Src1, MRI: *MRI);
982 if (Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) {
983 MI.setDesc(TII.get(Opcode: AMDGPU::COPY));
984 MI.removeOperand(OpNo: 2);
985 const auto &RC =
986 IsVector ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
987 return RBI.constrainGenericRegister(Reg: Dst, RC, MRI&: *MRI) &&
988 RBI.constrainGenericRegister(Reg: Src0, RC, MRI&: *MRI);
989 }
990
991 return selectS16MergeToS32(MI);
992}
993
994bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const {
995 const MachineOperand &MO = I.getOperand(i: 0);
996
997 // FIXME: Interface for getConstrainedRegClassForReg needs work. The
998 // regbank check here is to know why getConstrainedRegClassForReg failed.
999 const TargetRegisterClass *RC =
1000 TRI.getConstrainedRegClassForReg(Reg: MO.getReg(), MRI: *MRI);
1001 if ((!RC && !MRI->getRegBankOrNull(Reg: MO.getReg())) ||
1002 (RC && RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI))) {
1003 I.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1004 return true;
1005 }
1006
1007 return false;
1008}
1009
1010bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
1011 MachineBasicBlock *BB = I.getParent();
1012
1013 Register DstReg = I.getOperand(i: 0).getReg();
1014 Register Src0Reg = I.getOperand(i: 1).getReg();
1015 Register Src1Reg = I.getOperand(i: 2).getReg();
1016 LLT Src1Ty = MRI->getType(Reg: Src1Reg);
1017
1018 unsigned DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
1019 unsigned InsSize = Src1Ty.getSizeInBits();
1020
1021 int64_t Offset = I.getOperand(i: 3).getImm();
1022
1023 // FIXME: These cases should have been illegal and unnecessary to check here.
1024 if (Offset % 32 != 0 || InsSize % 32 != 0)
1025 return false;
1026
1027 // Currently not handled by getSubRegFromChannel.
1028 if (InsSize > 128)
1029 return false;
1030
1031 unsigned SubReg = TRI.getSubRegFromChannel(Channel: Offset / 32, NumRegs: InsSize / 32);
1032 if (SubReg == AMDGPU::NoSubRegister)
1033 return false;
1034
1035 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1036 const TargetRegisterClass *DstRC =
1037 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
1038 if (!DstRC)
1039 return false;
1040
1041 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0Reg, MRI: *MRI, TRI);
1042 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1Reg, MRI: *MRI, TRI);
1043 const TargetRegisterClass *Src0RC =
1044 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *Src0Bank);
1045 const TargetRegisterClass *Src1RC =
1046 TRI.getRegClassForSizeOnBank(Size: InsSize, Bank: *Src1Bank);
1047
1048 // Deal with weird cases where the class only partially supports the subreg
1049 // index.
1050 Src0RC = TRI.getSubClassWithSubReg(Src0RC, SubReg);
1051 if (!Src0RC || !Src1RC)
1052 return false;
1053
1054 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
1055 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: *Src0RC, MRI&: *MRI) ||
1056 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: *Src1RC, MRI&: *MRI))
1057 return false;
1058
1059 const DebugLoc &DL = I.getDebugLoc();
1060 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1061 .addReg(RegNo: Src0Reg)
1062 .addReg(RegNo: Src1Reg)
1063 .addImm(Val: SubReg);
1064
1065 I.eraseFromParent();
1066 return true;
1067}
1068
1069bool AMDGPUInstructionSelector::selectG_SBFX_UBFX(MachineInstr &MI) const {
1070 Register DstReg = MI.getOperand(i: 0).getReg();
1071 Register SrcReg = MI.getOperand(i: 1).getReg();
1072 Register OffsetReg = MI.getOperand(i: 2).getReg();
1073 Register WidthReg = MI.getOperand(i: 3).getReg();
1074
1075 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID &&
1076 "scalar BFX instructions are expanded in regbankselect");
1077 assert(MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 &&
1078 "64-bit vector BFX instructions are expanded in regbankselect");
1079
1080 const DebugLoc &DL = MI.getDebugLoc();
1081 MachineBasicBlock *MBB = MI.getParent();
1082
1083 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SBFX;
1084 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
1085 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
1086 .addReg(RegNo: SrcReg)
1087 .addReg(RegNo: OffsetReg)
1088 .addReg(RegNo: WidthReg);
1089 MI.eraseFromParent();
1090 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1091 return true;
1092}
1093
1094bool AMDGPUInstructionSelector::selectInterpP1F16(MachineInstr &MI) const {
1095 if (STI.getLDSBankCount() != 16)
1096 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1097
1098 Register Dst = MI.getOperand(i: 0).getReg();
1099 Register Src0 = MI.getOperand(i: 2).getReg();
1100 Register M0Val = MI.getOperand(i: 6).getReg();
1101 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
1102 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI) ||
1103 !RBI.constrainGenericRegister(Reg: Src0, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
1104 return false;
1105
1106 // This requires 2 instructions. It is possible to write a pattern to support
1107 // this, but the generated isel emitter doesn't correctly deal with multiple
1108 // output instructions using the same physical register input. The copy to m0
1109 // is incorrectly placed before the second instruction.
1110 //
1111 // TODO: Match source modifiers.
1112
1113 Register InterpMov = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1114 const DebugLoc &DL = MI.getDebugLoc();
1115 MachineBasicBlock *MBB = MI.getParent();
1116
1117 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1118 .addReg(RegNo: M0Val);
1119 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_MOV_F32), DestReg: InterpMov)
1120 .addImm(Val: 2)
1121 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1122 .addImm(Val: MI.getOperand(i: 3).getImm()); // $attrchan
1123
1124 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_P1LV_F16), DestReg: Dst)
1125 .addImm(Val: 0) // $src0_modifiers
1126 .addReg(RegNo: Src0) // $src0
1127 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1128 .addImm(Val: MI.getOperand(i: 3).getImm()) // $attrchan
1129 .addImm(Val: 0) // $src2_modifiers
1130 .addReg(RegNo: InterpMov) // $src2 - 2 f16 values selected by high
1131 .addImm(Val: MI.getOperand(i: 5).getImm()) // $high
1132 .addImm(Val: 0) // $clamp
1133 .addImm(Val: 0); // $omod
1134
1135 MI.eraseFromParent();
1136 return true;
1137}
1138
1139// Writelane is special in that it can use SGPR and M0 (which would normally
1140// count as using the constant bus twice - but in this case it is allowed since
1141// the lane selector doesn't count as a use of the constant bus). However, it is
1142// still required to abide by the 1 SGPR rule. Fix this up if we might have
1143// multiple SGPRs.
1144bool AMDGPUInstructionSelector::selectWritelane(MachineInstr &MI) const {
1145 // With a constant bus limit of at least 2, there's no issue.
1146 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_WRITELANE_B32) > 1)
1147 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1148
1149 MachineBasicBlock *MBB = MI.getParent();
1150 const DebugLoc &DL = MI.getDebugLoc();
1151 Register VDst = MI.getOperand(i: 0).getReg();
1152 Register Val = MI.getOperand(i: 2).getReg();
1153 Register LaneSelect = MI.getOperand(i: 3).getReg();
1154 Register VDstIn = MI.getOperand(i: 4).getReg();
1155
1156 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_WRITELANE_B32), DestReg: VDst);
1157
1158 std::optional<ValueAndVReg> ConstSelect =
1159 getIConstantVRegValWithLookThrough(VReg: LaneSelect, MRI: *MRI);
1160 if (ConstSelect) {
1161 // The selector has to be an inline immediate, so we can use whatever for
1162 // the other operands.
1163 MIB.addReg(RegNo: Val);
1164 MIB.addImm(Val: ConstSelect->Value.getSExtValue() &
1165 maskTrailingOnes<uint64_t>(N: STI.getWavefrontSizeLog2()));
1166 } else {
1167 std::optional<ValueAndVReg> ConstVal =
1168 getIConstantVRegValWithLookThrough(VReg: Val, MRI: *MRI);
1169
1170 // If the value written is an inline immediate, we can get away without a
1171 // copy to m0.
1172 if (ConstVal && AMDGPU::isInlinableLiteral32(Literal: ConstVal->Value.getSExtValue(),
1173 HasInv2Pi: STI.hasInv2PiInlineImm())) {
1174 MIB.addImm(Val: ConstVal->Value.getSExtValue());
1175 MIB.addReg(RegNo: LaneSelect);
1176 } else {
1177 MIB.addReg(RegNo: Val);
1178
1179 // If the lane selector was originally in a VGPR and copied with
1180 // readfirstlane, there's a hazard to read the same SGPR from the
1181 // VALU. Constrain to a different SGPR to help avoid needing a nop later.
1182 RBI.constrainGenericRegister(Reg: LaneSelect, RC: AMDGPU::SReg_32_XM0RegClass, MRI&: *MRI);
1183
1184 BuildMI(BB&: *MBB, I&: *MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1185 .addReg(RegNo: LaneSelect);
1186 MIB.addReg(RegNo: AMDGPU::M0);
1187 }
1188 }
1189
1190 MIB.addReg(RegNo: VDstIn);
1191
1192 MI.eraseFromParent();
1193 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1194 return true;
1195}
1196
1197// We need to handle this here because tablegen doesn't support matching
1198// instructions with multiple outputs.
1199bool AMDGPUInstructionSelector::selectDivScale(MachineInstr &MI) const {
1200 Register Dst0 = MI.getOperand(i: 0).getReg();
1201 Register Dst1 = MI.getOperand(i: 1).getReg();
1202
1203 LLT Ty = MRI->getType(Reg: Dst0);
1204 unsigned Opc;
1205 if (Ty == LLT::scalar(SizeInBits: 32))
1206 Opc = AMDGPU::V_DIV_SCALE_F32_e64;
1207 else if (Ty == LLT::scalar(SizeInBits: 64))
1208 Opc = AMDGPU::V_DIV_SCALE_F64_e64;
1209 else
1210 return false;
1211
1212 // TODO: Match source modifiers.
1213
1214 const DebugLoc &DL = MI.getDebugLoc();
1215 MachineBasicBlock *MBB = MI.getParent();
1216
1217 Register Numer = MI.getOperand(i: 3).getReg();
1218 Register Denom = MI.getOperand(i: 4).getReg();
1219 unsigned ChooseDenom = MI.getOperand(i: 5).getImm();
1220
1221 Register Src0 = ChooseDenom != 0 ? Numer : Denom;
1222
1223 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
1224 .addDef(RegNo: Dst1)
1225 .addImm(Val: 0) // $src0_modifiers
1226 .addUse(RegNo: Src0) // $src0
1227 .addImm(Val: 0) // $src1_modifiers
1228 .addUse(RegNo: Denom) // $src1
1229 .addImm(Val: 0) // $src2_modifiers
1230 .addUse(RegNo: Numer) // $src2
1231 .addImm(Val: 0) // $clamp
1232 .addImm(Val: 0); // $omod
1233
1234 MI.eraseFromParent();
1235 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1236 return true;
1237}
1238
1239bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
1240 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
1241 switch (IntrinsicID) {
1242 case Intrinsic::amdgcn_if_break: {
1243 MachineBasicBlock *BB = I.getParent();
1244
1245 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1246 // SelectionDAG uses for wave32 vs wave64.
1247 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_IF_BREAK))
1248 .add(MO: I.getOperand(i: 0))
1249 .add(MO: I.getOperand(i: 2))
1250 .add(MO: I.getOperand(i: 3))
1251 .setOperandDead(3); // implicit-def $scc
1252
1253 Register DstReg = I.getOperand(i: 0).getReg();
1254 Register Src0Reg = I.getOperand(i: 2).getReg();
1255 Register Src1Reg = I.getOperand(i: 3).getReg();
1256
1257 I.eraseFromParent();
1258
1259 for (Register Reg : { DstReg, Src0Reg, Src1Reg })
1260 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1261
1262 return true;
1263 }
1264 case Intrinsic::amdgcn_interp_p1_f16:
1265 return selectInterpP1F16(MI&: I);
1266 case Intrinsic::amdgcn_wqm:
1267 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::WQM);
1268 case Intrinsic::amdgcn_softwqm:
1269 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::SOFT_WQM);
1270 case Intrinsic::amdgcn_strict_wwm:
1271 case Intrinsic::amdgcn_wwm:
1272 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WWM);
1273 case Intrinsic::amdgcn_strict_wqm:
1274 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WQM);
1275 case Intrinsic::amdgcn_writelane:
1276 return selectWritelane(MI&: I);
1277 case Intrinsic::amdgcn_div_scale:
1278 return selectDivScale(MI&: I);
1279 case Intrinsic::amdgcn_ballot:
1280 return selectBallot(I);
1281 case Intrinsic::amdgcn_reloc_constant:
1282 return selectRelocConstant(I);
1283 case Intrinsic::amdgcn_groupstaticsize:
1284 return selectGroupStaticSize(I);
1285 case Intrinsic::returnaddress:
1286 return selectReturnAddress(I);
1287 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
1288 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
1289 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
1290 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
1291 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
1292 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
1293 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
1294 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
1295 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
1296 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
1297 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
1298 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
1299 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
1300 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
1301 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
1302 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
1303 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
1304 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
1305 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
1306 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
1307 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
1308 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
1309 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
1310 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
1311 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
1312 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
1313 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
1314 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
1315 return selectSMFMACIntrin(I);
1316 case Intrinsic::amdgcn_permlane16_swap:
1317 case Intrinsic::amdgcn_permlane32_swap:
1318 return selectPermlaneSwapIntrin(I, IntrID: IntrinsicID);
1319 case Intrinsic::amdgcn_wave_shuffle:
1320 return selectWaveShuffleIntrin(I);
1321 default:
1322 return selectImpl(I, CoverageInfo&: *CoverageInfo);
1323 }
1324}
1325
1326static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size,
1327 const GCNSubtarget &ST) {
1328 if (Size != 16 && Size != 32 && Size != 64)
1329 return -1;
1330
1331 if (Size == 16 && !ST.has16BitInsts())
1332 return -1;
1333
1334 const auto Select = [&](unsigned S16Opc, unsigned TrueS16Opc,
1335 unsigned FakeS16Opc, unsigned S32Opc,
1336 unsigned S64Opc) {
1337 if (Size == 16)
1338 return ST.hasTrue16BitInsts()
1339 ? ST.useRealTrue16Insts() ? TrueS16Opc : FakeS16Opc
1340 : S16Opc;
1341 if (Size == 32)
1342 return S32Opc;
1343 return S64Opc;
1344 };
1345
1346 switch (P) {
1347 default:
1348 llvm_unreachable("Unknown condition code!");
1349 case CmpInst::ICMP_NE:
1350 return Select(AMDGPU::V_CMP_NE_U16_e64, AMDGPU::V_CMP_NE_U16_t16_e64,
1351 AMDGPU::V_CMP_NE_U16_fake16_e64, AMDGPU::V_CMP_NE_U32_e64,
1352 AMDGPU::V_CMP_NE_U64_e64);
1353 case CmpInst::ICMP_EQ:
1354 return Select(AMDGPU::V_CMP_EQ_U16_e64, AMDGPU::V_CMP_EQ_U16_t16_e64,
1355 AMDGPU::V_CMP_EQ_U16_fake16_e64, AMDGPU::V_CMP_EQ_U32_e64,
1356 AMDGPU::V_CMP_EQ_U64_e64);
1357 case CmpInst::ICMP_SGT:
1358 return Select(AMDGPU::V_CMP_GT_I16_e64, AMDGPU::V_CMP_GT_I16_t16_e64,
1359 AMDGPU::V_CMP_GT_I16_fake16_e64, AMDGPU::V_CMP_GT_I32_e64,
1360 AMDGPU::V_CMP_GT_I64_e64);
1361 case CmpInst::ICMP_SGE:
1362 return Select(AMDGPU::V_CMP_GE_I16_e64, AMDGPU::V_CMP_GE_I16_t16_e64,
1363 AMDGPU::V_CMP_GE_I16_fake16_e64, AMDGPU::V_CMP_GE_I32_e64,
1364 AMDGPU::V_CMP_GE_I64_e64);
1365 case CmpInst::ICMP_SLT:
1366 return Select(AMDGPU::V_CMP_LT_I16_e64, AMDGPU::V_CMP_LT_I16_t16_e64,
1367 AMDGPU::V_CMP_LT_I16_fake16_e64, AMDGPU::V_CMP_LT_I32_e64,
1368 AMDGPU::V_CMP_LT_I64_e64);
1369 case CmpInst::ICMP_SLE:
1370 return Select(AMDGPU::V_CMP_LE_I16_e64, AMDGPU::V_CMP_LE_I16_t16_e64,
1371 AMDGPU::V_CMP_LE_I16_fake16_e64, AMDGPU::V_CMP_LE_I32_e64,
1372 AMDGPU::V_CMP_LE_I64_e64);
1373 case CmpInst::ICMP_UGT:
1374 return Select(AMDGPU::V_CMP_GT_U16_e64, AMDGPU::V_CMP_GT_U16_t16_e64,
1375 AMDGPU::V_CMP_GT_U16_fake16_e64, AMDGPU::V_CMP_GT_U32_e64,
1376 AMDGPU::V_CMP_GT_U64_e64);
1377 case CmpInst::ICMP_UGE:
1378 return Select(AMDGPU::V_CMP_GE_U16_e64, AMDGPU::V_CMP_GE_U16_t16_e64,
1379 AMDGPU::V_CMP_GE_U16_fake16_e64, AMDGPU::V_CMP_GE_U32_e64,
1380 AMDGPU::V_CMP_GE_U64_e64);
1381 case CmpInst::ICMP_ULT:
1382 return Select(AMDGPU::V_CMP_LT_U16_e64, AMDGPU::V_CMP_LT_U16_t16_e64,
1383 AMDGPU::V_CMP_LT_U16_fake16_e64, AMDGPU::V_CMP_LT_U32_e64,
1384 AMDGPU::V_CMP_LT_U64_e64);
1385 case CmpInst::ICMP_ULE:
1386 return Select(AMDGPU::V_CMP_LE_U16_e64, AMDGPU::V_CMP_LE_U16_t16_e64,
1387 AMDGPU::V_CMP_LE_U16_fake16_e64, AMDGPU::V_CMP_LE_U32_e64,
1388 AMDGPU::V_CMP_LE_U64_e64);
1389
1390 case CmpInst::FCMP_OEQ:
1391 return Select(AMDGPU::V_CMP_EQ_F16_e64, AMDGPU::V_CMP_EQ_F16_t16_e64,
1392 AMDGPU::V_CMP_EQ_F16_fake16_e64, AMDGPU::V_CMP_EQ_F32_e64,
1393 AMDGPU::V_CMP_EQ_F64_e64);
1394 case CmpInst::FCMP_OGT:
1395 return Select(AMDGPU::V_CMP_GT_F16_e64, AMDGPU::V_CMP_GT_F16_t16_e64,
1396 AMDGPU::V_CMP_GT_F16_fake16_e64, AMDGPU::V_CMP_GT_F32_e64,
1397 AMDGPU::V_CMP_GT_F64_e64);
1398 case CmpInst::FCMP_OGE:
1399 return Select(AMDGPU::V_CMP_GE_F16_e64, AMDGPU::V_CMP_GE_F16_t16_e64,
1400 AMDGPU::V_CMP_GE_F16_fake16_e64, AMDGPU::V_CMP_GE_F32_e64,
1401 AMDGPU::V_CMP_GE_F64_e64);
1402 case CmpInst::FCMP_OLT:
1403 return Select(AMDGPU::V_CMP_LT_F16_e64, AMDGPU::V_CMP_LT_F16_t16_e64,
1404 AMDGPU::V_CMP_LT_F16_fake16_e64, AMDGPU::V_CMP_LT_F32_e64,
1405 AMDGPU::V_CMP_LT_F64_e64);
1406 case CmpInst::FCMP_OLE:
1407 return Select(AMDGPU::V_CMP_LE_F16_e64, AMDGPU::V_CMP_LE_F16_t16_e64,
1408 AMDGPU::V_CMP_LE_F16_fake16_e64, AMDGPU::V_CMP_LE_F32_e64,
1409 AMDGPU::V_CMP_LE_F64_e64);
1410 case CmpInst::FCMP_ONE:
1411 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1412 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1413 AMDGPU::V_CMP_NEQ_F64_e64);
1414 case CmpInst::FCMP_ORD:
1415 return Select(AMDGPU::V_CMP_O_F16_e64, AMDGPU::V_CMP_O_F16_t16_e64,
1416 AMDGPU::V_CMP_O_F16_fake16_e64, AMDGPU::V_CMP_O_F32_e64,
1417 AMDGPU::V_CMP_O_F64_e64);
1418 case CmpInst::FCMP_UNO:
1419 return Select(AMDGPU::V_CMP_U_F16_e64, AMDGPU::V_CMP_U_F16_t16_e64,
1420 AMDGPU::V_CMP_U_F16_fake16_e64, AMDGPU::V_CMP_U_F32_e64,
1421 AMDGPU::V_CMP_U_F64_e64);
1422 case CmpInst::FCMP_UEQ:
1423 return Select(AMDGPU::V_CMP_NLG_F16_e64, AMDGPU::V_CMP_NLG_F16_t16_e64,
1424 AMDGPU::V_CMP_NLG_F16_fake16_e64, AMDGPU::V_CMP_NLG_F32_e64,
1425 AMDGPU::V_CMP_NLG_F64_e64);
1426 case CmpInst::FCMP_UGT:
1427 return Select(AMDGPU::V_CMP_NLE_F16_e64, AMDGPU::V_CMP_NLE_F16_t16_e64,
1428 AMDGPU::V_CMP_NLE_F16_fake16_e64, AMDGPU::V_CMP_NLE_F32_e64,
1429 AMDGPU::V_CMP_NLE_F64_e64);
1430 case CmpInst::FCMP_UGE:
1431 return Select(AMDGPU::V_CMP_NLT_F16_e64, AMDGPU::V_CMP_NLT_F16_t16_e64,
1432 AMDGPU::V_CMP_NLT_F16_fake16_e64, AMDGPU::V_CMP_NLT_F32_e64,
1433 AMDGPU::V_CMP_NLT_F64_e64);
1434 case CmpInst::FCMP_ULT:
1435 return Select(AMDGPU::V_CMP_NGE_F16_e64, AMDGPU::V_CMP_NGE_F16_t16_e64,
1436 AMDGPU::V_CMP_NGE_F16_fake16_e64, AMDGPU::V_CMP_NGE_F32_e64,
1437 AMDGPU::V_CMP_NGE_F64_e64);
1438 case CmpInst::FCMP_ULE:
1439 return Select(AMDGPU::V_CMP_NGT_F16_e64, AMDGPU::V_CMP_NGT_F16_t16_e64,
1440 AMDGPU::V_CMP_NGT_F16_fake16_e64, AMDGPU::V_CMP_NGT_F32_e64,
1441 AMDGPU::V_CMP_NGT_F64_e64);
1442 case CmpInst::FCMP_UNE:
1443 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1444 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1445 AMDGPU::V_CMP_NEQ_F64_e64);
1446 case CmpInst::FCMP_TRUE:
1447 return Select(AMDGPU::V_CMP_TRU_F16_e64, AMDGPU::V_CMP_TRU_F16_t16_e64,
1448 AMDGPU::V_CMP_TRU_F16_fake16_e64, AMDGPU::V_CMP_TRU_F32_e64,
1449 AMDGPU::V_CMP_TRU_F64_e64);
1450 case CmpInst::FCMP_FALSE:
1451 return Select(AMDGPU::V_CMP_F_F16_e64, AMDGPU::V_CMP_F_F16_t16_e64,
1452 AMDGPU::V_CMP_F_F16_fake16_e64, AMDGPU::V_CMP_F_F32_e64,
1453 AMDGPU::V_CMP_F_F64_e64);
1454 }
1455}
1456
1457int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
1458 unsigned Size) const {
1459 if (Size == 64) {
1460 if (!STI.hasScalarCompareEq64())
1461 return -1;
1462
1463 switch (P) {
1464 case CmpInst::ICMP_NE:
1465 return AMDGPU::S_CMP_LG_U64;
1466 case CmpInst::ICMP_EQ:
1467 return AMDGPU::S_CMP_EQ_U64;
1468 default:
1469 return -1;
1470 }
1471 }
1472
1473 if (Size == 32) {
1474 switch (P) {
1475 case CmpInst::ICMP_NE:
1476 return AMDGPU::S_CMP_LG_U32;
1477 case CmpInst::ICMP_EQ:
1478 return AMDGPU::S_CMP_EQ_U32;
1479 case CmpInst::ICMP_SGT:
1480 return AMDGPU::S_CMP_GT_I32;
1481 case CmpInst::ICMP_SGE:
1482 return AMDGPU::S_CMP_GE_I32;
1483 case CmpInst::ICMP_SLT:
1484 return AMDGPU::S_CMP_LT_I32;
1485 case CmpInst::ICMP_SLE:
1486 return AMDGPU::S_CMP_LE_I32;
1487 case CmpInst::ICMP_UGT:
1488 return AMDGPU::S_CMP_GT_U32;
1489 case CmpInst::ICMP_UGE:
1490 return AMDGPU::S_CMP_GE_U32;
1491 case CmpInst::ICMP_ULT:
1492 return AMDGPU::S_CMP_LT_U32;
1493 case CmpInst::ICMP_ULE:
1494 return AMDGPU::S_CMP_LE_U32;
1495 case CmpInst::FCMP_OEQ:
1496 return AMDGPU::S_CMP_EQ_F32;
1497 case CmpInst::FCMP_OGT:
1498 return AMDGPU::S_CMP_GT_F32;
1499 case CmpInst::FCMP_OGE:
1500 return AMDGPU::S_CMP_GE_F32;
1501 case CmpInst::FCMP_OLT:
1502 return AMDGPU::S_CMP_LT_F32;
1503 case CmpInst::FCMP_OLE:
1504 return AMDGPU::S_CMP_LE_F32;
1505 case CmpInst::FCMP_ONE:
1506 return AMDGPU::S_CMP_LG_F32;
1507 case CmpInst::FCMP_ORD:
1508 return AMDGPU::S_CMP_O_F32;
1509 case CmpInst::FCMP_UNO:
1510 return AMDGPU::S_CMP_U_F32;
1511 case CmpInst::FCMP_UEQ:
1512 return AMDGPU::S_CMP_NLG_F32;
1513 case CmpInst::FCMP_UGT:
1514 return AMDGPU::S_CMP_NLE_F32;
1515 case CmpInst::FCMP_UGE:
1516 return AMDGPU::S_CMP_NLT_F32;
1517 case CmpInst::FCMP_ULT:
1518 return AMDGPU::S_CMP_NGE_F32;
1519 case CmpInst::FCMP_ULE:
1520 return AMDGPU::S_CMP_NGT_F32;
1521 case CmpInst::FCMP_UNE:
1522 return AMDGPU::S_CMP_NEQ_F32;
1523 default:
1524 llvm_unreachable("Unknown condition code!");
1525 }
1526 }
1527
1528 if (Size == 16) {
1529 if (!STI.hasSALUFloatInsts())
1530 return -1;
1531
1532 switch (P) {
1533 case CmpInst::FCMP_OEQ:
1534 return AMDGPU::S_CMP_EQ_F16;
1535 case CmpInst::FCMP_OGT:
1536 return AMDGPU::S_CMP_GT_F16;
1537 case CmpInst::FCMP_OGE:
1538 return AMDGPU::S_CMP_GE_F16;
1539 case CmpInst::FCMP_OLT:
1540 return AMDGPU::S_CMP_LT_F16;
1541 case CmpInst::FCMP_OLE:
1542 return AMDGPU::S_CMP_LE_F16;
1543 case CmpInst::FCMP_ONE:
1544 return AMDGPU::S_CMP_LG_F16;
1545 case CmpInst::FCMP_ORD:
1546 return AMDGPU::S_CMP_O_F16;
1547 case CmpInst::FCMP_UNO:
1548 return AMDGPU::S_CMP_U_F16;
1549 case CmpInst::FCMP_UEQ:
1550 return AMDGPU::S_CMP_NLG_F16;
1551 case CmpInst::FCMP_UGT:
1552 return AMDGPU::S_CMP_NLE_F16;
1553 case CmpInst::FCMP_UGE:
1554 return AMDGPU::S_CMP_NLT_F16;
1555 case CmpInst::FCMP_ULT:
1556 return AMDGPU::S_CMP_NGE_F16;
1557 case CmpInst::FCMP_ULE:
1558 return AMDGPU::S_CMP_NGT_F16;
1559 case CmpInst::FCMP_UNE:
1560 return AMDGPU::S_CMP_NEQ_F16;
1561 default:
1562 llvm_unreachable("Unknown condition code!");
1563 }
1564 }
1565
1566 return -1;
1567}
1568
1569bool AMDGPUInstructionSelector::selectG_ICMP_or_FCMP(MachineInstr &I) const {
1570
1571 MachineBasicBlock *BB = I.getParent();
1572 const DebugLoc &DL = I.getDebugLoc();
1573
1574 Register SrcReg = I.getOperand(i: 2).getReg();
1575 unsigned Size = RBI.getSizeInBits(Reg: SrcReg, MRI: *MRI, TRI);
1576
1577 auto Pred = (CmpInst::Predicate)I.getOperand(i: 1).getPredicate();
1578
1579 Register CCReg = I.getOperand(i: 0).getReg();
1580 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
1581 int Opcode = getS_CMPOpcode(P: Pred, Size);
1582 if (Opcode == -1)
1583 return false;
1584 MachineInstr *ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode))
1585 .add(MO: I.getOperand(i: 2))
1586 .add(MO: I.getOperand(i: 3));
1587 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg)
1588 .addReg(RegNo: AMDGPU::SCC);
1589 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1590 bool Ret =
1591 RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
1592 I.eraseFromParent();
1593 return Ret;
1594 }
1595
1596 if (I.getOpcode() == AMDGPU::G_FCMP)
1597 return false;
1598
1599 int Opcode = getV_CMPOpcode(P: Pred, Size, ST: *Subtarget);
1600 if (Opcode == -1)
1601 return false;
1602
1603 MachineInstrBuilder ICmp;
1604 // t16 instructions
1605 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
1606 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1607 .addImm(Val: 0)
1608 .add(MO: I.getOperand(i: 2))
1609 .addImm(Val: 0)
1610 .add(MO: I.getOperand(i: 3))
1611 .addImm(Val: 0); // op_sel
1612 } else {
1613 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1614 .add(MO: I.getOperand(i: 2))
1615 .add(MO: I.getOperand(i: 3));
1616 }
1617
1618 RBI.constrainGenericRegister(Reg: ICmp->getOperand(i: 0).getReg(),
1619 RC: *TRI.getBoolRC(), MRI&: *MRI);
1620 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1621 I.eraseFromParent();
1622 return true;
1623}
1624
1625// Ballot has to zero bits in input lane-mask that are zero in current exec,
1626// Done as AND with exec. For inputs that are results of instruction that
1627// implicitly use same exec, for example compares in same basic block or SCC to
1628// VCC copy, use copy.
1629static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI,
1630 MachineBasicBlock *MBB) {
1631 MachineInstr *MI = MRI.getVRegDef(Reg);
1632 if (MI->getParent() != MBB)
1633 return false;
1634
1635 // Lane mask generated by SCC to VCC copy.
1636 if (MI->getOpcode() == AMDGPU::COPY) {
1637 auto DstRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 0).getReg());
1638 auto SrcRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 1).getReg());
1639 if (DstRB && SrcRB && DstRB->getID() == AMDGPU::VCCRegBankID &&
1640 SrcRB->getID() == AMDGPU::SGPRRegBankID)
1641 return true;
1642 }
1643
1644 // Lane mask generated by SCC to VCC copy
1645 if (MI->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
1646 return true;
1647
1648 // Lane mask generated using compare with same exec.
1649 if (isa<GAnyCmp>(Val: MI))
1650 return true;
1651
1652 Register LHS, RHS;
1653 // Look through AND.
1654 if (mi_match(R: Reg, MRI, P: m_GAnd(L: m_Reg(R&: LHS), R: m_Reg(R&: RHS))))
1655 return isLaneMaskFromSameBlock(Reg: LHS, MRI, MBB) ||
1656 isLaneMaskFromSameBlock(Reg: RHS, MRI, MBB);
1657
1658 return false;
1659}
1660
1661bool AMDGPUInstructionSelector::selectBallot(MachineInstr &I) const {
1662 MachineBasicBlock *BB = I.getParent();
1663 const DebugLoc &DL = I.getDebugLoc();
1664 Register DstReg = I.getOperand(i: 0).getReg();
1665 Register SrcReg = I.getOperand(i: 2).getReg();
1666 const unsigned BallotSize = MRI->getType(Reg: DstReg).getSizeInBits();
1667 const unsigned WaveSize = STI.getWavefrontSize();
1668
1669 // In the common case, the return type matches the wave size.
1670 // However we also support emitting i64 ballots in wave32 mode.
1671 if (BallotSize != WaveSize && (BallotSize != 64 || WaveSize != 32))
1672 return false;
1673
1674 std::optional<ValueAndVReg> Arg =
1675 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI);
1676
1677 Register Dst = DstReg;
1678 // i64 ballot on Wave32: new Dst(i32) for WaveSize ballot.
1679 if (BallotSize != WaveSize) {
1680 Dst = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
1681 }
1682
1683 if (Arg) {
1684 const int64_t Value = Arg->Value.getZExtValue();
1685 if (Value == 0) {
1686 // Dst = S_MOV 0
1687 unsigned Opcode = WaveSize == 64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
1688 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: Dst).addImm(Val: 0);
1689 } else {
1690 // Dst = COPY EXEC
1691 assert(Value == 1);
1692 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: TRI.getExec());
1693 }
1694 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1695 return false;
1696 } else {
1697 if (isLaneMaskFromSameBlock(Reg: SrcReg, MRI&: *MRI, MBB: BB)) {
1698 // Dst = COPY SrcReg
1699 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: SrcReg);
1700 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1701 return false;
1702 } else {
1703 // Dst = S_AND SrcReg, EXEC
1704 unsigned AndOpc = WaveSize == 64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
1705 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: Dst)
1706 .addReg(RegNo: SrcReg)
1707 .addReg(RegNo: TRI.getExec())
1708 .setOperandDead(3); // Dead scc
1709 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
1710 }
1711 }
1712
1713 // i64 ballot on Wave32: zero-extend i32 ballot to i64.
1714 if (BallotSize != WaveSize) {
1715 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1716 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg).addImm(Val: 0);
1717 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
1718 .addReg(RegNo: Dst)
1719 .addImm(Val: AMDGPU::sub0)
1720 .addReg(RegNo: HiReg)
1721 .addImm(Val: AMDGPU::sub1);
1722 }
1723
1724 I.eraseFromParent();
1725 return true;
1726}
1727
1728bool AMDGPUInstructionSelector::selectRelocConstant(MachineInstr &I) const {
1729 Register DstReg = I.getOperand(i: 0).getReg();
1730 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1731 const TargetRegisterClass *DstRC = TRI.getRegClassForSizeOnBank(Size: 32, Bank: *DstBank);
1732 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1733 return false;
1734
1735 const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID;
1736
1737 Module *M = MF->getFunction().getParent();
1738 const MDNode *Metadata = I.getOperand(i: 2).getMetadata();
1739 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
1740 auto *RelocSymbol = cast<GlobalVariable>(
1741 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
1742
1743 MachineBasicBlock *BB = I.getParent();
1744 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(),
1745 MCID: TII.get(Opcode: IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DestReg: DstReg)
1746 .addGlobalAddress(GV: RelocSymbol, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1747
1748 I.eraseFromParent();
1749 return true;
1750}
1751
1752bool AMDGPUInstructionSelector::selectGroupStaticSize(MachineInstr &I) const {
1753 Triple::OSType OS = MF->getTarget().getTargetTriple().getOS();
1754
1755 Register DstReg = I.getOperand(i: 0).getReg();
1756 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1757 unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ?
1758 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1759
1760 MachineBasicBlock *MBB = I.getParent();
1761 const DebugLoc &DL = I.getDebugLoc();
1762
1763 auto MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Mov), DestReg: DstReg);
1764
1765 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
1766 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1767 MIB.addImm(Val: MFI->getLDSSize());
1768 } else {
1769 Module *M = MF->getFunction().getParent();
1770 const GlobalValue *GV =
1771 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::amdgcn_groupstaticsize);
1772 MIB.addGlobalAddress(GV, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1773 }
1774
1775 I.eraseFromParent();
1776 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1777 return true;
1778}
1779
1780bool AMDGPUInstructionSelector::selectReturnAddress(MachineInstr &I) const {
1781 MachineBasicBlock *MBB = I.getParent();
1782 MachineFunction &MF = *MBB->getParent();
1783 const DebugLoc &DL = I.getDebugLoc();
1784
1785 Register DstReg = I.getOperand(i: 0).getReg();
1786 unsigned Depth = I.getOperand(i: 2).getImm();
1787
1788 const TargetRegisterClass *RC =
1789 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
1790 if (!RC->hasSubClassEq(RC: &AMDGPU::SGPR_64RegClass) ||
1791 !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
1792 return false;
1793
1794 // Check for kernel and shader functions
1795 if (Depth != 0 ||
1796 MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
1797 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg)
1798 .addImm(Val: 0);
1799 I.eraseFromParent();
1800 return true;
1801 }
1802
1803 MachineFrameInfo &MFI = MF.getFrameInfo();
1804 // There is a call to @llvm.returnaddress in this function
1805 MFI.setReturnAddressIsTaken(true);
1806
1807 // Get the return address reg and mark it as an implicit live-in
1808 Register ReturnAddrReg = TRI.getReturnAddressReg(MF);
1809 Register LiveIn = getFunctionLiveInPhysReg(MF, TII, PhysReg: ReturnAddrReg,
1810 RC: AMDGPU::SReg_64RegClass, DL);
1811 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
1812 .addReg(RegNo: LiveIn);
1813 I.eraseFromParent();
1814 return true;
1815}
1816
1817bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const {
1818 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1819 // SelectionDAG uses for wave32 vs wave64.
1820 MachineBasicBlock *BB = MI.getParent();
1821 BuildMI(BB&: *BB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_END_CF))
1822 .add(MO: MI.getOperand(i: 1))
1823 .setOperandDead(2); // implicit-def $scc
1824
1825 Register Reg = MI.getOperand(i: 1).getReg();
1826 MI.eraseFromParent();
1827
1828 if (!MRI->getRegClassOrNull(Reg))
1829 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1830 return true;
1831}
1832
1833bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic(
1834 MachineInstr &MI, Intrinsic::ID IntrID) const {
1835 MachineBasicBlock *MBB = MI.getParent();
1836 MachineFunction *MF = MBB->getParent();
1837 const DebugLoc &DL = MI.getDebugLoc();
1838
1839 unsigned IndexOperand = MI.getOperand(i: 7).getImm();
1840 bool WaveRelease = MI.getOperand(i: 8).getImm() != 0;
1841 bool WaveDone = MI.getOperand(i: 9).getImm() != 0;
1842
1843 if (WaveDone && !WaveRelease) {
1844 // TODO: Move this to IR verifier
1845 const Function &Fn = MF->getFunction();
1846 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1847 Fn, "ds_ordered_count: wave_done requires wave_release", DL));
1848 }
1849
1850 unsigned OrderedCountIndex = IndexOperand & 0x3f;
1851 IndexOperand &= ~0x3f;
1852 unsigned CountDw = 0;
1853
1854 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) {
1855 CountDw = (IndexOperand >> 24) & 0xf;
1856 IndexOperand &= ~(0xf << 24);
1857
1858 if (CountDw < 1 || CountDw > 4) {
1859 const Function &Fn = MF->getFunction();
1860 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1861 Fn, "ds_ordered_count: dword count must be between 1 and 4", DL));
1862 CountDw = 1;
1863 }
1864 }
1865
1866 if (IndexOperand) {
1867 const Function &Fn = MF->getFunction();
1868 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1869 Fn, "ds_ordered_count: bad index operand", DL));
1870 }
1871
1872 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
1873 unsigned ShaderType = SIInstrInfo::getDSShaderTypeValue(MF: *MF);
1874
1875 unsigned Offset0 = OrderedCountIndex << 2;
1876 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
1877
1878 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10)
1879 Offset1 |= (CountDw - 1) << 6;
1880
1881 if (STI.getGeneration() < AMDGPUSubtarget::GFX11)
1882 Offset1 |= ShaderType << 2;
1883
1884 unsigned Offset = Offset0 | (Offset1 << 8);
1885
1886 Register M0Val = MI.getOperand(i: 2).getReg();
1887 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1888 .addReg(RegNo: M0Val);
1889
1890 Register DstReg = MI.getOperand(i: 0).getReg();
1891 Register ValReg = MI.getOperand(i: 3).getReg();
1892 MachineInstrBuilder DS =
1893 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_ORDERED_COUNT), DestReg: DstReg)
1894 .addReg(RegNo: ValReg)
1895 .addImm(Val: Offset)
1896 .cloneMemRefs(OtherMI: MI);
1897
1898 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
1899 return false;
1900
1901 constrainSelectedInstRegOperands(I&: *DS, TII, TRI, RBI);
1902 MI.eraseFromParent();
1903 return true;
1904}
1905
1906static unsigned gwsIntrinToOpcode(unsigned IntrID) {
1907 switch (IntrID) {
1908 case Intrinsic::amdgcn_ds_gws_init:
1909 return AMDGPU::DS_GWS_INIT;
1910 case Intrinsic::amdgcn_ds_gws_barrier:
1911 return AMDGPU::DS_GWS_BARRIER;
1912 case Intrinsic::amdgcn_ds_gws_sema_v:
1913 return AMDGPU::DS_GWS_SEMA_V;
1914 case Intrinsic::amdgcn_ds_gws_sema_br:
1915 return AMDGPU::DS_GWS_SEMA_BR;
1916 case Intrinsic::amdgcn_ds_gws_sema_p:
1917 return AMDGPU::DS_GWS_SEMA_P;
1918 case Intrinsic::amdgcn_ds_gws_sema_release_all:
1919 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
1920 default:
1921 llvm_unreachable("not a gws intrinsic");
1922 }
1923}
1924
1925bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
1926 Intrinsic::ID IID) const {
1927 if (!STI.hasGWS() || (IID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
1928 !STI.hasGWSSemaReleaseAll()))
1929 return false;
1930
1931 // intrinsic ID, vsrc, offset
1932 const bool HasVSrc = MI.getNumOperands() == 3;
1933 assert(HasVSrc || MI.getNumOperands() == 2);
1934
1935 Register BaseOffset = MI.getOperand(i: HasVSrc ? 2 : 1).getReg();
1936 const RegisterBank *OffsetRB = RBI.getRegBank(Reg: BaseOffset, MRI: *MRI, TRI);
1937 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID)
1938 return false;
1939
1940 MachineInstr *OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
1941 unsigned ImmOffset;
1942
1943 MachineBasicBlock *MBB = MI.getParent();
1944 const DebugLoc &DL = MI.getDebugLoc();
1945
1946 MachineInstr *Readfirstlane = nullptr;
1947
1948 // If we legalized the VGPR input, strip out the readfirstlane to analyze the
1949 // incoming offset, in case there's an add of a constant. We'll have to put it
1950 // back later.
1951 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) {
1952 Readfirstlane = OffsetDef;
1953 BaseOffset = OffsetDef->getOperand(i: 1).getReg();
1954 OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
1955 }
1956
1957 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) {
1958 // If we have a constant offset, try to use the 0 in m0 as the base.
1959 // TODO: Look into changing the default m0 initialization value. If the
1960 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
1961 // the immediate offset.
1962
1963 ImmOffset = OffsetDef->getOperand(i: 1).getCImm()->getZExtValue();
1964 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
1965 .addImm(Val: 0);
1966 } else {
1967 std::tie(args&: BaseOffset, args&: ImmOffset) =
1968 AMDGPU::getBaseWithConstantOffset(MRI&: *MRI, Reg: BaseOffset, ValueTracking: VT);
1969
1970 if (Readfirstlane) {
1971 // We have the constant offset now, so put the readfirstlane back on the
1972 // variable component.
1973 if (!RBI.constrainGenericRegister(Reg: BaseOffset, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
1974 return false;
1975
1976 Readfirstlane->getOperand(i: 1).setReg(BaseOffset);
1977 BaseOffset = Readfirstlane->getOperand(i: 0).getReg();
1978 } else {
1979 if (!RBI.constrainGenericRegister(Reg: BaseOffset,
1980 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
1981 return false;
1982 }
1983
1984 Register M0Base = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1985 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: M0Base)
1986 .addReg(RegNo: BaseOffset)
1987 .addImm(Val: 16)
1988 .setOperandDead(3); // Dead scc
1989
1990 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1991 .addReg(RegNo: M0Base);
1992 }
1993
1994 // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
1995 // offset field) % 64. Some versions of the programming guide omit the m0
1996 // part, or claim it's from offset 0.
1997
1998 unsigned Opc = gwsIntrinToOpcode(IntrID: IID);
1999 const MCInstrDesc &InstrDesc = TII.get(Opcode: Opc);
2000
2001 if (HasVSrc) {
2002 Register VSrc = MI.getOperand(i: 1).getReg();
2003
2004 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
2005 const TargetRegisterClass *DataRC = TII.getRegClass(MCID: InstrDesc, OpNum: Data0Idx);
2006 const TargetRegisterClass *SubRC =
2007 TRI.getSubRegisterClass(DataRC, AMDGPU::sub0);
2008
2009 if (!SubRC) {
2010 // 32-bit normal case.
2011 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *DataRC, MRI&: *MRI))
2012 return false;
2013
2014 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2015 .addReg(RegNo: VSrc)
2016 .addImm(Val: ImmOffset)
2017 .cloneMemRefs(OtherMI: MI);
2018 } else {
2019 // Requires even register alignment, so create 64-bit value and pad the
2020 // top half with undef.
2021 Register DataReg = MRI->createVirtualRegister(RegClass: DataRC);
2022 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *SubRC, MRI&: *MRI))
2023 return false;
2024
2025 Register UndefReg = MRI->createVirtualRegister(RegClass: SubRC);
2026 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2027 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DataReg)
2028 .addReg(RegNo: VSrc)
2029 .addImm(Val: AMDGPU::sub0)
2030 .addReg(RegNo: UndefReg)
2031 .addImm(Val: AMDGPU::sub1);
2032
2033 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2034 .addReg(RegNo: DataReg)
2035 .addImm(Val: ImmOffset)
2036 .cloneMemRefs(OtherMI: MI);
2037 }
2038 } else {
2039 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2040 .addImm(Val: ImmOffset)
2041 .cloneMemRefs(OtherMI: MI);
2042 }
2043
2044 MI.eraseFromParent();
2045 return true;
2046}
2047
2048bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
2049 bool IsAppend) const {
2050 Register PtrBase = MI.getOperand(i: 2).getReg();
2051 LLT PtrTy = MRI->getType(Reg: PtrBase);
2052 bool IsGDS = PtrTy.getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2053
2054 unsigned Offset;
2055 std::tie(args&: PtrBase, args&: Offset) = selectDS1Addr1OffsetImpl(Root&: MI.getOperand(i: 2));
2056
2057 // TODO: Should this try to look through readfirstlane like GWS?
2058 if (!isDSOffsetLegal(Base: PtrBase, Offset)) {
2059 PtrBase = MI.getOperand(i: 2).getReg();
2060 Offset = 0;
2061 }
2062
2063 MachineBasicBlock *MBB = MI.getParent();
2064 const DebugLoc &DL = MI.getDebugLoc();
2065 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2066
2067 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
2068 .addReg(RegNo: PtrBase);
2069 if (!RBI.constrainGenericRegister(Reg: PtrBase, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
2070 return false;
2071
2072 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: MI.getOperand(i: 0).getReg())
2073 .addImm(Val: Offset)
2074 .addImm(Val: IsGDS ? -1 : 0)
2075 .cloneMemRefs(OtherMI: MI);
2076 MI.eraseFromParent();
2077 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2078 return true;
2079}
2080
2081bool AMDGPUInstructionSelector::selectInitWholeWave(MachineInstr &MI) const {
2082 MachineFunction *MF = MI.getMF();
2083 SIMachineFunctionInfo *MFInfo = MF->getInfo<SIMachineFunctionInfo>();
2084
2085 MFInfo->setInitWholeWave();
2086 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
2087}
2088
2089static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE,
2090 bool &IsTexFail) {
2091 if (TexFailCtrl)
2092 IsTexFail = true;
2093
2094 TFE = TexFailCtrl & 0x1;
2095 TexFailCtrl &= ~(uint64_t)0x1;
2096 LWE = TexFailCtrl & 0x2;
2097 TexFailCtrl &= ~(uint64_t)0x2;
2098
2099 return TexFailCtrl == 0;
2100}
2101
2102bool AMDGPUInstructionSelector::selectImageIntrinsic(
2103 MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
2104 MachineBasicBlock *MBB = MI.getParent();
2105 const DebugLoc &DL = MI.getDebugLoc();
2106 unsigned IntrOpcode = Intr->BaseOpcode;
2107
2108 // For image atomic: use no-return opcode if result is unused.
2109 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode) {
2110 Register ResultDef = MI.getOperand(i: 0).getReg();
2111 if (MRI->use_nodbg_empty(RegNo: ResultDef))
2112 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
2113 }
2114
2115 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
2116 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
2117
2118 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
2119 const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI);
2120 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI);
2121 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
2122 const bool IsGFX13Plus = AMDGPU::isGFX13Plus(STI);
2123
2124 const unsigned ArgOffset = MI.getNumExplicitDefs() + 1;
2125
2126 Register VDataIn = AMDGPU::NoRegister;
2127 Register VDataOut = AMDGPU::NoRegister;
2128 LLT VDataTy;
2129 int NumVDataDwords = -1;
2130 bool IsD16 = MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 ||
2131 MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16;
2132
2133 bool Unorm;
2134 if (!BaseOpcode->Sampler)
2135 Unorm = true;
2136 else
2137 Unorm = MI.getOperand(i: ArgOffset + Intr->UnormIndex).getImm() != 0;
2138
2139 bool TFE;
2140 bool LWE;
2141 bool IsTexFail = false;
2142 if (!parseTexFail(TexFailCtrl: MI.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex).getImm(),
2143 TFE, LWE, IsTexFail))
2144 return false;
2145
2146 const int Flags = MI.getOperand(i: ArgOffset + Intr->NumArgs).getImm();
2147 const bool IsA16 = (Flags & 1) != 0;
2148 const bool IsG16 = (Flags & 2) != 0;
2149
2150 // A16 implies 16 bit gradients if subtarget doesn't support G16
2151 if (IsA16 && !STI.hasG16() && !IsG16)
2152 return false;
2153
2154 unsigned DMask = 0;
2155 unsigned DMaskLanes = 0;
2156
2157 if (BaseOpcode->Atomic) {
2158 if (!BaseOpcode->NoReturn)
2159 VDataOut = MI.getOperand(i: 0).getReg();
2160 VDataIn = MI.getOperand(i: 2).getReg();
2161 LLT Ty = MRI->getType(Reg: VDataIn);
2162
2163 // Be careful to allow atomic swap on 16-bit element vectors.
2164 const bool Is64Bit = BaseOpcode->AtomicX2 ?
2165 Ty.getSizeInBits() == 128 :
2166 Ty.getSizeInBits() == 64;
2167
2168 if (BaseOpcode->AtomicX2) {
2169 assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister);
2170
2171 DMask = Is64Bit ? 0xf : 0x3;
2172 NumVDataDwords = Is64Bit ? 4 : 2;
2173 } else {
2174 DMask = Is64Bit ? 0x3 : 0x1;
2175 NumVDataDwords = Is64Bit ? 2 : 1;
2176 }
2177 } else {
2178 DMask = MI.getOperand(i: ArgOffset + Intr->DMaskIndex).getImm();
2179 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
2180
2181 if (BaseOpcode->Store) {
2182 VDataIn = MI.getOperand(i: 1).getReg();
2183 VDataTy = MRI->getType(Reg: VDataIn);
2184 NumVDataDwords = (VDataTy.getSizeInBits() + 31) / 32;
2185 } else if (BaseOpcode->NoReturn) {
2186 NumVDataDwords = 0;
2187 } else {
2188 VDataOut = MI.getOperand(i: 0).getReg();
2189 VDataTy = MRI->getType(Reg: VDataOut);
2190 NumVDataDwords = DMaskLanes;
2191
2192 if (IsD16 && !STI.hasUnpackedD16VMem())
2193 NumVDataDwords = (DMaskLanes + 1) / 2;
2194 }
2195 }
2196
2197 // Set G16 opcode
2198 if (Subtarget->hasG16() && IsG16) {
2199 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
2200 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
2201 assert(G16MappingInfo);
2202 IntrOpcode = G16MappingInfo->G16; // set opcode to variant with _g16
2203 }
2204
2205 // TODO: Check this in verifier.
2206 assert((!IsTexFail || DMaskLanes >= 1) && "should have legalized this");
2207
2208 unsigned CPol = MI.getOperand(i: ArgOffset + Intr->CachePolicyIndex).getImm();
2209 // Keep GLC only when the atomic's result is actually used.
2210 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
2211 CPol |= AMDGPU::CPol::GLC;
2212 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
2213 AMDGPU::CPol::VOLATILE))
2214 return false;
2215
2216 int NumVAddrRegs = 0;
2217 int NumVAddrDwords = 0;
2218 for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
2219 // Skip the $noregs and 0s inserted during legalization.
2220 MachineOperand &AddrOp = MI.getOperand(i: ArgOffset + I);
2221 if (!AddrOp.isReg())
2222 continue; // XXX - Break?
2223
2224 Register Addr = AddrOp.getReg();
2225 if (!Addr)
2226 break;
2227
2228 ++NumVAddrRegs;
2229 NumVAddrDwords += (MRI->getType(Reg: Addr).getSizeInBits() + 31) / 32;
2230 }
2231
2232 // The legalizer preprocessed the intrinsic arguments. If we aren't using
2233 // NSA, these should have been packed into a single value in the first
2234 // address register
2235 const bool UseNSA =
2236 NumVAddrRegs != 1 &&
2237 (STI.hasPartialNSAEncoding() ? NumVAddrDwords >= NumVAddrRegs
2238 : NumVAddrDwords == NumVAddrRegs);
2239 if (UseNSA && !STI.hasFeature(Feature: AMDGPU::FeatureNSAEncoding)) {
2240 LLVM_DEBUG(dbgs() << "Trying to use NSA on non-NSA target\n");
2241 return false;
2242 }
2243
2244 if (IsTexFail)
2245 ++NumVDataDwords;
2246
2247 int Opcode = -1;
2248 if (IsGFX13Plus) {
2249 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx13,
2250 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2251 } else if (IsGFX12Plus) {
2252 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
2253 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2254 } else if (IsGFX11Plus) {
2255 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2256 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
2257 : AMDGPU::MIMGEncGfx11Default,
2258 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2259 } else if (IsGFX10Plus) {
2260 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2261 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
2262 : AMDGPU::MIMGEncGfx10Default,
2263 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2264 } else {
2265 if (Subtarget->hasGFX90AInsts()) {
2266 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
2267 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2268 if (Opcode == -1) {
2269 LLVM_DEBUG(
2270 dbgs()
2271 << "requested image instruction is not supported on this GPU\n");
2272 return false;
2273 }
2274 }
2275 if (Opcode == -1 &&
2276 STI.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
2277 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
2278 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2279 if (Opcode == -1)
2280 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
2281 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2282 }
2283 if (Opcode == -1)
2284 return false;
2285
2286 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode))
2287 .cloneMemRefs(OtherMI: MI);
2288
2289 if (VDataOut) {
2290 if (BaseOpcode->AtomicX2) {
2291 const bool Is64 = MRI->getType(Reg: VDataOut).getSizeInBits() == 64;
2292
2293 Register TmpReg = MRI->createVirtualRegister(
2294 RegClass: Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass);
2295 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0;
2296
2297 MIB.addDef(RegNo: TmpReg);
2298 if (!MRI->use_empty(RegNo: VDataOut)) {
2299 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VDataOut)
2300 .addReg(RegNo: TmpReg, Flags: RegState::Kill, SubReg);
2301 }
2302
2303 } else {
2304 MIB.addDef(RegNo: VDataOut); // vdata output
2305 }
2306 }
2307
2308 if (VDataIn)
2309 MIB.addReg(RegNo: VDataIn); // vdata input
2310
2311 for (int I = 0; I != NumVAddrRegs; ++I) {
2312 MachineOperand &SrcOp = MI.getOperand(i: ArgOffset + Intr->VAddrStart + I);
2313 if (SrcOp.isReg()) {
2314 assert(SrcOp.getReg() != 0);
2315 MIB.addReg(RegNo: SrcOp.getReg());
2316 }
2317 }
2318
2319 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->RsrcIndex).getReg());
2320 if (BaseOpcode->Sampler)
2321 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->SampIndex).getReg());
2322
2323 MIB.addImm(Val: DMask); // dmask
2324
2325 if (IsGFX10Plus)
2326 MIB.addImm(Val: DimInfo->Encoding);
2327 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::unorm))
2328 MIB.addImm(Val: Unorm);
2329
2330 MIB.addImm(Val: CPol);
2331 MIB.addImm(Val: IsA16 && // a16 or r128
2332 STI.hasFeature(Feature: AMDGPU::FeatureR128A16) ? -1 : 0);
2333 if (IsGFX10Plus)
2334 MIB.addImm(Val: IsA16 ? -1 : 0);
2335
2336 if (!Subtarget->hasGFX90AInsts()) {
2337 MIB.addImm(Val: TFE); // tfe
2338 } else if (TFE) {
2339 LLVM_DEBUG(dbgs() << "TFE is not supported on this GPU\n");
2340 return false;
2341 }
2342
2343 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::lwe))
2344 MIB.addImm(Val: LWE); // lwe
2345 if (!IsGFX10Plus)
2346 MIB.addImm(Val: DimInfo->DA ? -1 : 0);
2347 if (BaseOpcode->HasD16)
2348 MIB.addImm(Val: IsD16 ? -1 : 0);
2349
2350 MI.eraseFromParent();
2351 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2352 TII.enforceOperandRCAlignment(MI&: *MIB, OpName: AMDGPU::OpName::vaddr);
2353 return true;
2354}
2355
2356// We need to handle this here because tablegen doesn't support matching
2357// instructions with multiple outputs.
2358bool AMDGPUInstructionSelector::selectDSBvhStackIntrinsic(
2359 MachineInstr &MI) const {
2360 Register Dst0 = MI.getOperand(i: 0).getReg();
2361 Register Dst1 = MI.getOperand(i: 1).getReg();
2362
2363 const DebugLoc &DL = MI.getDebugLoc();
2364 MachineBasicBlock *MBB = MI.getParent();
2365
2366 Register Addr = MI.getOperand(i: 3).getReg();
2367 Register Data0 = MI.getOperand(i: 4).getReg();
2368 Register Data1 = MI.getOperand(i: 5).getReg();
2369 unsigned Offset = MI.getOperand(i: 6).getImm();
2370
2371 unsigned Opc;
2372 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
2373 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2374 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2375 Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
2376 break;
2377 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2378 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32;
2379 break;
2380 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2381 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64;
2382 break;
2383 }
2384
2385 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
2386 .addDef(RegNo: Dst1)
2387 .addUse(RegNo: Addr)
2388 .addUse(RegNo: Data0)
2389 .addUse(RegNo: Data1)
2390 .addImm(Val: Offset)
2391 .cloneMemRefs(OtherMI: MI);
2392
2393 MI.eraseFromParent();
2394 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2395 return true;
2396}
2397
2398bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
2399 MachineInstr &I) const {
2400 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
2401 switch (IntrinsicID) {
2402 case Intrinsic::amdgcn_end_cf:
2403 return selectEndCfIntrinsic(MI&: I);
2404 case Intrinsic::amdgcn_ds_ordered_add:
2405 case Intrinsic::amdgcn_ds_ordered_swap:
2406 return selectDSOrderedIntrinsic(MI&: I, IntrID: IntrinsicID);
2407 case Intrinsic::amdgcn_ds_gws_init:
2408 case Intrinsic::amdgcn_ds_gws_barrier:
2409 case Intrinsic::amdgcn_ds_gws_sema_v:
2410 case Intrinsic::amdgcn_ds_gws_sema_br:
2411 case Intrinsic::amdgcn_ds_gws_sema_p:
2412 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2413 return selectDSGWSIntrinsic(MI&: I, IID: IntrinsicID);
2414 case Intrinsic::amdgcn_ds_append:
2415 return selectDSAppendConsume(MI&: I, IsAppend: true);
2416 case Intrinsic::amdgcn_ds_consume:
2417 return selectDSAppendConsume(MI&: I, IsAppend: false);
2418 case Intrinsic::amdgcn_init_whole_wave:
2419 return selectInitWholeWave(MI&: I);
2420 case Intrinsic::amdgcn_raw_buffer_load_lds:
2421 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
2422 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
2423 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
2424 case Intrinsic::amdgcn_struct_buffer_load_lds:
2425 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
2426 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
2427 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
2428 return selectBufferLoadLds(MI&: I);
2429 // Until we can store both the address space of the global and the LDS
2430 // arguments by having tto MachineMemOperands on an intrinsic, we just trust
2431 // that the argument is a global pointer (buffer pointers have been handled by
2432 // a LLVM IR-level lowering).
2433 case Intrinsic::amdgcn_load_to_lds:
2434 case Intrinsic::amdgcn_load_async_to_lds:
2435 case Intrinsic::amdgcn_global_load_lds:
2436 case Intrinsic::amdgcn_global_load_async_lds:
2437 return selectGlobalLoadLds(MI&: I);
2438 case Intrinsic::amdgcn_tensor_load_to_lds:
2439 case Intrinsic::amdgcn_tensor_store_from_lds:
2440 return selectTensorLoadStore(MI&: I, IID: IntrinsicID);
2441 case Intrinsic::amdgcn_asyncmark:
2442 case Intrinsic::amdgcn_wait_asyncmark:
2443 if (!Subtarget->hasAsyncMark())
2444 return false;
2445 break;
2446 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2447 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2448 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2449 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2450 return selectDSBvhStackIntrinsic(MI&: I);
2451 case Intrinsic::amdgcn_s_alloc_vgpr: {
2452 // S_ALLOC_VGPR doesn't have a destination register, it just implicitly sets
2453 // SCC. We then need to COPY it into the result vreg.
2454 MachineBasicBlock *MBB = I.getParent();
2455 const DebugLoc &DL = I.getDebugLoc();
2456
2457 Register ResReg = I.getOperand(i: 0).getReg();
2458
2459 MachineInstr *AllocMI = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ALLOC_VGPR))
2460 .add(MO: I.getOperand(i: 2));
2461 (void)BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: ResReg)
2462 .addReg(RegNo: AMDGPU::SCC);
2463 I.eraseFromParent();
2464 constrainSelectedInstRegOperands(I&: *AllocMI, TII, TRI, RBI);
2465 return RBI.constrainGenericRegister(Reg: ResReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2466 }
2467 case Intrinsic::amdgcn_s_barrier_init:
2468 case Intrinsic::amdgcn_s_barrier_signal_var:
2469 return selectNamedBarrierInit(I, IID: IntrinsicID);
2470 case Intrinsic::amdgcn_s_wakeup_barrier:
2471 case Intrinsic::amdgcn_s_barrier_join:
2472 case Intrinsic::amdgcn_s_get_named_barrier_state:
2473 return selectNamedBarrierInst(I, IID: IntrinsicID);
2474 case Intrinsic::amdgcn_s_get_barrier_state:
2475 return selectSGetBarrierState(I, IID: IntrinsicID);
2476 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
2477 return selectSBarrierSignalIsfirst(I, IID: IntrinsicID);
2478 }
2479 return selectImpl(I, CoverageInfo&: *CoverageInfo);
2480}
2481
2482bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
2483 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
2484 return true;
2485
2486 MachineBasicBlock *BB = I.getParent();
2487 const DebugLoc &DL = I.getDebugLoc();
2488
2489 Register DstReg = I.getOperand(i: 0).getReg();
2490 unsigned Size = RBI.getSizeInBits(Reg: DstReg, MRI: *MRI, TRI);
2491 assert(Size <= 32 || Size == 64);
2492 const MachineOperand &CCOp = I.getOperand(i: 1);
2493 Register CCReg = CCOp.getReg();
2494 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
2495 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 :
2496 AMDGPU::S_CSELECT_B32;
2497 MachineInstr *CopySCC = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
2498 .addReg(RegNo: CCReg);
2499
2500 // The generic constrainSelectedInstRegOperands doesn't work for the scc register
2501 // bank, because it does not cover the register class that we used to represent
2502 // for it. So we need to manually set the register class here.
2503 if (!MRI->getRegClassOrNull(Reg: CCReg))
2504 MRI->setRegClass(Reg: CCReg, RC: TRI.getConstrainedRegClassForReg(Reg: CCReg, MRI: *MRI));
2505 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
2506 .add(MO: I.getOperand(i: 2))
2507 .add(MO: I.getOperand(i: 3));
2508
2509 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2510 constrainSelectedInstRegOperands(I&: *CopySCC, TII, TRI, RBI);
2511 I.eraseFromParent();
2512 return true;
2513 }
2514
2515 // Wide VGPR select should have been split in RegBankSelect.
2516 if (Size > 32)
2517 return false;
2518
2519 MachineInstr *Select =
2520 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
2521 .addImm(Val: 0)
2522 .add(MO: I.getOperand(i: 3))
2523 .addImm(Val: 0)
2524 .add(MO: I.getOperand(i: 2))
2525 .add(MO: I.getOperand(i: 1));
2526
2527 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2528 I.eraseFromParent();
2529 return true;
2530}
2531
2532bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
2533 Register DstReg = I.getOperand(i: 0).getReg();
2534 Register SrcReg = I.getOperand(i: 1).getReg();
2535 const LLT DstTy = MRI->getType(Reg: DstReg);
2536 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2537 const LLT S1 = LLT::scalar(SizeInBits: 1);
2538
2539 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2540 const RegisterBank *DstRB;
2541 if (DstTy == S1) {
2542 // This is a special case. We don't treat s1 for legalization artifacts as
2543 // vcc booleans.
2544 DstRB = SrcRB;
2545 } else {
2546 DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2547 if (SrcRB != DstRB)
2548 return false;
2549 }
2550
2551 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
2552
2553 unsigned DstSize = DstTy.getSizeInBits();
2554 unsigned SrcSize = SrcTy.getSizeInBits();
2555
2556 const TargetRegisterClass *SrcRC =
2557 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcRB);
2558 const TargetRegisterClass *DstRC =
2559 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
2560 if (!SrcRC || !DstRC)
2561 return false;
2562
2563 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
2564 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
2565 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
2566 return false;
2567 }
2568
2569 if (DstRC == &AMDGPU::VGPR_16RegClass && SrcSize == 32) {
2570 assert(STI.useRealTrue16Insts());
2571 const DebugLoc &DL = I.getDebugLoc();
2572 MachineBasicBlock *MBB = I.getParent();
2573 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
2574 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::lo16);
2575 I.eraseFromParent();
2576 return true;
2577 }
2578
2579 if (DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) && SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
2580 MachineBasicBlock *MBB = I.getParent();
2581 const DebugLoc &DL = I.getDebugLoc();
2582
2583 Register LoReg = MRI->createVirtualRegister(RegClass: DstRC);
2584 Register HiReg = MRI->createVirtualRegister(RegClass: DstRC);
2585 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
2586 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
2587 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
2588 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
2589
2590 if (IsVALU && STI.hasSDWA()) {
2591 // Write the low 16-bits of the high element into the high 16-bits of the
2592 // low element.
2593 MachineInstr *MovSDWA =
2594 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: DstReg)
2595 .addImm(Val: 0) // $src0_modifiers
2596 .addReg(RegNo: HiReg) // $src0
2597 .addImm(Val: 0) // $clamp
2598 .addImm(Val: AMDGPU::SDWA::WORD_1) // $dst_sel
2599 .addImm(Val: AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused
2600 .addImm(Val: AMDGPU::SDWA::WORD_0) // $src0_sel
2601 .addReg(RegNo: LoReg, Flags: RegState::Implicit);
2602 MovSDWA->tieOperands(DefIdx: 0, UseIdx: MovSDWA->getNumOperands() - 1);
2603 } else {
2604 Register TmpReg0 = MRI->createVirtualRegister(RegClass: DstRC);
2605 Register TmpReg1 = MRI->createVirtualRegister(RegClass: DstRC);
2606 Register ImmReg = MRI->createVirtualRegister(RegClass: DstRC);
2607 if (IsVALU) {
2608 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: TmpReg0)
2609 .addImm(Val: 16)
2610 .addReg(RegNo: HiReg);
2611 } else {
2612 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg0)
2613 .addReg(RegNo: HiReg)
2614 .addImm(Val: 16)
2615 .setOperandDead(3); // Dead scc
2616 }
2617
2618 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2619 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
2620 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32;
2621
2622 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: ImmReg)
2623 .addImm(Val: 0xffff);
2624 auto And = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: TmpReg1)
2625 .addReg(RegNo: LoReg)
2626 .addReg(RegNo: ImmReg);
2627 auto Or = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: OrOpc), DestReg: DstReg)
2628 .addReg(RegNo: TmpReg0)
2629 .addReg(RegNo: TmpReg1);
2630
2631 if (!IsVALU) {
2632 And.setOperandDead(3); // Dead scc
2633 Or.setOperandDead(3); // Dead scc
2634 }
2635 }
2636
2637 I.eraseFromParent();
2638 return true;
2639 }
2640
2641 if (!DstTy.isScalar())
2642 return false;
2643
2644 if (SrcSize > 32) {
2645 unsigned SubRegIdx = DstSize < 32
2646 ? static_cast<unsigned>(AMDGPU::sub0)
2647 : TRI.getSubRegFromChannel(Channel: 0, NumRegs: DstSize / 32);
2648 if (SubRegIdx == AMDGPU::NoSubRegister)
2649 return false;
2650
2651 // Deal with weird cases where the class only partially supports the subreg
2652 // index.
2653 const TargetRegisterClass *SrcWithSubRC
2654 = TRI.getSubClassWithSubReg(SrcRC, SubRegIdx);
2655 if (!SrcWithSubRC)
2656 return false;
2657
2658 if (SrcWithSubRC != SrcRC) {
2659 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcWithSubRC, MRI&: *MRI))
2660 return false;
2661 }
2662
2663 I.getOperand(i: 1).setSubReg(SubRegIdx);
2664 }
2665
2666 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
2667 return true;
2668}
2669
2670/// \returns true if a bitmask for \p Size bits will be an inline immediate.
2671static bool shouldUseAndMask(unsigned Size, unsigned &Mask) {
2672 Mask = maskTrailingOnes<unsigned>(N: Size);
2673 int SignedMask = static_cast<int>(Mask);
2674 return SignedMask >= -16 && SignedMask <= 64;
2675}
2676
2677// Like RegisterBankInfo::getRegBank, but don't assume vcc for s1.
2678const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank(
2679 Register Reg, const MachineRegisterInfo &MRI,
2680 const TargetRegisterInfo &TRI) const {
2681 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
2682 if (auto *RB = dyn_cast<const RegisterBank *>(Val: RegClassOrBank))
2683 return RB;
2684
2685 // Ignore the type, since we don't use vcc in artifacts.
2686 if (auto *RC = dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank))
2687 return &RBI.getRegBankFromRegClass(RC: *RC, LLT());
2688 return nullptr;
2689}
2690
2691bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
2692 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG;
2693 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg;
2694 const DebugLoc &DL = I.getDebugLoc();
2695 MachineBasicBlock &MBB = *I.getParent();
2696 const Register DstReg = I.getOperand(i: 0).getReg();
2697 const Register SrcReg = I.getOperand(i: 1).getReg();
2698
2699 const LLT DstTy = MRI->getType(Reg: DstReg);
2700 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2701 const unsigned SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG ?
2702 I.getOperand(i: 2).getImm() : SrcTy.getSizeInBits();
2703 const unsigned DstSize = DstTy.getSizeInBits();
2704 if (!DstTy.isScalar())
2705 return false;
2706
2707 // Artifact casts should never use vcc.
2708 const RegisterBank *SrcBank = getArtifactRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2709
2710 // FIXME: This should probably be illegal and split earlier.
2711 if (I.getOpcode() == AMDGPU::G_ANYEXT) {
2712 if (DstSize <= 32)
2713 return selectCOPY(I);
2714
2715 const TargetRegisterClass *SrcRC =
2716 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcBank);
2717 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2718 const TargetRegisterClass *DstRC =
2719 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
2720
2721 Register UndefReg = MRI->createVirtualRegister(RegClass: SrcRC);
2722 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2723 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2724 .addReg(RegNo: SrcReg)
2725 .addImm(Val: AMDGPU::sub0)
2726 .addReg(RegNo: UndefReg)
2727 .addImm(Val: AMDGPU::sub1);
2728 I.eraseFromParent();
2729
2730 return RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) &&
2731 RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI);
2732 }
2733
2734 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) {
2735 // 64-bit should have been split up in RegBankSelect
2736
2737 // Try to use an and with a mask if it will save code size.
2738 unsigned Mask;
2739 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2740 MachineInstr *ExtI =
2741 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: DstReg)
2742 .addImm(Val: Mask)
2743 .addReg(RegNo: SrcReg);
2744 I.eraseFromParent();
2745 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2746 return true;
2747 }
2748
2749 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2750 MachineInstr *ExtI =
2751 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE), DestReg: DstReg)
2752 .addReg(RegNo: SrcReg)
2753 .addImm(Val: 0) // Offset
2754 .addImm(Val: SrcSize); // Width
2755 I.eraseFromParent();
2756 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2757 return true;
2758 }
2759
2760 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
2761 const TargetRegisterClass &SrcRC = InReg && DstSize > 32 ?
2762 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass;
2763 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: SrcRC, MRI&: *MRI))
2764 return false;
2765
2766 if (Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) {
2767 const unsigned SextOpc = SrcSize == 8 ?
2768 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16;
2769 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: SextOpc), DestReg: DstReg)
2770 .addReg(RegNo: SrcReg);
2771 I.eraseFromParent();
2772 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2773 }
2774
2775 // Using a single 32-bit SALU to calculate the high half is smaller than
2776 // S_BFE with a literal constant operand.
2777 if (DstSize > 32 && SrcSize == 32) {
2778 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2779 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2780 if (Signed) {
2781 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ASHR_I32), DestReg: HiReg)
2782 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2783 .addImm(Val: 31)
2784 .setOperandDead(3); // Dead scc
2785 } else {
2786 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg)
2787 .addImm(Val: 0);
2788 }
2789 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2790 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2791 .addImm(Val: AMDGPU::sub0)
2792 .addReg(RegNo: HiReg)
2793 .addImm(Val: AMDGPU::sub1);
2794 I.eraseFromParent();
2795 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass,
2796 MRI&: *MRI);
2797 }
2798
2799 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
2800 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2801
2802 // Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width.
2803 if (DstSize > 32 && (SrcSize <= 32 || InReg)) {
2804 // We need a 64-bit register source, but the high bits don't matter.
2805 Register ExtReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
2806 Register UndefReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2807 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2808
2809 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2810 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: ExtReg)
2811 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2812 .addImm(Val: AMDGPU::sub0)
2813 .addReg(RegNo: UndefReg)
2814 .addImm(Val: AMDGPU::sub1);
2815
2816 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE64), DestReg: DstReg)
2817 .addReg(RegNo: ExtReg)
2818 .addImm(Val: SrcSize << 16);
2819
2820 I.eraseFromParent();
2821 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI);
2822 }
2823
2824 unsigned Mask;
2825 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2826 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: DstReg)
2827 .addReg(RegNo: SrcReg)
2828 .addImm(Val: Mask)
2829 .setOperandDead(3); // Dead scc
2830 } else {
2831 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE32), DestReg: DstReg)
2832 .addReg(RegNo: SrcReg)
2833 .addImm(Val: SrcSize << 16);
2834 }
2835
2836 I.eraseFromParent();
2837 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2838 }
2839
2840 return false;
2841}
2842
2843static Register stripCopy(Register Reg, MachineRegisterInfo &MRI) {
2844 return getDefSrcRegIgnoringCopies(Reg, MRI)->Reg;
2845}
2846
2847static Register stripBitCast(Register Reg, MachineRegisterInfo &MRI) {
2848 Register BitcastSrc;
2849 if (mi_match(R: Reg, MRI, P: m_GBitcast(Src: m_Reg(R&: BitcastSrc))))
2850 Reg = BitcastSrc;
2851 return Reg;
2852}
2853
2854static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In,
2855 Register &Out) {
2856 // When unmerging a register that is composed of 2 x 16-bit values allow to
2857 // use an extract hi instruction for the upper 16 bits. We only need to check
2858 // the size of `In` as all defs are guaranteed to be the same type for
2859 // GUnmerge.
2860 GUnmerge *Unmerge;
2861 if (mi_match(R: In, MRI, P: m_GUnmerge(Inst&: Unmerge))) {
2862 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(i: 1).getReg() == In &&
2863 MRI.getType(Reg: In).getSizeInBits() == 16) {
2864 Out = stripBitCast(Reg: Unmerge->getSourceReg(), MRI);
2865 return true;
2866 }
2867 }
2868
2869 Register Trunc;
2870 if (!mi_match(R: In, MRI, P: m_GTrunc(Src: m_Reg(R&: Trunc))))
2871 return false;
2872
2873 Register LShlSrc;
2874 Register Cst;
2875 if (mi_match(R: Trunc, MRI, P: m_GLShr(L: m_Reg(R&: LShlSrc), R: m_Reg(R&: Cst)))) {
2876 Cst = stripCopy(Reg: Cst, MRI);
2877 if (mi_match(R: Cst, MRI, P: m_SpecificICst(RequestedValue: 16))) {
2878 Out = stripBitCast(Reg: LShlSrc, MRI);
2879 return true;
2880 }
2881 }
2882
2883 ArrayRef<int> Mask;
2884 Register Src1;
2885 if (!mi_match(R: Trunc, MRI, P: m_GShuffleVector(Src1: m_Reg(R&: Src1), Src2: m_Reg(), Mask)))
2886 return false;
2887
2888 assert(MRI.getType(Src1) == LLT::fixed_vector(2, 16));
2889 assert(Mask.size() == 2);
2890
2891 if (Mask[0] == 1 && Mask[1] <= 1) {
2892 Out = Trunc;
2893 return true;
2894 }
2895
2896 return false;
2897}
2898
2899static bool isExtractLoElt(MachineRegisterInfo &MRI, Register In,
2900 Register &Out) {
2901 // There could be a bitcast between the extraction and its use.
2902 In = stripBitCast(Reg: In, MRI);
2903
2904 // The first def of a 2 x 16-bit unmerge is the low half of its source.
2905 if (auto *Unmerge = dyn_cast<GUnmerge>(Val: MRI.getVRegDef(Reg: In))) {
2906 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(i: 0).getReg() == In &&
2907 MRI.getType(Reg: In).getSizeInBits() == 16) {
2908 Out = stripBitCast(Reg: Unmerge->getSourceReg(), MRI);
2909 return true;
2910 }
2911 }
2912
2913 // A truncation from 32 to 16 bits keeps the low half in place.
2914 Register Trunc;
2915 if (mi_match(R: In, MRI, P: m_GTrunc(Src: m_Reg(R&: Trunc))) &&
2916 MRI.getType(Reg: Trunc).getSizeInBits() == 32) {
2917 Out = stripBitCast(Reg: Trunc, MRI);
2918 return true;
2919 }
2920
2921 return false;
2922}
2923
2924bool AMDGPUInstructionSelector::selectG_FPEXT(MachineInstr &I) const {
2925 if (!Subtarget->hasSALUFloatInsts())
2926 return false;
2927
2928 Register Dst = I.getOperand(i: 0).getReg();
2929 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2930 if (DstRB->getID() != AMDGPU::SGPRRegBankID)
2931 return false;
2932
2933 Register Src = I.getOperand(i: 1).getReg();
2934
2935 if (MRI->getType(Reg: Dst) == LLT::scalar(SizeInBits: 32) &&
2936 MRI->getType(Reg: Src) == LLT::scalar(SizeInBits: 16)) {
2937 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
2938 MachineBasicBlock *BB = I.getParent();
2939 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_CVT_HI_F32_F16), DestReg: Dst)
2940 .addUse(RegNo: Src);
2941 I.eraseFromParent();
2942 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2943 }
2944 }
2945
2946 return false;
2947}
2948
2949bool AMDGPUInstructionSelector::selectG_FNEG(MachineInstr &MI) const {
2950 // Only manually handle the f64 SGPR case.
2951 //
2952 // FIXME: This is a workaround for 2.5 different tablegen problems. Because
2953 // the bit ops theoretically have a second result due to the implicit def of
2954 // SCC, the GlobalISelEmitter is overly conservative and rejects it. Fixing
2955 // that is easy by disabling the check. The result works, but uses a
2956 // nonsensical sreg32orlds_and_sreg_1 regclass.
2957 //
2958 // The DAG emitter is more problematic, and incorrectly adds both S_XOR_B32 to
2959 // the variadic REG_SEQUENCE operands.
2960
2961 Register Dst = MI.getOperand(i: 0).getReg();
2962 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2963 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
2964 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
2965 return false;
2966
2967 Register Src = MI.getOperand(i: 1).getReg();
2968 MachineInstr *Fabs = getOpcodeDef(Opcode: TargetOpcode::G_FABS, Reg: Src, MRI: *MRI);
2969 if (Fabs)
2970 Src = Fabs->getOperand(i: 1).getReg();
2971
2972 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
2973 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
2974 return false;
2975
2976 MachineBasicBlock *BB = MI.getParent();
2977 const DebugLoc &DL = MI.getDebugLoc();
2978 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2979 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2980 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2981 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2982
2983 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
2984 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
2985 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
2986 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
2987 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
2988 .addImm(Val: 0x80000000);
2989
2990 // Set or toggle sign bit.
2991 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32;
2992 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: OpReg)
2993 .addReg(RegNo: HiReg)
2994 .addReg(RegNo: ConstReg)
2995 .setOperandDead(3); // Dead scc
2996 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
2997 .addReg(RegNo: LoReg)
2998 .addImm(Val: AMDGPU::sub0)
2999 .addReg(RegNo: OpReg)
3000 .addImm(Val: AMDGPU::sub1);
3001 MI.eraseFromParent();
3002 return true;
3003}
3004
3005// FIXME: This is a workaround for the same tablegen problems as G_FNEG
3006bool AMDGPUInstructionSelector::selectG_FABS(MachineInstr &MI) const {
3007 Register Dst = MI.getOperand(i: 0).getReg();
3008 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
3009 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
3010 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
3011 return false;
3012
3013 Register Src = MI.getOperand(i: 1).getReg();
3014 MachineBasicBlock *BB = MI.getParent();
3015 const DebugLoc &DL = MI.getDebugLoc();
3016 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3017 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3018 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3019 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3020
3021 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
3022 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
3023 return false;
3024
3025 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3026 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
3027 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3028 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
3029 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
3030 .addImm(Val: 0x7fffffff);
3031
3032 // Clear sign bit.
3033 // TODO: Should this used S_BITSET0_*?
3034 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: OpReg)
3035 .addReg(RegNo: HiReg)
3036 .addReg(RegNo: ConstReg)
3037 .setOperandDead(3); // Dead scc
3038 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
3039 .addReg(RegNo: LoReg)
3040 .addImm(Val: AMDGPU::sub0)
3041 .addReg(RegNo: OpReg)
3042 .addImm(Val: AMDGPU::sub1);
3043
3044 MI.eraseFromParent();
3045 return true;
3046}
3047
3048static bool isConstant(const MachineInstr &MI) {
3049 return MI.getOpcode() == TargetOpcode::G_CONSTANT;
3050}
3051
3052void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
3053 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const {
3054
3055 unsigned OpNo = Load.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1;
3056 const MachineInstr *PtrMI =
3057 MRI.getUniqueVRegDef(Reg: Load.getOperand(i: OpNo).getReg());
3058
3059 assert(PtrMI);
3060
3061 if (PtrMI->getOpcode() != TargetOpcode::G_PTR_ADD)
3062 return;
3063
3064 GEPInfo GEPInfo;
3065
3066 for (unsigned i = 1; i != 3; ++i) {
3067 const MachineOperand &GEPOp = PtrMI->getOperand(i);
3068 const MachineInstr *OpDef = MRI.getUniqueVRegDef(Reg: GEPOp.getReg());
3069 assert(OpDef);
3070 if (i == 2 && isConstant(MI: *OpDef)) {
3071 // TODO: Could handle constant base + variable offset, but a combine
3072 // probably should have commuted it.
3073 assert(GEPInfo.Imm == 0);
3074 GEPInfo.Imm = OpDef->getOperand(i: 1).getCImm()->getSExtValue();
3075 continue;
3076 }
3077 const RegisterBank *OpBank = RBI.getRegBank(Reg: GEPOp.getReg(), MRI, TRI);
3078 if (OpBank->getID() == AMDGPU::SGPRRegBankID)
3079 GEPInfo.SgprParts.push_back(Elt: GEPOp.getReg());
3080 else
3081 GEPInfo.VgprParts.push_back(Elt: GEPOp.getReg());
3082 }
3083
3084 AddrInfo.push_back(Elt: GEPInfo);
3085 getAddrModeInfo(Load: *PtrMI, MRI, AddrInfo);
3086}
3087
3088bool AMDGPUInstructionSelector::isSGPR(Register Reg) const {
3089 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID;
3090}
3091
3092bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const {
3093 if (!MI.hasOneMemOperand())
3094 return false;
3095
3096 const MachineMemOperand *MMO = *MI.memoperands_begin();
3097 const Value *Ptr = MMO->getValue();
3098
3099 // UndefValue means this is a load of a kernel input. These are uniform.
3100 // Sometimes LDS instructions have constant pointers.
3101 // If Ptr is null, then that means this mem operand contains a
3102 // PseudoSourceValue like GOT.
3103 if (!Ptr || isa<UndefValue, Argument, Constant, GlobalValue>(Val: Ptr))
3104 return true;
3105
3106 if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
3107 return true;
3108
3109 if (MI.getOpcode() == AMDGPU::G_PREFETCH)
3110 return RBI.getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI: *MRI, TRI)->getID() ==
3111 AMDGPU::SGPRRegBankID;
3112
3113 const Instruction *I = dyn_cast<Instruction>(Val: Ptr);
3114 return I && I->getMetadata(Kind: "amdgpu.uniform");
3115}
3116
3117bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const {
3118 for (const GEPInfo &GEPInfo : AddrInfo) {
3119 if (!GEPInfo.VgprParts.empty())
3120 return true;
3121 }
3122 return false;
3123}
3124
3125void AMDGPUInstructionSelector::initM0(MachineInstr &I) const {
3126 const LLT PtrTy = MRI->getType(Reg: I.getOperand(i: 1).getReg());
3127 unsigned AS = PtrTy.getAddressSpace();
3128 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) &&
3129 STI.ldsRequiresM0Init()) {
3130 MachineBasicBlock *BB = I.getParent();
3131
3132 // If DS instructions require M0 initialization, insert it before selecting.
3133 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
3134 .addImm(Val: -1);
3135 }
3136}
3137
3138bool AMDGPUInstructionSelector::selectG_LOAD_STORE_ATOMICRMW(
3139 MachineInstr &I) const {
3140 initM0(I);
3141 return selectImpl(I, CoverageInfo&: *CoverageInfo);
3142}
3143
3144static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI) {
3145 if (Reg.isPhysical())
3146 return false;
3147
3148 MachineInstr &MI = *MRI.getUniqueVRegDef(Reg);
3149 const unsigned Opcode = MI.getOpcode();
3150
3151 if (Opcode == AMDGPU::COPY)
3152 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI);
3153
3154 if (Opcode == AMDGPU::G_AND || Opcode == AMDGPU::G_OR ||
3155 Opcode == AMDGPU::G_XOR)
3156 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI) &&
3157 isVCmpResult(Reg: MI.getOperand(i: 2).getReg(), MRI);
3158
3159 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI))
3160 return GI->is(ID: Intrinsic::amdgcn_class);
3161
3162 return Opcode == AMDGPU::G_ICMP || Opcode == AMDGPU::G_FCMP;
3163}
3164
3165bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
3166 MachineBasicBlock *BB = I.getParent();
3167 MachineOperand &CondOp = I.getOperand(i: 0);
3168 Register CondReg = CondOp.getReg();
3169 const DebugLoc &DL = I.getDebugLoc();
3170
3171 unsigned BrOpcode;
3172 Register CondPhysReg;
3173 const TargetRegisterClass *ConstrainRC;
3174
3175 // In SelectionDAG, we inspect the IR block for uniformity metadata to decide
3176 // whether the branch is uniform when selecting the instruction. In
3177 // GlobalISel, we should push that decision into RegBankSelect. Assume for now
3178 // RegBankSelect knows what it's doing if the branch condition is scc, even
3179 // though it currently does not.
3180 if (!isVCC(Reg: CondReg, MRI: *MRI)) {
3181 if (MRI->getType(Reg: CondReg) != LLT::scalar(SizeInBits: 32))
3182 return false;
3183
3184 CondPhysReg = AMDGPU::SCC;
3185 BrOpcode = AMDGPU::S_CBRANCH_SCC1;
3186 ConstrainRC = &AMDGPU::SReg_32RegClass;
3187 } else {
3188 // FIXME: Should scc->vcc copies and with exec?
3189
3190 // Unless the value of CondReg is a result of a V_CMP* instruction then we
3191 // need to insert an and with exec.
3192 if (!isVCmpResult(Reg: CondReg, MRI&: *MRI)) {
3193 const bool Is64 = STI.isWave64();
3194 const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
3195 const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO;
3196
3197 Register TmpReg = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
3198 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: TmpReg)
3199 .addReg(RegNo: CondReg)
3200 .addReg(RegNo: Exec)
3201 .setOperandDead(3); // Dead scc
3202 CondReg = TmpReg;
3203 }
3204
3205 CondPhysReg = TRI.getVCC();
3206 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ;
3207 ConstrainRC = TRI.getBoolRC();
3208 }
3209
3210 if (!MRI->getRegClassOrNull(Reg: CondReg))
3211 MRI->setRegClass(Reg: CondReg, RC: ConstrainRC);
3212
3213 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CondPhysReg)
3214 .addReg(RegNo: CondReg);
3215 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: BrOpcode))
3216 .addMBB(MBB: I.getOperand(i: 1).getMBB());
3217
3218 I.eraseFromParent();
3219 return true;
3220}
3221
3222bool AMDGPUInstructionSelector::selectG_GLOBAL_VALUE(
3223 MachineInstr &I) const {
3224 Register DstReg = I.getOperand(i: 0).getReg();
3225 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3226 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3227 I.setDesc(TII.get(Opcode: IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32));
3228 if (IsVGPR)
3229 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
3230
3231 return RBI.constrainGenericRegister(
3232 Reg: DstReg, RC: IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, MRI&: *MRI);
3233}
3234
3235bool AMDGPUInstructionSelector::selectG_PTRMASK(MachineInstr &I) const {
3236 Register DstReg = I.getOperand(i: 0).getReg();
3237 Register SrcReg = I.getOperand(i: 1).getReg();
3238 Register MaskReg = I.getOperand(i: 2).getReg();
3239 LLT Ty = MRI->getType(Reg: DstReg);
3240 LLT MaskTy = MRI->getType(Reg: MaskReg);
3241 MachineBasicBlock *BB = I.getParent();
3242 const DebugLoc &DL = I.getDebugLoc();
3243
3244 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3245 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3246 const RegisterBank *MaskRB = RBI.getRegBank(Reg: MaskReg, MRI: *MRI, TRI);
3247 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3248 if (DstRB != SrcRB) // Should only happen for hand written MIR.
3249 return false;
3250
3251 // Try to avoid emitting a bit operation when we only need to touch half of
3252 // the 64-bit pointer.
3253 APInt MaskOnes = VT->getKnownOnes(R: MaskReg).zext(width: 64);
3254 const APInt MaskHi32 = APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32);
3255 const APInt MaskLo32 = APInt::getLowBitsSet(numBits: 64, loBitsSet: 32);
3256
3257 const bool CanCopyLow32 = (MaskOnes & MaskLo32) == MaskLo32;
3258 const bool CanCopyHi32 = (MaskOnes & MaskHi32) == MaskHi32;
3259
3260 if (!IsVGPR && Ty.getSizeInBits() == 64 &&
3261 !CanCopyLow32 && !CanCopyHi32) {
3262 auto MIB = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B64), DestReg: DstReg)
3263 .addReg(RegNo: SrcReg)
3264 .addReg(RegNo: MaskReg)
3265 .setOperandDead(3); // Dead scc
3266 I.eraseFromParent();
3267 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3268 return true;
3269 }
3270
3271 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
3272 const TargetRegisterClass &RegRC
3273 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
3274
3275 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *DstRB);
3276 const TargetRegisterClass *SrcRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *SrcRB);
3277 const TargetRegisterClass *MaskRC =
3278 TRI.getRegClassForTypeOnBank(Ty: MaskTy, Bank: *MaskRB);
3279
3280 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3281 !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3282 !RBI.constrainGenericRegister(Reg: MaskReg, RC: *MaskRC, MRI&: *MRI))
3283 return false;
3284
3285 if (Ty.getSizeInBits() == 32) {
3286 assert(MaskTy.getSizeInBits() == 32 &&
3287 "ptrmask should have been narrowed during legalize");
3288
3289 auto NewOp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: DstReg)
3290 .addReg(RegNo: SrcReg)
3291 .addReg(RegNo: MaskReg);
3292
3293 if (!IsVGPR)
3294 NewOp.setOperandDead(3); // Dead scc
3295 I.eraseFromParent();
3296 return true;
3297 }
3298
3299 Register HiReg = MRI->createVirtualRegister(RegClass: &RegRC);
3300 Register LoReg = MRI->createVirtualRegister(RegClass: &RegRC);
3301
3302 // Extract the subregisters from the source pointer.
3303 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3304 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
3305 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3306 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
3307
3308 Register MaskedLo, MaskedHi;
3309
3310 if (CanCopyLow32) {
3311 // If all the bits in the low half are 1, we only need a copy for it.
3312 MaskedLo = LoReg;
3313 } else {
3314 // Extract the mask subregister and apply the and.
3315 Register MaskLo = MRI->createVirtualRegister(RegClass: &RegRC);
3316 MaskedLo = MRI->createVirtualRegister(RegClass: &RegRC);
3317
3318 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskLo)
3319 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub0);
3320 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedLo)
3321 .addReg(RegNo: LoReg)
3322 .addReg(RegNo: MaskLo);
3323 }
3324
3325 if (CanCopyHi32) {
3326 // If all the bits in the high half are 1, we only need a copy for it.
3327 MaskedHi = HiReg;
3328 } else {
3329 Register MaskHi = MRI->createVirtualRegister(RegClass: &RegRC);
3330 MaskedHi = MRI->createVirtualRegister(RegClass: &RegRC);
3331
3332 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskHi)
3333 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub1);
3334 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedHi)
3335 .addReg(RegNo: HiReg)
3336 .addReg(RegNo: MaskHi);
3337 }
3338
3339 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
3340 .addReg(RegNo: MaskedLo)
3341 .addImm(Val: AMDGPU::sub0)
3342 .addReg(RegNo: MaskedHi)
3343 .addImm(Val: AMDGPU::sub1);
3344 I.eraseFromParent();
3345 return true;
3346}
3347
3348/// Return the register to use for the index value, and the subregister to use
3349/// for the indirectly accessed register.
3350static std::pair<Register, unsigned>
3351computeIndirectRegIndex(MachineRegisterInfo &MRI, const SIRegisterInfo &TRI,
3352 const TargetRegisterClass *SuperRC, Register IdxReg,
3353 unsigned EltSize, GISelValueTracking &ValueTracking) {
3354 Register IdxBaseReg;
3355 int Offset;
3356
3357 std::tie(args&: IdxBaseReg, args&: Offset) =
3358 AMDGPU::getBaseWithConstantOffset(MRI, Reg: IdxReg, ValueTracking: &ValueTracking);
3359 if (IdxBaseReg == AMDGPU::NoRegister) {
3360 // This will happen if the index is a known constant. This should ordinarily
3361 // be legalized out, but handle it as a register just in case.
3362 assert(Offset == 0);
3363 IdxBaseReg = IdxReg;
3364 }
3365
3366 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SuperRC, EltSize);
3367
3368 // Skip out of bounds offsets, or else we would end up using an undefined
3369 // register.
3370 if (static_cast<unsigned>(Offset) >= SubRegs.size())
3371 return std::pair(IdxReg, SubRegs[0]);
3372 return std::pair(IdxBaseReg, SubRegs[Offset]);
3373}
3374
3375bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
3376 MachineInstr &MI) const {
3377 Register DstReg = MI.getOperand(i: 0).getReg();
3378 Register SrcReg = MI.getOperand(i: 1).getReg();
3379 Register IdxReg = MI.getOperand(i: 2).getReg();
3380
3381 LLT DstTy = MRI->getType(Reg: DstReg);
3382 LLT SrcTy = MRI->getType(Reg: SrcReg);
3383
3384 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3385 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3386 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3387
3388 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3389 // into a waterfall loop.
3390 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3391 return false;
3392
3393 const TargetRegisterClass *SrcRC =
3394 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcRB);
3395 const TargetRegisterClass *DstRC =
3396 TRI.getRegClassForTypeOnBank(Ty: DstTy, Bank: *DstRB);
3397 if (!SrcRC || !DstRC)
3398 return false;
3399 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3400 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3401 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3402 return false;
3403
3404 MachineBasicBlock *BB = MI.getParent();
3405 const DebugLoc &DL = MI.getDebugLoc();
3406 const bool Is64 = DstTy.getSizeInBits() == 64;
3407
3408 unsigned SubReg;
3409 std::tie(args&: IdxReg, args&: SubReg) = computeIndirectRegIndex(
3410 MRI&: *MRI, TRI, SuperRC: SrcRC, IdxReg, EltSize: DstTy.getSizeInBits() / 8, ValueTracking&: *VT);
3411
3412 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) {
3413 if (DstTy.getSizeInBits() != 32 && !Is64)
3414 return false;
3415
3416 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3417 .addReg(RegNo: IdxReg);
3418
3419 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32;
3420 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
3421 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3422 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3423 MI.eraseFromParent();
3424 return true;
3425 }
3426
3427 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32)
3428 return false;
3429
3430 if (!STI.useVGPRIndexMode()) {
3431 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3432 .addReg(RegNo: IdxReg);
3433 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: DstReg)
3434 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3435 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3436 MI.eraseFromParent();
3437 return true;
3438 }
3439
3440 const MCInstrDesc &GPRIDXDesc =
3441 TII.getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *SrcRC), IsIndirectSrc: true);
3442 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3443 .addReg(RegNo: SrcReg)
3444 .addReg(RegNo: IdxReg)
3445 .addImm(Val: SubReg);
3446
3447 MI.eraseFromParent();
3448 return true;
3449}
3450
3451// TODO: Fold insert_vector_elt (extract_vector_elt) into movrelsd
3452bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT(
3453 MachineInstr &MI) const {
3454 Register DstReg = MI.getOperand(i: 0).getReg();
3455 Register VecReg = MI.getOperand(i: 1).getReg();
3456 Register ValReg = MI.getOperand(i: 2).getReg();
3457 Register IdxReg = MI.getOperand(i: 3).getReg();
3458
3459 LLT VecTy = MRI->getType(Reg: DstReg);
3460 LLT ValTy = MRI->getType(Reg: ValReg);
3461 unsigned VecSize = VecTy.getSizeInBits();
3462 unsigned ValSize = ValTy.getSizeInBits();
3463
3464 const RegisterBank *VecRB = RBI.getRegBank(Reg: VecReg, MRI: *MRI, TRI);
3465 const RegisterBank *ValRB = RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
3466 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3467
3468 assert(VecTy.getElementType() == ValTy);
3469
3470 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3471 // into a waterfall loop.
3472 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3473 return false;
3474
3475 const TargetRegisterClass *VecRC =
3476 TRI.getRegClassForTypeOnBank(Ty: VecTy, Bank: *VecRB);
3477 const TargetRegisterClass *ValRC =
3478 TRI.getRegClassForTypeOnBank(Ty: ValTy, Bank: *ValRB);
3479
3480 if (!RBI.constrainGenericRegister(Reg: VecReg, RC: *VecRC, MRI&: *MRI) ||
3481 !RBI.constrainGenericRegister(Reg: DstReg, RC: *VecRC, MRI&: *MRI) ||
3482 !RBI.constrainGenericRegister(Reg: ValReg, RC: *ValRC, MRI&: *MRI) ||
3483 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3484 return false;
3485
3486 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32)
3487 return false;
3488
3489 unsigned SubReg;
3490 std::tie(args&: IdxReg, args&: SubReg) =
3491 computeIndirectRegIndex(MRI&: *MRI, TRI, SuperRC: VecRC, IdxReg, EltSize: ValSize / 8, ValueTracking&: *VT);
3492
3493 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID &&
3494 STI.useVGPRIndexMode();
3495
3496 MachineBasicBlock *BB = MI.getParent();
3497 const DebugLoc &DL = MI.getDebugLoc();
3498
3499 if (!IndexMode) {
3500 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3501 .addReg(RegNo: IdxReg);
3502
3503 const MCInstrDesc &RegWriteOp = TII.getIndirectRegWriteMovRelPseudo(
3504 VecSize, EltSize: ValSize, IsSGPR: VecRB->getID() == AMDGPU::SGPRRegBankID);
3505 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: RegWriteOp, DestReg: DstReg)
3506 .addReg(RegNo: VecReg)
3507 .addReg(RegNo: ValReg)
3508 .addImm(Val: SubReg);
3509 MI.eraseFromParent();
3510 return true;
3511 }
3512
3513 const MCInstrDesc &GPRIDXDesc =
3514 TII.getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
3515 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3516 .addReg(RegNo: VecReg)
3517 .addReg(RegNo: ValReg)
3518 .addReg(RegNo: IdxReg)
3519 .addImm(Val: SubReg);
3520
3521 MI.eraseFromParent();
3522 return true;
3523}
3524
3525static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
3526 switch (Intr) {
3527 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
3528 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
3529 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
3530 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
3531 case Intrinsic::amdgcn_load_async_to_lds:
3532 case Intrinsic::amdgcn_global_load_async_lds:
3533 return true;
3534 }
3535 return false;
3536}
3537
3538bool AMDGPUInstructionSelector::selectBufferLoadLds(MachineInstr &MI) const {
3539 if (!Subtarget->hasVMemToLDSLoad())
3540 return false;
3541 unsigned Opc;
3542 unsigned Size = MI.getOperand(i: 3).getImm();
3543 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3544
3545 // The struct intrinsic variants add one additional operand over raw.
3546 const bool HasVIndex = MI.getNumOperands() == 9;
3547 Register VIndex;
3548 int OpOffset = 0;
3549 if (HasVIndex) {
3550 VIndex = MI.getOperand(i: 4).getReg();
3551 OpOffset = 1;
3552 }
3553
3554 Register VOffset = MI.getOperand(i: 4 + OpOffset).getReg();
3555 std::optional<ValueAndVReg> MaybeVOffset =
3556 getIConstantVRegValWithLookThrough(VReg: VOffset, MRI: *MRI);
3557 const bool HasVOffset = !MaybeVOffset || MaybeVOffset->Value.getZExtValue();
3558
3559 switch (Size) {
3560 default:
3561 return false;
3562 case 1:
3563 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
3564 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
3565 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
3566 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
3567 break;
3568 case 2:
3569 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
3570 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
3571 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
3572 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
3573 break;
3574 case 4:
3575 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
3576 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
3577 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
3578 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
3579 break;
3580 case 12:
3581 if (!Subtarget->hasLDSLoadB96_B128())
3582 return false;
3583
3584 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
3585 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
3586 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
3587 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
3588 break;
3589 case 16:
3590 if (!Subtarget->hasLDSLoadB96_B128())
3591 return false;
3592
3593 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
3594 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
3595 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
3596 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
3597 break;
3598 }
3599
3600 MachineBasicBlock *MBB = MI.getParent();
3601 const DebugLoc &DL = MI.getDebugLoc();
3602 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3603 .add(MO: MI.getOperand(i: 2));
3604
3605 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc));
3606
3607 if (HasVIndex && HasVOffset) {
3608 Register IdxReg = MRI->createVirtualRegister(RegClass: TRI.getVGPR64Class());
3609 BuildMI(BB&: *MBB, I: &*MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: IdxReg)
3610 .addReg(RegNo: VIndex)
3611 .addImm(Val: AMDGPU::sub0)
3612 .addReg(RegNo: VOffset)
3613 .addImm(Val: AMDGPU::sub1);
3614
3615 MIB.addReg(RegNo: IdxReg);
3616 } else if (HasVIndex) {
3617 MIB.addReg(RegNo: VIndex);
3618 } else if (HasVOffset) {
3619 MIB.addReg(RegNo: VOffset);
3620 }
3621
3622 MIB.add(MO: MI.getOperand(i: 1)); // rsrc
3623 MIB.add(MO: MI.getOperand(i: 5 + OpOffset)); // soffset
3624 MIB.add(MO: MI.getOperand(i: 6 + OpOffset)); // imm offset
3625 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
3626 unsigned Aux = MI.getOperand(i: 7 + OpOffset).getImm();
3627 MIB.addImm(Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL
3628 : AMDGPU::CPol::ALL_pregfx12)); // cpol
3629 MIB.addImm(
3630 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
3631 ? 1
3632 : 0); // swz
3633 MIB.addImm(Val: isAsyncLDSDMA(Intr: IntrinsicID));
3634
3635 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3636 // Don't set the offset value here because the pointer points to the base of
3637 // the buffer.
3638 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3639
3640 MachinePointerInfo StorePtrI = LoadPtrI;
3641 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3642 AddressSpace: AMDGPUAS::BUFFER_RESOURCE));
3643 LoadPtrI.AddrSpace = AMDGPUAS::BUFFER_RESOURCE;
3644 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3645
3646 auto F = LoadMMO->getFlags() &
3647 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3648 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3649 Size, BaseAlignment: LoadMMO->getBaseAlign());
3650
3651 MachineMemOperand *StoreMMO =
3652 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3653 Size: sizeof(int32_t), BaseAlignment: LoadMMO->getBaseAlign());
3654
3655 MIB.setMemRefs({LoadMMO, StoreMMO});
3656
3657 MI.eraseFromParent();
3658 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3659 return true;
3660}
3661
3662/// Match a zero extend from a 32-bit value to 64-bits.
3663Register AMDGPUInstructionSelector::matchZeroExtendFromS32(Register Reg) const {
3664 Register ZExtSrc;
3665 if (mi_match(R: Reg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrc))))
3666 return MRI->getType(Reg: ZExtSrc) == LLT::scalar(SizeInBits: 32) ? ZExtSrc : Register();
3667
3668 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 0)
3669 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3670 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3671 return Register();
3672
3673 assert(Def->getNumOperands() == 3 &&
3674 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3675 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_ZeroInt())) {
3676 return Def->getOperand(i: 1).getReg();
3677 }
3678
3679 return Register();
3680}
3681
3682/// Match a sign extend from a 32-bit value to 64-bits.
3683Register AMDGPUInstructionSelector::matchSignExtendFromS32(Register Reg) const {
3684 Register SExtSrc;
3685 if (mi_match(R: Reg, MRI: *MRI, P: m_GSExt(Src: m_Reg(R&: SExtSrc))))
3686 return MRI->getType(Reg: SExtSrc) == LLT::scalar(SizeInBits: 32) ? SExtSrc : Register();
3687
3688 // Match legalized form %sext = G_MERGE_VALUES (s32 %x), G_ASHR((S32 %x, 31))
3689 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3690 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3691 return Register();
3692
3693 assert(Def->getNumOperands() == 3 &&
3694 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3695 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI,
3696 P: m_GAShr(L: m_SpecificReg(RequestedReg: Def->getOperand(i: 1).getReg()),
3697 R: m_SpecificICst(RequestedValue: 31))))
3698 return Def->getOperand(i: 1).getReg();
3699
3700 Register ZextSrc = matchZeroExtendFromS32(Reg);
3701 if (ZextSrc && VT->signBitIsZero(Op: ZextSrc))
3702 return ZextSrc;
3703
3704 return Register();
3705}
3706
3707/// Match a zero extend from a 32-bit value to 64-bits, or \p Reg itself if it
3708/// is 32-bit.
3709Register
3710AMDGPUInstructionSelector::matchZeroExtendFromS32OrS32(Register Reg) const {
3711 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3712 : matchZeroExtendFromS32(Reg);
3713}
3714
3715/// Match a sign extend from a 32-bit value to 64-bits, or \p Reg itself if it
3716/// is 32-bit.
3717Register
3718AMDGPUInstructionSelector::matchSignExtendFromS32OrS32(Register Reg) const {
3719 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3720 : matchSignExtendFromS32(Reg);
3721}
3722
3723Register
3724AMDGPUInstructionSelector::matchExtendFromS32OrS32(Register Reg,
3725 bool IsSigned) const {
3726 if (IsSigned)
3727 return matchSignExtendFromS32OrS32(Reg);
3728
3729 return matchZeroExtendFromS32OrS32(Reg);
3730}
3731
3732Register AMDGPUInstructionSelector::matchAnyExtendFromS32(Register Reg) const {
3733 Register AnyExtSrc;
3734 if (mi_match(R: Reg, MRI: *MRI, P: m_GAnyExt(Src: m_Reg(R&: AnyExtSrc))))
3735 return MRI->getType(Reg: AnyExtSrc) == LLT::scalar(SizeInBits: 32) ? AnyExtSrc : Register();
3736
3737 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 G_IMPLICIT_DEF)
3738 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3739 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3740 return Register();
3741
3742 assert(Def->getNumOperands() == 3 &&
3743 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3744
3745 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_GImplicitDef()))
3746 return Def->getOperand(i: 1).getReg();
3747
3748 return Register();
3749}
3750
3751bool AMDGPUInstructionSelector::selectGlobalLoadLds(MachineInstr &MI) const{
3752 if (!Subtarget->hasVMemToLDSLoad())
3753 return false;
3754
3755 unsigned Opc;
3756 unsigned Size = MI.getOperand(i: 3).getImm();
3757 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3758
3759 switch (Size) {
3760 default:
3761 return false;
3762 case 1:
3763 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
3764 break;
3765 case 2:
3766 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
3767 break;
3768 case 4:
3769 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
3770 break;
3771 case 12:
3772 if (!Subtarget->hasLDSLoadB96_B128())
3773 return false;
3774 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
3775 break;
3776 case 16:
3777 if (!Subtarget->hasLDSLoadB96_B128())
3778 return false;
3779 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
3780 break;
3781 }
3782
3783 MachineBasicBlock *MBB = MI.getParent();
3784 const DebugLoc &DL = MI.getDebugLoc();
3785 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3786 .add(MO: MI.getOperand(i: 2));
3787
3788 Register Addr = MI.getOperand(i: 1).getReg();
3789 Register VOffset;
3790 // Try to split SAddr and VOffset. Global and LDS pointers share the same
3791 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
3792 if (!isSGPR(Reg: Addr)) {
3793 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
3794 if (isSGPR(Reg: AddrDef->Reg)) {
3795 Addr = AddrDef->Reg;
3796 } else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
3797 Register SAddr =
3798 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
3799 if (isSGPR(Reg: SAddr)) {
3800 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
3801 if (Register Off = matchZeroExtendFromS32(Reg: PtrBaseOffset)) {
3802 Addr = SAddr;
3803 VOffset = Off;
3804 }
3805 }
3806 }
3807 }
3808
3809 if (isSGPR(Reg: Addr)) {
3810 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
3811 if (!VOffset) {
3812 VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
3813 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
3814 .addImm(Val: 0);
3815 }
3816 }
3817
3818 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc))
3819 .addReg(RegNo: Addr);
3820
3821 if (isSGPR(Reg: Addr))
3822 MIB.addReg(RegNo: VOffset);
3823
3824 MIB.add(MO: MI.getOperand(i: 4)); // offset
3825
3826 unsigned Aux = MI.getOperand(i: 5).getImm();
3827 MIB.addImm(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS); // cpol
3828 MIB.addImm(Val: isAsyncLDSDMA(Intr: IntrinsicID));
3829
3830 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3831 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3832 LoadPtrI.Offset = MI.getOperand(i: 4).getImm();
3833 MachinePointerInfo StorePtrI = LoadPtrI;
3834 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3835 AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
3836 LoadPtrI.AddrSpace = AMDGPUAS::GLOBAL_ADDRESS;
3837 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3838 auto F = LoadMMO->getFlags() &
3839 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3840 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3841 Size, BaseAlignment: LoadMMO->getBaseAlign());
3842 MachineMemOperand *StoreMMO =
3843 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3844 Size: sizeof(int32_t), BaseAlignment: Align(4));
3845
3846 MIB.setMemRefs({LoadMMO, StoreMMO});
3847
3848 MI.eraseFromParent();
3849 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3850 return true;
3851}
3852
3853bool AMDGPUInstructionSelector::selectTensorLoadStore(MachineInstr &MI,
3854 Intrinsic::ID IID) const {
3855 bool IsLoad = IID == Intrinsic::amdgcn_tensor_load_to_lds;
3856 unsigned Opc =
3857 IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d4 : AMDGPU::TENSOR_STORE_FROM_LDS_d4;
3858 int NumGroups = 4;
3859
3860 // A lamda function to check whether an operand is a vector of all 0s.
3861 const auto isAllZeros = [&](MachineOperand &Opnd) {
3862 const MachineInstr *DefMI = MRI->getVRegDef(Reg: Opnd.getReg());
3863 if (!DefMI)
3864 return false;
3865 return llvm::isBuildVectorAllZeros(MI: *DefMI, MRI: *MRI, AllowUndef: true);
3866 };
3867
3868 // Use _D2 version if both group 2 and 3 are zero-initialized.
3869 if (isAllZeros(MI.getOperand(i: 3)) && isAllZeros(MI.getOperand(i: 4))) {
3870 NumGroups = 2;
3871 Opc = IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d2
3872 : AMDGPU::TENSOR_STORE_FROM_LDS_d2;
3873 }
3874
3875 // TODO: Handle the fifth group: MI.getOpetand(5), which is silently ignored
3876 // for now because all existing targets only support up to 4 groups.
3877 MachineBasicBlock *MBB = MI.getParent();
3878 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
3879 .add(MO: MI.getOperand(i: 1)) // D# group 0
3880 .add(MO: MI.getOperand(i: 2)); // D# group 1
3881
3882 if (NumGroups >= 4) { // Has at least 4 groups
3883 MIB.add(MO: MI.getOperand(i: 3)) // D# group 2
3884 .add(MO: MI.getOperand(i: 4)); // D# group 3
3885 }
3886
3887 MIB.addImm(Val: 0) // r128
3888 .add(MO: MI.getOperand(i: 6)); // cpol
3889
3890 MI.eraseFromParent();
3891 return true;
3892}
3893
3894bool AMDGPUInstructionSelector::selectBVHIntersectRayIntrinsic(
3895 MachineInstr &MI) const {
3896 unsigned OpcodeOpIdx =
3897 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY ? 1 : 3;
3898 MI.setDesc(TII.get(Opcode: MI.getOperand(i: OpcodeOpIdx).getImm()));
3899 MI.removeOperand(OpNo: OpcodeOpIdx);
3900 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
3901 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
3902 return true;
3903}
3904
3905// FIXME: This should be removed and let the patterns select. We just need the
3906// AGPR/VGPR combination versions.
3907bool AMDGPUInstructionSelector::selectSMFMACIntrin(MachineInstr &MI) const {
3908 unsigned Opc;
3909 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
3910 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
3911 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64;
3912 break;
3913 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
3914 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64;
3915 break;
3916 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
3917 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64;
3918 break;
3919 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
3920 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64;
3921 break;
3922 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
3923 Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64;
3924 break;
3925 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
3926 Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64;
3927 break;
3928 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
3929 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64;
3930 break;
3931 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
3932 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64;
3933 break;
3934 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
3935 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64;
3936 break;
3937 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
3938 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64;
3939 break;
3940 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
3941 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64;
3942 break;
3943 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
3944 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64;
3945 break;
3946 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
3947 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64;
3948 break;
3949 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
3950 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64;
3951 break;
3952 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
3953 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_F16_e64;
3954 break;
3955 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
3956 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_F16_e64;
3957 break;
3958 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
3959 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF16_e64;
3960 break;
3961 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
3962 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF16_e64;
3963 break;
3964 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
3965 Opc = AMDGPU::V_SMFMAC_I32_16X16X128_I8_e64;
3966 break;
3967 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
3968 Opc = AMDGPU::V_SMFMAC_I32_32X32X64_I8_e64;
3969 break;
3970 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
3971 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_BF8_e64;
3972 break;
3973 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
3974 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_FP8_e64;
3975 break;
3976 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
3977 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_BF8_e64;
3978 break;
3979 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
3980 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_FP8_e64;
3981 break;
3982 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
3983 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_BF8_e64;
3984 break;
3985 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
3986 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_FP8_e64;
3987 break;
3988 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
3989 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_BF8_e64;
3990 break;
3991 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
3992 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_FP8_e64;
3993 break;
3994 default:
3995 llvm_unreachable("unhandled smfmac intrinsic");
3996 }
3997
3998 auto VDst_In = MI.getOperand(i: 4);
3999
4000 MI.setDesc(TII.get(Opcode: Opc));
4001 MI.removeOperand(OpNo: 4); // VDst_In
4002 MI.removeOperand(OpNo: 1); // Intrinsic ID
4003 MI.addOperand(Op: VDst_In); // Readd VDst_In to the end
4004 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
4005 const MCInstrDesc &MCID = MI.getDesc();
4006 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
4007 MI.getOperand(i: 0).setIsEarlyClobber(true);
4008 }
4009 return true;
4010}
4011
4012bool AMDGPUInstructionSelector::selectPermlaneSwapIntrin(
4013 MachineInstr &MI, Intrinsic::ID IntrID) const {
4014 if (IntrID == Intrinsic::amdgcn_permlane16_swap &&
4015 !Subtarget->hasPermlane16Swap())
4016 return false;
4017 if (IntrID == Intrinsic::amdgcn_permlane32_swap &&
4018 !Subtarget->hasPermlane32Swap())
4019 return false;
4020
4021 unsigned Opcode = IntrID == Intrinsic::amdgcn_permlane16_swap
4022 ? AMDGPU::V_PERMLANE16_SWAP_B32_e64
4023 : AMDGPU::V_PERMLANE32_SWAP_B32_e64;
4024
4025 MI.removeOperand(OpNo: 2);
4026 MI.setDesc(TII.get(Opcode));
4027 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
4028
4029 MachineOperand &FI = MI.getOperand(i: 4);
4030 FI.setImm(FI.getImm() ? AMDGPU::DPP::DPP_FI_1 : AMDGPU::DPP::DPP_FI_0);
4031
4032 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
4033 return true;
4034}
4035
4036bool AMDGPUInstructionSelector::selectWaveAddress(MachineInstr &MI) const {
4037 Register DstReg = MI.getOperand(i: 0).getReg();
4038 Register SrcReg = MI.getOperand(i: 1).getReg();
4039 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4040 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4041 MachineBasicBlock *MBB = MI.getParent();
4042 const DebugLoc &DL = MI.getDebugLoc();
4043
4044 if (IsVALU) {
4045 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: DstReg)
4046 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4047 .addReg(RegNo: SrcReg);
4048 } else {
4049 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: DstReg)
4050 .addReg(RegNo: SrcReg)
4051 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4052 .setOperandDead(3); // Dead scc
4053 }
4054
4055 const TargetRegisterClass &RC =
4056 IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
4057 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
4058 return false;
4059
4060 MI.eraseFromParent();
4061 return true;
4062}
4063
4064bool AMDGPUInstructionSelector::selectWaveShuffleIntrin(
4065 MachineInstr &MI) const {
4066 assert(MI.getNumOperands() == 4);
4067 MachineBasicBlock *MBB = MI.getParent();
4068 const DebugLoc &DL = MI.getDebugLoc();
4069
4070 Register DstReg = MI.getOperand(i: 0).getReg();
4071 Register ValReg = MI.getOperand(i: 2).getReg();
4072 Register IdxReg = MI.getOperand(i: 3).getReg();
4073
4074 const LLT DstTy = MRI->getType(Reg: DstReg);
4075 unsigned DstSize = DstTy.getSizeInBits();
4076 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4077 const TargetRegisterClass *DstRC =
4078 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
4079
4080 if (DstTy != LLT::scalar(SizeInBits: 32))
4081 return false;
4082
4083 if (!Subtarget->supportsBPermute())
4084 return false;
4085
4086 // If we can bpermute across the whole wave, then just do that
4087 if (Subtarget->supportsWaveWideBPermute()) {
4088 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4089 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4090 .addImm(Val: 2)
4091 .addReg(RegNo: IdxReg);
4092
4093 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: DstReg)
4094 .addReg(RegNo: ShiftIdxReg)
4095 .addReg(RegNo: ValReg)
4096 .addImm(Val: 0);
4097 } else {
4098 // Otherwise, we need to make use of whole wave mode
4099 assert(Subtarget->isWave64());
4100
4101 // Set inactive lanes to poison
4102 Register UndefValReg =
4103 MRI->createVirtualRegister(RegClass: TRI.getRegClass(i: AMDGPU::SReg_32RegClassID));
4104 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefValReg);
4105
4106 Register UndefExecReg = MRI->createVirtualRegister(
4107 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4108 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExecReg);
4109
4110 Register PoisonValReg = MRI->createVirtualRegister(RegClass: DstRC);
4111 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonValReg)
4112 .addImm(Val: 0)
4113 .addReg(RegNo: ValReg)
4114 .addImm(Val: 0)
4115 .addReg(RegNo: UndefValReg)
4116 .addReg(RegNo: UndefExecReg);
4117
4118 // ds_bpermute requires index to be multiplied by 4
4119 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4120 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4121 .addImm(Val: 2)
4122 .addReg(RegNo: IdxReg);
4123
4124 Register PoisonIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4125 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonIdxReg)
4126 .addImm(Val: 0)
4127 .addReg(RegNo: ShiftIdxReg)
4128 .addImm(Val: 0)
4129 .addReg(RegNo: UndefValReg)
4130 .addReg(RegNo: UndefExecReg);
4131
4132 Register PoisonUnshiftedIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4133 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32),
4134 DestReg: PoisonUnshiftedIdxReg)
4135 .addImm(Val: 0)
4136 .addReg(RegNo: IdxReg)
4137 .addImm(Val: 0)
4138 .addReg(RegNo: UndefValReg)
4139 .addReg(RegNo: UndefExecReg);
4140
4141 // Get permutation of each half, then we'll select which one to use
4142 Register SameSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4143 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: SameSidePermReg)
4144 .addReg(RegNo: PoisonIdxReg)
4145 .addReg(RegNo: PoisonValReg)
4146 .addImm(Val: 0);
4147
4148 Register SwappedValReg = MRI->createVirtualRegister(RegClass: DstRC);
4149 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_PERMLANE64_B32), DestReg: SwappedValReg)
4150 .addReg(RegNo: PoisonValReg);
4151
4152 Register OppSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4153 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: OppSidePermReg)
4154 .addReg(RegNo: PoisonIdxReg)
4155 .addReg(RegNo: SwappedValReg)
4156 .addImm(Val: 0);
4157
4158 Register WWMSwapPermReg = MRI->createVirtualRegister(RegClass: DstRC);
4159 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::STRICT_WWM), DestReg: WWMSwapPermReg)
4160 .addReg(RegNo: OppSidePermReg);
4161
4162 // Select which side to take the permute from
4163 // We can get away with only using mbcnt_lo here since we're only
4164 // trying to detect which side of 32 each lane is on, and mbcnt_lo
4165 // returns 32 for lanes 32-63.
4166 Register ThreadIDReg = MRI->createVirtualRegister(RegClass: DstRC);
4167 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64), DestReg: ThreadIDReg)
4168 .addImm(Val: -1)
4169 .addImm(Val: 0);
4170
4171 Register XORReg = MRI->createVirtualRegister(RegClass: DstRC);
4172 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_XOR_B32_e64), DestReg: XORReg)
4173 .addReg(RegNo: ThreadIDReg)
4174 .addReg(RegNo: PoisonUnshiftedIdxReg);
4175
4176 Register ANDReg = MRI->createVirtualRegister(RegClass: DstRC);
4177 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: ANDReg)
4178 .addReg(RegNo: XORReg)
4179 .addImm(Val: 32);
4180
4181 Register CompareReg = MRI->createVirtualRegister(
4182 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4183 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CompareReg)
4184 .addReg(RegNo: ANDReg)
4185 .addImm(Val: 0);
4186
4187 // Finally do the selection
4188 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
4189 .addImm(Val: 0)
4190 .addReg(RegNo: WWMSwapPermReg)
4191 .addImm(Val: 0)
4192 .addReg(RegNo: SameSidePermReg)
4193 .addReg(RegNo: CompareReg);
4194 }
4195
4196 MI.eraseFromParent();
4197 return true;
4198}
4199
4200// Match BITOP3 operation and return a number of matched instructions plus
4201// truth table.
4202static std::pair<unsigned, uint8_t> BitOp3_Op(Register R,
4203 SmallVectorImpl<Register> &Src,
4204 const MachineRegisterInfo &MRI) {
4205 unsigned NumOpcodes = 0;
4206 uint8_t LHSBits, RHSBits;
4207
4208 auto getOperandBits = [&Src, R, &MRI](Register Op, uint8_t &Bits) -> bool {
4209 // Define truth table given Src0, Src1, Src2 bits permutations:
4210 // 0 0 0
4211 // 0 0 1
4212 // 0 1 0
4213 // 0 1 1
4214 // 1 0 0
4215 // 1 0 1
4216 // 1 1 0
4217 // 1 1 1
4218 const uint8_t SrcBits[3] = { 0xf0, 0xcc, 0xaa };
4219
4220 if (mi_match(R: Op, MRI, P: m_AllOnesInt())) {
4221 Bits = 0xff;
4222 return true;
4223 }
4224 if (mi_match(R: Op, MRI, P: m_ZeroInt())) {
4225 Bits = 0;
4226 return true;
4227 }
4228
4229 for (unsigned I = 0; I < Src.size(); ++I) {
4230 // Try to find existing reused operand
4231 if (Src[I] == Op) {
4232 Bits = SrcBits[I];
4233 return true;
4234 }
4235 // Try to replace parent operator
4236 if (Src[I] == R) {
4237 Bits = SrcBits[I];
4238 Src[I] = Op;
4239 return true;
4240 }
4241 }
4242
4243 if (Src.size() == 3) {
4244 // No room left for operands. Try one last time, there can be a 'not' of
4245 // one of our source operands. In this case we can compute the bits
4246 // without growing Src vector.
4247 Register LHS;
4248 if (mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: LHS)))) {
4249 LHS = getSrcRegIgnoringCopies(Reg: LHS, MRI);
4250 for (unsigned I = 0; I < Src.size(); ++I) {
4251 if (Src[I] == LHS) {
4252 Bits = ~SrcBits[I];
4253 return true;
4254 }
4255 }
4256 }
4257
4258 return false;
4259 }
4260
4261 Bits = SrcBits[Src.size()];
4262 Src.push_back(Elt: Op);
4263 return true;
4264 };
4265
4266 MachineInstr *MI = MRI.getVRegDef(Reg: R);
4267 switch (MI->getOpcode()) {
4268 case TargetOpcode::G_AND:
4269 case TargetOpcode::G_OR:
4270 case TargetOpcode::G_XOR: {
4271 Register LHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 1).getReg(), MRI);
4272 Register RHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 2).getReg(), MRI);
4273
4274 SmallVector<Register, 3> Backup(Src.begin(), Src.end());
4275 if (!getOperandBits(LHS, LHSBits) ||
4276 !getOperandBits(RHS, RHSBits)) {
4277 Src = std::move(Backup);
4278 return std::make_pair(x: 0, y: 0);
4279 }
4280
4281 // Recursion is naturally limited by the size of the operand vector.
4282 //
4283 // When LHS and RHS share a common sub-expression, one side's recursion
4284 // may decompose that sub-expression and replace the Src slot the other
4285 // side occupies with sub-operands via the "replace parent" path in
4286 // getOperandBits. The other side's cached bit-pattern then refers to a
4287 // slot whose contents changed, producing a wrong truth table.
4288 //
4289 // We detect this in three ways:
4290 // (A) If LHS recursed, its truth table is valid against the Src state
4291 // when LHS recursion completed (SrcAfterLHS). If RHS recursion
4292 // then mutates a Src slot that LHSBits depends on, LHSBits is
4293 // stale.
4294 // (B) If RHS did not recurse, RHSBits came from getOperandBits and
4295 // refers to a specific Src slot. If that slot's contents changed
4296 // (by either recursion), RHSBits is stale.
4297 // (C) Symmetrically for LHS if it did not recurse.
4298 SmallVector<Register, 3> SrcBeforeRecurse(Src.begin(), Src.end());
4299 uint8_t LHSBitsOrig = LHSBits;
4300 uint8_t RHSBitsOrig = RHSBits;
4301
4302 auto LHSOp = BitOp3_Op(R: LHS, Src, MRI);
4303 if (LHSOp.first) {
4304 NumOpcodes += LHSOp.first;
4305 LHSBits = LHSOp.second;
4306 }
4307
4308 SmallVector<Register, 3> SrcAfterLHS(Src.begin(), Src.end());
4309
4310 auto RHSOp = BitOp3_Op(R: RHS, Src, MRI);
4311 if (RHSOp.first) {
4312 NumOpcodes += RHSOp.first;
4313 RHSBits = RHSOp.second;
4314 }
4315
4316 // dependsOnSlot: true iff the truth table TT varies with slot Slot.
4317 auto dependsOnSlot = [](uint8_t TT, int Slot) -> bool {
4318 if (Slot < 0 || Slot > 2)
4319 return false;
4320 const uint8_t Masks[3] = {0x0f, 0x33, 0x55};
4321 const int Shifts[3] = {4, 2, 1};
4322 return ((TT ^ (TT >> Shifts[Slot])) & Masks[Slot]) != 0;
4323 };
4324
4325 // findSlot: locate the Src slot a getOperandBits result depends on,
4326 // including negated (NOT) patterns that getOperandBits resolves via
4327 // the ~SrcBits[I] shortcut.
4328 const uint8_t SrcBitsConst[3] = {0xf0, 0xcc, 0xaa};
4329 auto findSlot = [&](uint8_t Bits, Register Op,
4330 const SmallVectorImpl<Register> &S) -> int {
4331 Register NegatedInner;
4332 bool IsNegationOp = mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: NegatedInner)));
4333 if (IsNegationOp)
4334 NegatedInner = getSrcRegIgnoringCopies(Reg: NegatedInner, MRI);
4335 for (int I = 0; I < (int)S.size(); I++) {
4336 if (Bits == SrcBitsConst[I] && S[I] == Op)
4337 return I;
4338 if (IsNegationOp && Bits == (uint8_t)~SrcBitsConst[I] &&
4339 S[I] == NegatedInner)
4340 return I;
4341 }
4342 return -1;
4343 };
4344
4345 bool Stale = false;
4346
4347 // (A) LHS recursed: its truth table is against SrcAfterLHS.
4348 // Check if RHS recursion mutated a slot that LHSBits uses.
4349 if (LHSOp.first) {
4350 for (int I = 0; I < (int)SrcAfterLHS.size() && I < 3; I++) {
4351 if (I < (int)Src.size() && Src[I] != SrcAfterLHS[I] &&
4352 dependsOnSlot(LHSBits, I)) {
4353 Stale = true;
4354 break;
4355 }
4356 }
4357 }
4358
4359 // (B) RHS did not recurse: RHSBits from getOperandBits is against
4360 // SrcBeforeRecurse. Check if that slot was mutated since then.
4361 if (!Stale && !RHSOp.first) {
4362 int Slot = findSlot(RHSBitsOrig, RHS, SrcBeforeRecurse);
4363 if (Slot >= 0 &&
4364 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4365 Stale = true;
4366 }
4367
4368 // (C) LHS did not recurse: LHSBits from getOperandBits is against
4369 // SrcBeforeRecurse. Check if that slot was mutated since then.
4370 if (!Stale && !LHSOp.first) {
4371 int Slot = findSlot(LHSBitsOrig, LHS, SrcBeforeRecurse);
4372 if (Slot >= 0 &&
4373 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4374 Stale = true;
4375 }
4376
4377 if (Stale) {
4378 Src = std::move(SrcBeforeRecurse);
4379 LHSBits = LHSBitsOrig;
4380 RHSBits = RHSBitsOrig;
4381 NumOpcodes = 0;
4382 }
4383 break;
4384 }
4385 default:
4386 return std::make_pair(x: 0, y: 0);
4387 }
4388
4389 uint8_t TTbl;
4390 switch (MI->getOpcode()) {
4391 case TargetOpcode::G_AND:
4392 TTbl = LHSBits & RHSBits;
4393 break;
4394 case TargetOpcode::G_OR:
4395 TTbl = LHSBits | RHSBits;
4396 break;
4397 case TargetOpcode::G_XOR:
4398 TTbl = LHSBits ^ RHSBits;
4399 break;
4400 default:
4401 break;
4402 }
4403
4404 return std::make_pair(x: NumOpcodes + 1, y&: TTbl);
4405}
4406
4407bool AMDGPUInstructionSelector::selectBITOP3(MachineInstr &MI) const {
4408 if (!Subtarget->hasBitOp3Insts())
4409 return false;
4410
4411 Register DstReg = MI.getOperand(i: 0).getReg();
4412 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4413 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4414 if (!IsVALU)
4415 return false;
4416
4417 SmallVector<Register, 3> Src;
4418 uint8_t TTbl;
4419 unsigned NumOpcodes;
4420
4421 std::tie(args&: NumOpcodes, args&: TTbl) = BitOp3_Op(R: DstReg, Src, MRI: *MRI);
4422
4423 // Src.empty() case can happen if all operands are all zero or all ones.
4424 // Normally it shall be optimized out before reaching this.
4425 if (NumOpcodes < 2 || Src.empty())
4426 return false;
4427
4428 // RegBankSelect splits wider VALU logic ops and widens 1-bit ones, so only
4429 // 16 and 32 bit types reach here. Note that <2 x i16> is 32 bits wide.
4430 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
4431 assert((Size == 16 || Size == 32) && "unexpected VALU logic op size");
4432 const bool IsB32 = Size == 32;
4433 if (NumOpcodes == 2 && IsB32) {
4434 // Avoid using BITOP3 for OR3, XOR3, AND_OR. This is not faster but makes
4435 // asm more readable. This cannot be modeled with AddedComplexity because
4436 // selector does not know how many operations did we match.
4437 if (mi_match(MI, MRI: *MRI, P: m_GXor(L: m_GXor(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4438 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GOr(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4439 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GAnd(L: m_Reg(), R: m_Reg()), R: m_Reg())))
4440 return false;
4441 } else if (NumOpcodes < 4) {
4442 // For a uniform case threshold should be higher to account for moves
4443 // between VGPRs and SGPRs. It needs one operand in a VGPR, rest two can be
4444 // in SGPRs and a readtfirstlane after.
4445 return false;
4446 }
4447
4448 unsigned Opc = IsB32 ? AMDGPU::V_BITOP3_B32_e64 : AMDGPU::V_BITOP3_B16_e64;
4449 if (!IsB32 && STI.hasTrue16BitInsts())
4450 Opc = STI.useRealTrue16Insts() ? AMDGPU::V_BITOP3_B16_gfx1250_t16_e64
4451 : AMDGPU::V_BITOP3_B16_gfx1250_fake16_e64;
4452 unsigned CBL = STI.getConstantBusLimit(Opcode: Opc);
4453 MachineBasicBlock *MBB = MI.getParent();
4454 const DebugLoc &DL = MI.getDebugLoc();
4455
4456 for (unsigned I = 0; I < Src.size(); ++I) {
4457 const RegisterBank *RB = RBI.getRegBank(Reg: Src[I], MRI: *MRI, TRI);
4458 if (RB->getID() != AMDGPU::SGPRRegBankID)
4459 continue;
4460 if (CBL > 0) {
4461 --CBL;
4462 continue;
4463 }
4464 Register NewReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
4465 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: NewReg)
4466 .addReg(RegNo: Src[I]);
4467 Src[I] = NewReg;
4468 }
4469
4470 // Last operand can be ignored, turning a ternary operation into a binary.
4471 // For example: (~a & b & c) | (~a & b & ~c) -> (~a & b). We can replace
4472 // 'c' with 'a' here without changing the answer. In some pathological
4473 // cases it should be possible to get an operation with a single operand
4474 // too if optimizer would not catch it.
4475 while (Src.size() < 3)
4476 Src.push_back(Elt: Src[0]);
4477
4478 auto MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg);
4479 if (!IsB32)
4480 MIB.addImm(Val: 0); // src_mod0
4481 MIB.addReg(RegNo: Src[0]);
4482 if (!IsB32)
4483 MIB.addImm(Val: 0); // src_mod1
4484 MIB.addReg(RegNo: Src[1]);
4485 if (!IsB32)
4486 MIB.addImm(Val: 0); // src_mod2
4487 MIB.addReg(RegNo: Src[2])
4488 .addImm(Val: TTbl);
4489 if (!IsB32)
4490 MIB.addImm(Val: 0); // op_sel
4491
4492 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
4493 MI.eraseFromParent();
4494
4495 return true;
4496}
4497
4498bool AMDGPUInstructionSelector::selectWriteRegister(MachineInstr &MI) const {
4499 const MDString *RegStr =
4500 cast<MDString>(Val: MI.getOperand(i: 0).getMetadata()->getOperand(I: 0));
4501 Register SrcReg = MI.getOperand(i: 1).getReg();
4502 LLT Ty = MRI->getType(Reg: SrcReg);
4503
4504 Register PhysReg = Subtarget->getTargetLowering()->getRegisterByName(
4505 RegName: RegStr->getString().data(), VT: Ty, MF: *MF);
4506 if (!PhysReg) {
4507 const Function &Fn = MF->getFunction();
4508 Fn.getContext().diagnose(DI: DiagnosticInfoGenericWithLoc(
4509 "invalid register \"" + Twine(RegStr->getString()) +
4510 "\" for llvm.write_register",
4511 Fn, MI.getDebugLoc()));
4512 MI.eraseFromParent();
4513 return true;
4514 }
4515
4516 if (!RBI.constrainGenericRegister(
4517 Reg: SrcReg, RC: *TRI.getSGPRClassForBitWidth(BitWidth: Ty.getSizeInBits()), MRI&: *MRI))
4518 return false;
4519
4520 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhysReg)
4521 .addReg(RegNo: SrcReg);
4522 MI.eraseFromParent();
4523 return true;
4524}
4525
4526bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const {
4527 Register SrcReg = MI.getOperand(i: 0).getReg();
4528 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
4529 return false;
4530
4531 MachineInstr *DefMI = MRI->getVRegDef(Reg: SrcReg);
4532 Register SP =
4533 Subtarget->getTargetLowering()->getStackPointerRegisterToSaveRestore();
4534 Register WaveAddr = getWaveAddress(Def: DefMI);
4535 MachineBasicBlock *MBB = MI.getParent();
4536 const DebugLoc &DL = MI.getDebugLoc();
4537
4538 if (!WaveAddr) {
4539 WaveAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
4540 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: WaveAddr)
4541 .addReg(RegNo: SrcReg)
4542 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4543 .setOperandDead(3); // Dead scc
4544 }
4545
4546 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: SP)
4547 .addReg(RegNo: WaveAddr);
4548
4549 MI.eraseFromParent();
4550 return true;
4551}
4552
4553bool AMDGPUInstructionSelector::select(MachineInstr &I) {
4554
4555 if (!I.isPreISelOpcode()) {
4556 if (I.isCopy())
4557 return selectCOPY(I);
4558 return true;
4559 }
4560
4561 switch (I.getOpcode()) {
4562 case TargetOpcode::G_AND:
4563 case TargetOpcode::G_OR:
4564 case TargetOpcode::G_XOR:
4565 if (selectBITOP3(MI&: I))
4566 return true;
4567 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4568 return true;
4569 return selectG_AND_OR_XOR(I);
4570 case TargetOpcode::G_ADD:
4571 case TargetOpcode::G_SUB:
4572 case TargetOpcode::G_PTR_ADD:
4573 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4574 return true;
4575 return selectG_ADD_SUB(I);
4576 case TargetOpcode::G_UADDO:
4577 case TargetOpcode::G_USUBO:
4578 case TargetOpcode::G_UADDE:
4579 case TargetOpcode::G_USUBE:
4580 return selectG_UADDO_USUBO_UADDE_USUBE(I);
4581 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4582 case AMDGPU::G_AMDGPU_MAD_I64_I32:
4583 return selectG_AMDGPU_MAD_64_32(I);
4584 case TargetOpcode::G_INTTOPTR:
4585 case TargetOpcode::G_BITCAST:
4586 case TargetOpcode::G_PTRTOINT:
4587 case TargetOpcode::G_FREEZE:
4588 return selectCOPY(I);
4589 case TargetOpcode::G_FNEG:
4590 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4591 return true;
4592 return selectG_FNEG(MI&: I);
4593 case TargetOpcode::G_FABS:
4594 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4595 return true;
4596 return selectG_FABS(MI&: I);
4597 case TargetOpcode::G_EXTRACT:
4598 return selectG_EXTRACT(I);
4599 case TargetOpcode::G_MERGE_VALUES:
4600 case TargetOpcode::G_CONCAT_VECTORS:
4601 return selectG_MERGE_VALUES(MI&: I);
4602 case TargetOpcode::G_UNMERGE_VALUES:
4603 return selectG_UNMERGE_VALUES(MI&: I);
4604 case TargetOpcode::G_BUILD_VECTOR:
4605 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
4606 return selectG_BUILD_VECTOR(MI&: I);
4607 case TargetOpcode::G_IMPLICIT_DEF:
4608 return selectG_IMPLICIT_DEF(I);
4609 case TargetOpcode::G_INSERT:
4610 return selectG_INSERT(I);
4611 case TargetOpcode::G_INTRINSIC:
4612 case TargetOpcode::G_INTRINSIC_CONVERGENT:
4613 return selectG_INTRINSIC(I);
4614 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
4615 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
4616 return selectG_INTRINSIC_W_SIDE_EFFECTS(I);
4617 case TargetOpcode::G_ICMP:
4618 case TargetOpcode::G_FCMP:
4619 if (selectG_ICMP_or_FCMP(I))
4620 return true;
4621 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4622 case TargetOpcode::G_LOAD:
4623 case TargetOpcode::G_ZEXTLOAD:
4624 case TargetOpcode::G_SEXTLOAD:
4625 case TargetOpcode::G_STORE:
4626 case TargetOpcode::G_ATOMIC_CMPXCHG:
4627 case TargetOpcode::G_ATOMICRMW_XCHG:
4628 case TargetOpcode::G_ATOMICRMW_ADD:
4629 case TargetOpcode::G_ATOMICRMW_SUB:
4630 case TargetOpcode::G_ATOMICRMW_AND:
4631 case TargetOpcode::G_ATOMICRMW_OR:
4632 case TargetOpcode::G_ATOMICRMW_XOR:
4633 case TargetOpcode::G_ATOMICRMW_MIN:
4634 case TargetOpcode::G_ATOMICRMW_MAX:
4635 case TargetOpcode::G_ATOMICRMW_UMIN:
4636 case TargetOpcode::G_ATOMICRMW_UMAX:
4637 case TargetOpcode::G_ATOMICRMW_UINC_WRAP:
4638 case TargetOpcode::G_ATOMICRMW_UDEC_WRAP:
4639 case TargetOpcode::G_ATOMICRMW_USUB_COND:
4640 case TargetOpcode::G_ATOMICRMW_USUB_SAT:
4641 case TargetOpcode::G_ATOMICRMW_FADD:
4642 case TargetOpcode::G_ATOMICRMW_FMIN:
4643 case TargetOpcode::G_ATOMICRMW_FMAX:
4644 return selectG_LOAD_STORE_ATOMICRMW(I);
4645 case TargetOpcode::G_SELECT:
4646 return selectG_SELECT(I);
4647 case TargetOpcode::G_TRUNC:
4648 return selectG_TRUNC(I);
4649 case TargetOpcode::G_SEXT:
4650 case TargetOpcode::G_ZEXT:
4651 case TargetOpcode::G_ANYEXT:
4652 case TargetOpcode::G_SEXT_INREG:
4653 // This is a workaround. For extension from type i1, `selectImpl()` uses
4654 // patterns from TD file and generates an illegal VGPR to SGPR COPY as type
4655 // i1 can only be hold in a SGPR class.
4656 if (MRI->getType(Reg: I.getOperand(i: 1).getReg()) != LLT::scalar(SizeInBits: 1) &&
4657 selectImpl(I, CoverageInfo&: *CoverageInfo))
4658 return true;
4659 return selectG_SZA_EXT(I);
4660 case TargetOpcode::G_FPEXT:
4661 if (selectG_FPEXT(I))
4662 return true;
4663 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4664 case TargetOpcode::G_BRCOND:
4665 return selectG_BRCOND(I);
4666 case TargetOpcode::G_GLOBAL_VALUE:
4667 return selectG_GLOBAL_VALUE(I);
4668 case TargetOpcode::G_PTRMASK:
4669 return selectG_PTRMASK(I);
4670 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4671 return selectG_EXTRACT_VECTOR_ELT(MI&: I);
4672 case TargetOpcode::G_INSERT_VECTOR_ELT:
4673 return selectG_INSERT_VECTOR_ELT(MI&: I);
4674 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
4675 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
4676 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
4677 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
4678 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
4679 const AMDGPU::ImageDimIntrinsicInfo *Intr =
4680 AMDGPU::getImageDimIntrinsicInfo(Intr: AMDGPU::getIntrinsicID(I));
4681 assert(Intr && "not an image intrinsic with image pseudo");
4682 return selectImageIntrinsic(MI&: I, Intr);
4683 }
4684 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY:
4685 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
4686 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
4687 return selectBVHIntersectRayIntrinsic(MI&: I);
4688 case AMDGPU::G_SBFX:
4689 case AMDGPU::G_UBFX:
4690 return selectG_SBFX_UBFX(MI&: I);
4691 case AMDGPU::G_SI_CALL:
4692 I.setDesc(TII.get(Opcode: AMDGPU::SI_CALL));
4693 return true;
4694 case AMDGPU::G_AMDGPU_WAVE_ADDRESS:
4695 return selectWaveAddress(MI&: I);
4696 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN: {
4697 I.setDesc(TII.get(Opcode: AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN));
4698 return true;
4699 }
4700 case AMDGPU::G_STACKRESTORE:
4701 return selectStackRestore(MI&: I);
4702 case TargetOpcode::G_WRITE_REGISTER:
4703 return selectWriteRegister(MI&: I);
4704 case AMDGPU::G_PHI:
4705 return selectPHI(I);
4706 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
4707 return selectCOPY_SCC_VCC(I);
4708 case AMDGPU::G_AMDGPU_COPY_VCC_SCC:
4709 return selectCOPY_VCC_SCC(I);
4710 case AMDGPU::G_AMDGPU_READANYLANE:
4711 return selectReadAnyLane(I);
4712 case TargetOpcode::G_CONSTANT:
4713 case TargetOpcode::G_FCONSTANT:
4714 default:
4715 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4716 }
4717 return false;
4718}
4719
4720InstructionSelector::ComplexRendererFns
4721AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const {
4722 return {{
4723 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4724 }};
4725
4726}
4727
4728std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3ModsImpl(
4729 Register Src, bool IsCanonicalizing, bool AllowAbs, bool OpSel) const {
4730 unsigned Mods = 0;
4731 MachineInstr *MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4732
4733 if (MI->getOpcode() == AMDGPU::G_FNEG) {
4734 Src = MI->getOperand(i: 1).getReg();
4735 Mods |= SISrcMods::NEG;
4736 MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4737 } else if (MI->getOpcode() == AMDGPU::G_FSUB && IsCanonicalizing) {
4738 // Fold fsub [+-]0 into fneg. This may not have folded depending on the
4739 // denormal mode, but we're implicitly canonicalizing in a source operand.
4740 const ConstantFP *LHS =
4741 getConstantFPVRegVal(VReg: MI->getOperand(i: 1).getReg(), MRI: *MRI);
4742 if (LHS && LHS->isZero()) {
4743 Mods |= SISrcMods::NEG;
4744 Src = MI->getOperand(i: 2).getReg();
4745 }
4746 }
4747
4748 if (AllowAbs && MI->getOpcode() == AMDGPU::G_FABS) {
4749 Src = MI->getOperand(i: 1).getReg();
4750 Mods |= SISrcMods::ABS;
4751 }
4752
4753 if (OpSel)
4754 Mods |= SISrcMods::OP_SEL_0;
4755
4756 return std::pair(Src, Mods);
4757}
4758
4759std::pair<Register, unsigned>
4760AMDGPUInstructionSelector::selectVOP3PModsF32Impl(Register Src) const {
4761 unsigned Mods;
4762 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src);
4763 Mods |= SISrcMods::OP_SEL_1;
4764 return std::pair(Src, Mods);
4765}
4766
4767Register AMDGPUInstructionSelector::copyToVGPRIfSrcFolded(
4768 Register Src, unsigned Mods, MachineOperand Root, MachineInstr *InsertPt,
4769 bool ForceVGPR) const {
4770 if ((Mods != 0 || ForceVGPR) &&
4771 RBI.getRegBank(Reg: Src, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) {
4772
4773 // If we looked through copies to find source modifiers on an SGPR operand,
4774 // we now have an SGPR register source. To avoid potentially violating the
4775 // constant bus restriction, we need to insert a copy to a VGPR.
4776 Register VGPRSrc = MRI->cloneVirtualRegister(VReg: Root.getReg());
4777 BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
4778 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VGPRSrc)
4779 .addReg(RegNo: Src);
4780 Src = VGPRSrc;
4781 }
4782
4783 return Src;
4784}
4785
4786/// Some instructions must have 32-bit sources. With real true16 instructions a
4787/// 16-bit VALU value lives in a VGPR_16, which they cannot read, so place it in
4788/// the low half of a new 32-bit VGPR.
4789Register
4790AMDGPUInstructionSelector::widenSrcIfVGPR16(Register Src,
4791 MachineInstr *InsertPt) const {
4792 if (!Subtarget->useRealTrue16Insts() || MRI->getType(Reg: Src) != LLT::scalar(SizeInBits: 16))
4793 return Src;
4794
4795 const RegisterBank *SrcRB = RBI.getRegBank(Reg: Src, MRI: *MRI, TRI);
4796 if (!SrcRB || SrcRB->getID() != AMDGPU::VGPRRegBankID)
4797 return Src;
4798
4799 MachineIRBuilder B(*InsertPt);
4800
4801 Register ImpDefReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
4802 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF).addDef(RegNo: ImpDefReg);
4803
4804 Register DstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
4805 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
4806 .addDef(RegNo: DstReg)
4807 .addReg(RegNo: Src)
4808 .addImm(Val: AMDGPU::lo16)
4809 .addReg(RegNo: ImpDefReg)
4810 .addImm(Val: AMDGPU::hi16);
4811
4812 return DstReg;
4813}
4814
4815///
4816/// This will select either an SGPR or VGPR operand and will save us from
4817/// having to write an extra tablegen pattern.
4818InstructionSelector::ComplexRendererFns
4819AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const {
4820 return {{
4821 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4822 }};
4823}
4824
4825InstructionSelector::ComplexRendererFns
4826AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const {
4827 Register Src;
4828 unsigned Mods;
4829 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4830
4831 return {{
4832 [=](MachineInstrBuilder &MIB) {
4833 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4834 },
4835 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4836 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4837 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4838 }};
4839}
4840
4841InstructionSelector::ComplexRendererFns
4842AMDGPUInstructionSelector::selectVOP3BMods0(MachineOperand &Root) const {
4843 Register Src;
4844 unsigned Mods;
4845 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
4846 /*IsCanonicalizing=*/true,
4847 /*AllowAbs=*/false);
4848
4849 return {{
4850 [=](MachineInstrBuilder &MIB) {
4851 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4852 },
4853 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4854 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4855 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4856 }};
4857}
4858
4859InstructionSelector::ComplexRendererFns
4860AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const {
4861 return {{
4862 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); },
4863 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4864 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4865 }};
4866}
4867
4868InstructionSelector::ComplexRendererFns
4869AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const {
4870 Register Src;
4871 unsigned Mods;
4872 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4873
4874 return {{
4875 [=](MachineInstrBuilder &MIB) {
4876 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4877 },
4878 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4879 }};
4880}
4881
4882InstructionSelector::ComplexRendererFns
4883AMDGPUInstructionSelector::selectVOP3ModsNonCanonicalizing(
4884 MachineOperand &Root) const {
4885 Register Src;
4886 unsigned Mods;
4887 std::tie(args&: Src, args&: Mods) =
4888 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/false);
4889
4890 return {{
4891 [=](MachineInstrBuilder &MIB) {
4892 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4893 },
4894 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4895 }};
4896}
4897
4898InstructionSelector::ComplexRendererFns
4899AMDGPUInstructionSelector::selectVOP3BMods(MachineOperand &Root) const {
4900 Register Src;
4901 unsigned Mods;
4902 std::tie(args&: Src, args&: Mods) =
4903 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/true,
4904 /*AllowAbs=*/false);
4905
4906 return {{
4907 [=](MachineInstrBuilder &MIB) {
4908 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4909 },
4910 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4911 }};
4912}
4913
4914InstructionSelector::ComplexRendererFns
4915AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
4916 Register Reg = Root.getReg();
4917 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
4918 if (Def->getOpcode() == AMDGPU::G_FNEG || Def->getOpcode() == AMDGPU::G_FABS)
4919 return {};
4920 return {{
4921 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
4922 }};
4923}
4924
4925enum class SrcStatus {
4926 IS_SAME,
4927 IS_UPPER_HALF,
4928 IS_LOWER_HALF,
4929 IS_UPPER_HALF_NEG,
4930 // This means current op = [op_upper, op_lower] and src = -op_lower.
4931 IS_LOWER_HALF_NEG,
4932 IS_HI_NEG,
4933 // This means current op = [op_upper, op_lower] and src = [op_upper,
4934 // -op_lower].
4935 IS_LO_NEG,
4936 IS_BOTH_NEG,
4937 INVALID,
4938 NEG_START = IS_UPPER_HALF_NEG,
4939 NEG_END = IS_BOTH_NEG,
4940 HALF_START = IS_UPPER_HALF,
4941 HALF_END = IS_LOWER_HALF_NEG
4942};
4943/// Test if the MI is truncating to half, such as `%reg0:n = G_TRUNC %reg1:2n`
4944static bool isTruncHalf(const MachineInstr *MI,
4945 const MachineRegisterInfo &MRI) {
4946 if (MI->getOpcode() != AMDGPU::G_TRUNC)
4947 return false;
4948
4949 unsigned DstSize = MRI.getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
4950 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4951 return DstSize * 2 == SrcSize;
4952}
4953
4954/// Test if the MI is logic shift right with half bits,
4955/// such as `%reg0:2n =G_LSHR %reg1:2n, CONST(n)`
4956static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
4957 if (MI->getOpcode() != AMDGPU::G_LSHR)
4958 return false;
4959
4960 Register ShiftSrc;
4961 std::optional<ValueAndVReg> ShiftAmt;
4962 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
4963 P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
4964 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4965 unsigned Shift = ShiftAmt->Value.getZExtValue();
4966 return Shift * 2 == SrcSize;
4967 }
4968 return false;
4969}
4970
4971/// Test if the MI is shift left with half bits,
4972/// such as `%reg0:2n =G_SHL %reg1:2n, CONST(n)`
4973static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
4974 if (MI->getOpcode() != AMDGPU::G_SHL)
4975 return false;
4976
4977 Register ShiftSrc;
4978 std::optional<ValueAndVReg> ShiftAmt;
4979 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
4980 P: m_GShl(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
4981 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4982 unsigned Shift = ShiftAmt->Value.getZExtValue();
4983 return Shift * 2 == SrcSize;
4984 }
4985 return false;
4986}
4987
4988/// Test function, if the MI is `%reg0:n, %reg1:n = G_UNMERGE_VALUES %reg2:2n`
4989static bool isUnmergeHalf(const MachineInstr *MI,
4990 const MachineRegisterInfo &MRI) {
4991 if (MI->getOpcode() != AMDGPU::G_UNMERGE_VALUES)
4992 return false;
4993 return MI->getNumOperands() == 3 && MI->getOperand(i: 0).isDef() &&
4994 MI->getOperand(i: 1).isDef() && !MI->getOperand(i: 2).isDef();
4995}
4996
4997enum class TypeClass { VECTOR_OF_TWO, SCALAR, NONE_OF_LISTED };
4998
4999static TypeClass isVectorOfTwoOrScalar(Register Reg,
5000 const MachineRegisterInfo &MRI) {
5001 LLT OpTy = MRI.getType(Reg);
5002 if (OpTy.isScalar())
5003 return TypeClass::SCALAR;
5004 if (OpTy.isVector() && OpTy.getNumElements() == 2)
5005 return TypeClass::VECTOR_OF_TWO;
5006 return TypeClass::NONE_OF_LISTED;
5007}
5008
5009static SrcStatus getNegStatus(Register Reg, SrcStatus S,
5010 const MachineRegisterInfo &MRI) {
5011 TypeClass NegType = isVectorOfTwoOrScalar(Reg, MRI);
5012 if (NegType != TypeClass::VECTOR_OF_TWO && NegType != TypeClass::SCALAR)
5013 return SrcStatus::INVALID;
5014
5015 switch (S) {
5016 case SrcStatus::IS_SAME:
5017 if (NegType == TypeClass::VECTOR_OF_TWO) {
5018 // Vector of 2:
5019 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
5020 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
5021 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5022 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
5023 return SrcStatus::IS_BOTH_NEG;
5024 }
5025 if (NegType == TypeClass::SCALAR) {
5026 // Scalar:
5027 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
5028 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
5029 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5030 // [SrcHi, SrcLo] = [-OpHi, OpLo]
5031 return SrcStatus::IS_HI_NEG;
5032 }
5033 break;
5034 case SrcStatus::IS_HI_NEG:
5035 if (NegType == TypeClass::VECTOR_OF_TWO) {
5036 // Vector of 2:
5037 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5038 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
5039 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5040 // [SrcHi, SrcLo] = [-(-OpHi), -OpLo] = [OpHi, -OpLo]
5041 return SrcStatus::IS_LO_NEG;
5042 }
5043 if (NegType == TypeClass::SCALAR) {
5044 // Scalar:
5045 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5046 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
5047 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5048 // [SrcHi, SrcLo] = [-(-OpHi), OpLo] = [OpHi, OpLo]
5049 return SrcStatus::IS_SAME;
5050 }
5051 break;
5052 case SrcStatus::IS_LO_NEG:
5053 if (NegType == TypeClass::VECTOR_OF_TWO) {
5054 // Vector of 2:
5055 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5056 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5057 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5058 // [SrcHi, SrcLo] = [-OpHi, -(-OpLo)] = [-OpHi, OpLo]
5059 return SrcStatus::IS_HI_NEG;
5060 }
5061 if (NegType == TypeClass::SCALAR) {
5062 // Scalar:
5063 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5064 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5065 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5066 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
5067 return SrcStatus::IS_BOTH_NEG;
5068 }
5069 break;
5070 case SrcStatus::IS_BOTH_NEG:
5071 if (NegType == TypeClass::VECTOR_OF_TWO) {
5072 // Vector of 2:
5073 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5074 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5075 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5076 // [SrcHi, SrcLo] = [OpHi, OpLo]
5077 return SrcStatus::IS_SAME;
5078 }
5079 if (NegType == TypeClass::SCALAR) {
5080 // Scalar:
5081 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5082 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5083 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5084 // [SrcHi, SrcLo] = [OpHi, -OpLo]
5085 return SrcStatus::IS_LO_NEG;
5086 }
5087 break;
5088 case SrcStatus::IS_UPPER_HALF:
5089 // Vector of 2:
5090 // Src = CurrUpper
5091 // Curr = [CurrUpper, CurrLower]
5092 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5093 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5094 // Src = -OpUpper
5095 //
5096 // Scalar:
5097 // Src = CurrUpper
5098 // Curr = [CurrUpper, CurrLower]
5099 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5100 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5101 // Src = -OpUpper
5102 return SrcStatus::IS_UPPER_HALF_NEG;
5103 case SrcStatus::IS_LOWER_HALF:
5104 if (NegType == TypeClass::VECTOR_OF_TWO) {
5105 // Vector of 2:
5106 // Src = CurrLower
5107 // Curr = [CurrUpper, CurrLower]
5108 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5109 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5110 // Src = -OpLower
5111 return SrcStatus::IS_LOWER_HALF_NEG;
5112 }
5113 if (NegType == TypeClass::SCALAR) {
5114 // Scalar:
5115 // Src = CurrLower
5116 // Curr = [CurrUpper, CurrLower]
5117 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5118 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5119 // Src = OpLower
5120 return SrcStatus::IS_LOWER_HALF;
5121 }
5122 break;
5123 case SrcStatus::IS_UPPER_HALF_NEG:
5124 // Vector of 2:
5125 // Src = -CurrUpper
5126 // Curr = [CurrUpper, CurrLower]
5127 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5128 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5129 // Src = -(-OpUpper) = OpUpper
5130 //
5131 // Scalar:
5132 // Src = -CurrUpper
5133 // Curr = [CurrUpper, CurrLower]
5134 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5135 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5136 // Src = -(-OpUpper) = OpUpper
5137 return SrcStatus::IS_UPPER_HALF;
5138 case SrcStatus::IS_LOWER_HALF_NEG:
5139 if (NegType == TypeClass::VECTOR_OF_TWO) {
5140 // Vector of 2:
5141 // Src = -CurrLower
5142 // Curr = [CurrUpper, CurrLower]
5143 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5144 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5145 // Src = -(-OpLower) = OpLower
5146 return SrcStatus::IS_LOWER_HALF;
5147 }
5148 if (NegType == TypeClass::SCALAR) {
5149 // Scalar:
5150 // Src = -CurrLower
5151 // Curr = [CurrUpper, CurrLower]
5152 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5153 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5154 // Src = -OpLower
5155 return SrcStatus::IS_LOWER_HALF_NEG;
5156 }
5157 break;
5158 default:
5159 break;
5160 }
5161 llvm_unreachable("unexpected SrcStatus & NegType combination");
5162}
5163
5164static std::optional<std::pair<Register, SrcStatus>>
5165calcNextStatus(std::pair<Register, SrcStatus> Curr,
5166 const MachineRegisterInfo &MRI) {
5167 const MachineInstr *MI = MRI.getVRegDef(Reg: Curr.first);
5168
5169 unsigned Opc = MI->getOpcode();
5170
5171 // Handle general Opc cases.
5172 switch (Opc) {
5173 case AMDGPU::G_BITCAST:
5174 return std::optional<std::pair<Register, SrcStatus>>(
5175 {MI->getOperand(i: 1).getReg(), Curr.second});
5176 case AMDGPU::COPY:
5177 if (MI->getOperand(i: 1).getReg().isPhysical())
5178 return std::nullopt;
5179 return std::optional<std::pair<Register, SrcStatus>>(
5180 {MI->getOperand(i: 1).getReg(), Curr.second});
5181 case AMDGPU::G_FNEG: {
5182 SrcStatus Stat = getNegStatus(Reg: Curr.first, S: Curr.second, MRI);
5183 if (Stat == SrcStatus::INVALID)
5184 return std::nullopt;
5185 return std::optional<std::pair<Register, SrcStatus>>(
5186 {MI->getOperand(i: 1).getReg(), Stat});
5187 }
5188 default:
5189 break;
5190 }
5191
5192 // Calc next Stat from current Stat.
5193 switch (Curr.second) {
5194 case SrcStatus::IS_SAME:
5195 if (isTruncHalf(MI, MRI))
5196 return std::optional<std::pair<Register, SrcStatus>>(
5197 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5198 else if (isUnmergeHalf(MI, MRI)) {
5199 if (Curr.first == MI->getOperand(i: 0).getReg())
5200 return std::optional<std::pair<Register, SrcStatus>>(
5201 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF});
5202 return std::optional<std::pair<Register, SrcStatus>>(
5203 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF});
5204 }
5205 break;
5206 case SrcStatus::IS_HI_NEG:
5207 if (isTruncHalf(MI, MRI)) {
5208 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5209 // [CurrHi, CurrLo] = trunc [OpUpper, OpLower] = OpLower
5210 // = [OpLowerHi, OpLowerLo]
5211 // Src = [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5212 // = [-OpLowerHi, OpLowerLo]
5213 // = -OpLower
5214 return std::optional<std::pair<Register, SrcStatus>>(
5215 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5216 }
5217 if (isUnmergeHalf(MI, MRI)) {
5218 if (Curr.first == MI->getOperand(i: 0).getReg())
5219 return std::optional<std::pair<Register, SrcStatus>>(
5220 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5221 return std::optional<std::pair<Register, SrcStatus>>(
5222 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5223 }
5224 break;
5225 case SrcStatus::IS_UPPER_HALF:
5226 if (isShlHalf(MI, MRI))
5227 return std::optional<std::pair<Register, SrcStatus>>(
5228 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5229 break;
5230 case SrcStatus::IS_LOWER_HALF:
5231 if (isLshrHalf(MI, MRI))
5232 return std::optional<std::pair<Register, SrcStatus>>(
5233 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF});
5234 break;
5235 case SrcStatus::IS_UPPER_HALF_NEG:
5236 if (isShlHalf(MI, MRI))
5237 return std::optional<std::pair<Register, SrcStatus>>(
5238 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5239 break;
5240 case SrcStatus::IS_LOWER_HALF_NEG:
5241 if (isLshrHalf(MI, MRI))
5242 return std::optional<std::pair<Register, SrcStatus>>(
5243 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5244 break;
5245 default:
5246 break;
5247 }
5248 return std::nullopt;
5249}
5250
5251/// This is used to control valid status that current MI supports. For example,
5252/// non floating point intrinsic such as @llvm.amdgcn.sdot2 does not support NEG
5253/// bit on VOP3P.
5254/// The class can be further extended to recognize support on SEL, NEG, ABS bit
5255/// for different MI on different arch
5256class SearchOptions {
5257private:
5258 bool HasNeg = false;
5259 // Assume all complex pattern of VOP3P have opsel.
5260 bool HasOpsel = true;
5261
5262public:
5263 SearchOptions(Register Reg, const MachineRegisterInfo &MRI) {
5264 const MachineInstr *MI = MRI.getVRegDef(Reg);
5265 unsigned Opc = MI->getOpcode();
5266
5267 if (Opc == TargetOpcode::G_INTRINSIC) {
5268 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: *MI).getIntrinsicID();
5269 // Only float point intrinsic has neg & neg_hi bits.
5270 if (IntrinsicID == Intrinsic::amdgcn_fdot2)
5271 HasNeg = true;
5272 } else if (TargetInstrInfo::isGenericOpcode(Opc)) {
5273 // Keep same for generic op.
5274 HasNeg = true;
5275 }
5276 }
5277 bool checkOptions(SrcStatus Stat) const {
5278 if (!HasNeg &&
5279 (Stat >= SrcStatus::NEG_START && Stat <= SrcStatus::NEG_END)) {
5280 return false;
5281 }
5282 if (!HasOpsel &&
5283 (Stat >= SrcStatus::HALF_START && Stat <= SrcStatus::HALF_END)) {
5284 return false;
5285 }
5286 return true;
5287 }
5288};
5289
5290static SmallVector<std::pair<Register, SrcStatus>>
5291getSrcStats(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5292 int MaxDepth = 3) {
5293 int Depth = 0;
5294 auto Curr = calcNextStatus(Curr: {Reg, SrcStatus::IS_SAME}, MRI);
5295 SmallVector<std::pair<Register, SrcStatus>> Statlist;
5296
5297 while (Depth <= MaxDepth && Curr.has_value()) {
5298 Depth++;
5299 if (SO.checkOptions(Stat: Curr.value().second))
5300 Statlist.push_back(Elt: Curr.value());
5301 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5302 }
5303
5304 return Statlist;
5305}
5306
5307static std::pair<Register, SrcStatus>
5308getLastSameOrNeg(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5309 int MaxDepth = 3) {
5310 int Depth = 0;
5311 std::pair<Register, SrcStatus> LastSameOrNeg = {Reg, SrcStatus::IS_SAME};
5312 auto Curr = calcNextStatus(Curr: LastSameOrNeg, MRI);
5313
5314 while (Depth <= MaxDepth && Curr.has_value()) {
5315 Depth++;
5316 SrcStatus Stat = Curr.value().second;
5317 if (SO.checkOptions(Stat)) {
5318 if (Stat == SrcStatus::IS_SAME || Stat == SrcStatus::IS_HI_NEG ||
5319 Stat == SrcStatus::IS_LO_NEG || Stat == SrcStatus::IS_BOTH_NEG)
5320 LastSameOrNeg = Curr.value();
5321 }
5322 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5323 }
5324
5325 return LastSameOrNeg;
5326}
5327
5328static bool isSameBitWidth(Register Reg1, Register Reg2,
5329 const MachineRegisterInfo &MRI) {
5330 unsigned Width1 = MRI.getType(Reg: Reg1).getSizeInBits();
5331 unsigned Width2 = MRI.getType(Reg: Reg2).getSizeInBits();
5332 return Width1 == Width2;
5333}
5334
5335static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods) {
5336 // SrcStatus::IS_LOWER_HALF remain 0.
5337 if (HiStat == SrcStatus::IS_UPPER_HALF_NEG) {
5338 Mods ^= SISrcMods::NEG_HI;
5339 Mods |= SISrcMods::OP_SEL_1;
5340 } else if (HiStat == SrcStatus::IS_UPPER_HALF)
5341 Mods |= SISrcMods::OP_SEL_1;
5342 else if (HiStat == SrcStatus::IS_LOWER_HALF_NEG)
5343 Mods ^= SISrcMods::NEG_HI;
5344 else if (HiStat == SrcStatus::IS_HI_NEG)
5345 Mods ^= SISrcMods::NEG_HI;
5346
5347 if (LoStat == SrcStatus::IS_UPPER_HALF_NEG) {
5348 Mods ^= SISrcMods::NEG;
5349 Mods |= SISrcMods::OP_SEL_0;
5350 } else if (LoStat == SrcStatus::IS_UPPER_HALF)
5351 Mods |= SISrcMods::OP_SEL_0;
5352 else if (LoStat == SrcStatus::IS_LOWER_HALF_NEG)
5353 Mods |= SISrcMods::NEG;
5354 else if (LoStat == SrcStatus::IS_HI_NEG)
5355 Mods ^= SISrcMods::NEG;
5356
5357 return Mods;
5358}
5359
5360static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg,
5361 Register RootReg, const SIInstrInfo &TII,
5362 const MachineRegisterInfo &MRI) {
5363 auto IsHalfState = [](SrcStatus S) {
5364 return S == SrcStatus::IS_UPPER_HALF || S == SrcStatus::IS_UPPER_HALF_NEG ||
5365 S == SrcStatus::IS_LOWER_HALF || S == SrcStatus::IS_LOWER_HALF_NEG;
5366 };
5367 return isSameBitWidth(Reg1: NewReg, Reg2: RootReg, MRI) && IsHalfState(LoStat) &&
5368 IsHalfState(HiStat);
5369}
5370
5371std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
5372 Register RootReg, const MachineRegisterInfo &MRI, bool IsDOT) const {
5373 unsigned Mods = 0;
5374 // No modification if Root type is not form of <2 x Type>.
5375 if (isVectorOfTwoOrScalar(Reg: RootReg, MRI) != TypeClass::VECTOR_OF_TWO) {
5376 Mods |= SISrcMods::OP_SEL_1;
5377 return {RootReg, Mods};
5378 }
5379
5380 SearchOptions SO(RootReg, MRI);
5381
5382 std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(Reg: RootReg, MRI, SO);
5383
5384 if (Stat.second == SrcStatus::IS_BOTH_NEG)
5385 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
5386 else if (Stat.second == SrcStatus::IS_HI_NEG)
5387 Mods ^= SISrcMods::NEG_HI;
5388 else if (Stat.second == SrcStatus::IS_LO_NEG)
5389 Mods ^= SISrcMods::NEG;
5390
5391 // 64-bit VOP3P instructions do not have OPSEL or ABS. Bail on v2f64 or v2i64.
5392 // TODO: Select NEG_LO and NEG_HI modifiers from BUILD_VECTOR.
5393 if (MRI.getType(Reg: RootReg).getSizeInBits() == 128) {
5394 Mods |= SISrcMods::OP_SEL_1; // Just the default, OPSEL unsupported.
5395 return {Stat.first, Mods};
5396 }
5397
5398 GBuildVector *MI;
5399 if (!mi_match(R: Stat.first, MRI, P: m_GBuildVector(Inst&: MI)) ||
5400 MI->getNumOperands() != 3 || (IsDOT && Subtarget->hasDOTOpSelHazard())) {
5401 Mods |= SISrcMods::OP_SEL_1;
5402 return {Stat.first, Mods};
5403 }
5404
5405 SmallVector<std::pair<Register, SrcStatus>> StatlistHi =
5406 getSrcStats(Reg: MI->getOperand(i: 2).getReg(), MRI, SO);
5407
5408 if (StatlistHi.empty()) {
5409 Mods |= SISrcMods::OP_SEL_1;
5410 return {Stat.first, Mods};
5411 }
5412
5413 SmallVector<std::pair<Register, SrcStatus>> StatlistLo =
5414 getSrcStats(Reg: MI->getOperand(i: 1).getReg(), MRI, SO);
5415
5416 if (StatlistLo.empty()) {
5417 Mods |= SISrcMods::OP_SEL_1;
5418 return {Stat.first, Mods};
5419 }
5420
5421 for (int I = StatlistHi.size() - 1; I >= 0; I--) {
5422 for (int J = StatlistLo.size() - 1; J >= 0; J--) {
5423 if (StatlistHi[I].first == StatlistLo[J].first &&
5424 isValidToPack(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second,
5425 NewReg: StatlistHi[I].first, RootReg, TII, MRI))
5426 return {StatlistHi[I].first,
5427 updateMods(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second, Mods)};
5428 }
5429 }
5430 // Packed instructions do not have abs modifiers.
5431 Mods |= SISrcMods::OP_SEL_1;
5432
5433 return {Stat.first, Mods};
5434}
5435
5436// Removed unused function `getAllKindImm` to eliminate dead code.
5437
5438static bool checkRB(Register Reg, unsigned int RBNo,
5439 const AMDGPURegisterBankInfo &RBI,
5440 const MachineRegisterInfo &MRI,
5441 const TargetRegisterInfo &TRI) {
5442 const RegisterBank *RB = RBI.getRegBank(Reg, MRI, TRI);
5443 return RB->getID() == RBNo;
5444}
5445
5446// This function is used to get the correct register bank for returned reg.
5447// Assume:
5448// 1. VOP3P is always legal for VGPR.
5449// 2. RootOp's regbank is legal.
5450// Thus
5451// 1. If RootOp is SGPR, then NewOp can be SGPR or VGPR.
5452// 2. If RootOp is VGPR, then NewOp must be VGPR.
5453static Register
5454getLegalRegBank(Register NewReg, Register RootReg, MachineInstr &Use,
5455 const AMDGPURegisterBankInfo &RBI, MachineRegisterInfo &MRI,
5456 const TargetRegisterInfo &TRI, const SIInstrInfo &TII) {
5457 // RootOp can only be VGPR or SGPR (some hand written cases such as.
5458 // inst-select-ashr.v2s16.mir::ashr_v2s16_vs).
5459 if (checkRB(Reg: RootReg, RBNo: AMDGPU::SGPRRegBankID, RBI, MRI, TRI) ||
5460 checkRB(Reg: NewReg, RBNo: AMDGPU::VGPRRegBankID, RBI, MRI, TRI))
5461 return NewReg;
5462
5463 if (mi_match(R: RootReg, MRI, P: m_Copy(Src: m_SpecificReg(RequestedReg: NewReg)))) {
5464 // RootOp is VGPR, NewOp is not VGPR, but RootOp = COPY NewOp.
5465 return RootReg;
5466 }
5467
5468 Register DstReg = MRI.cloneVirtualRegister(VReg: RootReg);
5469 MachineInstrBuilder MIB = BuildMI(BB&: *Use.getParent(), I&: Use, MIMD: Use.getDebugLoc(),
5470 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
5471 .addReg(RegNo: NewReg);
5472
5473 // Only accept VGPR.
5474 return MIB->getOperand(i: 0).getReg();
5475}
5476
5477InstructionSelector::ComplexRendererFns
5478AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
5479 bool IsDOT) const {
5480 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5481 Register Reg;
5482 unsigned Mods;
5483 std::tie(args&: Reg, args&: Mods) = selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT);
5484
5485 Reg = getLegalRegBank(NewReg: Reg, RootReg: Root.getReg(), Use&: *Root.getParent(), RBI, MRI, TRI,
5486 TII);
5487 return {{
5488 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
5489 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5490 }};
5491}
5492
5493InstructionSelector::ComplexRendererFns
5494AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
5495
5496 return selectVOP3PRetHelper(Root);
5497}
5498
5499InstructionSelector::ComplexRendererFns
5500AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
5501
5502 return selectVOP3PRetHelper(Root, IsDOT: true);
5503}
5504
5505InstructionSelector::ComplexRendererFns
5506AMDGPUInstructionSelector::selectVOP3PNoModsDOT(MachineOperand &Root) const {
5507 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5508 Register Src;
5509 unsigned Mods;
5510 std::tie(args&: Src, args&: Mods) = selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT: true /*IsDOT*/);
5511 if (Mods != SISrcMods::OP_SEL_1)
5512 return {};
5513
5514 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5515}
5516
5517InstructionSelector::ComplexRendererFns
5518AMDGPUInstructionSelector::selectVOP3PModsF32(MachineOperand &Root) const {
5519 Register Src;
5520 unsigned Mods;
5521 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5522
5523 return {{
5524 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5525 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5526 }};
5527}
5528
5529InstructionSelector::ComplexRendererFns
5530AMDGPUInstructionSelector::selectVOP3PNoModsF32(MachineOperand &Root) const {
5531 Register Src;
5532 unsigned Mods;
5533 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5534 if (Mods != SISrcMods::OP_SEL_1)
5535 return {};
5536
5537 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5538}
5539
5540InstructionSelector::ComplexRendererFns
5541AMDGPUInstructionSelector::selectWMMAOpSelVOP3PMods(
5542 MachineOperand &Root) const {
5543 assert((Root.isImm() && (Root.getImm() == -1 || Root.getImm() == 0)) &&
5544 "expected i1 value");
5545 unsigned Mods = SISrcMods::OP_SEL_1;
5546 if (Root.getImm() != 0)
5547 Mods |= SISrcMods::OP_SEL_0;
5548
5549 return {{
5550 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5551 }};
5552}
5553
5554static Register buildRegSequence(SmallVectorImpl<Register> &Elts,
5555 MachineInstr *InsertPt,
5556 MachineRegisterInfo &MRI) {
5557 const TargetRegisterClass *DstRegClass;
5558 switch (Elts.size()) {
5559 case 8:
5560 DstRegClass = &AMDGPU::VReg_256RegClass;
5561 break;
5562 case 4:
5563 DstRegClass = &AMDGPU::VReg_128RegClass;
5564 break;
5565 case 2:
5566 DstRegClass = &AMDGPU::VReg_64RegClass;
5567 break;
5568 default:
5569 llvm_unreachable("unhandled Reg sequence size");
5570 }
5571
5572 MachineIRBuilder B(*InsertPt);
5573 auto MIB = B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
5574 .addDef(RegNo: MRI.createVirtualRegister(RegClass: DstRegClass));
5575 for (unsigned i = 0; i < Elts.size(); ++i) {
5576 MIB.addReg(RegNo: Elts[i]);
5577 MIB.addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: i));
5578 }
5579 return MIB->getOperand(i: 0).getReg();
5580}
5581
5582static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods,
5583 SmallVectorImpl<Register> &Elts, Register &Src,
5584 MachineInstr *InsertPt,
5585 MachineRegisterInfo &MRI) {
5586 if (ModOpcode == TargetOpcode::G_FNEG) {
5587 Mods |= SISrcMods::NEG;
5588 // Check if all elements also have abs modifier
5589 SmallVector<Register, 8> NegAbsElts;
5590 for (auto El : Elts) {
5591 Register FabsSrc;
5592 if (!mi_match(R: El, MRI, P: m_GFabs(Src: m_Reg(R&: FabsSrc))))
5593 break;
5594 NegAbsElts.push_back(Elt: FabsSrc);
5595 }
5596 if (Elts.size() != NegAbsElts.size()) {
5597 // Neg
5598 Src = buildRegSequence(Elts, InsertPt, MRI);
5599 } else {
5600 // Neg and Abs
5601 Mods |= SISrcMods::NEG_HI;
5602 Src = buildRegSequence(Elts&: NegAbsElts, InsertPt, MRI);
5603 }
5604 } else {
5605 assert(ModOpcode == TargetOpcode::G_FABS);
5606 // Abs
5607 Mods |= SISrcMods::NEG_HI;
5608 Src = buildRegSequence(Elts, InsertPt, MRI);
5609 }
5610}
5611
5612InstructionSelector::ComplexRendererFns
5613AMDGPUInstructionSelector::selectWMMAModsF32NegAbs(MachineOperand &Root) const {
5614 Register Src = Root.getReg();
5615 unsigned Mods = SISrcMods::OP_SEL_1;
5616 SmallVector<Register, 8> EltsF32;
5617
5618 GBuildVector *BV;
5619 if (mi_match(R: Src, MRI: *MRI, P: m_GBuildVector(Inst&: BV))) {
5620 assert(BV->getNumSources() > 0);
5621 // Based on first element decide which mod we match, neg or abs
5622 MachineInstr *ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: 0));
5623 unsigned ModOpcode = (ElF32->getOpcode() == AMDGPU::G_FNEG)
5624 ? AMDGPU::G_FNEG
5625 : AMDGPU::G_FABS;
5626 for (unsigned i = 0; i < BV->getNumSources(); ++i) {
5627 ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: i));
5628 if (ElF32->getOpcode() != ModOpcode)
5629 break;
5630 EltsF32.push_back(Elt: ElF32->getOperand(i: 1).getReg());
5631 }
5632
5633 // All elements had ModOpcode modifier
5634 if (BV->getNumSources() == EltsF32.size()) {
5635 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsF32, Src, InsertPt: Root.getParent(),
5636 MRI&: *MRI);
5637 }
5638 }
5639
5640 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5641 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5642}
5643
5644InstructionSelector::ComplexRendererFns
5645AMDGPUInstructionSelector::selectWMMAModsF16Neg(MachineOperand &Root) const {
5646 Register Src = Root.getReg();
5647 unsigned Mods = SISrcMods::OP_SEL_1;
5648 SmallVector<Register, 8> EltsV2F16;
5649
5650 GConcatVectors *CV;
5651 if (mi_match(R: Src, MRI: *MRI, P: m_GConcatVectors(Inst&: CV))) {
5652 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5653 Register FNegSrc;
5654 if (!mi_match(R: CV->getSourceReg(I: i), MRI: *MRI, P: m_GFNeg(Src: m_Reg(R&: FNegSrc))))
5655 break;
5656 EltsV2F16.push_back(Elt: FNegSrc);
5657 }
5658
5659 // All elements had ModOpcode modifier
5660 if (CV->getNumSources() == EltsV2F16.size()) {
5661 Mods |= SISrcMods::NEG;
5662 Mods |= SISrcMods::NEG_HI;
5663 Src = buildRegSequence(Elts&: EltsV2F16, InsertPt: Root.getParent(), MRI&: *MRI);
5664 }
5665 }
5666
5667 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5668 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5669}
5670
5671InstructionSelector::ComplexRendererFns
5672AMDGPUInstructionSelector::selectWMMAModsF16NegAbs(MachineOperand &Root) const {
5673 Register Src = Root.getReg();
5674 unsigned Mods = SISrcMods::OP_SEL_1;
5675 SmallVector<Register, 8> EltsV2F16;
5676
5677 GConcatVectors *CV;
5678 if (mi_match(R: Src, MRI: *MRI, P: m_GConcatVectors(Inst&: CV))) {
5679 assert(CV->getNumSources() > 0);
5680 MachineInstr *ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: 0));
5681 // Based on first element decide which mod we match, neg or abs
5682 unsigned ModOpcode = (ElV2F16->getOpcode() == AMDGPU::G_FNEG)
5683 ? AMDGPU::G_FNEG
5684 : AMDGPU::G_FABS;
5685
5686 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5687 ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: i));
5688 if (ElV2F16->getOpcode() != ModOpcode)
5689 break;
5690 EltsV2F16.push_back(Elt: ElV2F16->getOperand(i: 1).getReg());
5691 }
5692
5693 // All elements had ModOpcode modifier
5694 if (CV->getNumSources() == EltsV2F16.size()) {
5695 MachineIRBuilder B(*Root.getParent());
5696 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsV2F16, Src, InsertPt: Root.getParent(),
5697 MRI&: *MRI);
5698 }
5699 }
5700
5701 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5702 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5703}
5704
5705InstructionSelector::ComplexRendererFns
5706AMDGPUInstructionSelector::selectWMMAVISrc(MachineOperand &Root) const {
5707 std::optional<FPValueAndVReg> FPValReg;
5708 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GFCstOrSplat(FPValReg))) {
5709 if (TII.isInlineConstant(Imm: FPValReg->Value)) {
5710 return {{[=](MachineInstrBuilder &MIB) {
5711 MIB.addImm(Val: FPValReg->Value.bitcastToAPInt().getSExtValue());
5712 }}};
5713 }
5714 // Non-inlineable splat floats should not fall-through for integer immediate
5715 // checks.
5716 return {};
5717 }
5718
5719 APInt ICst;
5720 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICstOrSplat(Cst&: ICst))) {
5721 if (TII.isInlineConstant(Imm: ICst)) {
5722 return {
5723 {[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ICst.getSExtValue()); }}};
5724 }
5725 }
5726
5727 return {};
5728}
5729
5730InstructionSelector::ComplexRendererFns
5731AMDGPUInstructionSelector::selectSWMMACIndex8(MachineOperand &Root) const {
5732 Register Src =
5733 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5734 unsigned Key = 0;
5735
5736 Register ShiftSrc;
5737 std::optional<ValueAndVReg> ShiftAmt;
5738 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5739 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5740 ShiftAmt->Value.getZExtValue() % 8 == 0) {
5741 Key = ShiftAmt->Value.getZExtValue() / 8;
5742 Src = ShiftSrc;
5743 }
5744
5745 return {{
5746 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5747 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5748 }};
5749}
5750
5751InstructionSelector::ComplexRendererFns
5752AMDGPUInstructionSelector::selectSWMMACIndex16(MachineOperand &Root) const {
5753
5754 Register Src =
5755 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5756 unsigned Key = 0;
5757
5758 Register ShiftSrc;
5759 std::optional<ValueAndVReg> ShiftAmt;
5760 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5761 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5762 ShiftAmt->Value.getZExtValue() == 16) {
5763 Src = ShiftSrc;
5764 Key = 1;
5765 }
5766
5767 return {{
5768 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5769 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5770 }};
5771}
5772
5773InstructionSelector::ComplexRendererFns
5774AMDGPUInstructionSelector::selectSWMMACIndex32(MachineOperand &Root) const {
5775 Register Src =
5776 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5777 unsigned Key = 0;
5778
5779 Register S32 = matchZeroExtendFromS32(Reg: Src);
5780 if (!S32)
5781 S32 = matchAnyExtendFromS32(Reg: Src);
5782
5783 if (S32) {
5784 const MachineInstr *Def = getDefIgnoringCopies(Reg: S32, MRI: *MRI);
5785 if (Def->getOpcode() == TargetOpcode::G_UNMERGE_VALUES) {
5786 assert(Def->getNumOperands() == 3);
5787 Register DstReg1 = Def->getOperand(i: 1).getReg();
5788 if (mi_match(R: S32, MRI: *MRI,
5789 P: m_any_of(preds: m_SpecificReg(RequestedReg: DstReg1), preds: m_Copy(Src: m_Reg(R&: DstReg1))))) {
5790 Src = Def->getOperand(i: 2).getReg();
5791 Key = 1;
5792 }
5793 }
5794 }
5795
5796 return {{
5797 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5798 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5799 }};
5800}
5801
5802InstructionSelector::ComplexRendererFns
5803AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const {
5804 Register Src;
5805 unsigned Mods;
5806 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
5807
5808 Register ExtractSrc;
5809 if (!Subtarget->useRealTrue16Insts() &&
5810 MRI->getType(Reg: Root.getReg()).getSizeInBits() == 16 &&
5811 isExtractHiElt(MRI&: *MRI, In: Src, Out&: ExtractSrc)) {
5812 Src = ExtractSrc;
5813 Mods |= SISrcMods::OP_SEL_0;
5814 }
5815
5816 return {{
5817 [=](MachineInstrBuilder &MIB) {
5818 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
5819 },
5820 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5821 }};
5822}
5823
5824// FIXME-TRUE16 remove when fake16 is removed
5825InstructionSelector::ComplexRendererFns
5826AMDGPUInstructionSelector::selectVINTERPMods(MachineOperand &Root) const {
5827 Register Src;
5828 unsigned Mods;
5829 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5830 /*IsCanonicalizing=*/true,
5831 /*AllowAbs=*/false,
5832 /*OpSel=*/false);
5833
5834 return {{
5835 [=](MachineInstrBuilder &MIB) {
5836 MIB.addReg(
5837 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5838 },
5839 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5840 }};
5841}
5842
5843InstructionSelector::ComplexRendererFns
5844AMDGPUInstructionSelector::selectVINTERPModsHi(MachineOperand &Root) const {
5845 Register Src;
5846 unsigned Mods;
5847 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5848 /*IsCanonicalizing=*/true,
5849 /*AllowAbs=*/false,
5850 /*OpSel=*/true);
5851
5852 return {{
5853 [=](MachineInstrBuilder &MIB) {
5854 MIB.addReg(
5855 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5856 },
5857 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5858 }};
5859}
5860
5861// Given \p Offset and load specified by the \p Root operand check if \p Offset
5862// is a multiple of the load byte size. If it is update \p Offset to a
5863// pre-scaled value and return true.
5864bool AMDGPUInstructionSelector::selectScaleOffset(MachineOperand &Root,
5865 Register &Offset,
5866 bool IsSigned) const {
5867 if (!Subtarget->hasScaleOffset())
5868 return false;
5869
5870 const MachineInstr &MI = *Root.getParent();
5871 MachineMemOperand *MMO = *MI.memoperands_begin();
5872
5873 if (!MMO->getSize().hasValue())
5874 return false;
5875
5876 uint64_t Size = MMO->getSize().getValue();
5877
5878 Register OffsetReg = matchExtendFromS32OrS32(Reg: Offset, IsSigned);
5879 if (!OffsetReg)
5880 OffsetReg = Offset;
5881
5882 if (auto Def = getDefSrcRegIgnoringCopies(Reg: OffsetReg, MRI: *MRI))
5883 OffsetReg = Def->Reg;
5884
5885 Register Op0;
5886 MachineInstr *Mul;
5887 bool ScaleOffset =
5888 (isPowerOf2_64(Value: Size) &&
5889 mi_match(R: OffsetReg, MRI: *MRI,
5890 P: m_GShl(L: m_Reg(R&: Op0),
5891 R: m_any_of(preds: m_SpecificICst(RequestedValue: Log2_64(Value: Size)),
5892 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Log2_64(Value: Size))))))) ||
5893 mi_match(R: OffsetReg, MRI: *MRI,
5894 P: m_GMul(L: m_Reg(R&: Op0), R: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5895 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) ||
5896 mi_match(
5897 R: OffsetReg, MRI: *MRI,
5898 P: m_BinOp(Opcode: IsSigned ? AMDGPU::S_MUL_I64_I32_PSEUDO : AMDGPU::S_MUL_U64,
5899 L: m_Reg(R&: Op0), R: m_SpecificICst(RequestedValue: Size))) ||
5900 // Match G_AMDGPU_MAD_U64_U32 offset, c, 0
5901 (mi_match(R: OffsetReg, MRI: *MRI, P: m_MInstr(MI&: Mul)) &&
5902 (Mul->getOpcode() == (IsSigned ? AMDGPU::G_AMDGPU_MAD_I64_I32
5903 : AMDGPU::G_AMDGPU_MAD_U64_U32) ||
5904 (IsSigned && Mul->getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32 &&
5905 VT->signBitIsZero(Op: Mul->getOperand(i: 2).getReg()))) &&
5906 mi_match(R: Mul->getOperand(i: 4).getReg(), MRI: *MRI, P: m_ZeroInt()) &&
5907 mi_match(R: Mul->getOperand(i: 3).getReg(), MRI: *MRI,
5908 P: m_GTrunc(Src: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5909 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) &&
5910 mi_match(R: Mul->getOperand(i: 2).getReg(), MRI: *MRI, P: m_Reg(R&: Op0)));
5911
5912 if (ScaleOffset)
5913 Offset = Op0;
5914
5915 return ScaleOffset;
5916}
5917
5918bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root,
5919 Register &Base,
5920 Register *SOffset,
5921 int64_t *Offset,
5922 bool *ScaleOffset) const {
5923 MachineInstr *MI = Root.getParent();
5924 MachineBasicBlock *MBB = MI->getParent();
5925
5926 // FIXME: We should shrink the GEP if the offset is known to be <= 32-bits,
5927 // then we can select all ptr + 32-bit offsets.
5928 SmallVector<GEPInfo, 4> AddrInfo;
5929 getAddrModeInfo(Load: *MI, MRI: *MRI, AddrInfo);
5930
5931 if (AddrInfo.empty())
5932 return false;
5933
5934 const GEPInfo &GEPI = AddrInfo[0];
5935 std::optional<int64_t> EncodedImm;
5936
5937 if (ScaleOffset)
5938 *ScaleOffset = false;
5939
5940 if (SOffset && Offset) {
5941 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
5942 /*HasSOffset=*/true);
5943 if (GEPI.SgprParts.size() == 1 && GEPI.Imm != 0 && EncodedImm &&
5944 AddrInfo.size() > 1) {
5945 const GEPInfo &GEPI2 = AddrInfo[1];
5946 if (GEPI2.SgprParts.size() == 2 && GEPI2.Imm == 0) {
5947 Register OffsetReg = GEPI2.SgprParts[1];
5948 if (ScaleOffset)
5949 *ScaleOffset =
5950 selectScaleOffset(Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */);
5951 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
5952 if (OffsetReg) {
5953 Base = GEPI2.SgprParts[0];
5954 *SOffset = OffsetReg;
5955 *Offset = *EncodedImm;
5956 if (*Offset >= 0 || !AMDGPU::hasSMRDSignedImmOffset(ST: STI))
5957 return true;
5958
5959 // For unbuffered smem loads, it is illegal for the Immediate Offset
5960 // to be negative if the resulting (Offset + (M0 or SOffset or zero)
5961 // is negative. Handle the case where the Immediate Offset + SOffset
5962 // is negative.
5963 auto SKnown = VT->getKnownBits(R: *SOffset);
5964 if (*Offset + SKnown.getMinValue().getSExtValue() < 0)
5965 return false;
5966
5967 return true;
5968 }
5969 }
5970 }
5971 return false;
5972 }
5973
5974 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
5975 /*HasSOffset=*/false);
5976 if (Offset && GEPI.SgprParts.size() == 1 && EncodedImm) {
5977 Base = GEPI.SgprParts[0];
5978 *Offset = *EncodedImm;
5979 return true;
5980 }
5981
5982 // SGPR offset is unsigned.
5983 if (SOffset && GEPI.SgprParts.size() == 1 && isUInt<32>(x: GEPI.Imm) &&
5984 GEPI.Imm != 0) {
5985 // If we make it this far we have a load with an 32-bit immediate offset.
5986 // It is OK to select this using a sgpr offset, because we have already
5987 // failed trying to select this load into one of the _IMM variants since
5988 // the _IMM Patterns are considered before the _SGPR patterns.
5989 Base = GEPI.SgprParts[0];
5990 *SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5991 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: *SOffset)
5992 .addImm(Val: GEPI.Imm);
5993 return true;
5994 }
5995
5996 if (SOffset && GEPI.SgprParts.size() && GEPI.Imm == 0) {
5997 Register OffsetReg = GEPI.SgprParts[1];
5998 if (ScaleOffset)
5999 *ScaleOffset = selectScaleOffset(Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */);
6000 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
6001 if (OffsetReg) {
6002 Base = GEPI.SgprParts[0];
6003 *SOffset = OffsetReg;
6004 return true;
6005 }
6006 }
6007
6008 return false;
6009}
6010
6011InstructionSelector::ComplexRendererFns
6012AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
6013 Register Base;
6014 int64_t Offset;
6015 if (!selectSmrdOffset(Root, Base, /* SOffset= */ nullptr, Offset: &Offset,
6016 /* ScaleOffset */ nullptr))
6017 return std::nullopt;
6018
6019 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6020 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }}};
6021}
6022
6023InstructionSelector::ComplexRendererFns
6024AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const {
6025 SmallVector<GEPInfo, 4> AddrInfo;
6026 getAddrModeInfo(Load: *Root.getParent(), MRI: *MRI, AddrInfo);
6027
6028 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
6029 return std::nullopt;
6030
6031 const GEPInfo &GEPInfo = AddrInfo[0];
6032 Register PtrReg = GEPInfo.SgprParts[0];
6033 std::optional<int64_t> EncodedImm =
6034 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: GEPInfo.Imm);
6035 if (!EncodedImm)
6036 return std::nullopt;
6037
6038 return {{
6039 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrReg); },
6040 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); }
6041 }};
6042}
6043
6044InstructionSelector::ComplexRendererFns
6045AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
6046 Register Base, SOffset;
6047 bool ScaleOffset;
6048 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, /* Offset= */ nullptr,
6049 ScaleOffset: &ScaleOffset))
6050 return std::nullopt;
6051
6052 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6053 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6054 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
6055 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
6056}
6057
6058InstructionSelector::ComplexRendererFns
6059AMDGPUInstructionSelector::selectSmrdSgprImm(MachineOperand &Root) const {
6060 Register Base, SOffset;
6061 int64_t Offset;
6062 bool ScaleOffset;
6063 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, Offset: &Offset, ScaleOffset: &ScaleOffset))
6064 return std::nullopt;
6065
6066 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6067 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6068 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
6069 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
6070 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
6071}
6072
6073std::pair<Register, int> AMDGPUInstructionSelector::selectFlatOffsetImpl(
6074 MachineOperand &Root, AMDGPU::FlatAddrSpace FlatVariant) const {
6075 MachineInstr *MI = Root.getParent();
6076
6077 auto Default = std::pair(Root.getReg(), 0);
6078
6079 if (!STI.hasFlatInstOffsets())
6080 return Default;
6081
6082 Register PtrBase;
6083 int64_t ConstOffset;
6084 bool IsInBounds;
6085 std::tie(args&: PtrBase, args&: ConstOffset, args&: IsInBounds) =
6086 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6087
6088 // Adding the offset to the base address with an immediate in a FLAT
6089 // instruction must not change the memory aperture in which the address falls.
6090 // Therefore we can only fold offsets from inbounds GEPs into FLAT
6091 // instructions.
6092 if (ConstOffset == 0 ||
6093 (FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch &&
6094 !isFlatScratchBaseLegal(Addr: Root.getReg())) ||
6095 (FlatVariant == AMDGPU::FlatAddrSpace::FLAT && !IsInBounds))
6096 return Default;
6097
6098 unsigned AddrSpace = (*MI->memoperands_begin())->getAddrSpace();
6099 if (!TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace, FlatVariant))
6100 return Default;
6101
6102 return std::pair(PtrBase, ConstOffset);
6103}
6104
6105InstructionSelector::ComplexRendererFns
6106AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const {
6107 auto PtrWithOffset = selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FLAT);
6108
6109 return {{
6110 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6111 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6112 }};
6113}
6114
6115InstructionSelector::ComplexRendererFns
6116AMDGPUInstructionSelector::selectGlobalOffset(MachineOperand &Root) const {
6117 auto PtrWithOffset =
6118 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6119
6120 return {{
6121 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6122 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6123 }};
6124}
6125
6126InstructionSelector::ComplexRendererFns
6127AMDGPUInstructionSelector::selectScratchOffset(MachineOperand &Root) const {
6128 auto PtrWithOffset =
6129 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch);
6130
6131 return {{
6132 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6133 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6134 }};
6135}
6136
6137// Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
6138InstructionSelector::ComplexRendererFns
6139AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root,
6140 unsigned CPolBits,
6141 bool NeedIOffset) const {
6142 Register Addr = Root.getReg();
6143 Register PtrBase;
6144 int64_t ConstOffset;
6145 int64_t ImmOffset = 0;
6146
6147 // Match the immediate offset first, which canonically is moved as low as
6148 // possible.
6149 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6150 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6151
6152 if (ConstOffset != 0) {
6153 if (NeedIOffset &&
6154 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6155 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal)) {
6156 Addr = PtrBase;
6157 ImmOffset = ConstOffset;
6158 } else {
6159 auto PtrBaseDef = getDefSrcRegIgnoringCopies(Reg: PtrBase, MRI: *MRI);
6160 if (isSGPR(Reg: PtrBaseDef->Reg)) {
6161 // Offset is too large.
6162 //
6163 // saddr + large_offset -> saddr +
6164 // (voffset = large_offset & ~MaxOffset) +
6165 // (large_offset & MaxOffset);
6166 int64_t SplitImmOffset = 0, RemainderOffset = ConstOffset;
6167 if (NeedIOffset) {
6168 std::tie(args&: SplitImmOffset, args&: RemainderOffset) =
6169 TII.splitFlatOffset(COffsetVal: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6170 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6171 }
6172
6173 if (Subtarget->hasSignedGVSOffset() ? isInt<32>(x: RemainderOffset)
6174 : isUInt<32>(x: RemainderOffset)) {
6175 MachineInstr *MI = Root.getParent();
6176 MachineBasicBlock *MBB = MI->getParent();
6177 Register HighBits =
6178 MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6179
6180 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6181 DestReg: HighBits)
6182 .addImm(Val: RemainderOffset);
6183
6184 if (NeedIOffset)
6185 return {{
6186 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrBase); }, // saddr
6187 [=](MachineInstrBuilder &MIB) {
6188 MIB.addReg(RegNo: HighBits);
6189 }, // voffset
6190 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: SplitImmOffset); },
6191 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6192 }};
6193 return {{
6194 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrBase); }, // saddr
6195 [=](MachineInstrBuilder &MIB) {
6196 MIB.addReg(RegNo: HighBits);
6197 }, // voffset
6198 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6199 }};
6200 }
6201
6202 // We are adding a 64 bit SGPR and a constant. If constant bus limit
6203 // is 1 we would need to perform 1 or 2 extra moves for each half of
6204 // the constant and it is better to do a scalar add and then issue a
6205 // single VALU instruction to materialize zero. Otherwise it is less
6206 // instructions to perform VALU adds with immediates or inline literals.
6207 unsigned NumLiterals =
6208 !TII.isInlineConstant(Imm: APInt(32, Lo_32(Value: ConstOffset))) +
6209 !TII.isInlineConstant(Imm: APInt(32, Hi_32(Value: ConstOffset)));
6210 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_ADD_U32_e64) > NumLiterals)
6211 return std::nullopt;
6212 }
6213 }
6214 }
6215
6216 // Match the variable offset.
6217 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6218 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6219 // Look through the SGPR->VGPR copy.
6220 Register SAddr =
6221 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
6222
6223 if (isSGPR(Reg: SAddr)) {
6224 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
6225
6226 // It's possible voffset is an SGPR here, but the copy to VGPR will be
6227 // inserted later.
6228 bool ScaleOffset = selectScaleOffset(Root, Offset&: PtrBaseOffset,
6229 IsSigned: Subtarget->hasSignedGVSOffset());
6230 if (Register VOffset = matchExtendFromS32OrS32(
6231 Reg: PtrBaseOffset, IsSigned: Subtarget->hasSignedGVSOffset())) {
6232 if (NeedIOffset)
6233 return {{[=](MachineInstrBuilder &MIB) { // saddr
6234 MIB.addReg(RegNo: SAddr);
6235 },
6236 [=](MachineInstrBuilder &MIB) { // voffset
6237 MIB.addReg(RegNo: VOffset);
6238 },
6239 [=](MachineInstrBuilder &MIB) { // offset
6240 MIB.addImm(Val: ImmOffset);
6241 },
6242 [=](MachineInstrBuilder &MIB) { // cpol
6243 MIB.addImm(Val: CPolBits |
6244 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6245 }}};
6246 return {{[=](MachineInstrBuilder &MIB) { // saddr
6247 MIB.addReg(RegNo: SAddr);
6248 },
6249 [=](MachineInstrBuilder &MIB) { // voffset
6250 MIB.addReg(RegNo: VOffset);
6251 },
6252 [=](MachineInstrBuilder &MIB) { // cpol
6253 MIB.addImm(Val: CPolBits |
6254 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6255 }}};
6256 }
6257 }
6258 }
6259
6260 // FIXME: We should probably have folded COPY (G_IMPLICIT_DEF) earlier, and
6261 // drop this.
6262 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF ||
6263 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(Reg: AddrDef->Reg))
6264 return std::nullopt;
6265
6266 // It's cheaper to materialize a single 32-bit zero for vaddr than the two
6267 // moves required to copy a 64-bit SGPR to VGPR.
6268 MachineInstr *MI = Root.getParent();
6269 MachineBasicBlock *MBB = MI->getParent();
6270 Register VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6271
6272 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
6273 .addImm(Val: 0);
6274
6275 if (NeedIOffset)
6276 return {{
6277 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6278 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6279 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6280 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6281 }};
6282 return {{
6283 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6284 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6285 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6286 }};
6287}
6288
6289InstructionSelector::ComplexRendererFns
6290AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root) const {
6291 return selectGlobalSAddr(Root, CPolBits: 0);
6292}
6293
6294InstructionSelector::ComplexRendererFns
6295AMDGPUInstructionSelector::selectGlobalSAddrCPol(MachineOperand &Root) const {
6296 const MachineInstr &I = *Root.getParent();
6297
6298 // We are assuming CPol is always the last operand of the intrinsic.
6299 auto PassedCPol =
6300 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6301 return selectGlobalSAddr(Root, CPolBits: PassedCPol);
6302}
6303
6304InstructionSelector::ComplexRendererFns
6305AMDGPUInstructionSelector::selectGlobalSAddrCPolM0(MachineOperand &Root) const {
6306 const MachineInstr &I = *Root.getParent();
6307
6308 // We are assuming CPol is second from last operand of the intrinsic.
6309 auto PassedCPol =
6310 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6311 return selectGlobalSAddr(Root, CPolBits: PassedCPol);
6312}
6313
6314InstructionSelector::ComplexRendererFns
6315AMDGPUInstructionSelector::selectGlobalSAddrGLC(MachineOperand &Root) const {
6316 return selectGlobalSAddr(Root, CPolBits: AMDGPU::CPol::GLC);
6317}
6318
6319InstructionSelector::ComplexRendererFns
6320AMDGPUInstructionSelector::selectGlobalSAddrNoIOffset(
6321 MachineOperand &Root) const {
6322 const MachineInstr &I = *Root.getParent();
6323
6324 // We are assuming CPol is always the last operand of the intrinsic.
6325 auto PassedCPol =
6326 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6327 return selectGlobalSAddr(Root, CPolBits: PassedCPol, NeedIOffset: false);
6328}
6329
6330InstructionSelector::ComplexRendererFns
6331AMDGPUInstructionSelector::selectGlobalSAddrNoIOffsetM0(
6332 MachineOperand &Root) const {
6333 const MachineInstr &I = *Root.getParent();
6334
6335 // We are assuming CPol is second from last operand of the intrinsic.
6336 auto PassedCPol =
6337 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6338 return selectGlobalSAddr(Root, CPolBits: PassedCPol, NeedIOffset: false);
6339}
6340
6341InstructionSelector::ComplexRendererFns
6342AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const {
6343 Register Addr = Root.getReg();
6344 Register PtrBase;
6345 int64_t ConstOffset;
6346 int64_t ImmOffset = 0;
6347
6348 // Match the immediate offset first, which canonically is moved as low as
6349 // possible.
6350 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6351 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6352
6353 if (ConstOffset != 0 && isFlatScratchBaseLegal(Addr) &&
6354 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6355 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6356 Addr = PtrBase;
6357 ImmOffset = ConstOffset;
6358 }
6359
6360 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6361 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6362 int FI = AddrDef->MI->getOperand(i: 1).getIndex();
6363 return {{
6364 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6365 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6366 }};
6367 }
6368
6369 Register SAddr = AddrDef->Reg;
6370
6371 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6372 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6373 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6374 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6375 auto RHSDef = getDefSrcRegIgnoringCopies(Reg: RHS, MRI: *MRI);
6376
6377 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX &&
6378 isSGPR(Reg: RHSDef->Reg)) {
6379 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6380 MachineInstr &I = *Root.getParent();
6381 MachineBasicBlock *BB = I.getParent();
6382 const DebugLoc &DL = I.getDebugLoc();
6383 SAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6384
6385 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_I32), DestReg: SAddr)
6386 .addFrameIndex(Idx: FI)
6387 .addReg(RegNo: RHSDef->Reg)
6388 .setOperandDead(3); // Dead scc
6389 }
6390 }
6391
6392 if (!isSGPR(Reg: SAddr))
6393 return std::nullopt;
6394
6395 return {{
6396 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SAddr); }, // saddr
6397 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6398 }};
6399}
6400
6401// Check whether the flat scratch SVS swizzle bug affects this access.
6402bool AMDGPUInstructionSelector::checkFlatScratchSVSSwizzleBug(
6403 Register VAddr, Register SAddr, uint64_t ImmOffset) const {
6404 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
6405 return false;
6406
6407 // The bug affects the swizzling of SVS accesses if there is any carry out
6408 // from the two low order bits (i.e. from bit 1 into bit 2) when adding
6409 // voffset to (soffset + inst_offset).
6410 auto VKnown = VT->getKnownBits(R: VAddr);
6411 auto SKnown = KnownBits::add(LHS: VT->getKnownBits(R: SAddr),
6412 RHS: KnownBits::makeConstant(C: APInt(32, ImmOffset)));
6413 uint64_t VMax = VKnown.getMaxValue().getZExtValue();
6414 uint64_t SMax = SKnown.getMaxValue().getZExtValue();
6415 return (VMax & 3) + (SMax & 3) >= 4;
6416}
6417
6418InstructionSelector::ComplexRendererFns
6419AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const {
6420 Register Addr = Root.getReg();
6421 Register PtrBase;
6422 int64_t ConstOffset;
6423 int64_t ImmOffset = 0;
6424
6425 // Match the immediate offset first, which canonically is moved as low as
6426 // possible.
6427 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6428 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6429
6430 Register OrigAddr = Addr;
6431 if (ConstOffset != 0 &&
6432 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6433 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6434 Addr = PtrBase;
6435 ImmOffset = ConstOffset;
6436 }
6437
6438 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6439 if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD)
6440 return std::nullopt;
6441
6442 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6443 if (RBI.getRegBank(Reg: RHS, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
6444 return std::nullopt;
6445
6446 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6447 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6448
6449 if (OrigAddr != Addr) {
6450 if (!isFlatScratchBaseLegalSVImm(Addr: OrigAddr))
6451 return std::nullopt;
6452 } else {
6453 if (!isFlatScratchBaseLegalSV(Addr: OrigAddr))
6454 return std::nullopt;
6455 }
6456
6457 if (checkFlatScratchSVSSwizzleBug(VAddr: RHS, SAddr: LHS, ImmOffset))
6458 return std::nullopt;
6459
6460 unsigned CPol = selectScaleOffset(Root, Offset&: RHS, IsSigned: true /* IsSigned */)
6461 ? AMDGPU::CPol::SCAL
6462 : 0;
6463
6464 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6465 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6466 return {{
6467 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6468 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6469 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6470 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6471 }};
6472 }
6473
6474 if (!isSGPR(Reg: LHS))
6475 if (auto Def = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI))
6476 LHS = Def->Reg;
6477
6478 if (!isSGPR(Reg: LHS))
6479 return std::nullopt;
6480
6481 return {{
6482 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6483 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: LHS); }, // saddr
6484 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6485 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6486 }};
6487}
6488
6489InstructionSelector::ComplexRendererFns
6490AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
6491 MachineInstr *MI = Root.getParent();
6492 MachineBasicBlock *MBB = MI->getParent();
6493 MachineFunction *MF = MBB->getParent();
6494 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6495
6496 int64_t Offset = 0;
6497 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) &&
6498 Offset != AMDGPU::getNullPointerValue(AS: AMDGPUAS::PRIVATE_ADDRESS)) {
6499 Register HighBits = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6500
6501 // TODO: Should this be inside the render function? The iterator seems to
6502 // move.
6503 const int64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
6504 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6505 DestReg: HighBits)
6506 .addImm(Val: Offset & ~MaxOffset);
6507
6508 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6509 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6510 },
6511 [=](MachineInstrBuilder &MIB) { // vaddr
6512 MIB.addReg(RegNo: HighBits);
6513 },
6514 [=](MachineInstrBuilder &MIB) { // soffset
6515 // Use constant zero for soffset and rely on eliminateFrameIndex
6516 // to choose the appropriate frame register if need be.
6517 MIB.addImm(Val: 0);
6518 },
6519 [=](MachineInstrBuilder &MIB) { // offset
6520 MIB.addImm(Val: Offset & MaxOffset);
6521 }}};
6522 }
6523
6524 assert(Offset == 0 || Offset == -1);
6525
6526 // Try to fold a frame index directly into the MUBUF vaddr field, and any
6527 // offsets.
6528 std::optional<int> FI;
6529 Register VAddr = Root.getReg();
6530
6531 Register PtrBase;
6532 int64_t ConstOffset;
6533 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6534 getPtrBaseWithConstantOffset(Root: VAddr, MRI: *MRI);
6535 int MatchedFI;
6536 if (ConstOffset != 0) {
6537 if (TII.isLegalMUBUFImmOffset(Imm: ConstOffset) &&
6538 (!STI.privateMemoryResourceIsRangeChecked() ||
6539 VT->signBitIsZero(Op: PtrBase))) {
6540 if (mi_match(R: PtrBase, MRI: *MRI, P: m_GFrameIndex(FI&: MatchedFI)))
6541 FI = MatchedFI;
6542 else
6543 VAddr = PtrBase;
6544 Offset = ConstOffset;
6545 }
6546 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GFrameIndex(FI&: MatchedFI))) {
6547 FI = MatchedFI;
6548 }
6549
6550 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6551 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6552 },
6553 [=](MachineInstrBuilder &MIB) { // vaddr
6554 if (FI)
6555 MIB.addFrameIndex(Idx: *FI);
6556 else
6557 MIB.addReg(RegNo: VAddr);
6558 },
6559 [=](MachineInstrBuilder &MIB) { // soffset
6560 // Use constant zero for soffset and rely on eliminateFrameIndex
6561 // to choose the appropriate frame register if need be.
6562 MIB.addImm(Val: 0);
6563 },
6564 [=](MachineInstrBuilder &MIB) { // offset
6565 MIB.addImm(Val: Offset);
6566 }}};
6567}
6568
6569bool AMDGPUInstructionSelector::isDSOffsetLegal(Register Base,
6570 int64_t Offset) const {
6571 if (!isUInt<16>(x: Offset))
6572 return false;
6573
6574 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6575 return true;
6576
6577 // On Southern Islands instruction with a negative base value and an offset
6578 // don't seem to work.
6579 return VT->signBitIsZero(Op: Base);
6580}
6581
6582bool AMDGPUInstructionSelector::isDSOffset2Legal(Register Base, int64_t Offset0,
6583 int64_t Offset1,
6584 unsigned Size) const {
6585 if (Offset0 % Size != 0 || Offset1 % Size != 0)
6586 return false;
6587 if (!isUInt<8>(x: Offset0 / Size) || !isUInt<8>(x: Offset1 / Size))
6588 return false;
6589
6590 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6591 return true;
6592
6593 // On Southern Islands instruction with a negative base value and an offset
6594 // don't seem to work.
6595 return VT->signBitIsZero(Op: Base);
6596}
6597
6598// Return whether the operation has NoUnsignedWrap property.
6599static bool isNoUnsignedWrap(MachineInstr *Addr) {
6600 return Addr->getOpcode() == TargetOpcode::G_OR ||
6601 (Addr->getOpcode() == TargetOpcode::G_PTR_ADD &&
6602 Addr->getFlag(Flag: MachineInstr::NoUWrap));
6603}
6604
6605// Check that the base address of flat scratch load/store in the form of `base +
6606// offset` is legal to be put in SGPR/VGPR (i.e. unsigned per hardware
6607// requirement). We always treat the first operand as the base address here.
6608bool AMDGPUInstructionSelector::isFlatScratchBaseLegal(Register Addr) const {
6609 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6610
6611 if (isNoUnsignedWrap(Addr: AddrMI))
6612 return true;
6613
6614 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6615 // values.
6616 if (STI.hasSignedScratchOffsets())
6617 return true;
6618
6619 Register LHS = AddrMI->getOperand(i: 1).getReg();
6620 Register RHS = AddrMI->getOperand(i: 2).getReg();
6621
6622 if (AddrMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
6623 std::optional<ValueAndVReg> RhsValReg =
6624 getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
6625 // If the immediate offset is negative and within certain range, the base
6626 // address cannot also be negative. If the base is also negative, the sum
6627 // would be either negative or much larger than the valid range of scratch
6628 // memory a thread can access.
6629 if (RhsValReg && RhsValReg->Value.getSExtValue() < 0 &&
6630 RhsValReg->Value.getSExtValue() > -0x40000000)
6631 return true;
6632 }
6633
6634 return VT->signBitIsZero(Op: LHS);
6635}
6636
6637// Check address value in SGPR/VGPR are legal for flat scratch in the form
6638// of: SGPR + VGPR.
6639bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSV(Register Addr) const {
6640 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6641
6642 if (isNoUnsignedWrap(Addr: AddrMI))
6643 return true;
6644
6645 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6646 // values.
6647 if (STI.hasSignedScratchOffsets())
6648 return true;
6649
6650 Register LHS = AddrMI->getOperand(i: 1).getReg();
6651 Register RHS = AddrMI->getOperand(i: 2).getReg();
6652 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6653}
6654
6655// Check address value in SGPR/VGPR are legal for flat scratch in the form
6656// of: SGPR + VGPR + Imm.
6657bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSVImm(
6658 Register Addr) const {
6659 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6660 // values.
6661 if (STI.hasSignedScratchOffsets())
6662 return true;
6663
6664 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6665 Register Base = AddrMI->getOperand(i: 1).getReg();
6666 std::optional<DefinitionAndSourceRegister> BaseDef =
6667 getDefSrcRegIgnoringCopies(Reg: Base, MRI: *MRI);
6668 std::optional<ValueAndVReg> RHSOffset =
6669 getIConstantVRegValWithLookThrough(VReg: AddrMI->getOperand(i: 2).getReg(), MRI: *MRI);
6670 assert(RHSOffset);
6671
6672 // If the immediate offset is negative and within certain range, the base
6673 // address cannot also be negative. If the base is also negative, the sum
6674 // would be either negative or much larger than the valid range of scratch
6675 // memory a thread can access.
6676 if (isNoUnsignedWrap(Addr: BaseDef->MI) &&
6677 (isNoUnsignedWrap(Addr: AddrMI) ||
6678 (RHSOffset->Value.getSExtValue() < 0 &&
6679 RHSOffset->Value.getSExtValue() > -0x40000000)))
6680 return true;
6681
6682 Register LHS = BaseDef->MI->getOperand(i: 1).getReg();
6683 Register RHS = BaseDef->MI->getOperand(i: 2).getReg();
6684 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6685}
6686
6687bool AMDGPUInstructionSelector::isUnneededShiftMask(const MachineInstr &MI,
6688 unsigned ShAmtBits) const {
6689 assert(MI.getOpcode() == TargetOpcode::G_AND);
6690
6691 std::optional<APInt> RHS =
6692 getIConstantVRegVal(VReg: MI.getOperand(i: 2).getReg(), MRI: *MRI);
6693 if (!RHS)
6694 return false;
6695
6696 if (RHS->countr_one() >= ShAmtBits)
6697 return true;
6698
6699 const APInt &LHSKnownZeros = VT->getKnownZeroes(R: MI.getOperand(i: 1).getReg());
6700 return (LHSKnownZeros | *RHS).countr_one() >= ShAmtBits;
6701}
6702
6703InstructionSelector::ComplexRendererFns
6704AMDGPUInstructionSelector::selectMUBUFScratchOffset(
6705 MachineOperand &Root) const {
6706 Register Reg = Root.getReg();
6707 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6708
6709 std::optional<DefinitionAndSourceRegister> Def =
6710 getDefSrcRegIgnoringCopies(Reg, MRI: *MRI);
6711 assert(Def && "this shouldn't be an optional result");
6712 Reg = Def->Reg;
6713
6714 if (Register WaveBase = getWaveAddress(Def: Def->MI)) {
6715 return {{
6716 [=](MachineInstrBuilder &MIB) { // rsrc
6717 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6718 },
6719 [=](MachineInstrBuilder &MIB) { // soffset
6720 MIB.addReg(RegNo: WaveBase);
6721 },
6722 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // offset
6723 }};
6724 }
6725
6726 int64_t Offset = 0;
6727
6728 // FIXME: Copy check is a hack
6729 Register BasePtr;
6730 if (mi_match(R: Reg, MRI: *MRI,
6731 P: m_GPtrAdd(L: m_Reg(R&: BasePtr),
6732 R: m_any_of(preds: m_ICst(Cst&: Offset), preds: m_Copy(Src: m_ICst(Cst&: Offset)))))) {
6733 if (!TII.isLegalMUBUFImmOffset(Imm: Offset))
6734 return {};
6735 MachineInstr *BasePtrDef = getDefIgnoringCopies(Reg: BasePtr, MRI: *MRI);
6736 Register WaveBase = getWaveAddress(Def: BasePtrDef);
6737 if (!WaveBase)
6738 return {};
6739
6740 return {{
6741 [=](MachineInstrBuilder &MIB) { // rsrc
6742 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6743 },
6744 [=](MachineInstrBuilder &MIB) { // soffset
6745 MIB.addReg(RegNo: WaveBase);
6746 },
6747 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6748 }};
6749 }
6750
6751 if (!mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) ||
6752 !TII.isLegalMUBUFImmOffset(Imm: Offset))
6753 return {};
6754
6755 return {{
6756 [=](MachineInstrBuilder &MIB) { // rsrc
6757 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6758 },
6759 [=](MachineInstrBuilder &MIB) { // soffset
6760 MIB.addImm(Val: 0);
6761 },
6762 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6763 }};
6764}
6765
6766std::pair<Register, unsigned>
6767AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(
6768 MachineOperand &Root) const {
6769 int64_t ConstAddr = 0;
6770
6771 Register PtrBase;
6772 int64_t Offset;
6773 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6774 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6775
6776 if (Offset) {
6777 if (isDSOffsetLegal(Base: PtrBase, Offset)) {
6778 // (add n0, c0)
6779 return std::pair(PtrBase, Offset);
6780 }
6781 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GSub(L: m_Reg(), R: m_Reg()))) {
6782 // TODO
6783
6784 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6785 // TODO
6786 }
6787
6788 return std::pair(Root.getReg(), 0);
6789}
6790
6791InstructionSelector::ComplexRendererFns
6792AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
6793 Register Reg;
6794 unsigned Offset;
6795 std::tie(args&: Reg, args&: Offset) = selectDS1Addr1OffsetImpl(Root);
6796 return {{
6797 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6798 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }
6799 }};
6800}
6801
6802InstructionSelector::ComplexRendererFns
6803AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const {
6804 return selectDSReadWrite2(Root, size: 4);
6805}
6806
6807InstructionSelector::ComplexRendererFns
6808AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(MachineOperand &Root) const {
6809 return selectDSReadWrite2(Root, size: 8);
6810}
6811
6812InstructionSelector::ComplexRendererFns
6813AMDGPUInstructionSelector::selectDSReadWrite2(MachineOperand &Root,
6814 unsigned Size) const {
6815 Register Reg;
6816 unsigned Offset;
6817 std::tie(args&: Reg, args&: Offset) = selectDSReadWrite2Impl(Root, size: Size);
6818 return {{
6819 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6820 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
6821 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset+1); }
6822 }};
6823}
6824
6825std::pair<Register, unsigned>
6826AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root,
6827 unsigned Size) const {
6828 int64_t ConstAddr = 0;
6829
6830 Register PtrBase;
6831 int64_t Offset;
6832 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6833 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6834
6835 if (Offset) {
6836 int64_t OffsetValue0 = Offset;
6837 int64_t OffsetValue1 = Offset + Size;
6838 if (isDSOffset2Legal(Base: PtrBase, Offset0: OffsetValue0, Offset1: OffsetValue1, Size)) {
6839 // (add n0, c0)
6840 return std::pair(PtrBase, OffsetValue0 / Size);
6841 }
6842 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GSub(L: m_Reg(), R: m_Reg()))) {
6843 // TODO
6844
6845 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6846 // TODO
6847 }
6848
6849 return std::pair(Root.getReg(), 0);
6850}
6851
6852/// If \p Root is a G_PTR_ADD with a G_CONSTANT on the right hand side, return
6853/// the base value with the constant offset, and if the offset computation is
6854/// known to be inbounds. There may be intervening copies between \p Root and
6855/// the identified constant. Returns \p Root, 0, false if this does not match
6856/// the pattern.
6857std::tuple<Register, int64_t, bool>
6858AMDGPUInstructionSelector::getPtrBaseWithConstantOffset(
6859 Register Root, const MachineRegisterInfo &MRI) const {
6860 MachineInstr *RootI = getDefIgnoringCopies(Reg: Root, MRI);
6861 if (RootI->getOpcode() != TargetOpcode::G_PTR_ADD)
6862 return {Root, 0, false};
6863
6864 MachineOperand &RHS = RootI->getOperand(i: 2);
6865 std::optional<ValueAndVReg> MaybeOffset =
6866 getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI);
6867 if (!MaybeOffset)
6868 return {Root, 0, false};
6869 bool IsInBounds = RootI->getFlag(Flag: MachineInstr::MIFlag::InBounds);
6870 return {RootI->getOperand(i: 1).getReg(), MaybeOffset->Value.getSExtValue(),
6871 IsInBounds};
6872}
6873
6874static void addZeroImm(MachineInstrBuilder &MIB) {
6875 MIB.addImm(Val: 0);
6876}
6877
6878/// Return a resource descriptor for use with an arbitrary 64-bit pointer. If \p
6879/// BasePtr is not valid, a null base pointer will be used.
6880static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6881 uint32_t FormatLo, uint32_t FormatHi,
6882 Register BasePtr) {
6883 Register RSrc2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6884 Register RSrc3 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6885 Register RSrcHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6886 Register RSrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
6887
6888 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6889 .addDef(RegNo: RSrc2)
6890 .addImm(Val: FormatLo);
6891 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6892 .addDef(RegNo: RSrc3)
6893 .addImm(Val: FormatHi);
6894
6895 // Build the half of the subregister with the constants before building the
6896 // full 128-bit register. If we are building multiple resource descriptors,
6897 // this will allow CSEing of the 2-component register.
6898 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6899 .addDef(RegNo: RSrcHi)
6900 .addReg(RegNo: RSrc2)
6901 .addImm(Val: AMDGPU::sub0)
6902 .addReg(RegNo: RSrc3)
6903 .addImm(Val: AMDGPU::sub1);
6904
6905 Register RSrcLo = BasePtr;
6906 if (!BasePtr) {
6907 RSrcLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6908 B.buildInstr(Opcode: AMDGPU::S_MOV_B64)
6909 .addDef(RegNo: RSrcLo)
6910 .addImm(Val: 0);
6911 }
6912
6913 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6914 .addDef(RegNo: RSrc)
6915 .addReg(RegNo: RSrcLo)
6916 .addImm(Val: AMDGPU::sub0_sub1)
6917 .addReg(RegNo: RSrcHi)
6918 .addImm(Val: AMDGPU::sub2_sub3);
6919
6920 return RSrc;
6921}
6922
6923static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6924 const SIInstrInfo &TII, Register BasePtr) {
6925 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
6926
6927 // FIXME: Why are half the "default" bits ignored based on the addressing
6928 // mode?
6929 return buildRSRC(B, MRI, FormatLo: 0, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
6930}
6931
6932static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6933 const SIInstrInfo &TII, Register BasePtr) {
6934 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
6935
6936 // FIXME: Why are half the "default" bits ignored based on the addressing
6937 // mode?
6938 return buildRSRC(B, MRI, FormatLo: -1, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
6939}
6940
6941AMDGPUInstructionSelector::MUBUFAddressData
6942AMDGPUInstructionSelector::parseMUBUFAddress(Register Src) const {
6943 MUBUFAddressData Data;
6944 Data.N0 = Src;
6945
6946 Register PtrBase;
6947 int64_t Offset;
6948
6949 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6950 getPtrBaseWithConstantOffset(Root: Src, MRI: *MRI);
6951 if (isUInt<32>(x: Offset)) {
6952 Data.N0 = PtrBase;
6953 Data.Offset = Offset;
6954 }
6955
6956 if (MachineInstr *InputAdd
6957 = getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Data.N0, MRI: *MRI)) {
6958 Data.N2 = InputAdd->getOperand(i: 1).getReg();
6959 Data.N3 = InputAdd->getOperand(i: 2).getReg();
6960
6961 // FIXME: Need to fix extra SGPR->VGPRcopies inserted
6962 // FIXME: Don't know this was defined by operand 0
6963 //
6964 // TODO: Remove this when we have copy folding optimizations after
6965 // RegBankSelect.
6966 Data.N2 = getDefIgnoringCopies(Reg: Data.N2, MRI: *MRI)->getOperand(i: 0).getReg();
6967 Data.N3 = getDefIgnoringCopies(Reg: Data.N3, MRI: *MRI)->getOperand(i: 0).getReg();
6968 }
6969
6970 return Data;
6971}
6972
6973/// Return if the addr64 mubuf mode should be used for the given address.
6974bool AMDGPUInstructionSelector::shouldUseAddr64(MUBUFAddressData Addr) const {
6975 // (ptr_add N2, N3) -> addr64, or
6976 // (ptr_add (ptr_add N2, N3), C1) -> addr64
6977 if (Addr.N2)
6978 return true;
6979
6980 const RegisterBank *N0Bank = RBI.getRegBank(Reg: Addr.N0, MRI: *MRI, TRI);
6981 return N0Bank->getID() == AMDGPU::VGPRRegBankID;
6982}
6983
6984/// Split an immediate offset \p ImmOffset depending on whether it fits in the
6985/// immediate field. Modifies \p ImmOffset and sets \p SOffset to the variable
6986/// component.
6987void AMDGPUInstructionSelector::splitIllegalMUBUFOffset(
6988 MachineIRBuilder &B, Register &SOffset, int64_t &ImmOffset) const {
6989 if (TII.isLegalMUBUFImmOffset(Imm: ImmOffset))
6990 return;
6991
6992 // Illegal offset, store it in soffset.
6993 SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6994 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6995 .addDef(RegNo: SOffset)
6996 .addImm(Val: ImmOffset);
6997 ImmOffset = 0;
6998}
6999
7000bool AMDGPUInstructionSelector::selectMUBUFAddr64Impl(
7001 MachineOperand &Root, Register &VAddr, Register &RSrcReg,
7002 Register &SOffset, int64_t &Offset) const {
7003 // FIXME: Predicates should stop this from reaching here.
7004 // addr64 bit was removed for volcanic islands.
7005 if (!STI.hasAddr64() || STI.useFlatForGlobal())
7006 return false;
7007
7008 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
7009 if (!shouldUseAddr64(Addr: AddrData))
7010 return false;
7011
7012 Register N0 = AddrData.N0;
7013 Register N2 = AddrData.N2;
7014 Register N3 = AddrData.N3;
7015 Offset = AddrData.Offset;
7016
7017 // Base pointer for the SRD.
7018 Register SRDPtr;
7019
7020 if (N2) {
7021 if (RBI.getRegBank(Reg: N2, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7022 assert(N3);
7023 if (RBI.getRegBank(Reg: N3, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7024 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
7025 // addr64, and construct the default resource from a 0 address.
7026 VAddr = N0;
7027 } else {
7028 SRDPtr = N3;
7029 VAddr = N2;
7030 }
7031 } else {
7032 // N2 is not divergent.
7033 SRDPtr = N2;
7034 VAddr = N3;
7035 }
7036 } else if (RBI.getRegBank(Reg: N0, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7037 // Use the default null pointer in the resource
7038 VAddr = N0;
7039 } else {
7040 // N0 -> offset, or
7041 // (N0 + C1) -> offset
7042 SRDPtr = N0;
7043 }
7044
7045 MachineIRBuilder B(*Root.getParent());
7046 RSrcReg = buildAddr64RSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
7047 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
7048 return true;
7049}
7050
7051bool AMDGPUInstructionSelector::selectMUBUFOffsetImpl(
7052 MachineOperand &Root, Register &RSrcReg, Register &SOffset,
7053 int64_t &Offset) const {
7054
7055 // FIXME: Pattern should not reach here.
7056 if (STI.useFlatForGlobal())
7057 return false;
7058
7059 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
7060 if (shouldUseAddr64(Addr: AddrData))
7061 return false;
7062
7063 // N0 -> offset, or
7064 // (N0 + C1) -> offset
7065 Register SRDPtr = AddrData.N0;
7066 Offset = AddrData.Offset;
7067
7068 // TODO: Look through extensions for 32-bit soffset.
7069 MachineIRBuilder B(*Root.getParent());
7070
7071 RSrcReg = buildOffsetSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
7072 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
7073 return true;
7074}
7075
7076InstructionSelector::ComplexRendererFns
7077AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const {
7078 Register VAddr;
7079 Register RSrcReg;
7080 Register SOffset;
7081 int64_t Offset = 0;
7082
7083 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset))
7084 return {};
7085
7086 // FIXME: Use defaulted operands for trailing 0s and remove from the complex
7087 // pattern.
7088 return {{
7089 [=](MachineInstrBuilder &MIB) { // rsrc
7090 MIB.addReg(RegNo: RSrcReg);
7091 },
7092 [=](MachineInstrBuilder &MIB) { // vaddr
7093 MIB.addReg(RegNo: VAddr);
7094 },
7095 [=](MachineInstrBuilder &MIB) { // soffset
7096 if (SOffset)
7097 MIB.addReg(RegNo: SOffset);
7098 else if (STI.hasRestrictedSOffset())
7099 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7100 else
7101 MIB.addImm(Val: 0);
7102 },
7103 [=](MachineInstrBuilder &MIB) { // offset
7104 MIB.addImm(Val: Offset);
7105 },
7106 addZeroImm, // cpol
7107 addZeroImm, // tfe
7108 addZeroImm // swz
7109 }};
7110}
7111
7112InstructionSelector::ComplexRendererFns
7113AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const {
7114 Register RSrcReg;
7115 Register SOffset;
7116 int64_t Offset = 0;
7117
7118 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset))
7119 return {};
7120
7121 return {{
7122 [=](MachineInstrBuilder &MIB) { // rsrc
7123 MIB.addReg(RegNo: RSrcReg);
7124 },
7125 [=](MachineInstrBuilder &MIB) { // soffset
7126 if (SOffset)
7127 MIB.addReg(RegNo: SOffset);
7128 else if (STI.hasRestrictedSOffset())
7129 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7130 else
7131 MIB.addImm(Val: 0);
7132 },
7133 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }, // offset
7134 addZeroImm, // cpol
7135 addZeroImm, // tfe
7136 addZeroImm, // swz
7137 }};
7138}
7139
7140InstructionSelector::ComplexRendererFns
7141AMDGPUInstructionSelector::selectBUFSOffset(MachineOperand &Root) const {
7142
7143 Register SOffset = Root.getReg();
7144
7145 if (STI.hasRestrictedSOffset() && mi_match(R: SOffset, MRI: *MRI, P: m_ZeroInt()))
7146 SOffset = AMDGPU::SGPR_NULL;
7147
7148 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); }}};
7149}
7150
7151/// Get an immediate that must be 32-bits, and treated as zero extended.
7152static std::optional<uint64_t>
7153getConstantZext32Val(Register Reg, const MachineRegisterInfo &MRI) {
7154 // getIConstantVRegVal sexts any values, so see if that matters.
7155 std::optional<int64_t> OffsetVal = getIConstantVRegSExtVal(VReg: Reg, MRI);
7156 if (!OffsetVal || !isInt<32>(x: *OffsetVal))
7157 return std::nullopt;
7158 return Lo_32(Value: *OffsetVal);
7159}
7160
7161InstructionSelector::ComplexRendererFns
7162AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const {
7163 std::optional<uint64_t> OffsetVal =
7164 Root.isImm() ? Root.getImm() : getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7165 if (!OffsetVal)
7166 return {};
7167
7168 std::optional<int64_t> EncodedImm =
7169 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: *OffsetVal, IsBuffer: true);
7170 if (!EncodedImm)
7171 return {};
7172
7173 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7174}
7175
7176InstructionSelector::ComplexRendererFns
7177AMDGPUInstructionSelector::selectSMRDBufferImm32(MachineOperand &Root) const {
7178 assert(STI.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
7179
7180 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7181 if (!OffsetVal)
7182 return {};
7183
7184 std::optional<int64_t> EncodedImm =
7185 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: *OffsetVal);
7186 if (!EncodedImm)
7187 return {};
7188
7189 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7190}
7191
7192InstructionSelector::ComplexRendererFns
7193AMDGPUInstructionSelector::selectSMRDBufferSgprImm(MachineOperand &Root) const {
7194 // Match the (soffset + offset) pair as a 32-bit register base and
7195 // an immediate offset.
7196 Register SOffset;
7197 unsigned Offset;
7198 std::tie(args&: SOffset, args&: Offset) = AMDGPU::getBaseWithConstantOffset(
7199 MRI&: *MRI, Reg: Root.getReg(), ValueTracking: VT, /*CheckNUW*/ true);
7200 if (!SOffset)
7201 return std::nullopt;
7202
7203 std::optional<int64_t> EncodedOffset =
7204 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: Offset, /* IsBuffer */ true);
7205 if (!EncodedOffset)
7206 return std::nullopt;
7207
7208 assert(MRI->getType(SOffset).getSizeInBits() == 32);
7209 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
7210 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedOffset); }}};
7211}
7212
7213std::pair<Register, unsigned>
7214AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(MachineOperand &Root,
7215 bool &Matched) const {
7216 Matched = false;
7217
7218 Register Src;
7219 unsigned Mods;
7220 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
7221
7222 if (mi_match(R: Src, MRI: *MRI, P: m_GFPExt(Src: m_Reg(R&: Src)))) {
7223 assert(MRI->getType(Src) == LLT::scalar(16));
7224
7225 // Only change Src if src modifier could be gained. In such cases new Src
7226 // could be sgpr but this does not violate constant bus restriction for
7227 // instruction that is being selected.
7228 Src = stripBitCast(Reg: Src, MRI&: *MRI);
7229
7230 const auto CheckAbsNeg = [&]() {
7231 // Be careful about folding modifiers if we already have an abs. fneg is
7232 // applied last, so we don't want to apply an earlier fneg.
7233 if ((Mods & SISrcMods::ABS) == 0) {
7234 unsigned ModsTmp;
7235 std::tie(args&: Src, args&: ModsTmp) = selectVOP3ModsImpl(Src);
7236
7237 if ((ModsTmp & SISrcMods::NEG) != 0)
7238 Mods ^= SISrcMods::NEG;
7239
7240 if ((ModsTmp & SISrcMods::ABS) != 0)
7241 Mods |= SISrcMods::ABS;
7242 }
7243 };
7244
7245 CheckAbsNeg();
7246
7247 // op_sel/op_sel_hi decide the source type and source.
7248 // If the source's op_sel_hi is set, it indicates to do a conversion from
7249 // fp16. If the sources's op_sel is set, it picks the high half of the
7250 // source register.
7251
7252 Mods |= SISrcMods::OP_SEL_1;
7253
7254 // The instruction reads a 32-bit source and selects a half of it, so look
7255 // for the 32-bit register the 16-bit value is a half of.
7256 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
7257 // Src is now the 32-bit source and op_sel picks its high half.
7258 Mods |= SISrcMods::OP_SEL_0;
7259 CheckAbsNeg();
7260 } else if (isExtractLoElt(MRI&: *MRI, In: Src, Out&: Src)) {
7261 // op_sel already picks the low half, so the 32-bit source can be used
7262 // directly. Unlike the high half, only an fneg/fabs that acts on each
7263 // 16-bit element can be folded here: one that acts on the 32-bit value
7264 // touches bit 31 and leaves the low half alone.
7265 if (MRI->getType(Reg: Src).isFixedVector(NumElements: 2, ScalarSize: 16))
7266 CheckAbsNeg();
7267 }
7268 // Otherwise Src is genuinely 16 bits wide and widenSrcIfVGPR16 widens it
7269 // when the operand is rendered.
7270
7271 Matched = true;
7272 }
7273
7274 return {Src, Mods};
7275}
7276
7277InstructionSelector::ComplexRendererFns
7278AMDGPUInstructionSelector::selectVOP3PMadMixModsExt(
7279 MachineOperand &Root) const {
7280 Register Src;
7281 unsigned Mods;
7282 bool Matched;
7283 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7284 if (!Matched)
7285 return {};
7286
7287 return {{
7288 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7289 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7290 }};
7291}
7292
7293InstructionSelector::ComplexRendererFns
7294AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const {
7295 Register Src;
7296 unsigned Mods;
7297 bool Matched;
7298 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7299
7300 return {{
7301 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7302 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7303 }};
7304}
7305
7306InstructionSelector::ComplexRendererFns
7307AMDGPUInstructionSelector::selectVOP3PMadMixModsExtNeg(
7308 MachineOperand &Root) const {
7309 Register Src;
7310 unsigned Mods;
7311 bool Matched;
7312 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7313 if (!Matched)
7314 return {};
7315
7316 return {{
7317 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7318 [=](MachineInstrBuilder &MIB) {
7319 MIB.addImm(Val: Mods ^ SISrcMods::NEG);
7320 } // src_mods
7321 }};
7322}
7323
7324InstructionSelector::ComplexRendererFns
7325AMDGPUInstructionSelector::selectVOP3PMadMixModsNeg(
7326 MachineOperand &Root) const {
7327 Register Src;
7328 unsigned Mods;
7329 bool Matched;
7330 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7331
7332 return {{
7333 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7334 [=](MachineInstrBuilder &MIB) {
7335 MIB.addImm(Val: Mods ^ SISrcMods::NEG);
7336 } // src_mods
7337 }};
7338}
7339
7340bool AMDGPUInstructionSelector::selectSBarrierSignalIsfirst(
7341 MachineInstr &I, Intrinsic::ID IntrID) const {
7342 MachineBasicBlock *MBB = I.getParent();
7343 const DebugLoc &DL = I.getDebugLoc();
7344 Register CCReg = I.getOperand(i: 0).getReg();
7345
7346 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7347 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_EQ_U32)).addImm(Val: 0).addImm(Val: 0);
7348
7349 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM))
7350 .addImm(Val: I.getOperand(i: 2).getImm());
7351
7352 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg).addReg(RegNo: AMDGPU::SCC);
7353
7354 I.eraseFromParent();
7355 return RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32_XM0_XEXECRegClass,
7356 MRI&: *MRI);
7357}
7358
7359bool AMDGPUInstructionSelector::selectSGetBarrierState(
7360 MachineInstr &I, Intrinsic::ID IntrID) const {
7361 MachineBasicBlock *MBB = I.getParent();
7362 const DebugLoc &DL = I.getDebugLoc();
7363 const MachineOperand &BarOp = I.getOperand(i: 2);
7364 std::optional<int64_t> BarValImm =
7365 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7366
7367 if (!BarValImm) {
7368 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7369 .addReg(RegNo: BarOp.getReg());
7370 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7371 }
7372 MachineInstrBuilder MIB;
7373 unsigned Opc = BarValImm ? AMDGPU::S_GET_BARRIER_STATE_IMM
7374 : AMDGPU::S_GET_BARRIER_STATE_M0;
7375 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7376
7377 auto DstReg = I.getOperand(i: 0).getReg();
7378 const TargetRegisterClass *DstRC =
7379 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
7380 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7381 return false;
7382 MIB.addDef(RegNo: DstReg);
7383 if (BarValImm) {
7384 MIB.addImm(Val: *BarValImm);
7385 }
7386 I.eraseFromParent();
7387 return true;
7388}
7389
7390unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID) {
7391 if (HasInlineConst) {
7392 switch (IntrID) {
7393 default:
7394 llvm_unreachable("not a named barrier op");
7395 case Intrinsic::amdgcn_s_barrier_join:
7396 return AMDGPU::S_BARRIER_JOIN_IMM;
7397 case Intrinsic::amdgcn_s_wakeup_barrier:
7398 return AMDGPU::S_WAKEUP_BARRIER_IMM;
7399 case Intrinsic::amdgcn_s_get_named_barrier_state:
7400 return AMDGPU::S_GET_BARRIER_STATE_IMM;
7401 };
7402 } else {
7403 switch (IntrID) {
7404 default:
7405 llvm_unreachable("not a named barrier op");
7406 case Intrinsic::amdgcn_s_barrier_join:
7407 return AMDGPU::S_BARRIER_JOIN_M0;
7408 case Intrinsic::amdgcn_s_wakeup_barrier:
7409 return AMDGPU::S_WAKEUP_BARRIER_M0;
7410 case Intrinsic::amdgcn_s_get_named_barrier_state:
7411 return AMDGPU::S_GET_BARRIER_STATE_M0;
7412 };
7413 }
7414}
7415
7416bool AMDGPUInstructionSelector::selectNamedBarrierInit(
7417 MachineInstr &I, Intrinsic::ID IntrID) const {
7418 MachineBasicBlock *MBB = I.getParent();
7419 const DebugLoc &DL = I.getDebugLoc();
7420 const MachineOperand &BarOp = I.getOperand(i: 1);
7421 const MachineOperand &CntOp = I.getOperand(i: 2);
7422
7423 // A member count of 0 means "keep existing member count". That plus a known
7424 // constant value for the barrier ID lets us use the immarg form.
7425 if (IntrID == Intrinsic::amdgcn_s_barrier_signal_var) {
7426 std::optional<int64_t> CntImm =
7427 getIConstantVRegSExtVal(VReg: CntOp.getReg(), MRI: *MRI);
7428 if (CntImm && *CntImm == 0) {
7429 std::optional<int64_t> BarValImm =
7430 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7431 if (BarValImm) {
7432 uint32_t BarID = *BarValImm & 0x3F;
7433 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_IMM))
7434 .addImm(Val: BarID);
7435 I.eraseFromParent();
7436 return true;
7437 }
7438 }
7439 }
7440
7441 // BarID = BarOp & 0x3F
7442 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7443 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7444 .add(MO: BarOp)
7445 .addImm(Val: 0x3F)
7446 .setOperandDead(3); // Dead scc
7447
7448 // MO = ((CntOp & 0x3F) << shAmt) | BarID
7449 Register TmpReg2 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7450 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg2)
7451 .add(MO: CntOp)
7452 .addImm(Val: 0x3F)
7453 .setOperandDead(3); // Dead scc
7454
7455 Register TmpReg3 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7456 constexpr unsigned ShAmt = 16;
7457 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg3)
7458 .addReg(RegNo: TmpReg2)
7459 .addImm(Val: ShAmt)
7460 .setOperandDead(3); // Dead scc
7461
7462 Register TmpReg4 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7463 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B32), DestReg: TmpReg4)
7464 .addReg(RegNo: TmpReg1)
7465 .addReg(RegNo: TmpReg3)
7466 .setOperandDead(3); // Dead scc;
7467
7468 auto CopyMIB =
7469 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0).addReg(RegNo: TmpReg4);
7470 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7471
7472 unsigned Opc = IntrID == Intrinsic::amdgcn_s_barrier_init
7473 ? AMDGPU::S_BARRIER_INIT_M0
7474 : AMDGPU::S_BARRIER_SIGNAL_M0;
7475 MachineInstrBuilder MIB;
7476 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7477
7478 I.eraseFromParent();
7479 return true;
7480}
7481
7482bool AMDGPUInstructionSelector::selectNamedBarrierInst(
7483 MachineInstr &I, Intrinsic::ID IntrID) const {
7484 MachineBasicBlock *MBB = I.getParent();
7485 const DebugLoc &DL = I.getDebugLoc();
7486 MachineOperand BarOp = IntrID == Intrinsic::amdgcn_s_get_named_barrier_state
7487 ? I.getOperand(i: 2)
7488 : I.getOperand(i: 1);
7489 std::optional<int64_t> BarValImm =
7490 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7491
7492 if (!BarValImm) {
7493 // BarID = BarOp & 0x3F
7494 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7495 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7496 .addReg(RegNo: BarOp.getReg())
7497 .addImm(Val: 0x3F)
7498 .setOperandDead(3); // Dead scc;
7499
7500 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7501 .addReg(RegNo: TmpReg1);
7502 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7503 }
7504
7505 MachineInstrBuilder MIB;
7506 unsigned Opc = getNamedBarrierOp(HasInlineConst: BarValImm.has_value(), IntrID);
7507 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7508
7509 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
7510 auto DstReg = I.getOperand(i: 0).getReg();
7511 const TargetRegisterClass *DstRC =
7512 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
7513 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7514 return false;
7515 MIB.addDef(RegNo: DstReg);
7516 }
7517
7518 if (BarValImm) {
7519 uint32_t BarId = *BarValImm & 0x3F;
7520 MIB.addImm(Val: BarId);
7521 }
7522
7523 I.eraseFromParent();
7524 return true;
7525}
7526
7527void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB,
7528 const MachineInstr &MI,
7529 int OpIdx) const {
7530 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7531 "Expected G_CONSTANT");
7532 MIB.addImm(Val: -MI.getOperand(i: 1).getCImm()->getSExtValue());
7533}
7534
7535void AMDGPUInstructionSelector::renderBitcastFPImm(MachineInstrBuilder &MIB,
7536 const MachineInstr &MI,
7537 int OpIdx) const {
7538 const MachineOperand &Op = MI.getOperand(i: 1);
7539 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1);
7540 MIB.addImm(Val: Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
7541}
7542
7543void AMDGPUInstructionSelector::renderCountTrailingOnesImm(
7544 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7545 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7546 "Expected G_CONSTANT");
7547 MIB.addImm(Val: MI.getOperand(i: 1).getCImm()->getValue().countTrailingOnes());
7548}
7549
7550/// This only really exists to satisfy DAG type checking machinery, so is a
7551/// no-op here.
7552void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB,
7553 const MachineInstr &MI,
7554 int OpIdx) const {
7555 const MachineOperand &Op = MI.getOperand(i: OpIdx);
7556 int64_t Imm;
7557 if (Op.isReg() && mi_match(R: Op.getReg(), MRI: *MRI, P: m_ICst(Cst&: Imm)))
7558 MIB.addImm(Val: Imm);
7559 else
7560 MIB.addImm(Val: Op.getImm());
7561}
7562
7563void AMDGPUInstructionSelector::renderZextBoolTImm(MachineInstrBuilder &MIB,
7564 const MachineInstr &MI,
7565 int OpIdx) const {
7566 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() != 0);
7567}
7568
7569void AMDGPUInstructionSelector::renderOpSelTImm(MachineInstrBuilder &MIB,
7570 const MachineInstr &MI,
7571 int OpIdx) const {
7572 assert(OpIdx >= 0 && "expected to match an immediate operand");
7573 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7574}
7575
7576void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_0(
7577 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7578 assert(OpIdx >= 0 && "expected to match an immediate operand");
7579 MIB.addImm(
7580 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7581}
7582
7583void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_1(
7584 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7585 assert(OpIdx >= 0 && "expected to match an immediate operand");
7586 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x1)
7587 ? (int64_t)(SISrcMods::OP_SEL_0 | SISrcMods::DST_OP_SEL)
7588 : (int64_t)SISrcMods::DST_OP_SEL);
7589}
7590
7591void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_0(
7592 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7593 assert(OpIdx >= 0 && "expected to match an immediate operand");
7594 MIB.addImm(
7595 Val: (MI.getOperand(i: OpIdx).getImm() & 0x2) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7596}
7597
7598void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_1(
7599 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7600 assert(OpIdx >= 0 && "expected to match an immediate operand");
7601 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7602 ? (int64_t)(SISrcMods::OP_SEL_0)
7603 : 0);
7604}
7605
7606void AMDGPUInstructionSelector::renderDstSelToOpSelXForm(
7607 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7608 assert(OpIdx >= 0 && "expected to match an immediate operand");
7609 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::DST_OP_SEL)
7610 : 0);
7611}
7612
7613void AMDGPUInstructionSelector::renderSrcSelToOpSelXForm(
7614 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7615 assert(OpIdx >= 0 && "expected to match an immediate operand");
7616 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::OP_SEL_0)
7617 : 0);
7618}
7619
7620void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_2_0(
7621 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7622 assert(OpIdx >= 0 && "expected to match an immediate operand");
7623 MIB.addImm(
7624 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7625}
7626
7627void AMDGPUInstructionSelector::renderDstSelToOpSel3XFormXForm(
7628 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7629 assert(OpIdx >= 0 && "expected to match an immediate operand");
7630 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7631 ? (int64_t)SISrcMods::DST_OP_SEL
7632 : 0);
7633}
7634
7635void AMDGPUInstructionSelector::renderExtractCPol(MachineInstrBuilder &MIB,
7636 const MachineInstr &MI,
7637 int OpIdx) const {
7638 assert(OpIdx >= 0 && "expected to match an immediate operand");
7639 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() &
7640 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7641 : AMDGPU::CPol::ALL_pregfx12));
7642}
7643
7644void AMDGPUInstructionSelector::renderExtractSWZ(MachineInstrBuilder &MIB,
7645 const MachineInstr &MI,
7646 int OpIdx) const {
7647 assert(OpIdx >= 0 && "expected to match an immediate operand");
7648 const bool Swizzle = MI.getOperand(i: OpIdx).getImm() &
7649 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::SWZ
7650 : AMDGPU::CPol::SWZ_pregfx12);
7651 MIB.addImm(Val: Swizzle);
7652}
7653
7654void AMDGPUInstructionSelector::renderExtractCpolSetGLC(
7655 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7656 assert(OpIdx >= 0 && "expected to match an immediate operand");
7657 const uint32_t Cpol = MI.getOperand(i: OpIdx).getImm() &
7658 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7659 : AMDGPU::CPol::ALL_pregfx12);
7660 MIB.addImm(Val: Cpol | AMDGPU::CPol::GLC);
7661}
7662
7663void AMDGPUInstructionSelector::renderFPPow2ToExponent(MachineInstrBuilder &MIB,
7664 const MachineInstr &MI,
7665 int OpIdx) const {
7666 const APFloat &APF = MI.getOperand(i: 1).getFPImm()->getValueAPF();
7667 int ExpVal = APF.getExactLog2Abs();
7668 assert(ExpVal != INT_MIN);
7669 MIB.addImm(Val: ExpVal);
7670}
7671
7672void AMDGPUInstructionSelector::renderRoundMode(MachineInstrBuilder &MIB,
7673 const MachineInstr &MI,
7674 int OpIdx) const {
7675 // "round.towardzero" -> TowardZero 0 -> FP_ROUND_ROUND_TO_ZERO 3
7676 // "round.tonearest" -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
7677 // "round.upward" -> TowardPositive 2 -> FP_ROUND_ROUND_TO_INF 1
7678 // "round.downward -> TowardNegative 3 -> FP_ROUND_ROUND_TO_NEGINF 2
7679 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() + 3) % 4);
7680}
7681
7682void AMDGPUInstructionSelector::renderVOP3PModsNeg(MachineInstrBuilder &MIB,
7683 const MachineInstr &MI,
7684 int OpIdx) const {
7685 unsigned Mods = SISrcMods::OP_SEL_1;
7686 if (MI.getOperand(i: OpIdx).getImm())
7687 Mods ^= SISrcMods::NEG;
7688 MIB.addImm(Val: (int64_t)Mods);
7689}
7690
7691void AMDGPUInstructionSelector::renderVOP3PModsNegs(MachineInstrBuilder &MIB,
7692 const MachineInstr &MI,
7693 int OpIdx) const {
7694 unsigned Mods = SISrcMods::OP_SEL_1;
7695 if (MI.getOperand(i: OpIdx).getImm())
7696 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
7697 MIB.addImm(Val: (int64_t)Mods);
7698}
7699
7700void AMDGPUInstructionSelector::renderVOP3PModsNegAbs(MachineInstrBuilder &MIB,
7701 const MachineInstr &MI,
7702 int OpIdx) const {
7703 unsigned Val = MI.getOperand(i: OpIdx).getImm();
7704 unsigned Mods = SISrcMods::OP_SEL_1; // default: none
7705 if (Val == 1) // neg
7706 Mods ^= SISrcMods::NEG;
7707 if (Val == 2) // abs
7708 Mods ^= SISrcMods::ABS;
7709 if (Val == 3) // neg and abs
7710 Mods ^= (SISrcMods::NEG | SISrcMods::ABS);
7711 MIB.addImm(Val: (int64_t)Mods);
7712}
7713
7714void AMDGPUInstructionSelector::renderPrefetchLoc(MachineInstrBuilder &MIB,
7715 const MachineInstr &MI,
7716 int OpIdx) const {
7717 uint32_t V = MI.getOperand(i: 2).getImm();
7718 V = (AMDGPU::CPol::SCOPE_MASK - (V & AMDGPU::CPol::SCOPE_MASK))
7719 << AMDGPU::CPol::SCOPE_SHIFT;
7720 if (!Subtarget->hasSafeCUPrefetch())
7721 V = std::max(a: V, b: (uint32_t)AMDGPU::CPol::SCOPE_SE); // CU scope is unsafe
7722 MIB.addImm(Val: V);
7723}
7724
7725/// Convert from 2-bit value to enum values used for op_sel* source modifiers.
7726void AMDGPUInstructionSelector::renderScaledMAIIntrinsicOperand(
7727 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7728 unsigned Val = MI.getOperand(i: OpIdx).getImm();
7729 unsigned New = 0;
7730 if (Val & 0x1)
7731 New |= SISrcMods::OP_SEL_0;
7732 if (Val & 0x2)
7733 New |= SISrcMods::OP_SEL_1;
7734 MIB.addImm(Val: New);
7735}
7736
7737bool AMDGPUInstructionSelector::isInlineImmediate(const APInt &Imm) const {
7738 return TII.isInlineConstant(Imm);
7739}
7740
7741bool AMDGPUInstructionSelector::isInlineImmediate(const APFloat &Imm) const {
7742 return TII.isInlineConstant(Imm);
7743}
7744