1//===- AMDGPUInstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUInstructionSelector.h"
15#include "AMDGPU.h"
16#include "AMDGPUGlobalISelUtils.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPURegisterBankInfo.h"
19#include "SIMachineFunctionInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
22#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/IntrinsicsAMDGPU.h"
29#include <optional>
30
31#define DEBUG_TYPE "amdgpu-isel"
32
33using namespace llvm;
34using namespace MIPatternMatch;
35
36#define GET_GLOBALISEL_IMPL
37#define AMDGPUSubtarget GCNSubtarget
38#include "AMDGPUGenGlobalISel.inc"
39#undef GET_GLOBALISEL_IMPL
40#undef AMDGPUSubtarget
41
42AMDGPUInstructionSelector::AMDGPUInstructionSelector(
43 const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI)
44 : TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI), STI(STI),
45#define GET_GLOBALISEL_PREDICATES_INIT
46#include "AMDGPUGenGlobalISel.inc"
47#undef GET_GLOBALISEL_PREDICATES_INIT
48#define GET_GLOBALISEL_TEMPORARIES_INIT
49#include "AMDGPUGenGlobalISel.inc"
50#undef GET_GLOBALISEL_TEMPORARIES_INIT
51{
52}
53
54const char *AMDGPUInstructionSelector::getName() { return DEBUG_TYPE; }
55
56void AMDGPUInstructionSelector::setupMF(MachineFunction &MF,
57 GISelValueTracking *VT,
58 CodeGenCoverage *CoverageInfo,
59 ProfileSummaryInfo *PSI,
60 BlockFrequencyInfo *BFI) {
61 MRI = &MF.getRegInfo();
62 Subtarget = &MF.getSubtarget<GCNSubtarget>();
63 Subtarget->checkSubtargetFeatures(F: MF.getFunction());
64 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
65}
66
67// Return the wave level SGPR base address if this is a wave address.
68static Register getWaveAddress(const MachineInstr *Def) {
69 return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS
70 ? Def->getOperand(i: 1).getReg()
71 : Register();
72}
73
74bool AMDGPUInstructionSelector::isVCC(Register Reg,
75 const MachineRegisterInfo &MRI) const {
76 // The verifier is oblivious to s1 being a valid value for wavesize registers.
77 if (Reg.isPhysical())
78 return false;
79
80 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
81 const TargetRegisterClass *RC =
82 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
83 if (RC) {
84 const LLT Ty = MRI.getType(Reg);
85 if (!Ty.isValid() || Ty.getSizeInBits() != 1)
86 return false;
87 // G_TRUNC s1 result is never vcc.
88 return !mi_match(R: Reg, MRI, P: m_GTrunc(Src: m_Reg())) &&
89 RC->hasSuperClassEq(RC: TRI.getBoolRC());
90 }
91
92 const RegisterBank *RB = cast<const RegisterBank *>(Val: RegClassOrBank);
93 return RB->getID() == AMDGPU::VCCRegBankID;
94}
95
96bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(MachineInstr &MI,
97 unsigned NewOpc) const {
98 MI.setDesc(TII.get(Opcode: NewOpc));
99 MI.removeOperand(OpNo: 1); // Remove intrinsic ID.
100 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
101
102 Register DstReg = MI.getOperand(i: 0).getReg();
103 Register SrcReg = MI.getOperand(i: 1).getReg();
104
105 // TODO: This should be legalized to s32 if needed
106 if (MRI->getType(Reg: DstReg) == LLT::scalar(SizeInBits: 1))
107 return false;
108
109 const TargetRegisterClass *DstRC =
110 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
111 const TargetRegisterClass *SrcRC =
112 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
113 if (!DstRC || DstRC != SrcRC)
114 return false;
115
116 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
117 !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
118 return false;
119 const MCInstrDesc &MCID = MI.getDesc();
120 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
121 MI.getOperand(i: 0).setIsEarlyClobber(true);
122 }
123 return true;
124}
125
126bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
127 const DebugLoc &DL = I.getDebugLoc();
128 MachineBasicBlock *BB = I.getParent();
129 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
130
131 Register DstReg = I.getOperand(i: 0).getReg();
132 Register SrcReg = I.getOperand(i: 1).getReg();
133
134 if (isVCC(Reg: DstReg, MRI: *MRI)) {
135 if (SrcReg == AMDGPU::SCC) {
136 const TargetRegisterClass *RC =
137 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
138 if (!RC)
139 return true;
140 return RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI);
141 }
142
143 if (!isVCC(Reg: SrcReg, MRI: *MRI)) {
144 // TODO: Should probably leave the copy and let copyPhysReg expand it.
145 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI))
146 return false;
147
148 const TargetRegisterClass *SrcRC =
149 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
150
151 std::optional<ValueAndVReg> ConstVal =
152 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI, LookThroughInstrs: true);
153 if (ConstVal) {
154 unsigned MovOpc =
155 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
156 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: DstReg)
157 .addImm(Val: ConstVal->Value.getBoolValue() ? -1 : 0);
158 } else {
159 Register MaskedReg = MRI->createVirtualRegister(RegClass: SrcRC);
160
161 // We can't trust the high bits at this point, so clear them.
162
163 // TODO: Skip masking high bits if def is known boolean.
164
165 if (AMDGPU::getRegBitWidth(RCID: SrcRC->getID()) == 16) {
166 assert(Subtarget->useRealTrue16Insts());
167 const int64_t NoMods = 0;
168 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B16_t16_e64), DestReg: MaskedReg)
169 .addImm(Val: NoMods)
170 .addImm(Val: 1)
171 .addImm(Val: NoMods)
172 .addReg(RegNo: SrcReg)
173 .addImm(Val: NoMods);
174 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U16_t16_e64), DestReg: DstReg)
175 .addImm(Val: NoMods)
176 .addImm(Val: 0)
177 .addImm(Val: NoMods)
178 .addReg(RegNo: MaskedReg)
179 .addImm(Val: NoMods);
180 } else {
181 bool IsSGPR = TRI.isSGPRClass(RC: SrcRC);
182 unsigned AndOpc = IsSGPR ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32;
183 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: MaskedReg)
184 .addImm(Val: 1)
185 .addReg(RegNo: SrcReg);
186 if (IsSGPR)
187 And.setOperandDead(3); // Dead scc
188
189 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U32_e64), DestReg: DstReg)
190 .addImm(Val: 0)
191 .addReg(RegNo: MaskedReg);
192 }
193 }
194
195 if (!MRI->getRegClassOrNull(Reg: SrcReg))
196 MRI->setRegClass(Reg: SrcReg, RC: SrcRC);
197 I.eraseFromParent();
198 return true;
199 }
200
201 const TargetRegisterClass *RC =
202 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
203 if (RC && !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
204 return false;
205
206 return true;
207 }
208
209 for (const MachineOperand &MO : I.operands()) {
210 if (MO.getReg().isPhysical())
211 continue;
212
213 const TargetRegisterClass *RC =
214 TRI.getConstrainedRegClassForReg(Reg: MO.getReg(), MRI: *MRI);
215 if (!RC)
216 continue;
217 RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI);
218 }
219 return true;
220}
221
222bool AMDGPUInstructionSelector::selectCOPY_SCC_VCC(MachineInstr &I) const {
223 const DebugLoc &DL = I.getDebugLoc();
224 MachineBasicBlock *BB = I.getParent();
225 Register VCCReg = I.getOperand(i: 1).getReg();
226 MachineInstr *Cmp;
227
228 // Set SCC as a side effect with S_CMP or S_OR.
229 if (STI.hasScalarCompareEq64()) {
230 unsigned CmpOpc =
231 STI.isWave64() ? AMDGPU::S_CMP_LG_U64 : AMDGPU::S_CMP_LG_U32;
232 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: CmpOpc)).addReg(RegNo: VCCReg).addImm(Val: 0);
233 } else {
234 Register DeadDst = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
235 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B64), DestReg: DeadDst)
236 .addReg(RegNo: VCCReg)
237 .addReg(RegNo: VCCReg);
238 }
239
240 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
241
242 Register DstReg = I.getOperand(i: 0).getReg();
243 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: AMDGPU::SCC);
244
245 I.eraseFromParent();
246 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
247}
248
249bool AMDGPUInstructionSelector::selectCOPY_VCC_SCC(MachineInstr &I) const {
250 const DebugLoc &DL = I.getDebugLoc();
251 MachineBasicBlock *BB = I.getParent();
252
253 Register DstReg = I.getOperand(i: 0).getReg();
254 Register SrcReg = I.getOperand(i: 1).getReg();
255 std::optional<ValueAndVReg> Arg =
256 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 1).getReg(), MRI: *MRI);
257
258 if (Arg) {
259 const int64_t Value = Arg->Value.getZExtValue();
260 if (Value == 0) {
261 unsigned Opcode = STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
262 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DstReg).addImm(Val: 0);
263 } else {
264 assert(Value == 1);
265 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: TRI.getExec());
266 }
267 I.eraseFromParent();
268 return RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI);
269 }
270
271 // RegBankLegalize ensures that SrcReg is bool in reg (high bits are 0).
272 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC).addReg(RegNo: SrcReg);
273
274 unsigned SelectOpcode =
275 STI.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
276 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
277 .addReg(RegNo: TRI.getExec())
278 .addImm(Val: 0);
279
280 I.eraseFromParent();
281 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
282 return true;
283}
284
285bool AMDGPUInstructionSelector::selectReadAnyLane(MachineInstr &I) const {
286 Register DstReg = I.getOperand(i: 0).getReg();
287 Register SrcReg = I.getOperand(i: 1).getReg();
288
289 const DebugLoc &DL = I.getDebugLoc();
290 MachineBasicBlock *BB = I.getParent();
291
292 auto RFL = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
293 .addReg(RegNo: SrcReg);
294
295 I.eraseFromParent();
296 constrainSelectedInstRegOperands(I&: *RFL, TII, TRI, RBI);
297 return true;
298}
299
300bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
301 const Register DefReg = I.getOperand(i: 0).getReg();
302 const LLT DefTy = MRI->getType(Reg: DefReg);
303
304 // S1 G_PHIs should not be selected in instruction-select, instead:
305 // - divergent S1 G_PHI should go through lane mask merging algorithm
306 // and be fully inst-selected in AMDGPUGlobalISelDivergenceLowering
307 // - uniform S1 G_PHI should be lowered into S32 G_PHI in AMDGPURegBankSelect
308 if (DefTy == LLT::scalar(SizeInBits: 1))
309 return false;
310
311 // TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy)
312
313 const RegClassOrRegBank &RegClassOrBank =
314 MRI->getRegClassOrRegBank(Reg: DefReg);
315
316 const TargetRegisterClass *DefRC =
317 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
318 if (!DefRC) {
319 if (!DefTy.isValid()) {
320 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
321 return false;
322 }
323
324 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
325 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, Bank: RB);
326 if (!DefRC) {
327 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
328 return false;
329 }
330 }
331
332 // If inputs have register bank, assign corresponding reg class.
333 // Note: registers don't need to have the same reg bank.
334 for (unsigned i = 1; i != I.getNumOperands(); i += 2) {
335 const Register SrcReg = I.getOperand(i).getReg();
336
337 const RegisterBank *RB = MRI->getRegBankOrNull(Reg: SrcReg);
338 if (RB) {
339 const LLT SrcTy = MRI->getType(Reg: SrcReg);
340 const TargetRegisterClass *SrcRC =
341 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *RB);
342 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
343 return false;
344 }
345 }
346
347 I.setDesc(TII.get(Opcode: TargetOpcode::PHI));
348 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
349}
350
351MachineOperand
352AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
353 const TargetRegisterClass &SubRC,
354 unsigned SubIdx) const {
355
356 MachineInstr *MI = MO.getParent();
357 MachineBasicBlock *BB = MO.getParent()->getParent();
358 Register DstReg = MRI->createVirtualRegister(RegClass: &SubRC);
359
360 if (MO.isReg()) {
361 unsigned ComposedSubIdx = TRI.composeSubRegIndices(a: MO.getSubReg(), b: SubIdx);
362 Register Reg = MO.getReg();
363 BuildMI(BB&: *BB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
364 .addReg(RegNo: Reg, Flags: {}, SubReg: ComposedSubIdx);
365
366 return MachineOperand::CreateReg(Reg: DstReg, isDef: MO.isDef(), isImp: MO.isImplicit(),
367 isKill: MO.isKill(), isDead: MO.isDead(), isUndef: MO.isUndef(),
368 isEarlyClobber: MO.isEarlyClobber(), SubReg: 0, isDebug: MO.isDebug(),
369 isInternalRead: MO.isInternalRead());
370 }
371
372 assert(MO.isImm());
373
374 APInt Imm(64, MO.getImm());
375
376 switch (SubIdx) {
377 default:
378 llvm_unreachable("do not know to split immediate with this sub index.");
379 case AMDGPU::sub0:
380 return MachineOperand::CreateImm(Val: Imm.getLoBits(numBits: 32).getSExtValue());
381 case AMDGPU::sub1:
382 return MachineOperand::CreateImm(Val: Imm.getHiBits(numBits: 32).getSExtValue());
383 }
384}
385
386static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
387 switch (Opc) {
388 case AMDGPU::G_AND:
389 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
390 case AMDGPU::G_OR:
391 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32;
392 case AMDGPU::G_XOR:
393 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32;
394 default:
395 llvm_unreachable("not a bit op");
396 }
397}
398
399bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
400 Register DstReg = I.getOperand(i: 0).getReg();
401 unsigned Size = RBI.getSizeInBits(Reg: DstReg, MRI: *MRI, TRI);
402
403 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
404 if (DstRB->getID() != AMDGPU::SGPRRegBankID &&
405 DstRB->getID() != AMDGPU::VCCRegBankID)
406 return false;
407
408 bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID &&
409 STI.isWave64());
410 I.setDesc(TII.get(Opcode: getLogicalBitOpcode(Opc: I.getOpcode(), Is64)));
411
412 // Dead implicit-def of scc
413 I.addOperand(Op: MachineOperand::CreateReg(Reg: AMDGPU::SCC, isDef: true, // isDef
414 isImp: true, // isImp
415 isKill: false, // isKill
416 isDead: true)); // isDead
417 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
418 return true;
419}
420
421bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
422 MachineBasicBlock *BB = I.getParent();
423 MachineFunction *MF = BB->getParent();
424 Register DstReg = I.getOperand(i: 0).getReg();
425 const DebugLoc &DL = I.getDebugLoc();
426 LLT Ty = MRI->getType(Reg: DstReg);
427 if (Ty.isVector())
428 return false;
429
430 unsigned Size = Ty.getSizeInBits();
431 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
432 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID;
433 const bool Sub = I.getOpcode() == TargetOpcode::G_SUB;
434
435 if (Size == 32) {
436 if (IsSALU) {
437 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
438 MachineInstr *Add =
439 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
440 .add(MO: I.getOperand(i: 1))
441 .add(MO: I.getOperand(i: 2))
442 .setOperandDead(3); // Dead scc
443 I.eraseFromParent();
444 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
445 return true;
446 }
447
448 if (STI.hasAddNoCarryInsts()) {
449 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64;
450 I.setDesc(TII.get(Opcode: Opc));
451 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
452 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
453 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
454 return true;
455 }
456
457 const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64;
458
459 Register UnusedCarry = MRI->createVirtualRegister(RegClass: TRI.getWaveMaskRegClass());
460 MachineInstr *Add
461 = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
462 .addDef(RegNo: UnusedCarry, Flags: RegState::Dead)
463 .add(MO: I.getOperand(i: 1))
464 .add(MO: I.getOperand(i: 2))
465 .addImm(Val: 0);
466 I.eraseFromParent();
467 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
468 return true;
469 }
470
471 assert(!Sub && "illegal sub should not reach here");
472
473 const TargetRegisterClass &RC
474 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass;
475 const TargetRegisterClass &HalfRC
476 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass;
477
478 MachineOperand Lo1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
479 MachineOperand Lo2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
480 MachineOperand Hi1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
481 MachineOperand Hi2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
482
483 Register DstLo = MRI->createVirtualRegister(RegClass: &HalfRC);
484 Register DstHi = MRI->createVirtualRegister(RegClass: &HalfRC);
485
486 if (IsSALU) {
487 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_U32), DestReg: DstLo)
488 .add(MO: Lo1)
489 .add(MO: Lo2);
490 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADDC_U32), DestReg: DstHi)
491 .add(MO: Hi1)
492 .add(MO: Hi2)
493 .setOperandDead(3); // Dead scc
494 } else {
495 const TargetRegisterClass *CarryRC = TRI.getWaveMaskRegClass();
496 Register CarryReg = MRI->createVirtualRegister(RegClass: CarryRC);
497 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: DstLo)
498 .addDef(RegNo: CarryReg)
499 .add(MO: Lo1)
500 .add(MO: Lo2)
501 .addImm(Val: 0);
502 MachineInstr *Addc = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: DstHi)
503 .addDef(RegNo: MRI->createVirtualRegister(RegClass: CarryRC), Flags: RegState::Dead)
504 .add(MO: Hi1)
505 .add(MO: Hi2)
506 .addReg(RegNo: CarryReg, Flags: RegState::Kill)
507 .addImm(Val: 0);
508
509 constrainSelectedInstRegOperands(I&: *Addc, TII, TRI, RBI);
510 }
511
512 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
513 .addReg(RegNo: DstLo)
514 .addImm(Val: AMDGPU::sub0)
515 .addReg(RegNo: DstHi)
516 .addImm(Val: AMDGPU::sub1);
517
518
519 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
520 return false;
521
522 I.eraseFromParent();
523 return true;
524}
525
526bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE(
527 MachineInstr &I) const {
528 MachineBasicBlock *BB = I.getParent();
529 MachineFunction *MF = BB->getParent();
530 const DebugLoc &DL = I.getDebugLoc();
531 Register Dst0Reg = I.getOperand(i: 0).getReg();
532 Register Dst1Reg = I.getOperand(i: 1).getReg();
533 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO ||
534 I.getOpcode() == AMDGPU::G_UADDE;
535 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE ||
536 I.getOpcode() == AMDGPU::G_USUBE;
537
538 if (isVCC(Reg: Dst1Reg, MRI: *MRI)) {
539 unsigned NoCarryOpc =
540 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
541 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
542 I.setDesc(TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc));
543 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
544 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
545 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
546 return true;
547 }
548
549 Register Src0Reg = I.getOperand(i: 2).getReg();
550 Register Src1Reg = I.getOperand(i: 3).getReg();
551
552 if (HasCarryIn) {
553 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
554 .addReg(RegNo: I.getOperand(i: 4).getReg());
555 }
556
557 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
558 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
559
560 auto CarryInst = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc), DestReg: Dst0Reg)
561 .add(MO: I.getOperand(i: 2))
562 .add(MO: I.getOperand(i: 3));
563
564 if (MRI->use_nodbg_empty(RegNo: Dst1Reg)) {
565 CarryInst.setOperandDead(3); // Dead scc
566 } else {
567 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst1Reg)
568 .addReg(RegNo: AMDGPU::SCC);
569 if (!MRI->getRegClassOrNull(Reg: Dst1Reg))
570 MRI->setRegClass(Reg: Dst1Reg, RC: &AMDGPU::SReg_32RegClass);
571 }
572
573 if (!RBI.constrainGenericRegister(Reg: Dst0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
574 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
575 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
576 return false;
577
578 if (HasCarryIn &&
579 !RBI.constrainGenericRegister(Reg: I.getOperand(i: 4).getReg(),
580 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
581 return false;
582
583 I.eraseFromParent();
584 return true;
585}
586
587bool AMDGPUInstructionSelector::selectG_AMDGPU_MAD_64_32(
588 MachineInstr &I) const {
589 MachineBasicBlock *BB = I.getParent();
590 MachineFunction *MF = BB->getParent();
591 const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
592 bool UseNoCarry = Subtarget->hasMadNC64_32Insts() &&
593 MRI->use_nodbg_empty(RegNo: I.getOperand(i: 1).getReg());
594
595 unsigned Opc;
596 if (Subtarget->hasMADIntraFwdBug())
597 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64
598 : AMDGPU::V_MAD_I64_I32_gfx11_e64;
599 else if (UseNoCarry)
600 Opc = IsUnsigned ? AMDGPU::V_MAD_NC_U64_U32_e64
601 : AMDGPU::V_MAD_NC_I64_I32_e64;
602 else
603 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64;
604
605 if (UseNoCarry)
606 I.removeOperand(OpNo: 1);
607
608 I.setDesc(TII.get(Opcode: Opc));
609 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
610 I.addImplicitDefUseOperands(MF&: *MF);
611 I.getOperand(i: 0).setIsEarlyClobber(true);
612 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
613 return true;
614}
615
616// TODO: We should probably legalize these to only using 32-bit results.
617bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
618 MachineBasicBlock *BB = I.getParent();
619 Register DstReg = I.getOperand(i: 0).getReg();
620 Register SrcReg = I.getOperand(i: 1).getReg();
621 LLT DstTy = MRI->getType(Reg: DstReg);
622 LLT SrcTy = MRI->getType(Reg: SrcReg);
623 const unsigned SrcSize = SrcTy.getSizeInBits();
624 unsigned DstSize = DstTy.getSizeInBits();
625
626 // TODO: Should handle any multiple of 32 offset.
627 unsigned Offset = I.getOperand(i: 2).getImm();
628 if (Offset % 32 != 0 || DstSize > 128)
629 return false;
630
631 // 16-bit operations really use 32-bit registers.
632 // FIXME: Probably should not allow 16-bit G_EXTRACT results.
633 if (DstSize == 16)
634 DstSize = 32;
635
636 const TargetRegisterClass *DstRC =
637 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
638 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
639 return false;
640
641 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
642 const TargetRegisterClass *SrcRC =
643 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
644 if (!SrcRC)
645 return false;
646 unsigned SubReg = SIRegisterInfo::getSubRegFromChannel(Channel: Offset / 32,
647 NumRegs: DstSize / 32);
648 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubReg);
649 if (!SrcRC)
650 return false;
651
652 SrcReg = constrainOperandRegClass(MF: *MF, TRI, MRI&: *MRI, TII, RBI, InsertPt&: I,
653 RegClass: *SrcRC, RegMO&: I.getOperand(i: 1));
654 const DebugLoc &DL = I.getDebugLoc();
655 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
656 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
657
658 I.eraseFromParent();
659 return true;
660}
661
662bool AMDGPUInstructionSelector::selectS16MergeToS32(MachineInstr &MI) const {
663 Register Dst = MI.getOperand(i: 0).getReg();
664 Register Src0 = MI.getOperand(i: 1).getReg();
665 Register Src1 = MI.getOperand(i: 2).getReg();
666
667 LLT Src0Ty = MRI->getType(Reg: Src0);
668 LLT Src1Ty = MRI->getType(Reg: Src1);
669
670 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
671 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0, MRI: *MRI, TRI);
672 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1, MRI: *MRI, TRI);
673 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
674
675 Register ShiftSrc0;
676 Register ShiftSrc1;
677
678 const DebugLoc &DL = MI.getDebugLoc();
679 MachineBasicBlock *BB = MI.getParent();
680
681 // VGPR case
682 if (IsVector) {
683 // If source are both VGPR16, use REG_SEQUENCE with lo16/hi16 subregisters
684 if (Src0Bank->getID() == AMDGPU::VGPRRegBankID &&
685 Src1Bank->getID() == AMDGPU::VGPRRegBankID &&
686 Src0Ty == LLT::scalar(SizeInBits: 16) && Src1Ty == LLT::scalar(SizeInBits: 16)) {
687 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
688 .addReg(RegNo: Src0)
689 .addImm(Val: AMDGPU::lo16)
690 .addReg(RegNo: Src1)
691 .addImm(Val: AMDGPU::hi16);
692
693 if (!RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
694 return false;
695
696 MI.eraseFromParent();
697 return true;
698 }
699
700 // Otherwise, use V_LSHL_OR_B32_e64
701 Register TmpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
702 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: TmpReg)
703 .addImm(Val: 0xFFFF)
704 .addReg(RegNo: Src0);
705 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
706
707 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: Dst)
708 .addReg(RegNo: Src1)
709 .addImm(Val: 16)
710 .addReg(RegNo: TmpReg);
711 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
712
713 MI.eraseFromParent();
714 return true;
715 }
716
717 // SGPR case -> S_PACK_*_B32_B16
718 // With multiple uses of the shift, this will duplicate the shift and
719 // increase register pressure.
720 //
721 // (merge (lshr_oneuse $src0, 16), (lshr_oneuse $src1, 16)
722 // => (S_PACK_HH_B32_B16 $src0, $src1)
723 // (merge (lshr_oneuse SReg_32:$src0, 16), $src1)
724 // => (S_PACK_HL_B32_B16 $src0, $src1)
725 // (merge $src0, (lshr_oneuse SReg_32:$src1, 16))
726 // => (S_PACK_LH_B32_B16 $src0, $src1)
727 // (merge $src0, $src1)
728 // => (S_PACK_LL_B32_B16 $src0, $src1)
729
730 bool Shift0 = mi_match(
731 R: Src0, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc0), R: m_SpecificICst(RequestedValue: 16))));
732
733 bool Shift1 = mi_match(
734 R: Src1, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc1), R: m_SpecificICst(RequestedValue: 16))));
735
736 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16;
737 if (Shift0 && Shift1) {
738 Opc = AMDGPU::S_PACK_HH_B32_B16;
739 MI.getOperand(i: 1).setReg(ShiftSrc0);
740 MI.getOperand(i: 2).setReg(ShiftSrc1);
741 } else if (Shift1) {
742 Opc = AMDGPU::S_PACK_LH_B32_B16;
743 MI.getOperand(i: 2).setReg(ShiftSrc1);
744 } else if (Shift0) {
745 auto ConstSrc1 =
746 getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
747 if (ConstSrc1 && ConstSrc1->Value == 0) {
748 // build_vector_trunc (lshr $src0, 16), 0 -> s_lshr_b32 $src0, 16
749 auto MIB = BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: Dst)
750 .addReg(RegNo: ShiftSrc0)
751 .addImm(Val: 16)
752 .setOperandDead(3); // Dead scc
753
754 MI.eraseFromParent();
755 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
756 return true;
757 }
758 if (STI.hasSPackHL()) {
759 Opc = AMDGPU::S_PACK_HL_B32_B16;
760 MI.getOperand(i: 1).setReg(ShiftSrc0);
761 }
762 }
763
764 MI.setDesc(TII.get(Opcode: Opc));
765 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
766 return true;
767}
768
769// Pack each pair of s16 into an s32 with S_PACK_LL_B32_B16, then combine the
770// s32 pieces into the destination with a REG_SEQUENCE.
771bool AMDGPUInstructionSelector::selectS16MergeToWide(MachineInstr &MI) const {
772 MachineBasicBlock *BB = MI.getParent();
773 const DebugLoc &DL = MI.getDebugLoc();
774 Register DstReg = MI.getOperand(i: 0).getReg();
775 const unsigned DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
776 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
777 const unsigned NumSrc = MI.getNumOperands() - 1;
778
779 // Pack each pair of s16 sources into an s32.
780 SmallVector<Register, 8> S32Regs;
781 for (unsigned I = 0; I != NumSrc; I += 2) {
782 Register S32 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
783 auto Pack = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_PACK_LL_B32_B16), DestReg: S32)
784 .addReg(RegNo: MI.getOperand(i: I + 1).getReg())
785 .addReg(RegNo: MI.getOperand(i: I + 2).getReg());
786 constrainSelectedInstRegOperands(I&: *Pack, TII, TRI, RBI);
787 S32Regs.push_back(Elt: S32);
788 }
789
790 // Combine the s32 pieces into the destination with a REG_SEQUENCE.
791 const TargetRegisterClass *DstRC =
792 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
793 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
794 return false;
795 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, /*EltSize=*/4);
796 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
797 for (unsigned I = 0, E = S32Regs.size(); I != E; ++I)
798 MIB.addReg(RegNo: S32Regs[I]).addImm(Val: SubRegs[I]);
799
800 MI.eraseFromParent();
801 return true;
802}
803
804bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
805 MachineBasicBlock *BB = MI.getParent();
806 Register DstReg = MI.getOperand(i: 0).getReg();
807 LLT DstTy = MRI->getType(Reg: DstReg);
808 LLT SrcTy = MRI->getType(Reg: MI.getOperand(i: 1).getReg());
809
810 const unsigned SrcSize = SrcTy.getSizeInBits();
811 if (SrcSize < 32) {
812 // Handle s32 <- G_MERGE_VALUES s16, s16
813 if (SrcSize == 16 && DstTy.getSizeInBits() == 32 &&
814 MI.getNumOperands() == 3) {
815 return selectS16MergeToS32(MI);
816 }
817 // With true16 a scalar s16 is a register type, so a scalar wider than 32
818 // bits can be built from s16 pieces.
819 bool IsWideS16Merge = SrcSize == 16 && DstTy.getSizeInBits() > 32 &&
820 DstTy.getSizeInBits() % 32 == 0;
821
822 // SGPRs have no 16-bit subregisters, so pack pairs of s16 with S_PACK.
823 if (IsWideS16Merge &&
824 RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
825 return selectS16MergeToWide(MI);
826
827 // A VGPR wide s16 merge falls through to the generic path below.
828 if (!IsWideS16Merge)
829 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
830 }
831
832 const DebugLoc &DL = MI.getDebugLoc();
833 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
834 const unsigned DstSize = DstTy.getSizeInBits();
835 const TargetRegisterClass *DstRC =
836 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
837 if (!DstRC)
838 return false;
839
840 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, EltSize: SrcSize / 8);
841 MachineInstrBuilder MIB =
842 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
843 for (int I = 0, E = MI.getNumOperands() - 1; I != E; ++I) {
844 MachineOperand &Src = MI.getOperand(i: I + 1);
845 Register SrcReg = Src.getReg();
846 MIB.addReg(RegNo: SrcReg, Flags: getUndefRegState(B: Src.isUndef()));
847 MIB.addImm(Val: SubRegs[I]);
848
849 const TargetRegisterClass *SrcRC =
850 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
851 if (SrcRC && !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
852 return false;
853 }
854
855 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
856 return false;
857
858 MI.eraseFromParent();
859 return true;
860}
861
862bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
863 MachineBasicBlock *BB = MI.getParent();
864 const int NumDst = MI.getNumOperands() - 1;
865
866 MachineOperand &Src = MI.getOperand(i: NumDst);
867
868 Register SrcReg = Src.getReg();
869 Register DstReg0 = MI.getOperand(i: 0).getReg();
870 LLT DstTy = MRI->getType(Reg: DstReg0);
871 LLT SrcTy = MRI->getType(Reg: SrcReg);
872
873 const unsigned DstSize = DstTy.getSizeInBits();
874 const unsigned SrcSize = SrcTy.getSizeInBits();
875 const DebugLoc &DL = MI.getDebugLoc();
876 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
877
878 const TargetRegisterClass *SrcRC =
879 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
880 if (!SrcRC || !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
881 return false;
882
883 // Note we could have mixed SGPR and VGPR destination banks for an SGPR
884 // source, and this relies on the fact that the same subregister indices are
885 // used for both.
886 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SrcRC, EltSize: DstSize / 8);
887 for (int I = 0, E = NumDst; I != E; ++I) {
888 Register DstReg = MI.getOperand(i: I).getReg();
889 // hi16:sreg_32 is not allowed so explicitly shift upper 16-bits.
890 if (SrcBank->getID() == AMDGPU::SGPRRegBankID &&
891 SubRegs[I] == AMDGPU::hi16) {
892 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: DstReg)
893 .addReg(RegNo: SrcReg)
894 .addImm(Val: 16);
895 } else {
896 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
897 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegs[I]);
898 }
899
900 // Make sure the subregister index is valid for the source register.
901 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[I]);
902 if (!SrcRC || !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
903 return false;
904
905 const TargetRegisterClass *DstRC =
906 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
907 if (DstRC && !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
908 return false;
909 }
910
911 MI.eraseFromParent();
912 return true;
913}
914
915bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR(MachineInstr &MI) const {
916 assert(MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC ||
917 MI.getOpcode() == AMDGPU::G_BUILD_VECTOR);
918
919 Register Src0 = MI.getOperand(i: 1).getReg();
920 Register Src1 = MI.getOperand(i: 2).getReg();
921 LLT SrcTy = MRI->getType(Reg: Src0);
922 const unsigned SrcSize = SrcTy.getSizeInBits();
923
924 // BUILD_VECTOR with >=32 bits source is handled by MERGE_VALUE.
925 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR && SrcSize >= 32) {
926 return selectG_MERGE_VALUES(MI);
927 }
928
929 // Selection logic below is for V2S16 only.
930 // For G_BUILD_VECTOR_TRUNC, additionally check that the operands are s32.
931 Register Dst = MI.getOperand(i: 0).getReg();
932 if (MRI->getType(Reg: Dst) != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
933 (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC &&
934 SrcTy != LLT::scalar(SizeInBits: 32)))
935 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
936
937 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
938 if (DstBank->getID() == AMDGPU::AGPRRegBankID)
939 return false;
940
941 assert(DstBank->getID() == AMDGPU::SGPRRegBankID ||
942 DstBank->getID() == AMDGPU::VGPRRegBankID);
943 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
944
945 const DebugLoc &DL = MI.getDebugLoc();
946 MachineBasicBlock *BB = MI.getParent();
947
948 // First, before trying TableGen patterns, check if both sources are
949 // constants. In those cases, we can trivially compute the final constant
950 // and emit a simple move.
951 auto ConstSrc1 = getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
952 if (ConstSrc1) {
953 auto ConstSrc0 =
954 getAnyConstantVRegValWithLookThrough(VReg: Src0, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
955 if (ConstSrc0) {
956 const int64_t K0 = ConstSrc0->Value.getSExtValue();
957 const int64_t K1 = ConstSrc1->Value.getSExtValue();
958 uint32_t Lo16 = static_cast<uint32_t>(K0) & 0xffff;
959 uint32_t Hi16 = static_cast<uint32_t>(K1) & 0xffff;
960 uint32_t Imm = Lo16 | (Hi16 << 16);
961
962 // VALU
963 if (IsVector) {
964 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: Dst).addImm(Val: Imm);
965 MI.eraseFromParent();
966 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI);
967 }
968
969 // SALU
970 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: Dst).addImm(Val: Imm);
971 MI.eraseFromParent();
972 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
973 }
974 }
975
976 // Now try TableGen patterns.
977 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
978 return true;
979
980 // TODO: This should probably be a combine somewhere
981 // (build_vector $src0, undef) -> copy $src0
982 MachineInstr *Src1Def = getDefIgnoringCopies(Reg: Src1, MRI: *MRI);
983 if (Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) {
984 MI.setDesc(TII.get(Opcode: AMDGPU::COPY));
985 MI.removeOperand(OpNo: 2);
986 const auto &RC =
987 IsVector ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
988 return RBI.constrainGenericRegister(Reg: Dst, RC, MRI&: *MRI) &&
989 RBI.constrainGenericRegister(Reg: Src0, RC, MRI&: *MRI);
990 }
991
992 return selectS16MergeToS32(MI);
993}
994
995bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const {
996 const MachineOperand &MO = I.getOperand(i: 0);
997
998 // FIXME: Interface for getConstrainedRegClassForReg needs work. The
999 // regbank check here is to know why getConstrainedRegClassForReg failed.
1000 const TargetRegisterClass *RC =
1001 TRI.getConstrainedRegClassForReg(Reg: MO.getReg(), MRI: *MRI);
1002 if ((!RC && !MRI->getRegBankOrNull(Reg: MO.getReg())) ||
1003 (RC && RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI))) {
1004 I.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1005 return true;
1006 }
1007
1008 return false;
1009}
1010
1011bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
1012 MachineBasicBlock *BB = I.getParent();
1013
1014 Register DstReg = I.getOperand(i: 0).getReg();
1015 Register Src0Reg = I.getOperand(i: 1).getReg();
1016 Register Src1Reg = I.getOperand(i: 2).getReg();
1017 LLT Src1Ty = MRI->getType(Reg: Src1Reg);
1018
1019 unsigned DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
1020 unsigned InsSize = Src1Ty.getSizeInBits();
1021
1022 int64_t Offset = I.getOperand(i: 3).getImm();
1023
1024 // FIXME: These cases should have been illegal and unnecessary to check here.
1025 if (Offset % 32 != 0 || InsSize % 32 != 0)
1026 return false;
1027
1028 // Currently not handled by getSubRegFromChannel.
1029 if (InsSize > 128)
1030 return false;
1031
1032 unsigned SubReg = TRI.getSubRegFromChannel(Channel: Offset / 32, NumRegs: InsSize / 32);
1033 if (SubReg == AMDGPU::NoSubRegister)
1034 return false;
1035
1036 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1037 const TargetRegisterClass *DstRC =
1038 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
1039 if (!DstRC)
1040 return false;
1041
1042 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0Reg, MRI: *MRI, TRI);
1043 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1Reg, MRI: *MRI, TRI);
1044 const TargetRegisterClass *Src0RC =
1045 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *Src0Bank);
1046 const TargetRegisterClass *Src1RC =
1047 TRI.getRegClassForSizeOnBank(Size: InsSize, Bank: *Src1Bank);
1048
1049 // Deal with weird cases where the class only partially supports the subreg
1050 // index.
1051 Src0RC = TRI.getSubClassWithSubReg(Src0RC, SubReg);
1052 if (!Src0RC || !Src1RC)
1053 return false;
1054
1055 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
1056 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: *Src0RC, MRI&: *MRI) ||
1057 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: *Src1RC, MRI&: *MRI))
1058 return false;
1059
1060 const DebugLoc &DL = I.getDebugLoc();
1061 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1062 .addReg(RegNo: Src0Reg)
1063 .addReg(RegNo: Src1Reg)
1064 .addImm(Val: SubReg);
1065
1066 I.eraseFromParent();
1067 return true;
1068}
1069
1070bool AMDGPUInstructionSelector::selectG_SBFX_UBFX(MachineInstr &MI) const {
1071 Register DstReg = MI.getOperand(i: 0).getReg();
1072 Register SrcReg = MI.getOperand(i: 1).getReg();
1073 Register OffsetReg = MI.getOperand(i: 2).getReg();
1074 Register WidthReg = MI.getOperand(i: 3).getReg();
1075
1076 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID &&
1077 "scalar BFX instructions are expanded in regbankselect");
1078 assert(MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 &&
1079 "64-bit vector BFX instructions are expanded in regbankselect");
1080
1081 const DebugLoc &DL = MI.getDebugLoc();
1082 MachineBasicBlock *MBB = MI.getParent();
1083
1084 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SBFX;
1085 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
1086 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
1087 .addReg(RegNo: SrcReg)
1088 .addReg(RegNo: OffsetReg)
1089 .addReg(RegNo: WidthReg);
1090 MI.eraseFromParent();
1091 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1092 return true;
1093}
1094
1095bool AMDGPUInstructionSelector::selectInterpP1F16(MachineInstr &MI) const {
1096 if (STI.getLDSBankCount() != 16)
1097 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1098
1099 Register Dst = MI.getOperand(i: 0).getReg();
1100 Register Src0 = MI.getOperand(i: 2).getReg();
1101 Register M0Val = MI.getOperand(i: 6).getReg();
1102 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
1103 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI) ||
1104 !RBI.constrainGenericRegister(Reg: Src0, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
1105 return false;
1106
1107 // This requires 2 instructions. It is possible to write a pattern to support
1108 // this, but the generated isel emitter doesn't correctly deal with multiple
1109 // output instructions using the same physical register input. The copy to m0
1110 // is incorrectly placed before the second instruction.
1111 //
1112 // TODO: Match source modifiers.
1113
1114 Register InterpMov = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1115 const DebugLoc &DL = MI.getDebugLoc();
1116 MachineBasicBlock *MBB = MI.getParent();
1117
1118 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1119 .addReg(RegNo: M0Val);
1120 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_MOV_F32), DestReg: InterpMov)
1121 .addImm(Val: 2)
1122 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1123 .addImm(Val: MI.getOperand(i: 3).getImm()); // $attrchan
1124
1125 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_P1LV_F16), DestReg: Dst)
1126 .addImm(Val: 0) // $src0_modifiers
1127 .addReg(RegNo: Src0) // $src0
1128 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1129 .addImm(Val: MI.getOperand(i: 3).getImm()) // $attrchan
1130 .addImm(Val: 0) // $src2_modifiers
1131 .addReg(RegNo: InterpMov) // $src2 - 2 f16 values selected by high
1132 .addImm(Val: MI.getOperand(i: 5).getImm()) // $high
1133 .addImm(Val: 0) // $clamp
1134 .addImm(Val: 0); // $omod
1135
1136 MI.eraseFromParent();
1137 return true;
1138}
1139
1140// Writelane is special in that it can use SGPR and M0 (which would normally
1141// count as using the constant bus twice - but in this case it is allowed since
1142// the lane selector doesn't count as a use of the constant bus). However, it is
1143// still required to abide by the 1 SGPR rule. Fix this up if we might have
1144// multiple SGPRs.
1145bool AMDGPUInstructionSelector::selectWritelane(MachineInstr &MI) const {
1146 // With a constant bus limit of at least 2, there's no issue.
1147 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_WRITELANE_B32) > 1)
1148 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1149
1150 MachineBasicBlock *MBB = MI.getParent();
1151 const DebugLoc &DL = MI.getDebugLoc();
1152 Register VDst = MI.getOperand(i: 0).getReg();
1153 Register Val = MI.getOperand(i: 2).getReg();
1154 Register LaneSelect = MI.getOperand(i: 3).getReg();
1155 Register VDstIn = MI.getOperand(i: 4).getReg();
1156
1157 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_WRITELANE_B32), DestReg: VDst);
1158
1159 std::optional<ValueAndVReg> ConstSelect =
1160 getIConstantVRegValWithLookThrough(VReg: LaneSelect, MRI: *MRI);
1161 if (ConstSelect) {
1162 // The selector has to be an inline immediate, so we can use whatever for
1163 // the other operands.
1164 MIB.addReg(RegNo: Val);
1165 MIB.addImm(Val: ConstSelect->Value.getSExtValue() &
1166 maskTrailingOnes<uint64_t>(N: STI.getWavefrontSizeLog2()));
1167 } else {
1168 std::optional<ValueAndVReg> ConstVal =
1169 getIConstantVRegValWithLookThrough(VReg: Val, MRI: *MRI);
1170
1171 // If the value written is an inline immediate, we can get away without a
1172 // copy to m0.
1173 if (ConstVal && AMDGPU::isInlinableLiteral32(Literal: ConstVal->Value.getSExtValue(),
1174 HasInv2Pi: STI.hasInv2PiInlineImm())) {
1175 MIB.addImm(Val: ConstVal->Value.getSExtValue());
1176 MIB.addReg(RegNo: LaneSelect);
1177 } else {
1178 MIB.addReg(RegNo: Val);
1179
1180 // If the lane selector was originally in a VGPR and copied with
1181 // readfirstlane, there's a hazard to read the same SGPR from the
1182 // VALU. Constrain to a different SGPR to help avoid needing a nop later.
1183 RBI.constrainGenericRegister(Reg: LaneSelect, RC: AMDGPU::SReg_32_XM0RegClass, MRI&: *MRI);
1184
1185 BuildMI(BB&: *MBB, I&: *MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1186 .addReg(RegNo: LaneSelect);
1187 MIB.addReg(RegNo: AMDGPU::M0);
1188 }
1189 }
1190
1191 MIB.addReg(RegNo: VDstIn);
1192
1193 MI.eraseFromParent();
1194 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1195 return true;
1196}
1197
1198// We need to handle this here because tablegen doesn't support matching
1199// instructions with multiple outputs.
1200bool AMDGPUInstructionSelector::selectDivScale(MachineInstr &MI) const {
1201 Register Dst0 = MI.getOperand(i: 0).getReg();
1202 Register Dst1 = MI.getOperand(i: 1).getReg();
1203
1204 LLT Ty = MRI->getType(Reg: Dst0);
1205 unsigned Opc;
1206 if (Ty == LLT::scalar(SizeInBits: 32))
1207 Opc = AMDGPU::V_DIV_SCALE_F32_e64;
1208 else if (Ty == LLT::scalar(SizeInBits: 64))
1209 Opc = AMDGPU::V_DIV_SCALE_F64_e64;
1210 else
1211 return false;
1212
1213 // TODO: Match source modifiers.
1214
1215 const DebugLoc &DL = MI.getDebugLoc();
1216 MachineBasicBlock *MBB = MI.getParent();
1217
1218 Register Numer = MI.getOperand(i: 3).getReg();
1219 Register Denom = MI.getOperand(i: 4).getReg();
1220 unsigned ChooseDenom = MI.getOperand(i: 5).getImm();
1221
1222 Register Src0 = ChooseDenom != 0 ? Numer : Denom;
1223
1224 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
1225 .addDef(RegNo: Dst1)
1226 .addImm(Val: 0) // $src0_modifiers
1227 .addUse(RegNo: Src0) // $src0
1228 .addImm(Val: 0) // $src1_modifiers
1229 .addUse(RegNo: Denom) // $src1
1230 .addImm(Val: 0) // $src2_modifiers
1231 .addUse(RegNo: Numer) // $src2
1232 .addImm(Val: 0) // $clamp
1233 .addImm(Val: 0); // $omod
1234
1235 MI.eraseFromParent();
1236 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1237 return true;
1238}
1239
1240bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
1241 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
1242 switch (IntrinsicID) {
1243 case Intrinsic::amdgcn_if_break: {
1244 MachineBasicBlock *BB = I.getParent();
1245
1246 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1247 // SelectionDAG uses for wave32 vs wave64.
1248 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_IF_BREAK))
1249 .add(MO: I.getOperand(i: 0))
1250 .add(MO: I.getOperand(i: 2))
1251 .add(MO: I.getOperand(i: 3));
1252
1253 Register DstReg = I.getOperand(i: 0).getReg();
1254 Register Src0Reg = I.getOperand(i: 2).getReg();
1255 Register Src1Reg = I.getOperand(i: 3).getReg();
1256
1257 I.eraseFromParent();
1258
1259 for (Register Reg : { DstReg, Src0Reg, Src1Reg })
1260 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1261
1262 return true;
1263 }
1264 case Intrinsic::amdgcn_interp_p1_f16:
1265 return selectInterpP1F16(MI&: I);
1266 case Intrinsic::amdgcn_wqm:
1267 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::WQM);
1268 case Intrinsic::amdgcn_softwqm:
1269 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::SOFT_WQM);
1270 case Intrinsic::amdgcn_strict_wwm:
1271 case Intrinsic::amdgcn_wwm:
1272 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WWM);
1273 case Intrinsic::amdgcn_strict_wqm:
1274 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WQM);
1275 case Intrinsic::amdgcn_writelane:
1276 return selectWritelane(MI&: I);
1277 case Intrinsic::amdgcn_div_scale:
1278 return selectDivScale(MI&: I);
1279 case Intrinsic::amdgcn_ballot:
1280 return selectBallot(I);
1281 case Intrinsic::amdgcn_reloc_constant:
1282 return selectRelocConstant(I);
1283 case Intrinsic::amdgcn_groupstaticsize:
1284 return selectGroupStaticSize(I);
1285 case Intrinsic::returnaddress:
1286 return selectReturnAddress(I);
1287 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
1288 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
1289 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
1290 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
1291 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
1292 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
1293 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
1294 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
1295 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
1296 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
1297 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
1298 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
1299 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
1300 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
1301 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
1302 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
1303 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
1304 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
1305 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
1306 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
1307 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
1308 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
1309 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
1310 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
1311 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
1312 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
1313 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
1314 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
1315 return selectSMFMACIntrin(I);
1316 case Intrinsic::amdgcn_permlane16_swap:
1317 case Intrinsic::amdgcn_permlane32_swap:
1318 return selectPermlaneSwapIntrin(I, IntrID: IntrinsicID);
1319 case Intrinsic::amdgcn_wave_shuffle:
1320 return selectWaveShuffleIntrin(I);
1321 default:
1322 return selectImpl(I, CoverageInfo&: *CoverageInfo);
1323 }
1324}
1325
1326static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size,
1327 const GCNSubtarget &ST) {
1328 if (Size != 16 && Size != 32 && Size != 64)
1329 return -1;
1330
1331 if (Size == 16 && !ST.has16BitInsts())
1332 return -1;
1333
1334 const auto Select = [&](unsigned S16Opc, unsigned TrueS16Opc,
1335 unsigned FakeS16Opc, unsigned S32Opc,
1336 unsigned S64Opc) {
1337 if (Size == 16)
1338 return ST.hasTrue16BitInsts()
1339 ? ST.useRealTrue16Insts() ? TrueS16Opc : FakeS16Opc
1340 : S16Opc;
1341 if (Size == 32)
1342 return S32Opc;
1343 return S64Opc;
1344 };
1345
1346 switch (P) {
1347 default:
1348 llvm_unreachable("Unknown condition code!");
1349 case CmpInst::ICMP_NE:
1350 return Select(AMDGPU::V_CMP_NE_U16_e64, AMDGPU::V_CMP_NE_U16_t16_e64,
1351 AMDGPU::V_CMP_NE_U16_fake16_e64, AMDGPU::V_CMP_NE_U32_e64,
1352 AMDGPU::V_CMP_NE_U64_e64);
1353 case CmpInst::ICMP_EQ:
1354 return Select(AMDGPU::V_CMP_EQ_U16_e64, AMDGPU::V_CMP_EQ_U16_t16_e64,
1355 AMDGPU::V_CMP_EQ_U16_fake16_e64, AMDGPU::V_CMP_EQ_U32_e64,
1356 AMDGPU::V_CMP_EQ_U64_e64);
1357 case CmpInst::ICMP_SGT:
1358 return Select(AMDGPU::V_CMP_GT_I16_e64, AMDGPU::V_CMP_GT_I16_t16_e64,
1359 AMDGPU::V_CMP_GT_I16_fake16_e64, AMDGPU::V_CMP_GT_I32_e64,
1360 AMDGPU::V_CMP_GT_I64_e64);
1361 case CmpInst::ICMP_SGE:
1362 return Select(AMDGPU::V_CMP_GE_I16_e64, AMDGPU::V_CMP_GE_I16_t16_e64,
1363 AMDGPU::V_CMP_GE_I16_fake16_e64, AMDGPU::V_CMP_GE_I32_e64,
1364 AMDGPU::V_CMP_GE_I64_e64);
1365 case CmpInst::ICMP_SLT:
1366 return Select(AMDGPU::V_CMP_LT_I16_e64, AMDGPU::V_CMP_LT_I16_t16_e64,
1367 AMDGPU::V_CMP_LT_I16_fake16_e64, AMDGPU::V_CMP_LT_I32_e64,
1368 AMDGPU::V_CMP_LT_I64_e64);
1369 case CmpInst::ICMP_SLE:
1370 return Select(AMDGPU::V_CMP_LE_I16_e64, AMDGPU::V_CMP_LE_I16_t16_e64,
1371 AMDGPU::V_CMP_LE_I16_fake16_e64, AMDGPU::V_CMP_LE_I32_e64,
1372 AMDGPU::V_CMP_LE_I64_e64);
1373 case CmpInst::ICMP_UGT:
1374 return Select(AMDGPU::V_CMP_GT_U16_e64, AMDGPU::V_CMP_GT_U16_t16_e64,
1375 AMDGPU::V_CMP_GT_U16_fake16_e64, AMDGPU::V_CMP_GT_U32_e64,
1376 AMDGPU::V_CMP_GT_U64_e64);
1377 case CmpInst::ICMP_UGE:
1378 return Select(AMDGPU::V_CMP_GE_U16_e64, AMDGPU::V_CMP_GE_U16_t16_e64,
1379 AMDGPU::V_CMP_GE_U16_fake16_e64, AMDGPU::V_CMP_GE_U32_e64,
1380 AMDGPU::V_CMP_GE_U64_e64);
1381 case CmpInst::ICMP_ULT:
1382 return Select(AMDGPU::V_CMP_LT_U16_e64, AMDGPU::V_CMP_LT_U16_t16_e64,
1383 AMDGPU::V_CMP_LT_U16_fake16_e64, AMDGPU::V_CMP_LT_U32_e64,
1384 AMDGPU::V_CMP_LT_U64_e64);
1385 case CmpInst::ICMP_ULE:
1386 return Select(AMDGPU::V_CMP_LE_U16_e64, AMDGPU::V_CMP_LE_U16_t16_e64,
1387 AMDGPU::V_CMP_LE_U16_fake16_e64, AMDGPU::V_CMP_LE_U32_e64,
1388 AMDGPU::V_CMP_LE_U64_e64);
1389
1390 case CmpInst::FCMP_OEQ:
1391 return Select(AMDGPU::V_CMP_EQ_F16_e64, AMDGPU::V_CMP_EQ_F16_t16_e64,
1392 AMDGPU::V_CMP_EQ_F16_fake16_e64, AMDGPU::V_CMP_EQ_F32_e64,
1393 AMDGPU::V_CMP_EQ_F64_e64);
1394 case CmpInst::FCMP_OGT:
1395 return Select(AMDGPU::V_CMP_GT_F16_e64, AMDGPU::V_CMP_GT_F16_t16_e64,
1396 AMDGPU::V_CMP_GT_F16_fake16_e64, AMDGPU::V_CMP_GT_F32_e64,
1397 AMDGPU::V_CMP_GT_F64_e64);
1398 case CmpInst::FCMP_OGE:
1399 return Select(AMDGPU::V_CMP_GE_F16_e64, AMDGPU::V_CMP_GE_F16_t16_e64,
1400 AMDGPU::V_CMP_GE_F16_fake16_e64, AMDGPU::V_CMP_GE_F32_e64,
1401 AMDGPU::V_CMP_GE_F64_e64);
1402 case CmpInst::FCMP_OLT:
1403 return Select(AMDGPU::V_CMP_LT_F16_e64, AMDGPU::V_CMP_LT_F16_t16_e64,
1404 AMDGPU::V_CMP_LT_F16_fake16_e64, AMDGPU::V_CMP_LT_F32_e64,
1405 AMDGPU::V_CMP_LT_F64_e64);
1406 case CmpInst::FCMP_OLE:
1407 return Select(AMDGPU::V_CMP_LE_F16_e64, AMDGPU::V_CMP_LE_F16_t16_e64,
1408 AMDGPU::V_CMP_LE_F16_fake16_e64, AMDGPU::V_CMP_LE_F32_e64,
1409 AMDGPU::V_CMP_LE_F64_e64);
1410 case CmpInst::FCMP_ONE:
1411 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1412 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1413 AMDGPU::V_CMP_NEQ_F64_e64);
1414 case CmpInst::FCMP_ORD:
1415 return Select(AMDGPU::V_CMP_O_F16_e64, AMDGPU::V_CMP_O_F16_t16_e64,
1416 AMDGPU::V_CMP_O_F16_fake16_e64, AMDGPU::V_CMP_O_F32_e64,
1417 AMDGPU::V_CMP_O_F64_e64);
1418 case CmpInst::FCMP_UNO:
1419 return Select(AMDGPU::V_CMP_U_F16_e64, AMDGPU::V_CMP_U_F16_t16_e64,
1420 AMDGPU::V_CMP_U_F16_fake16_e64, AMDGPU::V_CMP_U_F32_e64,
1421 AMDGPU::V_CMP_U_F64_e64);
1422 case CmpInst::FCMP_UEQ:
1423 return Select(AMDGPU::V_CMP_NLG_F16_e64, AMDGPU::V_CMP_NLG_F16_t16_e64,
1424 AMDGPU::V_CMP_NLG_F16_fake16_e64, AMDGPU::V_CMP_NLG_F32_e64,
1425 AMDGPU::V_CMP_NLG_F64_e64);
1426 case CmpInst::FCMP_UGT:
1427 return Select(AMDGPU::V_CMP_NLE_F16_e64, AMDGPU::V_CMP_NLE_F16_t16_e64,
1428 AMDGPU::V_CMP_NLE_F16_fake16_e64, AMDGPU::V_CMP_NLE_F32_e64,
1429 AMDGPU::V_CMP_NLE_F64_e64);
1430 case CmpInst::FCMP_UGE:
1431 return Select(AMDGPU::V_CMP_NLT_F16_e64, AMDGPU::V_CMP_NLT_F16_t16_e64,
1432 AMDGPU::V_CMP_NLT_F16_fake16_e64, AMDGPU::V_CMP_NLT_F32_e64,
1433 AMDGPU::V_CMP_NLT_F64_e64);
1434 case CmpInst::FCMP_ULT:
1435 return Select(AMDGPU::V_CMP_NGE_F16_e64, AMDGPU::V_CMP_NGE_F16_t16_e64,
1436 AMDGPU::V_CMP_NGE_F16_fake16_e64, AMDGPU::V_CMP_NGE_F32_e64,
1437 AMDGPU::V_CMP_NGE_F64_e64);
1438 case CmpInst::FCMP_ULE:
1439 return Select(AMDGPU::V_CMP_NGT_F16_e64, AMDGPU::V_CMP_NGT_F16_t16_e64,
1440 AMDGPU::V_CMP_NGT_F16_fake16_e64, AMDGPU::V_CMP_NGT_F32_e64,
1441 AMDGPU::V_CMP_NGT_F64_e64);
1442 case CmpInst::FCMP_UNE:
1443 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1444 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1445 AMDGPU::V_CMP_NEQ_F64_e64);
1446 case CmpInst::FCMP_TRUE:
1447 return Select(AMDGPU::V_CMP_TRU_F16_e64, AMDGPU::V_CMP_TRU_F16_t16_e64,
1448 AMDGPU::V_CMP_TRU_F16_fake16_e64, AMDGPU::V_CMP_TRU_F32_e64,
1449 AMDGPU::V_CMP_TRU_F64_e64);
1450 case CmpInst::FCMP_FALSE:
1451 return Select(AMDGPU::V_CMP_F_F16_e64, AMDGPU::V_CMP_F_F16_t16_e64,
1452 AMDGPU::V_CMP_F_F16_fake16_e64, AMDGPU::V_CMP_F_F32_e64,
1453 AMDGPU::V_CMP_F_F64_e64);
1454 }
1455}
1456
1457int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
1458 unsigned Size) const {
1459 if (Size == 64) {
1460 if (!STI.hasScalarCompareEq64())
1461 return -1;
1462
1463 switch (P) {
1464 case CmpInst::ICMP_NE:
1465 return AMDGPU::S_CMP_LG_U64;
1466 case CmpInst::ICMP_EQ:
1467 return AMDGPU::S_CMP_EQ_U64;
1468 default:
1469 return -1;
1470 }
1471 }
1472
1473 if (Size == 32) {
1474 switch (P) {
1475 case CmpInst::ICMP_NE:
1476 return AMDGPU::S_CMP_LG_U32;
1477 case CmpInst::ICMP_EQ:
1478 return AMDGPU::S_CMP_EQ_U32;
1479 case CmpInst::ICMP_SGT:
1480 return AMDGPU::S_CMP_GT_I32;
1481 case CmpInst::ICMP_SGE:
1482 return AMDGPU::S_CMP_GE_I32;
1483 case CmpInst::ICMP_SLT:
1484 return AMDGPU::S_CMP_LT_I32;
1485 case CmpInst::ICMP_SLE:
1486 return AMDGPU::S_CMP_LE_I32;
1487 case CmpInst::ICMP_UGT:
1488 return AMDGPU::S_CMP_GT_U32;
1489 case CmpInst::ICMP_UGE:
1490 return AMDGPU::S_CMP_GE_U32;
1491 case CmpInst::ICMP_ULT:
1492 return AMDGPU::S_CMP_LT_U32;
1493 case CmpInst::ICMP_ULE:
1494 return AMDGPU::S_CMP_LE_U32;
1495 case CmpInst::FCMP_OEQ:
1496 return AMDGPU::S_CMP_EQ_F32;
1497 case CmpInst::FCMP_OGT:
1498 return AMDGPU::S_CMP_GT_F32;
1499 case CmpInst::FCMP_OGE:
1500 return AMDGPU::S_CMP_GE_F32;
1501 case CmpInst::FCMP_OLT:
1502 return AMDGPU::S_CMP_LT_F32;
1503 case CmpInst::FCMP_OLE:
1504 return AMDGPU::S_CMP_LE_F32;
1505 case CmpInst::FCMP_ONE:
1506 return AMDGPU::S_CMP_LG_F32;
1507 case CmpInst::FCMP_ORD:
1508 return AMDGPU::S_CMP_O_F32;
1509 case CmpInst::FCMP_UNO:
1510 return AMDGPU::S_CMP_U_F32;
1511 case CmpInst::FCMP_UEQ:
1512 return AMDGPU::S_CMP_NLG_F32;
1513 case CmpInst::FCMP_UGT:
1514 return AMDGPU::S_CMP_NLE_F32;
1515 case CmpInst::FCMP_UGE:
1516 return AMDGPU::S_CMP_NLT_F32;
1517 case CmpInst::FCMP_ULT:
1518 return AMDGPU::S_CMP_NGE_F32;
1519 case CmpInst::FCMP_ULE:
1520 return AMDGPU::S_CMP_NGT_F32;
1521 case CmpInst::FCMP_UNE:
1522 return AMDGPU::S_CMP_NEQ_F32;
1523 default:
1524 llvm_unreachable("Unknown condition code!");
1525 }
1526 }
1527
1528 if (Size == 16) {
1529 if (!STI.hasSALUFloatInsts())
1530 return -1;
1531
1532 switch (P) {
1533 case CmpInst::FCMP_OEQ:
1534 return AMDGPU::S_CMP_EQ_F16;
1535 case CmpInst::FCMP_OGT:
1536 return AMDGPU::S_CMP_GT_F16;
1537 case CmpInst::FCMP_OGE:
1538 return AMDGPU::S_CMP_GE_F16;
1539 case CmpInst::FCMP_OLT:
1540 return AMDGPU::S_CMP_LT_F16;
1541 case CmpInst::FCMP_OLE:
1542 return AMDGPU::S_CMP_LE_F16;
1543 case CmpInst::FCMP_ONE:
1544 return AMDGPU::S_CMP_LG_F16;
1545 case CmpInst::FCMP_ORD:
1546 return AMDGPU::S_CMP_O_F16;
1547 case CmpInst::FCMP_UNO:
1548 return AMDGPU::S_CMP_U_F16;
1549 case CmpInst::FCMP_UEQ:
1550 return AMDGPU::S_CMP_NLG_F16;
1551 case CmpInst::FCMP_UGT:
1552 return AMDGPU::S_CMP_NLE_F16;
1553 case CmpInst::FCMP_UGE:
1554 return AMDGPU::S_CMP_NLT_F16;
1555 case CmpInst::FCMP_ULT:
1556 return AMDGPU::S_CMP_NGE_F16;
1557 case CmpInst::FCMP_ULE:
1558 return AMDGPU::S_CMP_NGT_F16;
1559 case CmpInst::FCMP_UNE:
1560 return AMDGPU::S_CMP_NEQ_F16;
1561 default:
1562 llvm_unreachable("Unknown condition code!");
1563 }
1564 }
1565
1566 return -1;
1567}
1568
1569bool AMDGPUInstructionSelector::selectG_ICMP_or_FCMP(MachineInstr &I) const {
1570
1571 MachineBasicBlock *BB = I.getParent();
1572 const DebugLoc &DL = I.getDebugLoc();
1573
1574 Register SrcReg = I.getOperand(i: 2).getReg();
1575 unsigned Size = RBI.getSizeInBits(Reg: SrcReg, MRI: *MRI, TRI);
1576
1577 auto Pred = (CmpInst::Predicate)I.getOperand(i: 1).getPredicate();
1578
1579 Register CCReg = I.getOperand(i: 0).getReg();
1580 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
1581 int Opcode = getS_CMPOpcode(P: Pred, Size);
1582 if (Opcode == -1)
1583 return false;
1584 MachineInstr *ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode))
1585 .add(MO: I.getOperand(i: 2))
1586 .add(MO: I.getOperand(i: 3));
1587 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg)
1588 .addReg(RegNo: AMDGPU::SCC);
1589 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1590 bool Ret =
1591 RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
1592 I.eraseFromParent();
1593 return Ret;
1594 }
1595
1596 if (I.getOpcode() == AMDGPU::G_FCMP)
1597 return false;
1598
1599 int Opcode = getV_CMPOpcode(P: Pred, Size, ST: *Subtarget);
1600 if (Opcode == -1)
1601 return false;
1602
1603 MachineInstrBuilder ICmp;
1604 // t16 instructions
1605 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
1606 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1607 .addImm(Val: 0)
1608 .add(MO: I.getOperand(i: 2))
1609 .addImm(Val: 0)
1610 .add(MO: I.getOperand(i: 3))
1611 .addImm(Val: 0); // op_sel
1612 } else {
1613 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1614 .add(MO: I.getOperand(i: 2))
1615 .add(MO: I.getOperand(i: 3));
1616 }
1617
1618 RBI.constrainGenericRegister(Reg: ICmp->getOperand(i: 0).getReg(),
1619 RC: *TRI.getBoolRC(), MRI&: *MRI);
1620 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1621 I.eraseFromParent();
1622 return true;
1623}
1624
1625// Ballot has to zero bits in input lane-mask that are zero in current exec,
1626// Done as AND with exec. For inputs that are results of instruction that
1627// implicitly use same exec, for example compares in same basic block or SCC to
1628// VCC copy, use copy.
1629static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI,
1630 MachineBasicBlock *MBB) {
1631 MachineInstr *MI = MRI.getVRegDef(Reg);
1632 if (MI->getParent() != MBB)
1633 return false;
1634
1635 // Lane mask generated by SCC to VCC copy.
1636 if (MI->getOpcode() == AMDGPU::COPY) {
1637 auto DstRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 0).getReg());
1638 auto SrcRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 1).getReg());
1639 if (DstRB && SrcRB && DstRB->getID() == AMDGPU::VCCRegBankID &&
1640 SrcRB->getID() == AMDGPU::SGPRRegBankID)
1641 return true;
1642 }
1643
1644 // Lane mask generated by SCC to VCC copy
1645 if (MI->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
1646 return true;
1647
1648 // Lane mask generated using compare with same exec.
1649 if (isa<GAnyCmp>(Val: MI))
1650 return true;
1651
1652 Register LHS, RHS;
1653 // Look through AND.
1654 if (mi_match(R: Reg, MRI, P: m_GAnd(L: m_Reg(R&: LHS), R: m_Reg(R&: RHS))))
1655 return isLaneMaskFromSameBlock(Reg: LHS, MRI, MBB) ||
1656 isLaneMaskFromSameBlock(Reg: RHS, MRI, MBB);
1657
1658 return false;
1659}
1660
1661bool AMDGPUInstructionSelector::selectBallot(MachineInstr &I) const {
1662 MachineBasicBlock *BB = I.getParent();
1663 const DebugLoc &DL = I.getDebugLoc();
1664 Register DstReg = I.getOperand(i: 0).getReg();
1665 Register SrcReg = I.getOperand(i: 2).getReg();
1666 const unsigned BallotSize = MRI->getType(Reg: DstReg).getSizeInBits();
1667 const unsigned WaveSize = STI.getWavefrontSize();
1668
1669 // In the common case, the return type matches the wave size.
1670 // However we also support emitting i64 ballots in wave32 mode.
1671 if (BallotSize != WaveSize && (BallotSize != 64 || WaveSize != 32))
1672 return false;
1673
1674 std::optional<ValueAndVReg> Arg =
1675 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI);
1676
1677 Register Dst = DstReg;
1678 // i64 ballot on Wave32: new Dst(i32) for WaveSize ballot.
1679 if (BallotSize != WaveSize) {
1680 Dst = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
1681 }
1682
1683 if (Arg) {
1684 const int64_t Value = Arg->Value.getZExtValue();
1685 if (Value == 0) {
1686 // Dst = S_MOV 0
1687 unsigned Opcode = WaveSize == 64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
1688 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: Dst).addImm(Val: 0);
1689 } else {
1690 // Dst = COPY EXEC
1691 assert(Value == 1);
1692 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: TRI.getExec());
1693 }
1694 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1695 return false;
1696 } else {
1697 if (isLaneMaskFromSameBlock(Reg: SrcReg, MRI&: *MRI, MBB: BB)) {
1698 // Dst = COPY SrcReg
1699 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: SrcReg);
1700 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1701 return false;
1702 } else {
1703 // Dst = S_AND SrcReg, EXEC
1704 unsigned AndOpc = WaveSize == 64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
1705 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: Dst)
1706 .addReg(RegNo: SrcReg)
1707 .addReg(RegNo: TRI.getExec())
1708 .setOperandDead(3); // Dead scc
1709 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
1710 }
1711 }
1712
1713 // i64 ballot on Wave32: zero-extend i32 ballot to i64.
1714 if (BallotSize != WaveSize) {
1715 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1716 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg).addImm(Val: 0);
1717 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
1718 .addReg(RegNo: Dst)
1719 .addImm(Val: AMDGPU::sub0)
1720 .addReg(RegNo: HiReg)
1721 .addImm(Val: AMDGPU::sub1);
1722 }
1723
1724 I.eraseFromParent();
1725 return true;
1726}
1727
1728bool AMDGPUInstructionSelector::selectRelocConstant(MachineInstr &I) const {
1729 Register DstReg = I.getOperand(i: 0).getReg();
1730 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1731 const TargetRegisterClass *DstRC = TRI.getRegClassForSizeOnBank(Size: 32, Bank: *DstBank);
1732 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1733 return false;
1734
1735 const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID;
1736
1737 Module *M = MF->getFunction().getParent();
1738 const MDNode *Metadata = I.getOperand(i: 2).getMetadata();
1739 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
1740 auto *RelocSymbol = cast<GlobalVariable>(
1741 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
1742
1743 MachineBasicBlock *BB = I.getParent();
1744 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(),
1745 MCID: TII.get(Opcode: IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DestReg: DstReg)
1746 .addGlobalAddress(GV: RelocSymbol, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1747
1748 I.eraseFromParent();
1749 return true;
1750}
1751
1752bool AMDGPUInstructionSelector::selectGroupStaticSize(MachineInstr &I) const {
1753 Triple::OSType OS = MF->getTarget().getTargetTriple().getOS();
1754
1755 Register DstReg = I.getOperand(i: 0).getReg();
1756 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1757 unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ?
1758 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1759
1760 MachineBasicBlock *MBB = I.getParent();
1761 const DebugLoc &DL = I.getDebugLoc();
1762
1763 auto MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Mov), DestReg: DstReg);
1764
1765 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
1766 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1767 MIB.addImm(Val: MFI->getLDSSize());
1768 } else {
1769 Module *M = MF->getFunction().getParent();
1770 const GlobalValue *GV =
1771 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::amdgcn_groupstaticsize);
1772 MIB.addGlobalAddress(GV, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1773 }
1774
1775 I.eraseFromParent();
1776 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1777 return true;
1778}
1779
1780bool AMDGPUInstructionSelector::selectReturnAddress(MachineInstr &I) const {
1781 MachineBasicBlock *MBB = I.getParent();
1782 MachineFunction &MF = *MBB->getParent();
1783 const DebugLoc &DL = I.getDebugLoc();
1784
1785 Register DstReg = I.getOperand(i: 0).getReg();
1786 unsigned Depth = I.getOperand(i: 2).getImm();
1787
1788 const TargetRegisterClass *RC =
1789 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
1790 if (!RC->hasSubClassEq(RC: &AMDGPU::SGPR_64RegClass) ||
1791 !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
1792 return false;
1793
1794 // Check for kernel and shader functions
1795 if (Depth != 0 ||
1796 MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
1797 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg)
1798 .addImm(Val: 0);
1799 I.eraseFromParent();
1800 return true;
1801 }
1802
1803 MachineFrameInfo &MFI = MF.getFrameInfo();
1804 // There is a call to @llvm.returnaddress in this function
1805 MFI.setReturnAddressIsTaken(true);
1806
1807 // Get the return address reg and mark it as an implicit live-in
1808 Register ReturnAddrReg = TRI.getReturnAddressReg(MF);
1809 Register LiveIn = getFunctionLiveInPhysReg(MF, TII, PhysReg: ReturnAddrReg,
1810 RC: AMDGPU::SReg_64RegClass, DL);
1811 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
1812 .addReg(RegNo: LiveIn);
1813 I.eraseFromParent();
1814 return true;
1815}
1816
1817bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const {
1818 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1819 // SelectionDAG uses for wave32 vs wave64.
1820 MachineBasicBlock *BB = MI.getParent();
1821 BuildMI(BB&: *BB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_END_CF))
1822 .add(MO: MI.getOperand(i: 1));
1823
1824 Register Reg = MI.getOperand(i: 1).getReg();
1825 MI.eraseFromParent();
1826
1827 if (!MRI->getRegClassOrNull(Reg))
1828 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1829 return true;
1830}
1831
1832bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic(
1833 MachineInstr &MI, Intrinsic::ID IntrID) const {
1834 MachineBasicBlock *MBB = MI.getParent();
1835 MachineFunction *MF = MBB->getParent();
1836 const DebugLoc &DL = MI.getDebugLoc();
1837
1838 unsigned IndexOperand = MI.getOperand(i: 7).getImm();
1839 bool WaveRelease = MI.getOperand(i: 8).getImm() != 0;
1840 bool WaveDone = MI.getOperand(i: 9).getImm() != 0;
1841
1842 if (WaveDone && !WaveRelease) {
1843 // TODO: Move this to IR verifier
1844 const Function &Fn = MF->getFunction();
1845 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1846 Fn, "ds_ordered_count: wave_done requires wave_release", DL));
1847 }
1848
1849 unsigned OrderedCountIndex = IndexOperand & 0x3f;
1850 IndexOperand &= ~0x3f;
1851 unsigned CountDw = 0;
1852
1853 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) {
1854 CountDw = (IndexOperand >> 24) & 0xf;
1855 IndexOperand &= ~(0xf << 24);
1856
1857 if (CountDw < 1 || CountDw > 4) {
1858 const Function &Fn = MF->getFunction();
1859 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1860 Fn, "ds_ordered_count: dword count must be between 1 and 4", DL));
1861 CountDw = 1;
1862 }
1863 }
1864
1865 if (IndexOperand) {
1866 const Function &Fn = MF->getFunction();
1867 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1868 Fn, "ds_ordered_count: bad index operand", DL));
1869 }
1870
1871 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
1872 unsigned ShaderType = SIInstrInfo::getDSShaderTypeValue(MF: *MF);
1873
1874 unsigned Offset0 = OrderedCountIndex << 2;
1875 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
1876
1877 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10)
1878 Offset1 |= (CountDw - 1) << 6;
1879
1880 if (STI.getGeneration() < AMDGPUSubtarget::GFX11)
1881 Offset1 |= ShaderType << 2;
1882
1883 unsigned Offset = Offset0 | (Offset1 << 8);
1884
1885 Register M0Val = MI.getOperand(i: 2).getReg();
1886 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1887 .addReg(RegNo: M0Val);
1888
1889 Register DstReg = MI.getOperand(i: 0).getReg();
1890 Register ValReg = MI.getOperand(i: 3).getReg();
1891 MachineInstrBuilder DS =
1892 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_ORDERED_COUNT), DestReg: DstReg)
1893 .addReg(RegNo: ValReg)
1894 .addImm(Val: Offset)
1895 .cloneMemRefs(OtherMI: MI);
1896
1897 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
1898 return false;
1899
1900 constrainSelectedInstRegOperands(I&: *DS, TII, TRI, RBI);
1901 MI.eraseFromParent();
1902 return true;
1903}
1904
1905static unsigned gwsIntrinToOpcode(unsigned IntrID) {
1906 switch (IntrID) {
1907 case Intrinsic::amdgcn_ds_gws_init:
1908 return AMDGPU::DS_GWS_INIT;
1909 case Intrinsic::amdgcn_ds_gws_barrier:
1910 return AMDGPU::DS_GWS_BARRIER;
1911 case Intrinsic::amdgcn_ds_gws_sema_v:
1912 return AMDGPU::DS_GWS_SEMA_V;
1913 case Intrinsic::amdgcn_ds_gws_sema_br:
1914 return AMDGPU::DS_GWS_SEMA_BR;
1915 case Intrinsic::amdgcn_ds_gws_sema_p:
1916 return AMDGPU::DS_GWS_SEMA_P;
1917 case Intrinsic::amdgcn_ds_gws_sema_release_all:
1918 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
1919 default:
1920 llvm_unreachable("not a gws intrinsic");
1921 }
1922}
1923
1924bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
1925 Intrinsic::ID IID) const {
1926 if (!STI.hasGWS() || (IID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
1927 !STI.hasGWSSemaReleaseAll()))
1928 return false;
1929
1930 // intrinsic ID, vsrc, offset
1931 const bool HasVSrc = MI.getNumOperands() == 3;
1932 assert(HasVSrc || MI.getNumOperands() == 2);
1933
1934 Register BaseOffset = MI.getOperand(i: HasVSrc ? 2 : 1).getReg();
1935 const RegisterBank *OffsetRB = RBI.getRegBank(Reg: BaseOffset, MRI: *MRI, TRI);
1936 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID)
1937 return false;
1938
1939 MachineInstr *OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
1940 unsigned ImmOffset;
1941
1942 MachineBasicBlock *MBB = MI.getParent();
1943 const DebugLoc &DL = MI.getDebugLoc();
1944
1945 MachineInstr *Readfirstlane = nullptr;
1946
1947 // If we legalized the VGPR input, strip out the readfirstlane to analyze the
1948 // incoming offset, in case there's an add of a constant. We'll have to put it
1949 // back later.
1950 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) {
1951 Readfirstlane = OffsetDef;
1952 BaseOffset = OffsetDef->getOperand(i: 1).getReg();
1953 OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
1954 }
1955
1956 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) {
1957 // If we have a constant offset, try to use the 0 in m0 as the base.
1958 // TODO: Look into changing the default m0 initialization value. If the
1959 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
1960 // the immediate offset.
1961
1962 ImmOffset = OffsetDef->getOperand(i: 1).getCImm()->getZExtValue();
1963 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
1964 .addImm(Val: 0);
1965 } else {
1966 std::tie(args&: BaseOffset, args&: ImmOffset) =
1967 AMDGPU::getBaseWithConstantOffset(MRI&: *MRI, Reg: BaseOffset, ValueTracking: VT);
1968
1969 if (Readfirstlane) {
1970 // We have the constant offset now, so put the readfirstlane back on the
1971 // variable component.
1972 if (!RBI.constrainGenericRegister(Reg: BaseOffset, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
1973 return false;
1974
1975 Readfirstlane->getOperand(i: 1).setReg(BaseOffset);
1976 BaseOffset = Readfirstlane->getOperand(i: 0).getReg();
1977 } else {
1978 if (!RBI.constrainGenericRegister(Reg: BaseOffset,
1979 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
1980 return false;
1981 }
1982
1983 Register M0Base = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1984 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: M0Base)
1985 .addReg(RegNo: BaseOffset)
1986 .addImm(Val: 16)
1987 .setOperandDead(3); // Dead scc
1988
1989 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1990 .addReg(RegNo: M0Base);
1991 }
1992
1993 // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
1994 // offset field) % 64. Some versions of the programming guide omit the m0
1995 // part, or claim it's from offset 0.
1996
1997 unsigned Opc = gwsIntrinToOpcode(IntrID: IID);
1998 const MCInstrDesc &InstrDesc = TII.get(Opcode: Opc);
1999
2000 if (HasVSrc) {
2001 Register VSrc = MI.getOperand(i: 1).getReg();
2002
2003 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
2004 const TargetRegisterClass *DataRC = TII.getRegClass(MCID: InstrDesc, OpNum: Data0Idx);
2005 const TargetRegisterClass *SubRC =
2006 TRI.getSubRegisterClass(DataRC, AMDGPU::sub0);
2007
2008 if (!SubRC) {
2009 // 32-bit normal case.
2010 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *DataRC, MRI&: *MRI))
2011 return false;
2012
2013 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2014 .addReg(RegNo: VSrc)
2015 .addImm(Val: ImmOffset)
2016 .cloneMemRefs(OtherMI: MI);
2017 } else {
2018 // Requires even register alignment, so create 64-bit value and pad the
2019 // top half with undef.
2020 Register DataReg = MRI->createVirtualRegister(RegClass: DataRC);
2021 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *SubRC, MRI&: *MRI))
2022 return false;
2023
2024 Register UndefReg = MRI->createVirtualRegister(RegClass: SubRC);
2025 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2026 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DataReg)
2027 .addReg(RegNo: VSrc)
2028 .addImm(Val: AMDGPU::sub0)
2029 .addReg(RegNo: UndefReg)
2030 .addImm(Val: AMDGPU::sub1);
2031
2032 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2033 .addReg(RegNo: DataReg)
2034 .addImm(Val: ImmOffset)
2035 .cloneMemRefs(OtherMI: MI);
2036 }
2037 } else {
2038 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2039 .addImm(Val: ImmOffset)
2040 .cloneMemRefs(OtherMI: MI);
2041 }
2042
2043 MI.eraseFromParent();
2044 return true;
2045}
2046
2047bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
2048 bool IsAppend) const {
2049 Register PtrBase = MI.getOperand(i: 2).getReg();
2050 LLT PtrTy = MRI->getType(Reg: PtrBase);
2051 bool IsGDS = PtrTy.getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2052
2053 unsigned Offset;
2054 std::tie(args&: PtrBase, args&: Offset) = selectDS1Addr1OffsetImpl(Root&: MI.getOperand(i: 2));
2055
2056 // TODO: Should this try to look through readfirstlane like GWS?
2057 if (!isDSOffsetLegal(Base: PtrBase, Offset)) {
2058 PtrBase = MI.getOperand(i: 2).getReg();
2059 Offset = 0;
2060 }
2061
2062 MachineBasicBlock *MBB = MI.getParent();
2063 const DebugLoc &DL = MI.getDebugLoc();
2064 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2065
2066 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
2067 .addReg(RegNo: PtrBase);
2068 if (!RBI.constrainGenericRegister(Reg: PtrBase, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
2069 return false;
2070
2071 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: MI.getOperand(i: 0).getReg())
2072 .addImm(Val: Offset)
2073 .addImm(Val: IsGDS ? -1 : 0)
2074 .cloneMemRefs(OtherMI: MI);
2075 MI.eraseFromParent();
2076 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2077 return true;
2078}
2079
2080bool AMDGPUInstructionSelector::selectInitWholeWave(MachineInstr &MI) const {
2081 MachineFunction *MF = MI.getMF();
2082 SIMachineFunctionInfo *MFInfo = MF->getInfo<SIMachineFunctionInfo>();
2083
2084 MFInfo->setInitWholeWave();
2085 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
2086}
2087
2088static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE,
2089 bool &IsTexFail) {
2090 if (TexFailCtrl)
2091 IsTexFail = true;
2092
2093 TFE = TexFailCtrl & 0x1;
2094 TexFailCtrl &= ~(uint64_t)0x1;
2095 LWE = TexFailCtrl & 0x2;
2096 TexFailCtrl &= ~(uint64_t)0x2;
2097
2098 return TexFailCtrl == 0;
2099}
2100
2101bool AMDGPUInstructionSelector::selectImageIntrinsic(
2102 MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
2103 MachineBasicBlock *MBB = MI.getParent();
2104 const DebugLoc &DL = MI.getDebugLoc();
2105 unsigned IntrOpcode = Intr->BaseOpcode;
2106
2107 // For image atomic: use no-return opcode if result is unused.
2108 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode) {
2109 Register ResultDef = MI.getOperand(i: 0).getReg();
2110 if (MRI->use_nodbg_empty(RegNo: ResultDef))
2111 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
2112 }
2113
2114 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
2115 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
2116
2117 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
2118 const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI);
2119 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI);
2120 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
2121 const bool IsGFX13Plus = AMDGPU::isGFX13Plus(STI);
2122
2123 const unsigned ArgOffset = MI.getNumExplicitDefs() + 1;
2124
2125 Register VDataIn = AMDGPU::NoRegister;
2126 Register VDataOut = AMDGPU::NoRegister;
2127 LLT VDataTy;
2128 int NumVDataDwords = -1;
2129 bool IsD16 = MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 ||
2130 MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16;
2131
2132 bool Unorm;
2133 if (!BaseOpcode->Sampler)
2134 Unorm = true;
2135 else
2136 Unorm = MI.getOperand(i: ArgOffset + Intr->UnormIndex).getImm() != 0;
2137
2138 bool TFE;
2139 bool LWE;
2140 bool IsTexFail = false;
2141 if (!parseTexFail(TexFailCtrl: MI.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex).getImm(),
2142 TFE, LWE, IsTexFail))
2143 return false;
2144
2145 const int Flags = MI.getOperand(i: ArgOffset + Intr->NumArgs).getImm();
2146 const bool IsA16 = (Flags & 1) != 0;
2147 const bool IsG16 = (Flags & 2) != 0;
2148
2149 // A16 implies 16 bit gradients if subtarget doesn't support G16
2150 if (IsA16 && !STI.hasG16() && !IsG16)
2151 return false;
2152
2153 unsigned DMask = 0;
2154 unsigned DMaskLanes = 0;
2155
2156 if (BaseOpcode->Atomic) {
2157 if (!BaseOpcode->NoReturn)
2158 VDataOut = MI.getOperand(i: 0).getReg();
2159 VDataIn = MI.getOperand(i: 2).getReg();
2160 LLT Ty = MRI->getType(Reg: VDataIn);
2161
2162 // Be careful to allow atomic swap on 16-bit element vectors.
2163 const bool Is64Bit = BaseOpcode->AtomicX2 ?
2164 Ty.getSizeInBits() == 128 :
2165 Ty.getSizeInBits() == 64;
2166
2167 if (BaseOpcode->AtomicX2) {
2168 assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister);
2169
2170 DMask = Is64Bit ? 0xf : 0x3;
2171 NumVDataDwords = Is64Bit ? 4 : 2;
2172 } else {
2173 DMask = Is64Bit ? 0x3 : 0x1;
2174 NumVDataDwords = Is64Bit ? 2 : 1;
2175 }
2176 } else {
2177 DMask = MI.getOperand(i: ArgOffset + Intr->DMaskIndex).getImm();
2178 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
2179
2180 if (BaseOpcode->Store) {
2181 VDataIn = MI.getOperand(i: 1).getReg();
2182 VDataTy = MRI->getType(Reg: VDataIn);
2183 NumVDataDwords = (VDataTy.getSizeInBits() + 31) / 32;
2184 } else if (BaseOpcode->NoReturn) {
2185 NumVDataDwords = 0;
2186 } else {
2187 VDataOut = MI.getOperand(i: 0).getReg();
2188 VDataTy = MRI->getType(Reg: VDataOut);
2189 NumVDataDwords = DMaskLanes;
2190
2191 if (IsD16 && !STI.hasUnpackedD16VMem())
2192 NumVDataDwords = (DMaskLanes + 1) / 2;
2193 }
2194 }
2195
2196 // Set G16 opcode
2197 if (Subtarget->hasG16() && IsG16) {
2198 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
2199 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
2200 assert(G16MappingInfo);
2201 IntrOpcode = G16MappingInfo->G16; // set opcode to variant with _g16
2202 }
2203
2204 // TODO: Check this in verifier.
2205 assert((!IsTexFail || DMaskLanes >= 1) && "should have legalized this");
2206
2207 unsigned CPol = MI.getOperand(i: ArgOffset + Intr->CachePolicyIndex).getImm();
2208 // Keep GLC only when the atomic's result is actually used.
2209 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
2210 CPol |= AMDGPU::CPol::GLC;
2211 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
2212 AMDGPU::CPol::VOLATILE))
2213 return false;
2214
2215 int NumVAddrRegs = 0;
2216 int NumVAddrDwords = 0;
2217 for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
2218 // Skip the $noregs and 0s inserted during legalization.
2219 MachineOperand &AddrOp = MI.getOperand(i: ArgOffset + I);
2220 if (!AddrOp.isReg())
2221 continue; // XXX - Break?
2222
2223 Register Addr = AddrOp.getReg();
2224 if (!Addr)
2225 break;
2226
2227 ++NumVAddrRegs;
2228 NumVAddrDwords += (MRI->getType(Reg: Addr).getSizeInBits() + 31) / 32;
2229 }
2230
2231 // The legalizer preprocessed the intrinsic arguments. If we aren't using
2232 // NSA, these should have been packed into a single value in the first
2233 // address register
2234 const bool UseNSA =
2235 NumVAddrRegs != 1 &&
2236 (STI.hasPartialNSAEncoding() ? NumVAddrDwords >= NumVAddrRegs
2237 : NumVAddrDwords == NumVAddrRegs);
2238 if (UseNSA && !STI.hasFeature(Feature: AMDGPU::FeatureNSAEncoding)) {
2239 LLVM_DEBUG(dbgs() << "Trying to use NSA on non-NSA target\n");
2240 return false;
2241 }
2242
2243 if (IsTexFail)
2244 ++NumVDataDwords;
2245
2246 int Opcode = -1;
2247 if (IsGFX13Plus) {
2248 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx13,
2249 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2250 } else if (IsGFX12Plus) {
2251 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
2252 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2253 } else if (IsGFX11Plus) {
2254 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2255 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
2256 : AMDGPU::MIMGEncGfx11Default,
2257 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2258 } else if (IsGFX10Plus) {
2259 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2260 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
2261 : AMDGPU::MIMGEncGfx10Default,
2262 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2263 } else {
2264 if (Subtarget->hasGFX90AInsts()) {
2265 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
2266 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2267 if (Opcode == -1) {
2268 LLVM_DEBUG(
2269 dbgs()
2270 << "requested image instruction is not supported on this GPU\n");
2271 return false;
2272 }
2273 }
2274 if (Opcode == -1 &&
2275 STI.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
2276 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
2277 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2278 if (Opcode == -1)
2279 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
2280 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2281 }
2282 if (Opcode == -1)
2283 return false;
2284
2285 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode))
2286 .cloneMemRefs(OtherMI: MI);
2287
2288 if (VDataOut) {
2289 if (BaseOpcode->AtomicX2) {
2290 const bool Is64 = MRI->getType(Reg: VDataOut).getSizeInBits() == 64;
2291
2292 Register TmpReg = MRI->createVirtualRegister(
2293 RegClass: Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass);
2294 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0;
2295
2296 MIB.addDef(RegNo: TmpReg);
2297 if (!MRI->use_empty(RegNo: VDataOut)) {
2298 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VDataOut)
2299 .addReg(RegNo: TmpReg, Flags: RegState::Kill, SubReg);
2300 }
2301
2302 } else {
2303 MIB.addDef(RegNo: VDataOut); // vdata output
2304 }
2305 }
2306
2307 if (VDataIn)
2308 MIB.addReg(RegNo: VDataIn); // vdata input
2309
2310 for (int I = 0; I != NumVAddrRegs; ++I) {
2311 MachineOperand &SrcOp = MI.getOperand(i: ArgOffset + Intr->VAddrStart + I);
2312 if (SrcOp.isReg()) {
2313 assert(SrcOp.getReg() != 0);
2314 MIB.addReg(RegNo: SrcOp.getReg());
2315 }
2316 }
2317
2318 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->RsrcIndex).getReg());
2319 if (BaseOpcode->Sampler)
2320 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->SampIndex).getReg());
2321
2322 MIB.addImm(Val: DMask); // dmask
2323
2324 if (IsGFX10Plus)
2325 MIB.addImm(Val: DimInfo->Encoding);
2326 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::unorm))
2327 MIB.addImm(Val: Unorm);
2328
2329 MIB.addImm(Val: CPol);
2330 MIB.addImm(Val: IsA16 && // a16 or r128
2331 STI.hasFeature(Feature: AMDGPU::FeatureR128A16) ? -1 : 0);
2332 if (IsGFX10Plus)
2333 MIB.addImm(Val: IsA16 ? -1 : 0);
2334
2335 if (!Subtarget->hasGFX90AInsts()) {
2336 MIB.addImm(Val: TFE); // tfe
2337 } else if (TFE) {
2338 LLVM_DEBUG(dbgs() << "TFE is not supported on this GPU\n");
2339 return false;
2340 }
2341
2342 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::lwe))
2343 MIB.addImm(Val: LWE); // lwe
2344 if (!IsGFX10Plus)
2345 MIB.addImm(Val: DimInfo->DA ? -1 : 0);
2346 if (BaseOpcode->HasD16)
2347 MIB.addImm(Val: IsD16 ? -1 : 0);
2348
2349 MI.eraseFromParent();
2350 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2351 TII.enforceOperandRCAlignment(MI&: *MIB, OpName: AMDGPU::OpName::vaddr);
2352 return true;
2353}
2354
2355// We need to handle this here because tablegen doesn't support matching
2356// instructions with multiple outputs.
2357bool AMDGPUInstructionSelector::selectDSBvhStackIntrinsic(
2358 MachineInstr &MI) const {
2359 Register Dst0 = MI.getOperand(i: 0).getReg();
2360 Register Dst1 = MI.getOperand(i: 1).getReg();
2361
2362 const DebugLoc &DL = MI.getDebugLoc();
2363 MachineBasicBlock *MBB = MI.getParent();
2364
2365 Register Addr = MI.getOperand(i: 3).getReg();
2366 Register Data0 = MI.getOperand(i: 4).getReg();
2367 Register Data1 = MI.getOperand(i: 5).getReg();
2368 unsigned Offset = MI.getOperand(i: 6).getImm();
2369
2370 unsigned Opc;
2371 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
2372 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2373 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2374 Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
2375 break;
2376 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2377 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32;
2378 break;
2379 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2380 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64;
2381 break;
2382 }
2383
2384 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
2385 .addDef(RegNo: Dst1)
2386 .addUse(RegNo: Addr)
2387 .addUse(RegNo: Data0)
2388 .addUse(RegNo: Data1)
2389 .addImm(Val: Offset)
2390 .cloneMemRefs(OtherMI: MI);
2391
2392 MI.eraseFromParent();
2393 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2394 return true;
2395}
2396
2397bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
2398 MachineInstr &I) const {
2399 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
2400 switch (IntrinsicID) {
2401 case Intrinsic::amdgcn_end_cf:
2402 return selectEndCfIntrinsic(MI&: I);
2403 case Intrinsic::amdgcn_ds_ordered_add:
2404 case Intrinsic::amdgcn_ds_ordered_swap:
2405 return selectDSOrderedIntrinsic(MI&: I, IntrID: IntrinsicID);
2406 case Intrinsic::amdgcn_ds_gws_init:
2407 case Intrinsic::amdgcn_ds_gws_barrier:
2408 case Intrinsic::amdgcn_ds_gws_sema_v:
2409 case Intrinsic::amdgcn_ds_gws_sema_br:
2410 case Intrinsic::amdgcn_ds_gws_sema_p:
2411 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2412 return selectDSGWSIntrinsic(MI&: I, IID: IntrinsicID);
2413 case Intrinsic::amdgcn_ds_append:
2414 return selectDSAppendConsume(MI&: I, IsAppend: true);
2415 case Intrinsic::amdgcn_ds_consume:
2416 return selectDSAppendConsume(MI&: I, IsAppend: false);
2417 case Intrinsic::amdgcn_init_whole_wave:
2418 return selectInitWholeWave(MI&: I);
2419 case Intrinsic::amdgcn_raw_buffer_load_lds:
2420 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
2421 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
2422 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
2423 case Intrinsic::amdgcn_struct_buffer_load_lds:
2424 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
2425 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
2426 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
2427 return selectBufferLoadLds(MI&: I);
2428 // Until we can store both the address space of the global and the LDS
2429 // arguments by having tto MachineMemOperands on an intrinsic, we just trust
2430 // that the argument is a global pointer (buffer pointers have been handled by
2431 // a LLVM IR-level lowering).
2432 case Intrinsic::amdgcn_load_to_lds:
2433 case Intrinsic::amdgcn_load_async_to_lds:
2434 case Intrinsic::amdgcn_global_load_lds:
2435 case Intrinsic::amdgcn_global_load_async_lds:
2436 return selectGlobalLoadLds(MI&: I);
2437 case Intrinsic::amdgcn_tensor_load_to_lds:
2438 case Intrinsic::amdgcn_tensor_store_from_lds:
2439 return selectTensorLoadStore(MI&: I, IID: IntrinsicID);
2440 case Intrinsic::amdgcn_asyncmark:
2441 case Intrinsic::amdgcn_wait_asyncmark:
2442 if (!Subtarget->hasAsyncMark())
2443 return false;
2444 break;
2445 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2446 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2447 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2448 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2449 return selectDSBvhStackIntrinsic(MI&: I);
2450 case Intrinsic::amdgcn_s_alloc_vgpr: {
2451 // S_ALLOC_VGPR doesn't have a destination register, it just implicitly sets
2452 // SCC. We then need to COPY it into the result vreg.
2453 MachineBasicBlock *MBB = I.getParent();
2454 const DebugLoc &DL = I.getDebugLoc();
2455
2456 Register ResReg = I.getOperand(i: 0).getReg();
2457
2458 MachineInstr *AllocMI = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ALLOC_VGPR))
2459 .add(MO: I.getOperand(i: 2));
2460 (void)BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: ResReg)
2461 .addReg(RegNo: AMDGPU::SCC);
2462 I.eraseFromParent();
2463 constrainSelectedInstRegOperands(I&: *AllocMI, TII, TRI, RBI);
2464 return RBI.constrainGenericRegister(Reg: ResReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2465 }
2466 case Intrinsic::amdgcn_s_barrier_init:
2467 case Intrinsic::amdgcn_s_barrier_signal_var:
2468 return selectNamedBarrierInit(I, IID: IntrinsicID);
2469 case Intrinsic::amdgcn_s_wakeup_barrier:
2470 case Intrinsic::amdgcn_s_barrier_join:
2471 case Intrinsic::amdgcn_s_get_named_barrier_state:
2472 return selectNamedBarrierInst(I, IID: IntrinsicID);
2473 case Intrinsic::amdgcn_s_get_barrier_state:
2474 return selectSGetBarrierState(I, IID: IntrinsicID);
2475 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
2476 return selectSBarrierSignalIsfirst(I, IID: IntrinsicID);
2477 }
2478 return selectImpl(I, CoverageInfo&: *CoverageInfo);
2479}
2480
2481bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
2482 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
2483 return true;
2484
2485 MachineBasicBlock *BB = I.getParent();
2486 const DebugLoc &DL = I.getDebugLoc();
2487
2488 Register DstReg = I.getOperand(i: 0).getReg();
2489 unsigned Size = RBI.getSizeInBits(Reg: DstReg, MRI: *MRI, TRI);
2490 assert(Size <= 32 || Size == 64);
2491 const MachineOperand &CCOp = I.getOperand(i: 1);
2492 Register CCReg = CCOp.getReg();
2493 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
2494 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 :
2495 AMDGPU::S_CSELECT_B32;
2496 MachineInstr *CopySCC = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
2497 .addReg(RegNo: CCReg);
2498
2499 // The generic constrainSelectedInstRegOperands doesn't work for the scc register
2500 // bank, because it does not cover the register class that we used to represent
2501 // for it. So we need to manually set the register class here.
2502 if (!MRI->getRegClassOrNull(Reg: CCReg))
2503 MRI->setRegClass(Reg: CCReg, RC: TRI.getConstrainedRegClassForReg(Reg: CCReg, MRI: *MRI));
2504 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
2505 .add(MO: I.getOperand(i: 2))
2506 .add(MO: I.getOperand(i: 3));
2507
2508 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2509 constrainSelectedInstRegOperands(I&: *CopySCC, TII, TRI, RBI);
2510 I.eraseFromParent();
2511 return true;
2512 }
2513
2514 // Wide VGPR select should have been split in RegBankSelect.
2515 if (Size > 32)
2516 return false;
2517
2518 MachineInstr *Select =
2519 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
2520 .addImm(Val: 0)
2521 .add(MO: I.getOperand(i: 3))
2522 .addImm(Val: 0)
2523 .add(MO: I.getOperand(i: 2))
2524 .add(MO: I.getOperand(i: 1));
2525
2526 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2527 I.eraseFromParent();
2528 return true;
2529}
2530
2531bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
2532 Register DstReg = I.getOperand(i: 0).getReg();
2533 Register SrcReg = I.getOperand(i: 1).getReg();
2534 const LLT DstTy = MRI->getType(Reg: DstReg);
2535 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2536 const LLT S1 = LLT::scalar(SizeInBits: 1);
2537
2538 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2539 const RegisterBank *DstRB;
2540 if (DstTy == S1) {
2541 // This is a special case. We don't treat s1 for legalization artifacts as
2542 // vcc booleans.
2543 DstRB = SrcRB;
2544 } else {
2545 DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2546 if (SrcRB != DstRB)
2547 return false;
2548 }
2549
2550 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
2551
2552 unsigned DstSize = DstTy.getSizeInBits();
2553 unsigned SrcSize = SrcTy.getSizeInBits();
2554
2555 const TargetRegisterClass *SrcRC =
2556 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcRB);
2557 const TargetRegisterClass *DstRC =
2558 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
2559 if (!SrcRC || !DstRC)
2560 return false;
2561
2562 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
2563 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
2564 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
2565 return false;
2566 }
2567
2568 if (DstRC == &AMDGPU::VGPR_16RegClass && SrcSize == 32) {
2569 assert(STI.useRealTrue16Insts());
2570 const DebugLoc &DL = I.getDebugLoc();
2571 MachineBasicBlock *MBB = I.getParent();
2572 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
2573 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::lo16);
2574 I.eraseFromParent();
2575 return true;
2576 }
2577
2578 if (DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) && SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
2579 MachineBasicBlock *MBB = I.getParent();
2580 const DebugLoc &DL = I.getDebugLoc();
2581
2582 Register LoReg = MRI->createVirtualRegister(RegClass: DstRC);
2583 Register HiReg = MRI->createVirtualRegister(RegClass: DstRC);
2584 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
2585 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
2586 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
2587 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
2588
2589 if (IsVALU && STI.hasSDWA()) {
2590 // Write the low 16-bits of the high element into the high 16-bits of the
2591 // low element.
2592 MachineInstr *MovSDWA =
2593 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: DstReg)
2594 .addImm(Val: 0) // $src0_modifiers
2595 .addReg(RegNo: HiReg) // $src0
2596 .addImm(Val: 0) // $clamp
2597 .addImm(Val: AMDGPU::SDWA::WORD_1) // $dst_sel
2598 .addImm(Val: AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused
2599 .addImm(Val: AMDGPU::SDWA::WORD_0) // $src0_sel
2600 .addReg(RegNo: LoReg, Flags: RegState::Implicit);
2601 MovSDWA->tieOperands(DefIdx: 0, UseIdx: MovSDWA->getNumOperands() - 1);
2602 } else {
2603 Register TmpReg0 = MRI->createVirtualRegister(RegClass: DstRC);
2604 Register TmpReg1 = MRI->createVirtualRegister(RegClass: DstRC);
2605 Register ImmReg = MRI->createVirtualRegister(RegClass: DstRC);
2606 if (IsVALU) {
2607 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: TmpReg0)
2608 .addImm(Val: 16)
2609 .addReg(RegNo: HiReg);
2610 } else {
2611 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg0)
2612 .addReg(RegNo: HiReg)
2613 .addImm(Val: 16)
2614 .setOperandDead(3); // Dead scc
2615 }
2616
2617 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2618 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
2619 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32;
2620
2621 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: ImmReg)
2622 .addImm(Val: 0xffff);
2623 auto And = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: TmpReg1)
2624 .addReg(RegNo: LoReg)
2625 .addReg(RegNo: ImmReg);
2626 auto Or = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: OrOpc), DestReg: DstReg)
2627 .addReg(RegNo: TmpReg0)
2628 .addReg(RegNo: TmpReg1);
2629
2630 if (!IsVALU) {
2631 And.setOperandDead(3); // Dead scc
2632 Or.setOperandDead(3); // Dead scc
2633 }
2634 }
2635
2636 I.eraseFromParent();
2637 return true;
2638 }
2639
2640 if (!DstTy.isScalar())
2641 return false;
2642
2643 if (SrcSize > 32) {
2644 unsigned SubRegIdx = DstSize < 32
2645 ? static_cast<unsigned>(AMDGPU::sub0)
2646 : TRI.getSubRegFromChannel(Channel: 0, NumRegs: DstSize / 32);
2647 if (SubRegIdx == AMDGPU::NoSubRegister)
2648 return false;
2649
2650 // Deal with weird cases where the class only partially supports the subreg
2651 // index.
2652 const TargetRegisterClass *SrcWithSubRC
2653 = TRI.getSubClassWithSubReg(SrcRC, SubRegIdx);
2654 if (!SrcWithSubRC)
2655 return false;
2656
2657 if (SrcWithSubRC != SrcRC) {
2658 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcWithSubRC, MRI&: *MRI))
2659 return false;
2660 }
2661
2662 I.getOperand(i: 1).setSubReg(SubRegIdx);
2663 }
2664
2665 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
2666 return true;
2667}
2668
2669/// \returns true if a bitmask for \p Size bits will be an inline immediate.
2670static bool shouldUseAndMask(unsigned Size, unsigned &Mask) {
2671 Mask = maskTrailingOnes<unsigned>(N: Size);
2672 int SignedMask = static_cast<int>(Mask);
2673 return SignedMask >= -16 && SignedMask <= 64;
2674}
2675
2676// Like RegisterBankInfo::getRegBank, but don't assume vcc for s1.
2677const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank(
2678 Register Reg, const MachineRegisterInfo &MRI,
2679 const TargetRegisterInfo &TRI) const {
2680 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
2681 if (auto *RB = dyn_cast<const RegisterBank *>(Val: RegClassOrBank))
2682 return RB;
2683
2684 // Ignore the type, since we don't use vcc in artifacts.
2685 if (auto *RC = dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank))
2686 return &RBI.getRegBankFromRegClass(RC: *RC, LLT());
2687 return nullptr;
2688}
2689
2690bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
2691 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG;
2692 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg;
2693 const DebugLoc &DL = I.getDebugLoc();
2694 MachineBasicBlock &MBB = *I.getParent();
2695 const Register DstReg = I.getOperand(i: 0).getReg();
2696 const Register SrcReg = I.getOperand(i: 1).getReg();
2697
2698 const LLT DstTy = MRI->getType(Reg: DstReg);
2699 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2700 const unsigned SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG ?
2701 I.getOperand(i: 2).getImm() : SrcTy.getSizeInBits();
2702 const unsigned DstSize = DstTy.getSizeInBits();
2703 if (!DstTy.isScalar())
2704 return false;
2705
2706 // Artifact casts should never use vcc.
2707 const RegisterBank *SrcBank = getArtifactRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2708
2709 // FIXME: This should probably be illegal and split earlier.
2710 if (I.getOpcode() == AMDGPU::G_ANYEXT) {
2711 if (DstSize <= 32)
2712 return selectCOPY(I);
2713
2714 const TargetRegisterClass *SrcRC =
2715 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcBank);
2716 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2717 const TargetRegisterClass *DstRC =
2718 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
2719
2720 Register UndefReg = MRI->createVirtualRegister(RegClass: SrcRC);
2721 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2722 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2723 .addReg(RegNo: SrcReg)
2724 .addImm(Val: AMDGPU::sub0)
2725 .addReg(RegNo: UndefReg)
2726 .addImm(Val: AMDGPU::sub1);
2727 I.eraseFromParent();
2728
2729 return RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) &&
2730 RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI);
2731 }
2732
2733 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) {
2734 // 64-bit should have been split up in RegBankSelect
2735
2736 // Try to use an and with a mask if it will save code size.
2737 unsigned Mask;
2738 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2739 MachineInstr *ExtI =
2740 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: DstReg)
2741 .addImm(Val: Mask)
2742 .addReg(RegNo: SrcReg);
2743 I.eraseFromParent();
2744 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2745 return true;
2746 }
2747
2748 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2749 MachineInstr *ExtI =
2750 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE), DestReg: DstReg)
2751 .addReg(RegNo: SrcReg)
2752 .addImm(Val: 0) // Offset
2753 .addImm(Val: SrcSize); // Width
2754 I.eraseFromParent();
2755 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2756 return true;
2757 }
2758
2759 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
2760 const TargetRegisterClass &SrcRC = InReg && DstSize > 32 ?
2761 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass;
2762 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: SrcRC, MRI&: *MRI))
2763 return false;
2764
2765 if (Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) {
2766 const unsigned SextOpc = SrcSize == 8 ?
2767 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16;
2768 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: SextOpc), DestReg: DstReg)
2769 .addReg(RegNo: SrcReg);
2770 I.eraseFromParent();
2771 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2772 }
2773
2774 // Using a single 32-bit SALU to calculate the high half is smaller than
2775 // S_BFE with a literal constant operand.
2776 if (DstSize > 32 && SrcSize == 32) {
2777 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2778 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2779 if (Signed) {
2780 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ASHR_I32), DestReg: HiReg)
2781 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2782 .addImm(Val: 31)
2783 .setOperandDead(3); // Dead scc
2784 } else {
2785 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg)
2786 .addImm(Val: 0);
2787 }
2788 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2789 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2790 .addImm(Val: AMDGPU::sub0)
2791 .addReg(RegNo: HiReg)
2792 .addImm(Val: AMDGPU::sub1);
2793 I.eraseFromParent();
2794 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass,
2795 MRI&: *MRI);
2796 }
2797
2798 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
2799 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2800
2801 // Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width.
2802 if (DstSize > 32 && (SrcSize <= 32 || InReg)) {
2803 // We need a 64-bit register source, but the high bits don't matter.
2804 Register ExtReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
2805 Register UndefReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2806 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2807
2808 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2809 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: ExtReg)
2810 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2811 .addImm(Val: AMDGPU::sub0)
2812 .addReg(RegNo: UndefReg)
2813 .addImm(Val: AMDGPU::sub1);
2814
2815 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE64), DestReg: DstReg)
2816 .addReg(RegNo: ExtReg)
2817 .addImm(Val: SrcSize << 16);
2818
2819 I.eraseFromParent();
2820 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI);
2821 }
2822
2823 unsigned Mask;
2824 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2825 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: DstReg)
2826 .addReg(RegNo: SrcReg)
2827 .addImm(Val: Mask)
2828 .setOperandDead(3); // Dead scc
2829 } else {
2830 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE32), DestReg: DstReg)
2831 .addReg(RegNo: SrcReg)
2832 .addImm(Val: SrcSize << 16);
2833 }
2834
2835 I.eraseFromParent();
2836 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2837 }
2838
2839 return false;
2840}
2841
2842static Register stripCopy(Register Reg, MachineRegisterInfo &MRI) {
2843 return getDefSrcRegIgnoringCopies(Reg, MRI)->Reg;
2844}
2845
2846static Register stripBitCast(Register Reg, MachineRegisterInfo &MRI) {
2847 Register BitcastSrc;
2848 if (mi_match(R: Reg, MRI, P: m_GBitcast(Src: m_Reg(R&: BitcastSrc))))
2849 Reg = BitcastSrc;
2850 return Reg;
2851}
2852
2853static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In,
2854 Register &Out) {
2855 // When unmerging a register that is composed of 2 x 16-bit values allow to
2856 // use an extract hi instruction for the upper 16 bits. We only need to check
2857 // the size of `In` as all defs are guaranteed to be the same type for
2858 // GUnmerge.
2859 GUnmerge *Unmerge;
2860 if (mi_match(R: In, MRI, P: m_GUnmerge(Inst&: Unmerge))) {
2861 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(i: 1).getReg() == In &&
2862 MRI.getType(Reg: In).getSizeInBits() == 16) {
2863 Out = stripBitCast(Reg: Unmerge->getSourceReg(), MRI);
2864 return true;
2865 }
2866 }
2867
2868 Register Trunc;
2869 if (!mi_match(R: In, MRI, P: m_GTrunc(Src: m_Reg(R&: Trunc))))
2870 return false;
2871
2872 Register LShlSrc;
2873 Register Cst;
2874 if (mi_match(R: Trunc, MRI, P: m_GLShr(L: m_Reg(R&: LShlSrc), R: m_Reg(R&: Cst)))) {
2875 Cst = stripCopy(Reg: Cst, MRI);
2876 if (mi_match(R: Cst, MRI, P: m_SpecificICst(RequestedValue: 16))) {
2877 Out = stripBitCast(Reg: LShlSrc, MRI);
2878 return true;
2879 }
2880 }
2881
2882 ArrayRef<int> Mask;
2883 Register Src1;
2884 if (!mi_match(R: Trunc, MRI, P: m_GShuffleVector(Src1: m_Reg(R&: Src1), Src2: m_Reg(), Mask)))
2885 return false;
2886
2887 assert(MRI.getType(Src1) == LLT::fixed_vector(2, 16));
2888 assert(Mask.size() == 2);
2889
2890 if (Mask[0] == 1 && Mask[1] <= 1) {
2891 Out = Trunc;
2892 return true;
2893 }
2894
2895 return false;
2896}
2897
2898static bool isExtractLoElt(MachineRegisterInfo &MRI, Register In,
2899 Register &Out) {
2900 // There could be a bitcast between the extraction and its use.
2901 In = stripBitCast(Reg: In, MRI);
2902
2903 // The first def of a 2 x 16-bit unmerge is the low half of its source.
2904 if (auto *Unmerge = dyn_cast<GUnmerge>(Val: MRI.getVRegDef(Reg: In))) {
2905 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(i: 0).getReg() == In &&
2906 MRI.getType(Reg: In).getSizeInBits() == 16) {
2907 Out = stripBitCast(Reg: Unmerge->getSourceReg(), MRI);
2908 return true;
2909 }
2910 }
2911
2912 // A truncation from 32 to 16 bits keeps the low half in place.
2913 Register Trunc;
2914 if (mi_match(R: In, MRI, P: m_GTrunc(Src: m_Reg(R&: Trunc))) &&
2915 MRI.getType(Reg: Trunc).getSizeInBits() == 32) {
2916 Out = stripBitCast(Reg: Trunc, MRI);
2917 return true;
2918 }
2919
2920 return false;
2921}
2922
2923bool AMDGPUInstructionSelector::selectG_FPEXT(MachineInstr &I) const {
2924 if (!Subtarget->hasSALUFloatInsts())
2925 return false;
2926
2927 Register Dst = I.getOperand(i: 0).getReg();
2928 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2929 if (DstRB->getID() != AMDGPU::SGPRRegBankID)
2930 return false;
2931
2932 Register Src = I.getOperand(i: 1).getReg();
2933
2934 if (MRI->getType(Reg: Dst) == LLT::scalar(SizeInBits: 32) &&
2935 MRI->getType(Reg: Src) == LLT::scalar(SizeInBits: 16)) {
2936 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
2937 MachineBasicBlock *BB = I.getParent();
2938 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_CVT_HI_F32_F16), DestReg: Dst)
2939 .addUse(RegNo: Src);
2940 I.eraseFromParent();
2941 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2942 }
2943 }
2944
2945 return false;
2946}
2947
2948bool AMDGPUInstructionSelector::selectG_FNEG(MachineInstr &MI) const {
2949 // Only manually handle the f64 SGPR case.
2950 //
2951 // FIXME: This is a workaround for 2.5 different tablegen problems. Because
2952 // the bit ops theoretically have a second result due to the implicit def of
2953 // SCC, the GlobalISelEmitter is overly conservative and rejects it. Fixing
2954 // that is easy by disabling the check. The result works, but uses a
2955 // nonsensical sreg32orlds_and_sreg_1 regclass.
2956 //
2957 // The DAG emitter is more problematic, and incorrectly adds both S_XOR_B32 to
2958 // the variadic REG_SEQUENCE operands.
2959
2960 Register Dst = MI.getOperand(i: 0).getReg();
2961 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2962 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
2963 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
2964 return false;
2965
2966 Register Src = MI.getOperand(i: 1).getReg();
2967 MachineInstr *Fabs = getOpcodeDef(Opcode: TargetOpcode::G_FABS, Reg: Src, MRI: *MRI);
2968 if (Fabs)
2969 Src = Fabs->getOperand(i: 1).getReg();
2970
2971 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
2972 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
2973 return false;
2974
2975 MachineBasicBlock *BB = MI.getParent();
2976 const DebugLoc &DL = MI.getDebugLoc();
2977 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2978 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2979 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2980 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2981
2982 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
2983 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
2984 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
2985 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
2986 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
2987 .addImm(Val: 0x80000000);
2988
2989 // Set or toggle sign bit.
2990 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32;
2991 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: OpReg)
2992 .addReg(RegNo: HiReg)
2993 .addReg(RegNo: ConstReg)
2994 .setOperandDead(3); // Dead scc
2995 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
2996 .addReg(RegNo: LoReg)
2997 .addImm(Val: AMDGPU::sub0)
2998 .addReg(RegNo: OpReg)
2999 .addImm(Val: AMDGPU::sub1);
3000 MI.eraseFromParent();
3001 return true;
3002}
3003
3004// FIXME: This is a workaround for the same tablegen problems as G_FNEG
3005bool AMDGPUInstructionSelector::selectG_FABS(MachineInstr &MI) const {
3006 Register Dst = MI.getOperand(i: 0).getReg();
3007 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
3008 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
3009 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
3010 return false;
3011
3012 Register Src = MI.getOperand(i: 1).getReg();
3013 MachineBasicBlock *BB = MI.getParent();
3014 const DebugLoc &DL = MI.getDebugLoc();
3015 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3016 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3017 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3018 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3019
3020 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
3021 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
3022 return false;
3023
3024 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3025 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
3026 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3027 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
3028 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
3029 .addImm(Val: 0x7fffffff);
3030
3031 // Clear sign bit.
3032 // TODO: Should this used S_BITSET0_*?
3033 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: OpReg)
3034 .addReg(RegNo: HiReg)
3035 .addReg(RegNo: ConstReg)
3036 .setOperandDead(3); // Dead scc
3037 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
3038 .addReg(RegNo: LoReg)
3039 .addImm(Val: AMDGPU::sub0)
3040 .addReg(RegNo: OpReg)
3041 .addImm(Val: AMDGPU::sub1);
3042
3043 MI.eraseFromParent();
3044 return true;
3045}
3046
3047static bool isConstant(const MachineInstr &MI) {
3048 return MI.getOpcode() == TargetOpcode::G_CONSTANT;
3049}
3050
3051void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
3052 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const {
3053
3054 unsigned OpNo = Load.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1;
3055 const MachineInstr *PtrMI =
3056 MRI.getUniqueVRegDef(Reg: Load.getOperand(i: OpNo).getReg());
3057
3058 assert(PtrMI);
3059
3060 if (PtrMI->getOpcode() != TargetOpcode::G_PTR_ADD)
3061 return;
3062
3063 GEPInfo GEPInfo;
3064
3065 for (unsigned i = 1; i != 3; ++i) {
3066 const MachineOperand &GEPOp = PtrMI->getOperand(i);
3067 const MachineInstr *OpDef = MRI.getUniqueVRegDef(Reg: GEPOp.getReg());
3068 assert(OpDef);
3069 if (i == 2 && isConstant(MI: *OpDef)) {
3070 // TODO: Could handle constant base + variable offset, but a combine
3071 // probably should have commuted it.
3072 assert(GEPInfo.Imm == 0);
3073 GEPInfo.Imm = OpDef->getOperand(i: 1).getCImm()->getSExtValue();
3074 continue;
3075 }
3076 const RegisterBank *OpBank = RBI.getRegBank(Reg: GEPOp.getReg(), MRI, TRI);
3077 if (OpBank->getID() == AMDGPU::SGPRRegBankID)
3078 GEPInfo.SgprParts.push_back(Elt: GEPOp.getReg());
3079 else
3080 GEPInfo.VgprParts.push_back(Elt: GEPOp.getReg());
3081 }
3082
3083 AddrInfo.push_back(Elt: GEPInfo);
3084 getAddrModeInfo(Load: *PtrMI, MRI, AddrInfo);
3085}
3086
3087bool AMDGPUInstructionSelector::isSGPR(Register Reg) const {
3088 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID;
3089}
3090
3091bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const {
3092 if (!MI.hasOneMemOperand())
3093 return false;
3094
3095 const MachineMemOperand *MMO = *MI.memoperands_begin();
3096 const Value *Ptr = MMO->getValue();
3097
3098 // UndefValue means this is a load of a kernel input. These are uniform.
3099 // Sometimes LDS instructions have constant pointers.
3100 // If Ptr is null, then that means this mem operand contains a
3101 // PseudoSourceValue like GOT.
3102 if (!Ptr || isa<UndefValue, Argument, Constant, GlobalValue>(Val: Ptr))
3103 return true;
3104
3105 if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
3106 return true;
3107
3108 if (MI.getOpcode() == AMDGPU::G_PREFETCH)
3109 return RBI.getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI: *MRI, TRI)->getID() ==
3110 AMDGPU::SGPRRegBankID;
3111
3112 const Instruction *I = dyn_cast<Instruction>(Val: Ptr);
3113 return I && I->getMetadata(Kind: "amdgpu.uniform");
3114}
3115
3116bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const {
3117 for (const GEPInfo &GEPInfo : AddrInfo) {
3118 if (!GEPInfo.VgprParts.empty())
3119 return true;
3120 }
3121 return false;
3122}
3123
3124void AMDGPUInstructionSelector::initM0(MachineInstr &I) const {
3125 const LLT PtrTy = MRI->getType(Reg: I.getOperand(i: 1).getReg());
3126 unsigned AS = PtrTy.getAddressSpace();
3127 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) &&
3128 STI.ldsRequiresM0Init()) {
3129 MachineBasicBlock *BB = I.getParent();
3130
3131 // If DS instructions require M0 initialization, insert it before selecting.
3132 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
3133 .addImm(Val: -1);
3134 }
3135}
3136
3137bool AMDGPUInstructionSelector::selectG_LOAD_STORE_ATOMICRMW(
3138 MachineInstr &I) const {
3139 initM0(I);
3140 return selectImpl(I, CoverageInfo&: *CoverageInfo);
3141}
3142
3143static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI) {
3144 if (Reg.isPhysical())
3145 return false;
3146
3147 MachineInstr &MI = *MRI.getUniqueVRegDef(Reg);
3148 const unsigned Opcode = MI.getOpcode();
3149
3150 if (Opcode == AMDGPU::COPY)
3151 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI);
3152
3153 if (Opcode == AMDGPU::G_AND || Opcode == AMDGPU::G_OR ||
3154 Opcode == AMDGPU::G_XOR)
3155 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI) &&
3156 isVCmpResult(Reg: MI.getOperand(i: 2).getReg(), MRI);
3157
3158 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI))
3159 return GI->is(ID: Intrinsic::amdgcn_class);
3160
3161 return Opcode == AMDGPU::G_ICMP || Opcode == AMDGPU::G_FCMP;
3162}
3163
3164bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
3165 MachineBasicBlock *BB = I.getParent();
3166 MachineOperand &CondOp = I.getOperand(i: 0);
3167 Register CondReg = CondOp.getReg();
3168 const DebugLoc &DL = I.getDebugLoc();
3169
3170 unsigned BrOpcode;
3171 Register CondPhysReg;
3172 const TargetRegisterClass *ConstrainRC;
3173
3174 // In SelectionDAG, we inspect the IR block for uniformity metadata to decide
3175 // whether the branch is uniform when selecting the instruction. In
3176 // GlobalISel, we should push that decision into RegBankSelect. Assume for now
3177 // RegBankSelect knows what it's doing if the branch condition is scc, even
3178 // though it currently does not.
3179 if (!isVCC(Reg: CondReg, MRI: *MRI)) {
3180 if (MRI->getType(Reg: CondReg) != LLT::scalar(SizeInBits: 32))
3181 return false;
3182
3183 CondPhysReg = AMDGPU::SCC;
3184 BrOpcode = AMDGPU::S_CBRANCH_SCC1;
3185 ConstrainRC = &AMDGPU::SReg_32RegClass;
3186 } else {
3187 // FIXME: Should scc->vcc copies and with exec?
3188
3189 // Unless the value of CondReg is a result of a V_CMP* instruction then we
3190 // need to insert an and with exec.
3191 if (!isVCmpResult(Reg: CondReg, MRI&: *MRI)) {
3192 const bool Is64 = STI.isWave64();
3193 const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
3194 const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO;
3195
3196 Register TmpReg = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
3197 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: TmpReg)
3198 .addReg(RegNo: CondReg)
3199 .addReg(RegNo: Exec)
3200 .setOperandDead(3); // Dead scc
3201 CondReg = TmpReg;
3202 }
3203
3204 CondPhysReg = TRI.getVCC();
3205 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ;
3206 ConstrainRC = TRI.getBoolRC();
3207 }
3208
3209 if (!MRI->getRegClassOrNull(Reg: CondReg))
3210 MRI->setRegClass(Reg: CondReg, RC: ConstrainRC);
3211
3212 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CondPhysReg)
3213 .addReg(RegNo: CondReg);
3214 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: BrOpcode))
3215 .addMBB(MBB: I.getOperand(i: 1).getMBB());
3216
3217 I.eraseFromParent();
3218 return true;
3219}
3220
3221bool AMDGPUInstructionSelector::selectG_GLOBAL_VALUE(
3222 MachineInstr &I) const {
3223 Register DstReg = I.getOperand(i: 0).getReg();
3224 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3225 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3226 I.setDesc(TII.get(Opcode: IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32));
3227 if (IsVGPR)
3228 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
3229
3230 return RBI.constrainGenericRegister(
3231 Reg: DstReg, RC: IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, MRI&: *MRI);
3232}
3233
3234bool AMDGPUInstructionSelector::selectG_PTRMASK(MachineInstr &I) const {
3235 Register DstReg = I.getOperand(i: 0).getReg();
3236 Register SrcReg = I.getOperand(i: 1).getReg();
3237 Register MaskReg = I.getOperand(i: 2).getReg();
3238 LLT Ty = MRI->getType(Reg: DstReg);
3239 LLT MaskTy = MRI->getType(Reg: MaskReg);
3240 MachineBasicBlock *BB = I.getParent();
3241 const DebugLoc &DL = I.getDebugLoc();
3242
3243 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3244 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3245 const RegisterBank *MaskRB = RBI.getRegBank(Reg: MaskReg, MRI: *MRI, TRI);
3246 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3247 if (DstRB != SrcRB) // Should only happen for hand written MIR.
3248 return false;
3249
3250 // Try to avoid emitting a bit operation when we only need to touch half of
3251 // the 64-bit pointer.
3252 APInt MaskOnes = VT->getKnownOnes(R: MaskReg).zext(width: 64);
3253 const APInt MaskHi32 = APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32);
3254 const APInt MaskLo32 = APInt::getLowBitsSet(numBits: 64, loBitsSet: 32);
3255
3256 const bool CanCopyLow32 = (MaskOnes & MaskLo32) == MaskLo32;
3257 const bool CanCopyHi32 = (MaskOnes & MaskHi32) == MaskHi32;
3258
3259 if (!IsVGPR && Ty.getSizeInBits() == 64 &&
3260 !CanCopyLow32 && !CanCopyHi32) {
3261 auto MIB = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B64), DestReg: DstReg)
3262 .addReg(RegNo: SrcReg)
3263 .addReg(RegNo: MaskReg)
3264 .setOperandDead(3); // Dead scc
3265 I.eraseFromParent();
3266 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3267 return true;
3268 }
3269
3270 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
3271 const TargetRegisterClass &RegRC
3272 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
3273
3274 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *DstRB);
3275 const TargetRegisterClass *SrcRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *SrcRB);
3276 const TargetRegisterClass *MaskRC =
3277 TRI.getRegClassForTypeOnBank(Ty: MaskTy, Bank: *MaskRB);
3278
3279 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3280 !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3281 !RBI.constrainGenericRegister(Reg: MaskReg, RC: *MaskRC, MRI&: *MRI))
3282 return false;
3283
3284 if (Ty.getSizeInBits() == 32) {
3285 assert(MaskTy.getSizeInBits() == 32 &&
3286 "ptrmask should have been narrowed during legalize");
3287
3288 auto NewOp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: DstReg)
3289 .addReg(RegNo: SrcReg)
3290 .addReg(RegNo: MaskReg);
3291
3292 if (!IsVGPR)
3293 NewOp.setOperandDead(3); // Dead scc
3294 I.eraseFromParent();
3295 return true;
3296 }
3297
3298 Register HiReg = MRI->createVirtualRegister(RegClass: &RegRC);
3299 Register LoReg = MRI->createVirtualRegister(RegClass: &RegRC);
3300
3301 // Extract the subregisters from the source pointer.
3302 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3303 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
3304 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3305 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
3306
3307 Register MaskedLo, MaskedHi;
3308
3309 if (CanCopyLow32) {
3310 // If all the bits in the low half are 1, we only need a copy for it.
3311 MaskedLo = LoReg;
3312 } else {
3313 // Extract the mask subregister and apply the and.
3314 Register MaskLo = MRI->createVirtualRegister(RegClass: &RegRC);
3315 MaskedLo = MRI->createVirtualRegister(RegClass: &RegRC);
3316
3317 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskLo)
3318 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub0);
3319 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedLo)
3320 .addReg(RegNo: LoReg)
3321 .addReg(RegNo: MaskLo);
3322 }
3323
3324 if (CanCopyHi32) {
3325 // If all the bits in the high half are 1, we only need a copy for it.
3326 MaskedHi = HiReg;
3327 } else {
3328 Register MaskHi = MRI->createVirtualRegister(RegClass: &RegRC);
3329 MaskedHi = MRI->createVirtualRegister(RegClass: &RegRC);
3330
3331 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskHi)
3332 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub1);
3333 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedHi)
3334 .addReg(RegNo: HiReg)
3335 .addReg(RegNo: MaskHi);
3336 }
3337
3338 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
3339 .addReg(RegNo: MaskedLo)
3340 .addImm(Val: AMDGPU::sub0)
3341 .addReg(RegNo: MaskedHi)
3342 .addImm(Val: AMDGPU::sub1);
3343 I.eraseFromParent();
3344 return true;
3345}
3346
3347/// Return the register to use for the index value, and the subregister to use
3348/// for the indirectly accessed register.
3349static std::pair<Register, unsigned>
3350computeIndirectRegIndex(MachineRegisterInfo &MRI, const SIRegisterInfo &TRI,
3351 const TargetRegisterClass *SuperRC, Register IdxReg,
3352 unsigned EltSize, GISelValueTracking &ValueTracking) {
3353 Register IdxBaseReg;
3354 int Offset;
3355
3356 std::tie(args&: IdxBaseReg, args&: Offset) =
3357 AMDGPU::getBaseWithConstantOffset(MRI, Reg: IdxReg, ValueTracking: &ValueTracking);
3358 if (IdxBaseReg == AMDGPU::NoRegister) {
3359 // This will happen if the index is a known constant. This should ordinarily
3360 // be legalized out, but handle it as a register just in case.
3361 assert(Offset == 0);
3362 IdxBaseReg = IdxReg;
3363 }
3364
3365 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SuperRC, EltSize);
3366
3367 // Skip out of bounds offsets, or else we would end up using an undefined
3368 // register.
3369 if (static_cast<unsigned>(Offset) >= SubRegs.size())
3370 return std::pair(IdxReg, SubRegs[0]);
3371 return std::pair(IdxBaseReg, SubRegs[Offset]);
3372}
3373
3374bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
3375 MachineInstr &MI) const {
3376 Register DstReg = MI.getOperand(i: 0).getReg();
3377 Register SrcReg = MI.getOperand(i: 1).getReg();
3378 Register IdxReg = MI.getOperand(i: 2).getReg();
3379
3380 LLT DstTy = MRI->getType(Reg: DstReg);
3381 LLT SrcTy = MRI->getType(Reg: SrcReg);
3382
3383 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3384 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3385 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3386
3387 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3388 // into a waterfall loop.
3389 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3390 return false;
3391
3392 const TargetRegisterClass *SrcRC =
3393 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcRB);
3394 const TargetRegisterClass *DstRC =
3395 TRI.getRegClassForTypeOnBank(Ty: DstTy, Bank: *DstRB);
3396 if (!SrcRC || !DstRC)
3397 return false;
3398 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3399 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3400 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3401 return false;
3402
3403 MachineBasicBlock *BB = MI.getParent();
3404 const DebugLoc &DL = MI.getDebugLoc();
3405 const bool Is64 = DstTy.getSizeInBits() == 64;
3406
3407 unsigned SubReg;
3408 std::tie(args&: IdxReg, args&: SubReg) = computeIndirectRegIndex(
3409 MRI&: *MRI, TRI, SuperRC: SrcRC, IdxReg, EltSize: DstTy.getSizeInBits() / 8, ValueTracking&: *VT);
3410
3411 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) {
3412 if (DstTy.getSizeInBits() != 32 && !Is64)
3413 return false;
3414
3415 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3416 .addReg(RegNo: IdxReg);
3417
3418 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32;
3419 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
3420 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3421 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3422 MI.eraseFromParent();
3423 return true;
3424 }
3425
3426 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32)
3427 return false;
3428
3429 if (!STI.useVGPRIndexMode()) {
3430 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3431 .addReg(RegNo: IdxReg);
3432 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: DstReg)
3433 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3434 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3435 MI.eraseFromParent();
3436 return true;
3437 }
3438
3439 const MCInstrDesc &GPRIDXDesc =
3440 TII.getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *SrcRC), IsIndirectSrc: true);
3441 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3442 .addReg(RegNo: SrcReg)
3443 .addReg(RegNo: IdxReg)
3444 .addImm(Val: SubReg);
3445
3446 MI.eraseFromParent();
3447 return true;
3448}
3449
3450// TODO: Fold insert_vector_elt (extract_vector_elt) into movrelsd
3451bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT(
3452 MachineInstr &MI) const {
3453 Register DstReg = MI.getOperand(i: 0).getReg();
3454 Register VecReg = MI.getOperand(i: 1).getReg();
3455 Register ValReg = MI.getOperand(i: 2).getReg();
3456 Register IdxReg = MI.getOperand(i: 3).getReg();
3457
3458 LLT VecTy = MRI->getType(Reg: DstReg);
3459 LLT ValTy = MRI->getType(Reg: ValReg);
3460 unsigned VecSize = VecTy.getSizeInBits();
3461 unsigned ValSize = ValTy.getSizeInBits();
3462
3463 const RegisterBank *VecRB = RBI.getRegBank(Reg: VecReg, MRI: *MRI, TRI);
3464 const RegisterBank *ValRB = RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
3465 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3466
3467 assert(VecTy.getElementType() == ValTy);
3468
3469 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3470 // into a waterfall loop.
3471 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3472 return false;
3473
3474 const TargetRegisterClass *VecRC =
3475 TRI.getRegClassForTypeOnBank(Ty: VecTy, Bank: *VecRB);
3476 const TargetRegisterClass *ValRC =
3477 TRI.getRegClassForTypeOnBank(Ty: ValTy, Bank: *ValRB);
3478
3479 if (!RBI.constrainGenericRegister(Reg: VecReg, RC: *VecRC, MRI&: *MRI) ||
3480 !RBI.constrainGenericRegister(Reg: DstReg, RC: *VecRC, MRI&: *MRI) ||
3481 !RBI.constrainGenericRegister(Reg: ValReg, RC: *ValRC, MRI&: *MRI) ||
3482 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3483 return false;
3484
3485 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32)
3486 return false;
3487
3488 unsigned SubReg;
3489 std::tie(args&: IdxReg, args&: SubReg) =
3490 computeIndirectRegIndex(MRI&: *MRI, TRI, SuperRC: VecRC, IdxReg, EltSize: ValSize / 8, ValueTracking&: *VT);
3491
3492 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID &&
3493 STI.useVGPRIndexMode();
3494
3495 MachineBasicBlock *BB = MI.getParent();
3496 const DebugLoc &DL = MI.getDebugLoc();
3497
3498 if (!IndexMode) {
3499 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3500 .addReg(RegNo: IdxReg);
3501
3502 const MCInstrDesc &RegWriteOp = TII.getIndirectRegWriteMovRelPseudo(
3503 VecSize, EltSize: ValSize, IsSGPR: VecRB->getID() == AMDGPU::SGPRRegBankID);
3504 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: RegWriteOp, DestReg: DstReg)
3505 .addReg(RegNo: VecReg)
3506 .addReg(RegNo: ValReg)
3507 .addImm(Val: SubReg);
3508 MI.eraseFromParent();
3509 return true;
3510 }
3511
3512 const MCInstrDesc &GPRIDXDesc =
3513 TII.getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
3514 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3515 .addReg(RegNo: VecReg)
3516 .addReg(RegNo: ValReg)
3517 .addReg(RegNo: IdxReg)
3518 .addImm(Val: SubReg);
3519
3520 MI.eraseFromParent();
3521 return true;
3522}
3523
3524static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
3525 switch (Intr) {
3526 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
3527 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
3528 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
3529 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
3530 case Intrinsic::amdgcn_load_async_to_lds:
3531 case Intrinsic::amdgcn_global_load_async_lds:
3532 return true;
3533 }
3534 return false;
3535}
3536
3537bool AMDGPUInstructionSelector::selectBufferLoadLds(MachineInstr &MI) const {
3538 if (!Subtarget->hasVMemToLDSLoad())
3539 return false;
3540 unsigned Opc;
3541 unsigned Size = MI.getOperand(i: 3).getImm();
3542 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3543
3544 // The struct intrinsic variants add one additional operand over raw.
3545 const bool HasVIndex = MI.getNumOperands() == 9;
3546 Register VIndex;
3547 int OpOffset = 0;
3548 if (HasVIndex) {
3549 VIndex = MI.getOperand(i: 4).getReg();
3550 OpOffset = 1;
3551 }
3552
3553 Register VOffset = MI.getOperand(i: 4 + OpOffset).getReg();
3554 std::optional<ValueAndVReg> MaybeVOffset =
3555 getIConstantVRegValWithLookThrough(VReg: VOffset, MRI: *MRI);
3556 const bool HasVOffset = !MaybeVOffset || MaybeVOffset->Value.getZExtValue();
3557
3558 switch (Size) {
3559 default:
3560 return false;
3561 case 1:
3562 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
3563 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
3564 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
3565 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
3566 break;
3567 case 2:
3568 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
3569 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
3570 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
3571 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
3572 break;
3573 case 4:
3574 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
3575 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
3576 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
3577 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
3578 break;
3579 case 12:
3580 if (!Subtarget->hasLDSLoadB96_B128())
3581 return false;
3582
3583 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
3584 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
3585 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
3586 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
3587 break;
3588 case 16:
3589 if (!Subtarget->hasLDSLoadB96_B128())
3590 return false;
3591
3592 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
3593 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
3594 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
3595 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
3596 break;
3597 }
3598
3599 MachineBasicBlock *MBB = MI.getParent();
3600 const DebugLoc &DL = MI.getDebugLoc();
3601 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3602 .add(MO: MI.getOperand(i: 2));
3603
3604 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc));
3605
3606 if (HasVIndex && HasVOffset) {
3607 Register IdxReg = MRI->createVirtualRegister(RegClass: TRI.getVGPR64Class());
3608 BuildMI(BB&: *MBB, I: &*MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: IdxReg)
3609 .addReg(RegNo: VIndex)
3610 .addImm(Val: AMDGPU::sub0)
3611 .addReg(RegNo: VOffset)
3612 .addImm(Val: AMDGPU::sub1);
3613
3614 MIB.addReg(RegNo: IdxReg);
3615 } else if (HasVIndex) {
3616 MIB.addReg(RegNo: VIndex);
3617 } else if (HasVOffset) {
3618 MIB.addReg(RegNo: VOffset);
3619 }
3620
3621 MIB.add(MO: MI.getOperand(i: 1)); // rsrc
3622 MIB.add(MO: MI.getOperand(i: 5 + OpOffset)); // soffset
3623 MIB.add(MO: MI.getOperand(i: 6 + OpOffset)); // imm offset
3624 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
3625 unsigned Aux = MI.getOperand(i: 7 + OpOffset).getImm();
3626 MIB.addImm(Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL
3627 : AMDGPU::CPol::ALL_pregfx12)); // cpol
3628 MIB.addImm(
3629 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
3630 ? 1
3631 : 0); // swz
3632 MIB.addImm(Val: isAsyncLDSDMA(Intr: IntrinsicID));
3633
3634 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3635 // Don't set the offset value here because the pointer points to the base of
3636 // the buffer.
3637 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3638
3639 MachinePointerInfo StorePtrI = LoadPtrI;
3640 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3641 AddressSpace: AMDGPUAS::BUFFER_RESOURCE));
3642 LoadPtrI.AddrSpace = AMDGPUAS::BUFFER_RESOURCE;
3643 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3644
3645 auto F = LoadMMO->getFlags() &
3646 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3647 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3648 Size, BaseAlignment: LoadMMO->getBaseAlign());
3649
3650 MachineMemOperand *StoreMMO =
3651 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3652 Size: sizeof(int32_t), BaseAlignment: LoadMMO->getBaseAlign());
3653
3654 MIB.setMemRefs({LoadMMO, StoreMMO});
3655
3656 MI.eraseFromParent();
3657 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3658 return true;
3659}
3660
3661/// Match a zero extend from a 32-bit value to 64-bits.
3662Register AMDGPUInstructionSelector::matchZeroExtendFromS32(Register Reg) const {
3663 Register ZExtSrc;
3664 if (mi_match(R: Reg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrc))))
3665 return MRI->getType(Reg: ZExtSrc) == LLT::scalar(SizeInBits: 32) ? ZExtSrc : Register();
3666
3667 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 0)
3668 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3669 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3670 return Register();
3671
3672 assert(Def->getNumOperands() == 3 &&
3673 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3674 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_ZeroInt())) {
3675 return Def->getOperand(i: 1).getReg();
3676 }
3677
3678 return Register();
3679}
3680
3681/// Match a sign extend from a 32-bit value to 64-bits.
3682Register AMDGPUInstructionSelector::matchSignExtendFromS32(Register Reg) const {
3683 Register SExtSrc;
3684 if (mi_match(R: Reg, MRI: *MRI, P: m_GSExt(Src: m_Reg(R&: SExtSrc))))
3685 return MRI->getType(Reg: SExtSrc) == LLT::scalar(SizeInBits: 32) ? SExtSrc : Register();
3686
3687 // Match legalized form %sext = G_MERGE_VALUES (s32 %x), G_ASHR((S32 %x, 31))
3688 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3689 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3690 return Register();
3691
3692 assert(Def->getNumOperands() == 3 &&
3693 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3694 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI,
3695 P: m_GAShr(L: m_SpecificReg(RequestedReg: Def->getOperand(i: 1).getReg()),
3696 R: m_SpecificICst(RequestedValue: 31))))
3697 return Def->getOperand(i: 1).getReg();
3698
3699 Register ZextSrc = matchZeroExtendFromS32(Reg);
3700 if (ZextSrc && VT->signBitIsZero(Op: ZextSrc))
3701 return ZextSrc;
3702
3703 return Register();
3704}
3705
3706/// Match a zero extend from a 32-bit value to 64-bits, or \p Reg itself if it
3707/// is 32-bit.
3708Register
3709AMDGPUInstructionSelector::matchZeroExtendFromS32OrS32(Register Reg) const {
3710 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3711 : matchZeroExtendFromS32(Reg);
3712}
3713
3714/// Match a sign extend from a 32-bit value to 64-bits, or \p Reg itself if it
3715/// is 32-bit.
3716Register
3717AMDGPUInstructionSelector::matchSignExtendFromS32OrS32(Register Reg) const {
3718 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3719 : matchSignExtendFromS32(Reg);
3720}
3721
3722Register
3723AMDGPUInstructionSelector::matchExtendFromS32OrS32(Register Reg,
3724 bool IsSigned) const {
3725 if (IsSigned)
3726 return matchSignExtendFromS32OrS32(Reg);
3727
3728 return matchZeroExtendFromS32OrS32(Reg);
3729}
3730
3731Register AMDGPUInstructionSelector::matchAnyExtendFromS32(Register Reg) const {
3732 Register AnyExtSrc;
3733 if (mi_match(R: Reg, MRI: *MRI, P: m_GAnyExt(Src: m_Reg(R&: AnyExtSrc))))
3734 return MRI->getType(Reg: AnyExtSrc) == LLT::scalar(SizeInBits: 32) ? AnyExtSrc : Register();
3735
3736 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 G_IMPLICIT_DEF)
3737 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3738 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3739 return Register();
3740
3741 assert(Def->getNumOperands() == 3 &&
3742 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3743
3744 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_GImplicitDef()))
3745 return Def->getOperand(i: 1).getReg();
3746
3747 return Register();
3748}
3749
3750bool AMDGPUInstructionSelector::selectGlobalLoadLds(MachineInstr &MI) const{
3751 if (!Subtarget->hasVMemToLDSLoad())
3752 return false;
3753
3754 unsigned Opc;
3755 unsigned Size = MI.getOperand(i: 3).getImm();
3756 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3757
3758 switch (Size) {
3759 default:
3760 return false;
3761 case 1:
3762 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
3763 break;
3764 case 2:
3765 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
3766 break;
3767 case 4:
3768 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
3769 break;
3770 case 12:
3771 if (!Subtarget->hasLDSLoadB96_B128())
3772 return false;
3773 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
3774 break;
3775 case 16:
3776 if (!Subtarget->hasLDSLoadB96_B128())
3777 return false;
3778 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
3779 break;
3780 }
3781
3782 MachineBasicBlock *MBB = MI.getParent();
3783 const DebugLoc &DL = MI.getDebugLoc();
3784 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3785 .add(MO: MI.getOperand(i: 2));
3786
3787 Register Addr = MI.getOperand(i: 1).getReg();
3788 Register VOffset;
3789 // Try to split SAddr and VOffset. Global and LDS pointers share the same
3790 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
3791 if (!isSGPR(Reg: Addr)) {
3792 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
3793 if (isSGPR(Reg: AddrDef->Reg)) {
3794 Addr = AddrDef->Reg;
3795 } else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
3796 Register SAddr =
3797 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
3798 if (isSGPR(Reg: SAddr)) {
3799 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
3800 if (Register Off = matchZeroExtendFromS32(Reg: PtrBaseOffset)) {
3801 Addr = SAddr;
3802 VOffset = Off;
3803 }
3804 }
3805 }
3806 }
3807
3808 if (isSGPR(Reg: Addr)) {
3809 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
3810 if (!VOffset) {
3811 VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
3812 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
3813 .addImm(Val: 0);
3814 }
3815 }
3816
3817 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc))
3818 .addReg(RegNo: Addr);
3819
3820 if (isSGPR(Reg: Addr))
3821 MIB.addReg(RegNo: VOffset);
3822
3823 MIB.add(MO: MI.getOperand(i: 4)); // offset
3824
3825 unsigned Aux = MI.getOperand(i: 5).getImm();
3826 MIB.addImm(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS); // cpol
3827 MIB.addImm(Val: isAsyncLDSDMA(Intr: IntrinsicID));
3828
3829 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3830 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3831 LoadPtrI.Offset = MI.getOperand(i: 4).getImm();
3832 MachinePointerInfo StorePtrI = LoadPtrI;
3833 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3834 AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
3835 LoadPtrI.AddrSpace = AMDGPUAS::GLOBAL_ADDRESS;
3836 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3837 auto F = LoadMMO->getFlags() &
3838 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3839 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3840 Size, BaseAlignment: LoadMMO->getBaseAlign());
3841 MachineMemOperand *StoreMMO =
3842 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3843 Size: sizeof(int32_t), BaseAlignment: Align(4));
3844
3845 MIB.setMemRefs({LoadMMO, StoreMMO});
3846
3847 MI.eraseFromParent();
3848 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3849 return true;
3850}
3851
3852bool AMDGPUInstructionSelector::selectTensorLoadStore(MachineInstr &MI,
3853 Intrinsic::ID IID) const {
3854 bool IsLoad = IID == Intrinsic::amdgcn_tensor_load_to_lds;
3855 unsigned Opc =
3856 IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d4 : AMDGPU::TENSOR_STORE_FROM_LDS_d4;
3857 int NumGroups = 4;
3858
3859 // A lamda function to check whether an operand is a vector of all 0s.
3860 const auto isAllZeros = [&](MachineOperand &Opnd) {
3861 const MachineInstr *DefMI = MRI->getVRegDef(Reg: Opnd.getReg());
3862 if (!DefMI)
3863 return false;
3864 return llvm::isBuildVectorAllZeros(MI: *DefMI, MRI: *MRI, AllowUndef: true);
3865 };
3866
3867 // Use _D2 version if both group 2 and 3 are zero-initialized.
3868 if (isAllZeros(MI.getOperand(i: 3)) && isAllZeros(MI.getOperand(i: 4))) {
3869 NumGroups = 2;
3870 Opc = IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d2
3871 : AMDGPU::TENSOR_STORE_FROM_LDS_d2;
3872 }
3873
3874 // TODO: Handle the fifth group: MI.getOpetand(5), which is silently ignored
3875 // for now because all existing targets only support up to 4 groups.
3876 MachineBasicBlock *MBB = MI.getParent();
3877 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
3878 .add(MO: MI.getOperand(i: 1)) // D# group 0
3879 .add(MO: MI.getOperand(i: 2)); // D# group 1
3880
3881 if (NumGroups >= 4) { // Has at least 4 groups
3882 MIB.add(MO: MI.getOperand(i: 3)) // D# group 2
3883 .add(MO: MI.getOperand(i: 4)); // D# group 3
3884 }
3885
3886 MIB.addImm(Val: 0) // r128
3887 .add(MO: MI.getOperand(i: 6)); // cpol
3888
3889 MI.eraseFromParent();
3890 return true;
3891}
3892
3893bool AMDGPUInstructionSelector::selectBVHIntersectRayIntrinsic(
3894 MachineInstr &MI) const {
3895 unsigned OpcodeOpIdx =
3896 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY ? 1 : 3;
3897 MI.setDesc(TII.get(Opcode: MI.getOperand(i: OpcodeOpIdx).getImm()));
3898 MI.removeOperand(OpNo: OpcodeOpIdx);
3899 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
3900 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
3901 return true;
3902}
3903
3904// FIXME: This should be removed and let the patterns select. We just need the
3905// AGPR/VGPR combination versions.
3906bool AMDGPUInstructionSelector::selectSMFMACIntrin(MachineInstr &MI) const {
3907 unsigned Opc;
3908 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
3909 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
3910 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64;
3911 break;
3912 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
3913 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64;
3914 break;
3915 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
3916 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64;
3917 break;
3918 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
3919 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64;
3920 break;
3921 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
3922 Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64;
3923 break;
3924 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
3925 Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64;
3926 break;
3927 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
3928 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64;
3929 break;
3930 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
3931 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64;
3932 break;
3933 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
3934 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64;
3935 break;
3936 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
3937 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64;
3938 break;
3939 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
3940 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64;
3941 break;
3942 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
3943 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64;
3944 break;
3945 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
3946 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64;
3947 break;
3948 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
3949 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64;
3950 break;
3951 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
3952 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_F16_e64;
3953 break;
3954 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
3955 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_F16_e64;
3956 break;
3957 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
3958 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF16_e64;
3959 break;
3960 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
3961 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF16_e64;
3962 break;
3963 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
3964 Opc = AMDGPU::V_SMFMAC_I32_16X16X128_I8_e64;
3965 break;
3966 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
3967 Opc = AMDGPU::V_SMFMAC_I32_32X32X64_I8_e64;
3968 break;
3969 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
3970 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_BF8_e64;
3971 break;
3972 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
3973 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_FP8_e64;
3974 break;
3975 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
3976 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_BF8_e64;
3977 break;
3978 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
3979 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_FP8_e64;
3980 break;
3981 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
3982 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_BF8_e64;
3983 break;
3984 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
3985 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_FP8_e64;
3986 break;
3987 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
3988 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_BF8_e64;
3989 break;
3990 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
3991 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_FP8_e64;
3992 break;
3993 default:
3994 llvm_unreachable("unhandled smfmac intrinsic");
3995 }
3996
3997 auto VDst_In = MI.getOperand(i: 4);
3998
3999 MI.setDesc(TII.get(Opcode: Opc));
4000 MI.removeOperand(OpNo: 4); // VDst_In
4001 MI.removeOperand(OpNo: 1); // Intrinsic ID
4002 MI.addOperand(Op: VDst_In); // Readd VDst_In to the end
4003 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
4004 const MCInstrDesc &MCID = MI.getDesc();
4005 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
4006 MI.getOperand(i: 0).setIsEarlyClobber(true);
4007 }
4008 return true;
4009}
4010
4011bool AMDGPUInstructionSelector::selectPermlaneSwapIntrin(
4012 MachineInstr &MI, Intrinsic::ID IntrID) const {
4013 if (IntrID == Intrinsic::amdgcn_permlane16_swap &&
4014 !Subtarget->hasPermlane16Swap())
4015 return false;
4016 if (IntrID == Intrinsic::amdgcn_permlane32_swap &&
4017 !Subtarget->hasPermlane32Swap())
4018 return false;
4019
4020 unsigned Opcode = IntrID == Intrinsic::amdgcn_permlane16_swap
4021 ? AMDGPU::V_PERMLANE16_SWAP_B32_e64
4022 : AMDGPU::V_PERMLANE32_SWAP_B32_e64;
4023
4024 MI.removeOperand(OpNo: 2);
4025 MI.setDesc(TII.get(Opcode));
4026 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
4027
4028 MachineOperand &FI = MI.getOperand(i: 4);
4029 FI.setImm(FI.getImm() ? AMDGPU::DPP::DPP_FI_1 : AMDGPU::DPP::DPP_FI_0);
4030
4031 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
4032 return true;
4033}
4034
4035bool AMDGPUInstructionSelector::selectWaveAddress(MachineInstr &MI) const {
4036 Register DstReg = MI.getOperand(i: 0).getReg();
4037 Register SrcReg = MI.getOperand(i: 1).getReg();
4038 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4039 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4040 MachineBasicBlock *MBB = MI.getParent();
4041 const DebugLoc &DL = MI.getDebugLoc();
4042
4043 if (IsVALU) {
4044 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: DstReg)
4045 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4046 .addReg(RegNo: SrcReg);
4047 } else {
4048 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: DstReg)
4049 .addReg(RegNo: SrcReg)
4050 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4051 .setOperandDead(3); // Dead scc
4052 }
4053
4054 const TargetRegisterClass &RC =
4055 IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
4056 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
4057 return false;
4058
4059 MI.eraseFromParent();
4060 return true;
4061}
4062
4063bool AMDGPUInstructionSelector::selectWaveShuffleIntrin(
4064 MachineInstr &MI) const {
4065 assert(MI.getNumOperands() == 4);
4066 MachineBasicBlock *MBB = MI.getParent();
4067 const DebugLoc &DL = MI.getDebugLoc();
4068
4069 Register DstReg = MI.getOperand(i: 0).getReg();
4070 Register ValReg = MI.getOperand(i: 2).getReg();
4071 Register IdxReg = MI.getOperand(i: 3).getReg();
4072
4073 const LLT DstTy = MRI->getType(Reg: DstReg);
4074 unsigned DstSize = DstTy.getSizeInBits();
4075 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4076 const TargetRegisterClass *DstRC =
4077 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
4078
4079 if (DstTy != LLT::scalar(SizeInBits: 32))
4080 return false;
4081
4082 if (!Subtarget->supportsBPermute())
4083 return false;
4084
4085 // If we can bpermute across the whole wave, then just do that
4086 if (Subtarget->supportsWaveWideBPermute()) {
4087 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4088 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4089 .addImm(Val: 2)
4090 .addReg(RegNo: IdxReg);
4091
4092 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: DstReg)
4093 .addReg(RegNo: ShiftIdxReg)
4094 .addReg(RegNo: ValReg)
4095 .addImm(Val: 0);
4096 } else {
4097 // Otherwise, we need to make use of whole wave mode
4098 assert(Subtarget->isWave64());
4099
4100 // Set inactive lanes to poison
4101 Register UndefValReg =
4102 MRI->createVirtualRegister(RegClass: TRI.getRegClass(i: AMDGPU::SReg_32RegClassID));
4103 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefValReg);
4104
4105 Register UndefExecReg = MRI->createVirtualRegister(
4106 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4107 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExecReg);
4108
4109 Register PoisonValReg = MRI->createVirtualRegister(RegClass: DstRC);
4110 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonValReg)
4111 .addImm(Val: 0)
4112 .addReg(RegNo: ValReg)
4113 .addImm(Val: 0)
4114 .addReg(RegNo: UndefValReg)
4115 .addReg(RegNo: UndefExecReg);
4116
4117 // ds_bpermute requires index to be multiplied by 4
4118 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4119 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4120 .addImm(Val: 2)
4121 .addReg(RegNo: IdxReg);
4122
4123 Register PoisonIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4124 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonIdxReg)
4125 .addImm(Val: 0)
4126 .addReg(RegNo: ShiftIdxReg)
4127 .addImm(Val: 0)
4128 .addReg(RegNo: UndefValReg)
4129 .addReg(RegNo: UndefExecReg);
4130
4131 Register PoisonUnshiftedIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4132 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32),
4133 DestReg: PoisonUnshiftedIdxReg)
4134 .addImm(Val: 0)
4135 .addReg(RegNo: IdxReg)
4136 .addImm(Val: 0)
4137 .addReg(RegNo: UndefValReg)
4138 .addReg(RegNo: UndefExecReg);
4139
4140 // Get permutation of each half, then we'll select which one to use
4141 Register SameSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4142 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: SameSidePermReg)
4143 .addReg(RegNo: PoisonIdxReg)
4144 .addReg(RegNo: PoisonValReg)
4145 .addImm(Val: 0);
4146
4147 Register SwappedValReg = MRI->createVirtualRegister(RegClass: DstRC);
4148 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_PERMLANE64_B32), DestReg: SwappedValReg)
4149 .addReg(RegNo: PoisonValReg);
4150
4151 Register OppSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4152 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: OppSidePermReg)
4153 .addReg(RegNo: PoisonIdxReg)
4154 .addReg(RegNo: SwappedValReg)
4155 .addImm(Val: 0);
4156
4157 Register WWMSwapPermReg = MRI->createVirtualRegister(RegClass: DstRC);
4158 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::STRICT_WWM), DestReg: WWMSwapPermReg)
4159 .addReg(RegNo: OppSidePermReg);
4160
4161 // Select which side to take the permute from
4162 // We can get away with only using mbcnt_lo here since we're only
4163 // trying to detect which side of 32 each lane is on, and mbcnt_lo
4164 // returns 32 for lanes 32-63.
4165 Register ThreadIDReg = MRI->createVirtualRegister(RegClass: DstRC);
4166 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64), DestReg: ThreadIDReg)
4167 .addImm(Val: -1)
4168 .addImm(Val: 0);
4169
4170 Register XORReg = MRI->createVirtualRegister(RegClass: DstRC);
4171 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_XOR_B32_e64), DestReg: XORReg)
4172 .addReg(RegNo: ThreadIDReg)
4173 .addReg(RegNo: PoisonUnshiftedIdxReg);
4174
4175 Register ANDReg = MRI->createVirtualRegister(RegClass: DstRC);
4176 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: ANDReg)
4177 .addReg(RegNo: XORReg)
4178 .addImm(Val: 32);
4179
4180 Register CompareReg = MRI->createVirtualRegister(
4181 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4182 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CompareReg)
4183 .addReg(RegNo: ANDReg)
4184 .addImm(Val: 0);
4185
4186 // Finally do the selection
4187 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
4188 .addImm(Val: 0)
4189 .addReg(RegNo: WWMSwapPermReg)
4190 .addImm(Val: 0)
4191 .addReg(RegNo: SameSidePermReg)
4192 .addReg(RegNo: CompareReg);
4193 }
4194
4195 MI.eraseFromParent();
4196 return true;
4197}
4198
4199// Match BITOP3 operation and return a number of matched instructions plus
4200// truth table.
4201static std::pair<unsigned, uint8_t> BitOp3_Op(Register R,
4202 SmallVectorImpl<Register> &Src,
4203 const MachineRegisterInfo &MRI) {
4204 unsigned NumOpcodes = 0;
4205 uint8_t LHSBits, RHSBits;
4206
4207 auto getOperandBits = [&Src, R, &MRI](Register Op, uint8_t &Bits) -> bool {
4208 // Define truth table given Src0, Src1, Src2 bits permutations:
4209 // 0 0 0
4210 // 0 0 1
4211 // 0 1 0
4212 // 0 1 1
4213 // 1 0 0
4214 // 1 0 1
4215 // 1 1 0
4216 // 1 1 1
4217 const uint8_t SrcBits[3] = { 0xf0, 0xcc, 0xaa };
4218
4219 if (mi_match(R: Op, MRI, P: m_AllOnesInt())) {
4220 Bits = 0xff;
4221 return true;
4222 }
4223 if (mi_match(R: Op, MRI, P: m_ZeroInt())) {
4224 Bits = 0;
4225 return true;
4226 }
4227
4228 for (unsigned I = 0; I < Src.size(); ++I) {
4229 // Try to find existing reused operand
4230 if (Src[I] == Op) {
4231 Bits = SrcBits[I];
4232 return true;
4233 }
4234 // Try to replace parent operator
4235 if (Src[I] == R) {
4236 Bits = SrcBits[I];
4237 Src[I] = Op;
4238 return true;
4239 }
4240 }
4241
4242 if (Src.size() == 3) {
4243 // No room left for operands. Try one last time, there can be a 'not' of
4244 // one of our source operands. In this case we can compute the bits
4245 // without growing Src vector.
4246 Register LHS;
4247 if (mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: LHS)))) {
4248 LHS = getSrcRegIgnoringCopies(Reg: LHS, MRI);
4249 for (unsigned I = 0; I < Src.size(); ++I) {
4250 if (Src[I] == LHS) {
4251 Bits = ~SrcBits[I];
4252 return true;
4253 }
4254 }
4255 }
4256
4257 return false;
4258 }
4259
4260 Bits = SrcBits[Src.size()];
4261 Src.push_back(Elt: Op);
4262 return true;
4263 };
4264
4265 MachineInstr *MI = MRI.getVRegDef(Reg: R);
4266 switch (MI->getOpcode()) {
4267 case TargetOpcode::G_AND:
4268 case TargetOpcode::G_OR:
4269 case TargetOpcode::G_XOR: {
4270 Register LHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 1).getReg(), MRI);
4271 Register RHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 2).getReg(), MRI);
4272
4273 SmallVector<Register, 3> Backup(Src.begin(), Src.end());
4274 if (!getOperandBits(LHS, LHSBits) ||
4275 !getOperandBits(RHS, RHSBits)) {
4276 Src = std::move(Backup);
4277 return std::make_pair(x: 0, y: 0);
4278 }
4279
4280 // Recursion is naturally limited by the size of the operand vector.
4281 //
4282 // When LHS and RHS share a common sub-expression, one side's recursion
4283 // may decompose that sub-expression and replace the Src slot the other
4284 // side occupies with sub-operands via the "replace parent" path in
4285 // getOperandBits. The other side's cached bit-pattern then refers to a
4286 // slot whose contents changed, producing a wrong truth table.
4287 //
4288 // We detect this in three ways:
4289 // (A) If LHS recursed, its truth table is valid against the Src state
4290 // when LHS recursion completed (SrcAfterLHS). If RHS recursion
4291 // then mutates a Src slot that LHSBits depends on, LHSBits is
4292 // stale.
4293 // (B) If RHS did not recurse, RHSBits came from getOperandBits and
4294 // refers to a specific Src slot. If that slot's contents changed
4295 // (by either recursion), RHSBits is stale.
4296 // (C) Symmetrically for LHS if it did not recurse.
4297 SmallVector<Register, 3> SrcBeforeRecurse(Src.begin(), Src.end());
4298 uint8_t LHSBitsOrig = LHSBits;
4299 uint8_t RHSBitsOrig = RHSBits;
4300
4301 auto LHSOp = BitOp3_Op(R: LHS, Src, MRI);
4302 if (LHSOp.first) {
4303 NumOpcodes += LHSOp.first;
4304 LHSBits = LHSOp.second;
4305 }
4306
4307 SmallVector<Register, 3> SrcAfterLHS(Src.begin(), Src.end());
4308
4309 auto RHSOp = BitOp3_Op(R: RHS, Src, MRI);
4310 if (RHSOp.first) {
4311 NumOpcodes += RHSOp.first;
4312 RHSBits = RHSOp.second;
4313 }
4314
4315 // dependsOnSlot: true iff the truth table TT varies with slot Slot.
4316 auto dependsOnSlot = [](uint8_t TT, int Slot) -> bool {
4317 if (Slot < 0 || Slot > 2)
4318 return false;
4319 const uint8_t Masks[3] = {0x0f, 0x33, 0x55};
4320 const int Shifts[3] = {4, 2, 1};
4321 return ((TT ^ (TT >> Shifts[Slot])) & Masks[Slot]) != 0;
4322 };
4323
4324 // findSlot: locate the Src slot a getOperandBits result depends on,
4325 // including negated (NOT) patterns that getOperandBits resolves via
4326 // the ~SrcBits[I] shortcut.
4327 const uint8_t SrcBitsConst[3] = {0xf0, 0xcc, 0xaa};
4328 auto findSlot = [&](uint8_t Bits, Register Op,
4329 const SmallVectorImpl<Register> &S) -> int {
4330 Register NegatedInner;
4331 bool IsNegationOp = mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: NegatedInner)));
4332 if (IsNegationOp)
4333 NegatedInner = getSrcRegIgnoringCopies(Reg: NegatedInner, MRI);
4334 for (int I = 0; I < (int)S.size(); I++) {
4335 if (Bits == SrcBitsConst[I] && S[I] == Op)
4336 return I;
4337 if (IsNegationOp && Bits == (uint8_t)~SrcBitsConst[I] &&
4338 S[I] == NegatedInner)
4339 return I;
4340 }
4341 return -1;
4342 };
4343
4344 bool Stale = false;
4345
4346 // (A) LHS recursed: its truth table is against SrcAfterLHS.
4347 // Check if RHS recursion mutated a slot that LHSBits uses.
4348 if (LHSOp.first) {
4349 for (int I = 0; I < (int)SrcAfterLHS.size() && I < 3; I++) {
4350 if (I < (int)Src.size() && Src[I] != SrcAfterLHS[I] &&
4351 dependsOnSlot(LHSBits, I)) {
4352 Stale = true;
4353 break;
4354 }
4355 }
4356 }
4357
4358 // (B) RHS did not recurse: RHSBits from getOperandBits is against
4359 // SrcBeforeRecurse. Check if that slot was mutated since then.
4360 if (!Stale && !RHSOp.first) {
4361 int Slot = findSlot(RHSBitsOrig, RHS, SrcBeforeRecurse);
4362 if (Slot >= 0 &&
4363 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4364 Stale = true;
4365 }
4366
4367 // (C) LHS did not recurse: LHSBits from getOperandBits is against
4368 // SrcBeforeRecurse. Check if that slot was mutated since then.
4369 if (!Stale && !LHSOp.first) {
4370 int Slot = findSlot(LHSBitsOrig, LHS, SrcBeforeRecurse);
4371 if (Slot >= 0 &&
4372 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4373 Stale = true;
4374 }
4375
4376 if (Stale) {
4377 Src = std::move(SrcBeforeRecurse);
4378 LHSBits = LHSBitsOrig;
4379 RHSBits = RHSBitsOrig;
4380 NumOpcodes = 0;
4381 }
4382 break;
4383 }
4384 default:
4385 return std::make_pair(x: 0, y: 0);
4386 }
4387
4388 uint8_t TTbl;
4389 switch (MI->getOpcode()) {
4390 case TargetOpcode::G_AND:
4391 TTbl = LHSBits & RHSBits;
4392 break;
4393 case TargetOpcode::G_OR:
4394 TTbl = LHSBits | RHSBits;
4395 break;
4396 case TargetOpcode::G_XOR:
4397 TTbl = LHSBits ^ RHSBits;
4398 break;
4399 default:
4400 break;
4401 }
4402
4403 return std::make_pair(x: NumOpcodes + 1, y&: TTbl);
4404}
4405
4406bool AMDGPUInstructionSelector::selectBITOP3(MachineInstr &MI) const {
4407 if (!Subtarget->hasBitOp3Insts())
4408 return false;
4409
4410 Register DstReg = MI.getOperand(i: 0).getReg();
4411 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4412 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4413 if (!IsVALU)
4414 return false;
4415
4416 SmallVector<Register, 3> Src;
4417 uint8_t TTbl;
4418 unsigned NumOpcodes;
4419
4420 std::tie(args&: NumOpcodes, args&: TTbl) = BitOp3_Op(R: DstReg, Src, MRI: *MRI);
4421
4422 // Src.empty() case can happen if all operands are all zero or all ones.
4423 // Normally it shall be optimized out before reaching this.
4424 if (NumOpcodes < 2 || Src.empty())
4425 return false;
4426
4427 // RegBankSelect splits wider VALU logic ops and widens 1-bit ones, so only
4428 // 16 and 32 bit types reach here. Note that <2 x i16> is 32 bits wide.
4429 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
4430 assert((Size == 16 || Size == 32) && "unexpected VALU logic op size");
4431 const bool IsB32 = Size == 32;
4432 if (NumOpcodes == 2 && IsB32) {
4433 // Avoid using BITOP3 for OR3, XOR3, AND_OR. This is not faster but makes
4434 // asm more readable. This cannot be modeled with AddedComplexity because
4435 // selector does not know how many operations did we match.
4436 if (mi_match(MI, MRI: *MRI, P: m_GXor(L: m_GXor(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4437 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GOr(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4438 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GAnd(L: m_Reg(), R: m_Reg()), R: m_Reg())))
4439 return false;
4440 } else if (NumOpcodes < 4) {
4441 // For a uniform case threshold should be higher to account for moves
4442 // between VGPRs and SGPRs. It needs one operand in a VGPR, rest two can be
4443 // in SGPRs and a readtfirstlane after.
4444 return false;
4445 }
4446
4447 unsigned Opc = IsB32 ? AMDGPU::V_BITOP3_B32_e64 : AMDGPU::V_BITOP3_B16_e64;
4448 if (!IsB32 && STI.hasTrue16BitInsts())
4449 Opc = STI.useRealTrue16Insts() ? AMDGPU::V_BITOP3_B16_gfx1250_t16_e64
4450 : AMDGPU::V_BITOP3_B16_gfx1250_fake16_e64;
4451 unsigned CBL = STI.getConstantBusLimit(Opcode: Opc);
4452 MachineBasicBlock *MBB = MI.getParent();
4453 const DebugLoc &DL = MI.getDebugLoc();
4454
4455 for (unsigned I = 0; I < Src.size(); ++I) {
4456 const RegisterBank *RB = RBI.getRegBank(Reg: Src[I], MRI: *MRI, TRI);
4457 if (RB->getID() != AMDGPU::SGPRRegBankID)
4458 continue;
4459 if (CBL > 0) {
4460 --CBL;
4461 continue;
4462 }
4463 Register NewReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
4464 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: NewReg)
4465 .addReg(RegNo: Src[I]);
4466 Src[I] = NewReg;
4467 }
4468
4469 // Last operand can be ignored, turning a ternary operation into a binary.
4470 // For example: (~a & b & c) | (~a & b & ~c) -> (~a & b). We can replace
4471 // 'c' with 'a' here without changing the answer. In some pathological
4472 // cases it should be possible to get an operation with a single operand
4473 // too if optimizer would not catch it.
4474 while (Src.size() < 3)
4475 Src.push_back(Elt: Src[0]);
4476
4477 auto MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg);
4478 if (!IsB32)
4479 MIB.addImm(Val: 0); // src_mod0
4480 MIB.addReg(RegNo: Src[0]);
4481 if (!IsB32)
4482 MIB.addImm(Val: 0); // src_mod1
4483 MIB.addReg(RegNo: Src[1]);
4484 if (!IsB32)
4485 MIB.addImm(Val: 0); // src_mod2
4486 MIB.addReg(RegNo: Src[2])
4487 .addImm(Val: TTbl);
4488 if (!IsB32)
4489 MIB.addImm(Val: 0); // op_sel
4490
4491 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
4492 MI.eraseFromParent();
4493
4494 return true;
4495}
4496
4497bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const {
4498 Register SrcReg = MI.getOperand(i: 0).getReg();
4499 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
4500 return false;
4501
4502 MachineInstr *DefMI = MRI->getVRegDef(Reg: SrcReg);
4503 Register SP =
4504 Subtarget->getTargetLowering()->getStackPointerRegisterToSaveRestore();
4505 Register WaveAddr = getWaveAddress(Def: DefMI);
4506 MachineBasicBlock *MBB = MI.getParent();
4507 const DebugLoc &DL = MI.getDebugLoc();
4508
4509 if (!WaveAddr) {
4510 WaveAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
4511 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: WaveAddr)
4512 .addReg(RegNo: SrcReg)
4513 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4514 .setOperandDead(3); // Dead scc
4515 }
4516
4517 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: SP)
4518 .addReg(RegNo: WaveAddr);
4519
4520 MI.eraseFromParent();
4521 return true;
4522}
4523
4524bool AMDGPUInstructionSelector::select(MachineInstr &I) {
4525
4526 if (!I.isPreISelOpcode()) {
4527 if (I.isCopy())
4528 return selectCOPY(I);
4529 return true;
4530 }
4531
4532 switch (I.getOpcode()) {
4533 case TargetOpcode::G_AND:
4534 case TargetOpcode::G_OR:
4535 case TargetOpcode::G_XOR:
4536 if (selectBITOP3(MI&: I))
4537 return true;
4538 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4539 return true;
4540 return selectG_AND_OR_XOR(I);
4541 case TargetOpcode::G_ADD:
4542 case TargetOpcode::G_SUB:
4543 case TargetOpcode::G_PTR_ADD:
4544 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4545 return true;
4546 return selectG_ADD_SUB(I);
4547 case TargetOpcode::G_UADDO:
4548 case TargetOpcode::G_USUBO:
4549 case TargetOpcode::G_UADDE:
4550 case TargetOpcode::G_USUBE:
4551 return selectG_UADDO_USUBO_UADDE_USUBE(I);
4552 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4553 case AMDGPU::G_AMDGPU_MAD_I64_I32:
4554 return selectG_AMDGPU_MAD_64_32(I);
4555 case TargetOpcode::G_INTTOPTR:
4556 case TargetOpcode::G_BITCAST:
4557 case TargetOpcode::G_PTRTOINT:
4558 case TargetOpcode::G_FREEZE:
4559 return selectCOPY(I);
4560 case TargetOpcode::G_FNEG:
4561 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4562 return true;
4563 return selectG_FNEG(MI&: I);
4564 case TargetOpcode::G_FABS:
4565 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4566 return true;
4567 return selectG_FABS(MI&: I);
4568 case TargetOpcode::G_EXTRACT:
4569 return selectG_EXTRACT(I);
4570 case TargetOpcode::G_MERGE_VALUES:
4571 case TargetOpcode::G_CONCAT_VECTORS:
4572 return selectG_MERGE_VALUES(MI&: I);
4573 case TargetOpcode::G_UNMERGE_VALUES:
4574 return selectG_UNMERGE_VALUES(MI&: I);
4575 case TargetOpcode::G_BUILD_VECTOR:
4576 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
4577 return selectG_BUILD_VECTOR(MI&: I);
4578 case TargetOpcode::G_IMPLICIT_DEF:
4579 return selectG_IMPLICIT_DEF(I);
4580 case TargetOpcode::G_INSERT:
4581 return selectG_INSERT(I);
4582 case TargetOpcode::G_INTRINSIC:
4583 case TargetOpcode::G_INTRINSIC_CONVERGENT:
4584 return selectG_INTRINSIC(I);
4585 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
4586 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
4587 return selectG_INTRINSIC_W_SIDE_EFFECTS(I);
4588 case TargetOpcode::G_ICMP:
4589 case TargetOpcode::G_FCMP:
4590 if (selectG_ICMP_or_FCMP(I))
4591 return true;
4592 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4593 case TargetOpcode::G_LOAD:
4594 case TargetOpcode::G_ZEXTLOAD:
4595 case TargetOpcode::G_SEXTLOAD:
4596 case TargetOpcode::G_STORE:
4597 case TargetOpcode::G_ATOMIC_CMPXCHG:
4598 case TargetOpcode::G_ATOMICRMW_XCHG:
4599 case TargetOpcode::G_ATOMICRMW_ADD:
4600 case TargetOpcode::G_ATOMICRMW_SUB:
4601 case TargetOpcode::G_ATOMICRMW_AND:
4602 case TargetOpcode::G_ATOMICRMW_OR:
4603 case TargetOpcode::G_ATOMICRMW_XOR:
4604 case TargetOpcode::G_ATOMICRMW_MIN:
4605 case TargetOpcode::G_ATOMICRMW_MAX:
4606 case TargetOpcode::G_ATOMICRMW_UMIN:
4607 case TargetOpcode::G_ATOMICRMW_UMAX:
4608 case TargetOpcode::G_ATOMICRMW_UINC_WRAP:
4609 case TargetOpcode::G_ATOMICRMW_UDEC_WRAP:
4610 case TargetOpcode::G_ATOMICRMW_USUB_COND:
4611 case TargetOpcode::G_ATOMICRMW_USUB_SAT:
4612 case TargetOpcode::G_ATOMICRMW_FADD:
4613 case TargetOpcode::G_ATOMICRMW_FMIN:
4614 case TargetOpcode::G_ATOMICRMW_FMAX:
4615 return selectG_LOAD_STORE_ATOMICRMW(I);
4616 case TargetOpcode::G_SELECT:
4617 return selectG_SELECT(I);
4618 case TargetOpcode::G_TRUNC:
4619 return selectG_TRUNC(I);
4620 case TargetOpcode::G_SEXT:
4621 case TargetOpcode::G_ZEXT:
4622 case TargetOpcode::G_ANYEXT:
4623 case TargetOpcode::G_SEXT_INREG:
4624 // This is a workaround. For extension from type i1, `selectImpl()` uses
4625 // patterns from TD file and generates an illegal VGPR to SGPR COPY as type
4626 // i1 can only be hold in a SGPR class.
4627 if (MRI->getType(Reg: I.getOperand(i: 1).getReg()) != LLT::scalar(SizeInBits: 1) &&
4628 selectImpl(I, CoverageInfo&: *CoverageInfo))
4629 return true;
4630 return selectG_SZA_EXT(I);
4631 case TargetOpcode::G_FPEXT:
4632 if (selectG_FPEXT(I))
4633 return true;
4634 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4635 case TargetOpcode::G_BRCOND:
4636 return selectG_BRCOND(I);
4637 case TargetOpcode::G_GLOBAL_VALUE:
4638 return selectG_GLOBAL_VALUE(I);
4639 case TargetOpcode::G_PTRMASK:
4640 return selectG_PTRMASK(I);
4641 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4642 return selectG_EXTRACT_VECTOR_ELT(MI&: I);
4643 case TargetOpcode::G_INSERT_VECTOR_ELT:
4644 return selectG_INSERT_VECTOR_ELT(MI&: I);
4645 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
4646 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
4647 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
4648 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
4649 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
4650 const AMDGPU::ImageDimIntrinsicInfo *Intr =
4651 AMDGPU::getImageDimIntrinsicInfo(Intr: AMDGPU::getIntrinsicID(I));
4652 assert(Intr && "not an image intrinsic with image pseudo");
4653 return selectImageIntrinsic(MI&: I, Intr);
4654 }
4655 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY:
4656 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
4657 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
4658 return selectBVHIntersectRayIntrinsic(MI&: I);
4659 case AMDGPU::G_SBFX:
4660 case AMDGPU::G_UBFX:
4661 return selectG_SBFX_UBFX(MI&: I);
4662 case AMDGPU::G_SI_CALL:
4663 I.setDesc(TII.get(Opcode: AMDGPU::SI_CALL));
4664 return true;
4665 case AMDGPU::G_AMDGPU_WAVE_ADDRESS:
4666 return selectWaveAddress(MI&: I);
4667 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN: {
4668 I.setDesc(TII.get(Opcode: AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN));
4669 return true;
4670 }
4671 case AMDGPU::G_STACKRESTORE:
4672 return selectStackRestore(MI&: I);
4673 case AMDGPU::G_PHI:
4674 return selectPHI(I);
4675 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
4676 return selectCOPY_SCC_VCC(I);
4677 case AMDGPU::G_AMDGPU_COPY_VCC_SCC:
4678 return selectCOPY_VCC_SCC(I);
4679 case AMDGPU::G_AMDGPU_READANYLANE:
4680 return selectReadAnyLane(I);
4681 case TargetOpcode::G_CONSTANT:
4682 case TargetOpcode::G_FCONSTANT:
4683 default:
4684 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4685 }
4686 return false;
4687}
4688
4689InstructionSelector::ComplexRendererFns
4690AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const {
4691 return {{
4692 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4693 }};
4694
4695}
4696
4697std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3ModsImpl(
4698 Register Src, bool IsCanonicalizing, bool AllowAbs, bool OpSel) const {
4699 unsigned Mods = 0;
4700 MachineInstr *MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4701
4702 if (MI->getOpcode() == AMDGPU::G_FNEG) {
4703 Src = MI->getOperand(i: 1).getReg();
4704 Mods |= SISrcMods::NEG;
4705 MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4706 } else if (MI->getOpcode() == AMDGPU::G_FSUB && IsCanonicalizing) {
4707 // Fold fsub [+-]0 into fneg. This may not have folded depending on the
4708 // denormal mode, but we're implicitly canonicalizing in a source operand.
4709 const ConstantFP *LHS =
4710 getConstantFPVRegVal(VReg: MI->getOperand(i: 1).getReg(), MRI: *MRI);
4711 if (LHS && LHS->isZero()) {
4712 Mods |= SISrcMods::NEG;
4713 Src = MI->getOperand(i: 2).getReg();
4714 }
4715 }
4716
4717 if (AllowAbs && MI->getOpcode() == AMDGPU::G_FABS) {
4718 Src = MI->getOperand(i: 1).getReg();
4719 Mods |= SISrcMods::ABS;
4720 }
4721
4722 if (OpSel)
4723 Mods |= SISrcMods::OP_SEL_0;
4724
4725 return std::pair(Src, Mods);
4726}
4727
4728std::pair<Register, unsigned>
4729AMDGPUInstructionSelector::selectVOP3PModsF32Impl(Register Src) const {
4730 unsigned Mods;
4731 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src);
4732 Mods |= SISrcMods::OP_SEL_1;
4733 return std::pair(Src, Mods);
4734}
4735
4736Register AMDGPUInstructionSelector::copyToVGPRIfSrcFolded(
4737 Register Src, unsigned Mods, MachineOperand Root, MachineInstr *InsertPt,
4738 bool ForceVGPR) const {
4739 if ((Mods != 0 || ForceVGPR) &&
4740 RBI.getRegBank(Reg: Src, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) {
4741
4742 // If we looked through copies to find source modifiers on an SGPR operand,
4743 // we now have an SGPR register source. To avoid potentially violating the
4744 // constant bus restriction, we need to insert a copy to a VGPR.
4745 Register VGPRSrc = MRI->cloneVirtualRegister(VReg: Root.getReg());
4746 BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
4747 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VGPRSrc)
4748 .addReg(RegNo: Src);
4749 Src = VGPRSrc;
4750 }
4751
4752 return Src;
4753}
4754
4755/// Some instructions must have 32-bit sources. With real true16 instructions a
4756/// 16-bit VALU value lives in a VGPR_16, which they cannot read, so place it in
4757/// the low half of a new 32-bit VGPR.
4758Register
4759AMDGPUInstructionSelector::widenSrcIfVGPR16(Register Src,
4760 MachineInstr *InsertPt) const {
4761 if (!Subtarget->useRealTrue16Insts() || MRI->getType(Reg: Src) != LLT::scalar(SizeInBits: 16))
4762 return Src;
4763
4764 const RegisterBank *SrcRB = RBI.getRegBank(Reg: Src, MRI: *MRI, TRI);
4765 if (!SrcRB || SrcRB->getID() != AMDGPU::VGPRRegBankID)
4766 return Src;
4767
4768 MachineIRBuilder B(*InsertPt);
4769
4770 Register ImpDefReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
4771 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF).addDef(RegNo: ImpDefReg);
4772
4773 Register DstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
4774 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
4775 .addDef(RegNo: DstReg)
4776 .addReg(RegNo: Src)
4777 .addImm(Val: AMDGPU::lo16)
4778 .addReg(RegNo: ImpDefReg)
4779 .addImm(Val: AMDGPU::hi16);
4780
4781 return DstReg;
4782}
4783
4784///
4785/// This will select either an SGPR or VGPR operand and will save us from
4786/// having to write an extra tablegen pattern.
4787InstructionSelector::ComplexRendererFns
4788AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const {
4789 return {{
4790 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4791 }};
4792}
4793
4794InstructionSelector::ComplexRendererFns
4795AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const {
4796 Register Src;
4797 unsigned Mods;
4798 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4799
4800 return {{
4801 [=](MachineInstrBuilder &MIB) {
4802 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4803 },
4804 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4805 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4806 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4807 }};
4808}
4809
4810InstructionSelector::ComplexRendererFns
4811AMDGPUInstructionSelector::selectVOP3BMods0(MachineOperand &Root) const {
4812 Register Src;
4813 unsigned Mods;
4814 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
4815 /*IsCanonicalizing=*/true,
4816 /*AllowAbs=*/false);
4817
4818 return {{
4819 [=](MachineInstrBuilder &MIB) {
4820 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4821 },
4822 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4823 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4824 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4825 }};
4826}
4827
4828InstructionSelector::ComplexRendererFns
4829AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const {
4830 return {{
4831 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); },
4832 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4833 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4834 }};
4835}
4836
4837InstructionSelector::ComplexRendererFns
4838AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const {
4839 Register Src;
4840 unsigned Mods;
4841 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4842
4843 return {{
4844 [=](MachineInstrBuilder &MIB) {
4845 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4846 },
4847 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4848 }};
4849}
4850
4851InstructionSelector::ComplexRendererFns
4852AMDGPUInstructionSelector::selectVOP3ModsNonCanonicalizing(
4853 MachineOperand &Root) const {
4854 Register Src;
4855 unsigned Mods;
4856 std::tie(args&: Src, args&: Mods) =
4857 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/false);
4858
4859 return {{
4860 [=](MachineInstrBuilder &MIB) {
4861 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4862 },
4863 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4864 }};
4865}
4866
4867InstructionSelector::ComplexRendererFns
4868AMDGPUInstructionSelector::selectVOP3BMods(MachineOperand &Root) const {
4869 Register Src;
4870 unsigned Mods;
4871 std::tie(args&: Src, args&: Mods) =
4872 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/true,
4873 /*AllowAbs=*/false);
4874
4875 return {{
4876 [=](MachineInstrBuilder &MIB) {
4877 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4878 },
4879 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4880 }};
4881}
4882
4883InstructionSelector::ComplexRendererFns
4884AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
4885 Register Reg = Root.getReg();
4886 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
4887 if (Def->getOpcode() == AMDGPU::G_FNEG || Def->getOpcode() == AMDGPU::G_FABS)
4888 return {};
4889 return {{
4890 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
4891 }};
4892}
4893
4894enum class SrcStatus {
4895 IS_SAME,
4896 IS_UPPER_HALF,
4897 IS_LOWER_HALF,
4898 IS_UPPER_HALF_NEG,
4899 // This means current op = [op_upper, op_lower] and src = -op_lower.
4900 IS_LOWER_HALF_NEG,
4901 IS_HI_NEG,
4902 // This means current op = [op_upper, op_lower] and src = [op_upper,
4903 // -op_lower].
4904 IS_LO_NEG,
4905 IS_BOTH_NEG,
4906 INVALID,
4907 NEG_START = IS_UPPER_HALF_NEG,
4908 NEG_END = IS_BOTH_NEG,
4909 HALF_START = IS_UPPER_HALF,
4910 HALF_END = IS_LOWER_HALF_NEG
4911};
4912/// Test if the MI is truncating to half, such as `%reg0:n = G_TRUNC %reg1:2n`
4913static bool isTruncHalf(const MachineInstr *MI,
4914 const MachineRegisterInfo &MRI) {
4915 if (MI->getOpcode() != AMDGPU::G_TRUNC)
4916 return false;
4917
4918 unsigned DstSize = MRI.getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
4919 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4920 return DstSize * 2 == SrcSize;
4921}
4922
4923/// Test if the MI is logic shift right with half bits,
4924/// such as `%reg0:2n =G_LSHR %reg1:2n, CONST(n)`
4925static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
4926 if (MI->getOpcode() != AMDGPU::G_LSHR)
4927 return false;
4928
4929 Register ShiftSrc;
4930 std::optional<ValueAndVReg> ShiftAmt;
4931 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
4932 P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
4933 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4934 unsigned Shift = ShiftAmt->Value.getZExtValue();
4935 return Shift * 2 == SrcSize;
4936 }
4937 return false;
4938}
4939
4940/// Test if the MI is shift left with half bits,
4941/// such as `%reg0:2n =G_SHL %reg1:2n, CONST(n)`
4942static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
4943 if (MI->getOpcode() != AMDGPU::G_SHL)
4944 return false;
4945
4946 Register ShiftSrc;
4947 std::optional<ValueAndVReg> ShiftAmt;
4948 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
4949 P: m_GShl(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
4950 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4951 unsigned Shift = ShiftAmt->Value.getZExtValue();
4952 return Shift * 2 == SrcSize;
4953 }
4954 return false;
4955}
4956
4957/// Test function, if the MI is `%reg0:n, %reg1:n = G_UNMERGE_VALUES %reg2:2n`
4958static bool isUnmergeHalf(const MachineInstr *MI,
4959 const MachineRegisterInfo &MRI) {
4960 if (MI->getOpcode() != AMDGPU::G_UNMERGE_VALUES)
4961 return false;
4962 return MI->getNumOperands() == 3 && MI->getOperand(i: 0).isDef() &&
4963 MI->getOperand(i: 1).isDef() && !MI->getOperand(i: 2).isDef();
4964}
4965
4966enum class TypeClass { VECTOR_OF_TWO, SCALAR, NONE_OF_LISTED };
4967
4968static TypeClass isVectorOfTwoOrScalar(Register Reg,
4969 const MachineRegisterInfo &MRI) {
4970 LLT OpTy = MRI.getType(Reg);
4971 if (OpTy.isScalar())
4972 return TypeClass::SCALAR;
4973 if (OpTy.isVector() && OpTy.getNumElements() == 2)
4974 return TypeClass::VECTOR_OF_TWO;
4975 return TypeClass::NONE_OF_LISTED;
4976}
4977
4978static SrcStatus getNegStatus(Register Reg, SrcStatus S,
4979 const MachineRegisterInfo &MRI) {
4980 TypeClass NegType = isVectorOfTwoOrScalar(Reg, MRI);
4981 if (NegType != TypeClass::VECTOR_OF_TWO && NegType != TypeClass::SCALAR)
4982 return SrcStatus::INVALID;
4983
4984 switch (S) {
4985 case SrcStatus::IS_SAME:
4986 if (NegType == TypeClass::VECTOR_OF_TWO) {
4987 // Vector of 2:
4988 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
4989 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
4990 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
4991 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
4992 return SrcStatus::IS_BOTH_NEG;
4993 }
4994 if (NegType == TypeClass::SCALAR) {
4995 // Scalar:
4996 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
4997 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
4998 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
4999 // [SrcHi, SrcLo] = [-OpHi, OpLo]
5000 return SrcStatus::IS_HI_NEG;
5001 }
5002 break;
5003 case SrcStatus::IS_HI_NEG:
5004 if (NegType == TypeClass::VECTOR_OF_TWO) {
5005 // Vector of 2:
5006 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5007 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
5008 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5009 // [SrcHi, SrcLo] = [-(-OpHi), -OpLo] = [OpHi, -OpLo]
5010 return SrcStatus::IS_LO_NEG;
5011 }
5012 if (NegType == TypeClass::SCALAR) {
5013 // Scalar:
5014 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5015 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
5016 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5017 // [SrcHi, SrcLo] = [-(-OpHi), OpLo] = [OpHi, OpLo]
5018 return SrcStatus::IS_SAME;
5019 }
5020 break;
5021 case SrcStatus::IS_LO_NEG:
5022 if (NegType == TypeClass::VECTOR_OF_TWO) {
5023 // Vector of 2:
5024 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5025 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5026 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5027 // [SrcHi, SrcLo] = [-OpHi, -(-OpLo)] = [-OpHi, OpLo]
5028 return SrcStatus::IS_HI_NEG;
5029 }
5030 if (NegType == TypeClass::SCALAR) {
5031 // Scalar:
5032 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5033 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5034 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5035 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
5036 return SrcStatus::IS_BOTH_NEG;
5037 }
5038 break;
5039 case SrcStatus::IS_BOTH_NEG:
5040 if (NegType == TypeClass::VECTOR_OF_TWO) {
5041 // Vector of 2:
5042 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5043 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5044 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5045 // [SrcHi, SrcLo] = [OpHi, OpLo]
5046 return SrcStatus::IS_SAME;
5047 }
5048 if (NegType == TypeClass::SCALAR) {
5049 // Scalar:
5050 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5051 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5052 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5053 // [SrcHi, SrcLo] = [OpHi, -OpLo]
5054 return SrcStatus::IS_LO_NEG;
5055 }
5056 break;
5057 case SrcStatus::IS_UPPER_HALF:
5058 // Vector of 2:
5059 // Src = CurrUpper
5060 // Curr = [CurrUpper, CurrLower]
5061 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5062 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5063 // Src = -OpUpper
5064 //
5065 // Scalar:
5066 // Src = CurrUpper
5067 // Curr = [CurrUpper, CurrLower]
5068 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5069 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5070 // Src = -OpUpper
5071 return SrcStatus::IS_UPPER_HALF_NEG;
5072 case SrcStatus::IS_LOWER_HALF:
5073 if (NegType == TypeClass::VECTOR_OF_TWO) {
5074 // Vector of 2:
5075 // Src = CurrLower
5076 // Curr = [CurrUpper, CurrLower]
5077 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5078 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5079 // Src = -OpLower
5080 return SrcStatus::IS_LOWER_HALF_NEG;
5081 }
5082 if (NegType == TypeClass::SCALAR) {
5083 // Scalar:
5084 // Src = CurrLower
5085 // Curr = [CurrUpper, CurrLower]
5086 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5087 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5088 // Src = OpLower
5089 return SrcStatus::IS_LOWER_HALF;
5090 }
5091 break;
5092 case SrcStatus::IS_UPPER_HALF_NEG:
5093 // Vector of 2:
5094 // Src = -CurrUpper
5095 // Curr = [CurrUpper, CurrLower]
5096 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5097 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5098 // Src = -(-OpUpper) = OpUpper
5099 //
5100 // Scalar:
5101 // Src = -CurrUpper
5102 // Curr = [CurrUpper, CurrLower]
5103 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5104 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5105 // Src = -(-OpUpper) = OpUpper
5106 return SrcStatus::IS_UPPER_HALF;
5107 case SrcStatus::IS_LOWER_HALF_NEG:
5108 if (NegType == TypeClass::VECTOR_OF_TWO) {
5109 // Vector of 2:
5110 // Src = -CurrLower
5111 // Curr = [CurrUpper, CurrLower]
5112 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5113 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5114 // Src = -(-OpLower) = OpLower
5115 return SrcStatus::IS_LOWER_HALF;
5116 }
5117 if (NegType == TypeClass::SCALAR) {
5118 // Scalar:
5119 // Src = -CurrLower
5120 // Curr = [CurrUpper, CurrLower]
5121 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5122 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5123 // Src = -OpLower
5124 return SrcStatus::IS_LOWER_HALF_NEG;
5125 }
5126 break;
5127 default:
5128 break;
5129 }
5130 llvm_unreachable("unexpected SrcStatus & NegType combination");
5131}
5132
5133static std::optional<std::pair<Register, SrcStatus>>
5134calcNextStatus(std::pair<Register, SrcStatus> Curr,
5135 const MachineRegisterInfo &MRI) {
5136 const MachineInstr *MI = MRI.getVRegDef(Reg: Curr.first);
5137
5138 unsigned Opc = MI->getOpcode();
5139
5140 // Handle general Opc cases.
5141 switch (Opc) {
5142 case AMDGPU::G_BITCAST:
5143 return std::optional<std::pair<Register, SrcStatus>>(
5144 {MI->getOperand(i: 1).getReg(), Curr.second});
5145 case AMDGPU::COPY:
5146 if (MI->getOperand(i: 1).getReg().isPhysical())
5147 return std::nullopt;
5148 return std::optional<std::pair<Register, SrcStatus>>(
5149 {MI->getOperand(i: 1).getReg(), Curr.second});
5150 case AMDGPU::G_FNEG: {
5151 SrcStatus Stat = getNegStatus(Reg: Curr.first, S: Curr.second, MRI);
5152 if (Stat == SrcStatus::INVALID)
5153 return std::nullopt;
5154 return std::optional<std::pair<Register, SrcStatus>>(
5155 {MI->getOperand(i: 1).getReg(), Stat});
5156 }
5157 default:
5158 break;
5159 }
5160
5161 // Calc next Stat from current Stat.
5162 switch (Curr.second) {
5163 case SrcStatus::IS_SAME:
5164 if (isTruncHalf(MI, MRI))
5165 return std::optional<std::pair<Register, SrcStatus>>(
5166 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5167 else if (isUnmergeHalf(MI, MRI)) {
5168 if (Curr.first == MI->getOperand(i: 0).getReg())
5169 return std::optional<std::pair<Register, SrcStatus>>(
5170 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF});
5171 return std::optional<std::pair<Register, SrcStatus>>(
5172 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF});
5173 }
5174 break;
5175 case SrcStatus::IS_HI_NEG:
5176 if (isTruncHalf(MI, MRI)) {
5177 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5178 // [CurrHi, CurrLo] = trunc [OpUpper, OpLower] = OpLower
5179 // = [OpLowerHi, OpLowerLo]
5180 // Src = [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5181 // = [-OpLowerHi, OpLowerLo]
5182 // = -OpLower
5183 return std::optional<std::pair<Register, SrcStatus>>(
5184 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5185 }
5186 if (isUnmergeHalf(MI, MRI)) {
5187 if (Curr.first == MI->getOperand(i: 0).getReg())
5188 return std::optional<std::pair<Register, SrcStatus>>(
5189 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5190 return std::optional<std::pair<Register, SrcStatus>>(
5191 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5192 }
5193 break;
5194 case SrcStatus::IS_UPPER_HALF:
5195 if (isShlHalf(MI, MRI))
5196 return std::optional<std::pair<Register, SrcStatus>>(
5197 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5198 break;
5199 case SrcStatus::IS_LOWER_HALF:
5200 if (isLshrHalf(MI, MRI))
5201 return std::optional<std::pair<Register, SrcStatus>>(
5202 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF});
5203 break;
5204 case SrcStatus::IS_UPPER_HALF_NEG:
5205 if (isShlHalf(MI, MRI))
5206 return std::optional<std::pair<Register, SrcStatus>>(
5207 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5208 break;
5209 case SrcStatus::IS_LOWER_HALF_NEG:
5210 if (isLshrHalf(MI, MRI))
5211 return std::optional<std::pair<Register, SrcStatus>>(
5212 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5213 break;
5214 default:
5215 break;
5216 }
5217 return std::nullopt;
5218}
5219
5220/// This is used to control valid status that current MI supports. For example,
5221/// non floating point intrinsic such as @llvm.amdgcn.sdot2 does not support NEG
5222/// bit on VOP3P.
5223/// The class can be further extended to recognize support on SEL, NEG, ABS bit
5224/// for different MI on different arch
5225class SearchOptions {
5226private:
5227 bool HasNeg = false;
5228 // Assume all complex pattern of VOP3P have opsel.
5229 bool HasOpsel = true;
5230
5231public:
5232 SearchOptions(Register Reg, const MachineRegisterInfo &MRI) {
5233 const MachineInstr *MI = MRI.getVRegDef(Reg);
5234 unsigned Opc = MI->getOpcode();
5235
5236 if (Opc == TargetOpcode::G_INTRINSIC) {
5237 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: *MI).getIntrinsicID();
5238 // Only float point intrinsic has neg & neg_hi bits.
5239 if (IntrinsicID == Intrinsic::amdgcn_fdot2)
5240 HasNeg = true;
5241 } else if (TargetInstrInfo::isGenericOpcode(Opc)) {
5242 // Keep same for generic op.
5243 HasNeg = true;
5244 }
5245 }
5246 bool checkOptions(SrcStatus Stat) const {
5247 if (!HasNeg &&
5248 (Stat >= SrcStatus::NEG_START && Stat <= SrcStatus::NEG_END)) {
5249 return false;
5250 }
5251 if (!HasOpsel &&
5252 (Stat >= SrcStatus::HALF_START && Stat <= SrcStatus::HALF_END)) {
5253 return false;
5254 }
5255 return true;
5256 }
5257};
5258
5259static SmallVector<std::pair<Register, SrcStatus>>
5260getSrcStats(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5261 int MaxDepth = 3) {
5262 int Depth = 0;
5263 auto Curr = calcNextStatus(Curr: {Reg, SrcStatus::IS_SAME}, MRI);
5264 SmallVector<std::pair<Register, SrcStatus>> Statlist;
5265
5266 while (Depth <= MaxDepth && Curr.has_value()) {
5267 Depth++;
5268 if (SO.checkOptions(Stat: Curr.value().second))
5269 Statlist.push_back(Elt: Curr.value());
5270 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5271 }
5272
5273 return Statlist;
5274}
5275
5276static std::pair<Register, SrcStatus>
5277getLastSameOrNeg(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5278 int MaxDepth = 3) {
5279 int Depth = 0;
5280 std::pair<Register, SrcStatus> LastSameOrNeg = {Reg, SrcStatus::IS_SAME};
5281 auto Curr = calcNextStatus(Curr: LastSameOrNeg, MRI);
5282
5283 while (Depth <= MaxDepth && Curr.has_value()) {
5284 Depth++;
5285 SrcStatus Stat = Curr.value().second;
5286 if (SO.checkOptions(Stat)) {
5287 if (Stat == SrcStatus::IS_SAME || Stat == SrcStatus::IS_HI_NEG ||
5288 Stat == SrcStatus::IS_LO_NEG || Stat == SrcStatus::IS_BOTH_NEG)
5289 LastSameOrNeg = Curr.value();
5290 }
5291 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5292 }
5293
5294 return LastSameOrNeg;
5295}
5296
5297static bool isSameBitWidth(Register Reg1, Register Reg2,
5298 const MachineRegisterInfo &MRI) {
5299 unsigned Width1 = MRI.getType(Reg: Reg1).getSizeInBits();
5300 unsigned Width2 = MRI.getType(Reg: Reg2).getSizeInBits();
5301 return Width1 == Width2;
5302}
5303
5304static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods) {
5305 // SrcStatus::IS_LOWER_HALF remain 0.
5306 if (HiStat == SrcStatus::IS_UPPER_HALF_NEG) {
5307 Mods ^= SISrcMods::NEG_HI;
5308 Mods |= SISrcMods::OP_SEL_1;
5309 } else if (HiStat == SrcStatus::IS_UPPER_HALF)
5310 Mods |= SISrcMods::OP_SEL_1;
5311 else if (HiStat == SrcStatus::IS_LOWER_HALF_NEG)
5312 Mods ^= SISrcMods::NEG_HI;
5313 else if (HiStat == SrcStatus::IS_HI_NEG)
5314 Mods ^= SISrcMods::NEG_HI;
5315
5316 if (LoStat == SrcStatus::IS_UPPER_HALF_NEG) {
5317 Mods ^= SISrcMods::NEG;
5318 Mods |= SISrcMods::OP_SEL_0;
5319 } else if (LoStat == SrcStatus::IS_UPPER_HALF)
5320 Mods |= SISrcMods::OP_SEL_0;
5321 else if (LoStat == SrcStatus::IS_LOWER_HALF_NEG)
5322 Mods |= SISrcMods::NEG;
5323 else if (LoStat == SrcStatus::IS_HI_NEG)
5324 Mods ^= SISrcMods::NEG;
5325
5326 return Mods;
5327}
5328
5329static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg,
5330 Register RootReg, const SIInstrInfo &TII,
5331 const MachineRegisterInfo &MRI) {
5332 auto IsHalfState = [](SrcStatus S) {
5333 return S == SrcStatus::IS_UPPER_HALF || S == SrcStatus::IS_UPPER_HALF_NEG ||
5334 S == SrcStatus::IS_LOWER_HALF || S == SrcStatus::IS_LOWER_HALF_NEG;
5335 };
5336 return isSameBitWidth(Reg1: NewReg, Reg2: RootReg, MRI) && IsHalfState(LoStat) &&
5337 IsHalfState(HiStat);
5338}
5339
5340std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
5341 Register RootReg, const MachineRegisterInfo &MRI, bool IsDOT) const {
5342 unsigned Mods = 0;
5343 // No modification if Root type is not form of <2 x Type>.
5344 if (isVectorOfTwoOrScalar(Reg: RootReg, MRI) != TypeClass::VECTOR_OF_TWO) {
5345 Mods |= SISrcMods::OP_SEL_1;
5346 return {RootReg, Mods};
5347 }
5348
5349 SearchOptions SO(RootReg, MRI);
5350
5351 std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(Reg: RootReg, MRI, SO);
5352
5353 if (Stat.second == SrcStatus::IS_BOTH_NEG)
5354 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
5355 else if (Stat.second == SrcStatus::IS_HI_NEG)
5356 Mods ^= SISrcMods::NEG_HI;
5357 else if (Stat.second == SrcStatus::IS_LO_NEG)
5358 Mods ^= SISrcMods::NEG;
5359
5360 // 64-bit VOP3P instructions do not have OPSEL or ABS. Bail on v2f64 or v2i64.
5361 // TODO: Select NEG_LO and NEG_HI modifiers from BUILD_VECTOR.
5362 if (MRI.getType(Reg: RootReg).getSizeInBits() == 128) {
5363 Mods |= SISrcMods::OP_SEL_1; // Just the default, OPSEL unsupported.
5364 return {Stat.first, Mods};
5365 }
5366
5367 GBuildVector *MI;
5368 if (!mi_match(R: Stat.first, MRI, P: m_GBuildVector(Inst&: MI)) ||
5369 MI->getNumOperands() != 3 || (IsDOT && Subtarget->hasDOTOpSelHazard())) {
5370 Mods |= SISrcMods::OP_SEL_1;
5371 return {Stat.first, Mods};
5372 }
5373
5374 SmallVector<std::pair<Register, SrcStatus>> StatlistHi =
5375 getSrcStats(Reg: MI->getOperand(i: 2).getReg(), MRI, SO);
5376
5377 if (StatlistHi.empty()) {
5378 Mods |= SISrcMods::OP_SEL_1;
5379 return {Stat.first, Mods};
5380 }
5381
5382 SmallVector<std::pair<Register, SrcStatus>> StatlistLo =
5383 getSrcStats(Reg: MI->getOperand(i: 1).getReg(), MRI, SO);
5384
5385 if (StatlistLo.empty()) {
5386 Mods |= SISrcMods::OP_SEL_1;
5387 return {Stat.first, Mods};
5388 }
5389
5390 for (int I = StatlistHi.size() - 1; I >= 0; I--) {
5391 for (int J = StatlistLo.size() - 1; J >= 0; J--) {
5392 if (StatlistHi[I].first == StatlistLo[J].first &&
5393 isValidToPack(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second,
5394 NewReg: StatlistHi[I].first, RootReg, TII, MRI))
5395 return {StatlistHi[I].first,
5396 updateMods(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second, Mods)};
5397 }
5398 }
5399 // Packed instructions do not have abs modifiers.
5400 Mods |= SISrcMods::OP_SEL_1;
5401
5402 return {Stat.first, Mods};
5403}
5404
5405// Removed unused function `getAllKindImm` to eliminate dead code.
5406
5407static bool checkRB(Register Reg, unsigned int RBNo,
5408 const AMDGPURegisterBankInfo &RBI,
5409 const MachineRegisterInfo &MRI,
5410 const TargetRegisterInfo &TRI) {
5411 const RegisterBank *RB = RBI.getRegBank(Reg, MRI, TRI);
5412 return RB->getID() == RBNo;
5413}
5414
5415// This function is used to get the correct register bank for returned reg.
5416// Assume:
5417// 1. VOP3P is always legal for VGPR.
5418// 2. RootOp's regbank is legal.
5419// Thus
5420// 1. If RootOp is SGPR, then NewOp can be SGPR or VGPR.
5421// 2. If RootOp is VGPR, then NewOp must be VGPR.
5422static Register
5423getLegalRegBank(Register NewReg, Register RootReg, MachineInstr &Use,
5424 const AMDGPURegisterBankInfo &RBI, MachineRegisterInfo &MRI,
5425 const TargetRegisterInfo &TRI, const SIInstrInfo &TII) {
5426 // RootOp can only be VGPR or SGPR (some hand written cases such as.
5427 // inst-select-ashr.v2s16.mir::ashr_v2s16_vs).
5428 if (checkRB(Reg: RootReg, RBNo: AMDGPU::SGPRRegBankID, RBI, MRI, TRI) ||
5429 checkRB(Reg: NewReg, RBNo: AMDGPU::VGPRRegBankID, RBI, MRI, TRI))
5430 return NewReg;
5431
5432 if (mi_match(R: RootReg, MRI, P: m_Copy(Src: m_SpecificReg(RequestedReg: NewReg)))) {
5433 // RootOp is VGPR, NewOp is not VGPR, but RootOp = COPY NewOp.
5434 return RootReg;
5435 }
5436
5437 Register DstReg = MRI.cloneVirtualRegister(VReg: RootReg);
5438 MachineInstrBuilder MIB = BuildMI(BB&: *Use.getParent(), I&: Use, MIMD: Use.getDebugLoc(),
5439 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
5440 .addReg(RegNo: NewReg);
5441
5442 // Only accept VGPR.
5443 return MIB->getOperand(i: 0).getReg();
5444}
5445
5446InstructionSelector::ComplexRendererFns
5447AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
5448 bool IsDOT) const {
5449 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5450 Register Reg;
5451 unsigned Mods;
5452 std::tie(args&: Reg, args&: Mods) = selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT);
5453
5454 Reg = getLegalRegBank(NewReg: Reg, RootReg: Root.getReg(), Use&: *Root.getParent(), RBI, MRI, TRI,
5455 TII);
5456 return {{
5457 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
5458 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5459 }};
5460}
5461
5462InstructionSelector::ComplexRendererFns
5463AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
5464
5465 return selectVOP3PRetHelper(Root);
5466}
5467
5468InstructionSelector::ComplexRendererFns
5469AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
5470
5471 return selectVOP3PRetHelper(Root, IsDOT: true);
5472}
5473
5474InstructionSelector::ComplexRendererFns
5475AMDGPUInstructionSelector::selectVOP3PNoModsDOT(MachineOperand &Root) const {
5476 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5477 Register Src;
5478 unsigned Mods;
5479 std::tie(args&: Src, args&: Mods) = selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT: true /*IsDOT*/);
5480 if (Mods != SISrcMods::OP_SEL_1)
5481 return {};
5482
5483 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5484}
5485
5486InstructionSelector::ComplexRendererFns
5487AMDGPUInstructionSelector::selectVOP3PModsF32(MachineOperand &Root) const {
5488 Register Src;
5489 unsigned Mods;
5490 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5491
5492 return {{
5493 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5494 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5495 }};
5496}
5497
5498InstructionSelector::ComplexRendererFns
5499AMDGPUInstructionSelector::selectVOP3PNoModsF32(MachineOperand &Root) const {
5500 Register Src;
5501 unsigned Mods;
5502 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5503 if (Mods != SISrcMods::OP_SEL_1)
5504 return {};
5505
5506 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5507}
5508
5509InstructionSelector::ComplexRendererFns
5510AMDGPUInstructionSelector::selectWMMAOpSelVOP3PMods(
5511 MachineOperand &Root) const {
5512 assert((Root.isImm() && (Root.getImm() == -1 || Root.getImm() == 0)) &&
5513 "expected i1 value");
5514 unsigned Mods = SISrcMods::OP_SEL_1;
5515 if (Root.getImm() != 0)
5516 Mods |= SISrcMods::OP_SEL_0;
5517
5518 return {{
5519 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5520 }};
5521}
5522
5523static Register buildRegSequence(SmallVectorImpl<Register> &Elts,
5524 MachineInstr *InsertPt,
5525 MachineRegisterInfo &MRI) {
5526 const TargetRegisterClass *DstRegClass;
5527 switch (Elts.size()) {
5528 case 8:
5529 DstRegClass = &AMDGPU::VReg_256RegClass;
5530 break;
5531 case 4:
5532 DstRegClass = &AMDGPU::VReg_128RegClass;
5533 break;
5534 case 2:
5535 DstRegClass = &AMDGPU::VReg_64RegClass;
5536 break;
5537 default:
5538 llvm_unreachable("unhandled Reg sequence size");
5539 }
5540
5541 MachineIRBuilder B(*InsertPt);
5542 auto MIB = B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
5543 .addDef(RegNo: MRI.createVirtualRegister(RegClass: DstRegClass));
5544 for (unsigned i = 0; i < Elts.size(); ++i) {
5545 MIB.addReg(RegNo: Elts[i]);
5546 MIB.addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: i));
5547 }
5548 return MIB->getOperand(i: 0).getReg();
5549}
5550
5551static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods,
5552 SmallVectorImpl<Register> &Elts, Register &Src,
5553 MachineInstr *InsertPt,
5554 MachineRegisterInfo &MRI) {
5555 if (ModOpcode == TargetOpcode::G_FNEG) {
5556 Mods |= SISrcMods::NEG;
5557 // Check if all elements also have abs modifier
5558 SmallVector<Register, 8> NegAbsElts;
5559 for (auto El : Elts) {
5560 Register FabsSrc;
5561 if (!mi_match(R: El, MRI, P: m_GFabs(Src: m_Reg(R&: FabsSrc))))
5562 break;
5563 NegAbsElts.push_back(Elt: FabsSrc);
5564 }
5565 if (Elts.size() != NegAbsElts.size()) {
5566 // Neg
5567 Src = buildRegSequence(Elts, InsertPt, MRI);
5568 } else {
5569 // Neg and Abs
5570 Mods |= SISrcMods::NEG_HI;
5571 Src = buildRegSequence(Elts&: NegAbsElts, InsertPt, MRI);
5572 }
5573 } else {
5574 assert(ModOpcode == TargetOpcode::G_FABS);
5575 // Abs
5576 Mods |= SISrcMods::NEG_HI;
5577 Src = buildRegSequence(Elts, InsertPt, MRI);
5578 }
5579}
5580
5581InstructionSelector::ComplexRendererFns
5582AMDGPUInstructionSelector::selectWMMAModsF32NegAbs(MachineOperand &Root) const {
5583 Register Src = Root.getReg();
5584 unsigned Mods = SISrcMods::OP_SEL_1;
5585 SmallVector<Register, 8> EltsF32;
5586
5587 GBuildVector *BV;
5588 if (mi_match(R: Src, MRI: *MRI, P: m_GBuildVector(Inst&: BV))) {
5589 assert(BV->getNumSources() > 0);
5590 // Based on first element decide which mod we match, neg or abs
5591 MachineInstr *ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: 0));
5592 unsigned ModOpcode = (ElF32->getOpcode() == AMDGPU::G_FNEG)
5593 ? AMDGPU::G_FNEG
5594 : AMDGPU::G_FABS;
5595 for (unsigned i = 0; i < BV->getNumSources(); ++i) {
5596 ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: i));
5597 if (ElF32->getOpcode() != ModOpcode)
5598 break;
5599 EltsF32.push_back(Elt: ElF32->getOperand(i: 1).getReg());
5600 }
5601
5602 // All elements had ModOpcode modifier
5603 if (BV->getNumSources() == EltsF32.size()) {
5604 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsF32, Src, InsertPt: Root.getParent(),
5605 MRI&: *MRI);
5606 }
5607 }
5608
5609 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5610 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5611}
5612
5613InstructionSelector::ComplexRendererFns
5614AMDGPUInstructionSelector::selectWMMAModsF16Neg(MachineOperand &Root) const {
5615 Register Src = Root.getReg();
5616 unsigned Mods = SISrcMods::OP_SEL_1;
5617 SmallVector<Register, 8> EltsV2F16;
5618
5619 GConcatVectors *CV;
5620 if (mi_match(R: Src, MRI: *MRI, P: m_GConcatVectors(Inst&: CV))) {
5621 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5622 Register FNegSrc;
5623 if (!mi_match(R: CV->getSourceReg(I: i), MRI: *MRI, P: m_GFNeg(Src: m_Reg(R&: FNegSrc))))
5624 break;
5625 EltsV2F16.push_back(Elt: FNegSrc);
5626 }
5627
5628 // All elements had ModOpcode modifier
5629 if (CV->getNumSources() == EltsV2F16.size()) {
5630 Mods |= SISrcMods::NEG;
5631 Mods |= SISrcMods::NEG_HI;
5632 Src = buildRegSequence(Elts&: EltsV2F16, InsertPt: Root.getParent(), MRI&: *MRI);
5633 }
5634 }
5635
5636 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5637 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5638}
5639
5640InstructionSelector::ComplexRendererFns
5641AMDGPUInstructionSelector::selectWMMAModsF16NegAbs(MachineOperand &Root) const {
5642 Register Src = Root.getReg();
5643 unsigned Mods = SISrcMods::OP_SEL_1;
5644 SmallVector<Register, 8> EltsV2F16;
5645
5646 GConcatVectors *CV;
5647 if (mi_match(R: Src, MRI: *MRI, P: m_GConcatVectors(Inst&: CV))) {
5648 assert(CV->getNumSources() > 0);
5649 MachineInstr *ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: 0));
5650 // Based on first element decide which mod we match, neg or abs
5651 unsigned ModOpcode = (ElV2F16->getOpcode() == AMDGPU::G_FNEG)
5652 ? AMDGPU::G_FNEG
5653 : AMDGPU::G_FABS;
5654
5655 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5656 ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: i));
5657 if (ElV2F16->getOpcode() != ModOpcode)
5658 break;
5659 EltsV2F16.push_back(Elt: ElV2F16->getOperand(i: 1).getReg());
5660 }
5661
5662 // All elements had ModOpcode modifier
5663 if (CV->getNumSources() == EltsV2F16.size()) {
5664 MachineIRBuilder B(*Root.getParent());
5665 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsV2F16, Src, InsertPt: Root.getParent(),
5666 MRI&: *MRI);
5667 }
5668 }
5669
5670 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5671 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5672}
5673
5674InstructionSelector::ComplexRendererFns
5675AMDGPUInstructionSelector::selectWMMAVISrc(MachineOperand &Root) const {
5676 std::optional<FPValueAndVReg> FPValReg;
5677 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GFCstOrSplat(FPValReg))) {
5678 if (TII.isInlineConstant(Imm: FPValReg->Value)) {
5679 return {{[=](MachineInstrBuilder &MIB) {
5680 MIB.addImm(Val: FPValReg->Value.bitcastToAPInt().getSExtValue());
5681 }}};
5682 }
5683 // Non-inlineable splat floats should not fall-through for integer immediate
5684 // checks.
5685 return {};
5686 }
5687
5688 APInt ICst;
5689 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICstOrSplat(Cst&: ICst))) {
5690 if (TII.isInlineConstant(Imm: ICst)) {
5691 return {
5692 {[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ICst.getSExtValue()); }}};
5693 }
5694 }
5695
5696 return {};
5697}
5698
5699InstructionSelector::ComplexRendererFns
5700AMDGPUInstructionSelector::selectSWMMACIndex8(MachineOperand &Root) const {
5701 Register Src =
5702 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5703 unsigned Key = 0;
5704
5705 Register ShiftSrc;
5706 std::optional<ValueAndVReg> ShiftAmt;
5707 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5708 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5709 ShiftAmt->Value.getZExtValue() % 8 == 0) {
5710 Key = ShiftAmt->Value.getZExtValue() / 8;
5711 Src = ShiftSrc;
5712 }
5713
5714 return {{
5715 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5716 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5717 }};
5718}
5719
5720InstructionSelector::ComplexRendererFns
5721AMDGPUInstructionSelector::selectSWMMACIndex16(MachineOperand &Root) const {
5722
5723 Register Src =
5724 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5725 unsigned Key = 0;
5726
5727 Register ShiftSrc;
5728 std::optional<ValueAndVReg> ShiftAmt;
5729 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5730 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5731 ShiftAmt->Value.getZExtValue() == 16) {
5732 Src = ShiftSrc;
5733 Key = 1;
5734 }
5735
5736 return {{
5737 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5738 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5739 }};
5740}
5741
5742InstructionSelector::ComplexRendererFns
5743AMDGPUInstructionSelector::selectSWMMACIndex32(MachineOperand &Root) const {
5744 Register Src =
5745 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5746 unsigned Key = 0;
5747
5748 Register S32 = matchZeroExtendFromS32(Reg: Src);
5749 if (!S32)
5750 S32 = matchAnyExtendFromS32(Reg: Src);
5751
5752 if (S32) {
5753 const MachineInstr *Def = getDefIgnoringCopies(Reg: S32, MRI: *MRI);
5754 if (Def->getOpcode() == TargetOpcode::G_UNMERGE_VALUES) {
5755 assert(Def->getNumOperands() == 3);
5756 Register DstReg1 = Def->getOperand(i: 1).getReg();
5757 if (mi_match(R: S32, MRI: *MRI,
5758 P: m_any_of(preds: m_SpecificReg(RequestedReg: DstReg1), preds: m_Copy(Src: m_Reg(R&: DstReg1))))) {
5759 Src = Def->getOperand(i: 2).getReg();
5760 Key = 1;
5761 }
5762 }
5763 }
5764
5765 return {{
5766 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5767 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5768 }};
5769}
5770
5771InstructionSelector::ComplexRendererFns
5772AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const {
5773 Register Src;
5774 unsigned Mods;
5775 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
5776
5777 // FIXME: Handle op_sel
5778 return {{
5779 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5780 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5781 }};
5782}
5783
5784// FIXME-TRUE16 remove when fake16 is removed
5785InstructionSelector::ComplexRendererFns
5786AMDGPUInstructionSelector::selectVINTERPMods(MachineOperand &Root) const {
5787 Register Src;
5788 unsigned Mods;
5789 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5790 /*IsCanonicalizing=*/true,
5791 /*AllowAbs=*/false,
5792 /*OpSel=*/false);
5793
5794 return {{
5795 [=](MachineInstrBuilder &MIB) {
5796 MIB.addReg(
5797 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5798 },
5799 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5800 }};
5801}
5802
5803InstructionSelector::ComplexRendererFns
5804AMDGPUInstructionSelector::selectVINTERPModsHi(MachineOperand &Root) const {
5805 Register Src;
5806 unsigned Mods;
5807 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5808 /*IsCanonicalizing=*/true,
5809 /*AllowAbs=*/false,
5810 /*OpSel=*/true);
5811
5812 return {{
5813 [=](MachineInstrBuilder &MIB) {
5814 MIB.addReg(
5815 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5816 },
5817 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5818 }};
5819}
5820
5821// Given \p Offset and load specified by the \p Root operand check if \p Offset
5822// is a multiple of the load byte size. If it is update \p Offset to a
5823// pre-scaled value and return true.
5824bool AMDGPUInstructionSelector::selectScaleOffset(MachineOperand &Root,
5825 Register &Offset,
5826 bool IsSigned) const {
5827 if (!Subtarget->hasScaleOffset())
5828 return false;
5829
5830 const MachineInstr &MI = *Root.getParent();
5831 MachineMemOperand *MMO = *MI.memoperands_begin();
5832
5833 if (!MMO->getSize().hasValue())
5834 return false;
5835
5836 uint64_t Size = MMO->getSize().getValue();
5837
5838 Register OffsetReg = matchExtendFromS32OrS32(Reg: Offset, IsSigned);
5839 if (!OffsetReg)
5840 OffsetReg = Offset;
5841
5842 if (auto Def = getDefSrcRegIgnoringCopies(Reg: OffsetReg, MRI: *MRI))
5843 OffsetReg = Def->Reg;
5844
5845 Register Op0;
5846 MachineInstr *Mul;
5847 bool ScaleOffset =
5848 (isPowerOf2_64(Value: Size) &&
5849 mi_match(R: OffsetReg, MRI: *MRI,
5850 P: m_GShl(L: m_Reg(R&: Op0),
5851 R: m_any_of(preds: m_SpecificICst(RequestedValue: Log2_64(Value: Size)),
5852 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Log2_64(Value: Size))))))) ||
5853 mi_match(R: OffsetReg, MRI: *MRI,
5854 P: m_GMul(L: m_Reg(R&: Op0), R: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5855 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) ||
5856 mi_match(
5857 R: OffsetReg, MRI: *MRI,
5858 P: m_BinOp(Opcode: IsSigned ? AMDGPU::S_MUL_I64_I32_PSEUDO : AMDGPU::S_MUL_U64,
5859 L: m_Reg(R&: Op0), R: m_SpecificICst(RequestedValue: Size))) ||
5860 // Match G_AMDGPU_MAD_U64_U32 offset, c, 0
5861 (mi_match(R: OffsetReg, MRI: *MRI, P: m_MInstr(MI&: Mul)) &&
5862 (Mul->getOpcode() == (IsSigned ? AMDGPU::G_AMDGPU_MAD_I64_I32
5863 : AMDGPU::G_AMDGPU_MAD_U64_U32) ||
5864 (IsSigned && Mul->getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32 &&
5865 VT->signBitIsZero(Op: Mul->getOperand(i: 2).getReg()))) &&
5866 mi_match(R: Mul->getOperand(i: 4).getReg(), MRI: *MRI, P: m_ZeroInt()) &&
5867 mi_match(R: Mul->getOperand(i: 3).getReg(), MRI: *MRI,
5868 P: m_GTrunc(Src: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5869 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) &&
5870 mi_match(R: Mul->getOperand(i: 2).getReg(), MRI: *MRI, P: m_Reg(R&: Op0)));
5871
5872 if (ScaleOffset)
5873 Offset = Op0;
5874
5875 return ScaleOffset;
5876}
5877
5878bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root,
5879 Register &Base,
5880 Register *SOffset,
5881 int64_t *Offset,
5882 bool *ScaleOffset) const {
5883 MachineInstr *MI = Root.getParent();
5884 MachineBasicBlock *MBB = MI->getParent();
5885
5886 // FIXME: We should shrink the GEP if the offset is known to be <= 32-bits,
5887 // then we can select all ptr + 32-bit offsets.
5888 SmallVector<GEPInfo, 4> AddrInfo;
5889 getAddrModeInfo(Load: *MI, MRI: *MRI, AddrInfo);
5890
5891 if (AddrInfo.empty())
5892 return false;
5893
5894 const GEPInfo &GEPI = AddrInfo[0];
5895 std::optional<int64_t> EncodedImm;
5896
5897 if (ScaleOffset)
5898 *ScaleOffset = false;
5899
5900 if (SOffset && Offset) {
5901 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
5902 /*HasSOffset=*/true);
5903 if (GEPI.SgprParts.size() == 1 && GEPI.Imm != 0 && EncodedImm &&
5904 AddrInfo.size() > 1) {
5905 const GEPInfo &GEPI2 = AddrInfo[1];
5906 if (GEPI2.SgprParts.size() == 2 && GEPI2.Imm == 0) {
5907 Register OffsetReg = GEPI2.SgprParts[1];
5908 if (ScaleOffset)
5909 *ScaleOffset =
5910 selectScaleOffset(Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */);
5911 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
5912 if (OffsetReg) {
5913 Base = GEPI2.SgprParts[0];
5914 *SOffset = OffsetReg;
5915 *Offset = *EncodedImm;
5916 if (*Offset >= 0 || !AMDGPU::hasSMRDSignedImmOffset(ST: STI))
5917 return true;
5918
5919 // For unbuffered smem loads, it is illegal for the Immediate Offset
5920 // to be negative if the resulting (Offset + (M0 or SOffset or zero)
5921 // is negative. Handle the case where the Immediate Offset + SOffset
5922 // is negative.
5923 auto SKnown = VT->getKnownBits(R: *SOffset);
5924 if (*Offset + SKnown.getMinValue().getSExtValue() < 0)
5925 return false;
5926
5927 return true;
5928 }
5929 }
5930 }
5931 return false;
5932 }
5933
5934 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
5935 /*HasSOffset=*/false);
5936 if (Offset && GEPI.SgprParts.size() == 1 && EncodedImm) {
5937 Base = GEPI.SgprParts[0];
5938 *Offset = *EncodedImm;
5939 return true;
5940 }
5941
5942 // SGPR offset is unsigned.
5943 if (SOffset && GEPI.SgprParts.size() == 1 && isUInt<32>(x: GEPI.Imm) &&
5944 GEPI.Imm != 0) {
5945 // If we make it this far we have a load with an 32-bit immediate offset.
5946 // It is OK to select this using a sgpr offset, because we have already
5947 // failed trying to select this load into one of the _IMM variants since
5948 // the _IMM Patterns are considered before the _SGPR patterns.
5949 Base = GEPI.SgprParts[0];
5950 *SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5951 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: *SOffset)
5952 .addImm(Val: GEPI.Imm);
5953 return true;
5954 }
5955
5956 if (SOffset && GEPI.SgprParts.size() && GEPI.Imm == 0) {
5957 Register OffsetReg = GEPI.SgprParts[1];
5958 if (ScaleOffset)
5959 *ScaleOffset = selectScaleOffset(Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */);
5960 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
5961 if (OffsetReg) {
5962 Base = GEPI.SgprParts[0];
5963 *SOffset = OffsetReg;
5964 return true;
5965 }
5966 }
5967
5968 return false;
5969}
5970
5971InstructionSelector::ComplexRendererFns
5972AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
5973 Register Base;
5974 int64_t Offset;
5975 if (!selectSmrdOffset(Root, Base, /* SOffset= */ nullptr, Offset: &Offset,
5976 /* ScaleOffset */ nullptr))
5977 return std::nullopt;
5978
5979 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
5980 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }}};
5981}
5982
5983InstructionSelector::ComplexRendererFns
5984AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const {
5985 SmallVector<GEPInfo, 4> AddrInfo;
5986 getAddrModeInfo(Load: *Root.getParent(), MRI: *MRI, AddrInfo);
5987
5988 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
5989 return std::nullopt;
5990
5991 const GEPInfo &GEPInfo = AddrInfo[0];
5992 Register PtrReg = GEPInfo.SgprParts[0];
5993 std::optional<int64_t> EncodedImm =
5994 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: GEPInfo.Imm);
5995 if (!EncodedImm)
5996 return std::nullopt;
5997
5998 return {{
5999 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrReg); },
6000 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); }
6001 }};
6002}
6003
6004InstructionSelector::ComplexRendererFns
6005AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
6006 Register Base, SOffset;
6007 bool ScaleOffset;
6008 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, /* Offset= */ nullptr,
6009 ScaleOffset: &ScaleOffset))
6010 return std::nullopt;
6011
6012 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6013 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6014 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
6015 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
6016}
6017
6018InstructionSelector::ComplexRendererFns
6019AMDGPUInstructionSelector::selectSmrdSgprImm(MachineOperand &Root) const {
6020 Register Base, SOffset;
6021 int64_t Offset;
6022 bool ScaleOffset;
6023 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, Offset: &Offset, ScaleOffset: &ScaleOffset))
6024 return std::nullopt;
6025
6026 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6027 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6028 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
6029 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
6030 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
6031}
6032
6033std::pair<Register, int> AMDGPUInstructionSelector::selectFlatOffsetImpl(
6034 MachineOperand &Root, AMDGPU::FlatAddrSpace FlatVariant) const {
6035 MachineInstr *MI = Root.getParent();
6036
6037 auto Default = std::pair(Root.getReg(), 0);
6038
6039 if (!STI.hasFlatInstOffsets())
6040 return Default;
6041
6042 Register PtrBase;
6043 int64_t ConstOffset;
6044 bool IsInBounds;
6045 std::tie(args&: PtrBase, args&: ConstOffset, args&: IsInBounds) =
6046 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6047
6048 // Adding the offset to the base address with an immediate in a FLAT
6049 // instruction must not change the memory aperture in which the address falls.
6050 // Therefore we can only fold offsets from inbounds GEPs into FLAT
6051 // instructions.
6052 if (ConstOffset == 0 ||
6053 (FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch &&
6054 !isFlatScratchBaseLegal(Addr: Root.getReg())) ||
6055 (FlatVariant == AMDGPU::FlatAddrSpace::FLAT && !IsInBounds))
6056 return Default;
6057
6058 unsigned AddrSpace = (*MI->memoperands_begin())->getAddrSpace();
6059 if (!TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace, FlatVariant))
6060 return Default;
6061
6062 return std::pair(PtrBase, ConstOffset);
6063}
6064
6065InstructionSelector::ComplexRendererFns
6066AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const {
6067 auto PtrWithOffset = selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FLAT);
6068
6069 return {{
6070 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6071 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6072 }};
6073}
6074
6075InstructionSelector::ComplexRendererFns
6076AMDGPUInstructionSelector::selectGlobalOffset(MachineOperand &Root) const {
6077 auto PtrWithOffset =
6078 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6079
6080 return {{
6081 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6082 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6083 }};
6084}
6085
6086InstructionSelector::ComplexRendererFns
6087AMDGPUInstructionSelector::selectScratchOffset(MachineOperand &Root) const {
6088 auto PtrWithOffset =
6089 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch);
6090
6091 return {{
6092 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6093 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6094 }};
6095}
6096
6097// Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
6098InstructionSelector::ComplexRendererFns
6099AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root,
6100 unsigned CPolBits,
6101 bool NeedIOffset) const {
6102 Register Addr = Root.getReg();
6103 Register PtrBase;
6104 int64_t ConstOffset;
6105 int64_t ImmOffset = 0;
6106
6107 // Match the immediate offset first, which canonically is moved as low as
6108 // possible.
6109 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6110 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6111
6112 if (ConstOffset != 0) {
6113 if (NeedIOffset &&
6114 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6115 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal)) {
6116 Addr = PtrBase;
6117 ImmOffset = ConstOffset;
6118 } else {
6119 auto PtrBaseDef = getDefSrcRegIgnoringCopies(Reg: PtrBase, MRI: *MRI);
6120 if (isSGPR(Reg: PtrBaseDef->Reg)) {
6121 if (ConstOffset > 0) {
6122 // Offset is too large.
6123 //
6124 // saddr + large_offset -> saddr +
6125 // (voffset = large_offset & ~MaxOffset) +
6126 // (large_offset & MaxOffset);
6127 int64_t SplitImmOffset = 0, RemainderOffset = ConstOffset;
6128 if (NeedIOffset) {
6129 std::tie(args&: SplitImmOffset, args&: RemainderOffset) =
6130 TII.splitFlatOffset(COffsetVal: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6131 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6132 }
6133
6134 if (Subtarget->hasSignedGVSOffset() ? isInt<32>(x: RemainderOffset)
6135 : isUInt<32>(x: RemainderOffset)) {
6136 MachineInstr *MI = Root.getParent();
6137 MachineBasicBlock *MBB = MI->getParent();
6138 Register HighBits =
6139 MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6140
6141 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6142 DestReg: HighBits)
6143 .addImm(Val: RemainderOffset);
6144
6145 if (NeedIOffset)
6146 return {{
6147 [=](MachineInstrBuilder &MIB) {
6148 MIB.addReg(RegNo: PtrBase);
6149 }, // saddr
6150 [=](MachineInstrBuilder &MIB) {
6151 MIB.addReg(RegNo: HighBits);
6152 }, // voffset
6153 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: SplitImmOffset); },
6154 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6155 }};
6156 return {{
6157 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrBase); }, // saddr
6158 [=](MachineInstrBuilder &MIB) {
6159 MIB.addReg(RegNo: HighBits);
6160 }, // voffset
6161 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6162 }};
6163 }
6164 }
6165
6166 // We are adding a 64 bit SGPR and a constant. If constant bus limit
6167 // is 1 we would need to perform 1 or 2 extra moves for each half of
6168 // the constant and it is better to do a scalar add and then issue a
6169 // single VALU instruction to materialize zero. Otherwise it is less
6170 // instructions to perform VALU adds with immediates or inline literals.
6171 unsigned NumLiterals =
6172 !TII.isInlineConstant(Imm: APInt(32, Lo_32(Value: ConstOffset))) +
6173 !TII.isInlineConstant(Imm: APInt(32, Hi_32(Value: ConstOffset)));
6174 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_ADD_U32_e64) > NumLiterals)
6175 return std::nullopt;
6176 }
6177 }
6178 }
6179
6180 // Match the variable offset.
6181 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6182 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6183 // Look through the SGPR->VGPR copy.
6184 Register SAddr =
6185 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
6186
6187 if (isSGPR(Reg: SAddr)) {
6188 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
6189
6190 // It's possible voffset is an SGPR here, but the copy to VGPR will be
6191 // inserted later.
6192 bool ScaleOffset = selectScaleOffset(Root, Offset&: PtrBaseOffset,
6193 IsSigned: Subtarget->hasSignedGVSOffset());
6194 if (Register VOffset = matchExtendFromS32OrS32(
6195 Reg: PtrBaseOffset, IsSigned: Subtarget->hasSignedGVSOffset())) {
6196 if (NeedIOffset)
6197 return {{[=](MachineInstrBuilder &MIB) { // saddr
6198 MIB.addReg(RegNo: SAddr);
6199 },
6200 [=](MachineInstrBuilder &MIB) { // voffset
6201 MIB.addReg(RegNo: VOffset);
6202 },
6203 [=](MachineInstrBuilder &MIB) { // offset
6204 MIB.addImm(Val: ImmOffset);
6205 },
6206 [=](MachineInstrBuilder &MIB) { // cpol
6207 MIB.addImm(Val: CPolBits |
6208 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6209 }}};
6210 return {{[=](MachineInstrBuilder &MIB) { // saddr
6211 MIB.addReg(RegNo: SAddr);
6212 },
6213 [=](MachineInstrBuilder &MIB) { // voffset
6214 MIB.addReg(RegNo: VOffset);
6215 },
6216 [=](MachineInstrBuilder &MIB) { // cpol
6217 MIB.addImm(Val: CPolBits |
6218 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6219 }}};
6220 }
6221 }
6222 }
6223
6224 // FIXME: We should probably have folded COPY (G_IMPLICIT_DEF) earlier, and
6225 // drop this.
6226 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF ||
6227 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(Reg: AddrDef->Reg))
6228 return std::nullopt;
6229
6230 // It's cheaper to materialize a single 32-bit zero for vaddr than the two
6231 // moves required to copy a 64-bit SGPR to VGPR.
6232 MachineInstr *MI = Root.getParent();
6233 MachineBasicBlock *MBB = MI->getParent();
6234 Register VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6235
6236 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
6237 .addImm(Val: 0);
6238
6239 if (NeedIOffset)
6240 return {{
6241 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6242 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6243 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6244 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6245 }};
6246 return {{
6247 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6248 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6249 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6250 }};
6251}
6252
6253InstructionSelector::ComplexRendererFns
6254AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root) const {
6255 return selectGlobalSAddr(Root, CPolBits: 0);
6256}
6257
6258InstructionSelector::ComplexRendererFns
6259AMDGPUInstructionSelector::selectGlobalSAddrCPol(MachineOperand &Root) const {
6260 const MachineInstr &I = *Root.getParent();
6261
6262 // We are assuming CPol is always the last operand of the intrinsic.
6263 auto PassedCPol =
6264 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6265 return selectGlobalSAddr(Root, CPolBits: PassedCPol);
6266}
6267
6268InstructionSelector::ComplexRendererFns
6269AMDGPUInstructionSelector::selectGlobalSAddrCPolM0(MachineOperand &Root) const {
6270 const MachineInstr &I = *Root.getParent();
6271
6272 // We are assuming CPol is second from last operand of the intrinsic.
6273 auto PassedCPol =
6274 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6275 return selectGlobalSAddr(Root, CPolBits: PassedCPol);
6276}
6277
6278InstructionSelector::ComplexRendererFns
6279AMDGPUInstructionSelector::selectGlobalSAddrGLC(MachineOperand &Root) const {
6280 return selectGlobalSAddr(Root, CPolBits: AMDGPU::CPol::GLC);
6281}
6282
6283InstructionSelector::ComplexRendererFns
6284AMDGPUInstructionSelector::selectGlobalSAddrNoIOffset(
6285 MachineOperand &Root) const {
6286 const MachineInstr &I = *Root.getParent();
6287
6288 // We are assuming CPol is always the last operand of the intrinsic.
6289 auto PassedCPol =
6290 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6291 return selectGlobalSAddr(Root, CPolBits: PassedCPol, NeedIOffset: false);
6292}
6293
6294InstructionSelector::ComplexRendererFns
6295AMDGPUInstructionSelector::selectGlobalSAddrNoIOffsetM0(
6296 MachineOperand &Root) const {
6297 const MachineInstr &I = *Root.getParent();
6298
6299 // We are assuming CPol is second from last operand of the intrinsic.
6300 auto PassedCPol =
6301 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6302 return selectGlobalSAddr(Root, CPolBits: PassedCPol, NeedIOffset: false);
6303}
6304
6305InstructionSelector::ComplexRendererFns
6306AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const {
6307 Register Addr = Root.getReg();
6308 Register PtrBase;
6309 int64_t ConstOffset;
6310 int64_t ImmOffset = 0;
6311
6312 // Match the immediate offset first, which canonically is moved as low as
6313 // possible.
6314 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6315 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6316
6317 if (ConstOffset != 0 && isFlatScratchBaseLegal(Addr) &&
6318 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6319 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6320 Addr = PtrBase;
6321 ImmOffset = ConstOffset;
6322 }
6323
6324 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6325 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6326 int FI = AddrDef->MI->getOperand(i: 1).getIndex();
6327 return {{
6328 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6329 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6330 }};
6331 }
6332
6333 Register SAddr = AddrDef->Reg;
6334
6335 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6336 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6337 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6338 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6339 auto RHSDef = getDefSrcRegIgnoringCopies(Reg: RHS, MRI: *MRI);
6340
6341 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX &&
6342 isSGPR(Reg: RHSDef->Reg)) {
6343 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6344 MachineInstr &I = *Root.getParent();
6345 MachineBasicBlock *BB = I.getParent();
6346 const DebugLoc &DL = I.getDebugLoc();
6347 SAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6348
6349 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_I32), DestReg: SAddr)
6350 .addFrameIndex(Idx: FI)
6351 .addReg(RegNo: RHSDef->Reg)
6352 .setOperandDead(3); // Dead scc
6353 }
6354 }
6355
6356 if (!isSGPR(Reg: SAddr))
6357 return std::nullopt;
6358
6359 return {{
6360 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SAddr); }, // saddr
6361 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6362 }};
6363}
6364
6365// Check whether the flat scratch SVS swizzle bug affects this access.
6366bool AMDGPUInstructionSelector::checkFlatScratchSVSSwizzleBug(
6367 Register VAddr, Register SAddr, uint64_t ImmOffset) const {
6368 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
6369 return false;
6370
6371 // The bug affects the swizzling of SVS accesses if there is any carry out
6372 // from the two low order bits (i.e. from bit 1 into bit 2) when adding
6373 // voffset to (soffset + inst_offset).
6374 auto VKnown = VT->getKnownBits(R: VAddr);
6375 auto SKnown = KnownBits::add(LHS: VT->getKnownBits(R: SAddr),
6376 RHS: KnownBits::makeConstant(C: APInt(32, ImmOffset)));
6377 uint64_t VMax = VKnown.getMaxValue().getZExtValue();
6378 uint64_t SMax = SKnown.getMaxValue().getZExtValue();
6379 return (VMax & 3) + (SMax & 3) >= 4;
6380}
6381
6382InstructionSelector::ComplexRendererFns
6383AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const {
6384 Register Addr = Root.getReg();
6385 Register PtrBase;
6386 int64_t ConstOffset;
6387 int64_t ImmOffset = 0;
6388
6389 // Match the immediate offset first, which canonically is moved as low as
6390 // possible.
6391 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6392 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6393
6394 Register OrigAddr = Addr;
6395 if (ConstOffset != 0 &&
6396 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6397 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6398 Addr = PtrBase;
6399 ImmOffset = ConstOffset;
6400 }
6401
6402 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6403 if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD)
6404 return std::nullopt;
6405
6406 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6407 if (RBI.getRegBank(Reg: RHS, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
6408 return std::nullopt;
6409
6410 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6411 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6412
6413 if (OrigAddr != Addr) {
6414 if (!isFlatScratchBaseLegalSVImm(Addr: OrigAddr))
6415 return std::nullopt;
6416 } else {
6417 if (!isFlatScratchBaseLegalSV(Addr: OrigAddr))
6418 return std::nullopt;
6419 }
6420
6421 if (checkFlatScratchSVSSwizzleBug(VAddr: RHS, SAddr: LHS, ImmOffset))
6422 return std::nullopt;
6423
6424 unsigned CPol = selectScaleOffset(Root, Offset&: RHS, IsSigned: true /* IsSigned */)
6425 ? AMDGPU::CPol::SCAL
6426 : 0;
6427
6428 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6429 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6430 return {{
6431 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6432 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6433 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6434 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6435 }};
6436 }
6437
6438 if (!isSGPR(Reg: LHS))
6439 if (auto Def = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI))
6440 LHS = Def->Reg;
6441
6442 if (!isSGPR(Reg: LHS))
6443 return std::nullopt;
6444
6445 return {{
6446 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6447 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: LHS); }, // saddr
6448 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6449 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6450 }};
6451}
6452
6453InstructionSelector::ComplexRendererFns
6454AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
6455 MachineInstr *MI = Root.getParent();
6456 MachineBasicBlock *MBB = MI->getParent();
6457 MachineFunction *MF = MBB->getParent();
6458 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6459
6460 int64_t Offset = 0;
6461 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) &&
6462 Offset != AMDGPU::getNullPointerValue(AS: AMDGPUAS::PRIVATE_ADDRESS)) {
6463 Register HighBits = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6464
6465 // TODO: Should this be inside the render function? The iterator seems to
6466 // move.
6467 const int64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
6468 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6469 DestReg: HighBits)
6470 .addImm(Val: Offset & ~MaxOffset);
6471
6472 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6473 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6474 },
6475 [=](MachineInstrBuilder &MIB) { // vaddr
6476 MIB.addReg(RegNo: HighBits);
6477 },
6478 [=](MachineInstrBuilder &MIB) { // soffset
6479 // Use constant zero for soffset and rely on eliminateFrameIndex
6480 // to choose the appropriate frame register if need be.
6481 MIB.addImm(Val: 0);
6482 },
6483 [=](MachineInstrBuilder &MIB) { // offset
6484 MIB.addImm(Val: Offset & MaxOffset);
6485 }}};
6486 }
6487
6488 assert(Offset == 0 || Offset == -1);
6489
6490 // Try to fold a frame index directly into the MUBUF vaddr field, and any
6491 // offsets.
6492 std::optional<int> FI;
6493 Register VAddr = Root.getReg();
6494
6495 Register PtrBase;
6496 int64_t ConstOffset;
6497 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6498 getPtrBaseWithConstantOffset(Root: VAddr, MRI: *MRI);
6499 int MatchedFI;
6500 if (ConstOffset != 0) {
6501 if (TII.isLegalMUBUFImmOffset(Imm: ConstOffset) &&
6502 (!STI.privateMemoryResourceIsRangeChecked() ||
6503 VT->signBitIsZero(Op: PtrBase))) {
6504 if (mi_match(R: PtrBase, MRI: *MRI, P: m_GFrameIndex(FI&: MatchedFI)))
6505 FI = MatchedFI;
6506 else
6507 VAddr = PtrBase;
6508 Offset = ConstOffset;
6509 }
6510 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GFrameIndex(FI&: MatchedFI))) {
6511 FI = MatchedFI;
6512 }
6513
6514 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6515 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6516 },
6517 [=](MachineInstrBuilder &MIB) { // vaddr
6518 if (FI)
6519 MIB.addFrameIndex(Idx: *FI);
6520 else
6521 MIB.addReg(RegNo: VAddr);
6522 },
6523 [=](MachineInstrBuilder &MIB) { // soffset
6524 // Use constant zero for soffset and rely on eliminateFrameIndex
6525 // to choose the appropriate frame register if need be.
6526 MIB.addImm(Val: 0);
6527 },
6528 [=](MachineInstrBuilder &MIB) { // offset
6529 MIB.addImm(Val: Offset);
6530 }}};
6531}
6532
6533bool AMDGPUInstructionSelector::isDSOffsetLegal(Register Base,
6534 int64_t Offset) const {
6535 if (!isUInt<16>(x: Offset))
6536 return false;
6537
6538 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6539 return true;
6540
6541 // On Southern Islands instruction with a negative base value and an offset
6542 // don't seem to work.
6543 return VT->signBitIsZero(Op: Base);
6544}
6545
6546bool AMDGPUInstructionSelector::isDSOffset2Legal(Register Base, int64_t Offset0,
6547 int64_t Offset1,
6548 unsigned Size) const {
6549 if (Offset0 % Size != 0 || Offset1 % Size != 0)
6550 return false;
6551 if (!isUInt<8>(x: Offset0 / Size) || !isUInt<8>(x: Offset1 / Size))
6552 return false;
6553
6554 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6555 return true;
6556
6557 // On Southern Islands instruction with a negative base value and an offset
6558 // don't seem to work.
6559 return VT->signBitIsZero(Op: Base);
6560}
6561
6562// Return whether the operation has NoUnsignedWrap property.
6563static bool isNoUnsignedWrap(MachineInstr *Addr) {
6564 return Addr->getOpcode() == TargetOpcode::G_OR ||
6565 (Addr->getOpcode() == TargetOpcode::G_PTR_ADD &&
6566 Addr->getFlag(Flag: MachineInstr::NoUWrap));
6567}
6568
6569// Check that the base address of flat scratch load/store in the form of `base +
6570// offset` is legal to be put in SGPR/VGPR (i.e. unsigned per hardware
6571// requirement). We always treat the first operand as the base address here.
6572bool AMDGPUInstructionSelector::isFlatScratchBaseLegal(Register Addr) const {
6573 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6574
6575 if (isNoUnsignedWrap(Addr: AddrMI))
6576 return true;
6577
6578 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6579 // values.
6580 if (STI.hasSignedScratchOffsets())
6581 return true;
6582
6583 Register LHS = AddrMI->getOperand(i: 1).getReg();
6584 Register RHS = AddrMI->getOperand(i: 2).getReg();
6585
6586 if (AddrMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
6587 std::optional<ValueAndVReg> RhsValReg =
6588 getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
6589 // If the immediate offset is negative and within certain range, the base
6590 // address cannot also be negative. If the base is also negative, the sum
6591 // would be either negative or much larger than the valid range of scratch
6592 // memory a thread can access.
6593 if (RhsValReg && RhsValReg->Value.getSExtValue() < 0 &&
6594 RhsValReg->Value.getSExtValue() > -0x40000000)
6595 return true;
6596 }
6597
6598 return VT->signBitIsZero(Op: LHS);
6599}
6600
6601// Check address value in SGPR/VGPR are legal for flat scratch in the form
6602// of: SGPR + VGPR.
6603bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSV(Register Addr) const {
6604 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6605
6606 if (isNoUnsignedWrap(Addr: AddrMI))
6607 return true;
6608
6609 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6610 // values.
6611 if (STI.hasSignedScratchOffsets())
6612 return true;
6613
6614 Register LHS = AddrMI->getOperand(i: 1).getReg();
6615 Register RHS = AddrMI->getOperand(i: 2).getReg();
6616 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6617}
6618
6619// Check address value in SGPR/VGPR are legal for flat scratch in the form
6620// of: SGPR + VGPR + Imm.
6621bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSVImm(
6622 Register Addr) const {
6623 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6624 // values.
6625 if (STI.hasSignedScratchOffsets())
6626 return true;
6627
6628 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6629 Register Base = AddrMI->getOperand(i: 1).getReg();
6630 std::optional<DefinitionAndSourceRegister> BaseDef =
6631 getDefSrcRegIgnoringCopies(Reg: Base, MRI: *MRI);
6632 std::optional<ValueAndVReg> RHSOffset =
6633 getIConstantVRegValWithLookThrough(VReg: AddrMI->getOperand(i: 2).getReg(), MRI: *MRI);
6634 assert(RHSOffset);
6635
6636 // If the immediate offset is negative and within certain range, the base
6637 // address cannot also be negative. If the base is also negative, the sum
6638 // would be either negative or much larger than the valid range of scratch
6639 // memory a thread can access.
6640 if (isNoUnsignedWrap(Addr: BaseDef->MI) &&
6641 (isNoUnsignedWrap(Addr: AddrMI) ||
6642 (RHSOffset->Value.getSExtValue() < 0 &&
6643 RHSOffset->Value.getSExtValue() > -0x40000000)))
6644 return true;
6645
6646 Register LHS = BaseDef->MI->getOperand(i: 1).getReg();
6647 Register RHS = BaseDef->MI->getOperand(i: 2).getReg();
6648 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6649}
6650
6651bool AMDGPUInstructionSelector::isUnneededShiftMask(const MachineInstr &MI,
6652 unsigned ShAmtBits) const {
6653 assert(MI.getOpcode() == TargetOpcode::G_AND);
6654
6655 std::optional<APInt> RHS =
6656 getIConstantVRegVal(VReg: MI.getOperand(i: 2).getReg(), MRI: *MRI);
6657 if (!RHS)
6658 return false;
6659
6660 if (RHS->countr_one() >= ShAmtBits)
6661 return true;
6662
6663 const APInt &LHSKnownZeros = VT->getKnownZeroes(R: MI.getOperand(i: 1).getReg());
6664 return (LHSKnownZeros | *RHS).countr_one() >= ShAmtBits;
6665}
6666
6667InstructionSelector::ComplexRendererFns
6668AMDGPUInstructionSelector::selectMUBUFScratchOffset(
6669 MachineOperand &Root) const {
6670 Register Reg = Root.getReg();
6671 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6672
6673 std::optional<DefinitionAndSourceRegister> Def =
6674 getDefSrcRegIgnoringCopies(Reg, MRI: *MRI);
6675 assert(Def && "this shouldn't be an optional result");
6676 Reg = Def->Reg;
6677
6678 if (Register WaveBase = getWaveAddress(Def: Def->MI)) {
6679 return {{
6680 [=](MachineInstrBuilder &MIB) { // rsrc
6681 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6682 },
6683 [=](MachineInstrBuilder &MIB) { // soffset
6684 MIB.addReg(RegNo: WaveBase);
6685 },
6686 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // offset
6687 }};
6688 }
6689
6690 int64_t Offset = 0;
6691
6692 // FIXME: Copy check is a hack
6693 Register BasePtr;
6694 if (mi_match(R: Reg, MRI: *MRI,
6695 P: m_GPtrAdd(L: m_Reg(R&: BasePtr),
6696 R: m_any_of(preds: m_ICst(Cst&: Offset), preds: m_Copy(Src: m_ICst(Cst&: Offset)))))) {
6697 if (!TII.isLegalMUBUFImmOffset(Imm: Offset))
6698 return {};
6699 MachineInstr *BasePtrDef = getDefIgnoringCopies(Reg: BasePtr, MRI: *MRI);
6700 Register WaveBase = getWaveAddress(Def: BasePtrDef);
6701 if (!WaveBase)
6702 return {};
6703
6704 return {{
6705 [=](MachineInstrBuilder &MIB) { // rsrc
6706 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6707 },
6708 [=](MachineInstrBuilder &MIB) { // soffset
6709 MIB.addReg(RegNo: WaveBase);
6710 },
6711 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6712 }};
6713 }
6714
6715 if (!mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) ||
6716 !TII.isLegalMUBUFImmOffset(Imm: Offset))
6717 return {};
6718
6719 return {{
6720 [=](MachineInstrBuilder &MIB) { // rsrc
6721 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6722 },
6723 [=](MachineInstrBuilder &MIB) { // soffset
6724 MIB.addImm(Val: 0);
6725 },
6726 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6727 }};
6728}
6729
6730std::pair<Register, unsigned>
6731AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(
6732 MachineOperand &Root) const {
6733 int64_t ConstAddr = 0;
6734
6735 Register PtrBase;
6736 int64_t Offset;
6737 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6738 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6739
6740 if (Offset) {
6741 if (isDSOffsetLegal(Base: PtrBase, Offset)) {
6742 // (add n0, c0)
6743 return std::pair(PtrBase, Offset);
6744 }
6745 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GSub(L: m_Reg(), R: m_Reg()))) {
6746 // TODO
6747
6748 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6749 // TODO
6750 }
6751
6752 return std::pair(Root.getReg(), 0);
6753}
6754
6755InstructionSelector::ComplexRendererFns
6756AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
6757 Register Reg;
6758 unsigned Offset;
6759 std::tie(args&: Reg, args&: Offset) = selectDS1Addr1OffsetImpl(Root);
6760 return {{
6761 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6762 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }
6763 }};
6764}
6765
6766InstructionSelector::ComplexRendererFns
6767AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const {
6768 return selectDSReadWrite2(Root, size: 4);
6769}
6770
6771InstructionSelector::ComplexRendererFns
6772AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(MachineOperand &Root) const {
6773 return selectDSReadWrite2(Root, size: 8);
6774}
6775
6776InstructionSelector::ComplexRendererFns
6777AMDGPUInstructionSelector::selectDSReadWrite2(MachineOperand &Root,
6778 unsigned Size) const {
6779 Register Reg;
6780 unsigned Offset;
6781 std::tie(args&: Reg, args&: Offset) = selectDSReadWrite2Impl(Root, size: Size);
6782 return {{
6783 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6784 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
6785 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset+1); }
6786 }};
6787}
6788
6789std::pair<Register, unsigned>
6790AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root,
6791 unsigned Size) const {
6792 int64_t ConstAddr = 0;
6793
6794 Register PtrBase;
6795 int64_t Offset;
6796 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6797 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6798
6799 if (Offset) {
6800 int64_t OffsetValue0 = Offset;
6801 int64_t OffsetValue1 = Offset + Size;
6802 if (isDSOffset2Legal(Base: PtrBase, Offset0: OffsetValue0, Offset1: OffsetValue1, Size)) {
6803 // (add n0, c0)
6804 return std::pair(PtrBase, OffsetValue0 / Size);
6805 }
6806 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GSub(L: m_Reg(), R: m_Reg()))) {
6807 // TODO
6808
6809 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6810 // TODO
6811 }
6812
6813 return std::pair(Root.getReg(), 0);
6814}
6815
6816/// If \p Root is a G_PTR_ADD with a G_CONSTANT on the right hand side, return
6817/// the base value with the constant offset, and if the offset computation is
6818/// known to be inbounds. There may be intervening copies between \p Root and
6819/// the identified constant. Returns \p Root, 0, false if this does not match
6820/// the pattern.
6821std::tuple<Register, int64_t, bool>
6822AMDGPUInstructionSelector::getPtrBaseWithConstantOffset(
6823 Register Root, const MachineRegisterInfo &MRI) const {
6824 MachineInstr *RootI = getDefIgnoringCopies(Reg: Root, MRI);
6825 if (RootI->getOpcode() != TargetOpcode::G_PTR_ADD)
6826 return {Root, 0, false};
6827
6828 MachineOperand &RHS = RootI->getOperand(i: 2);
6829 std::optional<ValueAndVReg> MaybeOffset =
6830 getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI);
6831 if (!MaybeOffset)
6832 return {Root, 0, false};
6833 bool IsInBounds = RootI->getFlag(Flag: MachineInstr::MIFlag::InBounds);
6834 return {RootI->getOperand(i: 1).getReg(), MaybeOffset->Value.getSExtValue(),
6835 IsInBounds};
6836}
6837
6838static void addZeroImm(MachineInstrBuilder &MIB) {
6839 MIB.addImm(Val: 0);
6840}
6841
6842/// Return a resource descriptor for use with an arbitrary 64-bit pointer. If \p
6843/// BasePtr is not valid, a null base pointer will be used.
6844static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6845 uint32_t FormatLo, uint32_t FormatHi,
6846 Register BasePtr) {
6847 Register RSrc2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6848 Register RSrc3 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6849 Register RSrcHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6850 Register RSrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
6851
6852 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6853 .addDef(RegNo: RSrc2)
6854 .addImm(Val: FormatLo);
6855 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6856 .addDef(RegNo: RSrc3)
6857 .addImm(Val: FormatHi);
6858
6859 // Build the half of the subregister with the constants before building the
6860 // full 128-bit register. If we are building multiple resource descriptors,
6861 // this will allow CSEing of the 2-component register.
6862 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6863 .addDef(RegNo: RSrcHi)
6864 .addReg(RegNo: RSrc2)
6865 .addImm(Val: AMDGPU::sub0)
6866 .addReg(RegNo: RSrc3)
6867 .addImm(Val: AMDGPU::sub1);
6868
6869 Register RSrcLo = BasePtr;
6870 if (!BasePtr) {
6871 RSrcLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6872 B.buildInstr(Opcode: AMDGPU::S_MOV_B64)
6873 .addDef(RegNo: RSrcLo)
6874 .addImm(Val: 0);
6875 }
6876
6877 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6878 .addDef(RegNo: RSrc)
6879 .addReg(RegNo: RSrcLo)
6880 .addImm(Val: AMDGPU::sub0_sub1)
6881 .addReg(RegNo: RSrcHi)
6882 .addImm(Val: AMDGPU::sub2_sub3);
6883
6884 return RSrc;
6885}
6886
6887static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6888 const SIInstrInfo &TII, Register BasePtr) {
6889 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
6890
6891 // FIXME: Why are half the "default" bits ignored based on the addressing
6892 // mode?
6893 return buildRSRC(B, MRI, FormatLo: 0, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
6894}
6895
6896static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6897 const SIInstrInfo &TII, Register BasePtr) {
6898 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
6899
6900 // FIXME: Why are half the "default" bits ignored based on the addressing
6901 // mode?
6902 return buildRSRC(B, MRI, FormatLo: -1, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
6903}
6904
6905AMDGPUInstructionSelector::MUBUFAddressData
6906AMDGPUInstructionSelector::parseMUBUFAddress(Register Src) const {
6907 MUBUFAddressData Data;
6908 Data.N0 = Src;
6909
6910 Register PtrBase;
6911 int64_t Offset;
6912
6913 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6914 getPtrBaseWithConstantOffset(Root: Src, MRI: *MRI);
6915 if (isUInt<32>(x: Offset)) {
6916 Data.N0 = PtrBase;
6917 Data.Offset = Offset;
6918 }
6919
6920 if (MachineInstr *InputAdd
6921 = getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Data.N0, MRI: *MRI)) {
6922 Data.N2 = InputAdd->getOperand(i: 1).getReg();
6923 Data.N3 = InputAdd->getOperand(i: 2).getReg();
6924
6925 // FIXME: Need to fix extra SGPR->VGPRcopies inserted
6926 // FIXME: Don't know this was defined by operand 0
6927 //
6928 // TODO: Remove this when we have copy folding optimizations after
6929 // RegBankSelect.
6930 Data.N2 = getDefIgnoringCopies(Reg: Data.N2, MRI: *MRI)->getOperand(i: 0).getReg();
6931 Data.N3 = getDefIgnoringCopies(Reg: Data.N3, MRI: *MRI)->getOperand(i: 0).getReg();
6932 }
6933
6934 return Data;
6935}
6936
6937/// Return if the addr64 mubuf mode should be used for the given address.
6938bool AMDGPUInstructionSelector::shouldUseAddr64(MUBUFAddressData Addr) const {
6939 // (ptr_add N2, N3) -> addr64, or
6940 // (ptr_add (ptr_add N2, N3), C1) -> addr64
6941 if (Addr.N2)
6942 return true;
6943
6944 const RegisterBank *N0Bank = RBI.getRegBank(Reg: Addr.N0, MRI: *MRI, TRI);
6945 return N0Bank->getID() == AMDGPU::VGPRRegBankID;
6946}
6947
6948/// Split an immediate offset \p ImmOffset depending on whether it fits in the
6949/// immediate field. Modifies \p ImmOffset and sets \p SOffset to the variable
6950/// component.
6951void AMDGPUInstructionSelector::splitIllegalMUBUFOffset(
6952 MachineIRBuilder &B, Register &SOffset, int64_t &ImmOffset) const {
6953 if (TII.isLegalMUBUFImmOffset(Imm: ImmOffset))
6954 return;
6955
6956 // Illegal offset, store it in soffset.
6957 SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6958 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6959 .addDef(RegNo: SOffset)
6960 .addImm(Val: ImmOffset);
6961 ImmOffset = 0;
6962}
6963
6964bool AMDGPUInstructionSelector::selectMUBUFAddr64Impl(
6965 MachineOperand &Root, Register &VAddr, Register &RSrcReg,
6966 Register &SOffset, int64_t &Offset) const {
6967 // FIXME: Predicates should stop this from reaching here.
6968 // addr64 bit was removed for volcanic islands.
6969 if (!STI.hasAddr64() || STI.useFlatForGlobal())
6970 return false;
6971
6972 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
6973 if (!shouldUseAddr64(Addr: AddrData))
6974 return false;
6975
6976 Register N0 = AddrData.N0;
6977 Register N2 = AddrData.N2;
6978 Register N3 = AddrData.N3;
6979 Offset = AddrData.Offset;
6980
6981 // Base pointer for the SRD.
6982 Register SRDPtr;
6983
6984 if (N2) {
6985 if (RBI.getRegBank(Reg: N2, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
6986 assert(N3);
6987 if (RBI.getRegBank(Reg: N3, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
6988 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
6989 // addr64, and construct the default resource from a 0 address.
6990 VAddr = N0;
6991 } else {
6992 SRDPtr = N3;
6993 VAddr = N2;
6994 }
6995 } else {
6996 // N2 is not divergent.
6997 SRDPtr = N2;
6998 VAddr = N3;
6999 }
7000 } else if (RBI.getRegBank(Reg: N0, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7001 // Use the default null pointer in the resource
7002 VAddr = N0;
7003 } else {
7004 // N0 -> offset, or
7005 // (N0 + C1) -> offset
7006 SRDPtr = N0;
7007 }
7008
7009 MachineIRBuilder B(*Root.getParent());
7010 RSrcReg = buildAddr64RSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
7011 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
7012 return true;
7013}
7014
7015bool AMDGPUInstructionSelector::selectMUBUFOffsetImpl(
7016 MachineOperand &Root, Register &RSrcReg, Register &SOffset,
7017 int64_t &Offset) const {
7018
7019 // FIXME: Pattern should not reach here.
7020 if (STI.useFlatForGlobal())
7021 return false;
7022
7023 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
7024 if (shouldUseAddr64(Addr: AddrData))
7025 return false;
7026
7027 // N0 -> offset, or
7028 // (N0 + C1) -> offset
7029 Register SRDPtr = AddrData.N0;
7030 Offset = AddrData.Offset;
7031
7032 // TODO: Look through extensions for 32-bit soffset.
7033 MachineIRBuilder B(*Root.getParent());
7034
7035 RSrcReg = buildOffsetSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
7036 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
7037 return true;
7038}
7039
7040InstructionSelector::ComplexRendererFns
7041AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const {
7042 Register VAddr;
7043 Register RSrcReg;
7044 Register SOffset;
7045 int64_t Offset = 0;
7046
7047 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset))
7048 return {};
7049
7050 // FIXME: Use defaulted operands for trailing 0s and remove from the complex
7051 // pattern.
7052 return {{
7053 [=](MachineInstrBuilder &MIB) { // rsrc
7054 MIB.addReg(RegNo: RSrcReg);
7055 },
7056 [=](MachineInstrBuilder &MIB) { // vaddr
7057 MIB.addReg(RegNo: VAddr);
7058 },
7059 [=](MachineInstrBuilder &MIB) { // soffset
7060 if (SOffset)
7061 MIB.addReg(RegNo: SOffset);
7062 else if (STI.hasRestrictedSOffset())
7063 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7064 else
7065 MIB.addImm(Val: 0);
7066 },
7067 [=](MachineInstrBuilder &MIB) { // offset
7068 MIB.addImm(Val: Offset);
7069 },
7070 addZeroImm, // cpol
7071 addZeroImm, // tfe
7072 addZeroImm // swz
7073 }};
7074}
7075
7076InstructionSelector::ComplexRendererFns
7077AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const {
7078 Register RSrcReg;
7079 Register SOffset;
7080 int64_t Offset = 0;
7081
7082 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset))
7083 return {};
7084
7085 return {{
7086 [=](MachineInstrBuilder &MIB) { // rsrc
7087 MIB.addReg(RegNo: RSrcReg);
7088 },
7089 [=](MachineInstrBuilder &MIB) { // soffset
7090 if (SOffset)
7091 MIB.addReg(RegNo: SOffset);
7092 else if (STI.hasRestrictedSOffset())
7093 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7094 else
7095 MIB.addImm(Val: 0);
7096 },
7097 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }, // offset
7098 addZeroImm, // cpol
7099 addZeroImm, // tfe
7100 addZeroImm, // swz
7101 }};
7102}
7103
7104InstructionSelector::ComplexRendererFns
7105AMDGPUInstructionSelector::selectBUFSOffset(MachineOperand &Root) const {
7106
7107 Register SOffset = Root.getReg();
7108
7109 if (STI.hasRestrictedSOffset() && mi_match(R: SOffset, MRI: *MRI, P: m_ZeroInt()))
7110 SOffset = AMDGPU::SGPR_NULL;
7111
7112 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); }}};
7113}
7114
7115/// Get an immediate that must be 32-bits, and treated as zero extended.
7116static std::optional<uint64_t>
7117getConstantZext32Val(Register Reg, const MachineRegisterInfo &MRI) {
7118 // getIConstantVRegVal sexts any values, so see if that matters.
7119 std::optional<int64_t> OffsetVal = getIConstantVRegSExtVal(VReg: Reg, MRI);
7120 if (!OffsetVal || !isInt<32>(x: *OffsetVal))
7121 return std::nullopt;
7122 return Lo_32(Value: *OffsetVal);
7123}
7124
7125InstructionSelector::ComplexRendererFns
7126AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const {
7127 std::optional<uint64_t> OffsetVal =
7128 Root.isImm() ? Root.getImm() : getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7129 if (!OffsetVal)
7130 return {};
7131
7132 std::optional<int64_t> EncodedImm =
7133 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: *OffsetVal, IsBuffer: true);
7134 if (!EncodedImm)
7135 return {};
7136
7137 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7138}
7139
7140InstructionSelector::ComplexRendererFns
7141AMDGPUInstructionSelector::selectSMRDBufferImm32(MachineOperand &Root) const {
7142 assert(STI.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
7143
7144 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7145 if (!OffsetVal)
7146 return {};
7147
7148 std::optional<int64_t> EncodedImm =
7149 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: *OffsetVal);
7150 if (!EncodedImm)
7151 return {};
7152
7153 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7154}
7155
7156InstructionSelector::ComplexRendererFns
7157AMDGPUInstructionSelector::selectSMRDBufferSgprImm(MachineOperand &Root) const {
7158 // Match the (soffset + offset) pair as a 32-bit register base and
7159 // an immediate offset.
7160 Register SOffset;
7161 unsigned Offset;
7162 std::tie(args&: SOffset, args&: Offset) = AMDGPU::getBaseWithConstantOffset(
7163 MRI&: *MRI, Reg: Root.getReg(), ValueTracking: VT, /*CheckNUW*/ true);
7164 if (!SOffset)
7165 return std::nullopt;
7166
7167 std::optional<int64_t> EncodedOffset =
7168 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: Offset, /* IsBuffer */ true);
7169 if (!EncodedOffset)
7170 return std::nullopt;
7171
7172 assert(MRI->getType(SOffset).getSizeInBits() == 32);
7173 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
7174 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedOffset); }}};
7175}
7176
7177std::pair<Register, unsigned>
7178AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(MachineOperand &Root,
7179 bool &Matched) const {
7180 Matched = false;
7181
7182 Register Src;
7183 unsigned Mods;
7184 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
7185
7186 if (mi_match(R: Src, MRI: *MRI, P: m_GFPExt(Src: m_Reg(R&: Src)))) {
7187 assert(MRI->getType(Src) == LLT::scalar(16));
7188
7189 // Only change Src if src modifier could be gained. In such cases new Src
7190 // could be sgpr but this does not violate constant bus restriction for
7191 // instruction that is being selected.
7192 Src = stripBitCast(Reg: Src, MRI&: *MRI);
7193
7194 const auto CheckAbsNeg = [&]() {
7195 // Be careful about folding modifiers if we already have an abs. fneg is
7196 // applied last, so we don't want to apply an earlier fneg.
7197 if ((Mods & SISrcMods::ABS) == 0) {
7198 unsigned ModsTmp;
7199 std::tie(args&: Src, args&: ModsTmp) = selectVOP3ModsImpl(Src);
7200
7201 if ((ModsTmp & SISrcMods::NEG) != 0)
7202 Mods ^= SISrcMods::NEG;
7203
7204 if ((ModsTmp & SISrcMods::ABS) != 0)
7205 Mods |= SISrcMods::ABS;
7206 }
7207 };
7208
7209 CheckAbsNeg();
7210
7211 // op_sel/op_sel_hi decide the source type and source.
7212 // If the source's op_sel_hi is set, it indicates to do a conversion from
7213 // fp16. If the sources's op_sel is set, it picks the high half of the
7214 // source register.
7215
7216 Mods |= SISrcMods::OP_SEL_1;
7217
7218 // The instruction reads a 32-bit source and selects a half of it, so look
7219 // for the 32-bit register the 16-bit value is a half of.
7220 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
7221 // Src is now the 32-bit source and op_sel picks its high half.
7222 Mods |= SISrcMods::OP_SEL_0;
7223 CheckAbsNeg();
7224 } else {
7225 // op_sel already picks the low half, so use the 32-bit source directly if
7226 // the 16-bit value is the low half of one. Otherwise Src is genuinely 16
7227 // bits wide and widenSrcIfVGPR16 widens it when the operand is
7228 // rendered.
7229 isExtractLoElt(MRI&: *MRI, In: Src, Out&: Src);
7230 }
7231
7232 Matched = true;
7233 }
7234
7235 return {Src, Mods};
7236}
7237
7238InstructionSelector::ComplexRendererFns
7239AMDGPUInstructionSelector::selectVOP3PMadMixModsExt(
7240 MachineOperand &Root) const {
7241 Register Src;
7242 unsigned Mods;
7243 bool Matched;
7244 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7245 if (!Matched)
7246 return {};
7247
7248 return {{
7249 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7250 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7251 }};
7252}
7253
7254InstructionSelector::ComplexRendererFns
7255AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const {
7256 Register Src;
7257 unsigned Mods;
7258 bool Matched;
7259 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7260
7261 return {{
7262 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7263 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7264 }};
7265}
7266
7267InstructionSelector::ComplexRendererFns
7268AMDGPUInstructionSelector::selectVOP3PMadMixModsExtNeg(
7269 MachineOperand &Root) const {
7270 Register Src;
7271 unsigned Mods;
7272 bool Matched;
7273 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7274 if (!Matched)
7275 return {};
7276
7277 return {{
7278 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7279 [=](MachineInstrBuilder &MIB) {
7280 MIB.addImm(Val: Mods ^ SISrcMods::NEG);
7281 } // src_mods
7282 }};
7283}
7284
7285InstructionSelector::ComplexRendererFns
7286AMDGPUInstructionSelector::selectVOP3PMadMixModsNeg(
7287 MachineOperand &Root) const {
7288 Register Src;
7289 unsigned Mods;
7290 bool Matched;
7291 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7292
7293 return {{
7294 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7295 [=](MachineInstrBuilder &MIB) {
7296 MIB.addImm(Val: Mods ^ SISrcMods::NEG);
7297 } // src_mods
7298 }};
7299}
7300
7301bool AMDGPUInstructionSelector::selectSBarrierSignalIsfirst(
7302 MachineInstr &I, Intrinsic::ID IntrID) const {
7303 MachineBasicBlock *MBB = I.getParent();
7304 const DebugLoc &DL = I.getDebugLoc();
7305 Register CCReg = I.getOperand(i: 0).getReg();
7306
7307 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7308 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_EQ_U32)).addImm(Val: 0).addImm(Val: 0);
7309
7310 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM))
7311 .addImm(Val: I.getOperand(i: 2).getImm());
7312
7313 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg).addReg(RegNo: AMDGPU::SCC);
7314
7315 I.eraseFromParent();
7316 return RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32_XM0_XEXECRegClass,
7317 MRI&: *MRI);
7318}
7319
7320bool AMDGPUInstructionSelector::selectSGetBarrierState(
7321 MachineInstr &I, Intrinsic::ID IntrID) const {
7322 MachineBasicBlock *MBB = I.getParent();
7323 const DebugLoc &DL = I.getDebugLoc();
7324 const MachineOperand &BarOp = I.getOperand(i: 2);
7325 std::optional<int64_t> BarValImm =
7326 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7327
7328 if (!BarValImm) {
7329 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7330 .addReg(RegNo: BarOp.getReg());
7331 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7332 }
7333 MachineInstrBuilder MIB;
7334 unsigned Opc = BarValImm ? AMDGPU::S_GET_BARRIER_STATE_IMM
7335 : AMDGPU::S_GET_BARRIER_STATE_M0;
7336 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7337
7338 auto DstReg = I.getOperand(i: 0).getReg();
7339 const TargetRegisterClass *DstRC =
7340 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
7341 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7342 return false;
7343 MIB.addDef(RegNo: DstReg);
7344 if (BarValImm) {
7345 MIB.addImm(Val: *BarValImm);
7346 }
7347 I.eraseFromParent();
7348 return true;
7349}
7350
7351unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID) {
7352 if (HasInlineConst) {
7353 switch (IntrID) {
7354 default:
7355 llvm_unreachable("not a named barrier op");
7356 case Intrinsic::amdgcn_s_barrier_join:
7357 return AMDGPU::S_BARRIER_JOIN_IMM;
7358 case Intrinsic::amdgcn_s_wakeup_barrier:
7359 return AMDGPU::S_WAKEUP_BARRIER_IMM;
7360 case Intrinsic::amdgcn_s_get_named_barrier_state:
7361 return AMDGPU::S_GET_BARRIER_STATE_IMM;
7362 };
7363 } else {
7364 switch (IntrID) {
7365 default:
7366 llvm_unreachable("not a named barrier op");
7367 case Intrinsic::amdgcn_s_barrier_join:
7368 return AMDGPU::S_BARRIER_JOIN_M0;
7369 case Intrinsic::amdgcn_s_wakeup_barrier:
7370 return AMDGPU::S_WAKEUP_BARRIER_M0;
7371 case Intrinsic::amdgcn_s_get_named_barrier_state:
7372 return AMDGPU::S_GET_BARRIER_STATE_M0;
7373 };
7374 }
7375}
7376
7377bool AMDGPUInstructionSelector::selectNamedBarrierInit(
7378 MachineInstr &I, Intrinsic::ID IntrID) const {
7379 MachineBasicBlock *MBB = I.getParent();
7380 const DebugLoc &DL = I.getDebugLoc();
7381 const MachineOperand &BarOp = I.getOperand(i: 1);
7382 const MachineOperand &CntOp = I.getOperand(i: 2);
7383
7384 // A member count of 0 means "keep existing member count". That plus a known
7385 // constant value for the barrier ID lets us use the immarg form.
7386 if (IntrID == Intrinsic::amdgcn_s_barrier_signal_var) {
7387 std::optional<int64_t> CntImm =
7388 getIConstantVRegSExtVal(VReg: CntOp.getReg(), MRI: *MRI);
7389 if (CntImm && *CntImm == 0) {
7390 std::optional<int64_t> BarValImm =
7391 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7392 if (BarValImm) {
7393 uint32_t BarID = *BarValImm & 0x3F;
7394 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_IMM))
7395 .addImm(Val: BarID);
7396 I.eraseFromParent();
7397 return true;
7398 }
7399 }
7400 }
7401
7402 // BarID = BarOp & 0x3F
7403 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7404 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7405 .add(MO: BarOp)
7406 .addImm(Val: 0x3F)
7407 .setOperandDead(3); // Dead scc
7408
7409 // MO = ((CntOp & 0x3F) << shAmt) | BarID
7410 Register TmpReg2 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7411 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg2)
7412 .add(MO: CntOp)
7413 .addImm(Val: 0x3F)
7414 .setOperandDead(3); // Dead scc
7415
7416 Register TmpReg3 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7417 constexpr unsigned ShAmt = 16;
7418 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg3)
7419 .addReg(RegNo: TmpReg2)
7420 .addImm(Val: ShAmt)
7421 .setOperandDead(3); // Dead scc
7422
7423 Register TmpReg4 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7424 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B32), DestReg: TmpReg4)
7425 .addReg(RegNo: TmpReg1)
7426 .addReg(RegNo: TmpReg3)
7427 .setOperandDead(3); // Dead scc;
7428
7429 auto CopyMIB =
7430 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0).addReg(RegNo: TmpReg4);
7431 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7432
7433 unsigned Opc = IntrID == Intrinsic::amdgcn_s_barrier_init
7434 ? AMDGPU::S_BARRIER_INIT_M0
7435 : AMDGPU::S_BARRIER_SIGNAL_M0;
7436 MachineInstrBuilder MIB;
7437 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7438
7439 I.eraseFromParent();
7440 return true;
7441}
7442
7443bool AMDGPUInstructionSelector::selectNamedBarrierInst(
7444 MachineInstr &I, Intrinsic::ID IntrID) const {
7445 MachineBasicBlock *MBB = I.getParent();
7446 const DebugLoc &DL = I.getDebugLoc();
7447 MachineOperand BarOp = IntrID == Intrinsic::amdgcn_s_get_named_barrier_state
7448 ? I.getOperand(i: 2)
7449 : I.getOperand(i: 1);
7450 std::optional<int64_t> BarValImm =
7451 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7452
7453 if (!BarValImm) {
7454 // BarID = BarOp & 0x3F
7455 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7456 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7457 .addReg(RegNo: BarOp.getReg())
7458 .addImm(Val: 0x3F)
7459 .setOperandDead(3); // Dead scc;
7460
7461 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7462 .addReg(RegNo: TmpReg1);
7463 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7464 }
7465
7466 MachineInstrBuilder MIB;
7467 unsigned Opc = getNamedBarrierOp(HasInlineConst: BarValImm.has_value(), IntrID);
7468 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7469
7470 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
7471 auto DstReg = I.getOperand(i: 0).getReg();
7472 const TargetRegisterClass *DstRC =
7473 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
7474 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7475 return false;
7476 MIB.addDef(RegNo: DstReg);
7477 }
7478
7479 if (BarValImm) {
7480 uint32_t BarId = *BarValImm & 0x3F;
7481 MIB.addImm(Val: BarId);
7482 }
7483
7484 I.eraseFromParent();
7485 return true;
7486}
7487
7488void AMDGPUInstructionSelector::renderTruncImm32(MachineInstrBuilder &MIB,
7489 const MachineInstr &MI,
7490 int OpIdx) const {
7491 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7492 "Expected G_CONSTANT");
7493 MIB.addImm(Val: MI.getOperand(i: 1).getCImm()->getSExtValue());
7494}
7495
7496void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB,
7497 const MachineInstr &MI,
7498 int OpIdx) const {
7499 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7500 "Expected G_CONSTANT");
7501 MIB.addImm(Val: -MI.getOperand(i: 1).getCImm()->getSExtValue());
7502}
7503
7504void AMDGPUInstructionSelector::renderBitcastFPImm(MachineInstrBuilder &MIB,
7505 const MachineInstr &MI,
7506 int OpIdx) const {
7507 const MachineOperand &Op = MI.getOperand(i: 1);
7508 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1);
7509 MIB.addImm(Val: Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
7510}
7511
7512void AMDGPUInstructionSelector::renderCountTrailingOnesImm(
7513 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7514 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7515 "Expected G_CONSTANT");
7516 MIB.addImm(Val: MI.getOperand(i: 1).getCImm()->getValue().countTrailingOnes());
7517}
7518
7519/// This only really exists to satisfy DAG type checking machinery, so is a
7520/// no-op here.
7521void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB,
7522 const MachineInstr &MI,
7523 int OpIdx) const {
7524 const MachineOperand &Op = MI.getOperand(i: OpIdx);
7525 int64_t Imm;
7526 if (Op.isReg() && mi_match(R: Op.getReg(), MRI: *MRI, P: m_ICst(Cst&: Imm)))
7527 MIB.addImm(Val: Imm);
7528 else
7529 MIB.addImm(Val: Op.getImm());
7530}
7531
7532void AMDGPUInstructionSelector::renderZextBoolTImm(MachineInstrBuilder &MIB,
7533 const MachineInstr &MI,
7534 int OpIdx) const {
7535 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() != 0);
7536}
7537
7538void AMDGPUInstructionSelector::renderOpSelTImm(MachineInstrBuilder &MIB,
7539 const MachineInstr &MI,
7540 int OpIdx) const {
7541 assert(OpIdx >= 0 && "expected to match an immediate operand");
7542 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7543}
7544
7545void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_0(
7546 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7547 assert(OpIdx >= 0 && "expected to match an immediate operand");
7548 MIB.addImm(
7549 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7550}
7551
7552void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_1(
7553 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7554 assert(OpIdx >= 0 && "expected to match an immediate operand");
7555 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x1)
7556 ? (int64_t)(SISrcMods::OP_SEL_0 | SISrcMods::DST_OP_SEL)
7557 : (int64_t)SISrcMods::DST_OP_SEL);
7558}
7559
7560void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_0(
7561 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7562 assert(OpIdx >= 0 && "expected to match an immediate operand");
7563 MIB.addImm(
7564 Val: (MI.getOperand(i: OpIdx).getImm() & 0x2) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7565}
7566
7567void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_1(
7568 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7569 assert(OpIdx >= 0 && "expected to match an immediate operand");
7570 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7571 ? (int64_t)(SISrcMods::OP_SEL_0)
7572 : 0);
7573}
7574
7575void AMDGPUInstructionSelector::renderDstSelToOpSelXForm(
7576 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7577 assert(OpIdx >= 0 && "expected to match an immediate operand");
7578 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::DST_OP_SEL)
7579 : 0);
7580}
7581
7582void AMDGPUInstructionSelector::renderSrcSelToOpSelXForm(
7583 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7584 assert(OpIdx >= 0 && "expected to match an immediate operand");
7585 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::OP_SEL_0)
7586 : 0);
7587}
7588
7589void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_2_0(
7590 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7591 assert(OpIdx >= 0 && "expected to match an immediate operand");
7592 MIB.addImm(
7593 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7594}
7595
7596void AMDGPUInstructionSelector::renderDstSelToOpSel3XFormXForm(
7597 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7598 assert(OpIdx >= 0 && "expected to match an immediate operand");
7599 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7600 ? (int64_t)SISrcMods::DST_OP_SEL
7601 : 0);
7602}
7603
7604void AMDGPUInstructionSelector::renderExtractCPol(MachineInstrBuilder &MIB,
7605 const MachineInstr &MI,
7606 int OpIdx) const {
7607 assert(OpIdx >= 0 && "expected to match an immediate operand");
7608 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() &
7609 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7610 : AMDGPU::CPol::ALL_pregfx12));
7611}
7612
7613void AMDGPUInstructionSelector::renderExtractSWZ(MachineInstrBuilder &MIB,
7614 const MachineInstr &MI,
7615 int OpIdx) const {
7616 assert(OpIdx >= 0 && "expected to match an immediate operand");
7617 const bool Swizzle = MI.getOperand(i: OpIdx).getImm() &
7618 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::SWZ
7619 : AMDGPU::CPol::SWZ_pregfx12);
7620 MIB.addImm(Val: Swizzle);
7621}
7622
7623void AMDGPUInstructionSelector::renderExtractCpolSetGLC(
7624 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7625 assert(OpIdx >= 0 && "expected to match an immediate operand");
7626 const uint32_t Cpol = MI.getOperand(i: OpIdx).getImm() &
7627 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7628 : AMDGPU::CPol::ALL_pregfx12);
7629 MIB.addImm(Val: Cpol | AMDGPU::CPol::GLC);
7630}
7631
7632void AMDGPUInstructionSelector::renderFPPow2ToExponent(MachineInstrBuilder &MIB,
7633 const MachineInstr &MI,
7634 int OpIdx) const {
7635 const APFloat &APF = MI.getOperand(i: 1).getFPImm()->getValueAPF();
7636 int ExpVal = APF.getExactLog2Abs();
7637 assert(ExpVal != INT_MIN);
7638 MIB.addImm(Val: ExpVal);
7639}
7640
7641void AMDGPUInstructionSelector::renderRoundMode(MachineInstrBuilder &MIB,
7642 const MachineInstr &MI,
7643 int OpIdx) const {
7644 // "round.towardzero" -> TowardZero 0 -> FP_ROUND_ROUND_TO_ZERO 3
7645 // "round.tonearest" -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
7646 // "round.upward" -> TowardPositive 2 -> FP_ROUND_ROUND_TO_INF 1
7647 // "round.downward -> TowardNegative 3 -> FP_ROUND_ROUND_TO_NEGINF 2
7648 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() + 3) % 4);
7649}
7650
7651void AMDGPUInstructionSelector::renderVOP3PModsNeg(MachineInstrBuilder &MIB,
7652 const MachineInstr &MI,
7653 int OpIdx) const {
7654 unsigned Mods = SISrcMods::OP_SEL_1;
7655 if (MI.getOperand(i: OpIdx).getImm())
7656 Mods ^= SISrcMods::NEG;
7657 MIB.addImm(Val: (int64_t)Mods);
7658}
7659
7660void AMDGPUInstructionSelector::renderVOP3PModsNegs(MachineInstrBuilder &MIB,
7661 const MachineInstr &MI,
7662 int OpIdx) const {
7663 unsigned Mods = SISrcMods::OP_SEL_1;
7664 if (MI.getOperand(i: OpIdx).getImm())
7665 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
7666 MIB.addImm(Val: (int64_t)Mods);
7667}
7668
7669void AMDGPUInstructionSelector::renderVOP3PModsNegAbs(MachineInstrBuilder &MIB,
7670 const MachineInstr &MI,
7671 int OpIdx) const {
7672 unsigned Val = MI.getOperand(i: OpIdx).getImm();
7673 unsigned Mods = SISrcMods::OP_SEL_1; // default: none
7674 if (Val == 1) // neg
7675 Mods ^= SISrcMods::NEG;
7676 if (Val == 2) // abs
7677 Mods ^= SISrcMods::ABS;
7678 if (Val == 3) // neg and abs
7679 Mods ^= (SISrcMods::NEG | SISrcMods::ABS);
7680 MIB.addImm(Val: (int64_t)Mods);
7681}
7682
7683void AMDGPUInstructionSelector::renderPrefetchLoc(MachineInstrBuilder &MIB,
7684 const MachineInstr &MI,
7685 int OpIdx) const {
7686 uint32_t V = MI.getOperand(i: 2).getImm();
7687 V = (AMDGPU::CPol::SCOPE_MASK - (V & AMDGPU::CPol::SCOPE_MASK))
7688 << AMDGPU::CPol::SCOPE_SHIFT;
7689 if (!Subtarget->hasSafeCUPrefetch())
7690 V = std::max(a: V, b: (uint32_t)AMDGPU::CPol::SCOPE_SE); // CU scope is unsafe
7691 MIB.addImm(Val: V);
7692}
7693
7694/// Convert from 2-bit value to enum values used for op_sel* source modifiers.
7695void AMDGPUInstructionSelector::renderScaledMAIIntrinsicOperand(
7696 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7697 unsigned Val = MI.getOperand(i: OpIdx).getImm();
7698 unsigned New = 0;
7699 if (Val & 0x1)
7700 New |= SISrcMods::OP_SEL_0;
7701 if (Val & 0x2)
7702 New |= SISrcMods::OP_SEL_1;
7703 MIB.addImm(Val: New);
7704}
7705
7706bool AMDGPUInstructionSelector::isInlineImmediate(const APInt &Imm) const {
7707 return TII.isInlineConstant(Imm);
7708}
7709
7710bool AMDGPUInstructionSelector::isInlineImmediate(const APFloat &Imm) const {
7711 return TII.isInlineConstant(Imm);
7712}
7713