1//===- AMDGPUInstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUInstructionSelector.h"
15#include "AMDGPU.h"
16#include "AMDGPUGlobalISelUtils.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPURegisterBankInfo.h"
19#include "SIMachineFunctionInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
22#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/IntrinsicsAMDGPU.h"
29#include <optional>
30
31#define DEBUG_TYPE "amdgpu-isel"
32
33using namespace llvm;
34using namespace MIPatternMatch;
35
36#define GET_GLOBALISEL_IMPL
37#define AMDGPUSubtarget GCNSubtarget
38#include "AMDGPUGenGlobalISel.inc"
39#undef GET_GLOBALISEL_IMPL
40#undef AMDGPUSubtarget
41
42AMDGPUInstructionSelector::AMDGPUInstructionSelector(
43 const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI)
44 : TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI), STI(STI),
45#define GET_GLOBALISEL_PREDICATES_INIT
46#include "AMDGPUGenGlobalISel.inc"
47#undef GET_GLOBALISEL_PREDICATES_INIT
48#define GET_GLOBALISEL_TEMPORARIES_INIT
49#include "AMDGPUGenGlobalISel.inc"
50#undef GET_GLOBALISEL_TEMPORARIES_INIT
51{
52}
53
54const char *AMDGPUInstructionSelector::getName() { return DEBUG_TYPE; }
55
56void AMDGPUInstructionSelector::setupMF(MachineFunction &MF,
57 GISelValueTracking *VT,
58 CodeGenCoverage *CoverageInfo,
59 ProfileSummaryInfo *PSI,
60 BlockFrequencyInfo *BFI) {
61 MRI = &MF.getRegInfo();
62 Subtarget = &MF.getSubtarget<GCNSubtarget>();
63 Subtarget->checkSubtargetFeatures(F: MF.getFunction());
64 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
65}
66
67// Return the wave level SGPR base address if this is a wave address.
68static Register getWaveAddress(const MachineInstr *Def) {
69 return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS
70 ? Def->getOperand(i: 1).getReg()
71 : Register();
72}
73
74bool AMDGPUInstructionSelector::isVCC(Register Reg,
75 const MachineRegisterInfo &MRI) const {
76 // The verifier is oblivious to s1 being a valid value for wavesize registers.
77 if (Reg.isPhysical())
78 return false;
79
80 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
81 const TargetRegisterClass *RC =
82 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
83 if (RC) {
84 const LLT Ty = MRI.getType(Reg);
85 if (!Ty.isValid() || Ty.getSizeInBits() != 1)
86 return false;
87 // G_TRUNC s1 result is never vcc.
88 return !mi_match(R: Reg, MRI, P: m_GTrunc(Src: m_Reg())) &&
89 RC->hasSuperClassEq(RC: TRI.getBoolRC());
90 }
91
92 const RegisterBank *RB = cast<const RegisterBank *>(Val: RegClassOrBank);
93 return RB->getID() == AMDGPU::VCCRegBankID;
94}
95
96bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(MachineInstr &MI,
97 unsigned NewOpc) const {
98 MI.setDesc(TII.get(Opcode: NewOpc));
99 MI.removeOperand(OpNo: 1); // Remove intrinsic ID.
100 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
101
102 Register DstReg = MI.getOperand(i: 0).getReg();
103 Register SrcReg = MI.getOperand(i: 1).getReg();
104
105 // TODO: This should be legalized to s32 if needed
106 if (MRI->getType(Reg: DstReg) == LLT::scalar(SizeInBits: 1))
107 return false;
108
109 const TargetRegisterClass *DstRC =
110 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
111 const TargetRegisterClass *SrcRC =
112 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
113 if (!DstRC || DstRC != SrcRC)
114 return false;
115
116 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
117 !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
118 return false;
119 const MCInstrDesc &MCID = MI.getDesc();
120 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
121 MI.getOperand(i: 0).setIsEarlyClobber(true);
122 }
123 return true;
124}
125
126bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
127 const DebugLoc &DL = I.getDebugLoc();
128 MachineBasicBlock *BB = I.getParent();
129 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
130
131 Register DstReg = I.getOperand(i: 0).getReg();
132 Register SrcReg = I.getOperand(i: 1).getReg();
133
134 if (isVCC(Reg: DstReg, MRI: *MRI)) {
135 if (SrcReg == AMDGPU::SCC) {
136 const TargetRegisterClass *RC =
137 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
138 if (!RC)
139 return true;
140 return RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI);
141 }
142
143 if (!isVCC(Reg: SrcReg, MRI: *MRI)) {
144 // TODO: Should probably leave the copy and let copyPhysReg expand it.
145 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI))
146 return false;
147
148 const TargetRegisterClass *SrcRC =
149 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
150
151 std::optional<ValueAndVReg> ConstVal =
152 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI, LookThroughInstrs: true);
153 if (ConstVal) {
154 unsigned MovOpc =
155 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
156 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: DstReg)
157 .addImm(Val: ConstVal->Value.getBoolValue() ? -1 : 0);
158 } else {
159 Register MaskedReg = MRI->createVirtualRegister(RegClass: SrcRC);
160
161 // We can't trust the high bits at this point, so clear them.
162
163 // TODO: Skip masking high bits if def is known boolean.
164
165 if (AMDGPU::getRegBitWidth(RCID: SrcRC->getID()) == 16) {
166 assert(Subtarget->useRealTrue16Insts());
167 const int64_t NoMods = 0;
168 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B16_t16_e64), DestReg: MaskedReg)
169 .addImm(Val: NoMods)
170 .addImm(Val: 1)
171 .addImm(Val: NoMods)
172 .addReg(RegNo: SrcReg)
173 .addImm(Val: NoMods);
174 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U16_t16_e64), DestReg: DstReg)
175 .addImm(Val: NoMods)
176 .addImm(Val: 0)
177 .addImm(Val: NoMods)
178 .addReg(RegNo: MaskedReg)
179 .addImm(Val: NoMods);
180 } else {
181 bool IsSGPR = TRI.isSGPRClass(RC: SrcRC);
182 unsigned AndOpc = IsSGPR ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32;
183 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: MaskedReg)
184 .addImm(Val: 1)
185 .addReg(RegNo: SrcReg);
186 if (IsSGPR)
187 And.setOperandDead(3); // Dead scc
188
189 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U32_e64), DestReg: DstReg)
190 .addImm(Val: 0)
191 .addReg(RegNo: MaskedReg);
192 }
193 }
194
195 if (!MRI->getRegClassOrNull(Reg: SrcReg))
196 MRI->setRegClass(Reg: SrcReg, RC: SrcRC);
197 I.eraseFromParent();
198 return true;
199 }
200
201 const TargetRegisterClass *RC =
202 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
203 if (RC && !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
204 return false;
205
206 return true;
207 }
208
209 for (const MachineOperand &MO : I.operands()) {
210 if (MO.getReg().isPhysical())
211 continue;
212
213 const TargetRegisterClass *RC =
214 TRI.getConstrainedRegClassForReg(Reg: MO.getReg(), MRI: *MRI);
215 if (!RC)
216 continue;
217 RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI);
218 }
219 return true;
220}
221
222bool AMDGPUInstructionSelector::selectCOPY_SCC_VCC(MachineInstr &I) const {
223 const DebugLoc &DL = I.getDebugLoc();
224 MachineBasicBlock *BB = I.getParent();
225 Register VCCReg = I.getOperand(i: 1).getReg();
226 MachineInstr *Cmp;
227
228 // Set SCC as a side effect with S_CMP or S_OR.
229 if (STI.hasScalarCompareEq64()) {
230 unsigned CmpOpc =
231 STI.isWave64() ? AMDGPU::S_CMP_LG_U64 : AMDGPU::S_CMP_LG_U32;
232 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: CmpOpc)).addReg(RegNo: VCCReg).addImm(Val: 0);
233 } else {
234 Register DeadDst = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
235 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B64), DestReg: DeadDst)
236 .addReg(RegNo: VCCReg)
237 .addReg(RegNo: VCCReg);
238 }
239
240 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
241
242 Register DstReg = I.getOperand(i: 0).getReg();
243 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: AMDGPU::SCC);
244
245 I.eraseFromParent();
246 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
247}
248
249bool AMDGPUInstructionSelector::selectCOPY_VCC_SCC(MachineInstr &I) const {
250 const DebugLoc &DL = I.getDebugLoc();
251 MachineBasicBlock *BB = I.getParent();
252
253 Register DstReg = I.getOperand(i: 0).getReg();
254 Register SrcReg = I.getOperand(i: 1).getReg();
255 std::optional<ValueAndVReg> Arg =
256 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 1).getReg(), MRI: *MRI);
257
258 if (Arg) {
259 const int64_t Value = Arg->Value.getZExtValue();
260 if (Value == 0) {
261 unsigned Opcode = STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
262 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DstReg).addImm(Val: 0);
263 } else {
264 assert(Value == 1);
265 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: TRI.getExec());
266 }
267 I.eraseFromParent();
268 return RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI);
269 }
270
271 // RegBankLegalize ensures that SrcReg is bool in reg (high bits are 0).
272 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC).addReg(RegNo: SrcReg);
273
274 unsigned SelectOpcode =
275 STI.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
276 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
277 .addReg(RegNo: TRI.getExec())
278 .addImm(Val: 0);
279
280 I.eraseFromParent();
281 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
282 return true;
283}
284
285bool AMDGPUInstructionSelector::selectReadAnyLane(MachineInstr &I) const {
286 Register DstReg = I.getOperand(i: 0).getReg();
287 Register SrcReg = I.getOperand(i: 1).getReg();
288
289 const DebugLoc &DL = I.getDebugLoc();
290 MachineBasicBlock *BB = I.getParent();
291
292 auto RFL = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
293 .addReg(RegNo: SrcReg);
294
295 I.eraseFromParent();
296 constrainSelectedInstRegOperands(I&: *RFL, TII, TRI, RBI);
297 return true;
298}
299
300bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
301 const Register DefReg = I.getOperand(i: 0).getReg();
302 const LLT DefTy = MRI->getType(Reg: DefReg);
303
304 // S1 G_PHIs should not be selected in instruction-select, instead:
305 // - divergent S1 G_PHI should go through lane mask merging algorithm
306 // and be fully inst-selected in AMDGPUGlobalISelDivergenceLowering
307 // - uniform S1 G_PHI should be lowered into S32 G_PHI in AMDGPURegBankSelect
308 if (DefTy == LLT::scalar(SizeInBits: 1))
309 return false;
310
311 // TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy)
312
313 const RegClassOrRegBank &RegClassOrBank =
314 MRI->getRegClassOrRegBank(Reg: DefReg);
315
316 const TargetRegisterClass *DefRC =
317 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
318 if (!DefRC) {
319 if (!DefTy.isValid()) {
320 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
321 return false;
322 }
323
324 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
325 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, Bank: RB);
326 if (!DefRC) {
327 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
328 return false;
329 }
330 }
331
332 // If inputs have register bank, assign corresponding reg class.
333 // Note: registers don't need to have the same reg bank.
334 for (unsigned i = 1; i != I.getNumOperands(); i += 2) {
335 const Register SrcReg = I.getOperand(i).getReg();
336
337 const RegisterBank *RB = MRI->getRegBankOrNull(Reg: SrcReg);
338 if (RB) {
339 const LLT SrcTy = MRI->getType(Reg: SrcReg);
340 const TargetRegisterClass *SrcRC =
341 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *RB);
342 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
343 return false;
344 }
345 }
346
347 I.setDesc(TII.get(Opcode: TargetOpcode::PHI));
348 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
349}
350
351MachineOperand
352AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
353 const TargetRegisterClass &SubRC,
354 unsigned SubIdx) const {
355
356 MachineInstr *MI = MO.getParent();
357 MachineBasicBlock *BB = MO.getParent()->getParent();
358 Register DstReg = MRI->createVirtualRegister(RegClass: &SubRC);
359
360 if (MO.isReg()) {
361 unsigned ComposedSubIdx = TRI.composeSubRegIndices(a: MO.getSubReg(), b: SubIdx);
362 Register Reg = MO.getReg();
363 BuildMI(BB&: *BB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
364 .addReg(RegNo: Reg, Flags: {}, SubReg: ComposedSubIdx);
365
366 return MachineOperand::CreateReg(Reg: DstReg, isDef: MO.isDef(), isImp: MO.isImplicit(),
367 isKill: MO.isKill(), isDead: MO.isDead(), isUndef: MO.isUndef(),
368 isEarlyClobber: MO.isEarlyClobber(), SubReg: 0, isDebug: MO.isDebug(),
369 isInternalRead: MO.isInternalRead());
370 }
371
372 assert(MO.isImm());
373
374 APInt Imm(64, MO.getImm());
375
376 switch (SubIdx) {
377 default:
378 llvm_unreachable("do not know to split immediate with this sub index.");
379 case AMDGPU::sub0:
380 return MachineOperand::CreateImm(Val: Imm.getLoBits(numBits: 32).getSExtValue());
381 case AMDGPU::sub1:
382 return MachineOperand::CreateImm(Val: Imm.getHiBits(numBits: 32).getSExtValue());
383 }
384}
385
386static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
387 switch (Opc) {
388 case AMDGPU::G_AND:
389 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
390 case AMDGPU::G_OR:
391 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32;
392 case AMDGPU::G_XOR:
393 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32;
394 default:
395 llvm_unreachable("not a bit op");
396 }
397}
398
399bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
400 Register DstReg = I.getOperand(i: 0).getReg();
401 unsigned Size = RBI.getSizeInBits(Reg: DstReg, MRI: *MRI, TRI);
402
403 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
404 if (DstRB->getID() != AMDGPU::SGPRRegBankID &&
405 DstRB->getID() != AMDGPU::VCCRegBankID)
406 return false;
407
408 bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID &&
409 STI.isWave64());
410 I.setDesc(TII.get(Opcode: getLogicalBitOpcode(Opc: I.getOpcode(), Is64)));
411
412 // Dead implicit-def of scc
413 I.addOperand(Op: MachineOperand::CreateReg(Reg: AMDGPU::SCC, isDef: true, // isDef
414 isImp: true, // isImp
415 isKill: false, // isKill
416 isDead: true)); // isDead
417 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
418 return true;
419}
420
421bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
422 MachineBasicBlock *BB = I.getParent();
423 MachineFunction *MF = BB->getParent();
424 Register DstReg = I.getOperand(i: 0).getReg();
425 const DebugLoc &DL = I.getDebugLoc();
426 LLT Ty = MRI->getType(Reg: DstReg);
427 if (Ty.isVector())
428 return false;
429
430 unsigned Size = Ty.getSizeInBits();
431 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
432 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID;
433 const bool Sub = I.getOpcode() == TargetOpcode::G_SUB;
434
435 if (Size == 32) {
436 if (IsSALU) {
437 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
438 MachineInstr *Add =
439 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
440 .add(MO: I.getOperand(i: 1))
441 .add(MO: I.getOperand(i: 2))
442 .setOperandDead(3); // Dead scc
443 I.eraseFromParent();
444 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
445 return true;
446 }
447
448 if (STI.hasAddNoCarryInsts()) {
449 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64;
450 I.setDesc(TII.get(Opcode: Opc));
451 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
452 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
453 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
454 return true;
455 }
456
457 const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64;
458
459 Register UnusedCarry = MRI->createVirtualRegister(RegClass: TRI.getWaveMaskRegClass());
460 MachineInstr *Add
461 = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
462 .addDef(RegNo: UnusedCarry, Flags: RegState::Dead)
463 .add(MO: I.getOperand(i: 1))
464 .add(MO: I.getOperand(i: 2))
465 .addImm(Val: 0);
466 I.eraseFromParent();
467 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
468 return true;
469 }
470
471 assert(!Sub && "illegal sub should not reach here");
472
473 const TargetRegisterClass &RC
474 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass;
475 const TargetRegisterClass &HalfRC
476 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass;
477
478 MachineOperand Lo1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
479 MachineOperand Lo2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
480 MachineOperand Hi1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
481 MachineOperand Hi2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
482
483 Register DstLo = MRI->createVirtualRegister(RegClass: &HalfRC);
484 Register DstHi = MRI->createVirtualRegister(RegClass: &HalfRC);
485
486 if (IsSALU) {
487 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_U32), DestReg: DstLo)
488 .add(MO: Lo1)
489 .add(MO: Lo2);
490 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADDC_U32), DestReg: DstHi)
491 .add(MO: Hi1)
492 .add(MO: Hi2)
493 .setOperandDead(3); // Dead scc
494 } else {
495 const TargetRegisterClass *CarryRC = TRI.getWaveMaskRegClass();
496 Register CarryReg = MRI->createVirtualRegister(RegClass: CarryRC);
497 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: DstLo)
498 .addDef(RegNo: CarryReg)
499 .add(MO: Lo1)
500 .add(MO: Lo2)
501 .addImm(Val: 0);
502 MachineInstr *Addc =
503 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: DstHi)
504 .addDef(RegNo: MRI->createVirtualRegister(RegClass: CarryRC), Flags: RegState::Dead)
505 .add(MO: Hi1)
506 .add(MO: Hi2)
507 .addReg(RegNo: CarryReg)
508 .addImm(Val: 0);
509
510 constrainSelectedInstRegOperands(I&: *Addc, TII, TRI, RBI);
511 }
512
513 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
514 .addReg(RegNo: DstLo)
515 .addImm(Val: AMDGPU::sub0)
516 .addReg(RegNo: DstHi)
517 .addImm(Val: AMDGPU::sub1);
518
519
520 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
521 return false;
522
523 I.eraseFromParent();
524 return true;
525}
526
527bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE(
528 MachineInstr &I) const {
529 MachineBasicBlock *BB = I.getParent();
530 MachineFunction *MF = BB->getParent();
531 const DebugLoc &DL = I.getDebugLoc();
532 Register Dst0Reg = I.getOperand(i: 0).getReg();
533 Register Dst1Reg = I.getOperand(i: 1).getReg();
534 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO ||
535 I.getOpcode() == AMDGPU::G_UADDE;
536 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE ||
537 I.getOpcode() == AMDGPU::G_USUBE;
538
539 if (isVCC(Reg: Dst1Reg, MRI: *MRI)) {
540 unsigned NoCarryOpc =
541 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
542 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
543 I.setDesc(TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc));
544 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
545 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
546 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
547 return true;
548 }
549
550 Register Src0Reg = I.getOperand(i: 2).getReg();
551 Register Src1Reg = I.getOperand(i: 3).getReg();
552
553 if (HasCarryIn) {
554 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
555 .addReg(RegNo: I.getOperand(i: 4).getReg());
556 }
557
558 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
559 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
560
561 auto CarryInst = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc), DestReg: Dst0Reg)
562 .add(MO: I.getOperand(i: 2))
563 .add(MO: I.getOperand(i: 3));
564
565 if (MRI->use_nodbg_empty(RegNo: Dst1Reg)) {
566 CarryInst.setOperandDead(3); // Dead scc
567 } else {
568 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst1Reg)
569 .addReg(RegNo: AMDGPU::SCC);
570 if (!MRI->getRegClassOrNull(Reg: Dst1Reg))
571 MRI->setRegClass(Reg: Dst1Reg, RC: &AMDGPU::SReg_32RegClass);
572 }
573
574 if (!RBI.constrainGenericRegister(Reg: Dst0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
575 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
576 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
577 return false;
578
579 if (HasCarryIn &&
580 !RBI.constrainGenericRegister(Reg: I.getOperand(i: 4).getReg(),
581 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
582 return false;
583
584 I.eraseFromParent();
585 return true;
586}
587
588bool AMDGPUInstructionSelector::selectG_AMDGPU_MAD_64_32(
589 MachineInstr &I) const {
590 MachineBasicBlock *BB = I.getParent();
591 MachineFunction *MF = BB->getParent();
592 const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
593 bool UseNoCarry = Subtarget->hasMadNC64_32Insts() &&
594 MRI->use_nodbg_empty(RegNo: I.getOperand(i: 1).getReg());
595
596 unsigned Opc;
597 if (Subtarget->hasMADIntraFwdBug())
598 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64
599 : AMDGPU::V_MAD_I64_I32_gfx11_e64;
600 else if (UseNoCarry)
601 Opc = IsUnsigned ? AMDGPU::V_MAD_NC_U64_U32_e64
602 : AMDGPU::V_MAD_NC_I64_I32_e64;
603 else
604 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64;
605
606 if (UseNoCarry)
607 I.removeOperand(OpNo: 1);
608
609 I.setDesc(TII.get(Opcode: Opc));
610 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
611 I.addImplicitDefUseOperands(MF&: *MF);
612 I.getOperand(i: 0).setIsEarlyClobber(true);
613 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
614 return true;
615}
616
617// TODO: We should probably legalize these to only using 32-bit results.
618bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
619 MachineBasicBlock *BB = I.getParent();
620 Register DstReg = I.getOperand(i: 0).getReg();
621 Register SrcReg = I.getOperand(i: 1).getReg();
622 LLT DstTy = MRI->getType(Reg: DstReg);
623 LLT SrcTy = MRI->getType(Reg: SrcReg);
624 const unsigned SrcSize = SrcTy.getSizeInBits();
625 unsigned DstSize = DstTy.getSizeInBits();
626
627 // TODO: Should handle any multiple of 32 offset.
628 unsigned Offset = I.getOperand(i: 2).getImm();
629 if (Offset % 32 != 0 || DstSize > 128)
630 return false;
631
632 // 16-bit operations really use 32-bit registers.
633 // FIXME: Probably should not allow 16-bit G_EXTRACT results.
634 if (DstSize == 16)
635 DstSize = 32;
636
637 const TargetRegisterClass *DstRC =
638 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
639 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
640 return false;
641
642 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
643 const TargetRegisterClass *SrcRC =
644 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
645 if (!SrcRC)
646 return false;
647 unsigned SubReg = SIRegisterInfo::getSubRegFromChannel(Channel: Offset / 32,
648 NumRegs: DstSize / 32);
649 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubReg);
650 if (!SrcRC)
651 return false;
652
653 SrcReg = constrainOperandRegClass(MF: *MF, TRI, MRI&: *MRI, TII, RBI, InsertPt&: I,
654 RegClass: *SrcRC, RegMO&: I.getOperand(i: 1));
655 const DebugLoc &DL = I.getDebugLoc();
656 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
657 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
658
659 I.eraseFromParent();
660 return true;
661}
662
663bool AMDGPUInstructionSelector::selectS16MergeToS32(MachineInstr &MI) const {
664 Register Dst = MI.getOperand(i: 0).getReg();
665 Register Src0 = MI.getOperand(i: 1).getReg();
666 Register Src1 = MI.getOperand(i: 2).getReg();
667
668 LLT Src0Ty = MRI->getType(Reg: Src0);
669 LLT Src1Ty = MRI->getType(Reg: Src1);
670
671 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
672 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0, MRI: *MRI, TRI);
673 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1, MRI: *MRI, TRI);
674 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
675
676 Register ShiftSrc0;
677 Register ShiftSrc1;
678
679 const DebugLoc &DL = MI.getDebugLoc();
680 MachineBasicBlock *BB = MI.getParent();
681
682 // VGPR case
683 if (IsVector) {
684 // If source are both VGPR16, use REG_SEQUENCE with lo16/hi16 subregisters
685 if (Src0Bank->getID() == AMDGPU::VGPRRegBankID &&
686 Src1Bank->getID() == AMDGPU::VGPRRegBankID &&
687 Src0Ty == LLT::scalar(SizeInBits: 16) && Src1Ty == LLT::scalar(SizeInBits: 16)) {
688 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
689 .addReg(RegNo: Src0)
690 .addImm(Val: AMDGPU::lo16)
691 .addReg(RegNo: Src1)
692 .addImm(Val: AMDGPU::hi16);
693
694 if (!RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
695 return false;
696
697 MI.eraseFromParent();
698 return true;
699 }
700
701 // Otherwise, use V_LSHL_OR_B32_e64
702 Register TmpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
703 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: TmpReg)
704 .addImm(Val: 0xFFFF)
705 .addReg(RegNo: Src0);
706 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
707
708 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: Dst)
709 .addReg(RegNo: Src1)
710 .addImm(Val: 16)
711 .addReg(RegNo: TmpReg);
712 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
713
714 MI.eraseFromParent();
715 return true;
716 }
717
718 // SGPR case -> S_PACK_*_B32_B16
719 // With multiple uses of the shift, this will duplicate the shift and
720 // increase register pressure.
721 //
722 // (merge (lshr_oneuse $src0, 16), (lshr_oneuse $src1, 16)
723 // => (S_PACK_HH_B32_B16 $src0, $src1)
724 // (merge (lshr_oneuse SReg_32:$src0, 16), $src1)
725 // => (S_PACK_HL_B32_B16 $src0, $src1)
726 // (merge $src0, (lshr_oneuse SReg_32:$src1, 16))
727 // => (S_PACK_LH_B32_B16 $src0, $src1)
728 // (merge $src0, $src1)
729 // => (S_PACK_LL_B32_B16 $src0, $src1)
730
731 bool Shift0 = mi_match(
732 R: Src0, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc0), R: m_SpecificICst(RequestedValue: 16))));
733
734 bool Shift1 = mi_match(
735 R: Src1, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc1), R: m_SpecificICst(RequestedValue: 16))));
736
737 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16;
738 if (Shift0 && Shift1) {
739 Opc = AMDGPU::S_PACK_HH_B32_B16;
740 MI.getOperand(i: 1).setReg(ShiftSrc0);
741 MI.getOperand(i: 2).setReg(ShiftSrc1);
742 } else if (Shift1) {
743 Opc = AMDGPU::S_PACK_LH_B32_B16;
744 MI.getOperand(i: 2).setReg(ShiftSrc1);
745 } else if (Shift0) {
746 auto ConstSrc1 =
747 getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
748 if (ConstSrc1 && ConstSrc1->Value == 0) {
749 // build_vector_trunc (lshr $src0, 16), 0 -> s_lshr_b32 $src0, 16
750 auto MIB = BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: Dst)
751 .addReg(RegNo: ShiftSrc0)
752 .addImm(Val: 16)
753 .setOperandDead(3); // Dead scc
754
755 MI.eraseFromParent();
756 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
757 return true;
758 }
759 if (STI.hasSPackHL()) {
760 Opc = AMDGPU::S_PACK_HL_B32_B16;
761 MI.getOperand(i: 1).setReg(ShiftSrc0);
762 }
763 }
764
765 MI.setDesc(TII.get(Opcode: Opc));
766 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
767 return true;
768}
769
770// Pack each pair of s16 into an s32 with S_PACK_LL_B32_B16, then combine the
771// s32 pieces into the destination with a REG_SEQUENCE.
772bool AMDGPUInstructionSelector::selectS16MergeToWide(MachineInstr &MI) const {
773 MachineBasicBlock *BB = MI.getParent();
774 const DebugLoc &DL = MI.getDebugLoc();
775 Register DstReg = MI.getOperand(i: 0).getReg();
776 const unsigned DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
777 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
778 const unsigned NumSrc = MI.getNumOperands() - 1;
779
780 // Pack each pair of s16 sources into an s32.
781 SmallVector<Register, 8> S32Regs;
782 for (unsigned I = 0; I != NumSrc; I += 2) {
783 Register S32 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
784 auto Pack = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_PACK_LL_B32_B16), DestReg: S32)
785 .addReg(RegNo: MI.getOperand(i: I + 1).getReg())
786 .addReg(RegNo: MI.getOperand(i: I + 2).getReg());
787 constrainSelectedInstRegOperands(I&: *Pack, TII, TRI, RBI);
788 S32Regs.push_back(Elt: S32);
789 }
790
791 // Combine the s32 pieces into the destination with a REG_SEQUENCE.
792 const TargetRegisterClass *DstRC =
793 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
794 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
795 return false;
796 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, /*EltSize=*/4);
797 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
798 for (unsigned I = 0, E = S32Regs.size(); I != E; ++I)
799 MIB.addReg(RegNo: S32Regs[I]).addImm(Val: SubRegs[I]);
800
801 MI.eraseFromParent();
802 return true;
803}
804
805bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
806 MachineBasicBlock *BB = MI.getParent();
807 Register DstReg = MI.getOperand(i: 0).getReg();
808 LLT DstTy = MRI->getType(Reg: DstReg);
809 LLT SrcTy = MRI->getType(Reg: MI.getOperand(i: 1).getReg());
810
811 const unsigned SrcSize = SrcTy.getSizeInBits();
812 if (SrcSize < 32) {
813 // Handle s32 <- G_MERGE_VALUES s16, s16
814 if (SrcSize == 16 && DstTy.getSizeInBits() == 32 &&
815 MI.getNumOperands() == 3) {
816 return selectS16MergeToS32(MI);
817 }
818 // With true16 a scalar s16 is a register type, so a scalar wider than 32
819 // bits can be built from s16 pieces.
820 bool IsWideS16Merge = SrcSize == 16 && DstTy.getSizeInBits() > 32 &&
821 DstTy.getSizeInBits() % 32 == 0;
822
823 // SGPRs have no 16-bit subregisters, so pack pairs of s16 with S_PACK.
824 if (IsWideS16Merge &&
825 RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
826 return selectS16MergeToWide(MI);
827
828 // A VGPR wide s16 merge falls through to the generic path below.
829 if (!IsWideS16Merge)
830 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
831 }
832
833 const DebugLoc &DL = MI.getDebugLoc();
834 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
835 const unsigned DstSize = DstTy.getSizeInBits();
836 const TargetRegisterClass *DstRC =
837 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
838 if (!DstRC)
839 return false;
840
841 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, EltSize: SrcSize / 8);
842 MachineInstrBuilder MIB =
843 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
844 for (int I = 0, E = MI.getNumOperands() - 1; I != E; ++I) {
845 MachineOperand &Src = MI.getOperand(i: I + 1);
846 Register SrcReg = Src.getReg();
847 MIB.addReg(RegNo: SrcReg, Flags: getUndefRegState(B: Src.isUndef()));
848 MIB.addImm(Val: SubRegs[I]);
849
850 const TargetRegisterClass *SrcRC =
851 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
852 if (SrcRC && !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
853 return false;
854 }
855
856 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
857 return false;
858
859 MI.eraseFromParent();
860 return true;
861}
862
863bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
864 MachineBasicBlock *BB = MI.getParent();
865 const int NumDst = MI.getNumOperands() - 1;
866
867 MachineOperand &Src = MI.getOperand(i: NumDst);
868
869 Register SrcReg = Src.getReg();
870 Register DstReg0 = MI.getOperand(i: 0).getReg();
871 LLT DstTy = MRI->getType(Reg: DstReg0);
872 LLT SrcTy = MRI->getType(Reg: SrcReg);
873
874 const unsigned DstSize = DstTy.getSizeInBits();
875 const unsigned SrcSize = SrcTy.getSizeInBits();
876 const DebugLoc &DL = MI.getDebugLoc();
877 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
878
879 const TargetRegisterClass *SrcRC =
880 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
881
882 // Note we could have mixed SGPR and VGPR destination banks for an SGPR
883 // source, and this relies on the fact that the same subregister indices are
884 // used for both.
885 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SrcRC, EltSize: DstSize / 8);
886 for (int I = 0, E = NumDst; I != E; ++I) {
887 Register DstReg = MI.getOperand(i: I).getReg();
888 // hi16:sreg_32 is not allowed so explicitly shift upper 16-bits.
889 if (SrcBank->getID() == AMDGPU::SGPRRegBankID &&
890 SubRegs[I] == AMDGPU::hi16) {
891 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: DstReg)
892 .addReg(RegNo: SrcReg)
893 .addImm(Val: 16);
894 } else {
895 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
896 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegs[I]);
897
898 // Make sure the subregister index is valid for the source register.
899 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[I]);
900 }
901
902 if (!SrcRC || !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
903 return false;
904
905 const TargetRegisterClass *DstRC =
906 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
907 if (DstRC && !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
908 return false;
909 }
910
911 MI.eraseFromParent();
912 return true;
913}
914
915bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR(MachineInstr &MI) const {
916 assert(MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC ||
917 MI.getOpcode() == AMDGPU::G_BUILD_VECTOR);
918
919 Register Src0 = MI.getOperand(i: 1).getReg();
920 Register Src1 = MI.getOperand(i: 2).getReg();
921 LLT SrcTy = MRI->getType(Reg: Src0);
922 const unsigned SrcSize = SrcTy.getSizeInBits();
923
924 // BUILD_VECTOR with >=32 bits source is handled by MERGE_VALUE.
925 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR && SrcSize >= 32) {
926 return selectG_MERGE_VALUES(MI);
927 }
928
929 // Selection logic below is for V2S16 only.
930 // For G_BUILD_VECTOR_TRUNC, additionally check that the operands are s32.
931 Register Dst = MI.getOperand(i: 0).getReg();
932 if (MRI->getType(Reg: Dst) != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
933 (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC &&
934 SrcTy != LLT::scalar(SizeInBits: 32)))
935 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
936
937 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
938 if (DstBank->getID() == AMDGPU::AGPRRegBankID)
939 return false;
940
941 assert(DstBank->getID() == AMDGPU::SGPRRegBankID ||
942 DstBank->getID() == AMDGPU::VGPRRegBankID);
943 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
944
945 const DebugLoc &DL = MI.getDebugLoc();
946 MachineBasicBlock *BB = MI.getParent();
947
948 // First, before trying TableGen patterns, check if both sources are
949 // constants. In those cases, we can trivially compute the final constant
950 // and emit a simple move.
951 auto ConstSrc1 = getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
952 if (ConstSrc1) {
953 auto ConstSrc0 =
954 getAnyConstantVRegValWithLookThrough(VReg: Src0, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
955 if (ConstSrc0) {
956 const int64_t K0 = ConstSrc0->Value.getSExtValue();
957 const int64_t K1 = ConstSrc1->Value.getSExtValue();
958 uint32_t Lo16 = static_cast<uint32_t>(K0) & 0xffff;
959 uint32_t Hi16 = static_cast<uint32_t>(K1) & 0xffff;
960 uint32_t Imm = Lo16 | (Hi16 << 16);
961
962 // VALU
963 if (IsVector) {
964 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: Dst).addImm(Val: Imm);
965 MI.eraseFromParent();
966 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI);
967 }
968
969 // SALU
970 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: Dst).addImm(Val: Imm);
971 MI.eraseFromParent();
972 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
973 }
974 }
975
976 // Now try TableGen patterns.
977 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
978 return true;
979
980 // TODO: This should probably be a combine somewhere
981 // (build_vector $src0, undef) -> copy $src0
982 MachineInstr *Src1Def = getDefIgnoringCopies(Reg: Src1, MRI: *MRI);
983 if (Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) {
984 MI.setDesc(TII.get(Opcode: AMDGPU::COPY));
985 MI.removeOperand(OpNo: 2);
986 const auto &RC =
987 IsVector ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
988 return RBI.constrainGenericRegister(Reg: Dst, RC, MRI&: *MRI) &&
989 RBI.constrainGenericRegister(Reg: Src0, RC, MRI&: *MRI);
990 }
991
992 return selectS16MergeToS32(MI);
993}
994
995bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const {
996 const MachineOperand &MO = I.getOperand(i: 0);
997
998 // FIXME: Interface for getConstrainedRegClassForReg needs work. The
999 // regbank check here is to know why getConstrainedRegClassForReg failed.
1000 const TargetRegisterClass *RC =
1001 TRI.getConstrainedRegClassForReg(Reg: MO.getReg(), MRI: *MRI);
1002 if ((!RC && !MRI->getRegBankOrNull(Reg: MO.getReg())) ||
1003 (RC && RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI))) {
1004 I.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1005 return true;
1006 }
1007
1008 return false;
1009}
1010
1011bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
1012 MachineBasicBlock *BB = I.getParent();
1013
1014 Register DstReg = I.getOperand(i: 0).getReg();
1015 Register Src0Reg = I.getOperand(i: 1).getReg();
1016 Register Src1Reg = I.getOperand(i: 2).getReg();
1017 LLT Src1Ty = MRI->getType(Reg: Src1Reg);
1018
1019 unsigned DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
1020 unsigned InsSize = Src1Ty.getSizeInBits();
1021
1022 int64_t Offset = I.getOperand(i: 3).getImm();
1023
1024 // FIXME: These cases should have been illegal and unnecessary to check here.
1025 if (Offset % 32 != 0 || InsSize % 32 != 0)
1026 return false;
1027
1028 // Currently not handled by getSubRegFromChannel.
1029 if (InsSize > 128)
1030 return false;
1031
1032 unsigned SubReg = TRI.getSubRegFromChannel(Channel: Offset / 32, NumRegs: InsSize / 32);
1033 if (SubReg == AMDGPU::NoSubRegister)
1034 return false;
1035
1036 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1037 const TargetRegisterClass *DstRC =
1038 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
1039 if (!DstRC)
1040 return false;
1041
1042 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0Reg, MRI: *MRI, TRI);
1043 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1Reg, MRI: *MRI, TRI);
1044 const TargetRegisterClass *Src0RC =
1045 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *Src0Bank);
1046 const TargetRegisterClass *Src1RC =
1047 TRI.getRegClassForSizeOnBank(Size: InsSize, Bank: *Src1Bank);
1048
1049 // Deal with weird cases where the class only partially supports the subreg
1050 // index.
1051 Src0RC = TRI.getSubClassWithSubReg(Src0RC, SubReg);
1052 if (!Src0RC || !Src1RC)
1053 return false;
1054
1055 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
1056 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: *Src0RC, MRI&: *MRI) ||
1057 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: *Src1RC, MRI&: *MRI))
1058 return false;
1059
1060 const DebugLoc &DL = I.getDebugLoc();
1061 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1062 .addReg(RegNo: Src0Reg)
1063 .addReg(RegNo: Src1Reg)
1064 .addImm(Val: SubReg);
1065
1066 I.eraseFromParent();
1067 return true;
1068}
1069
1070bool AMDGPUInstructionSelector::selectG_SBFX_UBFX(MachineInstr &MI) const {
1071 Register DstReg = MI.getOperand(i: 0).getReg();
1072 Register SrcReg = MI.getOperand(i: 1).getReg();
1073 Register OffsetReg = MI.getOperand(i: 2).getReg();
1074 Register WidthReg = MI.getOperand(i: 3).getReg();
1075
1076 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID &&
1077 "scalar BFX instructions are expanded in regbankselect");
1078 assert(MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 &&
1079 "64-bit vector BFX instructions are expanded in regbankselect");
1080
1081 const DebugLoc &DL = MI.getDebugLoc();
1082 MachineBasicBlock *MBB = MI.getParent();
1083
1084 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SBFX;
1085 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
1086 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
1087 .addReg(RegNo: SrcReg)
1088 .addReg(RegNo: OffsetReg)
1089 .addReg(RegNo: WidthReg);
1090 MI.eraseFromParent();
1091 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1092 return true;
1093}
1094
1095bool AMDGPUInstructionSelector::selectInterpP1F16(MachineInstr &MI) const {
1096 if (STI.getLDSBankCount() != 16)
1097 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1098
1099 Register Dst = MI.getOperand(i: 0).getReg();
1100 Register Src0 = MI.getOperand(i: 2).getReg();
1101 Register M0Val = MI.getOperand(i: 6).getReg();
1102 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
1103 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI) ||
1104 !RBI.constrainGenericRegister(Reg: Src0, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
1105 return false;
1106
1107 // This requires 2 instructions. It is possible to write a pattern to support
1108 // this, but the generated isel emitter doesn't correctly deal with multiple
1109 // output instructions using the same physical register input. The copy to m0
1110 // is incorrectly placed before the second instruction.
1111 //
1112 // TODO: Match source modifiers.
1113
1114 Register InterpMov = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1115 const DebugLoc &DL = MI.getDebugLoc();
1116 MachineBasicBlock *MBB = MI.getParent();
1117
1118 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1119 .addReg(RegNo: M0Val);
1120 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_MOV_F32), DestReg: InterpMov)
1121 .addImm(Val: 2)
1122 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1123 .addImm(Val: MI.getOperand(i: 3).getImm()); // $attrchan
1124
1125 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_P1LV_F16), DestReg: Dst)
1126 .addImm(Val: 0) // $src0_modifiers
1127 .addReg(RegNo: Src0) // $src0
1128 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1129 .addImm(Val: MI.getOperand(i: 3).getImm()) // $attrchan
1130 .addImm(Val: 0) // $src2_modifiers
1131 .addReg(RegNo: InterpMov) // $src2 - 2 f16 values selected by high
1132 .addImm(Val: MI.getOperand(i: 5).getImm()) // $high
1133 .addImm(Val: 0) // $clamp
1134 .addImm(Val: 0); // $omod
1135
1136 MI.eraseFromParent();
1137 return true;
1138}
1139
1140// Writelane is special in that it can use SGPR and M0 (which would normally
1141// count as using the constant bus twice - but in this case it is allowed since
1142// the lane selector doesn't count as a use of the constant bus). However, it is
1143// still required to abide by the 1 SGPR rule. Fix this up if we might have
1144// multiple SGPRs.
1145bool AMDGPUInstructionSelector::selectWritelane(MachineInstr &MI) const {
1146 // With a constant bus limit of at least 2, there's no issue.
1147 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_WRITELANE_B32) > 1)
1148 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1149
1150 MachineBasicBlock *MBB = MI.getParent();
1151 const DebugLoc &DL = MI.getDebugLoc();
1152 Register VDst = MI.getOperand(i: 0).getReg();
1153 Register Val = MI.getOperand(i: 2).getReg();
1154 Register LaneSelect = MI.getOperand(i: 3).getReg();
1155 Register VDstIn = MI.getOperand(i: 4).getReg();
1156
1157 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_WRITELANE_B32), DestReg: VDst);
1158
1159 std::optional<ValueAndVReg> ConstSelect =
1160 getIConstantVRegValWithLookThrough(VReg: LaneSelect, MRI: *MRI);
1161 if (ConstSelect) {
1162 // The selector has to be an inline immediate, so we can use whatever for
1163 // the other operands.
1164 MIB.addReg(RegNo: Val);
1165 MIB.addImm(Val: ConstSelect->Value.getSExtValue() &
1166 maskTrailingOnes<uint64_t>(N: STI.getWavefrontSizeLog2()));
1167 } else {
1168 std::optional<ValueAndVReg> ConstVal =
1169 getIConstantVRegValWithLookThrough(VReg: Val, MRI: *MRI);
1170
1171 // If the value written is an inline immediate, we can get away without a
1172 // copy to m0.
1173 if (ConstVal && AMDGPU::isInlinableLiteral32(Literal: ConstVal->Value.getSExtValue(),
1174 HasInv2Pi: STI.hasInv2PiInlineImm())) {
1175 MIB.addImm(Val: ConstVal->Value.getSExtValue());
1176 MIB.addReg(RegNo: LaneSelect);
1177 } else {
1178 MIB.addReg(RegNo: Val);
1179
1180 // If the lane selector was originally in a VGPR and copied with
1181 // readfirstlane, there's a hazard to read the same SGPR from the
1182 // VALU. Constrain to a different SGPR to help avoid needing a nop later.
1183 RBI.constrainGenericRegister(Reg: LaneSelect, RC: AMDGPU::SReg_32_XM0RegClass, MRI&: *MRI);
1184
1185 BuildMI(BB&: *MBB, I&: *MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1186 .addReg(RegNo: LaneSelect);
1187 MIB.addReg(RegNo: AMDGPU::M0);
1188 }
1189 }
1190
1191 MIB.addReg(RegNo: VDstIn);
1192
1193 MI.eraseFromParent();
1194 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1195 return true;
1196}
1197
1198// We need to handle this here because tablegen doesn't support matching
1199// instructions with multiple outputs.
1200bool AMDGPUInstructionSelector::selectDivScale(MachineInstr &MI) const {
1201 Register Dst0 = MI.getOperand(i: 0).getReg();
1202 Register Dst1 = MI.getOperand(i: 1).getReg();
1203
1204 LLT Ty = MRI->getType(Reg: Dst0);
1205 unsigned Opc;
1206 if (Ty == LLT::scalar(SizeInBits: 32))
1207 Opc = AMDGPU::V_DIV_SCALE_F32_e64;
1208 else if (Ty == LLT::scalar(SizeInBits: 64))
1209 Opc = AMDGPU::V_DIV_SCALE_F64_e64;
1210 else
1211 return false;
1212
1213 // TODO: Match source modifiers.
1214
1215 const DebugLoc &DL = MI.getDebugLoc();
1216 MachineBasicBlock *MBB = MI.getParent();
1217
1218 Register Numer = MI.getOperand(i: 3).getReg();
1219 Register Denom = MI.getOperand(i: 4).getReg();
1220 unsigned ChooseDenom = MI.getOperand(i: 5).getImm();
1221
1222 Register Src0 = ChooseDenom != 0 ? Numer : Denom;
1223
1224 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
1225 .addDef(RegNo: Dst1)
1226 .addImm(Val: 0) // $src0_modifiers
1227 .addUse(RegNo: Src0) // $src0
1228 .addImm(Val: 0) // $src1_modifiers
1229 .addUse(RegNo: Denom) // $src1
1230 .addImm(Val: 0) // $src2_modifiers
1231 .addUse(RegNo: Numer) // $src2
1232 .addImm(Val: 0) // $clamp
1233 .addImm(Val: 0); // $omod
1234
1235 MI.eraseFromParent();
1236 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1237 return true;
1238}
1239
1240bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
1241 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
1242 switch (IntrinsicID) {
1243 case Intrinsic::amdgcn_if_break: {
1244 MachineBasicBlock *BB = I.getParent();
1245
1246 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1247 // SelectionDAG uses for wave32 vs wave64.
1248 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_IF_BREAK))
1249 .add(MO: I.getOperand(i: 0))
1250 .add(MO: I.getOperand(i: 2))
1251 .add(MO: I.getOperand(i: 3))
1252 .setOperandDead(3); // implicit-def $scc
1253
1254 Register DstReg = I.getOperand(i: 0).getReg();
1255 Register Src0Reg = I.getOperand(i: 2).getReg();
1256 Register Src1Reg = I.getOperand(i: 3).getReg();
1257
1258 I.eraseFromParent();
1259
1260 for (Register Reg : { DstReg, Src0Reg, Src1Reg })
1261 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1262
1263 return true;
1264 }
1265 case Intrinsic::amdgcn_interp_p1_f16:
1266 return selectInterpP1F16(MI&: I);
1267 case Intrinsic::amdgcn_wqm:
1268 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::WQM);
1269 case Intrinsic::amdgcn_softwqm:
1270 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::SOFT_WQM);
1271 case Intrinsic::amdgcn_strict_wwm:
1272 case Intrinsic::amdgcn_wwm:
1273 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WWM);
1274 case Intrinsic::amdgcn_strict_wqm:
1275 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WQM);
1276 case Intrinsic::amdgcn_writelane:
1277 return selectWritelane(MI&: I);
1278 case Intrinsic::amdgcn_div_scale:
1279 return selectDivScale(MI&: I);
1280 case Intrinsic::amdgcn_ballot:
1281 return selectBallot(I);
1282 case Intrinsic::amdgcn_reloc_constant:
1283 return selectRelocConstant(I);
1284 case Intrinsic::amdgcn_groupstaticsize:
1285 return selectGroupStaticSize(I);
1286 case Intrinsic::returnaddress:
1287 return selectReturnAddress(I);
1288 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
1289 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
1290 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
1291 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
1292 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
1293 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
1294 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
1295 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
1296 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
1297 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
1298 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
1299 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
1300 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
1301 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
1302 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
1303 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
1304 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
1305 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
1306 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
1307 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
1308 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
1309 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
1310 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
1311 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
1312 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
1313 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
1314 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
1315 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
1316 return selectSMFMACIntrin(I);
1317 case Intrinsic::amdgcn_permlane16_swap:
1318 case Intrinsic::amdgcn_permlane32_swap:
1319 return selectPermlaneSwapIntrin(I, IntrID: IntrinsicID);
1320 case Intrinsic::amdgcn_wave_shuffle:
1321 return selectWaveShuffleIntrin(I);
1322 default:
1323 return selectImpl(I, CoverageInfo&: *CoverageInfo);
1324 }
1325}
1326
1327static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size,
1328 const GCNSubtarget &ST) {
1329 if (Size != 16 && Size != 32 && Size != 64)
1330 return -1;
1331
1332 if (Size == 16 && !ST.has16BitInsts())
1333 return -1;
1334
1335 const auto Select = [&](unsigned S16Opc, unsigned TrueS16Opc,
1336 unsigned FakeS16Opc, unsigned S32Opc,
1337 unsigned S64Opc) {
1338 if (Size == 16)
1339 return ST.hasTrue16BitInsts()
1340 ? ST.useRealTrue16Insts() ? TrueS16Opc : FakeS16Opc
1341 : S16Opc;
1342 if (Size == 32)
1343 return S32Opc;
1344 return S64Opc;
1345 };
1346
1347 switch (P) {
1348 default:
1349 llvm_unreachable("Unknown condition code!");
1350 case CmpInst::ICMP_NE:
1351 return Select(AMDGPU::V_CMP_NE_U16_e64, AMDGPU::V_CMP_NE_U16_t16_e64,
1352 AMDGPU::V_CMP_NE_U16_fake16_e64, AMDGPU::V_CMP_NE_U32_e64,
1353 AMDGPU::V_CMP_NE_U64_e64);
1354 case CmpInst::ICMP_EQ:
1355 return Select(AMDGPU::V_CMP_EQ_U16_e64, AMDGPU::V_CMP_EQ_U16_t16_e64,
1356 AMDGPU::V_CMP_EQ_U16_fake16_e64, AMDGPU::V_CMP_EQ_U32_e64,
1357 AMDGPU::V_CMP_EQ_U64_e64);
1358 case CmpInst::ICMP_SGT:
1359 return Select(AMDGPU::V_CMP_GT_I16_e64, AMDGPU::V_CMP_GT_I16_t16_e64,
1360 AMDGPU::V_CMP_GT_I16_fake16_e64, AMDGPU::V_CMP_GT_I32_e64,
1361 AMDGPU::V_CMP_GT_I64_e64);
1362 case CmpInst::ICMP_SGE:
1363 return Select(AMDGPU::V_CMP_GE_I16_e64, AMDGPU::V_CMP_GE_I16_t16_e64,
1364 AMDGPU::V_CMP_GE_I16_fake16_e64, AMDGPU::V_CMP_GE_I32_e64,
1365 AMDGPU::V_CMP_GE_I64_e64);
1366 case CmpInst::ICMP_SLT:
1367 return Select(AMDGPU::V_CMP_LT_I16_e64, AMDGPU::V_CMP_LT_I16_t16_e64,
1368 AMDGPU::V_CMP_LT_I16_fake16_e64, AMDGPU::V_CMP_LT_I32_e64,
1369 AMDGPU::V_CMP_LT_I64_e64);
1370 case CmpInst::ICMP_SLE:
1371 return Select(AMDGPU::V_CMP_LE_I16_e64, AMDGPU::V_CMP_LE_I16_t16_e64,
1372 AMDGPU::V_CMP_LE_I16_fake16_e64, AMDGPU::V_CMP_LE_I32_e64,
1373 AMDGPU::V_CMP_LE_I64_e64);
1374 case CmpInst::ICMP_UGT:
1375 return Select(AMDGPU::V_CMP_GT_U16_e64, AMDGPU::V_CMP_GT_U16_t16_e64,
1376 AMDGPU::V_CMP_GT_U16_fake16_e64, AMDGPU::V_CMP_GT_U32_e64,
1377 AMDGPU::V_CMP_GT_U64_e64);
1378 case CmpInst::ICMP_UGE:
1379 return Select(AMDGPU::V_CMP_GE_U16_e64, AMDGPU::V_CMP_GE_U16_t16_e64,
1380 AMDGPU::V_CMP_GE_U16_fake16_e64, AMDGPU::V_CMP_GE_U32_e64,
1381 AMDGPU::V_CMP_GE_U64_e64);
1382 case CmpInst::ICMP_ULT:
1383 return Select(AMDGPU::V_CMP_LT_U16_e64, AMDGPU::V_CMP_LT_U16_t16_e64,
1384 AMDGPU::V_CMP_LT_U16_fake16_e64, AMDGPU::V_CMP_LT_U32_e64,
1385 AMDGPU::V_CMP_LT_U64_e64);
1386 case CmpInst::ICMP_ULE:
1387 return Select(AMDGPU::V_CMP_LE_U16_e64, AMDGPU::V_CMP_LE_U16_t16_e64,
1388 AMDGPU::V_CMP_LE_U16_fake16_e64, AMDGPU::V_CMP_LE_U32_e64,
1389 AMDGPU::V_CMP_LE_U64_e64);
1390
1391 case CmpInst::FCMP_OEQ:
1392 return Select(AMDGPU::V_CMP_EQ_F16_e64, AMDGPU::V_CMP_EQ_F16_t16_e64,
1393 AMDGPU::V_CMP_EQ_F16_fake16_e64, AMDGPU::V_CMP_EQ_F32_e64,
1394 AMDGPU::V_CMP_EQ_F64_e64);
1395 case CmpInst::FCMP_OGT:
1396 return Select(AMDGPU::V_CMP_GT_F16_e64, AMDGPU::V_CMP_GT_F16_t16_e64,
1397 AMDGPU::V_CMP_GT_F16_fake16_e64, AMDGPU::V_CMP_GT_F32_e64,
1398 AMDGPU::V_CMP_GT_F64_e64);
1399 case CmpInst::FCMP_OGE:
1400 return Select(AMDGPU::V_CMP_GE_F16_e64, AMDGPU::V_CMP_GE_F16_t16_e64,
1401 AMDGPU::V_CMP_GE_F16_fake16_e64, AMDGPU::V_CMP_GE_F32_e64,
1402 AMDGPU::V_CMP_GE_F64_e64);
1403 case CmpInst::FCMP_OLT:
1404 return Select(AMDGPU::V_CMP_LT_F16_e64, AMDGPU::V_CMP_LT_F16_t16_e64,
1405 AMDGPU::V_CMP_LT_F16_fake16_e64, AMDGPU::V_CMP_LT_F32_e64,
1406 AMDGPU::V_CMP_LT_F64_e64);
1407 case CmpInst::FCMP_OLE:
1408 return Select(AMDGPU::V_CMP_LE_F16_e64, AMDGPU::V_CMP_LE_F16_t16_e64,
1409 AMDGPU::V_CMP_LE_F16_fake16_e64, AMDGPU::V_CMP_LE_F32_e64,
1410 AMDGPU::V_CMP_LE_F64_e64);
1411 case CmpInst::FCMP_ONE:
1412 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1413 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1414 AMDGPU::V_CMP_NEQ_F64_e64);
1415 case CmpInst::FCMP_ORD:
1416 return Select(AMDGPU::V_CMP_O_F16_e64, AMDGPU::V_CMP_O_F16_t16_e64,
1417 AMDGPU::V_CMP_O_F16_fake16_e64, AMDGPU::V_CMP_O_F32_e64,
1418 AMDGPU::V_CMP_O_F64_e64);
1419 case CmpInst::FCMP_UNO:
1420 return Select(AMDGPU::V_CMP_U_F16_e64, AMDGPU::V_CMP_U_F16_t16_e64,
1421 AMDGPU::V_CMP_U_F16_fake16_e64, AMDGPU::V_CMP_U_F32_e64,
1422 AMDGPU::V_CMP_U_F64_e64);
1423 case CmpInst::FCMP_UEQ:
1424 return Select(AMDGPU::V_CMP_NLG_F16_e64, AMDGPU::V_CMP_NLG_F16_t16_e64,
1425 AMDGPU::V_CMP_NLG_F16_fake16_e64, AMDGPU::V_CMP_NLG_F32_e64,
1426 AMDGPU::V_CMP_NLG_F64_e64);
1427 case CmpInst::FCMP_UGT:
1428 return Select(AMDGPU::V_CMP_NLE_F16_e64, AMDGPU::V_CMP_NLE_F16_t16_e64,
1429 AMDGPU::V_CMP_NLE_F16_fake16_e64, AMDGPU::V_CMP_NLE_F32_e64,
1430 AMDGPU::V_CMP_NLE_F64_e64);
1431 case CmpInst::FCMP_UGE:
1432 return Select(AMDGPU::V_CMP_NLT_F16_e64, AMDGPU::V_CMP_NLT_F16_t16_e64,
1433 AMDGPU::V_CMP_NLT_F16_fake16_e64, AMDGPU::V_CMP_NLT_F32_e64,
1434 AMDGPU::V_CMP_NLT_F64_e64);
1435 case CmpInst::FCMP_ULT:
1436 return Select(AMDGPU::V_CMP_NGE_F16_e64, AMDGPU::V_CMP_NGE_F16_t16_e64,
1437 AMDGPU::V_CMP_NGE_F16_fake16_e64, AMDGPU::V_CMP_NGE_F32_e64,
1438 AMDGPU::V_CMP_NGE_F64_e64);
1439 case CmpInst::FCMP_ULE:
1440 return Select(AMDGPU::V_CMP_NGT_F16_e64, AMDGPU::V_CMP_NGT_F16_t16_e64,
1441 AMDGPU::V_CMP_NGT_F16_fake16_e64, AMDGPU::V_CMP_NGT_F32_e64,
1442 AMDGPU::V_CMP_NGT_F64_e64);
1443 case CmpInst::FCMP_UNE:
1444 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1445 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1446 AMDGPU::V_CMP_NEQ_F64_e64);
1447 case CmpInst::FCMP_TRUE:
1448 return Select(AMDGPU::V_CMP_TRU_F16_e64, AMDGPU::V_CMP_TRU_F16_t16_e64,
1449 AMDGPU::V_CMP_TRU_F16_fake16_e64, AMDGPU::V_CMP_TRU_F32_e64,
1450 AMDGPU::V_CMP_TRU_F64_e64);
1451 case CmpInst::FCMP_FALSE:
1452 return Select(AMDGPU::V_CMP_F_F16_e64, AMDGPU::V_CMP_F_F16_t16_e64,
1453 AMDGPU::V_CMP_F_F16_fake16_e64, AMDGPU::V_CMP_F_F32_e64,
1454 AMDGPU::V_CMP_F_F64_e64);
1455 }
1456}
1457
1458int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
1459 unsigned Size) const {
1460 if (Size == 64) {
1461 if (!STI.hasScalarCompareEq64())
1462 return -1;
1463
1464 switch (P) {
1465 case CmpInst::ICMP_NE:
1466 return AMDGPU::S_CMP_LG_U64;
1467 case CmpInst::ICMP_EQ:
1468 return AMDGPU::S_CMP_EQ_U64;
1469 default:
1470 return -1;
1471 }
1472 }
1473
1474 if (Size == 32) {
1475 switch (P) {
1476 case CmpInst::ICMP_NE:
1477 return AMDGPU::S_CMP_LG_U32;
1478 case CmpInst::ICMP_EQ:
1479 return AMDGPU::S_CMP_EQ_U32;
1480 case CmpInst::ICMP_SGT:
1481 return AMDGPU::S_CMP_GT_I32;
1482 case CmpInst::ICMP_SGE:
1483 return AMDGPU::S_CMP_GE_I32;
1484 case CmpInst::ICMP_SLT:
1485 return AMDGPU::S_CMP_LT_I32;
1486 case CmpInst::ICMP_SLE:
1487 return AMDGPU::S_CMP_LE_I32;
1488 case CmpInst::ICMP_UGT:
1489 return AMDGPU::S_CMP_GT_U32;
1490 case CmpInst::ICMP_UGE:
1491 return AMDGPU::S_CMP_GE_U32;
1492 case CmpInst::ICMP_ULT:
1493 return AMDGPU::S_CMP_LT_U32;
1494 case CmpInst::ICMP_ULE:
1495 return AMDGPU::S_CMP_LE_U32;
1496 case CmpInst::FCMP_OEQ:
1497 return AMDGPU::S_CMP_EQ_F32;
1498 case CmpInst::FCMP_OGT:
1499 return AMDGPU::S_CMP_GT_F32;
1500 case CmpInst::FCMP_OGE:
1501 return AMDGPU::S_CMP_GE_F32;
1502 case CmpInst::FCMP_OLT:
1503 return AMDGPU::S_CMP_LT_F32;
1504 case CmpInst::FCMP_OLE:
1505 return AMDGPU::S_CMP_LE_F32;
1506 case CmpInst::FCMP_ONE:
1507 return AMDGPU::S_CMP_LG_F32;
1508 case CmpInst::FCMP_ORD:
1509 return AMDGPU::S_CMP_O_F32;
1510 case CmpInst::FCMP_UNO:
1511 return AMDGPU::S_CMP_U_F32;
1512 case CmpInst::FCMP_UEQ:
1513 return AMDGPU::S_CMP_NLG_F32;
1514 case CmpInst::FCMP_UGT:
1515 return AMDGPU::S_CMP_NLE_F32;
1516 case CmpInst::FCMP_UGE:
1517 return AMDGPU::S_CMP_NLT_F32;
1518 case CmpInst::FCMP_ULT:
1519 return AMDGPU::S_CMP_NGE_F32;
1520 case CmpInst::FCMP_ULE:
1521 return AMDGPU::S_CMP_NGT_F32;
1522 case CmpInst::FCMP_UNE:
1523 return AMDGPU::S_CMP_NEQ_F32;
1524 default:
1525 llvm_unreachable("Unknown condition code!");
1526 }
1527 }
1528
1529 if (Size == 16) {
1530 if (!STI.hasSALUFloatInsts())
1531 return -1;
1532
1533 switch (P) {
1534 case CmpInst::FCMP_OEQ:
1535 return AMDGPU::S_CMP_EQ_F16;
1536 case CmpInst::FCMP_OGT:
1537 return AMDGPU::S_CMP_GT_F16;
1538 case CmpInst::FCMP_OGE:
1539 return AMDGPU::S_CMP_GE_F16;
1540 case CmpInst::FCMP_OLT:
1541 return AMDGPU::S_CMP_LT_F16;
1542 case CmpInst::FCMP_OLE:
1543 return AMDGPU::S_CMP_LE_F16;
1544 case CmpInst::FCMP_ONE:
1545 return AMDGPU::S_CMP_LG_F16;
1546 case CmpInst::FCMP_ORD:
1547 return AMDGPU::S_CMP_O_F16;
1548 case CmpInst::FCMP_UNO:
1549 return AMDGPU::S_CMP_U_F16;
1550 case CmpInst::FCMP_UEQ:
1551 return AMDGPU::S_CMP_NLG_F16;
1552 case CmpInst::FCMP_UGT:
1553 return AMDGPU::S_CMP_NLE_F16;
1554 case CmpInst::FCMP_UGE:
1555 return AMDGPU::S_CMP_NLT_F16;
1556 case CmpInst::FCMP_ULT:
1557 return AMDGPU::S_CMP_NGE_F16;
1558 case CmpInst::FCMP_ULE:
1559 return AMDGPU::S_CMP_NGT_F16;
1560 case CmpInst::FCMP_UNE:
1561 return AMDGPU::S_CMP_NEQ_F16;
1562 default:
1563 llvm_unreachable("Unknown condition code!");
1564 }
1565 }
1566
1567 return -1;
1568}
1569
1570bool AMDGPUInstructionSelector::selectG_ICMP_or_FCMP(MachineInstr &I) const {
1571
1572 MachineBasicBlock *BB = I.getParent();
1573 const DebugLoc &DL = I.getDebugLoc();
1574
1575 Register SrcReg = I.getOperand(i: 2).getReg();
1576 unsigned Size = RBI.getSizeInBits(Reg: SrcReg, MRI: *MRI, TRI);
1577
1578 auto Pred = (CmpInst::Predicate)I.getOperand(i: 1).getPredicate();
1579
1580 Register CCReg = I.getOperand(i: 0).getReg();
1581 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
1582 int Opcode = getS_CMPOpcode(P: Pred, Size);
1583 if (Opcode == -1)
1584 return false;
1585 MachineInstr *ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode))
1586 .add(MO: I.getOperand(i: 2))
1587 .add(MO: I.getOperand(i: 3));
1588 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg)
1589 .addReg(RegNo: AMDGPU::SCC);
1590 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1591 bool Ret =
1592 RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
1593 I.eraseFromParent();
1594 return Ret;
1595 }
1596
1597 if (I.getOpcode() == AMDGPU::G_FCMP)
1598 return false;
1599
1600 int Opcode = getV_CMPOpcode(P: Pred, Size, ST: *Subtarget);
1601 if (Opcode == -1)
1602 return false;
1603
1604 MachineInstrBuilder ICmp;
1605 // t16 instructions
1606 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
1607 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1608 .addImm(Val: 0)
1609 .add(MO: I.getOperand(i: 2))
1610 .addImm(Val: 0)
1611 .add(MO: I.getOperand(i: 3))
1612 .addImm(Val: 0); // op_sel
1613 } else {
1614 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1615 .add(MO: I.getOperand(i: 2))
1616 .add(MO: I.getOperand(i: 3));
1617 }
1618
1619 RBI.constrainGenericRegister(Reg: ICmp->getOperand(i: 0).getReg(),
1620 RC: *TRI.getBoolRC(), MRI&: *MRI);
1621 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1622 I.eraseFromParent();
1623 return true;
1624}
1625
1626// Ballot has to zero bits in input lane-mask that are zero in current exec,
1627// Done as AND with exec. For inputs that are results of instruction that
1628// implicitly use same exec, for example compares in same basic block or SCC to
1629// VCC copy, use copy.
1630static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI,
1631 MachineBasicBlock *MBB) {
1632 MachineInstr *MI = MRI.getVRegDef(Reg);
1633 if (MI->getParent() != MBB)
1634 return false;
1635
1636 // Lane mask generated by SCC to VCC copy.
1637 if (MI->getOpcode() == AMDGPU::COPY) {
1638 auto DstRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 0).getReg());
1639 auto SrcRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 1).getReg());
1640 if (DstRB && SrcRB && DstRB->getID() == AMDGPU::VCCRegBankID &&
1641 SrcRB->getID() == AMDGPU::SGPRRegBankID)
1642 return true;
1643 }
1644
1645 // Lane mask generated by SCC to VCC copy
1646 if (MI->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
1647 return true;
1648
1649 // Lane mask generated using compare with same exec.
1650 if (isa<GAnyCmp>(Val: MI))
1651 return true;
1652
1653 Register LHS, RHS;
1654 // Look through AND.
1655 if (mi_match(R: Reg, MRI, P: m_GAnd(L: m_Reg(R&: LHS), R: m_Reg(R&: RHS))))
1656 return isLaneMaskFromSameBlock(Reg: LHS, MRI, MBB) ||
1657 isLaneMaskFromSameBlock(Reg: RHS, MRI, MBB);
1658
1659 return false;
1660}
1661
1662bool AMDGPUInstructionSelector::selectBallot(MachineInstr &I) const {
1663 MachineBasicBlock *BB = I.getParent();
1664 const DebugLoc &DL = I.getDebugLoc();
1665 Register DstReg = I.getOperand(i: 0).getReg();
1666 Register SrcReg = I.getOperand(i: 2).getReg();
1667 const unsigned BallotSize = MRI->getType(Reg: DstReg).getSizeInBits();
1668 const unsigned WaveSize = STI.getWavefrontSize();
1669
1670 // In the common case, the return type matches the wave size.
1671 // However we also support emitting i64 ballots in wave32 mode.
1672 if (BallotSize != WaveSize && (BallotSize != 64 || WaveSize != 32))
1673 return false;
1674
1675 std::optional<ValueAndVReg> Arg =
1676 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI);
1677
1678 Register Dst = DstReg;
1679 // i64 ballot on Wave32: new Dst(i32) for WaveSize ballot.
1680 if (BallotSize != WaveSize) {
1681 Dst = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
1682 }
1683
1684 if (Arg) {
1685 const int64_t Value = Arg->Value.getZExtValue();
1686 if (Value == 0) {
1687 // Dst = S_MOV 0
1688 unsigned Opcode = WaveSize == 64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
1689 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: Dst).addImm(Val: 0);
1690 } else {
1691 // Dst = COPY EXEC
1692 assert(Value == 1);
1693 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: TRI.getExec());
1694 }
1695 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1696 return false;
1697 } else {
1698 if (isLaneMaskFromSameBlock(Reg: SrcReg, MRI&: *MRI, MBB: BB)) {
1699 // Dst = COPY SrcReg
1700 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: SrcReg);
1701 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1702 return false;
1703 } else {
1704 // Dst = S_AND SrcReg, EXEC
1705 unsigned AndOpc = WaveSize == 64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
1706 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: Dst)
1707 .addReg(RegNo: SrcReg)
1708 .addReg(RegNo: TRI.getExec())
1709 .setOperandDead(3); // Dead scc
1710 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
1711 }
1712 }
1713
1714 // i64 ballot on Wave32: zero-extend i32 ballot to i64.
1715 if (BallotSize != WaveSize) {
1716 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1717 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg).addImm(Val: 0);
1718 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
1719 .addReg(RegNo: Dst)
1720 .addImm(Val: AMDGPU::sub0)
1721 .addReg(RegNo: HiReg)
1722 .addImm(Val: AMDGPU::sub1);
1723 }
1724
1725 I.eraseFromParent();
1726 return true;
1727}
1728
1729bool AMDGPUInstructionSelector::selectRelocConstant(MachineInstr &I) const {
1730 Register DstReg = I.getOperand(i: 0).getReg();
1731 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1732 const TargetRegisterClass *DstRC = TRI.getRegClassForSizeOnBank(Size: 32, Bank: *DstBank);
1733 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1734 return false;
1735
1736 const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID;
1737
1738 Module *M = MF->getFunction().getParent();
1739 const MDNode *Metadata = I.getOperand(i: 2).getMetadata();
1740 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
1741 auto *RelocSymbol = cast<GlobalVariable>(
1742 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
1743
1744 MachineBasicBlock *BB = I.getParent();
1745 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(),
1746 MCID: TII.get(Opcode: IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DestReg: DstReg)
1747 .addGlobalAddress(GV: RelocSymbol, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1748
1749 I.eraseFromParent();
1750 return true;
1751}
1752
1753bool AMDGPUInstructionSelector::selectGroupStaticSize(MachineInstr &I) const {
1754 Triple::OSType OS = MF->getTarget().getTargetTriple().getOS();
1755
1756 Register DstReg = I.getOperand(i: 0).getReg();
1757 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1758 unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ?
1759 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1760
1761 MachineBasicBlock *MBB = I.getParent();
1762 const DebugLoc &DL = I.getDebugLoc();
1763
1764 auto MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Mov), DestReg: DstReg);
1765
1766 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
1767 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1768 MIB.addImm(Val: MFI->getLDSSize());
1769 } else {
1770 Module *M = MF->getFunction().getParent();
1771 const GlobalValue *GV =
1772 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::amdgcn_groupstaticsize);
1773 MIB.addGlobalAddress(GV, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1774 }
1775
1776 I.eraseFromParent();
1777 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1778 return true;
1779}
1780
1781bool AMDGPUInstructionSelector::selectReturnAddress(MachineInstr &I) const {
1782 MachineBasicBlock *MBB = I.getParent();
1783 MachineFunction &MF = *MBB->getParent();
1784 const DebugLoc &DL = I.getDebugLoc();
1785
1786 Register DstReg = I.getOperand(i: 0).getReg();
1787 unsigned Depth = I.getOperand(i: 2).getImm();
1788
1789 const TargetRegisterClass *RC =
1790 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
1791 if (!RC->hasSubClassEq(RC: &AMDGPU::SGPR_64RegClass) ||
1792 !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
1793 return false;
1794
1795 // Check for kernel and shader functions
1796 if (Depth != 0 ||
1797 MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
1798 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg)
1799 .addImm(Val: 0);
1800 I.eraseFromParent();
1801 return true;
1802 }
1803
1804 MachineFrameInfo &MFI = MF.getFrameInfo();
1805 // There is a call to @llvm.returnaddress in this function
1806 MFI.setReturnAddressIsTaken(true);
1807
1808 // Get the return address reg and mark it as an implicit live-in
1809 Register ReturnAddrReg = TRI.getReturnAddressReg(MF);
1810 Register LiveIn = getFunctionLiveInPhysReg(MF, TII, PhysReg: ReturnAddrReg,
1811 RC: AMDGPU::SReg_64RegClass, DL);
1812 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
1813 .addReg(RegNo: LiveIn);
1814 I.eraseFromParent();
1815 return true;
1816}
1817
1818bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const {
1819 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1820 // SelectionDAG uses for wave32 vs wave64.
1821 MachineBasicBlock *BB = MI.getParent();
1822 BuildMI(BB&: *BB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_END_CF))
1823 .add(MO: MI.getOperand(i: 1))
1824 .setOperandDead(2); // implicit-def $scc
1825
1826 Register Reg = MI.getOperand(i: 1).getReg();
1827 MI.eraseFromParent();
1828
1829 if (!MRI->getRegClassOrNull(Reg))
1830 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1831 return true;
1832}
1833
1834bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic(
1835 MachineInstr &MI, Intrinsic::ID IntrID) const {
1836 MachineBasicBlock *MBB = MI.getParent();
1837 MachineFunction *MF = MBB->getParent();
1838 const DebugLoc &DL = MI.getDebugLoc();
1839
1840 unsigned IndexOperand = MI.getOperand(i: 7).getImm();
1841 bool WaveRelease = MI.getOperand(i: 8).getImm() != 0;
1842 bool WaveDone = MI.getOperand(i: 9).getImm() != 0;
1843
1844 if (WaveDone && !WaveRelease) {
1845 // TODO: Move this to IR verifier
1846 const Function &Fn = MF->getFunction();
1847 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1848 Fn, "ds_ordered_count: wave_done requires wave_release", DL));
1849 }
1850
1851 unsigned OrderedCountIndex = IndexOperand & 0x3f;
1852 IndexOperand &= ~0x3f;
1853 unsigned CountDw = 0;
1854
1855 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) {
1856 CountDw = (IndexOperand >> 24) & 0xf;
1857 IndexOperand &= ~(0xf << 24);
1858
1859 if (CountDw < 1 || CountDw > 4) {
1860 const Function &Fn = MF->getFunction();
1861 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1862 Fn, "ds_ordered_count: dword count must be between 1 and 4", DL));
1863 CountDw = 1;
1864 }
1865 }
1866
1867 if (IndexOperand) {
1868 const Function &Fn = MF->getFunction();
1869 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1870 Fn, "ds_ordered_count: bad index operand", DL));
1871 }
1872
1873 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
1874 unsigned ShaderType = SIInstrInfo::getDSShaderTypeValue(MF: *MF);
1875
1876 unsigned Offset0 = OrderedCountIndex << 2;
1877 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
1878
1879 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10)
1880 Offset1 |= (CountDw - 1) << 6;
1881
1882 if (STI.getGeneration() < AMDGPUSubtarget::GFX11)
1883 Offset1 |= ShaderType << 2;
1884
1885 unsigned Offset = Offset0 | (Offset1 << 8);
1886
1887 Register M0Val = MI.getOperand(i: 2).getReg();
1888 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1889 .addReg(RegNo: M0Val);
1890
1891 Register DstReg = MI.getOperand(i: 0).getReg();
1892 Register ValReg = MI.getOperand(i: 3).getReg();
1893 MachineInstrBuilder DS =
1894 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_ORDERED_COUNT), DestReg: DstReg)
1895 .addReg(RegNo: ValReg)
1896 .addImm(Val: Offset)
1897 .cloneMemRefs(OtherMI: MI);
1898
1899 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
1900 return false;
1901
1902 constrainSelectedInstRegOperands(I&: *DS, TII, TRI, RBI);
1903 MI.eraseFromParent();
1904 return true;
1905}
1906
1907static unsigned gwsIntrinToOpcode(unsigned IntrID) {
1908 switch (IntrID) {
1909 case Intrinsic::amdgcn_ds_gws_init:
1910 return AMDGPU::DS_GWS_INIT;
1911 case Intrinsic::amdgcn_ds_gws_barrier:
1912 return AMDGPU::DS_GWS_BARRIER;
1913 case Intrinsic::amdgcn_ds_gws_sema_v:
1914 return AMDGPU::DS_GWS_SEMA_V;
1915 case Intrinsic::amdgcn_ds_gws_sema_br:
1916 return AMDGPU::DS_GWS_SEMA_BR;
1917 case Intrinsic::amdgcn_ds_gws_sema_p:
1918 return AMDGPU::DS_GWS_SEMA_P;
1919 case Intrinsic::amdgcn_ds_gws_sema_release_all:
1920 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
1921 default:
1922 llvm_unreachable("not a gws intrinsic");
1923 }
1924}
1925
1926bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
1927 Intrinsic::ID IID) const {
1928 if (!STI.hasGWS() || (IID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
1929 !STI.hasGWSSemaReleaseAll()))
1930 return false;
1931
1932 // intrinsic ID, vsrc, offset
1933 const bool HasVSrc = MI.getNumOperands() == 3;
1934 assert(HasVSrc || MI.getNumOperands() == 2);
1935
1936 Register BaseOffset = MI.getOperand(i: HasVSrc ? 2 : 1).getReg();
1937 const RegisterBank *OffsetRB = RBI.getRegBank(Reg: BaseOffset, MRI: *MRI, TRI);
1938 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID)
1939 return false;
1940
1941 MachineInstr *OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
1942 unsigned ImmOffset;
1943
1944 MachineBasicBlock *MBB = MI.getParent();
1945 const DebugLoc &DL = MI.getDebugLoc();
1946
1947 MachineInstr *Readfirstlane = nullptr;
1948
1949 // If we legalized the VGPR input, strip out the readfirstlane to analyze the
1950 // incoming offset, in case there's an add of a constant. We'll have to put it
1951 // back later.
1952 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) {
1953 Readfirstlane = OffsetDef;
1954 BaseOffset = OffsetDef->getOperand(i: 1).getReg();
1955 OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
1956 }
1957
1958 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) {
1959 // If we have a constant offset, try to use the 0 in m0 as the base.
1960 // TODO: Look into changing the default m0 initialization value. If the
1961 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
1962 // the immediate offset.
1963
1964 ImmOffset = OffsetDef->getOperand(i: 1).getCImm()->getZExtValue();
1965 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
1966 .addImm(Val: 0);
1967 } else {
1968 std::tie(args&: BaseOffset, args&: ImmOffset) =
1969 AMDGPU::getBaseWithConstantOffset(MRI&: *MRI, Reg: BaseOffset, ValueTracking: VT);
1970
1971 if (Readfirstlane) {
1972 // We have the constant offset now, so put the readfirstlane back on the
1973 // variable component.
1974 if (!RBI.constrainGenericRegister(Reg: BaseOffset, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
1975 return false;
1976
1977 Readfirstlane->getOperand(i: 1).setReg(BaseOffset);
1978 BaseOffset = Readfirstlane->getOperand(i: 0).getReg();
1979 } else {
1980 if (!RBI.constrainGenericRegister(Reg: BaseOffset,
1981 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
1982 return false;
1983 }
1984
1985 Register M0Base = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1986 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: M0Base)
1987 .addReg(RegNo: BaseOffset)
1988 .addImm(Val: 16)
1989 .setOperandDead(3); // Dead scc
1990
1991 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1992 .addReg(RegNo: M0Base);
1993 }
1994
1995 // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
1996 // offset field) % 64. Some versions of the programming guide omit the m0
1997 // part, or claim it's from offset 0.
1998
1999 unsigned Opc = gwsIntrinToOpcode(IntrID: IID);
2000 const MCInstrDesc &InstrDesc = TII.get(Opcode: Opc);
2001
2002 if (HasVSrc) {
2003 Register VSrc = MI.getOperand(i: 1).getReg();
2004
2005 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
2006 const TargetRegisterClass *DataRC = TII.getRegClass(MCID: InstrDesc, OpNum: Data0Idx);
2007 const TargetRegisterClass *SubRC =
2008 TRI.getSubRegisterClass(DataRC, AMDGPU::sub0);
2009
2010 if (!SubRC) {
2011 // 32-bit normal case.
2012 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *DataRC, MRI&: *MRI))
2013 return false;
2014
2015 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2016 .addReg(RegNo: VSrc)
2017 .addImm(Val: ImmOffset)
2018 .cloneMemRefs(OtherMI: MI);
2019 } else {
2020 // Requires even register alignment, so create 64-bit value and pad the
2021 // top half with undef.
2022 Register DataReg = MRI->createVirtualRegister(RegClass: DataRC);
2023 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *SubRC, MRI&: *MRI))
2024 return false;
2025
2026 Register UndefReg = MRI->createVirtualRegister(RegClass: SubRC);
2027 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2028 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DataReg)
2029 .addReg(RegNo: VSrc)
2030 .addImm(Val: AMDGPU::sub0)
2031 .addReg(RegNo: UndefReg)
2032 .addImm(Val: AMDGPU::sub1);
2033
2034 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2035 .addReg(RegNo: DataReg)
2036 .addImm(Val: ImmOffset)
2037 .cloneMemRefs(OtherMI: MI);
2038 }
2039 } else {
2040 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2041 .addImm(Val: ImmOffset)
2042 .cloneMemRefs(OtherMI: MI);
2043 }
2044
2045 MI.eraseFromParent();
2046 return true;
2047}
2048
2049bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
2050 bool IsAppend) const {
2051 Register PtrBase = MI.getOperand(i: 2).getReg();
2052 LLT PtrTy = MRI->getType(Reg: PtrBase);
2053 bool IsGDS = PtrTy.getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2054
2055 unsigned Offset;
2056 std::tie(args&: PtrBase, args&: Offset) = selectDS1Addr1OffsetImpl(Root&: MI.getOperand(i: 2));
2057
2058 // TODO: Should this try to look through readfirstlane like GWS?
2059 if (!isDSOffsetLegal(Base: PtrBase, Offset)) {
2060 PtrBase = MI.getOperand(i: 2).getReg();
2061 Offset = 0;
2062 }
2063
2064 MachineBasicBlock *MBB = MI.getParent();
2065 const DebugLoc &DL = MI.getDebugLoc();
2066 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2067
2068 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
2069 .addReg(RegNo: PtrBase);
2070 if (!RBI.constrainGenericRegister(Reg: PtrBase, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
2071 return false;
2072
2073 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: MI.getOperand(i: 0).getReg())
2074 .addImm(Val: Offset)
2075 .addImm(Val: IsGDS ? -1 : 0)
2076 .cloneMemRefs(OtherMI: MI);
2077 MI.eraseFromParent();
2078 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2079 return true;
2080}
2081
2082bool AMDGPUInstructionSelector::selectInitWholeWave(MachineInstr &MI) const {
2083 MachineFunction *MF = MI.getMF();
2084 SIMachineFunctionInfo *MFInfo = MF->getInfo<SIMachineFunctionInfo>();
2085
2086 MFInfo->setInitWholeWave();
2087 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
2088}
2089
2090static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE,
2091 bool &IsTexFail) {
2092 if (TexFailCtrl)
2093 IsTexFail = true;
2094
2095 TFE = TexFailCtrl & 0x1;
2096 TexFailCtrl &= ~(uint64_t)0x1;
2097 LWE = TexFailCtrl & 0x2;
2098 TexFailCtrl &= ~(uint64_t)0x2;
2099
2100 return TexFailCtrl == 0;
2101}
2102
2103bool AMDGPUInstructionSelector::selectImageIntrinsic(
2104 MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
2105 MachineBasicBlock *MBB = MI.getParent();
2106 const DebugLoc &DL = MI.getDebugLoc();
2107 unsigned IntrOpcode = Intr->BaseOpcode;
2108
2109 // For image atomic: use no-return opcode if result is unused.
2110 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode) {
2111 Register ResultDef = MI.getOperand(i: 0).getReg();
2112 if (MRI->use_nodbg_empty(RegNo: ResultDef))
2113 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
2114 }
2115
2116 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
2117 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
2118
2119 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
2120 const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI);
2121 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI);
2122 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
2123 const bool IsGFX13Plus = AMDGPU::isGFX13Plus(STI);
2124
2125 const unsigned ArgOffset = MI.getNumExplicitDefs() + 1;
2126
2127 Register VDataIn = AMDGPU::NoRegister;
2128 Register VDataOut = AMDGPU::NoRegister;
2129 LLT VDataTy;
2130 int NumVDataDwords = -1;
2131 bool IsD16 = MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 ||
2132 MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16;
2133
2134 bool Unorm;
2135 if (!BaseOpcode->Sampler)
2136 Unorm = true;
2137 else
2138 Unorm = MI.getOperand(i: ArgOffset + Intr->UnormIndex).getImm() != 0;
2139
2140 bool TFE;
2141 bool LWE;
2142 bool IsTexFail = false;
2143 if (!parseTexFail(TexFailCtrl: MI.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex).getImm(),
2144 TFE, LWE, IsTexFail))
2145 return false;
2146
2147 const int Flags = MI.getOperand(i: ArgOffset + Intr->NumArgs).getImm();
2148 const bool IsA16 = (Flags & 1) != 0;
2149 const bool IsG16 = (Flags & 2) != 0;
2150
2151 // A16 implies 16 bit gradients if subtarget doesn't support G16
2152 if (IsA16 && !STI.hasG16() && !IsG16)
2153 return false;
2154
2155 unsigned DMask = 0;
2156 unsigned DMaskLanes = 0;
2157
2158 if (BaseOpcode->Atomic) {
2159 if (!BaseOpcode->NoReturn)
2160 VDataOut = MI.getOperand(i: 0).getReg();
2161 VDataIn = MI.getOperand(i: 2).getReg();
2162 LLT Ty = MRI->getType(Reg: VDataIn);
2163
2164 // Be careful to allow atomic swap on 16-bit element vectors.
2165 const bool Is64Bit = BaseOpcode->AtomicX2 ?
2166 Ty.getSizeInBits() == 128 :
2167 Ty.getSizeInBits() == 64;
2168
2169 if (BaseOpcode->AtomicX2) {
2170 assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister);
2171
2172 DMask = Is64Bit ? 0xf : 0x3;
2173 NumVDataDwords = Is64Bit ? 4 : 2;
2174 } else {
2175 DMask = Is64Bit ? 0x3 : 0x1;
2176 NumVDataDwords = Is64Bit ? 2 : 1;
2177 }
2178 } else {
2179 DMask = MI.getOperand(i: ArgOffset + Intr->DMaskIndex).getImm();
2180 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
2181
2182 if (BaseOpcode->Store) {
2183 VDataIn = MI.getOperand(i: 1).getReg();
2184 VDataTy = MRI->getType(Reg: VDataIn);
2185 NumVDataDwords = (VDataTy.getSizeInBits() + 31) / 32;
2186 } else if (BaseOpcode->NoReturn) {
2187 NumVDataDwords = 0;
2188 } else {
2189 VDataOut = MI.getOperand(i: 0).getReg();
2190 VDataTy = MRI->getType(Reg: VDataOut);
2191 NumVDataDwords = DMaskLanes;
2192
2193 if (IsD16 && !STI.hasUnpackedD16VMem())
2194 NumVDataDwords = (DMaskLanes + 1) / 2;
2195 }
2196 }
2197
2198 // Set G16 opcode
2199 if (Subtarget->hasG16() && IsG16) {
2200 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
2201 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
2202 assert(G16MappingInfo);
2203 IntrOpcode = G16MappingInfo->G16; // set opcode to variant with _g16
2204 }
2205
2206 // TODO: Check this in verifier.
2207 assert((!IsTexFail || DMaskLanes >= 1) && "should have legalized this");
2208
2209 unsigned CPol = MI.getOperand(i: ArgOffset + Intr->CachePolicyIndex).getImm();
2210 // Keep GLC only when the atomic's result is actually used.
2211 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
2212 CPol |= AMDGPU::CPol::GLC;
2213 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
2214 AMDGPU::CPol::VOLATILE))
2215 return false;
2216
2217 int NumVAddrRegs = 0;
2218 int NumVAddrDwords = 0;
2219 for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
2220 // Skip the $noregs and 0s inserted during legalization.
2221 MachineOperand &AddrOp = MI.getOperand(i: ArgOffset + I);
2222 if (!AddrOp.isReg())
2223 continue; // XXX - Break?
2224
2225 Register Addr = AddrOp.getReg();
2226 if (!Addr)
2227 break;
2228
2229 ++NumVAddrRegs;
2230 NumVAddrDwords += (MRI->getType(Reg: Addr).getSizeInBits() + 31) / 32;
2231 }
2232
2233 // The legalizer preprocessed the intrinsic arguments. If we aren't using
2234 // NSA, these should have been packed into a single value in the first
2235 // address register
2236 const bool UseNSA =
2237 NumVAddrRegs != 1 &&
2238 (STI.hasPartialNSAEncoding() ? NumVAddrDwords >= NumVAddrRegs
2239 : NumVAddrDwords == NumVAddrRegs);
2240 if (UseNSA && !STI.hasFeature(Feature: AMDGPU::FeatureNSAEncoding)) {
2241 LLVM_DEBUG(dbgs() << "Trying to use NSA on non-NSA target\n");
2242 return false;
2243 }
2244
2245 if (IsTexFail)
2246 ++NumVDataDwords;
2247
2248 int Opcode = -1;
2249 if (IsGFX13Plus) {
2250 Register RsrcReg = MI.getOperand(i: ArgOffset + Intr->RsrcIndex).getReg();
2251 const LLT I32 = LLT::integer(SizeInBits: 32);
2252 bool IndexedRsrc = MRI->getType(Reg: RsrcReg) == I32;
2253 bool IndexedSamp = false;
2254 if (BaseOpcode->Sampler) {
2255 Register SampReg = MI.getOperand(i: ArgOffset + Intr->SampIndex).getReg();
2256 IndexedSamp = MRI->getType(Reg: SampReg) == I32;
2257 }
2258 Opcode =
2259 AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx13, VDataDwords: NumVDataDwords,
2260 VAddrDwords: NumVAddrDwords, IndexedRsrc, IndexedSamp);
2261 } else if (IsGFX12Plus) {
2262 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
2263 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2264 } else if (IsGFX11Plus) {
2265 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2266 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
2267 : AMDGPU::MIMGEncGfx11Default,
2268 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2269 } else if (IsGFX10Plus) {
2270 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2271 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
2272 : AMDGPU::MIMGEncGfx10Default,
2273 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2274 } else {
2275 if (Subtarget->hasGFX90AInsts()) {
2276 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
2277 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2278 if (Opcode == -1) {
2279 LLVM_DEBUG(
2280 dbgs()
2281 << "requested image instruction is not supported on this GPU\n");
2282 return false;
2283 }
2284 }
2285 if (Opcode == -1 &&
2286 STI.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
2287 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
2288 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2289 if (Opcode == -1)
2290 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
2291 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2292 }
2293 if (Opcode == -1)
2294 return false;
2295
2296 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode))
2297 .cloneMemRefs(OtherMI: MI);
2298
2299 if (VDataOut) {
2300 if (BaseOpcode->AtomicX2) {
2301 const bool Is64 = MRI->getType(Reg: VDataOut).getSizeInBits() == 64;
2302
2303 Register TmpReg = MRI->createVirtualRegister(
2304 RegClass: Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass);
2305 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0;
2306
2307 MIB.addDef(RegNo: TmpReg);
2308 if (!MRI->use_empty(RegNo: VDataOut)) {
2309 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VDataOut)
2310 .addReg(RegNo: TmpReg, Flags: {}, SubReg);
2311 }
2312
2313 } else {
2314 MIB.addDef(RegNo: VDataOut); // vdata output
2315 }
2316 }
2317
2318 if (VDataIn)
2319 MIB.addReg(RegNo: VDataIn); // vdata input
2320
2321 for (int I = 0; I != NumVAddrRegs; ++I) {
2322 MachineOperand &SrcOp = MI.getOperand(i: ArgOffset + Intr->VAddrStart + I);
2323 if (SrcOp.isReg()) {
2324 assert(SrcOp.getReg() != 0);
2325 MIB.addReg(RegNo: SrcOp.getReg());
2326 }
2327 }
2328
2329 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->RsrcIndex).getReg());
2330 if (BaseOpcode->Sampler)
2331 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->SampIndex).getReg());
2332
2333 MIB.addImm(Val: DMask); // dmask
2334
2335 if (IsGFX10Plus)
2336 MIB.addImm(Val: DimInfo->Encoding);
2337 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::unorm))
2338 MIB.addImm(Val: Unorm);
2339
2340 MIB.addImm(Val: CPol);
2341 MIB.addImm(Val: IsA16 && // a16 or r128
2342 STI.hasFeature(Feature: AMDGPU::FeatureR128A16) ? -1 : 0);
2343 if (IsGFX10Plus)
2344 MIB.addImm(Val: IsA16 ? -1 : 0);
2345
2346 if (!Subtarget->hasGFX90AInsts()) {
2347 MIB.addImm(Val: TFE); // tfe
2348 } else if (TFE) {
2349 LLVM_DEBUG(dbgs() << "TFE is not supported on this GPU\n");
2350 return false;
2351 }
2352
2353 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::lwe))
2354 MIB.addImm(Val: LWE); // lwe
2355 if (!IsGFX10Plus)
2356 MIB.addImm(Val: DimInfo->DA ? -1 : 0);
2357 if (BaseOpcode->HasD16)
2358 MIB.addImm(Val: IsD16 ? -1 : 0);
2359
2360 MI.eraseFromParent();
2361 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2362 TII.enforceOperandRCAlignment(MI&: *MIB, OpName: AMDGPU::OpName::vaddr);
2363 return true;
2364}
2365
2366// We need to handle this here because tablegen doesn't support matching
2367// instructions with multiple outputs.
2368bool AMDGPUInstructionSelector::selectDSBvhStackIntrinsic(
2369 MachineInstr &MI) const {
2370 Register Dst0 = MI.getOperand(i: 0).getReg();
2371 Register Dst1 = MI.getOperand(i: 1).getReg();
2372
2373 const DebugLoc &DL = MI.getDebugLoc();
2374 MachineBasicBlock *MBB = MI.getParent();
2375
2376 Register Addr = MI.getOperand(i: 3).getReg();
2377 Register Data0 = MI.getOperand(i: 4).getReg();
2378 Register Data1 = MI.getOperand(i: 5).getReg();
2379 unsigned Offset = MI.getOperand(i: 6).getImm();
2380
2381 unsigned Opc;
2382 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
2383 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2384 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2385 Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
2386 break;
2387 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2388 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32;
2389 break;
2390 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2391 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64;
2392 break;
2393 }
2394
2395 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
2396 .addDef(RegNo: Dst1)
2397 .addUse(RegNo: Addr)
2398 .addUse(RegNo: Data0)
2399 .addUse(RegNo: Data1)
2400 .addImm(Val: Offset)
2401 .cloneMemRefs(OtherMI: MI);
2402
2403 MI.eraseFromParent();
2404 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2405 return true;
2406}
2407
2408bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
2409 MachineInstr &I) const {
2410 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
2411 switch (IntrinsicID) {
2412 case Intrinsic::amdgcn_end_cf:
2413 return selectEndCfIntrinsic(MI&: I);
2414 case Intrinsic::amdgcn_ds_ordered_add:
2415 case Intrinsic::amdgcn_ds_ordered_swap:
2416 return selectDSOrderedIntrinsic(MI&: I, IntrID: IntrinsicID);
2417 case Intrinsic::amdgcn_ds_gws_init:
2418 case Intrinsic::amdgcn_ds_gws_barrier:
2419 case Intrinsic::amdgcn_ds_gws_sema_v:
2420 case Intrinsic::amdgcn_ds_gws_sema_br:
2421 case Intrinsic::amdgcn_ds_gws_sema_p:
2422 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2423 return selectDSGWSIntrinsic(MI&: I, IID: IntrinsicID);
2424 case Intrinsic::amdgcn_ds_append:
2425 return selectDSAppendConsume(MI&: I, IsAppend: true);
2426 case Intrinsic::amdgcn_ds_consume:
2427 return selectDSAppendConsume(MI&: I, IsAppend: false);
2428 case Intrinsic::amdgcn_init_whole_wave:
2429 return selectInitWholeWave(MI&: I);
2430 case Intrinsic::amdgcn_raw_buffer_load_lds:
2431 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
2432 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
2433 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
2434 case Intrinsic::amdgcn_struct_buffer_load_lds:
2435 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
2436 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
2437 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
2438 return selectBufferLoadLds(MI&: I);
2439 // Until we can store both the address space of the global and the LDS
2440 // arguments by having tto MachineMemOperands on an intrinsic, we just trust
2441 // that the argument is a global pointer (buffer pointers have been handled by
2442 // a LLVM IR-level lowering).
2443 case Intrinsic::amdgcn_load_to_lds:
2444 case Intrinsic::amdgcn_load_async_to_lds:
2445 case Intrinsic::amdgcn_global_load_lds:
2446 case Intrinsic::amdgcn_global_load_async_lds:
2447 return selectGlobalLoadLds(MI&: I);
2448 case Intrinsic::amdgcn_tensor_load_to_lds:
2449 case Intrinsic::amdgcn_tensor_store_from_lds:
2450 return selectTensorLoadStore(MI&: I, IID: IntrinsicID);
2451 case Intrinsic::amdgcn_asyncmark:
2452 case Intrinsic::amdgcn_wait_asyncmark:
2453 if (!Subtarget->hasAsyncMark())
2454 return false;
2455 break;
2456 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2457 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2458 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2459 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2460 return selectDSBvhStackIntrinsic(MI&: I);
2461 case Intrinsic::amdgcn_s_alloc_vgpr: {
2462 // S_ALLOC_VGPR doesn't have a destination register, it just implicitly sets
2463 // SCC. We then need to COPY it into the result vreg.
2464 MachineBasicBlock *MBB = I.getParent();
2465 const DebugLoc &DL = I.getDebugLoc();
2466
2467 Register ResReg = I.getOperand(i: 0).getReg();
2468
2469 MachineInstr *AllocMI = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ALLOC_VGPR))
2470 .add(MO: I.getOperand(i: 2));
2471 (void)BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: ResReg)
2472 .addReg(RegNo: AMDGPU::SCC);
2473 I.eraseFromParent();
2474 constrainSelectedInstRegOperands(I&: *AllocMI, TII, TRI, RBI);
2475 return RBI.constrainGenericRegister(Reg: ResReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2476 }
2477 case Intrinsic::amdgcn_s_barrier_init:
2478 case Intrinsic::amdgcn_s_barrier_signal_var:
2479 return selectNamedBarrierInit(I, IID: IntrinsicID);
2480 case Intrinsic::amdgcn_s_wakeup_barrier:
2481 case Intrinsic::amdgcn_s_barrier_join:
2482 case Intrinsic::amdgcn_s_get_named_barrier_state:
2483 return selectNamedBarrierInst(I, IID: IntrinsicID);
2484 case Intrinsic::amdgcn_s_get_barrier_state:
2485 return selectSGetBarrierState(I, IID: IntrinsicID);
2486 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
2487 return selectSBarrierSignalIsfirst(I, IID: IntrinsicID);
2488 }
2489 return selectImpl(I, CoverageInfo&: *CoverageInfo);
2490}
2491
2492bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
2493 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
2494 return true;
2495
2496 MachineBasicBlock *BB = I.getParent();
2497 const DebugLoc &DL = I.getDebugLoc();
2498
2499 Register DstReg = I.getOperand(i: 0).getReg();
2500 unsigned Size = RBI.getSizeInBits(Reg: DstReg, MRI: *MRI, TRI);
2501 assert(Size <= 32 || Size == 64);
2502 const MachineOperand &CCOp = I.getOperand(i: 1);
2503 Register CCReg = CCOp.getReg();
2504 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
2505 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 :
2506 AMDGPU::S_CSELECT_B32;
2507 MachineInstr *CopySCC = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
2508 .addReg(RegNo: CCReg);
2509
2510 // The generic constrainSelectedInstRegOperands doesn't work for the scc register
2511 // bank, because it does not cover the register class that we used to represent
2512 // for it. So we need to manually set the register class here.
2513 if (!MRI->getRegClassOrNull(Reg: CCReg))
2514 MRI->setRegClass(Reg: CCReg, RC: TRI.getConstrainedRegClassForReg(Reg: CCReg, MRI: *MRI));
2515 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
2516 .add(MO: I.getOperand(i: 2))
2517 .add(MO: I.getOperand(i: 3));
2518
2519 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2520 constrainSelectedInstRegOperands(I&: *CopySCC, TII, TRI, RBI);
2521 I.eraseFromParent();
2522 return true;
2523 }
2524
2525 // Wide VGPR select should have been split in RegBankSelect.
2526 if (Size > 32)
2527 return false;
2528
2529 MachineInstr *Select =
2530 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
2531 .addImm(Val: 0)
2532 .add(MO: I.getOperand(i: 3))
2533 .addImm(Val: 0)
2534 .add(MO: I.getOperand(i: 2))
2535 .add(MO: I.getOperand(i: 1));
2536
2537 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2538 I.eraseFromParent();
2539 return true;
2540}
2541
2542bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
2543 Register DstReg = I.getOperand(i: 0).getReg();
2544 Register SrcReg = I.getOperand(i: 1).getReg();
2545 const LLT DstTy = MRI->getType(Reg: DstReg);
2546 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2547 const LLT S1 = LLT::scalar(SizeInBits: 1);
2548
2549 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2550 const RegisterBank *DstRB;
2551 if (DstTy == S1) {
2552 // This is a special case. We don't treat s1 for legalization artifacts as
2553 // vcc booleans.
2554 DstRB = SrcRB;
2555 } else {
2556 DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2557 if (SrcRB != DstRB)
2558 return false;
2559 }
2560
2561 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
2562
2563 unsigned DstSize = DstTy.getSizeInBits();
2564 unsigned SrcSize = SrcTy.getSizeInBits();
2565
2566 const TargetRegisterClass *SrcRC =
2567 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcRB);
2568 const TargetRegisterClass *DstRC =
2569 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
2570 if (!SrcRC || !DstRC)
2571 return false;
2572
2573 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
2574 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
2575 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
2576 return false;
2577 }
2578
2579 if (DstRC == &AMDGPU::VGPR_16RegClass && SrcSize == 32) {
2580 assert(STI.useRealTrue16Insts());
2581 const DebugLoc &DL = I.getDebugLoc();
2582 MachineBasicBlock *MBB = I.getParent();
2583 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
2584 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::lo16);
2585 I.eraseFromParent();
2586 return true;
2587 }
2588
2589 if (DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) && SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
2590 MachineBasicBlock *MBB = I.getParent();
2591 const DebugLoc &DL = I.getDebugLoc();
2592
2593 Register LoReg = MRI->createVirtualRegister(RegClass: DstRC);
2594 Register HiReg = MRI->createVirtualRegister(RegClass: DstRC);
2595 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
2596 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
2597 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
2598 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
2599
2600 if (IsVALU && STI.hasSDWA()) {
2601 // Write the low 16-bits of the high element into the high 16-bits of the
2602 // low element.
2603 MachineInstr *MovSDWA =
2604 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: DstReg)
2605 .addImm(Val: 0) // $src0_modifiers
2606 .addReg(RegNo: HiReg) // $src0
2607 .addImm(Val: 0) // $clamp
2608 .addImm(Val: AMDGPU::SDWA::WORD_1) // $dst_sel
2609 .addImm(Val: AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused
2610 .addImm(Val: AMDGPU::SDWA::WORD_0) // $src0_sel
2611 .addReg(RegNo: LoReg, Flags: RegState::Implicit);
2612 MovSDWA->tieOperands(DefIdx: 0, UseIdx: MovSDWA->getNumOperands() - 1);
2613 } else {
2614 Register TmpReg0 = MRI->createVirtualRegister(RegClass: DstRC);
2615 Register TmpReg1 = MRI->createVirtualRegister(RegClass: DstRC);
2616 Register ImmReg = MRI->createVirtualRegister(RegClass: DstRC);
2617 if (IsVALU) {
2618 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: TmpReg0)
2619 .addImm(Val: 16)
2620 .addReg(RegNo: HiReg);
2621 } else {
2622 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg0)
2623 .addReg(RegNo: HiReg)
2624 .addImm(Val: 16)
2625 .setOperandDead(3); // Dead scc
2626 }
2627
2628 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2629 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
2630 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32;
2631
2632 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: ImmReg)
2633 .addImm(Val: 0xffff);
2634 auto And = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: TmpReg1)
2635 .addReg(RegNo: LoReg)
2636 .addReg(RegNo: ImmReg);
2637 auto Or = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: OrOpc), DestReg: DstReg)
2638 .addReg(RegNo: TmpReg0)
2639 .addReg(RegNo: TmpReg1);
2640
2641 if (!IsVALU) {
2642 And.setOperandDead(3); // Dead scc
2643 Or.setOperandDead(3); // Dead scc
2644 }
2645 }
2646
2647 I.eraseFromParent();
2648 return true;
2649 }
2650
2651 if (!DstTy.isScalar())
2652 return false;
2653
2654 if (SrcSize > 32) {
2655 unsigned SubRegIdx = DstSize < 32
2656 ? static_cast<unsigned>(AMDGPU::sub0)
2657 : TRI.getSubRegFromChannel(Channel: 0, NumRegs: DstSize / 32);
2658 if (SubRegIdx == AMDGPU::NoSubRegister)
2659 return false;
2660
2661 // Deal with weird cases where the class only partially supports the subreg
2662 // index.
2663 const TargetRegisterClass *SrcWithSubRC
2664 = TRI.getSubClassWithSubReg(SrcRC, SubRegIdx);
2665 if (!SrcWithSubRC)
2666 return false;
2667
2668 if (SrcWithSubRC != SrcRC) {
2669 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcWithSubRC, MRI&: *MRI))
2670 return false;
2671 }
2672
2673 I.getOperand(i: 1).setSubReg(SubRegIdx);
2674 }
2675
2676 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
2677 return true;
2678}
2679
2680/// \returns true if a bitmask for \p Size bits will be an inline immediate.
2681static bool shouldUseAndMask(unsigned Size, unsigned &Mask) {
2682 Mask = maskTrailingOnes<unsigned>(N: Size);
2683 int SignedMask = static_cast<int>(Mask);
2684 return SignedMask >= -16 && SignedMask <= 64;
2685}
2686
2687// Like RegisterBankInfo::getRegBank, but don't assume vcc for s1.
2688const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank(
2689 Register Reg, const MachineRegisterInfo &MRI,
2690 const TargetRegisterInfo &TRI) const {
2691 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
2692 if (auto *RB = dyn_cast<const RegisterBank *>(Val: RegClassOrBank))
2693 return RB;
2694
2695 // Ignore the type, since we don't use vcc in artifacts.
2696 if (auto *RC = dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank))
2697 return &RBI.getRegBankFromRegClass(RC: *RC, LLT());
2698 return nullptr;
2699}
2700
2701bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
2702 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG;
2703 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg;
2704 const DebugLoc &DL = I.getDebugLoc();
2705 MachineBasicBlock &MBB = *I.getParent();
2706 const Register DstReg = I.getOperand(i: 0).getReg();
2707 const Register SrcReg = I.getOperand(i: 1).getReg();
2708
2709 const LLT DstTy = MRI->getType(Reg: DstReg);
2710 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2711 const unsigned SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG ?
2712 I.getOperand(i: 2).getImm() : SrcTy.getSizeInBits();
2713 const unsigned DstSize = DstTy.getSizeInBits();
2714 if (!DstTy.isScalar())
2715 return false;
2716
2717 // Artifact casts should never use vcc.
2718 const RegisterBank *SrcBank = getArtifactRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2719
2720 // FIXME: This should probably be illegal and split earlier.
2721 if (I.getOpcode() == AMDGPU::G_ANYEXT) {
2722 if (DstSize <= 32)
2723 return selectCOPY(I);
2724
2725 const TargetRegisterClass *SrcRC =
2726 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcBank);
2727 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2728 const TargetRegisterClass *DstRC =
2729 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
2730
2731 Register UndefReg = MRI->createVirtualRegister(RegClass: SrcRC);
2732 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2733 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2734 .addReg(RegNo: SrcReg)
2735 .addImm(Val: AMDGPU::sub0)
2736 .addReg(RegNo: UndefReg)
2737 .addImm(Val: AMDGPU::sub1);
2738 I.eraseFromParent();
2739
2740 return RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) &&
2741 RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI);
2742 }
2743
2744 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) {
2745 // 64-bit should have been split up in RegBankSelect
2746
2747 // Try to use an and with a mask if it will save code size.
2748 unsigned Mask;
2749 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2750 MachineInstr *ExtI =
2751 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: DstReg)
2752 .addImm(Val: Mask)
2753 .addReg(RegNo: SrcReg);
2754 I.eraseFromParent();
2755 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2756 return true;
2757 }
2758
2759 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2760 MachineInstr *ExtI =
2761 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE), DestReg: DstReg)
2762 .addReg(RegNo: SrcReg)
2763 .addImm(Val: 0) // Offset
2764 .addImm(Val: SrcSize); // Width
2765 I.eraseFromParent();
2766 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2767 return true;
2768 }
2769
2770 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
2771 const TargetRegisterClass &SrcRC = InReg && DstSize > 32 ?
2772 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass;
2773 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: SrcRC, MRI&: *MRI))
2774 return false;
2775
2776 if (Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) {
2777 const unsigned SextOpc = SrcSize == 8 ?
2778 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16;
2779 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: SextOpc), DestReg: DstReg)
2780 .addReg(RegNo: SrcReg);
2781 I.eraseFromParent();
2782 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2783 }
2784
2785 // Using a single 32-bit SALU to calculate the high half is smaller than
2786 // S_BFE with a literal constant operand.
2787 if (DstSize > 32 && SrcSize == 32) {
2788 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2789 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2790 if (Signed) {
2791 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ASHR_I32), DestReg: HiReg)
2792 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2793 .addImm(Val: 31)
2794 .setOperandDead(3); // Dead scc
2795 } else {
2796 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg)
2797 .addImm(Val: 0);
2798 }
2799 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2800 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2801 .addImm(Val: AMDGPU::sub0)
2802 .addReg(RegNo: HiReg)
2803 .addImm(Val: AMDGPU::sub1);
2804 I.eraseFromParent();
2805 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass,
2806 MRI&: *MRI);
2807 }
2808
2809 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
2810 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2811
2812 // Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width.
2813 if (DstSize > 32 && (SrcSize <= 32 || InReg)) {
2814 // We need a 64-bit register source, but the high bits don't matter.
2815 Register ExtReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
2816 Register UndefReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2817 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2818
2819 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2820 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: ExtReg)
2821 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2822 .addImm(Val: AMDGPU::sub0)
2823 .addReg(RegNo: UndefReg)
2824 .addImm(Val: AMDGPU::sub1);
2825
2826 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE64), DestReg: DstReg)
2827 .addReg(RegNo: ExtReg)
2828 .addImm(Val: SrcSize << 16);
2829
2830 I.eraseFromParent();
2831 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI);
2832 }
2833
2834 unsigned Mask;
2835 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2836 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: DstReg)
2837 .addReg(RegNo: SrcReg)
2838 .addImm(Val: Mask)
2839 .setOperandDead(3); // Dead scc
2840 } else {
2841 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE32), DestReg: DstReg)
2842 .addReg(RegNo: SrcReg)
2843 .addImm(Val: SrcSize << 16);
2844 }
2845
2846 I.eraseFromParent();
2847 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2848 }
2849
2850 return false;
2851}
2852
2853static Register stripCopy(Register Reg, MachineRegisterInfo &MRI) {
2854 return getDefSrcRegIgnoringCopies(Reg, MRI)->Reg;
2855}
2856
2857static Register stripBitCast(Register Reg, MachineRegisterInfo &MRI) {
2858 Register BitcastSrc;
2859 if (mi_match(R: Reg, MRI, P: m_GBitcast(Src: m_Reg(R&: BitcastSrc))))
2860 Reg = BitcastSrc;
2861 return Reg;
2862}
2863
2864static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In,
2865 Register &Out) {
2866 // When unmerging a register that is composed of 2 x 16-bit values allow to
2867 // use an extract hi instruction for the upper 16 bits. We only need to check
2868 // the size of `In` as all defs are guaranteed to be the same type for
2869 // GUnmerge.
2870 GUnmerge *Unmerge;
2871 if (mi_match(R: In, MRI, P: m_GUnmerge(Inst&: Unmerge))) {
2872 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(i: 1).getReg() == In &&
2873 MRI.getType(Reg: In).getSizeInBits() == 16) {
2874 Out = stripBitCast(Reg: Unmerge->getSourceReg(), MRI);
2875 return true;
2876 }
2877 }
2878
2879 Register Trunc;
2880 if (!mi_match(R: In, MRI, P: m_GTrunc(Src: m_Reg(R&: Trunc))))
2881 return false;
2882
2883 Register LShlSrc;
2884 Register Cst;
2885 if (mi_match(R: Trunc, MRI, P: m_GLShr(L: m_Reg(R&: LShlSrc), R: m_Reg(R&: Cst)))) {
2886 Cst = stripCopy(Reg: Cst, MRI);
2887 if (mi_match(R: Cst, MRI, P: m_SpecificICst(RequestedValue: 16))) {
2888 Out = stripBitCast(Reg: LShlSrc, MRI);
2889 return true;
2890 }
2891 }
2892
2893 ArrayRef<int> Mask;
2894 Register Src1;
2895 if (!mi_match(R: Trunc, MRI, P: m_GShuffleVector(Src1: m_Reg(R&: Src1), Src2: m_Reg(), Mask)))
2896 return false;
2897
2898 assert(MRI.getType(Src1) == LLT::fixed_vector(2, 16));
2899 assert(Mask.size() == 2);
2900
2901 if (Mask[0] == 1 && Mask[1] <= 1) {
2902 Out = Trunc;
2903 return true;
2904 }
2905
2906 return false;
2907}
2908
2909static bool isExtractLoElt(MachineRegisterInfo &MRI, Register In,
2910 Register &Out) {
2911 // There could be a bitcast between the extraction and its use.
2912 In = stripBitCast(Reg: In, MRI);
2913
2914 // The first def of a 2 x 16-bit unmerge is the low half of its source.
2915 if (auto *Unmerge = dyn_cast<GUnmerge>(Val: MRI.getVRegDef(Reg: In))) {
2916 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(i: 0).getReg() == In &&
2917 MRI.getType(Reg: In).getSizeInBits() == 16) {
2918 Out = stripBitCast(Reg: Unmerge->getSourceReg(), MRI);
2919 return true;
2920 }
2921 }
2922
2923 // A truncation from 32 to 16 bits keeps the low half in place.
2924 Register Trunc;
2925 if (mi_match(R: In, MRI, P: m_GTrunc(Src: m_Reg(R&: Trunc))) &&
2926 MRI.getType(Reg: Trunc).getSizeInBits() == 32) {
2927 Out = stripBitCast(Reg: Trunc, MRI);
2928 return true;
2929 }
2930
2931 return false;
2932}
2933
2934bool AMDGPUInstructionSelector::selectG_FPEXT(MachineInstr &I) const {
2935 if (!Subtarget->hasSALUFloatInsts())
2936 return false;
2937
2938 Register Dst = I.getOperand(i: 0).getReg();
2939 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2940 if (DstRB->getID() != AMDGPU::SGPRRegBankID)
2941 return false;
2942
2943 Register Src = I.getOperand(i: 1).getReg();
2944
2945 if (MRI->getType(Reg: Dst) == LLT::scalar(SizeInBits: 32) &&
2946 MRI->getType(Reg: Src) == LLT::scalar(SizeInBits: 16)) {
2947 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
2948 MachineBasicBlock *BB = I.getParent();
2949 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_CVT_HI_F32_F16), DestReg: Dst)
2950 .addUse(RegNo: Src);
2951 I.eraseFromParent();
2952 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2953 }
2954 }
2955
2956 return false;
2957}
2958
2959bool AMDGPUInstructionSelector::selectG_FNEG(MachineInstr &MI) const {
2960 // Only manually handle the f64 SGPR case.
2961 //
2962 // FIXME: This is a workaround for 2.5 different tablegen problems. Because
2963 // the bit ops theoretically have a second result due to the implicit def of
2964 // SCC, the GlobalISelEmitter is overly conservative and rejects it. Fixing
2965 // that is easy by disabling the check. The result works, but uses a
2966 // nonsensical sreg32orlds_and_sreg_1 regclass.
2967 //
2968 // The DAG emitter is more problematic, and incorrectly adds both S_XOR_B32 to
2969 // the variadic REG_SEQUENCE operands.
2970
2971 Register Dst = MI.getOperand(i: 0).getReg();
2972 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2973 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
2974 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
2975 return false;
2976
2977 Register Src = MI.getOperand(i: 1).getReg();
2978 MachineInstr *Fabs = getOpcodeDef(Opcode: TargetOpcode::G_FABS, Reg: Src, MRI: *MRI);
2979 if (Fabs)
2980 Src = Fabs->getOperand(i: 1).getReg();
2981
2982 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
2983 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
2984 return false;
2985
2986 MachineBasicBlock *BB = MI.getParent();
2987 const DebugLoc &DL = MI.getDebugLoc();
2988 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2989 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2990 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2991 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2992
2993 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
2994 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
2995 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
2996 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
2997 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
2998 .addImm(Val: 0x80000000);
2999
3000 // Set or toggle sign bit.
3001 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32;
3002 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: OpReg)
3003 .addReg(RegNo: HiReg)
3004 .addReg(RegNo: ConstReg)
3005 .setOperandDead(3); // Dead scc
3006 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
3007 .addReg(RegNo: LoReg)
3008 .addImm(Val: AMDGPU::sub0)
3009 .addReg(RegNo: OpReg)
3010 .addImm(Val: AMDGPU::sub1);
3011 MI.eraseFromParent();
3012 return true;
3013}
3014
3015// FIXME: This is a workaround for the same tablegen problems as G_FNEG
3016bool AMDGPUInstructionSelector::selectG_FABS(MachineInstr &MI) const {
3017 Register Dst = MI.getOperand(i: 0).getReg();
3018 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
3019 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
3020 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
3021 return false;
3022
3023 Register Src = MI.getOperand(i: 1).getReg();
3024 MachineBasicBlock *BB = MI.getParent();
3025 const DebugLoc &DL = MI.getDebugLoc();
3026 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3027 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3028 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3029 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3030
3031 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
3032 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
3033 return false;
3034
3035 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3036 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
3037 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3038 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
3039 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
3040 .addImm(Val: 0x7fffffff);
3041
3042 // Clear sign bit.
3043 // TODO: Should this used S_BITSET0_*?
3044 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: OpReg)
3045 .addReg(RegNo: HiReg)
3046 .addReg(RegNo: ConstReg)
3047 .setOperandDead(3); // Dead scc
3048 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
3049 .addReg(RegNo: LoReg)
3050 .addImm(Val: AMDGPU::sub0)
3051 .addReg(RegNo: OpReg)
3052 .addImm(Val: AMDGPU::sub1);
3053
3054 MI.eraseFromParent();
3055 return true;
3056}
3057
3058static bool isConstant(const MachineInstr &MI) {
3059 return MI.getOpcode() == TargetOpcode::G_CONSTANT;
3060}
3061
3062void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
3063 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const {
3064
3065 unsigned OpNo = Load.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1;
3066 const MachineInstr *PtrMI =
3067 MRI.getUniqueVRegDef(Reg: Load.getOperand(i: OpNo).getReg());
3068
3069 assert(PtrMI);
3070
3071 if (PtrMI->getOpcode() != TargetOpcode::G_PTR_ADD)
3072 return;
3073
3074 GEPInfo GEPInfo;
3075
3076 for (unsigned i = 1; i != 3; ++i) {
3077 const MachineOperand &GEPOp = PtrMI->getOperand(i);
3078 const MachineInstr *OpDef = MRI.getUniqueVRegDef(Reg: GEPOp.getReg());
3079 assert(OpDef);
3080 if (i == 2 && isConstant(MI: *OpDef)) {
3081 // TODO: Could handle constant base + variable offset, but a combine
3082 // probably should have commuted it.
3083 assert(GEPInfo.Imm == 0);
3084 GEPInfo.Imm = OpDef->getOperand(i: 1).getCImm()->getSExtValue();
3085 continue;
3086 }
3087 const RegisterBank *OpBank = RBI.getRegBank(Reg: GEPOp.getReg(), MRI, TRI);
3088 if (OpBank->getID() == AMDGPU::SGPRRegBankID)
3089 GEPInfo.SgprParts.push_back(Elt: GEPOp.getReg());
3090 else
3091 GEPInfo.VgprParts.push_back(Elt: GEPOp.getReg());
3092 }
3093
3094 AddrInfo.push_back(Elt: GEPInfo);
3095 getAddrModeInfo(Load: *PtrMI, MRI, AddrInfo);
3096}
3097
3098bool AMDGPUInstructionSelector::isSGPR(Register Reg) const {
3099 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID;
3100}
3101
3102bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const {
3103 if (!MI.hasOneMemOperand())
3104 return false;
3105
3106 const MachineMemOperand *MMO = *MI.memoperands_begin();
3107 const Value *Ptr = MMO->getValue();
3108
3109 // UndefValue means this is a load of a kernel input. These are uniform.
3110 // Sometimes LDS instructions have constant pointers.
3111 // If Ptr is null, then that means this mem operand contains a
3112 // PseudoSourceValue like GOT.
3113 if (!Ptr || isa<UndefValue, Argument, Constant, GlobalValue>(Val: Ptr))
3114 return true;
3115
3116 if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
3117 return true;
3118
3119 if (MI.getOpcode() == AMDGPU::G_PREFETCH)
3120 return RBI.getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI: *MRI, TRI)->getID() ==
3121 AMDGPU::SGPRRegBankID;
3122
3123 const Instruction *I = dyn_cast<Instruction>(Val: Ptr);
3124 return I && I->getMetadata(Kind: "amdgpu.uniform");
3125}
3126
3127bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const {
3128 for (const GEPInfo &GEPInfo : AddrInfo) {
3129 if (!GEPInfo.VgprParts.empty())
3130 return true;
3131 }
3132 return false;
3133}
3134
3135void AMDGPUInstructionSelector::initM0(MachineInstr &I) const {
3136 const LLT PtrTy = MRI->getType(Reg: I.getOperand(i: 1).getReg());
3137 unsigned AS = PtrTy.getAddressSpace();
3138 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) &&
3139 STI.ldsRequiresM0Init()) {
3140 MachineBasicBlock *BB = I.getParent();
3141
3142 // If DS instructions require M0 initialization, insert it before selecting.
3143 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
3144 .addImm(Val: -1);
3145 }
3146}
3147
3148bool AMDGPUInstructionSelector::selectG_LOAD_STORE_ATOMICRMW(
3149 MachineInstr &I) const {
3150 initM0(I);
3151 return selectImpl(I, CoverageInfo&: *CoverageInfo);
3152}
3153
3154static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI) {
3155 if (Reg.isPhysical())
3156 return false;
3157
3158 MachineInstr &MI = *MRI.getUniqueVRegDef(Reg);
3159 const unsigned Opcode = MI.getOpcode();
3160
3161 if (Opcode == AMDGPU::COPY)
3162 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI);
3163
3164 if (Opcode == AMDGPU::G_AND || Opcode == AMDGPU::G_OR ||
3165 Opcode == AMDGPU::G_XOR)
3166 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI) &&
3167 isVCmpResult(Reg: MI.getOperand(i: 2).getReg(), MRI);
3168
3169 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI))
3170 return GI->is(ID: Intrinsic::amdgcn_class);
3171
3172 return Opcode == AMDGPU::G_ICMP || Opcode == AMDGPU::G_FCMP;
3173}
3174
3175bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
3176 MachineBasicBlock *BB = I.getParent();
3177 MachineOperand &CondOp = I.getOperand(i: 0);
3178 Register CondReg = CondOp.getReg();
3179 const DebugLoc &DL = I.getDebugLoc();
3180
3181 unsigned BrOpcode;
3182 Register CondPhysReg;
3183 const TargetRegisterClass *ConstrainRC;
3184
3185 // In SelectionDAG, we inspect the IR block for uniformity metadata to decide
3186 // whether the branch is uniform when selecting the instruction. In
3187 // GlobalISel, we should push that decision into RegBankSelect. Assume for now
3188 // RegBankSelect knows what it's doing if the branch condition is scc, even
3189 // though it currently does not.
3190 if (!isVCC(Reg: CondReg, MRI: *MRI)) {
3191 if (MRI->getType(Reg: CondReg) != LLT::scalar(SizeInBits: 32))
3192 return false;
3193
3194 CondPhysReg = AMDGPU::SCC;
3195 BrOpcode = AMDGPU::S_CBRANCH_SCC1;
3196 ConstrainRC = &AMDGPU::SReg_32RegClass;
3197 } else {
3198 // FIXME: Should scc->vcc copies and with exec?
3199
3200 // Unless the value of CondReg is a result of a V_CMP* instruction then we
3201 // need to insert an and with exec.
3202 if (!isVCmpResult(Reg: CondReg, MRI&: *MRI)) {
3203 const bool Is64 = STI.isWave64();
3204 const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
3205 const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO;
3206
3207 Register TmpReg = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
3208 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: TmpReg)
3209 .addReg(RegNo: CondReg)
3210 .addReg(RegNo: Exec)
3211 .setOperandDead(3); // Dead scc
3212 CondReg = TmpReg;
3213 }
3214
3215 CondPhysReg = TRI.getVCC();
3216 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ;
3217 ConstrainRC = TRI.getBoolRC();
3218 }
3219
3220 if (!MRI->getRegClassOrNull(Reg: CondReg))
3221 MRI->setRegClass(Reg: CondReg, RC: ConstrainRC);
3222
3223 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CondPhysReg)
3224 .addReg(RegNo: CondReg);
3225 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: BrOpcode))
3226 .addMBB(MBB: I.getOperand(i: 1).getMBB());
3227
3228 I.eraseFromParent();
3229 return true;
3230}
3231
3232bool AMDGPUInstructionSelector::selectG_GLOBAL_VALUE(
3233 MachineInstr &I) const {
3234 Register DstReg = I.getOperand(i: 0).getReg();
3235 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3236 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3237 I.setDesc(TII.get(Opcode: IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32));
3238 if (IsVGPR)
3239 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
3240
3241 return RBI.constrainGenericRegister(
3242 Reg: DstReg, RC: IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, MRI&: *MRI);
3243}
3244
3245bool AMDGPUInstructionSelector::selectG_PTRMASK(MachineInstr &I) const {
3246 Register DstReg = I.getOperand(i: 0).getReg();
3247 Register SrcReg = I.getOperand(i: 1).getReg();
3248 Register MaskReg = I.getOperand(i: 2).getReg();
3249 LLT Ty = MRI->getType(Reg: DstReg);
3250 LLT MaskTy = MRI->getType(Reg: MaskReg);
3251 MachineBasicBlock *BB = I.getParent();
3252 const DebugLoc &DL = I.getDebugLoc();
3253
3254 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3255 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3256 const RegisterBank *MaskRB = RBI.getRegBank(Reg: MaskReg, MRI: *MRI, TRI);
3257 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3258 if (DstRB != SrcRB) // Should only happen for hand written MIR.
3259 return false;
3260
3261 // Try to avoid emitting a bit operation when we only need to touch half of
3262 // the 64-bit pointer.
3263 APInt MaskOnes = VT->getKnownOnes(R: MaskReg).zext(width: 64);
3264 const APInt MaskHi32 = APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32);
3265 const APInt MaskLo32 = APInt::getLowBitsSet(numBits: 64, loBitsSet: 32);
3266
3267 const bool CanCopyLow32 = (MaskOnes & MaskLo32) == MaskLo32;
3268 const bool CanCopyHi32 = (MaskOnes & MaskHi32) == MaskHi32;
3269
3270 if (!IsVGPR && Ty.getSizeInBits() == 64 &&
3271 !CanCopyLow32 && !CanCopyHi32) {
3272 auto MIB = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B64), DestReg: DstReg)
3273 .addReg(RegNo: SrcReg)
3274 .addReg(RegNo: MaskReg)
3275 .setOperandDead(3); // Dead scc
3276 I.eraseFromParent();
3277 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3278 return true;
3279 }
3280
3281 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
3282 const TargetRegisterClass &RegRC
3283 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
3284
3285 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *DstRB);
3286 const TargetRegisterClass *SrcRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *SrcRB);
3287 const TargetRegisterClass *MaskRC =
3288 TRI.getRegClassForTypeOnBank(Ty: MaskTy, Bank: *MaskRB);
3289
3290 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3291 !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3292 !RBI.constrainGenericRegister(Reg: MaskReg, RC: *MaskRC, MRI&: *MRI))
3293 return false;
3294
3295 if (Ty.getSizeInBits() == 32) {
3296 assert(MaskTy.getSizeInBits() == 32 &&
3297 "ptrmask should have been narrowed during legalize");
3298
3299 auto NewOp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: DstReg)
3300 .addReg(RegNo: SrcReg)
3301 .addReg(RegNo: MaskReg);
3302
3303 if (!IsVGPR)
3304 NewOp.setOperandDead(3); // Dead scc
3305 I.eraseFromParent();
3306 return true;
3307 }
3308
3309 Register HiReg = MRI->createVirtualRegister(RegClass: &RegRC);
3310 Register LoReg = MRI->createVirtualRegister(RegClass: &RegRC);
3311
3312 // Extract the subregisters from the source pointer.
3313 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3314 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
3315 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3316 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
3317
3318 Register MaskedLo, MaskedHi;
3319
3320 if (CanCopyLow32) {
3321 // If all the bits in the low half are 1, we only need a copy for it.
3322 MaskedLo = LoReg;
3323 } else {
3324 // Extract the mask subregister and apply the and.
3325 Register MaskLo = MRI->createVirtualRegister(RegClass: &RegRC);
3326 MaskedLo = MRI->createVirtualRegister(RegClass: &RegRC);
3327
3328 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskLo)
3329 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub0);
3330 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedLo)
3331 .addReg(RegNo: LoReg)
3332 .addReg(RegNo: MaskLo);
3333 }
3334
3335 if (CanCopyHi32) {
3336 // If all the bits in the high half are 1, we only need a copy for it.
3337 MaskedHi = HiReg;
3338 } else {
3339 Register MaskHi = MRI->createVirtualRegister(RegClass: &RegRC);
3340 MaskedHi = MRI->createVirtualRegister(RegClass: &RegRC);
3341
3342 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskHi)
3343 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub1);
3344 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedHi)
3345 .addReg(RegNo: HiReg)
3346 .addReg(RegNo: MaskHi);
3347 }
3348
3349 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
3350 .addReg(RegNo: MaskedLo)
3351 .addImm(Val: AMDGPU::sub0)
3352 .addReg(RegNo: MaskedHi)
3353 .addImm(Val: AMDGPU::sub1);
3354 I.eraseFromParent();
3355 return true;
3356}
3357
3358/// Return the register to use for the index value, and the subregister to use
3359/// for the indirectly accessed register.
3360static std::pair<Register, unsigned>
3361computeIndirectRegIndex(MachineRegisterInfo &MRI, const SIRegisterInfo &TRI,
3362 const TargetRegisterClass *SuperRC, Register IdxReg,
3363 unsigned EltSize, GISelValueTracking &ValueTracking) {
3364 Register IdxBaseReg;
3365 int Offset;
3366
3367 std::tie(args&: IdxBaseReg, args&: Offset) =
3368 AMDGPU::getBaseWithConstantOffset(MRI, Reg: IdxReg, ValueTracking: &ValueTracking);
3369 if (IdxBaseReg == AMDGPU::NoRegister) {
3370 // This will happen if the index is a known constant. This should ordinarily
3371 // be legalized out, but handle it as a register just in case.
3372 assert(Offset == 0);
3373 IdxBaseReg = IdxReg;
3374 }
3375
3376 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SuperRC, EltSize);
3377
3378 // Skip out of bounds offsets, or else we would end up using an undefined
3379 // register.
3380 if (static_cast<unsigned>(Offset) >= SubRegs.size())
3381 return std::pair(IdxReg, SubRegs[0]);
3382 return std::pair(IdxBaseReg, SubRegs[Offset]);
3383}
3384
3385bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
3386 MachineInstr &MI) const {
3387 Register DstReg = MI.getOperand(i: 0).getReg();
3388 Register SrcReg = MI.getOperand(i: 1).getReg();
3389 Register IdxReg = MI.getOperand(i: 2).getReg();
3390
3391 LLT DstTy = MRI->getType(Reg: DstReg);
3392 LLT SrcTy = MRI->getType(Reg: SrcReg);
3393
3394 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3395 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3396 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3397
3398 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3399 // into a waterfall loop.
3400 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3401 return false;
3402
3403 const TargetRegisterClass *SrcRC =
3404 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcRB);
3405 const TargetRegisterClass *DstRC =
3406 TRI.getRegClassForTypeOnBank(Ty: DstTy, Bank: *DstRB);
3407 if (!SrcRC || !DstRC)
3408 return false;
3409 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3410 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3411 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3412 return false;
3413
3414 MachineBasicBlock *BB = MI.getParent();
3415 const DebugLoc &DL = MI.getDebugLoc();
3416 const bool Is64 = DstTy.getSizeInBits() == 64;
3417
3418 unsigned SubReg;
3419 std::tie(args&: IdxReg, args&: SubReg) = computeIndirectRegIndex(
3420 MRI&: *MRI, TRI, SuperRC: SrcRC, IdxReg, EltSize: DstTy.getSizeInBits() / 8, ValueTracking&: *VT);
3421
3422 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) {
3423 if (DstTy.getSizeInBits() != 32 && !Is64)
3424 return false;
3425
3426 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3427 .addReg(RegNo: IdxReg);
3428
3429 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32;
3430 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
3431 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3432 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3433 MI.eraseFromParent();
3434 return true;
3435 }
3436
3437 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32)
3438 return false;
3439
3440 if (!STI.useVGPRIndexMode()) {
3441 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3442 .addReg(RegNo: IdxReg);
3443 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: DstReg)
3444 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3445 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3446 MI.eraseFromParent();
3447 return true;
3448 }
3449
3450 const MCInstrDesc &GPRIDXDesc =
3451 TII.getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *SrcRC), IsIndirectSrc: true);
3452 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3453 .addReg(RegNo: SrcReg)
3454 .addReg(RegNo: IdxReg)
3455 .addImm(Val: SubReg);
3456
3457 MI.eraseFromParent();
3458 return true;
3459}
3460
3461// TODO: Fold insert_vector_elt (extract_vector_elt) into movrelsd
3462bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT(
3463 MachineInstr &MI) const {
3464 Register DstReg = MI.getOperand(i: 0).getReg();
3465 Register VecReg = MI.getOperand(i: 1).getReg();
3466 Register ValReg = MI.getOperand(i: 2).getReg();
3467 Register IdxReg = MI.getOperand(i: 3).getReg();
3468
3469 LLT VecTy = MRI->getType(Reg: DstReg);
3470 LLT ValTy = MRI->getType(Reg: ValReg);
3471 unsigned VecSize = VecTy.getSizeInBits();
3472 unsigned ValSize = ValTy.getSizeInBits();
3473
3474 const RegisterBank *VecRB = RBI.getRegBank(Reg: VecReg, MRI: *MRI, TRI);
3475 const RegisterBank *ValRB = RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
3476 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3477
3478 assert(VecTy.getElementType() == ValTy);
3479
3480 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3481 // into a waterfall loop.
3482 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3483 return false;
3484
3485 const TargetRegisterClass *VecRC =
3486 TRI.getRegClassForTypeOnBank(Ty: VecTy, Bank: *VecRB);
3487 const TargetRegisterClass *ValRC =
3488 TRI.getRegClassForTypeOnBank(Ty: ValTy, Bank: *ValRB);
3489
3490 if (!RBI.constrainGenericRegister(Reg: VecReg, RC: *VecRC, MRI&: *MRI) ||
3491 !RBI.constrainGenericRegister(Reg: DstReg, RC: *VecRC, MRI&: *MRI) ||
3492 !RBI.constrainGenericRegister(Reg: ValReg, RC: *ValRC, MRI&: *MRI) ||
3493 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3494 return false;
3495
3496 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32)
3497 return false;
3498
3499 unsigned SubReg;
3500 std::tie(args&: IdxReg, args&: SubReg) =
3501 computeIndirectRegIndex(MRI&: *MRI, TRI, SuperRC: VecRC, IdxReg, EltSize: ValSize / 8, ValueTracking&: *VT);
3502
3503 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID &&
3504 STI.useVGPRIndexMode();
3505
3506 MachineBasicBlock *BB = MI.getParent();
3507 const DebugLoc &DL = MI.getDebugLoc();
3508
3509 if (!IndexMode) {
3510 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3511 .addReg(RegNo: IdxReg);
3512
3513 const MCInstrDesc &RegWriteOp = TII.getIndirectRegWriteMovRelPseudo(
3514 VecSize, EltSize: ValSize, IsSGPR: VecRB->getID() == AMDGPU::SGPRRegBankID);
3515 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: RegWriteOp, DestReg: DstReg)
3516 .addReg(RegNo: VecReg)
3517 .addReg(RegNo: ValReg)
3518 .addImm(Val: SubReg);
3519 MI.eraseFromParent();
3520 return true;
3521 }
3522
3523 const MCInstrDesc &GPRIDXDesc =
3524 TII.getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
3525 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3526 .addReg(RegNo: VecReg)
3527 .addReg(RegNo: ValReg)
3528 .addReg(RegNo: IdxReg)
3529 .addImm(Val: SubReg);
3530
3531 MI.eraseFromParent();
3532 return true;
3533}
3534
3535static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
3536 switch (Intr) {
3537 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
3538 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
3539 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
3540 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
3541 case Intrinsic::amdgcn_load_async_to_lds:
3542 case Intrinsic::amdgcn_global_load_async_lds:
3543 return true;
3544 }
3545 return false;
3546}
3547
3548bool AMDGPUInstructionSelector::selectBufferLoadLds(MachineInstr &MI) const {
3549 if (!Subtarget->hasVMemToLDSLoad())
3550 return false;
3551 unsigned Opc;
3552 unsigned Size = MI.getOperand(i: 3).getImm();
3553 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3554
3555 // The struct intrinsic variants add one additional operand over raw.
3556 const bool HasVIndex = MI.getNumOperands() == 9;
3557 Register VIndex;
3558 int OpOffset = 0;
3559 if (HasVIndex) {
3560 VIndex = MI.getOperand(i: 4).getReg();
3561 OpOffset = 1;
3562 }
3563
3564 Register VOffset = MI.getOperand(i: 4 + OpOffset).getReg();
3565 std::optional<ValueAndVReg> MaybeVOffset =
3566 getIConstantVRegValWithLookThrough(VReg: VOffset, MRI: *MRI);
3567 const bool HasVOffset = !MaybeVOffset || MaybeVOffset->Value.getZExtValue();
3568
3569 switch (Size) {
3570 default:
3571 return false;
3572 case 1:
3573 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
3574 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
3575 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
3576 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
3577 break;
3578 case 2:
3579 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
3580 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
3581 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
3582 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
3583 break;
3584 case 4:
3585 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
3586 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
3587 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
3588 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
3589 break;
3590 case 12:
3591 if (!Subtarget->hasLDSLoadB96_B128())
3592 return false;
3593
3594 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
3595 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
3596 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
3597 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
3598 break;
3599 case 16:
3600 if (!Subtarget->hasLDSLoadB96_B128())
3601 return false;
3602
3603 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
3604 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
3605 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
3606 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
3607 break;
3608 }
3609
3610 MachineBasicBlock *MBB = MI.getParent();
3611 const DebugLoc &DL = MI.getDebugLoc();
3612 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3613 .add(MO: MI.getOperand(i: 2));
3614
3615 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc));
3616
3617 if (HasVIndex && HasVOffset) {
3618 Register IdxReg = MRI->createVirtualRegister(RegClass: TRI.getVGPR64Class());
3619 BuildMI(BB&: *MBB, I: &*MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: IdxReg)
3620 .addReg(RegNo: VIndex)
3621 .addImm(Val: AMDGPU::sub0)
3622 .addReg(RegNo: VOffset)
3623 .addImm(Val: AMDGPU::sub1);
3624
3625 MIB.addReg(RegNo: IdxReg);
3626 } else if (HasVIndex) {
3627 MIB.addReg(RegNo: VIndex);
3628 } else if (HasVOffset) {
3629 MIB.addReg(RegNo: VOffset);
3630 }
3631
3632 MIB.add(MO: MI.getOperand(i: 1)); // rsrc
3633 MIB.add(MO: MI.getOperand(i: 5 + OpOffset)); // soffset
3634 MIB.add(MO: MI.getOperand(i: 6 + OpOffset)); // imm offset
3635 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
3636 unsigned Aux = MI.getOperand(i: 7 + OpOffset).getImm();
3637 MIB.addImm(Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL
3638 : AMDGPU::CPol::ALL_pregfx12)); // cpol
3639 MIB.addImm(
3640 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
3641 ? 1
3642 : 0); // swz
3643 MIB.addImm(Val: isAsyncLDSDMA(Intr: IntrinsicID));
3644
3645 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3646 // Don't set the offset value here because the pointer points to the base of
3647 // the buffer.
3648 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3649
3650 MachinePointerInfo StorePtrI = LoadPtrI;
3651 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3652 AddressSpace: AMDGPUAS::BUFFER_RESOURCE));
3653 LoadPtrI.AddrSpace = AMDGPUAS::BUFFER_RESOURCE;
3654 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3655
3656 auto F = LoadMMO->getFlags() &
3657 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3658 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3659 Size, BaseAlignment: LoadMMO->getBaseAlign());
3660
3661 MachineMemOperand *StoreMMO =
3662 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3663 Size: sizeof(int32_t), BaseAlignment: LoadMMO->getBaseAlign());
3664
3665 MIB.setMemRefs({LoadMMO, StoreMMO});
3666
3667 MI.eraseFromParent();
3668 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3669 return true;
3670}
3671
3672/// Match a zero extend from a 32-bit value to 64-bits.
3673Register AMDGPUInstructionSelector::matchZeroExtendFromS32(Register Reg) const {
3674 Register ZExtSrc;
3675 if (mi_match(R: Reg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrc))))
3676 return MRI->getType(Reg: ZExtSrc) == LLT::scalar(SizeInBits: 32) ? ZExtSrc : Register();
3677
3678 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 0)
3679 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3680 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3681 return Register();
3682
3683 assert(Def->getNumOperands() == 3 &&
3684 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3685 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_ZeroInt())) {
3686 return Def->getOperand(i: 1).getReg();
3687 }
3688
3689 return Register();
3690}
3691
3692/// Match a sign extend from a 32-bit value to 64-bits.
3693Register AMDGPUInstructionSelector::matchSignExtendFromS32(Register Reg) const {
3694 Register SExtSrc;
3695 if (mi_match(R: Reg, MRI: *MRI, P: m_GSExt(Src: m_Reg(R&: SExtSrc))))
3696 return MRI->getType(Reg: SExtSrc) == LLT::scalar(SizeInBits: 32) ? SExtSrc : Register();
3697
3698 // Match legalized form %sext = G_MERGE_VALUES (s32 %x), G_ASHR((S32 %x, 31))
3699 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3700 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3701 return Register();
3702
3703 assert(Def->getNumOperands() == 3 &&
3704 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3705 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI,
3706 P: m_GAShr(L: m_SpecificReg(RequestedReg: Def->getOperand(i: 1).getReg()),
3707 R: m_SpecificICst(RequestedValue: 31))))
3708 return Def->getOperand(i: 1).getReg();
3709
3710 Register ZextSrc = matchZeroExtendFromS32(Reg);
3711 if (ZextSrc && VT->signBitIsZero(Op: ZextSrc))
3712 return ZextSrc;
3713
3714 return Register();
3715}
3716
3717/// Match a zero extend from a 32-bit value to 64-bits, or \p Reg itself if it
3718/// is 32-bit.
3719Register
3720AMDGPUInstructionSelector::matchZeroExtendFromS32OrS32(Register Reg) const {
3721 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3722 : matchZeroExtendFromS32(Reg);
3723}
3724
3725/// Match a sign extend from a 32-bit value to 64-bits, or \p Reg itself if it
3726/// is 32-bit.
3727Register
3728AMDGPUInstructionSelector::matchSignExtendFromS32OrS32(Register Reg) const {
3729 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3730 : matchSignExtendFromS32(Reg);
3731}
3732
3733Register
3734AMDGPUInstructionSelector::matchExtendFromS32OrS32(Register Reg,
3735 bool IsSigned) const {
3736 if (IsSigned)
3737 return matchSignExtendFromS32OrS32(Reg);
3738
3739 return matchZeroExtendFromS32OrS32(Reg);
3740}
3741
3742Register AMDGPUInstructionSelector::matchAnyExtendFromS32(Register Reg) const {
3743 Register AnyExtSrc;
3744 if (mi_match(R: Reg, MRI: *MRI, P: m_GAnyExt(Src: m_Reg(R&: AnyExtSrc))))
3745 return MRI->getType(Reg: AnyExtSrc) == LLT::scalar(SizeInBits: 32) ? AnyExtSrc : Register();
3746
3747 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 G_IMPLICIT_DEF)
3748 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3749 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3750 return Register();
3751
3752 assert(Def->getNumOperands() == 3 &&
3753 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3754
3755 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_GImplicitDef()))
3756 return Def->getOperand(i: 1).getReg();
3757
3758 return Register();
3759}
3760
3761bool AMDGPUInstructionSelector::selectGlobalLoadLds(MachineInstr &MI) const{
3762 if (!Subtarget->hasVMemToLDSLoad())
3763 return false;
3764
3765 unsigned Opc;
3766 unsigned Size = MI.getOperand(i: 3).getImm();
3767 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3768
3769 switch (Size) {
3770 default:
3771 return false;
3772 case 1:
3773 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
3774 break;
3775 case 2:
3776 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
3777 break;
3778 case 4:
3779 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
3780 break;
3781 case 12:
3782 if (!Subtarget->hasLDSLoadB96_B128())
3783 return false;
3784 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
3785 break;
3786 case 16:
3787 if (!Subtarget->hasLDSLoadB96_B128())
3788 return false;
3789 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
3790 break;
3791 }
3792
3793 MachineBasicBlock *MBB = MI.getParent();
3794 const DebugLoc &DL = MI.getDebugLoc();
3795 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3796 .add(MO: MI.getOperand(i: 2));
3797
3798 Register Addr = MI.getOperand(i: 1).getReg();
3799 Register VOffset;
3800 // Try to split SAddr and VOffset. Global and LDS pointers share the same
3801 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
3802 if (!isSGPR(Reg: Addr)) {
3803 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
3804 if (isSGPR(Reg: AddrDef->Reg)) {
3805 Addr = AddrDef->Reg;
3806 } else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
3807 Register SAddr =
3808 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
3809 if (isSGPR(Reg: SAddr)) {
3810 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
3811 if (Register Off = matchZeroExtendFromS32(Reg: PtrBaseOffset)) {
3812 Addr = SAddr;
3813 VOffset = Off;
3814 }
3815 }
3816 }
3817 }
3818
3819 if (isSGPR(Reg: Addr)) {
3820 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
3821 if (!VOffset) {
3822 VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
3823 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
3824 .addImm(Val: 0);
3825 }
3826 }
3827
3828 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc))
3829 .addReg(RegNo: Addr);
3830
3831 if (isSGPR(Reg: Addr))
3832 MIB.addReg(RegNo: VOffset);
3833
3834 MIB.add(MO: MI.getOperand(i: 4)); // offset
3835
3836 unsigned Aux = MI.getOperand(i: 5).getImm();
3837 MIB.addImm(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS); // cpol
3838 MIB.addImm(Val: isAsyncLDSDMA(Intr: IntrinsicID));
3839
3840 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3841 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3842 LoadPtrI.Offset = MI.getOperand(i: 4).getImm();
3843 MachinePointerInfo StorePtrI = LoadPtrI;
3844 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3845 AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
3846 LoadPtrI.AddrSpace = AMDGPUAS::GLOBAL_ADDRESS;
3847 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3848 auto F = LoadMMO->getFlags() &
3849 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3850 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3851 Size, BaseAlignment: LoadMMO->getBaseAlign());
3852 MachineMemOperand *StoreMMO =
3853 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3854 Size: sizeof(int32_t), BaseAlignment: Align(4));
3855
3856 MIB.setMemRefs({LoadMMO, StoreMMO});
3857
3858 MI.eraseFromParent();
3859 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3860 return true;
3861}
3862
3863bool AMDGPUInstructionSelector::selectTensorLoadStore(MachineInstr &MI,
3864 Intrinsic::ID IID) const {
3865 bool IsLoad = IID == Intrinsic::amdgcn_tensor_load_to_lds;
3866 unsigned Opc =
3867 IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d4 : AMDGPU::TENSOR_STORE_FROM_LDS_d4;
3868 int NumGroups = 4;
3869
3870 // A lamda function to check whether an operand is a vector of all 0s.
3871 const auto isAllZeros = [&](MachineOperand &Opnd) {
3872 const MachineInstr *DefMI = MRI->getVRegDef(Reg: Opnd.getReg());
3873 if (!DefMI)
3874 return false;
3875 return llvm::isBuildVectorAllZeros(MI: *DefMI, MRI: *MRI, AllowUndef: true);
3876 };
3877
3878 // Use _D2 version if both group 2 and 3 are zero-initialized.
3879 if (isAllZeros(MI.getOperand(i: 3)) && isAllZeros(MI.getOperand(i: 4))) {
3880 NumGroups = 2;
3881 Opc = IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d2
3882 : AMDGPU::TENSOR_STORE_FROM_LDS_d2;
3883 }
3884
3885 // TODO: Handle the fifth group: MI.getOpetand(5), which is silently ignored
3886 // for now because all existing targets only support up to 4 groups.
3887 MachineBasicBlock *MBB = MI.getParent();
3888 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
3889 .add(MO: MI.getOperand(i: 1)) // D# group 0
3890 .add(MO: MI.getOperand(i: 2)); // D# group 1
3891
3892 if (NumGroups >= 4) { // Has at least 4 groups
3893 MIB.add(MO: MI.getOperand(i: 3)) // D# group 2
3894 .add(MO: MI.getOperand(i: 4)); // D# group 3
3895 }
3896
3897 MIB.addImm(Val: 0) // r128
3898 .add(MO: MI.getOperand(i: 6)); // cpol
3899
3900 MI.eraseFromParent();
3901 return true;
3902}
3903
3904bool AMDGPUInstructionSelector::selectBVHIntersectRayIntrinsic(
3905 MachineInstr &MI) const {
3906 unsigned OpcodeOpIdx =
3907 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY ? 1 : 3;
3908 MI.setDesc(TII.get(Opcode: MI.getOperand(i: OpcodeOpIdx).getImm()));
3909 MI.removeOperand(OpNo: OpcodeOpIdx);
3910 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
3911 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
3912 return true;
3913}
3914
3915// FIXME: This should be removed and let the patterns select. We just need the
3916// AGPR/VGPR combination versions.
3917bool AMDGPUInstructionSelector::selectSMFMACIntrin(MachineInstr &MI) const {
3918 unsigned Opc;
3919 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
3920 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
3921 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64;
3922 break;
3923 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
3924 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64;
3925 break;
3926 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
3927 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64;
3928 break;
3929 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
3930 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64;
3931 break;
3932 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
3933 Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64;
3934 break;
3935 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
3936 Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64;
3937 break;
3938 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
3939 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64;
3940 break;
3941 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
3942 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64;
3943 break;
3944 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
3945 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64;
3946 break;
3947 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
3948 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64;
3949 break;
3950 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
3951 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64;
3952 break;
3953 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
3954 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64;
3955 break;
3956 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
3957 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64;
3958 break;
3959 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
3960 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64;
3961 break;
3962 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
3963 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_F16_e64;
3964 break;
3965 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
3966 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_F16_e64;
3967 break;
3968 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
3969 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF16_e64;
3970 break;
3971 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
3972 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF16_e64;
3973 break;
3974 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
3975 Opc = AMDGPU::V_SMFMAC_I32_16X16X128_I8_e64;
3976 break;
3977 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
3978 Opc = AMDGPU::V_SMFMAC_I32_32X32X64_I8_e64;
3979 break;
3980 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
3981 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_BF8_e64;
3982 break;
3983 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
3984 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_FP8_e64;
3985 break;
3986 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
3987 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_BF8_e64;
3988 break;
3989 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
3990 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_FP8_e64;
3991 break;
3992 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
3993 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_BF8_e64;
3994 break;
3995 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
3996 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_FP8_e64;
3997 break;
3998 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
3999 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_BF8_e64;
4000 break;
4001 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
4002 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_FP8_e64;
4003 break;
4004 default:
4005 llvm_unreachable("unhandled smfmac intrinsic");
4006 }
4007
4008 auto VDst_In = MI.getOperand(i: 4);
4009
4010 MI.setDesc(TII.get(Opcode: Opc));
4011 MI.removeOperand(OpNo: 4); // VDst_In
4012 MI.removeOperand(OpNo: 1); // Intrinsic ID
4013 MI.addOperand(Op: VDst_In); // Readd VDst_In to the end
4014 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
4015 const MCInstrDesc &MCID = MI.getDesc();
4016 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
4017 MI.getOperand(i: 0).setIsEarlyClobber(true);
4018 }
4019 return true;
4020}
4021
4022bool AMDGPUInstructionSelector::selectPermlaneSwapIntrin(
4023 MachineInstr &MI, Intrinsic::ID IntrID) const {
4024 if (IntrID == Intrinsic::amdgcn_permlane16_swap &&
4025 !Subtarget->hasPermlane16Swap())
4026 return false;
4027 if (IntrID == Intrinsic::amdgcn_permlane32_swap &&
4028 !Subtarget->hasPermlane32Swap())
4029 return false;
4030
4031 unsigned Opcode = IntrID == Intrinsic::amdgcn_permlane16_swap
4032 ? AMDGPU::V_PERMLANE16_SWAP_B32_e64
4033 : AMDGPU::V_PERMLANE32_SWAP_B32_e64;
4034
4035 MI.removeOperand(OpNo: 2);
4036 MI.setDesc(TII.get(Opcode));
4037 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
4038
4039 MachineOperand &FI = MI.getOperand(i: 4);
4040 FI.setImm(FI.getImm() ? AMDGPU::DPP::DPP_FI_1 : AMDGPU::DPP::DPP_FI_0);
4041
4042 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
4043 return true;
4044}
4045
4046bool AMDGPUInstructionSelector::selectWaveAddress(MachineInstr &MI) const {
4047 Register DstReg = MI.getOperand(i: 0).getReg();
4048 Register SrcReg = MI.getOperand(i: 1).getReg();
4049 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4050 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4051 MachineBasicBlock *MBB = MI.getParent();
4052 const DebugLoc &DL = MI.getDebugLoc();
4053
4054 if (IsVALU) {
4055 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: DstReg)
4056 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4057 .addReg(RegNo: SrcReg);
4058 } else {
4059 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: DstReg)
4060 .addReg(RegNo: SrcReg)
4061 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4062 .setOperandDead(3); // Dead scc
4063 }
4064
4065 const TargetRegisterClass &RC =
4066 IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
4067 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
4068 return false;
4069
4070 MI.eraseFromParent();
4071 return true;
4072}
4073
4074bool AMDGPUInstructionSelector::selectWaveShuffleIntrin(
4075 MachineInstr &MI) const {
4076 assert(MI.getNumOperands() == 4);
4077 MachineBasicBlock *MBB = MI.getParent();
4078 const DebugLoc &DL = MI.getDebugLoc();
4079
4080 Register DstReg = MI.getOperand(i: 0).getReg();
4081 Register ValReg = MI.getOperand(i: 2).getReg();
4082 Register IdxReg = MI.getOperand(i: 3).getReg();
4083
4084 const LLT DstTy = MRI->getType(Reg: DstReg);
4085 unsigned DstSize = DstTy.getSizeInBits();
4086 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4087 const TargetRegisterClass *DstRC =
4088 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
4089
4090 if (DstTy != LLT::scalar(SizeInBits: 32))
4091 return false;
4092
4093 if (!Subtarget->supportsBPermute())
4094 return false;
4095
4096 // If we can bpermute across the whole wave, then just do that
4097 if (Subtarget->supportsWaveWideBPermute()) {
4098 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4099 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4100 .addImm(Val: 2)
4101 .addReg(RegNo: IdxReg);
4102
4103 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: DstReg)
4104 .addReg(RegNo: ShiftIdxReg)
4105 .addReg(RegNo: ValReg)
4106 .addImm(Val: 0);
4107 } else {
4108 // Otherwise, we need to make use of whole wave mode
4109 assert(Subtarget->isWave64());
4110
4111 // Set inactive lanes to poison
4112 Register UndefValReg =
4113 MRI->createVirtualRegister(RegClass: TRI.getRegClass(i: AMDGPU::SReg_32RegClassID));
4114 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefValReg);
4115
4116 Register UndefExecReg = MRI->createVirtualRegister(
4117 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4118 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExecReg);
4119
4120 Register PoisonValReg = MRI->createVirtualRegister(RegClass: DstRC);
4121 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonValReg)
4122 .addImm(Val: 0)
4123 .addReg(RegNo: ValReg)
4124 .addImm(Val: 0)
4125 .addReg(RegNo: UndefValReg)
4126 .addReg(RegNo: UndefExecReg);
4127
4128 // ds_bpermute requires index to be multiplied by 4
4129 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4130 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4131 .addImm(Val: 2)
4132 .addReg(RegNo: IdxReg);
4133
4134 Register PoisonIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4135 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonIdxReg)
4136 .addImm(Val: 0)
4137 .addReg(RegNo: ShiftIdxReg)
4138 .addImm(Val: 0)
4139 .addReg(RegNo: UndefValReg)
4140 .addReg(RegNo: UndefExecReg);
4141
4142 Register PoisonUnshiftedIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4143 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32),
4144 DestReg: PoisonUnshiftedIdxReg)
4145 .addImm(Val: 0)
4146 .addReg(RegNo: IdxReg)
4147 .addImm(Val: 0)
4148 .addReg(RegNo: UndefValReg)
4149 .addReg(RegNo: UndefExecReg);
4150
4151 // Get permutation of each half, then we'll select which one to use
4152 Register SameSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4153 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: SameSidePermReg)
4154 .addReg(RegNo: PoisonIdxReg)
4155 .addReg(RegNo: PoisonValReg)
4156 .addImm(Val: 0);
4157
4158 Register SwappedValReg = MRI->createVirtualRegister(RegClass: DstRC);
4159 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_PERMLANE64_B32), DestReg: SwappedValReg)
4160 .addReg(RegNo: PoisonValReg);
4161
4162 Register OppSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4163 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: OppSidePermReg)
4164 .addReg(RegNo: PoisonIdxReg)
4165 .addReg(RegNo: SwappedValReg)
4166 .addImm(Val: 0);
4167
4168 Register WWMSwapPermReg = MRI->createVirtualRegister(RegClass: DstRC);
4169 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::STRICT_WWM), DestReg: WWMSwapPermReg)
4170 .addReg(RegNo: OppSidePermReg);
4171
4172 // Select which side to take the permute from
4173 // We can get away with only using mbcnt_lo here since we're only
4174 // trying to detect which side of 32 each lane is on, and mbcnt_lo
4175 // returns 32 for lanes 32-63.
4176 Register ThreadIDReg = MRI->createVirtualRegister(RegClass: DstRC);
4177 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64), DestReg: ThreadIDReg)
4178 .addImm(Val: -1)
4179 .addImm(Val: 0);
4180
4181 Register XORReg = MRI->createVirtualRegister(RegClass: DstRC);
4182 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_XOR_B32_e64), DestReg: XORReg)
4183 .addReg(RegNo: ThreadIDReg)
4184 .addReg(RegNo: PoisonUnshiftedIdxReg);
4185
4186 Register ANDReg = MRI->createVirtualRegister(RegClass: DstRC);
4187 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: ANDReg)
4188 .addReg(RegNo: XORReg)
4189 .addImm(Val: 32);
4190
4191 Register CompareReg = MRI->createVirtualRegister(
4192 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4193 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CompareReg)
4194 .addReg(RegNo: ANDReg)
4195 .addImm(Val: 0);
4196
4197 // Finally do the selection
4198 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
4199 .addImm(Val: 0)
4200 .addReg(RegNo: WWMSwapPermReg)
4201 .addImm(Val: 0)
4202 .addReg(RegNo: SameSidePermReg)
4203 .addReg(RegNo: CompareReg);
4204 }
4205
4206 MI.eraseFromParent();
4207 return true;
4208}
4209
4210// Match BITOP3 operation and return a number of matched instructions plus
4211// truth table.
4212static std::pair<unsigned, uint8_t> BitOp3_Op(Register R,
4213 SmallVectorImpl<Register> &Src,
4214 const MachineRegisterInfo &MRI) {
4215 unsigned NumOpcodes = 0;
4216 uint8_t LHSBits, RHSBits;
4217
4218 auto getOperandBits = [&Src, R, &MRI](Register Op, uint8_t &Bits) -> bool {
4219 // Define truth table given Src0, Src1, Src2 bits permutations:
4220 // 0 0 0
4221 // 0 0 1
4222 // 0 1 0
4223 // 0 1 1
4224 // 1 0 0
4225 // 1 0 1
4226 // 1 1 0
4227 // 1 1 1
4228 const uint8_t SrcBits[3] = { 0xf0, 0xcc, 0xaa };
4229
4230 if (mi_match(R: Op, MRI, P: m_AllOnesInt())) {
4231 Bits = 0xff;
4232 return true;
4233 }
4234 if (mi_match(R: Op, MRI, P: m_ZeroInt())) {
4235 Bits = 0;
4236 return true;
4237 }
4238
4239 for (unsigned I = 0; I < Src.size(); ++I) {
4240 // Try to find existing reused operand
4241 if (Src[I] == Op) {
4242 Bits = SrcBits[I];
4243 return true;
4244 }
4245 // Try to replace parent operator
4246 if (Src[I] == R) {
4247 Bits = SrcBits[I];
4248 Src[I] = Op;
4249 return true;
4250 }
4251 }
4252
4253 if (Src.size() == 3) {
4254 // No room left for operands. Try one last time, there can be a 'not' of
4255 // one of our source operands. In this case we can compute the bits
4256 // without growing Src vector.
4257 Register LHS;
4258 if (mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: LHS)))) {
4259 LHS = getSrcRegIgnoringCopies(Reg: LHS, MRI);
4260 for (unsigned I = 0; I < Src.size(); ++I) {
4261 if (Src[I] == LHS) {
4262 Bits = ~SrcBits[I];
4263 return true;
4264 }
4265 }
4266 }
4267
4268 return false;
4269 }
4270
4271 Bits = SrcBits[Src.size()];
4272 Src.push_back(Elt: Op);
4273 return true;
4274 };
4275
4276 MachineInstr *MI = MRI.getVRegDef(Reg: R);
4277 switch (MI->getOpcode()) {
4278 case TargetOpcode::G_AND:
4279 case TargetOpcode::G_OR:
4280 case TargetOpcode::G_XOR: {
4281 Register LHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 1).getReg(), MRI);
4282 Register RHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 2).getReg(), MRI);
4283
4284 SmallVector<Register, 3> Backup(Src.begin(), Src.end());
4285 if (!getOperandBits(LHS, LHSBits) ||
4286 !getOperandBits(RHS, RHSBits)) {
4287 Src = std::move(Backup);
4288 return std::make_pair(x: 0, y: 0);
4289 }
4290
4291 // Recursion is naturally limited by the size of the operand vector.
4292 //
4293 // When LHS and RHS share a common sub-expression, one side's recursion
4294 // may decompose that sub-expression and replace the Src slot the other
4295 // side occupies with sub-operands via the "replace parent" path in
4296 // getOperandBits. The other side's cached bit-pattern then refers to a
4297 // slot whose contents changed, producing a wrong truth table.
4298 //
4299 // We detect this in three ways:
4300 // (A) If LHS recursed, its truth table is valid against the Src state
4301 // when LHS recursion completed (SrcAfterLHS). If RHS recursion
4302 // then mutates a Src slot that LHSBits depends on, LHSBits is
4303 // stale.
4304 // (B) If RHS did not recurse, RHSBits came from getOperandBits and
4305 // refers to a specific Src slot. If that slot's contents changed
4306 // (by either recursion), RHSBits is stale.
4307 // (C) Symmetrically for LHS if it did not recurse.
4308 SmallVector<Register, 3> SrcBeforeRecurse(Src.begin(), Src.end());
4309 uint8_t LHSBitsOrig = LHSBits;
4310 uint8_t RHSBitsOrig = RHSBits;
4311
4312 auto LHSOp = BitOp3_Op(R: LHS, Src, MRI);
4313 if (LHSOp.first) {
4314 NumOpcodes += LHSOp.first;
4315 LHSBits = LHSOp.second;
4316 }
4317
4318 SmallVector<Register, 3> SrcAfterLHS(Src.begin(), Src.end());
4319
4320 auto RHSOp = BitOp3_Op(R: RHS, Src, MRI);
4321 if (RHSOp.first) {
4322 NumOpcodes += RHSOp.first;
4323 RHSBits = RHSOp.second;
4324 }
4325
4326 // dependsOnSlot: true iff the truth table TT varies with slot Slot.
4327 auto dependsOnSlot = [](uint8_t TT, int Slot) -> bool {
4328 if (Slot < 0 || Slot > 2)
4329 return false;
4330 const uint8_t Masks[3] = {0x0f, 0x33, 0x55};
4331 const int Shifts[3] = {4, 2, 1};
4332 return ((TT ^ (TT >> Shifts[Slot])) & Masks[Slot]) != 0;
4333 };
4334
4335 // findSlot: locate the Src slot a getOperandBits result depends on,
4336 // including negated (NOT) patterns that getOperandBits resolves via
4337 // the ~SrcBits[I] shortcut.
4338 const uint8_t SrcBitsConst[3] = {0xf0, 0xcc, 0xaa};
4339 auto findSlot = [&](uint8_t Bits, Register Op,
4340 const SmallVectorImpl<Register> &S) -> int {
4341 Register NegatedInner;
4342 bool IsNegationOp = mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: NegatedInner)));
4343 if (IsNegationOp)
4344 NegatedInner = getSrcRegIgnoringCopies(Reg: NegatedInner, MRI);
4345 for (int I = 0; I < (int)S.size(); I++) {
4346 if (Bits == SrcBitsConst[I] && S[I] == Op)
4347 return I;
4348 if (IsNegationOp && Bits == (uint8_t)~SrcBitsConst[I] &&
4349 S[I] == NegatedInner)
4350 return I;
4351 }
4352 return -1;
4353 };
4354
4355 bool Stale = false;
4356
4357 // (A) LHS recursed: its truth table is against SrcAfterLHS.
4358 // Check if RHS recursion mutated a slot that LHSBits uses.
4359 if (LHSOp.first) {
4360 for (int I = 0; I < (int)SrcAfterLHS.size() && I < 3; I++) {
4361 if (I < (int)Src.size() && Src[I] != SrcAfterLHS[I] &&
4362 dependsOnSlot(LHSBits, I)) {
4363 Stale = true;
4364 break;
4365 }
4366 }
4367 }
4368
4369 // (B) RHS did not recurse: RHSBits from getOperandBits is against
4370 // SrcBeforeRecurse. Check if that slot was mutated since then.
4371 if (!Stale && !RHSOp.first) {
4372 int Slot = findSlot(RHSBitsOrig, RHS, SrcBeforeRecurse);
4373 if (Slot >= 0 &&
4374 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4375 Stale = true;
4376 }
4377
4378 // (C) LHS did not recurse: LHSBits from getOperandBits is against
4379 // SrcBeforeRecurse. Check if that slot was mutated since then.
4380 if (!Stale && !LHSOp.first) {
4381 int Slot = findSlot(LHSBitsOrig, LHS, SrcBeforeRecurse);
4382 if (Slot >= 0 &&
4383 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4384 Stale = true;
4385 }
4386
4387 if (Stale) {
4388 Src = std::move(SrcBeforeRecurse);
4389 LHSBits = LHSBitsOrig;
4390 RHSBits = RHSBitsOrig;
4391 NumOpcodes = 0;
4392 }
4393 break;
4394 }
4395 default:
4396 return std::make_pair(x: 0, y: 0);
4397 }
4398
4399 uint8_t TTbl;
4400 switch (MI->getOpcode()) {
4401 case TargetOpcode::G_AND:
4402 TTbl = LHSBits & RHSBits;
4403 break;
4404 case TargetOpcode::G_OR:
4405 TTbl = LHSBits | RHSBits;
4406 break;
4407 case TargetOpcode::G_XOR:
4408 TTbl = LHSBits ^ RHSBits;
4409 break;
4410 default:
4411 break;
4412 }
4413
4414 return std::make_pair(x: NumOpcodes + 1, y&: TTbl);
4415}
4416
4417bool AMDGPUInstructionSelector::selectBITOP3(MachineInstr &MI) const {
4418 if (!Subtarget->hasBitOp3Insts())
4419 return false;
4420
4421 Register DstReg = MI.getOperand(i: 0).getReg();
4422 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4423 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4424 if (!IsVALU)
4425 return false;
4426
4427 SmallVector<Register, 3> Src;
4428 uint8_t TTbl;
4429 unsigned NumOpcodes;
4430
4431 std::tie(args&: NumOpcodes, args&: TTbl) = BitOp3_Op(R: DstReg, Src, MRI: *MRI);
4432
4433 // Src.empty() case can happen if all operands are all zero or all ones.
4434 // Normally it shall be optimized out before reaching this.
4435 if (NumOpcodes < 2 || Src.empty())
4436 return false;
4437
4438 // RegBankSelect splits wider VALU logic ops and widens 1-bit ones, so only
4439 // 16 and 32 bit types reach here. Note that <2 x i16> is 32 bits wide.
4440 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
4441 assert((Size == 16 || Size == 32) && "unexpected VALU logic op size");
4442 const bool IsB32 = Size == 32;
4443 if (NumOpcodes == 2 && IsB32) {
4444 // Avoid using BITOP3 for OR3, XOR3, AND_OR. This is not faster but makes
4445 // asm more readable. This cannot be modeled with AddedComplexity because
4446 // selector does not know how many operations did we match.
4447 if (mi_match(MI, MRI: *MRI, P: m_GXor(L: m_GXor(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4448 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GOr(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4449 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GAnd(L: m_Reg(), R: m_Reg()), R: m_Reg())))
4450 return false;
4451 } else if (NumOpcodes < 4) {
4452 // For a uniform case threshold should be higher to account for moves
4453 // between VGPRs and SGPRs. It needs one operand in a VGPR, rest two can be
4454 // in SGPRs and a readtfirstlane after.
4455 return false;
4456 }
4457
4458 unsigned Opc = IsB32 ? AMDGPU::V_BITOP3_B32_e64 : AMDGPU::V_BITOP3_B16_e64;
4459 if (!IsB32 && STI.hasTrue16BitInsts())
4460 Opc = STI.useRealTrue16Insts() ? AMDGPU::V_BITOP3_B16_gfx1250_t16_e64
4461 : AMDGPU::V_BITOP3_B16_gfx1250_fake16_e64;
4462 unsigned CBL = STI.getConstantBusLimit(Opcode: Opc);
4463 MachineBasicBlock *MBB = MI.getParent();
4464 const DebugLoc &DL = MI.getDebugLoc();
4465
4466 for (unsigned I = 0; I < Src.size(); ++I) {
4467 const RegisterBank *RB = RBI.getRegBank(Reg: Src[I], MRI: *MRI, TRI);
4468 if (RB->getID() != AMDGPU::SGPRRegBankID)
4469 continue;
4470 if (CBL > 0) {
4471 --CBL;
4472 continue;
4473 }
4474 Register NewReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
4475 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: NewReg)
4476 .addReg(RegNo: Src[I]);
4477 Src[I] = NewReg;
4478 }
4479
4480 // Last operand can be ignored, turning a ternary operation into a binary.
4481 // For example: (~a & b & c) | (~a & b & ~c) -> (~a & b). We can replace
4482 // 'c' with 'a' here without changing the answer. In some pathological
4483 // cases it should be possible to get an operation with a single operand
4484 // too if optimizer would not catch it.
4485 while (Src.size() < 3)
4486 Src.push_back(Elt: Src[0]);
4487
4488 auto MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg);
4489 if (!IsB32)
4490 MIB.addImm(Val: 0); // src_mod0
4491 MIB.addReg(RegNo: Src[0]);
4492 if (!IsB32)
4493 MIB.addImm(Val: 0); // src_mod1
4494 MIB.addReg(RegNo: Src[1]);
4495 if (!IsB32)
4496 MIB.addImm(Val: 0); // src_mod2
4497 MIB.addReg(RegNo: Src[2])
4498 .addImm(Val: TTbl);
4499 if (!IsB32)
4500 MIB.addImm(Val: 0); // op_sel
4501
4502 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
4503 MI.eraseFromParent();
4504
4505 return true;
4506}
4507
4508bool AMDGPUInstructionSelector::selectWriteRegister(MachineInstr &MI) const {
4509 const MDString *RegStr =
4510 cast<MDString>(Val: MI.getOperand(i: 0).getMetadata()->getOperand(I: 0));
4511 Register SrcReg = MI.getOperand(i: 1).getReg();
4512 LLT Ty = MRI->getType(Reg: SrcReg);
4513
4514 Register PhysReg = Subtarget->getTargetLowering()->getRegisterByName(
4515 RegName: RegStr->getString().data(), VT: Ty, MF: *MF);
4516 if (!PhysReg) {
4517 const Function &Fn = MF->getFunction();
4518 Fn.getContext().diagnose(DI: DiagnosticInfoGenericWithLoc(
4519 "invalid register \"" + Twine(RegStr->getString()) +
4520 "\" for llvm.write_register",
4521 Fn, MI.getDebugLoc()));
4522 MI.eraseFromParent();
4523 return true;
4524 }
4525
4526 if (!RBI.constrainGenericRegister(
4527 Reg: SrcReg, RC: *TRI.getSGPRClassForBitWidth(BitWidth: Ty.getSizeInBits()), MRI&: *MRI))
4528 return false;
4529
4530 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhysReg)
4531 .addReg(RegNo: SrcReg);
4532 MI.eraseFromParent();
4533 return true;
4534}
4535
4536bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const {
4537 Register SrcReg = MI.getOperand(i: 0).getReg();
4538 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
4539 return false;
4540
4541 MachineInstr *DefMI = MRI->getVRegDef(Reg: SrcReg);
4542 Register SP =
4543 Subtarget->getTargetLowering()->getStackPointerRegisterToSaveRestore();
4544 Register WaveAddr = getWaveAddress(Def: DefMI);
4545 MachineBasicBlock *MBB = MI.getParent();
4546 const DebugLoc &DL = MI.getDebugLoc();
4547
4548 if (!WaveAddr) {
4549 WaveAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
4550 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: WaveAddr)
4551 .addReg(RegNo: SrcReg)
4552 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4553 .setOperandDead(3); // Dead scc
4554 }
4555
4556 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: SP)
4557 .addReg(RegNo: WaveAddr);
4558
4559 MI.eraseFromParent();
4560 return true;
4561}
4562
4563bool AMDGPUInstructionSelector::select(MachineInstr &I) {
4564
4565 if (!I.isPreISelOpcode()) {
4566 if (I.isCopy())
4567 return selectCOPY(I);
4568 return true;
4569 }
4570
4571 switch (I.getOpcode()) {
4572 case TargetOpcode::G_AND:
4573 case TargetOpcode::G_OR:
4574 case TargetOpcode::G_XOR:
4575 if (selectBITOP3(MI&: I))
4576 return true;
4577 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4578 return true;
4579 return selectG_AND_OR_XOR(I);
4580 case TargetOpcode::G_ADD:
4581 case TargetOpcode::G_SUB:
4582 case TargetOpcode::G_PTR_ADD:
4583 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4584 return true;
4585 return selectG_ADD_SUB(I);
4586 case TargetOpcode::G_UADDO:
4587 case TargetOpcode::G_USUBO:
4588 case TargetOpcode::G_UADDE:
4589 case TargetOpcode::G_USUBE:
4590 return selectG_UADDO_USUBO_UADDE_USUBE(I);
4591 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4592 case AMDGPU::G_AMDGPU_MAD_I64_I32:
4593 return selectG_AMDGPU_MAD_64_32(I);
4594 case TargetOpcode::G_INTTOPTR:
4595 case TargetOpcode::G_BITCAST:
4596 case TargetOpcode::G_PTRTOINT:
4597 case TargetOpcode::G_FREEZE:
4598 return selectCOPY(I);
4599 case TargetOpcode::G_FNEG:
4600 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4601 return true;
4602 return selectG_FNEG(MI&: I);
4603 case TargetOpcode::G_FABS:
4604 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4605 return true;
4606 return selectG_FABS(MI&: I);
4607 case TargetOpcode::G_EXTRACT:
4608 return selectG_EXTRACT(I);
4609 case TargetOpcode::G_MERGE_VALUES:
4610 case TargetOpcode::G_CONCAT_VECTORS:
4611 return selectG_MERGE_VALUES(MI&: I);
4612 case TargetOpcode::G_UNMERGE_VALUES:
4613 return selectG_UNMERGE_VALUES(MI&: I);
4614 case TargetOpcode::G_BUILD_VECTOR:
4615 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
4616 return selectG_BUILD_VECTOR(MI&: I);
4617 case TargetOpcode::G_IMPLICIT_DEF:
4618 return selectG_IMPLICIT_DEF(I);
4619 case TargetOpcode::G_INSERT:
4620 return selectG_INSERT(I);
4621 case TargetOpcode::G_INTRINSIC:
4622 case TargetOpcode::G_INTRINSIC_CONVERGENT:
4623 return selectG_INTRINSIC(I);
4624 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
4625 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
4626 return selectG_INTRINSIC_W_SIDE_EFFECTS(I);
4627 case TargetOpcode::G_ICMP:
4628 case TargetOpcode::G_FCMP:
4629 if (selectG_ICMP_or_FCMP(I))
4630 return true;
4631 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4632 case TargetOpcode::G_LOAD:
4633 case TargetOpcode::G_ZEXTLOAD:
4634 case TargetOpcode::G_SEXTLOAD:
4635 case TargetOpcode::G_STORE:
4636 case TargetOpcode::G_ATOMIC_CMPXCHG:
4637 case TargetOpcode::G_ATOMICRMW_XCHG:
4638 case TargetOpcode::G_ATOMICRMW_ADD:
4639 case TargetOpcode::G_ATOMICRMW_SUB:
4640 case TargetOpcode::G_ATOMICRMW_AND:
4641 case TargetOpcode::G_ATOMICRMW_OR:
4642 case TargetOpcode::G_ATOMICRMW_XOR:
4643 case TargetOpcode::G_ATOMICRMW_MIN:
4644 case TargetOpcode::G_ATOMICRMW_MAX:
4645 case TargetOpcode::G_ATOMICRMW_UMIN:
4646 case TargetOpcode::G_ATOMICRMW_UMAX:
4647 case TargetOpcode::G_ATOMICRMW_UINC_WRAP:
4648 case TargetOpcode::G_ATOMICRMW_UDEC_WRAP:
4649 case TargetOpcode::G_ATOMICRMW_USUB_COND:
4650 case TargetOpcode::G_ATOMICRMW_USUB_SAT:
4651 case TargetOpcode::G_ATOMICRMW_FADD:
4652 case TargetOpcode::G_ATOMICRMW_FMIN:
4653 case TargetOpcode::G_ATOMICRMW_FMAX:
4654 return selectG_LOAD_STORE_ATOMICRMW(I);
4655 case TargetOpcode::G_SELECT:
4656 return selectG_SELECT(I);
4657 case TargetOpcode::G_TRUNC:
4658 return selectG_TRUNC(I);
4659 case TargetOpcode::G_SEXT:
4660 case TargetOpcode::G_ZEXT:
4661 case TargetOpcode::G_ANYEXT:
4662 case TargetOpcode::G_SEXT_INREG:
4663 // This is a workaround. For extension from type i1, `selectImpl()` uses
4664 // patterns from TD file and generates an illegal VGPR to SGPR COPY as type
4665 // i1 can only be hold in a SGPR class.
4666 if (MRI->getType(Reg: I.getOperand(i: 1).getReg()) != LLT::scalar(SizeInBits: 1) &&
4667 selectImpl(I, CoverageInfo&: *CoverageInfo))
4668 return true;
4669 return selectG_SZA_EXT(I);
4670 case TargetOpcode::G_FPEXT:
4671 if (selectG_FPEXT(I))
4672 return true;
4673 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4674 case TargetOpcode::G_BRCOND:
4675 return selectG_BRCOND(I);
4676 case TargetOpcode::G_GLOBAL_VALUE:
4677 return selectG_GLOBAL_VALUE(I);
4678 case TargetOpcode::G_PTRMASK:
4679 return selectG_PTRMASK(I);
4680 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4681 return selectG_EXTRACT_VECTOR_ELT(MI&: I);
4682 case TargetOpcode::G_INSERT_VECTOR_ELT:
4683 return selectG_INSERT_VECTOR_ELT(MI&: I);
4684 case TargetOpcode::G_FMA: {
4685 if (!STI.hasPkF32InvalidOpSel())
4686 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4687
4688 Register VReg = I.getOperand(i: 0).getReg();
4689 if (!selectImpl(I, CoverageInfo&: *CoverageInfo))
4690 return false;
4691
4692 MachineInstr *Sel = MRI->getVRegDef(Reg: VReg);
4693 // Fix GFX950 v_pk_fma_f32 invalid op_sel by decomposing.
4694 if (Sel && Sel->getOpcode() == AMDGPU::V_PK_FMA_F32 &&
4695 !TII.isLegalPkFMAF32OpSel(MI: *Sel))
4696 TII.decomposePkFMAF32(MI&: *Sel, MRI&: *MRI);
4697
4698 return true;
4699 }
4700 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
4701 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
4702 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
4703 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
4704 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
4705 const AMDGPU::ImageDimIntrinsicInfo *Intr =
4706 AMDGPU::getImageDimIntrinsicInfo(Intr: AMDGPU::getIntrinsicID(I));
4707 assert(Intr && "not an image intrinsic with image pseudo");
4708 return selectImageIntrinsic(MI&: I, Intr);
4709 }
4710 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY:
4711 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
4712 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
4713 return selectBVHIntersectRayIntrinsic(MI&: I);
4714 case AMDGPU::G_SBFX:
4715 case AMDGPU::G_UBFX:
4716 return selectG_SBFX_UBFX(MI&: I);
4717 case AMDGPU::G_SI_CALL:
4718 I.setDesc(TII.get(Opcode: AMDGPU::SI_CALL));
4719 return true;
4720 case AMDGPU::G_AMDGPU_WAVE_ADDRESS:
4721 return selectWaveAddress(MI&: I);
4722 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN: {
4723 I.setDesc(TII.get(Opcode: AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN));
4724 return true;
4725 }
4726 case AMDGPU::G_STACKRESTORE:
4727 return selectStackRestore(MI&: I);
4728 case TargetOpcode::G_WRITE_REGISTER:
4729 return selectWriteRegister(MI&: I);
4730 case AMDGPU::G_PHI:
4731 return selectPHI(I);
4732 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
4733 return selectCOPY_SCC_VCC(I);
4734 case AMDGPU::G_AMDGPU_COPY_VCC_SCC:
4735 return selectCOPY_VCC_SCC(I);
4736 case AMDGPU::G_AMDGPU_READANYLANE:
4737 return selectReadAnyLane(I);
4738 case TargetOpcode::G_CONSTANT:
4739 case TargetOpcode::G_FCONSTANT:
4740 default:
4741 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4742 }
4743 return false;
4744}
4745
4746InstructionSelector::ComplexRendererFns
4747AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const {
4748 return {{
4749 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4750 }};
4751
4752}
4753
4754std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3ModsImpl(
4755 Register Src, bool IsCanonicalizing, bool AllowAbs, bool OpSel) const {
4756 unsigned Mods = 0;
4757 MachineInstr *MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4758
4759 if (MI->getOpcode() == AMDGPU::G_FNEG) {
4760 Src = MI->getOperand(i: 1).getReg();
4761 Mods |= SISrcMods::NEG;
4762 MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4763 } else if (MI->getOpcode() == AMDGPU::G_FSUB && IsCanonicalizing) {
4764 // Fold fsub [+-]0 into fneg. This may not have folded depending on the
4765 // denormal mode, but we're implicitly canonicalizing in a source operand.
4766 const ConstantFP *LHS =
4767 getConstantFPVRegVal(VReg: MI->getOperand(i: 1).getReg(), MRI: *MRI);
4768 if (LHS && LHS->isZero()) {
4769 Mods |= SISrcMods::NEG;
4770 Src = MI->getOperand(i: 2).getReg();
4771 }
4772 }
4773
4774 if (AllowAbs && MI->getOpcode() == AMDGPU::G_FABS) {
4775 Src = MI->getOperand(i: 1).getReg();
4776 Mods |= SISrcMods::ABS;
4777 }
4778
4779 if (OpSel)
4780 Mods |= SISrcMods::OP_SEL_0;
4781
4782 return std::pair(Src, Mods);
4783}
4784
4785std::pair<Register, unsigned>
4786AMDGPUInstructionSelector::selectVOP3PModsF32Impl(Register Src) const {
4787 unsigned Mods;
4788 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src);
4789 Mods |= SISrcMods::OP_SEL_1;
4790 return std::pair(Src, Mods);
4791}
4792
4793Register AMDGPUInstructionSelector::copyToVGPRIfSrcFolded(
4794 Register Src, unsigned Mods, MachineOperand Root, MachineInstr *InsertPt,
4795 bool ForceVGPR) const {
4796 if ((Mods != 0 || ForceVGPR) &&
4797 RBI.getRegBank(Reg: Src, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) {
4798
4799 // If we looked through copies to find source modifiers on an SGPR operand,
4800 // we now have an SGPR register source. To avoid potentially violating the
4801 // constant bus restriction, we need to insert a copy to a VGPR.
4802 Register VGPRSrc = MRI->cloneVirtualRegister(VReg: Root.getReg());
4803 BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
4804 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VGPRSrc)
4805 .addReg(RegNo: Src);
4806 Src = VGPRSrc;
4807 }
4808
4809 return Src;
4810}
4811
4812/// Some instructions must have 32-bit sources. With real true16 instructions a
4813/// 16-bit VALU value lives in a VGPR_16, which they cannot read, so place it in
4814/// the low half of a new 32-bit VGPR.
4815Register
4816AMDGPUInstructionSelector::widenSrcIfVGPR16(Register Src,
4817 MachineInstr *InsertPt) const {
4818 if (!Subtarget->useRealTrue16Insts() || MRI->getType(Reg: Src) != LLT::scalar(SizeInBits: 16))
4819 return Src;
4820
4821 const RegisterBank *SrcRB = RBI.getRegBank(Reg: Src, MRI: *MRI, TRI);
4822 if (!SrcRB || SrcRB->getID() != AMDGPU::VGPRRegBankID)
4823 return Src;
4824
4825 MachineIRBuilder B(*InsertPt);
4826
4827 Register ImpDefReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
4828 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF).addDef(RegNo: ImpDefReg);
4829
4830 Register DstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
4831 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
4832 .addDef(RegNo: DstReg)
4833 .addReg(RegNo: Src)
4834 .addImm(Val: AMDGPU::lo16)
4835 .addReg(RegNo: ImpDefReg)
4836 .addImm(Val: AMDGPU::hi16);
4837
4838 return DstReg;
4839}
4840
4841///
4842/// This will select either an SGPR or VGPR operand and will save us from
4843/// having to write an extra tablegen pattern.
4844InstructionSelector::ComplexRendererFns
4845AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const {
4846 return {{
4847 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4848 }};
4849}
4850
4851InstructionSelector::ComplexRendererFns
4852AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const {
4853 Register Src;
4854 unsigned Mods;
4855 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4856
4857 return {{
4858 [=](MachineInstrBuilder &MIB) {
4859 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4860 },
4861 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4862 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4863 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4864 }};
4865}
4866
4867InstructionSelector::ComplexRendererFns
4868AMDGPUInstructionSelector::selectVOP3BMods0(MachineOperand &Root) const {
4869 Register Src;
4870 unsigned Mods;
4871 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
4872 /*IsCanonicalizing=*/true,
4873 /*AllowAbs=*/false);
4874
4875 return {{
4876 [=](MachineInstrBuilder &MIB) {
4877 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4878 },
4879 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4880 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4881 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4882 }};
4883}
4884
4885InstructionSelector::ComplexRendererFns
4886AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const {
4887 return {{
4888 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); },
4889 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4890 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4891 }};
4892}
4893
4894InstructionSelector::ComplexRendererFns
4895AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const {
4896 Register Src;
4897 unsigned Mods;
4898 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4899
4900 return {{
4901 [=](MachineInstrBuilder &MIB) {
4902 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4903 },
4904 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4905 }};
4906}
4907
4908InstructionSelector::ComplexRendererFns
4909AMDGPUInstructionSelector::selectVOP3ModsNonCanonicalizing(
4910 MachineOperand &Root) const {
4911 Register Src;
4912 unsigned Mods;
4913 std::tie(args&: Src, args&: Mods) =
4914 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/false);
4915
4916 return {{
4917 [=](MachineInstrBuilder &MIB) {
4918 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4919 },
4920 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4921 }};
4922}
4923
4924InstructionSelector::ComplexRendererFns
4925AMDGPUInstructionSelector::selectVOP3BMods(MachineOperand &Root) const {
4926 Register Src;
4927 unsigned Mods;
4928 std::tie(args&: Src, args&: Mods) =
4929 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/true,
4930 /*AllowAbs=*/false);
4931
4932 return {{
4933 [=](MachineInstrBuilder &MIB) {
4934 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4935 },
4936 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4937 }};
4938}
4939
4940InstructionSelector::ComplexRendererFns
4941AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
4942 Register Reg = Root.getReg();
4943 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
4944 if (Def->getOpcode() == AMDGPU::G_FNEG || Def->getOpcode() == AMDGPU::G_FABS)
4945 return {};
4946 return {{
4947 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
4948 }};
4949}
4950
4951enum class SrcStatus {
4952 IS_SAME,
4953 IS_UPPER_HALF,
4954 IS_LOWER_HALF,
4955 IS_UPPER_HALF_NEG,
4956 // This means current op = [op_upper, op_lower] and src = -op_lower.
4957 IS_LOWER_HALF_NEG,
4958 IS_HI_NEG,
4959 // This means current op = [op_upper, op_lower] and src = [op_upper,
4960 // -op_lower].
4961 IS_LO_NEG,
4962 IS_BOTH_NEG,
4963 INVALID,
4964 NEG_START = IS_UPPER_HALF_NEG,
4965 NEG_END = IS_BOTH_NEG,
4966 HALF_START = IS_UPPER_HALF,
4967 HALF_END = IS_LOWER_HALF_NEG
4968};
4969/// Test if the MI is truncating to half, such as `%reg0:n = G_TRUNC %reg1:2n`
4970static bool isTruncHalf(const MachineInstr *MI,
4971 const MachineRegisterInfo &MRI) {
4972 if (MI->getOpcode() != AMDGPU::G_TRUNC)
4973 return false;
4974
4975 unsigned DstSize = MRI.getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
4976 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4977 return DstSize * 2 == SrcSize;
4978}
4979
4980/// Test if the MI is logic shift right with half bits,
4981/// such as `%reg0:2n =G_LSHR %reg1:2n, CONST(n)`
4982static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
4983 if (MI->getOpcode() != AMDGPU::G_LSHR)
4984 return false;
4985
4986 Register ShiftSrc;
4987 std::optional<ValueAndVReg> ShiftAmt;
4988 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
4989 P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
4990 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4991 unsigned Shift = ShiftAmt->Value.getZExtValue();
4992 return Shift * 2 == SrcSize;
4993 }
4994 return false;
4995}
4996
4997/// Test if the MI is shift left with half bits,
4998/// such as `%reg0:2n =G_SHL %reg1:2n, CONST(n)`
4999static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
5000 if (MI->getOpcode() != AMDGPU::G_SHL)
5001 return false;
5002
5003 Register ShiftSrc;
5004 std::optional<ValueAndVReg> ShiftAmt;
5005 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
5006 P: m_GShl(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
5007 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
5008 unsigned Shift = ShiftAmt->Value.getZExtValue();
5009 return Shift * 2 == SrcSize;
5010 }
5011 return false;
5012}
5013
5014/// Test function, if the MI is `%reg0:n, %reg1:n = G_UNMERGE_VALUES %reg2:2n`
5015static bool isUnmergeHalf(const MachineInstr *MI,
5016 const MachineRegisterInfo &MRI) {
5017 if (MI->getOpcode() != AMDGPU::G_UNMERGE_VALUES)
5018 return false;
5019 return MI->getNumOperands() == 3 && MI->getOperand(i: 0).isDef() &&
5020 MI->getOperand(i: 1).isDef() && !MI->getOperand(i: 2).isDef();
5021}
5022
5023enum class TypeClass { VECTOR_OF_TWO, SCALAR, NONE_OF_LISTED };
5024
5025static TypeClass isVectorOfTwoOrScalar(Register Reg,
5026 const MachineRegisterInfo &MRI) {
5027 LLT OpTy = MRI.getType(Reg);
5028 if (OpTy.isScalar())
5029 return TypeClass::SCALAR;
5030 if (OpTy.isVector() && OpTy.getNumElements() == 2)
5031 return TypeClass::VECTOR_OF_TWO;
5032 return TypeClass::NONE_OF_LISTED;
5033}
5034
5035static SrcStatus getNegStatus(Register Reg, SrcStatus S,
5036 const MachineRegisterInfo &MRI) {
5037 TypeClass NegType = isVectorOfTwoOrScalar(Reg, MRI);
5038 if (NegType != TypeClass::VECTOR_OF_TWO && NegType != TypeClass::SCALAR)
5039 return SrcStatus::INVALID;
5040
5041 switch (S) {
5042 case SrcStatus::IS_SAME:
5043 if (NegType == TypeClass::VECTOR_OF_TWO) {
5044 // Vector of 2:
5045 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
5046 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
5047 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5048 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
5049 return SrcStatus::IS_BOTH_NEG;
5050 }
5051 if (NegType == TypeClass::SCALAR) {
5052 // Scalar:
5053 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
5054 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
5055 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5056 // [SrcHi, SrcLo] = [-OpHi, OpLo]
5057 return SrcStatus::IS_HI_NEG;
5058 }
5059 break;
5060 case SrcStatus::IS_HI_NEG:
5061 if (NegType == TypeClass::VECTOR_OF_TWO) {
5062 // Vector of 2:
5063 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5064 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
5065 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5066 // [SrcHi, SrcLo] = [-(-OpHi), -OpLo] = [OpHi, -OpLo]
5067 return SrcStatus::IS_LO_NEG;
5068 }
5069 if (NegType == TypeClass::SCALAR) {
5070 // Scalar:
5071 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5072 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
5073 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5074 // [SrcHi, SrcLo] = [-(-OpHi), OpLo] = [OpHi, OpLo]
5075 return SrcStatus::IS_SAME;
5076 }
5077 break;
5078 case SrcStatus::IS_LO_NEG:
5079 if (NegType == TypeClass::VECTOR_OF_TWO) {
5080 // Vector of 2:
5081 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5082 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5083 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5084 // [SrcHi, SrcLo] = [-OpHi, -(-OpLo)] = [-OpHi, OpLo]
5085 return SrcStatus::IS_HI_NEG;
5086 }
5087 if (NegType == TypeClass::SCALAR) {
5088 // Scalar:
5089 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5090 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5091 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5092 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
5093 return SrcStatus::IS_BOTH_NEG;
5094 }
5095 break;
5096 case SrcStatus::IS_BOTH_NEG:
5097 if (NegType == TypeClass::VECTOR_OF_TWO) {
5098 // Vector of 2:
5099 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5100 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5101 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5102 // [SrcHi, SrcLo] = [OpHi, OpLo]
5103 return SrcStatus::IS_SAME;
5104 }
5105 if (NegType == TypeClass::SCALAR) {
5106 // Scalar:
5107 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5108 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5109 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5110 // [SrcHi, SrcLo] = [OpHi, -OpLo]
5111 return SrcStatus::IS_LO_NEG;
5112 }
5113 break;
5114 case SrcStatus::IS_UPPER_HALF:
5115 // Vector of 2:
5116 // Src = CurrUpper
5117 // Curr = [CurrUpper, CurrLower]
5118 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5119 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5120 // Src = -OpUpper
5121 //
5122 // Scalar:
5123 // Src = CurrUpper
5124 // Curr = [CurrUpper, CurrLower]
5125 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5126 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5127 // Src = -OpUpper
5128 return SrcStatus::IS_UPPER_HALF_NEG;
5129 case SrcStatus::IS_LOWER_HALF:
5130 if (NegType == TypeClass::VECTOR_OF_TWO) {
5131 // Vector of 2:
5132 // Src = CurrLower
5133 // Curr = [CurrUpper, CurrLower]
5134 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5135 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5136 // Src = -OpLower
5137 return SrcStatus::IS_LOWER_HALF_NEG;
5138 }
5139 if (NegType == TypeClass::SCALAR) {
5140 // Scalar:
5141 // Src = CurrLower
5142 // Curr = [CurrUpper, CurrLower]
5143 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5144 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5145 // Src = OpLower
5146 return SrcStatus::IS_LOWER_HALF;
5147 }
5148 break;
5149 case SrcStatus::IS_UPPER_HALF_NEG:
5150 // Vector of 2:
5151 // Src = -CurrUpper
5152 // Curr = [CurrUpper, CurrLower]
5153 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5154 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5155 // Src = -(-OpUpper) = OpUpper
5156 //
5157 // Scalar:
5158 // Src = -CurrUpper
5159 // Curr = [CurrUpper, CurrLower]
5160 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5161 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5162 // Src = -(-OpUpper) = OpUpper
5163 return SrcStatus::IS_UPPER_HALF;
5164 case SrcStatus::IS_LOWER_HALF_NEG:
5165 if (NegType == TypeClass::VECTOR_OF_TWO) {
5166 // Vector of 2:
5167 // Src = -CurrLower
5168 // Curr = [CurrUpper, CurrLower]
5169 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5170 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5171 // Src = -(-OpLower) = OpLower
5172 return SrcStatus::IS_LOWER_HALF;
5173 }
5174 if (NegType == TypeClass::SCALAR) {
5175 // Scalar:
5176 // Src = -CurrLower
5177 // Curr = [CurrUpper, CurrLower]
5178 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5179 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5180 // Src = -OpLower
5181 return SrcStatus::IS_LOWER_HALF_NEG;
5182 }
5183 break;
5184 default:
5185 break;
5186 }
5187 llvm_unreachable("unexpected SrcStatus & NegType combination");
5188}
5189
5190static std::optional<std::pair<Register, SrcStatus>>
5191calcNextStatus(std::pair<Register, SrcStatus> Curr,
5192 const MachineRegisterInfo &MRI) {
5193 const MachineInstr *MI = MRI.getVRegDef(Reg: Curr.first);
5194
5195 unsigned Opc = MI->getOpcode();
5196
5197 // Handle general Opc cases.
5198 switch (Opc) {
5199 case AMDGPU::G_BITCAST:
5200 return std::optional<std::pair<Register, SrcStatus>>(
5201 {MI->getOperand(i: 1).getReg(), Curr.second});
5202 case AMDGPU::COPY:
5203 if (MI->getOperand(i: 1).getReg().isPhysical())
5204 return std::nullopt;
5205 return std::optional<std::pair<Register, SrcStatus>>(
5206 {MI->getOperand(i: 1).getReg(), Curr.second});
5207 case AMDGPU::G_FNEG: {
5208 SrcStatus Stat = getNegStatus(Reg: Curr.first, S: Curr.second, MRI);
5209 if (Stat == SrcStatus::INVALID)
5210 return std::nullopt;
5211 return std::optional<std::pair<Register, SrcStatus>>(
5212 {MI->getOperand(i: 1).getReg(), Stat});
5213 }
5214 default:
5215 break;
5216 }
5217
5218 // Calc next Stat from current Stat.
5219 switch (Curr.second) {
5220 case SrcStatus::IS_SAME:
5221 if (isTruncHalf(MI, MRI))
5222 return std::optional<std::pair<Register, SrcStatus>>(
5223 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5224 else if (isUnmergeHalf(MI, MRI)) {
5225 if (Curr.first == MI->getOperand(i: 0).getReg())
5226 return std::optional<std::pair<Register, SrcStatus>>(
5227 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF});
5228 return std::optional<std::pair<Register, SrcStatus>>(
5229 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF});
5230 }
5231 break;
5232 case SrcStatus::IS_HI_NEG:
5233 if (isTruncHalf(MI, MRI)) {
5234 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5235 // [CurrHi, CurrLo] = trunc [OpUpper, OpLower] = OpLower
5236 // = [OpLowerHi, OpLowerLo]
5237 // Src = [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5238 // = [-OpLowerHi, OpLowerLo]
5239 // = -OpLower
5240 return std::optional<std::pair<Register, SrcStatus>>(
5241 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5242 }
5243 if (isUnmergeHalf(MI, MRI)) {
5244 if (Curr.first == MI->getOperand(i: 0).getReg())
5245 return std::optional<std::pair<Register, SrcStatus>>(
5246 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5247 return std::optional<std::pair<Register, SrcStatus>>(
5248 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5249 }
5250 break;
5251 case SrcStatus::IS_UPPER_HALF:
5252 if (isShlHalf(MI, MRI))
5253 return std::optional<std::pair<Register, SrcStatus>>(
5254 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5255 break;
5256 case SrcStatus::IS_LOWER_HALF:
5257 if (isLshrHalf(MI, MRI))
5258 return std::optional<std::pair<Register, SrcStatus>>(
5259 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF});
5260 break;
5261 case SrcStatus::IS_UPPER_HALF_NEG:
5262 if (isShlHalf(MI, MRI))
5263 return std::optional<std::pair<Register, SrcStatus>>(
5264 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5265 break;
5266 case SrcStatus::IS_LOWER_HALF_NEG:
5267 if (isLshrHalf(MI, MRI))
5268 return std::optional<std::pair<Register, SrcStatus>>(
5269 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5270 break;
5271 default:
5272 break;
5273 }
5274 return std::nullopt;
5275}
5276
5277/// This is used to control valid status that current MI supports. For example,
5278/// non floating point intrinsic such as @llvm.amdgcn.sdot2 does not support NEG
5279/// bit on VOP3P.
5280/// The class can be further extended to recognize support on SEL, NEG, ABS bit
5281/// for different MI on different arch
5282class SearchOptions {
5283private:
5284 bool HasNeg = false;
5285 // Assume all complex pattern of VOP3P have opsel.
5286 bool HasOpsel = true;
5287
5288public:
5289 SearchOptions(Register Reg, const MachineRegisterInfo &MRI) {
5290 const MachineInstr *MI = MRI.getVRegDef(Reg);
5291 unsigned Opc = MI->getOpcode();
5292
5293 if (Opc == TargetOpcode::G_INTRINSIC) {
5294 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: *MI).getIntrinsicID();
5295 // Only float point intrinsic has neg & neg_hi bits.
5296 if (IntrinsicID == Intrinsic::amdgcn_fdot2)
5297 HasNeg = true;
5298 } else if (TargetInstrInfo::isGenericOpcode(Opc)) {
5299 // Keep same for generic op.
5300 HasNeg = true;
5301 }
5302 }
5303 bool checkOptions(SrcStatus Stat) const {
5304 if (!HasNeg &&
5305 (Stat >= SrcStatus::NEG_START && Stat <= SrcStatus::NEG_END)) {
5306 return false;
5307 }
5308 if (!HasOpsel &&
5309 (Stat >= SrcStatus::HALF_START && Stat <= SrcStatus::HALF_END)) {
5310 return false;
5311 }
5312 return true;
5313 }
5314};
5315
5316static SmallVector<std::pair<Register, SrcStatus>>
5317getSrcStats(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5318 int MaxDepth = 3) {
5319 int Depth = 0;
5320 auto Curr = calcNextStatus(Curr: {Reg, SrcStatus::IS_SAME}, MRI);
5321 SmallVector<std::pair<Register, SrcStatus>> Statlist;
5322
5323 while (Depth <= MaxDepth && Curr.has_value()) {
5324 Depth++;
5325 if (SO.checkOptions(Stat: Curr.value().second))
5326 Statlist.push_back(Elt: Curr.value());
5327 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5328 }
5329
5330 return Statlist;
5331}
5332
5333static std::pair<Register, SrcStatus>
5334getLastSameOrNeg(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5335 int MaxDepth = 3) {
5336 int Depth = 0;
5337 std::pair<Register, SrcStatus> LastSameOrNeg = {Reg, SrcStatus::IS_SAME};
5338 auto Curr = calcNextStatus(Curr: LastSameOrNeg, MRI);
5339
5340 while (Depth <= MaxDepth && Curr.has_value()) {
5341 Depth++;
5342 SrcStatus Stat = Curr.value().second;
5343 if (SO.checkOptions(Stat)) {
5344 if (Stat == SrcStatus::IS_SAME || Stat == SrcStatus::IS_HI_NEG ||
5345 Stat == SrcStatus::IS_LO_NEG || Stat == SrcStatus::IS_BOTH_NEG)
5346 LastSameOrNeg = Curr.value();
5347 }
5348 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5349 }
5350
5351 return LastSameOrNeg;
5352}
5353
5354static bool isSameBitWidth(Register Reg1, Register Reg2,
5355 const MachineRegisterInfo &MRI) {
5356 unsigned Width1 = MRI.getType(Reg: Reg1).getSizeInBits();
5357 unsigned Width2 = MRI.getType(Reg: Reg2).getSizeInBits();
5358 return Width1 == Width2;
5359}
5360
5361static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods) {
5362 // SrcStatus::IS_LOWER_HALF remain 0.
5363 if (HiStat == SrcStatus::IS_UPPER_HALF_NEG) {
5364 Mods ^= SISrcMods::NEG_HI;
5365 Mods |= SISrcMods::OP_SEL_1;
5366 } else if (HiStat == SrcStatus::IS_UPPER_HALF)
5367 Mods |= SISrcMods::OP_SEL_1;
5368 else if (HiStat == SrcStatus::IS_LOWER_HALF_NEG)
5369 Mods ^= SISrcMods::NEG_HI;
5370 else if (HiStat == SrcStatus::IS_HI_NEG)
5371 Mods ^= SISrcMods::NEG_HI;
5372
5373 if (LoStat == SrcStatus::IS_UPPER_HALF_NEG) {
5374 Mods ^= SISrcMods::NEG;
5375 Mods |= SISrcMods::OP_SEL_0;
5376 } else if (LoStat == SrcStatus::IS_UPPER_HALF)
5377 Mods |= SISrcMods::OP_SEL_0;
5378 else if (LoStat == SrcStatus::IS_LOWER_HALF_NEG)
5379 Mods |= SISrcMods::NEG;
5380 else if (LoStat == SrcStatus::IS_HI_NEG)
5381 Mods ^= SISrcMods::NEG;
5382
5383 return Mods;
5384}
5385
5386static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg,
5387 Register RootReg, const SIInstrInfo &TII,
5388 const MachineRegisterInfo &MRI) {
5389 auto IsHalfState = [](SrcStatus S) {
5390 return S == SrcStatus::IS_UPPER_HALF || S == SrcStatus::IS_UPPER_HALF_NEG ||
5391 S == SrcStatus::IS_LOWER_HALF || S == SrcStatus::IS_LOWER_HALF_NEG;
5392 };
5393 return isSameBitWidth(Reg1: NewReg, Reg2: RootReg, MRI) && IsHalfState(LoStat) &&
5394 IsHalfState(HiStat);
5395}
5396
5397std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
5398 Register RootReg, const MachineRegisterInfo &MRI, bool IsDOT) const {
5399 unsigned Mods = 0;
5400 // No modification if Root type is not form of <2 x Type>.
5401 if (isVectorOfTwoOrScalar(Reg: RootReg, MRI) != TypeClass::VECTOR_OF_TWO) {
5402 Mods |= SISrcMods::OP_SEL_1;
5403 return {RootReg, Mods};
5404 }
5405
5406 SearchOptions SO(RootReg, MRI);
5407
5408 std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(Reg: RootReg, MRI, SO);
5409
5410 if (Stat.second == SrcStatus::IS_BOTH_NEG)
5411 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
5412 else if (Stat.second == SrcStatus::IS_HI_NEG)
5413 Mods ^= SISrcMods::NEG_HI;
5414 else if (Stat.second == SrcStatus::IS_LO_NEG)
5415 Mods ^= SISrcMods::NEG;
5416
5417 // 64-bit VOP3P instructions do not have OPSEL or ABS. Bail on v2f64 or v2i64.
5418 // TODO: Select NEG_LO and NEG_HI modifiers from BUILD_VECTOR.
5419 if (MRI.getType(Reg: RootReg).getSizeInBits() == 128) {
5420 Mods |= SISrcMods::OP_SEL_1; // Just the default, OPSEL unsupported.
5421 return {Stat.first, Mods};
5422 }
5423
5424 GBuildVector *MI;
5425 if (!mi_match(R: Stat.first, MRI, P: m_GBuildVector(Inst&: MI)) ||
5426 MI->getNumOperands() != 3 || (IsDOT && Subtarget->hasDOTOpSelHazard())) {
5427 Mods |= SISrcMods::OP_SEL_1;
5428 return {Stat.first, Mods};
5429 }
5430
5431 SmallVector<std::pair<Register, SrcStatus>> StatlistHi =
5432 getSrcStats(Reg: MI->getOperand(i: 2).getReg(), MRI, SO);
5433
5434 if (StatlistHi.empty()) {
5435 Mods |= SISrcMods::OP_SEL_1;
5436 return {Stat.first, Mods};
5437 }
5438
5439 SmallVector<std::pair<Register, SrcStatus>> StatlistLo =
5440 getSrcStats(Reg: MI->getOperand(i: 1).getReg(), MRI, SO);
5441
5442 if (StatlistLo.empty()) {
5443 Mods |= SISrcMods::OP_SEL_1;
5444 return {Stat.first, Mods};
5445 }
5446
5447 for (int I = StatlistHi.size() - 1; I >= 0; I--) {
5448 for (int J = StatlistLo.size() - 1; J >= 0; J--) {
5449 if (StatlistHi[I].first == StatlistLo[J].first &&
5450 isValidToPack(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second,
5451 NewReg: StatlistHi[I].first, RootReg, TII, MRI))
5452 return {StatlistHi[I].first,
5453 updateMods(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second, Mods)};
5454 }
5455 }
5456 // Packed instructions do not have abs modifiers.
5457 Mods |= SISrcMods::OP_SEL_1;
5458
5459 return {Stat.first, Mods};
5460}
5461
5462// Removed unused function `getAllKindImm` to eliminate dead code.
5463
5464static bool checkRB(Register Reg, unsigned int RBNo,
5465 const AMDGPURegisterBankInfo &RBI,
5466 const MachineRegisterInfo &MRI,
5467 const TargetRegisterInfo &TRI) {
5468 const RegisterBank *RB = RBI.getRegBank(Reg, MRI, TRI);
5469 return RB->getID() == RBNo;
5470}
5471
5472// This function is used to get the correct register bank for returned reg.
5473// Assume:
5474// 1. VOP3P is always legal for VGPR.
5475// 2. RootOp's regbank is legal.
5476// Thus
5477// 1. If RootOp is SGPR, then NewOp can be SGPR or VGPR.
5478// 2. If RootOp is VGPR, then NewOp must be VGPR.
5479static Register
5480getLegalRegBank(Register NewReg, Register RootReg, MachineInstr &Use,
5481 const AMDGPURegisterBankInfo &RBI, MachineRegisterInfo &MRI,
5482 const TargetRegisterInfo &TRI, const SIInstrInfo &TII) {
5483 // RootOp can only be VGPR or SGPR (some hand written cases such as.
5484 // inst-select-ashr.v2s16.mir::ashr_v2s16_vs).
5485 if (checkRB(Reg: RootReg, RBNo: AMDGPU::SGPRRegBankID, RBI, MRI, TRI) ||
5486 checkRB(Reg: NewReg, RBNo: AMDGPU::VGPRRegBankID, RBI, MRI, TRI))
5487 return NewReg;
5488
5489 if (mi_match(R: RootReg, MRI, P: m_Copy(Src: m_SpecificReg(RequestedReg: NewReg)))) {
5490 // RootOp is VGPR, NewOp is not VGPR, but RootOp = COPY NewOp.
5491 return RootReg;
5492 }
5493
5494 Register DstReg = MRI.cloneVirtualRegister(VReg: RootReg);
5495 MachineInstrBuilder MIB = BuildMI(BB&: *Use.getParent(), I&: Use, MIMD: Use.getDebugLoc(),
5496 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
5497 .addReg(RegNo: NewReg);
5498
5499 // Only accept VGPR.
5500 return MIB->getOperand(i: 0).getReg();
5501}
5502
5503InstructionSelector::ComplexRendererFns
5504AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
5505 bool IsDOT) const {
5506 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5507 Register Reg;
5508 unsigned Mods;
5509 std::tie(args&: Reg, args&: Mods) = selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT);
5510
5511 Reg = getLegalRegBank(NewReg: Reg, RootReg: Root.getReg(), Use&: *Root.getParent(), RBI, MRI, TRI,
5512 TII);
5513 return {{
5514 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
5515 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5516 }};
5517}
5518
5519InstructionSelector::ComplexRendererFns
5520AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
5521
5522 return selectVOP3PRetHelper(Root);
5523}
5524
5525InstructionSelector::ComplexRendererFns
5526AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
5527
5528 return selectVOP3PRetHelper(Root, IsDOT: true);
5529}
5530
5531InstructionSelector::ComplexRendererFns
5532AMDGPUInstructionSelector::selectVOP3PNoModsDOT(MachineOperand &Root) const {
5533 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5534 Register Src;
5535 unsigned Mods;
5536 std::tie(args&: Src, args&: Mods) = selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT: true /*IsDOT*/);
5537 if (Mods != SISrcMods::OP_SEL_1)
5538 return {};
5539
5540 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5541}
5542
5543InstructionSelector::ComplexRendererFns
5544AMDGPUInstructionSelector::selectVOP3PModsF32(MachineOperand &Root) const {
5545 Register Src;
5546 unsigned Mods;
5547 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5548
5549 return {{
5550 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5551 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5552 }};
5553}
5554
5555InstructionSelector::ComplexRendererFns
5556AMDGPUInstructionSelector::selectVOP3PNoModsF32(MachineOperand &Root) const {
5557 Register Src;
5558 unsigned Mods;
5559 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5560 if (Mods != SISrcMods::OP_SEL_1)
5561 return {};
5562
5563 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5564}
5565
5566InstructionSelector::ComplexRendererFns
5567AMDGPUInstructionSelector::selectWMMAOpSelVOP3PMods(
5568 MachineOperand &Root) const {
5569 assert((Root.isImm() && (Root.getImm() == -1 || Root.getImm() == 0)) &&
5570 "expected i1 value");
5571 unsigned Mods = SISrcMods::OP_SEL_1;
5572 if (Root.getImm() != 0)
5573 Mods |= SISrcMods::OP_SEL_0;
5574
5575 return {{
5576 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5577 }};
5578}
5579
5580static Register buildRegSequence(SmallVectorImpl<Register> &Elts,
5581 MachineInstr *InsertPt,
5582 MachineRegisterInfo &MRI) {
5583 const TargetRegisterClass *DstRegClass;
5584 switch (Elts.size()) {
5585 case 8:
5586 DstRegClass = &AMDGPU::VReg_256RegClass;
5587 break;
5588 case 4:
5589 DstRegClass = &AMDGPU::VReg_128RegClass;
5590 break;
5591 case 2:
5592 DstRegClass = &AMDGPU::VReg_64RegClass;
5593 break;
5594 default:
5595 llvm_unreachable("unhandled Reg sequence size");
5596 }
5597
5598 MachineIRBuilder B(*InsertPt);
5599 auto MIB = B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
5600 .addDef(RegNo: MRI.createVirtualRegister(RegClass: DstRegClass));
5601 for (unsigned i = 0; i < Elts.size(); ++i) {
5602 MIB.addReg(RegNo: Elts[i]);
5603 MIB.addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: i));
5604 }
5605 return MIB->getOperand(i: 0).getReg();
5606}
5607
5608static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods,
5609 SmallVectorImpl<Register> &Elts, Register &Src,
5610 MachineInstr *InsertPt,
5611 MachineRegisterInfo &MRI) {
5612 if (ModOpcode == TargetOpcode::G_FNEG) {
5613 Mods |= SISrcMods::NEG;
5614 // Check if all elements also have abs modifier
5615 SmallVector<Register, 8> NegAbsElts;
5616 for (auto El : Elts) {
5617 Register FabsSrc;
5618 if (!mi_match(R: El, MRI, P: m_GFabs(Src: m_Reg(R&: FabsSrc))))
5619 break;
5620 NegAbsElts.push_back(Elt: FabsSrc);
5621 }
5622 if (Elts.size() != NegAbsElts.size()) {
5623 // Neg
5624 Src = buildRegSequence(Elts, InsertPt, MRI);
5625 } else {
5626 // Neg and Abs
5627 Mods |= SISrcMods::NEG_HI;
5628 Src = buildRegSequence(Elts&: NegAbsElts, InsertPt, MRI);
5629 }
5630 } else {
5631 assert(ModOpcode == TargetOpcode::G_FABS);
5632 // Abs
5633 Mods |= SISrcMods::NEG_HI;
5634 Src = buildRegSequence(Elts, InsertPt, MRI);
5635 }
5636}
5637
5638InstructionSelector::ComplexRendererFns
5639AMDGPUInstructionSelector::selectWMMAModsF32NegAbs(MachineOperand &Root) const {
5640 Register Src = Root.getReg();
5641 unsigned Mods = SISrcMods::OP_SEL_1;
5642 SmallVector<Register, 8> EltsF32;
5643
5644 GBuildVector *BV;
5645 if (mi_match(R: Src, MRI: *MRI, P: m_GBuildVector(Inst&: BV))) {
5646 assert(BV->getNumSources() > 0);
5647 // Based on first element decide which mod we match, neg or abs
5648 MachineInstr *ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: 0));
5649 unsigned ModOpcode = (ElF32->getOpcode() == AMDGPU::G_FNEG)
5650 ? AMDGPU::G_FNEG
5651 : AMDGPU::G_FABS;
5652 for (unsigned i = 0; i < BV->getNumSources(); ++i) {
5653 ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: i));
5654 if (ElF32->getOpcode() != ModOpcode)
5655 break;
5656 EltsF32.push_back(Elt: ElF32->getOperand(i: 1).getReg());
5657 }
5658
5659 // All elements had ModOpcode modifier
5660 if (BV->getNumSources() == EltsF32.size()) {
5661 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsF32, Src, InsertPt: Root.getParent(),
5662 MRI&: *MRI);
5663 }
5664 }
5665
5666 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5667 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5668}
5669
5670InstructionSelector::ComplexRendererFns
5671AMDGPUInstructionSelector::selectWMMAModsF16Neg(MachineOperand &Root) const {
5672 Register Src = Root.getReg();
5673 unsigned Mods = SISrcMods::OP_SEL_1;
5674 SmallVector<Register, 8> EltsV2F16;
5675
5676 GConcatVectors *CV;
5677 if (mi_match(R: Src, MRI: *MRI, P: m_GConcatVectors(Inst&: CV))) {
5678 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5679 Register FNegSrc;
5680 if (!mi_match(R: CV->getSourceReg(I: i), MRI: *MRI, P: m_GFNeg(Src: m_Reg(R&: FNegSrc))))
5681 break;
5682 EltsV2F16.push_back(Elt: FNegSrc);
5683 }
5684
5685 // All elements had ModOpcode modifier
5686 if (CV->getNumSources() == EltsV2F16.size()) {
5687 Mods |= SISrcMods::NEG;
5688 Mods |= SISrcMods::NEG_HI;
5689 Src = buildRegSequence(Elts&: EltsV2F16, InsertPt: Root.getParent(), MRI&: *MRI);
5690 }
5691 }
5692
5693 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5694 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5695}
5696
5697InstructionSelector::ComplexRendererFns
5698AMDGPUInstructionSelector::selectWMMAModsF16NegAbs(MachineOperand &Root) const {
5699 Register Src = Root.getReg();
5700 unsigned Mods = SISrcMods::OP_SEL_1;
5701 SmallVector<Register, 8> EltsV2F16;
5702
5703 GConcatVectors *CV;
5704 if (mi_match(R: Src, MRI: *MRI, P: m_GConcatVectors(Inst&: CV))) {
5705 assert(CV->getNumSources() > 0);
5706 MachineInstr *ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: 0));
5707 // Based on first element decide which mod we match, neg or abs
5708 unsigned ModOpcode = (ElV2F16->getOpcode() == AMDGPU::G_FNEG)
5709 ? AMDGPU::G_FNEG
5710 : AMDGPU::G_FABS;
5711
5712 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5713 ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: i));
5714 if (ElV2F16->getOpcode() != ModOpcode)
5715 break;
5716 EltsV2F16.push_back(Elt: ElV2F16->getOperand(i: 1).getReg());
5717 }
5718
5719 // All elements had ModOpcode modifier
5720 if (CV->getNumSources() == EltsV2F16.size()) {
5721 MachineIRBuilder B(*Root.getParent());
5722 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsV2F16, Src, InsertPt: Root.getParent(),
5723 MRI&: *MRI);
5724 }
5725 }
5726
5727 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5728 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5729}
5730
5731InstructionSelector::ComplexRendererFns
5732AMDGPUInstructionSelector::selectWMMAVISrc(MachineOperand &Root) const {
5733 std::optional<FPValueAndVReg> FPValReg;
5734 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GFCstOrSplat(FPValReg))) {
5735 if (TII.isInlineConstant(Imm: FPValReg->Value)) {
5736 return {{[=](MachineInstrBuilder &MIB) {
5737 MIB.addImm(Val: FPValReg->Value.bitcastToAPInt().getSExtValue());
5738 }}};
5739 }
5740 // Non-inlineable splat floats should not fall-through for integer immediate
5741 // checks.
5742 return {};
5743 }
5744
5745 APInt ICst;
5746 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICstOrSplat(Cst&: ICst))) {
5747 if (TII.isInlineConstant(Imm: ICst)) {
5748 return {
5749 {[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ICst.getSExtValue()); }}};
5750 }
5751 }
5752
5753 return {};
5754}
5755
5756InstructionSelector::ComplexRendererFns
5757AMDGPUInstructionSelector::selectSWMMACIndex8(MachineOperand &Root) const {
5758 Register Src =
5759 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5760 unsigned Key = 0;
5761
5762 Register ShiftSrc;
5763 std::optional<ValueAndVReg> ShiftAmt;
5764 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5765 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5766 ShiftAmt->Value.getZExtValue() % 8 == 0) {
5767 Key = ShiftAmt->Value.getZExtValue() / 8;
5768 Src = ShiftSrc;
5769 }
5770
5771 return {{
5772 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5773 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5774 }};
5775}
5776
5777InstructionSelector::ComplexRendererFns
5778AMDGPUInstructionSelector::selectSWMMACIndex16(MachineOperand &Root) const {
5779
5780 Register Src =
5781 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5782 unsigned Key = 0;
5783
5784 Register ShiftSrc;
5785 std::optional<ValueAndVReg> ShiftAmt;
5786 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5787 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5788 ShiftAmt->Value.getZExtValue() == 16) {
5789 Src = ShiftSrc;
5790 Key = 1;
5791 }
5792
5793 return {{
5794 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5795 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5796 }};
5797}
5798
5799InstructionSelector::ComplexRendererFns
5800AMDGPUInstructionSelector::selectSWMMACIndex32(MachineOperand &Root) const {
5801 Register Src =
5802 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5803 unsigned Key = 0;
5804
5805 Register S32 = matchZeroExtendFromS32(Reg: Src);
5806 if (!S32)
5807 S32 = matchAnyExtendFromS32(Reg: Src);
5808
5809 if (S32) {
5810 const MachineInstr *Def = getDefIgnoringCopies(Reg: S32, MRI: *MRI);
5811 if (Def->getOpcode() == TargetOpcode::G_UNMERGE_VALUES) {
5812 assert(Def->getNumOperands() == 3);
5813 Register DstReg1 = Def->getOperand(i: 1).getReg();
5814 if (mi_match(R: S32, MRI: *MRI,
5815 P: m_any_of(preds: m_SpecificReg(RequestedReg: DstReg1), preds: m_Copy(Src: m_Reg(R&: DstReg1))))) {
5816 Src = Def->getOperand(i: 2).getReg();
5817 Key = 1;
5818 }
5819 }
5820 }
5821
5822 return {{
5823 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5824 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5825 }};
5826}
5827
5828InstructionSelector::ComplexRendererFns
5829AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const {
5830 Register Src;
5831 unsigned Mods;
5832 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
5833
5834 Register ExtractSrc;
5835 if (!Subtarget->useRealTrue16Insts() &&
5836 MRI->getType(Reg: Root.getReg()).getSizeInBits() == 16 &&
5837 isExtractHiElt(MRI&: *MRI, In: Src, Out&: ExtractSrc)) {
5838 Src = ExtractSrc;
5839 Mods |= SISrcMods::OP_SEL_0;
5840 }
5841
5842 return {{
5843 [=](MachineInstrBuilder &MIB) {
5844 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
5845 },
5846 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5847 }};
5848}
5849
5850// FIXME-TRUE16 remove when fake16 is removed
5851InstructionSelector::ComplexRendererFns
5852AMDGPUInstructionSelector::selectVINTERPMods(MachineOperand &Root) const {
5853 Register Src;
5854 unsigned Mods;
5855 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5856 /*IsCanonicalizing=*/true,
5857 /*AllowAbs=*/false,
5858 /*OpSel=*/false);
5859
5860 return {{
5861 [=](MachineInstrBuilder &MIB) {
5862 MIB.addReg(
5863 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5864 },
5865 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5866 }};
5867}
5868
5869InstructionSelector::ComplexRendererFns
5870AMDGPUInstructionSelector::selectVINTERPModsHi(MachineOperand &Root) const {
5871 Register Src;
5872 unsigned Mods;
5873 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5874 /*IsCanonicalizing=*/true,
5875 /*AllowAbs=*/false,
5876 /*OpSel=*/true);
5877
5878 return {{
5879 [=](MachineInstrBuilder &MIB) {
5880 MIB.addReg(
5881 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5882 },
5883 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5884 }};
5885}
5886
5887// Given \p Offset and load specified by the \p Root operand check if \p Offset
5888// is a multiple of the load byte size. If it is update \p Offset to a
5889// pre-scaled value and return true.
5890bool AMDGPUInstructionSelector::selectScaleOffset(MachineOperand &Root,
5891 Register &Offset,
5892 bool IsSigned) const {
5893 if (!Subtarget->hasScaleOffset())
5894 return false;
5895
5896 const MachineInstr &MI = *Root.getParent();
5897 MachineMemOperand *MMO = *MI.memoperands_begin();
5898
5899 if (!MMO->getSize().hasValue())
5900 return false;
5901
5902 uint64_t Size = MMO->getSize().getValue();
5903
5904 Register OffsetReg = matchExtendFromS32OrS32(Reg: Offset, IsSigned);
5905 if (!OffsetReg)
5906 OffsetReg = Offset;
5907
5908 if (auto Def = getDefSrcRegIgnoringCopies(Reg: OffsetReg, MRI: *MRI))
5909 OffsetReg = Def->Reg;
5910
5911 Register Op0;
5912 MachineInstr *Mul;
5913 bool ScaleOffset =
5914 (isPowerOf2_64(Value: Size) &&
5915 mi_match(R: OffsetReg, MRI: *MRI,
5916 P: m_GShl(L: m_Reg(R&: Op0),
5917 R: m_any_of(preds: m_SpecificICst(RequestedValue: Log2_64(Value: Size)),
5918 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Log2_64(Value: Size))))))) ||
5919 mi_match(R: OffsetReg, MRI: *MRI,
5920 P: m_GMul(L: m_Reg(R&: Op0), R: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5921 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) ||
5922 mi_match(
5923 R: OffsetReg, MRI: *MRI,
5924 P: m_BinOp(Opcode: IsSigned ? AMDGPU::S_MUL_I64_I32_PSEUDO : AMDGPU::S_MUL_U64,
5925 L: m_Reg(R&: Op0), R: m_SpecificICst(RequestedValue: Size))) ||
5926 // Match G_AMDGPU_MAD_U64_U32 offset, c, 0
5927 (mi_match(R: OffsetReg, MRI: *MRI, P: m_MInstr(MI&: Mul)) &&
5928 (Mul->getOpcode() == (IsSigned ? AMDGPU::G_AMDGPU_MAD_I64_I32
5929 : AMDGPU::G_AMDGPU_MAD_U64_U32) ||
5930 (IsSigned && Mul->getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32 &&
5931 VT->signBitIsZero(Op: Mul->getOperand(i: 2).getReg()))) &&
5932 mi_match(R: Mul->getOperand(i: 4).getReg(), MRI: *MRI, P: m_ZeroInt()) &&
5933 mi_match(R: Mul->getOperand(i: 3).getReg(), MRI: *MRI,
5934 P: m_GTrunc(Src: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5935 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) &&
5936 mi_match(R: Mul->getOperand(i: 2).getReg(), MRI: *MRI, P: m_Reg(R&: Op0)));
5937
5938 if (ScaleOffset)
5939 Offset = Op0;
5940
5941 return ScaleOffset;
5942}
5943
5944bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root,
5945 Register &Base,
5946 Register *SOffset,
5947 int64_t *Offset,
5948 bool *ScaleOffset) const {
5949 MachineInstr *MI = Root.getParent();
5950 MachineBasicBlock *MBB = MI->getParent();
5951
5952 // FIXME: We should shrink the GEP if the offset is known to be <= 32-bits,
5953 // then we can select all ptr + 32-bit offsets.
5954 SmallVector<GEPInfo, 4> AddrInfo;
5955 getAddrModeInfo(Load: *MI, MRI: *MRI, AddrInfo);
5956
5957 if (AddrInfo.empty())
5958 return false;
5959
5960 const GEPInfo &GEPI = AddrInfo[0];
5961 std::optional<int64_t> EncodedImm;
5962
5963 if (ScaleOffset)
5964 *ScaleOffset = false;
5965
5966 if (SOffset && Offset) {
5967 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
5968 /*HasSOffset=*/true);
5969 if (GEPI.SgprParts.size() == 1 && GEPI.Imm != 0 && EncodedImm &&
5970 AddrInfo.size() > 1) {
5971 const GEPInfo &GEPI2 = AddrInfo[1];
5972 if (GEPI2.SgprParts.size() == 2 && GEPI2.Imm == 0) {
5973 Register OffsetReg = GEPI2.SgprParts[1];
5974 if (ScaleOffset)
5975 *ScaleOffset =
5976 selectScaleOffset(Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */);
5977 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
5978 if (OffsetReg) {
5979 Base = GEPI2.SgprParts[0];
5980 *SOffset = OffsetReg;
5981 *Offset = *EncodedImm;
5982 if (*Offset >= 0 || !AMDGPU::hasSMRDSignedImmOffset(ST: STI))
5983 return true;
5984
5985 // For unbuffered smem loads, it is illegal for the Immediate Offset
5986 // to be negative if the resulting (Offset + (M0 or SOffset or zero)
5987 // is negative. Handle the case where the Immediate Offset + SOffset
5988 // is negative.
5989 auto SKnown = VT->getKnownBits(R: *SOffset);
5990 if (*Offset + SKnown.getMinValue().getSExtValue() < 0)
5991 return false;
5992
5993 return true;
5994 }
5995 }
5996 }
5997 return false;
5998 }
5999
6000 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
6001 /*HasSOffset=*/false);
6002 if (Offset && GEPI.SgprParts.size() == 1 && EncodedImm) {
6003 Base = GEPI.SgprParts[0];
6004 *Offset = *EncodedImm;
6005 return true;
6006 }
6007
6008 // SGPR offset is unsigned.
6009 if (SOffset && GEPI.SgprParts.size() == 1 && isUInt<32>(x: GEPI.Imm) &&
6010 GEPI.Imm != 0) {
6011 // If we make it this far we have a load with an 32-bit immediate offset.
6012 // It is OK to select this using a sgpr offset, because we have already
6013 // failed trying to select this load into one of the _IMM variants since
6014 // the _IMM Patterns are considered before the _SGPR patterns.
6015 Base = GEPI.SgprParts[0];
6016 *SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6017 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: *SOffset)
6018 .addImm(Val: GEPI.Imm);
6019 return true;
6020 }
6021
6022 if (SOffset && GEPI.SgprParts.size() && GEPI.Imm == 0) {
6023 Register OffsetReg = GEPI.SgprParts[1];
6024 if (ScaleOffset)
6025 *ScaleOffset = selectScaleOffset(Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */);
6026 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
6027 if (OffsetReg) {
6028 Base = GEPI.SgprParts[0];
6029 *SOffset = OffsetReg;
6030 return true;
6031 }
6032 }
6033
6034 return false;
6035}
6036
6037InstructionSelector::ComplexRendererFns
6038AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
6039 Register Base;
6040 int64_t Offset;
6041 if (!selectSmrdOffset(Root, Base, /* SOffset= */ nullptr, Offset: &Offset,
6042 /* ScaleOffset */ nullptr))
6043 return std::nullopt;
6044
6045 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6046 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }}};
6047}
6048
6049InstructionSelector::ComplexRendererFns
6050AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const {
6051 SmallVector<GEPInfo, 4> AddrInfo;
6052 getAddrModeInfo(Load: *Root.getParent(), MRI: *MRI, AddrInfo);
6053
6054 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
6055 return std::nullopt;
6056
6057 const GEPInfo &GEPInfo = AddrInfo[0];
6058 Register PtrReg = GEPInfo.SgprParts[0];
6059 std::optional<int64_t> EncodedImm =
6060 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: GEPInfo.Imm);
6061 if (!EncodedImm)
6062 return std::nullopt;
6063
6064 return {{
6065 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrReg); },
6066 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); }
6067 }};
6068}
6069
6070InstructionSelector::ComplexRendererFns
6071AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
6072 Register Base, SOffset;
6073 bool ScaleOffset;
6074 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, /* Offset= */ nullptr,
6075 ScaleOffset: &ScaleOffset))
6076 return std::nullopt;
6077
6078 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6079 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6080 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
6081 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
6082}
6083
6084InstructionSelector::ComplexRendererFns
6085AMDGPUInstructionSelector::selectSmrdSgprImm(MachineOperand &Root) const {
6086 Register Base, SOffset;
6087 int64_t Offset;
6088 bool ScaleOffset;
6089 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, Offset: &Offset, ScaleOffset: &ScaleOffset))
6090 return std::nullopt;
6091
6092 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6093 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6094 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
6095 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
6096 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
6097}
6098
6099std::pair<Register, int> AMDGPUInstructionSelector::selectFlatOffsetImpl(
6100 MachineOperand &Root, AMDGPU::FlatAddrSpace FlatVariant) const {
6101 MachineInstr *MI = Root.getParent();
6102
6103 auto Default = std::pair(Root.getReg(), 0);
6104
6105 if (!STI.hasFlatInstOffsets())
6106 return Default;
6107
6108 Register PtrBase;
6109 int64_t ConstOffset;
6110 bool IsInBounds;
6111 std::tie(args&: PtrBase, args&: ConstOffset, args&: IsInBounds) =
6112 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6113
6114 // Adding the offset to the base address with an immediate in a FLAT
6115 // instruction must not change the memory aperture in which the address falls.
6116 // Therefore we can only fold offsets from inbounds GEPs into FLAT
6117 // instructions.
6118 if (ConstOffset == 0 ||
6119 (FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch &&
6120 !isFlatScratchBaseLegal(Addr: Root.getReg())) ||
6121 (FlatVariant == AMDGPU::FlatAddrSpace::FLAT && !IsInBounds))
6122 return Default;
6123
6124 unsigned AddrSpace = (*MI->memoperands_begin())->getAddrSpace();
6125 if (!TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace, FlatVariant))
6126 return Default;
6127
6128 return std::pair(PtrBase, ConstOffset);
6129}
6130
6131InstructionSelector::ComplexRendererFns
6132AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const {
6133 auto PtrWithOffset = selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FLAT);
6134
6135 return {{
6136 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6137 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6138 }};
6139}
6140
6141InstructionSelector::ComplexRendererFns
6142AMDGPUInstructionSelector::selectGlobalOffset(MachineOperand &Root) const {
6143 auto PtrWithOffset =
6144 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6145
6146 return {{
6147 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6148 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6149 }};
6150}
6151
6152InstructionSelector::ComplexRendererFns
6153AMDGPUInstructionSelector::selectScratchOffset(MachineOperand &Root) const {
6154 auto PtrWithOffset =
6155 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch);
6156
6157 return {{
6158 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6159 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6160 }};
6161}
6162
6163// Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
6164InstructionSelector::ComplexRendererFns
6165AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root,
6166 unsigned CPolBits,
6167 bool NeedIOffset) const {
6168 Register Addr = Root.getReg();
6169 Register PtrBase;
6170 int64_t ConstOffset;
6171 int64_t ImmOffset = 0;
6172
6173 // Match the immediate offset first, which canonically is moved as low as
6174 // possible.
6175 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6176 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6177
6178 if (ConstOffset != 0) {
6179 if (NeedIOffset &&
6180 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6181 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal)) {
6182 Addr = PtrBase;
6183 ImmOffset = ConstOffset;
6184 } else {
6185 auto PtrBaseDef = getDefSrcRegIgnoringCopies(Reg: PtrBase, MRI: *MRI);
6186 if (isSGPR(Reg: PtrBaseDef->Reg)) {
6187 // Offset is too large.
6188 //
6189 // saddr + large_offset -> saddr +
6190 // (voffset = large_offset & ~MaxOffset) +
6191 // (large_offset & MaxOffset);
6192 int64_t SplitImmOffset = 0, RemainderOffset = ConstOffset;
6193 if (NeedIOffset) {
6194 std::tie(args&: SplitImmOffset, args&: RemainderOffset) =
6195 TII.splitFlatOffset(COffsetVal: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6196 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6197 }
6198
6199 if (Subtarget->hasSignedGVSOffset() ? isInt<32>(x: RemainderOffset)
6200 : isUInt<32>(x: RemainderOffset)) {
6201 MachineInstr *MI = Root.getParent();
6202 MachineBasicBlock *MBB = MI->getParent();
6203 Register HighBits =
6204 MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6205
6206 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6207 DestReg: HighBits)
6208 .addImm(Val: RemainderOffset);
6209
6210 if (NeedIOffset)
6211 return {{
6212 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrBase); }, // saddr
6213 [=](MachineInstrBuilder &MIB) {
6214 MIB.addReg(RegNo: HighBits);
6215 }, // voffset
6216 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: SplitImmOffset); },
6217 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6218 }};
6219 return {{
6220 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrBase); }, // saddr
6221 [=](MachineInstrBuilder &MIB) {
6222 MIB.addReg(RegNo: HighBits);
6223 }, // voffset
6224 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6225 }};
6226 }
6227
6228 // We are adding a 64 bit SGPR and a constant. If constant bus limit
6229 // is 1 we would need to perform 1 or 2 extra moves for each half of
6230 // the constant and it is better to do a scalar add and then issue a
6231 // single VALU instruction to materialize zero. Otherwise it is less
6232 // instructions to perform VALU adds with immediates or inline literals.
6233 unsigned NumLiterals =
6234 !TII.isInlineConstant(Imm: APInt(32, Lo_32(Value: ConstOffset))) +
6235 !TII.isInlineConstant(Imm: APInt(32, Hi_32(Value: ConstOffset)));
6236 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_ADD_U32_e64) > NumLiterals)
6237 return std::nullopt;
6238 }
6239 }
6240 }
6241
6242 // Match the variable offset.
6243 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6244 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6245 // Look through the SGPR->VGPR copy.
6246 Register SAddr =
6247 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
6248
6249 if (isSGPR(Reg: SAddr)) {
6250 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
6251
6252 // It's possible voffset is an SGPR here, but the copy to VGPR will be
6253 // inserted later.
6254 bool ScaleOffset = selectScaleOffset(Root, Offset&: PtrBaseOffset,
6255 IsSigned: Subtarget->hasSignedGVSOffset());
6256 if (Register VOffset = matchExtendFromS32OrS32(
6257 Reg: PtrBaseOffset, IsSigned: Subtarget->hasSignedGVSOffset())) {
6258 if (NeedIOffset)
6259 return {{[=](MachineInstrBuilder &MIB) { // saddr
6260 MIB.addReg(RegNo: SAddr);
6261 },
6262 [=](MachineInstrBuilder &MIB) { // voffset
6263 MIB.addReg(RegNo: VOffset);
6264 },
6265 [=](MachineInstrBuilder &MIB) { // offset
6266 MIB.addImm(Val: ImmOffset);
6267 },
6268 [=](MachineInstrBuilder &MIB) { // cpol
6269 MIB.addImm(Val: CPolBits |
6270 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6271 }}};
6272 return {{[=](MachineInstrBuilder &MIB) { // saddr
6273 MIB.addReg(RegNo: SAddr);
6274 },
6275 [=](MachineInstrBuilder &MIB) { // voffset
6276 MIB.addReg(RegNo: VOffset);
6277 },
6278 [=](MachineInstrBuilder &MIB) { // cpol
6279 MIB.addImm(Val: CPolBits |
6280 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6281 }}};
6282 }
6283 }
6284 }
6285
6286 // FIXME: We should probably have folded COPY (G_IMPLICIT_DEF) earlier, and
6287 // drop this.
6288 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF ||
6289 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(Reg: AddrDef->Reg))
6290 return std::nullopt;
6291
6292 // It's cheaper to materialize a single 32-bit zero for vaddr than the two
6293 // moves required to copy a 64-bit SGPR to VGPR.
6294 MachineInstr *MI = Root.getParent();
6295 MachineBasicBlock *MBB = MI->getParent();
6296 Register VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6297
6298 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
6299 .addImm(Val: 0);
6300
6301 if (NeedIOffset)
6302 return {{
6303 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6304 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6305 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6306 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6307 }};
6308 return {{
6309 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6310 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6311 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6312 }};
6313}
6314
6315InstructionSelector::ComplexRendererFns
6316AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root) const {
6317 return selectGlobalSAddr(Root, CPolBits: 0);
6318}
6319
6320InstructionSelector::ComplexRendererFns
6321AMDGPUInstructionSelector::selectGlobalSAddrCPol(MachineOperand &Root) const {
6322 const MachineInstr &I = *Root.getParent();
6323
6324 // We are assuming CPol is always the last operand of the intrinsic.
6325 auto PassedCPol =
6326 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6327 return selectGlobalSAddr(Root, CPolBits: PassedCPol);
6328}
6329
6330InstructionSelector::ComplexRendererFns
6331AMDGPUInstructionSelector::selectGlobalSAddrCPolM0(MachineOperand &Root) const {
6332 const MachineInstr &I = *Root.getParent();
6333
6334 // We are assuming CPol is second from last operand of the intrinsic.
6335 auto PassedCPol =
6336 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6337 return selectGlobalSAddr(Root, CPolBits: PassedCPol);
6338}
6339
6340InstructionSelector::ComplexRendererFns
6341AMDGPUInstructionSelector::selectGlobalSAddrGLC(MachineOperand &Root) const {
6342 return selectGlobalSAddr(Root, CPolBits: AMDGPU::CPol::GLC);
6343}
6344
6345InstructionSelector::ComplexRendererFns
6346AMDGPUInstructionSelector::selectGlobalSAddrNoIOffset(
6347 MachineOperand &Root) const {
6348 const MachineInstr &I = *Root.getParent();
6349
6350 // We are assuming CPol is always the last operand of the intrinsic.
6351 auto PassedCPol =
6352 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6353 return selectGlobalSAddr(Root, CPolBits: PassedCPol, NeedIOffset: false);
6354}
6355
6356InstructionSelector::ComplexRendererFns
6357AMDGPUInstructionSelector::selectGlobalSAddrNoIOffsetM0(
6358 MachineOperand &Root) const {
6359 const MachineInstr &I = *Root.getParent();
6360
6361 // We are assuming CPol is second from last operand of the intrinsic.
6362 auto PassedCPol =
6363 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6364 return selectGlobalSAddr(Root, CPolBits: PassedCPol, NeedIOffset: false);
6365}
6366
6367InstructionSelector::ComplexRendererFns
6368AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const {
6369 Register Addr = Root.getReg();
6370 Register PtrBase;
6371 int64_t ConstOffset;
6372 int64_t ImmOffset = 0;
6373
6374 // Match the immediate offset first, which canonically is moved as low as
6375 // possible.
6376 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6377 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6378
6379 if (ConstOffset != 0 && isFlatScratchBaseLegal(Addr) &&
6380 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6381 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6382 Addr = PtrBase;
6383 ImmOffset = ConstOffset;
6384 }
6385
6386 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6387 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6388 int FI = AddrDef->MI->getOperand(i: 1).getIndex();
6389 return {{
6390 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6391 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6392 }};
6393 }
6394
6395 Register SAddr = AddrDef->Reg;
6396
6397 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6398 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6399 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6400 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6401 auto RHSDef = getDefSrcRegIgnoringCopies(Reg: RHS, MRI: *MRI);
6402
6403 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX &&
6404 isSGPR(Reg: RHSDef->Reg)) {
6405 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6406 MachineInstr &I = *Root.getParent();
6407 MachineBasicBlock *BB = I.getParent();
6408 const DebugLoc &DL = I.getDebugLoc();
6409 SAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6410
6411 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_I32), DestReg: SAddr)
6412 .addFrameIndex(Idx: FI)
6413 .addReg(RegNo: RHSDef->Reg)
6414 .setOperandDead(3); // Dead scc
6415 }
6416 }
6417
6418 if (!isSGPR(Reg: SAddr))
6419 return std::nullopt;
6420
6421 return {{
6422 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SAddr); }, // saddr
6423 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6424 }};
6425}
6426
6427// Check whether the flat scratch SVS swizzle bug affects this access.
6428bool AMDGPUInstructionSelector::checkFlatScratchSVSSwizzleBug(
6429 Register VAddr, Register SAddr, uint64_t ImmOffset) const {
6430 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
6431 return false;
6432
6433 // The bug affects the swizzling of SVS accesses if there is any carry out
6434 // from the two low order bits (i.e. from bit 1 into bit 2) when adding
6435 // voffset to (soffset + inst_offset).
6436 auto VKnown = VT->getKnownBits(R: VAddr);
6437 auto SKnown = KnownBits::add(LHS: VT->getKnownBits(R: SAddr),
6438 RHS: KnownBits::makeConstant(C: APInt(32, ImmOffset)));
6439 uint64_t VMax = VKnown.getMaxValue().getZExtValue();
6440 uint64_t SMax = SKnown.getMaxValue().getZExtValue();
6441 return (VMax & 3) + (SMax & 3) >= 4;
6442}
6443
6444InstructionSelector::ComplexRendererFns
6445AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const {
6446 Register Addr = Root.getReg();
6447 Register PtrBase;
6448 int64_t ConstOffset;
6449 int64_t ImmOffset = 0;
6450
6451 // Match the immediate offset first, which canonically is moved as low as
6452 // possible.
6453 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6454 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6455
6456 Register OrigAddr = Addr;
6457 if (ConstOffset != 0 &&
6458 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6459 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6460 Addr = PtrBase;
6461 ImmOffset = ConstOffset;
6462 }
6463
6464 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6465 if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD)
6466 return std::nullopt;
6467
6468 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6469 if (RBI.getRegBank(Reg: RHS, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
6470 return std::nullopt;
6471
6472 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6473 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6474
6475 if (OrigAddr != Addr) {
6476 if (!isFlatScratchBaseLegalSVImm(Addr: OrigAddr))
6477 return std::nullopt;
6478 } else {
6479 if (!isFlatScratchBaseLegalSV(Addr: OrigAddr))
6480 return std::nullopt;
6481 }
6482
6483 if (checkFlatScratchSVSSwizzleBug(VAddr: RHS, SAddr: LHS, ImmOffset))
6484 return std::nullopt;
6485
6486 unsigned CPol = selectScaleOffset(Root, Offset&: RHS, IsSigned: true /* IsSigned */)
6487 ? AMDGPU::CPol::SCAL
6488 : 0;
6489
6490 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6491 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6492 return {{
6493 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6494 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6495 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6496 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6497 }};
6498 }
6499
6500 if (!isSGPR(Reg: LHS))
6501 if (auto Def = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI))
6502 LHS = Def->Reg;
6503
6504 if (!isSGPR(Reg: LHS))
6505 return std::nullopt;
6506
6507 return {{
6508 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6509 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: LHS); }, // saddr
6510 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6511 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6512 }};
6513}
6514
6515InstructionSelector::ComplexRendererFns
6516AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
6517 MachineInstr *MI = Root.getParent();
6518 MachineBasicBlock *MBB = MI->getParent();
6519 MachineFunction *MF = MBB->getParent();
6520 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6521
6522 int64_t Offset = 0;
6523 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) &&
6524 Offset != AMDGPU::getNullPointerValue(AS: AMDGPUAS::PRIVATE_ADDRESS)) {
6525 Register HighBits = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6526
6527 // TODO: Should this be inside the render function? The iterator seems to
6528 // move.
6529 const int64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
6530 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6531 DestReg: HighBits)
6532 .addImm(Val: Offset & ~MaxOffset);
6533
6534 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6535 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6536 },
6537 [=](MachineInstrBuilder &MIB) { // vaddr
6538 MIB.addReg(RegNo: HighBits);
6539 },
6540 [=](MachineInstrBuilder &MIB) { // soffset
6541 // Use constant zero for soffset and rely on eliminateFrameIndex
6542 // to choose the appropriate frame register if need be.
6543 MIB.addImm(Val: 0);
6544 },
6545 [=](MachineInstrBuilder &MIB) { // offset
6546 MIB.addImm(Val: Offset & MaxOffset);
6547 }}};
6548 }
6549
6550 assert(Offset == 0 || Offset == -1);
6551
6552 // Try to fold a frame index directly into the MUBUF vaddr field, and any
6553 // offsets.
6554 std::optional<int> FI;
6555 Register VAddr = Root.getReg();
6556
6557 Register PtrBase;
6558 int64_t ConstOffset;
6559 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6560 getPtrBaseWithConstantOffset(Root: VAddr, MRI: *MRI);
6561 int MatchedFI;
6562 if (ConstOffset != 0) {
6563 if (TII.isLegalMUBUFImmOffset(Imm: ConstOffset) &&
6564 (!STI.privateMemoryResourceIsRangeChecked() ||
6565 VT->signBitIsZero(Op: PtrBase))) {
6566 if (mi_match(R: PtrBase, MRI: *MRI, P: m_GFrameIndex(FI&: MatchedFI)))
6567 FI = MatchedFI;
6568 else
6569 VAddr = PtrBase;
6570 Offset = ConstOffset;
6571 }
6572 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GFrameIndex(FI&: MatchedFI))) {
6573 FI = MatchedFI;
6574 }
6575
6576 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6577 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6578 },
6579 [=](MachineInstrBuilder &MIB) { // vaddr
6580 if (FI)
6581 MIB.addFrameIndex(Idx: *FI);
6582 else
6583 MIB.addReg(RegNo: VAddr);
6584 },
6585 [=](MachineInstrBuilder &MIB) { // soffset
6586 // Use constant zero for soffset and rely on eliminateFrameIndex
6587 // to choose the appropriate frame register if need be.
6588 MIB.addImm(Val: 0);
6589 },
6590 [=](MachineInstrBuilder &MIB) { // offset
6591 MIB.addImm(Val: Offset);
6592 }}};
6593}
6594
6595bool AMDGPUInstructionSelector::isDSOffsetLegal(Register Base,
6596 int64_t Offset) const {
6597 if (!isUInt<16>(x: Offset))
6598 return false;
6599
6600 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6601 return true;
6602
6603 // On Southern Islands instruction with a negative base value and an offset
6604 // don't seem to work.
6605 return VT->signBitIsZero(Op: Base);
6606}
6607
6608bool AMDGPUInstructionSelector::isDSOffset2Legal(Register Base, int64_t Offset0,
6609 int64_t Offset1,
6610 unsigned Size) const {
6611 if (Offset0 % Size != 0 || Offset1 % Size != 0)
6612 return false;
6613 if (!isUInt<8>(x: Offset0 / Size) || !isUInt<8>(x: Offset1 / Size))
6614 return false;
6615
6616 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6617 return true;
6618
6619 // On Southern Islands instruction with a negative base value and an offset
6620 // don't seem to work.
6621 return VT->signBitIsZero(Op: Base);
6622}
6623
6624// Return whether the operation has NoUnsignedWrap property.
6625static bool isNoUnsignedWrap(MachineInstr *Addr) {
6626 return Addr->getOpcode() == TargetOpcode::G_OR ||
6627 (Addr->getOpcode() == TargetOpcode::G_PTR_ADD &&
6628 Addr->getFlag(Flag: MachineInstr::NoUWrap));
6629}
6630
6631// Check that the base address of flat scratch load/store in the form of `base +
6632// offset` is legal to be put in SGPR/VGPR (i.e. unsigned per hardware
6633// requirement). We always treat the first operand as the base address here.
6634bool AMDGPUInstructionSelector::isFlatScratchBaseLegal(Register Addr) const {
6635 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6636
6637 if (isNoUnsignedWrap(Addr: AddrMI))
6638 return true;
6639
6640 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6641 // values.
6642 if (STI.hasSignedScratchOffsets())
6643 return true;
6644
6645 Register LHS = AddrMI->getOperand(i: 1).getReg();
6646 Register RHS = AddrMI->getOperand(i: 2).getReg();
6647
6648 if (AddrMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
6649 std::optional<ValueAndVReg> RhsValReg =
6650 getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
6651 // If the immediate offset is negative and within certain range, the base
6652 // address cannot also be negative. If the base is also negative, the sum
6653 // would be either negative or much larger than the valid range of scratch
6654 // memory a thread can access.
6655 if (RhsValReg && RhsValReg->Value.getSExtValue() < 0 &&
6656 RhsValReg->Value.getSExtValue() > -0x40000000)
6657 return true;
6658 }
6659
6660 return VT->signBitIsZero(Op: LHS);
6661}
6662
6663// Check address value in SGPR/VGPR are legal for flat scratch in the form
6664// of: SGPR + VGPR.
6665bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSV(Register Addr) const {
6666 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6667
6668 if (isNoUnsignedWrap(Addr: AddrMI))
6669 return true;
6670
6671 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6672 // values.
6673 if (STI.hasSignedScratchOffsets())
6674 return true;
6675
6676 Register LHS = AddrMI->getOperand(i: 1).getReg();
6677 Register RHS = AddrMI->getOperand(i: 2).getReg();
6678 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6679}
6680
6681// Check address value in SGPR/VGPR are legal for flat scratch in the form
6682// of: SGPR + VGPR + Imm.
6683bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSVImm(
6684 Register Addr) const {
6685 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6686 // values.
6687 if (STI.hasSignedScratchOffsets())
6688 return true;
6689
6690 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6691 Register Base = AddrMI->getOperand(i: 1).getReg();
6692 std::optional<DefinitionAndSourceRegister> BaseDef =
6693 getDefSrcRegIgnoringCopies(Reg: Base, MRI: *MRI);
6694 std::optional<ValueAndVReg> RHSOffset =
6695 getIConstantVRegValWithLookThrough(VReg: AddrMI->getOperand(i: 2).getReg(), MRI: *MRI);
6696 assert(RHSOffset);
6697
6698 // If the immediate offset is negative and within certain range, the base
6699 // address cannot also be negative. If the base is also negative, the sum
6700 // would be either negative or much larger than the valid range of scratch
6701 // memory a thread can access.
6702 if (isNoUnsignedWrap(Addr: BaseDef->MI) &&
6703 (isNoUnsignedWrap(Addr: AddrMI) ||
6704 (RHSOffset->Value.getSExtValue() < 0 &&
6705 RHSOffset->Value.getSExtValue() > -0x40000000)))
6706 return true;
6707
6708 Register LHS = BaseDef->MI->getOperand(i: 1).getReg();
6709 Register RHS = BaseDef->MI->getOperand(i: 2).getReg();
6710 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6711}
6712
6713bool AMDGPUInstructionSelector::isUnneededShiftMask(const MachineInstr &MI,
6714 unsigned ShAmtBits) const {
6715 assert(MI.getOpcode() == TargetOpcode::G_AND);
6716
6717 // The mask constant is an SGPR G_CONSTANT copied to VGPR; the matcher looks
6718 // through that COPY, so look through it here too.
6719 std::optional<ValueAndVReg> RHS =
6720 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI: *MRI);
6721 if (!RHS)
6722 return false;
6723
6724 if (RHS->Value.countr_one() >= ShAmtBits)
6725 return true;
6726
6727 const APInt &LHSKnownZeros = VT->getKnownZeroes(R: MI.getOperand(i: 1).getReg());
6728 return (LHSKnownZeros | RHS->Value).countr_one() >= ShAmtBits;
6729}
6730
6731InstructionSelector::ComplexRendererFns
6732AMDGPUInstructionSelector::selectMUBUFScratchOffset(
6733 MachineOperand &Root) const {
6734 Register Reg = Root.getReg();
6735 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6736
6737 std::optional<DefinitionAndSourceRegister> Def =
6738 getDefSrcRegIgnoringCopies(Reg, MRI: *MRI);
6739 assert(Def && "this shouldn't be an optional result");
6740 Reg = Def->Reg;
6741
6742 if (Register WaveBase = getWaveAddress(Def: Def->MI)) {
6743 return {{
6744 [=](MachineInstrBuilder &MIB) { // rsrc
6745 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6746 },
6747 [=](MachineInstrBuilder &MIB) { // soffset
6748 MIB.addReg(RegNo: WaveBase);
6749 },
6750 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // offset
6751 }};
6752 }
6753
6754 int64_t Offset = 0;
6755
6756 // FIXME: Copy check is a hack
6757 Register BasePtr;
6758 if (mi_match(R: Reg, MRI: *MRI,
6759 P: m_GPtrAdd(L: m_Reg(R&: BasePtr),
6760 R: m_any_of(preds: m_ICst(Cst&: Offset), preds: m_Copy(Src: m_ICst(Cst&: Offset)))))) {
6761 if (!TII.isLegalMUBUFImmOffset(Imm: Offset))
6762 return {};
6763 MachineInstr *BasePtrDef = getDefIgnoringCopies(Reg: BasePtr, MRI: *MRI);
6764 Register WaveBase = getWaveAddress(Def: BasePtrDef);
6765 if (!WaveBase)
6766 return {};
6767
6768 return {{
6769 [=](MachineInstrBuilder &MIB) { // rsrc
6770 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6771 },
6772 [=](MachineInstrBuilder &MIB) { // soffset
6773 MIB.addReg(RegNo: WaveBase);
6774 },
6775 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6776 }};
6777 }
6778
6779 if (!mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) ||
6780 !TII.isLegalMUBUFImmOffset(Imm: Offset))
6781 return {};
6782
6783 return {{
6784 [=](MachineInstrBuilder &MIB) { // rsrc
6785 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6786 },
6787 [=](MachineInstrBuilder &MIB) { // soffset
6788 MIB.addImm(Val: 0);
6789 },
6790 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6791 }};
6792}
6793
6794std::pair<Register, unsigned>
6795AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(
6796 MachineOperand &Root) const {
6797 int64_t ConstAddr = 0;
6798
6799 Register PtrBase;
6800 int64_t Offset;
6801 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6802 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6803
6804 if (Offset) {
6805 if (isDSOffsetLegal(Base: PtrBase, Offset)) {
6806 // (add n0, c0)
6807 return std::pair(PtrBase, Offset);
6808 }
6809 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GSub(L: m_Reg(), R: m_Reg()))) {
6810 // TODO
6811
6812 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6813 // TODO
6814 }
6815
6816 return std::pair(Root.getReg(), 0);
6817}
6818
6819InstructionSelector::ComplexRendererFns
6820AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
6821 Register Reg;
6822 unsigned Offset;
6823 std::tie(args&: Reg, args&: Offset) = selectDS1Addr1OffsetImpl(Root);
6824 return {{
6825 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6826 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }
6827 }};
6828}
6829
6830InstructionSelector::ComplexRendererFns
6831AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const {
6832 return selectDSReadWrite2(Root, size: 4);
6833}
6834
6835InstructionSelector::ComplexRendererFns
6836AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(MachineOperand &Root) const {
6837 return selectDSReadWrite2(Root, size: 8);
6838}
6839
6840InstructionSelector::ComplexRendererFns
6841AMDGPUInstructionSelector::selectDSReadWrite2(MachineOperand &Root,
6842 unsigned Size) const {
6843 Register Reg;
6844 unsigned Offset;
6845 std::tie(args&: Reg, args&: Offset) = selectDSReadWrite2Impl(Root, size: Size);
6846 return {{
6847 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6848 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
6849 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset+1); }
6850 }};
6851}
6852
6853std::pair<Register, unsigned>
6854AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root,
6855 unsigned Size) const {
6856 int64_t ConstAddr = 0;
6857
6858 Register PtrBase;
6859 int64_t Offset;
6860 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6861 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6862
6863 if (Offset) {
6864 int64_t OffsetValue0 = Offset;
6865 int64_t OffsetValue1 = Offset + Size;
6866 if (isDSOffset2Legal(Base: PtrBase, Offset0: OffsetValue0, Offset1: OffsetValue1, Size)) {
6867 // (add n0, c0)
6868 return std::pair(PtrBase, OffsetValue0 / Size);
6869 }
6870 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GSub(L: m_Reg(), R: m_Reg()))) {
6871 // TODO
6872
6873 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6874 // TODO
6875 }
6876
6877 return std::pair(Root.getReg(), 0);
6878}
6879
6880/// If \p Root is a G_PTR_ADD with a G_CONSTANT on the right hand side, return
6881/// the base value with the constant offset, and if the offset computation is
6882/// known to be inbounds. There may be intervening copies between \p Root and
6883/// the identified constant. Returns \p Root, 0, false if this does not match
6884/// the pattern.
6885std::tuple<Register, int64_t, bool>
6886AMDGPUInstructionSelector::getPtrBaseWithConstantOffset(
6887 Register Root, const MachineRegisterInfo &MRI) const {
6888 MachineInstr *RootI = getDefIgnoringCopies(Reg: Root, MRI);
6889 if (RootI->getOpcode() != TargetOpcode::G_PTR_ADD)
6890 return {Root, 0, false};
6891
6892 MachineOperand &RHS = RootI->getOperand(i: 2);
6893 std::optional<ValueAndVReg> MaybeOffset =
6894 getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI);
6895 if (!MaybeOffset)
6896 return {Root, 0, false};
6897 bool IsInBounds = RootI->getFlag(Flag: MachineInstr::MIFlag::InBounds);
6898 return {RootI->getOperand(i: 1).getReg(), MaybeOffset->Value.getSExtValue(),
6899 IsInBounds};
6900}
6901
6902static void addZeroImm(MachineInstrBuilder &MIB) {
6903 MIB.addImm(Val: 0);
6904}
6905
6906/// Return a resource descriptor for use with an arbitrary 64-bit pointer. If \p
6907/// BasePtr is not valid, a null base pointer will be used.
6908static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6909 uint32_t FormatLo, uint32_t FormatHi,
6910 Register BasePtr) {
6911 Register RSrc2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6912 Register RSrc3 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6913 Register RSrcHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6914 Register RSrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
6915
6916 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6917 .addDef(RegNo: RSrc2)
6918 .addImm(Val: FormatLo);
6919 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6920 .addDef(RegNo: RSrc3)
6921 .addImm(Val: FormatHi);
6922
6923 // Build the half of the subregister with the constants before building the
6924 // full 128-bit register. If we are building multiple resource descriptors,
6925 // this will allow CSEing of the 2-component register.
6926 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6927 .addDef(RegNo: RSrcHi)
6928 .addReg(RegNo: RSrc2)
6929 .addImm(Val: AMDGPU::sub0)
6930 .addReg(RegNo: RSrc3)
6931 .addImm(Val: AMDGPU::sub1);
6932
6933 Register RSrcLo = BasePtr;
6934 if (!BasePtr) {
6935 RSrcLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6936 B.buildInstr(Opcode: AMDGPU::S_MOV_B64)
6937 .addDef(RegNo: RSrcLo)
6938 .addImm(Val: 0);
6939 }
6940
6941 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6942 .addDef(RegNo: RSrc)
6943 .addReg(RegNo: RSrcLo)
6944 .addImm(Val: AMDGPU::sub0_sub1)
6945 .addReg(RegNo: RSrcHi)
6946 .addImm(Val: AMDGPU::sub2_sub3);
6947
6948 return RSrc;
6949}
6950
6951static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6952 const SIInstrInfo &TII, Register BasePtr) {
6953 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
6954
6955 // FIXME: Why are half the "default" bits ignored based on the addressing
6956 // mode?
6957 return buildRSRC(B, MRI, FormatLo: 0, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
6958}
6959
6960static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6961 const SIInstrInfo &TII, Register BasePtr) {
6962 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
6963
6964 // FIXME: Why are half the "default" bits ignored based on the addressing
6965 // mode?
6966 return buildRSRC(B, MRI, FormatLo: -1, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
6967}
6968
6969AMDGPUInstructionSelector::MUBUFAddressData
6970AMDGPUInstructionSelector::parseMUBUFAddress(Register Src) const {
6971 MUBUFAddressData Data;
6972 Data.N0 = Src;
6973
6974 Register PtrBase;
6975 int64_t Offset;
6976
6977 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6978 getPtrBaseWithConstantOffset(Root: Src, MRI: *MRI);
6979 if (isUInt<32>(x: Offset)) {
6980 Data.N0 = PtrBase;
6981 Data.Offset = Offset;
6982 }
6983
6984 if (MachineInstr *InputAdd
6985 = getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Data.N0, MRI: *MRI)) {
6986 Data.N2 = InputAdd->getOperand(i: 1).getReg();
6987 Data.N3 = InputAdd->getOperand(i: 2).getReg();
6988
6989 // FIXME: Need to fix extra SGPR->VGPRcopies inserted
6990 // FIXME: Don't know this was defined by operand 0
6991 //
6992 // TODO: Remove this when we have copy folding optimizations after
6993 // RegBankSelect.
6994 Data.N2 = getDefIgnoringCopies(Reg: Data.N2, MRI: *MRI)->getOperand(i: 0).getReg();
6995 Data.N3 = getDefIgnoringCopies(Reg: Data.N3, MRI: *MRI)->getOperand(i: 0).getReg();
6996 }
6997
6998 return Data;
6999}
7000
7001/// Return if the addr64 mubuf mode should be used for the given address.
7002bool AMDGPUInstructionSelector::shouldUseAddr64(MUBUFAddressData Addr) const {
7003 // (ptr_add N2, N3) -> addr64, or
7004 // (ptr_add (ptr_add N2, N3), C1) -> addr64
7005 if (Addr.N2)
7006 return true;
7007
7008 const RegisterBank *N0Bank = RBI.getRegBank(Reg: Addr.N0, MRI: *MRI, TRI);
7009 return N0Bank->getID() == AMDGPU::VGPRRegBankID;
7010}
7011
7012/// Split an immediate offset \p ImmOffset depending on whether it fits in the
7013/// immediate field. Modifies \p ImmOffset and sets \p SOffset to the variable
7014/// component.
7015void AMDGPUInstructionSelector::splitIllegalMUBUFOffset(
7016 MachineIRBuilder &B, Register &SOffset, int64_t &ImmOffset) const {
7017 if (TII.isLegalMUBUFImmOffset(Imm: ImmOffset))
7018 return;
7019
7020 // Illegal offset, store it in soffset.
7021 SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7022 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
7023 .addDef(RegNo: SOffset)
7024 .addImm(Val: ImmOffset);
7025 ImmOffset = 0;
7026}
7027
7028bool AMDGPUInstructionSelector::selectMUBUFAddr64Impl(
7029 MachineOperand &Root, Register &VAddr, Register &RSrcReg,
7030 Register &SOffset, int64_t &Offset) const {
7031 // FIXME: Predicates should stop this from reaching here.
7032 // addr64 bit was removed for volcanic islands.
7033 if (!STI.hasAddr64() || STI.useFlatForGlobal())
7034 return false;
7035
7036 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
7037 if (!shouldUseAddr64(Addr: AddrData))
7038 return false;
7039
7040 Register N0 = AddrData.N0;
7041 Register N2 = AddrData.N2;
7042 Register N3 = AddrData.N3;
7043 Offset = AddrData.Offset;
7044
7045 // Base pointer for the SRD.
7046 Register SRDPtr;
7047
7048 if (N2) {
7049 if (RBI.getRegBank(Reg: N2, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7050 assert(N3);
7051 if (RBI.getRegBank(Reg: N3, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7052 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
7053 // addr64, and construct the default resource from a 0 address.
7054 VAddr = N0;
7055 } else {
7056 SRDPtr = N3;
7057 VAddr = N2;
7058 }
7059 } else {
7060 // N2 is not divergent.
7061 SRDPtr = N2;
7062 VAddr = N3;
7063 }
7064 } else if (RBI.getRegBank(Reg: N0, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7065 // Use the default null pointer in the resource
7066 VAddr = N0;
7067 } else {
7068 // N0 -> offset, or
7069 // (N0 + C1) -> offset
7070 SRDPtr = N0;
7071 }
7072
7073 MachineIRBuilder B(*Root.getParent());
7074 RSrcReg = buildAddr64RSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
7075 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
7076 return true;
7077}
7078
7079bool AMDGPUInstructionSelector::selectMUBUFOffsetImpl(
7080 MachineOperand &Root, Register &RSrcReg, Register &SOffset,
7081 int64_t &Offset) const {
7082
7083 // FIXME: Pattern should not reach here.
7084 if (STI.useFlatForGlobal())
7085 return false;
7086
7087 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
7088 if (shouldUseAddr64(Addr: AddrData))
7089 return false;
7090
7091 // N0 -> offset, or
7092 // (N0 + C1) -> offset
7093 Register SRDPtr = AddrData.N0;
7094 Offset = AddrData.Offset;
7095
7096 // TODO: Look through extensions for 32-bit soffset.
7097 MachineIRBuilder B(*Root.getParent());
7098
7099 RSrcReg = buildOffsetSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
7100 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
7101 return true;
7102}
7103
7104InstructionSelector::ComplexRendererFns
7105AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const {
7106 Register VAddr;
7107 Register RSrcReg;
7108 Register SOffset;
7109 int64_t Offset = 0;
7110
7111 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset))
7112 return {};
7113
7114 // FIXME: Use defaulted operands for trailing 0s and remove from the complex
7115 // pattern.
7116 return {{
7117 [=](MachineInstrBuilder &MIB) { // rsrc
7118 MIB.addReg(RegNo: RSrcReg);
7119 },
7120 [=](MachineInstrBuilder &MIB) { // vaddr
7121 MIB.addReg(RegNo: VAddr);
7122 },
7123 [=](MachineInstrBuilder &MIB) { // soffset
7124 if (SOffset)
7125 MIB.addReg(RegNo: SOffset);
7126 else if (STI.hasRestrictedSOffset())
7127 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7128 else
7129 MIB.addImm(Val: 0);
7130 },
7131 [=](MachineInstrBuilder &MIB) { // offset
7132 MIB.addImm(Val: Offset);
7133 },
7134 addZeroImm, // cpol
7135 addZeroImm, // tfe
7136 addZeroImm // swz
7137 }};
7138}
7139
7140InstructionSelector::ComplexRendererFns
7141AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const {
7142 Register RSrcReg;
7143 Register SOffset;
7144 int64_t Offset = 0;
7145
7146 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset))
7147 return {};
7148
7149 return {{
7150 [=](MachineInstrBuilder &MIB) { // rsrc
7151 MIB.addReg(RegNo: RSrcReg);
7152 },
7153 [=](MachineInstrBuilder &MIB) { // soffset
7154 if (SOffset)
7155 MIB.addReg(RegNo: SOffset);
7156 else if (STI.hasRestrictedSOffset())
7157 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7158 else
7159 MIB.addImm(Val: 0);
7160 },
7161 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }, // offset
7162 addZeroImm, // cpol
7163 addZeroImm, // tfe
7164 addZeroImm, // swz
7165 }};
7166}
7167
7168InstructionSelector::ComplexRendererFns
7169AMDGPUInstructionSelector::selectBUFSOffset(MachineOperand &Root) const {
7170
7171 Register SOffset = Root.getReg();
7172
7173 if (STI.hasRestrictedSOffset() && mi_match(R: SOffset, MRI: *MRI, P: m_ZeroInt()))
7174 SOffset = AMDGPU::SGPR_NULL;
7175
7176 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); }}};
7177}
7178
7179/// Get an immediate that must be 32-bits, and treated as zero extended.
7180static std::optional<uint64_t>
7181getConstantZext32Val(Register Reg, const MachineRegisterInfo &MRI) {
7182 // getIConstantVRegVal sexts any values, so see if that matters.
7183 std::optional<int64_t> OffsetVal = getIConstantVRegSExtVal(VReg: Reg, MRI);
7184 if (!OffsetVal || !isInt<32>(x: *OffsetVal))
7185 return std::nullopt;
7186 return Lo_32(Value: *OffsetVal);
7187}
7188
7189InstructionSelector::ComplexRendererFns
7190AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const {
7191 std::optional<uint64_t> OffsetVal =
7192 Root.isImm() ? Root.getImm() : getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7193 if (!OffsetVal)
7194 return {};
7195
7196 std::optional<int64_t> EncodedImm =
7197 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: *OffsetVal, IsBuffer: true);
7198 if (!EncodedImm)
7199 return {};
7200
7201 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7202}
7203
7204InstructionSelector::ComplexRendererFns
7205AMDGPUInstructionSelector::selectSMRDBufferImm32(MachineOperand &Root) const {
7206 assert(STI.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
7207
7208 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7209 if (!OffsetVal)
7210 return {};
7211
7212 std::optional<int64_t> EncodedImm =
7213 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: *OffsetVal);
7214 if (!EncodedImm)
7215 return {};
7216
7217 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7218}
7219
7220InstructionSelector::ComplexRendererFns
7221AMDGPUInstructionSelector::selectSMRDBufferSgprImm(MachineOperand &Root) const {
7222 // Match the (soffset + offset) pair as a 32-bit register base and
7223 // an immediate offset.
7224 Register SOffset;
7225 unsigned Offset;
7226 std::tie(args&: SOffset, args&: Offset) = AMDGPU::getBaseWithConstantOffset(
7227 MRI&: *MRI, Reg: Root.getReg(), ValueTracking: VT, /*CheckNUW*/ true);
7228 if (!SOffset)
7229 return std::nullopt;
7230
7231 std::optional<int64_t> EncodedOffset =
7232 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: Offset, /* IsBuffer */ true);
7233 if (!EncodedOffset)
7234 return std::nullopt;
7235
7236 assert(MRI->getType(SOffset).getSizeInBits() == 32);
7237 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
7238 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedOffset); }}};
7239}
7240
7241std::pair<Register, unsigned>
7242AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(MachineOperand &Root,
7243 bool &Matched) const {
7244 Matched = false;
7245
7246 Register Src;
7247 unsigned Mods;
7248 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
7249
7250 if (mi_match(R: Src, MRI: *MRI, P: m_GFPExt(Src: m_Reg(R&: Src)))) {
7251 assert(MRI->getType(Src) == LLT::scalar(16));
7252
7253 // Only change Src if src modifier could be gained. In such cases new Src
7254 // could be sgpr but this does not violate constant bus restriction for
7255 // instruction that is being selected.
7256 Src = stripBitCast(Reg: Src, MRI&: *MRI);
7257
7258 const auto CheckAbsNeg = [&]() {
7259 // Be careful about folding modifiers if we already have an abs. fneg is
7260 // applied last, so we don't want to apply an earlier fneg.
7261 if ((Mods & SISrcMods::ABS) == 0) {
7262 unsigned ModsTmp;
7263 std::tie(args&: Src, args&: ModsTmp) = selectVOP3ModsImpl(Src);
7264
7265 if ((ModsTmp & SISrcMods::NEG) != 0)
7266 Mods ^= SISrcMods::NEG;
7267
7268 if ((ModsTmp & SISrcMods::ABS) != 0)
7269 Mods |= SISrcMods::ABS;
7270 }
7271 };
7272
7273 CheckAbsNeg();
7274
7275 // op_sel/op_sel_hi decide the source type and source.
7276 // If the source's op_sel_hi is set, it indicates to do a conversion from
7277 // fp16. If the sources's op_sel is set, it picks the high half of the
7278 // source register.
7279
7280 Mods |= SISrcMods::OP_SEL_1;
7281
7282 // The instruction reads a 32-bit source and selects a half of it, so look
7283 // for the 32-bit register the 16-bit value is a half of.
7284 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
7285 // Src is now the 32-bit source and op_sel picks its high half.
7286 Mods |= SISrcMods::OP_SEL_0;
7287 CheckAbsNeg();
7288 } else if (isExtractLoElt(MRI&: *MRI, In: Src, Out&: Src)) {
7289 // op_sel already picks the low half, so the 32-bit source can be used
7290 // directly. Unlike the high half, only an fneg/fabs that acts on each
7291 // 16-bit element can be folded here: one that acts on the 32-bit value
7292 // touches bit 31 and leaves the low half alone.
7293 if (MRI->getType(Reg: Src).isFixedVector(NumElements: 2, ScalarSize: 16))
7294 CheckAbsNeg();
7295 }
7296 // Otherwise Src is genuinely 16 bits wide and widenSrcIfVGPR16 widens it
7297 // when the operand is rendered.
7298
7299 Matched = true;
7300 }
7301
7302 return {Src, Mods};
7303}
7304
7305InstructionSelector::ComplexRendererFns
7306AMDGPUInstructionSelector::selectVOP3PMadMixModsExt(
7307 MachineOperand &Root) const {
7308 Register Src;
7309 unsigned Mods;
7310 bool Matched;
7311 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7312 if (!Matched)
7313 return {};
7314
7315 return {{
7316 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7317 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7318 }};
7319}
7320
7321InstructionSelector::ComplexRendererFns
7322AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const {
7323 Register Src;
7324 unsigned Mods;
7325 bool Matched;
7326 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7327
7328 return {{
7329 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7330 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7331 }};
7332}
7333
7334InstructionSelector::ComplexRendererFns
7335AMDGPUInstructionSelector::selectVOP3PMadMixModsExtNeg(
7336 MachineOperand &Root) const {
7337 Register Src;
7338 unsigned Mods;
7339 bool Matched;
7340 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7341 if (!Matched)
7342 return {};
7343
7344 return {{
7345 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7346 [=](MachineInstrBuilder &MIB) {
7347 MIB.addImm(Val: Mods ^ SISrcMods::NEG);
7348 } // src_mods
7349 }};
7350}
7351
7352InstructionSelector::ComplexRendererFns
7353AMDGPUInstructionSelector::selectVOP3PMadMixModsNeg(
7354 MachineOperand &Root) const {
7355 Register Src;
7356 unsigned Mods;
7357 bool Matched;
7358 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7359
7360 return {{
7361 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7362 [=](MachineInstrBuilder &MIB) {
7363 MIB.addImm(Val: Mods ^ SISrcMods::NEG);
7364 } // src_mods
7365 }};
7366}
7367
7368bool AMDGPUInstructionSelector::selectSBarrierSignalIsfirst(
7369 MachineInstr &I, Intrinsic::ID IntrID) const {
7370 MachineBasicBlock *MBB = I.getParent();
7371 const DebugLoc &DL = I.getDebugLoc();
7372 Register CCReg = I.getOperand(i: 0).getReg();
7373
7374 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7375 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_EQ_U32)).addImm(Val: 0).addImm(Val: 0);
7376
7377 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM))
7378 .addImm(Val: I.getOperand(i: 2).getImm());
7379
7380 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg).addReg(RegNo: AMDGPU::SCC);
7381
7382 I.eraseFromParent();
7383 return RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32_XM0_XEXECRegClass,
7384 MRI&: *MRI);
7385}
7386
7387bool AMDGPUInstructionSelector::selectSGetBarrierState(
7388 MachineInstr &I, Intrinsic::ID IntrID) const {
7389 MachineBasicBlock *MBB = I.getParent();
7390 const DebugLoc &DL = I.getDebugLoc();
7391 const MachineOperand &BarOp = I.getOperand(i: 2);
7392 std::optional<int64_t> BarValImm =
7393 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7394
7395 if (!BarValImm) {
7396 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7397 .addReg(RegNo: BarOp.getReg());
7398 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7399 }
7400 MachineInstrBuilder MIB;
7401 unsigned Opc = BarValImm ? AMDGPU::S_GET_BARRIER_STATE_IMM
7402 : AMDGPU::S_GET_BARRIER_STATE_M0;
7403 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7404
7405 auto DstReg = I.getOperand(i: 0).getReg();
7406 const TargetRegisterClass *DstRC =
7407 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
7408 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7409 return false;
7410 MIB.addDef(RegNo: DstReg);
7411 if (BarValImm) {
7412 MIB.addImm(Val: *BarValImm);
7413 }
7414 I.eraseFromParent();
7415 return true;
7416}
7417
7418unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID) {
7419 if (HasInlineConst) {
7420 switch (IntrID) {
7421 default:
7422 llvm_unreachable("not a named barrier op");
7423 case Intrinsic::amdgcn_s_barrier_join:
7424 return AMDGPU::S_BARRIER_JOIN_IMM;
7425 case Intrinsic::amdgcn_s_wakeup_barrier:
7426 return AMDGPU::S_WAKEUP_BARRIER_IMM;
7427 case Intrinsic::amdgcn_s_get_named_barrier_state:
7428 return AMDGPU::S_GET_BARRIER_STATE_IMM;
7429 };
7430 } else {
7431 switch (IntrID) {
7432 default:
7433 llvm_unreachable("not a named barrier op");
7434 case Intrinsic::amdgcn_s_barrier_join:
7435 return AMDGPU::S_BARRIER_JOIN_M0;
7436 case Intrinsic::amdgcn_s_wakeup_barrier:
7437 return AMDGPU::S_WAKEUP_BARRIER_M0;
7438 case Intrinsic::amdgcn_s_get_named_barrier_state:
7439 return AMDGPU::S_GET_BARRIER_STATE_M0;
7440 };
7441 }
7442}
7443
7444bool AMDGPUInstructionSelector::selectNamedBarrierInit(
7445 MachineInstr &I, Intrinsic::ID IntrID) const {
7446 MachineBasicBlock *MBB = I.getParent();
7447 const DebugLoc &DL = I.getDebugLoc();
7448 const MachineOperand &BarOp = I.getOperand(i: 1);
7449 const MachineOperand &CntOp = I.getOperand(i: 2);
7450
7451 // A member count of 0 means "keep existing member count". That plus a known
7452 // constant value for the barrier ID lets us use the immarg form.
7453 if (IntrID == Intrinsic::amdgcn_s_barrier_signal_var) {
7454 std::optional<int64_t> CntImm =
7455 getIConstantVRegSExtVal(VReg: CntOp.getReg(), MRI: *MRI);
7456 if (CntImm && *CntImm == 0) {
7457 std::optional<int64_t> BarValImm =
7458 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7459 if (BarValImm) {
7460 uint32_t BarID = *BarValImm & 0x3F;
7461 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_IMM))
7462 .addImm(Val: BarID);
7463 I.eraseFromParent();
7464 return true;
7465 }
7466 }
7467 }
7468
7469 // BarID = BarOp & 0x3F
7470 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7471 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7472 .add(MO: BarOp)
7473 .addImm(Val: 0x3F)
7474 .setOperandDead(3); // Dead scc
7475
7476 // MO = ((CntOp & CntMask) << shAmt) | BarID
7477 unsigned CntMask =
7478 AMDGPU::getBarrierMemberCountBitMask(Version: AMDGPU::getIsaVersion(GPU: STI.getCPU()));
7479 Register TmpReg2 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7480 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg2)
7481 .add(MO: CntOp)
7482 .addImm(Val: CntMask)
7483 .setOperandDead(3); // Dead scc
7484
7485 Register TmpReg3 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7486 constexpr unsigned ShAmt = 16;
7487 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg3)
7488 .addReg(RegNo: TmpReg2)
7489 .addImm(Val: ShAmt)
7490 .setOperandDead(3); // Dead scc
7491
7492 Register TmpReg4 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7493 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B32), DestReg: TmpReg4)
7494 .addReg(RegNo: TmpReg1)
7495 .addReg(RegNo: TmpReg3)
7496 .setOperandDead(3); // Dead scc;
7497
7498 auto CopyMIB =
7499 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0).addReg(RegNo: TmpReg4);
7500 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7501
7502 unsigned Opc = IntrID == Intrinsic::amdgcn_s_barrier_init
7503 ? AMDGPU::S_BARRIER_INIT_M0
7504 : AMDGPU::S_BARRIER_SIGNAL_M0;
7505 MachineInstrBuilder MIB;
7506 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7507
7508 I.eraseFromParent();
7509 return true;
7510}
7511
7512bool AMDGPUInstructionSelector::selectNamedBarrierInst(
7513 MachineInstr &I, Intrinsic::ID IntrID) const {
7514 MachineBasicBlock *MBB = I.getParent();
7515 const DebugLoc &DL = I.getDebugLoc();
7516 MachineOperand BarOp = IntrID == Intrinsic::amdgcn_s_get_named_barrier_state
7517 ? I.getOperand(i: 2)
7518 : I.getOperand(i: 1);
7519 std::optional<int64_t> BarValImm =
7520 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7521
7522 if (!BarValImm) {
7523 // BarID = BarOp & 0x3F
7524 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7525 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7526 .addReg(RegNo: BarOp.getReg())
7527 .addImm(Val: 0x3F)
7528 .setOperandDead(3); // Dead scc;
7529
7530 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7531 .addReg(RegNo: TmpReg1);
7532 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7533 }
7534
7535 MachineInstrBuilder MIB;
7536 unsigned Opc = getNamedBarrierOp(HasInlineConst: BarValImm.has_value(), IntrID);
7537 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7538
7539 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
7540 auto DstReg = I.getOperand(i: 0).getReg();
7541 const TargetRegisterClass *DstRC =
7542 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
7543 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7544 return false;
7545 MIB.addDef(RegNo: DstReg);
7546 }
7547
7548 if (BarValImm) {
7549 uint32_t BarId = *BarValImm & 0x3F;
7550 MIB.addImm(Val: BarId);
7551 }
7552
7553 I.eraseFromParent();
7554 return true;
7555}
7556
7557void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB,
7558 const MachineInstr &MI,
7559 int OpIdx) const {
7560 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7561 "Expected G_CONSTANT");
7562 MIB.addImm(Val: -MI.getOperand(i: 1).getCImm()->getSExtValue());
7563}
7564
7565void AMDGPUInstructionSelector::renderBitcastFPImm(MachineInstrBuilder &MIB,
7566 const MachineInstr &MI,
7567 int OpIdx) const {
7568 const MachineOperand &Op = MI.getOperand(i: 1);
7569 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1);
7570 MIB.addImm(Val: Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
7571}
7572
7573void AMDGPUInstructionSelector::renderCountTrailingOnesImm(
7574 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7575 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7576 "Expected G_CONSTANT");
7577 MIB.addImm(Val: MI.getOperand(i: 1).getCImm()->getValue().countTrailingOnes());
7578}
7579
7580/// This only really exists to satisfy DAG type checking machinery, so is a
7581/// no-op here.
7582void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB,
7583 const MachineInstr &MI,
7584 int OpIdx) const {
7585 const MachineOperand &Op = MI.getOperand(i: OpIdx);
7586 int64_t Imm;
7587 if (Op.isReg() && mi_match(R: Op.getReg(), MRI: *MRI, P: m_ICst(Cst&: Imm)))
7588 MIB.addImm(Val: Imm);
7589 else
7590 MIB.addImm(Val: Op.getImm());
7591}
7592
7593void AMDGPUInstructionSelector::renderZextBoolTImm(MachineInstrBuilder &MIB,
7594 const MachineInstr &MI,
7595 int OpIdx) const {
7596 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() != 0);
7597}
7598
7599void AMDGPUInstructionSelector::renderOpSelTImm(MachineInstrBuilder &MIB,
7600 const MachineInstr &MI,
7601 int OpIdx) const {
7602 assert(OpIdx >= 0 && "expected to match an immediate operand");
7603 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7604}
7605
7606void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_0(
7607 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7608 assert(OpIdx >= 0 && "expected to match an immediate operand");
7609 MIB.addImm(
7610 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7611}
7612
7613void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_1(
7614 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7615 assert(OpIdx >= 0 && "expected to match an immediate operand");
7616 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x1)
7617 ? (int64_t)(SISrcMods::OP_SEL_0 | SISrcMods::DST_OP_SEL)
7618 : (int64_t)SISrcMods::DST_OP_SEL);
7619}
7620
7621void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_0(
7622 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7623 assert(OpIdx >= 0 && "expected to match an immediate operand");
7624 MIB.addImm(
7625 Val: (MI.getOperand(i: OpIdx).getImm() & 0x2) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7626}
7627
7628void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_1(
7629 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7630 assert(OpIdx >= 0 && "expected to match an immediate operand");
7631 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7632 ? (int64_t)(SISrcMods::OP_SEL_0)
7633 : 0);
7634}
7635
7636void AMDGPUInstructionSelector::renderDstSelToOpSelXForm(
7637 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7638 assert(OpIdx >= 0 && "expected to match an immediate operand");
7639 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::DST_OP_SEL)
7640 : 0);
7641}
7642
7643void AMDGPUInstructionSelector::renderSrcSelToOpSelXForm(
7644 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7645 assert(OpIdx >= 0 && "expected to match an immediate operand");
7646 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::OP_SEL_0)
7647 : 0);
7648}
7649
7650void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_2_0(
7651 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7652 assert(OpIdx >= 0 && "expected to match an immediate operand");
7653 MIB.addImm(
7654 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7655}
7656
7657void AMDGPUInstructionSelector::renderDstSelToOpSel3XFormXForm(
7658 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7659 assert(OpIdx >= 0 && "expected to match an immediate operand");
7660 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7661 ? (int64_t)SISrcMods::DST_OP_SEL
7662 : 0);
7663}
7664
7665void AMDGPUInstructionSelector::renderExtractCPol(MachineInstrBuilder &MIB,
7666 const MachineInstr &MI,
7667 int OpIdx) const {
7668 assert(OpIdx >= 0 && "expected to match an immediate operand");
7669 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() &
7670 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7671 : AMDGPU::CPol::ALL_pregfx12));
7672}
7673
7674void AMDGPUInstructionSelector::renderExtractSWZ(MachineInstrBuilder &MIB,
7675 const MachineInstr &MI,
7676 int OpIdx) const {
7677 assert(OpIdx >= 0 && "expected to match an immediate operand");
7678 const bool Swizzle = MI.getOperand(i: OpIdx).getImm() &
7679 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::SWZ
7680 : AMDGPU::CPol::SWZ_pregfx12);
7681 MIB.addImm(Val: Swizzle);
7682}
7683
7684void AMDGPUInstructionSelector::renderExtractCpolSetGLC(
7685 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7686 assert(OpIdx >= 0 && "expected to match an immediate operand");
7687 const uint32_t Cpol = MI.getOperand(i: OpIdx).getImm() &
7688 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7689 : AMDGPU::CPol::ALL_pregfx12);
7690 MIB.addImm(Val: Cpol | AMDGPU::CPol::GLC);
7691}
7692
7693void AMDGPUInstructionSelector::renderFPPow2ToExponent(MachineInstrBuilder &MIB,
7694 const MachineInstr &MI,
7695 int OpIdx) const {
7696 const APFloat &APF = MI.getOperand(i: 1).getFPImm()->getValueAPF();
7697 int ExpVal = APF.getExactLog2Abs();
7698 assert(ExpVal != INT_MIN);
7699 MIB.addImm(Val: ExpVal);
7700}
7701
7702void AMDGPUInstructionSelector::renderRoundMode(MachineInstrBuilder &MIB,
7703 const MachineInstr &MI,
7704 int OpIdx) const {
7705 // "round.towardzero" -> TowardZero 0 -> FP_ROUND_ROUND_TO_ZERO 3
7706 // "round.tonearest" -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
7707 // "round.upward" -> TowardPositive 2 -> FP_ROUND_ROUND_TO_INF 1
7708 // "round.downward -> TowardNegative 3 -> FP_ROUND_ROUND_TO_NEGINF 2
7709 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() + 3) % 4);
7710}
7711
7712void AMDGPUInstructionSelector::renderVOP3PModsNeg(MachineInstrBuilder &MIB,
7713 const MachineInstr &MI,
7714 int OpIdx) const {
7715 unsigned Mods = SISrcMods::OP_SEL_1;
7716 if (MI.getOperand(i: OpIdx).getImm())
7717 Mods ^= SISrcMods::NEG;
7718 MIB.addImm(Val: (int64_t)Mods);
7719}
7720
7721void AMDGPUInstructionSelector::renderVOP3PModsNegs(MachineInstrBuilder &MIB,
7722 const MachineInstr &MI,
7723 int OpIdx) const {
7724 unsigned Mods = SISrcMods::OP_SEL_1;
7725 if (MI.getOperand(i: OpIdx).getImm())
7726 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
7727 MIB.addImm(Val: (int64_t)Mods);
7728}
7729
7730void AMDGPUInstructionSelector::renderVOP3PModsNegAbs(MachineInstrBuilder &MIB,
7731 const MachineInstr &MI,
7732 int OpIdx) const {
7733 unsigned Val = MI.getOperand(i: OpIdx).getImm();
7734 unsigned Mods = SISrcMods::OP_SEL_1; // default: none
7735 if (Val == 1) // neg
7736 Mods ^= SISrcMods::NEG;
7737 if (Val == 2) // abs
7738 Mods ^= SISrcMods::ABS;
7739 if (Val == 3) // neg and abs
7740 Mods ^= (SISrcMods::NEG | SISrcMods::ABS);
7741 MIB.addImm(Val: (int64_t)Mods);
7742}
7743
7744void AMDGPUInstructionSelector::renderPrefetchLoc(MachineInstrBuilder &MIB,
7745 const MachineInstr &MI,
7746 int OpIdx) const {
7747 uint32_t V = MI.getOperand(i: 2).getImm();
7748 V = (AMDGPU::CPol::SCOPE_MASK - (V & AMDGPU::CPol::SCOPE_MASK))
7749 << AMDGPU::CPol::SCOPE_SHIFT;
7750 if (!Subtarget->hasSafeCUPrefetch())
7751 V = std::max(a: V, b: (uint32_t)AMDGPU::CPol::SCOPE_SE); // CU scope is unsafe
7752 MIB.addImm(Val: V);
7753}
7754
7755/// Convert from 2-bit value to enum values used for op_sel* source modifiers.
7756void AMDGPUInstructionSelector::renderScaledMAIIntrinsicOperand(
7757 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7758 unsigned Val = MI.getOperand(i: OpIdx).getImm();
7759 unsigned New = 0;
7760 if (Val & 0x1)
7761 New |= SISrcMods::OP_SEL_0;
7762 if (Val & 0x2)
7763 New |= SISrcMods::OP_SEL_1;
7764 MIB.addImm(Val: New);
7765}
7766
7767bool AMDGPUInstructionSelector::isInlineImmediate(const APInt &Imm) const {
7768 return TII.isInlineConstant(Imm);
7769}
7770
7771bool AMDGPUInstructionSelector::isInlineImmediate(const APFloat &Imm) const {
7772 return TII.isInlineConstant(Imm);
7773}
7774