1//===- AMDGPUInstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUInstructionSelector.h"
15#include "AMDGPU.h"
16#include "AMDGPUGlobalISelUtils.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPURegisterBankInfo.h"
19#include "SIMachineFunctionInfo.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
22#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
23#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
24#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
25#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/IntrinsicsAMDGPU.h"
29#include <optional>
30
31#define DEBUG_TYPE "amdgpu-isel"
32
33using namespace llvm;
34using namespace MIPatternMatch;
35
36#define GET_GLOBALISEL_IMPL
37#define AMDGPUSubtarget GCNSubtarget
38#include "AMDGPUGenGlobalISel.inc"
39#undef GET_GLOBALISEL_IMPL
40#undef AMDGPUSubtarget
41
42AMDGPUInstructionSelector::AMDGPUInstructionSelector(
43 const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI)
44 : TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI), STI(STI),
45#define GET_GLOBALISEL_PREDICATES_INIT
46#include "AMDGPUGenGlobalISel.inc"
47#undef GET_GLOBALISEL_PREDICATES_INIT
48#define GET_GLOBALISEL_TEMPORARIES_INIT
49#include "AMDGPUGenGlobalISel.inc"
50#undef GET_GLOBALISEL_TEMPORARIES_INIT
51{
52}
53
54const char *AMDGPUInstructionSelector::getName() { return DEBUG_TYPE; }
55
56void AMDGPUInstructionSelector::setupMF(MachineFunction &MF,
57 GISelValueTracking *VT,
58 CodeGenCoverage *CoverageInfo,
59 ProfileSummaryInfo *PSI,
60 BlockFrequencyInfo *BFI) {
61 MRI = &MF.getRegInfo();
62 Subtarget = &MF.getSubtarget<GCNSubtarget>();
63 Subtarget->checkSubtargetFeatures(F: MF.getFunction());
64 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
65}
66
67// Return the wave level SGPR base address if this is a wave address.
68static Register getWaveAddress(const MachineInstr *Def) {
69 return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS
70 ? Def->getOperand(i: 1).getReg()
71 : Register();
72}
73
74bool AMDGPUInstructionSelector::isVCC(Register Reg,
75 const MachineRegisterInfo &MRI) const {
76 // The verifier is oblivious to s1 being a valid value for wavesize registers.
77 if (Reg.isPhysical())
78 return false;
79
80 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
81 const TargetRegisterClass *RC =
82 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
83 if (RC) {
84 const LLT Ty = MRI.getType(Reg);
85 if (!Ty.isValid() || Ty.getSizeInBits() != 1)
86 return false;
87 // G_TRUNC s1 result is never vcc.
88 return !mi_match(R: Reg, MRI, P: m_GTrunc(Src: m_Reg())) &&
89 RC->hasSuperClassEq(RC: TRI.getBoolRC());
90 }
91
92 const RegisterBank *RB = cast<const RegisterBank *>(Val: RegClassOrBank);
93 return RB->getID() == AMDGPU::VCCRegBankID;
94}
95
96bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(MachineInstr &MI,
97 unsigned NewOpc) const {
98 MI.setDesc(TII.get(Opcode: NewOpc));
99 MI.removeOperand(OpNo: 1); // Remove intrinsic ID.
100 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
101
102 Register DstReg = MI.getOperand(i: 0).getReg();
103 Register SrcReg = MI.getOperand(i: 1).getReg();
104
105 // TODO: This should be legalized to s32 if needed
106 if (MRI->getType(Reg: DstReg) == LLT::scalar(SizeInBits: 1))
107 return false;
108
109 const TargetRegisterClass *DstRC =
110 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
111 const TargetRegisterClass *SrcRC =
112 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
113 if (!DstRC || DstRC != SrcRC)
114 return false;
115
116 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
117 !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
118 return false;
119 const MCInstrDesc &MCID = MI.getDesc();
120 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
121 MI.getOperand(i: 0).setIsEarlyClobber(true);
122 }
123 return true;
124}
125
126bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
127 const DebugLoc &DL = I.getDebugLoc();
128 MachineBasicBlock *BB = I.getParent();
129 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
130
131 Register DstReg = I.getOperand(i: 0).getReg();
132 Register SrcReg = I.getOperand(i: 1).getReg();
133
134 if (isVCC(Reg: DstReg, MRI: *MRI)) {
135 if (SrcReg == AMDGPU::SCC) {
136 const TargetRegisterClass *RC =
137 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
138 if (!RC)
139 return true;
140 return RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI);
141 }
142
143 if (!isVCC(Reg: SrcReg, MRI: *MRI)) {
144 // TODO: Should probably leave the copy and let copyPhysReg expand it.
145 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI))
146 return false;
147
148 const TargetRegisterClass *SrcRC =
149 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
150
151 std::optional<ValueAndVReg> ConstVal =
152 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI, LookThroughInstrs: true);
153 if (ConstVal) {
154 unsigned MovOpc =
155 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
156 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: DstReg)
157 .addImm(Val: ConstVal->Value.getBoolValue() ? -1 : 0);
158 } else {
159 Register MaskedReg = MRI->createVirtualRegister(RegClass: SrcRC);
160
161 // We can't trust the high bits at this point, so clear them.
162
163 // TODO: Skip masking high bits if def is known boolean.
164
165 if (AMDGPU::getRegBitWidth(RCID: SrcRC->getID()) == 16) {
166 assert(Subtarget->useRealTrue16Insts());
167 const int64_t NoMods = 0;
168 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B16_t16_e64), DestReg: MaskedReg)
169 .addImm(Val: NoMods)
170 .addImm(Val: 1)
171 .addImm(Val: NoMods)
172 .addReg(RegNo: SrcReg)
173 .addImm(Val: NoMods);
174 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U16_t16_e64), DestReg: DstReg)
175 .addImm(Val: NoMods)
176 .addImm(Val: 0)
177 .addImm(Val: NoMods)
178 .addReg(RegNo: MaskedReg)
179 .addImm(Val: NoMods);
180 } else {
181 bool IsSGPR = TRI.isSGPRClass(RC: SrcRC);
182 unsigned AndOpc = IsSGPR ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32;
183 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: MaskedReg)
184 .addImm(Val: 1)
185 .addReg(RegNo: SrcReg);
186 if (IsSGPR)
187 And.setOperandDead(3); // Dead scc
188
189 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U32_e64), DestReg: DstReg)
190 .addImm(Val: 0)
191 .addReg(RegNo: MaskedReg);
192 }
193 }
194
195 if (!MRI->getRegClassOrNull(Reg: SrcReg))
196 MRI->setRegClass(Reg: SrcReg, RC: SrcRC);
197 I.eraseFromParent();
198 return true;
199 }
200
201 const TargetRegisterClass *RC =
202 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
203 if (RC && !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
204 return false;
205
206 return true;
207 }
208
209 for (const MachineOperand &MO : I.operands()) {
210 if (MO.getReg().isPhysical())
211 continue;
212
213 const TargetRegisterClass *RC =
214 TRI.getConstrainedRegClassForReg(Reg: MO.getReg(), MRI: *MRI);
215 if (!RC)
216 continue;
217 RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI);
218 }
219 return true;
220}
221
222bool AMDGPUInstructionSelector::selectCOPY_SCC_VCC(MachineInstr &I) const {
223 const DebugLoc &DL = I.getDebugLoc();
224 MachineBasicBlock *BB = I.getParent();
225 Register VCCReg = I.getOperand(i: 1).getReg();
226 MachineInstr *Cmp;
227
228 // Set SCC as a side effect with S_CMP or S_OR.
229 if (STI.hasScalarCompareEq64()) {
230 unsigned CmpOpc =
231 STI.isWave64() ? AMDGPU::S_CMP_LG_U64 : AMDGPU::S_CMP_LG_U32;
232 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: CmpOpc)).addReg(RegNo: VCCReg).addImm(Val: 0);
233 } else {
234 Register DeadDst = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
235 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B64), DestReg: DeadDst)
236 .addReg(RegNo: VCCReg)
237 .addReg(RegNo: VCCReg);
238 }
239
240 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
241
242 Register DstReg = I.getOperand(i: 0).getReg();
243 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: AMDGPU::SCC);
244
245 I.eraseFromParent();
246 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
247}
248
249bool AMDGPUInstructionSelector::selectCOPY_VCC_SCC(MachineInstr &I) const {
250 const DebugLoc &DL = I.getDebugLoc();
251 MachineBasicBlock *BB = I.getParent();
252
253 Register DstReg = I.getOperand(i: 0).getReg();
254 Register SrcReg = I.getOperand(i: 1).getReg();
255 std::optional<ValueAndVReg> Arg =
256 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 1).getReg(), MRI: *MRI);
257
258 if (Arg) {
259 const int64_t Value = Arg->Value.getZExtValue();
260 if (Value == 0) {
261 unsigned Opcode = STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
262 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DstReg).addImm(Val: 0);
263 } else {
264 assert(Value == 1);
265 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: TRI.getExec());
266 }
267 I.eraseFromParent();
268 return RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI);
269 }
270
271 // RegBankLegalize ensures that SrcReg is bool in reg (high bits are 0).
272 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC).addReg(RegNo: SrcReg);
273
274 unsigned SelectOpcode =
275 STI.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
276 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
277 .addReg(RegNo: TRI.getExec())
278 .addImm(Val: 0);
279
280 I.eraseFromParent();
281 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
282 return true;
283}
284
285bool AMDGPUInstructionSelector::selectReadAnyLane(MachineInstr &I) const {
286 Register DstReg = I.getOperand(i: 0).getReg();
287 Register SrcReg = I.getOperand(i: 1).getReg();
288
289 const DebugLoc &DL = I.getDebugLoc();
290 MachineBasicBlock *BB = I.getParent();
291
292 auto RFL = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
293 .addReg(RegNo: SrcReg);
294
295 I.eraseFromParent();
296 constrainSelectedInstRegOperands(I&: *RFL, TII, TRI, RBI);
297 return true;
298}
299
300bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
301 const Register DefReg = I.getOperand(i: 0).getReg();
302 const LLT DefTy = MRI->getType(Reg: DefReg);
303
304 // S1 G_PHIs should not be selected in instruction-select, instead:
305 // - divergent S1 G_PHI should go through lane mask merging algorithm
306 // and be fully inst-selected in AMDGPUGlobalISelDivergenceLowering
307 // - uniform S1 G_PHI should be lowered into S32 G_PHI in AMDGPURegBankSelect
308 if (DefTy == LLT::scalar(SizeInBits: 1))
309 return false;
310
311 // TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy)
312
313 const RegClassOrRegBank &RegClassOrBank =
314 MRI->getRegClassOrRegBank(Reg: DefReg);
315
316 const TargetRegisterClass *DefRC =
317 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
318 if (!DefRC) {
319 if (!DefTy.isValid()) {
320 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
321 return false;
322 }
323
324 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
325 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, Bank: RB);
326 if (!DefRC) {
327 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
328 return false;
329 }
330 }
331
332 // If inputs have register bank, assign corresponding reg class.
333 // Note: registers don't need to have the same reg bank.
334 for (unsigned i = 1; i != I.getNumOperands(); i += 2) {
335 const Register SrcReg = I.getOperand(i).getReg();
336
337 const RegisterBank *RB = MRI->getRegBankOrNull(Reg: SrcReg);
338 if (RB) {
339 const LLT SrcTy = MRI->getType(Reg: SrcReg);
340 const TargetRegisterClass *SrcRC =
341 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *RB);
342 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
343 return false;
344 }
345 }
346
347 I.setDesc(TII.get(Opcode: TargetOpcode::PHI));
348 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
349}
350
351MachineOperand
352AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
353 const TargetRegisterClass &SubRC,
354 unsigned SubIdx) const {
355
356 MachineInstr *MI = MO.getParent();
357 MachineBasicBlock *BB = MO.getParent()->getParent();
358 Register DstReg = MRI->createVirtualRegister(RegClass: &SubRC);
359
360 if (MO.isReg()) {
361 unsigned ComposedSubIdx = TRI.composeSubRegIndices(a: MO.getSubReg(), b: SubIdx);
362 Register Reg = MO.getReg();
363 BuildMI(BB&: *BB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
364 .addReg(RegNo: Reg, Flags: {}, SubReg: ComposedSubIdx);
365
366 return MachineOperand::CreateReg(Reg: DstReg, isDef: MO.isDef(), isImp: MO.isImplicit(),
367 isKill: MO.isKill(), isDead: MO.isDead(), isUndef: MO.isUndef(),
368 isEarlyClobber: MO.isEarlyClobber(), SubReg: 0, isDebug: MO.isDebug(),
369 isInternalRead: MO.isInternalRead());
370 }
371
372 assert(MO.isImm());
373
374 APInt Imm(64, MO.getImm());
375
376 switch (SubIdx) {
377 default:
378 llvm_unreachable("do not know to split immediate with this sub index.");
379 case AMDGPU::sub0:
380 return MachineOperand::CreateImm(Val: Imm.getLoBits(numBits: 32).getSExtValue());
381 case AMDGPU::sub1:
382 return MachineOperand::CreateImm(Val: Imm.getHiBits(numBits: 32).getSExtValue());
383 }
384}
385
386static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
387 switch (Opc) {
388 case AMDGPU::G_AND:
389 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
390 case AMDGPU::G_OR:
391 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32;
392 case AMDGPU::G_XOR:
393 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32;
394 default:
395 llvm_unreachable("not a bit op");
396 }
397}
398
399bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
400 Register DstReg = I.getOperand(i: 0).getReg();
401 uint64_t Size = TRI.getRegSizeInBits(Reg: DstReg, MRI: *MRI);
402
403 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
404 if (DstRB->getID() != AMDGPU::SGPRRegBankID &&
405 DstRB->getID() != AMDGPU::VCCRegBankID)
406 return false;
407
408 bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID &&
409 STI.isWave64());
410 I.setDesc(TII.get(Opcode: getLogicalBitOpcode(Opc: I.getOpcode(), Is64)));
411
412 // Dead implicit-def of scc
413 I.addOperand(Op: MachineOperand::CreateReg(Reg: AMDGPU::SCC, isDef: true, // isDef
414 isImp: true, // isImp
415 isKill: false, // isKill
416 isDead: true)); // isDead
417 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
418 return true;
419}
420
421bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
422 MachineBasicBlock *BB = I.getParent();
423 MachineFunction *MF = BB->getParent();
424 Register DstReg = I.getOperand(i: 0).getReg();
425 const DebugLoc &DL = I.getDebugLoc();
426 LLT Ty = MRI->getType(Reg: DstReg);
427 if (Ty.isVector())
428 return false;
429
430 uint64_t Size = Ty.getSizeInBits();
431 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
432 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID;
433 const bool Sub = I.getOpcode() == TargetOpcode::G_SUB;
434
435 if (Size == 32) {
436 if (IsSALU) {
437 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
438 MachineInstr *Add =
439 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
440 .add(MO: I.getOperand(i: 1))
441 .add(MO: I.getOperand(i: 2))
442 .setOperandDead(3); // Dead scc
443 I.eraseFromParent();
444 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
445 return true;
446 }
447
448 if (STI.hasAddNoCarryInsts()) {
449 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64;
450 I.setDesc(TII.get(Opcode: Opc));
451 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
452 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
453 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
454 return true;
455 }
456
457 const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64;
458
459 Register UnusedCarry = MRI->createVirtualRegister(RegClass: TRI.getWaveMaskRegClass());
460 MachineInstr *Add
461 = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
462 .addDef(RegNo: UnusedCarry, Flags: RegState::Dead)
463 .add(MO: I.getOperand(i: 1))
464 .add(MO: I.getOperand(i: 2))
465 .addImm(Val: 0);
466 I.eraseFromParent();
467 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
468 return true;
469 }
470
471 assert(!Sub && "illegal sub should not reach here");
472
473 const TargetRegisterClass &RC
474 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass;
475 const TargetRegisterClass &HalfRC
476 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass;
477
478 MachineOperand Lo1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
479 MachineOperand Lo2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
480 MachineOperand Hi1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
481 MachineOperand Hi2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
482
483 Register DstLo = MRI->createVirtualRegister(RegClass: &HalfRC);
484 Register DstHi = MRI->createVirtualRegister(RegClass: &HalfRC);
485
486 if (IsSALU) {
487 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_U32), DestReg: DstLo)
488 .add(MO: Lo1)
489 .add(MO: Lo2);
490 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADDC_U32), DestReg: DstHi)
491 .add(MO: Hi1)
492 .add(MO: Hi2)
493 .setOperandDead(3); // Dead scc
494 } else {
495 const TargetRegisterClass *CarryRC = TRI.getWaveMaskRegClass();
496 Register CarryReg = MRI->createVirtualRegister(RegClass: CarryRC);
497 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: DstLo)
498 .addDef(RegNo: CarryReg)
499 .add(MO: Lo1)
500 .add(MO: Lo2)
501 .addImm(Val: 0);
502 MachineInstr *Addc =
503 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: DstHi)
504 .addDef(RegNo: MRI->createVirtualRegister(RegClass: CarryRC), Flags: RegState::Dead)
505 .add(MO: Hi1)
506 .add(MO: Hi2)
507 .addReg(RegNo: CarryReg)
508 .addImm(Val: 0);
509
510 constrainSelectedInstRegOperands(I&: *Addc, TII, TRI, RBI);
511 }
512
513 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
514 .addReg(RegNo: DstLo)
515 .addImm(Val: AMDGPU::sub0)
516 .addReg(RegNo: DstHi)
517 .addImm(Val: AMDGPU::sub1);
518
519
520 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
521 return false;
522
523 I.eraseFromParent();
524 return true;
525}
526
527bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE(
528 MachineInstr &I) const {
529 MachineBasicBlock *BB = I.getParent();
530 MachineFunction *MF = BB->getParent();
531 const DebugLoc &DL = I.getDebugLoc();
532 Register Dst0Reg = I.getOperand(i: 0).getReg();
533 Register Dst1Reg = I.getOperand(i: 1).getReg();
534 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO ||
535 I.getOpcode() == AMDGPU::G_UADDE;
536 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE ||
537 I.getOpcode() == AMDGPU::G_USUBE;
538
539 if (isVCC(Reg: Dst1Reg, MRI: *MRI)) {
540 unsigned NoCarryOpc =
541 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
542 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
543 I.setDesc(TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc));
544 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
545 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
546 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
547 return true;
548 }
549
550 Register Src0Reg = I.getOperand(i: 2).getReg();
551 Register Src1Reg = I.getOperand(i: 3).getReg();
552
553 if (HasCarryIn) {
554 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
555 .addReg(RegNo: I.getOperand(i: 4).getReg());
556 }
557
558 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
559 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
560
561 auto CarryInst = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc), DestReg: Dst0Reg)
562 .add(MO: I.getOperand(i: 2))
563 .add(MO: I.getOperand(i: 3));
564
565 if (MRI->use_nodbg_empty(RegNo: Dst1Reg)) {
566 CarryInst.setOperandDead(3); // Dead scc
567 } else {
568 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst1Reg)
569 .addReg(RegNo: AMDGPU::SCC);
570 if (!MRI->getRegClassOrNull(Reg: Dst1Reg))
571 MRI->setRegClass(Reg: Dst1Reg, RC: &AMDGPU::SReg_32RegClass);
572 }
573
574 if (!RBI.constrainGenericRegister(Reg: Dst0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
575 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
576 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
577 return false;
578
579 if (HasCarryIn &&
580 !RBI.constrainGenericRegister(Reg: I.getOperand(i: 4).getReg(),
581 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
582 return false;
583
584 I.eraseFromParent();
585 return true;
586}
587
588bool AMDGPUInstructionSelector::selectG_AMDGPU_MAD_64_32(
589 MachineInstr &I) const {
590 MachineBasicBlock *BB = I.getParent();
591 MachineFunction *MF = BB->getParent();
592 const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
593 bool UseNoCarry = Subtarget->hasMadNC64_32Insts() &&
594 MRI->use_nodbg_empty(RegNo: I.getOperand(i: 1).getReg());
595
596 unsigned Opc;
597 if (Subtarget->hasMADIntraFwdBug())
598 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64
599 : AMDGPU::V_MAD_I64_I32_gfx11_e64;
600 else if (UseNoCarry)
601 Opc = IsUnsigned ? AMDGPU::V_MAD_NC_U64_U32_e64
602 : AMDGPU::V_MAD_NC_I64_I32_e64;
603 else
604 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64;
605
606 if (UseNoCarry)
607 I.removeOperand(OpNo: 1);
608
609 I.setDesc(TII.get(Opcode: Opc));
610 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
611 I.addImplicitDefUseOperands(MF&: *MF);
612 I.getOperand(i: 0).setIsEarlyClobber(true);
613 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
614 return true;
615}
616
617// TODO: We should probably legalize these to only using 32-bit results.
618bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
619 MachineBasicBlock *BB = I.getParent();
620 Register DstReg = I.getOperand(i: 0).getReg();
621 Register SrcReg = I.getOperand(i: 1).getReg();
622 LLT DstTy = MRI->getType(Reg: DstReg);
623 LLT SrcTy = MRI->getType(Reg: SrcReg);
624 const uint64_t SrcSize = SrcTy.getSizeInBits();
625 uint64_t DstSize = DstTy.getSizeInBits();
626
627 // TODO: Should handle any multiple of 32 offset.
628 int64_t Offset = I.getOperand(i: 2).getImm();
629 if (Offset % 32 != 0 || DstSize > 128)
630 return false;
631
632 // 16-bit operations really use 32-bit registers.
633 // FIXME: Probably should not allow 16-bit G_EXTRACT results.
634 if (DstSize == 16)
635 DstSize = 32;
636
637 const TargetRegisterClass *DstRC =
638 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
639 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
640 return false;
641
642 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
643 const TargetRegisterClass *SrcRC =
644 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
645 if (!SrcRC)
646 return false;
647 unsigned SubReg = SIRegisterInfo::getSubRegFromChannel(Channel: Offset / 32,
648 NumRegs: DstSize / 32);
649 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubReg);
650 if (!SrcRC)
651 return false;
652
653 SrcReg = constrainOperandRegClass(MF: *MF, TRI, MRI&: *MRI, TII, RBI, InsertPt&: I,
654 RegClass: *SrcRC, RegMO&: I.getOperand(i: 1));
655 const DebugLoc &DL = I.getDebugLoc();
656 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
657 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
658
659 I.eraseFromParent();
660 return true;
661}
662
663bool AMDGPUInstructionSelector::selectS16MergeToS32(MachineInstr &MI) const {
664 Register Dst = MI.getOperand(i: 0).getReg();
665 Register Src0 = MI.getOperand(i: 1).getReg();
666 Register Src1 = MI.getOperand(i: 2).getReg();
667
668 LLT Src0Ty = MRI->getType(Reg: Src0);
669 LLT Src1Ty = MRI->getType(Reg: Src1);
670
671 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
672 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0, MRI: *MRI, TRI);
673 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1, MRI: *MRI, TRI);
674 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
675
676 Register ShiftSrc0;
677 Register ShiftSrc1;
678
679 const DebugLoc &DL = MI.getDebugLoc();
680 MachineBasicBlock *BB = MI.getParent();
681
682 // VGPR case
683 if (IsVector) {
684 // If source are both VGPR16, use REG_SEQUENCE with lo16/hi16 subregisters
685 if (Src0Bank->getID() == AMDGPU::VGPRRegBankID &&
686 Src1Bank->getID() == AMDGPU::VGPRRegBankID &&
687 Src0Ty == LLT::scalar(SizeInBits: 16) && Src1Ty == LLT::scalar(SizeInBits: 16)) {
688 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
689 .addReg(RegNo: Src0)
690 .addImm(Val: AMDGPU::lo16)
691 .addReg(RegNo: Src1)
692 .addImm(Val: AMDGPU::hi16);
693
694 if (!RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
695 return false;
696
697 MI.eraseFromParent();
698 return true;
699 }
700
701 // Otherwise, use V_LSHL_OR_B32_e64
702 Register TmpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
703 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: TmpReg)
704 .addImm(Val: 0xFFFF)
705 .addReg(RegNo: Src0);
706 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
707
708 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: Dst)
709 .addReg(RegNo: Src1)
710 .addImm(Val: 16)
711 .addReg(RegNo: TmpReg);
712 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
713
714 MI.eraseFromParent();
715 return true;
716 }
717
718 // SGPR case -> S_PACK_*_B32_B16
719 // With multiple uses of the shift, this will duplicate the shift and
720 // increase register pressure.
721 //
722 // (merge (lshr_oneuse $src0, 16), (lshr_oneuse $src1, 16)
723 // => (S_PACK_HH_B32_B16 $src0, $src1)
724 // (merge (lshr_oneuse SReg_32:$src0, 16), $src1)
725 // => (S_PACK_HL_B32_B16 $src0, $src1)
726 // (merge $src0, (lshr_oneuse SReg_32:$src1, 16))
727 // => (S_PACK_LH_B32_B16 $src0, $src1)
728 // (merge $src0, $src1)
729 // => (S_PACK_LL_B32_B16 $src0, $src1)
730
731 bool Shift0 = mi_match(
732 R: Src0, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc0), R: m_SpecificICst(RequestedValue: 16))));
733
734 bool Shift1 = mi_match(
735 R: Src1, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc1), R: m_SpecificICst(RequestedValue: 16))));
736
737 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16;
738 if (Shift0 && Shift1) {
739 Opc = AMDGPU::S_PACK_HH_B32_B16;
740 MI.getOperand(i: 1).setReg(ShiftSrc0);
741 MI.getOperand(i: 2).setReg(ShiftSrc1);
742 } else if (Shift1) {
743 Opc = AMDGPU::S_PACK_LH_B32_B16;
744 MI.getOperand(i: 2).setReg(ShiftSrc1);
745 } else if (Shift0) {
746 auto ConstSrc1 =
747 getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
748 if (ConstSrc1 && ConstSrc1->Value == 0) {
749 // build_vector_trunc (lshr $src0, 16), 0 -> s_lshr_b32 $src0, 16
750 auto MIB = BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: Dst)
751 .addReg(RegNo: ShiftSrc0)
752 .addImm(Val: 16)
753 .setOperandDead(3); // Dead scc
754
755 MI.eraseFromParent();
756 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
757 return true;
758 }
759 if (STI.hasSPackHL()) {
760 Opc = AMDGPU::S_PACK_HL_B32_B16;
761 MI.getOperand(i: 1).setReg(ShiftSrc0);
762 }
763 }
764
765 MI.setDesc(TII.get(Opcode: Opc));
766 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
767 return true;
768}
769
770// Pack each pair of s16 into an s32 with S_PACK_LL_B32_B16, then combine the
771// s32 pieces into the destination with a REG_SEQUENCE.
772bool AMDGPUInstructionSelector::selectS16MergeToWide(MachineInstr &MI) const {
773 MachineBasicBlock *BB = MI.getParent();
774 const DebugLoc &DL = MI.getDebugLoc();
775 Register DstReg = MI.getOperand(i: 0).getReg();
776 const uint64_t DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
777 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
778 const unsigned NumSrc = MI.getNumOperands() - 1;
779
780 // Pack each pair of s16 sources into an s32.
781 SmallVector<Register, 8> S32Regs;
782 for (unsigned I = 0; I != NumSrc; I += 2) {
783 Register S32 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
784 auto Pack = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_PACK_LL_B32_B16), DestReg: S32)
785 .addReg(RegNo: MI.getOperand(i: I + 1).getReg())
786 .addReg(RegNo: MI.getOperand(i: I + 2).getReg());
787 constrainSelectedInstRegOperands(I&: *Pack, TII, TRI, RBI);
788 S32Regs.push_back(Elt: S32);
789 }
790
791 // Combine the s32 pieces into the destination with a REG_SEQUENCE.
792 const TargetRegisterClass *DstRC =
793 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
794 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
795 return false;
796 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, /*EltSize=*/4);
797 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
798 for (size_t I = 0, E = S32Regs.size(); I != E; ++I)
799 MIB.addReg(RegNo: S32Regs[I]).addImm(Val: SubRegs[I]);
800
801 MI.eraseFromParent();
802 return true;
803}
804
805bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
806 MachineBasicBlock *BB = MI.getParent();
807 Register DstReg = MI.getOperand(i: 0).getReg();
808 LLT DstTy = MRI->getType(Reg: DstReg);
809 LLT SrcTy = MRI->getType(Reg: MI.getOperand(i: 1).getReg());
810
811 const uint64_t SrcSize = SrcTy.getSizeInBits();
812 if (SrcSize < 32) {
813 // Handle s32 <- G_MERGE_VALUES s16, s16
814 if (SrcSize == 16 && DstTy.getSizeInBits() == 32 &&
815 MI.getNumOperands() == 3) {
816 return selectS16MergeToS32(MI);
817 }
818 // With true16 a scalar s16 is a register type, so a scalar wider than 32
819 // bits can be built from s16 pieces.
820 bool IsWideS16Merge = SrcSize == 16 && DstTy.getSizeInBits() > 32 &&
821 DstTy.getSizeInBits() % 32 == 0;
822
823 // SGPRs have no 16-bit subregisters, so pack pairs of s16 with S_PACK.
824 if (IsWideS16Merge &&
825 RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
826 return selectS16MergeToWide(MI);
827
828 // A VGPR wide s16 merge falls through to the generic path below.
829 if (!IsWideS16Merge)
830 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
831 }
832
833 const DebugLoc &DL = MI.getDebugLoc();
834 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
835 const uint64_t DstSize = DstTy.getSizeInBits();
836 const TargetRegisterClass *DstRC =
837 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
838 if (!DstRC)
839 return false;
840
841 unsigned SrcSizeEnc = static_cast<unsigned>(SrcSize);
842 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, EltSize: SrcSizeEnc / 8);
843 MachineInstrBuilder MIB =
844 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
845 for (int I = 0, E = MI.getNumOperands() - 1; I != E; ++I) {
846 MachineOperand &Src = MI.getOperand(i: I + 1);
847 Register SrcReg = Src.getReg();
848 MIB.addReg(RegNo: SrcReg, Flags: getUndefRegState(B: Src.isUndef()));
849 MIB.addImm(Val: SubRegs[I]);
850
851 const TargetRegisterClass *SrcRC =
852 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
853 if (SrcRC && !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
854 return false;
855 }
856
857 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
858 return false;
859
860 MI.eraseFromParent();
861 return true;
862}
863
864bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
865 MachineBasicBlock *BB = MI.getParent();
866 const int NumDst = MI.getNumOperands() - 1;
867
868 MachineOperand &Src = MI.getOperand(i: NumDst);
869
870 Register SrcReg = Src.getReg();
871 Register DstReg0 = MI.getOperand(i: 0).getReg();
872 LLT DstTy = MRI->getType(Reg: DstReg0);
873 LLT SrcTy = MRI->getType(Reg: SrcReg);
874
875 const uint64_t DstSize = DstTy.getSizeInBits();
876 const uint64_t SrcSize = SrcTy.getSizeInBits();
877 const DebugLoc &DL = MI.getDebugLoc();
878 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
879
880 const TargetRegisterClass *SrcRC =
881 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
882
883 // Note we could have mixed SGPR and VGPR destination banks for an SGPR
884 // source, and this relies on the fact that the same subregister indices are
885 // used for both.
886 unsigned DstSizeEnc = static_cast<unsigned>(DstSize);
887 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SrcRC, EltSize: DstSizeEnc / 8);
888 for (int I = 0, E = NumDst; I != E; ++I) {
889 Register DstReg = MI.getOperand(i: I).getReg();
890 // hi16:sreg_32 is not allowed so explicitly shift upper 16-bits.
891 if (SrcBank->getID() == AMDGPU::SGPRRegBankID &&
892 SubRegs[I] == AMDGPU::hi16) {
893 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: DstReg)
894 .addReg(RegNo: SrcReg)
895 .addImm(Val: 16);
896 } else {
897 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
898 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegs[I]);
899
900 // Make sure the subregister index is valid for the source register.
901 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[I]);
902 }
903
904 if (!SrcRC || !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
905 return false;
906
907 const TargetRegisterClass *DstRC =
908 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
909 if (DstRC && !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
910 return false;
911 }
912
913 MI.eraseFromParent();
914 return true;
915}
916
917bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR(MachineInstr &MI) const {
918 assert(MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC ||
919 MI.getOpcode() == AMDGPU::G_BUILD_VECTOR);
920
921 Register Src0 = MI.getOperand(i: 1).getReg();
922 Register Src1 = MI.getOperand(i: 2).getReg();
923 LLT SrcTy = MRI->getType(Reg: Src0);
924 const uint64_t SrcSize = SrcTy.getSizeInBits();
925
926 // BUILD_VECTOR with >=32 bits source is handled by MERGE_VALUE.
927 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR && SrcSize >= 32) {
928 return selectG_MERGE_VALUES(MI);
929 }
930
931 // Selection logic below is for V2S16 only.
932 // For G_BUILD_VECTOR_TRUNC, additionally check that the operands are s32.
933 Register Dst = MI.getOperand(i: 0).getReg();
934 if (MRI->getType(Reg: Dst) != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
935 (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC &&
936 SrcTy != LLT::scalar(SizeInBits: 32)))
937 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
938
939 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
940 if (DstBank->getID() == AMDGPU::AGPRRegBankID)
941 return false;
942
943 assert(DstBank->getID() == AMDGPU::SGPRRegBankID ||
944 DstBank->getID() == AMDGPU::VGPRRegBankID);
945 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
946
947 const DebugLoc &DL = MI.getDebugLoc();
948 MachineBasicBlock *BB = MI.getParent();
949
950 // First, before trying TableGen patterns, check if both sources are
951 // constants. In those cases, we can trivially compute the final constant
952 // and emit a simple move.
953 auto ConstSrc1 = getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
954 if (ConstSrc1) {
955 auto ConstSrc0 =
956 getAnyConstantVRegValWithLookThrough(VReg: Src0, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
957 if (ConstSrc0) {
958 const int64_t K0 = ConstSrc0->Value.getSExtValue();
959 const int64_t K1 = ConstSrc1->Value.getSExtValue();
960 uint32_t Lo16 = static_cast<uint32_t>(K0) & 0xffff;
961 uint32_t Hi16 = static_cast<uint32_t>(K1) & 0xffff;
962 uint32_t Imm = Lo16 | (Hi16 << 16);
963
964 // VALU
965 if (IsVector) {
966 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: Dst).addImm(Val: Imm);
967 MI.eraseFromParent();
968 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI);
969 }
970
971 // SALU
972 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: Dst).addImm(Val: Imm);
973 MI.eraseFromParent();
974 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
975 }
976 }
977
978 // Now try TableGen patterns.
979 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
980 return true;
981
982 // TODO: This should probably be a combine somewhere
983 // (build_vector $src0, undef) -> copy $src0
984 MachineInstr *Src1Def = getDefIgnoringCopies(Reg: Src1, MRI: *MRI);
985 if (Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) {
986 MI.setDesc(TII.get(Opcode: AMDGPU::COPY));
987 MI.removeOperand(OpNo: 2);
988 const auto &RC =
989 IsVector ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
990 return RBI.constrainGenericRegister(Reg: Dst, RC, MRI&: *MRI) &&
991 RBI.constrainGenericRegister(Reg: Src0, RC, MRI&: *MRI);
992 }
993
994 return selectS16MergeToS32(MI);
995}
996
997bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const {
998 const MachineOperand &MO = I.getOperand(i: 0);
999
1000 // FIXME: Interface for getConstrainedRegClassForReg needs work. The
1001 // regbank check here is to know why getConstrainedRegClassForReg failed.
1002 const TargetRegisterClass *RC =
1003 TRI.getConstrainedRegClassForReg(Reg: MO.getReg(), MRI: *MRI);
1004 if ((!RC && !MRI->getRegBankOrNull(Reg: MO.getReg())) ||
1005 (RC && RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI))) {
1006 I.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1007 return true;
1008 }
1009
1010 return false;
1011}
1012
1013bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
1014 MachineBasicBlock *BB = I.getParent();
1015
1016 Register DstReg = I.getOperand(i: 0).getReg();
1017 Register Src0Reg = I.getOperand(i: 1).getReg();
1018 Register Src1Reg = I.getOperand(i: 2).getReg();
1019 LLT Src1Ty = MRI->getType(Reg: Src1Reg);
1020
1021 uint64_t DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
1022 uint64_t InsSize = Src1Ty.getSizeInBits();
1023
1024 int64_t Offset = I.getOperand(i: 3).getImm();
1025
1026 // FIXME: These cases should have been illegal and unnecessary to check here.
1027 if (Offset % 32 != 0 || InsSize % 32 != 0)
1028 return false;
1029
1030 // Currently not handled by getSubRegFromChannel.
1031 if (InsSize > 128)
1032 return false;
1033
1034 unsigned SubReg = TRI.getSubRegFromChannel(Channel: Offset / 32, NumRegs: InsSize / 32);
1035 if (SubReg == AMDGPU::NoSubRegister)
1036 return false;
1037
1038 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1039 const TargetRegisterClass *DstRC =
1040 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
1041 if (!DstRC)
1042 return false;
1043
1044 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0Reg, MRI: *MRI, TRI);
1045 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1Reg, MRI: *MRI, TRI);
1046 const TargetRegisterClass *Src0RC =
1047 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *Src0Bank);
1048 const TargetRegisterClass *Src1RC =
1049 TRI.getRegClassForSizeOnBank(Size: InsSize, Bank: *Src1Bank);
1050
1051 // Deal with weird cases where the class only partially supports the subreg
1052 // index.
1053 Src0RC = TRI.getSubClassWithSubReg(Src0RC, SubReg);
1054 if (!Src0RC || !Src1RC)
1055 return false;
1056
1057 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
1058 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: *Src0RC, MRI&: *MRI) ||
1059 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: *Src1RC, MRI&: *MRI))
1060 return false;
1061
1062 const DebugLoc &DL = I.getDebugLoc();
1063 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1064 .addReg(RegNo: Src0Reg)
1065 .addReg(RegNo: Src1Reg)
1066 .addImm(Val: SubReg);
1067
1068 I.eraseFromParent();
1069 return true;
1070}
1071
1072bool AMDGPUInstructionSelector::selectG_SBFX_UBFX(MachineInstr &MI) const {
1073 Register DstReg = MI.getOperand(i: 0).getReg();
1074 Register SrcReg = MI.getOperand(i: 1).getReg();
1075 Register OffsetReg = MI.getOperand(i: 2).getReg();
1076 Register WidthReg = MI.getOperand(i: 3).getReg();
1077
1078 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID &&
1079 "scalar BFX instructions are expanded in regbankselect");
1080 assert(MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 &&
1081 "64-bit vector BFX instructions are expanded in regbankselect");
1082
1083 const DebugLoc &DL = MI.getDebugLoc();
1084 MachineBasicBlock *MBB = MI.getParent();
1085
1086 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SBFX;
1087 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
1088 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
1089 .addReg(RegNo: SrcReg)
1090 .addReg(RegNo: OffsetReg)
1091 .addReg(RegNo: WidthReg);
1092 MI.eraseFromParent();
1093 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1094 return true;
1095}
1096
1097bool AMDGPUInstructionSelector::selectInterpP1F16(MachineInstr &MI) const {
1098 if (STI.getLDSBankCount() != 16)
1099 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1100
1101 Register Dst = MI.getOperand(i: 0).getReg();
1102 Register Src0 = MI.getOperand(i: 2).getReg();
1103 Register M0Val = MI.getOperand(i: 6).getReg();
1104 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
1105 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI) ||
1106 !RBI.constrainGenericRegister(Reg: Src0, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
1107 return false;
1108
1109 // This requires 2 instructions. It is possible to write a pattern to support
1110 // this, but the generated isel emitter doesn't correctly deal with multiple
1111 // output instructions using the same physical register input. The copy to m0
1112 // is incorrectly placed before the second instruction.
1113 //
1114 // TODO: Match source modifiers.
1115
1116 Register InterpMov = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1117 const DebugLoc &DL = MI.getDebugLoc();
1118 MachineBasicBlock *MBB = MI.getParent();
1119
1120 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1121 .addReg(RegNo: M0Val);
1122 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_MOV_F32), DestReg: InterpMov)
1123 .addImm(Val: 2)
1124 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1125 .addImm(Val: MI.getOperand(i: 3).getImm()); // $attrchan
1126
1127 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_P1LV_F16), DestReg: Dst)
1128 .addImm(Val: 0) // $src0_modifiers
1129 .addReg(RegNo: Src0) // $src0
1130 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1131 .addImm(Val: MI.getOperand(i: 3).getImm()) // $attrchan
1132 .addImm(Val: 0) // $src2_modifiers
1133 .addReg(RegNo: InterpMov) // $src2 - 2 f16 values selected by high
1134 .addImm(Val: MI.getOperand(i: 5).getImm()) // $high
1135 .addImm(Val: 0) // $clamp
1136 .addImm(Val: 0); // $omod
1137
1138 MI.eraseFromParent();
1139 return true;
1140}
1141
1142// Writelane is special in that it can use SGPR and M0 (which would normally
1143// count as using the constant bus twice - but in this case it is allowed since
1144// the lane selector doesn't count as a use of the constant bus). However, it is
1145// still required to abide by the 1 SGPR rule. Fix this up if we might have
1146// multiple SGPRs.
1147bool AMDGPUInstructionSelector::selectWritelane(MachineInstr &MI) const {
1148 // With a constant bus limit of at least 2, there's no issue.
1149 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_WRITELANE_B32) > 1)
1150 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1151
1152 MachineBasicBlock *MBB = MI.getParent();
1153 const DebugLoc &DL = MI.getDebugLoc();
1154 Register VDst = MI.getOperand(i: 0).getReg();
1155 Register Val = MI.getOperand(i: 2).getReg();
1156 Register LaneSelect = MI.getOperand(i: 3).getReg();
1157 Register VDstIn = MI.getOperand(i: 4).getReg();
1158
1159 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_WRITELANE_B32), DestReg: VDst);
1160
1161 std::optional<ValueAndVReg> ConstSelect =
1162 getIConstantVRegValWithLookThrough(VReg: LaneSelect, MRI: *MRI);
1163 if (ConstSelect) {
1164 // The selector has to be an inline immediate, so we can use whatever for
1165 // the other operands.
1166 MIB.addReg(RegNo: Val);
1167 MIB.addImm(Val: ConstSelect->Value.getSExtValue() &
1168 maskTrailingOnes<uint64_t>(N: STI.getWavefrontSizeLog2()));
1169 } else {
1170 std::optional<ValueAndVReg> ConstVal =
1171 getIConstantVRegValWithLookThrough(VReg: Val, MRI: *MRI);
1172
1173 // If the value written is an inline immediate, we can get away without a
1174 // copy to m0.
1175 if (ConstVal && AMDGPU::isInlinableLiteral32(
1176 Literal: static_cast<int32_t>(ConstVal->Value.getSExtValue()),
1177 HasInv2Pi: STI.hasInv2PiInlineImm())) {
1178 MIB.addImm(Val: ConstVal->Value.getSExtValue());
1179 MIB.addReg(RegNo: LaneSelect);
1180 } else {
1181 MIB.addReg(RegNo: Val);
1182
1183 // If the lane selector was originally in a VGPR and copied with
1184 // readfirstlane, there's a hazard to read the same SGPR from the
1185 // VALU. Constrain to a different SGPR to help avoid needing a nop later.
1186 RBI.constrainGenericRegister(Reg: LaneSelect, RC: AMDGPU::SReg_32_XM0RegClass, MRI&: *MRI);
1187
1188 BuildMI(BB&: *MBB, I&: *MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1189 .addReg(RegNo: LaneSelect);
1190 MIB.addReg(RegNo: AMDGPU::M0);
1191 }
1192 }
1193
1194 MIB.addReg(RegNo: VDstIn);
1195
1196 MI.eraseFromParent();
1197 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1198 return true;
1199}
1200
1201// We need to handle this here because tablegen doesn't support matching
1202// instructions with multiple outputs.
1203bool AMDGPUInstructionSelector::selectDivScale(MachineInstr &MI) const {
1204 Register Dst0 = MI.getOperand(i: 0).getReg();
1205 Register Dst1 = MI.getOperand(i: 1).getReg();
1206
1207 LLT Ty = MRI->getType(Reg: Dst0);
1208 unsigned Opc;
1209 if (Ty == LLT::scalar(SizeInBits: 32))
1210 Opc = AMDGPU::V_DIV_SCALE_F32_e64;
1211 else if (Ty == LLT::scalar(SizeInBits: 64))
1212 Opc = AMDGPU::V_DIV_SCALE_F64_e64;
1213 else
1214 return false;
1215
1216 // TODO: Match source modifiers.
1217
1218 const DebugLoc &DL = MI.getDebugLoc();
1219 MachineBasicBlock *MBB = MI.getParent();
1220
1221 Register Numer = MI.getOperand(i: 3).getReg();
1222 Register Denom = MI.getOperand(i: 4).getReg();
1223 unsigned ChooseDenom = static_cast<unsigned>(MI.getOperand(i: 5).getImm());
1224
1225 Register Src0 = ChooseDenom != 0 ? Numer : Denom;
1226
1227 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
1228 .addDef(RegNo: Dst1)
1229 .addImm(Val: 0) // $src0_modifiers
1230 .addUse(RegNo: Src0) // $src0
1231 .addImm(Val: 0) // $src1_modifiers
1232 .addUse(RegNo: Denom) // $src1
1233 .addImm(Val: 0) // $src2_modifiers
1234 .addUse(RegNo: Numer) // $src2
1235 .addImm(Val: 0) // $clamp
1236 .addImm(Val: 0); // $omod
1237
1238 MI.eraseFromParent();
1239 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1240 return true;
1241}
1242
1243bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
1244 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
1245 switch (IntrinsicID) {
1246 case Intrinsic::amdgcn_if_break: {
1247 MachineBasicBlock *BB = I.getParent();
1248
1249 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1250 // SelectionDAG uses for wave32 vs wave64.
1251 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_IF_BREAK))
1252 .add(MO: I.getOperand(i: 0))
1253 .add(MO: I.getOperand(i: 2))
1254 .add(MO: I.getOperand(i: 3))
1255 .setOperandDead(3); // implicit-def $scc
1256
1257 Register DstReg = I.getOperand(i: 0).getReg();
1258 Register Src0Reg = I.getOperand(i: 2).getReg();
1259 Register Src1Reg = I.getOperand(i: 3).getReg();
1260
1261 I.eraseFromParent();
1262
1263 for (Register Reg : { DstReg, Src0Reg, Src1Reg })
1264 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1265
1266 return true;
1267 }
1268 case Intrinsic::amdgcn_interp_p1_f16:
1269 return selectInterpP1F16(MI&: I);
1270 case Intrinsic::amdgcn_wqm:
1271 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::WQM);
1272 case Intrinsic::amdgcn_softwqm:
1273 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::SOFT_WQM);
1274 case Intrinsic::amdgcn_strict_wwm:
1275 case Intrinsic::amdgcn_wwm:
1276 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WWM);
1277 case Intrinsic::amdgcn_strict_wqm:
1278 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WQM);
1279 case Intrinsic::amdgcn_writelane:
1280 return selectWritelane(MI&: I);
1281 case Intrinsic::amdgcn_div_scale:
1282 return selectDivScale(MI&: I);
1283 case Intrinsic::amdgcn_ballot:
1284 return selectBallot(I);
1285 case Intrinsic::amdgcn_reloc_constant:
1286 return selectRelocConstant(I);
1287 case Intrinsic::amdgcn_groupstaticsize:
1288 return selectGroupStaticSize(I);
1289 case Intrinsic::returnaddress:
1290 return selectReturnAddress(I);
1291 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
1292 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
1293 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
1294 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
1295 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
1296 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
1297 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
1298 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
1299 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
1300 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
1301 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
1302 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
1303 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
1304 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
1305 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
1306 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
1307 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
1308 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
1309 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
1310 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
1311 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
1312 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
1313 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
1314 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
1315 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
1316 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
1317 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
1318 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
1319 return selectSMFMACIntrin(I);
1320 case Intrinsic::amdgcn_permlane16_swap:
1321 case Intrinsic::amdgcn_permlane32_swap:
1322 return selectPermlaneSwapIntrin(I, IntrID: IntrinsicID);
1323 case Intrinsic::amdgcn_wave_shuffle:
1324 return selectWaveShuffleIntrin(I);
1325 default:
1326 return selectImpl(I, CoverageInfo&: *CoverageInfo);
1327 }
1328}
1329
1330static int getV_CMPOpcode(CmpInst::Predicate P, uint64_t Size,
1331 const GCNSubtarget &ST) {
1332 if (Size != 16 && Size != 32 && Size != 64)
1333 return -1;
1334
1335 if (Size == 16 && !ST.has16BitInsts())
1336 return -1;
1337
1338 const auto Select = [&](unsigned S16Opc, unsigned TrueS16Opc,
1339 unsigned FakeS16Opc, unsigned S32Opc,
1340 unsigned S64Opc) {
1341 if (Size == 16)
1342 return ST.hasTrue16BitInsts()
1343 ? ST.useRealTrue16Insts() ? TrueS16Opc : FakeS16Opc
1344 : S16Opc;
1345 if (Size == 32)
1346 return S32Opc;
1347 return S64Opc;
1348 };
1349
1350 switch (P) {
1351 default:
1352 llvm_unreachable("Unknown condition code!");
1353 case CmpInst::ICMP_NE:
1354 return Select(AMDGPU::V_CMP_NE_U16_e64, AMDGPU::V_CMP_NE_U16_t16_e64,
1355 AMDGPU::V_CMP_NE_U16_fake16_e64, AMDGPU::V_CMP_NE_U32_e64,
1356 AMDGPU::V_CMP_NE_U64_e64);
1357 case CmpInst::ICMP_EQ:
1358 return Select(AMDGPU::V_CMP_EQ_U16_e64, AMDGPU::V_CMP_EQ_U16_t16_e64,
1359 AMDGPU::V_CMP_EQ_U16_fake16_e64, AMDGPU::V_CMP_EQ_U32_e64,
1360 AMDGPU::V_CMP_EQ_U64_e64);
1361 case CmpInst::ICMP_SGT:
1362 return Select(AMDGPU::V_CMP_GT_I16_e64, AMDGPU::V_CMP_GT_I16_t16_e64,
1363 AMDGPU::V_CMP_GT_I16_fake16_e64, AMDGPU::V_CMP_GT_I32_e64,
1364 AMDGPU::V_CMP_GT_I64_e64);
1365 case CmpInst::ICMP_SGE:
1366 return Select(AMDGPU::V_CMP_GE_I16_e64, AMDGPU::V_CMP_GE_I16_t16_e64,
1367 AMDGPU::V_CMP_GE_I16_fake16_e64, AMDGPU::V_CMP_GE_I32_e64,
1368 AMDGPU::V_CMP_GE_I64_e64);
1369 case CmpInst::ICMP_SLT:
1370 return Select(AMDGPU::V_CMP_LT_I16_e64, AMDGPU::V_CMP_LT_I16_t16_e64,
1371 AMDGPU::V_CMP_LT_I16_fake16_e64, AMDGPU::V_CMP_LT_I32_e64,
1372 AMDGPU::V_CMP_LT_I64_e64);
1373 case CmpInst::ICMP_SLE:
1374 return Select(AMDGPU::V_CMP_LE_I16_e64, AMDGPU::V_CMP_LE_I16_t16_e64,
1375 AMDGPU::V_CMP_LE_I16_fake16_e64, AMDGPU::V_CMP_LE_I32_e64,
1376 AMDGPU::V_CMP_LE_I64_e64);
1377 case CmpInst::ICMP_UGT:
1378 return Select(AMDGPU::V_CMP_GT_U16_e64, AMDGPU::V_CMP_GT_U16_t16_e64,
1379 AMDGPU::V_CMP_GT_U16_fake16_e64, AMDGPU::V_CMP_GT_U32_e64,
1380 AMDGPU::V_CMP_GT_U64_e64);
1381 case CmpInst::ICMP_UGE:
1382 return Select(AMDGPU::V_CMP_GE_U16_e64, AMDGPU::V_CMP_GE_U16_t16_e64,
1383 AMDGPU::V_CMP_GE_U16_fake16_e64, AMDGPU::V_CMP_GE_U32_e64,
1384 AMDGPU::V_CMP_GE_U64_e64);
1385 case CmpInst::ICMP_ULT:
1386 return Select(AMDGPU::V_CMP_LT_U16_e64, AMDGPU::V_CMP_LT_U16_t16_e64,
1387 AMDGPU::V_CMP_LT_U16_fake16_e64, AMDGPU::V_CMP_LT_U32_e64,
1388 AMDGPU::V_CMP_LT_U64_e64);
1389 case CmpInst::ICMP_ULE:
1390 return Select(AMDGPU::V_CMP_LE_U16_e64, AMDGPU::V_CMP_LE_U16_t16_e64,
1391 AMDGPU::V_CMP_LE_U16_fake16_e64, AMDGPU::V_CMP_LE_U32_e64,
1392 AMDGPU::V_CMP_LE_U64_e64);
1393
1394 case CmpInst::FCMP_OEQ:
1395 return Select(AMDGPU::V_CMP_EQ_F16_e64, AMDGPU::V_CMP_EQ_F16_t16_e64,
1396 AMDGPU::V_CMP_EQ_F16_fake16_e64, AMDGPU::V_CMP_EQ_F32_e64,
1397 AMDGPU::V_CMP_EQ_F64_e64);
1398 case CmpInst::FCMP_OGT:
1399 return Select(AMDGPU::V_CMP_GT_F16_e64, AMDGPU::V_CMP_GT_F16_t16_e64,
1400 AMDGPU::V_CMP_GT_F16_fake16_e64, AMDGPU::V_CMP_GT_F32_e64,
1401 AMDGPU::V_CMP_GT_F64_e64);
1402 case CmpInst::FCMP_OGE:
1403 return Select(AMDGPU::V_CMP_GE_F16_e64, AMDGPU::V_CMP_GE_F16_t16_e64,
1404 AMDGPU::V_CMP_GE_F16_fake16_e64, AMDGPU::V_CMP_GE_F32_e64,
1405 AMDGPU::V_CMP_GE_F64_e64);
1406 case CmpInst::FCMP_OLT:
1407 return Select(AMDGPU::V_CMP_LT_F16_e64, AMDGPU::V_CMP_LT_F16_t16_e64,
1408 AMDGPU::V_CMP_LT_F16_fake16_e64, AMDGPU::V_CMP_LT_F32_e64,
1409 AMDGPU::V_CMP_LT_F64_e64);
1410 case CmpInst::FCMP_OLE:
1411 return Select(AMDGPU::V_CMP_LE_F16_e64, AMDGPU::V_CMP_LE_F16_t16_e64,
1412 AMDGPU::V_CMP_LE_F16_fake16_e64, AMDGPU::V_CMP_LE_F32_e64,
1413 AMDGPU::V_CMP_LE_F64_e64);
1414 case CmpInst::FCMP_ONE:
1415 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1416 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1417 AMDGPU::V_CMP_NEQ_F64_e64);
1418 case CmpInst::FCMP_ORD:
1419 return Select(AMDGPU::V_CMP_O_F16_e64, AMDGPU::V_CMP_O_F16_t16_e64,
1420 AMDGPU::V_CMP_O_F16_fake16_e64, AMDGPU::V_CMP_O_F32_e64,
1421 AMDGPU::V_CMP_O_F64_e64);
1422 case CmpInst::FCMP_UNO:
1423 return Select(AMDGPU::V_CMP_U_F16_e64, AMDGPU::V_CMP_U_F16_t16_e64,
1424 AMDGPU::V_CMP_U_F16_fake16_e64, AMDGPU::V_CMP_U_F32_e64,
1425 AMDGPU::V_CMP_U_F64_e64);
1426 case CmpInst::FCMP_UEQ:
1427 return Select(AMDGPU::V_CMP_NLG_F16_e64, AMDGPU::V_CMP_NLG_F16_t16_e64,
1428 AMDGPU::V_CMP_NLG_F16_fake16_e64, AMDGPU::V_CMP_NLG_F32_e64,
1429 AMDGPU::V_CMP_NLG_F64_e64);
1430 case CmpInst::FCMP_UGT:
1431 return Select(AMDGPU::V_CMP_NLE_F16_e64, AMDGPU::V_CMP_NLE_F16_t16_e64,
1432 AMDGPU::V_CMP_NLE_F16_fake16_e64, AMDGPU::V_CMP_NLE_F32_e64,
1433 AMDGPU::V_CMP_NLE_F64_e64);
1434 case CmpInst::FCMP_UGE:
1435 return Select(AMDGPU::V_CMP_NLT_F16_e64, AMDGPU::V_CMP_NLT_F16_t16_e64,
1436 AMDGPU::V_CMP_NLT_F16_fake16_e64, AMDGPU::V_CMP_NLT_F32_e64,
1437 AMDGPU::V_CMP_NLT_F64_e64);
1438 case CmpInst::FCMP_ULT:
1439 return Select(AMDGPU::V_CMP_NGE_F16_e64, AMDGPU::V_CMP_NGE_F16_t16_e64,
1440 AMDGPU::V_CMP_NGE_F16_fake16_e64, AMDGPU::V_CMP_NGE_F32_e64,
1441 AMDGPU::V_CMP_NGE_F64_e64);
1442 case CmpInst::FCMP_ULE:
1443 return Select(AMDGPU::V_CMP_NGT_F16_e64, AMDGPU::V_CMP_NGT_F16_t16_e64,
1444 AMDGPU::V_CMP_NGT_F16_fake16_e64, AMDGPU::V_CMP_NGT_F32_e64,
1445 AMDGPU::V_CMP_NGT_F64_e64);
1446 case CmpInst::FCMP_UNE:
1447 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1448 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1449 AMDGPU::V_CMP_NEQ_F64_e64);
1450 case CmpInst::FCMP_TRUE:
1451 return Select(AMDGPU::V_CMP_TRU_F16_e64, AMDGPU::V_CMP_TRU_F16_t16_e64,
1452 AMDGPU::V_CMP_TRU_F16_fake16_e64, AMDGPU::V_CMP_TRU_F32_e64,
1453 AMDGPU::V_CMP_TRU_F64_e64);
1454 case CmpInst::FCMP_FALSE:
1455 return Select(AMDGPU::V_CMP_F_F16_e64, AMDGPU::V_CMP_F_F16_t16_e64,
1456 AMDGPU::V_CMP_F_F16_fake16_e64, AMDGPU::V_CMP_F_F32_e64,
1457 AMDGPU::V_CMP_F_F64_e64);
1458 }
1459}
1460
1461int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
1462 uint64_t Size) const {
1463 if (Size == 64) {
1464 if (!STI.hasScalarCompareEq64())
1465 return -1;
1466
1467 switch (P) {
1468 case CmpInst::ICMP_NE:
1469 return AMDGPU::S_CMP_LG_U64;
1470 case CmpInst::ICMP_EQ:
1471 return AMDGPU::S_CMP_EQ_U64;
1472 default:
1473 return -1;
1474 }
1475 }
1476
1477 if (Size == 32) {
1478 switch (P) {
1479 case CmpInst::ICMP_NE:
1480 return AMDGPU::S_CMP_LG_U32;
1481 case CmpInst::ICMP_EQ:
1482 return AMDGPU::S_CMP_EQ_U32;
1483 case CmpInst::ICMP_SGT:
1484 return AMDGPU::S_CMP_GT_I32;
1485 case CmpInst::ICMP_SGE:
1486 return AMDGPU::S_CMP_GE_I32;
1487 case CmpInst::ICMP_SLT:
1488 return AMDGPU::S_CMP_LT_I32;
1489 case CmpInst::ICMP_SLE:
1490 return AMDGPU::S_CMP_LE_I32;
1491 case CmpInst::ICMP_UGT:
1492 return AMDGPU::S_CMP_GT_U32;
1493 case CmpInst::ICMP_UGE:
1494 return AMDGPU::S_CMP_GE_U32;
1495 case CmpInst::ICMP_ULT:
1496 return AMDGPU::S_CMP_LT_U32;
1497 case CmpInst::ICMP_ULE:
1498 return AMDGPU::S_CMP_LE_U32;
1499 case CmpInst::FCMP_OEQ:
1500 return AMDGPU::S_CMP_EQ_F32;
1501 case CmpInst::FCMP_OGT:
1502 return AMDGPU::S_CMP_GT_F32;
1503 case CmpInst::FCMP_OGE:
1504 return AMDGPU::S_CMP_GE_F32;
1505 case CmpInst::FCMP_OLT:
1506 return AMDGPU::S_CMP_LT_F32;
1507 case CmpInst::FCMP_OLE:
1508 return AMDGPU::S_CMP_LE_F32;
1509 case CmpInst::FCMP_ONE:
1510 return AMDGPU::S_CMP_LG_F32;
1511 case CmpInst::FCMP_ORD:
1512 return AMDGPU::S_CMP_O_F32;
1513 case CmpInst::FCMP_UNO:
1514 return AMDGPU::S_CMP_U_F32;
1515 case CmpInst::FCMP_UEQ:
1516 return AMDGPU::S_CMP_NLG_F32;
1517 case CmpInst::FCMP_UGT:
1518 return AMDGPU::S_CMP_NLE_F32;
1519 case CmpInst::FCMP_UGE:
1520 return AMDGPU::S_CMP_NLT_F32;
1521 case CmpInst::FCMP_ULT:
1522 return AMDGPU::S_CMP_NGE_F32;
1523 case CmpInst::FCMP_ULE:
1524 return AMDGPU::S_CMP_NGT_F32;
1525 case CmpInst::FCMP_UNE:
1526 return AMDGPU::S_CMP_NEQ_F32;
1527 default:
1528 llvm_unreachable("Unknown condition code!");
1529 }
1530 }
1531
1532 if (Size == 16) {
1533 if (!STI.hasSALUFloatInsts())
1534 return -1;
1535
1536 switch (P) {
1537 case CmpInst::FCMP_OEQ:
1538 return AMDGPU::S_CMP_EQ_F16;
1539 case CmpInst::FCMP_OGT:
1540 return AMDGPU::S_CMP_GT_F16;
1541 case CmpInst::FCMP_OGE:
1542 return AMDGPU::S_CMP_GE_F16;
1543 case CmpInst::FCMP_OLT:
1544 return AMDGPU::S_CMP_LT_F16;
1545 case CmpInst::FCMP_OLE:
1546 return AMDGPU::S_CMP_LE_F16;
1547 case CmpInst::FCMP_ONE:
1548 return AMDGPU::S_CMP_LG_F16;
1549 case CmpInst::FCMP_ORD:
1550 return AMDGPU::S_CMP_O_F16;
1551 case CmpInst::FCMP_UNO:
1552 return AMDGPU::S_CMP_U_F16;
1553 case CmpInst::FCMP_UEQ:
1554 return AMDGPU::S_CMP_NLG_F16;
1555 case CmpInst::FCMP_UGT:
1556 return AMDGPU::S_CMP_NLE_F16;
1557 case CmpInst::FCMP_UGE:
1558 return AMDGPU::S_CMP_NLT_F16;
1559 case CmpInst::FCMP_ULT:
1560 return AMDGPU::S_CMP_NGE_F16;
1561 case CmpInst::FCMP_ULE:
1562 return AMDGPU::S_CMP_NGT_F16;
1563 case CmpInst::FCMP_UNE:
1564 return AMDGPU::S_CMP_NEQ_F16;
1565 default:
1566 llvm_unreachable("Unknown condition code!");
1567 }
1568 }
1569
1570 return -1;
1571}
1572
1573bool AMDGPUInstructionSelector::selectG_ICMP_or_FCMP(MachineInstr &I) const {
1574
1575 MachineBasicBlock *BB = I.getParent();
1576 const DebugLoc &DL = I.getDebugLoc();
1577
1578 Register SrcReg = I.getOperand(i: 2).getReg();
1579 uint64_t Size = TRI.getRegSizeInBits(Reg: SrcReg, MRI: *MRI);
1580
1581 auto Pred = (CmpInst::Predicate)I.getOperand(i: 1).getPredicate();
1582
1583 Register CCReg = I.getOperand(i: 0).getReg();
1584 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
1585 int Opcode = getS_CMPOpcode(P: Pred, Size);
1586 if (Opcode == -1)
1587 return false;
1588 MachineInstr *ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode))
1589 .add(MO: I.getOperand(i: 2))
1590 .add(MO: I.getOperand(i: 3));
1591 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg)
1592 .addReg(RegNo: AMDGPU::SCC);
1593 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1594 bool Ret =
1595 RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
1596 I.eraseFromParent();
1597 return Ret;
1598 }
1599
1600 if (I.getOpcode() == AMDGPU::G_FCMP)
1601 return false;
1602
1603 int Opcode = getV_CMPOpcode(P: Pred, Size, ST: *Subtarget);
1604 if (Opcode == -1)
1605 return false;
1606
1607 MachineInstrBuilder ICmp;
1608 // t16 instructions
1609 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
1610 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1611 .addImm(Val: 0)
1612 .add(MO: I.getOperand(i: 2))
1613 .addImm(Val: 0)
1614 .add(MO: I.getOperand(i: 3))
1615 .addImm(Val: 0); // op_sel
1616 } else {
1617 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1618 .add(MO: I.getOperand(i: 2))
1619 .add(MO: I.getOperand(i: 3));
1620 }
1621
1622 RBI.constrainGenericRegister(Reg: ICmp->getOperand(i: 0).getReg(),
1623 RC: *TRI.getBoolRC(), MRI&: *MRI);
1624 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1625 I.eraseFromParent();
1626 return true;
1627}
1628
1629// Ballot has to zero bits in input lane-mask that are zero in current exec,
1630// Done as AND with exec. For inputs that are results of instruction that
1631// implicitly use same exec, for example compares in same basic block or SCC to
1632// VCC copy, use copy.
1633static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI,
1634 MachineBasicBlock *MBB) {
1635 MachineInstr *MI = MRI.getVRegDef(Reg);
1636 if (MI->getParent() != MBB)
1637 return false;
1638
1639 // Lane mask generated by SCC to VCC copy.
1640 if (MI->getOpcode() == AMDGPU::COPY) {
1641 auto DstRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 0).getReg());
1642 auto SrcRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 1).getReg());
1643 if (DstRB && SrcRB && DstRB->getID() == AMDGPU::VCCRegBankID &&
1644 SrcRB->getID() == AMDGPU::SGPRRegBankID)
1645 return true;
1646 }
1647
1648 // Lane mask generated by SCC to VCC copy
1649 if (MI->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
1650 return true;
1651
1652 // Lane mask generated using compare with same exec.
1653 if (isa<GAnyCmp>(Val: MI))
1654 return true;
1655
1656 Register LHS, RHS;
1657 // Look through AND.
1658 if (mi_match(R: Reg, MRI, P: m_GAnd(L: m_Reg(R&: LHS), R: m_Reg(R&: RHS))))
1659 return isLaneMaskFromSameBlock(Reg: LHS, MRI, MBB) ||
1660 isLaneMaskFromSameBlock(Reg: RHS, MRI, MBB);
1661
1662 return false;
1663}
1664
1665bool AMDGPUInstructionSelector::selectBallot(MachineInstr &I) const {
1666 MachineBasicBlock *BB = I.getParent();
1667 const DebugLoc &DL = I.getDebugLoc();
1668 Register DstReg = I.getOperand(i: 0).getReg();
1669 Register SrcReg = I.getOperand(i: 2).getReg();
1670 const uint64_t BallotSize = MRI->getType(Reg: DstReg).getSizeInBits();
1671 const unsigned WaveSize = STI.getWavefrontSize();
1672
1673 // In the common case, the return type matches the wave size.
1674 // However we also support emitting i64 ballots in wave32 mode.
1675 if (BallotSize != WaveSize && (BallotSize != 64 || WaveSize != 32))
1676 return false;
1677
1678 std::optional<ValueAndVReg> Arg =
1679 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI);
1680
1681 Register Dst = DstReg;
1682 // i64 ballot on Wave32: new Dst(i32) for WaveSize ballot.
1683 if (BallotSize != WaveSize) {
1684 Dst = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
1685 }
1686
1687 if (Arg) {
1688 const int64_t Value = Arg->Value.getZExtValue();
1689 if (Value == 0) {
1690 // Dst = S_MOV 0
1691 unsigned Opcode = WaveSize == 64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
1692 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: Dst).addImm(Val: 0);
1693 } else {
1694 // Dst = COPY EXEC
1695 assert(Value == 1);
1696 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: TRI.getExec());
1697 }
1698 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1699 return false;
1700 } else {
1701 if (isLaneMaskFromSameBlock(Reg: SrcReg, MRI&: *MRI, MBB: BB)) {
1702 // Dst = COPY SrcReg
1703 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: SrcReg);
1704 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1705 return false;
1706 } else {
1707 // Dst = S_AND SrcReg, EXEC
1708 unsigned AndOpc = WaveSize == 64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
1709 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: Dst)
1710 .addReg(RegNo: SrcReg)
1711 .addReg(RegNo: TRI.getExec())
1712 .setOperandDead(3); // Dead scc
1713 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
1714 }
1715 }
1716
1717 // i64 ballot on Wave32: zero-extend i32 ballot to i64.
1718 if (BallotSize != WaveSize) {
1719 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1720 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg).addImm(Val: 0);
1721 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
1722 .addReg(RegNo: Dst)
1723 .addImm(Val: AMDGPU::sub0)
1724 .addReg(RegNo: HiReg)
1725 .addImm(Val: AMDGPU::sub1);
1726 }
1727
1728 I.eraseFromParent();
1729 return true;
1730}
1731
1732bool AMDGPUInstructionSelector::selectRelocConstant(MachineInstr &I) const {
1733 Register DstReg = I.getOperand(i: 0).getReg();
1734 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1735 const TargetRegisterClass *DstRC = TRI.getRegClassForSizeOnBank(Size: 32, Bank: *DstBank);
1736 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1737 return false;
1738
1739 const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID;
1740
1741 Module *M = MF->getFunction().getParent();
1742 const MDNode *Metadata = I.getOperand(i: 2).getMetadata();
1743 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
1744 auto *RelocSymbol = cast<GlobalVariable>(
1745 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
1746
1747 MachineBasicBlock *BB = I.getParent();
1748 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(),
1749 MCID: TII.get(Opcode: IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DestReg: DstReg)
1750 .addGlobalAddress(GV: RelocSymbol, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1751
1752 I.eraseFromParent();
1753 return true;
1754}
1755
1756bool AMDGPUInstructionSelector::selectGroupStaticSize(MachineInstr &I) const {
1757 Triple::OSType OS = MF->getTarget().getTargetTriple().getOS();
1758
1759 Register DstReg = I.getOperand(i: 0).getReg();
1760 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1761 unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ?
1762 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1763
1764 MachineBasicBlock *MBB = I.getParent();
1765 const DebugLoc &DL = I.getDebugLoc();
1766
1767 auto MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Mov), DestReg: DstReg);
1768
1769 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
1770 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1771 MIB.addImm(Val: MFI->getLDSSize());
1772 } else {
1773 Module *M = MF->getFunction().getParent();
1774 const GlobalValue *GV =
1775 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::amdgcn_groupstaticsize);
1776 MIB.addGlobalAddress(GV, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1777 }
1778
1779 I.eraseFromParent();
1780 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1781 return true;
1782}
1783
1784bool AMDGPUInstructionSelector::selectReturnAddress(MachineInstr &I) const {
1785 MachineBasicBlock *MBB = I.getParent();
1786 MachineFunction &MF = *MBB->getParent();
1787 const DebugLoc &DL = I.getDebugLoc();
1788
1789 Register DstReg = I.getOperand(i: 0).getReg();
1790 unsigned Depth = static_cast<unsigned>(I.getOperand(i: 2).getImm());
1791
1792 const TargetRegisterClass *RC =
1793 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
1794 if (!RC->hasSubClassEq(RC: &AMDGPU::SGPR_64RegClass) ||
1795 !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
1796 return false;
1797
1798 // Check for kernel and shader functions
1799 if (Depth != 0 ||
1800 MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
1801 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg)
1802 .addImm(Val: 0);
1803 I.eraseFromParent();
1804 return true;
1805 }
1806
1807 MachineFrameInfo &MFI = MF.getFrameInfo();
1808 // There is a call to @llvm.returnaddress in this function
1809 MFI.setReturnAddressIsTaken(true);
1810
1811 // Get the return address reg and mark it as an implicit live-in
1812 Register ReturnAddrReg = TRI.getReturnAddressReg(MF);
1813 Register LiveIn = getFunctionLiveInPhysReg(MF, TII, PhysReg: ReturnAddrReg,
1814 RC: AMDGPU::SReg_64RegClass, DL);
1815 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
1816 .addReg(RegNo: LiveIn);
1817 I.eraseFromParent();
1818 return true;
1819}
1820
1821bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const {
1822 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1823 // SelectionDAG uses for wave32 vs wave64.
1824 MachineBasicBlock *BB = MI.getParent();
1825 BuildMI(BB&: *BB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_END_CF))
1826 .add(MO: MI.getOperand(i: 1))
1827 .setOperandDead(2); // implicit-def $scc
1828
1829 Register Reg = MI.getOperand(i: 1).getReg();
1830 MI.eraseFromParent();
1831
1832 if (!MRI->getRegClassOrNull(Reg))
1833 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1834 return true;
1835}
1836
1837bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic(
1838 MachineInstr &MI, Intrinsic::ID IntrID) const {
1839 MachineBasicBlock *MBB = MI.getParent();
1840 MachineFunction *MF = MBB->getParent();
1841 const DebugLoc &DL = MI.getDebugLoc();
1842
1843 unsigned IndexOperand = static_cast<unsigned>(MI.getOperand(i: 7).getImm());
1844 bool WaveRelease = MI.getOperand(i: 8).getImm() != 0;
1845 bool WaveDone = MI.getOperand(i: 9).getImm() != 0;
1846
1847 if (WaveDone && !WaveRelease) {
1848 // TODO: Move this to IR verifier
1849 const Function &Fn = MF->getFunction();
1850 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1851 Fn, "ds_ordered_count: wave_done requires wave_release", DL));
1852 }
1853
1854 unsigned OrderedCountIndex = IndexOperand & 0x3f;
1855 IndexOperand &= ~0x3f;
1856 unsigned CountDw = 0;
1857
1858 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) {
1859 CountDw = (IndexOperand >> 24) & 0xf;
1860 IndexOperand &= ~(0xf << 24);
1861
1862 if (CountDw < 1 || CountDw > 4) {
1863 const Function &Fn = MF->getFunction();
1864 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1865 Fn, "ds_ordered_count: dword count must be between 1 and 4", DL));
1866 CountDw = 1;
1867 }
1868 }
1869
1870 if (IndexOperand) {
1871 const Function &Fn = MF->getFunction();
1872 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1873 Fn, "ds_ordered_count: bad index operand", DL));
1874 }
1875
1876 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
1877 unsigned ShaderType = SIInstrInfo::getDSShaderTypeValue(MF: *MF);
1878
1879 unsigned Offset0 = OrderedCountIndex << 2;
1880 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
1881
1882 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10)
1883 Offset1 |= (CountDw - 1) << 6;
1884
1885 if (STI.getGeneration() < AMDGPUSubtarget::GFX11)
1886 Offset1 |= ShaderType << 2;
1887
1888 unsigned Offset = Offset0 | (Offset1 << 8);
1889
1890 Register M0Val = MI.getOperand(i: 2).getReg();
1891 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1892 .addReg(RegNo: M0Val);
1893
1894 Register DstReg = MI.getOperand(i: 0).getReg();
1895 Register ValReg = MI.getOperand(i: 3).getReg();
1896 MachineInstrBuilder DS =
1897 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_ORDERED_COUNT), DestReg: DstReg)
1898 .addReg(RegNo: ValReg)
1899 .addImm(Val: Offset)
1900 .cloneMemRefs(OtherMI: MI);
1901
1902 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
1903 return false;
1904
1905 constrainSelectedInstRegOperands(I&: *DS, TII, TRI, RBI);
1906 MI.eraseFromParent();
1907 return true;
1908}
1909
1910static unsigned gwsIntrinToOpcode(unsigned IntrID) {
1911 switch (IntrID) {
1912 case Intrinsic::amdgcn_ds_gws_init:
1913 return AMDGPU::DS_GWS_INIT;
1914 case Intrinsic::amdgcn_ds_gws_barrier:
1915 return AMDGPU::DS_GWS_BARRIER;
1916 case Intrinsic::amdgcn_ds_gws_sema_v:
1917 return AMDGPU::DS_GWS_SEMA_V;
1918 case Intrinsic::amdgcn_ds_gws_sema_br:
1919 return AMDGPU::DS_GWS_SEMA_BR;
1920 case Intrinsic::amdgcn_ds_gws_sema_p:
1921 return AMDGPU::DS_GWS_SEMA_P;
1922 case Intrinsic::amdgcn_ds_gws_sema_release_all:
1923 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
1924 default:
1925 llvm_unreachable("not a gws intrinsic");
1926 }
1927}
1928
1929bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
1930 Intrinsic::ID IID) const {
1931 if (!STI.hasGWS() || (IID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
1932 !STI.hasGWSSemaReleaseAll()))
1933 return false;
1934
1935 // intrinsic ID, vsrc, offset
1936 const bool HasVSrc = MI.getNumOperands() == 3;
1937 assert(HasVSrc || MI.getNumOperands() == 2);
1938
1939 Register BaseOffset = MI.getOperand(i: HasVSrc ? 2 : 1).getReg();
1940 const RegisterBank *OffsetRB = RBI.getRegBank(Reg: BaseOffset, MRI: *MRI, TRI);
1941 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID)
1942 return false;
1943
1944 MachineInstr *OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
1945 unsigned ImmOffset;
1946
1947 MachineBasicBlock *MBB = MI.getParent();
1948 const DebugLoc &DL = MI.getDebugLoc();
1949
1950 MachineInstr *Readfirstlane = nullptr;
1951
1952 // If we legalized the VGPR input, strip out the readfirstlane to analyze the
1953 // incoming offset, in case there's an add of a constant. We'll have to put it
1954 // back later.
1955 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) {
1956 Readfirstlane = OffsetDef;
1957 BaseOffset = OffsetDef->getOperand(i: 1).getReg();
1958 OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
1959 }
1960
1961 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) {
1962 // If we have a constant offset, try to use the 0 in m0 as the base.
1963 // TODO: Look into changing the default m0 initialization value. If the
1964 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
1965 // the immediate offset.
1966
1967 ImmOffset = static_cast<unsigned>(
1968 OffsetDef->getOperand(i: 1).getCImm()->getZExtValue());
1969 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
1970 .addImm(Val: 0);
1971 } else {
1972 std::tie(args&: BaseOffset, args&: ImmOffset) =
1973 AMDGPU::getBaseWithConstantOffset(MRI&: *MRI, Reg: BaseOffset, ValueTracking: VT);
1974
1975 if (Readfirstlane) {
1976 // We have the constant offset now, so put the readfirstlane back on the
1977 // variable component.
1978 if (!RBI.constrainGenericRegister(Reg: BaseOffset, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
1979 return false;
1980
1981 Readfirstlane->getOperand(i: 1).setReg(BaseOffset);
1982 BaseOffset = Readfirstlane->getOperand(i: 0).getReg();
1983 } else {
1984 if (!RBI.constrainGenericRegister(Reg: BaseOffset,
1985 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
1986 return false;
1987 }
1988
1989 Register M0Base = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1990 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: M0Base)
1991 .addReg(RegNo: BaseOffset)
1992 .addImm(Val: 16)
1993 .setOperandDead(3); // Dead scc
1994
1995 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1996 .addReg(RegNo: M0Base);
1997 }
1998
1999 // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
2000 // offset field) % 64. Some versions of the programming guide omit the m0
2001 // part, or claim it's from offset 0.
2002
2003 unsigned Opc = gwsIntrinToOpcode(IntrID: IID);
2004 const MCInstrDesc &InstrDesc = TII.get(Opcode: Opc);
2005
2006 if (HasVSrc) {
2007 Register VSrc = MI.getOperand(i: 1).getReg();
2008
2009 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
2010 const TargetRegisterClass *DataRC = TII.getRegClass(MCID: InstrDesc, OpNum: Data0Idx);
2011 const TargetRegisterClass *SubRC =
2012 TRI.getSubRegisterClass(DataRC, AMDGPU::sub0);
2013
2014 if (!SubRC) {
2015 // 32-bit normal case.
2016 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *DataRC, MRI&: *MRI))
2017 return false;
2018
2019 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2020 .addReg(RegNo: VSrc)
2021 .addImm(Val: ImmOffset)
2022 .cloneMemRefs(OtherMI: MI);
2023 } else {
2024 // Requires even register alignment, so create 64-bit value and pad the
2025 // top half with undef.
2026 Register DataReg = MRI->createVirtualRegister(RegClass: DataRC);
2027 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *SubRC, MRI&: *MRI))
2028 return false;
2029
2030 Register UndefReg = MRI->createVirtualRegister(RegClass: SubRC);
2031 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2032 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DataReg)
2033 .addReg(RegNo: VSrc)
2034 .addImm(Val: AMDGPU::sub0)
2035 .addReg(RegNo: UndefReg)
2036 .addImm(Val: AMDGPU::sub1);
2037
2038 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2039 .addReg(RegNo: DataReg)
2040 .addImm(Val: ImmOffset)
2041 .cloneMemRefs(OtherMI: MI);
2042 }
2043 } else {
2044 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2045 .addImm(Val: ImmOffset)
2046 .cloneMemRefs(OtherMI: MI);
2047 }
2048
2049 MI.eraseFromParent();
2050 return true;
2051}
2052
2053bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
2054 bool IsAppend) const {
2055 Register PtrBase = MI.getOperand(i: 2).getReg();
2056 LLT PtrTy = MRI->getType(Reg: PtrBase);
2057 bool IsGDS = PtrTy.getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2058
2059 int64_t Offset;
2060 std::tie(args&: PtrBase, args&: Offset) = selectDS1Addr1OffsetImpl(Root&: MI.getOperand(i: 2));
2061
2062 // TODO: Should this try to look through readfirstlane like GWS?
2063 if (!isDSOffsetLegal(Base: PtrBase, Offset)) {
2064 PtrBase = MI.getOperand(i: 2).getReg();
2065 Offset = 0;
2066 }
2067
2068 MachineBasicBlock *MBB = MI.getParent();
2069 const DebugLoc &DL = MI.getDebugLoc();
2070 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2071
2072 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
2073 .addReg(RegNo: PtrBase);
2074 if (!RBI.constrainGenericRegister(Reg: PtrBase, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
2075 return false;
2076
2077 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: MI.getOperand(i: 0).getReg())
2078 .addImm(Val: Offset)
2079 .addImm(Val: IsGDS ? -1 : 0)
2080 .cloneMemRefs(OtherMI: MI);
2081 MI.eraseFromParent();
2082 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2083 return true;
2084}
2085
2086bool AMDGPUInstructionSelector::selectInitWholeWave(MachineInstr &MI) const {
2087 MachineFunction *MF = MI.getMF();
2088 SIMachineFunctionInfo *MFInfo = MF->getInfo<SIMachineFunctionInfo>();
2089
2090 MFInfo->setInitWholeWave();
2091 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
2092}
2093
2094static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE,
2095 bool &IsTexFail) {
2096 if (TexFailCtrl)
2097 IsTexFail = true;
2098
2099 TFE = TexFailCtrl & 0x1;
2100 TexFailCtrl &= ~(uint64_t)0x1;
2101 LWE = TexFailCtrl & 0x2;
2102 TexFailCtrl &= ~(uint64_t)0x2;
2103
2104 return TexFailCtrl == 0;
2105}
2106
2107bool AMDGPUInstructionSelector::selectImageIntrinsic(
2108 MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
2109 MachineBasicBlock *MBB = MI.getParent();
2110 const DebugLoc &DL = MI.getDebugLoc();
2111 unsigned IntrOpcode = Intr->BaseOpcode;
2112
2113 // For image atomic: use no-return opcode if result is unused.
2114 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode) {
2115 Register ResultDef = MI.getOperand(i: 0).getReg();
2116 if (MRI->use_nodbg_empty(RegNo: ResultDef))
2117 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
2118 }
2119
2120 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
2121 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
2122
2123 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
2124 const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI);
2125 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI);
2126 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
2127 const bool IsGFX13Plus = AMDGPU::isGFX13Plus(STI);
2128
2129 const unsigned ArgOffset = MI.getNumExplicitDefs() + 1;
2130
2131 Register VDataIn = AMDGPU::NoRegister;
2132 Register VDataOut = AMDGPU::NoRegister;
2133 LLT VDataTy;
2134 int NumVDataDwords = -1;
2135 bool IsD16 = MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 ||
2136 MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16;
2137
2138 bool Unorm;
2139 if (!BaseOpcode->Sampler)
2140 Unorm = true;
2141 else
2142 Unorm = MI.getOperand(i: ArgOffset + Intr->UnormIndex).getImm() != 0;
2143
2144 bool TFE;
2145 bool LWE;
2146 bool IsTexFail = false;
2147 if (!parseTexFail(TexFailCtrl: MI.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex).getImm(),
2148 TFE, LWE, IsTexFail))
2149 return false;
2150
2151 const int64_t Flags = MI.getOperand(i: ArgOffset + Intr->NumArgs).getImm();
2152 const bool IsA16 = (Flags & 1) != 0;
2153 const bool IsG16 = (Flags & 2) != 0;
2154
2155 // A16 implies 16 bit gradients if subtarget doesn't support G16
2156 if (IsA16 && !STI.hasG16() && !IsG16)
2157 return false;
2158
2159 unsigned DMask = 0;
2160 unsigned DMaskLanes = 0;
2161
2162 if (BaseOpcode->Atomic) {
2163 if (!BaseOpcode->NoReturn)
2164 VDataOut = MI.getOperand(i: 0).getReg();
2165 VDataIn = MI.getOperand(i: 2).getReg();
2166 LLT Ty = MRI->getType(Reg: VDataIn);
2167
2168 // Be careful to allow atomic swap on 16-bit element vectors.
2169 const bool Is64Bit = BaseOpcode->AtomicX2 ?
2170 Ty.getSizeInBits() == 128 :
2171 Ty.getSizeInBits() == 64;
2172
2173 if (BaseOpcode->AtomicX2) {
2174 assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister);
2175
2176 DMask = Is64Bit ? 0xf : 0x3;
2177 NumVDataDwords = Is64Bit ? 4 : 2;
2178 } else {
2179 DMask = Is64Bit ? 0x3 : 0x1;
2180 NumVDataDwords = Is64Bit ? 2 : 1;
2181 }
2182 } else {
2183 DMask = static_cast<unsigned>(
2184 MI.getOperand(i: ArgOffset + Intr->DMaskIndex).getImm());
2185 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
2186
2187 if (BaseOpcode->Store) {
2188 VDataIn = MI.getOperand(i: 1).getReg();
2189 VDataTy = MRI->getType(Reg: VDataIn);
2190 NumVDataDwords = static_cast<int>((VDataTy.getSizeInBits() + 31) / 32);
2191 } else if (BaseOpcode->NoReturn) {
2192 NumVDataDwords = 0;
2193 } else {
2194 VDataOut = MI.getOperand(i: 0).getReg();
2195 VDataTy = MRI->getType(Reg: VDataOut);
2196 NumVDataDwords = DMaskLanes;
2197
2198 if (IsD16 && !STI.hasUnpackedD16VMem())
2199 NumVDataDwords = (DMaskLanes + 1) / 2;
2200 }
2201 }
2202
2203 // Set G16 opcode
2204 if (Subtarget->hasG16() && IsG16) {
2205 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
2206 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
2207 assert(G16MappingInfo);
2208 IntrOpcode = G16MappingInfo->G16; // set opcode to variant with _g16
2209 }
2210
2211 // TODO: Check this in verifier.
2212 assert((!IsTexFail || DMaskLanes >= 1) && "should have legalized this");
2213
2214 unsigned CPol = static_cast<unsigned>(
2215 MI.getOperand(i: ArgOffset + Intr->CachePolicyIndex).getImm());
2216 // Keep GLC only when the atomic's result is actually used.
2217 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
2218 CPol |= AMDGPU::CPol::GLC;
2219 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
2220 AMDGPU::CPol::VOLATILE))
2221 return false;
2222
2223 int NumVAddrRegs = 0;
2224 int NumVAddrDwords = 0;
2225 for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
2226 // Skip the $noregs and 0s inserted during legalization.
2227 MachineOperand &AddrOp = MI.getOperand(i: ArgOffset + I);
2228 if (!AddrOp.isReg())
2229 continue; // XXX - Break?
2230
2231 Register Addr = AddrOp.getReg();
2232 if (!Addr)
2233 break;
2234
2235 ++NumVAddrRegs;
2236 NumVAddrDwords +=
2237 static_cast<int>((MRI->getType(Reg: Addr).getSizeInBits() + 31) / 32);
2238 }
2239
2240 // The legalizer preprocessed the intrinsic arguments. If we aren't using
2241 // NSA, these should have been packed into a single value in the first
2242 // address register
2243 const bool UseNSA =
2244 NumVAddrRegs != 1 &&
2245 (STI.hasPartialNSAEncoding() ? NumVAddrDwords >= NumVAddrRegs
2246 : NumVAddrDwords == NumVAddrRegs);
2247 if (UseNSA && !STI.hasFeature(Feature: AMDGPU::FeatureNSAEncoding)) {
2248 LLVM_DEBUG(dbgs() << "Trying to use NSA on non-NSA target\n");
2249 return false;
2250 }
2251
2252 if (IsTexFail)
2253 ++NumVDataDwords;
2254
2255 int Opcode = -1;
2256 if (IsGFX13Plus) {
2257 Register RsrcReg = MI.getOperand(i: ArgOffset + Intr->RsrcIndex).getReg();
2258 const LLT I32 = LLT::integer(SizeInBits: 32);
2259 bool IndexedRsrc = MRI->getType(Reg: RsrcReg) == I32;
2260 bool IndexedSamp = false;
2261 if (BaseOpcode->Sampler) {
2262 Register SampReg = MI.getOperand(i: ArgOffset + Intr->SampIndex).getReg();
2263 IndexedSamp = MRI->getType(Reg: SampReg) == I32;
2264 }
2265 Opcode =
2266 AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx13, VDataDwords: NumVDataDwords,
2267 VAddrDwords: NumVAddrDwords, IndexedRsrc, IndexedSamp);
2268 } else if (IsGFX12Plus) {
2269 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
2270 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2271 } else if (IsGFX11Plus) {
2272 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2273 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
2274 : AMDGPU::MIMGEncGfx11Default,
2275 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2276 } else if (IsGFX10Plus) {
2277 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2278 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
2279 : AMDGPU::MIMGEncGfx10Default,
2280 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2281 } else {
2282 if (Subtarget->hasGFX90AInsts()) {
2283 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
2284 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2285 if (Opcode == -1) {
2286 LLVM_DEBUG(
2287 dbgs()
2288 << "requested image instruction is not supported on this GPU\n");
2289 return false;
2290 }
2291 }
2292 if (Opcode == -1 &&
2293 STI.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
2294 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
2295 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2296 if (Opcode == -1)
2297 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
2298 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2299 }
2300 if (Opcode == -1)
2301 return false;
2302
2303 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode))
2304 .cloneMemRefs(OtherMI: MI);
2305
2306 if (VDataOut) {
2307 if (BaseOpcode->AtomicX2) {
2308 const bool Is64 = MRI->getType(Reg: VDataOut).getSizeInBits() == 64;
2309
2310 Register TmpReg = MRI->createVirtualRegister(
2311 RegClass: Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass);
2312 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0;
2313
2314 MIB.addDef(RegNo: TmpReg);
2315 if (!MRI->use_empty(RegNo: VDataOut)) {
2316 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VDataOut)
2317 .addReg(RegNo: TmpReg, Flags: {}, SubReg);
2318 }
2319
2320 } else {
2321 MIB.addDef(RegNo: VDataOut); // vdata output
2322 }
2323 }
2324
2325 if (VDataIn)
2326 MIB.addReg(RegNo: VDataIn); // vdata input
2327
2328 for (int I = 0; I != NumVAddrRegs; ++I) {
2329 MachineOperand &SrcOp = MI.getOperand(i: ArgOffset + Intr->VAddrStart + I);
2330 if (SrcOp.isReg()) {
2331 assert(SrcOp.getReg() != 0);
2332 MIB.addReg(RegNo: SrcOp.getReg());
2333 }
2334 }
2335
2336 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->RsrcIndex).getReg());
2337 if (BaseOpcode->Sampler)
2338 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->SampIndex).getReg());
2339
2340 MIB.addImm(Val: DMask); // dmask
2341
2342 if (IsGFX10Plus)
2343 MIB.addImm(Val: DimInfo->Encoding);
2344 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::unorm))
2345 MIB.addImm(Val: Unorm);
2346
2347 MIB.addImm(Val: CPol);
2348 MIB.addImm(Val: IsA16 && // a16 or r128
2349 STI.hasFeature(Feature: AMDGPU::FeatureR128A16) ? -1 : 0);
2350 if (IsGFX10Plus)
2351 MIB.addImm(Val: IsA16 ? -1 : 0);
2352
2353 if (!Subtarget->hasGFX90AInsts()) {
2354 MIB.addImm(Val: TFE); // tfe
2355 } else if (TFE) {
2356 LLVM_DEBUG(dbgs() << "TFE is not supported on this GPU\n");
2357 return false;
2358 }
2359
2360 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::lwe))
2361 MIB.addImm(Val: LWE); // lwe
2362 if (!IsGFX10Plus)
2363 MIB.addImm(Val: DimInfo->DA ? -1 : 0);
2364 if (BaseOpcode->HasD16)
2365 MIB.addImm(Val: IsD16 ? -1 : 0);
2366
2367 MI.eraseFromParent();
2368 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2369 TII.enforceOperandRCAlignment(MI&: *MIB, OpName: AMDGPU::OpName::vaddr);
2370 return true;
2371}
2372
2373// We need to handle this here because tablegen doesn't support matching
2374// instructions with multiple outputs.
2375bool AMDGPUInstructionSelector::selectDSBvhStackIntrinsic(
2376 MachineInstr &MI) const {
2377 Register Dst0 = MI.getOperand(i: 0).getReg();
2378 Register Dst1 = MI.getOperand(i: 1).getReg();
2379
2380 const DebugLoc &DL = MI.getDebugLoc();
2381 MachineBasicBlock *MBB = MI.getParent();
2382
2383 Register Addr = MI.getOperand(i: 3).getReg();
2384 Register Data0 = MI.getOperand(i: 4).getReg();
2385 Register Data1 = MI.getOperand(i: 5).getReg();
2386 unsigned Offset = static_cast<unsigned>(MI.getOperand(i: 6).getImm());
2387
2388 unsigned Opc;
2389 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
2390 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2391 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2392 Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
2393 break;
2394 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2395 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32;
2396 break;
2397 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2398 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64;
2399 break;
2400 }
2401
2402 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
2403 .addDef(RegNo: Dst1)
2404 .addUse(RegNo: Addr)
2405 .addUse(RegNo: Data0)
2406 .addUse(RegNo: Data1)
2407 .addImm(Val: Offset)
2408 .cloneMemRefs(OtherMI: MI);
2409
2410 MI.eraseFromParent();
2411 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2412 return true;
2413}
2414
2415bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
2416 MachineInstr &I) const {
2417 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
2418 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
2419 AMDGPU::lookupBufferIntrinsic(Intr: IntrinsicID);
2420 BufInfo && BufInfo->Kind == AMDGPU::BufferKindLoadToLDS)
2421 return selectBufferLoadLds(MI&: I, Info: *BufInfo);
2422
2423 switch (IntrinsicID) {
2424 case Intrinsic::amdgcn_end_cf:
2425 return selectEndCfIntrinsic(MI&: I);
2426 case Intrinsic::amdgcn_ds_ordered_add:
2427 case Intrinsic::amdgcn_ds_ordered_swap:
2428 return selectDSOrderedIntrinsic(MI&: I, IntrID: IntrinsicID);
2429 case Intrinsic::amdgcn_ds_gws_init:
2430 case Intrinsic::amdgcn_ds_gws_barrier:
2431 case Intrinsic::amdgcn_ds_gws_sema_v:
2432 case Intrinsic::amdgcn_ds_gws_sema_br:
2433 case Intrinsic::amdgcn_ds_gws_sema_p:
2434 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2435 return selectDSGWSIntrinsic(MI&: I, IID: IntrinsicID);
2436 case Intrinsic::amdgcn_ds_append:
2437 return selectDSAppendConsume(MI&: I, IsAppend: true);
2438 case Intrinsic::amdgcn_ds_consume:
2439 return selectDSAppendConsume(MI&: I, IsAppend: false);
2440 case Intrinsic::amdgcn_init_whole_wave:
2441 return selectInitWholeWave(MI&: I);
2442 // Until we can store both the address space of the global and the LDS
2443 // arguments by having tto MachineMemOperands on an intrinsic, we just trust
2444 // that the argument is a global pointer (buffer pointers have been handled by
2445 // a LLVM IR-level lowering).
2446 case Intrinsic::amdgcn_load_to_lds:
2447 case Intrinsic::amdgcn_load_async_to_lds:
2448 case Intrinsic::amdgcn_global_load_lds:
2449 case Intrinsic::amdgcn_global_load_async_lds:
2450 return selectGlobalLoadLds(MI&: I);
2451 case Intrinsic::amdgcn_tensor_load_to_lds:
2452 case Intrinsic::amdgcn_tensor_store_from_lds:
2453 return selectTensorLoadStore(MI&: I, IID: IntrinsicID);
2454 case Intrinsic::amdgcn_asyncmark:
2455 case Intrinsic::amdgcn_wait_asyncmark:
2456 if (!Subtarget->hasAsyncMark())
2457 return false;
2458 break;
2459 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2460 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2461 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2462 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2463 return selectDSBvhStackIntrinsic(MI&: I);
2464 case Intrinsic::amdgcn_s_alloc_vgpr: {
2465 // S_ALLOC_VGPR doesn't have a destination register, it just implicitly sets
2466 // SCC. We then need to COPY it into the result vreg.
2467 MachineBasicBlock *MBB = I.getParent();
2468 const DebugLoc &DL = I.getDebugLoc();
2469
2470 Register ResReg = I.getOperand(i: 0).getReg();
2471
2472 MachineInstr *AllocMI = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ALLOC_VGPR))
2473 .add(MO: I.getOperand(i: 2));
2474 (void)BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: ResReg)
2475 .addReg(RegNo: AMDGPU::SCC);
2476 I.eraseFromParent();
2477 constrainSelectedInstRegOperands(I&: *AllocMI, TII, TRI, RBI);
2478 return RBI.constrainGenericRegister(Reg: ResReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2479 }
2480 case Intrinsic::amdgcn_s_barrier_init:
2481 case Intrinsic::amdgcn_s_barrier_signal_var:
2482 return selectNamedBarrierInit(I, IID: IntrinsicID);
2483 case Intrinsic::amdgcn_s_wakeup_barrier:
2484 case Intrinsic::amdgcn_s_barrier_join:
2485 case Intrinsic::amdgcn_s_get_named_barrier_state:
2486 return selectNamedBarrierInst(I, IID: IntrinsicID);
2487 case Intrinsic::amdgcn_s_get_barrier_state:
2488 return selectSGetBarrierState(I, IID: IntrinsicID);
2489 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
2490 return selectSBarrierSignalIsfirst(I, IID: IntrinsicID);
2491 }
2492 return selectImpl(I, CoverageInfo&: *CoverageInfo);
2493}
2494
2495bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
2496 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
2497 return true;
2498
2499 MachineBasicBlock *BB = I.getParent();
2500 const DebugLoc &DL = I.getDebugLoc();
2501
2502 Register DstReg = I.getOperand(i: 0).getReg();
2503 uint64_t Size = TRI.getRegSizeInBits(Reg: DstReg, MRI: *MRI);
2504 assert(Size <= 32 || Size == 64);
2505 const MachineOperand &CCOp = I.getOperand(i: 1);
2506 Register CCReg = CCOp.getReg();
2507 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
2508 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 :
2509 AMDGPU::S_CSELECT_B32;
2510 MachineInstr *CopySCC = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
2511 .addReg(RegNo: CCReg);
2512
2513 // The generic constrainSelectedInstRegOperands doesn't work for the scc register
2514 // bank, because it does not cover the register class that we used to represent
2515 // for it. So we need to manually set the register class here.
2516 if (!MRI->getRegClassOrNull(Reg: CCReg))
2517 MRI->setRegClass(Reg: CCReg, RC: TRI.getConstrainedRegClassForReg(Reg: CCReg, MRI: *MRI));
2518 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
2519 .add(MO: I.getOperand(i: 2))
2520 .add(MO: I.getOperand(i: 3));
2521
2522 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2523 constrainSelectedInstRegOperands(I&: *CopySCC, TII, TRI, RBI);
2524 I.eraseFromParent();
2525 return true;
2526 }
2527
2528 // Wide VGPR select should have been split in RegBankSelect.
2529 if (Size > 32)
2530 return false;
2531
2532 MachineInstr *Select =
2533 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
2534 .addImm(Val: 0)
2535 .add(MO: I.getOperand(i: 3))
2536 .addImm(Val: 0)
2537 .add(MO: I.getOperand(i: 2))
2538 .add(MO: I.getOperand(i: 1));
2539
2540 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2541 I.eraseFromParent();
2542 return true;
2543}
2544
2545bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
2546 Register DstReg = I.getOperand(i: 0).getReg();
2547 Register SrcReg = I.getOperand(i: 1).getReg();
2548 const LLT DstTy = MRI->getType(Reg: DstReg);
2549 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2550 const LLT S1 = LLT::scalar(SizeInBits: 1);
2551
2552 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2553 const RegisterBank *DstRB;
2554 if (DstTy == S1) {
2555 // This is a special case. We don't treat s1 for legalization artifacts as
2556 // vcc booleans.
2557 DstRB = SrcRB;
2558 } else {
2559 DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2560 if (SrcRB != DstRB)
2561 return false;
2562 }
2563
2564 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
2565
2566 uint64_t DstSize = DstTy.getSizeInBits();
2567 uint64_t SrcSize = SrcTy.getSizeInBits();
2568
2569 const TargetRegisterClass *SrcRC =
2570 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcRB);
2571 const TargetRegisterClass *DstRC =
2572 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
2573 if (!SrcRC || !DstRC)
2574 return false;
2575
2576 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
2577 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
2578 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
2579 return false;
2580 }
2581
2582 if (DstRC == &AMDGPU::VGPR_16RegClass && SrcSize == 32) {
2583 assert(STI.useRealTrue16Insts());
2584 const DebugLoc &DL = I.getDebugLoc();
2585 MachineBasicBlock *MBB = I.getParent();
2586 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
2587 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::lo16);
2588 I.eraseFromParent();
2589 return true;
2590 }
2591
2592 if (DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) && SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
2593 MachineBasicBlock *MBB = I.getParent();
2594 const DebugLoc &DL = I.getDebugLoc();
2595
2596 Register LoReg = MRI->createVirtualRegister(RegClass: DstRC);
2597 Register HiReg = MRI->createVirtualRegister(RegClass: DstRC);
2598 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
2599 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
2600 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
2601 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
2602
2603 if (IsVALU && STI.hasSDWA()) {
2604 // Write the low 16-bits of the high element into the high 16-bits of the
2605 // low element.
2606 MachineInstr *MovSDWA =
2607 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: DstReg)
2608 .addImm(Val: 0) // $src0_modifiers
2609 .addReg(RegNo: HiReg) // $src0
2610 .addImm(Val: 0) // $clamp
2611 .addImm(Val: AMDGPU::SDWA::WORD_1) // $dst_sel
2612 .addImm(Val: AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused
2613 .addImm(Val: AMDGPU::SDWA::WORD_0) // $src0_sel
2614 .addReg(RegNo: LoReg, Flags: RegState::Implicit);
2615 MovSDWA->tieOperands(DefIdx: 0, UseIdx: MovSDWA->getNumOperands() - 1);
2616 } else {
2617 Register TmpReg0 = MRI->createVirtualRegister(RegClass: DstRC);
2618 Register TmpReg1 = MRI->createVirtualRegister(RegClass: DstRC);
2619 Register ImmReg = MRI->createVirtualRegister(RegClass: DstRC);
2620 if (IsVALU) {
2621 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: TmpReg0)
2622 .addImm(Val: 16)
2623 .addReg(RegNo: HiReg);
2624 } else {
2625 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg0)
2626 .addReg(RegNo: HiReg)
2627 .addImm(Val: 16)
2628 .setOperandDead(3); // Dead scc
2629 }
2630
2631 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2632 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
2633 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32;
2634
2635 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: ImmReg)
2636 .addImm(Val: 0xffff);
2637 auto And = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: TmpReg1)
2638 .addReg(RegNo: LoReg)
2639 .addReg(RegNo: ImmReg);
2640 auto Or = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: OrOpc), DestReg: DstReg)
2641 .addReg(RegNo: TmpReg0)
2642 .addReg(RegNo: TmpReg1);
2643
2644 if (!IsVALU) {
2645 And.setOperandDead(3); // Dead scc
2646 Or.setOperandDead(3); // Dead scc
2647 }
2648 }
2649
2650 I.eraseFromParent();
2651 return true;
2652 }
2653
2654 if (!DstTy.isScalar())
2655 return false;
2656
2657 if (SrcSize > 32) {
2658 unsigned SubRegIdx = DstSize < 32
2659 ? static_cast<unsigned>(AMDGPU::sub0)
2660 : TRI.getSubRegFromChannel(Channel: 0, NumRegs: DstSize / 32);
2661 if (SubRegIdx == AMDGPU::NoSubRegister)
2662 return false;
2663
2664 // Deal with weird cases where the class only partially supports the subreg
2665 // index.
2666 const TargetRegisterClass *SrcWithSubRC
2667 = TRI.getSubClassWithSubReg(SrcRC, SubRegIdx);
2668 if (!SrcWithSubRC)
2669 return false;
2670
2671 if (SrcWithSubRC != SrcRC) {
2672 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcWithSubRC, MRI&: *MRI))
2673 return false;
2674 }
2675
2676 I.getOperand(i: 1).setSubReg(SubRegIdx);
2677 }
2678
2679 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
2680 return true;
2681}
2682
2683/// \returns true if a bitmask for \p Size bits will be an inline immediate.
2684static bool shouldUseAndMask(uint64_t Size, unsigned &Mask) {
2685 Mask = maskTrailingOnes<unsigned>(N: static_cast<unsigned>(Size));
2686 int SignedMask = static_cast<int>(Mask);
2687 return SignedMask >= -16 && SignedMask <= 64;
2688}
2689
2690// Like RegisterBankInfo::getRegBank, but don't assume vcc for s1.
2691const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank(
2692 Register Reg, const MachineRegisterInfo &MRI,
2693 const TargetRegisterInfo &TRI) const {
2694 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
2695 if (auto *RB = dyn_cast<const RegisterBank *>(Val: RegClassOrBank))
2696 return RB;
2697
2698 // Ignore the type, since we don't use vcc in artifacts.
2699 if (auto *RC = dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank))
2700 return &RBI.getRegBankFromRegClass(RC: *RC, LLT());
2701 return nullptr;
2702}
2703
2704bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
2705 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG;
2706 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg;
2707 const DebugLoc &DL = I.getDebugLoc();
2708 MachineBasicBlock &MBB = *I.getParent();
2709 const Register DstReg = I.getOperand(i: 0).getReg();
2710 const Register SrcReg = I.getOperand(i: 1).getReg();
2711
2712 const LLT DstTy = MRI->getType(Reg: DstReg);
2713 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2714 const uint64_t SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG
2715 ? I.getOperand(i: 2).getImm()
2716 : SrcTy.getSizeInBits();
2717 const uint64_t DstSize = DstTy.getSizeInBits();
2718 if (!DstTy.isScalar())
2719 return false;
2720
2721 // Artifact casts should never use vcc.
2722 const RegisterBank *SrcBank = getArtifactRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2723
2724 // FIXME: This should probably be illegal and split earlier.
2725 if (I.getOpcode() == AMDGPU::G_ANYEXT) {
2726 if (DstSize <= 32)
2727 return selectCOPY(I);
2728
2729 const TargetRegisterClass *SrcRC =
2730 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcBank);
2731 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2732 const TargetRegisterClass *DstRC =
2733 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
2734
2735 Register UndefReg = MRI->createVirtualRegister(RegClass: SrcRC);
2736 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2737 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2738 .addReg(RegNo: SrcReg)
2739 .addImm(Val: AMDGPU::sub0)
2740 .addReg(RegNo: UndefReg)
2741 .addImm(Val: AMDGPU::sub1);
2742 I.eraseFromParent();
2743
2744 return RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) &&
2745 RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI);
2746 }
2747
2748 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) {
2749 // 64-bit should have been split up in RegBankSelect
2750
2751 // Try to use an and with a mask if it will save code size.
2752 unsigned Mask;
2753 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2754 MachineInstr *ExtI =
2755 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: DstReg)
2756 .addImm(Val: Mask)
2757 .addReg(RegNo: SrcReg);
2758 I.eraseFromParent();
2759 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2760 return true;
2761 }
2762
2763 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2764 MachineInstr *ExtI =
2765 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE), DestReg: DstReg)
2766 .addReg(RegNo: SrcReg)
2767 .addImm(Val: 0) // Offset
2768 .addImm(Val: SrcSize); // Width
2769 I.eraseFromParent();
2770 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2771 return true;
2772 }
2773
2774 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
2775 const TargetRegisterClass &SrcRC = InReg && DstSize > 32 ?
2776 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass;
2777 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: SrcRC, MRI&: *MRI))
2778 return false;
2779
2780 if (Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) {
2781 const unsigned SextOpc = SrcSize == 8 ?
2782 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16;
2783 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: SextOpc), DestReg: DstReg)
2784 .addReg(RegNo: SrcReg);
2785 I.eraseFromParent();
2786 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2787 }
2788
2789 // Using a single 32-bit SALU to calculate the high half is smaller than
2790 // S_BFE with a literal constant operand.
2791 if (DstSize > 32 && SrcSize == 32) {
2792 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2793 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2794 if (Signed) {
2795 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ASHR_I32), DestReg: HiReg)
2796 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2797 .addImm(Val: 31)
2798 .setOperandDead(3); // Dead scc
2799 } else {
2800 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg)
2801 .addImm(Val: 0);
2802 }
2803 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2804 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2805 .addImm(Val: AMDGPU::sub0)
2806 .addReg(RegNo: HiReg)
2807 .addImm(Val: AMDGPU::sub1);
2808 I.eraseFromParent();
2809 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass,
2810 MRI&: *MRI);
2811 }
2812
2813 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
2814 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2815
2816 // Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width.
2817 if (DstSize > 32 && (SrcSize <= 32 || InReg)) {
2818 // We need a 64-bit register source, but the high bits don't matter.
2819 Register ExtReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
2820 Register UndefReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2821 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2822
2823 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2824 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: ExtReg)
2825 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2826 .addImm(Val: AMDGPU::sub0)
2827 .addReg(RegNo: UndefReg)
2828 .addImm(Val: AMDGPU::sub1);
2829
2830 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE64), DestReg: DstReg)
2831 .addReg(RegNo: ExtReg)
2832 .addImm(Val: SrcSize << 16);
2833
2834 I.eraseFromParent();
2835 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI);
2836 }
2837
2838 unsigned Mask;
2839 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2840 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: DstReg)
2841 .addReg(RegNo: SrcReg)
2842 .addImm(Val: Mask)
2843 .setOperandDead(3); // Dead scc
2844 } else {
2845 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE32), DestReg: DstReg)
2846 .addReg(RegNo: SrcReg)
2847 .addImm(Val: SrcSize << 16);
2848 }
2849
2850 I.eraseFromParent();
2851 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2852 }
2853
2854 return false;
2855}
2856
2857static Register stripCopy(Register Reg, MachineRegisterInfo &MRI) {
2858 return getDefSrcRegIgnoringCopies(Reg, MRI)->Reg;
2859}
2860
2861static Register stripBitCast(Register Reg, MachineRegisterInfo &MRI) {
2862 Register BitcastSrc;
2863 if (mi_match(R: Reg, MRI, P: m_GBitcast(Src: m_Reg(R&: BitcastSrc))))
2864 Reg = BitcastSrc;
2865 return Reg;
2866}
2867
2868static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In,
2869 Register &Out) {
2870 // When unmerging a register that is composed of 2 x 16-bit values allow to
2871 // use an extract hi instruction for the upper 16 bits. We only need to check
2872 // the size of `In` as all defs are guaranteed to be the same type for
2873 // GUnmerge.
2874 GUnmerge *Unmerge;
2875 if (mi_match(R: In, MRI, P: m_GUnmerge(Inst&: Unmerge))) {
2876 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(i: 1).getReg() == In &&
2877 MRI.getType(Reg: In).getSizeInBits() == 16) {
2878 Out = stripBitCast(Reg: Unmerge->getSourceReg(), MRI);
2879 return true;
2880 }
2881 }
2882
2883 Register Trunc;
2884 if (!mi_match(R: In, MRI, P: m_GTrunc(Src: m_Reg(R&: Trunc))))
2885 return false;
2886
2887 Register LShlSrc;
2888 Register Cst;
2889 if (mi_match(R: Trunc, MRI, P: m_GLShr(L: m_Reg(R&: LShlSrc), R: m_Reg(R&: Cst)))) {
2890 Cst = stripCopy(Reg: Cst, MRI);
2891 if (mi_match(R: Cst, MRI, P: m_SpecificICst(RequestedValue: 16))) {
2892 Register Result = stripBitCast(Reg: LShlSrc, MRI);
2893 // op_sel can only select the high half of a 32-bit source. If the
2894 // shift source is wider (e.g. a 64-bit D16 image sample result), we
2895 // cannot fold it directly. Do not assign Out before this check since
2896 // the caller may pass the same register for both In and Out.
2897 if (MRI.getType(Reg: Result).getSizeInBits() != 32)
2898 return false;
2899 Out = Result;
2900 return true;
2901 }
2902 }
2903
2904 ArrayRef<int> Mask;
2905 Register Src1;
2906 if (!mi_match(R: Trunc, MRI, P: m_GShuffleVector(Src1: m_Reg(R&: Src1), Src2: m_Reg(), Mask)))
2907 return false;
2908
2909 assert(MRI.getType(Src1) == LLT::fixed_vector(2, 16));
2910 assert(Mask.size() == 2);
2911
2912 if (Mask[0] == 1 && Mask[1] <= 1) {
2913 Out = Trunc;
2914 return true;
2915 }
2916
2917 return false;
2918}
2919
2920static bool isExtractLoElt(MachineRegisterInfo &MRI, Register In,
2921 Register &Out) {
2922 // There could be a bitcast between the extraction and its use.
2923 In = stripBitCast(Reg: In, MRI);
2924
2925 // The first def of a 2 x 16-bit unmerge is the low half of its source.
2926 if (auto *Unmerge = dyn_cast<GUnmerge>(Val: MRI.getVRegDef(Reg: In))) {
2927 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(i: 0).getReg() == In &&
2928 MRI.getType(Reg: In).getSizeInBits() == 16) {
2929 Out = stripBitCast(Reg: Unmerge->getSourceReg(), MRI);
2930 return true;
2931 }
2932 }
2933
2934 // A truncation from 32 to 16 bits keeps the low half in place.
2935 Register Trunc;
2936 if (mi_match(R: In, MRI, P: m_GTrunc(Src: m_Reg(R&: Trunc))) &&
2937 MRI.getType(Reg: Trunc).getSizeInBits() == 32) {
2938 Out = stripBitCast(Reg: Trunc, MRI);
2939 return true;
2940 }
2941
2942 return false;
2943}
2944
2945bool AMDGPUInstructionSelector::selectG_FPEXT(MachineInstr &I) const {
2946 if (!Subtarget->hasSALUFloatInsts())
2947 return false;
2948
2949 Register Dst = I.getOperand(i: 0).getReg();
2950 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2951 if (DstRB->getID() != AMDGPU::SGPRRegBankID)
2952 return false;
2953
2954 Register Src = I.getOperand(i: 1).getReg();
2955
2956 if (MRI->getType(Reg: Dst) == LLT::scalar(SizeInBits: 32) &&
2957 MRI->getType(Reg: Src) == LLT::scalar(SizeInBits: 16)) {
2958 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
2959 MachineBasicBlock *BB = I.getParent();
2960 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_CVT_HI_F32_F16), DestReg: Dst)
2961 .addUse(RegNo: Src);
2962 I.eraseFromParent();
2963 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2964 }
2965 }
2966
2967 return false;
2968}
2969
2970bool AMDGPUInstructionSelector::selectG_FNEG(MachineInstr &MI) const {
2971 // Only manually handle the f64 SGPR case.
2972 //
2973 // FIXME: This is a workaround for 2.5 different tablegen problems. Because
2974 // the bit ops theoretically have a second result due to the implicit def of
2975 // SCC, the GlobalISelEmitter is overly conservative and rejects it. Fixing
2976 // that is easy by disabling the check. The result works, but uses a
2977 // nonsensical sreg32orlds_and_sreg_1 regclass.
2978 //
2979 // The DAG emitter is more problematic, and incorrectly adds both S_XOR_B32 to
2980 // the variadic REG_SEQUENCE operands.
2981
2982 Register Dst = MI.getOperand(i: 0).getReg();
2983 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2984 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
2985 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
2986 return false;
2987
2988 Register Src = MI.getOperand(i: 1).getReg();
2989 MachineInstr *Fabs = getOpcodeDef(Opcode: TargetOpcode::G_FABS, Reg: Src, MRI: *MRI);
2990 if (Fabs)
2991 Src = Fabs->getOperand(i: 1).getReg();
2992
2993 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
2994 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
2995 return false;
2996
2997 MachineBasicBlock *BB = MI.getParent();
2998 const DebugLoc &DL = MI.getDebugLoc();
2999 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3000 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3001 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3002 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3003
3004 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3005 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
3006 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3007 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
3008 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
3009 .addImm(Val: 0x80000000);
3010
3011 // Set or toggle sign bit.
3012 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32;
3013 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: OpReg)
3014 .addReg(RegNo: HiReg)
3015 .addReg(RegNo: ConstReg)
3016 .setOperandDead(3); // Dead scc
3017 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
3018 .addReg(RegNo: LoReg)
3019 .addImm(Val: AMDGPU::sub0)
3020 .addReg(RegNo: OpReg)
3021 .addImm(Val: AMDGPU::sub1);
3022 MI.eraseFromParent();
3023 return true;
3024}
3025
3026// FIXME: This is a workaround for the same tablegen problems as G_FNEG
3027bool AMDGPUInstructionSelector::selectG_FABS(MachineInstr &MI) const {
3028 Register Dst = MI.getOperand(i: 0).getReg();
3029 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
3030 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
3031 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
3032 return false;
3033
3034 Register Src = MI.getOperand(i: 1).getReg();
3035 MachineBasicBlock *BB = MI.getParent();
3036 const DebugLoc &DL = MI.getDebugLoc();
3037 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3038 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3039 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3040 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
3041
3042 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
3043 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
3044 return false;
3045
3046 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3047 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
3048 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3049 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
3050 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
3051 .addImm(Val: 0x7fffffff);
3052
3053 // Clear sign bit.
3054 // TODO: Should this used S_BITSET0_*?
3055 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: OpReg)
3056 .addReg(RegNo: HiReg)
3057 .addReg(RegNo: ConstReg)
3058 .setOperandDead(3); // Dead scc
3059 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
3060 .addReg(RegNo: LoReg)
3061 .addImm(Val: AMDGPU::sub0)
3062 .addReg(RegNo: OpReg)
3063 .addImm(Val: AMDGPU::sub1);
3064
3065 MI.eraseFromParent();
3066 return true;
3067}
3068
3069static bool isConstant(const MachineInstr &MI) {
3070 return MI.getOpcode() == TargetOpcode::G_CONSTANT;
3071}
3072
3073void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
3074 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const {
3075
3076 unsigned OpNo = Load.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1;
3077 const MachineInstr *PtrMI = MRI.getVRegDef(Reg: Load.getOperand(i: OpNo).getReg());
3078
3079 assert(PtrMI);
3080
3081 if (PtrMI->getOpcode() != TargetOpcode::G_PTR_ADD)
3082 return;
3083
3084 GEPInfo GEPInfo;
3085
3086 for (unsigned i = 1; i != 3; ++i) {
3087 const MachineOperand &GEPOp = PtrMI->getOperand(i);
3088 const MachineInstr *OpDef = MRI.getVRegDef(Reg: GEPOp.getReg());
3089 assert(OpDef);
3090 if (i == 2 && isConstant(MI: *OpDef)) {
3091 // TODO: Could handle constant base + variable offset, but a combine
3092 // probably should have commuted it.
3093 assert(GEPInfo.Imm == 0);
3094 GEPInfo.Imm = OpDef->getOperand(i: 1).getCImm()->getSExtValue();
3095 continue;
3096 }
3097 const RegisterBank *OpBank = RBI.getRegBank(Reg: GEPOp.getReg(), MRI, TRI);
3098 if (OpBank->getID() == AMDGPU::SGPRRegBankID)
3099 GEPInfo.SgprParts.push_back(Elt: GEPOp.getReg());
3100 else
3101 GEPInfo.VgprParts.push_back(Elt: GEPOp.getReg());
3102 }
3103
3104 AddrInfo.push_back(Elt: GEPInfo);
3105 getAddrModeInfo(Load: *PtrMI, MRI, AddrInfo);
3106}
3107
3108bool AMDGPUInstructionSelector::isSGPR(Register Reg) const {
3109 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID;
3110}
3111
3112bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const {
3113 if (!MI.hasOneMemOperand())
3114 return false;
3115
3116 const MachineMemOperand *MMO = *MI.memoperands_begin();
3117 const Value *Ptr = MMO->getValue();
3118
3119 // UndefValue means this is a load of a kernel input. These are uniform.
3120 // Sometimes LDS instructions have constant pointers.
3121 // If Ptr is null, then that means this mem operand contains a
3122 // PseudoSourceValue like GOT.
3123 if (!Ptr || isa<UndefValue, Argument, Constant, GlobalValue>(Val: Ptr))
3124 return true;
3125
3126 if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
3127 return true;
3128
3129 if (MI.getOpcode() == AMDGPU::G_PREFETCH)
3130 return RBI.getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI: *MRI, TRI)->getID() ==
3131 AMDGPU::SGPRRegBankID;
3132
3133 const Instruction *I = dyn_cast<Instruction>(Val: Ptr);
3134 return I && I->getMetadata(Kind: "amdgpu.uniform");
3135}
3136
3137bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const {
3138 for (const GEPInfo &GEPInfo : AddrInfo) {
3139 if (!GEPInfo.VgprParts.empty())
3140 return true;
3141 }
3142 return false;
3143}
3144
3145void AMDGPUInstructionSelector::initM0(MachineInstr &I) const {
3146 const LLT PtrTy = MRI->getType(Reg: I.getOperand(i: 1).getReg());
3147 unsigned AS = PtrTy.getAddressSpace();
3148 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) &&
3149 STI.ldsRequiresM0Init()) {
3150 MachineBasicBlock *BB = I.getParent();
3151
3152 // If DS instructions require M0 initialization, insert it before selecting.
3153 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
3154 .addImm(Val: -1);
3155 }
3156}
3157
3158bool AMDGPUInstructionSelector::selectG_LOAD_STORE_ATOMICRMW(
3159 MachineInstr &I) const {
3160 initM0(I);
3161 return selectImpl(I, CoverageInfo&: *CoverageInfo);
3162}
3163
3164static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI) {
3165 if (Reg.isPhysical())
3166 return false;
3167
3168 MachineInstr &MI = *MRI.getVRegDef(Reg);
3169 const unsigned Opcode = MI.getOpcode();
3170
3171 if (Opcode == AMDGPU::COPY)
3172 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI);
3173
3174 if (Opcode == AMDGPU::G_AND || Opcode == AMDGPU::G_OR ||
3175 Opcode == AMDGPU::G_XOR)
3176 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI) &&
3177 isVCmpResult(Reg: MI.getOperand(i: 2).getReg(), MRI);
3178
3179 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI))
3180 return GI->is(ID: Intrinsic::amdgcn_class);
3181
3182 return Opcode == AMDGPU::G_ICMP || Opcode == AMDGPU::G_FCMP;
3183}
3184
3185bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
3186 MachineBasicBlock *BB = I.getParent();
3187 MachineOperand &CondOp = I.getOperand(i: 0);
3188 Register CondReg = CondOp.getReg();
3189 const DebugLoc &DL = I.getDebugLoc();
3190
3191 unsigned BrOpcode;
3192 Register CondPhysReg;
3193 const TargetRegisterClass *ConstrainRC;
3194
3195 // In SelectionDAG, we inspect the IR block for uniformity metadata to decide
3196 // whether the branch is uniform when selecting the instruction. In
3197 // GlobalISel, we should push that decision into RegBankSelect. Assume for now
3198 // RegBankSelect knows what it's doing if the branch condition is scc, even
3199 // though it currently does not.
3200 if (!isVCC(Reg: CondReg, MRI: *MRI)) {
3201 if (MRI->getType(Reg: CondReg) != LLT::scalar(SizeInBits: 32))
3202 return false;
3203
3204 CondPhysReg = AMDGPU::SCC;
3205 BrOpcode = AMDGPU::S_CBRANCH_SCC1;
3206 ConstrainRC = &AMDGPU::SReg_32RegClass;
3207 } else {
3208 // FIXME: Should scc->vcc copies and with exec?
3209
3210 // Unless the value of CondReg is a result of a V_CMP* instruction then we
3211 // need to insert an and with exec.
3212 if (!isVCmpResult(Reg: CondReg, MRI&: *MRI)) {
3213 const bool Is64 = STI.isWave64();
3214 const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
3215 const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO;
3216
3217 Register TmpReg = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
3218 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: TmpReg)
3219 .addReg(RegNo: CondReg)
3220 .addReg(RegNo: Exec)
3221 .setOperandDead(3); // Dead scc
3222 CondReg = TmpReg;
3223 }
3224
3225 CondPhysReg = TRI.getVCC();
3226 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ;
3227 ConstrainRC = TRI.getBoolRC();
3228 }
3229
3230 if (!MRI->getRegClassOrNull(Reg: CondReg))
3231 MRI->setRegClass(Reg: CondReg, RC: ConstrainRC);
3232
3233 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CondPhysReg)
3234 .addReg(RegNo: CondReg);
3235 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: BrOpcode))
3236 .addMBB(MBB: I.getOperand(i: 1).getMBB());
3237
3238 I.eraseFromParent();
3239 return true;
3240}
3241
3242bool AMDGPUInstructionSelector::selectG_GLOBAL_VALUE(
3243 MachineInstr &I) const {
3244 Register DstReg = I.getOperand(i: 0).getReg();
3245 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3246 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3247 I.setDesc(TII.get(Opcode: IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32));
3248 if (IsVGPR)
3249 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
3250
3251 return RBI.constrainGenericRegister(
3252 Reg: DstReg, RC: IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, MRI&: *MRI);
3253}
3254
3255bool AMDGPUInstructionSelector::selectG_PTRMASK(MachineInstr &I) const {
3256 Register DstReg = I.getOperand(i: 0).getReg();
3257 Register SrcReg = I.getOperand(i: 1).getReg();
3258 Register MaskReg = I.getOperand(i: 2).getReg();
3259 LLT Ty = MRI->getType(Reg: DstReg);
3260 LLT MaskTy = MRI->getType(Reg: MaskReg);
3261 MachineBasicBlock *BB = I.getParent();
3262 const DebugLoc &DL = I.getDebugLoc();
3263
3264 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3265 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3266 const RegisterBank *MaskRB = RBI.getRegBank(Reg: MaskReg, MRI: *MRI, TRI);
3267 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3268 if (DstRB != SrcRB) // Should only happen for hand written MIR.
3269 return false;
3270
3271 // Try to avoid emitting a bit operation when we only need to touch half of
3272 // the 64-bit pointer.
3273 APInt MaskOnes = VT->getKnownOnes(R: MaskReg).zext(width: 64);
3274 const APInt MaskHi32 = APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32);
3275 const APInt MaskLo32 = APInt::getLowBitsSet(numBits: 64, loBitsSet: 32);
3276
3277 const bool CanCopyLow32 = (MaskOnes & MaskLo32) == MaskLo32;
3278 const bool CanCopyHi32 = (MaskOnes & MaskHi32) == MaskHi32;
3279
3280 if (!IsVGPR && Ty.getSizeInBits() == 64 &&
3281 !CanCopyLow32 && !CanCopyHi32) {
3282 auto MIB = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B64), DestReg: DstReg)
3283 .addReg(RegNo: SrcReg)
3284 .addReg(RegNo: MaskReg)
3285 .setOperandDead(3); // Dead scc
3286 I.eraseFromParent();
3287 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3288 return true;
3289 }
3290
3291 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
3292 const TargetRegisterClass &RegRC
3293 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
3294
3295 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *DstRB);
3296 const TargetRegisterClass *SrcRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *SrcRB);
3297 const TargetRegisterClass *MaskRC =
3298 TRI.getRegClassForTypeOnBank(Ty: MaskTy, Bank: *MaskRB);
3299
3300 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3301 !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3302 !RBI.constrainGenericRegister(Reg: MaskReg, RC: *MaskRC, MRI&: *MRI))
3303 return false;
3304
3305 if (Ty.getSizeInBits() == 32) {
3306 assert(MaskTy.getSizeInBits() == 32 &&
3307 "ptrmask should have been narrowed during legalize");
3308
3309 auto NewOp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: DstReg)
3310 .addReg(RegNo: SrcReg)
3311 .addReg(RegNo: MaskReg);
3312
3313 if (!IsVGPR)
3314 NewOp.setOperandDead(3); // Dead scc
3315 I.eraseFromParent();
3316 return true;
3317 }
3318
3319 Register HiReg = MRI->createVirtualRegister(RegClass: &RegRC);
3320 Register LoReg = MRI->createVirtualRegister(RegClass: &RegRC);
3321
3322 // Extract the subregisters from the source pointer.
3323 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3324 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
3325 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3326 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
3327
3328 Register MaskedLo, MaskedHi;
3329
3330 if (CanCopyLow32) {
3331 // If all the bits in the low half are 1, we only need a copy for it.
3332 MaskedLo = LoReg;
3333 } else {
3334 // Extract the mask subregister and apply the and.
3335 Register MaskLo = MRI->createVirtualRegister(RegClass: &RegRC);
3336 MaskedLo = MRI->createVirtualRegister(RegClass: &RegRC);
3337
3338 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskLo)
3339 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub0);
3340 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedLo)
3341 .addReg(RegNo: LoReg)
3342 .addReg(RegNo: MaskLo);
3343 }
3344
3345 if (CanCopyHi32) {
3346 // If all the bits in the high half are 1, we only need a copy for it.
3347 MaskedHi = HiReg;
3348 } else {
3349 Register MaskHi = MRI->createVirtualRegister(RegClass: &RegRC);
3350 MaskedHi = MRI->createVirtualRegister(RegClass: &RegRC);
3351
3352 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskHi)
3353 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub1);
3354 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedHi)
3355 .addReg(RegNo: HiReg)
3356 .addReg(RegNo: MaskHi);
3357 }
3358
3359 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
3360 .addReg(RegNo: MaskedLo)
3361 .addImm(Val: AMDGPU::sub0)
3362 .addReg(RegNo: MaskedHi)
3363 .addImm(Val: AMDGPU::sub1);
3364 I.eraseFromParent();
3365 return true;
3366}
3367
3368/// Return the register to use for the index value, and the subregister to use
3369/// for the indirectly accessed register.
3370static std::pair<Register, unsigned>
3371computeIndirectRegIndex(MachineRegisterInfo &MRI, const SIRegisterInfo &TRI,
3372 const TargetRegisterClass *SuperRC, Register IdxReg,
3373 unsigned EltSize, GISelValueTracking &ValueTracking) {
3374 Register IdxBaseReg;
3375 int Offset;
3376
3377 std::tie(args&: IdxBaseReg, args&: Offset) =
3378 AMDGPU::getBaseWithConstantOffset(MRI, Reg: IdxReg, ValueTracking: &ValueTracking);
3379 if (IdxBaseReg == AMDGPU::NoRegister) {
3380 // This will happen if the index is a known constant. This should ordinarily
3381 // be legalized out, but handle it as a register just in case.
3382 assert(Offset == 0);
3383 IdxBaseReg = IdxReg;
3384 }
3385
3386 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SuperRC, EltSize);
3387
3388 // Skip out of bounds offsets, or else we would end up using an undefined
3389 // register.
3390 if (static_cast<unsigned>(Offset) >= SubRegs.size())
3391 return std::pair(IdxReg, SubRegs[0]);
3392 return std::pair(IdxBaseReg, SubRegs[Offset]);
3393}
3394
3395bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
3396 MachineInstr &MI) const {
3397 Register DstReg = MI.getOperand(i: 0).getReg();
3398 Register SrcReg = MI.getOperand(i: 1).getReg();
3399 Register IdxReg = MI.getOperand(i: 2).getReg();
3400
3401 LLT DstTy = MRI->getType(Reg: DstReg);
3402 LLT SrcTy = MRI->getType(Reg: SrcReg);
3403
3404 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3405 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3406 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3407
3408 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3409 // into a waterfall loop.
3410 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3411 return false;
3412
3413 const TargetRegisterClass *SrcRC =
3414 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcRB);
3415 const TargetRegisterClass *DstRC =
3416 TRI.getRegClassForTypeOnBank(Ty: DstTy, Bank: *DstRB);
3417 if (!SrcRC || !DstRC)
3418 return false;
3419 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3420 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3421 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3422 return false;
3423
3424 MachineBasicBlock *BB = MI.getParent();
3425 const DebugLoc &DL = MI.getDebugLoc();
3426 const bool Is64 = DstTy.getSizeInBits() == 64;
3427
3428 unsigned SubReg;
3429 std::tie(args&: IdxReg, args&: SubReg) = computeIndirectRegIndex(
3430 MRI&: *MRI, TRI, SuperRC: SrcRC, IdxReg,
3431 EltSize: static_cast<unsigned>(DstTy.getSizeInBits() / 8), ValueTracking&: *VT);
3432
3433 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) {
3434 if (DstTy.getSizeInBits() != 32 && !Is64)
3435 return false;
3436
3437 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3438 .addReg(RegNo: IdxReg);
3439
3440 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32;
3441 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
3442 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3443 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3444 MI.eraseFromParent();
3445 return true;
3446 }
3447
3448 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32)
3449 return false;
3450
3451 if (!STI.useVGPRIndexMode()) {
3452 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3453 .addReg(RegNo: IdxReg);
3454 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: DstReg)
3455 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3456 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3457 MI.eraseFromParent();
3458 return true;
3459 }
3460
3461 const MCInstrDesc &GPRIDXDesc = TII.getIndirectGPRIDXPseudo(
3462 VecSize: static_cast<unsigned>(TRI.getRegSizeInBits(RC: *SrcRC)), IsIndirectSrc: true);
3463 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3464 .addReg(RegNo: SrcReg)
3465 .addReg(RegNo: IdxReg)
3466 .addImm(Val: SubReg);
3467
3468 MI.eraseFromParent();
3469 return true;
3470}
3471
3472// TODO: Fold insert_vector_elt (extract_vector_elt) into movrelsd
3473bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT(
3474 MachineInstr &MI) const {
3475 Register DstReg = MI.getOperand(i: 0).getReg();
3476 Register VecReg = MI.getOperand(i: 1).getReg();
3477 Register ValReg = MI.getOperand(i: 2).getReg();
3478 Register IdxReg = MI.getOperand(i: 3).getReg();
3479
3480 LLT VecTy = MRI->getType(Reg: DstReg);
3481 LLT ValTy = MRI->getType(Reg: ValReg);
3482 unsigned VecSize = static_cast<unsigned>(VecTy.getSizeInBits());
3483 unsigned ValSize = static_cast<unsigned>(ValTy.getSizeInBits());
3484
3485 const RegisterBank *VecRB = RBI.getRegBank(Reg: VecReg, MRI: *MRI, TRI);
3486 const RegisterBank *ValRB = RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
3487 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3488
3489 assert(VecTy.getElementType() == ValTy);
3490
3491 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3492 // into a waterfall loop.
3493 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3494 return false;
3495
3496 const TargetRegisterClass *VecRC =
3497 TRI.getRegClassForTypeOnBank(Ty: VecTy, Bank: *VecRB);
3498 const TargetRegisterClass *ValRC =
3499 TRI.getRegClassForTypeOnBank(Ty: ValTy, Bank: *ValRB);
3500
3501 if (!RBI.constrainGenericRegister(Reg: VecReg, RC: *VecRC, MRI&: *MRI) ||
3502 !RBI.constrainGenericRegister(Reg: DstReg, RC: *VecRC, MRI&: *MRI) ||
3503 !RBI.constrainGenericRegister(Reg: ValReg, RC: *ValRC, MRI&: *MRI) ||
3504 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3505 return false;
3506
3507 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32)
3508 return false;
3509
3510 unsigned SubReg;
3511 std::tie(args&: IdxReg, args&: SubReg) =
3512 computeIndirectRegIndex(MRI&: *MRI, TRI, SuperRC: VecRC, IdxReg, EltSize: ValSize / 8, ValueTracking&: *VT);
3513
3514 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID &&
3515 STI.useVGPRIndexMode();
3516
3517 MachineBasicBlock *BB = MI.getParent();
3518 const DebugLoc &DL = MI.getDebugLoc();
3519
3520 if (!IndexMode) {
3521 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3522 .addReg(RegNo: IdxReg);
3523
3524 const MCInstrDesc &RegWriteOp = TII.getIndirectRegWriteMovRelPseudo(
3525 VecSize, EltSize: ValSize, IsSGPR: VecRB->getID() == AMDGPU::SGPRRegBankID);
3526 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: RegWriteOp, DestReg: DstReg)
3527 .addReg(RegNo: VecReg)
3528 .addReg(RegNo: ValReg)
3529 .addImm(Val: SubReg);
3530 MI.eraseFromParent();
3531 return true;
3532 }
3533
3534 const MCInstrDesc &GPRIDXDesc = TII.getIndirectGPRIDXPseudo(
3535 VecSize: static_cast<unsigned>(TRI.getRegSizeInBits(RC: *VecRC)), IsIndirectSrc: false);
3536 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3537 .addReg(RegNo: VecReg)
3538 .addReg(RegNo: ValReg)
3539 .addReg(RegNo: IdxReg)
3540 .addImm(Val: SubReg);
3541
3542 MI.eraseFromParent();
3543 return true;
3544}
3545
3546static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
3547 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
3548 AMDGPU::lookupBufferIntrinsic(Intr))
3549 return BufInfo->IsAsync;
3550 switch (Intr) {
3551 case Intrinsic::amdgcn_load_async_to_lds:
3552 case Intrinsic::amdgcn_global_load_async_lds:
3553 return true;
3554 }
3555 return false;
3556}
3557
3558bool AMDGPUInstructionSelector::selectBufferLoadLds(
3559 MachineInstr &MI, const AMDGPU::BufferIntrinsicInfo &Info) const {
3560 if (!Subtarget->hasVMemToLDSLoad())
3561 return false;
3562 auto GetArg = [&](unsigned Arg) -> MachineOperand & {
3563 return MI.getOperand(i: AMDGPU::getMIROperandIdx(MI, Arg));
3564 };
3565 unsigned Opc;
3566 unsigned Size = static_cast<unsigned>(GetArg(*Info.getSizeArg()).getImm());
3567
3568 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
3569 const bool HasVIndex = VIndexArg.has_value();
3570 Register VIndex;
3571 if (HasVIndex)
3572 VIndex = GetArg(*VIndexArg).getReg();
3573
3574 Register VOffset = GetArg(*Info.getVOffsetArg()).getReg();
3575 std::optional<ValueAndVReg> MaybeVOffset =
3576 getIConstantVRegValWithLookThrough(VReg: VOffset, MRI: *MRI);
3577 const bool HasVOffset = !MaybeVOffset || MaybeVOffset->Value.getZExtValue();
3578
3579 switch (Size) {
3580 default:
3581 return false;
3582 case 1:
3583 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
3584 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
3585 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
3586 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
3587 break;
3588 case 2:
3589 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
3590 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
3591 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
3592 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
3593 break;
3594 case 4:
3595 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
3596 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
3597 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
3598 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
3599 break;
3600 case 12:
3601 if (!Subtarget->hasLDSLoadB96_B128())
3602 return false;
3603
3604 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
3605 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
3606 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
3607 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
3608 break;
3609 case 16:
3610 if (!Subtarget->hasLDSLoadB96_B128())
3611 return false;
3612
3613 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
3614 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
3615 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
3616 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
3617 break;
3618 }
3619
3620 MachineBasicBlock *MBB = MI.getParent();
3621 const DebugLoc &DL = MI.getDebugLoc();
3622 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3623 .add(MO: GetArg(*Info.getLDSPtrArg()));
3624
3625 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc));
3626
3627 if (HasVIndex && HasVOffset) {
3628 Register IdxReg = MRI->createVirtualRegister(RegClass: TRI.getVGPR64Class());
3629 BuildMI(BB&: *MBB, I: &*MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: IdxReg)
3630 .addReg(RegNo: VIndex)
3631 .addImm(Val: AMDGPU::sub0)
3632 .addReg(RegNo: VOffset)
3633 .addImm(Val: AMDGPU::sub1);
3634
3635 MIB.addReg(RegNo: IdxReg);
3636 } else if (HasVIndex) {
3637 MIB.addReg(RegNo: VIndex);
3638 } else if (HasVOffset) {
3639 MIB.addReg(RegNo: VOffset);
3640 }
3641
3642 MIB.add(MO: GetArg(Info.RsrcArg));
3643 MIB.add(MO: GetArg(*Info.getSOffsetArg()));
3644 MIB.add(MO: GetArg(*Info.getImmOffsetArg()));
3645 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
3646 int64_t Aux = GetArg(*Info.getAuxArg()).getImm();
3647 MIB.addImm(Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL
3648 : AMDGPU::CPol::ALL_pregfx12)); // cpol
3649 MIB.addImm(
3650 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
3651 ? 1
3652 : 0); // swz
3653 MIB.addImm(Val: Info.IsAsync);
3654
3655 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3656 // Don't set the offset value here because the pointer points to the base of
3657 // the buffer.
3658 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3659
3660 MachinePointerInfo StorePtrI = LoadPtrI;
3661 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3662 AddressSpace: AMDGPUAS::BUFFER_RESOURCE));
3663 LoadPtrI.AddrSpace = AMDGPUAS::BUFFER_RESOURCE;
3664 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3665
3666 auto F = LoadMMO->getFlags() &
3667 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3668 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3669 Size, BaseAlignment: LoadMMO->getBaseAlign());
3670
3671 MachineMemOperand *StoreMMO =
3672 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3673 Size: sizeof(int32_t), BaseAlignment: LoadMMO->getBaseAlign());
3674
3675 MIB.setMemRefs({LoadMMO, StoreMMO});
3676
3677 MI.eraseFromParent();
3678 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3679 return true;
3680}
3681
3682/// Match a zero extend from a 32-bit value to 64-bits.
3683Register AMDGPUInstructionSelector::matchZeroExtendFromS32(Register Reg) const {
3684 Register ZExtSrc;
3685 if (mi_match(R: Reg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrc))))
3686 return MRI->getType(Reg: ZExtSrc) == LLT::scalar(SizeInBits: 32) ? ZExtSrc : Register();
3687
3688 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 0)
3689 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3690 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3691 return Register();
3692
3693 assert(Def->getNumOperands() == 3 &&
3694 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3695 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_ZeroInt())) {
3696 return Def->getOperand(i: 1).getReg();
3697 }
3698
3699 return Register();
3700}
3701
3702/// Match a sign extend from a 32-bit value to 64-bits.
3703Register AMDGPUInstructionSelector::matchSignExtendFromS32(Register Reg) const {
3704 Register SExtSrc;
3705 if (mi_match(R: Reg, MRI: *MRI, P: m_GSExt(Src: m_Reg(R&: SExtSrc))))
3706 return MRI->getType(Reg: SExtSrc) == LLT::scalar(SizeInBits: 32) ? SExtSrc : Register();
3707
3708 // Match legalized form %sext = G_MERGE_VALUES (s32 %x), G_ASHR((S32 %x, 31))
3709 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3710 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3711 return Register();
3712
3713 assert(Def->getNumOperands() == 3 &&
3714 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3715 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI,
3716 P: m_GAShr(L: m_SpecificReg(RequestedReg: Def->getOperand(i: 1).getReg()),
3717 R: m_SpecificICst(RequestedValue: 31))))
3718 return Def->getOperand(i: 1).getReg();
3719
3720 Register ZextSrc = matchZeroExtendFromS32(Reg);
3721 if (ZextSrc && VT->signBitIsZero(Op: ZextSrc))
3722 return ZextSrc;
3723
3724 return Register();
3725}
3726
3727/// Match a zero extend from a 32-bit value to 64-bits, or \p Reg itself if it
3728/// is 32-bit.
3729Register
3730AMDGPUInstructionSelector::matchZeroExtendFromS32OrS32(Register Reg) const {
3731 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3732 : matchZeroExtendFromS32(Reg);
3733}
3734
3735/// Match a sign extend from a 32-bit value to 64-bits, or \p Reg itself if it
3736/// is 32-bit.
3737Register
3738AMDGPUInstructionSelector::matchSignExtendFromS32OrS32(Register Reg) const {
3739 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3740 : matchSignExtendFromS32(Reg);
3741}
3742
3743Register
3744AMDGPUInstructionSelector::matchExtendFromS32OrS32(Register Reg,
3745 bool IsSigned) const {
3746 if (IsSigned)
3747 return matchSignExtendFromS32OrS32(Reg);
3748
3749 return matchZeroExtendFromS32OrS32(Reg);
3750}
3751
3752Register AMDGPUInstructionSelector::matchAnyExtendFromS32(Register Reg) const {
3753 Register AnyExtSrc;
3754 if (mi_match(R: Reg, MRI: *MRI, P: m_GAnyExt(Src: m_Reg(R&: AnyExtSrc))))
3755 return MRI->getType(Reg: AnyExtSrc) == LLT::scalar(SizeInBits: 32) ? AnyExtSrc : Register();
3756
3757 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 G_IMPLICIT_DEF)
3758 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3759 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3760 return Register();
3761
3762 assert(Def->getNumOperands() == 3 &&
3763 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3764
3765 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_GImplicitDef()))
3766 return Def->getOperand(i: 1).getReg();
3767
3768 return Register();
3769}
3770
3771bool AMDGPUInstructionSelector::selectGlobalLoadLds(MachineInstr &MI) const{
3772 if (!Subtarget->hasVMemToLDSLoad())
3773 return false;
3774
3775 unsigned Opc;
3776 unsigned Size = static_cast<unsigned>(MI.getOperand(i: 3).getImm());
3777 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3778
3779 switch (Size) {
3780 default:
3781 return false;
3782 case 1:
3783 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
3784 break;
3785 case 2:
3786 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
3787 break;
3788 case 4:
3789 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
3790 break;
3791 case 12:
3792 if (!Subtarget->hasLDSLoadB96_B128())
3793 return false;
3794 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
3795 break;
3796 case 16:
3797 if (!Subtarget->hasLDSLoadB96_B128())
3798 return false;
3799 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
3800 break;
3801 }
3802
3803 MachineBasicBlock *MBB = MI.getParent();
3804 const DebugLoc &DL = MI.getDebugLoc();
3805 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3806 .add(MO: MI.getOperand(i: 2));
3807
3808 Register Addr = MI.getOperand(i: 1).getReg();
3809 Register VOffset;
3810 // Try to split SAddr and VOffset. Global and LDS pointers share the same
3811 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
3812 if (!isSGPR(Reg: Addr)) {
3813 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
3814 if (isSGPR(Reg: AddrDef->Reg)) {
3815 Addr = AddrDef->Reg;
3816 } else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
3817 Register SAddr =
3818 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
3819 if (isSGPR(Reg: SAddr)) {
3820 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
3821 if (Register Off = matchZeroExtendFromS32(Reg: PtrBaseOffset)) {
3822 Addr = SAddr;
3823 VOffset = Off;
3824 }
3825 }
3826 }
3827 }
3828
3829 if (isSGPR(Reg: Addr)) {
3830 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
3831 if (!VOffset) {
3832 VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
3833 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
3834 .addImm(Val: 0);
3835 }
3836 }
3837
3838 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc))
3839 .addReg(RegNo: Addr);
3840
3841 if (isSGPR(Reg: Addr))
3842 MIB.addReg(RegNo: VOffset);
3843
3844 MIB.add(MO: MI.getOperand(i: 4)); // offset
3845
3846 unsigned Aux = static_cast<unsigned>(MI.getOperand(i: 5).getImm());
3847 MIB.addImm(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS); // cpol
3848 MIB.addImm(Val: isAsyncLDSDMA(Intr: IntrinsicID));
3849
3850 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3851 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3852 LoadPtrI.Offset = MI.getOperand(i: 4).getImm();
3853 MachinePointerInfo StorePtrI = LoadPtrI;
3854 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3855 AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
3856 LoadPtrI.AddrSpace = AMDGPUAS::GLOBAL_ADDRESS;
3857 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3858 auto F = LoadMMO->getFlags() &
3859 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3860 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3861 Size, BaseAlignment: LoadMMO->getBaseAlign());
3862 MachineMemOperand *StoreMMO =
3863 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3864 Size: sizeof(int32_t), BaseAlignment: Align(4));
3865
3866 MIB.setMemRefs({LoadMMO, StoreMMO});
3867
3868 MI.eraseFromParent();
3869 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3870 return true;
3871}
3872
3873bool AMDGPUInstructionSelector::selectTensorLoadStore(MachineInstr &MI,
3874 Intrinsic::ID IID) const {
3875 bool IsLoad = IID == Intrinsic::amdgcn_tensor_load_to_lds;
3876 unsigned Opc =
3877 IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d4 : AMDGPU::TENSOR_STORE_FROM_LDS_d4;
3878 int NumGroups = 4;
3879
3880 // A lamda function to check whether an operand is a vector of all 0s.
3881 const auto isAllZeros = [&](MachineOperand &Opnd) {
3882 const MachineInstr *DefMI = MRI->getVRegDef(Reg: Opnd.getReg());
3883 if (!DefMI)
3884 return false;
3885 return llvm::isBuildVectorAllZeros(MI: *DefMI, MRI: *MRI, AllowUndef: true);
3886 };
3887
3888 // Use _D2 version if both group 2 and 3 are zero-initialized.
3889 if (isAllZeros(MI.getOperand(i: 3)) && isAllZeros(MI.getOperand(i: 4))) {
3890 NumGroups = 2;
3891 Opc = IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d2
3892 : AMDGPU::TENSOR_STORE_FROM_LDS_d2;
3893 }
3894
3895 // TODO: Handle the fifth group: MI.getOpetand(5), which is silently ignored
3896 // for now because all existing targets only support up to 4 groups.
3897 MachineBasicBlock *MBB = MI.getParent();
3898 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
3899 .add(MO: MI.getOperand(i: 1)) // D# group 0
3900 .add(MO: MI.getOperand(i: 2)); // D# group 1
3901
3902 if (NumGroups >= 4) { // Has at least 4 groups
3903 MIB.add(MO: MI.getOperand(i: 3)) // D# group 2
3904 .add(MO: MI.getOperand(i: 4)); // D# group 3
3905 }
3906
3907 MIB.addImm(Val: 0) // r128
3908 .add(MO: MI.getOperand(i: 6)); // cpol
3909
3910 MI.eraseFromParent();
3911 return true;
3912}
3913
3914bool AMDGPUInstructionSelector::selectBVHIntersectRayIntrinsic(
3915 MachineInstr &MI) const {
3916 unsigned OpcodeOpIdx =
3917 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY ? 1 : 3;
3918 MI.setDesc(
3919 TII.get(Opcode: static_cast<unsigned>(MI.getOperand(i: OpcodeOpIdx).getImm())));
3920 MI.removeOperand(OpNo: OpcodeOpIdx);
3921 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
3922 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
3923 return true;
3924}
3925
3926// FIXME: This should be removed and let the patterns select. We just need the
3927// AGPR/VGPR combination versions.
3928bool AMDGPUInstructionSelector::selectSMFMACIntrin(MachineInstr &MI) const {
3929 unsigned Opc;
3930 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
3931 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
3932 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64;
3933 break;
3934 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
3935 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64;
3936 break;
3937 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
3938 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64;
3939 break;
3940 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
3941 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64;
3942 break;
3943 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
3944 Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64;
3945 break;
3946 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
3947 Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64;
3948 break;
3949 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
3950 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64;
3951 break;
3952 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
3953 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64;
3954 break;
3955 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
3956 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64;
3957 break;
3958 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
3959 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64;
3960 break;
3961 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
3962 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64;
3963 break;
3964 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
3965 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64;
3966 break;
3967 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
3968 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64;
3969 break;
3970 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
3971 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64;
3972 break;
3973 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
3974 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_F16_e64;
3975 break;
3976 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
3977 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_F16_e64;
3978 break;
3979 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
3980 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF16_e64;
3981 break;
3982 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
3983 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF16_e64;
3984 break;
3985 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
3986 Opc = AMDGPU::V_SMFMAC_I32_16X16X128_I8_e64;
3987 break;
3988 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
3989 Opc = AMDGPU::V_SMFMAC_I32_32X32X64_I8_e64;
3990 break;
3991 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
3992 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_BF8_e64;
3993 break;
3994 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
3995 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_FP8_e64;
3996 break;
3997 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
3998 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_BF8_e64;
3999 break;
4000 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
4001 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_FP8_e64;
4002 break;
4003 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
4004 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_BF8_e64;
4005 break;
4006 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
4007 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_FP8_e64;
4008 break;
4009 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
4010 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_BF8_e64;
4011 break;
4012 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
4013 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_FP8_e64;
4014 break;
4015 default:
4016 llvm_unreachable("unhandled smfmac intrinsic");
4017 }
4018
4019 auto VDst_In = MI.getOperand(i: 4);
4020
4021 MI.setDesc(TII.get(Opcode: Opc));
4022 MI.removeOperand(OpNo: 4); // VDst_In
4023 MI.removeOperand(OpNo: 1); // Intrinsic ID
4024 MI.addOperand(Op: VDst_In); // Readd VDst_In to the end
4025 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
4026 const MCInstrDesc &MCID = MI.getDesc();
4027 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
4028 MI.getOperand(i: 0).setIsEarlyClobber(true);
4029 }
4030 return true;
4031}
4032
4033bool AMDGPUInstructionSelector::selectPermlaneSwapIntrin(
4034 MachineInstr &MI, Intrinsic::ID IntrID) const {
4035 if (IntrID == Intrinsic::amdgcn_permlane16_swap &&
4036 !Subtarget->hasPermlane16Swap())
4037 return false;
4038 if (IntrID == Intrinsic::amdgcn_permlane32_swap &&
4039 !Subtarget->hasPermlane32Swap())
4040 return false;
4041
4042 unsigned Opcode = IntrID == Intrinsic::amdgcn_permlane16_swap
4043 ? AMDGPU::V_PERMLANE16_SWAP_B32_e64
4044 : AMDGPU::V_PERMLANE32_SWAP_B32_e64;
4045
4046 MI.removeOperand(OpNo: 2);
4047 MI.setDesc(TII.get(Opcode));
4048 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
4049
4050 MachineOperand &FI = MI.getOperand(i: 4);
4051 FI.setImm(FI.getImm() ? AMDGPU::DPP::DPP_FI_1 : AMDGPU::DPP::DPP_FI_0);
4052
4053 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
4054 return true;
4055}
4056
4057bool AMDGPUInstructionSelector::selectWaveAddress(MachineInstr &MI) const {
4058 Register DstReg = MI.getOperand(i: 0).getReg();
4059 Register SrcReg = MI.getOperand(i: 1).getReg();
4060 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4061 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4062 MachineBasicBlock *MBB = MI.getParent();
4063 const DebugLoc &DL = MI.getDebugLoc();
4064
4065 if (IsVALU) {
4066 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: DstReg)
4067 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4068 .addReg(RegNo: SrcReg);
4069 } else {
4070 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: DstReg)
4071 .addReg(RegNo: SrcReg)
4072 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4073 .setOperandDead(3); // Dead scc
4074 }
4075
4076 const TargetRegisterClass &RC =
4077 IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
4078 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
4079 return false;
4080
4081 MI.eraseFromParent();
4082 return true;
4083}
4084
4085bool AMDGPUInstructionSelector::selectWaveShuffleIntrin(
4086 MachineInstr &MI) const {
4087 assert(MI.getNumOperands() == 4);
4088 MachineBasicBlock *MBB = MI.getParent();
4089 const DebugLoc &DL = MI.getDebugLoc();
4090
4091 Register DstReg = MI.getOperand(i: 0).getReg();
4092 Register ValReg = MI.getOperand(i: 2).getReg();
4093 Register IdxReg = MI.getOperand(i: 3).getReg();
4094
4095 const LLT DstTy = MRI->getType(Reg: DstReg);
4096 uint64_t DstSize = DstTy.getSizeInBits();
4097 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4098 const TargetRegisterClass *DstRC =
4099 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
4100
4101 if (DstTy != LLT::scalar(SizeInBits: 32))
4102 return false;
4103
4104 if (!Subtarget->supportsBPermute())
4105 return false;
4106
4107 // If we can bpermute across the whole wave, then just do that
4108 if (Subtarget->supportsWaveWideBPermute()) {
4109 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4110 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4111 .addImm(Val: 2)
4112 .addReg(RegNo: IdxReg);
4113
4114 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: DstReg)
4115 .addReg(RegNo: ShiftIdxReg)
4116 .addReg(RegNo: ValReg)
4117 .addImm(Val: 0);
4118 } else {
4119 // Otherwise, we need to make use of whole wave mode
4120 assert(Subtarget->isWave64());
4121
4122 // Set inactive lanes to poison
4123 Register UndefValReg =
4124 MRI->createVirtualRegister(RegClass: TRI.getRegClass(i: AMDGPU::SReg_32RegClassID));
4125 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefValReg);
4126
4127 Register UndefExecReg = MRI->createVirtualRegister(
4128 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4129 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExecReg);
4130
4131 Register PoisonValReg = MRI->createVirtualRegister(RegClass: DstRC);
4132 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonValReg)
4133 .addImm(Val: 0)
4134 .addReg(RegNo: ValReg)
4135 .addImm(Val: 0)
4136 .addReg(RegNo: UndefValReg)
4137 .addReg(RegNo: UndefExecReg);
4138
4139 // ds_bpermute requires index to be multiplied by 4
4140 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4141 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4142 .addImm(Val: 2)
4143 .addReg(RegNo: IdxReg);
4144
4145 Register PoisonIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4146 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonIdxReg)
4147 .addImm(Val: 0)
4148 .addReg(RegNo: ShiftIdxReg)
4149 .addImm(Val: 0)
4150 .addReg(RegNo: UndefValReg)
4151 .addReg(RegNo: UndefExecReg);
4152
4153 Register PoisonUnshiftedIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4154 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32),
4155 DestReg: PoisonUnshiftedIdxReg)
4156 .addImm(Val: 0)
4157 .addReg(RegNo: IdxReg)
4158 .addImm(Val: 0)
4159 .addReg(RegNo: UndefValReg)
4160 .addReg(RegNo: UndefExecReg);
4161
4162 // Get permutation of each half, then we'll select which one to use
4163 Register SameSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4164 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: SameSidePermReg)
4165 .addReg(RegNo: PoisonIdxReg)
4166 .addReg(RegNo: PoisonValReg)
4167 .addImm(Val: 0);
4168
4169 Register SwappedValReg = MRI->createVirtualRegister(RegClass: DstRC);
4170 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_PERMLANE64_B32), DestReg: SwappedValReg)
4171 .addReg(RegNo: PoisonValReg);
4172
4173 Register OppSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4174 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: OppSidePermReg)
4175 .addReg(RegNo: PoisonIdxReg)
4176 .addReg(RegNo: SwappedValReg)
4177 .addImm(Val: 0);
4178
4179 Register WWMSwapPermReg = MRI->createVirtualRegister(RegClass: DstRC);
4180 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::STRICT_WWM), DestReg: WWMSwapPermReg)
4181 .addReg(RegNo: OppSidePermReg);
4182
4183 // Select which side to take the permute from
4184 // We can get away with only using mbcnt_lo here since we're only
4185 // trying to detect which side of 32 each lane is on, and mbcnt_lo
4186 // returns 32 for lanes 32-63.
4187 Register ThreadIDReg = MRI->createVirtualRegister(RegClass: DstRC);
4188 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64), DestReg: ThreadIDReg)
4189 .addImm(Val: -1)
4190 .addImm(Val: 0);
4191
4192 Register XORReg = MRI->createVirtualRegister(RegClass: DstRC);
4193 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_XOR_B32_e64), DestReg: XORReg)
4194 .addReg(RegNo: ThreadIDReg)
4195 .addReg(RegNo: PoisonUnshiftedIdxReg);
4196
4197 Register ANDReg = MRI->createVirtualRegister(RegClass: DstRC);
4198 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: ANDReg)
4199 .addReg(RegNo: XORReg)
4200 .addImm(Val: 32);
4201
4202 Register CompareReg = MRI->createVirtualRegister(
4203 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4204 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CompareReg)
4205 .addReg(RegNo: ANDReg)
4206 .addImm(Val: 0);
4207
4208 // Finally do the selection
4209 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
4210 .addImm(Val: 0)
4211 .addReg(RegNo: WWMSwapPermReg)
4212 .addImm(Val: 0)
4213 .addReg(RegNo: SameSidePermReg)
4214 .addReg(RegNo: CompareReg);
4215 }
4216
4217 MI.eraseFromParent();
4218 return true;
4219}
4220
4221// Match BITOP3 operation and return a number of matched instructions plus
4222// truth table.
4223static std::pair<unsigned, uint8_t> BitOp3_Op(Register R,
4224 SmallVectorImpl<Register> &Src,
4225 const MachineRegisterInfo &MRI) {
4226 unsigned NumOpcodes = 0;
4227 uint8_t LHSBits, RHSBits;
4228
4229 auto getOperandBits = [&Src, R, &MRI](Register Op, uint8_t &Bits) -> bool {
4230 // Define truth table given Src0, Src1, Src2 bits permutations:
4231 // 0 0 0
4232 // 0 0 1
4233 // 0 1 0
4234 // 0 1 1
4235 // 1 0 0
4236 // 1 0 1
4237 // 1 1 0
4238 // 1 1 1
4239 const uint8_t SrcBits[3] = { 0xf0, 0xcc, 0xaa };
4240
4241 if (mi_match(R: Op, MRI, P: m_AllOnesInt())) {
4242 Bits = 0xff;
4243 return true;
4244 }
4245 if (mi_match(R: Op, MRI, P: m_ZeroInt())) {
4246 Bits = 0;
4247 return true;
4248 }
4249
4250 for (unsigned I = 0; I < Src.size(); ++I) {
4251 // Try to find existing reused operand
4252 if (Src[I] == Op) {
4253 Bits = SrcBits[I];
4254 return true;
4255 }
4256 // Try to replace parent operator
4257 if (Src[I] == R) {
4258 Bits = SrcBits[I];
4259 Src[I] = Op;
4260 return true;
4261 }
4262 }
4263
4264 if (Src.size() == 3) {
4265 // No room left for operands. Try one last time, there can be a 'not' of
4266 // one of our source operands. In this case we can compute the bits
4267 // without growing Src vector.
4268 Register LHS;
4269 if (mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: LHS)))) {
4270 LHS = getSrcRegIgnoringCopies(Reg: LHS, MRI);
4271 for (unsigned I = 0; I < Src.size(); ++I) {
4272 if (Src[I] == LHS) {
4273 Bits = ~SrcBits[I];
4274 return true;
4275 }
4276 }
4277 }
4278
4279 return false;
4280 }
4281
4282 Bits = SrcBits[Src.size()];
4283 Src.push_back(Elt: Op);
4284 return true;
4285 };
4286
4287 MachineInstr *MI = MRI.getVRegDef(Reg: R);
4288 switch (MI->getOpcode()) {
4289 case TargetOpcode::G_AND:
4290 case TargetOpcode::G_OR:
4291 case TargetOpcode::G_XOR: {
4292 Register LHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 1).getReg(), MRI);
4293 Register RHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 2).getReg(), MRI);
4294
4295 SmallVector<Register, 3> Backup(Src.begin(), Src.end());
4296 if (!getOperandBits(LHS, LHSBits) ||
4297 !getOperandBits(RHS, RHSBits)) {
4298 Src = std::move(Backup);
4299 return {};
4300 }
4301
4302 // Recursion is naturally limited by the size of the operand vector.
4303 //
4304 // When LHS and RHS share a common sub-expression, one side's recursion
4305 // may decompose that sub-expression and replace the Src slot the other
4306 // side occupies with sub-operands via the "replace parent" path in
4307 // getOperandBits. The other side's cached bit-pattern then refers to a
4308 // slot whose contents changed, producing a wrong truth table.
4309 //
4310 // We detect this in three ways:
4311 // (A) If LHS recursed, its truth table is valid against the Src state
4312 // when LHS recursion completed (SrcAfterLHS). If RHS recursion
4313 // then mutates a Src slot that LHSBits depends on, LHSBits is
4314 // stale.
4315 // (B) If RHS did not recurse, RHSBits came from getOperandBits and
4316 // refers to a specific Src slot. If that slot's contents changed
4317 // (by either recursion), RHSBits is stale.
4318 // (C) Symmetrically for LHS if it did not recurse.
4319 SmallVector<Register, 3> SrcBeforeRecurse(Src.begin(), Src.end());
4320 uint8_t LHSBitsOrig = LHSBits;
4321 uint8_t RHSBitsOrig = RHSBits;
4322
4323 auto LHSOp = BitOp3_Op(R: LHS, Src, MRI);
4324 if (LHSOp.first) {
4325 NumOpcodes += LHSOp.first;
4326 LHSBits = LHSOp.second;
4327 }
4328
4329 SmallVector<Register, 3> SrcAfterLHS(Src.begin(), Src.end());
4330
4331 auto RHSOp = BitOp3_Op(R: RHS, Src, MRI);
4332 if (RHSOp.first) {
4333 NumOpcodes += RHSOp.first;
4334 RHSBits = RHSOp.second;
4335 }
4336
4337 // dependsOnSlot: true iff the truth table TT varies with slot Slot.
4338 auto dependsOnSlot = [](uint8_t TT, int Slot) -> bool {
4339 if (Slot < 0 || Slot > 2)
4340 return false;
4341 const uint8_t Masks[3] = {0x0f, 0x33, 0x55};
4342 const int Shifts[3] = {4, 2, 1};
4343 return ((TT ^ (TT >> Shifts[Slot])) & Masks[Slot]) != 0;
4344 };
4345
4346 // findSlot: locate the Src slot a getOperandBits result depends on,
4347 // including negated (NOT) patterns that getOperandBits resolves via
4348 // the ~SrcBits[I] shortcut.
4349 const uint8_t SrcBitsConst[3] = {0xf0, 0xcc, 0xaa};
4350 auto findSlot = [&](uint8_t Bits, Register Op,
4351 const SmallVectorImpl<Register> &S) -> int {
4352 Register NegatedInner;
4353 bool IsNegationOp = mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: NegatedInner)));
4354 if (IsNegationOp)
4355 NegatedInner = getSrcRegIgnoringCopies(Reg: NegatedInner, MRI);
4356 for (int I = 0; I < (int)S.size(); I++) {
4357 if (Bits == SrcBitsConst[I] && S[I] == Op)
4358 return I;
4359 if (IsNegationOp && Bits == (uint8_t)~SrcBitsConst[I] &&
4360 S[I] == NegatedInner)
4361 return I;
4362 }
4363 return -1;
4364 };
4365
4366 bool Stale = false;
4367
4368 // (A) LHS recursed: its truth table is against SrcAfterLHS.
4369 // Check if RHS recursion mutated a slot that LHSBits uses.
4370 if (LHSOp.first) {
4371 for (int I = 0; I < (int)SrcAfterLHS.size() && I < 3; I++) {
4372 if (I < (int)Src.size() && Src[I] != SrcAfterLHS[I] &&
4373 dependsOnSlot(LHSBits, I)) {
4374 Stale = true;
4375 break;
4376 }
4377 }
4378 }
4379
4380 // (B) RHS did not recurse: RHSBits from getOperandBits is against
4381 // SrcBeforeRecurse. Check if that slot was mutated since then.
4382 if (!Stale && !RHSOp.first) {
4383 int Slot = findSlot(RHSBitsOrig, RHS, SrcBeforeRecurse);
4384 if (Slot >= 0 &&
4385 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4386 Stale = true;
4387 }
4388
4389 // (C) LHS did not recurse: LHSBits from getOperandBits is against
4390 // SrcBeforeRecurse. Check if that slot was mutated since then.
4391 if (!Stale && !LHSOp.first) {
4392 int Slot = findSlot(LHSBitsOrig, LHS, SrcBeforeRecurse);
4393 if (Slot >= 0 &&
4394 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4395 Stale = true;
4396 }
4397
4398 if (Stale) {
4399 Src = std::move(SrcBeforeRecurse);
4400 LHSBits = LHSBitsOrig;
4401 RHSBits = RHSBitsOrig;
4402 NumOpcodes = 0;
4403 }
4404 break;
4405 }
4406 default:
4407 return {};
4408 }
4409
4410 uint8_t TTbl;
4411 switch (MI->getOpcode()) {
4412 case TargetOpcode::G_AND:
4413 TTbl = LHSBits & RHSBits;
4414 break;
4415 case TargetOpcode::G_OR:
4416 TTbl = LHSBits | RHSBits;
4417 break;
4418 case TargetOpcode::G_XOR:
4419 TTbl = LHSBits ^ RHSBits;
4420 break;
4421 default:
4422 break;
4423 }
4424
4425 return std::make_pair(x: NumOpcodes + 1, y&: TTbl);
4426}
4427
4428bool AMDGPUInstructionSelector::selectBITOP3(MachineInstr &MI) const {
4429 if (!Subtarget->hasBitOp3Insts())
4430 return false;
4431
4432 Register DstReg = MI.getOperand(i: 0).getReg();
4433 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4434 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4435 if (!IsVALU)
4436 return false;
4437
4438 SmallVector<Register, 3> Src;
4439 uint8_t TTbl;
4440 unsigned NumOpcodes;
4441
4442 std::tie(args&: NumOpcodes, args&: TTbl) = BitOp3_Op(R: DstReg, Src, MRI: *MRI);
4443
4444 // Src.empty() case can happen if all operands are all zero or all ones.
4445 // Normally it shall be optimized out before reaching this.
4446 if (NumOpcodes < 2 || Src.empty())
4447 return false;
4448
4449 // RegBankSelect splits wider VALU logic ops and widens 1-bit ones, so only
4450 // 16 and 32 bit types reach here. Note that <2 x i16> is 32 bits wide.
4451 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
4452 assert((Size == 16 || Size == 32) && "unexpected VALU logic op size");
4453 const bool IsB32 = Size == 32;
4454 if (NumOpcodes == 2 && IsB32) {
4455 // Avoid using BITOP3 for OR3, XOR3, AND_OR. This is not faster but makes
4456 // asm more readable. This cannot be modeled with AddedComplexity because
4457 // selector does not know how many operations did we match.
4458 if (mi_match(MI, MRI: *MRI, P: m_GXor(L: m_GXor(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4459 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GOr(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4460 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GAnd(L: m_Reg(), R: m_Reg()), R: m_Reg())))
4461 return false;
4462 } else if (NumOpcodes < 4) {
4463 // For a uniform case threshold should be higher to account for moves
4464 // between VGPRs and SGPRs. It needs one operand in a VGPR, rest two can be
4465 // in SGPRs and a readtfirstlane after.
4466 return false;
4467 }
4468
4469 unsigned Opc = IsB32 ? AMDGPU::V_BITOP3_B32_e64 : AMDGPU::V_BITOP3_B16_e64;
4470 if (!IsB32 && STI.hasTrue16BitInsts())
4471 Opc = STI.useRealTrue16Insts() ? AMDGPU::V_BITOP3_B16_gfx1250_t16_e64
4472 : AMDGPU::V_BITOP3_B16_gfx1250_fake16_e64;
4473 unsigned CBL = STI.getConstantBusLimit(Opcode: Opc);
4474 MachineBasicBlock *MBB = MI.getParent();
4475 const DebugLoc &DL = MI.getDebugLoc();
4476
4477 for (unsigned I = 0; I < Src.size(); ++I) {
4478 const RegisterBank *RB = RBI.getRegBank(Reg: Src[I], MRI: *MRI, TRI);
4479 if (RB->getID() != AMDGPU::SGPRRegBankID)
4480 continue;
4481 if (CBL > 0) {
4482 --CBL;
4483 continue;
4484 }
4485 Register NewReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
4486 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: NewReg)
4487 .addReg(RegNo: Src[I]);
4488 Src[I] = NewReg;
4489 }
4490
4491 // Last operand can be ignored, turning a ternary operation into a binary.
4492 // For example: (~a & b & c) | (~a & b & ~c) -> (~a & b). We can replace
4493 // 'c' with 'a' here without changing the answer. In some pathological
4494 // cases it should be possible to get an operation with a single operand
4495 // too if optimizer would not catch it.
4496 while (Src.size() < 3)
4497 Src.push_back(Elt: Src[0]);
4498
4499 auto MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg);
4500 if (!IsB32)
4501 MIB.addImm(Val: 0); // src_mod0
4502 MIB.addReg(RegNo: Src[0]);
4503 if (!IsB32)
4504 MIB.addImm(Val: 0); // src_mod1
4505 MIB.addReg(RegNo: Src[1]);
4506 if (!IsB32)
4507 MIB.addImm(Val: 0); // src_mod2
4508 MIB.addReg(RegNo: Src[2])
4509 .addImm(Val: TTbl);
4510 if (!IsB32)
4511 MIB.addImm(Val: 0); // op_sel
4512
4513 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
4514 MI.eraseFromParent();
4515
4516 return true;
4517}
4518
4519bool AMDGPUInstructionSelector::selectWriteRegister(MachineInstr &MI) const {
4520 const MDString *RegStr =
4521 cast<MDString>(Val: MI.getOperand(i: 0).getMetadata()->getOperand(I: 0));
4522 Register SrcReg = MI.getOperand(i: 1).getReg();
4523 LLT Ty = MRI->getType(Reg: SrcReg);
4524
4525 Register PhysReg = Subtarget->getTargetLowering()->getRegisterByName(
4526 RegName: RegStr->getString().data(), VT: Ty, MF: *MF);
4527 if (!PhysReg) {
4528 const Function &Fn = MF->getFunction();
4529 Fn.getContext().diagnose(DI: DiagnosticInfoGenericWithLoc(
4530 "invalid register \"" + Twine(RegStr->getString()) +
4531 "\" for llvm.write_register",
4532 Fn, MI.getDebugLoc()));
4533 MI.eraseFromParent();
4534 return true;
4535 }
4536
4537 if (!RBI.constrainGenericRegister(
4538 Reg: SrcReg, RC: *TRI.getSGPRClassForBitWidth(BitWidth: Ty.getSizeInBits()), MRI&: *MRI))
4539 return false;
4540
4541 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: PhysReg)
4542 .addReg(RegNo: SrcReg);
4543 MI.eraseFromParent();
4544 return true;
4545}
4546
4547bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const {
4548 Register SrcReg = MI.getOperand(i: 0).getReg();
4549 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
4550 return false;
4551
4552 MachineInstr *DefMI = MRI->getVRegDef(Reg: SrcReg);
4553 Register SP =
4554 Subtarget->getTargetLowering()->getStackPointerRegisterToSaveRestore();
4555 Register WaveAddr = getWaveAddress(Def: DefMI);
4556 MachineBasicBlock *MBB = MI.getParent();
4557 const DebugLoc &DL = MI.getDebugLoc();
4558
4559 if (!WaveAddr) {
4560 WaveAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
4561 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: WaveAddr)
4562 .addReg(RegNo: SrcReg)
4563 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4564 .setOperandDead(3); // Dead scc
4565 }
4566
4567 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: SP)
4568 .addReg(RegNo: WaveAddr);
4569
4570 MI.eraseFromParent();
4571 return true;
4572}
4573
4574bool AMDGPUInstructionSelector::select(MachineInstr &I) {
4575
4576 if (!I.isPreISelOpcode()) {
4577 if (I.isCopy())
4578 return selectCOPY(I);
4579 return true;
4580 }
4581
4582 switch (I.getOpcode()) {
4583 case TargetOpcode::G_AND:
4584 case TargetOpcode::G_OR:
4585 case TargetOpcode::G_XOR:
4586 if (selectBITOP3(MI&: I))
4587 return true;
4588 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4589 return true;
4590 return selectG_AND_OR_XOR(I);
4591 case TargetOpcode::G_ADD:
4592 case TargetOpcode::G_SUB:
4593 case TargetOpcode::G_PTR_ADD:
4594 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4595 return true;
4596 return selectG_ADD_SUB(I);
4597 case TargetOpcode::G_UADDO:
4598 case TargetOpcode::G_USUBO:
4599 case TargetOpcode::G_UADDE:
4600 case TargetOpcode::G_USUBE:
4601 return selectG_UADDO_USUBO_UADDE_USUBE(I);
4602 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4603 case AMDGPU::G_AMDGPU_MAD_I64_I32:
4604 return selectG_AMDGPU_MAD_64_32(I);
4605 case TargetOpcode::G_INTTOPTR:
4606 case TargetOpcode::G_BITCAST:
4607 case TargetOpcode::G_PTRTOINT:
4608 case TargetOpcode::G_FREEZE:
4609 return selectCOPY(I);
4610 case TargetOpcode::G_FNEG:
4611 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4612 return true;
4613 return selectG_FNEG(MI&: I);
4614 case TargetOpcode::G_FABS:
4615 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4616 return true;
4617 return selectG_FABS(MI&: I);
4618 case TargetOpcode::G_EXTRACT:
4619 return selectG_EXTRACT(I);
4620 case TargetOpcode::G_MERGE_VALUES:
4621 case TargetOpcode::G_CONCAT_VECTORS:
4622 return selectG_MERGE_VALUES(MI&: I);
4623 case TargetOpcode::G_UNMERGE_VALUES:
4624 return selectG_UNMERGE_VALUES(MI&: I);
4625 case TargetOpcode::G_BUILD_VECTOR:
4626 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
4627 return selectG_BUILD_VECTOR(MI&: I);
4628 case TargetOpcode::G_IMPLICIT_DEF:
4629 return selectG_IMPLICIT_DEF(I);
4630 case TargetOpcode::G_INSERT:
4631 return selectG_INSERT(I);
4632 case TargetOpcode::G_INTRINSIC:
4633 case TargetOpcode::G_INTRINSIC_CONVERGENT:
4634 return selectG_INTRINSIC(I);
4635 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
4636 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
4637 return selectG_INTRINSIC_W_SIDE_EFFECTS(I);
4638 case TargetOpcode::G_ICMP:
4639 case TargetOpcode::G_FCMP:
4640 if (selectG_ICMP_or_FCMP(I))
4641 return true;
4642 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4643 case TargetOpcode::G_LOAD:
4644 case TargetOpcode::G_ZEXTLOAD:
4645 case TargetOpcode::G_SEXTLOAD:
4646 case TargetOpcode::G_STORE:
4647 case TargetOpcode::G_ATOMIC_CMPXCHG:
4648 case TargetOpcode::G_ATOMICRMW_XCHG:
4649 case TargetOpcode::G_ATOMICRMW_ADD:
4650 case TargetOpcode::G_ATOMICRMW_SUB:
4651 case TargetOpcode::G_ATOMICRMW_AND:
4652 case TargetOpcode::G_ATOMICRMW_OR:
4653 case TargetOpcode::G_ATOMICRMW_XOR:
4654 case TargetOpcode::G_ATOMICRMW_MIN:
4655 case TargetOpcode::G_ATOMICRMW_MAX:
4656 case TargetOpcode::G_ATOMICRMW_UMIN:
4657 case TargetOpcode::G_ATOMICRMW_UMAX:
4658 case TargetOpcode::G_ATOMICRMW_UINC_WRAP:
4659 case TargetOpcode::G_ATOMICRMW_UDEC_WRAP:
4660 case TargetOpcode::G_ATOMICRMW_USUB_COND:
4661 case TargetOpcode::G_ATOMICRMW_USUB_SAT:
4662 case TargetOpcode::G_ATOMICRMW_FADD:
4663 case TargetOpcode::G_ATOMICRMW_FMIN:
4664 case TargetOpcode::G_ATOMICRMW_FMAX:
4665 return selectG_LOAD_STORE_ATOMICRMW(I);
4666 case TargetOpcode::G_SELECT:
4667 return selectG_SELECT(I);
4668 case TargetOpcode::G_TRUNC:
4669 return selectG_TRUNC(I);
4670 case TargetOpcode::G_SEXT:
4671 case TargetOpcode::G_ZEXT:
4672 case TargetOpcode::G_ANYEXT:
4673 case TargetOpcode::G_SEXT_INREG:
4674 // This is a workaround. For extension from type i1, `selectImpl()` uses
4675 // patterns from TD file and generates an illegal VGPR to SGPR COPY as type
4676 // i1 can only be hold in a SGPR class.
4677 if (MRI->getType(Reg: I.getOperand(i: 1).getReg()) != LLT::scalar(SizeInBits: 1) &&
4678 selectImpl(I, CoverageInfo&: *CoverageInfo))
4679 return true;
4680 return selectG_SZA_EXT(I);
4681 case TargetOpcode::G_FPEXT:
4682 if (selectG_FPEXT(I))
4683 return true;
4684 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4685 case TargetOpcode::G_BRCOND:
4686 return selectG_BRCOND(I);
4687 case TargetOpcode::G_GLOBAL_VALUE:
4688 return selectG_GLOBAL_VALUE(I);
4689 case TargetOpcode::G_PTRMASK:
4690 return selectG_PTRMASK(I);
4691 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4692 return selectG_EXTRACT_VECTOR_ELT(MI&: I);
4693 case TargetOpcode::G_INSERT_VECTOR_ELT:
4694 return selectG_INSERT_VECTOR_ELT(MI&: I);
4695 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
4696 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
4697 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
4698 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
4699 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
4700 const AMDGPU::ImageDimIntrinsicInfo *Intr =
4701 AMDGPU::getImageDimIntrinsicInfo(Intr: AMDGPU::getIntrinsicID(I));
4702 assert(Intr && "not an image intrinsic with image pseudo");
4703 return selectImageIntrinsic(MI&: I, Intr);
4704 }
4705 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY:
4706 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
4707 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
4708 return selectBVHIntersectRayIntrinsic(MI&: I);
4709 case AMDGPU::G_SBFX:
4710 case AMDGPU::G_UBFX:
4711 return selectG_SBFX_UBFX(MI&: I);
4712 case AMDGPU::G_SI_CALL:
4713 I.setDesc(TII.get(Opcode: AMDGPU::SI_CALL));
4714 return true;
4715 case AMDGPU::G_AMDGPU_WAVE_ADDRESS:
4716 return selectWaveAddress(MI&: I);
4717 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN: {
4718 I.setDesc(TII.get(Opcode: AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN));
4719 return true;
4720 }
4721 case AMDGPU::G_STACKRESTORE:
4722 return selectStackRestore(MI&: I);
4723 case TargetOpcode::G_WRITE_REGISTER:
4724 return selectWriteRegister(MI&: I);
4725 case AMDGPU::G_PHI:
4726 return selectPHI(I);
4727 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
4728 return selectCOPY_SCC_VCC(I);
4729 case AMDGPU::G_AMDGPU_COPY_VCC_SCC:
4730 return selectCOPY_VCC_SCC(I);
4731 case AMDGPU::G_AMDGPU_READANYLANE:
4732 return selectReadAnyLane(I);
4733 case TargetOpcode::G_CONSTANT:
4734 case TargetOpcode::G_FCONSTANT:
4735 default:
4736 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4737 }
4738 return false;
4739}
4740
4741InstructionSelector::ComplexRendererFns
4742AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const {
4743 return {{
4744 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4745 }};
4746
4747}
4748
4749std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3ModsImpl(
4750 Register Src, bool IsCanonicalizing, bool AllowAbs, bool OpSel) const {
4751 unsigned Mods = 0;
4752 MachineInstr *MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4753
4754 if (MI->getOpcode() == AMDGPU::G_FNEG) {
4755 Src = MI->getOperand(i: 1).getReg();
4756 Mods |= SISrcMods::NEG;
4757 MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4758 } else if (MI->getOpcode() == AMDGPU::G_FSUB && IsCanonicalizing) {
4759 // Fold fsub [+-]0 into fneg. This may not have folded depending on the
4760 // denormal mode, but we're implicitly canonicalizing in a source operand.
4761 const ConstantFP *LHS =
4762 getConstantFPVRegVal(VReg: MI->getOperand(i: 1).getReg(), MRI: *MRI);
4763 if (LHS && LHS->isZero()) {
4764 Mods |= SISrcMods::NEG;
4765 Src = MI->getOperand(i: 2).getReg();
4766 }
4767 }
4768
4769 if (AllowAbs && MI->getOpcode() == AMDGPU::G_FABS) {
4770 Src = MI->getOperand(i: 1).getReg();
4771 Mods |= SISrcMods::ABS;
4772 }
4773
4774 if (OpSel)
4775 Mods |= SISrcMods::OP_SEL_0;
4776
4777 return std::pair(Src, Mods);
4778}
4779
4780std::pair<Register, unsigned>
4781AMDGPUInstructionSelector::selectVOP3PModsF32Impl(Register Src) const {
4782 unsigned Mods;
4783 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src);
4784 Mods |= SISrcMods::OP_SEL_1;
4785 return std::pair(Src, Mods);
4786}
4787
4788Register AMDGPUInstructionSelector::copyToVGPRIfSrcFolded(
4789 Register Src, unsigned Mods, MachineOperand Root, MachineInstr *InsertPt,
4790 bool ForceVGPR) const {
4791 if ((Mods != 0 || ForceVGPR) &&
4792 RBI.getRegBank(Reg: Src, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) {
4793
4794 // If we looked through copies to find source modifiers on an SGPR operand,
4795 // we now have an SGPR register source. To avoid potentially violating the
4796 // constant bus restriction, we need to insert a copy to a VGPR.
4797 Register VGPRSrc = MRI->cloneVirtualRegister(VReg: Root.getReg());
4798 BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
4799 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VGPRSrc)
4800 .addReg(RegNo: Src);
4801 Src = VGPRSrc;
4802 }
4803
4804 return Src;
4805}
4806
4807/// Some instructions must have 32-bit sources. With real true16 instructions a
4808/// 16-bit VALU value lives in a VGPR_16, which they cannot read, so place it in
4809/// the low half of a new 32-bit VGPR.
4810Register
4811AMDGPUInstructionSelector::widenSrcIfVGPR16(Register Src,
4812 MachineInstr *InsertPt) const {
4813 if (!Subtarget->useRealTrue16Insts() || MRI->getType(Reg: Src) != LLT::scalar(SizeInBits: 16))
4814 return Src;
4815
4816 const RegisterBank *SrcRB = RBI.getRegBank(Reg: Src, MRI: *MRI, TRI);
4817 if (!SrcRB || SrcRB->getID() != AMDGPU::VGPRRegBankID)
4818 return Src;
4819
4820 MachineIRBuilder B(*InsertPt);
4821
4822 Register ImpDefReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
4823 B.buildInstr(Opcode: TargetOpcode::IMPLICIT_DEF).addDef(RegNo: ImpDefReg);
4824
4825 Register DstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
4826 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
4827 .addDef(RegNo: DstReg)
4828 .addReg(RegNo: Src)
4829 .addImm(Val: AMDGPU::lo16)
4830 .addReg(RegNo: ImpDefReg)
4831 .addImm(Val: AMDGPU::hi16);
4832
4833 return DstReg;
4834}
4835
4836///
4837/// This will select either an SGPR or VGPR operand and will save us from
4838/// having to write an extra tablegen pattern.
4839InstructionSelector::ComplexRendererFns
4840AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const {
4841 return {{
4842 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4843 }};
4844}
4845
4846InstructionSelector::ComplexRendererFns
4847AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const {
4848 Register Src;
4849 unsigned Mods;
4850 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4851
4852 return {{
4853 [=](MachineInstrBuilder &MIB) {
4854 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4855 },
4856 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4857 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4858 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4859 }};
4860}
4861
4862InstructionSelector::ComplexRendererFns
4863AMDGPUInstructionSelector::selectVOP3BMods0(MachineOperand &Root) const {
4864 Register Src;
4865 unsigned Mods;
4866 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
4867 /*IsCanonicalizing=*/true,
4868 /*AllowAbs=*/false);
4869
4870 return {{
4871 [=](MachineInstrBuilder &MIB) {
4872 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4873 },
4874 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4875 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4876 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4877 }};
4878}
4879
4880InstructionSelector::ComplexRendererFns
4881AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const {
4882 return {{
4883 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); },
4884 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4885 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4886 }};
4887}
4888
4889InstructionSelector::ComplexRendererFns
4890AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const {
4891 Register Src;
4892 unsigned Mods;
4893 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4894
4895 return {{
4896 [=](MachineInstrBuilder &MIB) {
4897 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4898 },
4899 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4900 }};
4901}
4902
4903InstructionSelector::ComplexRendererFns
4904AMDGPUInstructionSelector::selectVOP3ModsNonCanonicalizing(
4905 MachineOperand &Root) const {
4906 Register Src;
4907 unsigned Mods;
4908 std::tie(args&: Src, args&: Mods) =
4909 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/false);
4910
4911 return {{
4912 [=](MachineInstrBuilder &MIB) {
4913 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4914 },
4915 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4916 }};
4917}
4918
4919InstructionSelector::ComplexRendererFns
4920AMDGPUInstructionSelector::selectVOP3BMods(MachineOperand &Root) const {
4921 Register Src;
4922 unsigned Mods;
4923 std::tie(args&: Src, args&: Mods) =
4924 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/true,
4925 /*AllowAbs=*/false);
4926
4927 return {{
4928 [=](MachineInstrBuilder &MIB) {
4929 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4930 },
4931 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4932 }};
4933}
4934
4935InstructionSelector::ComplexRendererFns
4936AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
4937 Register Reg = Root.getReg();
4938 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
4939 if (Def->getOpcode() == AMDGPU::G_FNEG || Def->getOpcode() == AMDGPU::G_FABS)
4940 return {};
4941 return {{
4942 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
4943 }};
4944}
4945
4946InstructionSelector::ComplexRendererFns
4947AMDGPUInstructionSelector::selectVOP3NoOpSelMods(MachineOperand &Root) const {
4948 Register Unused;
4949 if (isExtractHiElt(MRI&: *MRI, In: Root.getReg(), Out&: Unused))
4950 return {};
4951 return selectVOP3NoMods(Root);
4952}
4953
4954enum class SrcStatus {
4955 IS_SAME,
4956 IS_UPPER_HALF,
4957 IS_LOWER_HALF,
4958 IS_UPPER_HALF_NEG,
4959 // This means current op = [op_upper, op_lower] and src = -op_lower.
4960 IS_LOWER_HALF_NEG,
4961 IS_HI_NEG,
4962 // This means current op = [op_upper, op_lower] and src = [op_upper,
4963 // -op_lower].
4964 IS_LO_NEG,
4965 IS_BOTH_NEG,
4966 INVALID,
4967 NEG_START = IS_UPPER_HALF_NEG,
4968 NEG_END = IS_BOTH_NEG,
4969 HALF_START = IS_UPPER_HALF,
4970 HALF_END = IS_LOWER_HALF_NEG
4971};
4972/// Test if the MI is truncating to half, such as `%reg0:n = G_TRUNC %reg1:2n`
4973static bool isTruncHalf(const MachineInstr *MI,
4974 const MachineRegisterInfo &MRI) {
4975 if (MI->getOpcode() != AMDGPU::G_TRUNC)
4976 return false;
4977
4978 uint64_t DstSize = MRI.getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
4979 uint64_t SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4980 return DstSize * 2 == SrcSize;
4981}
4982
4983/// Test if the MI is logic shift right with half bits,
4984/// such as `%reg0:2n =G_LSHR %reg1:2n, CONST(n)`
4985static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
4986 if (MI->getOpcode() != AMDGPU::G_LSHR)
4987 return false;
4988
4989 Register ShiftSrc;
4990 std::optional<ValueAndVReg> ShiftAmt;
4991 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
4992 P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
4993 uint64_t SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4994 unsigned Shift = static_cast<unsigned>(ShiftAmt->Value.getZExtValue());
4995 return Shift * 2 == SrcSize;
4996 }
4997 return false;
4998}
4999
5000/// Test if the MI is shift left with half bits,
5001/// such as `%reg0:2n =G_SHL %reg1:2n, CONST(n)`
5002static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
5003 if (MI->getOpcode() != AMDGPU::G_SHL)
5004 return false;
5005
5006 Register ShiftSrc;
5007 std::optional<ValueAndVReg> ShiftAmt;
5008 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
5009 P: m_GShl(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
5010 uint64_t SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
5011 unsigned Shift = static_cast<unsigned>(ShiftAmt->Value.getZExtValue());
5012 return Shift * 2 == SrcSize;
5013 }
5014 return false;
5015}
5016
5017/// Test function, if the MI is `%reg0:n, %reg1:n = G_UNMERGE_VALUES %reg2:2n`
5018static bool isUnmergeHalf(const MachineInstr *MI,
5019 const MachineRegisterInfo &MRI) {
5020 if (MI->getOpcode() != AMDGPU::G_UNMERGE_VALUES)
5021 return false;
5022 return MI->getNumOperands() == 3 && MI->getOperand(i: 0).isDef() &&
5023 MI->getOperand(i: 1).isDef() && !MI->getOperand(i: 2).isDef();
5024}
5025
5026enum class TypeClass { VECTOR_OF_TWO, SCALAR, NONE_OF_LISTED };
5027
5028static TypeClass isVectorOfTwoOrScalar(Register Reg,
5029 const MachineRegisterInfo &MRI) {
5030 LLT OpTy = MRI.getType(Reg);
5031 if (OpTy.isScalar())
5032 return TypeClass::SCALAR;
5033 if (OpTy.isVector() && OpTy.getNumElements() == 2)
5034 return TypeClass::VECTOR_OF_TWO;
5035 return TypeClass::NONE_OF_LISTED;
5036}
5037
5038static SrcStatus getNegStatus(Register Reg, SrcStatus S,
5039 const MachineRegisterInfo &MRI) {
5040 TypeClass NegType = isVectorOfTwoOrScalar(Reg, MRI);
5041 if (NegType != TypeClass::VECTOR_OF_TWO && NegType != TypeClass::SCALAR)
5042 return SrcStatus::INVALID;
5043
5044 switch (S) {
5045 case SrcStatus::IS_SAME:
5046 if (NegType == TypeClass::VECTOR_OF_TWO) {
5047 // Vector of 2:
5048 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
5049 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
5050 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5051 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
5052 return SrcStatus::IS_BOTH_NEG;
5053 }
5054 if (NegType == TypeClass::SCALAR) {
5055 // Scalar:
5056 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
5057 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
5058 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5059 // [SrcHi, SrcLo] = [-OpHi, OpLo]
5060 return SrcStatus::IS_HI_NEG;
5061 }
5062 break;
5063 case SrcStatus::IS_HI_NEG:
5064 if (NegType == TypeClass::VECTOR_OF_TWO) {
5065 // Vector of 2:
5066 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5067 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
5068 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5069 // [SrcHi, SrcLo] = [-(-OpHi), -OpLo] = [OpHi, -OpLo]
5070 return SrcStatus::IS_LO_NEG;
5071 }
5072 if (NegType == TypeClass::SCALAR) {
5073 // Scalar:
5074 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5075 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
5076 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5077 // [SrcHi, SrcLo] = [-(-OpHi), OpLo] = [OpHi, OpLo]
5078 return SrcStatus::IS_SAME;
5079 }
5080 break;
5081 case SrcStatus::IS_LO_NEG:
5082 if (NegType == TypeClass::VECTOR_OF_TWO) {
5083 // Vector of 2:
5084 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5085 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5086 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5087 // [SrcHi, SrcLo] = [-OpHi, -(-OpLo)] = [-OpHi, OpLo]
5088 return SrcStatus::IS_HI_NEG;
5089 }
5090 if (NegType == TypeClass::SCALAR) {
5091 // Scalar:
5092 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
5093 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5094 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5095 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
5096 return SrcStatus::IS_BOTH_NEG;
5097 }
5098 break;
5099 case SrcStatus::IS_BOTH_NEG:
5100 if (NegType == TypeClass::VECTOR_OF_TWO) {
5101 // Vector of 2:
5102 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5103 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
5104 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
5105 // [SrcHi, SrcLo] = [OpHi, OpLo]
5106 return SrcStatus::IS_SAME;
5107 }
5108 if (NegType == TypeClass::SCALAR) {
5109 // Scalar:
5110 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
5111 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
5112 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
5113 // [SrcHi, SrcLo] = [OpHi, -OpLo]
5114 return SrcStatus::IS_LO_NEG;
5115 }
5116 break;
5117 case SrcStatus::IS_UPPER_HALF:
5118 // Vector of 2:
5119 // Src = CurrUpper
5120 // Curr = [CurrUpper, CurrLower]
5121 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5122 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5123 // Src = -OpUpper
5124 //
5125 // Scalar:
5126 // Src = CurrUpper
5127 // Curr = [CurrUpper, CurrLower]
5128 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5129 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5130 // Src = -OpUpper
5131 return SrcStatus::IS_UPPER_HALF_NEG;
5132 case SrcStatus::IS_LOWER_HALF:
5133 if (NegType == TypeClass::VECTOR_OF_TWO) {
5134 // Vector of 2:
5135 // Src = CurrLower
5136 // Curr = [CurrUpper, CurrLower]
5137 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5138 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5139 // Src = -OpLower
5140 return SrcStatus::IS_LOWER_HALF_NEG;
5141 }
5142 if (NegType == TypeClass::SCALAR) {
5143 // Scalar:
5144 // Src = CurrLower
5145 // Curr = [CurrUpper, CurrLower]
5146 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5147 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5148 // Src = OpLower
5149 return SrcStatus::IS_LOWER_HALF;
5150 }
5151 break;
5152 case SrcStatus::IS_UPPER_HALF_NEG:
5153 // Vector of 2:
5154 // Src = -CurrUpper
5155 // Curr = [CurrUpper, CurrLower]
5156 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5157 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5158 // Src = -(-OpUpper) = OpUpper
5159 //
5160 // Scalar:
5161 // Src = -CurrUpper
5162 // Curr = [CurrUpper, CurrLower]
5163 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5164 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5165 // Src = -(-OpUpper) = OpUpper
5166 return SrcStatus::IS_UPPER_HALF;
5167 case SrcStatus::IS_LOWER_HALF_NEG:
5168 if (NegType == TypeClass::VECTOR_OF_TWO) {
5169 // Vector of 2:
5170 // Src = -CurrLower
5171 // Curr = [CurrUpper, CurrLower]
5172 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5173 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5174 // Src = -(-OpLower) = OpLower
5175 return SrcStatus::IS_LOWER_HALF;
5176 }
5177 if (NegType == TypeClass::SCALAR) {
5178 // Scalar:
5179 // Src = -CurrLower
5180 // Curr = [CurrUpper, CurrLower]
5181 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5182 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5183 // Src = -OpLower
5184 return SrcStatus::IS_LOWER_HALF_NEG;
5185 }
5186 break;
5187 default:
5188 break;
5189 }
5190 llvm_unreachable("unexpected SrcStatus & NegType combination");
5191}
5192
5193static std::optional<std::pair<Register, SrcStatus>>
5194calcNextStatus(std::pair<Register, SrcStatus> Curr,
5195 const MachineRegisterInfo &MRI) {
5196 const MachineInstr *MI = MRI.getVRegDef(Reg: Curr.first);
5197
5198 unsigned Opc = MI->getOpcode();
5199
5200 // Handle general Opc cases.
5201 switch (Opc) {
5202 case AMDGPU::G_BITCAST:
5203 return std::optional<std::pair<Register, SrcStatus>>(
5204 {MI->getOperand(i: 1).getReg(), Curr.second});
5205 case AMDGPU::COPY:
5206 if (MI->getOperand(i: 1).getReg().isPhysical())
5207 return std::nullopt;
5208 return std::optional<std::pair<Register, SrcStatus>>(
5209 {MI->getOperand(i: 1).getReg(), Curr.second});
5210 case AMDGPU::G_FNEG: {
5211 SrcStatus Stat = getNegStatus(Reg: Curr.first, S: Curr.second, MRI);
5212 if (Stat == SrcStatus::INVALID)
5213 return std::nullopt;
5214 return std::optional<std::pair<Register, SrcStatus>>(
5215 {MI->getOperand(i: 1).getReg(), Stat});
5216 }
5217 default:
5218 break;
5219 }
5220
5221 // Calc next Stat from current Stat.
5222 switch (Curr.second) {
5223 case SrcStatus::IS_SAME:
5224 if (isTruncHalf(MI, MRI))
5225 return std::optional<std::pair<Register, SrcStatus>>(
5226 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5227 else if (isUnmergeHalf(MI, MRI)) {
5228 if (Curr.first == MI->getOperand(i: 0).getReg())
5229 return std::optional<std::pair<Register, SrcStatus>>(
5230 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF});
5231 return std::optional<std::pair<Register, SrcStatus>>(
5232 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF});
5233 }
5234 break;
5235 case SrcStatus::IS_HI_NEG:
5236 if (isTruncHalf(MI, MRI)) {
5237 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5238 // [CurrHi, CurrLo] = trunc [OpUpper, OpLower] = OpLower
5239 // = [OpLowerHi, OpLowerLo]
5240 // Src = [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5241 // = [-OpLowerHi, OpLowerLo]
5242 // = -OpLower
5243 return std::optional<std::pair<Register, SrcStatus>>(
5244 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5245 }
5246 if (isUnmergeHalf(MI, MRI)) {
5247 if (Curr.first == MI->getOperand(i: 0).getReg())
5248 return std::optional<std::pair<Register, SrcStatus>>(
5249 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5250 return std::optional<std::pair<Register, SrcStatus>>(
5251 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5252 }
5253 break;
5254 case SrcStatus::IS_UPPER_HALF:
5255 if (isShlHalf(MI, MRI))
5256 return std::optional<std::pair<Register, SrcStatus>>(
5257 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5258 break;
5259 case SrcStatus::IS_LOWER_HALF:
5260 if (isLshrHalf(MI, MRI))
5261 return std::optional<std::pair<Register, SrcStatus>>(
5262 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF});
5263 break;
5264 case SrcStatus::IS_UPPER_HALF_NEG:
5265 if (isShlHalf(MI, MRI))
5266 return std::optional<std::pair<Register, SrcStatus>>(
5267 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5268 break;
5269 case SrcStatus::IS_LOWER_HALF_NEG:
5270 if (isLshrHalf(MI, MRI))
5271 return std::optional<std::pair<Register, SrcStatus>>(
5272 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5273 break;
5274 default:
5275 break;
5276 }
5277 return std::nullopt;
5278}
5279
5280/// This is used to control valid status that current MI supports. For example,
5281/// non floating point intrinsic such as @llvm.amdgcn.sdot2 does not support NEG
5282/// bit on VOP3P.
5283/// The class can be further extended to recognize support on SEL, NEG, ABS bit
5284/// for different MI on different arch
5285class SearchOptions {
5286private:
5287 bool HasNeg = false;
5288 // Assume all complex pattern of VOP3P have opsel.
5289 bool HasOpsel = true;
5290
5291public:
5292 SearchOptions(Register Reg, const MachineRegisterInfo &MRI) {
5293 const MachineInstr *MI = MRI.getVRegDef(Reg);
5294 unsigned Opc = MI->getOpcode();
5295
5296 if (Opc == TargetOpcode::G_INTRINSIC) {
5297 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: *MI).getIntrinsicID();
5298 // Only float point intrinsic has neg & neg_hi bits.
5299 if (IntrinsicID == Intrinsic::amdgcn_fdot2)
5300 HasNeg = true;
5301 } else if (TargetInstrInfo::isGenericOpcode(Opc)) {
5302 // Keep same for generic op.
5303 HasNeg = true;
5304 }
5305 }
5306 bool checkOptions(SrcStatus Stat) const {
5307 if (!HasNeg &&
5308 (Stat >= SrcStatus::NEG_START && Stat <= SrcStatus::NEG_END)) {
5309 return false;
5310 }
5311 if (!HasOpsel &&
5312 (Stat >= SrcStatus::HALF_START && Stat <= SrcStatus::HALF_END)) {
5313 return false;
5314 }
5315 return true;
5316 }
5317};
5318
5319static SmallVector<std::pair<Register, SrcStatus>>
5320getSrcStats(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5321 int MaxDepth = 3) {
5322 int Depth = 0;
5323 auto Curr = calcNextStatus(Curr: {Reg, SrcStatus::IS_SAME}, MRI);
5324 SmallVector<std::pair<Register, SrcStatus>> Statlist;
5325
5326 while (Depth <= MaxDepth && Curr.has_value()) {
5327 Depth++;
5328 if (SO.checkOptions(Stat: Curr.value().second))
5329 Statlist.push_back(Elt: Curr.value());
5330 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5331 }
5332
5333 return Statlist;
5334}
5335
5336static std::pair<Register, SrcStatus>
5337getLastSameOrNeg(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5338 int MaxDepth = 3) {
5339 int Depth = 0;
5340 std::pair<Register, SrcStatus> LastSameOrNeg = {Reg, SrcStatus::IS_SAME};
5341 auto Curr = calcNextStatus(Curr: LastSameOrNeg, MRI);
5342
5343 while (Depth <= MaxDepth && Curr.has_value()) {
5344 Depth++;
5345 SrcStatus Stat = Curr.value().second;
5346 if (SO.checkOptions(Stat)) {
5347 if (Stat == SrcStatus::IS_SAME || Stat == SrcStatus::IS_HI_NEG ||
5348 Stat == SrcStatus::IS_LO_NEG || Stat == SrcStatus::IS_BOTH_NEG)
5349 LastSameOrNeg = Curr.value();
5350 }
5351 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5352 }
5353
5354 return LastSameOrNeg;
5355}
5356
5357static bool isSameBitWidth(Register Reg1, Register Reg2,
5358 const MachineRegisterInfo &MRI) {
5359 uint64_t Width1 = MRI.getType(Reg: Reg1).getSizeInBits();
5360 uint64_t Width2 = MRI.getType(Reg: Reg2).getSizeInBits();
5361 return Width1 == Width2;
5362}
5363
5364static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods) {
5365 // SrcStatus::IS_LOWER_HALF remain 0.
5366 if (HiStat == SrcStatus::IS_UPPER_HALF_NEG) {
5367 Mods ^= SISrcMods::NEG_HI;
5368 Mods |= SISrcMods::OP_SEL_1;
5369 } else if (HiStat == SrcStatus::IS_UPPER_HALF)
5370 Mods |= SISrcMods::OP_SEL_1;
5371 else if (HiStat == SrcStatus::IS_LOWER_HALF_NEG)
5372 Mods ^= SISrcMods::NEG_HI;
5373 else if (HiStat == SrcStatus::IS_HI_NEG)
5374 Mods ^= SISrcMods::NEG_HI;
5375
5376 if (LoStat == SrcStatus::IS_UPPER_HALF_NEG) {
5377 Mods ^= SISrcMods::NEG;
5378 Mods |= SISrcMods::OP_SEL_0;
5379 } else if (LoStat == SrcStatus::IS_UPPER_HALF)
5380 Mods |= SISrcMods::OP_SEL_0;
5381 else if (LoStat == SrcStatus::IS_LOWER_HALF_NEG)
5382 Mods |= SISrcMods::NEG;
5383 else if (LoStat == SrcStatus::IS_HI_NEG)
5384 Mods ^= SISrcMods::NEG;
5385
5386 return Mods;
5387}
5388
5389static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg,
5390 Register RootReg, const SIInstrInfo &TII,
5391 const MachineRegisterInfo &MRI) {
5392 auto IsHalfState = [](SrcStatus S) {
5393 return S == SrcStatus::IS_UPPER_HALF || S == SrcStatus::IS_UPPER_HALF_NEG ||
5394 S == SrcStatus::IS_LOWER_HALF || S == SrcStatus::IS_LOWER_HALF_NEG;
5395 };
5396 return isSameBitWidth(Reg1: NewReg, Reg2: RootReg, MRI) && IsHalfState(LoStat) &&
5397 IsHalfState(HiStat);
5398}
5399
5400std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
5401 Register RootReg, const MachineRegisterInfo &MRI, bool IsDOT,
5402 bool NoOpSel0) const {
5403 unsigned Mods = 0;
5404 // No modification if Root type is not form of <2 x Type>.
5405 if (isVectorOfTwoOrScalar(Reg: RootReg, MRI) != TypeClass::VECTOR_OF_TWO) {
5406 Mods |= SISrcMods::OP_SEL_1;
5407 return {RootReg, Mods};
5408 }
5409
5410 SearchOptions SO(RootReg, MRI);
5411
5412 std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(Reg: RootReg, MRI, SO);
5413
5414 if (Stat.second == SrcStatus::IS_BOTH_NEG)
5415 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
5416 else if (Stat.second == SrcStatus::IS_HI_NEG)
5417 Mods ^= SISrcMods::NEG_HI;
5418 else if (Stat.second == SrcStatus::IS_LO_NEG)
5419 Mods ^= SISrcMods::NEG;
5420
5421 // 64-bit VOP3P instructions do not have OPSEL or ABS. Bail on v2f64 or v2i64.
5422 // TODO: Select NEG_LO and NEG_HI modifiers from BUILD_VECTOR.
5423 if (MRI.getType(Reg: RootReg).getSizeInBits() == 128) {
5424 Mods |= SISrcMods::OP_SEL_1; // Just the default, OPSEL unsupported.
5425 return {Stat.first, Mods};
5426 }
5427
5428 GBuildVector *MI;
5429 if (!mi_match(R: Stat.first, MRI, P: m_GBuildVector(Inst&: MI)) ||
5430 MI->getNumOperands() != 3 || (IsDOT && Subtarget->hasDOTOpSelHazard())) {
5431 Mods |= SISrcMods::OP_SEL_1;
5432 return {Stat.first, Mods};
5433 }
5434
5435 SmallVector<std::pair<Register, SrcStatus>> StatlistHi =
5436 getSrcStats(Reg: MI->getOperand(i: 2).getReg(), MRI, SO);
5437
5438 if (StatlistHi.empty()) {
5439 Mods |= SISrcMods::OP_SEL_1;
5440 return {Stat.first, Mods};
5441 }
5442
5443 SmallVector<std::pair<Register, SrcStatus>> StatlistLo =
5444 getSrcStats(Reg: MI->getOperand(i: 1).getReg(), MRI, SO);
5445
5446 if (StatlistLo.empty()) {
5447 Mods |= SISrcMods::OP_SEL_1;
5448 return {Stat.first, Mods};
5449 }
5450
5451 for (int I = static_cast<int>(StatlistHi.size()) - 1; I >= 0; I--) {
5452 for (int J = static_cast<int>(StatlistLo.size()) - 1; J >= 0; J--) {
5453 if (StatlistHi[I].first == StatlistLo[J].first &&
5454 isValidToPack(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second,
5455 NewReg: StatlistHi[I].first, RootReg, TII, MRI)) {
5456 unsigned NewMods =
5457 updateMods(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second, Mods);
5458 if (NoOpSel0 && (NewMods & SISrcMods::OP_SEL_0))
5459 continue;
5460 return {StatlistHi[I].first, NewMods};
5461 }
5462 }
5463 }
5464 // Packed instructions do not have abs modifiers.
5465 Mods |= SISrcMods::OP_SEL_1;
5466
5467 return {Stat.first, Mods};
5468}
5469
5470// Removed unused function `getAllKindImm` to eliminate dead code.
5471
5472static bool checkRB(Register Reg, unsigned int RBNo,
5473 const AMDGPURegisterBankInfo &RBI,
5474 const MachineRegisterInfo &MRI,
5475 const TargetRegisterInfo &TRI) {
5476 const RegisterBank *RB = RBI.getRegBank(Reg, MRI, TRI);
5477 return RB->getID() == RBNo;
5478}
5479
5480// This function is used to get the correct register bank for returned reg.
5481// Assume:
5482// 1. VOP3P is always legal for VGPR.
5483// 2. RootOp's regbank is legal.
5484// Thus
5485// 1. If RootOp is SGPR, then NewOp can be SGPR or VGPR.
5486// 2. If RootOp is VGPR, then NewOp must be VGPR.
5487static Register
5488getLegalRegBank(Register NewReg, Register RootReg, MachineInstr &Use,
5489 const AMDGPURegisterBankInfo &RBI, MachineRegisterInfo &MRI,
5490 const TargetRegisterInfo &TRI, const SIInstrInfo &TII) {
5491 // RootOp can only be VGPR or SGPR (some hand written cases such as.
5492 // inst-select-ashr.v2s16.mir::ashr_v2s16_vs).
5493 if (checkRB(Reg: RootReg, RBNo: AMDGPU::SGPRRegBankID, RBI, MRI, TRI) ||
5494 checkRB(Reg: NewReg, RBNo: AMDGPU::VGPRRegBankID, RBI, MRI, TRI))
5495 return NewReg;
5496
5497 if (mi_match(R: RootReg, MRI, P: m_Copy(Src: m_SpecificReg(RequestedReg: NewReg)))) {
5498 // RootOp is VGPR, NewOp is not VGPR, but RootOp = COPY NewOp.
5499 return RootReg;
5500 }
5501
5502 Register DstReg = MRI.cloneVirtualRegister(VReg: RootReg);
5503 MachineInstrBuilder MIB = BuildMI(BB&: *Use.getParent(), I&: Use, MIMD: Use.getDebugLoc(),
5504 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
5505 .addReg(RegNo: NewReg);
5506
5507 // Only accept VGPR.
5508 return MIB->getOperand(i: 0).getReg();
5509}
5510
5511InstructionSelector::ComplexRendererFns
5512AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
5513 bool IsDOT,
5514 bool NoOpSel0) const {
5515 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5516 Register Reg;
5517 unsigned Mods;
5518 std::tie(args&: Reg, args&: Mods) =
5519 selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT, NoOpSel0);
5520
5521 return {{
5522 [=, &Root, &MRI](MachineInstrBuilder &MIB) {
5523 MIB.addReg(RegNo: getLegalRegBank(NewReg: Reg, RootReg: Root.getReg(), Use&: *MIB.getInstr(), RBI,
5524 MRI, TRI, TII));
5525 },
5526 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5527 }};
5528}
5529
5530InstructionSelector::ComplexRendererFns
5531AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
5532
5533 return selectVOP3PRetHelper(Root);
5534}
5535
5536InstructionSelector::ComplexRendererFns
5537AMDGPUInstructionSelector::selectVOP3PModsPkFmaOpSel0(
5538 MachineOperand &Root) const {
5539 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5540 LLT Ty = MRI.getType(Reg: Root.getReg());
5541 bool NoOpSel0 = Subtarget->hasPkF32InvalidOpSel() && Ty.isFixedVector() &&
5542 Ty.getNumElements() == 2 && Ty.getScalarSizeInBits() == 32;
5543 return selectVOP3PRetHelper(Root, /*IsDOT=*/false, NoOpSel0);
5544}
5545
5546InstructionSelector::ComplexRendererFns
5547AMDGPUInstructionSelector::selectVOP3PModsPkAddMulOpSel0(
5548 MachineOperand &Root) const {
5549 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5550 LLT Ty = MRI.getType(Reg: Root.getReg());
5551 if (Subtarget->hasPkF32InvalidOpSel() && Ty.isFixedVector() &&
5552 Ty.getNumElements() == 2 && Ty.getScalarSizeInBits() == 32 &&
5553 (selectVOP3PModsImpl(RootReg: Root.getReg(), MRI).second & SISrcMods::OP_SEL_0))
5554 return std::nullopt;
5555 return selectVOP3PRetHelper(Root);
5556}
5557
5558InstructionSelector::ComplexRendererFns
5559AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
5560
5561 return selectVOP3PRetHelper(Root, IsDOT: true);
5562}
5563
5564InstructionSelector::ComplexRendererFns
5565AMDGPUInstructionSelector::selectVOP3PNoModsDOT(MachineOperand &Root) const {
5566 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5567 Register Src;
5568 unsigned Mods;
5569 std::tie(args&: Src, args&: Mods) = selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT: true /*IsDOT*/);
5570 if (Mods != SISrcMods::OP_SEL_1)
5571 return {};
5572
5573 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5574}
5575
5576InstructionSelector::ComplexRendererFns
5577AMDGPUInstructionSelector::selectVOP3PModsF32(MachineOperand &Root) const {
5578 Register Src;
5579 unsigned Mods;
5580 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5581
5582 return {{
5583 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5584 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5585 }};
5586}
5587
5588InstructionSelector::ComplexRendererFns
5589AMDGPUInstructionSelector::selectVOP3PNoModsF32(MachineOperand &Root) const {
5590 Register Src;
5591 unsigned Mods;
5592 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5593 if (Mods != SISrcMods::OP_SEL_1)
5594 return {};
5595
5596 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5597}
5598
5599InstructionSelector::ComplexRendererFns
5600AMDGPUInstructionSelector::selectWMMAOpSelVOP3PMods(
5601 MachineOperand &Root) const {
5602 assert((Root.isImm() && (Root.getImm() == -1 || Root.getImm() == 0)) &&
5603 "expected i1 value");
5604 unsigned Mods = SISrcMods::OP_SEL_1;
5605 if (Root.getImm() != 0)
5606 Mods |= SISrcMods::OP_SEL_0;
5607
5608 return {{
5609 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5610 }};
5611}
5612
5613static Register buildRegSequence(SmallVectorImpl<Register> &Elts,
5614 MachineInstr *InsertPt,
5615 MachineRegisterInfo &MRI) {
5616 const TargetRegisterClass *DstRegClass;
5617 switch (Elts.size()) {
5618 case 8:
5619 DstRegClass = &AMDGPU::VReg_256RegClass;
5620 break;
5621 case 4:
5622 DstRegClass = &AMDGPU::VReg_128RegClass;
5623 break;
5624 case 2:
5625 DstRegClass = &AMDGPU::VReg_64RegClass;
5626 break;
5627 default:
5628 llvm_unreachable("unhandled Reg sequence size");
5629 }
5630
5631 MachineIRBuilder B(*InsertPt);
5632 auto MIB = B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
5633 .addDef(RegNo: MRI.createVirtualRegister(RegClass: DstRegClass));
5634 for (unsigned i = 0; i < Elts.size(); ++i) {
5635 MIB.addReg(RegNo: Elts[i]);
5636 MIB.addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: i));
5637 }
5638 return MIB->getOperand(i: 0).getReg();
5639}
5640
5641static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods,
5642 SmallVectorImpl<Register> &Elts, Register &Src,
5643 MachineInstr *InsertPt,
5644 MachineRegisterInfo &MRI) {
5645 if (ModOpcode == TargetOpcode::G_FNEG) {
5646 Mods |= SISrcMods::NEG;
5647 // Check if all elements also have abs modifier
5648 SmallVector<Register, 8> NegAbsElts;
5649 for (auto El : Elts) {
5650 Register FabsSrc;
5651 if (!mi_match(R: El, MRI, P: m_GFabs(Src: m_Reg(R&: FabsSrc))))
5652 break;
5653 NegAbsElts.push_back(Elt: FabsSrc);
5654 }
5655 if (Elts.size() != NegAbsElts.size()) {
5656 // Neg
5657 Src = buildRegSequence(Elts, InsertPt, MRI);
5658 } else {
5659 // Neg and Abs
5660 Mods |= SISrcMods::NEG_HI;
5661 Src = buildRegSequence(Elts&: NegAbsElts, InsertPt, MRI);
5662 }
5663 } else {
5664 assert(ModOpcode == TargetOpcode::G_FABS);
5665 // Abs
5666 Mods |= SISrcMods::NEG_HI;
5667 Src = buildRegSequence(Elts, InsertPt, MRI);
5668 }
5669}
5670
5671InstructionSelector::ComplexRendererFns
5672AMDGPUInstructionSelector::selectWMMAModsF32NegAbs(MachineOperand &Root) const {
5673 Register Src = Root.getReg();
5674 unsigned Mods = SISrcMods::OP_SEL_1;
5675 SmallVector<Register, 8> EltsF32;
5676
5677 GBuildVector *BV;
5678 if (mi_match(R: Src, MRI: *MRI, P: m_GBuildVector(Inst&: BV))) {
5679 assert(BV->getNumSources() > 0);
5680 // Based on first element decide which mod we match, neg or abs
5681 MachineInstr *ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: 0));
5682 unsigned ModOpcode = (ElF32->getOpcode() == AMDGPU::G_FNEG)
5683 ? AMDGPU::G_FNEG
5684 : AMDGPU::G_FABS;
5685 for (unsigned i = 0; i < BV->getNumSources(); ++i) {
5686 ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: i));
5687 if (ElF32->getOpcode() != ModOpcode)
5688 break;
5689 EltsF32.push_back(Elt: ElF32->getOperand(i: 1).getReg());
5690 }
5691
5692 // All elements had ModOpcode modifier
5693 if (BV->getNumSources() == EltsF32.size()) {
5694 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsF32, Src, InsertPt: Root.getParent(),
5695 MRI&: *MRI);
5696 }
5697 }
5698
5699 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5700 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5701}
5702
5703InstructionSelector::ComplexRendererFns
5704AMDGPUInstructionSelector::selectWMMAModsF16Neg(MachineOperand &Root) const {
5705 Register Src = Root.getReg();
5706 unsigned Mods = SISrcMods::OP_SEL_1;
5707 SmallVector<Register, 8> EltsV2F16;
5708
5709 GConcatVectors *CV;
5710 if (mi_match(R: Src, MRI: *MRI, P: m_GConcatVectors(Inst&: CV))) {
5711 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5712 Register FNegSrc;
5713 if (!mi_match(R: CV->getSourceReg(I: i), MRI: *MRI, P: m_GFNeg(Src: m_Reg(R&: FNegSrc))))
5714 break;
5715 EltsV2F16.push_back(Elt: FNegSrc);
5716 }
5717
5718 // All elements had ModOpcode modifier
5719 if (CV->getNumSources() == EltsV2F16.size()) {
5720 Mods |= SISrcMods::NEG;
5721 Mods |= SISrcMods::NEG_HI;
5722 Src = buildRegSequence(Elts&: EltsV2F16, InsertPt: Root.getParent(), MRI&: *MRI);
5723 }
5724 }
5725
5726 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5727 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5728}
5729
5730InstructionSelector::ComplexRendererFns
5731AMDGPUInstructionSelector::selectWMMAModsF16NegAbs(MachineOperand &Root) const {
5732 Register Src = Root.getReg();
5733 unsigned Mods = SISrcMods::OP_SEL_1;
5734 SmallVector<Register, 8> EltsV2F16;
5735
5736 GConcatVectors *CV;
5737 if (mi_match(R: Src, MRI: *MRI, P: m_GConcatVectors(Inst&: CV))) {
5738 assert(CV->getNumSources() > 0);
5739 MachineInstr *ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: 0));
5740 // Based on first element decide which mod we match, neg or abs
5741 unsigned ModOpcode = (ElV2F16->getOpcode() == AMDGPU::G_FNEG)
5742 ? AMDGPU::G_FNEG
5743 : AMDGPU::G_FABS;
5744
5745 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5746 ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: i));
5747 if (ElV2F16->getOpcode() != ModOpcode)
5748 break;
5749 EltsV2F16.push_back(Elt: ElV2F16->getOperand(i: 1).getReg());
5750 }
5751
5752 // All elements had ModOpcode modifier
5753 if (CV->getNumSources() == EltsV2F16.size()) {
5754 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsV2F16, Src, InsertPt: Root.getParent(),
5755 MRI&: *MRI);
5756 }
5757 }
5758
5759 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5760 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5761}
5762
5763InstructionSelector::ComplexRendererFns
5764AMDGPUInstructionSelector::selectWMMAVISrc(MachineOperand &Root) const {
5765 std::optional<FPValueAndVReg> FPValReg;
5766 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GFCstOrSplat(FPValReg))) {
5767 if (TII.isInlineConstant(Imm: FPValReg->Value)) {
5768 return {{[=](MachineInstrBuilder &MIB) {
5769 MIB.addImm(Val: FPValReg->Value.bitcastToAPInt().getSExtValue());
5770 }}};
5771 }
5772 // Non-inlineable splat floats should not fall-through for integer immediate
5773 // checks.
5774 return {};
5775 }
5776
5777 APInt ICst;
5778 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICstOrSplat(Cst&: ICst))) {
5779 if (TII.isInlineConstant(Imm: ICst)) {
5780 return {
5781 {[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ICst.getSExtValue()); }}};
5782 }
5783 }
5784
5785 return {};
5786}
5787
5788InstructionSelector::ComplexRendererFns
5789AMDGPUInstructionSelector::selectSWMMACIndex8(MachineOperand &Root) const {
5790 Register Src =
5791 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5792 unsigned Key = 0;
5793
5794 Register ShiftSrc;
5795 std::optional<ValueAndVReg> ShiftAmt;
5796 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5797 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5798 ShiftAmt->Value.getZExtValue() % 8 == 0) {
5799 Key = static_cast<unsigned>(ShiftAmt->Value.getZExtValue() / 8);
5800 Src = ShiftSrc;
5801 }
5802
5803 return {{
5804 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5805 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5806 }};
5807}
5808
5809InstructionSelector::ComplexRendererFns
5810AMDGPUInstructionSelector::selectSWMMACIndex16(MachineOperand &Root) const {
5811
5812 Register Src =
5813 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5814 unsigned Key = 0;
5815
5816 Register ShiftSrc;
5817 std::optional<ValueAndVReg> ShiftAmt;
5818 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5819 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5820 ShiftAmt->Value.getZExtValue() == 16) {
5821 Src = ShiftSrc;
5822 Key = 1;
5823 }
5824
5825 return {{
5826 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5827 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5828 }};
5829}
5830
5831InstructionSelector::ComplexRendererFns
5832AMDGPUInstructionSelector::selectSWMMACIndex32(MachineOperand &Root) const {
5833 Register Src =
5834 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5835 unsigned Key = 0;
5836
5837 Register S32 = matchZeroExtendFromS32(Reg: Src);
5838 if (!S32)
5839 S32 = matchAnyExtendFromS32(Reg: Src);
5840
5841 if (S32) {
5842 const MachineInstr *Def = getDefIgnoringCopies(Reg: S32, MRI: *MRI);
5843 if (Def->getOpcode() == TargetOpcode::G_UNMERGE_VALUES) {
5844 assert(Def->getNumOperands() == 3);
5845 Register DstReg1 = Def->getOperand(i: 1).getReg();
5846 if (mi_match(R: S32, MRI: *MRI,
5847 P: m_any_of(preds: m_SpecificReg(RequestedReg: DstReg1), preds: m_Copy(Src: m_Reg(R&: DstReg1))))) {
5848 Src = Def->getOperand(i: 2).getReg();
5849 Key = 1;
5850 }
5851 }
5852 }
5853
5854 return {{
5855 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5856 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5857 }};
5858}
5859
5860InstructionSelector::ComplexRendererFns
5861AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const {
5862 Register Src;
5863 unsigned Mods;
5864 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
5865
5866 Register ExtractSrc;
5867 if (!Subtarget->useRealTrue16Insts() &&
5868 MRI->getType(Reg: Root.getReg()).getSizeInBits() == 16 &&
5869 isExtractHiElt(MRI&: *MRI, In: Src, Out&: ExtractSrc)) {
5870 Src = ExtractSrc;
5871 Mods |= SISrcMods::OP_SEL_0;
5872 }
5873
5874 return {{
5875 [=](MachineInstrBuilder &MIB) {
5876 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
5877 },
5878 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5879 }};
5880}
5881
5882// FIXME-TRUE16 remove when fake16 is removed
5883InstructionSelector::ComplexRendererFns
5884AMDGPUInstructionSelector::selectVINTERPMods(MachineOperand &Root) const {
5885 Register Src;
5886 unsigned Mods;
5887 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5888 /*IsCanonicalizing=*/true,
5889 /*AllowAbs=*/false,
5890 /*OpSel=*/false);
5891
5892 return {{
5893 [=](MachineInstrBuilder &MIB) {
5894 MIB.addReg(
5895 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5896 },
5897 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5898 }};
5899}
5900
5901InstructionSelector::ComplexRendererFns
5902AMDGPUInstructionSelector::selectVINTERPModsHi(MachineOperand &Root) const {
5903 Register Src;
5904 unsigned Mods;
5905 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5906 /*IsCanonicalizing=*/true,
5907 /*AllowAbs=*/false,
5908 /*OpSel=*/true);
5909
5910 return {{
5911 [=](MachineInstrBuilder &MIB) {
5912 MIB.addReg(
5913 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5914 },
5915 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5916 }};
5917}
5918
5919// Given \p Offset and load specified by the \p Root operand check if \p Offset
5920// is a multiple of the load byte size. If it is update \p Offset to a
5921// pre-scaled value and return true.
5922bool AMDGPUInstructionSelector::selectScaleOffset(MachineOperand &Root,
5923 Register &Offset,
5924 bool IsSigned,
5925 bool IsSMEM) const {
5926 if (!Subtarget->hasScaleOffset())
5927 return false;
5928
5929 const MachineInstr &MI = *Root.getParent();
5930 MachineMemOperand *MMO = *MI.memoperands_begin();
5931
5932 if (!MMO->getSize().hasValue())
5933 return false;
5934
5935 uint64_t Size = MMO->getSize().getValue();
5936
5937 Register OffsetReg = matchExtendFromS32OrS32(Reg: Offset, IsSigned);
5938 bool IsExtended = OffsetReg && OffsetReg != Offset;
5939 if (!OffsetReg)
5940 OffsetReg = Offset;
5941
5942 if (auto Def = getDefSrcRegIgnoringCopies(Reg: OffsetReg, MRI: *MRI))
5943 OffsetReg = Def->Reg;
5944
5945 Register Op0;
5946 MachineInstr *Mul;
5947 bool ScaleOffset =
5948 (isPowerOf2_64(Value: Size) &&
5949 mi_match(R: OffsetReg, MRI: *MRI,
5950 P: m_GShl(L: m_Reg(R&: Op0),
5951 R: m_any_of(preds: m_SpecificICst(RequestedValue: Log2_64(Value: Size)),
5952 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Log2_64(Value: Size))))))) ||
5953 mi_match(R: OffsetReg, MRI: *MRI,
5954 P: m_GMul(L: m_Reg(R&: Op0), R: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5955 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) ||
5956 mi_match(
5957 R: OffsetReg, MRI: *MRI,
5958 P: m_BinOp(Opcode: IsSigned ? AMDGPU::S_MUL_I64_I32_PSEUDO : AMDGPU::S_MUL_U64,
5959 L: m_Reg(R&: Op0), R: m_SpecificICst(RequestedValue: Size))) ||
5960 // Match G_AMDGPU_MAD_U64_U32 offset, c, 0
5961 (mi_match(R: OffsetReg, MRI: *MRI, P: m_MInstr(MI&: Mul)) &&
5962 (Mul->getOpcode() == (IsSigned ? AMDGPU::G_AMDGPU_MAD_I64_I32
5963 : AMDGPU::G_AMDGPU_MAD_U64_U32) ||
5964 (IsSigned && Mul->getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32 &&
5965 VT->signBitIsZero(Op: Mul->getOperand(i: 2).getReg()))) &&
5966 mi_match(R: Mul->getOperand(i: 4).getReg(), MRI: *MRI, P: m_ZeroInt()) &&
5967 mi_match(R: Mul->getOperand(i: 3).getReg(), MRI: *MRI,
5968 P: m_GTrunc(Src: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5969 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) &&
5970 mi_match(R: Mul->getOperand(i: 2).getReg(), MRI: *MRI, P: m_Reg(R&: Op0)));
5971
5972 if (!ScaleOffset)
5973 return false;
5974
5975 // SMEM scales the offset in 32 bits, other instructions in 64 bits.
5976 unsigned Bits = Log2_64_Ceil(Value: Size);
5977 if (IsSMEM) {
5978 if (!IsExtended && MRI->getType(Reg: Offset).getSizeInBits() == 64) {
5979 KnownBits Known = VT->getKnownBits(R: Op0);
5980 if (Known.countMaxActiveBits() + Bits > 32)
5981 return false;
5982 }
5983 } else if (IsExtended) {
5984 const MachineInstr *Scale = MRI->getVRegDef(Reg: OffsetReg);
5985 if (IsSigned && !Scale->getFlag(Flag: MachineInstr::NoSWrap) &&
5986 VT->computeNumSignBits(R: Op0) <= Bits)
5987 return false;
5988 if (!IsSigned && !Scale->getFlag(Flag: MachineInstr::NoUWrap) &&
5989 VT->getKnownBits(R: Op0).countMinLeadingZeros() < Bits)
5990 return false;
5991 }
5992
5993 Offset = Op0;
5994 return true;
5995}
5996
5997bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root,
5998 Register &Base,
5999 Register *SOffset,
6000 int64_t *Offset,
6001 bool *ScaleOffset) const {
6002 MachineInstr *MI = Root.getParent();
6003 MachineBasicBlock *MBB = MI->getParent();
6004
6005 // FIXME: We should shrink the GEP if the offset is known to be <= 32-bits,
6006 // then we can select all ptr + 32-bit offsets.
6007 SmallVector<GEPInfo, 4> AddrInfo;
6008 getAddrModeInfo(Load: *MI, MRI: *MRI, AddrInfo);
6009
6010 if (AddrInfo.empty())
6011 return false;
6012
6013 const GEPInfo &GEPI = AddrInfo[0];
6014 std::optional<int64_t> EncodedImm;
6015
6016 if (ScaleOffset)
6017 *ScaleOffset = false;
6018
6019 if (SOffset && Offset) {
6020 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
6021 /*HasSOffset=*/true);
6022 if (GEPI.SgprParts.size() == 1 && GEPI.Imm != 0 && EncodedImm &&
6023 AddrInfo.size() > 1) {
6024 const GEPInfo &GEPI2 = AddrInfo[1];
6025 if (GEPI2.SgprParts.size() == 2 && GEPI2.Imm == 0) {
6026 Register OffsetReg = GEPI2.SgprParts[1];
6027 if (ScaleOffset)
6028 *ScaleOffset = selectScaleOffset(
6029 Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */, IsSMEM: true /* IsSMEM */);
6030 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
6031 if (OffsetReg) {
6032 Base = GEPI2.SgprParts[0];
6033 *SOffset = OffsetReg;
6034 *Offset = *EncodedImm;
6035 if (*Offset >= 0 || !AMDGPU::hasSMRDSignedImmOffset(ST: STI))
6036 return true;
6037
6038 // For unbuffered smem loads, it is illegal for the Immediate Offset
6039 // to be negative if the resulting (Offset + (M0 or SOffset or zero)
6040 // is negative. Handle the case where the Immediate Offset + SOffset
6041 // is negative.
6042 auto SKnown = VT->getKnownBits(R: *SOffset);
6043 if (*Offset + SKnown.getMinValue().getSExtValue() < 0)
6044 return false;
6045
6046 return true;
6047 }
6048 }
6049 }
6050 return false;
6051 }
6052
6053 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
6054 /*HasSOffset=*/false);
6055 if (Offset && GEPI.SgprParts.size() == 1 && EncodedImm) {
6056 Base = GEPI.SgprParts[0];
6057 *Offset = *EncodedImm;
6058 return true;
6059 }
6060
6061 // SGPR offset is unsigned.
6062 if (SOffset && GEPI.SgprParts.size() == 1 && isUInt<32>(x: GEPI.Imm) &&
6063 GEPI.Imm != 0) {
6064 // If we make it this far we have a load with an 32-bit immediate offset.
6065 // It is OK to select this using a sgpr offset, because we have already
6066 // failed trying to select this load into one of the _IMM variants since
6067 // the _IMM Patterns are considered before the _SGPR patterns.
6068 Base = GEPI.SgprParts[0];
6069 *SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6070 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: *SOffset)
6071 .addImm(Val: GEPI.Imm);
6072 return true;
6073 }
6074
6075 if (SOffset && GEPI.SgprParts.size() && GEPI.Imm == 0) {
6076 Register OffsetReg = GEPI.SgprParts[1];
6077 if (ScaleOffset)
6078 *ScaleOffset = selectScaleOffset(Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */,
6079 IsSMEM: true /* IsSMEM */);
6080 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
6081 if (OffsetReg) {
6082 Base = GEPI.SgprParts[0];
6083 *SOffset = OffsetReg;
6084 return true;
6085 }
6086 }
6087
6088 return false;
6089}
6090
6091InstructionSelector::ComplexRendererFns
6092AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
6093 Register Base;
6094 int64_t Offset;
6095 if (!selectSmrdOffset(Root, Base, /* SOffset= */ nullptr, Offset: &Offset,
6096 /* ScaleOffset */ nullptr))
6097 return std::nullopt;
6098
6099 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6100 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }}};
6101}
6102
6103InstructionSelector::ComplexRendererFns
6104AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const {
6105 SmallVector<GEPInfo, 4> AddrInfo;
6106 getAddrModeInfo(Load: *Root.getParent(), MRI: *MRI, AddrInfo);
6107
6108 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
6109 return std::nullopt;
6110
6111 const GEPInfo &GEPInfo = AddrInfo[0];
6112 Register PtrReg = GEPInfo.SgprParts[0];
6113 std::optional<int64_t> EncodedImm =
6114 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: GEPInfo.Imm);
6115 if (!EncodedImm)
6116 return std::nullopt;
6117
6118 return {{
6119 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrReg); },
6120 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); }
6121 }};
6122}
6123
6124InstructionSelector::ComplexRendererFns
6125AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
6126 Register Base, SOffset;
6127 bool ScaleOffset;
6128 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, /* Offset= */ nullptr,
6129 ScaleOffset: &ScaleOffset))
6130 return std::nullopt;
6131
6132 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6133 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6134 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
6135 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
6136}
6137
6138InstructionSelector::ComplexRendererFns
6139AMDGPUInstructionSelector::selectSmrdSgprImm(MachineOperand &Root) const {
6140 Register Base, SOffset;
6141 int64_t Offset;
6142 bool ScaleOffset;
6143 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, Offset: &Offset, ScaleOffset: &ScaleOffset))
6144 return std::nullopt;
6145
6146 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
6147 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
6148 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
6149 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
6150 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
6151}
6152
6153std::pair<Register, int64_t> AMDGPUInstructionSelector::selectFlatOffsetImpl(
6154 MachineOperand &Root, AMDGPU::FlatAddrSpace FlatVariant) const {
6155 MachineInstr *MI = Root.getParent();
6156
6157 auto Default = std::pair(Root.getReg(), 0);
6158
6159 if (!STI.hasFlatInstOffsets())
6160 return Default;
6161
6162 Register PtrBase;
6163 int64_t ConstOffset;
6164 bool IsInBounds;
6165 std::tie(args&: PtrBase, args&: ConstOffset, args&: IsInBounds) =
6166 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6167
6168 // Adding the offset to the base address with an immediate in a FLAT
6169 // instruction must not change the memory aperture in which the address falls.
6170 // Therefore we can only fold offsets from inbounds GEPs into FLAT
6171 // instructions.
6172 if (ConstOffset == 0 ||
6173 (FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch &&
6174 !isFlatScratchBaseLegal(Addr: Root.getReg())) ||
6175 (FlatVariant == AMDGPU::FlatAddrSpace::FLAT && !IsInBounds))
6176 return Default;
6177
6178 unsigned AddrSpace = (*MI->memoperands_begin())->getAddrSpace();
6179 if (!TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace, FlatVariant))
6180 return Default;
6181
6182 return std::pair(PtrBase, ConstOffset);
6183}
6184
6185InstructionSelector::ComplexRendererFns
6186AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const {
6187 auto PtrWithOffset = selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FLAT);
6188
6189 return {{
6190 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6191 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6192 }};
6193}
6194
6195InstructionSelector::ComplexRendererFns
6196AMDGPUInstructionSelector::selectGlobalOffset(MachineOperand &Root) const {
6197 auto PtrWithOffset =
6198 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6199
6200 return {{
6201 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6202 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6203 }};
6204}
6205
6206InstructionSelector::ComplexRendererFns
6207AMDGPUInstructionSelector::selectScratchOffset(MachineOperand &Root) const {
6208 auto PtrWithOffset =
6209 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch);
6210
6211 return {{
6212 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6213 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6214 }};
6215}
6216
6217// Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
6218InstructionSelector::ComplexRendererFns
6219AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root,
6220 unsigned CPolBits,
6221 bool NeedIOffset) const {
6222 Register Addr = Root.getReg();
6223 Register PtrBase;
6224 int64_t ConstOffset;
6225 int64_t ImmOffset = 0;
6226
6227 // Match the immediate offset first, which canonically is moved as low as
6228 // possible.
6229 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6230 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6231
6232 if (ConstOffset != 0) {
6233 if (NeedIOffset &&
6234 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6235 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal)) {
6236 Addr = PtrBase;
6237 ImmOffset = ConstOffset;
6238 } else {
6239 auto PtrBaseDef = getDefSrcRegIgnoringCopies(Reg: PtrBase, MRI: *MRI);
6240 if (isSGPR(Reg: PtrBaseDef->Reg)) {
6241 // Offset is too large.
6242 //
6243 // saddr + large_offset -> saddr +
6244 // (voffset = large_offset & ~MaxOffset) +
6245 // (large_offset & MaxOffset);
6246 int64_t SplitImmOffset = 0, RemainderOffset = ConstOffset;
6247 if (NeedIOffset) {
6248 std::tie(args&: SplitImmOffset, args&: RemainderOffset) =
6249 TII.splitFlatOffset(COffsetVal: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6250 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6251 }
6252
6253 if (Subtarget->hasSignedGVSOffset() ? isInt<32>(x: RemainderOffset)
6254 : isUInt<32>(x: RemainderOffset)) {
6255 MachineInstr *MI = Root.getParent();
6256 MachineBasicBlock *MBB = MI->getParent();
6257 Register HighBits =
6258 MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6259
6260 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6261 DestReg: HighBits)
6262 .addImm(Val: RemainderOffset);
6263
6264 if (NeedIOffset)
6265 return {{
6266 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrBase); }, // saddr
6267 [=](MachineInstrBuilder &MIB) {
6268 MIB.addReg(RegNo: HighBits);
6269 }, // voffset
6270 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: SplitImmOffset); },
6271 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6272 }};
6273 return {{
6274 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrBase); }, // saddr
6275 [=](MachineInstrBuilder &MIB) {
6276 MIB.addReg(RegNo: HighBits);
6277 }, // voffset
6278 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6279 }};
6280 }
6281
6282 // We are adding a 64 bit SGPR and a constant. If constant bus limit
6283 // is 1 we would need to perform 1 or 2 extra moves for each half of
6284 // the constant and it is better to do a scalar add and then issue a
6285 // single VALU instruction to materialize zero. Otherwise it is less
6286 // instructions to perform VALU adds with immediates or inline literals.
6287 unsigned NumLiterals =
6288 !TII.isInlineConstant(Imm: APInt(32, Lo_32(Value: ConstOffset))) +
6289 !TII.isInlineConstant(Imm: APInt(32, Hi_32(Value: ConstOffset)));
6290 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_ADD_U32_e64) > NumLiterals)
6291 return std::nullopt;
6292 }
6293 }
6294 }
6295
6296 // Match the variable offset.
6297 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6298 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6299 // Look through the SGPR->VGPR copy.
6300 Register SAddr =
6301 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
6302
6303 if (isSGPR(Reg: SAddr)) {
6304 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
6305
6306 // It's possible voffset is an SGPR here, but the copy to VGPR will be
6307 // inserted later.
6308 bool ScaleOffset = selectScaleOffset(Root, Offset&: PtrBaseOffset,
6309 IsSigned: Subtarget->hasSignedGVSOffset());
6310 if (Register VOffset = matchExtendFromS32OrS32(
6311 Reg: PtrBaseOffset, IsSigned: Subtarget->hasSignedGVSOffset())) {
6312 if (NeedIOffset)
6313 return {{[=](MachineInstrBuilder &MIB) { // saddr
6314 MIB.addReg(RegNo: SAddr);
6315 },
6316 [=](MachineInstrBuilder &MIB) { // voffset
6317 MIB.addReg(RegNo: VOffset);
6318 },
6319 [=](MachineInstrBuilder &MIB) { // offset
6320 MIB.addImm(Val: ImmOffset);
6321 },
6322 [=](MachineInstrBuilder &MIB) { // cpol
6323 MIB.addImm(Val: CPolBits |
6324 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6325 }}};
6326 return {{[=](MachineInstrBuilder &MIB) { // saddr
6327 MIB.addReg(RegNo: SAddr);
6328 },
6329 [=](MachineInstrBuilder &MIB) { // voffset
6330 MIB.addReg(RegNo: VOffset);
6331 },
6332 [=](MachineInstrBuilder &MIB) { // cpol
6333 MIB.addImm(Val: CPolBits |
6334 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6335 }}};
6336 }
6337 }
6338 }
6339
6340 // FIXME: We should probably have folded COPY (G_IMPLICIT_DEF) earlier, and
6341 // drop this.
6342 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF ||
6343 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(Reg: AddrDef->Reg))
6344 return std::nullopt;
6345
6346 // It's cheaper to materialize a single 32-bit zero for vaddr than the two
6347 // moves required to copy a 64-bit SGPR to VGPR.
6348 MachineInstr *MI = Root.getParent();
6349 MachineBasicBlock *MBB = MI->getParent();
6350 Register VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6351
6352 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
6353 .addImm(Val: 0);
6354
6355 if (NeedIOffset)
6356 return {{
6357 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6358 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6359 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6360 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6361 }};
6362 return {{
6363 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6364 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6365 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6366 }};
6367}
6368
6369InstructionSelector::ComplexRendererFns
6370AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root) const {
6371 return selectGlobalSAddr(Root, CPolBits: 0);
6372}
6373
6374InstructionSelector::ComplexRendererFns
6375AMDGPUInstructionSelector::selectGlobalSAddrCPol(MachineOperand &Root) const {
6376 const MachineInstr &I = *Root.getParent();
6377
6378 // We are assuming CPol is always the last operand of the intrinsic.
6379 auto PassedCPol =
6380 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6381 return selectGlobalSAddr(Root, CPolBits: static_cast<unsigned>(PassedCPol));
6382}
6383
6384InstructionSelector::ComplexRendererFns
6385AMDGPUInstructionSelector::selectGlobalSAddrCPolM0(MachineOperand &Root) const {
6386 const MachineInstr &I = *Root.getParent();
6387
6388 // We are assuming CPol is second from last operand of the intrinsic.
6389 auto PassedCPol =
6390 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6391 return selectGlobalSAddr(Root, CPolBits: static_cast<unsigned>(PassedCPol));
6392}
6393
6394InstructionSelector::ComplexRendererFns
6395AMDGPUInstructionSelector::selectGlobalSAddrGLC(MachineOperand &Root) const {
6396 return selectGlobalSAddr(Root, CPolBits: AMDGPU::CPol::GLC);
6397}
6398
6399InstructionSelector::ComplexRendererFns
6400AMDGPUInstructionSelector::selectGlobalSAddrNoIOffset(
6401 MachineOperand &Root) const {
6402 const MachineInstr &I = *Root.getParent();
6403
6404 // We are assuming CPol is always the last operand of the intrinsic.
6405 auto PassedCPol =
6406 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6407 return selectGlobalSAddr(Root, CPolBits: static_cast<unsigned>(PassedCPol), NeedIOffset: false);
6408}
6409
6410InstructionSelector::ComplexRendererFns
6411AMDGPUInstructionSelector::selectGlobalSAddrNoIOffsetM0(
6412 MachineOperand &Root) const {
6413 const MachineInstr &I = *Root.getParent();
6414
6415 // We are assuming CPol is second from last operand of the intrinsic.
6416 auto PassedCPol =
6417 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6418 return selectGlobalSAddr(Root, CPolBits: static_cast<unsigned>(PassedCPol), NeedIOffset: false);
6419}
6420
6421InstructionSelector::ComplexRendererFns
6422AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const {
6423 Register Addr = Root.getReg();
6424 Register PtrBase;
6425 int64_t ConstOffset;
6426 int64_t ImmOffset = 0;
6427
6428 // Match the immediate offset first, which canonically is moved as low as
6429 // possible.
6430 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6431 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6432
6433 if (ConstOffset != 0 && isFlatScratchBaseLegal(Addr) &&
6434 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6435 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6436 Addr = PtrBase;
6437 ImmOffset = ConstOffset;
6438 }
6439
6440 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6441 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6442 int FI = AddrDef->MI->getOperand(i: 1).getIndex();
6443 return {{
6444 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6445 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6446 }};
6447 }
6448
6449 Register SAddr = AddrDef->Reg;
6450
6451 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6452 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6453 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6454 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6455 auto RHSDef = getDefSrcRegIgnoringCopies(Reg: RHS, MRI: *MRI);
6456
6457 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX &&
6458 isSGPR(Reg: RHSDef->Reg)) {
6459 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6460 MachineInstr &I = *Root.getParent();
6461 MachineBasicBlock *BB = I.getParent();
6462 const DebugLoc &DL = I.getDebugLoc();
6463 SAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6464
6465 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_I32), DestReg: SAddr)
6466 .addFrameIndex(Idx: FI)
6467 .addReg(RegNo: RHSDef->Reg)
6468 .setOperandDead(3); // Dead scc
6469 }
6470 }
6471
6472 if (!isSGPR(Reg: SAddr))
6473 return std::nullopt;
6474
6475 return {{
6476 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SAddr); }, // saddr
6477 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6478 }};
6479}
6480
6481// Check whether the flat scratch SVS swizzle bug affects this access.
6482bool AMDGPUInstructionSelector::checkFlatScratchSVSSwizzleBug(
6483 Register VAddr, Register SAddr, uint64_t ImmOffset) const {
6484 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
6485 return false;
6486
6487 // The bug affects the swizzling of SVS accesses if there is any carry out
6488 // from the two low order bits (i.e. from bit 1 into bit 2) when adding
6489 // voffset to (soffset + inst_offset).
6490 auto VKnown = VT->getKnownBits(R: VAddr);
6491 auto SKnown = KnownBits::add(LHS: VT->getKnownBits(R: SAddr),
6492 RHS: KnownBits::makeConstant(C: APInt(32, ImmOffset)));
6493 uint64_t VMax = VKnown.getMaxValue().getZExtValue();
6494 uint64_t SMax = SKnown.getMaxValue().getZExtValue();
6495 return (VMax & 3) + (SMax & 3) >= 4;
6496}
6497
6498InstructionSelector::ComplexRendererFns
6499AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const {
6500 Register Addr = Root.getReg();
6501 Register PtrBase;
6502 int64_t ConstOffset;
6503 int64_t ImmOffset = 0;
6504
6505 // Match the immediate offset first, which canonically is moved as low as
6506 // possible.
6507 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6508 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6509
6510 Register OrigAddr = Addr;
6511 if (ConstOffset != 0 &&
6512 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6513 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6514 Addr = PtrBase;
6515 ImmOffset = ConstOffset;
6516 }
6517
6518 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6519 if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD)
6520 return std::nullopt;
6521
6522 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6523 if (RBI.getRegBank(Reg: RHS, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
6524 return std::nullopt;
6525
6526 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6527 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6528
6529 if (OrigAddr != Addr) {
6530 if (!isFlatScratchBaseLegalSVImm(Addr: OrigAddr))
6531 return std::nullopt;
6532 } else {
6533 if (!isFlatScratchBaseLegalSV(Addr: OrigAddr))
6534 return std::nullopt;
6535 }
6536
6537 if (checkFlatScratchSVSSwizzleBug(VAddr: RHS, SAddr: LHS, ImmOffset))
6538 return std::nullopt;
6539
6540 unsigned CPol = selectScaleOffset(Root, Offset&: RHS, IsSigned: true /* IsSigned */)
6541 ? AMDGPU::CPol::SCAL
6542 : 0;
6543
6544 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6545 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6546 return {{
6547 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6548 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6549 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6550 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6551 }};
6552 }
6553
6554 if (!isSGPR(Reg: LHS))
6555 if (auto Def = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI))
6556 LHS = Def->Reg;
6557
6558 if (!isSGPR(Reg: LHS))
6559 return std::nullopt;
6560
6561 return {{
6562 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6563 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: LHS); }, // saddr
6564 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6565 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6566 }};
6567}
6568
6569InstructionSelector::ComplexRendererFns
6570AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
6571 MachineInstr *MI = Root.getParent();
6572 MachineBasicBlock *MBB = MI->getParent();
6573 MachineFunction *MF = MBB->getParent();
6574 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6575
6576 int64_t Offset = 0;
6577 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) &&
6578 Offset != AMDGPU::getNullPointerValue(AS: AMDGPUAS::PRIVATE_ADDRESS)) {
6579 Register HighBits = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6580
6581 // TODO: Should this be inside the render function? The iterator seems to
6582 // move.
6583 const int64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
6584 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6585 DestReg: HighBits)
6586 .addImm(Val: Offset & ~MaxOffset);
6587
6588 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6589 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6590 },
6591 [=](MachineInstrBuilder &MIB) { // vaddr
6592 MIB.addReg(RegNo: HighBits);
6593 },
6594 [=](MachineInstrBuilder &MIB) { // soffset
6595 // Use constant zero for soffset and rely on eliminateFrameIndex
6596 // to choose the appropriate frame register if need be.
6597 MIB.addImm(Val: 0);
6598 },
6599 [=](MachineInstrBuilder &MIB) { // offset
6600 MIB.addImm(Val: Offset & MaxOffset);
6601 }}};
6602 }
6603
6604 assert(Offset == 0 || Offset == -1);
6605
6606 // Try to fold a frame index directly into the MUBUF vaddr field, and any
6607 // offsets.
6608 std::optional<int> FI;
6609 Register VAddr = Root.getReg();
6610
6611 Register PtrBase;
6612 int64_t ConstOffset;
6613 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6614 getPtrBaseWithConstantOffset(Root: VAddr, MRI: *MRI);
6615 int MatchedFI;
6616 if (ConstOffset != 0) {
6617 if (TII.isLegalMUBUFImmOffset(Imm: ConstOffset) &&
6618 (!STI.privateMemoryResourceIsRangeChecked() ||
6619 VT->signBitIsZero(Op: PtrBase))) {
6620 if (mi_match(R: PtrBase, MRI: *MRI, P: m_GFrameIndex(FI&: MatchedFI)))
6621 FI = MatchedFI;
6622 else
6623 VAddr = PtrBase;
6624 Offset = ConstOffset;
6625 }
6626 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GFrameIndex(FI&: MatchedFI))) {
6627 FI = MatchedFI;
6628 }
6629
6630 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6631 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6632 },
6633 [=](MachineInstrBuilder &MIB) { // vaddr
6634 if (FI)
6635 MIB.addFrameIndex(Idx: *FI);
6636 else
6637 MIB.addReg(RegNo: VAddr);
6638 },
6639 [=](MachineInstrBuilder &MIB) { // soffset
6640 // Use constant zero for soffset and rely on eliminateFrameIndex
6641 // to choose the appropriate frame register if need be.
6642 MIB.addImm(Val: 0);
6643 },
6644 [=](MachineInstrBuilder &MIB) { // offset
6645 MIB.addImm(Val: Offset);
6646 }}};
6647}
6648
6649bool AMDGPUInstructionSelector::isDSOffsetLegal(Register Base,
6650 int64_t Offset) const {
6651 if (!isUInt<16>(x: Offset))
6652 return false;
6653
6654 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6655 return true;
6656
6657 // On Southern Islands instruction with a negative base value and an offset
6658 // don't seem to work.
6659 return VT->signBitIsZero(Op: Base);
6660}
6661
6662bool AMDGPUInstructionSelector::isDSOffset2Legal(Register Base, int64_t Offset0,
6663 int64_t Offset1,
6664 unsigned Size) const {
6665 if (Offset0 % Size != 0 || Offset1 % Size != 0)
6666 return false;
6667 if (!isUInt<8>(x: Offset0 / Size) || !isUInt<8>(x: Offset1 / Size))
6668 return false;
6669
6670 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6671 return true;
6672
6673 // On Southern Islands instruction with a negative base value and an offset
6674 // don't seem to work.
6675 return VT->signBitIsZero(Op: Base);
6676}
6677
6678// Return whether the operation has NoUnsignedWrap property.
6679static bool isNoUnsignedWrap(MachineInstr *Addr) {
6680 return Addr->getOpcode() == TargetOpcode::G_OR ||
6681 (Addr->getOpcode() == TargetOpcode::G_PTR_ADD &&
6682 Addr->getFlag(Flag: MachineInstr::NoUWrap));
6683}
6684
6685// Check that the base address of flat scratch load/store in the form of `base +
6686// offset` is legal to be put in SGPR/VGPR (i.e. unsigned per hardware
6687// requirement). We always treat the first operand as the base address here.
6688bool AMDGPUInstructionSelector::isFlatScratchBaseLegal(Register Addr) const {
6689 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6690
6691 if (isNoUnsignedWrap(Addr: AddrMI))
6692 return true;
6693
6694 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6695 // values.
6696 if (STI.hasSignedScratchOffsets())
6697 return true;
6698
6699 Register LHS = AddrMI->getOperand(i: 1).getReg();
6700 Register RHS = AddrMI->getOperand(i: 2).getReg();
6701
6702 if (AddrMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
6703 std::optional<ValueAndVReg> RhsValReg =
6704 getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
6705 // If the immediate offset is negative and within certain range, the base
6706 // address cannot also be negative. If the base is also negative, the sum
6707 // would be either negative or much larger than the valid range of scratch
6708 // memory a thread can access.
6709 if (RhsValReg && RhsValReg->Value.getSExtValue() < 0 &&
6710 RhsValReg->Value.getSExtValue() > -0x40000000)
6711 return true;
6712 }
6713
6714 return VT->signBitIsZero(Op: LHS);
6715}
6716
6717// Check address value in SGPR/VGPR are legal for flat scratch in the form
6718// of: SGPR + VGPR.
6719bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSV(Register Addr) const {
6720 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6721
6722 if (isNoUnsignedWrap(Addr: AddrMI))
6723 return true;
6724
6725 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6726 // values.
6727 if (STI.hasSignedScratchOffsets())
6728 return true;
6729
6730 Register LHS = AddrMI->getOperand(i: 1).getReg();
6731 Register RHS = AddrMI->getOperand(i: 2).getReg();
6732 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6733}
6734
6735// Check address value in SGPR/VGPR are legal for flat scratch in the form
6736// of: SGPR + VGPR + Imm.
6737bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSVImm(
6738 Register Addr) const {
6739 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6740 // values.
6741 if (STI.hasSignedScratchOffsets())
6742 return true;
6743
6744 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6745 Register Base = AddrMI->getOperand(i: 1).getReg();
6746 std::optional<DefinitionAndSourceRegister> BaseDef =
6747 getDefSrcRegIgnoringCopies(Reg: Base, MRI: *MRI);
6748 std::optional<ValueAndVReg> RHSOffset =
6749 getIConstantVRegValWithLookThrough(VReg: AddrMI->getOperand(i: 2).getReg(), MRI: *MRI);
6750 assert(RHSOffset);
6751
6752 // If the immediate offset is negative and within certain range, the base
6753 // address cannot also be negative. If the base is also negative, the sum
6754 // would be either negative or much larger than the valid range of scratch
6755 // memory a thread can access.
6756 if (isNoUnsignedWrap(Addr: BaseDef->MI) &&
6757 (isNoUnsignedWrap(Addr: AddrMI) ||
6758 (RHSOffset->Value.getSExtValue() < 0 &&
6759 RHSOffset->Value.getSExtValue() > -0x40000000)))
6760 return true;
6761
6762 Register LHS = BaseDef->MI->getOperand(i: 1).getReg();
6763 Register RHS = BaseDef->MI->getOperand(i: 2).getReg();
6764 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6765}
6766
6767bool AMDGPUInstructionSelector::isUnneededShiftMask(const MachineInstr &MI,
6768 unsigned ShAmtBits) const {
6769 assert(MI.getOpcode() == TargetOpcode::G_AND);
6770
6771 // The mask constant is an SGPR G_CONSTANT copied to VGPR; the matcher looks
6772 // through that COPY, so look through it here too.
6773 std::optional<ValueAndVReg> RHS =
6774 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI: *MRI);
6775 if (!RHS)
6776 return false;
6777
6778 if (RHS->Value.countr_one() >= ShAmtBits)
6779 return true;
6780
6781 const APInt &LHSKnownZeros = VT->getKnownZeroes(R: MI.getOperand(i: 1).getReg());
6782 return (LHSKnownZeros | RHS->Value).countr_one() >= ShAmtBits;
6783}
6784
6785InstructionSelector::ComplexRendererFns
6786AMDGPUInstructionSelector::selectMUBUFScratchOffset(
6787 MachineOperand &Root) const {
6788 Register Reg = Root.getReg();
6789 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6790
6791 std::optional<DefinitionAndSourceRegister> Def =
6792 getDefSrcRegIgnoringCopies(Reg, MRI: *MRI);
6793 assert(Def && "this shouldn't be an optional result");
6794 Reg = Def->Reg;
6795
6796 if (Register WaveBase = getWaveAddress(Def: Def->MI)) {
6797 return {{
6798 [=](MachineInstrBuilder &MIB) { // rsrc
6799 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6800 },
6801 [=](MachineInstrBuilder &MIB) { // soffset
6802 MIB.addReg(RegNo: WaveBase);
6803 },
6804 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // offset
6805 }};
6806 }
6807
6808 int64_t Offset = 0;
6809
6810 // FIXME: Copy check is a hack
6811 Register BasePtr;
6812 if (mi_match(R: Reg, MRI: *MRI,
6813 P: m_GPtrAdd(L: m_Reg(R&: BasePtr),
6814 R: m_any_of(preds: m_ICst(Cst&: Offset), preds: m_Copy(Src: m_ICst(Cst&: Offset)))))) {
6815 if (!TII.isLegalMUBUFImmOffset(Imm: Offset))
6816 return {};
6817 MachineInstr *BasePtrDef = getDefIgnoringCopies(Reg: BasePtr, MRI: *MRI);
6818 Register WaveBase = getWaveAddress(Def: BasePtrDef);
6819 if (!WaveBase)
6820 return {};
6821
6822 return {{
6823 [=](MachineInstrBuilder &MIB) { // rsrc
6824 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6825 },
6826 [=](MachineInstrBuilder &MIB) { // soffset
6827 MIB.addReg(RegNo: WaveBase);
6828 },
6829 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6830 }};
6831 }
6832
6833 if (!mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) ||
6834 !TII.isLegalMUBUFImmOffset(Imm: Offset))
6835 return {};
6836
6837 return {{
6838 [=](MachineInstrBuilder &MIB) { // rsrc
6839 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6840 },
6841 [=](MachineInstrBuilder &MIB) { // soffset
6842 MIB.addImm(Val: 0);
6843 },
6844 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6845 }};
6846}
6847
6848std::pair<Register, int64_t>
6849AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(
6850 MachineOperand &Root) const {
6851 int64_t ConstAddr = 0;
6852
6853 Register PtrBase;
6854 int64_t Offset;
6855 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6856 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6857
6858 if (Offset) {
6859 if (isDSOffsetLegal(Base: PtrBase, Offset)) {
6860 // (add n0, c0)
6861 return std::pair(PtrBase, Offset);
6862 }
6863 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GSub(L: m_Reg(), R: m_Reg()))) {
6864 // TODO
6865
6866 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6867 // TODO
6868 }
6869
6870 return std::pair(Root.getReg(), 0);
6871}
6872
6873InstructionSelector::ComplexRendererFns
6874AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
6875 Register Reg;
6876 int64_t Offset;
6877 std::tie(args&: Reg, args&: Offset) = selectDS1Addr1OffsetImpl(Root);
6878 return {{
6879 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6880 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }
6881 }};
6882}
6883
6884InstructionSelector::ComplexRendererFns
6885AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const {
6886 return selectDSReadWrite2(Root, size: 4);
6887}
6888
6889InstructionSelector::ComplexRendererFns
6890AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(MachineOperand &Root) const {
6891 return selectDSReadWrite2(Root, size: 8);
6892}
6893
6894InstructionSelector::ComplexRendererFns
6895AMDGPUInstructionSelector::selectDSReadWrite2(MachineOperand &Root,
6896 unsigned Size) const {
6897 Register Reg;
6898 int64_t Offset;
6899 std::tie(args&: Reg, args&: Offset) = selectDSReadWrite2Impl(Root, size: Size);
6900 return {{
6901 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6902 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
6903 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset+1); }
6904 }};
6905}
6906
6907std::pair<Register, int64_t>
6908AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root,
6909 unsigned Size) const {
6910 int64_t ConstAddr = 0;
6911
6912 Register PtrBase;
6913 int64_t Offset;
6914 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6915 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6916
6917 if (Offset) {
6918 int64_t OffsetValue0 = Offset;
6919 int64_t OffsetValue1 = Offset + Size;
6920 if (isDSOffset2Legal(Base: PtrBase, Offset0: OffsetValue0, Offset1: OffsetValue1, Size)) {
6921 // (add n0, c0)
6922 return std::pair(PtrBase, OffsetValue0 / Size);
6923 }
6924 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GSub(L: m_Reg(), R: m_Reg()))) {
6925 // TODO
6926
6927 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6928 // TODO
6929 }
6930
6931 return std::pair(Root.getReg(), 0);
6932}
6933
6934/// If \p Root is a G_PTR_ADD with a G_CONSTANT on the right hand side, return
6935/// the base value with the constant offset, and if the offset computation is
6936/// known to be inbounds. There may be intervening copies between \p Root and
6937/// the identified constant. Returns \p Root, 0, false if this does not match
6938/// the pattern.
6939std::tuple<Register, int64_t, bool>
6940AMDGPUInstructionSelector::getPtrBaseWithConstantOffset(
6941 Register Root, const MachineRegisterInfo &MRI) const {
6942 MachineInstr *RootI = getDefIgnoringCopies(Reg: Root, MRI);
6943 if (RootI->getOpcode() != TargetOpcode::G_PTR_ADD)
6944 return {Root, 0, false};
6945
6946 MachineOperand &RHS = RootI->getOperand(i: 2);
6947 std::optional<ValueAndVReg> MaybeOffset =
6948 getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI);
6949 if (!MaybeOffset)
6950 return {Root, 0, false};
6951 bool IsInBounds = RootI->getFlag(Flag: MachineInstr::MIFlag::InBounds);
6952 return {RootI->getOperand(i: 1).getReg(), MaybeOffset->Value.getSExtValue(),
6953 IsInBounds};
6954}
6955
6956static void addZeroImm(MachineInstrBuilder &MIB) {
6957 MIB.addImm(Val: 0);
6958}
6959
6960/// Return a resource descriptor for use with an arbitrary 64-bit pointer. If \p
6961/// BasePtr is not valid, a null base pointer will be used.
6962static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6963 uint32_t FormatLo, uint32_t FormatHi,
6964 Register BasePtr) {
6965 Register RSrc2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6966 Register RSrc3 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6967 Register RSrcHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6968 Register RSrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
6969
6970 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6971 .addDef(RegNo: RSrc2)
6972 .addImm(Val: FormatLo);
6973 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6974 .addDef(RegNo: RSrc3)
6975 .addImm(Val: FormatHi);
6976
6977 // Build the half of the subregister with the constants before building the
6978 // full 128-bit register. If we are building multiple resource descriptors,
6979 // this will allow CSEing of the 2-component register.
6980 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6981 .addDef(RegNo: RSrcHi)
6982 .addReg(RegNo: RSrc2)
6983 .addImm(Val: AMDGPU::sub0)
6984 .addReg(RegNo: RSrc3)
6985 .addImm(Val: AMDGPU::sub1);
6986
6987 Register RSrcLo = BasePtr;
6988 if (!BasePtr) {
6989 RSrcLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6990 B.buildInstr(Opcode: AMDGPU::S_MOV_B64)
6991 .addDef(RegNo: RSrcLo)
6992 .addImm(Val: 0);
6993 }
6994
6995 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6996 .addDef(RegNo: RSrc)
6997 .addReg(RegNo: RSrcLo)
6998 .addImm(Val: AMDGPU::sub0_sub1)
6999 .addReg(RegNo: RSrcHi)
7000 .addImm(Val: AMDGPU::sub2_sub3);
7001
7002 return RSrc;
7003}
7004
7005static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
7006 const SIInstrInfo &TII, Register BasePtr) {
7007 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
7008
7009 // FIXME: Why are half the "default" bits ignored based on the addressing
7010 // mode?
7011 return buildRSRC(B, MRI, FormatLo: 0, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
7012}
7013
7014static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
7015 const SIInstrInfo &TII, Register BasePtr) {
7016 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
7017
7018 // FIXME: Why are half the "default" bits ignored based on the addressing
7019 // mode?
7020 return buildRSRC(B, MRI, FormatLo: -1, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
7021}
7022
7023AMDGPUInstructionSelector::MUBUFAddressData
7024AMDGPUInstructionSelector::parseMUBUFAddress(Register Src) const {
7025 MUBUFAddressData Data;
7026 Data.N0 = Src;
7027
7028 Register PtrBase;
7029 int64_t Offset;
7030
7031 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
7032 getPtrBaseWithConstantOffset(Root: Src, MRI: *MRI);
7033 if (isUInt<32>(x: Offset)) {
7034 Data.N0 = PtrBase;
7035 Data.Offset = Offset;
7036 }
7037
7038 if (MachineInstr *InputAdd
7039 = getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Data.N0, MRI: *MRI)) {
7040 Data.N2 = InputAdd->getOperand(i: 1).getReg();
7041 Data.N3 = InputAdd->getOperand(i: 2).getReg();
7042
7043 // FIXME: Need to fix extra SGPR->VGPRcopies inserted
7044 // FIXME: Don't know this was defined by operand 0
7045 //
7046 // TODO: Remove this when we have copy folding optimizations after
7047 // RegBankSelect.
7048 Data.N2 = getDefIgnoringCopies(Reg: Data.N2, MRI: *MRI)->getOperand(i: 0).getReg();
7049 Data.N3 = getDefIgnoringCopies(Reg: Data.N3, MRI: *MRI)->getOperand(i: 0).getReg();
7050 }
7051
7052 return Data;
7053}
7054
7055/// Return if the addr64 mubuf mode should be used for the given address.
7056bool AMDGPUInstructionSelector::shouldUseAddr64(MUBUFAddressData Addr) const {
7057 // (ptr_add N2, N3) -> addr64, or
7058 // (ptr_add (ptr_add N2, N3), C1) -> addr64
7059 if (Addr.N2)
7060 return true;
7061
7062 const RegisterBank *N0Bank = RBI.getRegBank(Reg: Addr.N0, MRI: *MRI, TRI);
7063 return N0Bank->getID() == AMDGPU::VGPRRegBankID;
7064}
7065
7066/// Split an immediate offset \p ImmOffset depending on whether it fits in the
7067/// immediate field. Modifies \p ImmOffset and sets \p SOffset to the variable
7068/// component.
7069void AMDGPUInstructionSelector::splitIllegalMUBUFOffset(
7070 MachineIRBuilder &B, Register &SOffset, int64_t &ImmOffset) const {
7071 if (TII.isLegalMUBUFImmOffset(Imm: ImmOffset))
7072 return;
7073
7074 // Illegal offset, store it in soffset.
7075 SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7076 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
7077 .addDef(RegNo: SOffset)
7078 .addImm(Val: ImmOffset);
7079 ImmOffset = 0;
7080}
7081
7082bool AMDGPUInstructionSelector::selectMUBUFAddr64Impl(
7083 MachineOperand &Root, Register &VAddr, Register &RSrcReg,
7084 Register &SOffset, int64_t &Offset) const {
7085 // FIXME: Predicates should stop this from reaching here.
7086 // addr64 bit was removed for volcanic islands.
7087 if (!STI.hasAddr64() || STI.useFlatForGlobal())
7088 return false;
7089
7090 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
7091 if (!shouldUseAddr64(Addr: AddrData))
7092 return false;
7093
7094 Register N0 = AddrData.N0;
7095 Register N2 = AddrData.N2;
7096 Register N3 = AddrData.N3;
7097 Offset = AddrData.Offset;
7098
7099 // Base pointer for the SRD.
7100 Register SRDPtr;
7101
7102 if (N2) {
7103 if (RBI.getRegBank(Reg: N2, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7104 assert(N3);
7105 if (RBI.getRegBank(Reg: N3, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7106 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
7107 // addr64, and construct the default resource from a 0 address.
7108 VAddr = N0;
7109 } else {
7110 SRDPtr = N3;
7111 VAddr = N2;
7112 }
7113 } else {
7114 // N2 is not divergent.
7115 SRDPtr = N2;
7116 VAddr = N3;
7117 }
7118 } else if (RBI.getRegBank(Reg: N0, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
7119 // Use the default null pointer in the resource
7120 VAddr = N0;
7121 } else {
7122 // N0 -> offset, or
7123 // (N0 + C1) -> offset
7124 SRDPtr = N0;
7125 }
7126
7127 MachineIRBuilder B(*Root.getParent());
7128 RSrcReg = buildAddr64RSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
7129 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
7130 return true;
7131}
7132
7133bool AMDGPUInstructionSelector::selectMUBUFOffsetImpl(
7134 MachineOperand &Root, Register &RSrcReg, Register &SOffset,
7135 int64_t &Offset) const {
7136
7137 // FIXME: Pattern should not reach here.
7138 if (STI.useFlatForGlobal())
7139 return false;
7140
7141 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
7142 if (shouldUseAddr64(Addr: AddrData))
7143 return false;
7144
7145 // N0 -> offset, or
7146 // (N0 + C1) -> offset
7147 Register SRDPtr = AddrData.N0;
7148 Offset = AddrData.Offset;
7149
7150 // TODO: Look through extensions for 32-bit soffset.
7151 MachineIRBuilder B(*Root.getParent());
7152
7153 RSrcReg = buildOffsetSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
7154 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
7155 return true;
7156}
7157
7158InstructionSelector::ComplexRendererFns
7159AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const {
7160 Register VAddr;
7161 Register RSrcReg;
7162 Register SOffset;
7163 int64_t Offset = 0;
7164
7165 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset))
7166 return {};
7167
7168 // FIXME: Use defaulted operands for trailing 0s and remove from the complex
7169 // pattern.
7170 return {{
7171 [=](MachineInstrBuilder &MIB) { // rsrc
7172 MIB.addReg(RegNo: RSrcReg);
7173 },
7174 [=](MachineInstrBuilder &MIB) { // vaddr
7175 MIB.addReg(RegNo: VAddr);
7176 },
7177 [=](MachineInstrBuilder &MIB) { // soffset
7178 if (SOffset)
7179 MIB.addReg(RegNo: SOffset);
7180 else if (STI.hasRestrictedSOffset())
7181 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7182 else
7183 MIB.addImm(Val: 0);
7184 },
7185 [=](MachineInstrBuilder &MIB) { // offset
7186 MIB.addImm(Val: Offset);
7187 },
7188 addZeroImm, // cpol
7189 addZeroImm, // tfe
7190 addZeroImm // swz
7191 }};
7192}
7193
7194InstructionSelector::ComplexRendererFns
7195AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const {
7196 Register RSrcReg;
7197 Register SOffset;
7198 int64_t Offset = 0;
7199
7200 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset))
7201 return {};
7202
7203 return {{
7204 [=](MachineInstrBuilder &MIB) { // rsrc
7205 MIB.addReg(RegNo: RSrcReg);
7206 },
7207 [=](MachineInstrBuilder &MIB) { // soffset
7208 if (SOffset)
7209 MIB.addReg(RegNo: SOffset);
7210 else if (STI.hasRestrictedSOffset())
7211 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7212 else
7213 MIB.addImm(Val: 0);
7214 },
7215 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }, // offset
7216 addZeroImm, // cpol
7217 addZeroImm, // tfe
7218 addZeroImm, // swz
7219 }};
7220}
7221
7222InstructionSelector::ComplexRendererFns
7223AMDGPUInstructionSelector::selectBUFSOffset(MachineOperand &Root) const {
7224
7225 Register SOffset = Root.getReg();
7226
7227 if (STI.hasRestrictedSOffset() && mi_match(R: SOffset, MRI: *MRI, P: m_ZeroInt()))
7228 SOffset = AMDGPU::SGPR_NULL;
7229
7230 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); }}};
7231}
7232
7233/// Get an immediate that must be 32-bits, and treated as zero extended.
7234static std::optional<uint64_t>
7235getConstantZext32Val(Register Reg, const MachineRegisterInfo &MRI) {
7236 // getIConstantVRegVal sexts any values, so see if that matters.
7237 std::optional<int64_t> OffsetVal = getIConstantVRegSExtVal(VReg: Reg, MRI);
7238 if (!OffsetVal || !isInt<32>(x: *OffsetVal))
7239 return std::nullopt;
7240 return Lo_32(Value: *OffsetVal);
7241}
7242
7243InstructionSelector::ComplexRendererFns
7244AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const {
7245 std::optional<uint64_t> OffsetVal =
7246 Root.isImm() ? Root.getImm() : getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7247 if (!OffsetVal)
7248 return {};
7249
7250 std::optional<int64_t> EncodedImm =
7251 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: *OffsetVal, IsBuffer: true);
7252 if (!EncodedImm)
7253 return {};
7254
7255 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7256}
7257
7258InstructionSelector::ComplexRendererFns
7259AMDGPUInstructionSelector::selectSMRDBufferImm32(MachineOperand &Root) const {
7260 assert(STI.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
7261
7262 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7263 if (!OffsetVal)
7264 return {};
7265
7266 std::optional<int64_t> EncodedImm =
7267 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: *OffsetVal);
7268 if (!EncodedImm)
7269 return {};
7270
7271 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7272}
7273
7274InstructionSelector::ComplexRendererFns
7275AMDGPUInstructionSelector::selectSMRDBufferSgprImm(MachineOperand &Root) const {
7276 // Match the (soffset + offset) pair as a 32-bit register base and
7277 // an immediate offset.
7278 Register SOffset;
7279 unsigned Offset;
7280 std::tie(args&: SOffset, args&: Offset) = AMDGPU::getBaseWithConstantOffset(
7281 MRI&: *MRI, Reg: Root.getReg(), ValueTracking: VT, /*CheckNUW*/ true);
7282 if (!SOffset)
7283 return std::nullopt;
7284
7285 std::optional<int64_t> EncodedOffset =
7286 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: Offset, /* IsBuffer */ true);
7287 if (!EncodedOffset)
7288 return std::nullopt;
7289
7290 assert(MRI->getType(SOffset).getSizeInBits() == 32);
7291 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
7292 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedOffset); }}};
7293}
7294
7295std::pair<Register, unsigned>
7296AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(MachineOperand &Root,
7297 bool &Matched) const {
7298 Matched = false;
7299
7300 Register Src;
7301 unsigned Mods;
7302 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
7303
7304 if (mi_match(R: Src, MRI: *MRI, P: m_GFPExt(Src: m_Reg(R&: Src)))) {
7305 assert(MRI->getType(Src) == LLT::scalar(16));
7306
7307 // Only change Src if src modifier could be gained. In such cases new Src
7308 // could be sgpr but this does not violate constant bus restriction for
7309 // instruction that is being selected.
7310 Src = stripBitCast(Reg: Src, MRI&: *MRI);
7311
7312 const auto CheckAbsNeg = [&]() {
7313 // Be careful about folding modifiers if we already have an abs. fneg is
7314 // applied last, so we don't want to apply an earlier fneg.
7315 if ((Mods & SISrcMods::ABS) == 0) {
7316 unsigned ModsTmp;
7317 std::tie(args&: Src, args&: ModsTmp) = selectVOP3ModsImpl(Src);
7318
7319 if ((ModsTmp & SISrcMods::NEG) != 0)
7320 Mods ^= SISrcMods::NEG;
7321
7322 if ((ModsTmp & SISrcMods::ABS) != 0)
7323 Mods |= SISrcMods::ABS;
7324 }
7325 };
7326
7327 CheckAbsNeg();
7328
7329 // op_sel/op_sel_hi decide the source type and source.
7330 // If the source's op_sel_hi is set, it indicates to do a conversion from
7331 // fp16. If the sources's op_sel is set, it picks the high half of the
7332 // source register.
7333
7334 Mods |= SISrcMods::OP_SEL_1;
7335
7336 // The instruction reads a 32-bit source and selects a half of it, so look
7337 // for the 32-bit register the 16-bit value is a half of.
7338 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
7339 // Src is now the 32-bit source and op_sel picks its high half.
7340 Mods |= SISrcMods::OP_SEL_0;
7341 CheckAbsNeg();
7342 } else if (isExtractLoElt(MRI&: *MRI, In: Src, Out&: Src)) {
7343 // op_sel already picks the low half, so the 32-bit source can be used
7344 // directly. Unlike the high half, only an fneg/fabs that acts on each
7345 // 16-bit element can be folded here: one that acts on the 32-bit value
7346 // touches bit 31 and leaves the low half alone.
7347 if (MRI->getType(Reg: Src).isFixedVector(NumElements: 2, ScalarSize: 16))
7348 CheckAbsNeg();
7349 }
7350 // Otherwise Src is genuinely 16 bits wide and widenSrcIfVGPR16 widens it
7351 // when the operand is rendered.
7352
7353 Matched = true;
7354 }
7355
7356 return {Src, Mods};
7357}
7358
7359InstructionSelector::ComplexRendererFns
7360AMDGPUInstructionSelector::selectVOP3PMadMixModsExt(
7361 MachineOperand &Root) const {
7362 Register Src;
7363 unsigned Mods;
7364 bool Matched;
7365 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7366 if (!Matched)
7367 return {};
7368
7369 return {{
7370 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7371 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7372 }};
7373}
7374
7375InstructionSelector::ComplexRendererFns
7376AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const {
7377 Register Src;
7378 unsigned Mods;
7379 bool Matched;
7380 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7381
7382 return {{
7383 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7384 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7385 }};
7386}
7387
7388InstructionSelector::ComplexRendererFns
7389AMDGPUInstructionSelector::selectVOP3PMadMixModsExtNeg(
7390 MachineOperand &Root) const {
7391 Register Src;
7392 unsigned Mods;
7393 bool Matched;
7394 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7395 if (!Matched)
7396 return {};
7397
7398 return {{
7399 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7400 [=](MachineInstrBuilder &MIB) {
7401 MIB.addImm(Val: Mods ^ SISrcMods::NEG);
7402 } // src_mods
7403 }};
7404}
7405
7406InstructionSelector::ComplexRendererFns
7407AMDGPUInstructionSelector::selectVOP3PMadMixModsNeg(
7408 MachineOperand &Root) const {
7409 Register Src;
7410 unsigned Mods;
7411 bool Matched;
7412 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7413
7414 return {{
7415 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: widenSrcIfVGPR16(Src, InsertPt: MIB)); },
7416 [=](MachineInstrBuilder &MIB) {
7417 MIB.addImm(Val: Mods ^ SISrcMods::NEG);
7418 } // src_mods
7419 }};
7420}
7421
7422bool AMDGPUInstructionSelector::selectSBarrierSignalIsfirst(
7423 MachineInstr &I, Intrinsic::ID IntrID) const {
7424 MachineBasicBlock *MBB = I.getParent();
7425 const DebugLoc &DL = I.getDebugLoc();
7426 Register CCReg = I.getOperand(i: 0).getReg();
7427
7428 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7429 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_EQ_U32)).addImm(Val: 0).addImm(Val: 0);
7430
7431 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM))
7432 .addImm(Val: I.getOperand(i: 2).getImm());
7433
7434 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg).addReg(RegNo: AMDGPU::SCC);
7435
7436 I.eraseFromParent();
7437 return RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32_XM0_XEXECRegClass,
7438 MRI&: *MRI);
7439}
7440
7441bool AMDGPUInstructionSelector::selectSGetBarrierState(
7442 MachineInstr &I, Intrinsic::ID IntrID) const {
7443 MachineBasicBlock *MBB = I.getParent();
7444 const DebugLoc &DL = I.getDebugLoc();
7445 const MachineOperand &BarOp = I.getOperand(i: 2);
7446 std::optional<int64_t> BarValImm =
7447 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7448
7449 if (!BarValImm) {
7450 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7451 .addReg(RegNo: BarOp.getReg());
7452 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7453 }
7454 MachineInstrBuilder MIB;
7455 unsigned Opc = BarValImm ? AMDGPU::S_GET_BARRIER_STATE_IMM
7456 : AMDGPU::S_GET_BARRIER_STATE_M0;
7457 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7458
7459 auto DstReg = I.getOperand(i: 0).getReg();
7460 const TargetRegisterClass *DstRC =
7461 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
7462 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7463 return false;
7464 MIB.addDef(RegNo: DstReg);
7465 if (BarValImm) {
7466 MIB.addImm(Val: *BarValImm);
7467 }
7468 I.eraseFromParent();
7469 return true;
7470}
7471
7472/// Return the barrier ID for \p BarOp if it's statically known, mapping poison
7473/// to 0 and masking off the high bits, since they're ignored as reserved.
7474static std::optional<uint32_t>
7475getConstantNamedBarrierNumber(const MachineOperand &BarOp,
7476 const MachineRegisterInfo &MRI) {
7477 Register BarReg = BarOp.getReg();
7478 if (std::optional<int64_t> BarValImm = getIConstantVRegSExtVal(VReg: BarReg, MRI))
7479 return *BarValImm & 0x3F;
7480 if (getOpcodeDef(Opcode: AMDGPU::G_IMPLICIT_DEF, Reg: BarReg, MRI))
7481 return 0;
7482 return std::nullopt;
7483}
7484
7485unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID) {
7486 if (HasInlineConst) {
7487 switch (IntrID) {
7488 default:
7489 llvm_unreachable("not a named barrier op");
7490 case Intrinsic::amdgcn_s_barrier_join:
7491 return AMDGPU::S_BARRIER_JOIN_IMM;
7492 case Intrinsic::amdgcn_s_wakeup_barrier:
7493 return AMDGPU::S_WAKEUP_BARRIER_IMM;
7494 case Intrinsic::amdgcn_s_get_named_barrier_state:
7495 return AMDGPU::S_GET_BARRIER_STATE_IMM;
7496 };
7497 } else {
7498 switch (IntrID) {
7499 default:
7500 llvm_unreachable("not a named barrier op");
7501 case Intrinsic::amdgcn_s_barrier_join:
7502 return AMDGPU::S_BARRIER_JOIN_M0;
7503 case Intrinsic::amdgcn_s_wakeup_barrier:
7504 return AMDGPU::S_WAKEUP_BARRIER_M0;
7505 case Intrinsic::amdgcn_s_get_named_barrier_state:
7506 return AMDGPU::S_GET_BARRIER_STATE_M0;
7507 };
7508 }
7509}
7510
7511bool AMDGPUInstructionSelector::selectNamedBarrierInit(
7512 MachineInstr &I, Intrinsic::ID IntrID) const {
7513 MachineBasicBlock *MBB = I.getParent();
7514 const DebugLoc &DL = I.getDebugLoc();
7515 const MachineOperand &BarOp = I.getOperand(i: 1);
7516 const MachineOperand &CntOp = I.getOperand(i: 2);
7517
7518 // A member count of 0 means "keep existing member count". That plus a known
7519 // constant value for the barrier ID lets us use the immarg form.
7520 if (IntrID == Intrinsic::amdgcn_s_barrier_signal_var) {
7521 std::optional<int64_t> CntImm =
7522 getIConstantVRegSExtVal(VReg: CntOp.getReg(), MRI: *MRI);
7523 if (CntImm && *CntImm == 0) {
7524 if (auto BarID = getConstantNamedBarrierNumber(BarOp, MRI: *MRI)) {
7525 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_IMM))
7526 .addImm(Val: *BarID);
7527 I.eraseFromParent();
7528 return true;
7529 }
7530 }
7531 }
7532
7533 // BarID = BarOp & 0x3F
7534 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7535 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7536 .add(MO: BarOp)
7537 .addImm(Val: 0x3F)
7538 .setOperandDead(3); // Dead scc
7539
7540 // MO = ((CntOp & CntMask) << shAmt) | BarID
7541 unsigned CntMask =
7542 AMDGPU::getBarrierMemberCountBitMask(Version: AMDGPU::getIsaVersion(GPU: STI.getCPU()));
7543 Register TmpReg2 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7544 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg2)
7545 .add(MO: CntOp)
7546 .addImm(Val: CntMask)
7547 .setOperandDead(3); // Dead scc
7548
7549 Register TmpReg3 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7550 constexpr unsigned ShAmt = 16;
7551 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg3)
7552 .addReg(RegNo: TmpReg2)
7553 .addImm(Val: ShAmt)
7554 .setOperandDead(3); // Dead scc
7555
7556 Register TmpReg4 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7557 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B32), DestReg: TmpReg4)
7558 .addReg(RegNo: TmpReg1)
7559 .addReg(RegNo: TmpReg3)
7560 .setOperandDead(3); // Dead scc;
7561
7562 auto CopyMIB =
7563 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0).addReg(RegNo: TmpReg4);
7564 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7565
7566 unsigned Opc = IntrID == Intrinsic::amdgcn_s_barrier_init
7567 ? AMDGPU::S_BARRIER_INIT_M0
7568 : AMDGPU::S_BARRIER_SIGNAL_M0;
7569 MachineInstrBuilder MIB;
7570 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7571
7572 I.eraseFromParent();
7573 return true;
7574}
7575
7576bool AMDGPUInstructionSelector::selectNamedBarrierInst(
7577 MachineInstr &I, Intrinsic::ID IntrID) const {
7578 MachineBasicBlock *MBB = I.getParent();
7579 const DebugLoc &DL = I.getDebugLoc();
7580 MachineOperand BarOp = IntrID == Intrinsic::amdgcn_s_get_named_barrier_state
7581 ? I.getOperand(i: 2)
7582 : I.getOperand(i: 1);
7583 std::optional<uint32_t> BarValImm =
7584 getConstantNamedBarrierNumber(BarOp, MRI: *MRI);
7585
7586 if (!BarValImm) {
7587 // BarID = BarOp & 0x3F
7588 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7589 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7590 .addReg(RegNo: BarOp.getReg())
7591 .addImm(Val: 0x3F)
7592 .setOperandDead(3); // Dead scc;
7593
7594 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7595 .addReg(RegNo: TmpReg1);
7596 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7597 }
7598
7599 MachineInstrBuilder MIB;
7600 unsigned Opc = getNamedBarrierOp(HasInlineConst: BarValImm.has_value(), IntrID);
7601 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7602
7603 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
7604 auto DstReg = I.getOperand(i: 0).getReg();
7605 const TargetRegisterClass *DstRC =
7606 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
7607 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7608 return false;
7609 MIB.addDef(RegNo: DstReg);
7610 }
7611
7612 if (BarValImm)
7613 MIB.addImm(Val: *BarValImm);
7614
7615 I.eraseFromParent();
7616 return true;
7617}
7618
7619void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB,
7620 const MachineInstr &MI,
7621 int OpIdx) const {
7622 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7623 "Expected G_CONSTANT");
7624 MIB.addImm(Val: -MI.getOperand(i: 1).getCImm()->getSExtValue());
7625}
7626
7627void AMDGPUInstructionSelector::renderBitcastFPImm(MachineInstrBuilder &MIB,
7628 const MachineInstr &MI,
7629 int OpIdx) const {
7630 const MachineOperand &Op = MI.getOperand(i: 1);
7631 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1);
7632 MIB.addImm(Val: Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
7633}
7634
7635void AMDGPUInstructionSelector::renderCountTrailingOnesImm(
7636 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7637 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7638 "Expected G_CONSTANT");
7639 MIB.addImm(Val: MI.getOperand(i: 1).getCImm()->getValue().countTrailingOnes());
7640}
7641
7642/// This only really exists to satisfy DAG type checking machinery, so is a
7643/// no-op here.
7644void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB,
7645 const MachineInstr &MI,
7646 int OpIdx) const {
7647 const MachineOperand &Op = MI.getOperand(i: OpIdx);
7648 int64_t Imm;
7649 if (Op.isReg() && mi_match(R: Op.getReg(), MRI: *MRI, P: m_ICst(Cst&: Imm)))
7650 MIB.addImm(Val: Imm);
7651 else
7652 MIB.addImm(Val: Op.getImm());
7653}
7654
7655void AMDGPUInstructionSelector::renderPrefetchLen(MachineInstrBuilder &MIB,
7656 const MachineInstr &MI,
7657 int OpIdx) const {
7658 Register Len = MI.getOperand(i: OpIdx).getReg();
7659 MIB.addImm(Val: *getIConstantVRegSExtVal(VReg: Len, MRI: *MRI) & 0x1f);
7660}
7661
7662void AMDGPUInstructionSelector::renderZextBoolTImm(MachineInstrBuilder &MIB,
7663 const MachineInstr &MI,
7664 int OpIdx) const {
7665 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() != 0);
7666}
7667
7668void AMDGPUInstructionSelector::renderOpSelTImm(MachineInstrBuilder &MIB,
7669 const MachineInstr &MI,
7670 int OpIdx) const {
7671 assert(OpIdx >= 0 && "expected to match an immediate operand");
7672 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7673}
7674
7675void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_0(
7676 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7677 assert(OpIdx >= 0 && "expected to match an immediate operand");
7678 MIB.addImm(
7679 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7680}
7681
7682void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_1(
7683 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7684 assert(OpIdx >= 0 && "expected to match an immediate operand");
7685 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x1)
7686 ? (int64_t)(SISrcMods::OP_SEL_0 | SISrcMods::DST_OP_SEL)
7687 : (int64_t)SISrcMods::DST_OP_SEL);
7688}
7689
7690void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_0(
7691 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7692 assert(OpIdx >= 0 && "expected to match an immediate operand");
7693 MIB.addImm(
7694 Val: (MI.getOperand(i: OpIdx).getImm() & 0x2) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7695}
7696
7697void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_1(
7698 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7699 assert(OpIdx >= 0 && "expected to match an immediate operand");
7700 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7701 ? (int64_t)(SISrcMods::OP_SEL_0)
7702 : 0);
7703}
7704
7705void AMDGPUInstructionSelector::renderDstSelToOpSelXForm(
7706 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7707 assert(OpIdx >= 0 && "expected to match an immediate operand");
7708 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::DST_OP_SEL)
7709 : 0);
7710}
7711
7712void AMDGPUInstructionSelector::renderSrcSelToOpSelXForm(
7713 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7714 assert(OpIdx >= 0 && "expected to match an immediate operand");
7715 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::OP_SEL_0)
7716 : 0);
7717}
7718
7719void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_2_0(
7720 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7721 assert(OpIdx >= 0 && "expected to match an immediate operand");
7722 MIB.addImm(
7723 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7724}
7725
7726void AMDGPUInstructionSelector::renderDstSelToOpSel3XFormXForm(
7727 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7728 assert(OpIdx >= 0 && "expected to match an immediate operand");
7729 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7730 ? (int64_t)SISrcMods::DST_OP_SEL
7731 : 0);
7732}
7733
7734void AMDGPUInstructionSelector::renderExtractCPol(MachineInstrBuilder &MIB,
7735 const MachineInstr &MI,
7736 int OpIdx) const {
7737 assert(OpIdx >= 0 && "expected to match an immediate operand");
7738 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() &
7739 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7740 : AMDGPU::CPol::ALL_pregfx12));
7741}
7742
7743void AMDGPUInstructionSelector::renderExtractSWZ(MachineInstrBuilder &MIB,
7744 const MachineInstr &MI,
7745 int OpIdx) const {
7746 assert(OpIdx >= 0 && "expected to match an immediate operand");
7747 const bool Swizzle = MI.getOperand(i: OpIdx).getImm() &
7748 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::SWZ
7749 : AMDGPU::CPol::SWZ_pregfx12);
7750 MIB.addImm(Val: Swizzle);
7751}
7752
7753void AMDGPUInstructionSelector::renderExtractCpolSetGLC(
7754 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7755 assert(OpIdx >= 0 && "expected to match an immediate operand");
7756 const uint32_t Cpol = MI.getOperand(i: OpIdx).getImm() &
7757 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7758 : AMDGPU::CPol::ALL_pregfx12);
7759 MIB.addImm(Val: Cpol | AMDGPU::CPol::GLC);
7760}
7761
7762void AMDGPUInstructionSelector::renderFPPow2ToExponent(MachineInstrBuilder &MIB,
7763 const MachineInstr &MI,
7764 int OpIdx) const {
7765 const APFloat &APF = MI.getOperand(i: 1).getFPImm()->getValueAPF();
7766 int ExpVal = APF.getExactLog2Abs();
7767 assert(ExpVal != INT_MIN);
7768 MIB.addImm(Val: ExpVal);
7769}
7770
7771void AMDGPUInstructionSelector::renderRoundMode(MachineInstrBuilder &MIB,
7772 const MachineInstr &MI,
7773 int OpIdx) const {
7774 // "round.towardzero" -> TowardZero 0 -> FP_ROUND_ROUND_TO_ZERO 3
7775 // "round.tonearest" -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
7776 // "round.upward" -> TowardPositive 2 -> FP_ROUND_ROUND_TO_INF 1
7777 // "round.downward -> TowardNegative 3 -> FP_ROUND_ROUND_TO_NEGINF 2
7778 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() + 3) % 4);
7779}
7780
7781void AMDGPUInstructionSelector::renderVOP3PModsNeg(MachineInstrBuilder &MIB,
7782 const MachineInstr &MI,
7783 int OpIdx) const {
7784 unsigned Mods = SISrcMods::OP_SEL_1;
7785 if (MI.getOperand(i: OpIdx).getImm())
7786 Mods ^= SISrcMods::NEG;
7787 MIB.addImm(Val: (int64_t)Mods);
7788}
7789
7790void AMDGPUInstructionSelector::renderVOP3PModsNegs(MachineInstrBuilder &MIB,
7791 const MachineInstr &MI,
7792 int OpIdx) const {
7793 unsigned Mods = SISrcMods::OP_SEL_1;
7794 if (MI.getOperand(i: OpIdx).getImm())
7795 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
7796 MIB.addImm(Val: (int64_t)Mods);
7797}
7798
7799void AMDGPUInstructionSelector::renderVOP3PModsNegAbs(MachineInstrBuilder &MIB,
7800 const MachineInstr &MI,
7801 int OpIdx) const {
7802 unsigned Val = static_cast<unsigned>(MI.getOperand(i: OpIdx).getImm());
7803 unsigned Mods = SISrcMods::OP_SEL_1; // default: none
7804 if (Val == 1) // neg
7805 Mods ^= SISrcMods::NEG;
7806 if (Val == 2) // abs
7807 Mods ^= SISrcMods::ABS;
7808 if (Val == 3) // neg and abs
7809 Mods ^= (SISrcMods::NEG | SISrcMods::ABS);
7810 MIB.addImm(Val: (int64_t)Mods);
7811}
7812
7813void AMDGPUInstructionSelector::renderPrefetchLoc(MachineInstrBuilder &MIB,
7814 const MachineInstr &MI,
7815 int OpIdx) const {
7816 uint32_t V = MI.getOperand(i: 2).getImm() & AMDGPU::CPol::SCOPE_MASK;
7817 V = (AMDGPU::CPol::SCOPE_MASK - V) << AMDGPU::CPol::SCOPE_SHIFT;
7818 if (!Subtarget->hasSafeCUPrefetch())
7819 V = std::max(a: V, b: (uint32_t)AMDGPU::CPol::SCOPE_SE); // CU scope is unsafe
7820 MIB.addImm(Val: V);
7821}
7822
7823/// Convert from 2-bit value to enum values used for op_sel* source modifiers.
7824void AMDGPUInstructionSelector::renderScaledMAIIntrinsicOperand(
7825 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7826 int64_t Val = MI.getOperand(i: OpIdx).getImm();
7827 unsigned New = 0;
7828 if (Val & 0x1)
7829 New |= SISrcMods::OP_SEL_0;
7830 if (Val & 0x2)
7831 New |= SISrcMods::OP_SEL_1;
7832 MIB.addImm(Val: New);
7833}
7834
7835bool AMDGPUInstructionSelector::isInlineImmediate(const APInt &Imm) const {
7836 return TII.isInlineConstant(Imm);
7837}
7838
7839bool AMDGPUInstructionSelector::isInlineImmediate(const APFloat &Imm) const {
7840 return TII.isInlineConstant(Imm);
7841}
7842