1//===- AMDGPUInstructionSelector.cpp ----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// AMDGPU.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUInstructionSelector.h"
15#include "AMDGPU.h"
16#include "AMDGPUGlobalISelUtils.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPURegisterBankInfo.h"
19#include "AMDGPUTargetMachine.h"
20#include "SIMachineFunctionInfo.h"
21#include "Utils/AMDGPUBaseInfo.h"
22#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
25#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
26#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/IntrinsicsAMDGPU.h"
30#include <optional>
31
32#define DEBUG_TYPE "amdgpu-isel"
33
34using namespace llvm;
35using namespace MIPatternMatch;
36
37#define GET_GLOBALISEL_IMPL
38#define AMDGPUSubtarget GCNSubtarget
39#include "AMDGPUGenGlobalISel.inc"
40#undef GET_GLOBALISEL_IMPL
41#undef AMDGPUSubtarget
42
43AMDGPUInstructionSelector::AMDGPUInstructionSelector(
44 const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI)
45 : TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI), STI(STI),
46#define GET_GLOBALISEL_PREDICATES_INIT
47#include "AMDGPUGenGlobalISel.inc"
48#undef GET_GLOBALISEL_PREDICATES_INIT
49#define GET_GLOBALISEL_TEMPORARIES_INIT
50#include "AMDGPUGenGlobalISel.inc"
51#undef GET_GLOBALISEL_TEMPORARIES_INIT
52{
53}
54
55const char *AMDGPUInstructionSelector::getName() { return DEBUG_TYPE; }
56
57void AMDGPUInstructionSelector::setupMF(MachineFunction &MF,
58 GISelValueTracking *VT,
59 CodeGenCoverage *CoverageInfo,
60 ProfileSummaryInfo *PSI,
61 BlockFrequencyInfo *BFI) {
62 MRI = &MF.getRegInfo();
63 Subtarget = &MF.getSubtarget<GCNSubtarget>();
64 Subtarget->checkSubtargetFeatures(F: MF.getFunction());
65 InstructionSelector::setupMF(mf&: MF, vt: VT, covinfo: CoverageInfo, psi: PSI, bfi: BFI);
66}
67
68// Return the wave level SGPR base address if this is a wave address.
69static Register getWaveAddress(const MachineInstr *Def) {
70 return Def->getOpcode() == AMDGPU::G_AMDGPU_WAVE_ADDRESS
71 ? Def->getOperand(i: 1).getReg()
72 : Register();
73}
74
75bool AMDGPUInstructionSelector::isVCC(Register Reg,
76 const MachineRegisterInfo &MRI) const {
77 // The verifier is oblivious to s1 being a valid value for wavesize registers.
78 if (Reg.isPhysical())
79 return false;
80
81 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
82 const TargetRegisterClass *RC =
83 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
84 if (RC) {
85 const LLT Ty = MRI.getType(Reg);
86 if (!Ty.isValid() || Ty.getSizeInBits() != 1)
87 return false;
88 // G_TRUNC s1 result is never vcc.
89 return !mi_match(R: Reg, MRI, P: m_GTrunc(Src: m_Reg())) &&
90 RC->hasSuperClassEq(RC: TRI.getBoolRC());
91 }
92
93 const RegisterBank *RB = cast<const RegisterBank *>(Val: RegClassOrBank);
94 return RB->getID() == AMDGPU::VCCRegBankID;
95}
96
97bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(MachineInstr &MI,
98 unsigned NewOpc) const {
99 MI.setDesc(TII.get(Opcode: NewOpc));
100 MI.removeOperand(OpNo: 1); // Remove intrinsic ID.
101 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
102
103 Register DstReg = MI.getOperand(i: 0).getReg();
104 Register SrcReg = MI.getOperand(i: 1).getReg();
105
106 // TODO: This should be legalized to s32 if needed
107 if (MRI->getType(Reg: DstReg) == LLT::scalar(SizeInBits: 1))
108 return false;
109
110 const TargetRegisterClass *DstRC =
111 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
112 const TargetRegisterClass *SrcRC =
113 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
114 if (!DstRC || DstRC != SrcRC)
115 return false;
116
117 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
118 !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
119 return false;
120 const MCInstrDesc &MCID = MI.getDesc();
121 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
122 MI.getOperand(i: 0).setIsEarlyClobber(true);
123 }
124 return true;
125}
126
127bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
128 const DebugLoc &DL = I.getDebugLoc();
129 MachineBasicBlock *BB = I.getParent();
130 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
131
132 Register DstReg = I.getOperand(i: 0).getReg();
133 Register SrcReg = I.getOperand(i: 1).getReg();
134
135 if (isVCC(Reg: DstReg, MRI: *MRI)) {
136 if (SrcReg == AMDGPU::SCC) {
137 const TargetRegisterClass *RC =
138 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
139 if (!RC)
140 return true;
141 return RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI);
142 }
143
144 if (!isVCC(Reg: SrcReg, MRI: *MRI)) {
145 // TODO: Should probably leave the copy and let copyPhysReg expand it.
146 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI))
147 return false;
148
149 const TargetRegisterClass *SrcRC =
150 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
151
152 std::optional<ValueAndVReg> ConstVal =
153 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI, LookThroughInstrs: true);
154 if (ConstVal) {
155 unsigned MovOpc =
156 STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
157 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: DstReg)
158 .addImm(Val: ConstVal->Value.getBoolValue() ? -1 : 0);
159 } else {
160 Register MaskedReg = MRI->createVirtualRegister(RegClass: SrcRC);
161
162 // We can't trust the high bits at this point, so clear them.
163
164 // TODO: Skip masking high bits if def is known boolean.
165
166 if (AMDGPU::getRegBitWidth(RCID: SrcRC->getID()) == 16) {
167 assert(Subtarget->useRealTrue16Insts());
168 const int64_t NoMods = 0;
169 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B16_t16_e64), DestReg: MaskedReg)
170 .addImm(Val: NoMods)
171 .addImm(Val: 1)
172 .addImm(Val: NoMods)
173 .addReg(RegNo: SrcReg)
174 .addImm(Val: NoMods);
175 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U16_t16_e64), DestReg: DstReg)
176 .addImm(Val: NoMods)
177 .addImm(Val: 0)
178 .addImm(Val: NoMods)
179 .addReg(RegNo: MaskedReg)
180 .addImm(Val: NoMods);
181 } else {
182 bool IsSGPR = TRI.isSGPRClass(RC: SrcRC);
183 unsigned AndOpc = IsSGPR ? AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32;
184 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: MaskedReg)
185 .addImm(Val: 1)
186 .addReg(RegNo: SrcReg);
187 if (IsSGPR)
188 And.setOperandDead(3); // Dead scc
189
190 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_NE_U32_e64), DestReg: DstReg)
191 .addImm(Val: 0)
192 .addReg(RegNo: MaskedReg);
193 }
194 }
195
196 if (!MRI->getRegClassOrNull(Reg: SrcReg))
197 MRI->setRegClass(Reg: SrcReg, RC: SrcRC);
198 I.eraseFromParent();
199 return true;
200 }
201
202 const TargetRegisterClass *RC =
203 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
204 if (RC && !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
205 return false;
206
207 return true;
208 }
209
210 for (const MachineOperand &MO : I.operands()) {
211 if (MO.getReg().isPhysical())
212 continue;
213
214 const TargetRegisterClass *RC =
215 TRI.getConstrainedRegClassForReg(Reg: MO.getReg(), MRI: *MRI);
216 if (!RC)
217 continue;
218 RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI);
219 }
220 return true;
221}
222
223bool AMDGPUInstructionSelector::selectCOPY_SCC_VCC(MachineInstr &I) const {
224 const DebugLoc &DL = I.getDebugLoc();
225 MachineBasicBlock *BB = I.getParent();
226 Register VCCReg = I.getOperand(i: 1).getReg();
227 MachineInstr *Cmp;
228
229 // Set SCC as a side effect with S_CMP or S_OR.
230 if (STI.hasScalarCompareEq64()) {
231 unsigned CmpOpc =
232 STI.isWave64() ? AMDGPU::S_CMP_LG_U64 : AMDGPU::S_CMP_LG_U32;
233 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: CmpOpc)).addReg(RegNo: VCCReg).addImm(Val: 0);
234 } else {
235 Register DeadDst = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
236 Cmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B64), DestReg: DeadDst)
237 .addReg(RegNo: VCCReg)
238 .addReg(RegNo: VCCReg);
239 }
240
241 constrainSelectedInstRegOperands(I&: *Cmp, TII, TRI, RBI);
242
243 Register DstReg = I.getOperand(i: 0).getReg();
244 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: AMDGPU::SCC);
245
246 I.eraseFromParent();
247 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
248}
249
250bool AMDGPUInstructionSelector::selectCOPY_VCC_SCC(MachineInstr &I) const {
251 const DebugLoc &DL = I.getDebugLoc();
252 MachineBasicBlock *BB = I.getParent();
253
254 Register DstReg = I.getOperand(i: 0).getReg();
255 Register SrcReg = I.getOperand(i: 1).getReg();
256 std::optional<ValueAndVReg> Arg =
257 getIConstantVRegValWithLookThrough(VReg: I.getOperand(i: 1).getReg(), MRI: *MRI);
258
259 if (Arg) {
260 const int64_t Value = Arg->Value.getZExtValue();
261 if (Value == 0) {
262 unsigned Opcode = STI.isWave64() ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
263 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: DstReg).addImm(Val: 0);
264 } else {
265 assert(Value == 1);
266 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg).addReg(RegNo: TRI.getExec());
267 }
268 I.eraseFromParent();
269 return RBI.constrainGenericRegister(Reg: DstReg, RC: *TRI.getBoolRC(), MRI&: *MRI);
270 }
271
272 // RegBankLegalize ensures that SrcReg is bool in reg (high bits are 0).
273 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC).addReg(RegNo: SrcReg);
274
275 unsigned SelectOpcode =
276 STI.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
277 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
278 .addReg(RegNo: TRI.getExec())
279 .addImm(Val: 0);
280
281 I.eraseFromParent();
282 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
283 return true;
284}
285
286bool AMDGPUInstructionSelector::selectReadAnyLane(MachineInstr &I) const {
287 Register DstReg = I.getOperand(i: 0).getReg();
288 Register SrcReg = I.getOperand(i: 1).getReg();
289
290 const DebugLoc &DL = I.getDebugLoc();
291 MachineBasicBlock *BB = I.getParent();
292
293 auto RFL = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
294 .addReg(RegNo: SrcReg);
295
296 I.eraseFromParent();
297 constrainSelectedInstRegOperands(I&: *RFL, TII, TRI, RBI);
298 return true;
299}
300
301bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
302 const Register DefReg = I.getOperand(i: 0).getReg();
303 const LLT DefTy = MRI->getType(Reg: DefReg);
304
305 // S1 G_PHIs should not be selected in instruction-select, instead:
306 // - divergent S1 G_PHI should go through lane mask merging algorithm
307 // and be fully inst-selected in AMDGPUGlobalISelDivergenceLowering
308 // - uniform S1 G_PHI should be lowered into S32 G_PHI in AMDGPURegBankSelect
309 if (DefTy == LLT::scalar(SizeInBits: 1))
310 return false;
311
312 // TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy)
313
314 const RegClassOrRegBank &RegClassOrBank =
315 MRI->getRegClassOrRegBank(Reg: DefReg);
316
317 const TargetRegisterClass *DefRC =
318 dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank);
319 if (!DefRC) {
320 if (!DefTy.isValid()) {
321 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
322 return false;
323 }
324
325 const RegisterBank &RB = *cast<const RegisterBank *>(Val: RegClassOrBank);
326 DefRC = TRI.getRegClassForTypeOnBank(Ty: DefTy, Bank: RB);
327 if (!DefRC) {
328 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
329 return false;
330 }
331 }
332
333 // If inputs have register bank, assign corresponding reg class.
334 // Note: registers don't need to have the same reg bank.
335 for (unsigned i = 1; i != I.getNumOperands(); i += 2) {
336 const Register SrcReg = I.getOperand(i).getReg();
337
338 const RegisterBank *RB = MRI->getRegBankOrNull(Reg: SrcReg);
339 if (RB) {
340 const LLT SrcTy = MRI->getType(Reg: SrcReg);
341 const TargetRegisterClass *SrcRC =
342 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *RB);
343 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
344 return false;
345 }
346 }
347
348 I.setDesc(TII.get(Opcode: TargetOpcode::PHI));
349 return RBI.constrainGenericRegister(Reg: DefReg, RC: *DefRC, MRI&: *MRI);
350}
351
352MachineOperand
353AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
354 const TargetRegisterClass &SubRC,
355 unsigned SubIdx) const {
356
357 MachineInstr *MI = MO.getParent();
358 MachineBasicBlock *BB = MO.getParent()->getParent();
359 Register DstReg = MRI->createVirtualRegister(RegClass: &SubRC);
360
361 if (MO.isReg()) {
362 unsigned ComposedSubIdx = TRI.composeSubRegIndices(a: MO.getSubReg(), b: SubIdx);
363 Register Reg = MO.getReg();
364 BuildMI(BB&: *BB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
365 .addReg(RegNo: Reg, Flags: {}, SubReg: ComposedSubIdx);
366
367 return MachineOperand::CreateReg(Reg: DstReg, isDef: MO.isDef(), isImp: MO.isImplicit(),
368 isKill: MO.isKill(), isDead: MO.isDead(), isUndef: MO.isUndef(),
369 isEarlyClobber: MO.isEarlyClobber(), SubReg: 0, isDebug: MO.isDebug(),
370 isInternalRead: MO.isInternalRead());
371 }
372
373 assert(MO.isImm());
374
375 APInt Imm(64, MO.getImm());
376
377 switch (SubIdx) {
378 default:
379 llvm_unreachable("do not know to split immediate with this sub index.");
380 case AMDGPU::sub0:
381 return MachineOperand::CreateImm(Val: Imm.getLoBits(numBits: 32).getSExtValue());
382 case AMDGPU::sub1:
383 return MachineOperand::CreateImm(Val: Imm.getHiBits(numBits: 32).getSExtValue());
384 }
385}
386
387static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
388 switch (Opc) {
389 case AMDGPU::G_AND:
390 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
391 case AMDGPU::G_OR:
392 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32;
393 case AMDGPU::G_XOR:
394 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32;
395 default:
396 llvm_unreachable("not a bit op");
397 }
398}
399
400bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
401 Register DstReg = I.getOperand(i: 0).getReg();
402 unsigned Size = RBI.getSizeInBits(Reg: DstReg, MRI: *MRI, TRI);
403
404 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
405 if (DstRB->getID() != AMDGPU::SGPRRegBankID &&
406 DstRB->getID() != AMDGPU::VCCRegBankID)
407 return false;
408
409 bool Is64 = Size > 32 || (DstRB->getID() == AMDGPU::VCCRegBankID &&
410 STI.isWave64());
411 I.setDesc(TII.get(Opcode: getLogicalBitOpcode(Opc: I.getOpcode(), Is64)));
412
413 // Dead implicit-def of scc
414 I.addOperand(Op: MachineOperand::CreateReg(Reg: AMDGPU::SCC, isDef: true, // isDef
415 isImp: true, // isImp
416 isKill: false, // isKill
417 isDead: true)); // isDead
418 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
419 return true;
420}
421
422bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
423 MachineBasicBlock *BB = I.getParent();
424 MachineFunction *MF = BB->getParent();
425 Register DstReg = I.getOperand(i: 0).getReg();
426 const DebugLoc &DL = I.getDebugLoc();
427 LLT Ty = MRI->getType(Reg: DstReg);
428 if (Ty.isVector())
429 return false;
430
431 unsigned Size = Ty.getSizeInBits();
432 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
433 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID;
434 const bool Sub = I.getOpcode() == TargetOpcode::G_SUB;
435
436 if (Size == 32) {
437 if (IsSALU) {
438 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32;
439 MachineInstr *Add =
440 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
441 .add(MO: I.getOperand(i: 1))
442 .add(MO: I.getOperand(i: 2))
443 .setOperandDead(3); // Dead scc
444 I.eraseFromParent();
445 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
446 return true;
447 }
448
449 if (STI.hasAddNoCarryInsts()) {
450 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64;
451 I.setDesc(TII.get(Opcode: Opc));
452 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
453 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
454 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
455 return true;
456 }
457
458 const unsigned Opc = Sub ? AMDGPU::V_SUB_CO_U32_e64 : AMDGPU::V_ADD_CO_U32_e64;
459
460 Register UnusedCarry = MRI->createVirtualRegister(RegClass: TRI.getWaveMaskRegClass());
461 MachineInstr *Add
462 = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
463 .addDef(RegNo: UnusedCarry, Flags: RegState::Dead)
464 .add(MO: I.getOperand(i: 1))
465 .add(MO: I.getOperand(i: 2))
466 .addImm(Val: 0);
467 I.eraseFromParent();
468 constrainSelectedInstRegOperands(I&: *Add, TII, TRI, RBI);
469 return true;
470 }
471
472 assert(!Sub && "illegal sub should not reach here");
473
474 const TargetRegisterClass &RC
475 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass;
476 const TargetRegisterClass &HalfRC
477 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass;
478
479 MachineOperand Lo1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
480 MachineOperand Lo2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub0));
481 MachineOperand Hi1(getSubOperand64(MO&: I.getOperand(i: 1), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
482 MachineOperand Hi2(getSubOperand64(MO&: I.getOperand(i: 2), SubRC: HalfRC, SubIdx: AMDGPU::sub1));
483
484 Register DstLo = MRI->createVirtualRegister(RegClass: &HalfRC);
485 Register DstHi = MRI->createVirtualRegister(RegClass: &HalfRC);
486
487 if (IsSALU) {
488 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_U32), DestReg: DstLo)
489 .add(MO: Lo1)
490 .add(MO: Lo2);
491 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADDC_U32), DestReg: DstHi)
492 .add(MO: Hi1)
493 .add(MO: Hi2)
494 .setOperandDead(3); // Dead scc
495 } else {
496 const TargetRegisterClass *CarryRC = TRI.getWaveMaskRegClass();
497 Register CarryReg = MRI->createVirtualRegister(RegClass: CarryRC);
498 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADD_CO_U32_e64), DestReg: DstLo)
499 .addDef(RegNo: CarryReg)
500 .add(MO: Lo1)
501 .add(MO: Lo2)
502 .addImm(Val: 0);
503 MachineInstr *Addc = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_ADDC_U32_e64), DestReg: DstHi)
504 .addDef(RegNo: MRI->createVirtualRegister(RegClass: CarryRC), Flags: RegState::Dead)
505 .add(MO: Hi1)
506 .add(MO: Hi2)
507 .addReg(RegNo: CarryReg, Flags: RegState::Kill)
508 .addImm(Val: 0);
509
510 constrainSelectedInstRegOperands(I&: *Addc, TII, TRI, RBI);
511 }
512
513 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
514 .addReg(RegNo: DstLo)
515 .addImm(Val: AMDGPU::sub0)
516 .addReg(RegNo: DstHi)
517 .addImm(Val: AMDGPU::sub1);
518
519
520 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
521 return false;
522
523 I.eraseFromParent();
524 return true;
525}
526
527bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE(
528 MachineInstr &I) const {
529 MachineBasicBlock *BB = I.getParent();
530 MachineFunction *MF = BB->getParent();
531 const DebugLoc &DL = I.getDebugLoc();
532 Register Dst0Reg = I.getOperand(i: 0).getReg();
533 Register Dst1Reg = I.getOperand(i: 1).getReg();
534 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO ||
535 I.getOpcode() == AMDGPU::G_UADDE;
536 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE ||
537 I.getOpcode() == AMDGPU::G_USUBE;
538
539 if (isVCC(Reg: Dst1Reg, MRI: *MRI)) {
540 unsigned NoCarryOpc =
541 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
542 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
543 I.setDesc(TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc));
544 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
545 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
546 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
547 return true;
548 }
549
550 Register Src0Reg = I.getOperand(i: 2).getReg();
551 Register Src1Reg = I.getOperand(i: 3).getReg();
552
553 if (HasCarryIn) {
554 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
555 .addReg(RegNo: I.getOperand(i: 4).getReg());
556 }
557
558 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
559 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
560
561 auto CarryInst = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: HasCarryIn ? CarryOpc : NoCarryOpc), DestReg: Dst0Reg)
562 .add(MO: I.getOperand(i: 2))
563 .add(MO: I.getOperand(i: 3));
564
565 if (MRI->use_nodbg_empty(RegNo: Dst1Reg)) {
566 CarryInst.setOperandDead(3); // Dead scc
567 } else {
568 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst1Reg)
569 .addReg(RegNo: AMDGPU::SCC);
570 if (!MRI->getRegClassOrNull(Reg: Dst1Reg))
571 MRI->setRegClass(Reg: Dst1Reg, RC: &AMDGPU::SReg_32RegClass);
572 }
573
574 if (!RBI.constrainGenericRegister(Reg: Dst0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
575 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
576 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
577 return false;
578
579 if (HasCarryIn &&
580 !RBI.constrainGenericRegister(Reg: I.getOperand(i: 4).getReg(),
581 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
582 return false;
583
584 I.eraseFromParent();
585 return true;
586}
587
588bool AMDGPUInstructionSelector::selectG_AMDGPU_MAD_64_32(
589 MachineInstr &I) const {
590 MachineBasicBlock *BB = I.getParent();
591 MachineFunction *MF = BB->getParent();
592 const bool IsUnsigned = I.getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32;
593 bool UseNoCarry = Subtarget->hasMadNC64_32Insts() &&
594 MRI->use_nodbg_empty(RegNo: I.getOperand(i: 1).getReg());
595
596 unsigned Opc;
597 if (Subtarget->hasMADIntraFwdBug())
598 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_gfx11_e64
599 : AMDGPU::V_MAD_I64_I32_gfx11_e64;
600 else if (UseNoCarry)
601 Opc = IsUnsigned ? AMDGPU::V_MAD_NC_U64_U32_e64
602 : AMDGPU::V_MAD_NC_I64_I32_e64;
603 else
604 Opc = IsUnsigned ? AMDGPU::V_MAD_U64_U32_e64 : AMDGPU::V_MAD_I64_I32_e64;
605
606 if (UseNoCarry)
607 I.removeOperand(OpNo: 1);
608
609 I.setDesc(TII.get(Opcode: Opc));
610 I.addOperand(MF&: *MF, Op: MachineOperand::CreateImm(Val: 0));
611 I.addImplicitDefUseOperands(MF&: *MF);
612 I.getOperand(i: 0).setIsEarlyClobber(true);
613 constrainSelectedInstRegOperands(I, TII, TRI, RBI);
614 return true;
615}
616
617// TODO: We should probably legalize these to only using 32-bit results.
618bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
619 MachineBasicBlock *BB = I.getParent();
620 Register DstReg = I.getOperand(i: 0).getReg();
621 Register SrcReg = I.getOperand(i: 1).getReg();
622 LLT DstTy = MRI->getType(Reg: DstReg);
623 LLT SrcTy = MRI->getType(Reg: SrcReg);
624 const unsigned SrcSize = SrcTy.getSizeInBits();
625 unsigned DstSize = DstTy.getSizeInBits();
626
627 // TODO: Should handle any multiple of 32 offset.
628 unsigned Offset = I.getOperand(i: 2).getImm();
629 if (Offset % 32 != 0 || DstSize > 128)
630 return false;
631
632 // 16-bit operations really use 32-bit registers.
633 // FIXME: Probably should not allow 16-bit G_EXTRACT results.
634 if (DstSize == 16)
635 DstSize = 32;
636
637 const TargetRegisterClass *DstRC =
638 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
639 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
640 return false;
641
642 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
643 const TargetRegisterClass *SrcRC =
644 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
645 if (!SrcRC)
646 return false;
647 unsigned SubReg = SIRegisterInfo::getSubRegFromChannel(Channel: Offset / 32,
648 NumRegs: DstSize / 32);
649 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubReg);
650 if (!SrcRC)
651 return false;
652
653 SrcReg = constrainOperandRegClass(MF: *MF, TRI, MRI&: *MRI, TII, RBI, InsertPt&: I,
654 RegClass: *SrcRC, RegMO&: I.getOperand(i: 1));
655 const DebugLoc &DL = I.getDebugLoc();
656 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
657 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
658
659 I.eraseFromParent();
660 return true;
661}
662
663bool AMDGPUInstructionSelector::selectS16MergeToS32(MachineInstr &MI) const {
664 Register Dst = MI.getOperand(i: 0).getReg();
665 Register Src0 = MI.getOperand(i: 1).getReg();
666 Register Src1 = MI.getOperand(i: 2).getReg();
667
668 LLT Src0Ty = MRI->getType(Reg: Src0);
669 LLT Src1Ty = MRI->getType(Reg: Src1);
670
671 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
672 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0, MRI: *MRI, TRI);
673 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1, MRI: *MRI, TRI);
674 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
675
676 Register ShiftSrc0;
677 Register ShiftSrc1;
678
679 const DebugLoc &DL = MI.getDebugLoc();
680 MachineBasicBlock *BB = MI.getParent();
681
682 // VGPR case
683 if (IsVector) {
684 // If source are both VGPR16, use REG_SEQUENCE with lo16/hi16 subregisters
685 if (Src0Bank->getID() == AMDGPU::VGPRRegBankID &&
686 Src1Bank->getID() == AMDGPU::VGPRRegBankID &&
687 Src0Ty == LLT::scalar(SizeInBits: 16) && Src1Ty == LLT::scalar(SizeInBits: 16)) {
688 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
689 .addReg(RegNo: Src0)
690 .addImm(Val: AMDGPU::lo16)
691 .addReg(RegNo: Src1)
692 .addImm(Val: AMDGPU::hi16);
693
694 if (!RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
695 return false;
696
697 MI.eraseFromParent();
698 return true;
699 }
700
701 // Otherwise, use V_LSHL_OR_B32_e64
702 Register TmpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
703 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: TmpReg)
704 .addImm(Val: 0xFFFF)
705 .addReg(RegNo: Src0);
706 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
707
708 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHL_OR_B32_e64), DestReg: Dst)
709 .addReg(RegNo: Src1)
710 .addImm(Val: 16)
711 .addReg(RegNo: TmpReg);
712 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
713
714 MI.eraseFromParent();
715 return true;
716 }
717
718 // SGPR case -> S_PACK_*_B32_B16
719 // With multiple uses of the shift, this will duplicate the shift and
720 // increase register pressure.
721 //
722 // (merge (lshr_oneuse $src0, 16), (lshr_oneuse $src1, 16)
723 // => (S_PACK_HH_B32_B16 $src0, $src1)
724 // (merge (lshr_oneuse SReg_32:$src0, 16), $src1)
725 // => (S_PACK_HL_B32_B16 $src0, $src1)
726 // (merge $src0, (lshr_oneuse SReg_32:$src1, 16))
727 // => (S_PACK_LH_B32_B16 $src0, $src1)
728 // (merge $src0, $src1)
729 // => (S_PACK_LL_B32_B16 $src0, $src1)
730
731 bool Shift0 = mi_match(
732 R: Src0, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc0), R: m_SpecificICst(RequestedValue: 16))));
733
734 bool Shift1 = mi_match(
735 R: Src1, MRI: *MRI, P: m_OneUse(SP: m_GLShr(L: m_Reg(R&: ShiftSrc1), R: m_SpecificICst(RequestedValue: 16))));
736
737 unsigned Opc = AMDGPU::S_PACK_LL_B32_B16;
738 if (Shift0 && Shift1) {
739 Opc = AMDGPU::S_PACK_HH_B32_B16;
740 MI.getOperand(i: 1).setReg(ShiftSrc0);
741 MI.getOperand(i: 2).setReg(ShiftSrc1);
742 } else if (Shift1) {
743 Opc = AMDGPU::S_PACK_LH_B32_B16;
744 MI.getOperand(i: 2).setReg(ShiftSrc1);
745 } else if (Shift0) {
746 auto ConstSrc1 =
747 getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
748 if (ConstSrc1 && ConstSrc1->Value == 0) {
749 // build_vector_trunc (lshr $src0, 16), 0 -> s_lshr_b32 $src0, 16
750 auto MIB = BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: Dst)
751 .addReg(RegNo: ShiftSrc0)
752 .addImm(Val: 16)
753 .setOperandDead(3); // Dead scc
754
755 MI.eraseFromParent();
756 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
757 return true;
758 }
759 if (STI.hasSPackHL()) {
760 Opc = AMDGPU::S_PACK_HL_B32_B16;
761 MI.getOperand(i: 1).setReg(ShiftSrc0);
762 }
763 }
764
765 MI.setDesc(TII.get(Opcode: Opc));
766 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
767 return true;
768}
769
770// Pack each pair of s16 into an s32 with S_PACK_LL_B32_B16, then combine the
771// s32 pieces into the destination with a REG_SEQUENCE.
772bool AMDGPUInstructionSelector::selectS16MergeToWide(MachineInstr &MI) const {
773 MachineBasicBlock *BB = MI.getParent();
774 const DebugLoc &DL = MI.getDebugLoc();
775 Register DstReg = MI.getOperand(i: 0).getReg();
776 const unsigned DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
777 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
778 const unsigned NumSrc = MI.getNumOperands() - 1;
779
780 // Pack each pair of s16 sources into an s32.
781 SmallVector<Register, 8> S32Regs;
782 for (unsigned I = 0; I != NumSrc; I += 2) {
783 Register S32 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
784 auto Pack = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_PACK_LL_B32_B16), DestReg: S32)
785 .addReg(RegNo: MI.getOperand(i: I + 1).getReg())
786 .addReg(RegNo: MI.getOperand(i: I + 2).getReg());
787 constrainSelectedInstRegOperands(I&: *Pack, TII, TRI, RBI);
788 S32Regs.push_back(Elt: S32);
789 }
790
791 // Combine the s32 pieces into the destination with a REG_SEQUENCE.
792 const TargetRegisterClass *DstRC =
793 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
794 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
795 return false;
796 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, /*EltSize=*/4);
797 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
798 for (unsigned I = 0, E = S32Regs.size(); I != E; ++I)
799 MIB.addReg(RegNo: S32Regs[I]).addImm(Val: SubRegs[I]);
800
801 MI.eraseFromParent();
802 return true;
803}
804
805bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
806 MachineBasicBlock *BB = MI.getParent();
807 Register DstReg = MI.getOperand(i: 0).getReg();
808 LLT DstTy = MRI->getType(Reg: DstReg);
809 LLT SrcTy = MRI->getType(Reg: MI.getOperand(i: 1).getReg());
810
811 const unsigned SrcSize = SrcTy.getSizeInBits();
812 if (SrcSize < 32) {
813 // Handle s32 <- G_MERGE_VALUES s16, s16
814 if (SrcSize == 16 && DstTy.getSizeInBits() == 32 &&
815 MI.getNumOperands() == 3) {
816 return selectS16MergeToS32(MI);
817 }
818 // With true16 a scalar s16 is a register type, so a scalar wider than 32
819 // bits can be built from s16 pieces.
820 bool IsWideS16Merge = SrcSize == 16 && DstTy.getSizeInBits() > 32 &&
821 DstTy.getSizeInBits() % 32 == 0;
822
823 // SGPRs have no 16-bit subregisters, so pack pairs of s16 with S_PACK.
824 if (IsWideS16Merge &&
825 RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
826 return selectS16MergeToWide(MI);
827
828 // A VGPR wide s16 merge falls through to the generic path below.
829 if (!IsWideS16Merge)
830 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
831 }
832
833 const DebugLoc &DL = MI.getDebugLoc();
834 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
835 const unsigned DstSize = DstTy.getSizeInBits();
836 const TargetRegisterClass *DstRC =
837 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
838 if (!DstRC)
839 return false;
840
841 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: DstRC, EltSize: SrcSize / 8);
842 MachineInstrBuilder MIB =
843 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: DstReg);
844 for (int I = 0, E = MI.getNumOperands() - 1; I != E; ++I) {
845 MachineOperand &Src = MI.getOperand(i: I + 1);
846 Register SrcReg = Src.getReg();
847 MIB.addReg(RegNo: SrcReg, Flags: getUndefRegState(B: Src.isUndef()));
848 MIB.addImm(Val: SubRegs[I]);
849
850 const TargetRegisterClass *SrcRC =
851 TRI.getConstrainedRegClassForReg(Reg: SrcReg, MRI: *MRI);
852 if (SrcRC && !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
853 return false;
854 }
855
856 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
857 return false;
858
859 MI.eraseFromParent();
860 return true;
861}
862
863bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
864 MachineBasicBlock *BB = MI.getParent();
865 const int NumDst = MI.getNumOperands() - 1;
866
867 MachineOperand &Src = MI.getOperand(i: NumDst);
868
869 Register SrcReg = Src.getReg();
870 Register DstReg0 = MI.getOperand(i: 0).getReg();
871 LLT DstTy = MRI->getType(Reg: DstReg0);
872 LLT SrcTy = MRI->getType(Reg: SrcReg);
873
874 const unsigned DstSize = DstTy.getSizeInBits();
875 const unsigned SrcSize = SrcTy.getSizeInBits();
876 const DebugLoc &DL = MI.getDebugLoc();
877 const RegisterBank *SrcBank = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
878
879 const TargetRegisterClass *SrcRC =
880 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcBank);
881 if (!SrcRC || !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
882 return false;
883
884 // Note we could have mixed SGPR and VGPR destination banks for an SGPR
885 // source, and this relies on the fact that the same subregister indices are
886 // used for both.
887 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SrcRC, EltSize: DstSize / 8);
888 for (int I = 0, E = NumDst; I != E; ++I) {
889 Register DstReg = MI.getOperand(i: I).getReg();
890 // hi16:sreg_32 is not allowed so explicitly shift upper 16-bits.
891 if (SrcBank->getID() == AMDGPU::SGPRRegBankID &&
892 SubRegs[I] == AMDGPU::hi16) {
893 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: DstReg)
894 .addReg(RegNo: SrcReg)
895 .addImm(Val: 16);
896 } else {
897 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::COPY), DestReg: DstReg)
898 .addReg(RegNo: SrcReg, Flags: {}, SubReg: SubRegs[I]);
899 }
900
901 // Make sure the subregister index is valid for the source register.
902 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[I]);
903 if (!SrcRC || !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI))
904 return false;
905
906 const TargetRegisterClass *DstRC =
907 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
908 if (DstRC && !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
909 return false;
910 }
911
912 MI.eraseFromParent();
913 return true;
914}
915
916bool AMDGPUInstructionSelector::selectG_BUILD_VECTOR(MachineInstr &MI) const {
917 assert(MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC ||
918 MI.getOpcode() == AMDGPU::G_BUILD_VECTOR);
919
920 Register Src0 = MI.getOperand(i: 1).getReg();
921 Register Src1 = MI.getOperand(i: 2).getReg();
922 LLT SrcTy = MRI->getType(Reg: Src0);
923 const unsigned SrcSize = SrcTy.getSizeInBits();
924
925 // BUILD_VECTOR with >=32 bits source is handled by MERGE_VALUE.
926 if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR && SrcSize >= 32) {
927 return selectG_MERGE_VALUES(MI);
928 }
929
930 // Selection logic below is for V2S16 only.
931 // For G_BUILD_VECTOR_TRUNC, additionally check that the operands are s32.
932 Register Dst = MI.getOperand(i: 0).getReg();
933 if (MRI->getType(Reg: Dst) != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
934 (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC &&
935 SrcTy != LLT::scalar(SizeInBits: 32)))
936 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
937
938 const RegisterBank *DstBank = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
939 if (DstBank->getID() == AMDGPU::AGPRRegBankID)
940 return false;
941
942 assert(DstBank->getID() == AMDGPU::SGPRRegBankID ||
943 DstBank->getID() == AMDGPU::VGPRRegBankID);
944 const bool IsVector = DstBank->getID() == AMDGPU::VGPRRegBankID;
945
946 const DebugLoc &DL = MI.getDebugLoc();
947 MachineBasicBlock *BB = MI.getParent();
948
949 // First, before trying TableGen patterns, check if both sources are
950 // constants. In those cases, we can trivially compute the final constant
951 // and emit a simple move.
952 auto ConstSrc1 = getAnyConstantVRegValWithLookThrough(VReg: Src1, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
953 if (ConstSrc1) {
954 auto ConstSrc0 =
955 getAnyConstantVRegValWithLookThrough(VReg: Src0, MRI: *MRI, LookThroughInstrs: true, LookThroughAnyExt: true);
956 if (ConstSrc0) {
957 const int64_t K0 = ConstSrc0->Value.getSExtValue();
958 const int64_t K1 = ConstSrc1->Value.getSExtValue();
959 uint32_t Lo16 = static_cast<uint32_t>(K0) & 0xffff;
960 uint32_t Hi16 = static_cast<uint32_t>(K1) & 0xffff;
961 uint32_t Imm = Lo16 | (Hi16 << 16);
962
963 // VALU
964 if (IsVector) {
965 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: Dst).addImm(Val: Imm);
966 MI.eraseFromParent();
967 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI);
968 }
969
970 // SALU
971 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: Dst).addImm(Val: Imm);
972 MI.eraseFromParent();
973 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
974 }
975 }
976
977 // Now try TableGen patterns.
978 if (selectImpl(I&: MI, CoverageInfo&: *CoverageInfo))
979 return true;
980
981 // TODO: This should probably be a combine somewhere
982 // (build_vector $src0, undef) -> copy $src0
983 MachineInstr *Src1Def = getDefIgnoringCopies(Reg: Src1, MRI: *MRI);
984 if (Src1Def->getOpcode() == AMDGPU::G_IMPLICIT_DEF) {
985 MI.setDesc(TII.get(Opcode: AMDGPU::COPY));
986 MI.removeOperand(OpNo: 2);
987 const auto &RC =
988 IsVector ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
989 return RBI.constrainGenericRegister(Reg: Dst, RC, MRI&: *MRI) &&
990 RBI.constrainGenericRegister(Reg: Src0, RC, MRI&: *MRI);
991 }
992
993 return selectS16MergeToS32(MI);
994}
995
996bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const {
997 const MachineOperand &MO = I.getOperand(i: 0);
998
999 // FIXME: Interface for getConstrainedRegClassForReg needs work. The
1000 // regbank check here is to know why getConstrainedRegClassForReg failed.
1001 const TargetRegisterClass *RC =
1002 TRI.getConstrainedRegClassForReg(Reg: MO.getReg(), MRI: *MRI);
1003 if ((!RC && !MRI->getRegBankOrNull(Reg: MO.getReg())) ||
1004 (RC && RBI.constrainGenericRegister(Reg: MO.getReg(), RC: *RC, MRI&: *MRI))) {
1005 I.setDesc(TII.get(Opcode: TargetOpcode::IMPLICIT_DEF));
1006 return true;
1007 }
1008
1009 return false;
1010}
1011
1012bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
1013 MachineBasicBlock *BB = I.getParent();
1014
1015 Register DstReg = I.getOperand(i: 0).getReg();
1016 Register Src0Reg = I.getOperand(i: 1).getReg();
1017 Register Src1Reg = I.getOperand(i: 2).getReg();
1018 LLT Src1Ty = MRI->getType(Reg: Src1Reg);
1019
1020 unsigned DstSize = MRI->getType(Reg: DstReg).getSizeInBits();
1021 unsigned InsSize = Src1Ty.getSizeInBits();
1022
1023 int64_t Offset = I.getOperand(i: 3).getImm();
1024
1025 // FIXME: These cases should have been illegal and unnecessary to check here.
1026 if (Offset % 32 != 0 || InsSize % 32 != 0)
1027 return false;
1028
1029 // Currently not handled by getSubRegFromChannel.
1030 if (InsSize > 128)
1031 return false;
1032
1033 unsigned SubReg = TRI.getSubRegFromChannel(Channel: Offset / 32, NumRegs: InsSize / 32);
1034 if (SubReg == AMDGPU::NoSubRegister)
1035 return false;
1036
1037 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1038 const TargetRegisterClass *DstRC =
1039 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
1040 if (!DstRC)
1041 return false;
1042
1043 const RegisterBank *Src0Bank = RBI.getRegBank(Reg: Src0Reg, MRI: *MRI, TRI);
1044 const RegisterBank *Src1Bank = RBI.getRegBank(Reg: Src1Reg, MRI: *MRI, TRI);
1045 const TargetRegisterClass *Src0RC =
1046 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *Src0Bank);
1047 const TargetRegisterClass *Src1RC =
1048 TRI.getRegClassForSizeOnBank(Size: InsSize, Bank: *Src1Bank);
1049
1050 // Deal with weird cases where the class only partially supports the subreg
1051 // index.
1052 Src0RC = TRI.getSubClassWithSubReg(Src0RC, SubReg);
1053 if (!Src0RC || !Src1RC)
1054 return false;
1055
1056 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
1057 !RBI.constrainGenericRegister(Reg: Src0Reg, RC: *Src0RC, MRI&: *MRI) ||
1058 !RBI.constrainGenericRegister(Reg: Src1Reg, RC: *Src1RC, MRI&: *MRI))
1059 return false;
1060
1061 const DebugLoc &DL = I.getDebugLoc();
1062 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: DstReg)
1063 .addReg(RegNo: Src0Reg)
1064 .addReg(RegNo: Src1Reg)
1065 .addImm(Val: SubReg);
1066
1067 I.eraseFromParent();
1068 return true;
1069}
1070
1071bool AMDGPUInstructionSelector::selectG_SBFX_UBFX(MachineInstr &MI) const {
1072 Register DstReg = MI.getOperand(i: 0).getReg();
1073 Register SrcReg = MI.getOperand(i: 1).getReg();
1074 Register OffsetReg = MI.getOperand(i: 2).getReg();
1075 Register WidthReg = MI.getOperand(i: 3).getReg();
1076
1077 assert(RBI.getRegBank(DstReg, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID &&
1078 "scalar BFX instructions are expanded in regbankselect");
1079 assert(MRI->getType(MI.getOperand(0).getReg()).getSizeInBits() == 32 &&
1080 "64-bit vector BFX instructions are expanded in regbankselect");
1081
1082 const DebugLoc &DL = MI.getDebugLoc();
1083 MachineBasicBlock *MBB = MI.getParent();
1084
1085 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SBFX;
1086 unsigned Opc = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
1087 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
1088 .addReg(RegNo: SrcReg)
1089 .addReg(RegNo: OffsetReg)
1090 .addReg(RegNo: WidthReg);
1091 MI.eraseFromParent();
1092 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1093 return true;
1094}
1095
1096bool AMDGPUInstructionSelector::selectInterpP1F16(MachineInstr &MI) const {
1097 if (STI.getLDSBankCount() != 16)
1098 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1099
1100 Register Dst = MI.getOperand(i: 0).getReg();
1101 Register Src0 = MI.getOperand(i: 2).getReg();
1102 Register M0Val = MI.getOperand(i: 6).getReg();
1103 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI) ||
1104 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI) ||
1105 !RBI.constrainGenericRegister(Reg: Src0, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
1106 return false;
1107
1108 // This requires 2 instructions. It is possible to write a pattern to support
1109 // this, but the generated isel emitter doesn't correctly deal with multiple
1110 // output instructions using the same physical register input. The copy to m0
1111 // is incorrectly placed before the second instruction.
1112 //
1113 // TODO: Match source modifiers.
1114
1115 Register InterpMov = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1116 const DebugLoc &DL = MI.getDebugLoc();
1117 MachineBasicBlock *MBB = MI.getParent();
1118
1119 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1120 .addReg(RegNo: M0Val);
1121 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_MOV_F32), DestReg: InterpMov)
1122 .addImm(Val: 2)
1123 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1124 .addImm(Val: MI.getOperand(i: 3).getImm()); // $attrchan
1125
1126 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_INTERP_P1LV_F16), DestReg: Dst)
1127 .addImm(Val: 0) // $src0_modifiers
1128 .addReg(RegNo: Src0) // $src0
1129 .addImm(Val: MI.getOperand(i: 4).getImm()) // $attr
1130 .addImm(Val: MI.getOperand(i: 3).getImm()) // $attrchan
1131 .addImm(Val: 0) // $src2_modifiers
1132 .addReg(RegNo: InterpMov) // $src2 - 2 f16 values selected by high
1133 .addImm(Val: MI.getOperand(i: 5).getImm()) // $high
1134 .addImm(Val: 0) // $clamp
1135 .addImm(Val: 0); // $omod
1136
1137 MI.eraseFromParent();
1138 return true;
1139}
1140
1141// Writelane is special in that it can use SGPR and M0 (which would normally
1142// count as using the constant bus twice - but in this case it is allowed since
1143// the lane selector doesn't count as a use of the constant bus). However, it is
1144// still required to abide by the 1 SGPR rule. Fix this up if we might have
1145// multiple SGPRs.
1146bool AMDGPUInstructionSelector::selectWritelane(MachineInstr &MI) const {
1147 // With a constant bus limit of at least 2, there's no issue.
1148 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_WRITELANE_B32) > 1)
1149 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
1150
1151 MachineBasicBlock *MBB = MI.getParent();
1152 const DebugLoc &DL = MI.getDebugLoc();
1153 Register VDst = MI.getOperand(i: 0).getReg();
1154 Register Val = MI.getOperand(i: 2).getReg();
1155 Register LaneSelect = MI.getOperand(i: 3).getReg();
1156 Register VDstIn = MI.getOperand(i: 4).getReg();
1157
1158 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_WRITELANE_B32), DestReg: VDst);
1159
1160 std::optional<ValueAndVReg> ConstSelect =
1161 getIConstantVRegValWithLookThrough(VReg: LaneSelect, MRI: *MRI);
1162 if (ConstSelect) {
1163 // The selector has to be an inline immediate, so we can use whatever for
1164 // the other operands.
1165 MIB.addReg(RegNo: Val);
1166 MIB.addImm(Val: ConstSelect->Value.getSExtValue() &
1167 maskTrailingOnes<uint64_t>(N: STI.getWavefrontSizeLog2()));
1168 } else {
1169 std::optional<ValueAndVReg> ConstVal =
1170 getIConstantVRegValWithLookThrough(VReg: Val, MRI: *MRI);
1171
1172 // If the value written is an inline immediate, we can get away without a
1173 // copy to m0.
1174 if (ConstVal && AMDGPU::isInlinableLiteral32(Literal: ConstVal->Value.getSExtValue(),
1175 HasInv2Pi: STI.hasInv2PiInlineImm())) {
1176 MIB.addImm(Val: ConstVal->Value.getSExtValue());
1177 MIB.addReg(RegNo: LaneSelect);
1178 } else {
1179 MIB.addReg(RegNo: Val);
1180
1181 // If the lane selector was originally in a VGPR and copied with
1182 // readfirstlane, there's a hazard to read the same SGPR from the
1183 // VALU. Constrain to a different SGPR to help avoid needing a nop later.
1184 RBI.constrainGenericRegister(Reg: LaneSelect, RC: AMDGPU::SReg_32_XM0RegClass, MRI&: *MRI);
1185
1186 BuildMI(BB&: *MBB, I&: *MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1187 .addReg(RegNo: LaneSelect);
1188 MIB.addReg(RegNo: AMDGPU::M0);
1189 }
1190 }
1191
1192 MIB.addReg(RegNo: VDstIn);
1193
1194 MI.eraseFromParent();
1195 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1196 return true;
1197}
1198
1199// We need to handle this here because tablegen doesn't support matching
1200// instructions with multiple outputs.
1201bool AMDGPUInstructionSelector::selectDivScale(MachineInstr &MI) const {
1202 Register Dst0 = MI.getOperand(i: 0).getReg();
1203 Register Dst1 = MI.getOperand(i: 1).getReg();
1204
1205 LLT Ty = MRI->getType(Reg: Dst0);
1206 unsigned Opc;
1207 if (Ty == LLT::scalar(SizeInBits: 32))
1208 Opc = AMDGPU::V_DIV_SCALE_F32_e64;
1209 else if (Ty == LLT::scalar(SizeInBits: 64))
1210 Opc = AMDGPU::V_DIV_SCALE_F64_e64;
1211 else
1212 return false;
1213
1214 // TODO: Match source modifiers.
1215
1216 const DebugLoc &DL = MI.getDebugLoc();
1217 MachineBasicBlock *MBB = MI.getParent();
1218
1219 Register Numer = MI.getOperand(i: 3).getReg();
1220 Register Denom = MI.getOperand(i: 4).getReg();
1221 unsigned ChooseDenom = MI.getOperand(i: 5).getImm();
1222
1223 Register Src0 = ChooseDenom != 0 ? Numer : Denom;
1224
1225 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
1226 .addDef(RegNo: Dst1)
1227 .addImm(Val: 0) // $src0_modifiers
1228 .addUse(RegNo: Src0) // $src0
1229 .addImm(Val: 0) // $src1_modifiers
1230 .addUse(RegNo: Denom) // $src1
1231 .addImm(Val: 0) // $src2_modifiers
1232 .addUse(RegNo: Numer) // $src2
1233 .addImm(Val: 0) // $clamp
1234 .addImm(Val: 0); // $omod
1235
1236 MI.eraseFromParent();
1237 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1238 return true;
1239}
1240
1241bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
1242 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
1243 switch (IntrinsicID) {
1244 case Intrinsic::amdgcn_if_break: {
1245 MachineBasicBlock *BB = I.getParent();
1246
1247 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1248 // SelectionDAG uses for wave32 vs wave64.
1249 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_IF_BREAK))
1250 .add(MO: I.getOperand(i: 0))
1251 .add(MO: I.getOperand(i: 2))
1252 .add(MO: I.getOperand(i: 3));
1253
1254 Register DstReg = I.getOperand(i: 0).getReg();
1255 Register Src0Reg = I.getOperand(i: 2).getReg();
1256 Register Src1Reg = I.getOperand(i: 3).getReg();
1257
1258 I.eraseFromParent();
1259
1260 for (Register Reg : { DstReg, Src0Reg, Src1Reg })
1261 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1262
1263 return true;
1264 }
1265 case Intrinsic::amdgcn_interp_p1_f16:
1266 return selectInterpP1F16(MI&: I);
1267 case Intrinsic::amdgcn_wqm:
1268 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::WQM);
1269 case Intrinsic::amdgcn_softwqm:
1270 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::SOFT_WQM);
1271 case Intrinsic::amdgcn_strict_wwm:
1272 case Intrinsic::amdgcn_wwm:
1273 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WWM);
1274 case Intrinsic::amdgcn_strict_wqm:
1275 return constrainCopyLikeIntrin(MI&: I, NewOpc: AMDGPU::STRICT_WQM);
1276 case Intrinsic::amdgcn_writelane:
1277 return selectWritelane(MI&: I);
1278 case Intrinsic::amdgcn_div_scale:
1279 return selectDivScale(MI&: I);
1280 case Intrinsic::amdgcn_ballot:
1281 return selectBallot(I);
1282 case Intrinsic::amdgcn_reloc_constant:
1283 return selectRelocConstant(I);
1284 case Intrinsic::amdgcn_groupstaticsize:
1285 return selectGroupStaticSize(I);
1286 case Intrinsic::returnaddress:
1287 return selectReturnAddress(I);
1288 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
1289 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
1290 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
1291 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
1292 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
1293 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
1294 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
1295 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
1296 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
1297 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
1298 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
1299 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
1300 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
1301 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
1302 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
1303 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
1304 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
1305 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
1306 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
1307 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
1308 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
1309 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
1310 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
1311 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
1312 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
1313 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
1314 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
1315 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
1316 return selectSMFMACIntrin(I);
1317 case Intrinsic::amdgcn_permlane16_swap:
1318 case Intrinsic::amdgcn_permlane32_swap:
1319 return selectPermlaneSwapIntrin(I, IntrID: IntrinsicID);
1320 case Intrinsic::amdgcn_wave_shuffle:
1321 return selectWaveShuffleIntrin(I);
1322 default:
1323 return selectImpl(I, CoverageInfo&: *CoverageInfo);
1324 }
1325}
1326
1327static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size,
1328 const GCNSubtarget &ST) {
1329 if (Size != 16 && Size != 32 && Size != 64)
1330 return -1;
1331
1332 if (Size == 16 && !ST.has16BitInsts())
1333 return -1;
1334
1335 const auto Select = [&](unsigned S16Opc, unsigned TrueS16Opc,
1336 unsigned FakeS16Opc, unsigned S32Opc,
1337 unsigned S64Opc) {
1338 if (Size == 16)
1339 return ST.hasTrue16BitInsts()
1340 ? ST.useRealTrue16Insts() ? TrueS16Opc : FakeS16Opc
1341 : S16Opc;
1342 if (Size == 32)
1343 return S32Opc;
1344 return S64Opc;
1345 };
1346
1347 switch (P) {
1348 default:
1349 llvm_unreachable("Unknown condition code!");
1350 case CmpInst::ICMP_NE:
1351 return Select(AMDGPU::V_CMP_NE_U16_e64, AMDGPU::V_CMP_NE_U16_t16_e64,
1352 AMDGPU::V_CMP_NE_U16_fake16_e64, AMDGPU::V_CMP_NE_U32_e64,
1353 AMDGPU::V_CMP_NE_U64_e64);
1354 case CmpInst::ICMP_EQ:
1355 return Select(AMDGPU::V_CMP_EQ_U16_e64, AMDGPU::V_CMP_EQ_U16_t16_e64,
1356 AMDGPU::V_CMP_EQ_U16_fake16_e64, AMDGPU::V_CMP_EQ_U32_e64,
1357 AMDGPU::V_CMP_EQ_U64_e64);
1358 case CmpInst::ICMP_SGT:
1359 return Select(AMDGPU::V_CMP_GT_I16_e64, AMDGPU::V_CMP_GT_I16_t16_e64,
1360 AMDGPU::V_CMP_GT_I16_fake16_e64, AMDGPU::V_CMP_GT_I32_e64,
1361 AMDGPU::V_CMP_GT_I64_e64);
1362 case CmpInst::ICMP_SGE:
1363 return Select(AMDGPU::V_CMP_GE_I16_e64, AMDGPU::V_CMP_GE_I16_t16_e64,
1364 AMDGPU::V_CMP_GE_I16_fake16_e64, AMDGPU::V_CMP_GE_I32_e64,
1365 AMDGPU::V_CMP_GE_I64_e64);
1366 case CmpInst::ICMP_SLT:
1367 return Select(AMDGPU::V_CMP_LT_I16_e64, AMDGPU::V_CMP_LT_I16_t16_e64,
1368 AMDGPU::V_CMP_LT_I16_fake16_e64, AMDGPU::V_CMP_LT_I32_e64,
1369 AMDGPU::V_CMP_LT_I64_e64);
1370 case CmpInst::ICMP_SLE:
1371 return Select(AMDGPU::V_CMP_LE_I16_e64, AMDGPU::V_CMP_LE_I16_t16_e64,
1372 AMDGPU::V_CMP_LE_I16_fake16_e64, AMDGPU::V_CMP_LE_I32_e64,
1373 AMDGPU::V_CMP_LE_I64_e64);
1374 case CmpInst::ICMP_UGT:
1375 return Select(AMDGPU::V_CMP_GT_U16_e64, AMDGPU::V_CMP_GT_U16_t16_e64,
1376 AMDGPU::V_CMP_GT_U16_fake16_e64, AMDGPU::V_CMP_GT_U32_e64,
1377 AMDGPU::V_CMP_GT_U64_e64);
1378 case CmpInst::ICMP_UGE:
1379 return Select(AMDGPU::V_CMP_GE_U16_e64, AMDGPU::V_CMP_GE_U16_t16_e64,
1380 AMDGPU::V_CMP_GE_U16_fake16_e64, AMDGPU::V_CMP_GE_U32_e64,
1381 AMDGPU::V_CMP_GE_U64_e64);
1382 case CmpInst::ICMP_ULT:
1383 return Select(AMDGPU::V_CMP_LT_U16_e64, AMDGPU::V_CMP_LT_U16_t16_e64,
1384 AMDGPU::V_CMP_LT_U16_fake16_e64, AMDGPU::V_CMP_LT_U32_e64,
1385 AMDGPU::V_CMP_LT_U64_e64);
1386 case CmpInst::ICMP_ULE:
1387 return Select(AMDGPU::V_CMP_LE_U16_e64, AMDGPU::V_CMP_LE_U16_t16_e64,
1388 AMDGPU::V_CMP_LE_U16_fake16_e64, AMDGPU::V_CMP_LE_U32_e64,
1389 AMDGPU::V_CMP_LE_U64_e64);
1390
1391 case CmpInst::FCMP_OEQ:
1392 return Select(AMDGPU::V_CMP_EQ_F16_e64, AMDGPU::V_CMP_EQ_F16_t16_e64,
1393 AMDGPU::V_CMP_EQ_F16_fake16_e64, AMDGPU::V_CMP_EQ_F32_e64,
1394 AMDGPU::V_CMP_EQ_F64_e64);
1395 case CmpInst::FCMP_OGT:
1396 return Select(AMDGPU::V_CMP_GT_F16_e64, AMDGPU::V_CMP_GT_F16_t16_e64,
1397 AMDGPU::V_CMP_GT_F16_fake16_e64, AMDGPU::V_CMP_GT_F32_e64,
1398 AMDGPU::V_CMP_GT_F64_e64);
1399 case CmpInst::FCMP_OGE:
1400 return Select(AMDGPU::V_CMP_GE_F16_e64, AMDGPU::V_CMP_GE_F16_t16_e64,
1401 AMDGPU::V_CMP_GE_F16_fake16_e64, AMDGPU::V_CMP_GE_F32_e64,
1402 AMDGPU::V_CMP_GE_F64_e64);
1403 case CmpInst::FCMP_OLT:
1404 return Select(AMDGPU::V_CMP_LT_F16_e64, AMDGPU::V_CMP_LT_F16_t16_e64,
1405 AMDGPU::V_CMP_LT_F16_fake16_e64, AMDGPU::V_CMP_LT_F32_e64,
1406 AMDGPU::V_CMP_LT_F64_e64);
1407 case CmpInst::FCMP_OLE:
1408 return Select(AMDGPU::V_CMP_LE_F16_e64, AMDGPU::V_CMP_LE_F16_t16_e64,
1409 AMDGPU::V_CMP_LE_F16_fake16_e64, AMDGPU::V_CMP_LE_F32_e64,
1410 AMDGPU::V_CMP_LE_F64_e64);
1411 case CmpInst::FCMP_ONE:
1412 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1413 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1414 AMDGPU::V_CMP_NEQ_F64_e64);
1415 case CmpInst::FCMP_ORD:
1416 return Select(AMDGPU::V_CMP_O_F16_e64, AMDGPU::V_CMP_O_F16_t16_e64,
1417 AMDGPU::V_CMP_O_F16_fake16_e64, AMDGPU::V_CMP_O_F32_e64,
1418 AMDGPU::V_CMP_O_F64_e64);
1419 case CmpInst::FCMP_UNO:
1420 return Select(AMDGPU::V_CMP_U_F16_e64, AMDGPU::V_CMP_U_F16_t16_e64,
1421 AMDGPU::V_CMP_U_F16_fake16_e64, AMDGPU::V_CMP_U_F32_e64,
1422 AMDGPU::V_CMP_U_F64_e64);
1423 case CmpInst::FCMP_UEQ:
1424 return Select(AMDGPU::V_CMP_NLG_F16_e64, AMDGPU::V_CMP_NLG_F16_t16_e64,
1425 AMDGPU::V_CMP_NLG_F16_fake16_e64, AMDGPU::V_CMP_NLG_F32_e64,
1426 AMDGPU::V_CMP_NLG_F64_e64);
1427 case CmpInst::FCMP_UGT:
1428 return Select(AMDGPU::V_CMP_NLE_F16_e64, AMDGPU::V_CMP_NLE_F16_t16_e64,
1429 AMDGPU::V_CMP_NLE_F16_fake16_e64, AMDGPU::V_CMP_NLE_F32_e64,
1430 AMDGPU::V_CMP_NLE_F64_e64);
1431 case CmpInst::FCMP_UGE:
1432 return Select(AMDGPU::V_CMP_NLT_F16_e64, AMDGPU::V_CMP_NLT_F16_t16_e64,
1433 AMDGPU::V_CMP_NLT_F16_fake16_e64, AMDGPU::V_CMP_NLT_F32_e64,
1434 AMDGPU::V_CMP_NLT_F64_e64);
1435 case CmpInst::FCMP_ULT:
1436 return Select(AMDGPU::V_CMP_NGE_F16_e64, AMDGPU::V_CMP_NGE_F16_t16_e64,
1437 AMDGPU::V_CMP_NGE_F16_fake16_e64, AMDGPU::V_CMP_NGE_F32_e64,
1438 AMDGPU::V_CMP_NGE_F64_e64);
1439 case CmpInst::FCMP_ULE:
1440 return Select(AMDGPU::V_CMP_NGT_F16_e64, AMDGPU::V_CMP_NGT_F16_t16_e64,
1441 AMDGPU::V_CMP_NGT_F16_fake16_e64, AMDGPU::V_CMP_NGT_F32_e64,
1442 AMDGPU::V_CMP_NGT_F64_e64);
1443 case CmpInst::FCMP_UNE:
1444 return Select(AMDGPU::V_CMP_NEQ_F16_e64, AMDGPU::V_CMP_NEQ_F16_t16_e64,
1445 AMDGPU::V_CMP_NEQ_F16_fake16_e64, AMDGPU::V_CMP_NEQ_F32_e64,
1446 AMDGPU::V_CMP_NEQ_F64_e64);
1447 case CmpInst::FCMP_TRUE:
1448 return Select(AMDGPU::V_CMP_TRU_F16_e64, AMDGPU::V_CMP_TRU_F16_t16_e64,
1449 AMDGPU::V_CMP_TRU_F16_fake16_e64, AMDGPU::V_CMP_TRU_F32_e64,
1450 AMDGPU::V_CMP_TRU_F64_e64);
1451 case CmpInst::FCMP_FALSE:
1452 return Select(AMDGPU::V_CMP_F_F16_e64, AMDGPU::V_CMP_F_F16_t16_e64,
1453 AMDGPU::V_CMP_F_F16_fake16_e64, AMDGPU::V_CMP_F_F32_e64,
1454 AMDGPU::V_CMP_F_F64_e64);
1455 }
1456}
1457
1458int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
1459 unsigned Size) const {
1460 if (Size == 64) {
1461 if (!STI.hasScalarCompareEq64())
1462 return -1;
1463
1464 switch (P) {
1465 case CmpInst::ICMP_NE:
1466 return AMDGPU::S_CMP_LG_U64;
1467 case CmpInst::ICMP_EQ:
1468 return AMDGPU::S_CMP_EQ_U64;
1469 default:
1470 return -1;
1471 }
1472 }
1473
1474 if (Size == 32) {
1475 switch (P) {
1476 case CmpInst::ICMP_NE:
1477 return AMDGPU::S_CMP_LG_U32;
1478 case CmpInst::ICMP_EQ:
1479 return AMDGPU::S_CMP_EQ_U32;
1480 case CmpInst::ICMP_SGT:
1481 return AMDGPU::S_CMP_GT_I32;
1482 case CmpInst::ICMP_SGE:
1483 return AMDGPU::S_CMP_GE_I32;
1484 case CmpInst::ICMP_SLT:
1485 return AMDGPU::S_CMP_LT_I32;
1486 case CmpInst::ICMP_SLE:
1487 return AMDGPU::S_CMP_LE_I32;
1488 case CmpInst::ICMP_UGT:
1489 return AMDGPU::S_CMP_GT_U32;
1490 case CmpInst::ICMP_UGE:
1491 return AMDGPU::S_CMP_GE_U32;
1492 case CmpInst::ICMP_ULT:
1493 return AMDGPU::S_CMP_LT_U32;
1494 case CmpInst::ICMP_ULE:
1495 return AMDGPU::S_CMP_LE_U32;
1496 case CmpInst::FCMP_OEQ:
1497 return AMDGPU::S_CMP_EQ_F32;
1498 case CmpInst::FCMP_OGT:
1499 return AMDGPU::S_CMP_GT_F32;
1500 case CmpInst::FCMP_OGE:
1501 return AMDGPU::S_CMP_GE_F32;
1502 case CmpInst::FCMP_OLT:
1503 return AMDGPU::S_CMP_LT_F32;
1504 case CmpInst::FCMP_OLE:
1505 return AMDGPU::S_CMP_LE_F32;
1506 case CmpInst::FCMP_ONE:
1507 return AMDGPU::S_CMP_LG_F32;
1508 case CmpInst::FCMP_ORD:
1509 return AMDGPU::S_CMP_O_F32;
1510 case CmpInst::FCMP_UNO:
1511 return AMDGPU::S_CMP_U_F32;
1512 case CmpInst::FCMP_UEQ:
1513 return AMDGPU::S_CMP_NLG_F32;
1514 case CmpInst::FCMP_UGT:
1515 return AMDGPU::S_CMP_NLE_F32;
1516 case CmpInst::FCMP_UGE:
1517 return AMDGPU::S_CMP_NLT_F32;
1518 case CmpInst::FCMP_ULT:
1519 return AMDGPU::S_CMP_NGE_F32;
1520 case CmpInst::FCMP_ULE:
1521 return AMDGPU::S_CMP_NGT_F32;
1522 case CmpInst::FCMP_UNE:
1523 return AMDGPU::S_CMP_NEQ_F32;
1524 default:
1525 llvm_unreachable("Unknown condition code!");
1526 }
1527 }
1528
1529 if (Size == 16) {
1530 if (!STI.hasSALUFloatInsts())
1531 return -1;
1532
1533 switch (P) {
1534 case CmpInst::FCMP_OEQ:
1535 return AMDGPU::S_CMP_EQ_F16;
1536 case CmpInst::FCMP_OGT:
1537 return AMDGPU::S_CMP_GT_F16;
1538 case CmpInst::FCMP_OGE:
1539 return AMDGPU::S_CMP_GE_F16;
1540 case CmpInst::FCMP_OLT:
1541 return AMDGPU::S_CMP_LT_F16;
1542 case CmpInst::FCMP_OLE:
1543 return AMDGPU::S_CMP_LE_F16;
1544 case CmpInst::FCMP_ONE:
1545 return AMDGPU::S_CMP_LG_F16;
1546 case CmpInst::FCMP_ORD:
1547 return AMDGPU::S_CMP_O_F16;
1548 case CmpInst::FCMP_UNO:
1549 return AMDGPU::S_CMP_U_F16;
1550 case CmpInst::FCMP_UEQ:
1551 return AMDGPU::S_CMP_NLG_F16;
1552 case CmpInst::FCMP_UGT:
1553 return AMDGPU::S_CMP_NLE_F16;
1554 case CmpInst::FCMP_UGE:
1555 return AMDGPU::S_CMP_NLT_F16;
1556 case CmpInst::FCMP_ULT:
1557 return AMDGPU::S_CMP_NGE_F16;
1558 case CmpInst::FCMP_ULE:
1559 return AMDGPU::S_CMP_NGT_F16;
1560 case CmpInst::FCMP_UNE:
1561 return AMDGPU::S_CMP_NEQ_F16;
1562 default:
1563 llvm_unreachable("Unknown condition code!");
1564 }
1565 }
1566
1567 return -1;
1568}
1569
1570bool AMDGPUInstructionSelector::selectG_ICMP_or_FCMP(MachineInstr &I) const {
1571
1572 MachineBasicBlock *BB = I.getParent();
1573 const DebugLoc &DL = I.getDebugLoc();
1574
1575 Register SrcReg = I.getOperand(i: 2).getReg();
1576 unsigned Size = RBI.getSizeInBits(Reg: SrcReg, MRI: *MRI, TRI);
1577
1578 auto Pred = (CmpInst::Predicate)I.getOperand(i: 1).getPredicate();
1579
1580 Register CCReg = I.getOperand(i: 0).getReg();
1581 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
1582 int Opcode = getS_CMPOpcode(P: Pred, Size);
1583 if (Opcode == -1)
1584 return false;
1585 MachineInstr *ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode))
1586 .add(MO: I.getOperand(i: 2))
1587 .add(MO: I.getOperand(i: 3));
1588 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg)
1589 .addReg(RegNo: AMDGPU::SCC);
1590 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1591 bool Ret =
1592 RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
1593 I.eraseFromParent();
1594 return Ret;
1595 }
1596
1597 if (I.getOpcode() == AMDGPU::G_FCMP)
1598 return false;
1599
1600 int Opcode = getV_CMPOpcode(P: Pred, Size, ST: *Subtarget);
1601 if (Opcode == -1)
1602 return false;
1603
1604 MachineInstrBuilder ICmp;
1605 // t16 instructions
1606 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0_modifiers)) {
1607 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1608 .addImm(Val: 0)
1609 .add(MO: I.getOperand(i: 2))
1610 .addImm(Val: 0)
1611 .add(MO: I.getOperand(i: 3))
1612 .addImm(Val: 0); // op_sel
1613 } else {
1614 ICmp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: I.getOperand(i: 0).getReg())
1615 .add(MO: I.getOperand(i: 2))
1616 .add(MO: I.getOperand(i: 3));
1617 }
1618
1619 RBI.constrainGenericRegister(Reg: ICmp->getOperand(i: 0).getReg(),
1620 RC: *TRI.getBoolRC(), MRI&: *MRI);
1621 constrainSelectedInstRegOperands(I&: *ICmp, TII, TRI, RBI);
1622 I.eraseFromParent();
1623 return true;
1624}
1625
1626// Ballot has to zero bits in input lane-mask that are zero in current exec,
1627// Done as AND with exec. For inputs that are results of instruction that
1628// implicitly use same exec, for example compares in same basic block or SCC to
1629// VCC copy, use copy.
1630static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI,
1631 MachineBasicBlock *MBB) {
1632 MachineInstr *MI = MRI.getVRegDef(Reg);
1633 if (MI->getParent() != MBB)
1634 return false;
1635
1636 // Lane mask generated by SCC to VCC copy.
1637 if (MI->getOpcode() == AMDGPU::COPY) {
1638 auto DstRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 0).getReg());
1639 auto SrcRB = MRI.getRegBankOrNull(Reg: MI->getOperand(i: 1).getReg());
1640 if (DstRB && SrcRB && DstRB->getID() == AMDGPU::VCCRegBankID &&
1641 SrcRB->getID() == AMDGPU::SGPRRegBankID)
1642 return true;
1643 }
1644
1645 // Lane mask generated by SCC to VCC copy
1646 if (MI->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
1647 return true;
1648
1649 // Lane mask generated using compare with same exec.
1650 if (isa<GAnyCmp>(Val: MI))
1651 return true;
1652
1653 Register LHS, RHS;
1654 // Look through AND.
1655 if (mi_match(R: Reg, MRI, P: m_GAnd(L: m_Reg(R&: LHS), R: m_Reg(R&: RHS))))
1656 return isLaneMaskFromSameBlock(Reg: LHS, MRI, MBB) ||
1657 isLaneMaskFromSameBlock(Reg: RHS, MRI, MBB);
1658
1659 return false;
1660}
1661
1662bool AMDGPUInstructionSelector::selectBallot(MachineInstr &I) const {
1663 MachineBasicBlock *BB = I.getParent();
1664 const DebugLoc &DL = I.getDebugLoc();
1665 Register DstReg = I.getOperand(i: 0).getReg();
1666 Register SrcReg = I.getOperand(i: 2).getReg();
1667 const unsigned BallotSize = MRI->getType(Reg: DstReg).getSizeInBits();
1668 const unsigned WaveSize = STI.getWavefrontSize();
1669
1670 // In the common case, the return type matches the wave size.
1671 // However we also support emitting i64 ballots in wave32 mode.
1672 if (BallotSize != WaveSize && (BallotSize != 64 || WaveSize != 32))
1673 return false;
1674
1675 std::optional<ValueAndVReg> Arg =
1676 getIConstantVRegValWithLookThrough(VReg: SrcReg, MRI: *MRI);
1677
1678 Register Dst = DstReg;
1679 // i64 ballot on Wave32: new Dst(i32) for WaveSize ballot.
1680 if (BallotSize != WaveSize) {
1681 Dst = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
1682 }
1683
1684 if (Arg) {
1685 const int64_t Value = Arg->Value.getZExtValue();
1686 if (Value == 0) {
1687 // Dst = S_MOV 0
1688 unsigned Opcode = WaveSize == 64 ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
1689 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: Dst).addImm(Val: 0);
1690 } else {
1691 // Dst = COPY EXEC
1692 assert(Value == 1);
1693 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: TRI.getExec());
1694 }
1695 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1696 return false;
1697 } else {
1698 if (isLaneMaskFromSameBlock(Reg: SrcReg, MRI&: *MRI, MBB: BB)) {
1699 // Dst = COPY SrcReg
1700 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: Dst).addReg(RegNo: SrcReg);
1701 if (!RBI.constrainGenericRegister(Reg: Dst, RC: *TRI.getBoolRC(), MRI&: *MRI))
1702 return false;
1703 } else {
1704 // Dst = S_AND SrcReg, EXEC
1705 unsigned AndOpc = WaveSize == 64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
1706 auto And = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: Dst)
1707 .addReg(RegNo: SrcReg)
1708 .addReg(RegNo: TRI.getExec())
1709 .setOperandDead(3); // Dead scc
1710 constrainSelectedInstRegOperands(I&: *And, TII, TRI, RBI);
1711 }
1712 }
1713
1714 // i64 ballot on Wave32: zero-extend i32 ballot to i64.
1715 if (BallotSize != WaveSize) {
1716 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1717 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg).addImm(Val: 0);
1718 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
1719 .addReg(RegNo: Dst)
1720 .addImm(Val: AMDGPU::sub0)
1721 .addReg(RegNo: HiReg)
1722 .addImm(Val: AMDGPU::sub1);
1723 }
1724
1725 I.eraseFromParent();
1726 return true;
1727}
1728
1729bool AMDGPUInstructionSelector::selectRelocConstant(MachineInstr &I) const {
1730 Register DstReg = I.getOperand(i: 0).getReg();
1731 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1732 const TargetRegisterClass *DstRC = TRI.getRegClassForSizeOnBank(Size: 32, Bank: *DstBank);
1733 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
1734 return false;
1735
1736 const bool IsVALU = DstBank->getID() == AMDGPU::VGPRRegBankID;
1737
1738 Module *M = MF->getFunction().getParent();
1739 const MDNode *Metadata = I.getOperand(i: 2).getMetadata();
1740 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
1741 auto *RelocSymbol = cast<GlobalVariable>(
1742 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
1743
1744 MachineBasicBlock *BB = I.getParent();
1745 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(),
1746 MCID: TII.get(Opcode: IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32), DestReg: DstReg)
1747 .addGlobalAddress(GV: RelocSymbol, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1748
1749 I.eraseFromParent();
1750 return true;
1751}
1752
1753bool AMDGPUInstructionSelector::selectGroupStaticSize(MachineInstr &I) const {
1754 Triple::OSType OS = MF->getTarget().getTargetTriple().getOS();
1755
1756 Register DstReg = I.getOperand(i: 0).getReg();
1757 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
1758 unsigned Mov = DstRB->getID() == AMDGPU::SGPRRegBankID ?
1759 AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
1760
1761 MachineBasicBlock *MBB = I.getParent();
1762 const DebugLoc &DL = I.getDebugLoc();
1763
1764 auto MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Mov), DestReg: DstReg);
1765
1766 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
1767 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1768 MIB.addImm(Val: MFI->getLDSSize());
1769 } else {
1770 Module *M = MF->getFunction().getParent();
1771 const GlobalValue *GV =
1772 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::amdgcn_groupstaticsize);
1773 MIB.addGlobalAddress(GV, Offset: 0, TargetFlags: SIInstrInfo::MO_ABS32_LO);
1774 }
1775
1776 I.eraseFromParent();
1777 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
1778 return true;
1779}
1780
1781bool AMDGPUInstructionSelector::selectReturnAddress(MachineInstr &I) const {
1782 MachineBasicBlock *MBB = I.getParent();
1783 MachineFunction &MF = *MBB->getParent();
1784 const DebugLoc &DL = I.getDebugLoc();
1785
1786 Register DstReg = I.getOperand(i: 0).getReg();
1787 unsigned Depth = I.getOperand(i: 2).getImm();
1788
1789 const TargetRegisterClass *RC =
1790 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
1791 if (!RC->hasSubClassEq(RC: &AMDGPU::SGPR_64RegClass) ||
1792 !RBI.constrainGenericRegister(Reg: DstReg, RC: *RC, MRI&: *MRI))
1793 return false;
1794
1795 // Check for kernel and shader functions
1796 if (Depth != 0 ||
1797 MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
1798 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg)
1799 .addImm(Val: 0);
1800 I.eraseFromParent();
1801 return true;
1802 }
1803
1804 MachineFrameInfo &MFI = MF.getFrameInfo();
1805 // There is a call to @llvm.returnaddress in this function
1806 MFI.setReturnAddressIsTaken(true);
1807
1808 // Get the return address reg and mark it as an implicit live-in
1809 Register ReturnAddrReg = TRI.getReturnAddressReg(MF);
1810 Register LiveIn = getFunctionLiveInPhysReg(MF, TII, PhysReg: ReturnAddrReg,
1811 RC: AMDGPU::SReg_64RegClass, DL);
1812 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
1813 .addReg(RegNo: LiveIn);
1814 I.eraseFromParent();
1815 return true;
1816}
1817
1818bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const {
1819 // FIXME: Manually selecting to avoid dealing with the SReg_1 trick
1820 // SelectionDAG uses for wave32 vs wave64.
1821 MachineBasicBlock *BB = MI.getParent();
1822 BuildMI(BB&: *BB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::SI_END_CF))
1823 .add(MO: MI.getOperand(i: 1));
1824
1825 Register Reg = MI.getOperand(i: 1).getReg();
1826 MI.eraseFromParent();
1827
1828 if (!MRI->getRegClassOrNull(Reg))
1829 MRI->setRegClass(Reg, RC: TRI.getWaveMaskRegClass());
1830 return true;
1831}
1832
1833bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic(
1834 MachineInstr &MI, Intrinsic::ID IntrID) const {
1835 MachineBasicBlock *MBB = MI.getParent();
1836 MachineFunction *MF = MBB->getParent();
1837 const DebugLoc &DL = MI.getDebugLoc();
1838
1839 unsigned IndexOperand = MI.getOperand(i: 7).getImm();
1840 bool WaveRelease = MI.getOperand(i: 8).getImm() != 0;
1841 bool WaveDone = MI.getOperand(i: 9).getImm() != 0;
1842
1843 if (WaveDone && !WaveRelease) {
1844 // TODO: Move this to IR verifier
1845 const Function &Fn = MF->getFunction();
1846 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1847 Fn, "ds_ordered_count: wave_done requires wave_release", DL));
1848 }
1849
1850 unsigned OrderedCountIndex = IndexOperand & 0x3f;
1851 IndexOperand &= ~0x3f;
1852 unsigned CountDw = 0;
1853
1854 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) {
1855 CountDw = (IndexOperand >> 24) & 0xf;
1856 IndexOperand &= ~(0xf << 24);
1857
1858 if (CountDw < 1 || CountDw > 4) {
1859 const Function &Fn = MF->getFunction();
1860 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1861 Fn, "ds_ordered_count: dword count must be between 1 and 4", DL));
1862 CountDw = 1;
1863 }
1864 }
1865
1866 if (IndexOperand) {
1867 const Function &Fn = MF->getFunction();
1868 Fn.getContext().diagnose(DI: DiagnosticInfoUnsupported(
1869 Fn, "ds_ordered_count: bad index operand", DL));
1870 }
1871
1872 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
1873 unsigned ShaderType = SIInstrInfo::getDSShaderTypeValue(MF: *MF);
1874
1875 unsigned Offset0 = OrderedCountIndex << 2;
1876 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
1877
1878 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10)
1879 Offset1 |= (CountDw - 1) << 6;
1880
1881 if (STI.getGeneration() < AMDGPUSubtarget::GFX11)
1882 Offset1 |= ShaderType << 2;
1883
1884 unsigned Offset = Offset0 | (Offset1 << 8);
1885
1886 Register M0Val = MI.getOperand(i: 2).getReg();
1887 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1888 .addReg(RegNo: M0Val);
1889
1890 Register DstReg = MI.getOperand(i: 0).getReg();
1891 Register ValReg = MI.getOperand(i: 3).getReg();
1892 MachineInstrBuilder DS =
1893 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_ORDERED_COUNT), DestReg: DstReg)
1894 .addReg(RegNo: ValReg)
1895 .addImm(Val: Offset)
1896 .cloneMemRefs(OtherMI: MI);
1897
1898 if (!RBI.constrainGenericRegister(Reg: M0Val, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
1899 return false;
1900
1901 constrainSelectedInstRegOperands(I&: *DS, TII, TRI, RBI);
1902 MI.eraseFromParent();
1903 return true;
1904}
1905
1906static unsigned gwsIntrinToOpcode(unsigned IntrID) {
1907 switch (IntrID) {
1908 case Intrinsic::amdgcn_ds_gws_init:
1909 return AMDGPU::DS_GWS_INIT;
1910 case Intrinsic::amdgcn_ds_gws_barrier:
1911 return AMDGPU::DS_GWS_BARRIER;
1912 case Intrinsic::amdgcn_ds_gws_sema_v:
1913 return AMDGPU::DS_GWS_SEMA_V;
1914 case Intrinsic::amdgcn_ds_gws_sema_br:
1915 return AMDGPU::DS_GWS_SEMA_BR;
1916 case Intrinsic::amdgcn_ds_gws_sema_p:
1917 return AMDGPU::DS_GWS_SEMA_P;
1918 case Intrinsic::amdgcn_ds_gws_sema_release_all:
1919 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
1920 default:
1921 llvm_unreachable("not a gws intrinsic");
1922 }
1923}
1924
1925bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
1926 Intrinsic::ID IID) const {
1927 if (!STI.hasGWS() || (IID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
1928 !STI.hasGWSSemaReleaseAll()))
1929 return false;
1930
1931 // intrinsic ID, vsrc, offset
1932 const bool HasVSrc = MI.getNumOperands() == 3;
1933 assert(HasVSrc || MI.getNumOperands() == 2);
1934
1935 Register BaseOffset = MI.getOperand(i: HasVSrc ? 2 : 1).getReg();
1936 const RegisterBank *OffsetRB = RBI.getRegBank(Reg: BaseOffset, MRI: *MRI, TRI);
1937 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID)
1938 return false;
1939
1940 MachineInstr *OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
1941 unsigned ImmOffset;
1942
1943 MachineBasicBlock *MBB = MI.getParent();
1944 const DebugLoc &DL = MI.getDebugLoc();
1945
1946 MachineInstr *Readfirstlane = nullptr;
1947
1948 // If we legalized the VGPR input, strip out the readfirstlane to analyze the
1949 // incoming offset, in case there's an add of a constant. We'll have to put it
1950 // back later.
1951 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) {
1952 Readfirstlane = OffsetDef;
1953 BaseOffset = OffsetDef->getOperand(i: 1).getReg();
1954 OffsetDef = getDefIgnoringCopies(Reg: BaseOffset, MRI: *MRI);
1955 }
1956
1957 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) {
1958 // If we have a constant offset, try to use the 0 in m0 as the base.
1959 // TODO: Look into changing the default m0 initialization value. If the
1960 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
1961 // the immediate offset.
1962
1963 ImmOffset = OffsetDef->getOperand(i: 1).getCImm()->getZExtValue();
1964 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
1965 .addImm(Val: 0);
1966 } else {
1967 std::tie(args&: BaseOffset, args&: ImmOffset) =
1968 AMDGPU::getBaseWithConstantOffset(MRI&: *MRI, Reg: BaseOffset, ValueTracking: VT);
1969
1970 if (Readfirstlane) {
1971 // We have the constant offset now, so put the readfirstlane back on the
1972 // variable component.
1973 if (!RBI.constrainGenericRegister(Reg: BaseOffset, RC: AMDGPU::VGPR_32RegClass, MRI&: *MRI))
1974 return false;
1975
1976 Readfirstlane->getOperand(i: 1).setReg(BaseOffset);
1977 BaseOffset = Readfirstlane->getOperand(i: 0).getReg();
1978 } else {
1979 if (!RBI.constrainGenericRegister(Reg: BaseOffset,
1980 RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
1981 return false;
1982 }
1983
1984 Register M0Base = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
1985 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: M0Base)
1986 .addReg(RegNo: BaseOffset)
1987 .addImm(Val: 16)
1988 .setOperandDead(3); // Dead scc
1989
1990 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
1991 .addReg(RegNo: M0Base);
1992 }
1993
1994 // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
1995 // offset field) % 64. Some versions of the programming guide omit the m0
1996 // part, or claim it's from offset 0.
1997
1998 unsigned Opc = gwsIntrinToOpcode(IntrID: IID);
1999 const MCInstrDesc &InstrDesc = TII.get(Opcode: Opc);
2000
2001 if (HasVSrc) {
2002 Register VSrc = MI.getOperand(i: 1).getReg();
2003
2004 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
2005 const TargetRegisterClass *DataRC = TII.getRegClass(MCID: InstrDesc, OpNum: Data0Idx);
2006 const TargetRegisterClass *SubRC =
2007 TRI.getSubRegisterClass(DataRC, AMDGPU::sub0);
2008
2009 if (!SubRC) {
2010 // 32-bit normal case.
2011 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *DataRC, MRI&: *MRI))
2012 return false;
2013
2014 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2015 .addReg(RegNo: VSrc)
2016 .addImm(Val: ImmOffset)
2017 .cloneMemRefs(OtherMI: MI);
2018 } else {
2019 // Requires even register alignment, so create 64-bit value and pad the
2020 // top half with undef.
2021 Register DataReg = MRI->createVirtualRegister(RegClass: DataRC);
2022 if (!RBI.constrainGenericRegister(Reg: VSrc, RC: *SubRC, MRI&: *MRI))
2023 return false;
2024
2025 Register UndefReg = MRI->createVirtualRegister(RegClass: SubRC);
2026 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2027 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DataReg)
2028 .addReg(RegNo: VSrc)
2029 .addImm(Val: AMDGPU::sub0)
2030 .addReg(RegNo: UndefReg)
2031 .addImm(Val: AMDGPU::sub1);
2032
2033 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2034 .addReg(RegNo: DataReg)
2035 .addImm(Val: ImmOffset)
2036 .cloneMemRefs(OtherMI: MI);
2037 }
2038 } else {
2039 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: InstrDesc)
2040 .addImm(Val: ImmOffset)
2041 .cloneMemRefs(OtherMI: MI);
2042 }
2043
2044 MI.eraseFromParent();
2045 return true;
2046}
2047
2048bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI,
2049 bool IsAppend) const {
2050 Register PtrBase = MI.getOperand(i: 2).getReg();
2051 LLT PtrTy = MRI->getType(Reg: PtrBase);
2052 bool IsGDS = PtrTy.getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2053
2054 unsigned Offset;
2055 std::tie(args&: PtrBase, args&: Offset) = selectDS1Addr1OffsetImpl(Root&: MI.getOperand(i: 2));
2056
2057 // TODO: Should this try to look through readfirstlane like GWS?
2058 if (!isDSOffsetLegal(Base: PtrBase, Offset)) {
2059 PtrBase = MI.getOperand(i: 2).getReg();
2060 Offset = 0;
2061 }
2062
2063 MachineBasicBlock *MBB = MI.getParent();
2064 const DebugLoc &DL = MI.getDebugLoc();
2065 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2066
2067 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
2068 .addReg(RegNo: PtrBase);
2069 if (!RBI.constrainGenericRegister(Reg: PtrBase, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
2070 return false;
2071
2072 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: MI.getOperand(i: 0).getReg())
2073 .addImm(Val: Offset)
2074 .addImm(Val: IsGDS ? -1 : 0)
2075 .cloneMemRefs(OtherMI: MI);
2076 MI.eraseFromParent();
2077 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2078 return true;
2079}
2080
2081bool AMDGPUInstructionSelector::selectInitWholeWave(MachineInstr &MI) const {
2082 MachineFunction *MF = MI.getMF();
2083 SIMachineFunctionInfo *MFInfo = MF->getInfo<SIMachineFunctionInfo>();
2084
2085 MFInfo->setInitWholeWave();
2086 return selectImpl(I&: MI, CoverageInfo&: *CoverageInfo);
2087}
2088
2089static bool parseTexFail(uint64_t TexFailCtrl, bool &TFE, bool &LWE,
2090 bool &IsTexFail) {
2091 if (TexFailCtrl)
2092 IsTexFail = true;
2093
2094 TFE = TexFailCtrl & 0x1;
2095 TexFailCtrl &= ~(uint64_t)0x1;
2096 LWE = TexFailCtrl & 0x2;
2097 TexFailCtrl &= ~(uint64_t)0x2;
2098
2099 return TexFailCtrl == 0;
2100}
2101
2102bool AMDGPUInstructionSelector::selectImageIntrinsic(
2103 MachineInstr &MI, const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
2104 MachineBasicBlock *MBB = MI.getParent();
2105 const DebugLoc &DL = MI.getDebugLoc();
2106 unsigned IntrOpcode = Intr->BaseOpcode;
2107
2108 // For image atomic: use no-return opcode if result is unused.
2109 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode) {
2110 Register ResultDef = MI.getOperand(i: 0).getReg();
2111 if (MRI->use_nodbg_empty(RegNo: ResultDef))
2112 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
2113 }
2114
2115 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
2116 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
2117
2118 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
2119 const bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI);
2120 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI);
2121 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
2122 const bool IsGFX13Plus = AMDGPU::isGFX13Plus(STI);
2123
2124 const unsigned ArgOffset = MI.getNumExplicitDefs() + 1;
2125
2126 Register VDataIn = AMDGPU::NoRegister;
2127 Register VDataOut = AMDGPU::NoRegister;
2128 LLT VDataTy;
2129 int NumVDataDwords = -1;
2130 bool IsD16 = MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16 ||
2131 MI.getOpcode() == AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16;
2132
2133 bool Unorm;
2134 if (!BaseOpcode->Sampler)
2135 Unorm = true;
2136 else
2137 Unorm = MI.getOperand(i: ArgOffset + Intr->UnormIndex).getImm() != 0;
2138
2139 bool TFE;
2140 bool LWE;
2141 bool IsTexFail = false;
2142 if (!parseTexFail(TexFailCtrl: MI.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex).getImm(),
2143 TFE, LWE, IsTexFail))
2144 return false;
2145
2146 const int Flags = MI.getOperand(i: ArgOffset + Intr->NumArgs).getImm();
2147 const bool IsA16 = (Flags & 1) != 0;
2148 const bool IsG16 = (Flags & 2) != 0;
2149
2150 // A16 implies 16 bit gradients if subtarget doesn't support G16
2151 if (IsA16 && !STI.hasG16() && !IsG16)
2152 return false;
2153
2154 unsigned DMask = 0;
2155 unsigned DMaskLanes = 0;
2156
2157 if (BaseOpcode->Atomic) {
2158 if (!BaseOpcode->NoReturn)
2159 VDataOut = MI.getOperand(i: 0).getReg();
2160 VDataIn = MI.getOperand(i: 2).getReg();
2161 LLT Ty = MRI->getType(Reg: VDataIn);
2162
2163 // Be careful to allow atomic swap on 16-bit element vectors.
2164 const bool Is64Bit = BaseOpcode->AtomicX2 ?
2165 Ty.getSizeInBits() == 128 :
2166 Ty.getSizeInBits() == 64;
2167
2168 if (BaseOpcode->AtomicX2) {
2169 assert(MI.getOperand(3).getReg() == AMDGPU::NoRegister);
2170
2171 DMask = Is64Bit ? 0xf : 0x3;
2172 NumVDataDwords = Is64Bit ? 4 : 2;
2173 } else {
2174 DMask = Is64Bit ? 0x3 : 0x1;
2175 NumVDataDwords = Is64Bit ? 2 : 1;
2176 }
2177 } else {
2178 DMask = MI.getOperand(i: ArgOffset + Intr->DMaskIndex).getImm();
2179 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
2180
2181 if (BaseOpcode->Store) {
2182 VDataIn = MI.getOperand(i: 1).getReg();
2183 VDataTy = MRI->getType(Reg: VDataIn);
2184 NumVDataDwords = (VDataTy.getSizeInBits() + 31) / 32;
2185 } else if (BaseOpcode->NoReturn) {
2186 NumVDataDwords = 0;
2187 } else {
2188 VDataOut = MI.getOperand(i: 0).getReg();
2189 VDataTy = MRI->getType(Reg: VDataOut);
2190 NumVDataDwords = DMaskLanes;
2191
2192 if (IsD16 && !STI.hasUnpackedD16VMem())
2193 NumVDataDwords = (DMaskLanes + 1) / 2;
2194 }
2195 }
2196
2197 // Set G16 opcode
2198 if (Subtarget->hasG16() && IsG16) {
2199 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
2200 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
2201 assert(G16MappingInfo);
2202 IntrOpcode = G16MappingInfo->G16; // set opcode to variant with _g16
2203 }
2204
2205 // TODO: Check this in verifier.
2206 assert((!IsTexFail || DMaskLanes >= 1) && "should have legalized this");
2207
2208 unsigned CPol = MI.getOperand(i: ArgOffset + Intr->CachePolicyIndex).getImm();
2209 // Keep GLC only when the atomic's result is actually used.
2210 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
2211 CPol |= AMDGPU::CPol::GLC;
2212 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
2213 AMDGPU::CPol::VOLATILE))
2214 return false;
2215
2216 int NumVAddrRegs = 0;
2217 int NumVAddrDwords = 0;
2218 for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
2219 // Skip the $noregs and 0s inserted during legalization.
2220 MachineOperand &AddrOp = MI.getOperand(i: ArgOffset + I);
2221 if (!AddrOp.isReg())
2222 continue; // XXX - Break?
2223
2224 Register Addr = AddrOp.getReg();
2225 if (!Addr)
2226 break;
2227
2228 ++NumVAddrRegs;
2229 NumVAddrDwords += (MRI->getType(Reg: Addr).getSizeInBits() + 31) / 32;
2230 }
2231
2232 // The legalizer preprocessed the intrinsic arguments. If we aren't using
2233 // NSA, these should have been packed into a single value in the first
2234 // address register
2235 const bool UseNSA =
2236 NumVAddrRegs != 1 &&
2237 (STI.hasPartialNSAEncoding() ? NumVAddrDwords >= NumVAddrRegs
2238 : NumVAddrDwords == NumVAddrRegs);
2239 if (UseNSA && !STI.hasFeature(Feature: AMDGPU::FeatureNSAEncoding)) {
2240 LLVM_DEBUG(dbgs() << "Trying to use NSA on non-NSA target\n");
2241 return false;
2242 }
2243
2244 if (IsTexFail)
2245 ++NumVDataDwords;
2246
2247 int Opcode = -1;
2248 if (IsGFX13Plus) {
2249 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx13,
2250 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2251 } else if (IsGFX12Plus) {
2252 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
2253 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2254 } else if (IsGFX11Plus) {
2255 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2256 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
2257 : AMDGPU::MIMGEncGfx11Default,
2258 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2259 } else if (IsGFX10Plus) {
2260 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
2261 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
2262 : AMDGPU::MIMGEncGfx10Default,
2263 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2264 } else {
2265 if (Subtarget->hasGFX90AInsts()) {
2266 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
2267 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2268 if (Opcode == -1) {
2269 LLVM_DEBUG(
2270 dbgs()
2271 << "requested image instruction is not supported on this GPU\n");
2272 return false;
2273 }
2274 }
2275 if (Opcode == -1 &&
2276 STI.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
2277 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
2278 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2279 if (Opcode == -1)
2280 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
2281 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
2282 }
2283 if (Opcode == -1)
2284 return false;
2285
2286 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode))
2287 .cloneMemRefs(OtherMI: MI);
2288
2289 if (VDataOut) {
2290 if (BaseOpcode->AtomicX2) {
2291 const bool Is64 = MRI->getType(Reg: VDataOut).getSizeInBits() == 64;
2292
2293 Register TmpReg = MRI->createVirtualRegister(
2294 RegClass: Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass);
2295 unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0;
2296
2297 MIB.addDef(RegNo: TmpReg);
2298 if (!MRI->use_empty(RegNo: VDataOut)) {
2299 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VDataOut)
2300 .addReg(RegNo: TmpReg, Flags: RegState::Kill, SubReg);
2301 }
2302
2303 } else {
2304 MIB.addDef(RegNo: VDataOut); // vdata output
2305 }
2306 }
2307
2308 if (VDataIn)
2309 MIB.addReg(RegNo: VDataIn); // vdata input
2310
2311 for (int I = 0; I != NumVAddrRegs; ++I) {
2312 MachineOperand &SrcOp = MI.getOperand(i: ArgOffset + Intr->VAddrStart + I);
2313 if (SrcOp.isReg()) {
2314 assert(SrcOp.getReg() != 0);
2315 MIB.addReg(RegNo: SrcOp.getReg());
2316 }
2317 }
2318
2319 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->RsrcIndex).getReg());
2320 if (BaseOpcode->Sampler)
2321 MIB.addReg(RegNo: MI.getOperand(i: ArgOffset + Intr->SampIndex).getReg());
2322
2323 MIB.addImm(Val: DMask); // dmask
2324
2325 if (IsGFX10Plus)
2326 MIB.addImm(Val: DimInfo->Encoding);
2327 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::unorm))
2328 MIB.addImm(Val: Unorm);
2329
2330 MIB.addImm(Val: CPol);
2331 MIB.addImm(Val: IsA16 && // a16 or r128
2332 STI.hasFeature(Feature: AMDGPU::FeatureR128A16) ? -1 : 0);
2333 if (IsGFX10Plus)
2334 MIB.addImm(Val: IsA16 ? -1 : 0);
2335
2336 if (!Subtarget->hasGFX90AInsts()) {
2337 MIB.addImm(Val: TFE); // tfe
2338 } else if (TFE) {
2339 LLVM_DEBUG(dbgs() << "TFE is not supported on this GPU\n");
2340 return false;
2341 }
2342
2343 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::lwe))
2344 MIB.addImm(Val: LWE); // lwe
2345 if (!IsGFX10Plus)
2346 MIB.addImm(Val: DimInfo->DA ? -1 : 0);
2347 if (BaseOpcode->HasD16)
2348 MIB.addImm(Val: IsD16 ? -1 : 0);
2349
2350 MI.eraseFromParent();
2351 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2352 TII.enforceOperandRCAlignment(MI&: *MIB, OpName: AMDGPU::OpName::vaddr);
2353 return true;
2354}
2355
2356// We need to handle this here because tablegen doesn't support matching
2357// instructions with multiple outputs.
2358bool AMDGPUInstructionSelector::selectDSBvhStackIntrinsic(
2359 MachineInstr &MI) const {
2360 Register Dst0 = MI.getOperand(i: 0).getReg();
2361 Register Dst1 = MI.getOperand(i: 1).getReg();
2362
2363 const DebugLoc &DL = MI.getDebugLoc();
2364 MachineBasicBlock *MBB = MI.getParent();
2365
2366 Register Addr = MI.getOperand(i: 3).getReg();
2367 Register Data0 = MI.getOperand(i: 4).getReg();
2368 Register Data1 = MI.getOperand(i: 5).getReg();
2369 unsigned Offset = MI.getOperand(i: 6).getImm();
2370
2371 unsigned Opc;
2372 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
2373 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2374 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2375 Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
2376 break;
2377 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2378 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32;
2379 break;
2380 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2381 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64;
2382 break;
2383 }
2384
2385 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: Dst0)
2386 .addDef(RegNo: Dst1)
2387 .addUse(RegNo: Addr)
2388 .addUse(RegNo: Data0)
2389 .addUse(RegNo: Data1)
2390 .addImm(Val: Offset)
2391 .cloneMemRefs(OtherMI: MI);
2392
2393 MI.eraseFromParent();
2394 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
2395 return true;
2396}
2397
2398bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
2399 MachineInstr &I) const {
2400 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: I).getIntrinsicID();
2401 switch (IntrinsicID) {
2402 case Intrinsic::amdgcn_end_cf:
2403 return selectEndCfIntrinsic(MI&: I);
2404 case Intrinsic::amdgcn_ds_ordered_add:
2405 case Intrinsic::amdgcn_ds_ordered_swap:
2406 return selectDSOrderedIntrinsic(MI&: I, IntrID: IntrinsicID);
2407 case Intrinsic::amdgcn_ds_gws_init:
2408 case Intrinsic::amdgcn_ds_gws_barrier:
2409 case Intrinsic::amdgcn_ds_gws_sema_v:
2410 case Intrinsic::amdgcn_ds_gws_sema_br:
2411 case Intrinsic::amdgcn_ds_gws_sema_p:
2412 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2413 return selectDSGWSIntrinsic(MI&: I, IID: IntrinsicID);
2414 case Intrinsic::amdgcn_ds_append:
2415 return selectDSAppendConsume(MI&: I, IsAppend: true);
2416 case Intrinsic::amdgcn_ds_consume:
2417 return selectDSAppendConsume(MI&: I, IsAppend: false);
2418 case Intrinsic::amdgcn_init_whole_wave:
2419 return selectInitWholeWave(MI&: I);
2420 case Intrinsic::amdgcn_raw_buffer_load_lds:
2421 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
2422 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
2423 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
2424 case Intrinsic::amdgcn_struct_buffer_load_lds:
2425 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
2426 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
2427 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
2428 return selectBufferLoadLds(MI&: I);
2429 // Until we can store both the address space of the global and the LDS
2430 // arguments by having tto MachineMemOperands on an intrinsic, we just trust
2431 // that the argument is a global pointer (buffer pointers have been handled by
2432 // a LLVM IR-level lowering).
2433 case Intrinsic::amdgcn_load_to_lds:
2434 case Intrinsic::amdgcn_load_async_to_lds:
2435 case Intrinsic::amdgcn_global_load_lds:
2436 case Intrinsic::amdgcn_global_load_async_lds:
2437 return selectGlobalLoadLds(MI&: I);
2438 case Intrinsic::amdgcn_tensor_load_to_lds:
2439 case Intrinsic::amdgcn_tensor_store_from_lds:
2440 return selectTensorLoadStore(MI&: I, IID: IntrinsicID);
2441 case Intrinsic::amdgcn_asyncmark:
2442 case Intrinsic::amdgcn_wait_asyncmark:
2443 if (!Subtarget->hasAsyncMark())
2444 return false;
2445 break;
2446 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
2447 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
2448 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
2449 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
2450 return selectDSBvhStackIntrinsic(MI&: I);
2451 case Intrinsic::amdgcn_s_alloc_vgpr: {
2452 // S_ALLOC_VGPR doesn't have a destination register, it just implicitly sets
2453 // SCC. We then need to COPY it into the result vreg.
2454 MachineBasicBlock *MBB = I.getParent();
2455 const DebugLoc &DL = I.getDebugLoc();
2456
2457 Register ResReg = I.getOperand(i: 0).getReg();
2458
2459 MachineInstr *AllocMI = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ALLOC_VGPR))
2460 .add(MO: I.getOperand(i: 2));
2461 (void)BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: ResReg)
2462 .addReg(RegNo: AMDGPU::SCC);
2463 I.eraseFromParent();
2464 constrainSelectedInstRegOperands(I&: *AllocMI, TII, TRI, RBI);
2465 return RBI.constrainGenericRegister(Reg: ResReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2466 }
2467 case Intrinsic::amdgcn_s_barrier_init:
2468 case Intrinsic::amdgcn_s_barrier_signal_var:
2469 return selectNamedBarrierInit(I, IID: IntrinsicID);
2470 case Intrinsic::amdgcn_s_wakeup_barrier:
2471 case Intrinsic::amdgcn_s_barrier_join:
2472 case Intrinsic::amdgcn_s_get_named_barrier_state:
2473 return selectNamedBarrierInst(I, IID: IntrinsicID);
2474 case Intrinsic::amdgcn_s_get_barrier_state:
2475 return selectSGetBarrierState(I, IID: IntrinsicID);
2476 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
2477 return selectSBarrierSignalIsfirst(I, IID: IntrinsicID);
2478 }
2479 return selectImpl(I, CoverageInfo&: *CoverageInfo);
2480}
2481
2482bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
2483 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
2484 return true;
2485
2486 MachineBasicBlock *BB = I.getParent();
2487 const DebugLoc &DL = I.getDebugLoc();
2488
2489 Register DstReg = I.getOperand(i: 0).getReg();
2490 unsigned Size = RBI.getSizeInBits(Reg: DstReg, MRI: *MRI, TRI);
2491 assert(Size <= 32 || Size == 64);
2492 const MachineOperand &CCOp = I.getOperand(i: 1);
2493 Register CCReg = CCOp.getReg();
2494 if (!isVCC(Reg: CCReg, MRI: *MRI)) {
2495 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 :
2496 AMDGPU::S_CSELECT_B32;
2497 MachineInstr *CopySCC = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
2498 .addReg(RegNo: CCReg);
2499
2500 // The generic constrainSelectedInstRegOperands doesn't work for the scc register
2501 // bank, because it does not cover the register class that we used to represent
2502 // for it. So we need to manually set the register class here.
2503 if (!MRI->getRegClassOrNull(Reg: CCReg))
2504 MRI->setRegClass(Reg: CCReg, RC: TRI.getConstrainedRegClassForReg(Reg: CCReg, MRI: *MRI));
2505 MachineInstr *Select = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: SelectOpcode), DestReg: DstReg)
2506 .add(MO: I.getOperand(i: 2))
2507 .add(MO: I.getOperand(i: 3));
2508
2509 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2510 constrainSelectedInstRegOperands(I&: *CopySCC, TII, TRI, RBI);
2511 I.eraseFromParent();
2512 return true;
2513 }
2514
2515 // Wide VGPR select should have been split in RegBankSelect.
2516 if (Size > 32)
2517 return false;
2518
2519 MachineInstr *Select =
2520 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
2521 .addImm(Val: 0)
2522 .add(MO: I.getOperand(i: 3))
2523 .addImm(Val: 0)
2524 .add(MO: I.getOperand(i: 2))
2525 .add(MO: I.getOperand(i: 1));
2526
2527 constrainSelectedInstRegOperands(I&: *Select, TII, TRI, RBI);
2528 I.eraseFromParent();
2529 return true;
2530}
2531
2532bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
2533 Register DstReg = I.getOperand(i: 0).getReg();
2534 Register SrcReg = I.getOperand(i: 1).getReg();
2535 const LLT DstTy = MRI->getType(Reg: DstReg);
2536 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2537 const LLT S1 = LLT::scalar(SizeInBits: 1);
2538
2539 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2540 const RegisterBank *DstRB;
2541 if (DstTy == S1) {
2542 // This is a special case. We don't treat s1 for legalization artifacts as
2543 // vcc booleans.
2544 DstRB = SrcRB;
2545 } else {
2546 DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2547 if (SrcRB != DstRB)
2548 return false;
2549 }
2550
2551 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
2552
2553 unsigned DstSize = DstTy.getSizeInBits();
2554 unsigned SrcSize = SrcTy.getSizeInBits();
2555
2556 const TargetRegisterClass *SrcRC =
2557 TRI.getRegClassForSizeOnBank(Size: SrcSize, Bank: *SrcRB);
2558 const TargetRegisterClass *DstRC =
2559 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
2560 if (!SrcRC || !DstRC)
2561 return false;
2562
2563 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
2564 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI)) {
2565 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
2566 return false;
2567 }
2568
2569 if (DstRC == &AMDGPU::VGPR_16RegClass && SrcSize == 32) {
2570 assert(STI.useRealTrue16Insts());
2571 const DebugLoc &DL = I.getDebugLoc();
2572 MachineBasicBlock *MBB = I.getParent();
2573 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
2574 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::lo16);
2575 I.eraseFromParent();
2576 return true;
2577 }
2578
2579 if (DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) && SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32)) {
2580 MachineBasicBlock *MBB = I.getParent();
2581 const DebugLoc &DL = I.getDebugLoc();
2582
2583 Register LoReg = MRI->createVirtualRegister(RegClass: DstRC);
2584 Register HiReg = MRI->createVirtualRegister(RegClass: DstRC);
2585 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
2586 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
2587 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
2588 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
2589
2590 if (IsVALU && STI.hasSDWA()) {
2591 // Write the low 16-bits of the high element into the high 16-bits of the
2592 // low element.
2593 MachineInstr *MovSDWA =
2594 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_sdwa), DestReg: DstReg)
2595 .addImm(Val: 0) // $src0_modifiers
2596 .addReg(RegNo: HiReg) // $src0
2597 .addImm(Val: 0) // $clamp
2598 .addImm(Val: AMDGPU::SDWA::WORD_1) // $dst_sel
2599 .addImm(Val: AMDGPU::SDWA::UNUSED_PRESERVE) // $dst_unused
2600 .addImm(Val: AMDGPU::SDWA::WORD_0) // $src0_sel
2601 .addReg(RegNo: LoReg, Flags: RegState::Implicit);
2602 MovSDWA->tieOperands(DefIdx: 0, UseIdx: MovSDWA->getNumOperands() - 1);
2603 } else {
2604 Register TmpReg0 = MRI->createVirtualRegister(RegClass: DstRC);
2605 Register TmpReg1 = MRI->createVirtualRegister(RegClass: DstRC);
2606 Register ImmReg = MRI->createVirtualRegister(RegClass: DstRC);
2607 if (IsVALU) {
2608 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: TmpReg0)
2609 .addImm(Val: 16)
2610 .addReg(RegNo: HiReg);
2611 } else {
2612 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg0)
2613 .addReg(RegNo: HiReg)
2614 .addImm(Val: 16)
2615 .setOperandDead(3); // Dead scc
2616 }
2617
2618 unsigned MovOpc = IsVALU ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2619 unsigned AndOpc = IsVALU ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
2620 unsigned OrOpc = IsVALU ? AMDGPU::V_OR_B32_e64 : AMDGPU::S_OR_B32;
2621
2622 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: MovOpc), DestReg: ImmReg)
2623 .addImm(Val: 0xffff);
2624 auto And = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: AndOpc), DestReg: TmpReg1)
2625 .addReg(RegNo: LoReg)
2626 .addReg(RegNo: ImmReg);
2627 auto Or = BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII.get(Opcode: OrOpc), DestReg: DstReg)
2628 .addReg(RegNo: TmpReg0)
2629 .addReg(RegNo: TmpReg1);
2630
2631 if (!IsVALU) {
2632 And.setOperandDead(3); // Dead scc
2633 Or.setOperandDead(3); // Dead scc
2634 }
2635 }
2636
2637 I.eraseFromParent();
2638 return true;
2639 }
2640
2641 if (!DstTy.isScalar())
2642 return false;
2643
2644 if (SrcSize > 32) {
2645 unsigned SubRegIdx = DstSize < 32
2646 ? static_cast<unsigned>(AMDGPU::sub0)
2647 : TRI.getSubRegFromChannel(Channel: 0, NumRegs: DstSize / 32);
2648 if (SubRegIdx == AMDGPU::NoSubRegister)
2649 return false;
2650
2651 // Deal with weird cases where the class only partially supports the subreg
2652 // index.
2653 const TargetRegisterClass *SrcWithSubRC
2654 = TRI.getSubClassWithSubReg(SrcRC, SubRegIdx);
2655 if (!SrcWithSubRC)
2656 return false;
2657
2658 if (SrcWithSubRC != SrcRC) {
2659 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcWithSubRC, MRI&: *MRI))
2660 return false;
2661 }
2662
2663 I.getOperand(i: 1).setSubReg(SubRegIdx);
2664 }
2665
2666 I.setDesc(TII.get(Opcode: TargetOpcode::COPY));
2667 return true;
2668}
2669
2670/// \returns true if a bitmask for \p Size bits will be an inline immediate.
2671static bool shouldUseAndMask(unsigned Size, unsigned &Mask) {
2672 Mask = maskTrailingOnes<unsigned>(N: Size);
2673 int SignedMask = static_cast<int>(Mask);
2674 return SignedMask >= -16 && SignedMask <= 64;
2675}
2676
2677// Like RegisterBankInfo::getRegBank, but don't assume vcc for s1.
2678const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank(
2679 Register Reg, const MachineRegisterInfo &MRI,
2680 const TargetRegisterInfo &TRI) const {
2681 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
2682 if (auto *RB = dyn_cast<const RegisterBank *>(Val: RegClassOrBank))
2683 return RB;
2684
2685 // Ignore the type, since we don't use vcc in artifacts.
2686 if (auto *RC = dyn_cast<const TargetRegisterClass *>(Val: RegClassOrBank))
2687 return &RBI.getRegBankFromRegClass(RC: *RC, LLT());
2688 return nullptr;
2689}
2690
2691bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
2692 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG;
2693 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg;
2694 const DebugLoc &DL = I.getDebugLoc();
2695 MachineBasicBlock &MBB = *I.getParent();
2696 const Register DstReg = I.getOperand(i: 0).getReg();
2697 const Register SrcReg = I.getOperand(i: 1).getReg();
2698
2699 const LLT DstTy = MRI->getType(Reg: DstReg);
2700 const LLT SrcTy = MRI->getType(Reg: SrcReg);
2701 const unsigned SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG ?
2702 I.getOperand(i: 2).getImm() : SrcTy.getSizeInBits();
2703 const unsigned DstSize = DstTy.getSizeInBits();
2704 if (!DstTy.isScalar())
2705 return false;
2706
2707 // Artifact casts should never use vcc.
2708 const RegisterBank *SrcBank = getArtifactRegBank(Reg: SrcReg, MRI: *MRI, TRI);
2709
2710 // FIXME: This should probably be illegal and split earlier.
2711 if (I.getOpcode() == AMDGPU::G_ANYEXT) {
2712 if (DstSize <= 32)
2713 return selectCOPY(I);
2714
2715 const TargetRegisterClass *SrcRC =
2716 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcBank);
2717 const RegisterBank *DstBank = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
2718 const TargetRegisterClass *DstRC =
2719 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstBank);
2720
2721 Register UndefReg = MRI->createVirtualRegister(RegClass: SrcRC);
2722 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2723 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2724 .addReg(RegNo: SrcReg)
2725 .addImm(Val: AMDGPU::sub0)
2726 .addReg(RegNo: UndefReg)
2727 .addImm(Val: AMDGPU::sub1);
2728 I.eraseFromParent();
2729
2730 return RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) &&
2731 RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI);
2732 }
2733
2734 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) {
2735 // 64-bit should have been split up in RegBankSelect
2736
2737 // Try to use an and with a mask if it will save code size.
2738 unsigned Mask;
2739 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2740 MachineInstr *ExtI =
2741 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e32), DestReg: DstReg)
2742 .addImm(Val: Mask)
2743 .addReg(RegNo: SrcReg);
2744 I.eraseFromParent();
2745 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2746 return true;
2747 }
2748
2749 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2750 MachineInstr *ExtI =
2751 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE), DestReg: DstReg)
2752 .addReg(RegNo: SrcReg)
2753 .addImm(Val: 0) // Offset
2754 .addImm(Val: SrcSize); // Width
2755 I.eraseFromParent();
2756 constrainSelectedInstRegOperands(I&: *ExtI, TII, TRI, RBI);
2757 return true;
2758 }
2759
2760 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
2761 const TargetRegisterClass &SrcRC = InReg && DstSize > 32 ?
2762 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass;
2763 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: SrcRC, MRI&: *MRI))
2764 return false;
2765
2766 if (Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) {
2767 const unsigned SextOpc = SrcSize == 8 ?
2768 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16;
2769 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: SextOpc), DestReg: DstReg)
2770 .addReg(RegNo: SrcReg);
2771 I.eraseFromParent();
2772 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2773 }
2774
2775 // Using a single 32-bit SALU to calculate the high half is smaller than
2776 // S_BFE with a literal constant operand.
2777 if (DstSize > 32 && SrcSize == 32) {
2778 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2779 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2780 if (Signed) {
2781 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ASHR_I32), DestReg: HiReg)
2782 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2783 .addImm(Val: 31)
2784 .setOperandDead(3); // Dead scc
2785 } else {
2786 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: HiReg)
2787 .addImm(Val: 0);
2788 }
2789 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
2790 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2791 .addImm(Val: AMDGPU::sub0)
2792 .addReg(RegNo: HiReg)
2793 .addImm(Val: AMDGPU::sub1);
2794 I.eraseFromParent();
2795 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass,
2796 MRI&: *MRI);
2797 }
2798
2799 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
2800 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2801
2802 // Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width.
2803 if (DstSize > 32 && (SrcSize <= 32 || InReg)) {
2804 // We need a 64-bit register source, but the high bits don't matter.
2805 Register ExtReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
2806 Register UndefReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2807 unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
2808
2809 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefReg);
2810 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: ExtReg)
2811 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
2812 .addImm(Val: AMDGPU::sub0)
2813 .addReg(RegNo: UndefReg)
2814 .addImm(Val: AMDGPU::sub1);
2815
2816 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE64), DestReg: DstReg)
2817 .addReg(RegNo: ExtReg)
2818 .addImm(Val: SrcSize << 16);
2819
2820 I.eraseFromParent();
2821 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI);
2822 }
2823
2824 unsigned Mask;
2825 if (!Signed && shouldUseAndMask(Size: SrcSize, Mask)) {
2826 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: DstReg)
2827 .addReg(RegNo: SrcReg)
2828 .addImm(Val: Mask)
2829 .setOperandDead(3); // Dead scc
2830 } else {
2831 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII.get(Opcode: BFE32), DestReg: DstReg)
2832 .addReg(RegNo: SrcReg)
2833 .addImm(Val: SrcSize << 16);
2834 }
2835
2836 I.eraseFromParent();
2837 return RBI.constrainGenericRegister(Reg: DstReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2838 }
2839
2840 return false;
2841}
2842
2843static Register stripCopy(Register Reg, MachineRegisterInfo &MRI) {
2844 return getDefSrcRegIgnoringCopies(Reg, MRI)->Reg;
2845}
2846
2847static Register stripBitCast(Register Reg, MachineRegisterInfo &MRI) {
2848 Register BitcastSrc;
2849 if (mi_match(R: Reg, MRI, P: m_GBitcast(Src: m_Reg(R&: BitcastSrc))))
2850 Reg = BitcastSrc;
2851 return Reg;
2852}
2853
2854static bool isExtractHiElt(MachineRegisterInfo &MRI, Register In,
2855 Register &Out) {
2856 // When unmerging a register that is composed of 2 x 16-bit values allow to
2857 // use an extract hi instruction for the upper 16 bits. We only need to check
2858 // the size of `In` as all defs are guaranteed to be the same type for
2859 // GUnmerge.
2860 GUnmerge *Unmerge;
2861 if (mi_match(R: In, MRI, P: m_GUnmerge(Inst&: Unmerge))) {
2862 if (Unmerge->getNumDefs() == 2 && Unmerge->getOperand(i: 1).getReg() == In &&
2863 MRI.getType(Reg: In).getSizeInBits() == 16) {
2864 Out = Unmerge->getSourceReg();
2865 return true;
2866 }
2867 }
2868
2869 Register Trunc;
2870 if (!mi_match(R: In, MRI, P: m_GTrunc(Src: m_Reg(R&: Trunc))))
2871 return false;
2872
2873 Register LShlSrc;
2874 Register Cst;
2875 if (mi_match(R: Trunc, MRI, P: m_GLShr(L: m_Reg(R&: LShlSrc), R: m_Reg(R&: Cst)))) {
2876 Cst = stripCopy(Reg: Cst, MRI);
2877 if (mi_match(R: Cst, MRI, P: m_SpecificICst(RequestedValue: 16))) {
2878 Out = stripBitCast(Reg: LShlSrc, MRI);
2879 return true;
2880 }
2881 }
2882
2883 ArrayRef<int> Mask;
2884 Register Src1;
2885 if (!mi_match(R: Trunc, MRI, P: m_GShuffleVector(Src1: m_Reg(R&: Src1), Src2: m_Reg(), Mask)))
2886 return false;
2887
2888 assert(MRI.getType(Src1) == LLT::fixed_vector(2, 16));
2889 assert(Mask.size() == 2);
2890
2891 if (Mask[0] == 1 && Mask[1] <= 1) {
2892 Out = Trunc;
2893 return true;
2894 }
2895
2896 return false;
2897}
2898
2899bool AMDGPUInstructionSelector::selectG_FPEXT(MachineInstr &I) const {
2900 if (!Subtarget->hasSALUFloatInsts())
2901 return false;
2902
2903 Register Dst = I.getOperand(i: 0).getReg();
2904 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2905 if (DstRB->getID() != AMDGPU::SGPRRegBankID)
2906 return false;
2907
2908 Register Src = I.getOperand(i: 1).getReg();
2909
2910 if (MRI->getType(Reg: Dst) == LLT::scalar(SizeInBits: 32) &&
2911 MRI->getType(Reg: Src) == LLT::scalar(SizeInBits: 16)) {
2912 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
2913 MachineBasicBlock *BB = I.getParent();
2914 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_CVT_HI_F32_F16), DestReg: Dst)
2915 .addUse(RegNo: Src);
2916 I.eraseFromParent();
2917 return RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI);
2918 }
2919 }
2920
2921 return false;
2922}
2923
2924bool AMDGPUInstructionSelector::selectG_FNEG(MachineInstr &MI) const {
2925 // Only manually handle the f64 SGPR case.
2926 //
2927 // FIXME: This is a workaround for 2.5 different tablegen problems. Because
2928 // the bit ops theoretically have a second result due to the implicit def of
2929 // SCC, the GlobalISelEmitter is overly conservative and rejects it. Fixing
2930 // that is easy by disabling the check. The result works, but uses a
2931 // nonsensical sreg32orlds_and_sreg_1 regclass.
2932 //
2933 // The DAG emitter is more problematic, and incorrectly adds both S_XOR_B32 to
2934 // the variadic REG_SEQUENCE operands.
2935
2936 Register Dst = MI.getOperand(i: 0).getReg();
2937 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2938 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
2939 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
2940 return false;
2941
2942 Register Src = MI.getOperand(i: 1).getReg();
2943 MachineInstr *Fabs = getOpcodeDef(Opcode: TargetOpcode::G_FABS, Reg: Src, MRI: *MRI);
2944 if (Fabs)
2945 Src = Fabs->getOperand(i: 1).getReg();
2946
2947 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
2948 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
2949 return false;
2950
2951 MachineBasicBlock *BB = MI.getParent();
2952 const DebugLoc &DL = MI.getDebugLoc();
2953 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2954 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2955 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2956 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2957
2958 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
2959 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
2960 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
2961 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
2962 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
2963 .addImm(Val: 0x80000000);
2964
2965 // Set or toggle sign bit.
2966 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32;
2967 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: OpReg)
2968 .addReg(RegNo: HiReg)
2969 .addReg(RegNo: ConstReg)
2970 .setOperandDead(3); // Dead scc
2971 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
2972 .addReg(RegNo: LoReg)
2973 .addImm(Val: AMDGPU::sub0)
2974 .addReg(RegNo: OpReg)
2975 .addImm(Val: AMDGPU::sub1);
2976 MI.eraseFromParent();
2977 return true;
2978}
2979
2980// FIXME: This is a workaround for the same tablegen problems as G_FNEG
2981bool AMDGPUInstructionSelector::selectG_FABS(MachineInstr &MI) const {
2982 Register Dst = MI.getOperand(i: 0).getReg();
2983 const RegisterBank *DstRB = RBI.getRegBank(Reg: Dst, MRI: *MRI, TRI);
2984 if (DstRB->getID() != AMDGPU::SGPRRegBankID ||
2985 MRI->getType(Reg: Dst) != LLT::scalar(SizeInBits: 64))
2986 return false;
2987
2988 Register Src = MI.getOperand(i: 1).getReg();
2989 MachineBasicBlock *BB = MI.getParent();
2990 const DebugLoc &DL = MI.getDebugLoc();
2991 Register LoReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2992 Register HiReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2993 Register ConstReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2994 Register OpReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
2995
2996 if (!RBI.constrainGenericRegister(Reg: Src, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI) ||
2997 !RBI.constrainGenericRegister(Reg: Dst, RC: AMDGPU::SReg_64RegClass, MRI&: *MRI))
2998 return false;
2999
3000 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3001 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub0);
3002 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3003 .addReg(RegNo: Src, Flags: {}, SubReg: AMDGPU::sub1);
3004 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: ConstReg)
3005 .addImm(Val: 0x7fffffff);
3006
3007 // Clear sign bit.
3008 // TODO: Should this used S_BITSET0_*?
3009 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: OpReg)
3010 .addReg(RegNo: HiReg)
3011 .addReg(RegNo: ConstReg)
3012 .setOperandDead(3); // Dead scc
3013 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
3014 .addReg(RegNo: LoReg)
3015 .addImm(Val: AMDGPU::sub0)
3016 .addReg(RegNo: OpReg)
3017 .addImm(Val: AMDGPU::sub1);
3018
3019 MI.eraseFromParent();
3020 return true;
3021}
3022
3023static bool isConstant(const MachineInstr &MI) {
3024 return MI.getOpcode() == TargetOpcode::G_CONSTANT;
3025}
3026
3027void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
3028 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const {
3029
3030 unsigned OpNo = Load.getOpcode() == AMDGPU::G_PREFETCH ? 0 : 1;
3031 const MachineInstr *PtrMI =
3032 MRI.getUniqueVRegDef(Reg: Load.getOperand(i: OpNo).getReg());
3033
3034 assert(PtrMI);
3035
3036 if (PtrMI->getOpcode() != TargetOpcode::G_PTR_ADD)
3037 return;
3038
3039 GEPInfo GEPInfo;
3040
3041 for (unsigned i = 1; i != 3; ++i) {
3042 const MachineOperand &GEPOp = PtrMI->getOperand(i);
3043 const MachineInstr *OpDef = MRI.getUniqueVRegDef(Reg: GEPOp.getReg());
3044 assert(OpDef);
3045 if (i == 2 && isConstant(MI: *OpDef)) {
3046 // TODO: Could handle constant base + variable offset, but a combine
3047 // probably should have commuted it.
3048 assert(GEPInfo.Imm == 0);
3049 GEPInfo.Imm = OpDef->getOperand(i: 1).getCImm()->getSExtValue();
3050 continue;
3051 }
3052 const RegisterBank *OpBank = RBI.getRegBank(Reg: GEPOp.getReg(), MRI, TRI);
3053 if (OpBank->getID() == AMDGPU::SGPRRegBankID)
3054 GEPInfo.SgprParts.push_back(Elt: GEPOp.getReg());
3055 else
3056 GEPInfo.VgprParts.push_back(Elt: GEPOp.getReg());
3057 }
3058
3059 AddrInfo.push_back(Elt: GEPInfo);
3060 getAddrModeInfo(Load: *PtrMI, MRI, AddrInfo);
3061}
3062
3063bool AMDGPUInstructionSelector::isSGPR(Register Reg) const {
3064 return RBI.getRegBank(Reg, MRI: *MRI, TRI)->getID() == AMDGPU::SGPRRegBankID;
3065}
3066
3067bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const {
3068 if (!MI.hasOneMemOperand())
3069 return false;
3070
3071 const MachineMemOperand *MMO = *MI.memoperands_begin();
3072 const Value *Ptr = MMO->getValue();
3073
3074 // UndefValue means this is a load of a kernel input. These are uniform.
3075 // Sometimes LDS instructions have constant pointers.
3076 // If Ptr is null, then that means this mem operand contains a
3077 // PseudoSourceValue like GOT.
3078 if (!Ptr || isa<UndefValue, Argument, Constant, GlobalValue>(Val: Ptr))
3079 return true;
3080
3081 if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
3082 return true;
3083
3084 if (MI.getOpcode() == AMDGPU::G_PREFETCH)
3085 return RBI.getRegBank(Reg: MI.getOperand(i: 0).getReg(), MRI: *MRI, TRI)->getID() ==
3086 AMDGPU::SGPRRegBankID;
3087
3088 const Instruction *I = dyn_cast<Instruction>(Val: Ptr);
3089 return I && I->getMetadata(Kind: "amdgpu.uniform");
3090}
3091
3092bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const {
3093 for (const GEPInfo &GEPInfo : AddrInfo) {
3094 if (!GEPInfo.VgprParts.empty())
3095 return true;
3096 }
3097 return false;
3098}
3099
3100void AMDGPUInstructionSelector::initM0(MachineInstr &I) const {
3101 const LLT PtrTy = MRI->getType(Reg: I.getOperand(i: 1).getReg());
3102 unsigned AS = PtrTy.getAddressSpace();
3103 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) &&
3104 STI.ldsRequiresM0Init()) {
3105 MachineBasicBlock *BB = I.getParent();
3106
3107 // If DS instructions require M0 initialization, insert it before selecting.
3108 BuildMI(BB&: *BB, I: &I, MIMD: I.getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: AMDGPU::M0)
3109 .addImm(Val: -1);
3110 }
3111}
3112
3113bool AMDGPUInstructionSelector::selectG_LOAD_STORE_ATOMICRMW(
3114 MachineInstr &I) const {
3115 initM0(I);
3116 return selectImpl(I, CoverageInfo&: *CoverageInfo);
3117}
3118
3119static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI) {
3120 if (Reg.isPhysical())
3121 return false;
3122
3123 MachineInstr &MI = *MRI.getUniqueVRegDef(Reg);
3124 const unsigned Opcode = MI.getOpcode();
3125
3126 if (Opcode == AMDGPU::COPY)
3127 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI);
3128
3129 if (Opcode == AMDGPU::G_AND || Opcode == AMDGPU::G_OR ||
3130 Opcode == AMDGPU::G_XOR)
3131 return isVCmpResult(Reg: MI.getOperand(i: 1).getReg(), MRI) &&
3132 isVCmpResult(Reg: MI.getOperand(i: 2).getReg(), MRI);
3133
3134 if (auto *GI = dyn_cast<GIntrinsic>(Val: &MI))
3135 return GI->is(ID: Intrinsic::amdgcn_class);
3136
3137 return Opcode == AMDGPU::G_ICMP || Opcode == AMDGPU::G_FCMP;
3138}
3139
3140bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
3141 MachineBasicBlock *BB = I.getParent();
3142 MachineOperand &CondOp = I.getOperand(i: 0);
3143 Register CondReg = CondOp.getReg();
3144 const DebugLoc &DL = I.getDebugLoc();
3145
3146 unsigned BrOpcode;
3147 Register CondPhysReg;
3148 const TargetRegisterClass *ConstrainRC;
3149
3150 // In SelectionDAG, we inspect the IR block for uniformity metadata to decide
3151 // whether the branch is uniform when selecting the instruction. In
3152 // GlobalISel, we should push that decision into RegBankSelect. Assume for now
3153 // RegBankSelect knows what it's doing if the branch condition is scc, even
3154 // though it currently does not.
3155 if (!isVCC(Reg: CondReg, MRI: *MRI)) {
3156 if (MRI->getType(Reg: CondReg) != LLT::scalar(SizeInBits: 32))
3157 return false;
3158
3159 CondPhysReg = AMDGPU::SCC;
3160 BrOpcode = AMDGPU::S_CBRANCH_SCC1;
3161 ConstrainRC = &AMDGPU::SReg_32RegClass;
3162 } else {
3163 // FIXME: Should scc->vcc copies and with exec?
3164
3165 // Unless the value of CondReg is a result of a V_CMP* instruction then we
3166 // need to insert an and with exec.
3167 if (!isVCmpResult(Reg: CondReg, MRI&: *MRI)) {
3168 const bool Is64 = STI.isWave64();
3169 const unsigned Opcode = Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32;
3170 const Register Exec = Is64 ? AMDGPU::EXEC : AMDGPU::EXEC_LO;
3171
3172 Register TmpReg = MRI->createVirtualRegister(RegClass: TRI.getBoolRC());
3173 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode), DestReg: TmpReg)
3174 .addReg(RegNo: CondReg)
3175 .addReg(RegNo: Exec)
3176 .setOperandDead(3); // Dead scc
3177 CondReg = TmpReg;
3178 }
3179
3180 CondPhysReg = TRI.getVCC();
3181 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ;
3182 ConstrainRC = TRI.getBoolRC();
3183 }
3184
3185 if (!MRI->getRegClassOrNull(Reg: CondReg))
3186 MRI->setRegClass(Reg: CondReg, RC: ConstrainRC);
3187
3188 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CondPhysReg)
3189 .addReg(RegNo: CondReg);
3190 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: BrOpcode))
3191 .addMBB(MBB: I.getOperand(i: 1).getMBB());
3192
3193 I.eraseFromParent();
3194 return true;
3195}
3196
3197bool AMDGPUInstructionSelector::selectG_GLOBAL_VALUE(
3198 MachineInstr &I) const {
3199 Register DstReg = I.getOperand(i: 0).getReg();
3200 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3201 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3202 I.setDesc(TII.get(Opcode: IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32));
3203 if (IsVGPR)
3204 I.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
3205
3206 return RBI.constrainGenericRegister(
3207 Reg: DstReg, RC: IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, MRI&: *MRI);
3208}
3209
3210bool AMDGPUInstructionSelector::selectG_PTRMASK(MachineInstr &I) const {
3211 Register DstReg = I.getOperand(i: 0).getReg();
3212 Register SrcReg = I.getOperand(i: 1).getReg();
3213 Register MaskReg = I.getOperand(i: 2).getReg();
3214 LLT Ty = MRI->getType(Reg: DstReg);
3215 LLT MaskTy = MRI->getType(Reg: MaskReg);
3216 MachineBasicBlock *BB = I.getParent();
3217 const DebugLoc &DL = I.getDebugLoc();
3218
3219 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3220 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3221 const RegisterBank *MaskRB = RBI.getRegBank(Reg: MaskReg, MRI: *MRI, TRI);
3222 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
3223 if (DstRB != SrcRB) // Should only happen for hand written MIR.
3224 return false;
3225
3226 // Try to avoid emitting a bit operation when we only need to touch half of
3227 // the 64-bit pointer.
3228 APInt MaskOnes = VT->getKnownOnes(R: MaskReg).zext(width: 64);
3229 const APInt MaskHi32 = APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32);
3230 const APInt MaskLo32 = APInt::getLowBitsSet(numBits: 64, loBitsSet: 32);
3231
3232 const bool CanCopyLow32 = (MaskOnes & MaskLo32) == MaskLo32;
3233 const bool CanCopyHi32 = (MaskOnes & MaskHi32) == MaskHi32;
3234
3235 if (!IsVGPR && Ty.getSizeInBits() == 64 &&
3236 !CanCopyLow32 && !CanCopyHi32) {
3237 auto MIB = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B64), DestReg: DstReg)
3238 .addReg(RegNo: SrcReg)
3239 .addReg(RegNo: MaskReg)
3240 .setOperandDead(3); // Dead scc
3241 I.eraseFromParent();
3242 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3243 return true;
3244 }
3245
3246 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
3247 const TargetRegisterClass &RegRC
3248 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
3249
3250 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *DstRB);
3251 const TargetRegisterClass *SrcRC = TRI.getRegClassForTypeOnBank(Ty, Bank: *SrcRB);
3252 const TargetRegisterClass *MaskRC =
3253 TRI.getRegClassForTypeOnBank(Ty: MaskTy, Bank: *MaskRB);
3254
3255 if (!RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3256 !RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3257 !RBI.constrainGenericRegister(Reg: MaskReg, RC: *MaskRC, MRI&: *MRI))
3258 return false;
3259
3260 if (Ty.getSizeInBits() == 32) {
3261 assert(MaskTy.getSizeInBits() == 32 &&
3262 "ptrmask should have been narrowed during legalize");
3263
3264 auto NewOp = BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: DstReg)
3265 .addReg(RegNo: SrcReg)
3266 .addReg(RegNo: MaskReg);
3267
3268 if (!IsVGPR)
3269 NewOp.setOperandDead(3); // Dead scc
3270 I.eraseFromParent();
3271 return true;
3272 }
3273
3274 Register HiReg = MRI->createVirtualRegister(RegClass: &RegRC);
3275 Register LoReg = MRI->createVirtualRegister(RegClass: &RegRC);
3276
3277 // Extract the subregisters from the source pointer.
3278 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: LoReg)
3279 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub0);
3280 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: HiReg)
3281 .addReg(RegNo: SrcReg, Flags: {}, SubReg: AMDGPU::sub1);
3282
3283 Register MaskedLo, MaskedHi;
3284
3285 if (CanCopyLow32) {
3286 // If all the bits in the low half are 1, we only need a copy for it.
3287 MaskedLo = LoReg;
3288 } else {
3289 // Extract the mask subregister and apply the and.
3290 Register MaskLo = MRI->createVirtualRegister(RegClass: &RegRC);
3291 MaskedLo = MRI->createVirtualRegister(RegClass: &RegRC);
3292
3293 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskLo)
3294 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub0);
3295 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedLo)
3296 .addReg(RegNo: LoReg)
3297 .addReg(RegNo: MaskLo);
3298 }
3299
3300 if (CanCopyHi32) {
3301 // If all the bits in the high half are 1, we only need a copy for it.
3302 MaskedHi = HiReg;
3303 } else {
3304 Register MaskHi = MRI->createVirtualRegister(RegClass: &RegRC);
3305 MaskedHi = MRI->createVirtualRegister(RegClass: &RegRC);
3306
3307 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: MaskHi)
3308 .addReg(RegNo: MaskReg, Flags: {}, SubReg: AMDGPU::sub1);
3309 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: NewOpc), DestReg: MaskedHi)
3310 .addReg(RegNo: HiReg)
3311 .addReg(RegNo: MaskHi);
3312 }
3313
3314 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg)
3315 .addReg(RegNo: MaskedLo)
3316 .addImm(Val: AMDGPU::sub0)
3317 .addReg(RegNo: MaskedHi)
3318 .addImm(Val: AMDGPU::sub1);
3319 I.eraseFromParent();
3320 return true;
3321}
3322
3323/// Return the register to use for the index value, and the subregister to use
3324/// for the indirectly accessed register.
3325static std::pair<Register, unsigned>
3326computeIndirectRegIndex(MachineRegisterInfo &MRI, const SIRegisterInfo &TRI,
3327 const TargetRegisterClass *SuperRC, Register IdxReg,
3328 unsigned EltSize, GISelValueTracking &ValueTracking) {
3329 Register IdxBaseReg;
3330 int Offset;
3331
3332 std::tie(args&: IdxBaseReg, args&: Offset) =
3333 AMDGPU::getBaseWithConstantOffset(MRI, Reg: IdxReg, ValueTracking: &ValueTracking);
3334 if (IdxBaseReg == AMDGPU::NoRegister) {
3335 // This will happen if the index is a known constant. This should ordinarily
3336 // be legalized out, but handle it as a register just in case.
3337 assert(Offset == 0);
3338 IdxBaseReg = IdxReg;
3339 }
3340
3341 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(RC: SuperRC, EltSize);
3342
3343 // Skip out of bounds offsets, or else we would end up using an undefined
3344 // register.
3345 if (static_cast<unsigned>(Offset) >= SubRegs.size())
3346 return std::pair(IdxReg, SubRegs[0]);
3347 return std::pair(IdxBaseReg, SubRegs[Offset]);
3348}
3349
3350bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
3351 MachineInstr &MI) const {
3352 Register DstReg = MI.getOperand(i: 0).getReg();
3353 Register SrcReg = MI.getOperand(i: 1).getReg();
3354 Register IdxReg = MI.getOperand(i: 2).getReg();
3355
3356 LLT DstTy = MRI->getType(Reg: DstReg);
3357 LLT SrcTy = MRI->getType(Reg: SrcReg);
3358
3359 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
3360 const RegisterBank *SrcRB = RBI.getRegBank(Reg: SrcReg, MRI: *MRI, TRI);
3361 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3362
3363 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3364 // into a waterfall loop.
3365 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3366 return false;
3367
3368 const TargetRegisterClass *SrcRC =
3369 TRI.getRegClassForTypeOnBank(Ty: SrcTy, Bank: *SrcRB);
3370 const TargetRegisterClass *DstRC =
3371 TRI.getRegClassForTypeOnBank(Ty: DstTy, Bank: *DstRB);
3372 if (!SrcRC || !DstRC)
3373 return false;
3374 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: *SrcRC, MRI&: *MRI) ||
3375 !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI) ||
3376 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3377 return false;
3378
3379 MachineBasicBlock *BB = MI.getParent();
3380 const DebugLoc &DL = MI.getDebugLoc();
3381 const bool Is64 = DstTy.getSizeInBits() == 64;
3382
3383 unsigned SubReg;
3384 std::tie(args&: IdxReg, args&: SubReg) = computeIndirectRegIndex(
3385 MRI&: *MRI, TRI, SuperRC: SrcRC, IdxReg, EltSize: DstTy.getSizeInBits() / 8, ValueTracking&: *VT);
3386
3387 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) {
3388 if (DstTy.getSizeInBits() != 32 && !Is64)
3389 return false;
3390
3391 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3392 .addReg(RegNo: IdxReg);
3393
3394 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32;
3395 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg)
3396 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3397 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3398 MI.eraseFromParent();
3399 return true;
3400 }
3401
3402 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32)
3403 return false;
3404
3405 if (!STI.useVGPRIndexMode()) {
3406 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3407 .addReg(RegNo: IdxReg);
3408 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: DstReg)
3409 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
3410 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
3411 MI.eraseFromParent();
3412 return true;
3413 }
3414
3415 const MCInstrDesc &GPRIDXDesc =
3416 TII.getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *SrcRC), IsIndirectSrc: true);
3417 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3418 .addReg(RegNo: SrcReg)
3419 .addReg(RegNo: IdxReg)
3420 .addImm(Val: SubReg);
3421
3422 MI.eraseFromParent();
3423 return true;
3424}
3425
3426// TODO: Fold insert_vector_elt (extract_vector_elt) into movrelsd
3427bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT(
3428 MachineInstr &MI) const {
3429 Register DstReg = MI.getOperand(i: 0).getReg();
3430 Register VecReg = MI.getOperand(i: 1).getReg();
3431 Register ValReg = MI.getOperand(i: 2).getReg();
3432 Register IdxReg = MI.getOperand(i: 3).getReg();
3433
3434 LLT VecTy = MRI->getType(Reg: DstReg);
3435 LLT ValTy = MRI->getType(Reg: ValReg);
3436 unsigned VecSize = VecTy.getSizeInBits();
3437 unsigned ValSize = ValTy.getSizeInBits();
3438
3439 const RegisterBank *VecRB = RBI.getRegBank(Reg: VecReg, MRI: *MRI, TRI);
3440 const RegisterBank *ValRB = RBI.getRegBank(Reg: ValReg, MRI: *MRI, TRI);
3441 const RegisterBank *IdxRB = RBI.getRegBank(Reg: IdxReg, MRI: *MRI, TRI);
3442
3443 assert(VecTy.getElementType() == ValTy);
3444
3445 // The index must be scalar. If it wasn't RegBankSelect should have moved this
3446 // into a waterfall loop.
3447 if (IdxRB->getID() != AMDGPU::SGPRRegBankID)
3448 return false;
3449
3450 const TargetRegisterClass *VecRC =
3451 TRI.getRegClassForTypeOnBank(Ty: VecTy, Bank: *VecRB);
3452 const TargetRegisterClass *ValRC =
3453 TRI.getRegClassForTypeOnBank(Ty: ValTy, Bank: *ValRB);
3454
3455 if (!RBI.constrainGenericRegister(Reg: VecReg, RC: *VecRC, MRI&: *MRI) ||
3456 !RBI.constrainGenericRegister(Reg: DstReg, RC: *VecRC, MRI&: *MRI) ||
3457 !RBI.constrainGenericRegister(Reg: ValReg, RC: *ValRC, MRI&: *MRI) ||
3458 !RBI.constrainGenericRegister(Reg: IdxReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
3459 return false;
3460
3461 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32)
3462 return false;
3463
3464 unsigned SubReg;
3465 std::tie(args&: IdxReg, args&: SubReg) =
3466 computeIndirectRegIndex(MRI&: *MRI, TRI, SuperRC: VecRC, IdxReg, EltSize: ValSize / 8, ValueTracking&: *VT);
3467
3468 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID &&
3469 STI.useVGPRIndexMode();
3470
3471 MachineBasicBlock *BB = MI.getParent();
3472 const DebugLoc &DL = MI.getDebugLoc();
3473
3474 if (!IndexMode) {
3475 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3476 .addReg(RegNo: IdxReg);
3477
3478 const MCInstrDesc &RegWriteOp = TII.getIndirectRegWriteMovRelPseudo(
3479 VecSize, EltSize: ValSize, IsSGPR: VecRB->getID() == AMDGPU::SGPRRegBankID);
3480 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: RegWriteOp, DestReg: DstReg)
3481 .addReg(RegNo: VecReg)
3482 .addReg(RegNo: ValReg)
3483 .addImm(Val: SubReg);
3484 MI.eraseFromParent();
3485 return true;
3486 }
3487
3488 const MCInstrDesc &GPRIDXDesc =
3489 TII.getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
3490 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: GPRIDXDesc, DestReg: DstReg)
3491 .addReg(RegNo: VecReg)
3492 .addReg(RegNo: ValReg)
3493 .addReg(RegNo: IdxReg)
3494 .addImm(Val: SubReg);
3495
3496 MI.eraseFromParent();
3497 return true;
3498}
3499
3500static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
3501 switch (Intr) {
3502 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
3503 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
3504 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
3505 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
3506 case Intrinsic::amdgcn_load_async_to_lds:
3507 case Intrinsic::amdgcn_global_load_async_lds:
3508 return true;
3509 }
3510 return false;
3511}
3512
3513bool AMDGPUInstructionSelector::selectBufferLoadLds(MachineInstr &MI) const {
3514 if (!Subtarget->hasVMemToLDSLoad())
3515 return false;
3516 unsigned Opc;
3517 unsigned Size = MI.getOperand(i: 3).getImm();
3518 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3519
3520 // The struct intrinsic variants add one additional operand over raw.
3521 const bool HasVIndex = MI.getNumOperands() == 9;
3522 Register VIndex;
3523 int OpOffset = 0;
3524 if (HasVIndex) {
3525 VIndex = MI.getOperand(i: 4).getReg();
3526 OpOffset = 1;
3527 }
3528
3529 Register VOffset = MI.getOperand(i: 4 + OpOffset).getReg();
3530 std::optional<ValueAndVReg> MaybeVOffset =
3531 getIConstantVRegValWithLookThrough(VReg: VOffset, MRI: *MRI);
3532 const bool HasVOffset = !MaybeVOffset || MaybeVOffset->Value.getZExtValue();
3533
3534 switch (Size) {
3535 default:
3536 return false;
3537 case 1:
3538 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
3539 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
3540 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
3541 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
3542 break;
3543 case 2:
3544 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
3545 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
3546 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
3547 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
3548 break;
3549 case 4:
3550 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
3551 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
3552 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
3553 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
3554 break;
3555 case 12:
3556 if (!Subtarget->hasLDSLoadB96_B128())
3557 return false;
3558
3559 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
3560 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
3561 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
3562 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
3563 break;
3564 case 16:
3565 if (!Subtarget->hasLDSLoadB96_B128())
3566 return false;
3567
3568 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
3569 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
3570 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
3571 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
3572 break;
3573 }
3574
3575 MachineBasicBlock *MBB = MI.getParent();
3576 const DebugLoc &DL = MI.getDebugLoc();
3577 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3578 .add(MO: MI.getOperand(i: 2));
3579
3580 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc));
3581
3582 if (HasVIndex && HasVOffset) {
3583 Register IdxReg = MRI->createVirtualRegister(RegClass: TRI.getVGPR64Class());
3584 BuildMI(BB&: *MBB, I: &*MIB, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: IdxReg)
3585 .addReg(RegNo: VIndex)
3586 .addImm(Val: AMDGPU::sub0)
3587 .addReg(RegNo: VOffset)
3588 .addImm(Val: AMDGPU::sub1);
3589
3590 MIB.addReg(RegNo: IdxReg);
3591 } else if (HasVIndex) {
3592 MIB.addReg(RegNo: VIndex);
3593 } else if (HasVOffset) {
3594 MIB.addReg(RegNo: VOffset);
3595 }
3596
3597 MIB.add(MO: MI.getOperand(i: 1)); // rsrc
3598 MIB.add(MO: MI.getOperand(i: 5 + OpOffset)); // soffset
3599 MIB.add(MO: MI.getOperand(i: 6 + OpOffset)); // imm offset
3600 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI);
3601 unsigned Aux = MI.getOperand(i: 7 + OpOffset).getImm();
3602 MIB.addImm(Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL
3603 : AMDGPU::CPol::ALL_pregfx12)); // cpol
3604 MIB.addImm(
3605 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
3606 ? 1
3607 : 0); // swz
3608 MIB.addImm(Val: isAsyncLDSDMA(Intr: IntrinsicID));
3609
3610 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3611 // Don't set the offset value here because the pointer points to the base of
3612 // the buffer.
3613 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3614
3615 MachinePointerInfo StorePtrI = LoadPtrI;
3616 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3617 AddressSpace: AMDGPUAS::BUFFER_RESOURCE));
3618 LoadPtrI.AddrSpace = AMDGPUAS::BUFFER_RESOURCE;
3619 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3620
3621 auto F = LoadMMO->getFlags() &
3622 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3623 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3624 Size, BaseAlignment: LoadMMO->getBaseAlign());
3625
3626 MachineMemOperand *StoreMMO =
3627 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3628 Size: sizeof(int32_t), BaseAlignment: LoadMMO->getBaseAlign());
3629
3630 MIB.setMemRefs({LoadMMO, StoreMMO});
3631
3632 MI.eraseFromParent();
3633 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3634 return true;
3635}
3636
3637/// Match a zero extend from a 32-bit value to 64-bits.
3638Register AMDGPUInstructionSelector::matchZeroExtendFromS32(Register Reg) const {
3639 Register ZExtSrc;
3640 if (mi_match(R: Reg, MRI: *MRI, P: m_GZExt(Src: m_Reg(R&: ZExtSrc))))
3641 return MRI->getType(Reg: ZExtSrc) == LLT::scalar(SizeInBits: 32) ? ZExtSrc : Register();
3642
3643 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 0)
3644 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3645 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3646 return Register();
3647
3648 assert(Def->getNumOperands() == 3 &&
3649 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3650 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_ZeroInt())) {
3651 return Def->getOperand(i: 1).getReg();
3652 }
3653
3654 return Register();
3655}
3656
3657/// Match a sign extend from a 32-bit value to 64-bits.
3658Register AMDGPUInstructionSelector::matchSignExtendFromS32(Register Reg) const {
3659 Register SExtSrc;
3660 if (mi_match(R: Reg, MRI: *MRI, P: m_GSExt(Src: m_Reg(R&: SExtSrc))))
3661 return MRI->getType(Reg: SExtSrc) == LLT::scalar(SizeInBits: 32) ? SExtSrc : Register();
3662
3663 // Match legalized form %sext = G_MERGE_VALUES (s32 %x), G_ASHR((S32 %x, 31))
3664 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3665 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3666 return Register();
3667
3668 assert(Def->getNumOperands() == 3 &&
3669 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3670 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI,
3671 P: m_GAShr(L: m_SpecificReg(RequestedReg: Def->getOperand(i: 1).getReg()),
3672 R: m_SpecificICst(RequestedValue: 31))))
3673 return Def->getOperand(i: 1).getReg();
3674
3675 if (VT->signBitIsZero(Op: Reg))
3676 return matchZeroExtendFromS32(Reg);
3677
3678 return Register();
3679}
3680
3681/// Match a zero extend from a 32-bit value to 64-bits, or \p Reg itself if it
3682/// is 32-bit.
3683Register
3684AMDGPUInstructionSelector::matchZeroExtendFromS32OrS32(Register Reg) const {
3685 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3686 : matchZeroExtendFromS32(Reg);
3687}
3688
3689/// Match a sign extend from a 32-bit value to 64-bits, or \p Reg itself if it
3690/// is 32-bit.
3691Register
3692AMDGPUInstructionSelector::matchSignExtendFromS32OrS32(Register Reg) const {
3693 return MRI->getType(Reg) == LLT::scalar(SizeInBits: 32) ? Reg
3694 : matchSignExtendFromS32(Reg);
3695}
3696
3697Register
3698AMDGPUInstructionSelector::matchExtendFromS32OrS32(Register Reg,
3699 bool IsSigned) const {
3700 if (IsSigned)
3701 return matchSignExtendFromS32OrS32(Reg);
3702
3703 return matchZeroExtendFromS32OrS32(Reg);
3704}
3705
3706Register AMDGPUInstructionSelector::matchAnyExtendFromS32(Register Reg) const {
3707 Register AnyExtSrc;
3708 if (mi_match(R: Reg, MRI: *MRI, P: m_GAnyExt(Src: m_Reg(R&: AnyExtSrc))))
3709 return MRI->getType(Reg: AnyExtSrc) == LLT::scalar(SizeInBits: 32) ? AnyExtSrc : Register();
3710
3711 // Match legalized form %zext = G_MERGE_VALUES (s32 %x), (s32 G_IMPLICIT_DEF)
3712 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
3713 if (Def->getOpcode() != AMDGPU::G_MERGE_VALUES)
3714 return Register();
3715
3716 assert(Def->getNumOperands() == 3 &&
3717 MRI->getType(Def->getOperand(0).getReg()) == LLT::scalar(64));
3718
3719 if (mi_match(R: Def->getOperand(i: 2).getReg(), MRI: *MRI, P: m_GImplicitDef()))
3720 return Def->getOperand(i: 1).getReg();
3721
3722 return Register();
3723}
3724
3725bool AMDGPUInstructionSelector::selectGlobalLoadLds(MachineInstr &MI) const{
3726 if (!Subtarget->hasVMemToLDSLoad())
3727 return false;
3728
3729 unsigned Opc;
3730 unsigned Size = MI.getOperand(i: 3).getImm();
3731 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
3732
3733 switch (Size) {
3734 default:
3735 return false;
3736 case 1:
3737 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
3738 break;
3739 case 2:
3740 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
3741 break;
3742 case 4:
3743 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
3744 break;
3745 case 12:
3746 if (!Subtarget->hasLDSLoadB96_B128())
3747 return false;
3748 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
3749 break;
3750 case 16:
3751 if (!Subtarget->hasLDSLoadB96_B128())
3752 return false;
3753 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
3754 break;
3755 }
3756
3757 MachineBasicBlock *MBB = MI.getParent();
3758 const DebugLoc &DL = MI.getDebugLoc();
3759 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
3760 .add(MO: MI.getOperand(i: 2));
3761
3762 Register Addr = MI.getOperand(i: 1).getReg();
3763 Register VOffset;
3764 // Try to split SAddr and VOffset. Global and LDS pointers share the same
3765 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
3766 if (!isSGPR(Reg: Addr)) {
3767 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
3768 if (isSGPR(Reg: AddrDef->Reg)) {
3769 Addr = AddrDef->Reg;
3770 } else if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
3771 Register SAddr =
3772 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
3773 if (isSGPR(Reg: SAddr)) {
3774 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
3775 if (Register Off = matchZeroExtendFromS32(Reg: PtrBaseOffset)) {
3776 Addr = SAddr;
3777 VOffset = Off;
3778 }
3779 }
3780 }
3781 }
3782
3783 if (isSGPR(Reg: Addr)) {
3784 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
3785 if (!VOffset) {
3786 VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
3787 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
3788 .addImm(Val: 0);
3789 }
3790 }
3791
3792 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: Opc))
3793 .addReg(RegNo: Addr);
3794
3795 if (isSGPR(Reg: Addr))
3796 MIB.addReg(RegNo: VOffset);
3797
3798 MIB.add(MO: MI.getOperand(i: 4)); // offset
3799
3800 unsigned Aux = MI.getOperand(i: 5).getImm();
3801 MIB.addImm(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS); // cpol
3802 MIB.addImm(Val: isAsyncLDSDMA(Intr: IntrinsicID));
3803
3804 MachineMemOperand *LoadMMO = *MI.memoperands_begin();
3805 MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo();
3806 LoadPtrI.Offset = MI.getOperand(i: 4).getImm();
3807 MachinePointerInfo StorePtrI = LoadPtrI;
3808 LoadPtrI.V = PoisonValue::get(T: PointerType::get(C&: MF->getFunction().getContext(),
3809 AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
3810 LoadPtrI.AddrSpace = AMDGPUAS::GLOBAL_ADDRESS;
3811 StorePtrI.AddrSpace = AMDGPUAS::LOCAL_ADDRESS;
3812 auto F = LoadMMO->getFlags() &
3813 ~(MachineMemOperand::MOStore | MachineMemOperand::MOLoad);
3814 LoadMMO = MF->getMachineMemOperand(PtrInfo: LoadPtrI, F: F | MachineMemOperand::MOLoad,
3815 Size, BaseAlignment: LoadMMO->getBaseAlign());
3816 MachineMemOperand *StoreMMO =
3817 MF->getMachineMemOperand(PtrInfo: StorePtrI, F: F | MachineMemOperand::MOStore,
3818 Size: sizeof(int32_t), BaseAlignment: Align(4));
3819
3820 MIB.setMemRefs({LoadMMO, StoreMMO});
3821
3822 MI.eraseFromParent();
3823 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
3824 return true;
3825}
3826
3827bool AMDGPUInstructionSelector::selectTensorLoadStore(MachineInstr &MI,
3828 Intrinsic::ID IID) const {
3829 bool IsLoad = IID == Intrinsic::amdgcn_tensor_load_to_lds;
3830 unsigned Opc =
3831 IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d4 : AMDGPU::TENSOR_STORE_FROM_LDS_d4;
3832 int NumGroups = 4;
3833
3834 // A lamda function to check whether an operand is a vector of all 0s.
3835 const auto isAllZeros = [&](MachineOperand &Opnd) {
3836 const MachineInstr *DefMI = MRI->getVRegDef(Reg: Opnd.getReg());
3837 if (!DefMI)
3838 return false;
3839 return llvm::isBuildVectorAllZeros(MI: *DefMI, MRI: *MRI, AllowUndef: true);
3840 };
3841
3842 // Use _D2 version if both group 2 and 3 are zero-initialized.
3843 if (isAllZeros(MI.getOperand(i: 3)) && isAllZeros(MI.getOperand(i: 4))) {
3844 NumGroups = 2;
3845 Opc = IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d2
3846 : AMDGPU::TENSOR_STORE_FROM_LDS_d2;
3847 }
3848
3849 // TODO: Handle the fifth group: MI.getOpetand(5), which is silently ignored
3850 // for now because all existing targets only support up to 4 groups.
3851 MachineBasicBlock *MBB = MI.getParent();
3852 auto MIB = BuildMI(BB&: *MBB, I: &MI, MIMD: MI.getDebugLoc(), MCID: TII.get(Opcode: Opc))
3853 .add(MO: MI.getOperand(i: 1)) // D# group 0
3854 .add(MO: MI.getOperand(i: 2)); // D# group 1
3855
3856 if (NumGroups >= 4) { // Has at least 4 groups
3857 MIB.add(MO: MI.getOperand(i: 3)) // D# group 2
3858 .add(MO: MI.getOperand(i: 4)); // D# group 3
3859 }
3860
3861 MIB.addImm(Val: 0) // r128
3862 .add(MO: MI.getOperand(i: 6)); // cpol
3863
3864 MI.eraseFromParent();
3865 return true;
3866}
3867
3868bool AMDGPUInstructionSelector::selectBVHIntersectRayIntrinsic(
3869 MachineInstr &MI) const {
3870 unsigned OpcodeOpIdx =
3871 MI.getOpcode() == AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY ? 1 : 3;
3872 MI.setDesc(TII.get(Opcode: MI.getOperand(i: OpcodeOpIdx).getImm()));
3873 MI.removeOperand(OpNo: OpcodeOpIdx);
3874 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
3875 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
3876 return true;
3877}
3878
3879// FIXME: This should be removed and let the patterns select. We just need the
3880// AGPR/VGPR combination versions.
3881bool AMDGPUInstructionSelector::selectSMFMACIntrin(MachineInstr &MI) const {
3882 unsigned Opc;
3883 switch (cast<GIntrinsic>(Val&: MI).getIntrinsicID()) {
3884 case Intrinsic::amdgcn_smfmac_f32_16x16x32_f16:
3885 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_F16_e64;
3886 break;
3887 case Intrinsic::amdgcn_smfmac_f32_32x32x16_f16:
3888 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_F16_e64;
3889 break;
3890 case Intrinsic::amdgcn_smfmac_f32_16x16x32_bf16:
3891 Opc = AMDGPU::V_SMFMAC_F32_16X16X32_BF16_e64;
3892 break;
3893 case Intrinsic::amdgcn_smfmac_f32_32x32x16_bf16:
3894 Opc = AMDGPU::V_SMFMAC_F32_32X32X16_BF16_e64;
3895 break;
3896 case Intrinsic::amdgcn_smfmac_i32_16x16x64_i8:
3897 Opc = AMDGPU::V_SMFMAC_I32_16X16X64_I8_e64;
3898 break;
3899 case Intrinsic::amdgcn_smfmac_i32_32x32x32_i8:
3900 Opc = AMDGPU::V_SMFMAC_I32_32X32X32_I8_e64;
3901 break;
3902 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_bf8:
3903 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_BF8_e64;
3904 break;
3905 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf8_fp8:
3906 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF8_FP8_e64;
3907 break;
3908 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_bf8:
3909 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_BF8_e64;
3910 break;
3911 case Intrinsic::amdgcn_smfmac_f32_16x16x64_fp8_fp8:
3912 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_FP8_FP8_e64;
3913 break;
3914 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_bf8:
3915 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_BF8_e64;
3916 break;
3917 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf8_fp8:
3918 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF8_FP8_e64;
3919 break;
3920 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_bf8:
3921 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_BF8_e64;
3922 break;
3923 case Intrinsic::amdgcn_smfmac_f32_32x32x32_fp8_fp8:
3924 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_FP8_FP8_e64;
3925 break;
3926 case Intrinsic::amdgcn_smfmac_f32_16x16x64_f16:
3927 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_F16_e64;
3928 break;
3929 case Intrinsic::amdgcn_smfmac_f32_32x32x32_f16:
3930 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_F16_e64;
3931 break;
3932 case Intrinsic::amdgcn_smfmac_f32_16x16x64_bf16:
3933 Opc = AMDGPU::V_SMFMAC_F32_16X16X64_BF16_e64;
3934 break;
3935 case Intrinsic::amdgcn_smfmac_f32_32x32x32_bf16:
3936 Opc = AMDGPU::V_SMFMAC_F32_32X32X32_BF16_e64;
3937 break;
3938 case Intrinsic::amdgcn_smfmac_i32_16x16x128_i8:
3939 Opc = AMDGPU::V_SMFMAC_I32_16X16X128_I8_e64;
3940 break;
3941 case Intrinsic::amdgcn_smfmac_i32_32x32x64_i8:
3942 Opc = AMDGPU::V_SMFMAC_I32_32X32X64_I8_e64;
3943 break;
3944 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_bf8:
3945 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_BF8_e64;
3946 break;
3947 case Intrinsic::amdgcn_smfmac_f32_16x16x128_bf8_fp8:
3948 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_BF8_FP8_e64;
3949 break;
3950 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_bf8:
3951 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_BF8_e64;
3952 break;
3953 case Intrinsic::amdgcn_smfmac_f32_16x16x128_fp8_fp8:
3954 Opc = AMDGPU::V_SMFMAC_F32_16X16X128_FP8_FP8_e64;
3955 break;
3956 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_bf8:
3957 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_BF8_e64;
3958 break;
3959 case Intrinsic::amdgcn_smfmac_f32_32x32x64_bf8_fp8:
3960 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_BF8_FP8_e64;
3961 break;
3962 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_bf8:
3963 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_BF8_e64;
3964 break;
3965 case Intrinsic::amdgcn_smfmac_f32_32x32x64_fp8_fp8:
3966 Opc = AMDGPU::V_SMFMAC_F32_32X32X64_FP8_FP8_e64;
3967 break;
3968 default:
3969 llvm_unreachable("unhandled smfmac intrinsic");
3970 }
3971
3972 auto VDst_In = MI.getOperand(i: 4);
3973
3974 MI.setDesc(TII.get(Opcode: Opc));
3975 MI.removeOperand(OpNo: 4); // VDst_In
3976 MI.removeOperand(OpNo: 1); // Intrinsic ID
3977 MI.addOperand(Op: VDst_In); // Readd VDst_In to the end
3978 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
3979 const MCInstrDesc &MCID = MI.getDesc();
3980 if (MCID.getOperandConstraint(OpNum: 0, Constraint: MCOI::EARLY_CLOBBER) != -1) {
3981 MI.getOperand(i: 0).setIsEarlyClobber(true);
3982 }
3983 return true;
3984}
3985
3986bool AMDGPUInstructionSelector::selectPermlaneSwapIntrin(
3987 MachineInstr &MI, Intrinsic::ID IntrID) const {
3988 if (IntrID == Intrinsic::amdgcn_permlane16_swap &&
3989 !Subtarget->hasPermlane16Swap())
3990 return false;
3991 if (IntrID == Intrinsic::amdgcn_permlane32_swap &&
3992 !Subtarget->hasPermlane32Swap())
3993 return false;
3994
3995 unsigned Opcode = IntrID == Intrinsic::amdgcn_permlane16_swap
3996 ? AMDGPU::V_PERMLANE16_SWAP_B32_e64
3997 : AMDGPU::V_PERMLANE32_SWAP_B32_e64;
3998
3999 MI.removeOperand(OpNo: 2);
4000 MI.setDesc(TII.get(Opcode));
4001 MI.addOperand(MF&: *MF, Op: MachineOperand::CreateReg(Reg: AMDGPU::EXEC, isDef: false, isImp: true));
4002
4003 MachineOperand &FI = MI.getOperand(i: 4);
4004 FI.setImm(FI.getImm() ? AMDGPU::DPP::DPP_FI_1 : AMDGPU::DPP::DPP_FI_0);
4005
4006 constrainSelectedInstRegOperands(I&: MI, TII, TRI, RBI);
4007 return true;
4008}
4009
4010bool AMDGPUInstructionSelector::selectWaveAddress(MachineInstr &MI) const {
4011 Register DstReg = MI.getOperand(i: 0).getReg();
4012 Register SrcReg = MI.getOperand(i: 1).getReg();
4013 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4014 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4015 MachineBasicBlock *MBB = MI.getParent();
4016 const DebugLoc &DL = MI.getDebugLoc();
4017
4018 if (IsVALU) {
4019 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHRREV_B32_e64), DestReg: DstReg)
4020 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4021 .addReg(RegNo: SrcReg);
4022 } else {
4023 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: DstReg)
4024 .addReg(RegNo: SrcReg)
4025 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4026 .setOperandDead(3); // Dead scc
4027 }
4028
4029 const TargetRegisterClass &RC =
4030 IsVALU ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
4031 if (!RBI.constrainGenericRegister(Reg: DstReg, RC, MRI&: *MRI))
4032 return false;
4033
4034 MI.eraseFromParent();
4035 return true;
4036}
4037
4038bool AMDGPUInstructionSelector::selectWaveShuffleIntrin(
4039 MachineInstr &MI) const {
4040 assert(MI.getNumOperands() == 4);
4041 MachineBasicBlock *MBB = MI.getParent();
4042 const DebugLoc &DL = MI.getDebugLoc();
4043
4044 Register DstReg = MI.getOperand(i: 0).getReg();
4045 Register ValReg = MI.getOperand(i: 2).getReg();
4046 Register IdxReg = MI.getOperand(i: 3).getReg();
4047
4048 const LLT DstTy = MRI->getType(Reg: DstReg);
4049 unsigned DstSize = DstTy.getSizeInBits();
4050 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4051 const TargetRegisterClass *DstRC =
4052 TRI.getRegClassForSizeOnBank(Size: DstSize, Bank: *DstRB);
4053
4054 if (DstTy != LLT::scalar(SizeInBits: 32))
4055 return false;
4056
4057 if (!Subtarget->supportsBPermute())
4058 return false;
4059
4060 // If we can bpermute across the whole wave, then just do that
4061 if (Subtarget->supportsWaveWideBPermute()) {
4062 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4063 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4064 .addImm(Val: 2)
4065 .addReg(RegNo: IdxReg);
4066
4067 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: DstReg)
4068 .addReg(RegNo: ShiftIdxReg)
4069 .addReg(RegNo: ValReg)
4070 .addImm(Val: 0);
4071 } else {
4072 // Otherwise, we need to make use of whole wave mode
4073 assert(Subtarget->isWave64());
4074
4075 // Set inactive lanes to poison
4076 Register UndefValReg =
4077 MRI->createVirtualRegister(RegClass: TRI.getRegClass(i: AMDGPU::SReg_32RegClassID));
4078 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefValReg);
4079
4080 Register UndefExecReg = MRI->createVirtualRegister(
4081 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4082 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExecReg);
4083
4084 Register PoisonValReg = MRI->createVirtualRegister(RegClass: DstRC);
4085 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonValReg)
4086 .addImm(Val: 0)
4087 .addReg(RegNo: ValReg)
4088 .addImm(Val: 0)
4089 .addReg(RegNo: UndefValReg)
4090 .addReg(RegNo: UndefExecReg);
4091
4092 // ds_bpermute requires index to be multiplied by 4
4093 Register ShiftIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4094 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_LSHLREV_B32_e64), DestReg: ShiftIdxReg)
4095 .addImm(Val: 2)
4096 .addReg(RegNo: IdxReg);
4097
4098 Register PoisonIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4099 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32), DestReg: PoisonIdxReg)
4100 .addImm(Val: 0)
4101 .addReg(RegNo: ShiftIdxReg)
4102 .addImm(Val: 0)
4103 .addReg(RegNo: UndefValReg)
4104 .addReg(RegNo: UndefExecReg);
4105
4106 Register PoisonUnshiftedIdxReg = MRI->createVirtualRegister(RegClass: DstRC);
4107 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_SET_INACTIVE_B32),
4108 DestReg: PoisonUnshiftedIdxReg)
4109 .addImm(Val: 0)
4110 .addReg(RegNo: IdxReg)
4111 .addImm(Val: 0)
4112 .addReg(RegNo: UndefValReg)
4113 .addReg(RegNo: UndefExecReg);
4114
4115 // Get permutation of each half, then we'll select which one to use
4116 Register SameSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4117 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: SameSidePermReg)
4118 .addReg(RegNo: PoisonIdxReg)
4119 .addReg(RegNo: PoisonValReg)
4120 .addImm(Val: 0);
4121
4122 Register SwappedValReg = MRI->createVirtualRegister(RegClass: DstRC);
4123 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_PERMLANE64_B32), DestReg: SwappedValReg)
4124 .addReg(RegNo: PoisonValReg);
4125
4126 Register OppSidePermReg = MRI->createVirtualRegister(RegClass: DstRC);
4127 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::DS_BPERMUTE_B32), DestReg: OppSidePermReg)
4128 .addReg(RegNo: PoisonIdxReg)
4129 .addReg(RegNo: SwappedValReg)
4130 .addImm(Val: 0);
4131
4132 Register WWMSwapPermReg = MRI->createVirtualRegister(RegClass: DstRC);
4133 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::STRICT_WWM), DestReg: WWMSwapPermReg)
4134 .addReg(RegNo: OppSidePermReg);
4135
4136 // Select which side to take the permute from
4137 // We can get away with only using mbcnt_lo here since we're only
4138 // trying to detect which side of 32 each lane is on, and mbcnt_lo
4139 // returns 32 for lanes 32-63.
4140 Register ThreadIDReg = MRI->createVirtualRegister(RegClass: DstRC);
4141 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64), DestReg: ThreadIDReg)
4142 .addImm(Val: -1)
4143 .addImm(Val: 0);
4144
4145 Register XORReg = MRI->createVirtualRegister(RegClass: DstRC);
4146 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_XOR_B32_e64), DestReg: XORReg)
4147 .addReg(RegNo: ThreadIDReg)
4148 .addReg(RegNo: PoisonUnshiftedIdxReg);
4149
4150 Register ANDReg = MRI->createVirtualRegister(RegClass: DstRC);
4151 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_AND_B32_e64), DestReg: ANDReg)
4152 .addReg(RegNo: XORReg)
4153 .addImm(Val: 32);
4154
4155 Register CompareReg = MRI->createVirtualRegister(
4156 RegClass: TRI.getRegClass(i: AMDGPU::SReg_64_XEXECRegClassID));
4157 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CompareReg)
4158 .addReg(RegNo: ANDReg)
4159 .addImm(Val: 0);
4160
4161 // Finally do the selection
4162 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstReg)
4163 .addImm(Val: 0)
4164 .addReg(RegNo: WWMSwapPermReg)
4165 .addImm(Val: 0)
4166 .addReg(RegNo: SameSidePermReg)
4167 .addReg(RegNo: CompareReg);
4168 }
4169
4170 MI.eraseFromParent();
4171 return true;
4172}
4173
4174// Match BITOP3 operation and return a number of matched instructions plus
4175// truth table.
4176static std::pair<unsigned, uint8_t> BitOp3_Op(Register R,
4177 SmallVectorImpl<Register> &Src,
4178 const MachineRegisterInfo &MRI) {
4179 unsigned NumOpcodes = 0;
4180 uint8_t LHSBits, RHSBits;
4181
4182 auto getOperandBits = [&Src, R, &MRI](Register Op, uint8_t &Bits) -> bool {
4183 // Define truth table given Src0, Src1, Src2 bits permutations:
4184 // 0 0 0
4185 // 0 0 1
4186 // 0 1 0
4187 // 0 1 1
4188 // 1 0 0
4189 // 1 0 1
4190 // 1 1 0
4191 // 1 1 1
4192 const uint8_t SrcBits[3] = { 0xf0, 0xcc, 0xaa };
4193
4194 if (mi_match(R: Op, MRI, P: m_AllOnesInt())) {
4195 Bits = 0xff;
4196 return true;
4197 }
4198 if (mi_match(R: Op, MRI, P: m_ZeroInt())) {
4199 Bits = 0;
4200 return true;
4201 }
4202
4203 for (unsigned I = 0; I < Src.size(); ++I) {
4204 // Try to find existing reused operand
4205 if (Src[I] == Op) {
4206 Bits = SrcBits[I];
4207 return true;
4208 }
4209 // Try to replace parent operator
4210 if (Src[I] == R) {
4211 Bits = SrcBits[I];
4212 Src[I] = Op;
4213 return true;
4214 }
4215 }
4216
4217 if (Src.size() == 3) {
4218 // No room left for operands. Try one last time, there can be a 'not' of
4219 // one of our source operands. In this case we can compute the bits
4220 // without growing Src vector.
4221 Register LHS;
4222 if (mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: LHS)))) {
4223 LHS = getSrcRegIgnoringCopies(Reg: LHS, MRI);
4224 for (unsigned I = 0; I < Src.size(); ++I) {
4225 if (Src[I] == LHS) {
4226 Bits = ~SrcBits[I];
4227 return true;
4228 }
4229 }
4230 }
4231
4232 return false;
4233 }
4234
4235 Bits = SrcBits[Src.size()];
4236 Src.push_back(Elt: Op);
4237 return true;
4238 };
4239
4240 MachineInstr *MI = MRI.getVRegDef(Reg: R);
4241 switch (MI->getOpcode()) {
4242 case TargetOpcode::G_AND:
4243 case TargetOpcode::G_OR:
4244 case TargetOpcode::G_XOR: {
4245 Register LHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 1).getReg(), MRI);
4246 Register RHS = getSrcRegIgnoringCopies(Reg: MI->getOperand(i: 2).getReg(), MRI);
4247
4248 SmallVector<Register, 3> Backup(Src.begin(), Src.end());
4249 if (!getOperandBits(LHS, LHSBits) ||
4250 !getOperandBits(RHS, RHSBits)) {
4251 Src = std::move(Backup);
4252 return std::make_pair(x: 0, y: 0);
4253 }
4254
4255 // Recursion is naturally limited by the size of the operand vector.
4256 //
4257 // When LHS and RHS share a common sub-expression, one side's recursion
4258 // may decompose that sub-expression and replace the Src slot the other
4259 // side occupies with sub-operands via the "replace parent" path in
4260 // getOperandBits. The other side's cached bit-pattern then refers to a
4261 // slot whose contents changed, producing a wrong truth table.
4262 //
4263 // We detect this in three ways:
4264 // (A) If LHS recursed, its truth table is valid against the Src state
4265 // when LHS recursion completed (SrcAfterLHS). If RHS recursion
4266 // then mutates a Src slot that LHSBits depends on, LHSBits is
4267 // stale.
4268 // (B) If RHS did not recurse, RHSBits came from getOperandBits and
4269 // refers to a specific Src slot. If that slot's contents changed
4270 // (by either recursion), RHSBits is stale.
4271 // (C) Symmetrically for LHS if it did not recurse.
4272 SmallVector<Register, 3> SrcBeforeRecurse(Src.begin(), Src.end());
4273 uint8_t LHSBitsOrig = LHSBits;
4274 uint8_t RHSBitsOrig = RHSBits;
4275
4276 auto LHSOp = BitOp3_Op(R: LHS, Src, MRI);
4277 if (LHSOp.first) {
4278 NumOpcodes += LHSOp.first;
4279 LHSBits = LHSOp.second;
4280 }
4281
4282 SmallVector<Register, 3> SrcAfterLHS(Src.begin(), Src.end());
4283
4284 auto RHSOp = BitOp3_Op(R: RHS, Src, MRI);
4285 if (RHSOp.first) {
4286 NumOpcodes += RHSOp.first;
4287 RHSBits = RHSOp.second;
4288 }
4289
4290 // dependsOnSlot: true iff the truth table TT varies with slot Slot.
4291 auto dependsOnSlot = [](uint8_t TT, int Slot) -> bool {
4292 if (Slot < 0 || Slot > 2)
4293 return false;
4294 const uint8_t Masks[3] = {0x0f, 0x33, 0x55};
4295 const int Shifts[3] = {4, 2, 1};
4296 return ((TT ^ (TT >> Shifts[Slot])) & Masks[Slot]) != 0;
4297 };
4298
4299 // findSlot: locate the Src slot a getOperandBits result depends on,
4300 // including negated (NOT) patterns that getOperandBits resolves via
4301 // the ~SrcBits[I] shortcut.
4302 const uint8_t SrcBitsConst[3] = {0xf0, 0xcc, 0xaa};
4303 auto findSlot = [&](uint8_t Bits, Register Op,
4304 const SmallVectorImpl<Register> &S) -> int {
4305 Register NegatedInner;
4306 bool IsNegationOp = mi_match(R: Op, MRI, P: m_Not(Src: m_Reg(R&: NegatedInner)));
4307 if (IsNegationOp)
4308 NegatedInner = getSrcRegIgnoringCopies(Reg: NegatedInner, MRI);
4309 for (int I = 0; I < (int)S.size(); I++) {
4310 if (Bits == SrcBitsConst[I] && S[I] == Op)
4311 return I;
4312 if (IsNegationOp && Bits == (uint8_t)~SrcBitsConst[I] &&
4313 S[I] == NegatedInner)
4314 return I;
4315 }
4316 return -1;
4317 };
4318
4319 bool Stale = false;
4320
4321 // (A) LHS recursed: its truth table is against SrcAfterLHS.
4322 // Check if RHS recursion mutated a slot that LHSBits uses.
4323 if (LHSOp.first) {
4324 for (int I = 0; I < (int)SrcAfterLHS.size() && I < 3; I++) {
4325 if (I < (int)Src.size() && Src[I] != SrcAfterLHS[I] &&
4326 dependsOnSlot(LHSBits, I)) {
4327 Stale = true;
4328 break;
4329 }
4330 }
4331 }
4332
4333 // (B) RHS did not recurse: RHSBits from getOperandBits is against
4334 // SrcBeforeRecurse. Check if that slot was mutated since then.
4335 if (!Stale && !RHSOp.first) {
4336 int Slot = findSlot(RHSBitsOrig, RHS, SrcBeforeRecurse);
4337 if (Slot >= 0 &&
4338 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4339 Stale = true;
4340 }
4341
4342 // (C) LHS did not recurse: LHSBits from getOperandBits is against
4343 // SrcBeforeRecurse. Check if that slot was mutated since then.
4344 if (!Stale && !LHSOp.first) {
4345 int Slot = findSlot(LHSBitsOrig, LHS, SrcBeforeRecurse);
4346 if (Slot >= 0 &&
4347 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4348 Stale = true;
4349 }
4350
4351 if (Stale) {
4352 Src = std::move(SrcBeforeRecurse);
4353 LHSBits = LHSBitsOrig;
4354 RHSBits = RHSBitsOrig;
4355 NumOpcodes = 0;
4356 }
4357 break;
4358 }
4359 default:
4360 return std::make_pair(x: 0, y: 0);
4361 }
4362
4363 uint8_t TTbl;
4364 switch (MI->getOpcode()) {
4365 case TargetOpcode::G_AND:
4366 TTbl = LHSBits & RHSBits;
4367 break;
4368 case TargetOpcode::G_OR:
4369 TTbl = LHSBits | RHSBits;
4370 break;
4371 case TargetOpcode::G_XOR:
4372 TTbl = LHSBits ^ RHSBits;
4373 break;
4374 default:
4375 break;
4376 }
4377
4378 return std::make_pair(x: NumOpcodes + 1, y&: TTbl);
4379}
4380
4381bool AMDGPUInstructionSelector::selectBITOP3(MachineInstr &MI) const {
4382 if (!Subtarget->hasBitOp3Insts())
4383 return false;
4384
4385 Register DstReg = MI.getOperand(i: 0).getReg();
4386 const RegisterBank *DstRB = RBI.getRegBank(Reg: DstReg, MRI: *MRI, TRI);
4387 const bool IsVALU = DstRB->getID() == AMDGPU::VGPRRegBankID;
4388 if (!IsVALU)
4389 return false;
4390
4391 SmallVector<Register, 3> Src;
4392 uint8_t TTbl;
4393 unsigned NumOpcodes;
4394
4395 std::tie(args&: NumOpcodes, args&: TTbl) = BitOp3_Op(R: DstReg, Src, MRI: *MRI);
4396
4397 // Src.empty() case can happen if all operands are all zero or all ones.
4398 // Normally it shall be optimized out before reaching this.
4399 if (NumOpcodes < 2 || Src.empty())
4400 return false;
4401
4402 // RegBankSelect splits wider VALU logic ops and widens 1-bit ones, so only
4403 // 16 and 32 bit types reach here. Note that <2 x i16> is 32 bits wide.
4404 unsigned Size = MRI->getType(Reg: DstReg).getSizeInBits();
4405 assert((Size == 16 || Size == 32) && "unexpected VALU logic op size");
4406 const bool IsB32 = Size == 32;
4407 if (NumOpcodes == 2 && IsB32) {
4408 // Avoid using BITOP3 for OR3, XOR3, AND_OR. This is not faster but makes
4409 // asm more readable. This cannot be modeled with AddedComplexity because
4410 // selector does not know how many operations did we match.
4411 if (mi_match(MI, MRI: *MRI, P: m_GXor(L: m_GXor(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4412 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GOr(L: m_Reg(), R: m_Reg()), R: m_Reg())) ||
4413 mi_match(MI, MRI: *MRI, P: m_GOr(L: m_GAnd(L: m_Reg(), R: m_Reg()), R: m_Reg())))
4414 return false;
4415 } else if (NumOpcodes < 4) {
4416 // For a uniform case threshold should be higher to account for moves
4417 // between VGPRs and SGPRs. It needs one operand in a VGPR, rest two can be
4418 // in SGPRs and a readtfirstlane after.
4419 return false;
4420 }
4421
4422 unsigned Opc = IsB32 ? AMDGPU::V_BITOP3_B32_e64 : AMDGPU::V_BITOP3_B16_e64;
4423 if (!IsB32 && STI.hasTrue16BitInsts())
4424 Opc = STI.useRealTrue16Insts() ? AMDGPU::V_BITOP3_B16_gfx1250_t16_e64
4425 : AMDGPU::V_BITOP3_B16_gfx1250_fake16_e64;
4426 unsigned CBL = STI.getConstantBusLimit(Opcode: Opc);
4427 MachineBasicBlock *MBB = MI.getParent();
4428 const DebugLoc &DL = MI.getDebugLoc();
4429
4430 for (unsigned I = 0; I < Src.size(); ++I) {
4431 const RegisterBank *RB = RBI.getRegBank(Reg: Src[I], MRI: *MRI, TRI);
4432 if (RB->getID() != AMDGPU::SGPRRegBankID)
4433 continue;
4434 if (CBL > 0) {
4435 --CBL;
4436 continue;
4437 }
4438 Register NewReg = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
4439 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: NewReg)
4440 .addReg(RegNo: Src[I]);
4441 Src[I] = NewReg;
4442 }
4443
4444 // Last operand can be ignored, turning a ternary operation into a binary.
4445 // For example: (~a & b & c) | (~a & b & ~c) -> (~a & b). We can replace
4446 // 'c' with 'a' here without changing the answer. In some pathological
4447 // cases it should be possible to get an operation with a single operand
4448 // too if optimizer would not catch it.
4449 while (Src.size() < 3)
4450 Src.push_back(Elt: Src[0]);
4451
4452 auto MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: Opc), DestReg: DstReg);
4453 if (!IsB32)
4454 MIB.addImm(Val: 0); // src_mod0
4455 MIB.addReg(RegNo: Src[0]);
4456 if (!IsB32)
4457 MIB.addImm(Val: 0); // src_mod1
4458 MIB.addReg(RegNo: Src[1]);
4459 if (!IsB32)
4460 MIB.addImm(Val: 0); // src_mod2
4461 MIB.addReg(RegNo: Src[2])
4462 .addImm(Val: TTbl);
4463 if (!IsB32)
4464 MIB.addImm(Val: 0); // op_sel
4465
4466 constrainSelectedInstRegOperands(I&: *MIB, TII, TRI, RBI);
4467 MI.eraseFromParent();
4468
4469 return true;
4470}
4471
4472bool AMDGPUInstructionSelector::selectStackRestore(MachineInstr &MI) const {
4473 Register SrcReg = MI.getOperand(i: 0).getReg();
4474 if (!RBI.constrainGenericRegister(Reg: SrcReg, RC: AMDGPU::SReg_32RegClass, MRI&: *MRI))
4475 return false;
4476
4477 MachineInstr *DefMI = MRI->getVRegDef(Reg: SrcReg);
4478 Register SP =
4479 Subtarget->getTargetLowering()->getStackPointerRegisterToSaveRestore();
4480 Register WaveAddr = getWaveAddress(Def: DefMI);
4481 MachineBasicBlock *MBB = MI.getParent();
4482 const DebugLoc &DL = MI.getDebugLoc();
4483
4484 if (!WaveAddr) {
4485 WaveAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
4486 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: WaveAddr)
4487 .addReg(RegNo: SrcReg)
4488 .addImm(Val: Subtarget->getWavefrontSizeLog2())
4489 .setOperandDead(3); // Dead scc
4490 }
4491
4492 BuildMI(BB&: *MBB, I: &MI, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: SP)
4493 .addReg(RegNo: WaveAddr);
4494
4495 MI.eraseFromParent();
4496 return true;
4497}
4498
4499bool AMDGPUInstructionSelector::select(MachineInstr &I) {
4500
4501 if (!I.isPreISelOpcode()) {
4502 if (I.isCopy())
4503 return selectCOPY(I);
4504 return true;
4505 }
4506
4507 switch (I.getOpcode()) {
4508 case TargetOpcode::G_AND:
4509 case TargetOpcode::G_OR:
4510 case TargetOpcode::G_XOR:
4511 if (selectBITOP3(MI&: I))
4512 return true;
4513 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4514 return true;
4515 return selectG_AND_OR_XOR(I);
4516 case TargetOpcode::G_ADD:
4517 case TargetOpcode::G_SUB:
4518 case TargetOpcode::G_PTR_ADD:
4519 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4520 return true;
4521 return selectG_ADD_SUB(I);
4522 case TargetOpcode::G_UADDO:
4523 case TargetOpcode::G_USUBO:
4524 case TargetOpcode::G_UADDE:
4525 case TargetOpcode::G_USUBE:
4526 return selectG_UADDO_USUBO_UADDE_USUBE(I);
4527 case AMDGPU::G_AMDGPU_MAD_U64_U32:
4528 case AMDGPU::G_AMDGPU_MAD_I64_I32:
4529 return selectG_AMDGPU_MAD_64_32(I);
4530 case TargetOpcode::G_INTTOPTR:
4531 case TargetOpcode::G_BITCAST:
4532 case TargetOpcode::G_PTRTOINT:
4533 case TargetOpcode::G_FREEZE:
4534 return selectCOPY(I);
4535 case TargetOpcode::G_FNEG:
4536 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4537 return true;
4538 return selectG_FNEG(MI&: I);
4539 case TargetOpcode::G_FABS:
4540 if (selectImpl(I, CoverageInfo&: *CoverageInfo))
4541 return true;
4542 return selectG_FABS(MI&: I);
4543 case TargetOpcode::G_EXTRACT:
4544 return selectG_EXTRACT(I);
4545 case TargetOpcode::G_MERGE_VALUES:
4546 case TargetOpcode::G_CONCAT_VECTORS:
4547 return selectG_MERGE_VALUES(MI&: I);
4548 case TargetOpcode::G_UNMERGE_VALUES:
4549 return selectG_UNMERGE_VALUES(MI&: I);
4550 case TargetOpcode::G_BUILD_VECTOR:
4551 case TargetOpcode::G_BUILD_VECTOR_TRUNC:
4552 return selectG_BUILD_VECTOR(MI&: I);
4553 case TargetOpcode::G_IMPLICIT_DEF:
4554 return selectG_IMPLICIT_DEF(I);
4555 case TargetOpcode::G_INSERT:
4556 return selectG_INSERT(I);
4557 case TargetOpcode::G_INTRINSIC:
4558 case TargetOpcode::G_INTRINSIC_CONVERGENT:
4559 return selectG_INTRINSIC(I);
4560 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
4561 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
4562 return selectG_INTRINSIC_W_SIDE_EFFECTS(I);
4563 case TargetOpcode::G_ICMP:
4564 case TargetOpcode::G_FCMP:
4565 if (selectG_ICMP_or_FCMP(I))
4566 return true;
4567 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4568 case TargetOpcode::G_LOAD:
4569 case TargetOpcode::G_ZEXTLOAD:
4570 case TargetOpcode::G_SEXTLOAD:
4571 case TargetOpcode::G_STORE:
4572 case TargetOpcode::G_ATOMIC_CMPXCHG:
4573 case TargetOpcode::G_ATOMICRMW_XCHG:
4574 case TargetOpcode::G_ATOMICRMW_ADD:
4575 case TargetOpcode::G_ATOMICRMW_SUB:
4576 case TargetOpcode::G_ATOMICRMW_AND:
4577 case TargetOpcode::G_ATOMICRMW_OR:
4578 case TargetOpcode::G_ATOMICRMW_XOR:
4579 case TargetOpcode::G_ATOMICRMW_MIN:
4580 case TargetOpcode::G_ATOMICRMW_MAX:
4581 case TargetOpcode::G_ATOMICRMW_UMIN:
4582 case TargetOpcode::G_ATOMICRMW_UMAX:
4583 case TargetOpcode::G_ATOMICRMW_UINC_WRAP:
4584 case TargetOpcode::G_ATOMICRMW_UDEC_WRAP:
4585 case TargetOpcode::G_ATOMICRMW_USUB_COND:
4586 case TargetOpcode::G_ATOMICRMW_USUB_SAT:
4587 case TargetOpcode::G_ATOMICRMW_FADD:
4588 case TargetOpcode::G_ATOMICRMW_FMIN:
4589 case TargetOpcode::G_ATOMICRMW_FMAX:
4590 return selectG_LOAD_STORE_ATOMICRMW(I);
4591 case TargetOpcode::G_SELECT:
4592 return selectG_SELECT(I);
4593 case TargetOpcode::G_TRUNC:
4594 return selectG_TRUNC(I);
4595 case TargetOpcode::G_SEXT:
4596 case TargetOpcode::G_ZEXT:
4597 case TargetOpcode::G_ANYEXT:
4598 case TargetOpcode::G_SEXT_INREG:
4599 // This is a workaround. For extension from type i1, `selectImpl()` uses
4600 // patterns from TD file and generates an illegal VGPR to SGPR COPY as type
4601 // i1 can only be hold in a SGPR class.
4602 if (MRI->getType(Reg: I.getOperand(i: 1).getReg()) != LLT::scalar(SizeInBits: 1) &&
4603 selectImpl(I, CoverageInfo&: *CoverageInfo))
4604 return true;
4605 return selectG_SZA_EXT(I);
4606 case TargetOpcode::G_FPEXT:
4607 if (selectG_FPEXT(I))
4608 return true;
4609 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4610 case TargetOpcode::G_BRCOND:
4611 return selectG_BRCOND(I);
4612 case TargetOpcode::G_GLOBAL_VALUE:
4613 return selectG_GLOBAL_VALUE(I);
4614 case TargetOpcode::G_PTRMASK:
4615 return selectG_PTRMASK(I);
4616 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
4617 return selectG_EXTRACT_VECTOR_ELT(MI&: I);
4618 case TargetOpcode::G_INSERT_VECTOR_ELT:
4619 return selectG_INSERT_VECTOR_ELT(MI&: I);
4620 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD:
4621 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16:
4622 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET:
4623 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE:
4624 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16: {
4625 const AMDGPU::ImageDimIntrinsicInfo *Intr =
4626 AMDGPU::getImageDimIntrinsicInfo(Intr: AMDGPU::getIntrinsicID(I));
4627 assert(Intr && "not an image intrinsic with image pseudo");
4628 return selectImageIntrinsic(MI&: I, Intr);
4629 }
4630 case AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY:
4631 case AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY:
4632 case AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY:
4633 return selectBVHIntersectRayIntrinsic(MI&: I);
4634 case AMDGPU::G_SBFX:
4635 case AMDGPU::G_UBFX:
4636 return selectG_SBFX_UBFX(MI&: I);
4637 case AMDGPU::G_SI_CALL:
4638 I.setDesc(TII.get(Opcode: AMDGPU::SI_CALL));
4639 return true;
4640 case AMDGPU::G_AMDGPU_WAVE_ADDRESS:
4641 return selectWaveAddress(MI&: I);
4642 case AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN: {
4643 I.setDesc(TII.get(Opcode: AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN));
4644 return true;
4645 }
4646 case AMDGPU::G_STACKRESTORE:
4647 return selectStackRestore(MI&: I);
4648 case AMDGPU::G_PHI:
4649 return selectPHI(I);
4650 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
4651 return selectCOPY_SCC_VCC(I);
4652 case AMDGPU::G_AMDGPU_COPY_VCC_SCC:
4653 return selectCOPY_VCC_SCC(I);
4654 case AMDGPU::G_AMDGPU_READANYLANE:
4655 return selectReadAnyLane(I);
4656 case TargetOpcode::G_CONSTANT:
4657 case TargetOpcode::G_FCONSTANT:
4658 default:
4659 return selectImpl(I, CoverageInfo&: *CoverageInfo);
4660 }
4661 return false;
4662}
4663
4664InstructionSelector::ComplexRendererFns
4665AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const {
4666 return {{
4667 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4668 }};
4669
4670}
4671
4672std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3ModsImpl(
4673 Register Src, bool IsCanonicalizing, bool AllowAbs, bool OpSel) const {
4674 unsigned Mods = 0;
4675 MachineInstr *MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4676
4677 if (MI->getOpcode() == AMDGPU::G_FNEG) {
4678 Src = MI->getOperand(i: 1).getReg();
4679 Mods |= SISrcMods::NEG;
4680 MI = getDefIgnoringCopies(Reg: Src, MRI: *MRI);
4681 } else if (MI->getOpcode() == AMDGPU::G_FSUB && IsCanonicalizing) {
4682 // Fold fsub [+-]0 into fneg. This may not have folded depending on the
4683 // denormal mode, but we're implicitly canonicalizing in a source operand.
4684 const ConstantFP *LHS =
4685 getConstantFPVRegVal(VReg: MI->getOperand(i: 1).getReg(), MRI: *MRI);
4686 if (LHS && LHS->isZero()) {
4687 Mods |= SISrcMods::NEG;
4688 Src = MI->getOperand(i: 2).getReg();
4689 }
4690 }
4691
4692 if (AllowAbs && MI->getOpcode() == AMDGPU::G_FABS) {
4693 Src = MI->getOperand(i: 1).getReg();
4694 Mods |= SISrcMods::ABS;
4695 }
4696
4697 if (OpSel)
4698 Mods |= SISrcMods::OP_SEL_0;
4699
4700 return std::pair(Src, Mods);
4701}
4702
4703std::pair<Register, unsigned>
4704AMDGPUInstructionSelector::selectVOP3PModsF32Impl(Register Src) const {
4705 unsigned Mods;
4706 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src);
4707 Mods |= SISrcMods::OP_SEL_1;
4708 return std::pair(Src, Mods);
4709}
4710
4711Register AMDGPUInstructionSelector::copyToVGPRIfSrcFolded(
4712 Register Src, unsigned Mods, MachineOperand Root, MachineInstr *InsertPt,
4713 bool ForceVGPR) const {
4714 if ((Mods != 0 || ForceVGPR) &&
4715 RBI.getRegBank(Reg: Src, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID) {
4716
4717 // If we looked through copies to find source modifiers on an SGPR operand,
4718 // we now have an SGPR register source. To avoid potentially violating the
4719 // constant bus restriction, we need to insert a copy to a VGPR.
4720 Register VGPRSrc = MRI->cloneVirtualRegister(VReg: Root.getReg());
4721 BuildMI(BB&: *InsertPt->getParent(), I: InsertPt, MIMD: InsertPt->getDebugLoc(),
4722 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: VGPRSrc)
4723 .addReg(RegNo: Src);
4724 Src = VGPRSrc;
4725 }
4726
4727 return Src;
4728}
4729
4730///
4731/// This will select either an SGPR or VGPR operand and will save us from
4732/// having to write an extra tablegen pattern.
4733InstructionSelector::ComplexRendererFns
4734AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const {
4735 return {{
4736 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); }
4737 }};
4738}
4739
4740InstructionSelector::ComplexRendererFns
4741AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const {
4742 Register Src;
4743 unsigned Mods;
4744 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4745
4746 return {{
4747 [=](MachineInstrBuilder &MIB) {
4748 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4749 },
4750 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4751 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4752 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4753 }};
4754}
4755
4756InstructionSelector::ComplexRendererFns
4757AMDGPUInstructionSelector::selectVOP3BMods0(MachineOperand &Root) const {
4758 Register Src;
4759 unsigned Mods;
4760 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
4761 /*IsCanonicalizing=*/true,
4762 /*AllowAbs=*/false);
4763
4764 return {{
4765 [=](MachineInstrBuilder &MIB) {
4766 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4767 },
4768 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
4769 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4770 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4771 }};
4772}
4773
4774InstructionSelector::ComplexRendererFns
4775AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const {
4776 return {{
4777 [=](MachineInstrBuilder &MIB) { MIB.add(MO: Root); },
4778 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); }, // clamp
4779 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // omod
4780 }};
4781}
4782
4783InstructionSelector::ComplexRendererFns
4784AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const {
4785 Register Src;
4786 unsigned Mods;
4787 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
4788
4789 return {{
4790 [=](MachineInstrBuilder &MIB) {
4791 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4792 },
4793 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4794 }};
4795}
4796
4797InstructionSelector::ComplexRendererFns
4798AMDGPUInstructionSelector::selectVOP3ModsNonCanonicalizing(
4799 MachineOperand &Root) const {
4800 Register Src;
4801 unsigned Mods;
4802 std::tie(args&: Src, args&: Mods) =
4803 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/false);
4804
4805 return {{
4806 [=](MachineInstrBuilder &MIB) {
4807 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4808 },
4809 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4810 }};
4811}
4812
4813InstructionSelector::ComplexRendererFns
4814AMDGPUInstructionSelector::selectVOP3BMods(MachineOperand &Root) const {
4815 Register Src;
4816 unsigned Mods;
4817 std::tie(args&: Src, args&: Mods) =
4818 selectVOP3ModsImpl(Src: Root.getReg(), /*IsCanonicalizing=*/true,
4819 /*AllowAbs=*/false);
4820
4821 return {{
4822 [=](MachineInstrBuilder &MIB) {
4823 MIB.addReg(RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB));
4824 },
4825 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
4826 }};
4827}
4828
4829InstructionSelector::ComplexRendererFns
4830AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const {
4831 Register Reg = Root.getReg();
4832 const MachineInstr *Def = getDefIgnoringCopies(Reg, MRI: *MRI);
4833 if (Def->getOpcode() == AMDGPU::G_FNEG || Def->getOpcode() == AMDGPU::G_FABS)
4834 return {};
4835 return {{
4836 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
4837 }};
4838}
4839
4840enum class SrcStatus {
4841 IS_SAME,
4842 IS_UPPER_HALF,
4843 IS_LOWER_HALF,
4844 IS_UPPER_HALF_NEG,
4845 // This means current op = [op_upper, op_lower] and src = -op_lower.
4846 IS_LOWER_HALF_NEG,
4847 IS_HI_NEG,
4848 // This means current op = [op_upper, op_lower] and src = [op_upper,
4849 // -op_lower].
4850 IS_LO_NEG,
4851 IS_BOTH_NEG,
4852 INVALID,
4853 NEG_START = IS_UPPER_HALF_NEG,
4854 NEG_END = IS_BOTH_NEG,
4855 HALF_START = IS_UPPER_HALF,
4856 HALF_END = IS_LOWER_HALF_NEG
4857};
4858/// Test if the MI is truncating to half, such as `%reg0:n = G_TRUNC %reg1:2n`
4859static bool isTruncHalf(const MachineInstr *MI,
4860 const MachineRegisterInfo &MRI) {
4861 if (MI->getOpcode() != AMDGPU::G_TRUNC)
4862 return false;
4863
4864 unsigned DstSize = MRI.getType(Reg: MI->getOperand(i: 0).getReg()).getSizeInBits();
4865 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4866 return DstSize * 2 == SrcSize;
4867}
4868
4869/// Test if the MI is logic shift right with half bits,
4870/// such as `%reg0:2n =G_LSHR %reg1:2n, CONST(n)`
4871static bool isLshrHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
4872 if (MI->getOpcode() != AMDGPU::G_LSHR)
4873 return false;
4874
4875 Register ShiftSrc;
4876 std::optional<ValueAndVReg> ShiftAmt;
4877 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
4878 P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
4879 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4880 unsigned Shift = ShiftAmt->Value.getZExtValue();
4881 return Shift * 2 == SrcSize;
4882 }
4883 return false;
4884}
4885
4886/// Test if the MI is shift left with half bits,
4887/// such as `%reg0:2n =G_SHL %reg1:2n, CONST(n)`
4888static bool isShlHalf(const MachineInstr *MI, const MachineRegisterInfo &MRI) {
4889 if (MI->getOpcode() != AMDGPU::G_SHL)
4890 return false;
4891
4892 Register ShiftSrc;
4893 std::optional<ValueAndVReg> ShiftAmt;
4894 if (mi_match(R: MI->getOperand(i: 0).getReg(), MRI,
4895 P: m_GShl(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt)))) {
4896 unsigned SrcSize = MRI.getType(Reg: MI->getOperand(i: 1).getReg()).getSizeInBits();
4897 unsigned Shift = ShiftAmt->Value.getZExtValue();
4898 return Shift * 2 == SrcSize;
4899 }
4900 return false;
4901}
4902
4903/// Test function, if the MI is `%reg0:n, %reg1:n = G_UNMERGE_VALUES %reg2:2n`
4904static bool isUnmergeHalf(const MachineInstr *MI,
4905 const MachineRegisterInfo &MRI) {
4906 if (MI->getOpcode() != AMDGPU::G_UNMERGE_VALUES)
4907 return false;
4908 return MI->getNumOperands() == 3 && MI->getOperand(i: 0).isDef() &&
4909 MI->getOperand(i: 1).isDef() && !MI->getOperand(i: 2).isDef();
4910}
4911
4912enum class TypeClass { VECTOR_OF_TWO, SCALAR, NONE_OF_LISTED };
4913
4914static TypeClass isVectorOfTwoOrScalar(Register Reg,
4915 const MachineRegisterInfo &MRI) {
4916 LLT OpTy = MRI.getType(Reg);
4917 if (OpTy.isScalar())
4918 return TypeClass::SCALAR;
4919 if (OpTy.isVector() && OpTy.getNumElements() == 2)
4920 return TypeClass::VECTOR_OF_TWO;
4921 return TypeClass::NONE_OF_LISTED;
4922}
4923
4924static SrcStatus getNegStatus(Register Reg, SrcStatus S,
4925 const MachineRegisterInfo &MRI) {
4926 TypeClass NegType = isVectorOfTwoOrScalar(Reg, MRI);
4927 if (NegType != TypeClass::VECTOR_OF_TWO && NegType != TypeClass::SCALAR)
4928 return SrcStatus::INVALID;
4929
4930 switch (S) {
4931 case SrcStatus::IS_SAME:
4932 if (NegType == TypeClass::VECTOR_OF_TWO) {
4933 // Vector of 2:
4934 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
4935 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
4936 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
4937 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
4938 return SrcStatus::IS_BOTH_NEG;
4939 }
4940 if (NegType == TypeClass::SCALAR) {
4941 // Scalar:
4942 // [SrcHi, SrcLo] = [CurrHi, CurrLo]
4943 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
4944 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
4945 // [SrcHi, SrcLo] = [-OpHi, OpLo]
4946 return SrcStatus::IS_HI_NEG;
4947 }
4948 break;
4949 case SrcStatus::IS_HI_NEG:
4950 if (NegType == TypeClass::VECTOR_OF_TWO) {
4951 // Vector of 2:
4952 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
4953 // [CurrHi, CurrLo] = neg [OpHi, OpLo](2 x Type)
4954 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
4955 // [SrcHi, SrcLo] = [-(-OpHi), -OpLo] = [OpHi, -OpLo]
4956 return SrcStatus::IS_LO_NEG;
4957 }
4958 if (NegType == TypeClass::SCALAR) {
4959 // Scalar:
4960 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
4961 // [CurrHi, CurrLo] = neg [OpHi, OpLo](Type)
4962 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
4963 // [SrcHi, SrcLo] = [-(-OpHi), OpLo] = [OpHi, OpLo]
4964 return SrcStatus::IS_SAME;
4965 }
4966 break;
4967 case SrcStatus::IS_LO_NEG:
4968 if (NegType == TypeClass::VECTOR_OF_TWO) {
4969 // Vector of 2:
4970 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
4971 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
4972 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
4973 // [SrcHi, SrcLo] = [-OpHi, -(-OpLo)] = [-OpHi, OpLo]
4974 return SrcStatus::IS_HI_NEG;
4975 }
4976 if (NegType == TypeClass::SCALAR) {
4977 // Scalar:
4978 // [SrcHi, SrcLo] = [CurrHi, -CurrLo]
4979 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
4980 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
4981 // [SrcHi, SrcLo] = [-OpHi, -OpLo]
4982 return SrcStatus::IS_BOTH_NEG;
4983 }
4984 break;
4985 case SrcStatus::IS_BOTH_NEG:
4986 if (NegType == TypeClass::VECTOR_OF_TWO) {
4987 // Vector of 2:
4988 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
4989 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](2 x Type)
4990 // [CurrHi, CurrLo] = [-OpHi, -OpLo](2 x Type)
4991 // [SrcHi, SrcLo] = [OpHi, OpLo]
4992 return SrcStatus::IS_SAME;
4993 }
4994 if (NegType == TypeClass::SCALAR) {
4995 // Scalar:
4996 // [SrcHi, SrcLo] = [-CurrHi, -CurrLo]
4997 // [CurrHi, CurrLo] = fneg [OpHi, OpLo](Type)
4998 // [CurrHi, CurrLo] = [-OpHi, OpLo](Type)
4999 // [SrcHi, SrcLo] = [OpHi, -OpLo]
5000 return SrcStatus::IS_LO_NEG;
5001 }
5002 break;
5003 case SrcStatus::IS_UPPER_HALF:
5004 // Vector of 2:
5005 // Src = CurrUpper
5006 // Curr = [CurrUpper, CurrLower]
5007 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5008 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5009 // Src = -OpUpper
5010 //
5011 // Scalar:
5012 // Src = CurrUpper
5013 // Curr = [CurrUpper, CurrLower]
5014 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5015 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5016 // Src = -OpUpper
5017 return SrcStatus::IS_UPPER_HALF_NEG;
5018 case SrcStatus::IS_LOWER_HALF:
5019 if (NegType == TypeClass::VECTOR_OF_TWO) {
5020 // Vector of 2:
5021 // Src = CurrLower
5022 // Curr = [CurrUpper, CurrLower]
5023 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5024 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5025 // Src = -OpLower
5026 return SrcStatus::IS_LOWER_HALF_NEG;
5027 }
5028 if (NegType == TypeClass::SCALAR) {
5029 // Scalar:
5030 // Src = CurrLower
5031 // Curr = [CurrUpper, CurrLower]
5032 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5033 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5034 // Src = OpLower
5035 return SrcStatus::IS_LOWER_HALF;
5036 }
5037 break;
5038 case SrcStatus::IS_UPPER_HALF_NEG:
5039 // Vector of 2:
5040 // Src = -CurrUpper
5041 // Curr = [CurrUpper, CurrLower]
5042 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5043 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5044 // Src = -(-OpUpper) = OpUpper
5045 //
5046 // Scalar:
5047 // Src = -CurrUpper
5048 // Curr = [CurrUpper, CurrLower]
5049 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5050 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5051 // Src = -(-OpUpper) = OpUpper
5052 return SrcStatus::IS_UPPER_HALF;
5053 case SrcStatus::IS_LOWER_HALF_NEG:
5054 if (NegType == TypeClass::VECTOR_OF_TWO) {
5055 // Vector of 2:
5056 // Src = -CurrLower
5057 // Curr = [CurrUpper, CurrLower]
5058 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](2 x Type)
5059 // [CurrUpper, CurrLower] = [-OpUpper, -OpLower](2 x Type)
5060 // Src = -(-OpLower) = OpLower
5061 return SrcStatus::IS_LOWER_HALF;
5062 }
5063 if (NegType == TypeClass::SCALAR) {
5064 // Scalar:
5065 // Src = -CurrLower
5066 // Curr = [CurrUpper, CurrLower]
5067 // [CurrUpper, CurrLower] = fneg [OpUpper, OpLower](Type)
5068 // [CurrUpper, CurrLower] = [-OpUpper, OpLower](Type)
5069 // Src = -OpLower
5070 return SrcStatus::IS_LOWER_HALF_NEG;
5071 }
5072 break;
5073 default:
5074 break;
5075 }
5076 llvm_unreachable("unexpected SrcStatus & NegType combination");
5077}
5078
5079static std::optional<std::pair<Register, SrcStatus>>
5080calcNextStatus(std::pair<Register, SrcStatus> Curr,
5081 const MachineRegisterInfo &MRI) {
5082 const MachineInstr *MI = MRI.getVRegDef(Reg: Curr.first);
5083
5084 unsigned Opc = MI->getOpcode();
5085
5086 // Handle general Opc cases.
5087 switch (Opc) {
5088 case AMDGPU::G_BITCAST:
5089 return std::optional<std::pair<Register, SrcStatus>>(
5090 {MI->getOperand(i: 1).getReg(), Curr.second});
5091 case AMDGPU::COPY:
5092 if (MI->getOperand(i: 1).getReg().isPhysical())
5093 return std::nullopt;
5094 return std::optional<std::pair<Register, SrcStatus>>(
5095 {MI->getOperand(i: 1).getReg(), Curr.second});
5096 case AMDGPU::G_FNEG: {
5097 SrcStatus Stat = getNegStatus(Reg: Curr.first, S: Curr.second, MRI);
5098 if (Stat == SrcStatus::INVALID)
5099 return std::nullopt;
5100 return std::optional<std::pair<Register, SrcStatus>>(
5101 {MI->getOperand(i: 1).getReg(), Stat});
5102 }
5103 default:
5104 break;
5105 }
5106
5107 // Calc next Stat from current Stat.
5108 switch (Curr.second) {
5109 case SrcStatus::IS_SAME:
5110 if (isTruncHalf(MI, MRI))
5111 return std::optional<std::pair<Register, SrcStatus>>(
5112 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5113 else if (isUnmergeHalf(MI, MRI)) {
5114 if (Curr.first == MI->getOperand(i: 0).getReg())
5115 return std::optional<std::pair<Register, SrcStatus>>(
5116 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF});
5117 return std::optional<std::pair<Register, SrcStatus>>(
5118 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF});
5119 }
5120 break;
5121 case SrcStatus::IS_HI_NEG:
5122 if (isTruncHalf(MI, MRI)) {
5123 // [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5124 // [CurrHi, CurrLo] = trunc [OpUpper, OpLower] = OpLower
5125 // = [OpLowerHi, OpLowerLo]
5126 // Src = [SrcHi, SrcLo] = [-CurrHi, CurrLo]
5127 // = [-OpLowerHi, OpLowerLo]
5128 // = -OpLower
5129 return std::optional<std::pair<Register, SrcStatus>>(
5130 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5131 }
5132 if (isUnmergeHalf(MI, MRI)) {
5133 if (Curr.first == MI->getOperand(i: 0).getReg())
5134 return std::optional<std::pair<Register, SrcStatus>>(
5135 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5136 return std::optional<std::pair<Register, SrcStatus>>(
5137 {MI->getOperand(i: 2).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5138 }
5139 break;
5140 case SrcStatus::IS_UPPER_HALF:
5141 if (isShlHalf(MI, MRI))
5142 return std::optional<std::pair<Register, SrcStatus>>(
5143 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF});
5144 break;
5145 case SrcStatus::IS_LOWER_HALF:
5146 if (isLshrHalf(MI, MRI))
5147 return std::optional<std::pair<Register, SrcStatus>>(
5148 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF});
5149 break;
5150 case SrcStatus::IS_UPPER_HALF_NEG:
5151 if (isShlHalf(MI, MRI))
5152 return std::optional<std::pair<Register, SrcStatus>>(
5153 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_LOWER_HALF_NEG});
5154 break;
5155 case SrcStatus::IS_LOWER_HALF_NEG:
5156 if (isLshrHalf(MI, MRI))
5157 return std::optional<std::pair<Register, SrcStatus>>(
5158 {MI->getOperand(i: 1).getReg(), SrcStatus::IS_UPPER_HALF_NEG});
5159 break;
5160 default:
5161 break;
5162 }
5163 return std::nullopt;
5164}
5165
5166/// This is used to control valid status that current MI supports. For example,
5167/// non floating point intrinsic such as @llvm.amdgcn.sdot2 does not support NEG
5168/// bit on VOP3P.
5169/// The class can be further extended to recognize support on SEL, NEG, ABS bit
5170/// for different MI on different arch
5171class SearchOptions {
5172private:
5173 bool HasNeg = false;
5174 // Assume all complex pattern of VOP3P have opsel.
5175 bool HasOpsel = true;
5176
5177public:
5178 SearchOptions(Register Reg, const MachineRegisterInfo &MRI) {
5179 const MachineInstr *MI = MRI.getVRegDef(Reg);
5180 unsigned Opc = MI->getOpcode();
5181
5182 if (Opc == TargetOpcode::G_INTRINSIC) {
5183 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val: *MI).getIntrinsicID();
5184 // Only float point intrinsic has neg & neg_hi bits.
5185 if (IntrinsicID == Intrinsic::amdgcn_fdot2)
5186 HasNeg = true;
5187 } else if (TargetInstrInfo::isGenericOpcode(Opc)) {
5188 // Keep same for generic op.
5189 HasNeg = true;
5190 }
5191 }
5192 bool checkOptions(SrcStatus Stat) const {
5193 if (!HasNeg &&
5194 (Stat >= SrcStatus::NEG_START && Stat <= SrcStatus::NEG_END)) {
5195 return false;
5196 }
5197 if (!HasOpsel &&
5198 (Stat >= SrcStatus::HALF_START && Stat <= SrcStatus::HALF_END)) {
5199 return false;
5200 }
5201 return true;
5202 }
5203};
5204
5205static SmallVector<std::pair<Register, SrcStatus>>
5206getSrcStats(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5207 int MaxDepth = 3) {
5208 int Depth = 0;
5209 auto Curr = calcNextStatus(Curr: {Reg, SrcStatus::IS_SAME}, MRI);
5210 SmallVector<std::pair<Register, SrcStatus>> Statlist;
5211
5212 while (Depth <= MaxDepth && Curr.has_value()) {
5213 Depth++;
5214 if (SO.checkOptions(Stat: Curr.value().second))
5215 Statlist.push_back(Elt: Curr.value());
5216 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5217 }
5218
5219 return Statlist;
5220}
5221
5222static std::pair<Register, SrcStatus>
5223getLastSameOrNeg(Register Reg, const MachineRegisterInfo &MRI, SearchOptions SO,
5224 int MaxDepth = 3) {
5225 int Depth = 0;
5226 std::pair<Register, SrcStatus> LastSameOrNeg = {Reg, SrcStatus::IS_SAME};
5227 auto Curr = calcNextStatus(Curr: LastSameOrNeg, MRI);
5228
5229 while (Depth <= MaxDepth && Curr.has_value()) {
5230 Depth++;
5231 SrcStatus Stat = Curr.value().second;
5232 if (SO.checkOptions(Stat)) {
5233 if (Stat == SrcStatus::IS_SAME || Stat == SrcStatus::IS_HI_NEG ||
5234 Stat == SrcStatus::IS_LO_NEG || Stat == SrcStatus::IS_BOTH_NEG)
5235 LastSameOrNeg = Curr.value();
5236 }
5237 Curr = calcNextStatus(Curr: Curr.value(), MRI);
5238 }
5239
5240 return LastSameOrNeg;
5241}
5242
5243static bool isSameBitWidth(Register Reg1, Register Reg2,
5244 const MachineRegisterInfo &MRI) {
5245 unsigned Width1 = MRI.getType(Reg: Reg1).getSizeInBits();
5246 unsigned Width2 = MRI.getType(Reg: Reg2).getSizeInBits();
5247 return Width1 == Width2;
5248}
5249
5250static unsigned updateMods(SrcStatus HiStat, SrcStatus LoStat, unsigned Mods) {
5251 // SrcStatus::IS_LOWER_HALF remain 0.
5252 if (HiStat == SrcStatus::IS_UPPER_HALF_NEG) {
5253 Mods ^= SISrcMods::NEG_HI;
5254 Mods |= SISrcMods::OP_SEL_1;
5255 } else if (HiStat == SrcStatus::IS_UPPER_HALF)
5256 Mods |= SISrcMods::OP_SEL_1;
5257 else if (HiStat == SrcStatus::IS_LOWER_HALF_NEG)
5258 Mods ^= SISrcMods::NEG_HI;
5259 else if (HiStat == SrcStatus::IS_HI_NEG)
5260 Mods ^= SISrcMods::NEG_HI;
5261
5262 if (LoStat == SrcStatus::IS_UPPER_HALF_NEG) {
5263 Mods ^= SISrcMods::NEG;
5264 Mods |= SISrcMods::OP_SEL_0;
5265 } else if (LoStat == SrcStatus::IS_UPPER_HALF)
5266 Mods |= SISrcMods::OP_SEL_0;
5267 else if (LoStat == SrcStatus::IS_LOWER_HALF_NEG)
5268 Mods |= SISrcMods::NEG;
5269 else if (LoStat == SrcStatus::IS_HI_NEG)
5270 Mods ^= SISrcMods::NEG;
5271
5272 return Mods;
5273}
5274
5275static bool isValidToPack(SrcStatus HiStat, SrcStatus LoStat, Register NewReg,
5276 Register RootReg, const SIInstrInfo &TII,
5277 const MachineRegisterInfo &MRI) {
5278 auto IsHalfState = [](SrcStatus S) {
5279 return S == SrcStatus::IS_UPPER_HALF || S == SrcStatus::IS_UPPER_HALF_NEG ||
5280 S == SrcStatus::IS_LOWER_HALF || S == SrcStatus::IS_LOWER_HALF_NEG;
5281 };
5282 return isSameBitWidth(Reg1: NewReg, Reg2: RootReg, MRI) && IsHalfState(LoStat) &&
5283 IsHalfState(HiStat);
5284}
5285
5286std::pair<Register, unsigned> AMDGPUInstructionSelector::selectVOP3PModsImpl(
5287 Register RootReg, const MachineRegisterInfo &MRI, bool IsDOT) const {
5288 unsigned Mods = 0;
5289 // No modification if Root type is not form of <2 x Type>.
5290 if (isVectorOfTwoOrScalar(Reg: RootReg, MRI) != TypeClass::VECTOR_OF_TWO) {
5291 Mods |= SISrcMods::OP_SEL_1;
5292 return {RootReg, Mods};
5293 }
5294
5295 SearchOptions SO(RootReg, MRI);
5296
5297 std::pair<Register, SrcStatus> Stat = getLastSameOrNeg(Reg: RootReg, MRI, SO);
5298
5299 if (Stat.second == SrcStatus::IS_BOTH_NEG)
5300 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
5301 else if (Stat.second == SrcStatus::IS_HI_NEG)
5302 Mods ^= SISrcMods::NEG_HI;
5303 else if (Stat.second == SrcStatus::IS_LO_NEG)
5304 Mods ^= SISrcMods::NEG;
5305
5306 // 64-bit VOP3P instructions do not have OPSEL or ABS. Bail on v2f64 or v2i64.
5307 // TODO: Select NEG_LO and NEG_HI modifiers from BUILD_VECTOR.
5308 if (MRI.getType(Reg: RootReg).getSizeInBits() == 128) {
5309 Mods |= SISrcMods::OP_SEL_1; // Just the default, OPSEL unsupported.
5310 return {Stat.first, Mods};
5311 }
5312
5313 GBuildVector *MI;
5314 if (!mi_match(R: Stat.first, MRI, P: m_GBuildVector(Inst&: MI)) ||
5315 MI->getNumOperands() != 3 || (IsDOT && Subtarget->hasDOTOpSelHazard())) {
5316 Mods |= SISrcMods::OP_SEL_1;
5317 return {Stat.first, Mods};
5318 }
5319
5320 SmallVector<std::pair<Register, SrcStatus>> StatlistHi =
5321 getSrcStats(Reg: MI->getOperand(i: 2).getReg(), MRI, SO);
5322
5323 if (StatlistHi.empty()) {
5324 Mods |= SISrcMods::OP_SEL_1;
5325 return {Stat.first, Mods};
5326 }
5327
5328 SmallVector<std::pair<Register, SrcStatus>> StatlistLo =
5329 getSrcStats(Reg: MI->getOperand(i: 1).getReg(), MRI, SO);
5330
5331 if (StatlistLo.empty()) {
5332 Mods |= SISrcMods::OP_SEL_1;
5333 return {Stat.first, Mods};
5334 }
5335
5336 for (int I = StatlistHi.size() - 1; I >= 0; I--) {
5337 for (int J = StatlistLo.size() - 1; J >= 0; J--) {
5338 if (StatlistHi[I].first == StatlistLo[J].first &&
5339 isValidToPack(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second,
5340 NewReg: StatlistHi[I].first, RootReg, TII, MRI))
5341 return {StatlistHi[I].first,
5342 updateMods(HiStat: StatlistHi[I].second, LoStat: StatlistLo[J].second, Mods)};
5343 }
5344 }
5345 // Packed instructions do not have abs modifiers.
5346 Mods |= SISrcMods::OP_SEL_1;
5347
5348 return {Stat.first, Mods};
5349}
5350
5351// Removed unused function `getAllKindImm` to eliminate dead code.
5352
5353static bool checkRB(Register Reg, unsigned int RBNo,
5354 const AMDGPURegisterBankInfo &RBI,
5355 const MachineRegisterInfo &MRI,
5356 const TargetRegisterInfo &TRI) {
5357 const RegisterBank *RB = RBI.getRegBank(Reg, MRI, TRI);
5358 return RB->getID() == RBNo;
5359}
5360
5361// This function is used to get the correct register bank for returned reg.
5362// Assume:
5363// 1. VOP3P is always legal for VGPR.
5364// 2. RootOp's regbank is legal.
5365// Thus
5366// 1. If RootOp is SGPR, then NewOp can be SGPR or VGPR.
5367// 2. If RootOp is VGPR, then NewOp must be VGPR.
5368static Register
5369getLegalRegBank(Register NewReg, Register RootReg, MachineInstr &Use,
5370 const AMDGPURegisterBankInfo &RBI, MachineRegisterInfo &MRI,
5371 const TargetRegisterInfo &TRI, const SIInstrInfo &TII) {
5372 // RootOp can only be VGPR or SGPR (some hand written cases such as.
5373 // inst-select-ashr.v2s16.mir::ashr_v2s16_vs).
5374 if (checkRB(Reg: RootReg, RBNo: AMDGPU::SGPRRegBankID, RBI, MRI, TRI) ||
5375 checkRB(Reg: NewReg, RBNo: AMDGPU::VGPRRegBankID, RBI, MRI, TRI))
5376 return NewReg;
5377
5378 if (mi_match(R: RootReg, MRI, P: m_Copy(Src: m_SpecificReg(RequestedReg: NewReg)))) {
5379 // RootOp is VGPR, NewOp is not VGPR, but RootOp = COPY NewOp.
5380 return RootReg;
5381 }
5382
5383 Register DstReg = MRI.cloneVirtualRegister(VReg: RootReg);
5384 MachineInstrBuilder MIB = BuildMI(BB&: *Use.getParent(), I&: Use, MIMD: Use.getDebugLoc(),
5385 MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: DstReg)
5386 .addReg(RegNo: NewReg);
5387
5388 // Only accept VGPR.
5389 return MIB->getOperand(i: 0).getReg();
5390}
5391
5392InstructionSelector::ComplexRendererFns
5393AMDGPUInstructionSelector::selectVOP3PRetHelper(MachineOperand &Root,
5394 bool IsDOT) const {
5395 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5396 Register Reg;
5397 unsigned Mods;
5398 std::tie(args&: Reg, args&: Mods) = selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT);
5399
5400 Reg = getLegalRegBank(NewReg: Reg, RootReg: Root.getReg(), Use&: *Root.getParent(), RBI, MRI, TRI,
5401 TII);
5402 return {{
5403 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
5404 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5405 }};
5406}
5407
5408InstructionSelector::ComplexRendererFns
5409AMDGPUInstructionSelector::selectVOP3PMods(MachineOperand &Root) const {
5410
5411 return selectVOP3PRetHelper(Root);
5412}
5413
5414InstructionSelector::ComplexRendererFns
5415AMDGPUInstructionSelector::selectVOP3PModsDOT(MachineOperand &Root) const {
5416
5417 return selectVOP3PRetHelper(Root, IsDOT: true);
5418}
5419
5420InstructionSelector::ComplexRendererFns
5421AMDGPUInstructionSelector::selectVOP3PNoModsDOT(MachineOperand &Root) const {
5422 MachineRegisterInfo &MRI = Root.getParent()->getMF()->getRegInfo();
5423 Register Src;
5424 unsigned Mods;
5425 std::tie(args&: Src, args&: Mods) = selectVOP3PModsImpl(RootReg: Root.getReg(), MRI, IsDOT: true /*IsDOT*/);
5426 if (Mods != SISrcMods::OP_SEL_1)
5427 return {};
5428
5429 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5430}
5431
5432InstructionSelector::ComplexRendererFns
5433AMDGPUInstructionSelector::selectVOP3PModsF32(MachineOperand &Root) const {
5434 Register Src;
5435 unsigned Mods;
5436 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5437
5438 return {{
5439 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5440 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5441 }};
5442}
5443
5444InstructionSelector::ComplexRendererFns
5445AMDGPUInstructionSelector::selectVOP3PNoModsF32(MachineOperand &Root) const {
5446 Register Src;
5447 unsigned Mods;
5448 std::tie(args&: Src, args&: Mods) = selectVOP3PModsF32Impl(Src: Root.getReg());
5449 if (Mods != SISrcMods::OP_SEL_1)
5450 return {};
5451
5452 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); }}};
5453}
5454
5455InstructionSelector::ComplexRendererFns
5456AMDGPUInstructionSelector::selectWMMAOpSelVOP3PMods(
5457 MachineOperand &Root) const {
5458 assert((Root.isImm() && (Root.getImm() == -1 || Root.getImm() == 0)) &&
5459 "expected i1 value");
5460 unsigned Mods = SISrcMods::OP_SEL_1;
5461 if (Root.getImm() != 0)
5462 Mods |= SISrcMods::OP_SEL_0;
5463
5464 return {{
5465 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5466 }};
5467}
5468
5469static Register buildRegSequence(SmallVectorImpl<Register> &Elts,
5470 MachineInstr *InsertPt,
5471 MachineRegisterInfo &MRI) {
5472 const TargetRegisterClass *DstRegClass;
5473 switch (Elts.size()) {
5474 case 8:
5475 DstRegClass = &AMDGPU::VReg_256RegClass;
5476 break;
5477 case 4:
5478 DstRegClass = &AMDGPU::VReg_128RegClass;
5479 break;
5480 case 2:
5481 DstRegClass = &AMDGPU::VReg_64RegClass;
5482 break;
5483 default:
5484 llvm_unreachable("unhandled Reg sequence size");
5485 }
5486
5487 MachineIRBuilder B(*InsertPt);
5488 auto MIB = B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
5489 .addDef(RegNo: MRI.createVirtualRegister(RegClass: DstRegClass));
5490 for (unsigned i = 0; i < Elts.size(); ++i) {
5491 MIB.addReg(RegNo: Elts[i]);
5492 MIB.addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: i));
5493 }
5494 return MIB->getOperand(i: 0).getReg();
5495}
5496
5497static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods,
5498 SmallVectorImpl<Register> &Elts, Register &Src,
5499 MachineInstr *InsertPt,
5500 MachineRegisterInfo &MRI) {
5501 if (ModOpcode == TargetOpcode::G_FNEG) {
5502 Mods |= SISrcMods::NEG;
5503 // Check if all elements also have abs modifier
5504 SmallVector<Register, 8> NegAbsElts;
5505 for (auto El : Elts) {
5506 Register FabsSrc;
5507 if (!mi_match(R: El, MRI, P: m_GFabs(Src: m_Reg(R&: FabsSrc))))
5508 break;
5509 NegAbsElts.push_back(Elt: FabsSrc);
5510 }
5511 if (Elts.size() != NegAbsElts.size()) {
5512 // Neg
5513 Src = buildRegSequence(Elts, InsertPt, MRI);
5514 } else {
5515 // Neg and Abs
5516 Mods |= SISrcMods::NEG_HI;
5517 Src = buildRegSequence(Elts&: NegAbsElts, InsertPt, MRI);
5518 }
5519 } else {
5520 assert(ModOpcode == TargetOpcode::G_FABS);
5521 // Abs
5522 Mods |= SISrcMods::NEG_HI;
5523 Src = buildRegSequence(Elts, InsertPt, MRI);
5524 }
5525}
5526
5527InstructionSelector::ComplexRendererFns
5528AMDGPUInstructionSelector::selectWMMAModsF32NegAbs(MachineOperand &Root) const {
5529 Register Src = Root.getReg();
5530 unsigned Mods = SISrcMods::OP_SEL_1;
5531 SmallVector<Register, 8> EltsF32;
5532
5533 GBuildVector *BV;
5534 if (mi_match(R: Src, MRI: *MRI, P: m_GBuildVector(Inst&: BV))) {
5535 assert(BV->getNumSources() > 0);
5536 // Based on first element decide which mod we match, neg or abs
5537 MachineInstr *ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: 0));
5538 unsigned ModOpcode = (ElF32->getOpcode() == AMDGPU::G_FNEG)
5539 ? AMDGPU::G_FNEG
5540 : AMDGPU::G_FABS;
5541 for (unsigned i = 0; i < BV->getNumSources(); ++i) {
5542 ElF32 = MRI->getVRegDef(Reg: BV->getSourceReg(I: i));
5543 if (ElF32->getOpcode() != ModOpcode)
5544 break;
5545 EltsF32.push_back(Elt: ElF32->getOperand(i: 1).getReg());
5546 }
5547
5548 // All elements had ModOpcode modifier
5549 if (BV->getNumSources() == EltsF32.size()) {
5550 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsF32, Src, InsertPt: Root.getParent(),
5551 MRI&: *MRI);
5552 }
5553 }
5554
5555 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5556 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5557}
5558
5559InstructionSelector::ComplexRendererFns
5560AMDGPUInstructionSelector::selectWMMAModsF16Neg(MachineOperand &Root) const {
5561 Register Src = Root.getReg();
5562 unsigned Mods = SISrcMods::OP_SEL_1;
5563 SmallVector<Register, 8> EltsV2F16;
5564
5565 GConcatVectors *CV;
5566 if (mi_match(R: Src, MRI: *MRI, P: m_GConcatVectors(Inst&: CV))) {
5567 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5568 Register FNegSrc;
5569 if (!mi_match(R: CV->getSourceReg(I: i), MRI: *MRI, P: m_GFNeg(Src: m_Reg(R&: FNegSrc))))
5570 break;
5571 EltsV2F16.push_back(Elt: FNegSrc);
5572 }
5573
5574 // All elements had ModOpcode modifier
5575 if (CV->getNumSources() == EltsV2F16.size()) {
5576 Mods |= SISrcMods::NEG;
5577 Mods |= SISrcMods::NEG_HI;
5578 Src = buildRegSequence(Elts&: EltsV2F16, InsertPt: Root.getParent(), MRI&: *MRI);
5579 }
5580 }
5581
5582 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5583 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5584}
5585
5586InstructionSelector::ComplexRendererFns
5587AMDGPUInstructionSelector::selectWMMAModsF16NegAbs(MachineOperand &Root) const {
5588 Register Src = Root.getReg();
5589 unsigned Mods = SISrcMods::OP_SEL_1;
5590 SmallVector<Register, 8> EltsV2F16;
5591
5592 GConcatVectors *CV;
5593 if (mi_match(R: Src, MRI: *MRI, P: m_GConcatVectors(Inst&: CV))) {
5594 assert(CV->getNumSources() > 0);
5595 MachineInstr *ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: 0));
5596 // Based on first element decide which mod we match, neg or abs
5597 unsigned ModOpcode = (ElV2F16->getOpcode() == AMDGPU::G_FNEG)
5598 ? AMDGPU::G_FNEG
5599 : AMDGPU::G_FABS;
5600
5601 for (unsigned i = 0; i < CV->getNumSources(); ++i) {
5602 ElV2F16 = MRI->getVRegDef(Reg: CV->getSourceReg(I: i));
5603 if (ElV2F16->getOpcode() != ModOpcode)
5604 break;
5605 EltsV2F16.push_back(Elt: ElV2F16->getOperand(i: 1).getReg());
5606 }
5607
5608 // All elements had ModOpcode modifier
5609 if (CV->getNumSources() == EltsV2F16.size()) {
5610 MachineIRBuilder B(*Root.getParent());
5611 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsV2F16, Src, InsertPt: Root.getParent(),
5612 MRI&: *MRI);
5613 }
5614 }
5615
5616 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5617 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }}};
5618}
5619
5620InstructionSelector::ComplexRendererFns
5621AMDGPUInstructionSelector::selectWMMAVISrc(MachineOperand &Root) const {
5622 std::optional<FPValueAndVReg> FPValReg;
5623 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GFCstOrSplat(FPValReg))) {
5624 if (TII.isInlineConstant(Imm: FPValReg->Value)) {
5625 return {{[=](MachineInstrBuilder &MIB) {
5626 MIB.addImm(Val: FPValReg->Value.bitcastToAPInt().getSExtValue());
5627 }}};
5628 }
5629 // Non-inlineable splat floats should not fall-through for integer immediate
5630 // checks.
5631 return {};
5632 }
5633
5634 APInt ICst;
5635 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICstOrSplat(Cst&: ICst))) {
5636 if (TII.isInlineConstant(Imm: ICst)) {
5637 return {
5638 {[=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ICst.getSExtValue()); }}};
5639 }
5640 }
5641
5642 return {};
5643}
5644
5645InstructionSelector::ComplexRendererFns
5646AMDGPUInstructionSelector::selectSWMMACIndex8(MachineOperand &Root) const {
5647 Register Src =
5648 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5649 unsigned Key = 0;
5650
5651 Register ShiftSrc;
5652 std::optional<ValueAndVReg> ShiftAmt;
5653 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5654 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5655 ShiftAmt->Value.getZExtValue() % 8 == 0) {
5656 Key = ShiftAmt->Value.getZExtValue() / 8;
5657 Src = ShiftSrc;
5658 }
5659
5660 return {{
5661 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5662 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5663 }};
5664}
5665
5666InstructionSelector::ComplexRendererFns
5667AMDGPUInstructionSelector::selectSWMMACIndex16(MachineOperand &Root) const {
5668
5669 Register Src =
5670 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5671 unsigned Key = 0;
5672
5673 Register ShiftSrc;
5674 std::optional<ValueAndVReg> ShiftAmt;
5675 if (mi_match(R: Src, MRI: *MRI, P: m_GLShr(L: m_Reg(R&: ShiftSrc), R: m_GCst(ValReg&: ShiftAmt))) &&
5676 MRI->getType(Reg: ShiftSrc).getSizeInBits() == 32 &&
5677 ShiftAmt->Value.getZExtValue() == 16) {
5678 Src = ShiftSrc;
5679 Key = 1;
5680 }
5681
5682 return {{
5683 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5684 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5685 }};
5686}
5687
5688InstructionSelector::ComplexRendererFns
5689AMDGPUInstructionSelector::selectSWMMACIndex32(MachineOperand &Root) const {
5690 Register Src =
5691 getDefIgnoringCopies(Reg: Root.getReg(), MRI: *MRI)->getOperand(i: 0).getReg();
5692 unsigned Key = 0;
5693
5694 Register S32 = matchZeroExtendFromS32(Reg: Src);
5695 if (!S32)
5696 S32 = matchAnyExtendFromS32(Reg: Src);
5697
5698 if (S32) {
5699 const MachineInstr *Def = getDefIgnoringCopies(Reg: S32, MRI: *MRI);
5700 if (Def->getOpcode() == TargetOpcode::G_UNMERGE_VALUES) {
5701 assert(Def->getNumOperands() == 3);
5702 Register DstReg1 = Def->getOperand(i: 1).getReg();
5703 if (mi_match(R: S32, MRI: *MRI,
5704 P: m_any_of(preds: m_SpecificReg(RequestedReg: DstReg1), preds: m_Copy(Src: m_Reg(R&: DstReg1))))) {
5705 Src = Def->getOperand(i: 2).getReg();
5706 Key = 1;
5707 }
5708 }
5709 }
5710
5711 return {{
5712 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5713 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Key); } // index_key
5714 }};
5715}
5716
5717InstructionSelector::ComplexRendererFns
5718AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const {
5719 Register Src;
5720 unsigned Mods;
5721 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
5722
5723 // FIXME: Handle op_sel
5724 return {{
5725 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
5726 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
5727 }};
5728}
5729
5730// FIXME-TRUE16 remove when fake16 is removed
5731InstructionSelector::ComplexRendererFns
5732AMDGPUInstructionSelector::selectVINTERPMods(MachineOperand &Root) const {
5733 Register Src;
5734 unsigned Mods;
5735 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5736 /*IsCanonicalizing=*/true,
5737 /*AllowAbs=*/false,
5738 /*OpSel=*/false);
5739
5740 return {{
5741 [=](MachineInstrBuilder &MIB) {
5742 MIB.addReg(
5743 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5744 },
5745 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5746 }};
5747}
5748
5749InstructionSelector::ComplexRendererFns
5750AMDGPUInstructionSelector::selectVINTERPModsHi(MachineOperand &Root) const {
5751 Register Src;
5752 unsigned Mods;
5753 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg(),
5754 /*IsCanonicalizing=*/true,
5755 /*AllowAbs=*/false,
5756 /*OpSel=*/true);
5757
5758 return {{
5759 [=](MachineInstrBuilder &MIB) {
5760 MIB.addReg(
5761 RegNo: copyToVGPRIfSrcFolded(Src, Mods, Root, InsertPt: MIB, /* ForceVGPR */ true));
5762 },
5763 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); }, // src0_mods
5764 }};
5765}
5766
5767// Given \p Offset and load specified by the \p Root operand check if \p Offset
5768// is a multiple of the load byte size. If it is update \p Offset to a
5769// pre-scaled value and return true.
5770bool AMDGPUInstructionSelector::selectScaleOffset(MachineOperand &Root,
5771 Register &Offset,
5772 bool IsSigned) const {
5773 if (!Subtarget->hasScaleOffset())
5774 return false;
5775
5776 const MachineInstr &MI = *Root.getParent();
5777 MachineMemOperand *MMO = *MI.memoperands_begin();
5778
5779 if (!MMO->getSize().hasValue())
5780 return false;
5781
5782 uint64_t Size = MMO->getSize().getValue();
5783
5784 Register OffsetReg = matchExtendFromS32OrS32(Reg: Offset, IsSigned);
5785 if (!OffsetReg)
5786 OffsetReg = Offset;
5787
5788 if (auto Def = getDefSrcRegIgnoringCopies(Reg: OffsetReg, MRI: *MRI))
5789 OffsetReg = Def->Reg;
5790
5791 Register Op0;
5792 MachineInstr *Mul;
5793 bool ScaleOffset =
5794 (isPowerOf2_64(Value: Size) &&
5795 mi_match(R: OffsetReg, MRI: *MRI,
5796 P: m_GShl(L: m_Reg(R&: Op0),
5797 R: m_any_of(preds: m_SpecificICst(RequestedValue: Log2_64(Value: Size)),
5798 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Log2_64(Value: Size))))))) ||
5799 mi_match(R: OffsetReg, MRI: *MRI,
5800 P: m_GMul(L: m_Reg(R&: Op0), R: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5801 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) ||
5802 mi_match(
5803 R: OffsetReg, MRI: *MRI,
5804 P: m_BinOp(Opcode: IsSigned ? AMDGPU::S_MUL_I64_I32_PSEUDO : AMDGPU::S_MUL_U64,
5805 L: m_Reg(R&: Op0), R: m_SpecificICst(RequestedValue: Size))) ||
5806 // Match G_AMDGPU_MAD_U64_U32 offset, c, 0
5807 (mi_match(R: OffsetReg, MRI: *MRI, P: m_MInstr(MI&: Mul)) &&
5808 (Mul->getOpcode() == (IsSigned ? AMDGPU::G_AMDGPU_MAD_I64_I32
5809 : AMDGPU::G_AMDGPU_MAD_U64_U32) ||
5810 (IsSigned && Mul->getOpcode() == AMDGPU::G_AMDGPU_MAD_U64_U32 &&
5811 VT->signBitIsZero(Op: Mul->getOperand(i: 2).getReg()))) &&
5812 mi_match(R: Mul->getOperand(i: 4).getReg(), MRI: *MRI, P: m_ZeroInt()) &&
5813 mi_match(R: Mul->getOperand(i: 3).getReg(), MRI: *MRI,
5814 P: m_GTrunc(Src: m_any_of(preds: m_SpecificICst(RequestedValue: Size),
5815 preds: m_Copy(Src: m_SpecificICst(RequestedValue: Size))))) &&
5816 mi_match(R: Mul->getOperand(i: 2).getReg(), MRI: *MRI, P: m_Reg(R&: Op0)));
5817
5818 if (ScaleOffset)
5819 Offset = Op0;
5820
5821 return ScaleOffset;
5822}
5823
5824bool AMDGPUInstructionSelector::selectSmrdOffset(MachineOperand &Root,
5825 Register &Base,
5826 Register *SOffset,
5827 int64_t *Offset,
5828 bool *ScaleOffset) const {
5829 MachineInstr *MI = Root.getParent();
5830 MachineBasicBlock *MBB = MI->getParent();
5831
5832 // FIXME: We should shrink the GEP if the offset is known to be <= 32-bits,
5833 // then we can select all ptr + 32-bit offsets.
5834 SmallVector<GEPInfo, 4> AddrInfo;
5835 getAddrModeInfo(Load: *MI, MRI: *MRI, AddrInfo);
5836
5837 if (AddrInfo.empty())
5838 return false;
5839
5840 const GEPInfo &GEPI = AddrInfo[0];
5841 std::optional<int64_t> EncodedImm;
5842
5843 if (ScaleOffset)
5844 *ScaleOffset = false;
5845
5846 if (SOffset && Offset) {
5847 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
5848 /*HasSOffset=*/true);
5849 if (GEPI.SgprParts.size() == 1 && GEPI.Imm != 0 && EncodedImm &&
5850 AddrInfo.size() > 1) {
5851 const GEPInfo &GEPI2 = AddrInfo[1];
5852 if (GEPI2.SgprParts.size() == 2 && GEPI2.Imm == 0) {
5853 Register OffsetReg = GEPI2.SgprParts[1];
5854 if (ScaleOffset)
5855 *ScaleOffset =
5856 selectScaleOffset(Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */);
5857 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
5858 if (OffsetReg) {
5859 Base = GEPI2.SgprParts[0];
5860 *SOffset = OffsetReg;
5861 *Offset = *EncodedImm;
5862 if (*Offset >= 0 || !AMDGPU::hasSMRDSignedImmOffset(ST: STI))
5863 return true;
5864
5865 // For unbuffered smem loads, it is illegal for the Immediate Offset
5866 // to be negative if the resulting (Offset + (M0 or SOffset or zero)
5867 // is negative. Handle the case where the Immediate Offset + SOffset
5868 // is negative.
5869 auto SKnown = VT->getKnownBits(R: *SOffset);
5870 if (*Offset + SKnown.getMinValue().getSExtValue() < 0)
5871 return false;
5872
5873 return true;
5874 }
5875 }
5876 }
5877 return false;
5878 }
5879
5880 EncodedImm = AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: GEPI.Imm, /*IsBuffer=*/false,
5881 /*HasSOffset=*/false);
5882 if (Offset && GEPI.SgprParts.size() == 1 && EncodedImm) {
5883 Base = GEPI.SgprParts[0];
5884 *Offset = *EncodedImm;
5885 return true;
5886 }
5887
5888 // SGPR offset is unsigned.
5889 if (SOffset && GEPI.SgprParts.size() == 1 && isUInt<32>(x: GEPI.Imm) &&
5890 GEPI.Imm != 0) {
5891 // If we make it this far we have a load with an 32-bit immediate offset.
5892 // It is OK to select this using a sgpr offset, because we have already
5893 // failed trying to select this load into one of the _IMM variants since
5894 // the _IMM Patterns are considered before the _SGPR patterns.
5895 Base = GEPI.SgprParts[0];
5896 *SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5897 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::S_MOV_B32), DestReg: *SOffset)
5898 .addImm(Val: GEPI.Imm);
5899 return true;
5900 }
5901
5902 if (SOffset && GEPI.SgprParts.size() && GEPI.Imm == 0) {
5903 Register OffsetReg = GEPI.SgprParts[1];
5904 if (ScaleOffset)
5905 *ScaleOffset = selectScaleOffset(Root, Offset&: OffsetReg, IsSigned: false /* IsSigned */);
5906 OffsetReg = matchZeroExtendFromS32OrS32(Reg: OffsetReg);
5907 if (OffsetReg) {
5908 Base = GEPI.SgprParts[0];
5909 *SOffset = OffsetReg;
5910 return true;
5911 }
5912 }
5913
5914 return false;
5915}
5916
5917InstructionSelector::ComplexRendererFns
5918AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
5919 Register Base;
5920 int64_t Offset;
5921 if (!selectSmrdOffset(Root, Base, /* SOffset= */ nullptr, Offset: &Offset,
5922 /* ScaleOffset */ nullptr))
5923 return std::nullopt;
5924
5925 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
5926 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }}};
5927}
5928
5929InstructionSelector::ComplexRendererFns
5930AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const {
5931 SmallVector<GEPInfo, 4> AddrInfo;
5932 getAddrModeInfo(Load: *Root.getParent(), MRI: *MRI, AddrInfo);
5933
5934 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
5935 return std::nullopt;
5936
5937 const GEPInfo &GEPInfo = AddrInfo[0];
5938 Register PtrReg = GEPInfo.SgprParts[0];
5939 std::optional<int64_t> EncodedImm =
5940 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: GEPInfo.Imm);
5941 if (!EncodedImm)
5942 return std::nullopt;
5943
5944 return {{
5945 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrReg); },
5946 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); }
5947 }};
5948}
5949
5950InstructionSelector::ComplexRendererFns
5951AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
5952 Register Base, SOffset;
5953 bool ScaleOffset;
5954 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, /* Offset= */ nullptr,
5955 ScaleOffset: &ScaleOffset))
5956 return std::nullopt;
5957
5958 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
5959 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
5960 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
5961 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
5962}
5963
5964InstructionSelector::ComplexRendererFns
5965AMDGPUInstructionSelector::selectSmrdSgprImm(MachineOperand &Root) const {
5966 Register Base, SOffset;
5967 int64_t Offset;
5968 bool ScaleOffset;
5969 if (!selectSmrdOffset(Root, Base, SOffset: &SOffset, Offset: &Offset, ScaleOffset: &ScaleOffset))
5970 return std::nullopt;
5971
5972 unsigned CPol = ScaleOffset ? AMDGPU::CPol::SCAL : 0;
5973 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Base); },
5974 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
5975 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
5976 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); }}};
5977}
5978
5979std::pair<Register, int> AMDGPUInstructionSelector::selectFlatOffsetImpl(
5980 MachineOperand &Root, AMDGPU::FlatAddrSpace FlatVariant) const {
5981 MachineInstr *MI = Root.getParent();
5982
5983 auto Default = std::pair(Root.getReg(), 0);
5984
5985 if (!STI.hasFlatInstOffsets())
5986 return Default;
5987
5988 Register PtrBase;
5989 int64_t ConstOffset;
5990 bool IsInBounds;
5991 std::tie(args&: PtrBase, args&: ConstOffset, args&: IsInBounds) =
5992 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
5993
5994 // Adding the offset to the base address with an immediate in a FLAT
5995 // instruction must not change the memory aperture in which the address falls.
5996 // Therefore we can only fold offsets from inbounds GEPs into FLAT
5997 // instructions.
5998 if (ConstOffset == 0 ||
5999 (FlatVariant == AMDGPU::FlatAddrSpace::FlatScratch &&
6000 !isFlatScratchBaseLegal(Addr: Root.getReg())) ||
6001 (FlatVariant == AMDGPU::FlatAddrSpace::FLAT && !IsInBounds))
6002 return Default;
6003
6004 unsigned AddrSpace = (*MI->memoperands_begin())->getAddrSpace();
6005 if (!TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace, FlatVariant))
6006 return Default;
6007
6008 return std::pair(PtrBase, ConstOffset);
6009}
6010
6011InstructionSelector::ComplexRendererFns
6012AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const {
6013 auto PtrWithOffset = selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FLAT);
6014
6015 return {{
6016 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6017 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6018 }};
6019}
6020
6021InstructionSelector::ComplexRendererFns
6022AMDGPUInstructionSelector::selectGlobalOffset(MachineOperand &Root) const {
6023 auto PtrWithOffset =
6024 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6025
6026 return {{
6027 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6028 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6029 }};
6030}
6031
6032InstructionSelector::ComplexRendererFns
6033AMDGPUInstructionSelector::selectScratchOffset(MachineOperand &Root) const {
6034 auto PtrWithOffset =
6035 selectFlatOffsetImpl(Root, FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch);
6036
6037 return {{
6038 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrWithOffset.first); },
6039 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: PtrWithOffset.second); },
6040 }};
6041}
6042
6043// Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
6044InstructionSelector::ComplexRendererFns
6045AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root,
6046 unsigned CPolBits,
6047 bool NeedIOffset) const {
6048 Register Addr = Root.getReg();
6049 Register PtrBase;
6050 int64_t ConstOffset;
6051 int64_t ImmOffset = 0;
6052
6053 // Match the immediate offset first, which canonically is moved as low as
6054 // possible.
6055 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6056 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6057
6058 if (ConstOffset != 0) {
6059 if (NeedIOffset &&
6060 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6061 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal)) {
6062 Addr = PtrBase;
6063 ImmOffset = ConstOffset;
6064 } else {
6065 auto PtrBaseDef = getDefSrcRegIgnoringCopies(Reg: PtrBase, MRI: *MRI);
6066 if (isSGPR(Reg: PtrBaseDef->Reg)) {
6067 if (ConstOffset > 0) {
6068 // Offset is too large.
6069 //
6070 // saddr + large_offset -> saddr +
6071 // (voffset = large_offset & ~MaxOffset) +
6072 // (large_offset & MaxOffset);
6073 int64_t SplitImmOffset = 0, RemainderOffset = ConstOffset;
6074 if (NeedIOffset) {
6075 std::tie(args&: SplitImmOffset, args&: RemainderOffset) =
6076 TII.splitFlatOffset(COffsetVal: ConstOffset, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
6077 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
6078 }
6079
6080 if (Subtarget->hasSignedGVSOffset() ? isInt<32>(x: RemainderOffset)
6081 : isUInt<32>(x: RemainderOffset)) {
6082 MachineInstr *MI = Root.getParent();
6083 MachineBasicBlock *MBB = MI->getParent();
6084 Register HighBits =
6085 MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6086
6087 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6088 DestReg: HighBits)
6089 .addImm(Val: RemainderOffset);
6090
6091 if (NeedIOffset)
6092 return {{
6093 [=](MachineInstrBuilder &MIB) {
6094 MIB.addReg(RegNo: PtrBase);
6095 }, // saddr
6096 [=](MachineInstrBuilder &MIB) {
6097 MIB.addReg(RegNo: HighBits);
6098 }, // voffset
6099 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: SplitImmOffset); },
6100 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6101 }};
6102 return {{
6103 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: PtrBase); }, // saddr
6104 [=](MachineInstrBuilder &MIB) {
6105 MIB.addReg(RegNo: HighBits);
6106 }, // voffset
6107 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); },
6108 }};
6109 }
6110 }
6111
6112 // We are adding a 64 bit SGPR and a constant. If constant bus limit
6113 // is 1 we would need to perform 1 or 2 extra moves for each half of
6114 // the constant and it is better to do a scalar add and then issue a
6115 // single VALU instruction to materialize zero. Otherwise it is less
6116 // instructions to perform VALU adds with immediates or inline literals.
6117 unsigned NumLiterals =
6118 !TII.isInlineConstant(Imm: APInt(32, Lo_32(Value: ConstOffset))) +
6119 !TII.isInlineConstant(Imm: APInt(32, Hi_32(Value: ConstOffset)));
6120 if (STI.getConstantBusLimit(Opcode: AMDGPU::V_ADD_U32_e64) > NumLiterals)
6121 return std::nullopt;
6122 }
6123 }
6124 }
6125
6126 // Match the variable offset.
6127 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6128 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6129 // Look through the SGPR->VGPR copy.
6130 Register SAddr =
6131 getSrcRegIgnoringCopies(Reg: AddrDef->MI->getOperand(i: 1).getReg(), MRI: *MRI);
6132
6133 if (isSGPR(Reg: SAddr)) {
6134 Register PtrBaseOffset = AddrDef->MI->getOperand(i: 2).getReg();
6135
6136 // It's possible voffset is an SGPR here, but the copy to VGPR will be
6137 // inserted later.
6138 bool ScaleOffset = selectScaleOffset(Root, Offset&: PtrBaseOffset,
6139 IsSigned: Subtarget->hasSignedGVSOffset());
6140 if (Register VOffset = matchExtendFromS32OrS32(
6141 Reg: PtrBaseOffset, IsSigned: Subtarget->hasSignedGVSOffset())) {
6142 if (NeedIOffset)
6143 return {{[=](MachineInstrBuilder &MIB) { // saddr
6144 MIB.addReg(RegNo: SAddr);
6145 },
6146 [=](MachineInstrBuilder &MIB) { // voffset
6147 MIB.addReg(RegNo: VOffset);
6148 },
6149 [=](MachineInstrBuilder &MIB) { // offset
6150 MIB.addImm(Val: ImmOffset);
6151 },
6152 [=](MachineInstrBuilder &MIB) { // cpol
6153 MIB.addImm(Val: CPolBits |
6154 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6155 }}};
6156 return {{[=](MachineInstrBuilder &MIB) { // saddr
6157 MIB.addReg(RegNo: SAddr);
6158 },
6159 [=](MachineInstrBuilder &MIB) { // voffset
6160 MIB.addReg(RegNo: VOffset);
6161 },
6162 [=](MachineInstrBuilder &MIB) { // cpol
6163 MIB.addImm(Val: CPolBits |
6164 (ScaleOffset ? AMDGPU::CPol::SCAL : 0));
6165 }}};
6166 }
6167 }
6168 }
6169
6170 // FIXME: We should probably have folded COPY (G_IMPLICIT_DEF) earlier, and
6171 // drop this.
6172 if (AddrDef->MI->getOpcode() == AMDGPU::G_IMPLICIT_DEF ||
6173 AddrDef->MI->getOpcode() == AMDGPU::G_CONSTANT || !isSGPR(Reg: AddrDef->Reg))
6174 return std::nullopt;
6175
6176 // It's cheaper to materialize a single 32-bit zero for vaddr than the two
6177 // moves required to copy a 64-bit SGPR to VGPR.
6178 MachineInstr *MI = Root.getParent();
6179 MachineBasicBlock *MBB = MI->getParent();
6180 Register VOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6181
6182 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: VOffset)
6183 .addImm(Val: 0);
6184
6185 if (NeedIOffset)
6186 return {{
6187 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6188 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6189 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6190 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6191 }};
6192 return {{
6193 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: AddrDef->Reg); }, // saddr
6194 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: VOffset); }, // voffset
6195 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPolBits); } // cpol
6196 }};
6197}
6198
6199InstructionSelector::ComplexRendererFns
6200AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root) const {
6201 return selectGlobalSAddr(Root, CPolBits: 0);
6202}
6203
6204InstructionSelector::ComplexRendererFns
6205AMDGPUInstructionSelector::selectGlobalSAddrCPol(MachineOperand &Root) const {
6206 const MachineInstr &I = *Root.getParent();
6207
6208 // We are assuming CPol is always the last operand of the intrinsic.
6209 auto PassedCPol =
6210 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6211 return selectGlobalSAddr(Root, CPolBits: PassedCPol);
6212}
6213
6214InstructionSelector::ComplexRendererFns
6215AMDGPUInstructionSelector::selectGlobalSAddrCPolM0(MachineOperand &Root) const {
6216 const MachineInstr &I = *Root.getParent();
6217
6218 // We are assuming CPol is second from last operand of the intrinsic.
6219 auto PassedCPol =
6220 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6221 return selectGlobalSAddr(Root, CPolBits: PassedCPol);
6222}
6223
6224InstructionSelector::ComplexRendererFns
6225AMDGPUInstructionSelector::selectGlobalSAddrGLC(MachineOperand &Root) const {
6226 return selectGlobalSAddr(Root, CPolBits: AMDGPU::CPol::GLC);
6227}
6228
6229InstructionSelector::ComplexRendererFns
6230AMDGPUInstructionSelector::selectGlobalSAddrNoIOffset(
6231 MachineOperand &Root) const {
6232 const MachineInstr &I = *Root.getParent();
6233
6234 // We are assuming CPol is always the last operand of the intrinsic.
6235 auto PassedCPol =
6236 I.getOperand(i: I.getNumOperands() - 1).getImm() & ~AMDGPU::CPol::SCAL;
6237 return selectGlobalSAddr(Root, CPolBits: PassedCPol, NeedIOffset: false);
6238}
6239
6240InstructionSelector::ComplexRendererFns
6241AMDGPUInstructionSelector::selectGlobalSAddrNoIOffsetM0(
6242 MachineOperand &Root) const {
6243 const MachineInstr &I = *Root.getParent();
6244
6245 // We are assuming CPol is second from last operand of the intrinsic.
6246 auto PassedCPol =
6247 I.getOperand(i: I.getNumOperands() - 2).getImm() & ~AMDGPU::CPol::SCAL;
6248 return selectGlobalSAddr(Root, CPolBits: PassedCPol, NeedIOffset: false);
6249}
6250
6251InstructionSelector::ComplexRendererFns
6252AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const {
6253 Register Addr = Root.getReg();
6254 Register PtrBase;
6255 int64_t ConstOffset;
6256 int64_t ImmOffset = 0;
6257
6258 // Match the immediate offset first, which canonically is moved as low as
6259 // possible.
6260 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6261 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6262
6263 if (ConstOffset != 0 && isFlatScratchBaseLegal(Addr) &&
6264 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6265 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6266 Addr = PtrBase;
6267 ImmOffset = ConstOffset;
6268 }
6269
6270 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6271 if (AddrDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6272 int FI = AddrDef->MI->getOperand(i: 1).getIndex();
6273 return {{
6274 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6275 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6276 }};
6277 }
6278
6279 Register SAddr = AddrDef->Reg;
6280
6281 if (AddrDef->MI->getOpcode() == AMDGPU::G_PTR_ADD) {
6282 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6283 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6284 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6285 auto RHSDef = getDefSrcRegIgnoringCopies(Reg: RHS, MRI: *MRI);
6286
6287 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX &&
6288 isSGPR(Reg: RHSDef->Reg)) {
6289 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6290 MachineInstr &I = *Root.getParent();
6291 MachineBasicBlock *BB = I.getParent();
6292 const DebugLoc &DL = I.getDebugLoc();
6293 SAddr = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6294
6295 BuildMI(BB&: *BB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_ADD_I32), DestReg: SAddr)
6296 .addFrameIndex(Idx: FI)
6297 .addReg(RegNo: RHSDef->Reg)
6298 .setOperandDead(3); // Dead scc
6299 }
6300 }
6301
6302 if (!isSGPR(Reg: SAddr))
6303 return std::nullopt;
6304
6305 return {{
6306 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SAddr); }, // saddr
6307 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); } // offset
6308 }};
6309}
6310
6311// Check whether the flat scratch SVS swizzle bug affects this access.
6312bool AMDGPUInstructionSelector::checkFlatScratchSVSSwizzleBug(
6313 Register VAddr, Register SAddr, uint64_t ImmOffset) const {
6314 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
6315 return false;
6316
6317 // The bug affects the swizzling of SVS accesses if there is any carry out
6318 // from the two low order bits (i.e. from bit 1 into bit 2) when adding
6319 // voffset to (soffset + inst_offset).
6320 auto VKnown = VT->getKnownBits(R: VAddr);
6321 auto SKnown = KnownBits::add(LHS: VT->getKnownBits(R: SAddr),
6322 RHS: KnownBits::makeConstant(C: APInt(32, ImmOffset)));
6323 uint64_t VMax = VKnown.getMaxValue().getZExtValue();
6324 uint64_t SMax = SKnown.getMaxValue().getZExtValue();
6325 return (VMax & 3) + (SMax & 3) >= 4;
6326}
6327
6328InstructionSelector::ComplexRendererFns
6329AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const {
6330 Register Addr = Root.getReg();
6331 Register PtrBase;
6332 int64_t ConstOffset;
6333 int64_t ImmOffset = 0;
6334
6335 // Match the immediate offset first, which canonically is moved as low as
6336 // possible.
6337 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6338 getPtrBaseWithConstantOffset(Root: Addr, MRI: *MRI);
6339
6340 Register OrigAddr = Addr;
6341 if (ConstOffset != 0 &&
6342 TII.isLegalFLATOffset(Offset: ConstOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
6343 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
6344 Addr = PtrBase;
6345 ImmOffset = ConstOffset;
6346 }
6347
6348 auto AddrDef = getDefSrcRegIgnoringCopies(Reg: Addr, MRI: *MRI);
6349 if (AddrDef->MI->getOpcode() != AMDGPU::G_PTR_ADD)
6350 return std::nullopt;
6351
6352 Register RHS = AddrDef->MI->getOperand(i: 2).getReg();
6353 if (RBI.getRegBank(Reg: RHS, MRI: *MRI, TRI)->getID() != AMDGPU::VGPRRegBankID)
6354 return std::nullopt;
6355
6356 Register LHS = AddrDef->MI->getOperand(i: 1).getReg();
6357 auto LHSDef = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI);
6358
6359 if (OrigAddr != Addr) {
6360 if (!isFlatScratchBaseLegalSVImm(Addr: OrigAddr))
6361 return std::nullopt;
6362 } else {
6363 if (!isFlatScratchBaseLegalSV(Addr: OrigAddr))
6364 return std::nullopt;
6365 }
6366
6367 if (checkFlatScratchSVSSwizzleBug(VAddr: RHS, SAddr: LHS, ImmOffset))
6368 return std::nullopt;
6369
6370 unsigned CPol = selectScaleOffset(Root, Offset&: RHS, IsSigned: true /* IsSigned */)
6371 ? AMDGPU::CPol::SCAL
6372 : 0;
6373
6374 if (LHSDef->MI->getOpcode() == AMDGPU::G_FRAME_INDEX) {
6375 int FI = LHSDef->MI->getOperand(i: 1).getIndex();
6376 return {{
6377 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6378 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(Idx: FI); }, // saddr
6379 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6380 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6381 }};
6382 }
6383
6384 if (!isSGPR(Reg: LHS))
6385 if (auto Def = getDefSrcRegIgnoringCopies(Reg: LHS, MRI: *MRI))
6386 LHS = Def->Reg;
6387
6388 if (!isSGPR(Reg: LHS))
6389 return std::nullopt;
6390
6391 return {{
6392 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: RHS); }, // vaddr
6393 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: LHS); }, // saddr
6394 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: ImmOffset); }, // offset
6395 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: CPol); } // cpol
6396 }};
6397}
6398
6399InstructionSelector::ComplexRendererFns
6400AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
6401 MachineInstr *MI = Root.getParent();
6402 MachineBasicBlock *MBB = MI->getParent();
6403 MachineFunction *MF = MBB->getParent();
6404 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6405
6406 int64_t Offset = 0;
6407 if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) &&
6408 Offset != AMDGPU::getNullPointerValue(AS: AMDGPUAS::PRIVATE_ADDRESS)) {
6409 Register HighBits = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6410
6411 // TODO: Should this be inside the render function? The iterator seems to
6412 // move.
6413 const int64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
6414 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: TII.get(Opcode: AMDGPU::V_MOV_B32_e32),
6415 DestReg: HighBits)
6416 .addImm(Val: Offset & ~MaxOffset);
6417
6418 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6419 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6420 },
6421 [=](MachineInstrBuilder &MIB) { // vaddr
6422 MIB.addReg(RegNo: HighBits);
6423 },
6424 [=](MachineInstrBuilder &MIB) { // soffset
6425 // Use constant zero for soffset and rely on eliminateFrameIndex
6426 // to choose the appropriate frame register if need be.
6427 MIB.addImm(Val: 0);
6428 },
6429 [=](MachineInstrBuilder &MIB) { // offset
6430 MIB.addImm(Val: Offset & MaxOffset);
6431 }}};
6432 }
6433
6434 assert(Offset == 0 || Offset == -1);
6435
6436 // Try to fold a frame index directly into the MUBUF vaddr field, and any
6437 // offsets.
6438 std::optional<int> FI;
6439 Register VAddr = Root.getReg();
6440
6441 Register PtrBase;
6442 int64_t ConstOffset;
6443 std::tie(args&: PtrBase, args&: ConstOffset, args: std::ignore) =
6444 getPtrBaseWithConstantOffset(Root: VAddr, MRI: *MRI);
6445 int MatchedFI;
6446 if (ConstOffset != 0) {
6447 if (TII.isLegalMUBUFImmOffset(Imm: ConstOffset) &&
6448 (!STI.privateMemoryResourceIsRangeChecked() ||
6449 VT->signBitIsZero(Op: PtrBase))) {
6450 if (mi_match(R: PtrBase, MRI: *MRI, P: m_GFrameIndex(FI&: MatchedFI)))
6451 FI = MatchedFI;
6452 else
6453 VAddr = PtrBase;
6454 Offset = ConstOffset;
6455 }
6456 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GFrameIndex(FI&: MatchedFI))) {
6457 FI = MatchedFI;
6458 }
6459
6460 return {{[=](MachineInstrBuilder &MIB) { // rsrc
6461 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6462 },
6463 [=](MachineInstrBuilder &MIB) { // vaddr
6464 if (FI)
6465 MIB.addFrameIndex(Idx: *FI);
6466 else
6467 MIB.addReg(RegNo: VAddr);
6468 },
6469 [=](MachineInstrBuilder &MIB) { // soffset
6470 // Use constant zero for soffset and rely on eliminateFrameIndex
6471 // to choose the appropriate frame register if need be.
6472 MIB.addImm(Val: 0);
6473 },
6474 [=](MachineInstrBuilder &MIB) { // offset
6475 MIB.addImm(Val: Offset);
6476 }}};
6477}
6478
6479bool AMDGPUInstructionSelector::isDSOffsetLegal(Register Base,
6480 int64_t Offset) const {
6481 if (!isUInt<16>(x: Offset))
6482 return false;
6483
6484 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6485 return true;
6486
6487 // On Southern Islands instruction with a negative base value and an offset
6488 // don't seem to work.
6489 return VT->signBitIsZero(Op: Base);
6490}
6491
6492bool AMDGPUInstructionSelector::isDSOffset2Legal(Register Base, int64_t Offset0,
6493 int64_t Offset1,
6494 unsigned Size) const {
6495 if (Offset0 % Size != 0 || Offset1 % Size != 0)
6496 return false;
6497 if (!isUInt<8>(x: Offset0 / Size) || !isUInt<8>(x: Offset1 / Size))
6498 return false;
6499
6500 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled())
6501 return true;
6502
6503 // On Southern Islands instruction with a negative base value and an offset
6504 // don't seem to work.
6505 return VT->signBitIsZero(Op: Base);
6506}
6507
6508// Return whether the operation has NoUnsignedWrap property.
6509static bool isNoUnsignedWrap(MachineInstr *Addr) {
6510 return Addr->getOpcode() == TargetOpcode::G_OR ||
6511 (Addr->getOpcode() == TargetOpcode::G_PTR_ADD &&
6512 Addr->getFlag(Flag: MachineInstr::NoUWrap));
6513}
6514
6515// Check that the base address of flat scratch load/store in the form of `base +
6516// offset` is legal to be put in SGPR/VGPR (i.e. unsigned per hardware
6517// requirement). We always treat the first operand as the base address here.
6518bool AMDGPUInstructionSelector::isFlatScratchBaseLegal(Register Addr) const {
6519 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6520
6521 if (isNoUnsignedWrap(Addr: AddrMI))
6522 return true;
6523
6524 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6525 // values.
6526 if (STI.hasSignedScratchOffsets())
6527 return true;
6528
6529 Register LHS = AddrMI->getOperand(i: 1).getReg();
6530 Register RHS = AddrMI->getOperand(i: 2).getReg();
6531
6532 if (AddrMI->getOpcode() == TargetOpcode::G_PTR_ADD) {
6533 std::optional<ValueAndVReg> RhsValReg =
6534 getIConstantVRegValWithLookThrough(VReg: RHS, MRI: *MRI);
6535 // If the immediate offset is negative and within certain range, the base
6536 // address cannot also be negative. If the base is also negative, the sum
6537 // would be either negative or much larger than the valid range of scratch
6538 // memory a thread can access.
6539 if (RhsValReg && RhsValReg->Value.getSExtValue() < 0 &&
6540 RhsValReg->Value.getSExtValue() > -0x40000000)
6541 return true;
6542 }
6543
6544 return VT->signBitIsZero(Op: LHS);
6545}
6546
6547// Check address value in SGPR/VGPR are legal for flat scratch in the form
6548// of: SGPR + VGPR.
6549bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSV(Register Addr) const {
6550 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6551
6552 if (isNoUnsignedWrap(Addr: AddrMI))
6553 return true;
6554
6555 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6556 // values.
6557 if (STI.hasSignedScratchOffsets())
6558 return true;
6559
6560 Register LHS = AddrMI->getOperand(i: 1).getReg();
6561 Register RHS = AddrMI->getOperand(i: 2).getReg();
6562 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6563}
6564
6565// Check address value in SGPR/VGPR are legal for flat scratch in the form
6566// of: SGPR + VGPR + Imm.
6567bool AMDGPUInstructionSelector::isFlatScratchBaseLegalSVImm(
6568 Register Addr) const {
6569 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
6570 // values.
6571 if (STI.hasSignedScratchOffsets())
6572 return true;
6573
6574 MachineInstr *AddrMI = getDefIgnoringCopies(Reg: Addr, MRI: *MRI);
6575 Register Base = AddrMI->getOperand(i: 1).getReg();
6576 std::optional<DefinitionAndSourceRegister> BaseDef =
6577 getDefSrcRegIgnoringCopies(Reg: Base, MRI: *MRI);
6578 std::optional<ValueAndVReg> RHSOffset =
6579 getIConstantVRegValWithLookThrough(VReg: AddrMI->getOperand(i: 2).getReg(), MRI: *MRI);
6580 assert(RHSOffset);
6581
6582 // If the immediate offset is negative and within certain range, the base
6583 // address cannot also be negative. If the base is also negative, the sum
6584 // would be either negative or much larger than the valid range of scratch
6585 // memory a thread can access.
6586 if (isNoUnsignedWrap(Addr: BaseDef->MI) &&
6587 (isNoUnsignedWrap(Addr: AddrMI) ||
6588 (RHSOffset->Value.getSExtValue() < 0 &&
6589 RHSOffset->Value.getSExtValue() > -0x40000000)))
6590 return true;
6591
6592 Register LHS = BaseDef->MI->getOperand(i: 1).getReg();
6593 Register RHS = BaseDef->MI->getOperand(i: 2).getReg();
6594 return VT->signBitIsZero(Op: RHS) && VT->signBitIsZero(Op: LHS);
6595}
6596
6597bool AMDGPUInstructionSelector::isUnneededShiftMask(const MachineInstr &MI,
6598 unsigned ShAmtBits) const {
6599 assert(MI.getOpcode() == TargetOpcode::G_AND);
6600
6601 std::optional<APInt> RHS =
6602 getIConstantVRegVal(VReg: MI.getOperand(i: 2).getReg(), MRI: *MRI);
6603 if (!RHS)
6604 return false;
6605
6606 if (RHS->countr_one() >= ShAmtBits)
6607 return true;
6608
6609 const APInt &LHSKnownZeros = VT->getKnownZeroes(R: MI.getOperand(i: 1).getReg());
6610 return (LHSKnownZeros | *RHS).countr_one() >= ShAmtBits;
6611}
6612
6613InstructionSelector::ComplexRendererFns
6614AMDGPUInstructionSelector::selectMUBUFScratchOffset(
6615 MachineOperand &Root) const {
6616 Register Reg = Root.getReg();
6617 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
6618
6619 std::optional<DefinitionAndSourceRegister> Def =
6620 getDefSrcRegIgnoringCopies(Reg, MRI: *MRI);
6621 assert(Def && "this shouldn't be an optional result");
6622 Reg = Def->Reg;
6623
6624 if (Register WaveBase = getWaveAddress(Def: Def->MI)) {
6625 return {{
6626 [=](MachineInstrBuilder &MIB) { // rsrc
6627 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6628 },
6629 [=](MachineInstrBuilder &MIB) { // soffset
6630 MIB.addReg(RegNo: WaveBase);
6631 },
6632 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: 0); } // offset
6633 }};
6634 }
6635
6636 int64_t Offset = 0;
6637
6638 // FIXME: Copy check is a hack
6639 Register BasePtr;
6640 if (mi_match(R: Reg, MRI: *MRI,
6641 P: m_GPtrAdd(L: m_Reg(R&: BasePtr),
6642 R: m_any_of(preds: m_ICst(Cst&: Offset), preds: m_Copy(Src: m_ICst(Cst&: Offset)))))) {
6643 if (!TII.isLegalMUBUFImmOffset(Imm: Offset))
6644 return {};
6645 MachineInstr *BasePtrDef = getDefIgnoringCopies(Reg: BasePtr, MRI: *MRI);
6646 Register WaveBase = getWaveAddress(Def: BasePtrDef);
6647 if (!WaveBase)
6648 return {};
6649
6650 return {{
6651 [=](MachineInstrBuilder &MIB) { // rsrc
6652 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6653 },
6654 [=](MachineInstrBuilder &MIB) { // soffset
6655 MIB.addReg(RegNo: WaveBase);
6656 },
6657 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6658 }};
6659 }
6660
6661 if (!mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: Offset)) ||
6662 !TII.isLegalMUBUFImmOffset(Imm: Offset))
6663 return {};
6664
6665 return {{
6666 [=](MachineInstrBuilder &MIB) { // rsrc
6667 MIB.addReg(RegNo: Info->getScratchRSrcReg());
6668 },
6669 [=](MachineInstrBuilder &MIB) { // soffset
6670 MIB.addImm(Val: 0);
6671 },
6672 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); } // offset
6673 }};
6674}
6675
6676std::pair<Register, unsigned>
6677AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(
6678 MachineOperand &Root) const {
6679 int64_t ConstAddr = 0;
6680
6681 Register PtrBase;
6682 int64_t Offset;
6683 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6684 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6685
6686 if (Offset) {
6687 if (isDSOffsetLegal(Base: PtrBase, Offset)) {
6688 // (add n0, c0)
6689 return std::pair(PtrBase, Offset);
6690 }
6691 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GSub(L: m_Reg(), R: m_Reg()))) {
6692 // TODO
6693
6694 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6695 // TODO
6696 }
6697
6698 return std::pair(Root.getReg(), 0);
6699}
6700
6701InstructionSelector::ComplexRendererFns
6702AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
6703 Register Reg;
6704 unsigned Offset;
6705 std::tie(args&: Reg, args&: Offset) = selectDS1Addr1OffsetImpl(Root);
6706 return {{
6707 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6708 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }
6709 }};
6710}
6711
6712InstructionSelector::ComplexRendererFns
6713AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const {
6714 return selectDSReadWrite2(Root, size: 4);
6715}
6716
6717InstructionSelector::ComplexRendererFns
6718AMDGPUInstructionSelector::selectDS128Bit8ByteAligned(MachineOperand &Root) const {
6719 return selectDSReadWrite2(Root, size: 8);
6720}
6721
6722InstructionSelector::ComplexRendererFns
6723AMDGPUInstructionSelector::selectDSReadWrite2(MachineOperand &Root,
6724 unsigned Size) const {
6725 Register Reg;
6726 unsigned Offset;
6727 std::tie(args&: Reg, args&: Offset) = selectDSReadWrite2Impl(Root, size: Size);
6728 return {{
6729 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Reg); },
6730 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); },
6731 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset+1); }
6732 }};
6733}
6734
6735std::pair<Register, unsigned>
6736AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root,
6737 unsigned Size) const {
6738 int64_t ConstAddr = 0;
6739
6740 Register PtrBase;
6741 int64_t Offset;
6742 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6743 getPtrBaseWithConstantOffset(Root: Root.getReg(), MRI: *MRI);
6744
6745 if (Offset) {
6746 int64_t OffsetValue0 = Offset;
6747 int64_t OffsetValue1 = Offset + Size;
6748 if (isDSOffset2Legal(Base: PtrBase, Offset0: OffsetValue0, Offset1: OffsetValue1, Size)) {
6749 // (add n0, c0)
6750 return std::pair(PtrBase, OffsetValue0 / Size);
6751 }
6752 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_GSub(L: m_Reg(), R: m_Reg()))) {
6753 // TODO
6754
6755 } else if (mi_match(R: Root.getReg(), MRI: *MRI, P: m_ICst(Cst&: ConstAddr))) {
6756 // TODO
6757 }
6758
6759 return std::pair(Root.getReg(), 0);
6760}
6761
6762/// If \p Root is a G_PTR_ADD with a G_CONSTANT on the right hand side, return
6763/// the base value with the constant offset, and if the offset computation is
6764/// known to be inbounds. There may be intervening copies between \p Root and
6765/// the identified constant. Returns \p Root, 0, false if this does not match
6766/// the pattern.
6767std::tuple<Register, int64_t, bool>
6768AMDGPUInstructionSelector::getPtrBaseWithConstantOffset(
6769 Register Root, const MachineRegisterInfo &MRI) const {
6770 MachineInstr *RootI = getDefIgnoringCopies(Reg: Root, MRI);
6771 if (RootI->getOpcode() != TargetOpcode::G_PTR_ADD)
6772 return {Root, 0, false};
6773
6774 MachineOperand &RHS = RootI->getOperand(i: 2);
6775 std::optional<ValueAndVReg> MaybeOffset =
6776 getIConstantVRegValWithLookThrough(VReg: RHS.getReg(), MRI);
6777 if (!MaybeOffset)
6778 return {Root, 0, false};
6779 bool IsInBounds = RootI->getFlag(Flag: MachineInstr::MIFlag::InBounds);
6780 return {RootI->getOperand(i: 1).getReg(), MaybeOffset->Value.getSExtValue(),
6781 IsInBounds};
6782}
6783
6784static void addZeroImm(MachineInstrBuilder &MIB) {
6785 MIB.addImm(Val: 0);
6786}
6787
6788/// Return a resource descriptor for use with an arbitrary 64-bit pointer. If \p
6789/// BasePtr is not valid, a null base pointer will be used.
6790static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6791 uint32_t FormatLo, uint32_t FormatHi,
6792 Register BasePtr) {
6793 Register RSrc2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6794 Register RSrc3 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6795 Register RSrcHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6796 Register RSrc = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_128RegClass);
6797
6798 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6799 .addDef(RegNo: RSrc2)
6800 .addImm(Val: FormatLo);
6801 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6802 .addDef(RegNo: RSrc3)
6803 .addImm(Val: FormatHi);
6804
6805 // Build the half of the subregister with the constants before building the
6806 // full 128-bit register. If we are building multiple resource descriptors,
6807 // this will allow CSEing of the 2-component register.
6808 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6809 .addDef(RegNo: RSrcHi)
6810 .addReg(RegNo: RSrc2)
6811 .addImm(Val: AMDGPU::sub0)
6812 .addReg(RegNo: RSrc3)
6813 .addImm(Val: AMDGPU::sub1);
6814
6815 Register RSrcLo = BasePtr;
6816 if (!BasePtr) {
6817 RSrcLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6818 B.buildInstr(Opcode: AMDGPU::S_MOV_B64)
6819 .addDef(RegNo: RSrcLo)
6820 .addImm(Val: 0);
6821 }
6822
6823 B.buildInstr(Opcode: AMDGPU::REG_SEQUENCE)
6824 .addDef(RegNo: RSrc)
6825 .addReg(RegNo: RSrcLo)
6826 .addImm(Val: AMDGPU::sub0_sub1)
6827 .addReg(RegNo: RSrcHi)
6828 .addImm(Val: AMDGPU::sub2_sub3);
6829
6830 return RSrc;
6831}
6832
6833static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6834 const SIInstrInfo &TII, Register BasePtr) {
6835 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
6836
6837 // FIXME: Why are half the "default" bits ignored based on the addressing
6838 // mode?
6839 return buildRSRC(B, MRI, FormatLo: 0, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
6840}
6841
6842static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI,
6843 const SIInstrInfo &TII, Register BasePtr) {
6844 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat();
6845
6846 // FIXME: Why are half the "default" bits ignored based on the addressing
6847 // mode?
6848 return buildRSRC(B, MRI, FormatLo: -1, FormatHi: Hi_32(Value: DefaultFormat), BasePtr);
6849}
6850
6851AMDGPUInstructionSelector::MUBUFAddressData
6852AMDGPUInstructionSelector::parseMUBUFAddress(Register Src) const {
6853 MUBUFAddressData Data;
6854 Data.N0 = Src;
6855
6856 Register PtrBase;
6857 int64_t Offset;
6858
6859 std::tie(args&: PtrBase, args&: Offset, args: std::ignore) =
6860 getPtrBaseWithConstantOffset(Root: Src, MRI: *MRI);
6861 if (isUInt<32>(x: Offset)) {
6862 Data.N0 = PtrBase;
6863 Data.Offset = Offset;
6864 }
6865
6866 if (MachineInstr *InputAdd
6867 = getOpcodeDef(Opcode: TargetOpcode::G_PTR_ADD, Reg: Data.N0, MRI: *MRI)) {
6868 Data.N2 = InputAdd->getOperand(i: 1).getReg();
6869 Data.N3 = InputAdd->getOperand(i: 2).getReg();
6870
6871 // FIXME: Need to fix extra SGPR->VGPRcopies inserted
6872 // FIXME: Don't know this was defined by operand 0
6873 //
6874 // TODO: Remove this when we have copy folding optimizations after
6875 // RegBankSelect.
6876 Data.N2 = getDefIgnoringCopies(Reg: Data.N2, MRI: *MRI)->getOperand(i: 0).getReg();
6877 Data.N3 = getDefIgnoringCopies(Reg: Data.N3, MRI: *MRI)->getOperand(i: 0).getReg();
6878 }
6879
6880 return Data;
6881}
6882
6883/// Return if the addr64 mubuf mode should be used for the given address.
6884bool AMDGPUInstructionSelector::shouldUseAddr64(MUBUFAddressData Addr) const {
6885 // (ptr_add N2, N3) -> addr64, or
6886 // (ptr_add (ptr_add N2, N3), C1) -> addr64
6887 if (Addr.N2)
6888 return true;
6889
6890 const RegisterBank *N0Bank = RBI.getRegBank(Reg: Addr.N0, MRI: *MRI, TRI);
6891 return N0Bank->getID() == AMDGPU::VGPRRegBankID;
6892}
6893
6894/// Split an immediate offset \p ImmOffset depending on whether it fits in the
6895/// immediate field. Modifies \p ImmOffset and sets \p SOffset to the variable
6896/// component.
6897void AMDGPUInstructionSelector::splitIllegalMUBUFOffset(
6898 MachineIRBuilder &B, Register &SOffset, int64_t &ImmOffset) const {
6899 if (TII.isLegalMUBUFImmOffset(Imm: ImmOffset))
6900 return;
6901
6902 // Illegal offset, store it in soffset.
6903 SOffset = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6904 B.buildInstr(Opcode: AMDGPU::S_MOV_B32)
6905 .addDef(RegNo: SOffset)
6906 .addImm(Val: ImmOffset);
6907 ImmOffset = 0;
6908}
6909
6910bool AMDGPUInstructionSelector::selectMUBUFAddr64Impl(
6911 MachineOperand &Root, Register &VAddr, Register &RSrcReg,
6912 Register &SOffset, int64_t &Offset) const {
6913 // FIXME: Predicates should stop this from reaching here.
6914 // addr64 bit was removed for volcanic islands.
6915 if (!STI.hasAddr64() || STI.useFlatForGlobal())
6916 return false;
6917
6918 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
6919 if (!shouldUseAddr64(Addr: AddrData))
6920 return false;
6921
6922 Register N0 = AddrData.N0;
6923 Register N2 = AddrData.N2;
6924 Register N3 = AddrData.N3;
6925 Offset = AddrData.Offset;
6926
6927 // Base pointer for the SRD.
6928 Register SRDPtr;
6929
6930 if (N2) {
6931 if (RBI.getRegBank(Reg: N2, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
6932 assert(N3);
6933 if (RBI.getRegBank(Reg: N3, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
6934 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
6935 // addr64, and construct the default resource from a 0 address.
6936 VAddr = N0;
6937 } else {
6938 SRDPtr = N3;
6939 VAddr = N2;
6940 }
6941 } else {
6942 // N2 is not divergent.
6943 SRDPtr = N2;
6944 VAddr = N3;
6945 }
6946 } else if (RBI.getRegBank(Reg: N0, MRI: *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) {
6947 // Use the default null pointer in the resource
6948 VAddr = N0;
6949 } else {
6950 // N0 -> offset, or
6951 // (N0 + C1) -> offset
6952 SRDPtr = N0;
6953 }
6954
6955 MachineIRBuilder B(*Root.getParent());
6956 RSrcReg = buildAddr64RSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
6957 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
6958 return true;
6959}
6960
6961bool AMDGPUInstructionSelector::selectMUBUFOffsetImpl(
6962 MachineOperand &Root, Register &RSrcReg, Register &SOffset,
6963 int64_t &Offset) const {
6964
6965 // FIXME: Pattern should not reach here.
6966 if (STI.useFlatForGlobal())
6967 return false;
6968
6969 MUBUFAddressData AddrData = parseMUBUFAddress(Src: Root.getReg());
6970 if (shouldUseAddr64(Addr: AddrData))
6971 return false;
6972
6973 // N0 -> offset, or
6974 // (N0 + C1) -> offset
6975 Register SRDPtr = AddrData.N0;
6976 Offset = AddrData.Offset;
6977
6978 // TODO: Look through extensions for 32-bit soffset.
6979 MachineIRBuilder B(*Root.getParent());
6980
6981 RSrcReg = buildOffsetSrc(B, MRI&: *MRI, TII, BasePtr: SRDPtr);
6982 splitIllegalMUBUFOffset(B, SOffset, ImmOffset&: Offset);
6983 return true;
6984}
6985
6986InstructionSelector::ComplexRendererFns
6987AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const {
6988 Register VAddr;
6989 Register RSrcReg;
6990 Register SOffset;
6991 int64_t Offset = 0;
6992
6993 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset))
6994 return {};
6995
6996 // FIXME: Use defaulted operands for trailing 0s and remove from the complex
6997 // pattern.
6998 return {{
6999 [=](MachineInstrBuilder &MIB) { // rsrc
7000 MIB.addReg(RegNo: RSrcReg);
7001 },
7002 [=](MachineInstrBuilder &MIB) { // vaddr
7003 MIB.addReg(RegNo: VAddr);
7004 },
7005 [=](MachineInstrBuilder &MIB) { // soffset
7006 if (SOffset)
7007 MIB.addReg(RegNo: SOffset);
7008 else if (STI.hasRestrictedSOffset())
7009 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7010 else
7011 MIB.addImm(Val: 0);
7012 },
7013 [=](MachineInstrBuilder &MIB) { // offset
7014 MIB.addImm(Val: Offset);
7015 },
7016 addZeroImm, // cpol
7017 addZeroImm, // tfe
7018 addZeroImm // swz
7019 }};
7020}
7021
7022InstructionSelector::ComplexRendererFns
7023AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const {
7024 Register RSrcReg;
7025 Register SOffset;
7026 int64_t Offset = 0;
7027
7028 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset))
7029 return {};
7030
7031 return {{
7032 [=](MachineInstrBuilder &MIB) { // rsrc
7033 MIB.addReg(RegNo: RSrcReg);
7034 },
7035 [=](MachineInstrBuilder &MIB) { // soffset
7036 if (SOffset)
7037 MIB.addReg(RegNo: SOffset);
7038 else if (STI.hasRestrictedSOffset())
7039 MIB.addReg(RegNo: AMDGPU::SGPR_NULL);
7040 else
7041 MIB.addImm(Val: 0);
7042 },
7043 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Offset); }, // offset
7044 addZeroImm, // cpol
7045 addZeroImm, // tfe
7046 addZeroImm, // swz
7047 }};
7048}
7049
7050InstructionSelector::ComplexRendererFns
7051AMDGPUInstructionSelector::selectBUFSOffset(MachineOperand &Root) const {
7052
7053 Register SOffset = Root.getReg();
7054
7055 if (STI.hasRestrictedSOffset() && mi_match(R: SOffset, MRI: *MRI, P: m_ZeroInt()))
7056 SOffset = AMDGPU::SGPR_NULL;
7057
7058 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); }}};
7059}
7060
7061/// Get an immediate that must be 32-bits, and treated as zero extended.
7062static std::optional<uint64_t>
7063getConstantZext32Val(Register Reg, const MachineRegisterInfo &MRI) {
7064 // getIConstantVRegVal sexts any values, so see if that matters.
7065 std::optional<int64_t> OffsetVal = getIConstantVRegSExtVal(VReg: Reg, MRI);
7066 if (!OffsetVal || !isInt<32>(x: *OffsetVal))
7067 return std::nullopt;
7068 return Lo_32(Value: *OffsetVal);
7069}
7070
7071InstructionSelector::ComplexRendererFns
7072AMDGPUInstructionSelector::selectSMRDBufferImm(MachineOperand &Root) const {
7073 std::optional<uint64_t> OffsetVal =
7074 Root.isImm() ? Root.getImm() : getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7075 if (!OffsetVal)
7076 return {};
7077
7078 std::optional<int64_t> EncodedImm =
7079 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: *OffsetVal, IsBuffer: true);
7080 if (!EncodedImm)
7081 return {};
7082
7083 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7084}
7085
7086InstructionSelector::ComplexRendererFns
7087AMDGPUInstructionSelector::selectSMRDBufferImm32(MachineOperand &Root) const {
7088 assert(STI.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
7089
7090 std::optional<uint64_t> OffsetVal = getConstantZext32Val(Reg: Root.getReg(), MRI: *MRI);
7091 if (!OffsetVal)
7092 return {};
7093
7094 std::optional<int64_t> EncodedImm =
7095 AMDGPU::getSMRDEncodedLiteralOffset32(ST: STI, ByteOffset: *OffsetVal);
7096 if (!EncodedImm)
7097 return {};
7098
7099 return {{ [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedImm); } }};
7100}
7101
7102InstructionSelector::ComplexRendererFns
7103AMDGPUInstructionSelector::selectSMRDBufferSgprImm(MachineOperand &Root) const {
7104 // Match the (soffset + offset) pair as a 32-bit register base and
7105 // an immediate offset.
7106 Register SOffset;
7107 unsigned Offset;
7108 std::tie(args&: SOffset, args&: Offset) = AMDGPU::getBaseWithConstantOffset(
7109 MRI&: *MRI, Reg: Root.getReg(), ValueTracking: VT, /*CheckNUW*/ true);
7110 if (!SOffset)
7111 return std::nullopt;
7112
7113 std::optional<int64_t> EncodedOffset =
7114 AMDGPU::getSMRDEncodedOffset(ST: STI, ByteOffset: Offset, /* IsBuffer */ true);
7115 if (!EncodedOffset)
7116 return std::nullopt;
7117
7118 assert(MRI->getType(SOffset).getSizeInBits() == 32);
7119 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: SOffset); },
7120 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: *EncodedOffset); }}};
7121}
7122
7123std::pair<Register, unsigned>
7124AMDGPUInstructionSelector::selectVOP3PMadMixModsImpl(MachineOperand &Root,
7125 bool &Matched) const {
7126 Matched = false;
7127
7128 Register Src;
7129 unsigned Mods;
7130 std::tie(args&: Src, args&: Mods) = selectVOP3ModsImpl(Src: Root.getReg());
7131
7132 if (mi_match(R: Src, MRI: *MRI, P: m_GFPExt(Src: m_Reg(R&: Src)))) {
7133 assert(MRI->getType(Src) == LLT::scalar(16));
7134
7135 // Only change Src if src modifier could be gained. In such cases new Src
7136 // could be sgpr but this does not violate constant bus restriction for
7137 // instruction that is being selected.
7138 Src = stripBitCast(Reg: Src, MRI&: *MRI);
7139
7140 const auto CheckAbsNeg = [&]() {
7141 // Be careful about folding modifiers if we already have an abs. fneg is
7142 // applied last, so we don't want to apply an earlier fneg.
7143 if ((Mods & SISrcMods::ABS) == 0) {
7144 unsigned ModsTmp;
7145 std::tie(args&: Src, args&: ModsTmp) = selectVOP3ModsImpl(Src);
7146
7147 if ((ModsTmp & SISrcMods::NEG) != 0)
7148 Mods ^= SISrcMods::NEG;
7149
7150 if ((ModsTmp & SISrcMods::ABS) != 0)
7151 Mods |= SISrcMods::ABS;
7152 }
7153 };
7154
7155 CheckAbsNeg();
7156
7157 // op_sel/op_sel_hi decide the source type and source.
7158 // If the source's op_sel_hi is set, it indicates to do a conversion from
7159 // fp16. If the sources's op_sel is set, it picks the high half of the
7160 // source register.
7161
7162 Mods |= SISrcMods::OP_SEL_1;
7163
7164 if (isExtractHiElt(MRI&: *MRI, In: Src, Out&: Src)) {
7165 Mods |= SISrcMods::OP_SEL_0;
7166 CheckAbsNeg();
7167 }
7168
7169 Matched = true;
7170 }
7171
7172 return {Src, Mods};
7173}
7174
7175InstructionSelector::ComplexRendererFns
7176AMDGPUInstructionSelector::selectVOP3PMadMixModsExt(
7177 MachineOperand &Root) const {
7178 Register Src;
7179 unsigned Mods;
7180 bool Matched;
7181 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7182 if (!Matched)
7183 return {};
7184
7185 return {{
7186 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
7187 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7188 }};
7189}
7190
7191InstructionSelector::ComplexRendererFns
7192AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const {
7193 Register Src;
7194 unsigned Mods;
7195 bool Matched;
7196 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7197
7198 return {{
7199 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
7200 [=](MachineInstrBuilder &MIB) { MIB.addImm(Val: Mods); } // src_mods
7201 }};
7202}
7203
7204InstructionSelector::ComplexRendererFns
7205AMDGPUInstructionSelector::selectVOP3PMadMixModsExtNeg(
7206 MachineOperand &Root) const {
7207 Register Src;
7208 unsigned Mods;
7209 bool Matched;
7210 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7211 if (!Matched)
7212 return {};
7213
7214 return {{
7215 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
7216 [=](MachineInstrBuilder &MIB) {
7217 MIB.addImm(Val: Mods ^ SISrcMods::NEG);
7218 } // src_mods
7219 }};
7220}
7221
7222InstructionSelector::ComplexRendererFns
7223AMDGPUInstructionSelector::selectVOP3PMadMixModsNeg(
7224 MachineOperand &Root) const {
7225 Register Src;
7226 unsigned Mods;
7227 bool Matched;
7228 std::tie(args&: Src, args&: Mods) = selectVOP3PMadMixModsImpl(Root, Matched);
7229
7230 return {{
7231 [=](MachineInstrBuilder &MIB) { MIB.addReg(RegNo: Src); },
7232 [=](MachineInstrBuilder &MIB) {
7233 MIB.addImm(Val: Mods ^ SISrcMods::NEG);
7234 } // src_mods
7235 }};
7236}
7237
7238bool AMDGPUInstructionSelector::selectSBarrierSignalIsfirst(
7239 MachineInstr &I, Intrinsic::ID IntrID) const {
7240 MachineBasicBlock *MBB = I.getParent();
7241 const DebugLoc &DL = I.getDebugLoc();
7242 Register CCReg = I.getOperand(i: 0).getReg();
7243
7244 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7245 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_CMP_EQ_U32)).addImm(Val: 0).addImm(Val: 0);
7246
7247 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM))
7248 .addImm(Val: I.getOperand(i: 2).getImm());
7249
7250 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: CCReg).addReg(RegNo: AMDGPU::SCC);
7251
7252 I.eraseFromParent();
7253 return RBI.constrainGenericRegister(Reg: CCReg, RC: AMDGPU::SReg_32_XM0_XEXECRegClass,
7254 MRI&: *MRI);
7255}
7256
7257bool AMDGPUInstructionSelector::selectSGetBarrierState(
7258 MachineInstr &I, Intrinsic::ID IntrID) const {
7259 MachineBasicBlock *MBB = I.getParent();
7260 const DebugLoc &DL = I.getDebugLoc();
7261 const MachineOperand &BarOp = I.getOperand(i: 2);
7262 std::optional<int64_t> BarValImm =
7263 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7264
7265 if (!BarValImm) {
7266 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7267 .addReg(RegNo: BarOp.getReg());
7268 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7269 }
7270 MachineInstrBuilder MIB;
7271 unsigned Opc = BarValImm ? AMDGPU::S_GET_BARRIER_STATE_IMM
7272 : AMDGPU::S_GET_BARRIER_STATE_M0;
7273 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7274
7275 auto DstReg = I.getOperand(i: 0).getReg();
7276 const TargetRegisterClass *DstRC =
7277 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
7278 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7279 return false;
7280 MIB.addDef(RegNo: DstReg);
7281 if (BarValImm) {
7282 MIB.addImm(Val: *BarValImm);
7283 }
7284 I.eraseFromParent();
7285 return true;
7286}
7287
7288unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID) {
7289 if (HasInlineConst) {
7290 switch (IntrID) {
7291 default:
7292 llvm_unreachable("not a named barrier op");
7293 case Intrinsic::amdgcn_s_barrier_join:
7294 return AMDGPU::S_BARRIER_JOIN_IMM;
7295 case Intrinsic::amdgcn_s_wakeup_barrier:
7296 return AMDGPU::S_WAKEUP_BARRIER_IMM;
7297 case Intrinsic::amdgcn_s_get_named_barrier_state:
7298 return AMDGPU::S_GET_BARRIER_STATE_IMM;
7299 };
7300 } else {
7301 switch (IntrID) {
7302 default:
7303 llvm_unreachable("not a named barrier op");
7304 case Intrinsic::amdgcn_s_barrier_join:
7305 return AMDGPU::S_BARRIER_JOIN_M0;
7306 case Intrinsic::amdgcn_s_wakeup_barrier:
7307 return AMDGPU::S_WAKEUP_BARRIER_M0;
7308 case Intrinsic::amdgcn_s_get_named_barrier_state:
7309 return AMDGPU::S_GET_BARRIER_STATE_M0;
7310 };
7311 }
7312}
7313
7314bool AMDGPUInstructionSelector::selectNamedBarrierInit(
7315 MachineInstr &I, Intrinsic::ID IntrID) const {
7316 MachineBasicBlock *MBB = I.getParent();
7317 const DebugLoc &DL = I.getDebugLoc();
7318 const MachineOperand &BarOp = I.getOperand(i: 1);
7319 const MachineOperand &CntOp = I.getOperand(i: 2);
7320
7321 // A member count of 0 means "keep existing member count". That plus a known
7322 // constant value for the barrier ID lets us use the immarg form.
7323 if (IntrID == Intrinsic::amdgcn_s_barrier_signal_var) {
7324 std::optional<int64_t> CntImm =
7325 getIConstantVRegSExtVal(VReg: CntOp.getReg(), MRI: *MRI);
7326 if (CntImm && *CntImm == 0) {
7327 std::optional<int64_t> BarValImm =
7328 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7329 if (BarValImm) {
7330 auto BarID = ((*BarValImm) >> 4) & 0x3F;
7331 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_BARRIER_SIGNAL_IMM))
7332 .addImm(Val: BarID);
7333 I.eraseFromParent();
7334 return true;
7335 }
7336 }
7337 }
7338
7339 // BarID = (BarOp >> 4) & 0x3F
7340 Register TmpReg0 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7341 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: TmpReg0)
7342 .add(MO: BarOp)
7343 .addImm(Val: 4u)
7344 .setOperandDead(3); // Dead scc
7345
7346 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7347 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7348 .addReg(RegNo: TmpReg0)
7349 .addImm(Val: 0x3F)
7350 .setOperandDead(3); // Dead scc
7351
7352 // MO = ((CntOp & 0x3F) << shAmt) | BarID
7353 Register TmpReg2 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7354 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg2)
7355 .add(MO: CntOp)
7356 .addImm(Val: 0x3F)
7357 .setOperandDead(3); // Dead scc
7358
7359 Register TmpReg3 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7360 constexpr unsigned ShAmt = 16;
7361 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHL_B32), DestReg: TmpReg3)
7362 .addReg(RegNo: TmpReg2)
7363 .addImm(Val: ShAmt)
7364 .setOperandDead(3); // Dead scc
7365
7366 Register TmpReg4 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7367 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_OR_B32), DestReg: TmpReg4)
7368 .addReg(RegNo: TmpReg1)
7369 .addReg(RegNo: TmpReg3)
7370 .setOperandDead(3); // Dead scc;
7371
7372 auto CopyMIB =
7373 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0).addReg(RegNo: TmpReg4);
7374 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7375
7376 unsigned Opc = IntrID == Intrinsic::amdgcn_s_barrier_init
7377 ? AMDGPU::S_BARRIER_INIT_M0
7378 : AMDGPU::S_BARRIER_SIGNAL_M0;
7379 MachineInstrBuilder MIB;
7380 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7381
7382 I.eraseFromParent();
7383 return true;
7384}
7385
7386bool AMDGPUInstructionSelector::selectNamedBarrierInst(
7387 MachineInstr &I, Intrinsic::ID IntrID) const {
7388 MachineBasicBlock *MBB = I.getParent();
7389 const DebugLoc &DL = I.getDebugLoc();
7390 MachineOperand BarOp = IntrID == Intrinsic::amdgcn_s_get_named_barrier_state
7391 ? I.getOperand(i: 2)
7392 : I.getOperand(i: 1);
7393 std::optional<int64_t> BarValImm =
7394 getIConstantVRegSExtVal(VReg: BarOp.getReg(), MRI: *MRI);
7395
7396 if (!BarValImm) {
7397 // BarID = (BarOp >> 4) & 0x3F
7398 Register TmpReg0 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7399 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_LSHR_B32), DestReg: TmpReg0)
7400 .addReg(RegNo: BarOp.getReg())
7401 .addImm(Val: 4u)
7402 .setOperandDead(3); // Dead scc;
7403
7404 Register TmpReg1 = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7405 BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::S_AND_B32), DestReg: TmpReg1)
7406 .addReg(RegNo: TmpReg0)
7407 .addImm(Val: 0x3F)
7408 .setOperandDead(3); // Dead scc;
7409
7410 auto CopyMIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
7411 .addReg(RegNo: TmpReg1);
7412 constrainSelectedInstRegOperands(I&: *CopyMIB, TII, TRI, RBI);
7413 }
7414
7415 MachineInstrBuilder MIB;
7416 unsigned Opc = getNamedBarrierOp(HasInlineConst: BarValImm.has_value(), IntrID);
7417 MIB = BuildMI(BB&: *MBB, I: &I, MIMD: DL, MCID: TII.get(Opcode: Opc));
7418
7419 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
7420 auto DstReg = I.getOperand(i: 0).getReg();
7421 const TargetRegisterClass *DstRC =
7422 TRI.getConstrainedRegClassForReg(Reg: DstReg, MRI: *MRI);
7423 if (!DstRC || !RBI.constrainGenericRegister(Reg: DstReg, RC: *DstRC, MRI&: *MRI))
7424 return false;
7425 MIB.addDef(RegNo: DstReg);
7426 }
7427
7428 if (BarValImm) {
7429 auto BarId = ((*BarValImm) >> 4) & 0x3F;
7430 MIB.addImm(Val: BarId);
7431 }
7432
7433 I.eraseFromParent();
7434 return true;
7435}
7436
7437void AMDGPUInstructionSelector::renderTruncImm32(MachineInstrBuilder &MIB,
7438 const MachineInstr &MI,
7439 int OpIdx) const {
7440 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7441 "Expected G_CONSTANT");
7442 MIB.addImm(Val: MI.getOperand(i: 1).getCImm()->getSExtValue());
7443}
7444
7445void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB,
7446 const MachineInstr &MI,
7447 int OpIdx) const {
7448 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7449 "Expected G_CONSTANT");
7450 MIB.addImm(Val: -MI.getOperand(i: 1).getCImm()->getSExtValue());
7451}
7452
7453void AMDGPUInstructionSelector::renderBitcastFPImm(MachineInstrBuilder &MIB,
7454 const MachineInstr &MI,
7455 int OpIdx) const {
7456 const MachineOperand &Op = MI.getOperand(i: 1);
7457 assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1);
7458 MIB.addImm(Val: Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
7459}
7460
7461void AMDGPUInstructionSelector::renderCountTrailingOnesImm(
7462 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7463 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
7464 "Expected G_CONSTANT");
7465 MIB.addImm(Val: MI.getOperand(i: 1).getCImm()->getValue().countTrailingOnes());
7466}
7467
7468/// This only really exists to satisfy DAG type checking machinery, so is a
7469/// no-op here.
7470void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB,
7471 const MachineInstr &MI,
7472 int OpIdx) const {
7473 const MachineOperand &Op = MI.getOperand(i: OpIdx);
7474 int64_t Imm;
7475 if (Op.isReg() && mi_match(R: Op.getReg(), MRI: *MRI, P: m_ICst(Cst&: Imm)))
7476 MIB.addImm(Val: Imm);
7477 else
7478 MIB.addImm(Val: Op.getImm());
7479}
7480
7481void AMDGPUInstructionSelector::renderZextBoolTImm(MachineInstrBuilder &MIB,
7482 const MachineInstr &MI,
7483 int OpIdx) const {
7484 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() != 0);
7485}
7486
7487void AMDGPUInstructionSelector::renderOpSelTImm(MachineInstrBuilder &MIB,
7488 const MachineInstr &MI,
7489 int OpIdx) const {
7490 assert(OpIdx >= 0 && "expected to match an immediate operand");
7491 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7492}
7493
7494void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_0(
7495 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7496 assert(OpIdx >= 0 && "expected to match an immediate operand");
7497 MIB.addImm(
7498 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7499}
7500
7501void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_0_1(
7502 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7503 assert(OpIdx >= 0 && "expected to match an immediate operand");
7504 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x1)
7505 ? (int64_t)(SISrcMods::OP_SEL_0 | SISrcMods::DST_OP_SEL)
7506 : (int64_t)SISrcMods::DST_OP_SEL);
7507}
7508
7509void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_0(
7510 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7511 assert(OpIdx >= 0 && "expected to match an immediate operand");
7512 MIB.addImm(
7513 Val: (MI.getOperand(i: OpIdx).getImm() & 0x2) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7514}
7515
7516void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_1_1(
7517 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7518 assert(OpIdx >= 0 && "expected to match an immediate operand");
7519 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7520 ? (int64_t)(SISrcMods::OP_SEL_0)
7521 : 0);
7522}
7523
7524void AMDGPUInstructionSelector::renderDstSelToOpSelXForm(
7525 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7526 assert(OpIdx >= 0 && "expected to match an immediate operand");
7527 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::DST_OP_SEL)
7528 : 0);
7529}
7530
7531void AMDGPUInstructionSelector::renderSrcSelToOpSelXForm(
7532 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7533 assert(OpIdx >= 0 && "expected to match an immediate operand");
7534 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() ? (int64_t)(SISrcMods::OP_SEL_0)
7535 : 0);
7536}
7537
7538void AMDGPUInstructionSelector::renderSrcAndDstSelToOpSelXForm_2_0(
7539 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7540 assert(OpIdx >= 0 && "expected to match an immediate operand");
7541 MIB.addImm(
7542 Val: (MI.getOperand(i: OpIdx).getImm() & 0x1) ? (int64_t)SISrcMods::OP_SEL_0 : 0);
7543}
7544
7545void AMDGPUInstructionSelector::renderDstSelToOpSel3XFormXForm(
7546 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7547 assert(OpIdx >= 0 && "expected to match an immediate operand");
7548 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() & 0x2)
7549 ? (int64_t)SISrcMods::DST_OP_SEL
7550 : 0);
7551}
7552
7553void AMDGPUInstructionSelector::renderExtractCPol(MachineInstrBuilder &MIB,
7554 const MachineInstr &MI,
7555 int OpIdx) const {
7556 assert(OpIdx >= 0 && "expected to match an immediate operand");
7557 MIB.addImm(Val: MI.getOperand(i: OpIdx).getImm() &
7558 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7559 : AMDGPU::CPol::ALL_pregfx12));
7560}
7561
7562void AMDGPUInstructionSelector::renderExtractSWZ(MachineInstrBuilder &MIB,
7563 const MachineInstr &MI,
7564 int OpIdx) const {
7565 assert(OpIdx >= 0 && "expected to match an immediate operand");
7566 const bool Swizzle = MI.getOperand(i: OpIdx).getImm() &
7567 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::SWZ
7568 : AMDGPU::CPol::SWZ_pregfx12);
7569 MIB.addImm(Val: Swizzle);
7570}
7571
7572void AMDGPUInstructionSelector::renderExtractCpolSetGLC(
7573 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7574 assert(OpIdx >= 0 && "expected to match an immediate operand");
7575 const uint32_t Cpol = MI.getOperand(i: OpIdx).getImm() &
7576 (AMDGPU::isGFX12Plus(STI) ? AMDGPU::CPol::ALL
7577 : AMDGPU::CPol::ALL_pregfx12);
7578 MIB.addImm(Val: Cpol | AMDGPU::CPol::GLC);
7579}
7580
7581void AMDGPUInstructionSelector::renderFPPow2ToExponent(MachineInstrBuilder &MIB,
7582 const MachineInstr &MI,
7583 int OpIdx) const {
7584 const APFloat &APF = MI.getOperand(i: 1).getFPImm()->getValueAPF();
7585 int ExpVal = APF.getExactLog2Abs();
7586 assert(ExpVal != INT_MIN);
7587 MIB.addImm(Val: ExpVal);
7588}
7589
7590void AMDGPUInstructionSelector::renderRoundMode(MachineInstrBuilder &MIB,
7591 const MachineInstr &MI,
7592 int OpIdx) const {
7593 // "round.towardzero" -> TowardZero 0 -> FP_ROUND_ROUND_TO_ZERO 3
7594 // "round.tonearest" -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
7595 // "round.upward" -> TowardPositive 2 -> FP_ROUND_ROUND_TO_INF 1
7596 // "round.downward -> TowardNegative 3 -> FP_ROUND_ROUND_TO_NEGINF 2
7597 MIB.addImm(Val: (MI.getOperand(i: OpIdx).getImm() + 3) % 4);
7598}
7599
7600void AMDGPUInstructionSelector::renderVOP3PModsNeg(MachineInstrBuilder &MIB,
7601 const MachineInstr &MI,
7602 int OpIdx) const {
7603 unsigned Mods = SISrcMods::OP_SEL_1;
7604 if (MI.getOperand(i: OpIdx).getImm())
7605 Mods ^= SISrcMods::NEG;
7606 MIB.addImm(Val: (int64_t)Mods);
7607}
7608
7609void AMDGPUInstructionSelector::renderVOP3PModsNegs(MachineInstrBuilder &MIB,
7610 const MachineInstr &MI,
7611 int OpIdx) const {
7612 unsigned Mods = SISrcMods::OP_SEL_1;
7613 if (MI.getOperand(i: OpIdx).getImm())
7614 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
7615 MIB.addImm(Val: (int64_t)Mods);
7616}
7617
7618void AMDGPUInstructionSelector::renderVOP3PModsNegAbs(MachineInstrBuilder &MIB,
7619 const MachineInstr &MI,
7620 int OpIdx) const {
7621 unsigned Val = MI.getOperand(i: OpIdx).getImm();
7622 unsigned Mods = SISrcMods::OP_SEL_1; // default: none
7623 if (Val == 1) // neg
7624 Mods ^= SISrcMods::NEG;
7625 if (Val == 2) // abs
7626 Mods ^= SISrcMods::ABS;
7627 if (Val == 3) // neg and abs
7628 Mods ^= (SISrcMods::NEG | SISrcMods::ABS);
7629 MIB.addImm(Val: (int64_t)Mods);
7630}
7631
7632void AMDGPUInstructionSelector::renderPrefetchLoc(MachineInstrBuilder &MIB,
7633 const MachineInstr &MI,
7634 int OpIdx) const {
7635 uint32_t V = MI.getOperand(i: 2).getImm();
7636 V = (AMDGPU::CPol::SCOPE_MASK - (V & AMDGPU::CPol::SCOPE_MASK))
7637 << AMDGPU::CPol::SCOPE_SHIFT;
7638 if (!Subtarget->hasSafeCUPrefetch())
7639 V = std::max(a: V, b: (uint32_t)AMDGPU::CPol::SCOPE_SE); // CU scope is unsafe
7640 MIB.addImm(Val: V);
7641}
7642
7643/// Convert from 2-bit value to enum values used for op_sel* source modifiers.
7644void AMDGPUInstructionSelector::renderScaledMAIIntrinsicOperand(
7645 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
7646 unsigned Val = MI.getOperand(i: OpIdx).getImm();
7647 unsigned New = 0;
7648 if (Val & 0x1)
7649 New |= SISrcMods::OP_SEL_0;
7650 if (Val & 0x2)
7651 New |= SISrcMods::OP_SEL_1;
7652 MIB.addImm(Val: New);
7653}
7654
7655bool AMDGPUInstructionSelector::isInlineImmediate(const APInt &Imm) const {
7656 return TII.isInlineConstant(Imm);
7657}
7658
7659bool AMDGPUInstructionSelector::isInlineImmediate(const APFloat &Imm) const {
7660 return TII.isInlineConstant(Imm);
7661}
7662