1//===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//==-----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines an instruction selector for the AMDGPU target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUISelDAGToDAG.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPUSubtarget.h"
18#include "MCTargetDesc/R600MCTargetDesc.h"
19#include "R600RegisterInfo.h"
20#include "SIISelLowering.h"
21#include "SIMachineFunctionInfo.h"
22#include "llvm/Analysis/UniformityAnalysis.h"
23#include "llvm/CodeGen/FunctionLoweringInfo.h"
24#include "llvm/CodeGen/SelectionDAG.h"
25#include "llvm/CodeGen/SelectionDAGISel.h"
26#include "llvm/CodeGen/SelectionDAGNodes.h"
27#include "llvm/IR/IntrinsicsAMDGPU.h"
28#include "llvm/InitializePasses.h"
29#include "llvm/Support/ErrorHandling.h"
30
31#ifdef EXPENSIVE_CHECKS
32#include "llvm/Analysis/LoopInfo.h"
33#include "llvm/IR/Dominators.h"
34#endif
35
36#define DEBUG_TYPE "amdgpu-isel"
37
38using namespace llvm;
39
40//===----------------------------------------------------------------------===//
41// Instruction Selector Implementation
42//===----------------------------------------------------------------------===//
43
44namespace {
45static SDValue stripBitcast(SDValue Val) {
46 return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(i: 0) : Val;
47}
48
49// Figure out if this is really an extract of the high 16-bits of a dword.
50static bool isExtractHiElt(SDValue In, SDValue &Out) {
51 In = stripBitcast(Val: In);
52
53 if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
54 if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(Val: In.getOperand(i: 1))) {
55 if (!Idx->isOne())
56 return false;
57 Out = In.getOperand(i: 0);
58 return true;
59 }
60 }
61
62 if (In.getOpcode() != ISD::TRUNCATE)
63 return false;
64
65 SDValue Srl = In.getOperand(i: 0);
66 if (Srl.getOpcode() == ISD::SRL) {
67 if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Val: Srl.getOperand(i: 1))) {
68 if (ShiftAmt->getZExtValue() == 16) {
69 Out = stripBitcast(Val: Srl.getOperand(i: 0));
70 return true;
71 }
72 }
73 }
74
75 return false;
76}
77
78static SDValue createVOP3PSrc32FromLo16(SDValue Lo, SDValue Src,
79 llvm::SelectionDAG *CurDAG,
80 const GCNSubtarget *Subtarget) {
81 if (!Subtarget->useRealTrue16Insts()) {
82 return Lo;
83 }
84
85 SDValue NewSrc;
86 SDLoc SL(Lo);
87
88 if (Lo->isDivergent()) {
89 SDValue Undef = SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF,
90 dl: SL, VT: Lo.getValueType()),
91 0);
92 const SDValue Ops[] = {
93 CurDAG->getTargetConstant(Val: AMDGPU::VGPR_32RegClassID, DL: SL, VT: MVT::i32), Lo,
94 CurDAG->getTargetConstant(Val: AMDGPU::lo16, DL: SL, VT: MVT::i16), Undef,
95 CurDAG->getTargetConstant(Val: AMDGPU::hi16, DL: SL, VT: MVT::i16)};
96
97 NewSrc = SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: SL,
98 VT: Src.getValueType(), Ops),
99 0);
100 } else {
101 // the S_MOV is needed since the Lo could still be a VGPR16.
102 // With S_MOV, isel insert a "sgpr32 = copy vgpr16" and we reply on
103 // the fixvgpr2sgprcopy pass to legalize it
104 NewSrc = SDValue(
105 CurDAG->getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: SL, VT: Src.getValueType(), Op1: Lo),
106 0);
107 }
108
109 return NewSrc;
110}
111
112// Look through operations that obscure just looking at the low 16-bits of the
113// same register.
114static SDValue stripExtractLoElt(SDValue In) {
115 if (In.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
116 SDValue Idx = In.getOperand(i: 1);
117 if (isNullConstant(V: Idx) && In.getValueSizeInBits() <= 32)
118 return In.getOperand(i: 0);
119 }
120
121 if (In.getOpcode() == ISD::TRUNCATE) {
122 SDValue Src = In.getOperand(i: 0);
123 if (Src.getValueType().getSizeInBits() == 32)
124 return stripBitcast(Val: Src);
125 }
126
127 return In;
128}
129
130static SDValue emitRegSequence(llvm::SelectionDAG &CurDAG, unsigned DstRegClass,
131 EVT DstTy, ArrayRef<SDValue> Elts,
132 ArrayRef<unsigned> SubRegClass,
133 const SDLoc &DL) {
134 assert(Elts.size() == SubRegClass.size() && "array size mismatch");
135 unsigned NumElts = Elts.size();
136 SmallVector<SDValue, 17> Ops(2 * NumElts + 1);
137 Ops[0] = (CurDAG.getTargetConstant(Val: DstRegClass, DL, VT: MVT::i32));
138 for (unsigned i = 0; i < NumElts; ++i) {
139 Ops[2 * i + 1] = Elts[i];
140 Ops[2 * i + 2] = CurDAG.getTargetConstant(Val: SubRegClass[i], DL, VT: MVT::i32);
141 }
142 return SDValue(
143 CurDAG.getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: DstTy, Ops), 0);
144}
145
146} // end anonymous namespace
147
148INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISelLegacy, "amdgpu-isel",
149 "AMDGPU DAG->DAG Pattern Instruction Selection", false,
150 false)
151INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysisLegacy)
152INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
153#ifdef EXPENSIVE_CHECKS
154INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
155INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
156#endif
157INITIALIZE_PASS_END(AMDGPUDAGToDAGISelLegacy, "amdgpu-isel",
158 "AMDGPU DAG->DAG Pattern Instruction Selection", false,
159 false)
160
161/// This pass converts a legalized DAG into a AMDGPU-specific
162// DAG, ready for instruction scheduling.
163FunctionPass *llvm::createAMDGPUISelDag(TargetMachine &TM,
164 CodeGenOptLevel OptLevel) {
165 return new AMDGPUDAGToDAGISelLegacy(TM, OptLevel);
166}
167
168AMDGPUDAGToDAGISel::AMDGPUDAGToDAGISel(TargetMachine &TM,
169 CodeGenOptLevel OptLevel)
170 : SelectionDAGISel(TM, OptLevel) {}
171
172bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
173 Subtarget = &MF.getSubtarget<GCNSubtarget>();
174 Subtarget->checkSubtargetFeatures(F: MF.getFunction());
175 Mode = SIModeRegisterDefaults(MF.getFunction(), *Subtarget);
176 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
177}
178
179bool AMDGPUDAGToDAGISel::fp16SrcZerosHighBits(unsigned Opc) const {
180 // XXX - only need to list legal operations.
181 switch (Opc) {
182 case ISD::POISON:
183 return true;
184 case ISD::FADD:
185 case ISD::FSUB:
186 case ISD::FMUL:
187 case ISD::FDIV:
188 case ISD::FREM:
189 case ISD::FCANONICALIZE:
190 case ISD::UINT_TO_FP:
191 case ISD::SINT_TO_FP:
192 case ISD::FABS:
193 // Fabs is lowered to a bit operation, but it's an and which will clear the
194 // high bits anyway.
195 case ISD::FSQRT:
196 case ISD::FSIN:
197 case ISD::FCOS:
198 case ISD::FPOWI:
199 case ISD::FPOW:
200 case ISD::FLOG:
201 case ISD::FLOG2:
202 case ISD::FLOG10:
203 case ISD::FEXP:
204 case ISD::FEXP2:
205 case ISD::FCEIL:
206 case ISD::FTRUNC:
207 case ISD::FRINT:
208 case ISD::FNEARBYINT:
209 case ISD::FROUNDEVEN:
210 case ISD::FROUND:
211 case ISD::FFLOOR:
212 case ISD::FMINNUM:
213 case ISD::FMAXNUM:
214 case ISD::FLDEXP:
215 case AMDGPUISD::FRACT:
216 case AMDGPUISD::CLAMP:
217 case AMDGPUISD::COS_HW:
218 case AMDGPUISD::SIN_HW:
219 case AMDGPUISD::FMIN3:
220 case AMDGPUISD::FMAX3:
221 case AMDGPUISD::FMED3:
222 case AMDGPUISD::FMAD_FTZ:
223 case AMDGPUISD::RCP:
224 case AMDGPUISD::RSQ:
225 case AMDGPUISD::RCP_IFLAG:
226 // On gfx10, all 16-bit instructions preserve the high bits.
227 return Subtarget->getGeneration() <= AMDGPUSubtarget::GFX9;
228 case ISD::FP_ROUND:
229 // We may select fptrunc (fma/mad) to mad_mixlo, which does not zero the
230 // high bits on gfx9.
231 // TODO: If we had the source node we could see if the source was fma/mad
232 return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
233 case ISD::FMA:
234 case ISD::FMAD:
235 case AMDGPUISD::DIV_FIXUP:
236 return Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
237 default:
238 // fcopysign, select and others may be lowered to 32-bit bit operations
239 // which don't zero the high bits.
240 return false;
241 }
242}
243
244bool AMDGPUDAGToDAGISelLegacy::runOnMachineFunction(MachineFunction &MF) {
245#ifdef EXPENSIVE_CHECKS
246 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
247 LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
248 for (auto &L : LI->getLoopsInPreorder()) {
249 assert(L->isLCSSAForm(DT));
250 }
251#endif
252 return SelectionDAGISelLegacy::runOnMachineFunction(MF);
253}
254
255void AMDGPUDAGToDAGISelLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
256 AU.addRequired<UniformityInfoWrapperPass>();
257#ifdef EXPENSIVE_CHECKS
258 AU.addRequired<DominatorTreeWrapperPass>();
259 AU.addRequired<LoopInfoWrapperPass>();
260#endif
261 SelectionDAGISelLegacy::getAnalysisUsage(AU);
262}
263
264bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
265 assert(Subtarget->d16PreservesUnusedBits());
266 MVT VT = N->getValueType(ResNo: 0).getSimpleVT();
267 if (VT != MVT::v2i16 && VT != MVT::v2f16)
268 return false;
269
270 SDValue Lo = N->getOperand(Num: 0);
271 SDValue Hi = N->getOperand(Num: 1);
272
273 LoadSDNode *LdHi = dyn_cast<LoadSDNode>(Val: stripBitcast(Val: Hi));
274
275 // build_vector lo, (load ptr) -> load_d16_hi ptr, lo
276 // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo
277 // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo
278
279 // Need to check for possible indirect dependencies on the other half of the
280 // vector to avoid introducing a cycle.
281 if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(N: Lo.getNode())) {
282 SDVTList VTList = CurDAG->getVTList(VT1: VT, VT2: MVT::Other);
283
284 SDValue TiedIn = CurDAG->getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT, Operand: Lo);
285 SDValue Ops[] = {
286 LdHi->getChain(), LdHi->getBasePtr(), TiedIn
287 };
288
289 unsigned LoadOp = AMDGPUISD::LOAD_D16_HI;
290 if (LdHi->getMemoryVT() == MVT::i8) {
291 LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ?
292 AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8;
293 } else {
294 assert(LdHi->getMemoryVT() == MVT::i16);
295 }
296
297 SDValue NewLoadHi =
298 CurDAG->getMemIntrinsicNode(Opcode: LoadOp, dl: SDLoc(LdHi), VTList,
299 Ops, MemVT: LdHi->getMemoryVT(),
300 MMO: LdHi->getMemOperand());
301
302 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: NewLoadHi);
303 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(LdHi, 1), To: NewLoadHi.getValue(R: 1));
304 return true;
305 }
306
307 // build_vector (load ptr), hi -> load_d16_lo ptr, hi
308 // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi
309 // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi
310 LoadSDNode *LdLo = dyn_cast<LoadSDNode>(Val: stripBitcast(Val: Lo));
311 if (LdLo && Lo.hasOneUse()) {
312 SDValue TiedIn = getHi16Elt(In: Hi);
313 if (!TiedIn || LdLo->isPredecessorOf(N: TiedIn.getNode()))
314 return false;
315
316 SDVTList VTList = CurDAG->getVTList(VT1: VT, VT2: MVT::Other);
317 unsigned LoadOp = AMDGPUISD::LOAD_D16_LO;
318 if (LdLo->getMemoryVT() == MVT::i8) {
319 LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ?
320 AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8;
321 } else {
322 assert(LdLo->getMemoryVT() == MVT::i16);
323 }
324
325 TiedIn = CurDAG->getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT, Operand: TiedIn);
326
327 SDValue Ops[] = {
328 LdLo->getChain(), LdLo->getBasePtr(), TiedIn
329 };
330
331 SDValue NewLoadLo =
332 CurDAG->getMemIntrinsicNode(Opcode: LoadOp, dl: SDLoc(LdLo), VTList,
333 Ops, MemVT: LdLo->getMemoryVT(),
334 MMO: LdLo->getMemOperand());
335
336 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: NewLoadLo);
337 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(LdLo, 1), To: NewLoadLo.getValue(R: 1));
338 return true;
339 }
340
341 return false;
342}
343
344void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
345 if (!Subtarget->d16PreservesUnusedBits())
346 return;
347
348 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
349
350 bool MadeChange = false;
351 while (Position != CurDAG->allnodes_begin()) {
352 SDNode *N = &*--Position;
353 if (N->use_empty())
354 continue;
355
356 switch (N->getOpcode()) {
357 case ISD::BUILD_VECTOR:
358 // TODO: Match load d16 from shl (extload:i16), 16
359 MadeChange |= matchLoadD16FromBuildVector(N);
360 break;
361 default:
362 break;
363 }
364 }
365
366 if (MadeChange) {
367 CurDAG->RemoveDeadNodes();
368 LLVM_DEBUG(dbgs() << "After PreProcess:\n";
369 CurDAG->dump(););
370 }
371}
372
373bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N) const {
374 if (N->isUndef())
375 return true;
376
377 const SIInstrInfo *TII = Subtarget->getInstrInfo();
378 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N))
379 return TII->isInlineConstant(Imm: C->getAPIntValue());
380
381 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: N))
382 return TII->isInlineConstant(Imm: C->getValueAPF());
383
384 return false;
385}
386
387/// Determine the register class for \p OpNo
388/// \returns The register class of the virtual register that will be used for
389/// the given operand number \OpNo or NULL if the register class cannot be
390/// determined.
391const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
392 unsigned OpNo) const {
393 if (!N->isMachineOpcode()) {
394 if (N->getOpcode() == ISD::CopyToReg) {
395 Register Reg = cast<RegisterSDNode>(Val: N->getOperand(Num: 1))->getReg();
396 if (Reg.isVirtual()) {
397 MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo();
398 return MRI.getRegClass(Reg);
399 }
400
401 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
402 return TRI->getPhysRegBaseClass(Reg);
403 }
404
405 return nullptr;
406 }
407
408 switch (N->getMachineOpcode()) {
409 default: {
410 const SIInstrInfo *TII = Subtarget->getInstrInfo();
411 const MCInstrDesc &Desc = TII->get(Opcode: N->getMachineOpcode());
412 unsigned OpIdx = Desc.getNumDefs() + OpNo;
413 if (OpIdx >= Desc.getNumOperands())
414 return nullptr;
415
416 int16_t RegClass = TII->getOpRegClassID(OpInfo: Desc.operands()[OpIdx]);
417 if (RegClass == -1)
418 return nullptr;
419
420 return Subtarget->getRegisterInfo()->getRegClass(i: RegClass);
421 }
422 case AMDGPU::REG_SEQUENCE: {
423 unsigned RCID = N->getConstantOperandVal(Num: 0);
424 const TargetRegisterClass *SuperRC =
425 Subtarget->getRegisterInfo()->getRegClass(i: RCID);
426
427 SDValue SubRegOp = N->getOperand(Num: OpNo + 1);
428 unsigned SubRegIdx = SubRegOp->getAsZExtVal();
429 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
430 SubRegIdx);
431 }
432 }
433}
434
435SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain,
436 SDValue Glue) const {
437 SmallVector <SDValue, 8> Ops;
438 Ops.push_back(Elt: NewChain); // Replace the chain.
439 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
440 Ops.push_back(Elt: N->getOperand(Num: i));
441
442 Ops.push_back(Elt: Glue);
443 return CurDAG->MorphNodeTo(N, Opc: N->getOpcode(), VTs: N->getVTList(), Ops);
444}
445
446SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
447 const SITargetLowering& Lowering =
448 *static_cast<const SITargetLowering*>(getTargetLowering());
449
450 assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain");
451
452 SDValue M0 = Lowering.copyToM0(DAG&: *CurDAG, Chain: N->getOperand(Num: 0), DL: SDLoc(N), V: Val);
453 return glueCopyToOp(N, NewChain: M0, Glue: M0.getValue(R: 1));
454}
455
456SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const {
457 unsigned AS = cast<MemSDNode>(Val: N)->getAddressSpace();
458 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
459 if (Subtarget->ldsRequiresM0Init())
460 return glueCopyToM0(
461 N, Val: CurDAG->getSignedTargetConstant(Val: -1, DL: SDLoc(N), VT: MVT::i32));
462 } else if (AS == AMDGPUAS::REGION_ADDRESS) {
463 MachineFunction &MF = CurDAG->getMachineFunction();
464 unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize();
465 return
466 glueCopyToM0(N, Val: CurDAG->getTargetConstant(Val: Value, DL: SDLoc(N), VT: MVT::i32));
467 }
468 return N;
469}
470
471MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm,
472 EVT VT) const {
473 SDNode *Lo = CurDAG->getMachineNode(
474 Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32,
475 Op1: CurDAG->getTargetConstant(Val: Lo_32(Value: Imm), DL, VT: MVT::i32));
476 SDNode *Hi = CurDAG->getMachineNode(
477 Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32,
478 Op1: CurDAG->getTargetConstant(Val: Hi_32(Value: Imm), DL, VT: MVT::i32));
479 const SDValue Ops[] = {
480 CurDAG->getTargetConstant(Val: AMDGPU::SReg_64RegClassID, DL, VT: MVT::i32),
481 SDValue(Lo, 0), CurDAG->getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
482 SDValue(Hi, 0), CurDAG->getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32)};
483
484 return CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT, Ops);
485}
486
487SDNode *AMDGPUDAGToDAGISel::packConstantV2I16(const SDNode *N,
488 SelectionDAG &DAG) const {
489 // TODO: Handle undef as zero
490
491 assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
492 uint32_t LHSVal, RHSVal;
493 if (getConstantValue(N: N->getOperand(Num: 0), Out&: LHSVal) &&
494 getConstantValue(N: N->getOperand(Num: 1), Out&: RHSVal)) {
495 SDLoc SL(N);
496 uint32_t K = (LHSVal & 0xffff) | (RHSVal << 16);
497 return DAG.getMachineNode(
498 Opcode: isVGPRImm(N) ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32, dl: SL,
499 VT: N->getValueType(ResNo: 0), Op1: DAG.getTargetConstant(Val: K, DL: SL, VT: MVT::i32));
500 }
501
502 return nullptr;
503}
504
505void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
506 EVT VT = N->getValueType(ResNo: 0);
507 unsigned NumVectorElts = VT.getVectorNumElements();
508 EVT EltVT = VT.getVectorElementType();
509 SDLoc DL(N);
510 SDValue RegClass = CurDAG->getTargetConstant(Val: RegClassID, DL, VT: MVT::i32);
511
512 if (NumVectorElts == 1) {
513 CurDAG->SelectNodeTo(N, MachineOpc: AMDGPU::COPY_TO_REGCLASS, VT: EltVT, Op1: N->getOperand(Num: 0),
514 Op2: RegClass);
515 return;
516 }
517
518 bool IsGCN = CurDAG->getSubtarget().getTargetTriple().isAMDGCN();
519 if (IsGCN && Subtarget->has64BitLiterals() && VT.getSizeInBits() == 64 &&
520 CurDAG->isConstantValueOfAnyType(N: SDValue(N, 0))) {
521 uint64_t C = 0;
522 bool AllConst = true;
523 unsigned EltSize = EltVT.getSizeInBits();
524 for (unsigned I = 0; I < NumVectorElts; ++I) {
525 SDValue Op = N->getOperand(Num: I);
526 if (Op.isUndef()) {
527 AllConst = false;
528 break;
529 }
530 uint64_t Val;
531 if (ConstantFPSDNode *CF = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
532 Val = CF->getValueAPF().bitcastToAPInt().getZExtValue();
533 } else
534 Val = cast<ConstantSDNode>(Val&: Op)->getZExtValue();
535 C |= Val << (EltSize * I);
536 }
537 if (AllConst) {
538 SDValue CV = CurDAG->getTargetConstant(Val: C, DL, VT: MVT::i64);
539 MachineSDNode *Copy =
540 CurDAG->getMachineNode(Opcode: AMDGPU::S_MOV_B64_IMM_PSEUDO, dl: DL, VT, Op1: CV);
541 CurDAG->SelectNodeTo(N, MachineOpc: AMDGPU::COPY_TO_REGCLASS, VT, Op1: SDValue(Copy, 0),
542 Op2: RegClass);
543 return;
544 }
545 }
546
547 assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not "
548 "supported yet");
549 // 32 = Max Num Vector Elements
550 // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
551 // 1 = Vector Register Class
552 SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
553
554 RegSeqArgs[0] = CurDAG->getTargetConstant(Val: RegClassID, DL, VT: MVT::i32);
555 bool IsRegSeq = true;
556 unsigned NOps = N->getNumOperands();
557 unsigned EltSizeInRegs = EltVT.getSizeInBits() / 32;
558 assert(IsGCN || EltSizeInRegs == 1);
559 for (unsigned i = 0; i < NOps; i++) {
560 // XXX: Why is this here?
561 if (isa<RegisterSDNode>(Val: N->getOperand(Num: i))) {
562 IsRegSeq = false;
563 break;
564 }
565 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(
566 Channel: i * EltSizeInRegs, NumRegs: EltSizeInRegs)
567 : R600RegisterInfo::getSubRegFromChannel(Channel: i);
568 RegSeqArgs[1 + (2 * i)] = N->getOperand(Num: i);
569 RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Val: Sub, DL, VT: MVT::i32);
570 }
571 if (NOps != NumVectorElts) {
572 // Fill in the missing undef elements if this was a scalar_to_vector.
573 assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
574 MachineSDNode *ImpDef = CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF,
575 dl: DL, VT: EltVT);
576 for (unsigned i = NOps; i < NumVectorElts; ++i) {
577 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(
578 Channel: i * EltSizeInRegs, NumRegs: EltSizeInRegs)
579 : R600RegisterInfo::getSubRegFromChannel(Channel: i);
580 RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
581 RegSeqArgs[1 + (2 * i) + 1] =
582 CurDAG->getTargetConstant(Val: Sub, DL, VT: MVT::i32);
583 }
584 }
585
586 if (!IsRegSeq)
587 SelectCode(N);
588 CurDAG->SelectNodeTo(N, MachineOpc: AMDGPU::REG_SEQUENCE, VTs: N->getVTList(), Ops: RegSeqArgs);
589}
590
591void AMDGPUDAGToDAGISel::SelectVectorShuffle(SDNode *N) {
592 EVT VT = N->getValueType(ResNo: 0);
593 EVT EltVT = VT.getVectorElementType();
594
595 // TODO: Handle 16-bit element vectors with even aligned masks.
596 if (!Subtarget->hasPkMovB32() || !EltVT.bitsEq(VT: MVT::i32) ||
597 VT.getVectorNumElements() != 2) {
598 SelectCode(N);
599 return;
600 }
601
602 auto *SVN = cast<ShuffleVectorSDNode>(Val: N);
603
604 SDValue Src0 = SVN->getOperand(Num: 0);
605 SDValue Src1 = SVN->getOperand(Num: 1);
606 ArrayRef<int> Mask = SVN->getMask();
607 SDLoc DL(N);
608
609 assert(Src0.getValueType().getVectorNumElements() == 2 && Mask.size() == 2 &&
610 Mask[0] < 4 && Mask[1] < 4);
611
612 SDValue VSrc0 = Mask[0] < 2 ? Src0 : Src1;
613 SDValue VSrc1 = Mask[1] < 2 ? Src0 : Src1;
614 unsigned Src0SubReg = Mask[0] & 1 ? AMDGPU::sub1 : AMDGPU::sub0;
615 unsigned Src1SubReg = Mask[1] & 1 ? AMDGPU::sub1 : AMDGPU::sub0;
616
617 if (Mask[0] < 0) {
618 Src0SubReg = Src1SubReg;
619 MachineSDNode *ImpDef =
620 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT);
621 VSrc0 = SDValue(ImpDef, 0);
622 }
623
624 if (Mask[1] < 0) {
625 Src1SubReg = Src0SubReg;
626 MachineSDNode *ImpDef =
627 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT);
628 VSrc1 = SDValue(ImpDef, 0);
629 }
630
631 // SGPR case needs to lower to copies.
632 //
633 // Also use subregister extract when we can directly blend the registers with
634 // a simple subregister copy.
635 //
636 // TODO: Maybe we should fold this out earlier
637 if (N->isDivergent() && Src0SubReg == AMDGPU::sub1 &&
638 Src1SubReg == AMDGPU::sub0) {
639 // The low element of the result always comes from src0.
640 // The high element of the result always comes from src1.
641 // op_sel selects the high half of src0.
642 // op_sel_hi selects the high half of src1.
643
644 unsigned Src0OpSel =
645 Src0SubReg == AMDGPU::sub1 ? SISrcMods::OP_SEL_0 : SISrcMods::NONE;
646 unsigned Src1OpSel =
647 Src1SubReg == AMDGPU::sub1 ? SISrcMods::OP_SEL_0 : SISrcMods::NONE;
648
649 // Enable op_sel_hi to avoid printing it. This should have no effect on the
650 // result.
651 Src0OpSel |= SISrcMods::OP_SEL_1;
652 Src1OpSel |= SISrcMods::OP_SEL_1;
653
654 SDValue Src0OpSelVal = CurDAG->getTargetConstant(Val: Src0OpSel, DL, VT: MVT::i32);
655 SDValue Src1OpSelVal = CurDAG->getTargetConstant(Val: Src1OpSel, DL, VT: MVT::i32);
656 SDValue ZeroMods = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
657
658 CurDAG->SelectNodeTo(N, MachineOpc: AMDGPU::V_PK_MOV_B32, VTs: N->getVTList(),
659 Ops: {Src0OpSelVal, VSrc0, Src1OpSelVal, VSrc1,
660 ZeroMods, // clamp
661 ZeroMods, // op_sel
662 ZeroMods, // op_sel_hi
663 ZeroMods, // neg_lo
664 ZeroMods}); // neg_hi
665 return;
666 }
667
668 SDValue ResultElt0 =
669 CurDAG->getTargetExtractSubreg(SRIdx: Src0SubReg, DL, VT: EltVT, Operand: VSrc0);
670 SDValue ResultElt1 =
671 CurDAG->getTargetExtractSubreg(SRIdx: Src1SubReg, DL, VT: EltVT, Operand: VSrc1);
672
673 const SDValue Ops[] = {
674 CurDAG->getTargetConstant(Val: AMDGPU::SReg_64RegClassID, DL, VT: MVT::i32),
675 ResultElt0, CurDAG->getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
676 ResultElt1, CurDAG->getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32)};
677 CurDAG->SelectNodeTo(N, MachineOpc: TargetOpcode::REG_SEQUENCE, VT, Ops);
678}
679
680void AMDGPUDAGToDAGISel::Select(SDNode *N) {
681 unsigned int Opc = N->getOpcode();
682 if (N->isMachineOpcode()) {
683 N->setNodeId(-1);
684 return; // Already selected.
685 }
686
687 // isa<MemSDNode> almost works but is slightly too permissive for some DS
688 // intrinsics.
689 if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(Val: N)) {
690 N = glueCopyToM0LDSInit(N);
691 SelectCode(N);
692 return;
693 }
694
695 switch (Opc) {
696 default:
697 break;
698 case ISD::UADDO_CARRY:
699 case ISD::USUBO_CARRY:
700 if (N->getValueType(ResNo: 0) == MVT::i64) {
701 SelectAddcSubbI64(N);
702 return;
703 }
704
705 if (N->getValueType(ResNo: 0) != MVT::i32)
706 break;
707
708 SelectAddcSubb(N);
709 return;
710 case ISD::UADDO:
711 case ISD::USUBO: {
712 if (N->getValueType(ResNo: 0) == MVT::i64) {
713 SelectAddcSubbI64(N);
714 return;
715 }
716
717 SelectUADDO_USUBO(N);
718 return;
719 }
720 case AMDGPUISD::FMUL_W_CHAIN: {
721 SelectFMUL_W_CHAIN(N);
722 return;
723 }
724 case AMDGPUISD::FMA_W_CHAIN: {
725 SelectFMA_W_CHAIN(N);
726 return;
727 }
728
729 case ISD::SCALAR_TO_VECTOR:
730 case ISD::BUILD_VECTOR: {
731 EVT VT = N->getValueType(ResNo: 0);
732 unsigned NumVectorElts = VT.getVectorNumElements();
733 if (VT.getScalarSizeInBits() == 16) {
734 if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) {
735 if (SDNode *Packed = packConstantV2I16(N, DAG&: *CurDAG)) {
736 ReplaceNode(F: N, T: Packed);
737 return;
738 }
739 }
740
741 break;
742 }
743
744 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
745 EVT EltTy = VT.getVectorElementType();
746 assert(EltTy.bitsEq(MVT::i32) || EltTy.bitsEq(MVT::i64));
747 unsigned VecInBits = NumVectorElts * EltTy.getScalarSizeInBits();
748 const TargetRegisterClass *RegClass =
749 N->isDivergent() ? TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: VecInBits)
750 : SIRegisterInfo::getSGPRClassForBitWidth(BitWidth: VecInBits);
751
752 SelectBuildVector(N, RegClassID: RegClass->getID());
753 return;
754 }
755 case ISD::VECTOR_SHUFFLE:
756 SelectVectorShuffle(N);
757 return;
758 case ISD::BUILD_PAIR: {
759 SDValue RC, SubReg0, SubReg1;
760 SDLoc DL(N);
761 if (N->getValueType(ResNo: 0) == MVT::i128) {
762 RC = CurDAG->getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32);
763 SubReg0 = CurDAG->getTargetConstant(Val: AMDGPU::sub0_sub1, DL, VT: MVT::i32);
764 SubReg1 = CurDAG->getTargetConstant(Val: AMDGPU::sub2_sub3, DL, VT: MVT::i32);
765 } else if (N->getValueType(ResNo: 0) == MVT::i64) {
766 RC = CurDAG->getTargetConstant(Val: AMDGPU::SReg_64RegClassID, DL, VT: MVT::i32);
767 SubReg0 = CurDAG->getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32);
768 SubReg1 = CurDAG->getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32);
769 } else {
770 llvm_unreachable("Unhandled value type for BUILD_PAIR");
771 }
772 const SDValue Ops[] = { RC, N->getOperand(Num: 0), SubReg0,
773 N->getOperand(Num: 1), SubReg1 };
774 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL,
775 VT: N->getValueType(ResNo: 0), Ops));
776 return;
777 }
778
779 case ISD::Constant:
780 case ISD::ConstantFP: {
781 if (N->getValueType(ResNo: 0).getSizeInBits() != 64 || isInlineImmediate(N) ||
782 Subtarget->has64BitLiterals())
783 break;
784
785 uint64_t Imm;
786 if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(Val: N)) {
787 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
788 if (AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: true))
789 break;
790 } else {
791 ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
792 Imm = C->getZExtValue();
793 if (AMDGPU::isValid32BitLiteral(Val: Imm, IsFP64: false))
794 break;
795 }
796
797 SDLoc DL(N);
798 ReplaceNode(F: N, T: buildSMovImm64(DL, Imm, VT: N->getValueType(ResNo: 0)));
799 return;
800 }
801 case AMDGPUISD::BFE_I32:
802 case AMDGPUISD::BFE_U32: {
803 // There is a scalar version available, but unlike the vector version which
804 // has a separate operand for the offset and width, the scalar version packs
805 // the width and offset into a single operand. Try to move to the scalar
806 // version if the offsets are constant, so that we can try to keep extended
807 // loads of kernel arguments in SGPRs.
808
809 // TODO: Technically we could try to pattern match scalar bitshifts of
810 // dynamic values, but it's probably not useful.
811 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
812 if (!Offset)
813 break;
814
815 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
816 if (!Width)
817 break;
818
819 bool Signed = Opc == AMDGPUISD::BFE_I32;
820
821 uint32_t OffsetVal = Offset->getZExtValue();
822 uint32_t WidthVal = Width->getZExtValue();
823
824 ReplaceNode(F: N, T: getBFE32(IsSigned: Signed, DL: SDLoc(N), Val: N->getOperand(Num: 0), Offset: OffsetVal,
825 Width: WidthVal));
826 return;
827 }
828 case AMDGPUISD::DIV_SCALE: {
829 SelectDIV_SCALE(N);
830 return;
831 }
832 case AMDGPUISD::MAD_I64_I32:
833 case AMDGPUISD::MAD_U64_U32: {
834 SelectMAD_64_32(N);
835 return;
836 }
837 case ISD::SMUL_LOHI:
838 case ISD::UMUL_LOHI:
839 return SelectMUL_LOHI(N);
840 case ISD::CopyToReg: {
841 const SITargetLowering& Lowering =
842 *static_cast<const SITargetLowering*>(getTargetLowering());
843 N = Lowering.legalizeTargetIndependentNode(Node: N, DAG&: *CurDAG);
844 break;
845 }
846 case ISD::AND:
847 case ISD::SRL:
848 case ISD::SRA:
849 case ISD::SIGN_EXTEND_INREG:
850 if (N->getValueType(ResNo: 0) != MVT::i32)
851 break;
852
853 SelectS_BFE(N);
854 return;
855 case ISD::BRCOND:
856 SelectBRCOND(N);
857 return;
858 case ISD::FP_EXTEND:
859 SelectFP_EXTEND(N);
860 return;
861 case AMDGPUISD::CVT_PKRTZ_F16_F32:
862 case AMDGPUISD::CVT_PKNORM_I16_F32:
863 case AMDGPUISD::CVT_PKNORM_U16_F32:
864 case AMDGPUISD::CVT_PK_U16_U32:
865 case AMDGPUISD::CVT_PK_I16_I32: {
866 // Hack around using a legal type if f16 is illegal.
867 if (N->getValueType(ResNo: 0) == MVT::i32) {
868 MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16;
869 N = CurDAG->MorphNodeTo(N, Opc: N->getOpcode(), VTs: CurDAG->getVTList(VT: NewVT),
870 Ops: { N->getOperand(Num: 0), N->getOperand(Num: 1) });
871 SelectCode(N);
872 return;
873 }
874
875 break;
876 }
877 case ISD::INTRINSIC_W_CHAIN: {
878 SelectINTRINSIC_W_CHAIN(N);
879 return;
880 }
881 case ISD::INTRINSIC_WO_CHAIN: {
882 SelectINTRINSIC_WO_CHAIN(N);
883 return;
884 }
885 case ISD::INTRINSIC_VOID: {
886 SelectINTRINSIC_VOID(N);
887 return;
888 }
889 case AMDGPUISD::WAVE_ADDRESS: {
890 SelectWAVE_ADDRESS(N);
891 return;
892 }
893 case ISD::STACKRESTORE: {
894 SelectSTACKRESTORE(N);
895 return;
896 }
897 }
898
899 SelectCode(N);
900}
901
902bool AMDGPUDAGToDAGISel::isSDWAOperand(const SDNode *N) const {
903 if (!Subtarget->hasSDWA())
904 return false;
905
906 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
907 EVT VT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
908 return VT.getScalarSizeInBits() == 8 || VT.getScalarSizeInBits() == 16;
909 }
910
911 if (N->getOpcode() == ISD::AND)
912 if (auto *RHS = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1)))
913 return RHS->getZExtValue() == 0xFF || RHS->getZExtValue() == 0xFFFF;
914
915 if (N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL)
916 if (auto *RHS = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1)))
917 return (RHS->getZExtValue() % 8) == 0;
918
919 return false;
920}
921
922bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
923 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
924 const Instruction *Term = BB->getTerminator();
925 return Term->getMetadata(Kind: "amdgpu.uniform") ||
926 Term->getMetadata(Kind: "structurizecfg.uniform");
927}
928
929bool AMDGPUDAGToDAGISel::isUnneededShiftMask(const SDNode *N,
930 unsigned ShAmtBits) const {
931 assert(N->getOpcode() == ISD::AND);
932
933 const APInt &RHS = N->getConstantOperandAPInt(Num: 1);
934 if (RHS.countr_one() >= ShAmtBits)
935 return true;
936
937 const APInt &LHSKnownZeros = CurDAG->computeKnownBits(Op: N->getOperand(Num: 0)).Zero;
938 return (LHSKnownZeros | RHS).countr_one() >= ShAmtBits;
939}
940
941static bool getBaseWithOffsetUsingSplitOR(SelectionDAG &DAG, SDValue Addr,
942 SDValue &N0, SDValue &N1) {
943 if (Addr.getValueType() == MVT::i64 && Addr.getOpcode() == ISD::BITCAST &&
944 Addr.getOperand(i: 0).getOpcode() == ISD::BUILD_VECTOR) {
945 // As we split 64-bit `or` earlier, it's complicated pattern to match, i.e.
946 // (i64 (bitcast (v2i32 (build_vector
947 // (or (extract_vector_elt V, 0), OFFSET),
948 // (extract_vector_elt V, 1)))))
949 SDValue Lo = Addr.getOperand(i: 0).getOperand(i: 0);
950 if (Lo.getOpcode() == ISD::OR && DAG.isBaseWithConstantOffset(Op: Lo)) {
951 SDValue BaseLo = Lo.getOperand(i: 0);
952 SDValue BaseHi = Addr.getOperand(i: 0).getOperand(i: 1);
953 // Check that split base (Lo and Hi) are extracted from the same one.
954 if (BaseLo.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
955 BaseHi.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
956 BaseLo.getOperand(i: 0) == BaseHi.getOperand(i: 0) &&
957 // Lo is statically extracted from index 0.
958 isa<ConstantSDNode>(Val: BaseLo.getOperand(i: 1)) &&
959 BaseLo.getConstantOperandVal(i: 1) == 0 &&
960 // Hi is statically extracted from index 0.
961 isa<ConstantSDNode>(Val: BaseHi.getOperand(i: 1)) &&
962 BaseHi.getConstantOperandVal(i: 1) == 1) {
963 N0 = BaseLo.getOperand(i: 0).getOperand(i: 0);
964 N1 = Lo.getOperand(i: 1);
965 return true;
966 }
967 }
968 }
969 return false;
970}
971
972bool AMDGPUDAGToDAGISel::isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
973 SDValue &RHS) const {
974 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
975 LHS = Addr.getOperand(i: 0);
976 RHS = Addr.getOperand(i: 1);
977 return true;
978 }
979
980 if (getBaseWithOffsetUsingSplitOR(DAG&: *CurDAG, Addr, N0&: LHS, N1&: RHS)) {
981 assert(LHS && RHS && isa<ConstantSDNode>(RHS));
982 return true;
983 }
984
985 return false;
986}
987
988StringRef AMDGPUDAGToDAGISelLegacy::getPassName() const {
989 return "AMDGPU DAG->DAG Pattern Instruction Selection";
990}
991
992AMDGPUISelDAGToDAGPass::AMDGPUISelDAGToDAGPass(TargetMachine &TM)
993 : SelectionDAGISelPass(
994 std::make_unique<AMDGPUDAGToDAGISel>(args&: TM, args: TM.getOptLevel())) {}
995
996PreservedAnalyses
997AMDGPUISelDAGToDAGPass::run(MachineFunction &MF,
998 MachineFunctionAnalysisManager &MFAM) {
999 auto &FAM = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(IR&: MF)
1000 .getManager();
1001 auto &F = MF.getFunction();
1002 // UniformityInfoAnalysis is optional in generic dag isel,
1003 // AMDGPUISelDAGToDAGPass requires it, calculate it explicitly.
1004 FAM.getResult<UniformityInfoAnalysis>(IR&: F);
1005#ifdef EXPENSIVE_CHECKS
1006 DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F);
1007 LoopInfo &LI = FAM.getResult<LoopAnalysis>(F);
1008 for (auto &L : LI.getLoopsInPreorder())
1009 assert(L->isLCSSAForm(DT) && "Loop is not in LCSSA form!");
1010#endif
1011 return SelectionDAGISelPass::run(MF, MFAM);
1012}
1013
1014//===----------------------------------------------------------------------===//
1015// Complex Patterns
1016//===----------------------------------------------------------------------===//
1017
1018bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
1019 SDValue &Offset) {
1020 return false;
1021}
1022
1023bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
1024 SDValue &Offset) {
1025 ConstantSDNode *C;
1026 SDLoc DL(Addr);
1027
1028 if ((C = dyn_cast<ConstantSDNode>(Val&: Addr))) {
1029 Base = CurDAG->getRegister(Reg: R600::INDIRECT_BASE_ADDR, VT: MVT::i32);
1030 Offset = CurDAG->getTargetConstant(Val: C->getZExtValue(), DL, VT: MVT::i32);
1031 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
1032 (C = dyn_cast<ConstantSDNode>(Val: Addr.getOperand(i: 0)))) {
1033 Base = CurDAG->getRegister(Reg: R600::INDIRECT_BASE_ADDR, VT: MVT::i32);
1034 Offset = CurDAG->getTargetConstant(Val: C->getZExtValue(), DL, VT: MVT::i32);
1035 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
1036 (C = dyn_cast<ConstantSDNode>(Val: Addr.getOperand(i: 1)))) {
1037 Base = Addr.getOperand(i: 0);
1038 Offset = CurDAG->getTargetConstant(Val: C->getZExtValue(), DL, VT: MVT::i32);
1039 } else {
1040 Base = Addr;
1041 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1042 }
1043
1044 return true;
1045}
1046
1047SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val,
1048 const SDLoc &DL) const {
1049 SDNode *Mov = CurDAG->getMachineNode(
1050 Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32,
1051 Op1: CurDAG->getTargetConstant(Val, DL, VT: MVT::i32));
1052 return SDValue(Mov, 0);
1053}
1054
1055void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) {
1056 SDValue LHS = N->getOperand(Num: 0);
1057 SDValue RHS = N->getOperand(Num: 1);
1058 SDValue CI = N->getOperand(Num: 2);
1059
1060 if (N->isDivergent()) {
1061 unsigned Opc = N->getOpcode() == ISD::UADDO_CARRY ? AMDGPU::V_ADDC_U32_e64
1062 : AMDGPU::V_SUBB_U32_e64;
1063 CurDAG->SelectNodeTo(
1064 N, MachineOpc: Opc, VTs: N->getVTList(),
1065 Ops: {LHS, RHS, CI,
1066 CurDAG->getTargetConstant(Val: 0, DL: {}, VT: MVT::i1) /*clamp bit*/});
1067 } else {
1068 unsigned Opc = N->getOpcode() == ISD::UADDO_CARRY ? AMDGPU::S_ADD_CO_PSEUDO
1069 : AMDGPU::S_SUB_CO_PSEUDO;
1070 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VTs: N->getVTList(), Ops: {LHS, RHS, CI});
1071 }
1072}
1073
1074void AMDGPUDAGToDAGISel::SelectAddcSubbI64(SDNode *N) {
1075 SDLoc DL(N);
1076 SDValue LHS = N->getOperand(Num: 0);
1077 SDValue RHS = N->getOperand(Num: 1);
1078
1079 unsigned Opcode = N->getOpcode();
1080 bool ConsumeCarry = Opcode == ISD::UADDO_CARRY || Opcode == ISD::USUBO_CARRY;
1081 bool IsAdd = Opcode == ISD::UADDO || Opcode == ISD::UADDO_CARRY;
1082
1083 SDValue Sub0 = CurDAG->getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32);
1084 SDValue Sub1 = CurDAG->getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32);
1085
1086 SDNode *Lo0 = CurDAG->getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
1087 VT: MVT::i32, Op1: LHS, Op2: Sub0);
1088 SDNode *Hi0 = CurDAG->getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
1089 VT: MVT::i32, Op1: LHS, Op2: Sub1);
1090
1091 SDNode *Lo1 = CurDAG->getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
1092 VT: MVT::i32, Op1: RHS, Op2: Sub0);
1093 SDNode *Hi1 = CurDAG->getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
1094 VT: MVT::i32, Op1: RHS, Op2: Sub1);
1095
1096 SDVTList VTList = CurDAG->getVTList(VT1: MVT::i32, VT2: N->getValueType(ResNo: 1));
1097
1098 static const unsigned NoCarryOpcMap[2][2] = {
1099 {AMDGPU::S_USUBO_PSEUDO, AMDGPU::S_UADDO_PSEUDO},
1100 {AMDGPU::V_SUB_CO_U32_e64, AMDGPU::V_ADD_CO_U32_e64}};
1101 static const unsigned CarryOpcMap[2][2] = {
1102 {AMDGPU::S_SUB_CO_PSEUDO, AMDGPU::S_ADD_CO_PSEUDO},
1103 {AMDGPU::V_SUBB_U32_e64, AMDGPU::V_ADDC_U32_e64}};
1104
1105 bool IsVALU = N->isDivergent();
1106
1107 unsigned NoCarryOpc = NoCarryOpcMap[IsVALU][IsAdd];
1108 unsigned CarryOpc = CarryOpcMap[IsVALU][IsAdd];
1109 SDValue Clamp = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1);
1110
1111 SDNode *AddLo;
1112 if (!ConsumeCarry) {
1113 if (IsVALU) {
1114 SDValue Args[] = {SDValue(Lo0, 0), SDValue(Lo1, 0), Clamp};
1115 AddLo = CurDAG->getMachineNode(Opcode: NoCarryOpc, dl: DL, VTs: VTList, Ops: Args);
1116 } else {
1117 SDValue Args[] = {SDValue(Lo0, 0), SDValue(Lo1, 0)};
1118 AddLo = CurDAG->getMachineNode(Opcode: NoCarryOpc, dl: DL, VTs: VTList, Ops: Args);
1119 }
1120 } else {
1121 if (IsVALU) {
1122 SDValue Args[] = {SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(Num: 2),
1123 Clamp};
1124 AddLo = CurDAG->getMachineNode(Opcode: CarryOpc, dl: DL, VTs: VTList, Ops: Args);
1125 } else {
1126 SDValue Args[] = {SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(Num: 2)};
1127 AddLo = CurDAG->getMachineNode(Opcode: CarryOpc, dl: DL, VTs: VTList, Ops: Args);
1128 }
1129 }
1130
1131 SDNode *AddHi;
1132 if (IsVALU) {
1133 SDValue Args[] = {SDValue(Hi0, 0), SDValue(Hi1, 0), SDValue(AddLo, 1),
1134 Clamp};
1135 AddHi = CurDAG->getMachineNode(Opcode: CarryOpc, dl: DL, VTs: VTList, Ops: Args);
1136 } else {
1137 SDValue Args[] = {SDValue(Hi0, 0), SDValue(Hi1, 0), SDValue(AddLo, 1)};
1138 AddHi = CurDAG->getMachineNode(Opcode: CarryOpc, dl: DL, VTs: VTList, Ops: Args);
1139 }
1140
1141 unsigned RC = IsVALU ? AMDGPU::VReg_64RegClassID : AMDGPU::SReg_64RegClassID;
1142 SDValue RegSequenceArgs[] = {CurDAG->getTargetConstant(Val: RC, DL, VT: MVT::i32),
1143 SDValue(AddLo, 0), Sub0, SDValue(AddHi, 0),
1144 Sub1};
1145 SDNode *RegSequence = CurDAG->getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL,
1146 VT: MVT::i64, Ops: RegSequenceArgs);
1147
1148 ReplaceUses(F: SDValue(N, 1), T: SDValue(AddHi, 1));
1149 ReplaceNode(F: N, T: RegSequence);
1150}
1151
1152void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
1153 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
1154 // carry out despite the _i32 name. These were renamed in VI to _U32.
1155 // FIXME: We should probably rename the opcodes here.
1156 bool IsAdd = N->getOpcode() == ISD::UADDO;
1157 bool IsVALU = N->isDivergent();
1158
1159 for (SDNode::user_iterator UI = N->user_begin(), E = N->user_end(); UI != E;
1160 ++UI)
1161 if (UI.getUse().getResNo() == 1) {
1162 if (UI->isMachineOpcode()) {
1163 if (UI->getMachineOpcode() !=
1164 (IsAdd ? AMDGPU::S_ADD_CO_PSEUDO : AMDGPU::S_SUB_CO_PSEUDO)) {
1165 IsVALU = true;
1166 break;
1167 }
1168 } else {
1169 if (UI->getOpcode() != (IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY)) {
1170 IsVALU = true;
1171 break;
1172 }
1173 }
1174 }
1175
1176 if (IsVALU) {
1177 unsigned Opc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
1178
1179 CurDAG->SelectNodeTo(
1180 N, MachineOpc: Opc, VTs: N->getVTList(),
1181 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1),
1182 CurDAG->getTargetConstant(Val: 0, DL: {}, VT: MVT::i1) /*clamp bit*/});
1183 } else {
1184 unsigned Opc = IsAdd ? AMDGPU::S_UADDO_PSEUDO : AMDGPU::S_USUBO_PSEUDO;
1185
1186 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VTs: N->getVTList(),
1187 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1)});
1188 }
1189}
1190
1191void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) {
1192 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod
1193 SDValue Ops[10];
1194
1195 SelectVOP3Mods0(In: N->getOperand(Num: 1), Src&: Ops[1], SrcMods&: Ops[0], Clamp&: Ops[6], Omod&: Ops[7]);
1196 SelectVOP3Mods(In: N->getOperand(Num: 2), Src&: Ops[3], SrcMods&: Ops[2]);
1197 SelectVOP3Mods(In: N->getOperand(Num: 3), Src&: Ops[5], SrcMods&: Ops[4]);
1198 Ops[8] = N->getOperand(Num: 0);
1199 Ops[9] = N->getOperand(Num: 4);
1200
1201 // If there are no source modifiers, prefer fmac over fma because it can use
1202 // the smaller VOP2 encoding.
1203 bool UseFMAC = Subtarget->hasDLInsts() &&
1204 cast<ConstantSDNode>(Val&: Ops[0])->isZero() &&
1205 cast<ConstantSDNode>(Val&: Ops[2])->isZero() &&
1206 cast<ConstantSDNode>(Val&: Ops[4])->isZero();
1207 unsigned Opcode = UseFMAC ? AMDGPU::V_FMAC_F32_e64 : AMDGPU::V_FMA_F32_e64;
1208 CurDAG->SelectNodeTo(N, MachineOpc: Opcode, VTs: N->getVTList(), Ops);
1209}
1210
1211void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) {
1212 // src0_modifiers, src0, src1_modifiers, src1, clamp, omod
1213 SDValue Ops[8];
1214
1215 SelectVOP3Mods0(In: N->getOperand(Num: 1), Src&: Ops[1], SrcMods&: Ops[0], Clamp&: Ops[4], Omod&: Ops[5]);
1216 SelectVOP3Mods(In: N->getOperand(Num: 2), Src&: Ops[3], SrcMods&: Ops[2]);
1217 Ops[6] = N->getOperand(Num: 0);
1218 Ops[7] = N->getOperand(Num: 3);
1219
1220 CurDAG->SelectNodeTo(N, MachineOpc: AMDGPU::V_MUL_F32_e64, VTs: N->getVTList(), Ops);
1221}
1222
1223// We need to handle this here because tablegen doesn't support matching
1224// instructions with multiple outputs.
1225void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
1226 EVT VT = N->getValueType(ResNo: 0);
1227
1228 assert(VT == MVT::f32 || VT == MVT::f64);
1229
1230 unsigned Opc
1231 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64_e64 : AMDGPU::V_DIV_SCALE_F32_e64;
1232
1233 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp,
1234 // omod
1235 SDValue Ops[8];
1236 SelectVOP3BMods0(In: N->getOperand(Num: 0), Src&: Ops[1], SrcMods&: Ops[0], Clamp&: Ops[6], Omod&: Ops[7]);
1237 SelectVOP3BMods(In: N->getOperand(Num: 1), Src&: Ops[3], SrcMods&: Ops[2]);
1238 SelectVOP3BMods(In: N->getOperand(Num: 2), Src&: Ops[5], SrcMods&: Ops[4]);
1239 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VTs: N->getVTList(), Ops);
1240}
1241
1242// We need to handle this here because tablegen doesn't support matching
1243// instructions with multiple outputs.
1244void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
1245 SDLoc SL(N);
1246 bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32;
1247 unsigned Opc;
1248 bool UseNoCarry = Subtarget->hasMadNC64_32Insts() && !N->hasAnyUseOfValue(Value: 1);
1249 if (Subtarget->hasMADIntraFwdBug())
1250 Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
1251 : AMDGPU::V_MAD_U64_U32_gfx11_e64;
1252 else if (UseNoCarry)
1253 Opc = Signed ? AMDGPU::V_MAD_NC_I64_I32_e64 : AMDGPU::V_MAD_NC_U64_U32_e64;
1254 else
1255 Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1256
1257 SDValue Clamp = CurDAG->getTargetConstant(Val: 0, DL: SL, VT: MVT::i1);
1258 SDValue Ops[] = { N->getOperand(Num: 0), N->getOperand(Num: 1), N->getOperand(Num: 2),
1259 Clamp };
1260
1261 if (UseNoCarry) {
1262 MachineSDNode *Mad = CurDAG->getMachineNode(Opcode: Opc, dl: SL, VT: MVT::i64, Ops);
1263 ReplaceUses(F: SDValue(N, 0), T: SDValue(Mad, 0));
1264 CurDAG->RemoveDeadNode(N);
1265 return;
1266 }
1267
1268 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VTs: N->getVTList(), Ops);
1269}
1270
1271// We need to handle this here because tablegen doesn't support matching
1272// instructions with multiple outputs.
1273void AMDGPUDAGToDAGISel::SelectMUL_LOHI(SDNode *N) {
1274 SDLoc SL(N);
1275 bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
1276 SDVTList VTList;
1277 unsigned Opc;
1278 if (Subtarget->hasMadNC64_32Insts()) {
1279 VTList = CurDAG->getVTList(VT: MVT::i64);
1280 Opc = Signed ? AMDGPU::V_MAD_NC_I64_I32_e64 : AMDGPU::V_MAD_NC_U64_U32_e64;
1281 } else {
1282 VTList = CurDAG->getVTList(VT1: MVT::i64, VT2: MVT::i1);
1283 if (Subtarget->hasMADIntraFwdBug()) {
1284 Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
1285 : AMDGPU::V_MAD_U64_U32_gfx11_e64;
1286 } else {
1287 Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
1288 }
1289 }
1290
1291 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL: SL, VT: MVT::i64);
1292 SDValue Clamp = CurDAG->getTargetConstant(Val: 0, DL: SL, VT: MVT::i1);
1293 SDValue Ops[] = {N->getOperand(Num: 0), N->getOperand(Num: 1), Zero, Clamp};
1294 SDNode *Mad = CurDAG->getMachineNode(Opcode: Opc, dl: SL, VTs: VTList, Ops);
1295 if (!SDValue(N, 0).use_empty()) {
1296 SDValue Sub0 = CurDAG->getTargetConstant(Val: AMDGPU::sub0, DL: SL, VT: MVT::i32);
1297 SDNode *Lo = CurDAG->getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: SL,
1298 VT: MVT::i32, Op1: SDValue(Mad, 0), Op2: Sub0);
1299 ReplaceUses(F: SDValue(N, 0), T: SDValue(Lo, 0));
1300 }
1301 if (!SDValue(N, 1).use_empty()) {
1302 SDValue Sub1 = CurDAG->getTargetConstant(Val: AMDGPU::sub1, DL: SL, VT: MVT::i32);
1303 SDNode *Hi = CurDAG->getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: SL,
1304 VT: MVT::i32, Op1: SDValue(Mad, 0), Op2: Sub1);
1305 ReplaceUses(F: SDValue(N, 1), T: SDValue(Hi, 0));
1306 }
1307 CurDAG->RemoveDeadNode(N);
1308}
1309
1310bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset) const {
1311 if (!isUInt<16>(x: Offset))
1312 return false;
1313
1314 if (!Base || Subtarget->hasUsableDSOffset() ||
1315 Subtarget->unsafeDSOffsetFoldingEnabled())
1316 return true;
1317
1318 // On Southern Islands instruction with a negative base value and an offset
1319 // don't seem to work.
1320 return CurDAG->SignBitIsZero(Op: Base);
1321}
1322
1323bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1324 SDValue &Offset) const {
1325 SDLoc DL(Addr);
1326 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
1327 SDValue N0 = Addr.getOperand(i: 0);
1328 SDValue N1 = Addr.getOperand(i: 1);
1329 ConstantSDNode *C1 = cast<ConstantSDNode>(Val&: N1);
1330 if (isDSOffsetLegal(Base: N0, Offset: C1->getSExtValue())) {
1331 // (add n0, c0)
1332 Base = N0;
1333 Offset = CurDAG->getTargetConstant(Val: C1->getZExtValue(), DL, VT: MVT::i16);
1334 return true;
1335 }
1336 } else if (Addr.getOpcode() == ISD::SUB) {
1337 // sub C, x -> add (sub 0, x), C
1338 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Addr.getOperand(i: 0))) {
1339 int64_t ByteOffset = C->getSExtValue();
1340 if (isDSOffsetLegal(Base: SDValue(), Offset: ByteOffset)) {
1341 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1342
1343 // XXX - This is kind of hacky. Create a dummy sub node so we can check
1344 // the known bits in isDSOffsetLegal. We need to emit the selected node
1345 // here, so this is thrown away.
1346 SDValue Sub = CurDAG->getNode(Opcode: ISD::SUB, DL, VT: MVT::i32,
1347 N1: Zero, N2: Addr.getOperand(i: 1));
1348
1349 if (isDSOffsetLegal(Base: Sub, Offset: ByteOffset)) {
1350 SmallVector<SDValue, 3> Opnds;
1351 Opnds.push_back(Elt: Zero);
1352 Opnds.push_back(Elt: Addr.getOperand(i: 1));
1353
1354 // FIXME: Select to VOP3 version for with-carry.
1355 unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1356 if (Subtarget->hasAddNoCarryInsts()) {
1357 SubOp = AMDGPU::V_SUB_U32_e64;
1358 Opnds.push_back(
1359 Elt: CurDAG->getTargetConstant(Val: 0, DL: {}, VT: MVT::i1)); // clamp bit
1360 }
1361
1362 MachineSDNode *MachineSub =
1363 CurDAG->getMachineNode(Opcode: SubOp, dl: DL, VT: MVT::i32, Ops: Opnds);
1364
1365 Base = SDValue(MachineSub, 0);
1366 Offset = CurDAG->getTargetConstant(Val: ByteOffset, DL, VT: MVT::i16);
1367 return true;
1368 }
1369 }
1370 }
1371 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Val&: Addr)) {
1372 // If we have a constant address, prefer to put the constant into the
1373 // offset. This can save moves to load the constant address since multiple
1374 // operations can share the zero base address register, and enables merging
1375 // into read2 / write2 instructions.
1376
1377 SDLoc DL(Addr);
1378
1379 if (isDSOffsetLegal(Base: SDValue(), Offset: CAddr->getZExtValue())) {
1380 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1381 MachineSDNode *MovZero = CurDAG->getMachineNode(Opcode: AMDGPU::V_MOV_B32_e32,
1382 dl: DL, VT: MVT::i32, Op1: Zero);
1383 Base = SDValue(MovZero, 0);
1384 Offset = CurDAG->getTargetConstant(Val: CAddr->getZExtValue(), DL, VT: MVT::i16);
1385 return true;
1386 }
1387 }
1388
1389 // default case
1390 Base = Addr;
1391 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT: MVT::i16);
1392 return true;
1393}
1394
1395bool AMDGPUDAGToDAGISel::isDSOffset2Legal(SDValue Base, unsigned Offset0,
1396 unsigned Offset1,
1397 unsigned Size) const {
1398 if (Offset0 % Size != 0 || Offset1 % Size != 0)
1399 return false;
1400 if (!isUInt<8>(x: Offset0 / Size) || !isUInt<8>(x: Offset1 / Size))
1401 return false;
1402
1403 if (!Base || Subtarget->hasUsableDSOffset() ||
1404 Subtarget->unsafeDSOffsetFoldingEnabled())
1405 return true;
1406
1407 // On Southern Islands instruction with a negative base value and an offset
1408 // don't seem to work.
1409 return CurDAG->SignBitIsZero(Op: Base);
1410}
1411
1412// Return whether the operation has NoUnsignedWrap property.
1413static bool isNoUnsignedWrap(SDValue Addr) {
1414 return (Addr.getOpcode() == ISD::ADD &&
1415 Addr->getFlags().hasNoUnsignedWrap()) ||
1416 Addr->getOpcode() == ISD::OR;
1417}
1418
1419// Check that the base address of flat scratch load/store in the form of `base +
1420// offset` is legal to be put in SGPR/VGPR (i.e. unsigned per hardware
1421// requirement). We always treat the first operand as the base address here.
1422bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegal(SDValue Addr) const {
1423 if (isNoUnsignedWrap(Addr))
1424 return true;
1425
1426 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
1427 // values.
1428 if (Subtarget->hasSignedScratchOffsets())
1429 return true;
1430
1431 auto LHS = Addr.getOperand(i: 0);
1432 auto RHS = Addr.getOperand(i: 1);
1433
1434 // If the immediate offset is negative and within certain range, the base
1435 // address cannot also be negative. If the base is also negative, the sum
1436 // would be either negative or much larger than the valid range of scratch
1437 // memory a thread can access.
1438 ConstantSDNode *ImmOp = nullptr;
1439 if (Addr.getOpcode() == ISD::ADD && (ImmOp = dyn_cast<ConstantSDNode>(Val&: RHS))) {
1440 if (ImmOp->getSExtValue() < 0 && ImmOp->getSExtValue() > -0x40000000)
1441 return true;
1442 }
1443
1444 return CurDAG->SignBitIsZero(Op: LHS);
1445}
1446
1447// Check address value in SGPR/VGPR are legal for flat scratch in the form
1448// of: SGPR + VGPR.
1449bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegalSV(SDValue Addr) const {
1450 if (isNoUnsignedWrap(Addr))
1451 return true;
1452
1453 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
1454 // values.
1455 if (Subtarget->hasSignedScratchOffsets())
1456 return true;
1457
1458 auto LHS = Addr.getOperand(i: 0);
1459 auto RHS = Addr.getOperand(i: 1);
1460 return CurDAG->SignBitIsZero(Op: RHS) && CurDAG->SignBitIsZero(Op: LHS);
1461}
1462
1463// Check address value in SGPR/VGPR are legal for flat scratch in the form
1464// of: SGPR + VGPR + Imm.
1465bool AMDGPUDAGToDAGISel::isFlatScratchBaseLegalSVImm(SDValue Addr) const {
1466 // Starting with GFX12, VADDR and SADDR fields in VSCRATCH can use negative
1467 // values.
1468 if (AMDGPU::isGFX12Plus(STI: *Subtarget))
1469 return true;
1470
1471 auto Base = Addr.getOperand(i: 0);
1472 auto *RHSImm = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1));
1473 // If the immediate offset is negative and within certain range, the base
1474 // address cannot also be negative. If the base is also negative, the sum
1475 // would be either negative or much larger than the valid range of scratch
1476 // memory a thread can access.
1477 if (isNoUnsignedWrap(Addr: Base) &&
1478 (isNoUnsignedWrap(Addr) ||
1479 (RHSImm->getSExtValue() < 0 && RHSImm->getSExtValue() > -0x40000000)))
1480 return true;
1481
1482 auto LHS = Base.getOperand(i: 0);
1483 auto RHS = Base.getOperand(i: 1);
1484 return CurDAG->SignBitIsZero(Op: RHS) && CurDAG->SignBitIsZero(Op: LHS);
1485}
1486
1487// TODO: If offset is too big, put low 16-bit into offset.
1488bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1489 SDValue &Offset0,
1490 SDValue &Offset1) const {
1491 return SelectDSReadWrite2(Ptr: Addr, Base, Offset0, Offset1, Size: 4);
1492}
1493
1494bool AMDGPUDAGToDAGISel::SelectDS128Bit8ByteAligned(SDValue Addr, SDValue &Base,
1495 SDValue &Offset0,
1496 SDValue &Offset1) const {
1497 return SelectDSReadWrite2(Ptr: Addr, Base, Offset0, Offset1, Size: 8);
1498}
1499
1500bool AMDGPUDAGToDAGISel::SelectDSReadWrite2(SDValue Addr, SDValue &Base,
1501 SDValue &Offset0, SDValue &Offset1,
1502 unsigned Size) const {
1503 SDLoc DL(Addr);
1504
1505 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
1506 SDValue N0 = Addr.getOperand(i: 0);
1507 SDValue N1 = Addr.getOperand(i: 1);
1508 ConstantSDNode *C1 = cast<ConstantSDNode>(Val&: N1);
1509 unsigned OffsetValue0 = C1->getZExtValue();
1510 unsigned OffsetValue1 = OffsetValue0 + Size;
1511
1512 // (add n0, c0)
1513 if (isDSOffset2Legal(Base: N0, Offset0: OffsetValue0, Offset1: OffsetValue1, Size)) {
1514 Base = N0;
1515 Offset0 = CurDAG->getTargetConstant(Val: OffsetValue0 / Size, DL, VT: MVT::i32);
1516 Offset1 = CurDAG->getTargetConstant(Val: OffsetValue1 / Size, DL, VT: MVT::i32);
1517 return true;
1518 }
1519 } else if (Addr.getOpcode() == ISD::SUB) {
1520 // sub C, x -> add (sub 0, x), C
1521 if (const ConstantSDNode *C =
1522 dyn_cast<ConstantSDNode>(Val: Addr.getOperand(i: 0))) {
1523 unsigned OffsetValue0 = C->getZExtValue();
1524 unsigned OffsetValue1 = OffsetValue0 + Size;
1525
1526 if (isDSOffset2Legal(Base: SDValue(), Offset0: OffsetValue0, Offset1: OffsetValue1, Size)) {
1527 SDLoc DL(Addr);
1528 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1529
1530 // XXX - This is kind of hacky. Create a dummy sub node so we can check
1531 // the known bits in isDSOffsetLegal. We need to emit the selected node
1532 // here, so this is thrown away.
1533 SDValue Sub =
1534 CurDAG->getNode(Opcode: ISD::SUB, DL, VT: MVT::i32, N1: Zero, N2: Addr.getOperand(i: 1));
1535
1536 if (isDSOffset2Legal(Base: Sub, Offset0: OffsetValue0, Offset1: OffsetValue1, Size)) {
1537 SmallVector<SDValue, 3> Opnds;
1538 Opnds.push_back(Elt: Zero);
1539 Opnds.push_back(Elt: Addr.getOperand(i: 1));
1540 unsigned SubOp = AMDGPU::V_SUB_CO_U32_e32;
1541 if (Subtarget->hasAddNoCarryInsts()) {
1542 SubOp = AMDGPU::V_SUB_U32_e64;
1543 Opnds.push_back(
1544 Elt: CurDAG->getTargetConstant(Val: 0, DL: {}, VT: MVT::i1)); // clamp bit
1545 }
1546
1547 MachineSDNode *MachineSub = CurDAG->getMachineNode(
1548 Opcode: SubOp, dl: DL, VT: MVT::getIntegerVT(BitWidth: Size * 8), Ops: Opnds);
1549
1550 Base = SDValue(MachineSub, 0);
1551 Offset0 =
1552 CurDAG->getTargetConstant(Val: OffsetValue0 / Size, DL, VT: MVT::i32);
1553 Offset1 =
1554 CurDAG->getTargetConstant(Val: OffsetValue1 / Size, DL, VT: MVT::i32);
1555 return true;
1556 }
1557 }
1558 }
1559 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Val&: Addr)) {
1560 unsigned OffsetValue0 = CAddr->getZExtValue();
1561 unsigned OffsetValue1 = OffsetValue0 + Size;
1562
1563 if (isDSOffset2Legal(Base: SDValue(), Offset0: OffsetValue0, Offset1: OffsetValue1, Size)) {
1564 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1565 MachineSDNode *MovZero =
1566 CurDAG->getMachineNode(Opcode: AMDGPU::V_MOV_B32_e32, dl: DL, VT: MVT::i32, Op1: Zero);
1567 Base = SDValue(MovZero, 0);
1568 Offset0 = CurDAG->getTargetConstant(Val: OffsetValue0 / Size, DL, VT: MVT::i32);
1569 Offset1 = CurDAG->getTargetConstant(Val: OffsetValue1 / Size, DL, VT: MVT::i32);
1570 return true;
1571 }
1572 }
1573
1574 // default case
1575
1576 Base = Addr;
1577 Offset0 = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1578 Offset1 = CurDAG->getTargetConstant(Val: 1, DL, VT: MVT::i32);
1579 return true;
1580}
1581
1582bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, SDValue &VAddr,
1583 SDValue &SOffset, SDValue &Offset,
1584 SDValue &Offen, SDValue &Idxen,
1585 SDValue &Addr64) const {
1586 // Subtarget prefers to use flat instruction
1587 // FIXME: This should be a pattern predicate and not reach here
1588 if (Subtarget->useFlatForGlobal())
1589 return false;
1590
1591 SDLoc DL(Addr);
1592
1593 Idxen = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1);
1594 Offen = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1);
1595 Addr64 = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1);
1596 SOffset = Subtarget->hasRestrictedSOffset()
1597 ? CurDAG->getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32)
1598 : CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1599
1600 ConstantSDNode *C1 = nullptr;
1601 SDValue N0 = Addr;
1602 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
1603 C1 = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1));
1604 if (isUInt<32>(x: C1->getZExtValue()))
1605 N0 = Addr.getOperand(i: 0);
1606 else
1607 C1 = nullptr;
1608 }
1609
1610 if (N0->isAnyAdd()) {
1611 // (add N2, N3) -> addr64, or
1612 // (add (add N2, N3), C1) -> addr64
1613 SDValue N2 = N0.getOperand(i: 0);
1614 SDValue N3 = N0.getOperand(i: 1);
1615 Addr64 = CurDAG->getTargetConstant(Val: 1, DL, VT: MVT::i1);
1616
1617 if (N2->isDivergent()) {
1618 if (N3->isDivergent()) {
1619 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
1620 // addr64, and construct the resource from a 0 address.
1621 Ptr = SDValue(buildSMovImm64(DL, Imm: 0, VT: MVT::v2i32), 0);
1622 VAddr = N0;
1623 } else {
1624 // N2 is divergent, N3 is not.
1625 Ptr = N3;
1626 VAddr = N2;
1627 }
1628 } else {
1629 // N2 is not divergent.
1630 Ptr = N2;
1631 VAddr = N3;
1632 }
1633 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1634 } else if (N0->isDivergent()) {
1635 // N0 is divergent. Use it as the addr64, and construct the resource from a
1636 // 0 address.
1637 Ptr = SDValue(buildSMovImm64(DL, Imm: 0, VT: MVT::v2i32), 0);
1638 VAddr = N0;
1639 Addr64 = CurDAG->getTargetConstant(Val: 1, DL, VT: MVT::i1);
1640 } else {
1641 // N0 -> offset, or
1642 // (N0 + C1) -> offset
1643 VAddr = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1644 Ptr = N0;
1645 }
1646
1647 if (!C1) {
1648 // No offset.
1649 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1650 return true;
1651 }
1652
1653 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1654 if (TII->isLegalMUBUFImmOffset(Imm: C1->getZExtValue())) {
1655 // Legal offset for instruction.
1656 Offset = CurDAG->getTargetConstant(Val: C1->getZExtValue(), DL, VT: MVT::i32);
1657 return true;
1658 }
1659
1660 // Illegal offset, store it in soffset.
1661 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1662 SOffset =
1663 SDValue(CurDAG->getMachineNode(
1664 Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32,
1665 Op1: CurDAG->getTargetConstant(Val: C1->getZExtValue(), DL, VT: MVT::i32)),
1666 0);
1667 return true;
1668}
1669
1670bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1671 SDValue &VAddr, SDValue &SOffset,
1672 SDValue &Offset) const {
1673 SDValue Ptr, Offen, Idxen, Addr64;
1674
1675 // addr64 bit was removed for volcanic islands.
1676 // FIXME: This should be a pattern predicate and not reach here
1677 if (!Subtarget->hasAddr64())
1678 return false;
1679
1680 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1681 return false;
1682
1683 ConstantSDNode *C = cast<ConstantSDNode>(Val&: Addr64);
1684 if (C->getSExtValue()) {
1685 SDLoc DL(Addr);
1686
1687 const SITargetLowering& Lowering =
1688 *static_cast<const SITargetLowering*>(getTargetLowering());
1689
1690 SRsrc = SDValue(Lowering.wrapAddr64Rsrc(DAG&: *CurDAG, DL, Ptr), 0);
1691 return true;
1692 }
1693
1694 return false;
1695}
1696
1697std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1698 SDLoc DL(N);
1699
1700 auto *FI = dyn_cast<FrameIndexSDNode>(Val&: N);
1701 SDValue TFI =
1702 FI ? CurDAG->getTargetFrameIndex(FI: FI->getIndex(), VT: FI->getValueType(ResNo: 0)) : N;
1703
1704 // We rebase the base address into an absolute stack address and hence
1705 // use constant 0 for soffset. This value must be retained until
1706 // frame elimination and eliminateFrameIndex will choose the appropriate
1707 // frame register if need be.
1708 return std::pair(TFI, CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32));
1709}
1710
1711bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
1712 SDValue Addr, SDValue &Rsrc,
1713 SDValue &VAddr, SDValue &SOffset,
1714 SDValue &ImmOffset) const {
1715
1716 SDLoc DL(Addr);
1717 MachineFunction &MF = CurDAG->getMachineFunction();
1718 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1719
1720 Rsrc = CurDAG->getRegister(Reg: Info->getScratchRSrcReg(), VT: MVT::v4i32);
1721
1722 if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Val&: Addr)) {
1723 int64_t Imm = CAddr->getSExtValue();
1724 const int64_t NullPtr =
1725 AMDGPU::getNullPointerValue(AS: AMDGPUAS::PRIVATE_ADDRESS);
1726 // Don't fold null pointer.
1727 if (Imm != NullPtr) {
1728 const int64_t MaxOffset = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
1729 SDValue HighBits =
1730 CurDAG->getTargetConstant(Val: Imm & ~MaxOffset, DL, VT: MVT::i32);
1731 MachineSDNode *MovHighBits = CurDAG->getMachineNode(
1732 Opcode: AMDGPU::V_MOV_B32_e32, dl: DL, VT: MVT::i32, Op1: HighBits);
1733 VAddr = SDValue(MovHighBits, 0);
1734
1735 SOffset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1736 ImmOffset = CurDAG->getTargetConstant(Val: Imm & MaxOffset, DL, VT: MVT::i32);
1737 return true;
1738 }
1739 }
1740
1741 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
1742 // (add n0, c1)
1743
1744 SDValue N0 = Addr.getOperand(i: 0);
1745 uint64_t C1 = Addr.getConstantOperandVal(i: 1);
1746
1747 // Offsets in vaddr must be positive if range checking is enabled.
1748 //
1749 // The total computation of vaddr + soffset + offset must not overflow. If
1750 // vaddr is negative, even if offset is 0 the sgpr offset add will end up
1751 // overflowing.
1752 //
1753 // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would
1754 // always perform a range check. If a negative vaddr base index was used,
1755 // this would fail the range check. The overall address computation would
1756 // compute a valid address, but this doesn't happen due to the range
1757 // check. For out-of-bounds MUBUF loads, a 0 is returned.
1758 //
1759 // Therefore it should be safe to fold any VGPR offset on gfx9 into the
1760 // MUBUF vaddr, but not on older subtargets which can only do this if the
1761 // sign bit is known 0.
1762 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1763 if (TII->isLegalMUBUFImmOffset(Imm: C1) &&
1764 (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1765 CurDAG->SignBitIsZero(Op: N0))) {
1766 std::tie(args&: VAddr, args&: SOffset) = foldFrameIndex(N: N0);
1767 ImmOffset = CurDAG->getTargetConstant(Val: C1, DL, VT: MVT::i32);
1768 return true;
1769 }
1770 }
1771
1772 // (node)
1773 std::tie(args&: VAddr, args&: SOffset) = foldFrameIndex(N: Addr);
1774 ImmOffset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1775 return true;
1776}
1777
1778static bool IsCopyFromSGPR(const SIRegisterInfo &TRI, SDValue Val) {
1779 if (Val.getOpcode() != ISD::CopyFromReg)
1780 return false;
1781 auto Reg = cast<RegisterSDNode>(Val: Val.getOperand(i: 1))->getReg();
1782 if (!Reg.isPhysical())
1783 return false;
1784 const auto *RC = TRI.getPhysRegBaseClass(Reg);
1785 return RC && TRI.isSGPRClass(RC);
1786}
1787
1788bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
1789 SDValue Addr,
1790 SDValue &SRsrc,
1791 SDValue &SOffset,
1792 SDValue &Offset) const {
1793 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
1794 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1795 MachineFunction &MF = CurDAG->getMachineFunction();
1796 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1797 SDLoc DL(Addr);
1798
1799 // CopyFromReg <sgpr>
1800 if (IsCopyFromSGPR(TRI: *TRI, Val: Addr)) {
1801 SRsrc = CurDAG->getRegister(Reg: Info->getScratchRSrcReg(), VT: MVT::v4i32);
1802 SOffset = Addr;
1803 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1804 return true;
1805 }
1806
1807 ConstantSDNode *CAddr;
1808 if (Addr.getOpcode() == ISD::ADD) {
1809 // Add (CopyFromReg <sgpr>) <constant>
1810 CAddr = dyn_cast<ConstantSDNode>(Val: Addr.getOperand(i: 1));
1811 if (!CAddr || !TII->isLegalMUBUFImmOffset(Imm: CAddr->getZExtValue()))
1812 return false;
1813 if (!IsCopyFromSGPR(TRI: *TRI, Val: Addr.getOperand(i: 0)))
1814 return false;
1815
1816 SOffset = Addr.getOperand(i: 0);
1817 } else if ((CAddr = dyn_cast<ConstantSDNode>(Val&: Addr)) &&
1818 TII->isLegalMUBUFImmOffset(Imm: CAddr->getZExtValue())) {
1819 // <constant>
1820 SOffset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
1821 } else {
1822 return false;
1823 }
1824
1825 SRsrc = CurDAG->getRegister(Reg: Info->getScratchRSrcReg(), VT: MVT::v4i32);
1826
1827 Offset = CurDAG->getTargetConstant(Val: CAddr->getZExtValue(), DL, VT: MVT::i32);
1828 return true;
1829}
1830
1831bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1832 SDValue &SOffset, SDValue &Offset
1833 ) const {
1834 SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1835 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1836
1837 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64))
1838 return false;
1839
1840 if (!cast<ConstantSDNode>(Val&: Offen)->getSExtValue() &&
1841 !cast<ConstantSDNode>(Val&: Idxen)->getSExtValue() &&
1842 !cast<ConstantSDNode>(Val&: Addr64)->getSExtValue()) {
1843 uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1844 maskTrailingOnes<uint64_t>(N: 32); // Size
1845 SDLoc DL(Addr);
1846
1847 const SITargetLowering& Lowering =
1848 *static_cast<const SITargetLowering*>(getTargetLowering());
1849
1850 SRsrc = SDValue(Lowering.buildRSRC(DAG&: *CurDAG, DL, Ptr, RsrcDword1: 0, RsrcDword2And3: Rsrc), 0);
1851 return true;
1852 }
1853 return false;
1854}
1855
1856bool AMDGPUDAGToDAGISel::SelectBUFSOffset(SDValue ByteOffsetNode,
1857 SDValue &SOffset) const {
1858 if (Subtarget->hasRestrictedSOffset() && isNullConstant(V: ByteOffsetNode)) {
1859 SOffset = CurDAG->getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32);
1860 return true;
1861 }
1862
1863 SOffset = ByteOffsetNode;
1864 return true;
1865}
1866
1867// Find a load or store from corresponding pattern root.
1868// Roots may be build_vector, bitconvert or their combinations.
1869static MemSDNode* findMemSDNode(SDNode *N) {
1870 N = AMDGPUTargetLowering::stripBitcast(Val: SDValue(N,0)).getNode();
1871 if (MemSDNode *MN = dyn_cast<MemSDNode>(Val: N))
1872 return MN;
1873 assert(isa<BuildVectorSDNode>(N));
1874 for (SDValue V : N->op_values())
1875 if (MemSDNode *MN =
1876 dyn_cast<MemSDNode>(Val: AMDGPUTargetLowering::stripBitcast(Val: V)))
1877 return MN;
1878 llvm_unreachable("cannot find MemSDNode in the pattern!");
1879}
1880
1881bool AMDGPUDAGToDAGISel::SelectFlatOffsetImpl(
1882 SDNode *N, SDValue Addr, SDValue &VAddr, SDValue &Offset,
1883 AMDGPU::FlatAddrSpace FlatVariant) const {
1884 using AMDGPU::FlatAddrSpace;
1885 int64_t OffsetVal = 0;
1886
1887 unsigned AS = findMemSDNode(N)->getAddressSpace();
1888
1889 bool CanHaveFlatSegmentOffsetBug =
1890 Subtarget->hasFlatSegmentOffsetBug() &&
1891 FlatVariant == FlatAddrSpace::FLAT &&
1892 (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS);
1893
1894 if (Subtarget->hasFlatInstOffsets() && !CanHaveFlatSegmentOffsetBug) {
1895 SDValue N0, N1;
1896 if (isBaseWithConstantOffset64(Addr, LHS&: N0, RHS&: N1) &&
1897 (FlatVariant != FlatAddrSpace::FlatScratch ||
1898 isFlatScratchBaseLegal(Addr))) {
1899 int64_t COffsetVal = cast<ConstantSDNode>(Val&: N1)->getSExtValue();
1900
1901 // Adding the offset to the base address in a FLAT instruction must not
1902 // change the memory aperture in which the address falls. Therefore we can
1903 // only fold offsets from inbounds GEPs into FLAT instructions.
1904 bool IsInBounds =
1905 Addr.getOpcode() == ISD::PTRADD && Addr->getFlags().hasInBounds();
1906 if (COffsetVal == 0 || FlatVariant != FlatAddrSpace::FLAT || IsInBounds) {
1907 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1908 if (TII->isLegalFLATOffset(Offset: COffsetVal, AddrSpace: AS, FlatVariant)) {
1909 Addr = N0;
1910 OffsetVal = COffsetVal;
1911 } else {
1912 // If the offset doesn't fit, put the low bits into the offset field
1913 // and add the rest.
1914 //
1915 // For a FLAT instruction the hardware decides whether to access
1916 // global/scratch/shared memory based on the high bits of vaddr,
1917 // ignoring the offset field, so we have to ensure that when we add
1918 // remainder to vaddr it still points into the same underlying object.
1919 // The easiest way to do that is to make sure that we split the offset
1920 // into two pieces that are both >= 0 or both <= 0.
1921
1922 SDLoc DL(N);
1923 uint64_t RemainderOffset;
1924
1925 std::tie(args&: OffsetVal, args&: RemainderOffset) =
1926 TII->splitFlatOffset(COffsetVal, AddrSpace: AS, FlatVariant);
1927
1928 SDValue AddOffsetLo =
1929 getMaterializedScalarImm32(Val: Lo_32(Value: RemainderOffset), DL);
1930 SDValue Clamp = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1);
1931
1932 if (Addr.getValueType().getSizeInBits() == 32) {
1933 SmallVector<SDValue, 3> Opnds;
1934 Opnds.push_back(Elt: N0);
1935 Opnds.push_back(Elt: AddOffsetLo);
1936 unsigned AddOp = AMDGPU::V_ADD_CO_U32_e32;
1937 if (Subtarget->hasAddNoCarryInsts()) {
1938 AddOp = AMDGPU::V_ADD_U32_e64;
1939 Opnds.push_back(Elt: Clamp);
1940 }
1941 Addr =
1942 SDValue(CurDAG->getMachineNode(Opcode: AddOp, dl: DL, VT: MVT::i32, Ops: Opnds), 0);
1943 } else {
1944 // TODO: Should this try to use a scalar add pseudo if the base
1945 // address is uniform and saddr is usable?
1946 SDValue Sub0 =
1947 CurDAG->getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32);
1948 SDValue Sub1 =
1949 CurDAG->getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32);
1950
1951 SDNode *N0Lo = CurDAG->getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG,
1952 dl: DL, VT: MVT::i32, Op1: N0, Op2: Sub0);
1953 SDNode *N0Hi = CurDAG->getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG,
1954 dl: DL, VT: MVT::i32, Op1: N0, Op2: Sub1);
1955
1956 SDValue AddOffsetHi =
1957 getMaterializedScalarImm32(Val: Hi_32(Value: RemainderOffset), DL);
1958
1959 SDVTList VTs = CurDAG->getVTList(VT1: MVT::i32, VT2: MVT::i1);
1960
1961 SDNode *Add =
1962 CurDAG->getMachineNode(Opcode: AMDGPU::V_ADD_CO_U32_e64, dl: DL, VTs,
1963 Ops: {AddOffsetLo, SDValue(N0Lo, 0), Clamp});
1964
1965 SDNode *Addc = CurDAG->getMachineNode(
1966 Opcode: AMDGPU::V_ADDC_U32_e64, dl: DL, VTs,
1967 Ops: {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp});
1968
1969 SDValue RegSequenceArgs[] = {
1970 CurDAG->getTargetConstant(Val: AMDGPU::VReg_64RegClassID, DL,
1971 VT: MVT::i32),
1972 SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1};
1973
1974 Addr = SDValue(CurDAG->getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL,
1975 VT: MVT::i64, Ops: RegSequenceArgs),
1976 0);
1977 }
1978 }
1979 }
1980 }
1981 }
1982
1983 VAddr = Addr;
1984 Offset = CurDAG->getSignedTargetConstant(Val: OffsetVal, DL: SDLoc(), VT: MVT::i32);
1985 return true;
1986}
1987
1988bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N, SDValue Addr,
1989 SDValue &VAddr,
1990 SDValue &Offset) const {
1991 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset,
1992 FlatVariant: AMDGPU::FlatAddrSpace::FLAT);
1993}
1994
1995bool AMDGPUDAGToDAGISel::SelectGlobalOffset(SDNode *N, SDValue Addr,
1996 SDValue &VAddr,
1997 SDValue &Offset) const {
1998 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset,
1999 FlatVariant: AMDGPU::FlatAddrSpace::FlatGlobal);
2000}
2001
2002bool AMDGPUDAGToDAGISel::SelectScratchOffset(SDNode *N, SDValue Addr,
2003 SDValue &VAddr,
2004 SDValue &Offset) const {
2005 return SelectFlatOffsetImpl(N, Addr, VAddr, Offset,
2006 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch);
2007}
2008
2009// If this matches *_extend i32:x, return x
2010// Otherwise if the value is I32 returns x.
2011static SDValue matchExtFromI32orI32(SDValue Op, bool IsSigned,
2012 const SelectionDAG *DAG) {
2013 if (Op.getValueType() == MVT::i32)
2014 return Op;
2015
2016 if (Op.getOpcode() != (IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND) &&
2017 Op.getOpcode() != ISD::ANY_EXTEND &&
2018 !(Op.getOpcode() == (IsSigned ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND) &&
2019 DAG->SignBitIsZero(Op: Op.getOperand(i: 0))))
2020 return SDValue();
2021
2022 SDValue ExtSrc = Op.getOperand(i: 0);
2023 return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
2024}
2025
2026// Match (64-bit SGPR base) + (zext vgpr offset) + sext(imm offset)
2027// or (64-bit SGPR base) + (sext vgpr offset) + sext(imm offset)
2028bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, SDValue Addr,
2029 SDValue &SAddr, SDValue &VOffset,
2030 SDValue &Offset, bool &ScaleOffset,
2031 bool NeedIOffset) const {
2032 using AMDGPU::FlatAddrSpace;
2033 int64_t ImmOffset = 0;
2034 ScaleOffset = false;
2035
2036 // Match the immediate offset first, which canonically is moved as low as
2037 // possible.
2038
2039 SDValue LHS, RHS;
2040 if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
2041 int64_t COffsetVal = cast<ConstantSDNode>(Val&: RHS)->getSExtValue();
2042 const SIInstrInfo *TII = Subtarget->getInstrInfo();
2043
2044 if (NeedIOffset &&
2045 TII->isLegalFLATOffset(Offset: COffsetVal, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS,
2046 FlatVariant: FlatAddrSpace::FlatGlobal)) {
2047 Addr = LHS;
2048 ImmOffset = COffsetVal;
2049 } else if (!LHS->isDivergent()) {
2050 if (COffsetVal > 0) {
2051 SDLoc SL(N);
2052 // saddr + large_offset -> saddr +
2053 // (voffset = large_offset & ~MaxOffset) +
2054 // (large_offset & MaxOffset);
2055 int64_t SplitImmOffset = 0, RemainderOffset = COffsetVal;
2056 if (NeedIOffset) {
2057 std::tie(args&: SplitImmOffset, args&: RemainderOffset) = TII->splitFlatOffset(
2058 COffsetVal, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS, FlatVariant: FlatAddrSpace::FlatGlobal);
2059 }
2060
2061 if (Subtarget->hasSignedGVSOffset() ? isInt<32>(x: RemainderOffset)
2062 : isUInt<32>(x: RemainderOffset)) {
2063 SDNode *VMov = CurDAG->getMachineNode(
2064 Opcode: AMDGPU::V_MOV_B32_e32, dl: SL, VT: MVT::i32,
2065 Op1: CurDAG->getTargetConstant(Val: RemainderOffset, DL: SDLoc(), VT: MVT::i32));
2066 VOffset = SDValue(VMov, 0);
2067 SAddr = LHS;
2068 Offset = CurDAG->getTargetConstant(Val: SplitImmOffset, DL: SDLoc(), VT: MVT::i32);
2069 return true;
2070 }
2071 }
2072
2073 // We are adding a 64 bit SGPR and a constant. If constant bus limit
2074 // is 1 we would need to perform 1 or 2 extra moves for each half of
2075 // the constant and it is better to do a scalar add and then issue a
2076 // single VALU instruction to materialize zero. Otherwise it is less
2077 // instructions to perform VALU adds with immediates or inline literals.
2078 unsigned NumLiterals =
2079 !TII->isInlineConstant(Imm: APInt(32, Lo_32(Value: COffsetVal))) +
2080 !TII->isInlineConstant(Imm: APInt(32, Hi_32(Value: COffsetVal)));
2081 if (Subtarget->getConstantBusLimit(Opcode: AMDGPU::V_ADD_U32_e64) > NumLiterals)
2082 return false;
2083 }
2084 }
2085
2086 // Match the variable offset.
2087 if (Addr->isAnyAdd()) {
2088 LHS = Addr.getOperand(i: 0);
2089
2090 if (!LHS->isDivergent()) {
2091 // add (i64 sgpr), (*_extend (i32 vgpr))
2092 RHS = Addr.getOperand(i: 1);
2093 ScaleOffset = SelectScaleOffset(N, Offset&: RHS, IsSigned: Subtarget->hasSignedGVSOffset());
2094 if (SDValue ExtRHS = matchExtFromI32orI32(
2095 Op: RHS, IsSigned: Subtarget->hasSignedGVSOffset(), DAG: CurDAG)) {
2096 SAddr = LHS;
2097 VOffset = ExtRHS;
2098 }
2099 }
2100
2101 RHS = Addr.getOperand(i: 1);
2102 if (!SAddr && !RHS->isDivergent()) {
2103 // add (*_extend (i32 vgpr)), (i64 sgpr)
2104 ScaleOffset = SelectScaleOffset(N, Offset&: LHS, IsSigned: Subtarget->hasSignedGVSOffset());
2105 if (SDValue ExtLHS = matchExtFromI32orI32(
2106 Op: LHS, IsSigned: Subtarget->hasSignedGVSOffset(), DAG: CurDAG)) {
2107 SAddr = RHS;
2108 VOffset = ExtLHS;
2109 }
2110 }
2111
2112 if (SAddr) {
2113 Offset = CurDAG->getSignedTargetConstant(Val: ImmOffset, DL: SDLoc(), VT: MVT::i32);
2114 return true;
2115 }
2116 }
2117
2118 if (Subtarget->hasScaleOffset() &&
2119 (Addr.getOpcode() == (Subtarget->hasSignedGVSOffset()
2120 ? AMDGPUISD::MAD_I64_I32
2121 : AMDGPUISD::MAD_U64_U32) ||
2122 (Addr.getOpcode() == AMDGPUISD::MAD_U64_U32 &&
2123 CurDAG->SignBitIsZero(Op: Addr.getOperand(i: 0)))) &&
2124 Addr.getOperand(i: 0)->isDivergent() &&
2125 isa<ConstantSDNode>(Val: Addr.getOperand(i: 1)) &&
2126 !Addr.getOperand(i: 2)->isDivergent()) {
2127 // mad_u64_u32 (i32 vgpr), (i32 c), (i64 sgpr)
2128 unsigned Size =
2129 (unsigned)cast<MemSDNode>(Val: N)->getMemoryVT().getFixedSizeInBits() / 8;
2130 ScaleOffset = Addr.getConstantOperandVal(i: 1) == Size;
2131 if (ScaleOffset) {
2132 SAddr = Addr.getOperand(i: 2);
2133 VOffset = Addr.getOperand(i: 0);
2134 Offset = CurDAG->getTargetConstant(Val: ImmOffset, DL: SDLoc(), VT: MVT::i32);
2135 return true;
2136 }
2137 }
2138
2139 if (Addr->isDivergent() || Addr.isUndef() || isa<ConstantSDNode>(Val: Addr))
2140 return false;
2141
2142 // It's cheaper to materialize a single 32-bit zero for vaddr than the two
2143 // moves required to copy a 64-bit SGPR to VGPR.
2144 SAddr = Addr;
2145 SDNode *VMov =
2146 CurDAG->getMachineNode(Opcode: AMDGPU::V_MOV_B32_e32, dl: SDLoc(Addr), VT: MVT::i32,
2147 Op1: CurDAG->getTargetConstant(Val: 0, DL: SDLoc(), VT: MVT::i32));
2148 VOffset = SDValue(VMov, 0);
2149 Offset = CurDAG->getSignedTargetConstant(Val: ImmOffset, DL: SDLoc(), VT: MVT::i32);
2150 return true;
2151}
2152
2153bool AMDGPUDAGToDAGISel::SelectGlobalSAddr(SDNode *N, SDValue Addr,
2154 SDValue &SAddr, SDValue &VOffset,
2155 SDValue &Offset,
2156 SDValue &CPol) const {
2157 bool ScaleOffset;
2158 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, Offset, ScaleOffset))
2159 return false;
2160
2161 CPol = CurDAG->getTargetConstant(Val: ScaleOffset ? AMDGPU::CPol::SCAL : 0,
2162 DL: SDLoc(), VT: MVT::i32);
2163 return true;
2164}
2165
2166bool AMDGPUDAGToDAGISel::SelectGlobalSAddrCPol(SDNode *N, SDValue Addr,
2167 SDValue &SAddr, SDValue &VOffset,
2168 SDValue &Offset,
2169 SDValue &CPol) const {
2170 bool ScaleOffset;
2171 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, Offset, ScaleOffset))
2172 return false;
2173
2174 // We are assuming CPol is always the last operand of the intrinsic.
2175 auto PassedCPol =
2176 N->getConstantOperandVal(Num: N->getNumOperands() - 1) & ~AMDGPU::CPol::SCAL;
2177 CPol = CurDAG->getTargetConstant(
2178 Val: (ScaleOffset ? AMDGPU::CPol::SCAL : 0) | PassedCPol, DL: SDLoc(), VT: MVT::i32);
2179 return true;
2180}
2181
2182bool AMDGPUDAGToDAGISel::SelectGlobalSAddrCPolM0(SDNode *N, SDValue Addr,
2183 SDValue &SAddr,
2184 SDValue &VOffset,
2185 SDValue &Offset,
2186 SDValue &CPol) const {
2187 bool ScaleOffset;
2188 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, Offset, ScaleOffset))
2189 return false;
2190
2191 // We are assuming CPol is second from last operand of the intrinsic.
2192 auto PassedCPol =
2193 N->getConstantOperandVal(Num: N->getNumOperands() - 2) & ~AMDGPU::CPol::SCAL;
2194 CPol = CurDAG->getTargetConstant(
2195 Val: (ScaleOffset ? AMDGPU::CPol::SCAL : 0) | PassedCPol, DL: SDLoc(), VT: MVT::i32);
2196 return true;
2197}
2198
2199bool AMDGPUDAGToDAGISel::SelectGlobalSAddrGLC(SDNode *N, SDValue Addr,
2200 SDValue &SAddr, SDValue &VOffset,
2201 SDValue &Offset,
2202 SDValue &CPol) const {
2203 bool ScaleOffset;
2204 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, Offset, ScaleOffset))
2205 return false;
2206
2207 unsigned CPolVal = (ScaleOffset ? AMDGPU::CPol::SCAL : 0) | AMDGPU::CPol::GLC;
2208 CPol = CurDAG->getTargetConstant(Val: CPolVal, DL: SDLoc(), VT: MVT::i32);
2209 return true;
2210}
2211
2212bool AMDGPUDAGToDAGISel::SelectGlobalSAddrNoIOffset(SDNode *N, SDValue Addr,
2213 SDValue &SAddr,
2214 SDValue &VOffset,
2215 SDValue &CPol) const {
2216 bool ScaleOffset;
2217 SDValue DummyOffset;
2218 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, Offset&: DummyOffset, ScaleOffset,
2219 NeedIOffset: false))
2220 return false;
2221
2222 // We are assuming CPol is always the last operand of the intrinsic.
2223 auto PassedCPol =
2224 N->getConstantOperandVal(Num: N->getNumOperands() - 1) & ~AMDGPU::CPol::SCAL;
2225 CPol = CurDAG->getTargetConstant(
2226 Val: (ScaleOffset ? AMDGPU::CPol::SCAL : 0) | PassedCPol, DL: SDLoc(), VT: MVT::i32);
2227 return true;
2228}
2229
2230bool AMDGPUDAGToDAGISel::SelectGlobalSAddrNoIOffsetM0(SDNode *N, SDValue Addr,
2231 SDValue &SAddr,
2232 SDValue &VOffset,
2233 SDValue &CPol) const {
2234 bool ScaleOffset;
2235 SDValue DummyOffset;
2236 if (!SelectGlobalSAddr(N, Addr, SAddr, VOffset, Offset&: DummyOffset, ScaleOffset,
2237 NeedIOffset: false))
2238 return false;
2239
2240 // We are assuming CPol is second from last operand of the intrinsic.
2241 auto PassedCPol =
2242 N->getConstantOperandVal(Num: N->getNumOperands() - 2) & ~AMDGPU::CPol::SCAL;
2243 CPol = CurDAG->getTargetConstant(
2244 Val: (ScaleOffset ? AMDGPU::CPol::SCAL : 0) | PassedCPol, DL: SDLoc(), VT: MVT::i32);
2245 return true;
2246}
2247
2248static SDValue SelectSAddrFI(SelectionDAG *CurDAG, SDValue SAddr) {
2249 if (auto *FI = dyn_cast<FrameIndexSDNode>(Val&: SAddr)) {
2250 SAddr = CurDAG->getTargetFrameIndex(FI: FI->getIndex(), VT: FI->getValueType(ResNo: 0));
2251 } else if (SAddr.getOpcode() == ISD::ADD &&
2252 isa<FrameIndexSDNode>(Val: SAddr.getOperand(i: 0))) {
2253 // Materialize this into a scalar move for scalar address to avoid
2254 // readfirstlane.
2255 auto *FI = cast<FrameIndexSDNode>(Val: SAddr.getOperand(i: 0));
2256 SDValue TFI = CurDAG->getTargetFrameIndex(FI: FI->getIndex(),
2257 VT: FI->getValueType(ResNo: 0));
2258 SAddr = SDValue(CurDAG->getMachineNode(Opcode: AMDGPU::S_ADD_I32, dl: SDLoc(SAddr),
2259 VT: MVT::i32, Op1: TFI, Op2: SAddr.getOperand(i: 1)),
2260 0);
2261 }
2262
2263 return SAddr;
2264}
2265
2266// Match (32-bit SGPR base) + sext(imm offset)
2267bool AMDGPUDAGToDAGISel::SelectScratchSAddr(SDNode *Parent, SDValue Addr,
2268 SDValue &SAddr,
2269 SDValue &Offset) const {
2270 using AMDGPU::FlatAddrSpace;
2271 if (Addr->isDivergent())
2272 return false;
2273
2274 SDLoc DL(Addr);
2275
2276 int64_t COffsetVal = 0;
2277
2278 if (CurDAG->isBaseWithConstantOffset(Op: Addr) && isFlatScratchBaseLegal(Addr)) {
2279 COffsetVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
2280 SAddr = Addr.getOperand(i: 0);
2281 } else {
2282 SAddr = Addr;
2283 }
2284
2285 SAddr = SelectSAddrFI(CurDAG, SAddr);
2286
2287 const SIInstrInfo *TII = Subtarget->getInstrInfo();
2288
2289 if (!TII->isLegalFLATOffset(Offset: COffsetVal, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
2290 FlatVariant: FlatAddrSpace::FlatScratch)) {
2291 int64_t SplitImmOffset, RemainderOffset;
2292 std::tie(args&: SplitImmOffset, args&: RemainderOffset) = TII->splitFlatOffset(
2293 COffsetVal, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS, FlatVariant: FlatAddrSpace::FlatScratch);
2294
2295 COffsetVal = SplitImmOffset;
2296
2297 SDValue AddOffset =
2298 SAddr.getOpcode() == ISD::TargetFrameIndex
2299 ? getMaterializedScalarImm32(Val: Lo_32(Value: RemainderOffset), DL)
2300 : CurDAG->getSignedTargetConstant(Val: RemainderOffset, DL, VT: MVT::i32);
2301 SAddr = SDValue(CurDAG->getMachineNode(Opcode: AMDGPU::S_ADD_I32, dl: DL, VT: MVT::i32,
2302 Op1: SAddr, Op2: AddOffset),
2303 0);
2304 }
2305
2306 Offset = CurDAG->getSignedTargetConstant(Val: COffsetVal, DL, VT: MVT::i32);
2307
2308 return true;
2309}
2310
2311// Check whether the flat scratch SVS swizzle bug affects this access.
2312bool AMDGPUDAGToDAGISel::checkFlatScratchSVSSwizzleBug(
2313 SDValue VAddr, SDValue SAddr, uint64_t ImmOffset) const {
2314 if (!Subtarget->hasFlatScratchSVSSwizzleBug())
2315 return false;
2316
2317 // The bug affects the swizzling of SVS accesses if there is any carry out
2318 // from the two low order bits (i.e. from bit 1 into bit 2) when adding
2319 // voffset to (soffset + inst_offset).
2320 KnownBits VKnown = CurDAG->computeKnownBits(Op: VAddr);
2321 KnownBits SKnown =
2322 KnownBits::add(LHS: CurDAG->computeKnownBits(Op: SAddr),
2323 RHS: KnownBits::makeConstant(C: APInt(32, ImmOffset,
2324 /*isSigned=*/true)));
2325 uint64_t VMax = VKnown.getMaxValue().getZExtValue();
2326 uint64_t SMax = SKnown.getMaxValue().getZExtValue();
2327 return (VMax & 3) + (SMax & 3) >= 4;
2328}
2329
2330bool AMDGPUDAGToDAGISel::SelectScratchSVAddr(SDNode *N, SDValue Addr,
2331 SDValue &VAddr, SDValue &SAddr,
2332 SDValue &Offset,
2333 SDValue &CPol) const {
2334 int64_t ImmOffset = 0;
2335
2336 SDValue LHS, RHS;
2337 SDValue OrigAddr = Addr;
2338 if (isBaseWithConstantOffset64(Addr, LHS, RHS)) {
2339 int64_t COffsetVal = cast<ConstantSDNode>(Val&: RHS)->getSExtValue();
2340 const SIInstrInfo *TII = Subtarget->getInstrInfo();
2341
2342 if (TII->isLegalFLATOffset(Offset: COffsetVal, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
2343 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch)) {
2344 Addr = LHS;
2345 ImmOffset = COffsetVal;
2346 } else if (!LHS->isDivergent() && COffsetVal > 0) {
2347 SDLoc SL(N);
2348 // saddr + large_offset -> saddr + (vaddr = large_offset & ~MaxOffset) +
2349 // (large_offset & MaxOffset);
2350 int64_t SplitImmOffset, RemainderOffset;
2351 std::tie(args&: SplitImmOffset, args&: RemainderOffset) =
2352 TII->splitFlatOffset(COffsetVal, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
2353 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch);
2354
2355 if (isUInt<32>(x: RemainderOffset)) {
2356 SDNode *VMov = CurDAG->getMachineNode(
2357 Opcode: AMDGPU::V_MOV_B32_e32, dl: SL, VT: MVT::i32,
2358 Op1: CurDAG->getTargetConstant(Val: RemainderOffset, DL: SDLoc(), VT: MVT::i32));
2359 VAddr = SDValue(VMov, 0);
2360 SAddr = LHS;
2361 if (!isFlatScratchBaseLegal(Addr))
2362 return false;
2363 if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, ImmOffset: SplitImmOffset))
2364 return false;
2365 Offset = CurDAG->getTargetConstant(Val: SplitImmOffset, DL: SDLoc(), VT: MVT::i32);
2366 CPol = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(), VT: MVT::i32);
2367 return true;
2368 }
2369 }
2370 }
2371
2372 if (Addr.getOpcode() != ISD::ADD)
2373 return false;
2374
2375 LHS = Addr.getOperand(i: 0);
2376 RHS = Addr.getOperand(i: 1);
2377
2378 if (!LHS->isDivergent() && RHS->isDivergent()) {
2379 SAddr = LHS;
2380 VAddr = RHS;
2381 } else if (!RHS->isDivergent() && LHS->isDivergent()) {
2382 SAddr = RHS;
2383 VAddr = LHS;
2384 } else {
2385 return false;
2386 }
2387
2388 if (OrigAddr != Addr) {
2389 if (!isFlatScratchBaseLegalSVImm(Addr: OrigAddr))
2390 return false;
2391 } else {
2392 if (!isFlatScratchBaseLegalSV(Addr: OrigAddr))
2393 return false;
2394 }
2395
2396 if (checkFlatScratchSVSSwizzleBug(VAddr, SAddr, ImmOffset))
2397 return false;
2398 SAddr = SelectSAddrFI(CurDAG, SAddr);
2399 Offset = CurDAG->getSignedTargetConstant(Val: ImmOffset, DL: SDLoc(), VT: MVT::i32);
2400
2401 bool ScaleOffset = SelectScaleOffset(N, Offset&: VAddr, IsSigned: true /* IsSigned */);
2402 CPol = CurDAG->getTargetConstant(Val: ScaleOffset ? AMDGPU::CPol::SCAL : 0,
2403 DL: SDLoc(), VT: MVT::i32);
2404 return true;
2405}
2406
2407// For unbuffered smem loads, it is illegal for the Immediate Offset to be
2408// negative if the resulting (Offset + (M0 or SOffset or zero) is negative.
2409// Handle the case where the Immediate Offset + SOffset is negative.
2410bool AMDGPUDAGToDAGISel::isSOffsetLegalWithImmOffset(SDValue *SOffset,
2411 bool Imm32Only,
2412 bool IsBuffer,
2413 int64_t ImmOffset) const {
2414 if (!IsBuffer && !Imm32Only && ImmOffset < 0 &&
2415 AMDGPU::hasSMRDSignedImmOffset(ST: *Subtarget)) {
2416 KnownBits SKnown = CurDAG->computeKnownBits(Op: *SOffset);
2417 if (ImmOffset + SKnown.getMinValue().getSExtValue() < 0)
2418 return false;
2419 }
2420
2421 return true;
2422}
2423
2424// Given \p Offset and load node \p N check if an \p Offset is a multiple of
2425// the load byte size. If it is update \p Offset to a pre-scaled value and
2426// return true.
2427bool AMDGPUDAGToDAGISel::SelectScaleOffset(SDNode *N, SDValue &Offset,
2428 bool IsSigned) const {
2429 bool ScaleOffset = false;
2430 if (!Subtarget->hasScaleOffset() || !Offset)
2431 return false;
2432
2433 unsigned Size =
2434 (unsigned)cast<MemSDNode>(Val: N)->getMemoryVT().getFixedSizeInBits() / 8;
2435
2436 SDValue Off = Offset;
2437 if (SDValue Ext = matchExtFromI32orI32(Op: Offset, IsSigned, DAG: CurDAG))
2438 Off = Ext;
2439
2440 if (isPowerOf2_32(Value: Size) && Off.getOpcode() == ISD::SHL) {
2441 if (auto *C = dyn_cast<ConstantSDNode>(Val: Off.getOperand(i: 1)))
2442 ScaleOffset = C->getZExtValue() == Log2_32(Value: Size);
2443 } else if (Offset.getOpcode() == ISD::MUL ||
2444 (IsSigned && Offset.getOpcode() == AMDGPUISD::MUL_I24) ||
2445 Offset.getOpcode() == AMDGPUISD::MUL_U24 ||
2446 (Offset.isMachineOpcode() &&
2447 Offset.getMachineOpcode() ==
2448 (IsSigned ? AMDGPU::S_MUL_I64_I32_PSEUDO
2449 : AMDGPU::S_MUL_U64_U32_PSEUDO))) {
2450 if (auto *C = dyn_cast<ConstantSDNode>(Val: Offset.getOperand(i: 1)))
2451 ScaleOffset = C->getZExtValue() == Size;
2452 }
2453
2454 if (ScaleOffset)
2455 Offset = Off.getOperand(i: 0);
2456
2457 return ScaleOffset;
2458}
2459
2460// Match an immediate (if Offset is not null) or an SGPR (if SOffset is
2461// not null) offset. If Imm32Only is true, match only 32-bit immediate
2462// offsets available on CI.
2463bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDNode *N, SDValue ByteOffsetNode,
2464 SDValue *SOffset, SDValue *Offset,
2465 bool Imm32Only, bool IsBuffer,
2466 bool HasSOffset, int64_t ImmOffset,
2467 bool *ScaleOffset) const {
2468 assert((!SOffset || !Offset) &&
2469 "Cannot match both soffset and offset at the same time!");
2470
2471 if (ScaleOffset) {
2472 assert(N && SOffset);
2473
2474 *ScaleOffset = SelectScaleOffset(N, Offset&: ByteOffsetNode, IsSigned: false /* IsSigned */);
2475 }
2476
2477 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: ByteOffsetNode);
2478 if (!C) {
2479 if (!SOffset)
2480 return false;
2481
2482 if (ByteOffsetNode.getValueType().isScalarInteger() &&
2483 ByteOffsetNode.getValueType().getSizeInBits() == 32) {
2484 *SOffset = ByteOffsetNode;
2485 return isSOffsetLegalWithImmOffset(SOffset, Imm32Only, IsBuffer,
2486 ImmOffset);
2487 }
2488 if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) {
2489 if (ByteOffsetNode.getOperand(i: 0).getValueType().getSizeInBits() == 32) {
2490 *SOffset = ByteOffsetNode.getOperand(i: 0);
2491 return isSOffsetLegalWithImmOffset(SOffset, Imm32Only, IsBuffer,
2492 ImmOffset);
2493 }
2494 }
2495 return false;
2496 }
2497
2498 SDLoc SL(ByteOffsetNode);
2499
2500 // GFX9 and GFX10 have signed byte immediate offsets. The immediate
2501 // offset for S_BUFFER instructions is unsigned.
2502 int64_t ByteOffset = IsBuffer ? C->getZExtValue() : C->getSExtValue();
2503 std::optional<int64_t> EncodedOffset = AMDGPU::getSMRDEncodedOffset(
2504 ST: *Subtarget, ByteOffset, IsBuffer, HasSOffset);
2505 if (EncodedOffset && Offset && !Imm32Only) {
2506 *Offset = CurDAG->getSignedTargetConstant(Val: *EncodedOffset, DL: SL, VT: MVT::i32);
2507 return true;
2508 }
2509
2510 // SGPR and literal offsets are unsigned.
2511 if (ByteOffset < 0)
2512 return false;
2513
2514 EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(ST: *Subtarget, ByteOffset);
2515 if (EncodedOffset && Offset && Imm32Only) {
2516 *Offset = CurDAG->getTargetConstant(Val: *EncodedOffset, DL: SL, VT: MVT::i32);
2517 return true;
2518 }
2519
2520 if (!isUInt<32>(x: ByteOffset) && !isInt<32>(x: ByteOffset))
2521 return false;
2522
2523 if (SOffset) {
2524 SDValue C32Bit = CurDAG->getTargetConstant(Val: ByteOffset, DL: SL, VT: MVT::i32);
2525 *SOffset = SDValue(
2526 CurDAG->getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: SL, VT: MVT::i32, Op1: C32Bit), 0);
2527 return true;
2528 }
2529
2530 return false;
2531}
2532
2533SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
2534 if (Addr.getValueType() != MVT::i32)
2535 return Addr;
2536
2537 // Zero-extend a 32-bit address.
2538 SDLoc SL(Addr);
2539
2540 const MachineFunction &MF = CurDAG->getMachineFunction();
2541 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2542 unsigned AddrHiVal = Info->get32BitAddressHighBits();
2543 SDValue AddrHi = CurDAG->getTargetConstant(Val: AddrHiVal, DL: SL, VT: MVT::i32);
2544
2545 const SDValue Ops[] = {
2546 CurDAG->getTargetConstant(Val: AMDGPU::SReg_64_XEXECRegClassID, DL: SL, VT: MVT::i32),
2547 Addr,
2548 CurDAG->getTargetConstant(Val: AMDGPU::sub0, DL: SL, VT: MVT::i32),
2549 SDValue(CurDAG->getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: SL, VT: MVT::i32, Op1: AddrHi),
2550 0),
2551 CurDAG->getTargetConstant(Val: AMDGPU::sub1, DL: SL, VT: MVT::i32),
2552 };
2553
2554 return SDValue(CurDAG->getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: SL, VT: MVT::i64,
2555 Ops), 0);
2556}
2557
2558// Match a base and an immediate (if Offset is not null) or an SGPR (if
2559// SOffset is not null) or an immediate+SGPR offset. If Imm32Only is
2560// true, match only 32-bit immediate offsets available on CI.
2561bool AMDGPUDAGToDAGISel::SelectSMRDBaseOffset(SDNode *N, SDValue Addr,
2562 SDValue &SBase, SDValue *SOffset,
2563 SDValue *Offset, bool Imm32Only,
2564 bool IsBuffer, bool HasSOffset,
2565 int64_t ImmOffset,
2566 bool *ScaleOffset) const {
2567 if (SOffset && Offset) {
2568 assert(!Imm32Only && !IsBuffer);
2569 SDValue B;
2570
2571 if (!SelectSMRDBaseOffset(N, Addr, SBase&: B, SOffset: nullptr, Offset, Imm32Only: false, IsBuffer: false, HasSOffset: true))
2572 return false;
2573
2574 int64_t ImmOff = 0;
2575 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: *Offset))
2576 ImmOff = C->getSExtValue();
2577
2578 return SelectSMRDBaseOffset(N, Addr: B, SBase, SOffset, Offset: nullptr, Imm32Only: false, IsBuffer: false,
2579 HasSOffset: true, ImmOffset: ImmOff, ScaleOffset);
2580 }
2581
2582 // A 32-bit (address + offset) should not cause unsigned 32-bit integer
2583 // wraparound, because s_load instructions perform the addition in 64 bits.
2584 if (Addr.getValueType() == MVT::i32 && Addr.getOpcode() == ISD::ADD &&
2585 !Addr->getFlags().hasNoUnsignedWrap())
2586 return false;
2587
2588 SDValue N0, N1;
2589 // Extract the base and offset if possible.
2590 if (Addr->isAnyAdd() || CurDAG->isADDLike(Op: Addr)) {
2591 N0 = Addr.getOperand(i: 0);
2592 N1 = Addr.getOperand(i: 1);
2593 } else if (getBaseWithOffsetUsingSplitOR(DAG&: *CurDAG, Addr, N0, N1)) {
2594 assert(N0 && N1 && isa<ConstantSDNode>(N1));
2595 }
2596 if (!N0 || !N1)
2597 return false;
2598
2599 if (SelectSMRDOffset(N, ByteOffsetNode: N1, SOffset, Offset, Imm32Only, IsBuffer, HasSOffset,
2600 ImmOffset, ScaleOffset)) {
2601 SBase = N0;
2602 return true;
2603 }
2604 if (SelectSMRDOffset(N, ByteOffsetNode: N0, SOffset, Offset, Imm32Only, IsBuffer, HasSOffset,
2605 ImmOffset, ScaleOffset)) {
2606 SBase = N1;
2607 return true;
2608 }
2609 return false;
2610}
2611
2612bool AMDGPUDAGToDAGISel::SelectSMRD(SDNode *N, SDValue Addr, SDValue &SBase,
2613 SDValue *SOffset, SDValue *Offset,
2614 bool Imm32Only, bool *ScaleOffset) const {
2615 if (SelectSMRDBaseOffset(N, Addr, SBase, SOffset, Offset, Imm32Only,
2616 /* IsBuffer */ false, /* HasSOffset */ false,
2617 /* ImmOffset */ 0, ScaleOffset)) {
2618 SBase = Expand32BitAddress(Addr: SBase);
2619 return true;
2620 }
2621
2622 if (Addr.getValueType() == MVT::i32 && Offset && !SOffset) {
2623 SBase = Expand32BitAddress(Addr);
2624 *Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT: MVT::i32);
2625 return true;
2626 }
2627
2628 return false;
2629}
2630
2631bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
2632 SDValue &Offset) const {
2633 return SelectSMRD(/* N */ nullptr, Addr, SBase, /* SOffset */ nullptr,
2634 Offset: &Offset);
2635}
2636
2637bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
2638 SDValue &Offset) const {
2639 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2640 return SelectSMRD(/* N */ nullptr, Addr, SBase, /* SOffset */ nullptr,
2641 Offset: &Offset, /* Imm32Only */ true);
2642}
2643
2644bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDNode *N, SDValue Addr, SDValue &SBase,
2645 SDValue &SOffset, SDValue &CPol) const {
2646 bool ScaleOffset;
2647 if (!SelectSMRD(N, Addr, SBase, SOffset: &SOffset, /* Offset */ nullptr,
2648 /* Imm32Only */ false, ScaleOffset: &ScaleOffset))
2649 return false;
2650
2651 CPol = CurDAG->getTargetConstant(Val: ScaleOffset ? AMDGPU::CPol::SCAL : 0,
2652 DL: SDLoc(N), VT: MVT::i32);
2653 return true;
2654}
2655
2656bool AMDGPUDAGToDAGISel::SelectSMRDSgprImm(SDNode *N, SDValue Addr,
2657 SDValue &SBase, SDValue &SOffset,
2658 SDValue &Offset,
2659 SDValue &CPol) const {
2660 bool ScaleOffset;
2661 if (!SelectSMRD(N, Addr, SBase, SOffset: &SOffset, Offset: &Offset, Imm32Only: false, ScaleOffset: &ScaleOffset))
2662 return false;
2663
2664 CPol = CurDAG->getTargetConstant(Val: ScaleOffset ? AMDGPU::CPol::SCAL : 0,
2665 DL: SDLoc(N), VT: MVT::i32);
2666 return true;
2667}
2668
2669bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue N, SDValue &Offset) const {
2670 return SelectSMRDOffset(/* N */ nullptr, ByteOffsetNode: N, /* SOffset */ nullptr, Offset: &Offset,
2671 /* Imm32Only */ false, /* IsBuffer */ true);
2672}
2673
2674bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue N,
2675 SDValue &Offset) const {
2676 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS);
2677 return SelectSMRDOffset(/* N */ nullptr, ByteOffsetNode: N, /* SOffset */ nullptr, Offset: &Offset,
2678 /* Imm32Only */ true, /* IsBuffer */ true);
2679}
2680
2681bool AMDGPUDAGToDAGISel::SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,
2682 SDValue &Offset) const {
2683 // Match the (soffset + offset) pair as a 32-bit register base and
2684 // an immediate offset.
2685 return N.getValueType() == MVT::i32 &&
2686 SelectSMRDBaseOffset(/* N */ nullptr, Addr: N, /* SBase */ SOffset,
2687 /* SOffset*/ nullptr, Offset: &Offset,
2688 /* Imm32Only */ false, /* IsBuffer */ true);
2689}
2690
2691bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
2692 SDValue &Base,
2693 SDValue &Offset) const {
2694 SDLoc DL(Index);
2695
2696 if (CurDAG->isBaseWithConstantOffset(Op: Index)) {
2697 SDValue N0 = Index.getOperand(i: 0);
2698 SDValue N1 = Index.getOperand(i: 1);
2699 ConstantSDNode *C1 = cast<ConstantSDNode>(Val&: N1);
2700
2701 // (add n0, c0)
2702 // Don't peel off the offset (c0) if doing so could possibly lead
2703 // the base (n0) to be negative.
2704 // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset.
2705 if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(Op: N0) ||
2706 (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) {
2707 Base = N0;
2708 Offset = CurDAG->getTargetConstant(Val: C1->getZExtValue(), DL, VT: MVT::i32);
2709 return true;
2710 }
2711 }
2712
2713 if (isa<ConstantSDNode>(Val: Index))
2714 return false;
2715
2716 Base = Index;
2717 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
2718 return true;
2719}
2720
2721SDNode *AMDGPUDAGToDAGISel::getBFE32(bool IsSigned, const SDLoc &DL,
2722 SDValue Val, uint32_t Offset,
2723 uint32_t Width) {
2724 if (Val->isDivergent()) {
2725 unsigned Opcode = IsSigned ? AMDGPU::V_BFE_I32_e64 : AMDGPU::V_BFE_U32_e64;
2726 SDValue Off = CurDAG->getTargetConstant(Val: Offset, DL, VT: MVT::i32);
2727 SDValue W = CurDAG->getTargetConstant(Val: Width, DL, VT: MVT::i32);
2728
2729 return CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::i32, Op1: Val, Op2: Off, Op3: W);
2730 }
2731 unsigned Opcode = IsSigned ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
2732 // Transformation function, pack the offset and width of a BFE into
2733 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
2734 // source, bits [5:0] contain the offset and bits [22:16] the width.
2735 uint32_t PackedVal = Offset | (Width << 16);
2736 SDValue PackedConst = CurDAG->getTargetConstant(Val: PackedVal, DL, VT: MVT::i32);
2737
2738 return CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::i32, Op1: Val, Op2: PackedConst);
2739}
2740
2741void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
2742 // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
2743 // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
2744 // Predicate: 0 < b <= c < 32
2745
2746 const SDValue &Shl = N->getOperand(Num: 0);
2747 ConstantSDNode *B = dyn_cast<ConstantSDNode>(Val: Shl->getOperand(Num: 1));
2748 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
2749
2750 if (B && C) {
2751 uint32_t BVal = B->getZExtValue();
2752 uint32_t CVal = C->getZExtValue();
2753
2754 if (0 < BVal && BVal <= CVal && CVal < 32) {
2755 bool Signed = N->getOpcode() == ISD::SRA;
2756 ReplaceNode(F: N, T: getBFE32(IsSigned: Signed, DL: SDLoc(N), Val: Shl.getOperand(i: 0), Offset: CVal - BVal,
2757 Width: 32 - CVal));
2758 return;
2759 }
2760 }
2761 SelectCode(N);
2762}
2763
2764void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
2765 switch (N->getOpcode()) {
2766 case ISD::AND:
2767 if (N->getOperand(Num: 0).getOpcode() == ISD::SRL) {
2768 // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
2769 // Predicate: isMask(mask)
2770 const SDValue &Srl = N->getOperand(Num: 0);
2771 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Val: Srl.getOperand(i: 1));
2772 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
2773
2774 if (Shift && Mask) {
2775 uint32_t ShiftVal = Shift->getZExtValue();
2776 uint32_t MaskVal = Mask->getZExtValue();
2777
2778 if (isMask_32(Value: MaskVal)) {
2779 uint32_t WidthVal = llvm::popcount(Value: MaskVal);
2780 ReplaceNode(F: N, T: getBFE32(IsSigned: false, DL: SDLoc(N), Val: Srl.getOperand(i: 0), Offset: ShiftVal,
2781 Width: WidthVal));
2782 return;
2783 }
2784 }
2785 }
2786 break;
2787 case ISD::SRL:
2788 if (N->getOperand(Num: 0).getOpcode() == ISD::AND) {
2789 // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
2790 // Predicate: isMask(mask >> b)
2791 const SDValue &And = N->getOperand(Num: 0);
2792 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
2793 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Val: And->getOperand(Num: 1));
2794
2795 if (Shift && Mask) {
2796 uint32_t ShiftVal = Shift->getZExtValue();
2797 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
2798
2799 if (isMask_32(Value: MaskVal)) {
2800 uint32_t WidthVal = llvm::popcount(Value: MaskVal);
2801 ReplaceNode(F: N, T: getBFE32(IsSigned: false, DL: SDLoc(N), Val: And.getOperand(i: 0), Offset: ShiftVal,
2802 Width: WidthVal));
2803 return;
2804 }
2805 }
2806 } else if (N->getOperand(Num: 0).getOpcode() == ISD::SHL) {
2807 SelectS_BFEFromShifts(N);
2808 return;
2809 }
2810 break;
2811 case ISD::SRA:
2812 if (N->getOperand(Num: 0).getOpcode() == ISD::SHL) {
2813 SelectS_BFEFromShifts(N);
2814 return;
2815 }
2816 break;
2817
2818 case ISD::SIGN_EXTEND_INREG: {
2819 // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
2820 SDValue Src = N->getOperand(Num: 0);
2821 if (Src.getOpcode() != ISD::SRL)
2822 break;
2823
2824 const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Val: Src.getOperand(i: 1));
2825 if (!Amt)
2826 break;
2827
2828 unsigned Width = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getSizeInBits();
2829 ReplaceNode(F: N, T: getBFE32(IsSigned: true, DL: SDLoc(N), Val: Src.getOperand(i: 0),
2830 Offset: Amt->getZExtValue(), Width));
2831 return;
2832 }
2833 }
2834
2835 SelectCode(N);
2836}
2837
2838bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
2839 assert(N->getOpcode() == ISD::BRCOND);
2840 if (!N->hasOneUse())
2841 return false;
2842
2843 SDValue Cond = N->getOperand(Num: 1);
2844 if (Cond.getOpcode() == ISD::CopyToReg)
2845 Cond = Cond.getOperand(i: 2);
2846
2847 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
2848 return false;
2849
2850 MVT VT = Cond.getOperand(i: 0).getSimpleValueType();
2851 if (VT == MVT::i32)
2852 return true;
2853
2854 if (VT == MVT::i64) {
2855 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
2856 return (CC == ISD::SETEQ || CC == ISD::SETNE) &&
2857 Subtarget->hasScalarCompareEq64();
2858 }
2859
2860 if ((VT == MVT::f16 || VT == MVT::f32) && Subtarget->hasSALUFloatInsts())
2861 return true;
2862
2863 return false;
2864}
2865
2866static SDValue combineBallotPattern(SDValue VCMP, bool &Negate) {
2867 assert(VCMP->getOpcode() == AMDGPUISD::SETCC);
2868 // Special case for amdgcn.ballot:
2869 // %Cond = i1 (and/or combination of i1 ISD::SETCCs)
2870 // %VCMP = i(WaveSize) AMDGPUISD::SETCC (ext %Cond), 0, setne/seteq
2871 // =>
2872 // Use i1 %Cond value instead of i(WaveSize) %VCMP.
2873 // This is possible because divergent ISD::SETCC is selected as V_CMP and
2874 // Cond becomes a i(WaveSize) full mask value.
2875 // Note that ballot doesn't use SETEQ condition but its easy to support it
2876 // here for completeness, so in this case Negate is set true on return.
2877 auto VCMP_CC = cast<CondCodeSDNode>(Val: VCMP.getOperand(i: 2))->get();
2878 if ((VCMP_CC == ISD::SETEQ || VCMP_CC == ISD::SETNE) &&
2879 isNullConstant(V: VCMP.getOperand(i: 1))) {
2880
2881 auto Cond = VCMP.getOperand(i: 0);
2882 if (ISD::isExtOpcode(Opcode: Cond->getOpcode())) // Skip extension.
2883 Cond = Cond.getOperand(i: 0);
2884
2885 if (isBoolSGPR(V: Cond)) {
2886 Negate = VCMP_CC == ISD::SETEQ;
2887 return Cond;
2888 }
2889 }
2890 return SDValue();
2891}
2892
2893void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
2894 SDValue Cond = N->getOperand(Num: 1);
2895
2896 if (Cond.isUndef()) {
2897 CurDAG->SelectNodeTo(N, MachineOpc: AMDGPU::SI_BR_UNDEF, VT: MVT::Other,
2898 Op1: N->getOperand(Num: 2), Op2: N->getOperand(Num: 0));
2899 return;
2900 }
2901
2902 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
2903
2904 bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
2905 bool AndExec = !UseSCCBr;
2906 bool Negate = false;
2907
2908 if (Cond.getOpcode() == ISD::SETCC &&
2909 Cond->getOperand(Num: 0)->getOpcode() == AMDGPUISD::SETCC) {
2910 SDValue VCMP = Cond->getOperand(Num: 0);
2911 auto CC = cast<CondCodeSDNode>(Val: Cond->getOperand(Num: 2))->get();
2912 if ((CC == ISD::SETEQ || CC == ISD::SETNE) &&
2913 isNullConstant(V: Cond->getOperand(Num: 1)) &&
2914 // We may encounter ballot.i64 in wave32 mode on -O0.
2915 VCMP.getValueType().getSizeInBits() == Subtarget->getWavefrontSize()) {
2916 // %VCMP = i(WaveSize) AMDGPUISD::SETCC ...
2917 // %C = i1 ISD::SETCC %VCMP, 0, setne/seteq
2918 // BRCOND i1 %C, %BB
2919 // =>
2920 // %VCMP = i(WaveSize) AMDGPUISD::SETCC ...
2921 // VCC = COPY i(WaveSize) %VCMP
2922 // S_CBRANCH_VCCNZ/VCCZ %BB
2923 Negate = CC == ISD::SETEQ;
2924 bool NegatedBallot = false;
2925 if (auto BallotCond = combineBallotPattern(VCMP, Negate&: NegatedBallot)) {
2926 Cond = BallotCond;
2927 UseSCCBr = !BallotCond->isDivergent();
2928 Negate = Negate ^ NegatedBallot;
2929 } else {
2930 // TODO: don't use SCC here assuming that AMDGPUISD::SETCC is always
2931 // selected as V_CMP, but this may change for uniform condition.
2932 Cond = VCMP;
2933 UseSCCBr = false;
2934 }
2935 }
2936 // Cond is either V_CMP resulted from AMDGPUISD::SETCC or a combination of
2937 // V_CMPs resulted from ballot or ballot has uniform condition and SCC is
2938 // used.
2939 AndExec = false;
2940 }
2941
2942 unsigned BrOp =
2943 UseSCCBr ? (Negate ? AMDGPU::S_CBRANCH_SCC0 : AMDGPU::S_CBRANCH_SCC1)
2944 : (Negate ? AMDGPU::S_CBRANCH_VCCZ : AMDGPU::S_CBRANCH_VCCNZ);
2945 Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC();
2946 SDLoc SL(N);
2947
2948 if (AndExec) {
2949 // This is the case that we are selecting to S_CBRANCH_VCCNZ. We have not
2950 // analyzed what generates the vcc value, so we do not know whether vcc
2951 // bits for disabled lanes are 0. Thus we need to mask out bits for
2952 // disabled lanes.
2953 //
2954 // For the case that we select S_CBRANCH_SCC1 and it gets
2955 // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls
2956 // SIInstrInfo::moveToVALU which inserts the S_AND).
2957 //
2958 // We could add an analysis of what generates the vcc value here and omit
2959 // the S_AND when is unnecessary. But it would be better to add a separate
2960 // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it
2961 // catches both cases.
2962 Cond = SDValue(
2963 CurDAG->getMachineNode(
2964 Opcode: Subtarget->isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64, dl: SL,
2965 VT: MVT::i1,
2966 Op1: CurDAG->getRegister(Reg: Subtarget->isWave32() ? AMDGPU::EXEC_LO
2967 : AMDGPU::EXEC,
2968 VT: MVT::i1),
2969 Op2: Cond),
2970 0);
2971 }
2972
2973 SDValue VCC = CurDAG->getCopyToReg(Chain: N->getOperand(Num: 0), dl: SL, Reg: CondReg, N: Cond);
2974 CurDAG->SelectNodeTo(N, MachineOpc: BrOp, VT: MVT::Other,
2975 Op1: N->getOperand(Num: 2), // Basic Block
2976 Op2: VCC.getValue(R: 0));
2977}
2978
2979void AMDGPUDAGToDAGISel::SelectFP_EXTEND(SDNode *N) {
2980 if (Subtarget->hasSALUFloatInsts() && N->getValueType(ResNo: 0) == MVT::f32 &&
2981 !N->isDivergent()) {
2982 SDValue Src = N->getOperand(Num: 0);
2983 if (Src.getValueType() == MVT::f16) {
2984 if (isExtractHiElt(In: Src, Out&: Src)) {
2985 CurDAG->SelectNodeTo(N, MachineOpc: AMDGPU::S_CVT_HI_F32_F16, VTs: N->getVTList(),
2986 Ops: {Src});
2987 return;
2988 }
2989 }
2990 }
2991
2992 SelectCode(N);
2993}
2994
2995void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
2996 // The address is assumed to be uniform, so if it ends up in a VGPR, it will
2997 // be copied to an SGPR with readfirstlane.
2998 unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
2999 AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
3000
3001 SDValue Chain = N->getOperand(Num: 0);
3002 SDValue Ptr = N->getOperand(Num: 2);
3003 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(Val: N);
3004 MachineMemOperand *MMO = M->getMemOperand();
3005 bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
3006
3007 SDValue Offset;
3008 if (CurDAG->isBaseWithConstantOffset(Op: Ptr)) {
3009 SDValue PtrBase = Ptr.getOperand(i: 0);
3010 SDValue PtrOffset = Ptr.getOperand(i: 1);
3011
3012 const APInt &OffsetVal = PtrOffset->getAsAPIntVal();
3013 if (isDSOffsetLegal(Base: PtrBase, Offset: OffsetVal.getZExtValue())) {
3014 N = glueCopyToM0(N, Val: PtrBase);
3015 Offset = CurDAG->getTargetConstant(Val: OffsetVal, DL: SDLoc(), VT: MVT::i32);
3016 }
3017 }
3018
3019 if (!Offset) {
3020 N = glueCopyToM0(N, Val: Ptr);
3021 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(), VT: MVT::i32);
3022 }
3023
3024 SDValue Ops[] = {
3025 Offset,
3026 CurDAG->getTargetConstant(Val: IsGDS, DL: SDLoc(), VT: MVT::i32),
3027 Chain,
3028 N->getOperand(Num: N->getNumOperands() - 1) // New glue
3029 };
3030
3031 SDNode *Selected = CurDAG->SelectNodeTo(N, MachineOpc: Opc, VTs: N->getVTList(), Ops);
3032 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Selected), NewMemRefs: {MMO});
3033}
3034
3035// We need to handle this here because tablegen doesn't support matching
3036// instructions with multiple outputs.
3037void AMDGPUDAGToDAGISel::SelectDSBvhStackIntrinsic(SDNode *N, unsigned IntrID) {
3038 unsigned Opc;
3039 switch (IntrID) {
3040 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
3041 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
3042 Opc = AMDGPU::DS_BVH_STACK_RTN_B32;
3043 break;
3044 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
3045 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP1_RTN_B32;
3046 break;
3047 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
3048 Opc = AMDGPU::DS_BVH_STACK_PUSH8_POP2_RTN_B64;
3049 break;
3050 }
3051 SDValue Ops[] = {N->getOperand(Num: 2), N->getOperand(Num: 3), N->getOperand(Num: 4),
3052 N->getOperand(Num: 5), N->getOperand(Num: 0)};
3053
3054 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(Val: N);
3055 MachineMemOperand *MMO = M->getMemOperand();
3056 SDNode *Selected = CurDAG->SelectNodeTo(N, MachineOpc: Opc, VTs: N->getVTList(), Ops);
3057 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Selected), NewMemRefs: {MMO});
3058}
3059
3060void AMDGPUDAGToDAGISel::SelectTensorLoadStore(SDNode *N, unsigned IntrID) {
3061 bool IsLoad = IntrID == Intrinsic::amdgcn_tensor_load_to_lds;
3062 unsigned Opc =
3063 IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d4 : AMDGPU::TENSOR_STORE_FROM_LDS_d4;
3064
3065 SmallVector<SDValue, 7> TensorOps;
3066 // First two groups
3067 TensorOps.push_back(Elt: N->getOperand(Num: 2)); // D# group 0
3068 TensorOps.push_back(Elt: N->getOperand(Num: 3)); // D# group 1
3069
3070 // Use _D2 version if both group 2 and 3 are zero-initialized.
3071 SDValue Group2 = N->getOperand(Num: 4);
3072 SDValue Group3 = N->getOperand(Num: 5);
3073 if (ISD::isBuildVectorAllZeros(N: Group2.getNode()) &&
3074 ISD::isBuildVectorAllZeros(N: Group3.getNode())) {
3075 Opc = IsLoad ? AMDGPU::TENSOR_LOAD_TO_LDS_d2
3076 : AMDGPU::TENSOR_STORE_FROM_LDS_d2;
3077 } else { // Has at least 4 groups
3078 TensorOps.push_back(Elt: Group2); // D# group 2
3079 TensorOps.push_back(Elt: Group3); // D# group 3
3080 }
3081
3082 // TODO: Handle the fifth group: N->getOperand(6), which is silently ignored
3083 // for now because all existing targets only support up to 4 groups.
3084 TensorOps.push_back(Elt: CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i1)); // r128
3085 TensorOps.push_back(Elt: N->getOperand(Num: 7)); // cache policy
3086 TensorOps.push_back(Elt: N->getOperand(Num: 0)); // chain
3087
3088 (void)CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT: MVT::Other, Ops: TensorOps);
3089}
3090
3091static unsigned gwsIntrinToOpcode(unsigned IntrID) {
3092 switch (IntrID) {
3093 case Intrinsic::amdgcn_ds_gws_init:
3094 return AMDGPU::DS_GWS_INIT;
3095 case Intrinsic::amdgcn_ds_gws_barrier:
3096 return AMDGPU::DS_GWS_BARRIER;
3097 case Intrinsic::amdgcn_ds_gws_sema_v:
3098 return AMDGPU::DS_GWS_SEMA_V;
3099 case Intrinsic::amdgcn_ds_gws_sema_br:
3100 return AMDGPU::DS_GWS_SEMA_BR;
3101 case Intrinsic::amdgcn_ds_gws_sema_p:
3102 return AMDGPU::DS_GWS_SEMA_P;
3103 case Intrinsic::amdgcn_ds_gws_sema_release_all:
3104 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
3105 default:
3106 llvm_unreachable("not a gws intrinsic");
3107 }
3108}
3109
3110void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) {
3111 if (!Subtarget->hasGWS() ||
3112 (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
3113 !Subtarget->hasGWSSemaReleaseAll())) {
3114 // Let this error.
3115 SelectCode(N);
3116 return;
3117 }
3118
3119 // Chain, intrinsic ID, vsrc, offset
3120 const bool HasVSrc = N->getNumOperands() == 4;
3121 assert(HasVSrc || N->getNumOperands() == 3);
3122
3123 SDLoc SL(N);
3124 SDValue BaseOffset = N->getOperand(Num: HasVSrc ? 3 : 2);
3125 int ImmOffset = 0;
3126 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(Val: N);
3127 MachineMemOperand *MMO = M->getMemOperand();
3128
3129 // Don't worry if the offset ends up in a VGPR. Only one lane will have
3130 // effect, so SIFixSGPRCopies will validly insert readfirstlane.
3131
3132 // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
3133 // offset field) % 64. Some versions of the programming guide omit the m0
3134 // part, or claim it's from offset 0.
3135 if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(Val&: BaseOffset)) {
3136 // If we have a constant offset, try to use the 0 in m0 as the base.
3137 // TODO: Look into changing the default m0 initialization value. If the
3138 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
3139 // the immediate offset.
3140 glueCopyToM0(N, Val: CurDAG->getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
3141 ImmOffset = ConstOffset->getZExtValue();
3142 } else {
3143 if (CurDAG->isBaseWithConstantOffset(Op: BaseOffset)) {
3144 ImmOffset = BaseOffset.getConstantOperandVal(i: 1);
3145 BaseOffset = BaseOffset.getOperand(i: 0);
3146 }
3147
3148 // Prefer to do the shift in an SGPR since it should be possible to use m0
3149 // as the result directly. If it's already an SGPR, it will be eliminated
3150 // later.
3151 SDNode *SGPROffset
3152 = CurDAG->getMachineNode(Opcode: AMDGPU::V_READFIRSTLANE_B32, dl: SL, VT: MVT::i32,
3153 Op1: BaseOffset);
3154 // Shift to offset in m0
3155 SDNode *M0Base
3156 = CurDAG->getMachineNode(Opcode: AMDGPU::S_LSHL_B32, dl: SL, VT: MVT::i32,
3157 Op1: SDValue(SGPROffset, 0),
3158 Op2: CurDAG->getTargetConstant(Val: 16, DL: SL, VT: MVT::i32));
3159 glueCopyToM0(N, Val: SDValue(M0Base, 0));
3160 }
3161
3162 SDValue Chain = N->getOperand(Num: 0);
3163 SDValue OffsetField = CurDAG->getTargetConstant(Val: ImmOffset, DL: SL, VT: MVT::i32);
3164
3165 const unsigned Opc = gwsIntrinToOpcode(IntrID);
3166
3167 const MCInstrDesc &InstrDesc = TII->get(Opcode: Opc);
3168 int Data0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::data0);
3169
3170 const TargetRegisterClass *DataRC = TII->getRegClass(MCID: InstrDesc, OpNum: Data0Idx);
3171
3172 SmallVector<SDValue, 5> Ops;
3173 if (HasVSrc) {
3174 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3175
3176 SDValue Data = N->getOperand(Num: 2);
3177 MVT DataVT = Data.getValueType().getSimpleVT();
3178 if (TRI->isTypeLegalForClass(RC: *DataRC, T: DataVT)) {
3179 // Normal 32-bit case.
3180 Ops.push_back(Elt: N->getOperand(Num: 2));
3181 } else {
3182 // Operand is really 32-bits, but requires 64-bit alignment, so use the
3183 // even aligned 64-bit register class.
3184 const SDValue RegSeqOps[] = {
3185 CurDAG->getTargetConstant(Val: DataRC->getID(), DL: SL, VT: MVT::i32), Data,
3186 CurDAG->getTargetConstant(Val: AMDGPU::sub0, DL: SL, VT: MVT::i32),
3187 SDValue(
3188 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: SL, VT: MVT::i32),
3189 0),
3190 CurDAG->getTargetConstant(Val: AMDGPU::sub1, DL: SL, VT: MVT::i32)};
3191
3192 Ops.push_back(Elt: SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE,
3193 dl: SL, VT: MVT::v2i32, Ops: RegSeqOps),
3194 0));
3195 }
3196 }
3197
3198 Ops.push_back(Elt: OffsetField);
3199 Ops.push_back(Elt: Chain);
3200
3201 SDNode *Selected = CurDAG->SelectNodeTo(N, MachineOpc: Opc, VTs: N->getVTList(), Ops);
3202 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Selected), NewMemRefs: {MMO});
3203}
3204
3205void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) {
3206 if (Subtarget->getLDSBankCount() != 16) {
3207 // This is a single instruction with a pattern.
3208 SelectCode(N);
3209 return;
3210 }
3211
3212 SDLoc DL(N);
3213
3214 // This requires 2 instructions. It is possible to write a pattern to support
3215 // this, but the generated isel emitter doesn't correctly deal with multiple
3216 // output instructions using the same physical register input. The copy to m0
3217 // is incorrectly placed before the second instruction.
3218 //
3219 // TODO: Match source modifiers.
3220 //
3221 // def : Pat <
3222 // (int_amdgcn_interp_p1_f16
3223 // (VOP3Mods f32:$src0, i32:$src0_modifiers),
3224 // (i32 timm:$attrchan), (i32 timm:$attr),
3225 // (i1 timm:$high), M0),
3226 // (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr,
3227 // timm:$attrchan, 0,
3228 // (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> {
3229 // let Predicates = [has16BankLDS];
3230 // }
3231
3232 // 16 bank LDS
3233 SDValue ToM0 = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: AMDGPU::M0,
3234 N: N->getOperand(Num: 5), Glue: SDValue());
3235
3236 SDVTList VTs = CurDAG->getVTList(VT1: MVT::f32, VT2: MVT::Other);
3237
3238 SDNode *InterpMov =
3239 CurDAG->getMachineNode(Opcode: AMDGPU::V_INTERP_MOV_F32, dl: DL, VTs, Ops: {
3240 CurDAG->getTargetConstant(Val: 2, DL, VT: MVT::i32), // P0
3241 N->getOperand(Num: 3), // Attr
3242 N->getOperand(Num: 2), // Attrchan
3243 ToM0.getValue(R: 1) // In glue
3244 });
3245
3246 SDNode *InterpP1LV =
3247 CurDAG->getMachineNode(Opcode: AMDGPU::V_INTERP_P1LV_F16, dl: DL, VT: MVT::f32, Ops: {
3248 CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32), // $src0_modifiers
3249 N->getOperand(Num: 1), // Src0
3250 N->getOperand(Num: 3), // Attr
3251 N->getOperand(Num: 2), // Attrchan
3252 CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32), // $src2_modifiers
3253 SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high
3254 N->getOperand(Num: 4), // high
3255 CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1), // $clamp
3256 CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32), // $omod
3257 SDValue(InterpMov, 1)
3258 });
3259
3260 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: SDValue(InterpP1LV, 0));
3261}
3262
3263void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
3264 unsigned IntrID = N->getConstantOperandVal(Num: 1);
3265 switch (IntrID) {
3266 case Intrinsic::amdgcn_ds_append:
3267 case Intrinsic::amdgcn_ds_consume: {
3268 if (N->getValueType(ResNo: 0) != MVT::i32)
3269 break;
3270 SelectDSAppendConsume(N, IntrID);
3271 return;
3272 }
3273 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
3274 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
3275 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
3276 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
3277 SelectDSBvhStackIntrinsic(N, IntrID);
3278 return;
3279 case Intrinsic::amdgcn_init_whole_wave:
3280 CurDAG->getMachineFunction()
3281 .getInfo<SIMachineFunctionInfo>()
3282 ->setInitWholeWave();
3283 break;
3284 }
3285
3286 SelectCode(N);
3287}
3288
3289void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) {
3290 unsigned IntrID = N->getConstantOperandVal(Num: 0);
3291 unsigned Opcode = AMDGPU::INSTRUCTION_LIST_END;
3292 SDNode *ConvGlueNode = N->getGluedNode();
3293 if (ConvGlueNode) {
3294 // FIXME: Possibly iterate over multiple glue nodes?
3295 assert(ConvGlueNode->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
3296 ConvGlueNode = ConvGlueNode->getOperand(Num: 0).getNode();
3297 ConvGlueNode =
3298 CurDAG->getMachineNode(Opcode: TargetOpcode::CONVERGENCECTRL_GLUE, dl: {},
3299 VT: MVT::Glue, Op1: SDValue(ConvGlueNode, 0));
3300 } else {
3301 ConvGlueNode = nullptr;
3302 }
3303 switch (IntrID) {
3304 case Intrinsic::amdgcn_wqm:
3305 Opcode = AMDGPU::WQM;
3306 break;
3307 case Intrinsic::amdgcn_softwqm:
3308 Opcode = AMDGPU::SOFT_WQM;
3309 break;
3310 case Intrinsic::amdgcn_wwm:
3311 case Intrinsic::amdgcn_strict_wwm:
3312 Opcode = AMDGPU::STRICT_WWM;
3313 break;
3314 case Intrinsic::amdgcn_strict_wqm:
3315 Opcode = AMDGPU::STRICT_WQM;
3316 break;
3317 case Intrinsic::amdgcn_interp_p1_f16:
3318 SelectInterpP1F16(N);
3319 return;
3320 case Intrinsic::amdgcn_permlane16_swap:
3321 case Intrinsic::amdgcn_permlane32_swap: {
3322 if ((IntrID == Intrinsic::amdgcn_permlane16_swap &&
3323 !Subtarget->hasPermlane16Swap()) ||
3324 (IntrID == Intrinsic::amdgcn_permlane32_swap &&
3325 !Subtarget->hasPermlane32Swap())) {
3326 SelectCode(N); // Hit the default error
3327 return;
3328 }
3329
3330 Opcode = IntrID == Intrinsic::amdgcn_permlane16_swap
3331 ? AMDGPU::V_PERMLANE16_SWAP_B32_e64
3332 : AMDGPU::V_PERMLANE32_SWAP_B32_e64;
3333
3334 SmallVector<SDValue, 4> NewOps(N->op_begin() + 1, N->op_end());
3335 if (ConvGlueNode)
3336 NewOps.push_back(Elt: SDValue(ConvGlueNode, 0));
3337
3338 bool FI = N->getConstantOperandVal(Num: 3);
3339 NewOps[2] = CurDAG->getTargetConstant(
3340 Val: FI ? AMDGPU::DPP::DPP_FI_1 : AMDGPU::DPP::DPP_FI_0, DL: SDLoc(), VT: MVT::i32);
3341
3342 CurDAG->SelectNodeTo(N, MachineOpc: Opcode, VTs: N->getVTList(), Ops: NewOps);
3343 return;
3344 }
3345 default:
3346 SelectCode(N);
3347 break;
3348 }
3349
3350 if (Opcode != AMDGPU::INSTRUCTION_LIST_END) {
3351 SDValue Src = N->getOperand(Num: 1);
3352 CurDAG->SelectNodeTo(N, MachineOpc: Opcode, VTs: N->getVTList(), Ops: {Src});
3353 }
3354
3355 if (ConvGlueNode) {
3356 SmallVector<SDValue, 4> NewOps(N->ops());
3357 NewOps.push_back(Elt: SDValue(ConvGlueNode, 0));
3358 CurDAG->MorphNodeTo(N, Opc: N->getOpcode(), VTs: N->getVTList(), Ops: NewOps);
3359 }
3360}
3361
3362void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) {
3363 unsigned IntrID = N->getConstantOperandVal(Num: 1);
3364 switch (IntrID) {
3365 case Intrinsic::amdgcn_ds_gws_init:
3366 case Intrinsic::amdgcn_ds_gws_barrier:
3367 case Intrinsic::amdgcn_ds_gws_sema_v:
3368 case Intrinsic::amdgcn_ds_gws_sema_br:
3369 case Intrinsic::amdgcn_ds_gws_sema_p:
3370 case Intrinsic::amdgcn_ds_gws_sema_release_all:
3371 SelectDS_GWS(N, IntrID);
3372 return;
3373 case Intrinsic::amdgcn_tensor_load_to_lds:
3374 case Intrinsic::amdgcn_tensor_store_from_lds:
3375 SelectTensorLoadStore(N, IntrID);
3376 return;
3377 default:
3378 break;
3379 }
3380
3381 SelectCode(N);
3382}
3383
3384void AMDGPUDAGToDAGISel::SelectWAVE_ADDRESS(SDNode *N) {
3385 SDValue Log2WaveSize =
3386 CurDAG->getTargetConstant(Val: Subtarget->getWavefrontSizeLog2(), DL: SDLoc(N), VT: MVT::i32);
3387 CurDAG->SelectNodeTo(N, MachineOpc: AMDGPU::S_LSHR_B32, VTs: N->getVTList(),
3388 Ops: {N->getOperand(Num: 0), Log2WaveSize});
3389}
3390
3391void AMDGPUDAGToDAGISel::SelectSTACKRESTORE(SDNode *N) {
3392 SDValue SrcVal = N->getOperand(Num: 1);
3393 if (SrcVal.getValueType() != MVT::i32) {
3394 SelectCode(N); // Emit default error
3395 return;
3396 }
3397
3398 SDValue CopyVal;
3399 Register SP = TLI->getStackPointerRegisterToSaveRestore();
3400 SDLoc SL(N);
3401
3402 if (SrcVal.getOpcode() == AMDGPUISD::WAVE_ADDRESS) {
3403 CopyVal = SrcVal.getOperand(i: 0);
3404 } else {
3405 SDValue Log2WaveSize = CurDAG->getTargetConstant(
3406 Val: Subtarget->getWavefrontSizeLog2(), DL: SL, VT: MVT::i32);
3407
3408 if (N->isDivergent()) {
3409 SrcVal = SDValue(CurDAG->getMachineNode(Opcode: AMDGPU::V_READFIRSTLANE_B32, dl: SL,
3410 VT: MVT::i32, Op1: SrcVal),
3411 0);
3412 }
3413
3414 CopyVal = SDValue(CurDAG->getMachineNode(Opcode: AMDGPU::S_LSHL_B32, dl: SL, VT: MVT::i32,
3415 Ops: {SrcVal, Log2WaveSize}),
3416 0);
3417 }
3418
3419 SDValue CopyToSP = CurDAG->getCopyToReg(Chain: N->getOperand(Num: 0), dl: SL, Reg: SP, N: CopyVal);
3420 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: CopyToSP);
3421}
3422
3423bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
3424 unsigned &Mods,
3425 bool IsCanonicalizing,
3426 bool AllowAbs) const {
3427 Mods = SISrcMods::NONE;
3428 Src = In;
3429
3430 if (Src.getOpcode() == ISD::FNEG) {
3431 Mods |= SISrcMods::NEG;
3432 Src = Src.getOperand(i: 0);
3433 } else if (Src.getOpcode() == ISD::FSUB && IsCanonicalizing) {
3434 // Fold fsub [+-]0 into fneg. This may not have folded depending on the
3435 // denormal mode, but we're implicitly canonicalizing in a source operand.
3436 auto *LHS = dyn_cast<ConstantFPSDNode>(Val: Src.getOperand(i: 0));
3437 if (LHS && LHS->isZero()) {
3438 Mods |= SISrcMods::NEG;
3439 Src = Src.getOperand(i: 1);
3440 }
3441 }
3442
3443 if (AllowAbs && Src.getOpcode() == ISD::FABS) {
3444 Mods |= SISrcMods::ABS;
3445 Src = Src.getOperand(i: 0);
3446 }
3447
3448 if (Mods != SISrcMods::NONE)
3449 return true;
3450
3451 // Convert various sign-bit masks on integers to src mods. Currently disabled
3452 // for 16-bit types as the codegen replaces the operand without adding a
3453 // srcmod. This is intentionally finding the cases where we are performing
3454 // float neg and abs on int types, the goal is not to obtain two's complement
3455 // neg or abs. Limit converison to select operands via the nonCanonalizing
3456 // pattern.
3457 // TODO: Add 16-bit support.
3458 if (IsCanonicalizing)
3459 return true;
3460
3461 // v2i32 xor/or/and are legal. A vselect using these instructions as operands
3462 // is scalarised into two selects with EXTRACT_VECTOR_ELT operands. Peek
3463 // through the extract to the bitwise op.
3464 SDValue PeekSrc =
3465 Src->getOpcode() == ISD::EXTRACT_VECTOR_ELT ? Src->getOperand(Num: 0) : Src;
3466 // Convert various sign-bit masks to src mods. Currently disabled for 16-bit
3467 // types as the codegen replaces the operand without adding a srcmod.
3468 // This is intentionally finding the cases where we are performing float neg
3469 // and abs on int types, the goal is not to obtain two's complement neg or
3470 // abs.
3471 // TODO: Add 16-bit support.
3472 unsigned Opc = PeekSrc.getOpcode();
3473 EVT VT = Src.getValueType();
3474 if ((Opc != ISD::AND && Opc != ISD::OR && Opc != ISD::XOR) ||
3475 (VT != MVT::i32 && VT != MVT::v2i32 && VT != MVT::i64))
3476 return true;
3477
3478 ConstantSDNode *CRHS = isConstOrConstSplat(N: PeekSrc->getOperand(Num: 1));
3479 if (!CRHS)
3480 return true;
3481
3482 auto ReplaceSrc = [&]() -> SDValue {
3483 if (Src->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
3484 return Src.getOperand(i: 0);
3485
3486 SDValue LHS = PeekSrc->getOperand(Num: 0);
3487 SDValue Index = Src->getOperand(Num: 1);
3488 return CurDAG->getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(Src),
3489 VT: Src.getValueType(), N1: LHS, N2: Index);
3490 };
3491
3492 // Recognise Srcmods:
3493 // (xor a, 0x80000000) or v2i32 (xor a, {0x80000000,0x80000000}) as NEG.
3494 // (and a, 0x7fffffff) or v2i32 (and a, {0x7fffffff,0x7fffffff}) as ABS.
3495 // (or a, 0x80000000) or v2i32 (or a, {0x80000000,0x80000000}) as NEG+ABS
3496 // SrcModifiers.
3497 if (Opc == ISD::XOR && CRHS->getAPIntValue().isSignMask()) {
3498 Mods |= SISrcMods::NEG;
3499 Src = ReplaceSrc();
3500 } else if (Opc == ISD::AND && AllowAbs &&
3501 CRHS->getAPIntValue().isMaxSignedValue()) {
3502 Mods |= SISrcMods::ABS;
3503 Src = ReplaceSrc();
3504 } else if (Opc == ISD::OR && AllowAbs && CRHS->getAPIntValue().isSignMask()) {
3505 Mods |= SISrcMods::ABS | SISrcMods::NEG;
3506 Src = ReplaceSrc();
3507 }
3508
3509 return true;
3510}
3511
3512bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
3513 SDValue &SrcMods) const {
3514 unsigned Mods;
3515 if (SelectVOP3ModsImpl(In, Src, Mods, /*IsCanonicalizing=*/true,
3516 /*AllowAbs=*/true)) {
3517 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
3518 return true;
3519 }
3520
3521 return false;
3522}
3523
3524bool AMDGPUDAGToDAGISel::SelectVOP3ModsNonCanonicalizing(
3525 SDValue In, SDValue &Src, SDValue &SrcMods) const {
3526 unsigned Mods;
3527 if (SelectVOP3ModsImpl(In, Src, Mods, /*IsCanonicalizing=*/false,
3528 /*AllowAbs=*/true)) {
3529 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
3530 return true;
3531 }
3532
3533 return false;
3534}
3535
3536bool AMDGPUDAGToDAGISel::SelectVOP3BMods(SDValue In, SDValue &Src,
3537 SDValue &SrcMods) const {
3538 unsigned Mods;
3539 if (SelectVOP3ModsImpl(In, Src, Mods,
3540 /*IsCanonicalizing=*/true,
3541 /*AllowAbs=*/false)) {
3542 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
3543 return true;
3544 }
3545
3546 return false;
3547}
3548
3549bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
3550 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
3551 return false;
3552
3553 Src = In;
3554 return true;
3555}
3556
3557bool AMDGPUDAGToDAGISel::SelectVINTERPModsImpl(SDValue In, SDValue &Src,
3558 SDValue &SrcMods,
3559 bool OpSel) const {
3560 unsigned Mods;
3561 if (SelectVOP3ModsImpl(In, Src, Mods,
3562 /*IsCanonicalizing=*/true,
3563 /*AllowAbs=*/false)) {
3564 if (OpSel)
3565 Mods |= SISrcMods::OP_SEL_0;
3566 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
3567 return true;
3568 }
3569
3570 return false;
3571}
3572
3573bool AMDGPUDAGToDAGISel::SelectVINTERPMods(SDValue In, SDValue &Src,
3574 SDValue &SrcMods) const {
3575 return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ false);
3576}
3577
3578bool AMDGPUDAGToDAGISel::SelectVINTERPModsHi(SDValue In, SDValue &Src,
3579 SDValue &SrcMods) const {
3580 return SelectVINTERPModsImpl(In, Src, SrcMods, /* OpSel */ true);
3581}
3582
3583bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
3584 SDValue &SrcMods, SDValue &Clamp,
3585 SDValue &Omod) const {
3586 SDLoc DL(In);
3587 Clamp = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1);
3588 Omod = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1);
3589
3590 return SelectVOP3Mods(In, Src, SrcMods);
3591}
3592
3593bool AMDGPUDAGToDAGISel::SelectVOP3BMods0(SDValue In, SDValue &Src,
3594 SDValue &SrcMods, SDValue &Clamp,
3595 SDValue &Omod) const {
3596 SDLoc DL(In);
3597 Clamp = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1);
3598 Omod = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1);
3599
3600 return SelectVOP3BMods(In, Src, SrcMods);
3601}
3602
3603bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
3604 SDValue &Clamp, SDValue &Omod) const {
3605 Src = In;
3606
3607 SDLoc DL(In);
3608 Clamp = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1);
3609 Omod = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i1);
3610
3611 return true;
3612}
3613
3614bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
3615 SDValue &SrcMods, bool IsDOT) const {
3616 unsigned Mods = SISrcMods::NONE;
3617 Src = In;
3618
3619 // TODO: Handle G_FSUB 0 as fneg
3620 if (Src.getOpcode() == ISD::FNEG) {
3621 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
3622 Src = Src.getOperand(i: 0);
3623 }
3624
3625 // 64-bit VOP3P instructions do not have OPSEL or ABS.
3626 bool HasOpSel = Src.getValueSizeInBits() != 128;
3627
3628 if (Src.getOpcode() == ISD::BUILD_VECTOR && Src.getNumOperands() == 2 &&
3629 (!IsDOT || !Subtarget->hasDOTOpSelHazard())) {
3630 unsigned VecMods = Mods;
3631
3632 SDValue Lo = stripBitcast(Val: Src.getOperand(i: 0));
3633 SDValue Hi = stripBitcast(Val: Src.getOperand(i: 1));
3634
3635 if (Lo.getOpcode() == ISD::FNEG) {
3636 Lo = stripBitcast(Val: Lo.getOperand(i: 0));
3637 Mods ^= SISrcMods::NEG;
3638 }
3639
3640 if (Hi.getOpcode() == ISD::FNEG) {
3641 Hi = stripBitcast(Val: Hi.getOperand(i: 0));
3642 Mods ^= SISrcMods::NEG_HI;
3643 }
3644
3645 if (HasOpSel) {
3646 if (isExtractHiElt(In: Lo, Out&: Lo))
3647 Mods |= SISrcMods::OP_SEL_0;
3648
3649 if (isExtractHiElt(In: Hi, Out&: Hi))
3650 Mods |= SISrcMods::OP_SEL_1;
3651 }
3652
3653 unsigned VecSize = Src.getValueSizeInBits();
3654 Lo = stripExtractLoElt(In: Lo);
3655 Hi = stripExtractLoElt(In: Hi);
3656
3657 if (Lo.getValueSizeInBits() > VecSize) {
3658 Lo = CurDAG->getTargetExtractSubreg(
3659 SRIdx: (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, DL: SDLoc(In),
3660 VT: MVT::getIntegerVT(BitWidth: VecSize), Operand: Lo);
3661 }
3662
3663 if (Hi.getValueSizeInBits() > VecSize) {
3664 Hi = CurDAG->getTargetExtractSubreg(
3665 SRIdx: (VecSize > 32) ? AMDGPU::sub0_sub1 : AMDGPU::sub0, DL: SDLoc(In),
3666 VT: MVT::getIntegerVT(BitWidth: VecSize), Operand: Hi);
3667 }
3668
3669 assert(Lo.getValueSizeInBits() <= VecSize &&
3670 Hi.getValueSizeInBits() <= VecSize);
3671
3672 if (Lo == Hi && !isInlineImmediate(N: Lo.getNode())) {
3673 // Really a scalar input. Just select from the low half of the register to
3674 // avoid packing.
3675
3676 if (VecSize == Lo.getValueSizeInBits()) {
3677 Src = Lo;
3678 } else if (VecSize == 32) {
3679 Src = createVOP3PSrc32FromLo16(Lo, Src, CurDAG, Subtarget);
3680 } else {
3681 assert((Lo.getValueSizeInBits() == 32 && VecSize == 64) ||
3682 (Lo.getValueSizeInBits() == 64 && VecSize == 128));
3683
3684 SDLoc SL(In);
3685 SDValue Undef = SDValue(
3686 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: SL,
3687 VT: Lo.getValueType()), 0);
3688 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3689 // <2 x 64> instructions do not have OPSEL and also replicate low 64
3690 // bits of a scalar input into high 64 bits. Use VGPRs in this case.
3691 // TODO: This fact can be exploited but we need to set proper OPSEL for
3692 // codegen folding purposes. It will not affect a final instruction.
3693 auto RC = Lo->isDivergent() ? TRI->getVGPRClassForBitWidth(BitWidth: VecSize)
3694 : TRI->getSGPRClassForBitWidth(BitWidth: VecSize);
3695 unsigned NumRegs = Lo.getValueSizeInBits() == 32 ? 1 : 2;
3696 const SDValue Ops[] = {
3697 CurDAG->getTargetConstant(Val: RC->getID(), DL: SL, VT: MVT::i32), Lo,
3698 CurDAG->getTargetConstant(Val: TRI->getSubRegFromChannel(Channel: 0, NumRegs), DL: SL,
3699 VT: MVT::i32),
3700 // For packed 64-bit ops without OPSEL support, a later pass will
3701 // optimize the splat sgpr patterns to save registers.
3702 HasOpSel ? Undef : Lo,
3703 CurDAG->getTargetConstant(
3704 Val: TRI->getSubRegFromChannel(Channel: NumRegs, NumRegs), DL: SL, VT: MVT::i32)};
3705
3706 Src = SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: SL,
3707 VT: Src.getValueType(), Ops), 0);
3708 // Check that both op_sel_0 and op_sel_1 are zero.
3709 assert(!(Mods & (SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1)));
3710 }
3711 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
3712 return true;
3713 }
3714
3715 if (VecSize == 64 && Lo == Hi && isa<ConstantFPSDNode>(Val: Lo)) {
3716 uint64_t Lit = cast<ConstantFPSDNode>(Val&: Lo)->getValueAPF()
3717 .bitcastToAPInt().getZExtValue();
3718 if (AMDGPU::isInlinableLiteral32(Literal: Lit, HasInv2Pi: Subtarget->hasInv2PiInlineImm())) {
3719 Src = CurDAG->getTargetConstant(Val: Lit, DL: SDLoc(In), VT: MVT::i64);
3720 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
3721 return true;
3722 }
3723 }
3724
3725 Mods = VecMods;
3726 } else if (Src.getOpcode() == ISD::VECTOR_SHUFFLE &&
3727 Src.getNumOperands() == 2) {
3728
3729 // TODO: We should repeat the build_vector source check above for the
3730 // vector_shuffle for negates and casts of individual elements.
3731
3732 assert(Src.getValueSizeInBits() != 128 &&
3733 "<2 x 64> VECTOR_SHUFFLE should not be legal.");
3734
3735 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Src);
3736 ArrayRef<int> Mask = SVN->getMask();
3737
3738 if (Mask[0] < 2 && Mask[1] < 2) {
3739 // src1 should be undef.
3740 SDValue ShuffleSrc = SVN->getOperand(Num: 0);
3741
3742 if (ShuffleSrc.getOpcode() == ISD::FNEG) {
3743 ShuffleSrc = ShuffleSrc.getOperand(i: 0);
3744 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
3745 }
3746
3747 if (Mask[0] == 1)
3748 Mods |= SISrcMods::OP_SEL_0;
3749 if (Mask[1] == 1)
3750 Mods |= SISrcMods::OP_SEL_1;
3751
3752 Src = ShuffleSrc;
3753 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
3754 return true;
3755 }
3756 }
3757
3758 // Packed instructions do not have abs modifiers.
3759 Mods |= SISrcMods::OP_SEL_1;
3760
3761 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
3762 return true;
3763}
3764
3765bool AMDGPUDAGToDAGISel::SelectVOP3PModsDOT(SDValue In, SDValue &Src,
3766 SDValue &SrcMods) const {
3767 return SelectVOP3PMods(In, Src, SrcMods, IsDOT: true);
3768}
3769
3770bool AMDGPUDAGToDAGISel::SelectVOP3PNoModsDOT(SDValue In, SDValue &Src) const {
3771 SDValue SrcTmp, SrcModsTmp;
3772 SelectVOP3PMods(In, Src&: SrcTmp, SrcMods&: SrcModsTmp, IsDOT: true);
3773 if (cast<ConstantSDNode>(Val&: SrcModsTmp)->getZExtValue() == SISrcMods::OP_SEL_1) {
3774 Src = SrcTmp;
3775 return true;
3776 }
3777
3778 return false;
3779}
3780
3781bool AMDGPUDAGToDAGISel::SelectVOP3PModsF32(SDValue In, SDValue &Src,
3782 SDValue &SrcMods) const {
3783 SelectVOP3Mods(In, Src, SrcMods);
3784 unsigned Mods = SISrcMods::OP_SEL_1;
3785 Mods |= cast<ConstantSDNode>(Val&: SrcMods)->getZExtValue();
3786 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
3787 return true;
3788}
3789
3790bool AMDGPUDAGToDAGISel::SelectVOP3PNoModsF32(SDValue In, SDValue &Src) const {
3791 SDValue SrcTmp, SrcModsTmp;
3792 SelectVOP3PModsF32(In, Src&: SrcTmp, SrcMods&: SrcModsTmp);
3793 if (cast<ConstantSDNode>(Val&: SrcModsTmp)->getZExtValue() == SISrcMods::OP_SEL_1) {
3794 Src = SrcTmp;
3795 return true;
3796 }
3797
3798 return false;
3799}
3800
3801bool AMDGPUDAGToDAGISel::SelectWMMAOpSelVOP3PMods(SDValue In,
3802 SDValue &Src) const {
3803 const ConstantSDNode *C = cast<ConstantSDNode>(Val&: In);
3804 assert(C->getAPIntValue().getBitWidth() == 1 && "expected i1 value");
3805
3806 unsigned Mods = SISrcMods::OP_SEL_1;
3807 unsigned SrcVal = C->getZExtValue();
3808 if (SrcVal == 1)
3809 Mods |= SISrcMods::OP_SEL_0;
3810
3811 Src = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
3812 return true;
3813}
3814
3815MachineSDNode *
3816AMDGPUDAGToDAGISel::buildRegSequence32(SmallVectorImpl<SDValue> &Elts,
3817 const SDLoc &DL) const {
3818 unsigned DstRegClass;
3819 EVT DstTy;
3820 switch (Elts.size()) {
3821 case 8:
3822 DstRegClass = AMDGPU::VReg_256RegClassID;
3823 DstTy = MVT::v8i32;
3824 break;
3825 case 4:
3826 DstRegClass = AMDGPU::VReg_128RegClassID;
3827 DstTy = MVT::v4i32;
3828 break;
3829 case 2:
3830 DstRegClass = AMDGPU::VReg_64RegClassID;
3831 DstTy = MVT::v2i32;
3832 break;
3833 default:
3834 llvm_unreachable("unhandled Reg sequence size");
3835 }
3836
3837 SmallVector<SDValue, 17> Ops;
3838 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: DstRegClass, DL, VT: MVT::i32));
3839 for (unsigned i = 0; i < Elts.size(); ++i) {
3840 Ops.push_back(Elt: Elts[i]);
3841 Ops.push_back(Elt: CurDAG->getTargetConstant(
3842 Val: SIRegisterInfo::getSubRegFromChannel(Channel: i), DL, VT: MVT::i32));
3843 }
3844 return CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: DstTy, Ops);
3845}
3846
3847MachineSDNode *
3848AMDGPUDAGToDAGISel::buildRegSequence16(SmallVectorImpl<SDValue> &Elts,
3849 const SDLoc &DL) const {
3850 SmallVector<SDValue, 8> PackedElts;
3851 assert("unhandled Reg sequence size" &&
3852 (Elts.size() == 8 || Elts.size() == 16));
3853
3854 // Pack 16-bit elements in pairs into 32-bit register. If both elements are
3855 // unpacked from 32-bit source use it, otherwise pack them using v_perm.
3856 for (unsigned i = 0; i < Elts.size(); i += 2) {
3857 SDValue LoSrc = stripExtractLoElt(In: stripBitcast(Val: Elts[i]));
3858 SDValue HiSrc;
3859 if (isExtractHiElt(In: Elts[i + 1], Out&: HiSrc) && LoSrc == HiSrc) {
3860 PackedElts.push_back(Elt: HiSrc);
3861 } else {
3862 if (Subtarget->useRealTrue16Insts()) {
3863 // FIXME-TRUE16. For now pack VGPR_32 for 16-bit source before
3864 // passing to v_perm_b32. Eventually we should use replace v_perm_b32
3865 // by reg_sequence.
3866 SDValue Undef = SDValue(
3867 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: MVT::i16),
3868 0);
3869 Elts[i] =
3870 emitRegSequence(CurDAG&: *CurDAG, DstRegClass: AMDGPU::VGPR_32RegClassID, DstTy: MVT::i32,
3871 Elts: {Elts[i], Undef}, SubRegClass: {AMDGPU::lo16, AMDGPU::hi16}, DL);
3872 Elts[i + 1] = emitRegSequence(CurDAG&: *CurDAG, DstRegClass: AMDGPU::VGPR_32RegClassID,
3873 DstTy: MVT::i32, Elts: {Elts[i + 1], Undef},
3874 SubRegClass: {AMDGPU::lo16, AMDGPU::hi16}, DL);
3875 }
3876 SDValue PackLoLo = CurDAG->getTargetConstant(Val: 0x05040100, DL, VT: MVT::i32);
3877 MachineSDNode *Packed =
3878 CurDAG->getMachineNode(Opcode: AMDGPU::V_PERM_B32_e64, dl: DL, VT: MVT::i32,
3879 Ops: {Elts[i + 1], Elts[i], PackLoLo});
3880 PackedElts.push_back(Elt: SDValue(Packed, 0));
3881 }
3882 }
3883 return buildRegSequence32(Elts&: PackedElts, DL);
3884}
3885
3886MachineSDNode *
3887AMDGPUDAGToDAGISel::buildRegSequence(SmallVectorImpl<SDValue> &Elts,
3888 const SDLoc &DL,
3889 unsigned ElementSize) const {
3890 if (ElementSize == 16)
3891 return buildRegSequence16(Elts, DL);
3892 if (ElementSize == 32)
3893 return buildRegSequence32(Elts, DL);
3894 llvm_unreachable("Unhandled element size");
3895}
3896
3897void AMDGPUDAGToDAGISel::selectWMMAModsNegAbs(unsigned ModOpcode,
3898 unsigned &Mods,
3899 SmallVectorImpl<SDValue> &Elts,
3900 SDValue &Src, const SDLoc &DL,
3901 unsigned ElementSize) const {
3902 if (ModOpcode == ISD::FNEG) {
3903 Mods |= SISrcMods::NEG;
3904 // Check if all elements also have abs modifier
3905 SmallVector<SDValue, 8> NegAbsElts;
3906 for (auto El : Elts) {
3907 if (El.getOpcode() != ISD::FABS)
3908 break;
3909 NegAbsElts.push_back(Elt: El->getOperand(Num: 0));
3910 }
3911 if (Elts.size() != NegAbsElts.size()) {
3912 // Neg
3913 Src = SDValue(buildRegSequence(Elts, DL, ElementSize), 0);
3914 } else {
3915 // Neg and Abs
3916 Mods |= SISrcMods::NEG_HI;
3917 Src = SDValue(buildRegSequence(Elts&: NegAbsElts, DL, ElementSize), 0);
3918 }
3919 } else {
3920 assert(ModOpcode == ISD::FABS);
3921 // Abs
3922 Mods |= SISrcMods::NEG_HI;
3923 Src = SDValue(buildRegSequence(Elts, DL, ElementSize), 0);
3924 }
3925}
3926
3927// Check all f16 elements for modifiers while looking through b32 and v2b16
3928// build vector, stop if element does not satisfy ModifierCheck.
3929static void
3930checkWMMAElementsModifiersF16(BuildVectorSDNode *BV,
3931 std::function<bool(SDValue)> ModifierCheck) {
3932 for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
3933 if (auto *F16Pair =
3934 dyn_cast<BuildVectorSDNode>(Val: stripBitcast(Val: BV->getOperand(Num: i)))) {
3935 for (unsigned i = 0; i < F16Pair->getNumOperands(); ++i) {
3936 SDValue ElF16 = stripBitcast(Val: F16Pair->getOperand(Num: i));
3937 if (!ModifierCheck(ElF16))
3938 break;
3939 }
3940 }
3941 }
3942}
3943
3944bool AMDGPUDAGToDAGISel::SelectWMMAModsF16Neg(SDValue In, SDValue &Src,
3945 SDValue &SrcMods) const {
3946 Src = In;
3947 unsigned Mods = SISrcMods::OP_SEL_1;
3948
3949 // mods are on f16 elements
3950 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: stripBitcast(Val: In))) {
3951 SmallVector<SDValue, 8> EltsF16;
3952
3953 checkWMMAElementsModifiersF16(BV, ModifierCheck: [&](SDValue Element) -> bool {
3954 if (Element.getOpcode() != ISD::FNEG)
3955 return false;
3956 EltsF16.push_back(Elt: Element.getOperand(i: 0));
3957 return true;
3958 });
3959
3960 // All elements have neg modifier
3961 if (BV->getNumOperands() * 2 == EltsF16.size()) {
3962 Src = SDValue(buildRegSequence16(Elts&: EltsF16, DL: SDLoc(In)), 0);
3963 Mods |= SISrcMods::NEG;
3964 Mods |= SISrcMods::NEG_HI;
3965 }
3966 }
3967
3968 // mods are on v2f16 elements
3969 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: stripBitcast(Val: In))) {
3970 SmallVector<SDValue, 8> EltsV2F16;
3971 for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
3972 SDValue ElV2f16 = stripBitcast(Val: BV->getOperand(Num: i));
3973 // Based on first element decide which mod we match, neg or abs
3974 if (ElV2f16.getOpcode() != ISD::FNEG)
3975 break;
3976 EltsV2F16.push_back(Elt: ElV2f16.getOperand(i: 0));
3977 }
3978
3979 // All pairs of elements have neg modifier
3980 if (BV->getNumOperands() == EltsV2F16.size()) {
3981 Src = SDValue(buildRegSequence32(Elts&: EltsV2F16, DL: SDLoc(In)), 0);
3982 Mods |= SISrcMods::NEG;
3983 Mods |= SISrcMods::NEG_HI;
3984 }
3985 }
3986
3987 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
3988 return true;
3989}
3990
3991bool AMDGPUDAGToDAGISel::SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
3992 SDValue &SrcMods) const {
3993 Src = In;
3994 unsigned Mods = SISrcMods::OP_SEL_1;
3995 unsigned ModOpcode;
3996
3997 // mods are on f16 elements
3998 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: stripBitcast(Val: In))) {
3999 SmallVector<SDValue, 8> EltsF16;
4000 checkWMMAElementsModifiersF16(BV, ModifierCheck: [&](SDValue ElF16) -> bool {
4001 // Based on first element decide which mod we match, neg or abs
4002 if (EltsF16.empty())
4003 ModOpcode = (ElF16.getOpcode() == ISD::FNEG) ? ISD::FNEG : ISD::FABS;
4004 if (ElF16.getOpcode() != ModOpcode)
4005 return false;
4006 EltsF16.push_back(Elt: ElF16.getOperand(i: 0));
4007 return true;
4008 });
4009
4010 // All elements have ModOpcode modifier
4011 if (BV->getNumOperands() * 2 == EltsF16.size())
4012 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsF16, Src, DL: SDLoc(In), ElementSize: 16);
4013 }
4014
4015 // mods are on v2f16 elements
4016 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: stripBitcast(Val: In))) {
4017 SmallVector<SDValue, 8> EltsV2F16;
4018
4019 for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
4020 SDValue ElV2f16 = stripBitcast(Val: BV->getOperand(Num: i));
4021 // Based on first element decide which mod we match, neg or abs
4022 if (EltsV2F16.empty())
4023 ModOpcode = (ElV2f16.getOpcode() == ISD::FNEG) ? ISD::FNEG : ISD::FABS;
4024 if (ElV2f16->getOpcode() != ModOpcode)
4025 break;
4026 EltsV2F16.push_back(Elt: ElV2f16->getOperand(Num: 0));
4027 }
4028
4029 // All elements have ModOpcode modifier
4030 if (BV->getNumOperands() == EltsV2F16.size())
4031 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsV2F16, Src, DL: SDLoc(In), ElementSize: 32);
4032 }
4033
4034 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
4035 return true;
4036}
4037
4038bool AMDGPUDAGToDAGISel::SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
4039 SDValue &SrcMods) const {
4040 Src = In;
4041 unsigned Mods = SISrcMods::OP_SEL_1;
4042 SmallVector<SDValue, 8> EltsF32;
4043
4044 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: stripBitcast(Val: In))) {
4045 assert(BV->getNumOperands() > 0);
4046 // Based on first element decide which mod we match, neg or abs
4047 SDValue ElF32 = stripBitcast(Val: BV->getOperand(Num: 0));
4048 unsigned ModOpcode =
4049 (ElF32.getOpcode() == ISD::FNEG) ? ISD::FNEG : ISD::FABS;
4050 for (unsigned i = 0; i < BV->getNumOperands(); ++i) {
4051 SDValue ElF32 = stripBitcast(Val: BV->getOperand(Num: i));
4052 if (ElF32.getOpcode() != ModOpcode)
4053 break;
4054 EltsF32.push_back(Elt: ElF32.getOperand(i: 0));
4055 }
4056
4057 // All elements had ModOpcode modifier
4058 if (BV->getNumOperands() == EltsF32.size())
4059 selectWMMAModsNegAbs(ModOpcode, Mods, Elts&: EltsF32, Src, DL: SDLoc(In), ElementSize: 32);
4060 }
4061
4062 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
4063 return true;
4064}
4065
4066bool AMDGPUDAGToDAGISel::SelectWMMAVISrc(SDValue In, SDValue &Src) const {
4067 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: In)) {
4068 BitVector UndefElements;
4069 if (SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements))
4070 if (isInlineImmediate(N: Splat.getNode())) {
4071 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Splat)) {
4072 unsigned Imm = C->getAPIntValue().getSExtValue();
4073 Src = CurDAG->getTargetConstant(Val: Imm, DL: SDLoc(In), VT: MVT::i32);
4074 return true;
4075 }
4076 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Splat)) {
4077 unsigned Imm = C->getValueAPF().bitcastToAPInt().getSExtValue();
4078 Src = CurDAG->getTargetConstant(Val: Imm, DL: SDLoc(In), VT: MVT::i32);
4079 return true;
4080 }
4081 llvm_unreachable("unhandled Constant node");
4082 }
4083 }
4084
4085 // 16 bit splat
4086 SDValue SplatSrc32 = stripBitcast(Val: In);
4087 if (auto *SplatSrc32BV = dyn_cast<BuildVectorSDNode>(Val&: SplatSrc32))
4088 if (SDValue Splat32 = SplatSrc32BV->getSplatValue()) {
4089 SDValue SplatSrc16 = stripBitcast(Val: Splat32);
4090 if (auto *SplatSrc16BV = dyn_cast<BuildVectorSDNode>(Val&: SplatSrc16))
4091 if (SDValue Splat = SplatSrc16BV->getSplatValue()) {
4092 const SIInstrInfo *TII = Subtarget->getInstrInfo();
4093 std::optional<APInt> RawValue;
4094 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Splat))
4095 RawValue = C->getValueAPF().bitcastToAPInt();
4096 else if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Splat))
4097 RawValue = C->getAPIntValue();
4098
4099 if (RawValue.has_value()) {
4100 EVT VT = In.getValueType().getScalarType();
4101 if (VT.getSimpleVT() == MVT::f16 || VT.getSimpleVT() == MVT::bf16) {
4102 APFloat FloatVal(VT.getSimpleVT() == MVT::f16
4103 ? APFloatBase::IEEEhalf()
4104 : APFloatBase::BFloat(),
4105 RawValue.value());
4106 if (TII->isInlineConstant(Imm: FloatVal)) {
4107 Src = CurDAG->getTargetConstant(Val: RawValue.value(), DL: SDLoc(In),
4108 VT: MVT::i16);
4109 return true;
4110 }
4111 } else if (VT.getSimpleVT() == MVT::i16) {
4112 if (TII->isInlineConstant(Imm: RawValue.value())) {
4113 Src = CurDAG->getTargetConstant(Val: RawValue.value(), DL: SDLoc(In),
4114 VT: MVT::i16);
4115 return true;
4116 }
4117 } else
4118 llvm_unreachable("unknown 16-bit type");
4119 }
4120 }
4121 }
4122
4123 // Currently f64 immediate vectors are represented as vectors of v2i32, with
4124 // different lo and hi 32-bit values even though double values are splated.
4125 // So we have to manually compare to determine whether it is splated.
4126 if (CurDAG->isConstantIntBuildVectorOrConstantInt(N: SplatSrc32)) {
4127 int64_t Imm64 = 0;
4128 for (unsigned i = 0; i < SplatSrc32->getNumOperands(); i += 2) {
4129 auto Lo32 = cast<ConstantSDNode>(Val: SplatSrc32->getOperand(Num: i));
4130 auto Hi32 = cast<ConstantSDNode>(Val: SplatSrc32->getOperand(Num: i + 1));
4131 int64_t LoImm = Lo32->getAPIntValue().getSExtValue();
4132 int64_t HiImm = Hi32->getAPIntValue().getSExtValue();
4133 int64_t Imm64I = (HiImm << 32) + LoImm;
4134 if (i == 0) {
4135 if (!isInlineImmediate(Imm: APInt(64, Imm64I)))
4136 return false;
4137 Imm64 = Imm64I;
4138 } else if (Imm64I != Imm64)
4139 return false;
4140 } // end for
4141
4142 Src = CurDAG->getTargetConstant(Val: Imm64, DL: SDLoc(In), VT: MVT::i64);
4143 return true;
4144 }
4145
4146 return false;
4147}
4148
4149bool AMDGPUDAGToDAGISel::SelectSWMMACIndex8(SDValue In, SDValue &Src,
4150 SDValue &IndexKey) const {
4151 unsigned Key = 0;
4152 Src = In;
4153
4154 if (In.getOpcode() == ISD::SRL) {
4155 const llvm::SDValue &ShiftSrc = In.getOperand(i: 0);
4156 ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Val: In.getOperand(i: 1));
4157 if (ShiftSrc.getValueType().getSizeInBits() == 32 && ShiftAmt &&
4158 ShiftAmt->getZExtValue() % 8 == 0) {
4159 Key = ShiftAmt->getZExtValue() / 8;
4160 Src = ShiftSrc;
4161 }
4162 }
4163
4164 IndexKey = CurDAG->getTargetConstant(Val: Key, DL: SDLoc(In), VT: MVT::i32);
4165 return true;
4166}
4167
4168bool AMDGPUDAGToDAGISel::SelectSWMMACIndex16(SDValue In, SDValue &Src,
4169 SDValue &IndexKey) const {
4170 unsigned Key = 0;
4171 Src = In;
4172
4173 if (In.getOpcode() == ISD::SRL) {
4174 const llvm::SDValue &ShiftSrc = In.getOperand(i: 0);
4175 ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Val: In.getOperand(i: 1));
4176 if (ShiftSrc.getValueType().getSizeInBits() == 32 && ShiftAmt &&
4177 ShiftAmt->getZExtValue() == 16) {
4178 Key = 1;
4179 Src = ShiftSrc;
4180 }
4181 }
4182
4183 IndexKey = CurDAG->getTargetConstant(Val: Key, DL: SDLoc(In), VT: MVT::i32);
4184 return true;
4185}
4186
4187bool AMDGPUDAGToDAGISel::SelectSWMMACIndex32(SDValue In, SDValue &Src,
4188 SDValue &IndexKey) const {
4189 unsigned Key = 0;
4190 Src = In;
4191
4192 SDValue InI32;
4193
4194 if (In.getOpcode() == ISD::ANY_EXTEND || In.getOpcode() == ISD::ZERO_EXTEND) {
4195 const SDValue &ExtendSrc = In.getOperand(i: 0);
4196 if (ExtendSrc.getValueSizeInBits() == 32)
4197 InI32 = ExtendSrc;
4198 } else if (In->getOpcode() == ISD::BITCAST) {
4199 const SDValue &CastSrc = In.getOperand(i: 0);
4200 if (CastSrc.getOpcode() == ISD::BUILD_VECTOR &&
4201 CastSrc.getOperand(i: 0).getValueSizeInBits() == 32) {
4202 ConstantSDNode *Zero = dyn_cast<ConstantSDNode>(Val: CastSrc.getOperand(i: 1));
4203 if (Zero && Zero->getZExtValue() == 0)
4204 InI32 = CastSrc.getOperand(i: 0);
4205 }
4206 }
4207
4208 if (InI32 && InI32.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
4209 const SDValue &ExtractVecEltSrc = InI32.getOperand(i: 0);
4210 ConstantSDNode *EltIdx = dyn_cast<ConstantSDNode>(Val: InI32.getOperand(i: 1));
4211 if (ExtractVecEltSrc.getValueSizeInBits() == 64 && EltIdx &&
4212 EltIdx->getZExtValue() == 1) {
4213 Key = 1;
4214 Src = ExtractVecEltSrc;
4215 }
4216 }
4217
4218 IndexKey = CurDAG->getTargetConstant(Val: Key, DL: SDLoc(In), VT: MVT::i32);
4219 return true;
4220}
4221
4222bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
4223 SDValue &SrcMods) const {
4224 Src = In;
4225 // FIXME: Handle op_sel
4226 SrcMods = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(In), VT: MVT::i32);
4227 return true;
4228}
4229
4230bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
4231 SDValue &SrcMods) const {
4232 // FIXME: Handle op_sel
4233 return SelectVOP3Mods(In, Src, SrcMods);
4234}
4235
4236// Match lowered fpext from bf16 to f32. This is a bit operation extending
4237// a 16-bit value with 16-bit of zeroes at LSB:
4238//
4239// 1. (f32 (bitcast (build_vector (i16 0), (i16 (bitcast bf16:val)))))
4240// 2. (f32 (bitcast (and i32:val, 0xffff0000))) -> IsExtractHigh = true
4241// 3. (f32 (bitcast (shl i32:va, 16) -> IsExtractHigh = false
4242static SDValue matchBF16FPExtendLike(SDValue Op, bool &IsExtractHigh) {
4243 if (Op.getValueType() != MVT::f32 || Op.getOpcode() != ISD::BITCAST)
4244 return SDValue();
4245 Op = Op.getOperand(i: 0);
4246
4247 IsExtractHigh = false;
4248 if (Op.getValueType() == MVT::v2i16 && Op.getOpcode() == ISD::BUILD_VECTOR) {
4249 auto Low16 = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 0));
4250 if (!Low16 || !Low16->isZero())
4251 return SDValue();
4252 Op = stripBitcast(Val: Op.getOperand(i: 1));
4253 if (Op.getValueType() != MVT::bf16)
4254 return SDValue();
4255 return Op;
4256 }
4257
4258 if (Op.getValueType() != MVT::i32)
4259 return SDValue();
4260
4261 if (Op.getOpcode() == ISD::AND) {
4262 if (auto Mask = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
4263 if (Mask->getZExtValue() == 0xffff0000) {
4264 IsExtractHigh = true;
4265 return Op.getOperand(i: 0);
4266 }
4267 }
4268 return SDValue();
4269 }
4270
4271 if (Op.getOpcode() == ISD::SHL) {
4272 if (auto Amt = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
4273 if (Amt->getZExtValue() == 16)
4274 return Op.getOperand(i: 0);
4275 }
4276 }
4277
4278 return SDValue();
4279}
4280
4281// The return value is not whether the match is possible (which it always is),
4282// but whether or not it a conversion is really used.
4283bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
4284 unsigned &Mods,
4285 MVT VT) const {
4286 Mods = 0;
4287 SelectVOP3ModsImpl(In, Src, Mods);
4288
4289 bool IsExtractHigh = false;
4290 if (Src.getOpcode() == ISD::FP_EXTEND) {
4291 Src = Src.getOperand(i: 0);
4292 } else if (VT == MVT::bf16) {
4293 SDValue B16 = matchBF16FPExtendLike(Op: Src, IsExtractHigh);
4294 if (!B16)
4295 return false;
4296 Src = B16;
4297 } else
4298 return false;
4299
4300 if (Src.getValueType() != VT &&
4301 (VT != MVT::bf16 || Src.getValueType() != MVT::i32))
4302 return false;
4303
4304 Src = stripBitcast(Val: Src);
4305
4306 // Be careful about folding modifiers if we already have an abs. fneg is
4307 // applied last, so we don't want to apply an earlier fneg.
4308 if ((Mods & SISrcMods::ABS) == 0) {
4309 unsigned ModsTmp;
4310 SelectVOP3ModsImpl(In: Src, Src, Mods&: ModsTmp);
4311
4312 if ((ModsTmp & SISrcMods::NEG) != 0)
4313 Mods ^= SISrcMods::NEG;
4314
4315 if ((ModsTmp & SISrcMods::ABS) != 0)
4316 Mods |= SISrcMods::ABS;
4317 }
4318
4319 // op_sel/op_sel_hi decide the source type and source.
4320 // If the source's op_sel_hi is set, it indicates to do a conversion from
4321 // fp16. If the sources's op_sel is set, it picks the high half of the source
4322 // register.
4323
4324 Mods |= SISrcMods::OP_SEL_1;
4325 if (Src.getValueSizeInBits() == 16) {
4326 if (isExtractHiElt(In: Src, Out&: Src)) {
4327 Mods |= SISrcMods::OP_SEL_0;
4328
4329 // TODO: Should we try to look for neg/abs here?
4330 return true;
4331 }
4332
4333 if (Src.getOpcode() == ISD::TRUNCATE &&
4334 Src.getOperand(i: 0).getValueType() == MVT::i32) {
4335 Src = Src.getOperand(i: 0);
4336 return true;
4337 }
4338
4339 if (Subtarget->useRealTrue16Insts())
4340 // In true16 mode, pack src to a 32bit
4341 Src = createVOP3PSrc32FromLo16(Lo: Src, Src: In, CurDAG, Subtarget);
4342 } else if (IsExtractHigh)
4343 Mods |= SISrcMods::OP_SEL_0;
4344
4345 return true;
4346}
4347
4348bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
4349 SDValue &SrcMods) const {
4350 unsigned Mods = 0;
4351 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, VT: MVT::f16))
4352 return false;
4353 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
4354 return true;
4355}
4356
4357bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
4358 SDValue &SrcMods) const {
4359 unsigned Mods = 0;
4360 SelectVOP3PMadMixModsImpl(In, Src, Mods, VT: MVT::f16);
4361 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
4362 return true;
4363}
4364
4365bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsExtNeg(SDValue In, SDValue &Src,
4366 SDValue &SrcMods) const {
4367 unsigned Mods = 0;
4368 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, VT: MVT::f16))
4369 return false;
4370 SrcMods =
4371 CurDAG->getTargetConstant(Val: Mods ^ SISrcMods::NEG, DL: SDLoc(In), VT: MVT::i32);
4372 return true;
4373}
4374
4375bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsNeg(SDValue In, SDValue &Src,
4376 SDValue &SrcMods) const {
4377 unsigned Mods = 0;
4378 SelectVOP3PMadMixModsImpl(In, Src, Mods, VT: MVT::f16);
4379 SrcMods =
4380 CurDAG->getTargetConstant(Val: Mods ^ SISrcMods::NEG, DL: SDLoc(In), VT: MVT::i32);
4381 return true;
4382}
4383
4384bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16ModsExt(SDValue In, SDValue &Src,
4385 SDValue &SrcMods) const {
4386 unsigned Mods = 0;
4387 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, VT: MVT::bf16))
4388 return false;
4389 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
4390 return true;
4391}
4392
4393bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16Mods(SDValue In, SDValue &Src,
4394 SDValue &SrcMods) const {
4395 unsigned Mods = 0;
4396 SelectVOP3PMadMixModsImpl(In, Src, Mods, VT: MVT::bf16);
4397 SrcMods = CurDAG->getTargetConstant(Val: Mods, DL: SDLoc(In), VT: MVT::i32);
4398 return true;
4399}
4400
4401bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16ModsExtNeg(
4402 SDValue In, SDValue &Src, SDValue &SrcMods) const {
4403 unsigned Mods = 0;
4404 if (!SelectVOP3PMadMixModsImpl(In, Src, Mods, VT: MVT::bf16))
4405 return false;
4406 SrcMods =
4407 CurDAG->getTargetConstant(Val: Mods ^ SISrcMods::NEG, DL: SDLoc(In), VT: MVT::i32);
4408 return true;
4409}
4410
4411bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixBF16ModsNeg(SDValue In, SDValue &Src,
4412 SDValue &SrcMods) const {
4413 unsigned Mods = 0;
4414 SelectVOP3PMadMixModsImpl(In, Src, Mods, VT: MVT::bf16);
4415 SrcMods =
4416 CurDAG->getTargetConstant(Val: Mods ^ SISrcMods::NEG, DL: SDLoc(In), VT: MVT::i32);
4417 return true;
4418}
4419
4420// Match BITOP3 operation and return a number of matched instructions plus
4421// truth table.
4422static std::pair<unsigned, uint8_t> BitOp3_Op(SDValue In,
4423 SmallVectorImpl<SDValue> &Src) {
4424 unsigned NumOpcodes = 0;
4425 uint8_t LHSBits, RHSBits;
4426
4427 auto getOperandBits = [&Src, In](SDValue Op, uint8_t &Bits) -> bool {
4428 // Define truth table given Src0, Src1, Src2 bits permutations:
4429 // 0 0 0
4430 // 0 0 1
4431 // 0 1 0
4432 // 0 1 1
4433 // 1 0 0
4434 // 1 0 1
4435 // 1 1 0
4436 // 1 1 1
4437 const uint8_t SrcBits[3] = { 0xf0, 0xcc, 0xaa };
4438
4439 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4440 if (C->isAllOnes()) {
4441 Bits = 0xff;
4442 return true;
4443 }
4444 if (C->isZero()) {
4445 Bits = 0;
4446 return true;
4447 }
4448 }
4449
4450 for (unsigned I = 0; I < Src.size(); ++I) {
4451 // Try to find existing reused operand
4452 if (Src[I] == Op) {
4453 Bits = SrcBits[I];
4454 return true;
4455 }
4456 // Try to replace parent operator
4457 if (Src[I] == In) {
4458 Bits = SrcBits[I];
4459 Src[I] = Op;
4460 return true;
4461 }
4462 }
4463
4464 if (Src.size() == 3) {
4465 // No room left for operands. Try one last time, there can be a 'not' of
4466 // one of our source operands. In this case we can compute the bits
4467 // without growing Src vector.
4468 if (Op.getOpcode() == ISD::XOR) {
4469 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
4470 if (C->isAllOnes()) {
4471 SDValue LHS = Op.getOperand(i: 0);
4472 for (unsigned I = 0; I < Src.size(); ++I) {
4473 if (Src[I] == LHS) {
4474 Bits = ~SrcBits[I];
4475 return true;
4476 }
4477 }
4478 }
4479 }
4480 }
4481
4482 return false;
4483 }
4484
4485 Bits = SrcBits[Src.size()];
4486 Src.push_back(Elt: Op);
4487 return true;
4488 };
4489
4490 switch (In.getOpcode()) {
4491 case ISD::AND:
4492 case ISD::OR:
4493 case ISD::XOR: {
4494 SDValue LHS = In.getOperand(i: 0);
4495 SDValue RHS = In.getOperand(i: 1);
4496
4497 SmallVector<SDValue, 3> Backup(Src.begin(), Src.end());
4498 if (!getOperandBits(LHS, LHSBits) ||
4499 !getOperandBits(RHS, RHSBits)) {
4500 Src = std::move(Backup);
4501 return std::make_pair(x: 0, y: 0);
4502 }
4503
4504 // Recursion is naturally limited by the size of the operand vector.
4505 //
4506 // When LHS and RHS share a common sub-expression, one side's recursion
4507 // may decompose that sub-expression and replace the Src slot the other
4508 // side occupies with sub-operands via the "replace parent" path in
4509 // getOperandBits. The other side's cached bit-pattern then refers to a
4510 // slot whose contents changed, producing a wrong truth table.
4511 //
4512 // We detect this in three ways:
4513 // (A) If LHS recursed, its truth table is valid against the Src state
4514 // when LHS recursion completed (SrcAfterLHS). If RHS recursion
4515 // then mutates a Src slot that LHSBits depends on, LHSBits is
4516 // stale.
4517 // (B) If RHS did not recurse, RHSBits came from getOperandBits and
4518 // refers to a specific Src slot. If that slot's contents changed
4519 // (by either recursion), RHSBits is stale.
4520 // (C) Symmetrically for LHS if it did not recurse.
4521 SmallVector<SDValue, 3> SrcBeforeRecurse(Src.begin(), Src.end());
4522 uint8_t LHSBitsOrig = LHSBits;
4523 uint8_t RHSBitsOrig = RHSBits;
4524
4525 auto LHSOp = BitOp3_Op(In: LHS, Src);
4526 if (LHSOp.first) {
4527 NumOpcodes += LHSOp.first;
4528 LHSBits = LHSOp.second;
4529 }
4530
4531 SmallVector<SDValue, 3> SrcAfterLHS(Src.begin(), Src.end());
4532
4533 auto RHSOp = BitOp3_Op(In: RHS, Src);
4534 if (RHSOp.first) {
4535 NumOpcodes += RHSOp.first;
4536 RHSBits = RHSOp.second;
4537 }
4538
4539 // dependsOnSlot: true iff the truth table TT varies with slot Slot.
4540 auto dependsOnSlot = [](uint8_t TT, int Slot) -> bool {
4541 if (Slot < 0 || Slot > 2)
4542 return false;
4543 const uint8_t Masks[3] = {0x0f, 0x33, 0x55};
4544 const int Shifts[3] = {4, 2, 1};
4545 return ((TT ^ (TT >> Shifts[Slot])) & Masks[Slot]) != 0;
4546 };
4547
4548 // findSlot: locate the Src slot a getOperandBits result depends on,
4549 // including negated (XOR with -1) patterns that getOperandBits
4550 // resolves via the NOT shortcut (~SrcBits[I]).
4551 const uint8_t SrcBitsConst[3] = {0xf0, 0xcc, 0xaa};
4552 auto findSlot = [&](uint8_t Bits, SDValue Op,
4553 const SmallVectorImpl<SDValue> &S) -> int {
4554 SDValue NegatedInner;
4555 bool IsNegationOp =
4556 Op.getOpcode() == ISD::XOR && isAllOnesConstant(V: Op.getOperand(i: 1));
4557 if (IsNegationOp)
4558 NegatedInner = Op.getOperand(i: 0);
4559 for (int I = 0; I < (int)S.size(); I++) {
4560 if (Bits == SrcBitsConst[I] && S[I] == Op)
4561 return I;
4562 if (IsNegationOp && Bits == (uint8_t)~SrcBitsConst[I] &&
4563 S[I] == NegatedInner)
4564 return I;
4565 }
4566 return -1;
4567 };
4568
4569 bool Stale = false;
4570
4571 // (A) LHS recursed: its truth table is against SrcAfterLHS.
4572 // Check if RHS recursion mutated a slot that LHSBits uses.
4573 if (LHSOp.first) {
4574 for (int I = 0; I < (int)SrcAfterLHS.size() && I < 3; I++) {
4575 if (I < (int)Src.size() && Src[I] != SrcAfterLHS[I] &&
4576 dependsOnSlot(LHSBits, I)) {
4577 Stale = true;
4578 break;
4579 }
4580 }
4581 }
4582
4583 // (B) RHS did not recurse: RHSBits from getOperandBits is against
4584 // SrcBeforeRecurse. Check if that slot was mutated since then.
4585 if (!Stale && !RHSOp.first) {
4586 int Slot = findSlot(RHSBitsOrig, RHS, SrcBeforeRecurse);
4587 if (Slot >= 0 &&
4588 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4589 Stale = true;
4590 }
4591
4592 // (C) LHS did not recurse: LHSBits from getOperandBits is against
4593 // SrcBeforeRecurse. Check if that slot was mutated since then.
4594 if (!Stale && !LHSOp.first) {
4595 int Slot = findSlot(LHSBitsOrig, LHS, SrcBeforeRecurse);
4596 if (Slot >= 0 &&
4597 (Slot >= (int)Src.size() || Src[Slot] != SrcBeforeRecurse[Slot]))
4598 Stale = true;
4599 }
4600
4601 if (Stale) {
4602 Src = std::move(SrcBeforeRecurse);
4603 LHSBits = LHSBitsOrig;
4604 RHSBits = RHSBitsOrig;
4605 NumOpcodes = 0;
4606 }
4607 break;
4608 }
4609 default:
4610 return std::make_pair(x: 0, y: 0);
4611 }
4612
4613 uint8_t TTbl;
4614 switch (In.getOpcode()) {
4615 case ISD::AND:
4616 TTbl = LHSBits & RHSBits;
4617 break;
4618 case ISD::OR:
4619 TTbl = LHSBits | RHSBits;
4620 break;
4621 case ISD::XOR:
4622 TTbl = LHSBits ^ RHSBits;
4623 break;
4624 default:
4625 break;
4626 }
4627
4628 return std::make_pair(x: NumOpcodes + 1, y&: TTbl);
4629}
4630
4631bool AMDGPUDAGToDAGISel::SelectBITOP3(SDValue In, SDValue &Src0, SDValue &Src1,
4632 SDValue &Src2, SDValue &Tbl) const {
4633 SmallVector<SDValue, 3> Src;
4634 uint8_t TTbl;
4635 unsigned NumOpcodes;
4636
4637 std::tie(args&: NumOpcodes, args&: TTbl) = BitOp3_Op(In, Src);
4638
4639 // Src.empty() case can happen if all operands are all zero or all ones.
4640 // Normally it shall be optimized out before reaching this.
4641 if (NumOpcodes < 2 || Src.empty())
4642 return false;
4643
4644 // For a uniform case threshold should be higher to account for moves between
4645 // VGPRs and SGPRs. It needs one operand in a VGPR, rest two can be in SGPRs
4646 // and a readtfirstlane after.
4647 if (NumOpcodes < 4 && !In->isDivergent())
4648 return false;
4649
4650 if (NumOpcodes == 2 && In.getValueType() == MVT::i32) {
4651 // Avoid using BITOP3 for OR3, XOR3, AND_OR. This is not faster but makes
4652 // asm more readable. This cannot be modeled with AddedComplexity because
4653 // selector does not know how many operations did we match.
4654 if ((In.getOpcode() == ISD::XOR || In.getOpcode() == ISD::OR) &&
4655 (In.getOperand(i: 0).getOpcode() == In.getOpcode() ||
4656 In.getOperand(i: 1).getOpcode() == In.getOpcode()))
4657 return false;
4658
4659 if (In.getOpcode() == ISD::OR &&
4660 (In.getOperand(i: 0).getOpcode() == ISD::AND ||
4661 In.getOperand(i: 1).getOpcode() == ISD::AND))
4662 return false;
4663 }
4664
4665 // Last operand can be ignored, turning a ternary operation into a binary.
4666 // For example: (~a & b & c) | (~a & b & ~c) -> (~a & b). We can replace
4667 // 'c' with 'a' here without changing the answer. In some pathological
4668 // cases it should be possible to get an operation with a single operand
4669 // too if optimizer would not catch it.
4670 while (Src.size() < 3)
4671 Src.push_back(Elt: Src[0]);
4672
4673 Src0 = Src[0];
4674 Src1 = Src[1];
4675 Src2 = Src[2];
4676
4677 Tbl = CurDAG->getTargetConstant(Val: TTbl, DL: SDLoc(In), VT: MVT::i32);
4678 return true;
4679}
4680
4681SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
4682 if (In.getOpcode() == ISD::POISON)
4683 return CurDAG->getPOISON(VT: MVT::i32);
4684
4685 if (In.getOpcode() == ISD::UNDEF)
4686 return CurDAG->getUNDEF(VT: MVT::i32);
4687
4688 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: In)) {
4689 SDLoc SL(In);
4690 return CurDAG->getConstant(Val: C->getZExtValue() << 16, DL: SL, VT: MVT::i32);
4691 }
4692
4693 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: In)) {
4694 SDLoc SL(In);
4695 return CurDAG->getConstant(
4696 Val: C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, DL: SL, VT: MVT::i32);
4697 }
4698
4699 SDValue Src;
4700 if (isExtractHiElt(In, Out&: Src))
4701 return Src;
4702
4703 return SDValue();
4704}
4705
4706bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
4707 assert(CurDAG->getTarget().getTargetTriple().isAMDGCN());
4708
4709 const SIRegisterInfo *SIRI = Subtarget->getRegisterInfo();
4710 const SIInstrInfo *SII = Subtarget->getInstrInfo();
4711
4712 unsigned Limit = 0;
4713 bool AllUsesAcceptSReg = true;
4714 for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
4715 Limit < 10 && U != E; ++U, ++Limit) {
4716 const TargetRegisterClass *RC =
4717 getOperandRegClass(N: U->getUser(), OpNo: U->getOperandNo());
4718
4719 // If the register class is unknown, it could be an unknown
4720 // register class that needs to be an SGPR, e.g. an inline asm
4721 // constraint
4722 if (!RC || SIRI->isSGPRClass(RC))
4723 return false;
4724
4725 if (RC != &AMDGPU::VS_32RegClass && RC != &AMDGPU::VS_64RegClass &&
4726 RC != &AMDGPU::VS_64_Align2RegClass) {
4727 AllUsesAcceptSReg = false;
4728 SDNode *User = U->getUser();
4729 if (User->isMachineOpcode()) {
4730 unsigned Opc = User->getMachineOpcode();
4731 const MCInstrDesc &Desc = SII->get(Opcode: Opc);
4732 if (Desc.isCommutable()) {
4733 unsigned OpIdx = Desc.getNumDefs() + U->getOperandNo();
4734 unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
4735 if (SII->findCommutedOpIndices(Desc, SrcOpIdx0&: OpIdx, SrcOpIdx1&: CommuteIdx1)) {
4736 unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs();
4737 const TargetRegisterClass *CommutedRC =
4738 getOperandRegClass(N: U->getUser(), OpNo: CommutedOpNo);
4739 if (CommutedRC == &AMDGPU::VS_32RegClass ||
4740 CommutedRC == &AMDGPU::VS_64RegClass ||
4741 CommutedRC == &AMDGPU::VS_64_Align2RegClass)
4742 AllUsesAcceptSReg = true;
4743 }
4744 }
4745 }
4746 // If "AllUsesAcceptSReg == false" so far we haven't succeeded
4747 // commuting current user. This means have at least one use
4748 // that strictly require VGPR. Thus, we will not attempt to commute
4749 // other user instructions.
4750 if (!AllUsesAcceptSReg)
4751 break;
4752 }
4753 }
4754 return !AllUsesAcceptSReg && (Limit < 10);
4755}
4756
4757bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode *N) const {
4758 const auto *Ld = cast<LoadSDNode>(Val: N);
4759 const MachineMemOperand *MMO = Ld->getMemOperand();
4760
4761 // FIXME: We ought to able able to take the direct isDivergent result. We
4762 // cannot rely on the MMO for a uniformity check, and should stop using
4763 // it. This is a hack for 2 ways that the IR divergence analysis is superior
4764 // to the DAG divergence: Recognizing shift-of-workitem-id as always
4765 // uniform, and isSingleLaneExecution. These should be handled in the DAG
4766 // version, and then this can be dropped.
4767 if (Ld->isDivergent() && !AMDGPU::isUniformMMO(MMO))
4768 return false;
4769
4770 return MMO->getSize().hasValue() &&
4771 Ld->getAlign() >=
4772 Align(std::min(a: MMO->getSize().getValue().getKnownMinValue(),
4773 b: uint64_t(4))) &&
4774 (MMO->isInvariant() ||
4775 (Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
4776 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
4777 (Subtarget->getScalarizeGlobalBehavior() &&
4778 Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
4779 Ld->isSimple() &&
4780 static_cast<const SITargetLowering *>(getTargetLowering())
4781 ->isMemOpHasNoClobberedMemOperand(N)));
4782}
4783
4784void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
4785 const AMDGPUTargetLowering& Lowering =
4786 *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
4787 bool IsModified = false;
4788 do {
4789 IsModified = false;
4790
4791 // Go over all selected nodes and try to fold them a bit more
4792 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin();
4793 while (Position != CurDAG->allnodes_end()) {
4794 SDNode *Node = &*Position++;
4795 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Val: Node);
4796 if (!MachineNode)
4797 continue;
4798
4799 SDNode *ResNode = Lowering.PostISelFolding(N: MachineNode, DAG&: *CurDAG);
4800 if (ResNode != Node) {
4801 if (ResNode)
4802 ReplaceUses(F: Node, T: ResNode);
4803 IsModified = true;
4804 }
4805 }
4806 CurDAG->RemoveDeadNodes();
4807 } while (IsModified);
4808}
4809
4810AMDGPUDAGToDAGISelLegacy::AMDGPUDAGToDAGISelLegacy(TargetMachine &TM,
4811 CodeGenOptLevel OptLevel)
4812 : SelectionDAGISelLegacy(
4813 ID, std::make_unique<AMDGPUDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
4814
4815char AMDGPUDAGToDAGISelLegacy::ID = 0;
4816