1//===-- AMDGPUISelDAGToDAG.h - A dag to dag inst selector for AMDGPU ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//==-----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines an instruction selector for the AMDGPU target.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
15#define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
16
17#include "AMDGPUSelectionDAGInfo.h"
18#include "GCNSubtarget.h"
19#include "SIMachineFunctionInfo.h"
20#include "SIModeRegisterDefaults.h"
21#include "llvm/Analysis/ValueTracking.h"
22#include "llvm/CodeGen/SelectionDAGISel.h"
23#include "llvm/Support/AMDGPUAddrSpace.h"
24#include "llvm/Target/TargetMachine.h"
25
26namespace llvm {
27
28static inline bool getConstantValue(SDValue N, uint32_t &Out) {
29 // This is only used for packed vectors, where using 0 for undef should
30 // always be good.
31 if (N.isUndef()) {
32 Out = 0;
33 return true;
34 }
35
36 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: N)) {
37 Out = C->getAPIntValue().getSExtValue();
38 return true;
39 }
40
41 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: N)) {
42 Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
43 return true;
44 }
45
46 return false;
47}
48
49/// AMDGPU specific code to select AMDGPU machine instructions for
50/// SelectionDAG operations.
51class AMDGPUDAGToDAGISel : public SelectionDAGISel {
52 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
53 // make the right decision when generating code for different targets.
54 const GCNSubtarget *Subtarget;
55
56 // Default FP mode for the current function.
57 SIModeRegisterDefaults Mode;
58
59 // Instructions that will be lowered with a final instruction that zeros the
60 // high result bits.
61 bool fp16SrcZerosHighBits(unsigned Opc) const;
62
63public:
64 AMDGPUDAGToDAGISel() = delete;
65
66 explicit AMDGPUDAGToDAGISel(TargetMachine &TM, CodeGenOptLevel OptLevel);
67
68 bool runOnMachineFunction(MachineFunction &MF) override;
69 bool matchLoadD16FromBuildVector(SDNode *N) const;
70 bool widenRegionLoad16(SDNode *N) const;
71 void PreprocessISelDAG() override;
72 void Select(SDNode *N) override;
73 void PostprocessISelDAG() override;
74 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
75 InlineAsm::ConstraintCode ConstraintID,
76 std::vector<SDValue> &OutOps) override;
77
78protected:
79 void SelectBuildVector(SDNode *N, unsigned RegClassID);
80 void SelectVectorShuffle(SDNode *N);
81 bool isSDWAOperand(const SDNode *N) const;
82
83private:
84 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
85
86 bool isInlineImmediate(const SDNode *N) const;
87
88 bool isInlineImmediate(const APInt &Imm) const {
89 return Subtarget->getInstrInfo()->isInlineConstant(Imm);
90 }
91
92 bool isInlineImmediate(const APFloat &Imm) const {
93 return Subtarget->getInstrInfo()->isInlineConstant(Imm);
94 }
95
96 bool isVGPRImm(const SDNode *N) const;
97 bool isUniformBr(const SDNode *N) const;
98 bool isVRegIdxFoldableAdd(const SDNode *N, unsigned NumDwords) const;
99
100 MachineSDNode *buildRegSequence16(SmallVectorImpl<SDValue> &Elts,
101 const SDLoc &DL) const;
102 MachineSDNode *buildRegSequence32(SmallVectorImpl<SDValue> &Elts,
103 const SDLoc &DL) const;
104 MachineSDNode *buildRegSequence(SmallVectorImpl<SDValue> &Elts,
105 const SDLoc &DL, unsigned ElementSize) const;
106
107 void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods,
108 SmallVectorImpl<SDValue> &Elts, SDValue &Src,
109 const SDLoc &DL, unsigned ElementSize) const;
110
111 // Returns true if ISD::AND SDNode `N`'s masking of the shift amount operand's
112 // `ShAmtBits` bits is unneeded.
113 bool isUnneededShiftMask(const SDNode *N, unsigned ShAmtBits) const;
114
115 bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
116 SDValue &RHS) const;
117
118 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
119
120 SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG) const;
121
122 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
123 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
124 SDNode *glueCopyToM0LDSInit(SDNode *N) const;
125
126 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
127 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
128 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
129 bool isDSOffsetLegal(SDValue Base, unsigned Offset) const;
130 bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1,
131 unsigned Size) const;
132
133 bool isFlatScratchBaseLegal(SDValue Addr) const;
134 bool isFlatScratchBaseLegalSV(SDValue Addr) const;
135 bool isFlatScratchBaseLegalSVImm(SDValue Addr) const;
136 bool isSOffsetLegalWithImmOffset(SDValue *SOffset, bool Imm32Only,
137 bool IsBuffer, int64_t ImmOffset = 0) const;
138
139 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
140 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
141 SDValue &Offset1) const;
142 bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
143 SDValue &Offset1) const;
144 bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,
145 SDValue &Offset1, unsigned Size) const;
146 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
147 SDValue &SOffset, SDValue &Offset, SDValue &Offen,
148 SDValue &Idxen, SDValue &Addr64) const;
149 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
150 SDValue &SOffset, SDValue &Offset) const;
151 bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc,
152 SDValue &VAddr, SDValue &SOffset,
153 SDValue &ImmOffset) const;
154 bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc,
155 SDValue &Soffset, SDValue &Offset) const;
156
157 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
158 SDValue &Offset) const;
159 bool SelectBUFSOffset(SDValue Addr, SDValue &SOffset) const;
160
161 bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr,
162 SDValue &Offset,
163 AMDGPU::FlatAddrSpace FlatVariant) const;
164 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
165 SDValue &Offset) const;
166 bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
167 SDValue &Offset) const;
168 bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
169 SDValue &Offset) const;
170 bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
171 SDValue &VOffset, SDValue &Offset, bool &ScaleOffset,
172 bool NeedIOffset = true) const;
173 bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
174 SDValue &VOffset, SDValue &Offset,
175 SDValue &CPol) const;
176 bool SelectGlobalSAddrCPol(SDNode *N, SDValue Addr, SDValue &SAddr,
177 SDValue &VOffset, SDValue &Offset,
178 SDValue &CPol) const;
179 bool SelectGlobalSAddrCPolM0(SDNode *N, SDValue Addr, SDValue &SAddr,
180 SDValue &VOffset, SDValue &Offset,
181 SDValue &CPol) const;
182 bool SelectGlobalSAddrGLC(SDNode *N, SDValue Addr, SDValue &SAddr,
183 SDValue &VOffset, SDValue &Offset,
184 SDValue &CPol) const;
185 bool SelectGlobalSAddrNoIOffset(SDNode *N, SDValue Addr, SDValue &SAddr,
186 SDValue &VOffset, SDValue &CPol) const;
187 bool SelectGlobalSAddrNoIOffsetM0(SDNode *N, SDValue Addr, SDValue &SAddr,
188 SDValue &VOffset, SDValue &CPol) const;
189 bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
190 SDValue &Offset) const;
191 bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,
192 uint64_t ImmOffset) const;
193 bool SelectScratchSVAddr(SDNode *N, SDValue Addr, SDValue &VAddr,
194 SDValue &SAddr, SDValue &Offset,
195 SDValue &CPol) const;
196
197 bool SelectSMRDOffset(SDNode *N, SDValue ByteOffsetNode, SDValue *SOffset,
198 SDValue *Offset, bool Imm32Only = false,
199 bool IsBuffer = false, bool HasSOffset = false,
200 int64_t ImmOffset = 0,
201 bool *ScaleOffset = nullptr) const;
202 SDValue Expand32BitAddress(SDValue Addr) const;
203 bool SelectSMRDBaseOffset(SDNode *N, SDValue Addr, SDValue &SBase,
204 SDValue *SOffset, SDValue *Offset,
205 bool Imm32Only = false, bool IsBuffer = false,
206 bool HasSOffset = false, int64_t ImmOffset = 0,
207 bool *ScaleOffset = nullptr) const;
208 bool SelectSMRD(SDNode *N, SDValue Addr, SDValue &SBase, SDValue *SOffset,
209 SDValue *Offset, bool Imm32Only = false,
210 bool *ScaleOffset = nullptr) const;
211 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
212 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
213 bool SelectScaleOffset(SDNode *N, SDValue &Offset, bool IsSigned,
214 bool IsSMEM = false) const;
215 bool SelectSMRDSgpr(SDNode *N, SDValue Addr, SDValue &SBase, SDValue &SOffset,
216 SDValue &CPol) const;
217 bool SelectSMRDSgprImm(SDNode *N, SDValue Addr, SDValue &SBase,
218 SDValue &SOffset, SDValue &Offset,
219 SDValue &CPol) const;
220 bool SelectSMRDBufferImm(SDValue N, SDValue &Offset) const;
221 bool SelectSMRDBufferImm32(SDValue N, SDValue &Offset) const;
222 bool SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,
223 SDValue &Offset) const;
224 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
225
226 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,
227 bool IsCanonicalizing = true,
228 bool AllowAbs = true) const;
229 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
230 bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src,
231 SDValue &SrcMods) const;
232 bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
233 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
234 bool SelectVOP3NoOpSelMods(SDValue In, SDValue &Src) const;
235 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
236 SDValue &Clamp, SDValue &Omod) const;
237 bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
238 SDValue &Clamp, SDValue &Omod) const;
239
240 bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,
241 bool OpSel) const;
242 bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
243 bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods) const;
244
245 bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,
246 SDValue &Omod) const;
247
248 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,
249 bool IsDOT = false, bool NoOpSel0 = false) const;
250 bool SelectVOP3PModsPkFmaOpSel0(SDValue In, SDValue &Src,
251 SDValue &SrcMods) const;
252 bool SelectVOP3PModsPkAddMulOpSel0(SDValue In, SDValue &Src,
253 SDValue &SrcMods) const;
254 bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods) const;
255 bool SelectVOP3PNoModsDOT(SDValue In, SDValue &Src) const;
256 bool SelectVOP3PModsF32(SDValue In, SDValue &Src, SDValue &SrcMods) const;
257 bool SelectVOP3PNoModsF32(SDValue In, SDValue &Src) const;
258
259 bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src) const;
260
261 bool SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
262 SDValue &SrcMods) const;
263 bool SelectWMMAModsF16Neg(SDValue In, SDValue &Src, SDValue &SrcMods) const;
264 bool SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
265 SDValue &SrcMods) const;
266 bool SelectWMMAVISrc(SDValue In, SDValue &Src) const;
267
268 bool SelectSWMMACIndex8(SDValue In, SDValue &Src, SDValue &IndexKey) const;
269 bool SelectSWMMACIndex16(SDValue In, SDValue &Src, SDValue &IndexKey) const;
270 bool SelectSWMMACIndex32(SDValue In, SDValue &Src, SDValue &IndexKey) const;
271
272 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
273
274 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
275 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods,
276 MVT VT) const;
277 bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
278 SDValue &SrcMods) const;
279 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
280 bool SelectVOP3PMadMixModsExtNeg(SDValue In, SDValue &Src,
281 SDValue &SrcMods) const;
282 bool SelectVOP3PMadMixModsNeg(SDValue In, SDValue &Src,
283 SDValue &SrcMods) const;
284 bool SelectVOP3PMadMixBF16ModsExt(SDValue In, SDValue &Src,
285 SDValue &SrcMods) const;
286 bool SelectVOP3PMadMixBF16Mods(SDValue In, SDValue &Src,
287 SDValue &SrcMods) const;
288 bool SelectVOP3PMadMixBF16ModsExtNeg(SDValue In, SDValue &Src,
289 SDValue &SrcMods) const;
290 bool SelectVOP3PMadMixBF16ModsNeg(SDValue In, SDValue &Src,
291 SDValue &SrcMods) const;
292
293 bool SelectBITOP3(SDValue In, SDValue &Src0, SDValue &Src1, SDValue &Src2,
294 SDValue &Tbl) const;
295
296 SDValue getHi16Elt(SDValue In) const;
297
298 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
299
300 void SelectAddcSubb(SDNode *N);
301 void SelectAddcSubbI64(SDNode *N);
302 void SelectUADDO_USUBO(SDNode *N);
303 void SelectDIV_SCALE(SDNode *N);
304 void SelectMAD_64_32(SDNode *N);
305 void SelectMUL_LOHI(SDNode *N);
306 void SelectFMA_W_CHAIN(SDNode *N);
307 void SelectFMUL_W_CHAIN(SDNode *N);
308 SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset,
309 uint32_t Width);
310 void SelectS_BFEFromShifts(SDNode *N);
311 void SelectS_BFE(SDNode *N);
312 bool isCBranchSCC(const SDNode *N) const;
313 void SelectBRCOND(SDNode *N);
314 void SelectFP_EXTEND(SDNode *N);
315 void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
316 void SelectDSBvhStackIntrinsic(SDNode *N, unsigned IntrID);
317 void SelectTensorLoadStore(SDNode *N, unsigned IntrID);
318 void SelectDS_GWS(SDNode *N, unsigned IntrID);
319 void SelectInterpP1F16(SDNode *N);
320 void SelectINTRINSIC_W_CHAIN(SDNode *N);
321 void SelectINTRINSIC_WO_CHAIN(SDNode *N);
322 void SelectINTRINSIC_VOID(SDNode *N);
323 void SelectWAVE_ADDRESS(SDNode *N);
324 void SelectSTACKRESTORE(SDNode *N);
325 void SelectWRITE_REGISTER(SDNode *N);
326
327protected:
328 // Include the pieces autogenerated from the target description.
329#include "AMDGPUGenDAGISel.inc"
330};
331
332class AMDGPUISelDAGToDAGPass : public SelectionDAGISelPass {
333public:
334 AMDGPUISelDAGToDAGPass(TargetMachine &TM);
335
336 PreservedAnalyses run(MachineFunction &MF,
337 MachineFunctionAnalysisManager &MFAM);
338};
339
340class AMDGPUDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
341public:
342 static char ID;
343
344 AMDGPUDAGToDAGISelLegacy(TargetMachine &TM, CodeGenOptLevel OptLevel);
345
346 bool runOnMachineFunction(MachineFunction &MF) override;
347 void getAnalysisUsage(AnalysisUsage &AU) const override;
348 StringRef getPassName() const override;
349};
350
351} // namespace llvm
352
353#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
354