1//===-- AMDGPUISelDAGToDAG.h - A dag to dag inst selector for AMDGPU ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//==-----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines an instruction selector for the AMDGPU target.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
15#define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
16
17#include "GCNSubtarget.h"
18#include "SIMachineFunctionInfo.h"
19#include "SIModeRegisterDefaults.h"
20#include "llvm/CodeGen/SelectionDAGISel.h"
21#include "llvm/Target/TargetMachine.h"
22
23using namespace llvm;
24
25namespace {
26
27static inline bool getConstantValue(SDValue N, uint32_t &Out) {
28 // This is only used for packed vectors, where using 0 for undef should
29 // always be good.
30 if (N.isUndef()) {
31 Out = 0;
32 return true;
33 }
34
35 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: N)) {
36 Out = C->getAPIntValue().getSExtValue();
37 return true;
38 }
39
40 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: N)) {
41 Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
42 return true;
43 }
44
45 return false;
46}
47
48// TODO: Handle undef as zero
49static inline SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG) {
50 assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
51 uint32_t LHSVal, RHSVal;
52 if (getConstantValue(N: N->getOperand(Num: 0), Out&: LHSVal) &&
53 getConstantValue(N: N->getOperand(Num: 1), Out&: RHSVal)) {
54 SDLoc SL(N);
55 uint32_t K = (LHSVal & 0xffff) | (RHSVal << 16);
56 return DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: SL, VT: N->getValueType(ResNo: 0),
57 Op1: DAG.getTargetConstant(Val: K, DL: SL, VT: MVT::i32));
58 }
59
60 return nullptr;
61}
62
63} // namespace
64
65/// AMDGPU specific code to select AMDGPU machine instructions for
66/// SelectionDAG operations.
67class AMDGPUDAGToDAGISel : public SelectionDAGISel {
68 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
69 // make the right decision when generating code for different targets.
70 const GCNSubtarget *Subtarget;
71
72 // Default FP mode for the current function.
73 SIModeRegisterDefaults Mode;
74
75 bool EnableLateStructurizeCFG;
76
77 // Instructions that will be lowered with a final instruction that zeros the
78 // high result bits.
79 bool fp16SrcZerosHighBits(unsigned Opc) const;
80
81public:
82 AMDGPUDAGToDAGISel() = delete;
83
84 explicit AMDGPUDAGToDAGISel(TargetMachine &TM, CodeGenOptLevel OptLevel);
85
86 bool runOnMachineFunction(MachineFunction &MF) override;
87 bool matchLoadD16FromBuildVector(SDNode *N) const;
88 void PreprocessISelDAG() override;
89 void Select(SDNode *N) override;
90 void PostprocessISelDAG() override;
91
92protected:
93 void SelectBuildVector(SDNode *N, unsigned RegClassID);
94
95private:
96 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
97
98 bool isInlineImmediate(const SDNode *N) const;
99
100 bool isInlineImmediate(const APInt &Imm) const {
101 return Subtarget->getInstrInfo()->isInlineConstant(Imm);
102 }
103
104 bool isInlineImmediate(const APFloat &Imm) const {
105 return Subtarget->getInstrInfo()->isInlineConstant(Imm);
106 }
107
108 bool isVGPRImm(const SDNode *N) const;
109 bool isUniformLoad(const SDNode *N) const;
110 bool isUniformBr(const SDNode *N) const;
111
112 // Returns true if ISD::AND SDNode `N`'s masking of the shift amount operand's
113 // `ShAmtBits` bits is unneeded.
114 bool isUnneededShiftMask(const SDNode *N, unsigned ShAmtBits) const;
115
116 bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
117 SDValue &RHS) const;
118
119 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
120
121 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
122 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
123 SDNode *glueCopyToM0LDSInit(SDNode *N) const;
124
125 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
126 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
127 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
128 bool isDSOffsetLegal(SDValue Base, unsigned Offset) const;
129 bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1,
130 unsigned Size) const;
131
132 bool isFlatScratchBaseLegal(SDValue Addr) const;
133 bool isFlatScratchBaseLegalSV(SDValue Addr) const;
134 bool isFlatScratchBaseLegalSVImm(SDValue Addr) const;
135 bool isSOffsetLegalWithImmOffset(SDValue *SOffset, bool Imm32Only,
136 bool IsBuffer, int64_t ImmOffset = 0) const;
137
138 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
139 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
140 SDValue &Offset1) const;
141 bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
142 SDValue &Offset1) const;
143 bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,
144 SDValue &Offset1, unsigned Size) const;
145 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
146 SDValue &SOffset, SDValue &Offset, SDValue &Offen,
147 SDValue &Idxen, SDValue &Addr64) const;
148 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
149 SDValue &SOffset, SDValue &Offset) const;
150 bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc,
151 SDValue &VAddr, SDValue &SOffset,
152 SDValue &ImmOffset) const;
153 bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc,
154 SDValue &Soffset, SDValue &Offset) const;
155
156 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
157 SDValue &Offset) const;
158 bool SelectBUFSOffset(SDValue Addr, SDValue &SOffset) const;
159
160 bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr,
161 SDValue &Offset, uint64_t FlatVariant) const;
162 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
163 SDValue &Offset) const;
164 bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
165 SDValue &Offset) const;
166 bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
167 SDValue &Offset) const;
168 bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
169 SDValue &VOffset, SDValue &Offset) const;
170 bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
171 SDValue &Offset) const;
172 bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,
173 uint64_t ImmOffset) const;
174 bool SelectScratchSVAddr(SDNode *N, SDValue Addr, SDValue &VAddr,
175 SDValue &SAddr, SDValue &Offset) const;
176
177 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue *SOffset,
178 SDValue *Offset, bool Imm32Only = false,
179 bool IsBuffer = false, bool HasSOffset = false,
180 int64_t ImmOffset = 0) const;
181 SDValue Expand32BitAddress(SDValue Addr) const;
182 bool SelectSMRDBaseOffset(SDValue Addr, SDValue &SBase, SDValue *SOffset,
183 SDValue *Offset, bool Imm32Only = false,
184 bool IsBuffer = false, bool HasSOffset = false,
185 int64_t ImmOffset = 0) const;
186 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue *SOffset,
187 SDValue *Offset, bool Imm32Only = false) const;
188 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
189 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
190 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &SOffset) const;
191 bool SelectSMRDSgprImm(SDValue Addr, SDValue &SBase, SDValue &SOffset,
192 SDValue &Offset) const;
193 bool SelectSMRDBufferImm(SDValue N, SDValue &Offset) const;
194 bool SelectSMRDBufferImm32(SDValue N, SDValue &Offset) const;
195 bool SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,
196 SDValue &Offset) const;
197 bool SelectSMRDPrefetchImm(SDValue Addr, SDValue &SBase,
198 SDValue &Offset) const;
199 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
200
201 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,
202 bool IsCanonicalizing = true,
203 bool AllowAbs = true) const;
204 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
205 bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src,
206 SDValue &SrcMods) const;
207 bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
208 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
209 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
210 SDValue &Clamp, SDValue &Omod) const;
211 bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
212 SDValue &Clamp, SDValue &Omod) const;
213 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
214 SDValue &Clamp, SDValue &Omod) const;
215
216 bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,
217 bool OpSel) const;
218 bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
219 bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods) const;
220
221 bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,
222 SDValue &Omod) const;
223
224 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,
225 bool IsDOT = false) const;
226 bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods) const;
227
228 bool SelectVOP3PModsNeg(SDValue In, SDValue &Src) const;
229 bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src) const;
230
231 bool SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
232 SDValue &SrcMods) const;
233 bool SelectWMMAModsF16Neg(SDValue In, SDValue &Src, SDValue &SrcMods) const;
234 bool SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
235 SDValue &SrcMods) const;
236 bool SelectWMMAVISrc(SDValue In, SDValue &Src) const;
237
238 bool SelectSWMMACIndex8(SDValue In, SDValue &Src, SDValue &IndexKey) const;
239 bool SelectSWMMACIndex16(SDValue In, SDValue &Src, SDValue &IndexKey) const;
240
241 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
242
243 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
244 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
245 unsigned &Mods) const;
246 bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
247 SDValue &SrcMods) const;
248 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
249
250 SDValue getHi16Elt(SDValue In) const;
251
252 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
253
254 void SelectADD_SUB_I64(SDNode *N);
255 void SelectAddcSubb(SDNode *N);
256 void SelectUADDO_USUBO(SDNode *N);
257 void SelectDIV_SCALE(SDNode *N);
258 void SelectMAD_64_32(SDNode *N);
259 void SelectMUL_LOHI(SDNode *N);
260 void SelectFMA_W_CHAIN(SDNode *N);
261 void SelectFMUL_W_CHAIN(SDNode *N);
262 SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset,
263 uint32_t Width);
264 void SelectS_BFEFromShifts(SDNode *N);
265 void SelectS_BFE(SDNode *N);
266 bool isCBranchSCC(const SDNode *N) const;
267 void SelectBRCOND(SDNode *N);
268 void SelectFMAD_FMA(SDNode *N);
269 void SelectFP_EXTEND(SDNode *N);
270 void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
271 void SelectDSBvhStackIntrinsic(SDNode *N);
272 void SelectDS_GWS(SDNode *N, unsigned IntrID);
273 void SelectInterpP1F16(SDNode *N);
274 void SelectINTRINSIC_W_CHAIN(SDNode *N);
275 void SelectINTRINSIC_WO_CHAIN(SDNode *N);
276 void SelectINTRINSIC_VOID(SDNode *N);
277 void SelectWAVE_ADDRESS(SDNode *N);
278 void SelectSTACKRESTORE(SDNode *N);
279
280protected:
281 // Include the pieces autogenerated from the target description.
282#include "AMDGPUGenDAGISel.inc"
283};
284
285class AMDGPUISelDAGToDAGPass : public SelectionDAGISelPass {
286public:
287 AMDGPUISelDAGToDAGPass(TargetMachine &TM);
288
289 PreservedAnalyses run(MachineFunction &MF,
290 MachineFunctionAnalysisManager &MFAM);
291};
292
293class AMDGPUDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
294public:
295 static char ID;
296
297 AMDGPUDAGToDAGISelLegacy(TargetMachine &TM, CodeGenOptLevel OptLevel);
298
299 bool runOnMachineFunction(MachineFunction &MF) override;
300 void getAnalysisUsage(AnalysisUsage &AU) const override;
301 StringRef getPassName() const override;
302};
303
304#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
305