1//===- AMDGPUInstructionSelector --------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file declares the targeting of the InstructionSelector class for
10/// AMDGPU.
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
14#define LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
15
16#include "SIDefines.h"
17#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
18#include "llvm/IR/InstrTypes.h"
19
20namespace {
21#define GET_GLOBALISEL_PREDICATE_BITSET
22#define AMDGPUSubtarget GCNSubtarget
23#include "AMDGPUGenGlobalISel.inc"
24#undef GET_GLOBALISEL_PREDICATE_BITSET
25#undef AMDGPUSubtarget
26}
27
28namespace llvm {
29
30namespace AMDGPU {
31struct ImageDimIntrinsicInfo;
32}
33
34class AMDGPURegisterBankInfo;
35class AMDGPUTargetMachine;
36class BlockFrequencyInfo;
37class ProfileSummaryInfo;
38class GCNSubtarget;
39class MachineInstr;
40class MachineIRBuilder;
41class MachineOperand;
42class MachineRegisterInfo;
43class RegisterBank;
44class SIInstrInfo;
45class SIRegisterInfo;
46class TargetRegisterClass;
47
48class AMDGPUInstructionSelector final : public InstructionSelector {
49private:
50 MachineRegisterInfo *MRI;
51 const GCNSubtarget *Subtarget;
52
53public:
54 AMDGPUInstructionSelector(const GCNSubtarget &STI,
55 const AMDGPURegisterBankInfo &RBI,
56 const AMDGPUTargetMachine &TM);
57
58 bool select(MachineInstr &I) override;
59 static const char *getName();
60
61 void setupMF(MachineFunction &MF, GISelKnownBits *KB,
62 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
63 BlockFrequencyInfo *BFI) override;
64
65private:
66 struct GEPInfo {
67 SmallVector<unsigned, 2> SgprParts;
68 SmallVector<unsigned, 2> VgprParts;
69 int64_t Imm = 0;
70 };
71
72 bool isSGPR(Register Reg) const;
73
74 bool isInstrUniform(const MachineInstr &MI) const;
75 bool isVCC(Register Reg, const MachineRegisterInfo &MRI) const;
76
77 const RegisterBank *getArtifactRegBank(
78 Register Reg, const MachineRegisterInfo &MRI,
79 const TargetRegisterInfo &TRI) const;
80
81 /// tblgen-erated 'select' implementation.
82 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
83
84 MachineOperand getSubOperand64(MachineOperand &MO,
85 const TargetRegisterClass &SubRC,
86 unsigned SubIdx) const;
87
88 bool constrainCopyLikeIntrin(MachineInstr &MI, unsigned NewOpc) const;
89 bool selectCOPY(MachineInstr &I) const;
90 bool selectPHI(MachineInstr &I) const;
91 bool selectG_TRUNC(MachineInstr &I) const;
92 bool selectG_SZA_EXT(MachineInstr &I) const;
93 bool selectG_FPEXT(MachineInstr &I) const;
94 bool selectG_CONSTANT(MachineInstr &I) const;
95 bool selectG_FNEG(MachineInstr &I) const;
96 bool selectG_FABS(MachineInstr &I) const;
97 bool selectG_AND_OR_XOR(MachineInstr &I) const;
98 bool selectG_ADD_SUB(MachineInstr &I) const;
99 bool selectG_UADDO_USUBO_UADDE_USUBE(MachineInstr &I) const;
100 bool selectG_AMDGPU_MAD_64_32(MachineInstr &I) const;
101 bool selectG_EXTRACT(MachineInstr &I) const;
102 bool selectG_FMA_FMAD(MachineInstr &I) const;
103 bool selectG_MERGE_VALUES(MachineInstr &I) const;
104 bool selectG_UNMERGE_VALUES(MachineInstr &I) const;
105 bool selectG_BUILD_VECTOR(MachineInstr &I) const;
106 bool selectG_IMPLICIT_DEF(MachineInstr &I) const;
107 bool selectG_INSERT(MachineInstr &I) const;
108 bool selectG_SBFX_UBFX(MachineInstr &I) const;
109
110 bool selectInterpP1F16(MachineInstr &MI) const;
111 bool selectWritelane(MachineInstr &MI) const;
112 bool selectDivScale(MachineInstr &MI) const;
113 bool selectIntrinsicCmp(MachineInstr &MI) const;
114 bool selectBallot(MachineInstr &I) const;
115 bool selectRelocConstant(MachineInstr &I) const;
116 bool selectGroupStaticSize(MachineInstr &I) const;
117 bool selectReturnAddress(MachineInstr &I) const;
118 bool selectG_INTRINSIC(MachineInstr &I) const;
119
120 bool selectEndCfIntrinsic(MachineInstr &MI) const;
121 bool selectDSOrderedIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;
122 bool selectDSGWSIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;
123 bool selectDSAppendConsume(MachineInstr &MI, bool IsAppend) const;
124 bool selectSBarrier(MachineInstr &MI) const;
125 bool selectDSBvhStackIntrinsic(MachineInstr &MI) const;
126
127 bool selectImageIntrinsic(MachineInstr &MI,
128 const AMDGPU::ImageDimIntrinsicInfo *Intr) const;
129 bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I) const;
130 int getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) const;
131 bool selectG_ICMP_or_FCMP(MachineInstr &I) const;
132 bool hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const;
133 void getAddrModeInfo(const MachineInstr &Load, const MachineRegisterInfo &MRI,
134 SmallVectorImpl<GEPInfo> &AddrInfo) const;
135
136 void initM0(MachineInstr &I) const;
137 bool selectG_LOAD_STORE_ATOMICRMW(MachineInstr &I) const;
138 bool selectG_SELECT(MachineInstr &I) const;
139 bool selectG_BRCOND(MachineInstr &I) const;
140 bool selectG_GLOBAL_VALUE(MachineInstr &I) const;
141 bool selectG_PTRMASK(MachineInstr &I) const;
142 bool selectG_EXTRACT_VECTOR_ELT(MachineInstr &I) const;
143 bool selectG_INSERT_VECTOR_ELT(MachineInstr &I) const;
144 bool selectBufferLoadLds(MachineInstr &MI) const;
145 bool selectGlobalLoadLds(MachineInstr &MI) const;
146 bool selectBVHIntrinsic(MachineInstr &I) const;
147 bool selectSMFMACIntrin(MachineInstr &I) const;
148 bool selectWaveAddress(MachineInstr &I) const;
149 bool selectStackRestore(MachineInstr &MI) const;
150 bool selectNamedBarrierInst(MachineInstr &I, Intrinsic::ID IID) const;
151 bool selectSBarrierSignalIsfirst(MachineInstr &I, Intrinsic::ID IID) const;
152 bool selectSBarrierLeave(MachineInstr &I) const;
153
154 std::pair<Register, unsigned> selectVOP3ModsImpl(MachineOperand &Root,
155 bool IsCanonicalizing = true,
156 bool AllowAbs = true,
157 bool OpSel = false) const;
158
159 Register copyToVGPRIfSrcFolded(Register Src, unsigned Mods,
160 MachineOperand Root, MachineInstr *InsertPt,
161 bool ForceVGPR = false) const;
162
163 InstructionSelector::ComplexRendererFns
164 selectVCSRC(MachineOperand &Root) const;
165
166 InstructionSelector::ComplexRendererFns
167 selectVSRC0(MachineOperand &Root) const;
168
169 InstructionSelector::ComplexRendererFns
170 selectVOP3Mods0(MachineOperand &Root) const;
171 InstructionSelector::ComplexRendererFns
172 selectVOP3BMods0(MachineOperand &Root) const;
173 InstructionSelector::ComplexRendererFns
174 selectVOP3OMods(MachineOperand &Root) const;
175 InstructionSelector::ComplexRendererFns
176 selectVOP3Mods(MachineOperand &Root) const;
177 InstructionSelector::ComplexRendererFns
178 selectVOP3ModsNonCanonicalizing(MachineOperand &Root) const;
179 InstructionSelector::ComplexRendererFns
180 selectVOP3BMods(MachineOperand &Root) const;
181
182 ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
183
184 std::pair<Register, unsigned>
185 selectVOP3PModsImpl(Register Src, const MachineRegisterInfo &MRI,
186 bool IsDOT = false) const;
187
188 InstructionSelector::ComplexRendererFns
189 selectVOP3PMods(MachineOperand &Root) const;
190
191 InstructionSelector::ComplexRendererFns
192 selectVOP3PModsDOT(MachineOperand &Root) const;
193
194 InstructionSelector::ComplexRendererFns
195 selectVOP3PModsNeg(MachineOperand &Root) const;
196
197 InstructionSelector::ComplexRendererFns
198 selectWMMAOpSelVOP3PMods(MachineOperand &Root) const;
199
200 InstructionSelector::ComplexRendererFns
201 selectWMMAModsF32NegAbs(MachineOperand &Root) const;
202 InstructionSelector::ComplexRendererFns
203 selectWMMAModsF16Neg(MachineOperand &Root) const;
204 InstructionSelector::ComplexRendererFns
205 selectWMMAModsF16NegAbs(MachineOperand &Root) const;
206 InstructionSelector::ComplexRendererFns
207 selectWMMAVISrc(MachineOperand &Root) const;
208 InstructionSelector::ComplexRendererFns
209 selectSWMMACIndex8(MachineOperand &Root) const;
210 InstructionSelector::ComplexRendererFns
211 selectSWMMACIndex16(MachineOperand &Root) const;
212
213 InstructionSelector::ComplexRendererFns
214 selectVOP3OpSelMods(MachineOperand &Root) const;
215
216 InstructionSelector::ComplexRendererFns
217 selectVINTERPMods(MachineOperand &Root) const;
218 InstructionSelector::ComplexRendererFns
219 selectVINTERPModsHi(MachineOperand &Root) const;
220
221 bool selectSmrdOffset(MachineOperand &Root, Register &Base, Register *SOffset,
222 int64_t *Offset) const;
223 InstructionSelector::ComplexRendererFns
224 selectSmrdImm(MachineOperand &Root) const;
225 InstructionSelector::ComplexRendererFns
226 selectSmrdImm32(MachineOperand &Root) const;
227 InstructionSelector::ComplexRendererFns
228 selectSmrdSgpr(MachineOperand &Root) const;
229 InstructionSelector::ComplexRendererFns
230 selectSmrdSgprImm(MachineOperand &Root) const;
231
232 std::pair<Register, int> selectFlatOffsetImpl(MachineOperand &Root,
233 uint64_t FlatVariant) const;
234
235 InstructionSelector::ComplexRendererFns
236 selectFlatOffset(MachineOperand &Root) const;
237 InstructionSelector::ComplexRendererFns
238 selectGlobalOffset(MachineOperand &Root) const;
239 InstructionSelector::ComplexRendererFns
240 selectScratchOffset(MachineOperand &Root) const;
241
242 InstructionSelector::ComplexRendererFns
243 selectGlobalSAddr(MachineOperand &Root) const;
244
245 InstructionSelector::ComplexRendererFns
246 selectScratchSAddr(MachineOperand &Root) const;
247 bool checkFlatScratchSVSSwizzleBug(Register VAddr, Register SAddr,
248 uint64_t ImmOffset) const;
249 InstructionSelector::ComplexRendererFns
250 selectScratchSVAddr(MachineOperand &Root) const;
251
252 InstructionSelector::ComplexRendererFns
253 selectMUBUFScratchOffen(MachineOperand &Root) const;
254 InstructionSelector::ComplexRendererFns
255 selectMUBUFScratchOffset(MachineOperand &Root) const;
256
257 bool isDSOffsetLegal(Register Base, int64_t Offset) const;
258 bool isDSOffset2Legal(Register Base, int64_t Offset0, int64_t Offset1,
259 unsigned Size) const;
260 bool isFlatScratchBaseLegal(Register Addr) const;
261 bool isFlatScratchBaseLegalSV(Register Addr) const;
262 bool isFlatScratchBaseLegalSVImm(Register Addr) const;
263
264 std::pair<Register, unsigned>
265 selectDS1Addr1OffsetImpl(MachineOperand &Root) const;
266 InstructionSelector::ComplexRendererFns
267 selectDS1Addr1Offset(MachineOperand &Root) const;
268
269 InstructionSelector::ComplexRendererFns
270 selectDS64Bit4ByteAligned(MachineOperand &Root) const;
271
272 InstructionSelector::ComplexRendererFns
273 selectDS128Bit8ByteAligned(MachineOperand &Root) const;
274
275 std::pair<Register, unsigned> selectDSReadWrite2Impl(MachineOperand &Root,
276 unsigned size) const;
277 InstructionSelector::ComplexRendererFns
278 selectDSReadWrite2(MachineOperand &Root, unsigned size) const;
279
280 std::pair<Register, int64_t>
281 getPtrBaseWithConstantOffset(Register Root,
282 const MachineRegisterInfo &MRI) const;
283
284 // Parse out a chain of up to two g_ptr_add instructions.
285 // g_ptr_add (n0, _)
286 // g_ptr_add (n0, (n1 = g_ptr_add n2, n3))
287 struct MUBUFAddressData {
288 Register N0, N2, N3;
289 int64_t Offset = 0;
290 };
291
292 bool shouldUseAddr64(MUBUFAddressData AddrData) const;
293
294 void splitIllegalMUBUFOffset(MachineIRBuilder &B,
295 Register &SOffset, int64_t &ImmOffset) const;
296
297 MUBUFAddressData parseMUBUFAddress(Register Src) const;
298
299 bool selectMUBUFAddr64Impl(MachineOperand &Root, Register &VAddr,
300 Register &RSrcReg, Register &SOffset,
301 int64_t &Offset) const;
302
303 bool selectMUBUFOffsetImpl(MachineOperand &Root, Register &RSrcReg,
304 Register &SOffset, int64_t &Offset) const;
305
306 InstructionSelector::ComplexRendererFns
307 selectBUFSOffset(MachineOperand &Root) const;
308
309 InstructionSelector::ComplexRendererFns
310 selectMUBUFAddr64(MachineOperand &Root) const;
311
312 InstructionSelector::ComplexRendererFns
313 selectMUBUFOffset(MachineOperand &Root) const;
314
315 ComplexRendererFns selectSMRDBufferImm(MachineOperand &Root) const;
316 ComplexRendererFns selectSMRDBufferImm32(MachineOperand &Root) const;
317 ComplexRendererFns selectSMRDBufferSgprImm(MachineOperand &Root) const;
318
319 std::pair<Register, unsigned> selectVOP3PMadMixModsImpl(MachineOperand &Root,
320 bool &Matched) const;
321 ComplexRendererFns selectVOP3PMadMixModsExt(MachineOperand &Root) const;
322 ComplexRendererFns selectVOP3PMadMixMods(MachineOperand &Root) const;
323
324 void renderTruncImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
325 int OpIdx = -1) const;
326
327 void renderTruncTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
328 int OpIdx) const;
329
330 void renderOpSelTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
331 int OpIdx) const;
332
333 void renderNegateImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
334 int OpIdx) const;
335
336 void renderBitcastImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
337 int OpIdx) const;
338
339 void renderPopcntImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
340 int OpIdx) const;
341 void renderExtractCPol(MachineInstrBuilder &MIB, const MachineInstr &MI,
342 int OpIdx) const;
343 void renderExtractSWZ(MachineInstrBuilder &MIB, const MachineInstr &MI,
344 int OpIdx) const;
345 void renderExtractCpolSetGLC(MachineInstrBuilder &MIB, const MachineInstr &MI,
346 int OpIdx) const;
347
348 void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,
349 int OpIdx) const;
350
351 void renderFPPow2ToExponent(MachineInstrBuilder &MIB, const MachineInstr &MI,
352 int OpIdx) const;
353
354 bool isInlineImmediate(const APInt &Imm) const;
355 bool isInlineImmediate(const APFloat &Imm) const;
356
357 // Returns true if TargetOpcode::G_AND MachineInstr `MI`'s masking of the
358 // shift amount operand's `ShAmtBits` bits is unneeded.
359 bool isUnneededShiftMask(const MachineInstr &MI, unsigned ShAmtBits) const;
360
361 const SIInstrInfo &TII;
362 const SIRegisterInfo &TRI;
363 const AMDGPURegisterBankInfo &RBI;
364 const AMDGPUTargetMachine &TM;
365 const GCNSubtarget &STI;
366 bool EnableLateStructurizeCFG;
367#define GET_GLOBALISEL_PREDICATES_DECL
368#define AMDGPUSubtarget GCNSubtarget
369#include "AMDGPUGenGlobalISel.inc"
370#undef GET_GLOBALISEL_PREDICATES_DECL
371#undef AMDGPUSubtarget
372
373#define GET_GLOBALISEL_TEMPORARIES_DECL
374#include "AMDGPUGenGlobalISel.inc"
375#undef GET_GLOBALISEL_TEMPORARIES_DECL
376};
377
378} // End llvm namespace.
379#endif
380