1//===- AMDGPUInstructionSelector --------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file declares the targeting of the InstructionSelector class for
10/// AMDGPU.
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
14#define LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
15
16#include "SIDefines.h"
17#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
18#include "llvm/IR/InstrTypes.h"
19
20namespace {
21#define GET_GLOBALISEL_PREDICATE_BITSET
22#define AMDGPUSubtarget GCNSubtarget
23#include "AMDGPUGenGlobalISel.inc"
24#undef GET_GLOBALISEL_PREDICATE_BITSET
25#undef AMDGPUSubtarget
26}
27
28namespace llvm {
29
30namespace AMDGPU {
31struct ImageDimIntrinsicInfo;
32}
33
34class AMDGPURegisterBankInfo;
35class AMDGPUTargetMachine;
36class BlockFrequencyInfo;
37class ProfileSummaryInfo;
38class GCNSubtarget;
39class MachineInstr;
40class MachineIRBuilder;
41class MachineOperand;
42class MachineRegisterInfo;
43class RegisterBank;
44class SIInstrInfo;
45class SIRegisterInfo;
46class TargetRegisterClass;
47
48class AMDGPUInstructionSelector final : public InstructionSelector {
49private:
50 MachineRegisterInfo *MRI;
51 const GCNSubtarget *Subtarget;
52
53public:
54 AMDGPUInstructionSelector(const GCNSubtarget &STI,
55 const AMDGPURegisterBankInfo &RBI,
56 const AMDGPUTargetMachine &TM);
57
58 bool select(MachineInstr &I) override;
59 static const char *getName();
60
61 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
62 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
63 BlockFrequencyInfo *BFI) override;
64
65private:
66 struct GEPInfo {
67 SmallVector<unsigned, 2> SgprParts;
68 SmallVector<unsigned, 2> VgprParts;
69 int64_t Imm = 0;
70 };
71
72 bool isSGPR(Register Reg) const;
73
74 bool isInstrUniform(const MachineInstr &MI) const;
75 bool isVCC(Register Reg, const MachineRegisterInfo &MRI) const;
76
77 const RegisterBank *getArtifactRegBank(
78 Register Reg, const MachineRegisterInfo &MRI,
79 const TargetRegisterInfo &TRI) const;
80
81 /// tblgen-erated 'select' implementation.
82 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
83
84 MachineOperand getSubOperand64(MachineOperand &MO,
85 const TargetRegisterClass &SubRC,
86 unsigned SubIdx) const;
87
88 bool constrainCopyLikeIntrin(MachineInstr &MI, unsigned NewOpc) const;
89 bool selectCOPY(MachineInstr &I) const;
90 bool selectCOPY_SCC_VCC(MachineInstr &I) const;
91 bool selectCOPY_VCC_SCC(MachineInstr &I) const;
92 bool selectReadAnyLane(MachineInstr &I) const;
93 bool selectPHI(MachineInstr &I) const;
94 bool selectG_TRUNC(MachineInstr &I) const;
95 bool selectG_SZA_EXT(MachineInstr &I) const;
96 bool selectG_FPEXT(MachineInstr &I) const;
97 bool selectG_FNEG(MachineInstr &I) const;
98 bool selectG_FABS(MachineInstr &I) const;
99 bool selectG_AND_OR_XOR(MachineInstr &I) const;
100 bool selectG_ADD_SUB(MachineInstr &I) const;
101 bool selectG_UADDO_USUBO_UADDE_USUBE(MachineInstr &I) const;
102 bool selectG_AMDGPU_MAD_64_32(MachineInstr &I) const;
103 bool selectG_EXTRACT(MachineInstr &I) const;
104 bool selectG_FMA_FMAD(MachineInstr &I) const;
105 bool selectG_MERGE_VALUES(MachineInstr &I) const;
106 bool selectG_UNMERGE_VALUES(MachineInstr &I) const;
107 bool selectG_BUILD_VECTOR(MachineInstr &I) const;
108 bool selectG_IMPLICIT_DEF(MachineInstr &I) const;
109 bool selectG_INSERT(MachineInstr &I) const;
110 bool selectG_SBFX_UBFX(MachineInstr &I) const;
111
112 bool selectInterpP1F16(MachineInstr &MI) const;
113 bool selectWritelane(MachineInstr &MI) const;
114 bool selectDivScale(MachineInstr &MI) const;
115 bool selectIntrinsicCmp(MachineInstr &MI) const;
116 bool selectBallot(MachineInstr &I) const;
117 bool selectRelocConstant(MachineInstr &I) const;
118 bool selectGroupStaticSize(MachineInstr &I) const;
119 bool selectReturnAddress(MachineInstr &I) const;
120 bool selectG_INTRINSIC(MachineInstr &I) const;
121
122 bool selectEndCfIntrinsic(MachineInstr &MI) const;
123 bool selectDSOrderedIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;
124 bool selectDSGWSIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;
125 bool selectDSAppendConsume(MachineInstr &MI, bool IsAppend) const;
126 bool selectInitWholeWave(MachineInstr &MI) const;
127 bool selectDSBvhStackIntrinsic(MachineInstr &MI) const;
128
129 bool selectImageIntrinsic(MachineInstr &MI,
130 const AMDGPU::ImageDimIntrinsicInfo *Intr) const;
131 bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I) const;
132 int getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) const;
133 bool selectG_ICMP_or_FCMP(MachineInstr &I) const;
134 bool hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const;
135 void getAddrModeInfo(const MachineInstr &Load, const MachineRegisterInfo &MRI,
136 SmallVectorImpl<GEPInfo> &AddrInfo) const;
137
138 void initM0(MachineInstr &I) const;
139 bool selectG_LOAD_STORE_ATOMICRMW(MachineInstr &I) const;
140 bool selectG_SELECT(MachineInstr &I) const;
141 bool selectG_BRCOND(MachineInstr &I) const;
142 bool selectG_GLOBAL_VALUE(MachineInstr &I) const;
143 bool selectG_PTRMASK(MachineInstr &I) const;
144 bool selectG_EXTRACT_VECTOR_ELT(MachineInstr &I) const;
145 bool selectG_INSERT_VECTOR_ELT(MachineInstr &I) const;
146 bool selectBufferLoadLds(MachineInstr &MI) const;
147 bool selectGlobalLoadLds(MachineInstr &MI) const;
148 bool selectTensorLoadStore(MachineInstr &MI, Intrinsic::ID IID) const;
149 bool selectBVHIntersectRayIntrinsic(MachineInstr &I) const;
150 bool selectSMFMACIntrin(MachineInstr &I) const;
151 bool selectPermlaneSwapIntrin(MachineInstr &I, Intrinsic::ID IntrID) const;
152 bool selectWaveAddress(MachineInstr &I) const;
153 bool selectBITOP3(MachineInstr &I) const;
154 bool selectStackRestore(MachineInstr &MI) const;
155 bool selectNamedBarrierInit(MachineInstr &I, Intrinsic::ID IID) const;
156 bool selectNamedBarrierInst(MachineInstr &I, Intrinsic::ID IID) const;
157 bool selectSBarrierSignalIsfirst(MachineInstr &I, Intrinsic::ID IID) const;
158 bool selectSGetBarrierState(MachineInstr &I, Intrinsic::ID IID) const;
159 bool selectSBarrierLeave(MachineInstr &I) const;
160 bool selectWaveShuffleIntrin(MachineInstr &I) const;
161
162 std::pair<Register, unsigned> selectVOP3ModsImpl(Register Src,
163 bool IsCanonicalizing = true,
164 bool AllowAbs = true,
165 bool OpSel = false) const;
166
167 Register copyToVGPRIfSrcFolded(Register Src, unsigned Mods,
168 MachineOperand Root, MachineInstr *InsertPt,
169 bool ForceVGPR = false) const;
170
171 InstructionSelector::ComplexRendererFns
172 selectVCSRC(MachineOperand &Root) const;
173
174 InstructionSelector::ComplexRendererFns
175 selectVSRC0(MachineOperand &Root) const;
176
177 InstructionSelector::ComplexRendererFns
178 selectVOP3Mods0(MachineOperand &Root) const;
179 InstructionSelector::ComplexRendererFns
180 selectVOP3BMods0(MachineOperand &Root) const;
181 InstructionSelector::ComplexRendererFns
182 selectVOP3OMods(MachineOperand &Root) const;
183 InstructionSelector::ComplexRendererFns
184 selectVOP3Mods(MachineOperand &Root) const;
185 InstructionSelector::ComplexRendererFns
186 selectVOP3ModsNonCanonicalizing(MachineOperand &Root) const;
187 InstructionSelector::ComplexRendererFns
188 selectVOP3BMods(MachineOperand &Root) const;
189
190 ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
191
192 std::pair<Register, unsigned>
193 selectVOP3PModsImpl(Register RootReg, const MachineRegisterInfo &MRI,
194 bool IsDOT = false) const;
195 InstructionSelector::ComplexRendererFns
196 selectVOP3PRetHelper(MachineOperand &Root, bool IsDOT = false) const;
197
198 InstructionSelector::ComplexRendererFns
199 selectVOP3PMods(MachineOperand &Root) const;
200
201 InstructionSelector::ComplexRendererFns
202 selectVOP3PModsDOT(MachineOperand &Root) const;
203
204 InstructionSelector::ComplexRendererFns
205 selectWMMAOpSelVOP3PMods(MachineOperand &Root) const;
206
207 InstructionSelector::ComplexRendererFns
208 selectWMMAModsF32NegAbs(MachineOperand &Root) const;
209 InstructionSelector::ComplexRendererFns
210 selectWMMAModsF16Neg(MachineOperand &Root) const;
211 InstructionSelector::ComplexRendererFns
212 selectWMMAModsF16NegAbs(MachineOperand &Root) const;
213 InstructionSelector::ComplexRendererFns
214 selectWMMAVISrc(MachineOperand &Root) const;
215 InstructionSelector::ComplexRendererFns
216 selectSWMMACIndex8(MachineOperand &Root) const;
217 InstructionSelector::ComplexRendererFns
218 selectSWMMACIndex16(MachineOperand &Root) const;
219 InstructionSelector::ComplexRendererFns
220 selectSWMMACIndex32(MachineOperand &Root) const;
221
222 InstructionSelector::ComplexRendererFns
223 selectVOP3OpSelMods(MachineOperand &Root) const;
224
225 InstructionSelector::ComplexRendererFns
226 selectVINTERPMods(MachineOperand &Root) const;
227 InstructionSelector::ComplexRendererFns
228 selectVINTERPModsHi(MachineOperand &Root) const;
229
230 bool selectScaleOffset(MachineOperand &Root, Register &Offset,
231 bool IsSigned) const;
232 bool selectSmrdOffset(MachineOperand &Root, Register &Base, Register *SOffset,
233 int64_t *Offset, bool *ScaleOffset) const;
234 InstructionSelector::ComplexRendererFns
235 selectSmrdImm(MachineOperand &Root) const;
236 InstructionSelector::ComplexRendererFns
237 selectSmrdImm32(MachineOperand &Root) const;
238 InstructionSelector::ComplexRendererFns
239 selectSmrdSgpr(MachineOperand &Root) const;
240 InstructionSelector::ComplexRendererFns
241 selectSmrdSgprImm(MachineOperand &Root) const;
242
243 std::pair<Register, int> selectFlatOffsetImpl(MachineOperand &Root,
244 uint64_t FlatVariant) const;
245
246 InstructionSelector::ComplexRendererFns
247 selectFlatOffset(MachineOperand &Root) const;
248 InstructionSelector::ComplexRendererFns
249 selectGlobalOffset(MachineOperand &Root) const;
250 InstructionSelector::ComplexRendererFns
251 selectScratchOffset(MachineOperand &Root) const;
252
253 InstructionSelector::ComplexRendererFns
254 selectGlobalSAddr(MachineOperand &Root, unsigned CPolBits,
255 bool NeedIOffset = true) const;
256 InstructionSelector::ComplexRendererFns
257 selectGlobalSAddr(MachineOperand &Root) const;
258 InstructionSelector::ComplexRendererFns
259 selectGlobalSAddrCPol(MachineOperand &Root) const;
260 InstructionSelector::ComplexRendererFns
261 selectGlobalSAddrCPolM0(MachineOperand &Root) const;
262 InstructionSelector::ComplexRendererFns
263 selectGlobalSAddrGLC(MachineOperand &Root) const;
264 InstructionSelector::ComplexRendererFns
265 selectGlobalSAddrNoIOffset(MachineOperand &Root) const;
266 InstructionSelector::ComplexRendererFns
267 selectGlobalSAddrNoIOffsetM0(MachineOperand &Root) const;
268
269 InstructionSelector::ComplexRendererFns
270 selectScratchSAddr(MachineOperand &Root) const;
271 bool checkFlatScratchSVSSwizzleBug(Register VAddr, Register SAddr,
272 uint64_t ImmOffset) const;
273 InstructionSelector::ComplexRendererFns
274 selectScratchSVAddr(MachineOperand &Root) const;
275
276 InstructionSelector::ComplexRendererFns
277 selectMUBUFScratchOffen(MachineOperand &Root) const;
278 InstructionSelector::ComplexRendererFns
279 selectMUBUFScratchOffset(MachineOperand &Root) const;
280
281 bool isDSOffsetLegal(Register Base, int64_t Offset) const;
282 bool isDSOffset2Legal(Register Base, int64_t Offset0, int64_t Offset1,
283 unsigned Size) const;
284 bool isFlatScratchBaseLegal(Register Addr) const;
285 bool isFlatScratchBaseLegalSV(Register Addr) const;
286 bool isFlatScratchBaseLegalSVImm(Register Addr) const;
287
288 std::pair<Register, unsigned>
289 selectDS1Addr1OffsetImpl(MachineOperand &Root) const;
290 InstructionSelector::ComplexRendererFns
291 selectDS1Addr1Offset(MachineOperand &Root) const;
292
293 InstructionSelector::ComplexRendererFns
294 selectDS64Bit4ByteAligned(MachineOperand &Root) const;
295
296 InstructionSelector::ComplexRendererFns
297 selectDS128Bit8ByteAligned(MachineOperand &Root) const;
298
299 std::pair<Register, unsigned> selectDSReadWrite2Impl(MachineOperand &Root,
300 unsigned size) const;
301 InstructionSelector::ComplexRendererFns
302 selectDSReadWrite2(MachineOperand &Root, unsigned size) const;
303
304 std::tuple<Register, int64_t, bool>
305 getPtrBaseWithConstantOffset(Register Root,
306 const MachineRegisterInfo &MRI) const;
307
308 // Parse out a chain of up to two g_ptr_add instructions.
309 // g_ptr_add (n0, _)
310 // g_ptr_add (n0, (n1 = g_ptr_add n2, n3))
311 struct MUBUFAddressData {
312 Register N0, N2, N3;
313 int64_t Offset = 0;
314 };
315
316 bool shouldUseAddr64(MUBUFAddressData AddrData) const;
317
318 void splitIllegalMUBUFOffset(MachineIRBuilder &B,
319 Register &SOffset, int64_t &ImmOffset) const;
320
321 MUBUFAddressData parseMUBUFAddress(Register Src) const;
322
323 bool selectMUBUFAddr64Impl(MachineOperand &Root, Register &VAddr,
324 Register &RSrcReg, Register &SOffset,
325 int64_t &Offset) const;
326
327 bool selectMUBUFOffsetImpl(MachineOperand &Root, Register &RSrcReg,
328 Register &SOffset, int64_t &Offset) const;
329
330 InstructionSelector::ComplexRendererFns
331 selectBUFSOffset(MachineOperand &Root) const;
332
333 InstructionSelector::ComplexRendererFns
334 selectMUBUFAddr64(MachineOperand &Root) const;
335
336 InstructionSelector::ComplexRendererFns
337 selectMUBUFOffset(MachineOperand &Root) const;
338
339 ComplexRendererFns selectSMRDBufferImm(MachineOperand &Root) const;
340 ComplexRendererFns selectSMRDBufferImm32(MachineOperand &Root) const;
341 ComplexRendererFns selectSMRDBufferSgprImm(MachineOperand &Root) const;
342
343 std::pair<Register, unsigned> selectVOP3PMadMixModsImpl(MachineOperand &Root,
344 bool &Matched) const;
345 ComplexRendererFns selectVOP3PMadMixModsExt(MachineOperand &Root) const;
346 ComplexRendererFns selectVOP3PMadMixMods(MachineOperand &Root) const;
347
348 void renderTruncImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
349 int OpIdx = -1) const;
350
351 void renderTruncTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
352 int OpIdx) const;
353 void renderZextBoolTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
354 int OpIdx) const;
355
356 void renderOpSelTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
357 int OpIdx) const;
358
359 void renderSrcAndDstSelToOpSelXForm_0_0(MachineInstrBuilder &MIB,
360 const MachineInstr &MI,
361 int OpIdx) const;
362
363 void renderSrcAndDstSelToOpSelXForm_0_1(MachineInstrBuilder &MIB,
364 const MachineInstr &MI,
365 int OpIdx) const;
366
367 void renderSrcAndDstSelToOpSelXForm_1_0(MachineInstrBuilder &MIB,
368 const MachineInstr &MI,
369 int OpIdx) const;
370
371 void renderSrcAndDstSelToOpSelXForm_1_1(MachineInstrBuilder &MIB,
372 const MachineInstr &MI,
373 int OpIdx) const;
374
375 void renderDstSelToOpSelXForm(MachineInstrBuilder &MIB,
376 const MachineInstr &MI, int OpIdx) const;
377
378 void renderSrcSelToOpSelXForm(MachineInstrBuilder &MIB,
379 const MachineInstr &MI, int OpIdx) const;
380
381 void renderSrcAndDstSelToOpSelXForm_2_0(MachineInstrBuilder &MIB,
382 const MachineInstr &MI,
383 int OpIdx) const;
384
385 void renderDstSelToOpSel3XFormXForm(MachineInstrBuilder &MIB,
386 const MachineInstr &MI, int OpIdx) const;
387
388 void renderNegateImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
389 int OpIdx) const;
390
391 void renderBitcastFPImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
392 int OpIdx) const;
393
394 void renderBitcastFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
395 int OpIdx) const {
396 renderBitcastFPImm(MIB, MI, OpIdx);
397 }
398 void renderBitcastFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,
399 int OpIdx) const {
400 renderBitcastFPImm(MIB, MI, OpIdx);
401 }
402
403 void renderPopcntImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
404 int OpIdx) const;
405 void renderExtractCPol(MachineInstrBuilder &MIB, const MachineInstr &MI,
406 int OpIdx) const;
407 void renderExtractSWZ(MachineInstrBuilder &MIB, const MachineInstr &MI,
408 int OpIdx) const;
409 void renderExtractCpolSetGLC(MachineInstrBuilder &MIB, const MachineInstr &MI,
410 int OpIdx) const;
411
412 void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,
413 int OpIdx) const;
414
415 void renderFPPow2ToExponent(MachineInstrBuilder &MIB, const MachineInstr &MI,
416 int OpIdx) const;
417
418 void renderRoundMode(MachineInstrBuilder &MIB, const MachineInstr &MI,
419 int OpIdx) const;
420
421 void renderVOP3PModsNeg(MachineInstrBuilder &MIB, const MachineInstr &MI,
422 int OpIdx) const;
423 void renderVOP3PModsNegs(MachineInstrBuilder &MIB, const MachineInstr &MI,
424 int OpIdx) const;
425 void renderVOP3PModsNegAbs(MachineInstrBuilder &MIB, const MachineInstr &MI,
426 int OpIdx) const;
427
428 void renderPrefetchLoc(MachineInstrBuilder &MIB, const MachineInstr &MI,
429 int OpIdx) const;
430
431 void renderScaledMAIIntrinsicOperand(MachineInstrBuilder &MIB,
432 const MachineInstr &MI, int OpIdx) const;
433
434 bool isInlineImmediate(const APInt &Imm) const;
435 bool isInlineImmediate(const APFloat &Imm) const;
436
437 // Returns true if TargetOpcode::G_AND MachineInstr `MI`'s masking of the
438 // shift amount operand's `ShAmtBits` bits is unneeded.
439 bool isUnneededShiftMask(const MachineInstr &MI, unsigned ShAmtBits) const;
440
441 /// Match a zero extend from a 32-bit value to 64-bits.
442 Register matchZeroExtendFromS32(Register Reg) const;
443 /// Match a sign extend from a 32-bit value to 64-bits.
444 Register matchSignExtendFromS32(Register Reg) const;
445 /// Match a zero extend from a 32-bit value to 64-bits, or \p Reg itself if it
446 /// is 32-bit.
447 Register matchZeroExtendFromS32OrS32(Register Reg) const;
448 /// Match a sign extend from a 32-bit value to 64-bits, or \p Reg itself if it
449 /// is 32-bit.
450 Register matchSignExtendFromS32OrS32(Register Reg) const;
451 /// Match either sign or zero extend depending on the \p IsSigned from a
452 /// 32-bit value to 64-bits, or \p Reg itself if it is 32-bit.
453 Register matchExtendFromS32OrS32(Register Reg, bool IsSigned) const;
454 /// Match an any extend from a 32-bit value to 64-bit.
455 Register matchAnyExtendFromS32(Register Reg) const;
456
457 const SIInstrInfo &TII;
458 const SIRegisterInfo &TRI;
459 const AMDGPURegisterBankInfo &RBI;
460 const AMDGPUTargetMachine &TM;
461 const GCNSubtarget &STI;
462#define GET_GLOBALISEL_PREDICATES_DECL
463#define AMDGPUSubtarget GCNSubtarget
464#include "AMDGPUGenGlobalISel.inc"
465#undef GET_GLOBALISEL_PREDICATES_DECL
466#undef AMDGPUSubtarget
467
468#define GET_GLOBALISEL_TEMPORARIES_DECL
469#include "AMDGPUGenGlobalISel.inc"
470#undef GET_GLOBALISEL_TEMPORARIES_DECL
471};
472
473} // End llvm namespace.
474#endif
475