1//===- AMDGPUInstructionSelector --------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file declares the targeting of the InstructionSelector class for
10/// AMDGPU.
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
14#define LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H
15
16#include "SIDefines.h"
17#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
18#include "llvm/IR/InstrTypes.h"
19
20namespace {
21#define GET_GLOBALISEL_PREDICATE_BITSET
22#define AMDGPUSubtarget GCNSubtarget
23#include "AMDGPUGenGlobalISel.inc"
24#undef GET_GLOBALISEL_PREDICATE_BITSET
25#undef AMDGPUSubtarget
26}
27
28namespace llvm {
29
30namespace AMDGPU {
31struct ImageDimIntrinsicInfo;
32}
33
34class AMDGPURegisterBankInfo;
35class AMDGPUTargetMachine;
36class BlockFrequencyInfo;
37class ProfileSummaryInfo;
38class GCNSubtarget;
39class MachineInstr;
40class MachineIRBuilder;
41class MachineOperand;
42class MachineRegisterInfo;
43class RegisterBank;
44class SIInstrInfo;
45class SIRegisterInfo;
46class TargetRegisterClass;
47
48class AMDGPUInstructionSelector final : public InstructionSelector {
49private:
50 MachineRegisterInfo *MRI;
51 const GCNSubtarget *Subtarget;
52
53public:
54 AMDGPUInstructionSelector(const GCNSubtarget &STI,
55 const AMDGPURegisterBankInfo &RBI);
56
57 bool select(MachineInstr &I) override;
58 static const char *getName();
59
60 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
61 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
62 BlockFrequencyInfo *BFI) override;
63
64private:
65 struct GEPInfo {
66 SmallVector<unsigned, 2> SgprParts;
67 SmallVector<unsigned, 2> VgprParts;
68 int64_t Imm = 0;
69 };
70
71 bool isSGPR(Register Reg) const;
72
73 bool isInstrUniform(const MachineInstr &MI) const;
74 bool isVCC(Register Reg, const MachineRegisterInfo &MRI) const;
75
76 const RegisterBank *getArtifactRegBank(
77 Register Reg, const MachineRegisterInfo &MRI,
78 const TargetRegisterInfo &TRI) const;
79
80 /// tblgen-erated 'select' implementation.
81 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
82
83 MachineOperand getSubOperand64(MachineOperand &MO,
84 const TargetRegisterClass &SubRC,
85 unsigned SubIdx) const;
86
87 bool constrainCopyLikeIntrin(MachineInstr &MI, unsigned NewOpc) const;
88 bool selectCOPY(MachineInstr &I) const;
89 bool selectCOPY_SCC_VCC(MachineInstr &I) const;
90 bool selectCOPY_VCC_SCC(MachineInstr &I) const;
91 bool selectReadAnyLane(MachineInstr &I) const;
92 bool selectPHI(MachineInstr &I) const;
93 bool selectG_TRUNC(MachineInstr &I) const;
94 bool selectG_SZA_EXT(MachineInstr &I) const;
95 bool selectG_FPEXT(MachineInstr &I) const;
96 bool selectG_FNEG(MachineInstr &I) const;
97 bool selectG_FABS(MachineInstr &I) const;
98 bool selectG_AND_OR_XOR(MachineInstr &I) const;
99 bool selectG_ADD_SUB(MachineInstr &I) const;
100 bool selectG_UADDO_USUBO_UADDE_USUBE(MachineInstr &I) const;
101 bool selectG_AMDGPU_MAD_64_32(MachineInstr &I) const;
102 bool selectG_EXTRACT(MachineInstr &I) const;
103 bool selectS16MergeToS32(MachineInstr &MI) const;
104 bool selectG_MERGE_VALUES(MachineInstr &I) const;
105 bool selectG_UNMERGE_VALUES(MachineInstr &I) const;
106 bool selectG_BUILD_VECTOR(MachineInstr &I) const;
107 bool selectG_IMPLICIT_DEF(MachineInstr &I) const;
108 bool selectG_INSERT(MachineInstr &I) const;
109 bool selectG_SBFX_UBFX(MachineInstr &I) const;
110
111 bool selectInterpP1F16(MachineInstr &MI) const;
112 bool selectWritelane(MachineInstr &MI) const;
113 bool selectDivScale(MachineInstr &MI) const;
114 bool selectIntrinsicCmp(MachineInstr &MI) const;
115 bool selectBallot(MachineInstr &I) const;
116 bool selectRelocConstant(MachineInstr &I) const;
117 bool selectGroupStaticSize(MachineInstr &I) const;
118 bool selectReturnAddress(MachineInstr &I) const;
119 bool selectG_INTRINSIC(MachineInstr &I) const;
120
121 bool selectEndCfIntrinsic(MachineInstr &MI) const;
122 bool selectDSOrderedIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;
123 bool selectDSGWSIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;
124 bool selectDSAppendConsume(MachineInstr &MI, bool IsAppend) const;
125 bool selectInitWholeWave(MachineInstr &MI) const;
126 bool selectDSBvhStackIntrinsic(MachineInstr &MI) const;
127
128 bool selectImageIntrinsic(MachineInstr &MI,
129 const AMDGPU::ImageDimIntrinsicInfo *Intr) const;
130 bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I) const;
131 int getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) const;
132 bool selectG_ICMP_or_FCMP(MachineInstr &I) const;
133 bool hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const;
134 void getAddrModeInfo(const MachineInstr &Load, const MachineRegisterInfo &MRI,
135 SmallVectorImpl<GEPInfo> &AddrInfo) const;
136
137 void initM0(MachineInstr &I) const;
138 bool selectG_LOAD_STORE_ATOMICRMW(MachineInstr &I) const;
139 bool selectG_SELECT(MachineInstr &I) const;
140 bool selectG_BRCOND(MachineInstr &I) const;
141 bool selectG_GLOBAL_VALUE(MachineInstr &I) const;
142 bool selectG_PTRMASK(MachineInstr &I) const;
143 bool selectG_EXTRACT_VECTOR_ELT(MachineInstr &I) const;
144 bool selectG_INSERT_VECTOR_ELT(MachineInstr &I) const;
145 bool selectBufferLoadLds(MachineInstr &MI) const;
146 bool selectGlobalLoadLds(MachineInstr &MI) const;
147 bool selectTensorLoadStore(MachineInstr &MI, Intrinsic::ID IID) const;
148 bool selectBVHIntersectRayIntrinsic(MachineInstr &I) const;
149 bool selectSMFMACIntrin(MachineInstr &I) const;
150 bool selectPermlaneSwapIntrin(MachineInstr &I, Intrinsic::ID IntrID) const;
151 bool selectWaveAddress(MachineInstr &I) const;
152 bool selectBITOP3(MachineInstr &I) const;
153 bool selectStackRestore(MachineInstr &MI) const;
154 bool selectNamedBarrierInit(MachineInstr &I, Intrinsic::ID IID) const;
155 bool selectNamedBarrierInst(MachineInstr &I, Intrinsic::ID IID) const;
156 bool selectSBarrierSignalIsfirst(MachineInstr &I, Intrinsic::ID IID) const;
157 bool selectSGetBarrierState(MachineInstr &I, Intrinsic::ID IID) const;
158 bool selectWaveShuffleIntrin(MachineInstr &I) const;
159
160 std::pair<Register, unsigned> selectVOP3ModsImpl(Register Src,
161 bool IsCanonicalizing = true,
162 bool AllowAbs = true,
163 bool OpSel = false) const;
164 std::pair<Register, unsigned> selectVOP3PModsF32Impl(Register Src) const;
165
166 Register copyToVGPRIfSrcFolded(Register Src, unsigned Mods,
167 MachineOperand Root, MachineInstr *InsertPt,
168 bool ForceVGPR = false) const;
169
170 InstructionSelector::ComplexRendererFns
171 selectVCSRC(MachineOperand &Root) const;
172
173 InstructionSelector::ComplexRendererFns
174 selectVSRC0(MachineOperand &Root) const;
175
176 InstructionSelector::ComplexRendererFns
177 selectVOP3Mods0(MachineOperand &Root) const;
178 InstructionSelector::ComplexRendererFns
179 selectVOP3BMods0(MachineOperand &Root) const;
180 InstructionSelector::ComplexRendererFns
181 selectVOP3OMods(MachineOperand &Root) const;
182 InstructionSelector::ComplexRendererFns
183 selectVOP3Mods(MachineOperand &Root) const;
184 InstructionSelector::ComplexRendererFns
185 selectVOP3ModsNonCanonicalizing(MachineOperand &Root) const;
186 InstructionSelector::ComplexRendererFns
187 selectVOP3BMods(MachineOperand &Root) const;
188
189 ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;
190
191 std::pair<Register, unsigned>
192 selectVOP3PModsImpl(Register RootReg, const MachineRegisterInfo &MRI,
193 bool IsDOT = false) const;
194 InstructionSelector::ComplexRendererFns
195 selectVOP3PRetHelper(MachineOperand &Root, bool IsDOT = false) const;
196
197 InstructionSelector::ComplexRendererFns
198 selectVOP3PMods(MachineOperand &Root) const;
199
200 InstructionSelector::ComplexRendererFns
201 selectVOP3PModsDOT(MachineOperand &Root) const;
202 InstructionSelector::ComplexRendererFns
203 selectVOP3PNoModsDOT(MachineOperand &Root) const;
204 InstructionSelector::ComplexRendererFns
205 selectVOP3PModsF32(MachineOperand &Root) const;
206 InstructionSelector::ComplexRendererFns
207 selectVOP3PNoModsF32(MachineOperand &Root) const;
208
209 InstructionSelector::ComplexRendererFns
210 selectWMMAOpSelVOP3PMods(MachineOperand &Root) const;
211
212 InstructionSelector::ComplexRendererFns
213 selectWMMAModsF32NegAbs(MachineOperand &Root) const;
214 InstructionSelector::ComplexRendererFns
215 selectWMMAModsF16Neg(MachineOperand &Root) const;
216 InstructionSelector::ComplexRendererFns
217 selectWMMAModsF16NegAbs(MachineOperand &Root) const;
218 InstructionSelector::ComplexRendererFns
219 selectWMMAVISrc(MachineOperand &Root) const;
220 InstructionSelector::ComplexRendererFns
221 selectSWMMACIndex8(MachineOperand &Root) const;
222 InstructionSelector::ComplexRendererFns
223 selectSWMMACIndex16(MachineOperand &Root) const;
224 InstructionSelector::ComplexRendererFns
225 selectSWMMACIndex32(MachineOperand &Root) const;
226
227 InstructionSelector::ComplexRendererFns
228 selectVOP3OpSelMods(MachineOperand &Root) const;
229
230 InstructionSelector::ComplexRendererFns
231 selectVINTERPMods(MachineOperand &Root) const;
232 InstructionSelector::ComplexRendererFns
233 selectVINTERPModsHi(MachineOperand &Root) const;
234
235 bool selectScaleOffset(MachineOperand &Root, Register &Offset,
236 bool IsSigned) const;
237 bool selectSmrdOffset(MachineOperand &Root, Register &Base, Register *SOffset,
238 int64_t *Offset, bool *ScaleOffset) const;
239 InstructionSelector::ComplexRendererFns
240 selectSmrdImm(MachineOperand &Root) const;
241 InstructionSelector::ComplexRendererFns
242 selectSmrdImm32(MachineOperand &Root) const;
243 InstructionSelector::ComplexRendererFns
244 selectSmrdSgpr(MachineOperand &Root) const;
245 InstructionSelector::ComplexRendererFns
246 selectSmrdSgprImm(MachineOperand &Root) const;
247
248 std::pair<Register, int>
249 selectFlatOffsetImpl(MachineOperand &Root,
250 AMDGPU::FlatAddrSpace FlatVariant) const;
251
252 InstructionSelector::ComplexRendererFns
253 selectFlatOffset(MachineOperand &Root) const;
254 InstructionSelector::ComplexRendererFns
255 selectGlobalOffset(MachineOperand &Root) const;
256 InstructionSelector::ComplexRendererFns
257 selectScratchOffset(MachineOperand &Root) const;
258
259 InstructionSelector::ComplexRendererFns
260 selectGlobalSAddr(MachineOperand &Root, unsigned CPolBits,
261 bool NeedIOffset = true) const;
262 InstructionSelector::ComplexRendererFns
263 selectGlobalSAddr(MachineOperand &Root) const;
264 InstructionSelector::ComplexRendererFns
265 selectGlobalSAddrCPol(MachineOperand &Root) const;
266 InstructionSelector::ComplexRendererFns
267 selectGlobalSAddrCPolM0(MachineOperand &Root) const;
268 InstructionSelector::ComplexRendererFns
269 selectGlobalSAddrGLC(MachineOperand &Root) const;
270 InstructionSelector::ComplexRendererFns
271 selectGlobalSAddrNoIOffset(MachineOperand &Root) const;
272 InstructionSelector::ComplexRendererFns
273 selectGlobalSAddrNoIOffsetM0(MachineOperand &Root) const;
274
275 InstructionSelector::ComplexRendererFns
276 selectScratchSAddr(MachineOperand &Root) const;
277 bool checkFlatScratchSVSSwizzleBug(Register VAddr, Register SAddr,
278 uint64_t ImmOffset) const;
279 InstructionSelector::ComplexRendererFns
280 selectScratchSVAddr(MachineOperand &Root) const;
281
282 InstructionSelector::ComplexRendererFns
283 selectMUBUFScratchOffen(MachineOperand &Root) const;
284 InstructionSelector::ComplexRendererFns
285 selectMUBUFScratchOffset(MachineOperand &Root) const;
286
287 bool isDSOffsetLegal(Register Base, int64_t Offset) const;
288 bool isDSOffset2Legal(Register Base, int64_t Offset0, int64_t Offset1,
289 unsigned Size) const;
290 bool isFlatScratchBaseLegal(Register Addr) const;
291 bool isFlatScratchBaseLegalSV(Register Addr) const;
292 bool isFlatScratchBaseLegalSVImm(Register Addr) const;
293
294 std::pair<Register, unsigned>
295 selectDS1Addr1OffsetImpl(MachineOperand &Root) const;
296 InstructionSelector::ComplexRendererFns
297 selectDS1Addr1Offset(MachineOperand &Root) const;
298
299 InstructionSelector::ComplexRendererFns
300 selectDS64Bit4ByteAligned(MachineOperand &Root) const;
301
302 InstructionSelector::ComplexRendererFns
303 selectDS128Bit8ByteAligned(MachineOperand &Root) const;
304
305 std::pair<Register, unsigned> selectDSReadWrite2Impl(MachineOperand &Root,
306 unsigned size) const;
307 InstructionSelector::ComplexRendererFns
308 selectDSReadWrite2(MachineOperand &Root, unsigned size) const;
309
310 std::tuple<Register, int64_t, bool>
311 getPtrBaseWithConstantOffset(Register Root,
312 const MachineRegisterInfo &MRI) const;
313
314 // Parse out a chain of up to two g_ptr_add instructions.
315 // g_ptr_add (n0, _)
316 // g_ptr_add (n0, (n1 = g_ptr_add n2, n3))
317 struct MUBUFAddressData {
318 Register N0, N2, N3;
319 int64_t Offset = 0;
320 };
321
322 bool shouldUseAddr64(MUBUFAddressData AddrData) const;
323
324 void splitIllegalMUBUFOffset(MachineIRBuilder &B,
325 Register &SOffset, int64_t &ImmOffset) const;
326
327 MUBUFAddressData parseMUBUFAddress(Register Src) const;
328
329 bool selectMUBUFAddr64Impl(MachineOperand &Root, Register &VAddr,
330 Register &RSrcReg, Register &SOffset,
331 int64_t &Offset) const;
332
333 bool selectMUBUFOffsetImpl(MachineOperand &Root, Register &RSrcReg,
334 Register &SOffset, int64_t &Offset) const;
335
336 InstructionSelector::ComplexRendererFns
337 selectBUFSOffset(MachineOperand &Root) const;
338
339 InstructionSelector::ComplexRendererFns
340 selectMUBUFAddr64(MachineOperand &Root) const;
341
342 InstructionSelector::ComplexRendererFns
343 selectMUBUFOffset(MachineOperand &Root) const;
344
345 ComplexRendererFns selectSMRDBufferImm(MachineOperand &Root) const;
346 ComplexRendererFns selectSMRDBufferImm32(MachineOperand &Root) const;
347 ComplexRendererFns selectSMRDBufferSgprImm(MachineOperand &Root) const;
348
349 std::pair<Register, unsigned> selectVOP3PMadMixModsImpl(MachineOperand &Root,
350 bool &Matched) const;
351 ComplexRendererFns selectVOP3PMadMixModsExt(MachineOperand &Root) const;
352 ComplexRendererFns selectVOP3PMadMixMods(MachineOperand &Root) const;
353
354 void renderTruncImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
355 int OpIdx = -1) const;
356
357 void renderTruncTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
358 int OpIdx) const;
359 void renderZextBoolTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
360 int OpIdx) const;
361
362 void renderOpSelTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
363 int OpIdx) const;
364
365 void renderSrcAndDstSelToOpSelXForm_0_0(MachineInstrBuilder &MIB,
366 const MachineInstr &MI,
367 int OpIdx) const;
368
369 void renderSrcAndDstSelToOpSelXForm_0_1(MachineInstrBuilder &MIB,
370 const MachineInstr &MI,
371 int OpIdx) const;
372
373 void renderSrcAndDstSelToOpSelXForm_1_0(MachineInstrBuilder &MIB,
374 const MachineInstr &MI,
375 int OpIdx) const;
376
377 void renderSrcAndDstSelToOpSelXForm_1_1(MachineInstrBuilder &MIB,
378 const MachineInstr &MI,
379 int OpIdx) const;
380
381 void renderDstSelToOpSelXForm(MachineInstrBuilder &MIB,
382 const MachineInstr &MI, int OpIdx) const;
383
384 void renderSrcSelToOpSelXForm(MachineInstrBuilder &MIB,
385 const MachineInstr &MI, int OpIdx) const;
386
387 void renderSrcAndDstSelToOpSelXForm_2_0(MachineInstrBuilder &MIB,
388 const MachineInstr &MI,
389 int OpIdx) const;
390
391 void renderDstSelToOpSel3XFormXForm(MachineInstrBuilder &MIB,
392 const MachineInstr &MI, int OpIdx) const;
393
394 void renderNegateImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
395 int OpIdx) const;
396
397 void renderBitcastFPImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
398 int OpIdx) const;
399
400 void renderBitcastFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,
401 int OpIdx) const {
402 renderBitcastFPImm(MIB, MI, OpIdx);
403 }
404 void renderBitcastFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,
405 int OpIdx) const {
406 renderBitcastFPImm(MIB, MI, OpIdx);
407 }
408
409 void renderCountTrailingOnesImm(MachineInstrBuilder &MIB,
410 const MachineInstr &MI, int OpIdx) const;
411 void renderExtractCPol(MachineInstrBuilder &MIB, const MachineInstr &MI,
412 int OpIdx) const;
413 void renderExtractSWZ(MachineInstrBuilder &MIB, const MachineInstr &MI,
414 int OpIdx) const;
415 void renderExtractCpolSetGLC(MachineInstrBuilder &MIB, const MachineInstr &MI,
416 int OpIdx) const;
417
418 void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,
419 int OpIdx) const;
420
421 void renderFPPow2ToExponent(MachineInstrBuilder &MIB, const MachineInstr &MI,
422 int OpIdx) const;
423
424 void renderRoundMode(MachineInstrBuilder &MIB, const MachineInstr &MI,
425 int OpIdx) const;
426
427 void renderVOP3PModsNeg(MachineInstrBuilder &MIB, const MachineInstr &MI,
428 int OpIdx) const;
429 void renderVOP3PModsNegs(MachineInstrBuilder &MIB, const MachineInstr &MI,
430 int OpIdx) const;
431 void renderVOP3PModsNegAbs(MachineInstrBuilder &MIB, const MachineInstr &MI,
432 int OpIdx) const;
433
434 void renderPrefetchLoc(MachineInstrBuilder &MIB, const MachineInstr &MI,
435 int OpIdx) const;
436
437 void renderScaledMAIIntrinsicOperand(MachineInstrBuilder &MIB,
438 const MachineInstr &MI, int OpIdx) const;
439
440 bool isInlineImmediate(const APInt &Imm) const;
441 bool isInlineImmediate(const APFloat &Imm) const;
442
443 // Returns true if TargetOpcode::G_AND MachineInstr `MI`'s masking of the
444 // shift amount operand's `ShAmtBits` bits is unneeded.
445 bool isUnneededShiftMask(const MachineInstr &MI, unsigned ShAmtBits) const;
446
447 /// Match a zero extend from a 32-bit value to 64-bits.
448 Register matchZeroExtendFromS32(Register Reg) const;
449 /// Match a sign extend from a 32-bit value to 64-bits.
450 Register matchSignExtendFromS32(Register Reg) const;
451 /// Match a zero extend from a 32-bit value to 64-bits, or \p Reg itself if it
452 /// is 32-bit.
453 Register matchZeroExtendFromS32OrS32(Register Reg) const;
454 /// Match a sign extend from a 32-bit value to 64-bits, or \p Reg itself if it
455 /// is 32-bit.
456 Register matchSignExtendFromS32OrS32(Register Reg) const;
457 /// Match either sign or zero extend depending on the \p IsSigned from a
458 /// 32-bit value to 64-bits, or \p Reg itself if it is 32-bit.
459 Register matchExtendFromS32OrS32(Register Reg, bool IsSigned) const;
460 /// Match an any extend from a 32-bit value to 64-bit.
461 Register matchAnyExtendFromS32(Register Reg) const;
462
463 const SIInstrInfo &TII;
464 const SIRegisterInfo &TRI;
465 const AMDGPURegisterBankInfo &RBI;
466 const GCNSubtarget &STI;
467#define GET_GLOBALISEL_PREDICATES_DECL
468#define AMDGPUSubtarget GCNSubtarget
469#include "AMDGPUGenGlobalISel.inc"
470#undef GET_GLOBALISEL_PREDICATES_DECL
471#undef AMDGPUSubtarget
472
473#define GET_GLOBALISEL_TEMPORARIES_DECL
474#include "AMDGPUGenGlobalISel.inc"
475#undef GET_GLOBALISEL_TEMPORARIES_DECL
476};
477
478} // End llvm namespace.
479#endif
480