1//===- AMDGPUBaseInfo.h - Top level definitions for AMDGPU ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUBASEINFO_H
10#define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUBASEINFO_H
11
12#include "AMDGPUSubtarget.h"
13#include "SIDefines.h"
14#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/ADT/StringTable.h"
17#include "llvm/IR/CallingConv.h"
18#include "llvm/IR/InstrTypes.h"
19#include "llvm/IR/Module.h"
20#include "llvm/Support/Alignment.h"
21#include "llvm/TargetParser/AMDGPUTargetParser.h"
22#include <array>
23#include <functional>
24#include <optional>
25#include <utility>
26
27// Pull in OpName enum definition and getNamedOperandIdx() declaration.
28#define GET_INSTRINFO_OPERAND_ENUM
29#include "AMDGPUGenInstrInfo.inc"
30
31struct amd_kernel_code_t;
32
33namespace llvm {
34
35struct Align;
36class Argument;
37class Function;
38class GlobalValue;
39class MachineInstr;
40class MCInstrInfo;
41class MCRegisterClass;
42class MCRegisterInfo;
43class MCSubtargetInfo;
44class MDNode;
45class StringRef;
46class Triple;
47class raw_ostream;
48
49namespace AMDGPU {
50
51struct AMDGPUMCKernelCodeT;
52struct IsaVersion;
53
54/// Generic target versions emitted by this version of LLVM.
55///
56/// These numbers are incremented every time a codegen breaking change occurs
57/// within a generic family.
58namespace GenericVersion {
59static constexpr unsigned GFX9 = 1;
60static constexpr unsigned GFX9_4 = 1;
61static constexpr unsigned GFX10_1 = 1;
62static constexpr unsigned GFX10_3 = 1;
63static constexpr unsigned GFX11 = 1;
64static constexpr unsigned GFX11_7 = 1;
65static constexpr unsigned GFX12 = 1;
66static constexpr unsigned GFX12_5 = 1;
67static constexpr unsigned GFX13 = 1;
68} // namespace GenericVersion
69
70enum { AMDHSA_COV4 = 4, AMDHSA_COV5 = 5, AMDHSA_COV6 = 6 };
71
72enum class FPType { None, FP4, FP8 };
73
74/// \returns True if \p STI is AMDHSA.
75bool isHsaAbi(const MCSubtargetInfo &STI);
76
77/// \returns Code object version from the IR module flag.
78unsigned getAMDHSACodeObjectVersion(const Module &M);
79
80/// \returns Code object version from ELF's e_ident[EI_ABIVERSION].
81unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion);
82
83/// \returns The default HSA code object version. This should only be used when
84/// we lack a more accurate CodeObjectVersion value (e.g. from the IR module
85/// flag or a .amdhsa_code_object_version directive)
86unsigned getDefaultAMDHSACodeObjectVersion();
87
88/// \returns ABIVersion suitable for use in ELF's e_ident[EI_ABIVERSION]. \param
89/// CodeObjectVersion is a value returned by getAMDHSACodeObjectVersion().
90uint8_t getELFABIVersion(const Triple &OS, unsigned CodeObjectVersion);
91
92/// \returns The offset of the multigrid_sync_arg argument from implicitarg_ptr
93unsigned getMultigridSyncArgImplicitArgPosition(unsigned COV);
94
95/// \returns The offset of the hostcall pointer argument from implicitarg_ptr
96unsigned getHostcallImplicitArgPosition(unsigned COV);
97
98unsigned getDefaultQueueImplicitArgPosition(unsigned COV);
99unsigned getCompletionActionImplicitArgPosition(unsigned COV);
100
101struct GcnBufferFormatInfo {
102 unsigned Format;
103 unsigned BitsPerComp;
104 unsigned NumComponents;
105 unsigned NumFormat;
106 unsigned DataFormat;
107};
108
109struct MAIInstInfo {
110 uint32_t Opcode;
111 bool is_dgemm;
112 bool is_gfx940_xdl;
113};
114
115struct MFMA_F8F6F4_Info {
116 unsigned Opcode;
117 unsigned F8F8Opcode;
118 uint8_t NumRegsSrcA;
119 uint8_t NumRegsSrcB;
120};
121
122/// Normalized WMMA or SWMMAC family used to select co-execution rules.
123enum class WMMAVariant {
124 Unknown = 0,
125 IU8_16x16x64,
126 F8F6F4_16x16x128,
127 F8F6F4_16x16x128_BothF4,
128 FP8BF8_16x16x64,
129 F16BF16_16x16x32,
130 FP8BF8_16x16x128,
131 F4_32x16x128,
132};
133
134struct True16D16Info {
135 unsigned T16Op;
136 unsigned HiOp;
137 unsigned LoOp;
138};
139
140struct WMMAInstInfo {
141 uint32_t Opcode;
142 bool is_wmma_xdl;
143 bool HasMatrixScale;
144 WMMAVariant CoExecVariant;
145};
146
147#define GET_MIMGBaseOpcode_DECL
148#define GET_MIMGDim_DECL
149#define GET_MIMGEncoding_DECL
150#define GET_MIMGLZMapping_DECL
151#define GET_MIMGMIPMapping_DECL
152#define GET_MIMGBiASMapping_DECL
153#define GET_MAIInstInfoTable_DECL
154#define GET_isMFMA_F8F6F4Table_DECL
155#define GET_isCvtScaleF32_F32F16ToF8F4Table_DECL
156#define GET_True16D16Table_DECL
157#define GET_WMMAInstInfoTable_DECL
158#include "AMDGPUGenSearchableTables.inc"
159
160using TargetIDSetting = AMDGPU::TargetIDSetting;
161using TargetID = AMDGPU::TargetID;
162
163/// Construct TargetID from MCSubtargetInfo. \p FeatureString is used to
164/// determine explicitly requested xnack/sramecc settings.
165TargetID createAMDGPUTargetID(const MCSubtargetInfo &STI,
166 StringRef FeatureString);
167
168namespace IsaInfo {
169
170enum {
171 FIXED_NUM_SGPRS_FOR_INIT_BUG = AMDGPU::FIXED_NUM_SGPRS_FOR_INIT_BUG,
172 TRAP_NUM_SGPRS = 16
173};
174
175/// Returns true if \p Lhs and \p Rhs are incompatible (both specific but
176/// different).
177inline bool targetIDSettingsConflict(TargetIDSetting Lhs, TargetIDSetting Rhs) {
178 return Lhs != TargetIDSetting::Any && Rhs != TargetIDSetting::Any &&
179 Lhs != Rhs;
180}
181
182/// \returns Instruction cache line size in bytes for given subtarget \p STI.
183unsigned getInstCacheLineSize(const MCSubtargetInfo &STI);
184
185/// \returns Wavefront size for given subtarget \p STI.
186unsigned getWavefrontSize(const MCSubtargetInfo &STI);
187
188/// \returns Local memory size in bytes for given subtarget \p STI.
189unsigned getLocalMemorySize(const MCSubtargetInfo &STI);
190
191/// \returns Maximum addressable local memory size in bytes for given subtarget
192/// \p STI.
193unsigned getAddressableLocalMemorySize(const MCSubtargetInfo &STI);
194
195/// \returns Maximum number of work groups per compute unit for given subtarget
196/// \p STI and limited by given \p FlatWorkGroupSize.
197unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo &STI,
198 unsigned FlatWorkGroupSize);
199
200/// \returns Number of waves per execution unit required to support the given \p
201/// FlatWorkGroupSize.
202unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo &STI,
203 unsigned FlatWorkGroupSize);
204
205/// \returns Number of waves per work group for given subtarget \p STI and
206/// \p FlatWorkGroupSize.
207unsigned getWavesPerWorkGroup(const MCSubtargetInfo &STI,
208 unsigned FlatWorkGroupSize);
209
210/// \returns SGPR encoding granularity for given subtarget \p STI.
211unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI);
212
213/// \returns Minimum number of SGPRs that meets the given number of waves per
214/// execution unit requirement for given subtarget \p STI.
215unsigned getMinNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU);
216
217/// \returns Maximum number of SGPRs that meets the given number of waves per
218/// execution unit requirement for given subtarget \p STI.
219unsigned getMaxNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
220 bool Addressable);
221
222/// \returns Number of extra SGPRs implicitly required by given subtarget \p
223/// STI when the given special registers are used.
224unsigned getNumExtraSGPRs(const MCSubtargetInfo &STI, bool VCCUsed,
225 bool FlatScrUsed, bool XNACKUsed);
226
227/// \returns Number of SGPR blocks needed for given subtarget \p STI when
228/// \p NumSGPRs are used. \p NumSGPRs should already include any special
229/// register counts.
230unsigned getNumSGPRBlocks(const MCSubtargetInfo &STI, unsigned NumSGPRs);
231
232/// \returns VGPR allocation granularity for given subtarget \p STI.
233///
234/// For subtargets which support it, \p EnableWavefrontSize32 should match
235/// the ENABLE_WAVEFRONT_SIZE32 kernel descriptor field.
236unsigned
237getVGPRAllocGranule(const MCSubtargetInfo &STI, unsigned DynamicVGPRBlockSize,
238 std::optional<bool> EnableWavefrontSize32 = std::nullopt);
239
240/// \returns VGPR encoding granularity for given subtarget \p STI.
241///
242/// For subtargets which support it, \p EnableWavefrontSize32 should match
243/// the ENABLE_WAVEFRONT_SIZE32 kernel descriptor field.
244unsigned getVGPREncodingGranule(
245 const MCSubtargetInfo &STI,
246 std::optional<bool> EnableWavefrontSize32 = std::nullopt);
247
248/// For subtargets with a unified VGPR file and mixed ArchVGPR/AGPR usage,
249/// returns the allocation granule for ArchVGPRs.
250unsigned getArchVGPRAllocGranule();
251
252/// Maximum number of VGPR blocks that can be allocated in dynamic VGPR mode.
253static constexpr unsigned MaxDynamicVGPRBlocks = 8;
254
255/// \returns Addressable number of architectural VGPRs for a given subtarget \p
256/// STI.
257unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo &STI);
258
259/// \returns Addressable number of VGPRs for given subtarget \p STI.
260unsigned getAddressableNumVGPRs(const MCSubtargetInfo &STI,
261 unsigned DynamicVGPRBlockSize);
262
263/// \returns Minimum number of VGPRs that meets given number of waves per
264/// execution unit requirement for given subtarget \p STI.
265unsigned getMinNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
266 unsigned DynamicVGPRBlockSize);
267
268/// \returns Maximum number of VGPRs that meets given number of waves per
269/// execution unit requirement for given subtarget \p STI.
270unsigned getMaxNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
271 unsigned DynamicVGPRBlockSize);
272
273/// \returns Number of waves reachable for a given \p NumVGPRs usage for given
274/// subtarget \p STI.
275unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo &STI,
276 unsigned NumVGPRs,
277 unsigned DynamicVGPRBlockSize);
278
279/// \returns Number of waves reachable for a given \p NumVGPRs usage, \p Granule
280/// size, \p MaxWaves possible, and \p TotalNumVGPRs available.
281unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
282 unsigned MaxWaves,
283 unsigned TotalNumVGPRs);
284
285/// \returns Whether allocated SGPRs can reduce occupancy on subtarget \p STI
286/// (true pre-GFX10). One named capability so callers don't test the version.
287bool isSGPROccupancyLimited(const MCSubtargetInfo &STI);
288
289/// \returns SGPR-limited occupancy (waves per EU) for subtarget \p STI: the
290/// inverse of getMaxNumSGPRs(). Unlike getMaxNumSGPRs() the budget is not
291/// clamped to the addressable count, since the allocated count callers pass in
292/// can exceed it.
293unsigned getOccupancyWithNumSGPRs(const MCSubtargetInfo &STI, unsigned SGPRs);
294
295/// \returns SGPR-limited occupancy computed from explicit budget parameters
296/// (\p MaxWaves, \p TotalNumSGPRs, \p Granule, \p TrapReserve). Subtarget-free
297/// core shared by the overload above and the occupancy MCExpr. Callers must
298/// check isSGPROccupancyLimited() first.
299unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
300 unsigned TotalNumSGPRs, unsigned Granule,
301 unsigned TrapReserve);
302
303/// \returns Number of VGPR blocks that need to be allocated for the given
304/// subtarget \p STI when \p NumVGPRs are used.
305unsigned getAllocatedNumVGPRBlocks(
306 const MCSubtargetInfo &STI, unsigned NumVGPRs,
307 unsigned DynamicVGPRBlockSize,
308 std::optional<bool> EnableWavefrontSize32 = std::nullopt);
309
310} // end namespace IsaInfo
311
312// Represents a field in an encoded value.
313template <unsigned HighBit, unsigned LowBit, unsigned D = 0>
314struct EncodingField {
315 static_assert(HighBit >= LowBit, "Invalid bit range!");
316 static constexpr unsigned Offset = LowBit;
317 static constexpr unsigned Width = HighBit - LowBit + 1;
318
319 using ValueType = unsigned;
320 static_assert(Width <= sizeof(ValueType) * 8);
321 static constexpr ValueType Default = D;
322
323 ValueType Value;
324 constexpr EncodingField(ValueType Value) : Value(Value) {}
325
326 constexpr uint64_t encode() const { return Value; }
327 static ValueType decode(uint64_t Encoded) {
328 return static_cast<ValueType>(Encoded);
329 }
330};
331
332// Represents a single bit in an encoded value.
333template <unsigned Bit, unsigned D = 0>
334using EncodingBit = EncodingField<Bit, Bit, D>;
335
336// A helper for encoding and decoding multiple fields.
337template <typename... Fields> struct EncodingFields {
338 static constexpr uint64_t encode(Fields... Values) {
339 return ((Values.encode() << Values.Offset) | ...);
340 }
341
342 static std::tuple<typename Fields::ValueType...> decode(uint64_t Encoded) {
343 return {Fields::decode((Encoded >> Fields::Offset) &
344 maxUIntN(Fields::Width))...};
345 }
346};
347
348LLVM_READONLY
349inline bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx) {
350 return getNamedOperandIdx(Opcode, Name: NamedIdx) != -1;
351}
352
353LLVM_READONLY
354int32_t getSOPPWithRelaxation(uint32_t Opcode);
355
356struct MIMGBaseOpcodeInfo {
357 MIMGBaseOpcode BaseOpcode;
358 bool Store;
359 bool Atomic;
360 bool AtomicX2;
361 bool Sampler;
362 bool Gather4;
363
364 uint8_t NumExtraArgs;
365 bool Gradients;
366 bool G16;
367 bool Coordinates;
368 bool LodOrClampOrMip;
369 bool HasD16;
370 bool MSAA;
371 bool BVH;
372 bool A16;
373 bool NoReturn;
374 bool PointSampleAccel;
375};
376
377LLVM_READONLY
378const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc);
379
380LLVM_READONLY
381const MIMGBaseOpcodeInfo *getMIMGBaseOpcodeInfo(unsigned BaseOpcode);
382
383struct MIMGDimInfo {
384 MIMGDim Dim;
385 MIMGDim NonArrayDim;
386 uint8_t NumCoords;
387 uint8_t NumGradients;
388 bool MSAA;
389 bool DA;
390 uint8_t Encoding;
391 StringTable::Offset AsmSuffix;
392};
393
394LLVM_READONLY
395const MIMGDimInfo *getMIMGDimInfo(unsigned DimEnum);
396
397LLVM_READONLY StringRef getMIMGDimInfoStr(StringTable::Offset);
398
399LLVM_READONLY
400const MIMGDimInfo *getMIMGDimInfoByEncoding(uint8_t DimEnc);
401
402LLVM_READONLY
403const MIMGDimInfo *getMIMGDimInfoByAsmSuffix(StringRef AsmSuffix);
404
405struct MIMGLZMappingInfo {
406 MIMGBaseOpcode L;
407 MIMGBaseOpcode LZ;
408};
409
410struct MIMGMIPMappingInfo {
411 MIMGBaseOpcode MIP;
412 MIMGBaseOpcode NONMIP;
413};
414
415struct MIMGBiasMappingInfo {
416 MIMGBaseOpcode Bias;
417 MIMGBaseOpcode NoBias;
418};
419
420struct MIMGOffsetMappingInfo {
421 MIMGBaseOpcode Offset;
422 MIMGBaseOpcode NoOffset;
423};
424
425struct MIMGG16MappingInfo {
426 MIMGBaseOpcode G;
427 MIMGBaseOpcode G16;
428};
429
430LLVM_READONLY
431const MIMGLZMappingInfo *getMIMGLZMappingInfo(unsigned L);
432
433struct WMMAOpcodeMappingInfo {
434 unsigned Opcode2Addr;
435 unsigned Opcode3Addr;
436};
437
438LLVM_READONLY
439const MIMGMIPMappingInfo *getMIMGMIPMappingInfo(unsigned MIP);
440
441LLVM_READONLY
442const MIMGBiasMappingInfo *getMIMGBiasMappingInfo(unsigned Bias);
443
444LLVM_READONLY
445const MIMGOffsetMappingInfo *getMIMGOffsetMappingInfo(unsigned Offset);
446
447LLVM_READONLY
448const MIMGG16MappingInfo *getMIMGG16MappingInfo(unsigned G);
449
450LLVM_READONLY
451int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
452 unsigned VDataDwords, unsigned VAddrDwords,
453 bool IndexedRsrc = false, bool IndexedSamp = false);
454
455LLVM_READONLY
456int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels);
457
458LLVM_READONLY
459unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
460 const MIMGDimInfo *Dim, bool IsA16,
461 bool IsG16Supported);
462
463struct MIMGInfo {
464 uint32_t Opcode;
465 uint32_t BaseOpcode;
466 uint8_t MIMGEncoding;
467 uint8_t VDataDwords;
468 uint8_t VAddrDwords;
469 uint8_t VAddrOperands;
470 bool IndexedRsrc;
471 bool IndexedSamp;
472};
473
474LLVM_READONLY
475const MIMGInfo *getMIMGInfo(unsigned Opc);
476
477struct VLdStIdxOpcodeInfo {
478 unsigned Opcode;
479 uint16_t BitWidth;
480 bool IsStore;
481 bool IsGPRIdx;
482};
483
484LLVM_READONLY
485const VLdStIdxOpcodeInfo *getVLdStIdxOpcodeInfoByOpcode(unsigned Opc);
486
487LLVM_READONLY
488const VLdStIdxOpcodeInfo *
489getVLdStIdxOpcodeInfoByKey(unsigned BitWidth, bool IsStore, bool IsGPRIdx);
490
491LLVM_READONLY
492int getMTBUFBaseOpcode(unsigned Opc);
493
494LLVM_READONLY
495int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements);
496
497LLVM_READONLY
498int getMTBUFElements(unsigned Opc);
499
500LLVM_READONLY
501bool getMTBUFHasVAddr(unsigned Opc);
502
503LLVM_READONLY
504bool getMTBUFHasSrsrc(unsigned Opc);
505
506LLVM_READONLY
507bool getMTBUFHasSoffset(unsigned Opc);
508
509LLVM_READONLY
510int getMUBUFBaseOpcode(unsigned Opc);
511
512LLVM_READONLY
513int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements);
514
515LLVM_READONLY
516int getMUBUFElements(unsigned Opc);
517
518LLVM_READONLY
519bool getMUBUFHasVAddr(unsigned Opc);
520
521LLVM_READONLY
522bool getMUBUFHasSrsrc(unsigned Opc);
523
524LLVM_READONLY
525bool getMUBUFHasSoffset(unsigned Opc);
526
527LLVM_READONLY
528bool getMUBUFIsBufferInv(unsigned Opc);
529
530LLVM_READONLY
531bool getMUBUFTfe(unsigned Opc);
532
533LLVM_READONLY
534bool getSMEMIsBuffer(unsigned Opc);
535
536LLVM_READONLY
537bool getVOP1IsSingle(unsigned Opc);
538
539LLVM_READONLY
540bool getVOP2IsSingle(unsigned Opc);
541
542LLVM_READONLY
543bool getVOP3IsSingle(unsigned Opc);
544
545LLVM_READONLY
546bool isVOPC64DPP(unsigned Opc);
547
548LLVM_READONLY
549bool isVOPCAsmOnly(unsigned Opc);
550
551/// Returns true if this VOP3P opcode has no op_sel_hi field, in which case
552/// those bits are encoded as zero instead of defaulting to one.
553LLVM_READONLY
554bool hasNoOpSelHi(unsigned Opc);
555
556/// Returns true if MAI operation is a double precision GEMM.
557LLVM_READONLY
558bool getMAIIsDGEMM(unsigned Opc);
559
560LLVM_READONLY
561bool getMAIIsGFX940XDL(unsigned Opc);
562
563LLVM_READONLY
564bool getWMMAIsXDL(unsigned Opc);
565
566LLVM_READONLY
567bool getHasMatrixScale(unsigned Opc);
568
569// Get an equivalent BitOp3 for a binary logical \p Opc.
570// \returns BitOp3 modifier for the logical operation or zero.
571// Used in VOPD3 conversion.
572unsigned getBitOp2(unsigned Opc);
573
574struct CanBeVOPD {
575 bool X;
576 bool Y;
577};
578
579/// \returns SIEncodingFamily used for VOPD encoding on a \p ST.
580LLVM_READONLY
581unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST);
582
583LLVM_READONLY
584CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3);
585
586LLVM_READNONE
587uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal);
588
589LLVM_READONLY
590const MFMA_F8F6F4_Info *getMFMA_F8F6F4_WithFormatArgs(unsigned CBSZ,
591 unsigned BLGP,
592 unsigned F8F8Opcode);
593
594LLVM_READNONE
595uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt);
596
597LLVM_READONLY
598const MFMA_F8F6F4_Info *getWMMA_F8F6F4_WithFormatArgs(unsigned FmtA,
599 unsigned FmtB,
600 unsigned F8F8Opcode);
601
602/// \return true if this combination is listed as valid.
603LLVM_READONLY
604bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale,
605 unsigned BFmt, unsigned BScale);
606
607/// \returns the matrix_a_fmt and matrix_b_fmt operands of an f8f6f4 WMMA.
608/// Works on both MCInst and MachineInstr.
609template <typename InstT>
610std::pair<unsigned, unsigned> getWMMAMatrixFmts(const InstT &MI) {
611 unsigned Opc = MI.getOpcode();
612 int AIdx = getNamedOperandIdx(Opcode: Opc, Name: OpName::matrix_a_fmt);
613 int BIdx = getNamedOperandIdx(Opcode: Opc, Name: OpName::matrix_b_fmt);
614 assert(AIdx != -1 && BIdx != -1 && "expected an f8f6f4 WMMA");
615 return {static_cast<unsigned>(MI.getOperand(AIdx).getImm()),
616 static_cast<unsigned>(MI.getOperand(BIdx).getImm())};
617}
618
619/// \returns true if either matrix input of an f8f6f4 WMMA is f8.
620template <typename InstT> bool isWMMAAnyF8(const InstT &MI) {
621 auto [AFmt, BFmt] = getWMMAMatrixFmts(MI);
622 return AFmt <= WMMA::MATRIX_FMT_BF8 || BFmt <= WMMA::MATRIX_FMT_BF8;
623}
624
625/// \returns true if both matrix inputs of an f8f6f4 WMMA are f4.
626template <typename InstT> bool isWMMABothF4(const InstT &MI) {
627 auto [AFmt, BFmt] = getWMMAMatrixFmts(MI);
628 return AFmt == WMMA::MATRIX_FMT_FP4 && BFmt == WMMA::MATRIX_FMT_FP4;
629}
630
631LLVM_READONLY
632const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
633 uint8_t NumComponents,
634 uint8_t NumFormat,
635 const MCSubtargetInfo &STI);
636LLVM_READONLY
637const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
638 const MCSubtargetInfo &STI);
639
640LLVM_READONLY
641int32_t getMCOpcode(uint32_t Opcode, unsigned Gen);
642
643LLVM_READONLY
644unsigned getVOPDOpcode(unsigned Opc, bool VOPD3);
645
646LLVM_READONLY
647int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily,
648 bool VOPD3);
649
650LLVM_READONLY
651bool isVOPD(unsigned Opc);
652
653LLVM_READNONE
654bool isMAC(unsigned Opc);
655
656LLVM_READNONE
657bool isPermlane16(unsigned Opc);
658
659LLVM_READNONE
660bool isGenericAtomic(unsigned Opc);
661
662LLVM_READNONE
663bool isCvt_F32_Fp8_Bf8_e64(unsigned Opc);
664
665namespace VOPD {
666
667enum Component : unsigned {
668 DST = 0,
669 SRC0,
670 SRC1,
671 SRC2,
672
673 DST_NUM = 1,
674 MAX_SRC_NUM = 3,
675 MAX_OPR_NUM = DST_NUM + MAX_SRC_NUM
676};
677
678// LSB mask for VGPR banks per VOPD component operand.
679// 4 banks result in a mask 3, setting 2 lower bits.
680constexpr unsigned VOPD_VGPR_BANK_MASKS[] = {1, 3, 3, 1};
681constexpr unsigned VOPD3_VGPR_BANK_MASKS[] = {1, 3, 3, 3};
682// GFX11 VOPD interlock hazard requires SRC0/SRC1 to have
683// different parities, not just on different banks. Else,
684// non-deterministic forwarding error may occur.
685constexpr unsigned VOPD_GFX11_VGPR_BANK_MASKS[] = {1, 1, 1, 1};
686
687enum ComponentIndex : unsigned { X = 0, Y = 1 };
688constexpr unsigned COMPONENTS[] = {ComponentIndex::X, ComponentIndex::Y};
689constexpr unsigned COMPONENTS_NUM = 2;
690
691// Properties of VOPD components.
692class ComponentProps {
693private:
694 unsigned SrcOperandsNum = 0;
695 unsigned MandatoryLiteralIdx = ~0u;
696 bool HasSrc2Acc = false;
697 unsigned NumVOPD3Mods = 0;
698 unsigned Opcode = 0;
699 bool IsVOP3 = false;
700
701public:
702 ComponentProps() = default;
703 ComponentProps(const MCInstrDesc &OpDesc, bool VOP3Layout = false);
704
705 // Return the total number of src operands this component has.
706 unsigned getCompSrcOperandsNum() const { return SrcOperandsNum; }
707
708 // Return the number of src operands of this component visible to the parser.
709 unsigned getCompParsedSrcOperandsNum() const {
710 return SrcOperandsNum - HasSrc2Acc;
711 }
712
713 // Return true iif this component has a mandatory literal.
714 bool hasMandatoryLiteral() const { return MandatoryLiteralIdx != ~0u; }
715
716 // If this component has a mandatory literal, return component operand
717 // index of this literal (i.e. either Component::SRC1 or Component::SRC2).
718 unsigned getMandatoryLiteralCompOperandIndex() const {
719 assert(hasMandatoryLiteral());
720 return MandatoryLiteralIdx;
721 }
722
723 // Return true iif this component has operand
724 // with component index CompSrcIdx and this operand may be a register.
725 bool hasRegSrcOperand(unsigned CompSrcIdx) const {
726 assert(CompSrcIdx < Component::MAX_SRC_NUM);
727 return SrcOperandsNum > CompSrcIdx && !hasMandatoryLiteralAt(CompSrcIdx);
728 }
729
730 // Return true iif this component has tied src2.
731 bool hasSrc2Acc() const { return HasSrc2Acc; }
732
733 // Return a number of source modifiers if instruction is used in VOPD3.
734 unsigned getCompVOPD3ModsNum() const { return NumVOPD3Mods; }
735
736 // Return opcode of the component.
737 unsigned getOpcode() const { return Opcode; }
738
739 // Returns if component opcode is in VOP3 encoding.
740 unsigned isVOP3() const { return IsVOP3; }
741
742 // Return index of BitOp3 operand or -1.
743 int getBitOp3OperandIdx() const;
744
745private:
746 bool hasMandatoryLiteralAt(unsigned CompSrcIdx) const {
747 assert(CompSrcIdx < Component::MAX_SRC_NUM);
748 return MandatoryLiteralIdx == Component::DST_NUM + CompSrcIdx;
749 }
750};
751
752enum ComponentKind : unsigned {
753 SINGLE = 0, // A single VOP1 or VOP2 instruction which may be used in VOPD.
754 COMPONENT_X, // A VOPD instruction, X component.
755 COMPONENT_Y, // A VOPD instruction, Y component.
756 MAX = COMPONENT_Y
757};
758
759// Interface functions of this class map VOPD component operand indices
760// to indices of operands in MachineInstr/MCInst or parsed operands array.
761//
762// Note that this class operates with 3 kinds of indices:
763// - VOPD component operand indices (Component::DST, Component::SRC0, etc.);
764// - MC operand indices (they refer operands in a MachineInstr/MCInst);
765// - parsed operand indices (they refer operands in parsed operands array).
766//
767// For SINGLE components mapping between these indices is trivial.
768// But things get more complicated for COMPONENT_X and
769// COMPONENT_Y because these components share the same
770// MachineInstr/MCInst and the same parsed operands array.
771// Below is an example of component operand to parsed operand
772// mapping for the following instruction:
773//
774// v_dual_add_f32 v255, v4, v5 :: v_dual_mov_b32 v6, v1
775//
776// PARSED COMPONENT PARSED
777// COMPONENT OPERANDS OPERAND INDEX OPERAND INDEX
778// -------------------------------------------------------------------
779// "v_dual_add_f32" 0
780// v_dual_add_f32 v255 0 (DST) --> 1
781// v4 1 (SRC0) --> 2
782// v5 2 (SRC1) --> 3
783// "::" 4
784// "v_dual_mov_b32" 5
785// v_dual_mov_b32 v6 0 (DST) --> 6
786// v1 1 (SRC0) --> 7
787// -------------------------------------------------------------------
788//
789class ComponentLayout {
790private:
791 // Regular MachineInstr/MCInst operands are ordered as follows:
792 // dst, src0 [, other src operands]
793 // VOPD MachineInstr/MCInst operands are ordered as follows:
794 // dstX, dstY, src0X [, other OpX operands], src0Y [, other OpY operands]
795 // Each ComponentKind has operand indices defined below.
796 static constexpr unsigned MC_DST_IDX[] = {0, 0, 1};
797
798 // VOPD3 instructions may have 2 or 3 source modifiers, src2 modifier is not
799 // used if there is tied accumulator. Indexing of this array:
800 // MC_SRC_IDX[VOPD3ModsNum][SrcNo]. This returns an index for a SINGLE
801 // instruction layout, add 1 for COMPONENT_X or COMPONENT_Y. For the second
802 // component add OpX.MCSrcNum + OpX.VOPD3ModsNum.
803 // For VOPD1/VOPD2 use column with zero modifiers.
804 static constexpr unsigned SINGLE_MC_SRC_IDX[4][3] = {
805 {1, 2, 3}, {2, 3, 4}, {2, 4, 5}, {2, 4, 6}};
806
807 // Parsed operands of regular instructions are ordered as follows:
808 // Mnemo dst src0 [vsrc1 ...]
809 // Parsed VOPD operands are ordered as follows:
810 // OpXMnemo dstX src0X [vsrc1X|imm vsrc1X|vsrc1X imm] '::'
811 // OpYMnemo dstY src0Y [vsrc1Y|imm vsrc1Y|vsrc1Y imm]
812 // Each ComponentKind has operand indices defined below.
813 static constexpr unsigned PARSED_DST_IDX[] = {1, 1,
814 4 /* + OpX.ParsedSrcNum */};
815 static constexpr unsigned FIRST_PARSED_SRC_IDX[] = {
816 2, 2, 5 /* + OpX.ParsedSrcNum */};
817
818private:
819 const ComponentKind Kind;
820 const ComponentProps PrevComp;
821 const unsigned VOPD3ModsNum;
822 const int BitOp3Idx; // Index of bitop3 operand or -1
823
824public:
825 // Create layout for COMPONENT_X or SINGLE component.
826 ComponentLayout(ComponentKind Kind, unsigned VOPD3ModsNum, int BitOp3Idx)
827 : Kind(Kind), VOPD3ModsNum(VOPD3ModsNum), BitOp3Idx(BitOp3Idx) {
828 assert(Kind == ComponentKind::SINGLE || Kind == ComponentKind::COMPONENT_X);
829 }
830
831 // Create layout for COMPONENT_Y which depends on COMPONENT_X layout.
832 ComponentLayout(const ComponentProps &OpXProps, unsigned VOPD3ModsNum,
833 int BitOp3Idx)
834 : Kind(ComponentKind::COMPONENT_Y), PrevComp(OpXProps),
835 VOPD3ModsNum(VOPD3ModsNum), BitOp3Idx(BitOp3Idx) {}
836
837public:
838 // Return the index of dst operand in MCInst operands.
839 unsigned getIndexOfDstInMCOperands() const { return MC_DST_IDX[Kind]; }
840
841 // Return the index of the specified src operand in MCInst operands.
842 unsigned getIndexOfSrcInMCOperands(unsigned CompSrcIdx, bool VOPD3) const {
843 assert(CompSrcIdx < Component::MAX_SRC_NUM);
844
845 if (Kind == SINGLE && CompSrcIdx == 2 && BitOp3Idx != -1)
846 return BitOp3Idx;
847
848 if (VOPD3) {
849 return SINGLE_MC_SRC_IDX[VOPD3ModsNum][CompSrcIdx] + getPrevCompSrcNum() +
850 getPrevCompVOPD3ModsNum() + (Kind != SINGLE ? 1 : 0);
851 }
852
853 return SINGLE_MC_SRC_IDX[0][CompSrcIdx] + getPrevCompSrcNum() +
854 (Kind != SINGLE ? 1 : 0);
855 }
856
857 // Return the index of dst operand in the parsed operands array.
858 unsigned getIndexOfDstInParsedOperands() const {
859 return PARSED_DST_IDX[Kind] + getPrevCompParsedSrcNum();
860 }
861
862 // Return the index of the specified src operand in the parsed operands array.
863 unsigned getIndexOfSrcInParsedOperands(unsigned CompSrcIdx) const {
864 assert(CompSrcIdx < Component::MAX_SRC_NUM);
865 return FIRST_PARSED_SRC_IDX[Kind] + getPrevCompParsedSrcNum() + CompSrcIdx;
866 }
867
868private:
869 unsigned getPrevCompSrcNum() const {
870 return PrevComp.getCompSrcOperandsNum();
871 }
872 unsigned getPrevCompParsedSrcNum() const {
873 return PrevComp.getCompParsedSrcOperandsNum();
874 }
875 unsigned getPrevCompVOPD3ModsNum() const {
876 return PrevComp.getCompVOPD3ModsNum();
877 }
878};
879
880// Layout and properties of VOPD components.
881class ComponentInfo : public ComponentProps, public ComponentLayout {
882public:
883 // Create ComponentInfo for COMPONENT_X or SINGLE component.
884 ComponentInfo(const MCInstrDesc &OpDesc,
885 ComponentKind Kind = ComponentKind::SINGLE,
886 bool VOP3Layout = false)
887 : ComponentProps(OpDesc, VOP3Layout),
888 ComponentLayout(Kind, getCompVOPD3ModsNum(), getBitOp3OperandIdx()) {}
889
890 // Create ComponentInfo for COMPONENT_Y which depends on COMPONENT_X layout.
891 ComponentInfo(const MCInstrDesc &OpDesc, const ComponentProps &OpXProps,
892 bool VOP3Layout = false)
893 : ComponentProps(OpDesc, VOP3Layout),
894 ComponentLayout(OpXProps, getCompVOPD3ModsNum(),
895 getBitOp3OperandIdx()) {}
896
897 // Map component operand index to parsed operand index.
898 // Return 0 if the specified operand does not exist.
899 unsigned getIndexInParsedOperands(unsigned CompOprIdx) const;
900};
901
902// Properties of VOPD instructions.
903class InstInfo {
904private:
905 const ComponentInfo CompInfo[COMPONENTS_NUM];
906
907public:
908 using RegIndices = std::array<MCRegister, Component::MAX_OPR_NUM>;
909
910 InstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY)
911 : CompInfo{OpX, OpY} {}
912
913 InstInfo(const ComponentInfo &OprInfoX, const ComponentInfo &OprInfoY)
914 : CompInfo{OprInfoX, OprInfoY} {}
915
916 const ComponentInfo &operator[](size_t ComponentIdx) const {
917 assert(ComponentIdx < COMPONENTS_NUM);
918 return CompInfo[ComponentIdx];
919 }
920
921 // Check VOPD operands constraints.
922 // GetRegIdx(Component, MCOperandIdx) must return a VGPR register index
923 // for the specified component and MC operand. The callback must return 0
924 // if the operand is not a register or not a VGPR.
925 // If \p SkipSrc is set to true then constraints for source operands are not
926 // checked.
927 // If \p AllowSameVGPR is set then same VGPRs are allowed for X and Y sources
928 // even though it violates requirement to be from different banks.
929 // If \p VOPD3 is set to true both dst registers allowed to be either odd
930 // or even and instruction may have real src2 as opposed to tied accumulator.
931 // If \p HasGFX11InterlockHazard is set then X/Y SRC0 and SRC1 VGPRs
932 // must have different register-number parity.
933 bool
934 hasInvalidOperand(std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
935 const MCRegisterInfo &MRI, bool SkipSrc = false,
936 bool AllowSameVGPR = false, bool VOPD3 = false,
937 bool HasGFX11InterlockHazard = false) const {
938 return getInvalidCompOperandIndex(GetRegIdx, MRI, SkipSrc, AllowSameVGPR,
939 VOPD3, HasGFX11InterlockHazard)
940 .has_value();
941 }
942
943 // Check VOPD operands constraints.
944 // Return the index of an invalid component operand, if any.
945 // If \p SkipSrc is set to true then constraints for source operands are not
946 // checked except for being from the same halves of VGPR file on gfx1250.
947 // If \p AllowSameVGPR is set then same VGPRs are allowed for X and Y sources
948 // even though it violates requirement to be from different banks.
949 // If \p VOPD3 is set to true both dst registers allowed to be either odd
950 // or even and instruction may have real src2 as opposed to tied accumulator.
951 // If \p HasGFX11InterlockHazard is set then X/Y SRC0 and SRC1 VGPRs
952 // must have different register-number parity.
953 std::optional<unsigned> getInvalidCompOperandIndex(
954 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
955 const MCRegisterInfo &MRI, bool SkipSrc = false,
956 bool AllowSameVGPR = false, bool VOPD3 = false,
957 bool HasGFX11InterlockHazard = false) const;
958
959private:
960 RegIndices
961 getRegIndices(unsigned ComponentIdx,
962 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
963 bool VOPD3) const;
964};
965
966} // namespace VOPD
967
968LLVM_READONLY
969std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode);
970
971LLVM_READONLY
972// Get properties of 2 single VOP1/VOP2 instructions
973// used as components to create a VOPD instruction.
974VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY);
975
976LLVM_READONLY
977// Get properties of VOPD X and Y components.
978VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,
979 const MCInstrInfo *InstrInfo);
980
981LLVM_READONLY
982bool isAsyncStore(unsigned Opc);
983LLVM_READONLY
984bool isTensorStore(unsigned Opc);
985LLVM_READONLY
986unsigned getTemporalHintType(const MCInstrDesc TID);
987
988LLVM_READONLY
989bool isTrue16Inst(unsigned Opc);
990
991LLVM_READONLY
992FPType getFPDstSelType(unsigned Opc);
993
994bool isDPMACCInstruction(unsigned Opc);
995
996LLVM_READONLY
997unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc);
998
999void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &Header,
1000 const MCSubtargetInfo &STI);
1001
1002bool isReadOnlySegment(const GlobalValue *GV);
1003
1004/// \returns True if constants should be emitted to .text section for given
1005/// target triple \p TT, false otherwise.
1006bool shouldEmitConstantsToTextSection(const Triple &TT);
1007
1008/// Returns a valid charcode or 0 in the first entry if this is a valid physical
1009/// register name. Followed by the start register number, and the register
1010/// width. Does not validate the number of registers exists in the class. Unlike
1011/// parseAsmConstraintPhysReg, this does not expect the name to be wrapped in
1012/// "{}".
1013std::tuple<char, unsigned, unsigned> parseAsmPhysRegName(StringRef TupleString);
1014
1015/// Returns a valid charcode or 0 in the first entry if this is a valid physical
1016/// register constraint. Followed by the start register number, and the register
1017/// width. Does not validate the number of registers exists in the class.
1018std::tuple<char, unsigned, unsigned>
1019parseAsmConstraintPhysReg(StringRef Constraint);
1020
1021/// \returns A pair of integer values requested using \p F's \p Name attribute
1022/// in "first[,second]" format ("second" is optional unless \p OnlyFirstRequired
1023/// is false).
1024///
1025/// \returns \p Default if attribute is not present.
1026///
1027/// \returns \p Default and emits error if one of the requested values cannot be
1028/// converted to integer, or \p OnlyFirstRequired is false and "second" value is
1029/// not present.
1030std::pair<unsigned, unsigned>
1031getIntegerPairAttribute(const Function &F, StringRef Name,
1032 std::pair<unsigned, unsigned> Default,
1033 bool OnlyFirstRequired = false);
1034
1035/// \returns A pair of integer values requested using \p F's \p Name attribute
1036/// in "first[,second]" format ("second" is optional unless \p OnlyFirstRequired
1037/// is false).
1038///
1039/// \returns \p std::nullopt if attribute is not present.
1040///
1041/// \returns \p std::nullopt and emits error if one of the requested values
1042/// cannot be converted to integer, or \p OnlyFirstRequired is false and
1043/// "second" value is not present.
1044std::optional<std::pair<unsigned, std::optional<unsigned>>>
1045getIntegerPairAttribute(const Function &F, StringRef Name,
1046 bool OnlyFirstRequired = false);
1047
1048/// \returns Generate a vector of integer values requested using \p F's \p Name
1049/// attribute.
1050/// \returns A vector of size \p Size, with all elements set to \p DefaultVal,
1051/// if any error occurs. The corresponding error will also be emitted.
1052SmallVector<unsigned> getIntegerVecAttribute(const Function &F, StringRef Name,
1053 unsigned Size,
1054 unsigned DefaultVal);
1055/// Similar to the function above, but returns std::nullopt if any error occurs.
1056std::optional<SmallVector<unsigned>>
1057getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size);
1058
1059/// \returns The maximum number of workgroups for the function.
1060SmallVector<unsigned> getMaxNumWorkGroups(const Function &F);
1061
1062inline bool isTgSplitEnabled(const Function &F) {
1063 return F.hasFnAttribute(Kind: "amdgpu-tg-split");
1064}
1065
1066/// Checks if \p Val is inside \p MD, a !range-like metadata.
1067bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val);
1068
1069// The following methods are only meaningful on targets that support
1070// S_WAITCNT.
1071
1072/// \returns Vmcnt bit mask for given isa \p Version.
1073unsigned getVmcntBitMask(const IsaVersion &Version);
1074
1075/// \returns Expcnt bit mask for given isa \p Version.
1076unsigned getExpcntBitMask(const IsaVersion &Version);
1077
1078/// \returns Lgkmcnt bit mask for given isa \p Version.
1079unsigned getLgkmcntBitMask(const IsaVersion &Version);
1080
1081/// \returns Waitcnt bit mask for given isa \p Version.
1082unsigned getWaitcntBitMask(const IsaVersion &Version);
1083
1084/// \returns Decoded Vmcnt from given \p Waitcnt for given isa \p Version.
1085unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt);
1086
1087/// \returns Decoded Expcnt from given \p Waitcnt for given isa \p Version.
1088unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt);
1089
1090/// \returns Decoded Lgkmcnt from given \p Waitcnt for given isa \p Version.
1091unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt);
1092
1093/// \returns Decoded Loadcnt from given \p Waitcnt for given isa \p Version.
1094unsigned decodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt);
1095
1096/// \returns Decoded Storecnt from given \p Waitcnt for given isa \p Version.
1097unsigned decodeStorecnt(const IsaVersion &Version, unsigned Waitcnt);
1098
1099/// \returns Decoded Dscnt from given \p Waitcnt for given isa \p Version.
1100unsigned decodeDscnt(const IsaVersion &Version, unsigned Waitcnt);
1101
1102/// Decodes Vmcnt, Expcnt and Lgkmcnt from given \p Waitcnt for given isa
1103/// \p Version, and writes decoded values into \p Vmcnt, \p Expcnt and
1104/// \p Lgkmcnt respectively. Should not be used on gfx12+, the instruction
1105/// which needs it is deprecated
1106///
1107/// \details \p Vmcnt, \p Expcnt and \p Lgkmcnt are decoded as follows:
1108/// \p Vmcnt = \p Waitcnt[3:0] (pre-gfx9)
1109/// \p Vmcnt = \p Waitcnt[15:14,3:0] (gfx9,10)
1110/// \p Vmcnt = \p Waitcnt[15:10] (gfx11)
1111/// \p Expcnt = \p Waitcnt[6:4] (pre-gfx11)
1112/// \p Expcnt = \p Waitcnt[2:0] (gfx11)
1113/// \p Lgkmcnt = \p Waitcnt[11:8] (pre-gfx10)
1114/// \p Lgkmcnt = \p Waitcnt[13:8] (gfx10)
1115/// \p Lgkmcnt = \p Waitcnt[9:4] (gfx11)
1116///
1117void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt,
1118 unsigned &Expcnt, unsigned &Lgkmcnt);
1119
1120/// \returns \p Waitcnt with encoded \p Vmcnt for given isa \p Version.
1121unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
1122 unsigned Vmcnt);
1123
1124/// \returns \p Waitcnt with encoded \p Expcnt for given isa \p Version.
1125unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
1126 unsigned Expcnt);
1127
1128/// \returns \p Waitcnt with encoded \p Lgkmcnt for given isa \p Version.
1129unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
1130 unsigned Lgkmcnt);
1131
1132/// Encodes \p Vmcnt, \p Expcnt and \p Lgkmcnt into Waitcnt for given isa
1133/// \p Version. Should not be used on gfx12+, the instruction which needs
1134/// it is deprecated
1135///
1136/// \details \p Vmcnt, \p Expcnt and \p Lgkmcnt are encoded as follows:
1137/// Waitcnt[2:0] = \p Expcnt (gfx11+)
1138/// Waitcnt[3:0] = \p Vmcnt (pre-gfx9)
1139/// Waitcnt[3:0] = \p Vmcnt[3:0] (gfx9,10)
1140/// Waitcnt[6:4] = \p Expcnt (pre-gfx11)
1141/// Waitcnt[9:4] = \p Lgkmcnt (gfx11)
1142/// Waitcnt[11:8] = \p Lgkmcnt (pre-gfx10)
1143/// Waitcnt[13:8] = \p Lgkmcnt (gfx10)
1144/// Waitcnt[15:10] = \p Vmcnt (gfx11)
1145/// Waitcnt[15:14] = \p Vmcnt[5:4] (gfx9,10)
1146///
1147/// \returns Waitcnt with encoded \p Vmcnt, \p Expcnt and \p Lgkmcnt for given
1148/// isa \p Version.
1149///
1150unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt,
1151 unsigned Expcnt, unsigned Lgkmcnt);
1152
1153/// \returns Waitcnt with encoded \p Loadcnt and \p Dscnt for given isa \p
1154/// Version.
1155unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt,
1156 unsigned Dscnt);
1157
1158/// \returns Waitcnt with encoded \p Storecnt and \p Dscnt for given isa \p
1159/// Version.
1160unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt,
1161 unsigned Dscnt);
1162
1163// The following methods are only meaningful on targets that support
1164// S_WAIT_*CNT, introduced with gfx12.
1165
1166/// \returns Loadcnt bit mask for given isa \p Version.
1167/// Returns 0 for versions that do not support LOADcnt
1168unsigned getLoadcntBitMask(const IsaVersion &Version);
1169
1170/// \returns Samplecnt bit mask for given isa \p Version.
1171/// Returns 0 for versions that do not support SAMPLEcnt
1172unsigned getSamplecntBitMask(const IsaVersion &Version);
1173
1174/// \returns Bvhcnt bit mask for given isa \p Version.
1175/// Returns 0 for versions that do not support BVHcnt
1176unsigned getBvhcntBitMask(const IsaVersion &Version);
1177
1178/// \returns Asynccnt bit mask for given isa \p Version.
1179/// Returns 0 for versions that do not support Asynccnt
1180unsigned getAsynccntBitMask(const IsaVersion &Version);
1181
1182/// \returns Dscnt bit mask for given isa \p Version.
1183/// Returns 0 for versions that do not support DScnt
1184unsigned getDscntBitMask(const IsaVersion &Version);
1185
1186/// \returns Dscnt bit mask for given isa \p Version.
1187/// Returns 0 for versions that do not support KMcnt
1188unsigned getKmcntBitMask(const IsaVersion &Version);
1189
1190/// \returns Xcnt bit mask for given isa \p Version.
1191/// Returns 0 for versions that do not support Xcnt.
1192unsigned getXcntBitMask(const IsaVersion &Version);
1193
1194/// \return STOREcnt or VScnt bit mask for given isa \p Version.
1195/// returns 0 for versions that do not support STOREcnt or VScnt.
1196/// STOREcnt and VScnt are the same counter, the name used
1197/// depends on the ISA version.
1198unsigned getStorecntBitMask(const IsaVersion &Version);
1199
1200/// \returns barrier member count bit mask in M0 for given isa \p Version.
1201unsigned getBarrierMemberCountBitMask(const IsaVersion &Version);
1202
1203namespace Hwreg {
1204
1205using HwregId = EncodingField<5, 0>;
1206using HwregOffset = EncodingField<10, 6>;
1207
1208struct HwregSize : EncodingField<15, 11, 32> {
1209 using EncodingField::EncodingField;
1210 constexpr uint64_t encode() const { return Value - 1; }
1211 static ValueType decode(uint64_t Encoded) {
1212 return static_cast<ValueType>(Encoded + 1);
1213 }
1214};
1215
1216using HwregEncoding = EncodingFields<HwregId, HwregOffset, HwregSize>;
1217
1218} // namespace Hwreg
1219
1220namespace DepCtr {
1221
1222int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI);
1223int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask,
1224 const MCSubtargetInfo &STI);
1225bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal,
1226 const MCSubtargetInfo &STI);
1227bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val,
1228 bool &IsDefault, const MCSubtargetInfo &STI);
1229
1230/// \returns Maximum VaVdst value that can be encoded.
1231unsigned getVaVdstBitMask();
1232
1233/// \returns Maximum VaSdst value that can be encoded.
1234unsigned getVaSdstBitMask();
1235
1236/// \returns Maximum VaSsrc value that can be encoded.
1237unsigned getVaSsrcBitMask();
1238
1239/// \returns Maximum HoldCnt value that can be encoded.
1240unsigned getHoldCntBitMask(const IsaVersion &Version);
1241
1242/// \returns Maximum VmVsrc value that can be encoded.
1243unsigned getVmVsrcBitMask();
1244
1245/// \returns Maximum VaVcc value that can be encoded.
1246unsigned getVaVccBitMask();
1247
1248/// \returns Maximum SaSdst value that can be encoded.
1249unsigned getSaSdstBitMask();
1250
1251/// \returns Decoded VaVdst from given immediate \p Encoded.
1252unsigned decodeFieldVaVdst(unsigned Encoded);
1253
1254/// \returns Decoded VmVsrc from given immediate \p Encoded.
1255unsigned decodeFieldVmVsrc(unsigned Encoded);
1256
1257/// \returns Decoded SaSdst from given immediate \p Encoded.
1258unsigned decodeFieldSaSdst(unsigned Encoded);
1259
1260/// \returns Decoded VaSdst from given immediate \p Encoded.
1261unsigned decodeFieldVaSdst(unsigned Encoded);
1262
1263/// \returns Decoded VaVcc from given immediate \p Encoded.
1264unsigned decodeFieldVaVcc(unsigned Encoded);
1265
1266/// \returns Decoded SaSrc from given immediate \p Encoded.
1267unsigned decodeFieldVaSsrc(unsigned Encoded);
1268
1269/// \returns Decoded HoldCnt from given immediate \p Encoded.
1270unsigned decodeFieldHoldCnt(unsigned Encoded, const IsaVersion &Version);
1271
1272/// \returns \p VmVsrc as an encoded Depctr immediate.
1273unsigned encodeFieldVmVsrc(unsigned VmVsrc, const MCSubtargetInfo &STI);
1274
1275/// \returns \p Encoded combined with encoded \p VmVsrc.
1276unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc);
1277
1278/// \returns \p VaVdst as an encoded Depctr immediate.
1279unsigned encodeFieldVaVdst(unsigned VaVdst, const MCSubtargetInfo &STI);
1280
1281/// \returns \p Encoded combined with encoded \p VaVdst.
1282unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst);
1283
1284/// \returns \p SaSdst as an encoded Depctr immediate.
1285unsigned encodeFieldSaSdst(unsigned SaSdst, const MCSubtargetInfo &STI);
1286
1287/// \returns \p Encoded combined with encoded \p SaSdst.
1288unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst);
1289
1290/// \returns \p VaSdst as an encoded Depctr immediate.
1291unsigned encodeFieldVaSdst(unsigned VaSdst, const MCSubtargetInfo &STI);
1292
1293/// \returns \p Encoded combined with encoded \p VaSdst.
1294unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst);
1295
1296/// \returns \p VaVcc as an encoded Depctr immediate.
1297unsigned encodeFieldVaVcc(unsigned VaVcc, const MCSubtargetInfo &STI);
1298
1299/// \returns \p Encoded combined with encoded \p VaVcc.
1300unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc);
1301
1302/// \returns \p HoldCnt as an encoded Depctr immediate.
1303unsigned encodeFieldHoldCnt(unsigned HoldCnt, const MCSubtargetInfo &STI);
1304
1305/// \returns \p Encoded combined with encoded \p HoldCnt.
1306unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt,
1307 const IsaVersion &Version);
1308
1309/// \returns \p VaSsrc as an encoded Depctr immediate.
1310unsigned encodeFieldVaSsrc(unsigned VaSsrc, const MCSubtargetInfo &STI);
1311
1312/// \returns \p Encoded combined with encoded \p VaSsrc.
1313unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc);
1314
1315} // namespace DepCtr
1316
1317namespace Exp {
1318
1319bool getTgtName(unsigned Id, StringRef &Name, int &Index);
1320
1321LLVM_READONLY
1322unsigned getTgtId(const StringRef Name);
1323
1324LLVM_READNONE
1325bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI);
1326
1327} // namespace Exp
1328
1329namespace MTBUFFormat {
1330
1331LLVM_READNONE
1332int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt);
1333
1334void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt);
1335
1336int64_t getDfmt(const StringRef Name);
1337
1338StringRef getDfmtName(unsigned Id);
1339
1340int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI);
1341
1342StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI);
1343
1344bool isValidDfmtNfmt(unsigned Val, const MCSubtargetInfo &STI);
1345
1346bool isValidNfmt(unsigned Val, const MCSubtargetInfo &STI);
1347
1348int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI);
1349
1350StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI);
1351
1352bool isValidUnifiedFormat(unsigned Val, const MCSubtargetInfo &STI);
1353
1354int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt,
1355 const MCSubtargetInfo &STI);
1356
1357bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI);
1358
1359unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI);
1360
1361} // namespace MTBUFFormat
1362
1363namespace SendMsg {
1364
1365LLVM_READNONE
1366bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI);
1367
1368LLVM_READNONE
1369bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
1370 bool Strict = true);
1371
1372LLVM_READNONE
1373bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
1374 const MCSubtargetInfo &STI, bool Strict = true);
1375
1376LLVM_READNONE
1377bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI);
1378
1379LLVM_READNONE
1380bool msgSupportsStream(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI);
1381
1382void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,
1383 uint16_t &StreamId, const MCSubtargetInfo &STI);
1384
1385LLVM_READNONE
1386uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId);
1387
1388/// Returns true if the message does not use the m0 operand.
1389bool msgDoesNotUseM0(int64_t MsgId, const MCSubtargetInfo &STI);
1390
1391} // namespace SendMsg
1392
1393unsigned getInitialPSInputAddr(const Function &F);
1394
1395bool getHasColorExport(const Function &F);
1396
1397bool getHasDepthExport(const Function &F);
1398
1399// Returns the value of the "amdgpu-dynamic-vgpr-block-size" attribute, or 0 if
1400// the attribute is missing or its value is invalid.
1401unsigned getDynamicVGPRBlockSize(const Function &F);
1402
1403LLVM_READNONE
1404constexpr bool isShader(CallingConv::ID CC) {
1405 switch (CC) {
1406 case CallingConv::AMDGPU_VS:
1407 case CallingConv::AMDGPU_LS:
1408 case CallingConv::AMDGPU_HS:
1409 case CallingConv::AMDGPU_ES:
1410 case CallingConv::AMDGPU_GS:
1411 case CallingConv::AMDGPU_PS:
1412 case CallingConv::AMDGPU_CS_Chain:
1413 case CallingConv::AMDGPU_CS_ChainPreserve:
1414 case CallingConv::AMDGPU_CS:
1415 return true;
1416 default:
1417 return false;
1418 }
1419}
1420
1421LLVM_READNONE
1422constexpr bool isGraphics(CallingConv::ID CC) {
1423 return isShader(CC) || CC == CallingConv::AMDGPU_Gfx ||
1424 CC == CallingConv::AMDGPU_Gfx_WholeWave;
1425}
1426
1427LLVM_READNONE
1428constexpr bool isCompute(CallingConv::ID CC) {
1429 return !isGraphics(CC) || CC == CallingConv::AMDGPU_CS;
1430}
1431
1432LLVM_READNONE
1433constexpr bool isEntryFunctionCC(CallingConv::ID CC) {
1434 switch (CC) {
1435 case CallingConv::AMDGPU_KERNEL:
1436 case CallingConv::SPIR_KERNEL:
1437 case CallingConv::AMDGPU_VS:
1438 case CallingConv::AMDGPU_GS:
1439 case CallingConv::AMDGPU_PS:
1440 case CallingConv::AMDGPU_CS:
1441 case CallingConv::AMDGPU_ES:
1442 case CallingConv::AMDGPU_HS:
1443 case CallingConv::AMDGPU_LS:
1444 return true;
1445 default:
1446 return false;
1447 }
1448}
1449
1450LLVM_READNONE
1451constexpr bool isChainCC(CallingConv::ID CC) {
1452 switch (CC) {
1453 case CallingConv::AMDGPU_CS_Chain:
1454 case CallingConv::AMDGPU_CS_ChainPreserve:
1455 return true;
1456 default:
1457 return false;
1458 }
1459}
1460
1461// These functions are considered entrypoints into the current module, i.e. they
1462// are allowed to be called from outside the current module. This is different
1463// from isEntryFunctionCC, which is only true for functions that are entered by
1464// the hardware. Module entry points include all entry functions but also
1465// include functions that can be called from other functions inside or outside
1466// the current module. Module entry functions are allowed to allocate LDS.
1467//
1468// AMDGPU_CS_Chain is intended for externally callable chain functions, so it is
1469// treated as a module entrypoint. AMDGPU_CS_ChainPreserve is used for internal
1470// helper functions (e.g. retry helpers), so it is not a module entrypoint.
1471LLVM_READNONE
1472constexpr bool isModuleEntryFunctionCC(CallingConv::ID CC) {
1473 switch (CC) {
1474 case CallingConv::AMDGPU_Gfx:
1475 case CallingConv::AMDGPU_CS_Chain:
1476 return true;
1477 default:
1478 return isEntryFunctionCC(CC);
1479 }
1480}
1481
1482LLVM_READNONE
1483constexpr inline bool isKernel(CallingConv::ID CC) {
1484 switch (CC) {
1485 case CallingConv::AMDGPU_KERNEL:
1486 case CallingConv::SPIR_KERNEL:
1487 return true;
1488 default:
1489 return false;
1490 }
1491}
1492
1493inline bool isKernel(const Function &F) { return isKernel(CC: F.getCallingConv()); }
1494
1495LLVM_READNONE
1496constexpr bool canGuaranteeTCO(CallingConv::ID CC) {
1497 return CC == CallingConv::Fast;
1498}
1499
1500/// Return true if we might ever do TCO for calls with this calling convention.
1501LLVM_READNONE
1502constexpr bool mayTailCallThisCC(CallingConv::ID CC) {
1503 switch (CC) {
1504 case CallingConv::C:
1505 case CallingConv::AMDGPU_Gfx:
1506 case CallingConv::AMDGPU_Gfx_WholeWave:
1507 return true;
1508 default:
1509 return canGuaranteeTCO(CC);
1510 }
1511}
1512
1513bool hasMIMG_R128(const MCSubtargetInfo &STI);
1514bool hasA16(const MCSubtargetInfo &STI);
1515bool hasG16(const MCSubtargetInfo &STI);
1516bool hasPackedD16(const MCSubtargetInfo &STI);
1517bool hasGDS(const MCSubtargetInfo &STI);
1518unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler = false);
1519unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI);
1520
1521bool isSI(const MCSubtargetInfo &STI);
1522bool isCI(const MCSubtargetInfo &STI);
1523bool isVI(const MCSubtargetInfo &STI);
1524bool isGFX9(const MCSubtargetInfo &STI);
1525bool isGFX9_GFX10(const MCSubtargetInfo &STI);
1526bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI);
1527bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI);
1528bool isGFX8Plus(const MCSubtargetInfo &STI);
1529bool isGFX9Plus(const MCSubtargetInfo &STI);
1530bool isNotGFX9Plus(const MCSubtargetInfo &STI);
1531bool isGFX10(const MCSubtargetInfo &STI);
1532bool isGFX10_GFX11(const MCSubtargetInfo &STI);
1533bool isGFX10Plus(const MCSubtargetInfo &STI);
1534bool isNotGFX10Plus(const MCSubtargetInfo &STI);
1535bool isGFX10Before1030(const MCSubtargetInfo &STI);
1536bool isGFX11(const MCSubtargetInfo &STI);
1537bool isGFX11Plus(const MCSubtargetInfo &STI);
1538bool isGFX12(const MCSubtargetInfo &STI);
1539bool isGFX12Plus(const MCSubtargetInfo &STI);
1540bool isGFX1250(const MCSubtargetInfo &STI);
1541bool isGFX1250Plus(const MCSubtargetInfo &STI);
1542bool isGFX13(const MCSubtargetInfo &STI);
1543bool isGFX13Plus(const MCSubtargetInfo &STI);
1544
1545/// \returns true if a work-group's waves run on all four SIMD32s (one
1546/// contiguous LDS) and not just on two.
1547bool isFullSIMDMode(const MCSubtargetInfo &STI);
1548
1549bool supportsWGP(const MCSubtargetInfo &STI);
1550bool isNotGFX12Plus(const MCSubtargetInfo &STI);
1551bool isNotGFX11Plus(const MCSubtargetInfo &STI);
1552bool isGCN3Encoding(const MCSubtargetInfo &STI);
1553bool isGFX10_BEncoding(const MCSubtargetInfo &STI);
1554bool hasGFX10_3Insts(const MCSubtargetInfo &STI);
1555bool isGFX10_3_GFX11(const MCSubtargetInfo &STI);
1556bool isGFX90A(const MCSubtargetInfo &STI);
1557bool isGFX940(const MCSubtargetInfo &STI);
1558bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI);
1559bool hasMAIInsts(const MCSubtargetInfo &STI);
1560bool hasPopsExitingWaveID(const MCSubtargetInfo &STI);
1561
1562/// \returns true if the src_private_base and src_private_limit aperture
1563/// registers are available on \p STI. Targets with globally addressable
1564/// scratch have no private aperture and expose src_flat_scratch_base instead.
1565bool hasPrivateApertureRegs(const MCSubtargetInfo &STI);
1566
1567bool hasVOPD(const MCSubtargetInfo &STI);
1568bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI);
1569
1570uint32_t getTotalNumVGPRs(bool has90AInsts, uint32_t ArgNumAGPR,
1571 uint32_t ArgNumVGPR);
1572unsigned hasKernargPreload(const MCSubtargetInfo &STI);
1573bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST);
1574
1575/// Is Reg - scalar register
1576bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI);
1577
1578/// \returns true if \p Reg is an indexed-resource index register, i.e. either a
1579/// 32-bit SGPR (uniform-indexed form) or a Lo256 VGPR (per-lane indexed form).
1580bool isRsrcIndexReg(MCRegister Reg, const MCRegisterInfo &MRI);
1581
1582/// \returns if \p Reg occupies the high 16-bits of a 32-bit register.
1583bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI);
1584
1585/// If \p Reg is a pseudo reg, return the correct hardware register given
1586/// \p STI otherwise return \p Reg.
1587MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI);
1588
1589/// Convert hardware register \p Reg to a pseudo register
1590LLVM_READNONE
1591MCRegister mc2PseudoReg(MCRegister Reg);
1592
1593LLVM_READNONE
1594bool isInlineValue(MCRegister Reg);
1595
1596/// Is this an AMDGPU specific source operand? These include registers,
1597/// inline constants, literals and mandatory literals (KImm).
1598constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo) {
1599 return OpInfo.OperandType >= AMDGPU::OPERAND_SRC_FIRST &&
1600 OpInfo.OperandType <= AMDGPU::OPERAND_SRC_LAST;
1601}
1602
1603inline bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1604 return isSISrcOperand(OpInfo: Desc.operands()[OpNo]);
1605}
1606
1607/// Is this a scalar (i.e. not packed) bf16 source operand?
1608constexpr bool isBF16SrcOperand(const MCOperandInfo &OpInfo) {
1609 return OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_BF16 ||
1610 OpInfo.OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16;
1611}
1612
1613/// Is this a KImm operand?
1614bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo);
1615
1616/// Is this floating-point operand?
1617bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo);
1618
1619/// Get the size in bits of a register from the register class \p RC.
1620unsigned getRegBitWidth(unsigned RCID);
1621
1622/// Get the size in bits of a register from the register class \p RC.
1623unsigned getRegBitWidth(const MCRegisterClass &RC);
1624
1625LLVM_READNONE
1626inline unsigned getOperandSize(const MCOperandInfo &OpInfo) {
1627 switch (OpInfo.OperandType) {
1628 case AMDGPU::OPERAND_REG_IMM_INT32:
1629 case AMDGPU::OPERAND_REG_IMM_FP32:
1630 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1631 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1632 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1633 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1634 case AMDGPU::OPERAND_REG_IMM_V2INT32:
1635 case AMDGPU::OPERAND_REG_IMM_V2FP32:
1636 case AMDGPU::OPERAND_KIMM32:
1637 case AMDGPU::OPERAND_KIMM16: // mandatory literal is always size 4
1638 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
1639 return 4;
1640
1641 case AMDGPU::OPERAND_REG_IMM_INT64:
1642 case AMDGPU::OPERAND_REG_IMM_FP64:
1643 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1644 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1645 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
1646 case AMDGPU::OPERAND_REG_IMM_V2FP64:
1647 case AMDGPU::OPERAND_REG_IMM_V2INT64:
1648 case AMDGPU::OPERAND_KIMM64:
1649 return 8;
1650
1651 case AMDGPU::OPERAND_REG_IMM_INT16:
1652 case AMDGPU::OPERAND_REG_IMM_BF16:
1653 case AMDGPU::OPERAND_REG_IMM_FP16:
1654 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
1655 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1656 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
1657 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1658 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1659 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
1660 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1661 case AMDGPU::OPERAND_REG_IMM_V2INT16:
1662 case AMDGPU::OPERAND_REG_IMM_V2BF16:
1663 case AMDGPU::OPERAND_REG_IMM_V2FP16:
1664 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
1665 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
1666 return 2;
1667
1668 default:
1669 llvm_unreachable("unhandled operand type");
1670 }
1671}
1672
1673LLVM_READNONE
1674inline unsigned getOperandSize(const MCInstrDesc &Desc, unsigned OpNo) {
1675 return getOperandSize(OpInfo: Desc.operands()[OpNo]);
1676}
1677
1678/// Is this literal inlinable, and not one of the values intended for floating
1679/// point values.
1680LLVM_READNONE
1681inline bool isInlinableIntLiteral(int64_t Literal) {
1682 return Literal >= -16 && Literal <= 64;
1683}
1684
1685/// Is this literal inlinable
1686LLVM_READNONE
1687bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi);
1688
1689LLVM_READNONE
1690bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi);
1691
1692bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi,
1693 bool *IsBF16 = nullptr);
1694
1695LLVM_READNONE
1696bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi);
1697
1698LLVM_READNONE
1699bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi);
1700
1701LLVM_READNONE
1702std::optional<unsigned> getInlineEncodingV2I16(uint32_t Literal);
1703
1704LLVM_READNONE
1705std::optional<unsigned> getInlineEncodingV2BF16(uint32_t Literal);
1706
1707LLVM_READNONE
1708std::optional<unsigned> getInlineEncodingV2F16(uint32_t Literal);
1709
1710LLVM_READNONE
1711std::optional<unsigned> getPKFMACF16InlineEncoding(uint32_t Literal,
1712 bool IsGFX11Plus);
1713
1714LLVM_READNONE
1715bool isInlinableLiteralV216(uint32_t Literal, uint8_t OpType);
1716
1717LLVM_READNONE
1718bool isInlinableLiteralV2I16(uint32_t Literal);
1719
1720LLVM_READNONE
1721bool isInlinableLiteralV2BF16(uint32_t Literal);
1722
1723LLVM_READNONE
1724bool isInlinableLiteralV2F16(uint32_t Literal);
1725
1726LLVM_READNONE
1727bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus);
1728
1729LLVM_READNONE
1730bool isValid32BitLiteral(uint64_t Val, bool IsFP64);
1731
1732LLVM_READNONE
1733int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit);
1734
1735bool isArgPassedInSGPR(const Argument *Arg);
1736
1737bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo);
1738
1739/// The opcode is a packed fp32 instruction which only reads low 32 bits of
1740/// a scalar operand and propagates it to high channel.
1741LLVM_READONLY bool isPackedSingleSGPRFP32Inst(unsigned Opc);
1742
1743/// The opcode is a packed 64-bit instruction which only reads low 64 bits of
1744/// a scalar operand and propagates it to high channel.
1745LLVM_READONLY bool isPackedSingleSGPR64BitInst(unsigned Opc);
1746
1747/// Packed instructions that read a single SGPR for SGPR operands, except for
1748/// 64-bit elements which read two SGPRs.
1749LLVM_READONLY bool isSingleSGPRReadInst(unsigned Opc);
1750
1751LLVM_READONLY
1752bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
1753 int64_t EncodedOffset);
1754
1755LLVM_READONLY
1756bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
1757 int64_t EncodedOffset, bool IsBuffer);
1758
1759/// Convert \p ByteOffset to dwords if the subtarget uses dword SMRD immediate
1760/// offsets.
1761uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, uint64_t ByteOffset);
1762
1763/// \returns The encoding that will be used for \p ByteOffset in the
1764/// SMRD offset field, or std::nullopt if it won't fit. On GFX9 and GFX10
1765/// S_LOAD instructions have a signed offset, on other subtargets it is
1766/// unsigned. S_BUFFER has an unsigned offset for all subtargets.
1767std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
1768 int64_t ByteOffset, bool IsBuffer,
1769 bool HasSOffset = false);
1770
1771/// \return The encoding that can be used for a 32-bit literal offset in an SMRD
1772/// instruction. This is only useful on CI.s
1773std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
1774 int64_t ByteOffset);
1775
1776/// For pre-GFX12 FLAT instructions the offset must be positive;
1777/// MSB is ignored and forced to zero.
1778///
1779/// \return The number of bits available for the signed offset field in flat
1780/// instructions. Note that some forms of the instruction disallow negative
1781/// offsets.
1782unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST);
1783
1784LLVM_READNONE
1785inline bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC) {
1786 if (isGFX12(STI: ST))
1787 return DC >= DPP::ROW_SHARE_FIRST && DC <= DPP::ROW_SHARE_LAST;
1788 if (isGFX90A(STI: ST))
1789 return DC >= DPP::ROW_NEWBCAST_FIRST && DC <= DPP::ROW_NEWBCAST_LAST;
1790 return false;
1791}
1792
1793/// \returns true if an instruction is a DP ALU DPP without any 64-bit operands.
1794bool isDPALU_DPP32BitOpc(unsigned Opc);
1795
1796/// \returns true if an instruction is a DP ALU DPP.
1797bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
1798 const MCSubtargetInfo &ST);
1799
1800/// \returns true if the intrinsic is divergent
1801bool isIntrinsicSourceOfDivergence(unsigned IntrID);
1802
1803/// \returns true if the intrinsic is uniform
1804bool isIntrinsicAlwaysUniform(unsigned IntrID);
1805
1806/// Return true if we can easily prove that \p V is uniform at \p CtxI.
1807bool isTriviallyUniform(const Value *V, const Instruction *CtxI);
1808
1809/// Return true if we can easily prove that use \p U is uniform.
1810bool isTriviallyUniform(const Use &U);
1811
1812/// \returns true if the intrinsic executes an LDSDMA operation
1813bool isLDSDMAIntrinsic(unsigned IntrID);
1814
1815/// \returns the index of the f32 scale argument of a V_CVT_SCALEF32_*
1816/// conversion to a narrow format, or std::nullopt for any other intrinsic.
1817/// These instructions only read the bits of the scale given by
1818/// getExponentOnlyScaleDemandedBits().
1819std::optional<unsigned> getExponentOnlyScaleArgIdx(unsigned IntrID);
1820
1821/// \returns the bits of an exponent-only scale argument that are read: the
1822/// exponent field of the f32 value.
1823inline APInt getExponentOnlyScaleDemandedBits() {
1824 return APInt::getBitsSet(numBits: 32, loBit: 23, hiBit: 31);
1825}
1826
1827/// \returns a register class for the physical register \p Reg if it is a VGPR
1828/// or nullptr otherwise.
1829const MCRegisterClass *getVGPRPhysRegClass(MCRegister Reg,
1830 const MCRegisterInfo &MRI);
1831
1832/// \returns the MODE bits which have to be set by the S_SET_VGPR_MSB for the
1833/// physical register \p Reg.
1834unsigned getVGPREncodingMSBs(MCRegister Reg, const MCRegisterInfo &MRI);
1835
1836/// If \p Reg is a low VGPR return a corresponding high VGPR with \p MSBs set.
1837MCRegister getVGPRWithMSBs(MCRegister Reg, unsigned MSBs,
1838 const MCRegisterInfo &MRI);
1839
1840/// \returns VGPR MSBs encoded in a S_SETREG_IMM32_B32 \p MI if it sets
1841/// it. If \p HasSetregVGPRMSBFixup is true then size of the ID_MODE mask is
1842/// ignored.
1843std::optional<unsigned> convertSetRegImmToVgprMSBs(const MachineInstr &MI,
1844 bool HasSetregVGPRMSBFixup);
1845
1846/// \returns VGPR MSBs encoded in a S_SETREG_IMM32_B32 \p MI if it sets
1847/// it. If \p HasSetregVGPRMSBFixup is true then size of the ID_MODE mask is
1848/// ignored.
1849std::optional<unsigned> convertSetRegImmToVgprMSBs(const MCInst &MI,
1850 bool HasSetregVGPRMSBFixup);
1851
1852// Returns a table for the opcode with a given \p Desc to map the VGPR MSB
1853// set by the S_SET_VGPR_MSB to one of 4 sources. In case of VOPD returns 2
1854// maps, one for X and one for Y component.
1855std::pair<const AMDGPU::OpName *, const AMDGPU::OpName *>
1856getVGPRLoweringOperandTables(const MCInstrDesc &Desc);
1857
1858/// \returns true if a memory instruction supports scale_offset modifier.
1859bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode);
1860
1861class ClusterDimsAttr {
1862public:
1863 enum class Kind { Unknown, NoCluster, VariableDims, FixedDims };
1864
1865 ClusterDimsAttr() = default;
1866
1867 Kind getKind() const { return AttrKind; }
1868
1869 bool isUnknown() const { return getKind() == Kind::Unknown; }
1870
1871 bool isNoCluster() const { return getKind() == Kind::NoCluster; }
1872
1873 bool isFixedDims() const { return getKind() == Kind::FixedDims; }
1874
1875 bool isVariableDims() const { return getKind() == Kind::VariableDims; }
1876
1877 void setUnknown() { *this = ClusterDimsAttr(Kind::Unknown); }
1878
1879 void setVariableDims() { *this = ClusterDimsAttr(Kind::VariableDims); }
1880
1881 /// \returns the dims stored. Note that this function can only be called if
1882 /// the kind is \p Fixed.
1883 const std::array<unsigned, 3> &getDims() const;
1884
1885 bool operator==(const ClusterDimsAttr &RHS) const {
1886 return AttrKind == RHS.AttrKind && Dims == RHS.Dims;
1887 }
1888
1889 std::string to_string() const;
1890
1891 static ClusterDimsAttr get(const Function &F);
1892
1893private:
1894 enum Encoding { EncoNoCluster = 0, EncoVariableDims = 1024 };
1895
1896 ClusterDimsAttr(Kind AttrKind) : AttrKind(AttrKind) {}
1897
1898 std::array<unsigned, 3> Dims = {0, 0, 0};
1899
1900 Kind AttrKind = Kind::Unknown;
1901};
1902
1903/// Evaluate the constant-folded result of v_rcp for \p Val, accounting for
1904/// the hardware's denormal flushing on f32/f64 and its approximate rounding.
1905/// Returns std::nullopt if the hardware result is not guaranteed to match the
1906/// exact reciprocal.
1907std::optional<APFloat> evaluateRcp(const APFloat &Val);
1908
1909} // namespace AMDGPU
1910
1911raw_ostream &operator<<(raw_ostream &OS, const AMDGPU::TargetIDSetting S);
1912
1913} // end namespace llvm
1914
1915#endif // LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUBASEINFO_H
1916