1//===- AMDGPUBaseInfo.h - Top level definitions for AMDGPU ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUBASEINFO_H
10#define LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUBASEINFO_H
11
12#include "AMDGPUSubtarget.h"
13#include "SIDefines.h"
14#include "llvm/ADT/APFloat.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/ADT/StringTable.h"
17#include "llvm/IR/CallingConv.h"
18#include "llvm/IR/InstrTypes.h"
19#include "llvm/IR/Module.h"
20#include "llvm/Support/Alignment.h"
21#include "llvm/TargetParser/AMDGPUTargetParser.h"
22#include <array>
23#include <functional>
24#include <optional>
25#include <utility>
26
27// Pull in OpName enum definition and getNamedOperandIdx() declaration.
28#define GET_INSTRINFO_OPERAND_ENUM
29#include "AMDGPUGenInstrInfo.inc"
30
31struct amd_kernel_code_t;
32
33namespace llvm {
34
35struct Align;
36class Argument;
37class Function;
38class GlobalValue;
39class MachineInstr;
40class MCInstrInfo;
41class MCRegisterClass;
42class MCRegisterInfo;
43class MCSubtargetInfo;
44class MDNode;
45class StringRef;
46class Triple;
47class raw_ostream;
48
49namespace AMDGPU {
50
51struct AMDGPUMCKernelCodeT;
52struct IsaVersion;
53
54/// Generic target versions emitted by this version of LLVM.
55///
56/// These numbers are incremented every time a codegen breaking change occurs
57/// within a generic family.
58namespace GenericVersion {
59static constexpr unsigned GFX9 = 1;
60static constexpr unsigned GFX9_4 = 1;
61static constexpr unsigned GFX10_1 = 1;
62static constexpr unsigned GFX10_3 = 1;
63static constexpr unsigned GFX11 = 1;
64static constexpr unsigned GFX11_7 = 1;
65static constexpr unsigned GFX12 = 1;
66static constexpr unsigned GFX12_5 = 1;
67static constexpr unsigned GFX13 = 1;
68} // namespace GenericVersion
69
70enum { AMDHSA_COV4 = 4, AMDHSA_COV5 = 5, AMDHSA_COV6 = 6 };
71
72enum class FPType { None, FP4, FP8 };
73
74/// \returns True if \p STI is AMDHSA.
75bool isHsaAbi(const MCSubtargetInfo &STI);
76
77/// \returns Code object version from the IR module flag.
78unsigned getAMDHSACodeObjectVersion(const Module &M);
79
80/// \returns Code object version from ELF's e_ident[EI_ABIVERSION].
81unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion);
82
83/// \returns The default HSA code object version. This should only be used when
84/// we lack a more accurate CodeObjectVersion value (e.g. from the IR module
85/// flag or a .amdhsa_code_object_version directive)
86unsigned getDefaultAMDHSACodeObjectVersion();
87
88/// \returns ABIVersion suitable for use in ELF's e_ident[EI_ABIVERSION]. \param
89/// CodeObjectVersion is a value returned by getAMDHSACodeObjectVersion().
90uint8_t getELFABIVersion(const Triple &OS, unsigned CodeObjectVersion);
91
92/// \returns The offset of the multigrid_sync_arg argument from implicitarg_ptr
93unsigned getMultigridSyncArgImplicitArgPosition(unsigned COV);
94
95/// \returns The offset of the hostcall pointer argument from implicitarg_ptr
96unsigned getHostcallImplicitArgPosition(unsigned COV);
97
98unsigned getDefaultQueueImplicitArgPosition(unsigned COV);
99unsigned getCompletionActionImplicitArgPosition(unsigned COV);
100
101struct GcnBufferFormatInfo {
102 unsigned Format;
103 unsigned BitsPerComp;
104 unsigned NumComponents;
105 unsigned NumFormat;
106 unsigned DataFormat;
107};
108
109struct MAIInstInfo {
110 uint32_t Opcode;
111 bool is_dgemm;
112 bool is_gfx940_xdl;
113};
114
115struct MFMA_F8F6F4_Info {
116 unsigned Opcode;
117 unsigned F8F8Opcode;
118 uint8_t NumRegsSrcA;
119 uint8_t NumRegsSrcB;
120};
121
122struct CvtScaleF32_F32F16ToF8F4_Info {
123 unsigned Opcode;
124};
125
126/// Normalized WMMA or SWMMAC family used to select co-execution rules.
127enum class WMMAVariant {
128 Unknown = 0,
129 IU8_16x16x64,
130 F8F6F4_16x16x128,
131 F8F6F4_16x16x128_BothF4,
132 FP8BF8_16x16x64,
133 F16BF16_16x16x32,
134 FP8BF8_16x16x128,
135 F4_32x16x128,
136};
137
138struct True16D16Info {
139 unsigned T16Op;
140 unsigned HiOp;
141 unsigned LoOp;
142};
143
144struct WMMAInstInfo {
145 uint32_t Opcode;
146 bool is_wmma_xdl;
147 bool HasMatrixScale;
148 WMMAVariant CoExecVariant;
149};
150
151#define GET_MIMGBaseOpcode_DECL
152#define GET_MIMGDim_DECL
153#define GET_MIMGEncoding_DECL
154#define GET_MIMGLZMapping_DECL
155#define GET_MIMGMIPMapping_DECL
156#define GET_MIMGBiASMapping_DECL
157#define GET_MAIInstInfoTable_DECL
158#define GET_isMFMA_F8F6F4Table_DECL
159#define GET_isCvtScaleF32_F32F16ToF8F4Table_DECL
160#define GET_True16D16Table_DECL
161#define GET_WMMAInstInfoTable_DECL
162#include "AMDGPUGenSearchableTables.inc"
163
164using TargetIDSetting = AMDGPU::TargetIDSetting;
165using TargetID = AMDGPU::TargetID;
166
167/// Construct TargetID from MCSubtargetInfo. \p FeatureString is used to
168/// determine explicitly requested xnack/sramecc settings.
169TargetID createAMDGPUTargetID(const MCSubtargetInfo &STI,
170 StringRef FeatureString);
171
172namespace IsaInfo {
173
174enum {
175 FIXED_NUM_SGPRS_FOR_INIT_BUG = AMDGPU::FIXED_NUM_SGPRS_FOR_INIT_BUG,
176 TRAP_NUM_SGPRS = 16
177};
178
179/// Returns true if \p Lhs and \p Rhs are incompatible (both specific but
180/// different).
181inline bool targetIDSettingsConflict(TargetIDSetting Lhs, TargetIDSetting Rhs) {
182 return Lhs != TargetIDSetting::Any && Rhs != TargetIDSetting::Any &&
183 Lhs != Rhs;
184}
185
186/// \returns Instruction cache line size in bytes for given subtarget \p STI.
187unsigned getInstCacheLineSize(const MCSubtargetInfo &STI);
188
189/// \returns Wavefront size for given subtarget \p STI.
190unsigned getWavefrontSize(const MCSubtargetInfo &STI);
191
192/// \returns Local memory size in bytes for given subtarget \p STI.
193unsigned getLocalMemorySize(const MCSubtargetInfo &STI);
194
195/// \returns Maximum addressable local memory size in bytes for given subtarget
196/// \p STI.
197unsigned getAddressableLocalMemorySize(const MCSubtargetInfo &STI);
198
199/// \returns Maximum number of work groups per compute unit for given subtarget
200/// \p STI and limited by given \p FlatWorkGroupSize.
201unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo &STI,
202 unsigned FlatWorkGroupSize);
203
204/// \returns Number of waves per execution unit required to support the given \p
205/// FlatWorkGroupSize.
206unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo &STI,
207 unsigned FlatWorkGroupSize);
208
209/// \returns Number of waves per work group for given subtarget \p STI and
210/// \p FlatWorkGroupSize.
211unsigned getWavesPerWorkGroup(const MCSubtargetInfo &STI,
212 unsigned FlatWorkGroupSize);
213
214/// \returns SGPR encoding granularity for given subtarget \p STI.
215unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI);
216
217/// \returns Minimum number of SGPRs that meets the given number of waves per
218/// execution unit requirement for given subtarget \p STI.
219unsigned getMinNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU);
220
221/// \returns Maximum number of SGPRs that meets the given number of waves per
222/// execution unit requirement for given subtarget \p STI.
223unsigned getMaxNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
224 bool Addressable);
225
226/// \returns Number of extra SGPRs implicitly required by given subtarget \p
227/// STI when the given special registers are used.
228unsigned getNumExtraSGPRs(const MCSubtargetInfo &STI, bool VCCUsed,
229 bool FlatScrUsed, bool XNACKUsed);
230
231/// \returns Number of SGPR blocks needed for given subtarget \p STI when
232/// \p NumSGPRs are used. \p NumSGPRs should already include any special
233/// register counts.
234unsigned getNumSGPRBlocks(const MCSubtargetInfo &STI, unsigned NumSGPRs);
235
236/// \returns VGPR allocation granularity for given subtarget \p STI.
237///
238/// For subtargets which support it, \p EnableWavefrontSize32 should match
239/// the ENABLE_WAVEFRONT_SIZE32 kernel descriptor field.
240unsigned
241getVGPRAllocGranule(const MCSubtargetInfo &STI, unsigned DynamicVGPRBlockSize,
242 std::optional<bool> EnableWavefrontSize32 = std::nullopt);
243
244/// \returns VGPR encoding granularity for given subtarget \p STI.
245///
246/// For subtargets which support it, \p EnableWavefrontSize32 should match
247/// the ENABLE_WAVEFRONT_SIZE32 kernel descriptor field.
248unsigned getVGPREncodingGranule(
249 const MCSubtargetInfo &STI,
250 std::optional<bool> EnableWavefrontSize32 = std::nullopt);
251
252/// For subtargets with a unified VGPR file and mixed ArchVGPR/AGPR usage,
253/// returns the allocation granule for ArchVGPRs.
254unsigned getArchVGPRAllocGranule();
255
256/// Maximum number of VGPR blocks that can be allocated in dynamic VGPR mode.
257static constexpr unsigned MaxDynamicVGPRBlocks = 8;
258
259/// \returns Addressable number of architectural VGPRs for a given subtarget \p
260/// STI.
261unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo &STI);
262
263/// \returns Addressable number of VGPRs for given subtarget \p STI.
264unsigned getAddressableNumVGPRs(const MCSubtargetInfo &STI,
265 unsigned DynamicVGPRBlockSize);
266
267/// \returns Minimum number of VGPRs that meets given number of waves per
268/// execution unit requirement for given subtarget \p STI.
269unsigned getMinNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
270 unsigned DynamicVGPRBlockSize);
271
272/// \returns Maximum number of VGPRs that meets given number of waves per
273/// execution unit requirement for given subtarget \p STI.
274unsigned getMaxNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
275 unsigned DynamicVGPRBlockSize);
276
277/// \returns Number of waves reachable for a given \p NumVGPRs usage for given
278/// subtarget \p STI.
279unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo &STI,
280 unsigned NumVGPRs,
281 unsigned DynamicVGPRBlockSize);
282
283/// \returns Number of waves reachable for a given \p NumVGPRs usage, \p Granule
284/// size, \p MaxWaves possible, and \p TotalNumVGPRs available.
285unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
286 unsigned MaxWaves,
287 unsigned TotalNumVGPRs);
288
289/// \returns Whether allocated SGPRs can reduce occupancy on subtarget \p STI
290/// (true pre-GFX10). One named capability so callers don't test the version.
291bool isSGPROccupancyLimited(const MCSubtargetInfo &STI);
292
293/// \returns SGPR-limited occupancy (waves per EU) for subtarget \p STI: the
294/// inverse of getMaxNumSGPRs(). Unlike getMaxNumSGPRs() the budget is not
295/// clamped to the addressable count, since the allocated count callers pass in
296/// can exceed it.
297unsigned getOccupancyWithNumSGPRs(const MCSubtargetInfo &STI, unsigned SGPRs);
298
299/// \returns SGPR-limited occupancy computed from explicit budget parameters
300/// (\p MaxWaves, \p TotalNumSGPRs, \p Granule, \p TrapReserve). Subtarget-free
301/// core shared by the overload above and the occupancy MCExpr. Callers must
302/// check isSGPROccupancyLimited() first.
303unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
304 unsigned TotalNumSGPRs, unsigned Granule,
305 unsigned TrapReserve);
306
307/// \returns Number of VGPR blocks needed for given subtarget \p STI when
308/// \p NumVGPRs are used. We actually return the number of blocks -1, since
309/// that's what we encode.
310///
311/// For subtargets which support it, \p EnableWavefrontSize32 should match the
312/// ENABLE_WAVEFRONT_SIZE32 kernel descriptor field.
313unsigned getEncodedNumVGPRBlocks(
314 const MCSubtargetInfo &STI, unsigned NumVGPRs,
315 std::optional<bool> EnableWavefrontSize32 = std::nullopt);
316
317/// \returns Number of VGPR blocks that need to be allocated for the given
318/// subtarget \p STI when \p NumVGPRs are used.
319unsigned getAllocatedNumVGPRBlocks(
320 const MCSubtargetInfo &STI, unsigned NumVGPRs,
321 unsigned DynamicVGPRBlockSize,
322 std::optional<bool> EnableWavefrontSize32 = std::nullopt);
323
324} // end namespace IsaInfo
325
326// Represents a field in an encoded value.
327template <unsigned HighBit, unsigned LowBit, unsigned D = 0>
328struct EncodingField {
329 static_assert(HighBit >= LowBit, "Invalid bit range!");
330 static constexpr unsigned Offset = LowBit;
331 static constexpr unsigned Width = HighBit - LowBit + 1;
332
333 using ValueType = unsigned;
334 static constexpr ValueType Default = D;
335
336 ValueType Value;
337 constexpr EncodingField(ValueType Value) : Value(Value) {}
338
339 constexpr uint64_t encode() const { return Value; }
340 static ValueType decode(uint64_t Encoded) { return Encoded; }
341};
342
343// Represents a single bit in an encoded value.
344template <unsigned Bit, unsigned D = 0>
345using EncodingBit = EncodingField<Bit, Bit, D>;
346
347// A helper for encoding and decoding multiple fields.
348template <typename... Fields> struct EncodingFields {
349 static constexpr uint64_t encode(Fields... Values) {
350 return ((Values.encode() << Values.Offset) | ...);
351 }
352
353 static std::tuple<typename Fields::ValueType...> decode(uint64_t Encoded) {
354 return {Fields::decode((Encoded >> Fields::Offset) &
355 maxUIntN(Fields::Width))...};
356 }
357};
358
359LLVM_READONLY
360inline bool hasNamedOperand(uint64_t Opcode, OpName NamedIdx) {
361 return getNamedOperandIdx(Opcode, Name: NamedIdx) != -1;
362}
363
364LLVM_READONLY
365int32_t getSOPPWithRelaxation(uint32_t Opcode);
366
367struct MIMGBaseOpcodeInfo {
368 MIMGBaseOpcode BaseOpcode;
369 bool Store;
370 bool Atomic;
371 bool AtomicX2;
372 bool Sampler;
373 bool Gather4;
374
375 uint8_t NumExtraArgs;
376 bool Gradients;
377 bool G16;
378 bool Coordinates;
379 bool LodOrClampOrMip;
380 bool HasD16;
381 bool MSAA;
382 bool BVH;
383 bool A16;
384 bool NoReturn;
385 bool PointSampleAccel;
386};
387
388LLVM_READONLY
389const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc);
390
391LLVM_READONLY
392const MIMGBaseOpcodeInfo *getMIMGBaseOpcodeInfo(unsigned BaseOpcode);
393
394struct MIMGDimInfo {
395 MIMGDim Dim;
396 MIMGDim NonArrayDim;
397 uint8_t NumCoords;
398 uint8_t NumGradients;
399 bool MSAA;
400 bool DA;
401 uint8_t Encoding;
402 StringTable::Offset AsmSuffix;
403};
404
405LLVM_READONLY
406const MIMGDimInfo *getMIMGDimInfo(unsigned DimEnum);
407
408LLVM_READONLY StringRef getMIMGDimInfoStr(StringTable::Offset);
409
410LLVM_READONLY
411const MIMGDimInfo *getMIMGDimInfoByEncoding(uint8_t DimEnc);
412
413LLVM_READONLY
414const MIMGDimInfo *getMIMGDimInfoByAsmSuffix(StringRef AsmSuffix);
415
416struct MIMGLZMappingInfo {
417 MIMGBaseOpcode L;
418 MIMGBaseOpcode LZ;
419};
420
421struct MIMGMIPMappingInfo {
422 MIMGBaseOpcode MIP;
423 MIMGBaseOpcode NONMIP;
424};
425
426struct MIMGBiasMappingInfo {
427 MIMGBaseOpcode Bias;
428 MIMGBaseOpcode NoBias;
429};
430
431struct MIMGOffsetMappingInfo {
432 MIMGBaseOpcode Offset;
433 MIMGBaseOpcode NoOffset;
434};
435
436struct MIMGG16MappingInfo {
437 MIMGBaseOpcode G;
438 MIMGBaseOpcode G16;
439};
440
441LLVM_READONLY
442const MIMGLZMappingInfo *getMIMGLZMappingInfo(unsigned L);
443
444struct WMMAOpcodeMappingInfo {
445 unsigned Opcode2Addr;
446 unsigned Opcode3Addr;
447};
448
449LLVM_READONLY
450const MIMGMIPMappingInfo *getMIMGMIPMappingInfo(unsigned MIP);
451
452LLVM_READONLY
453const MIMGBiasMappingInfo *getMIMGBiasMappingInfo(unsigned Bias);
454
455LLVM_READONLY
456const MIMGOffsetMappingInfo *getMIMGOffsetMappingInfo(unsigned Offset);
457
458LLVM_READONLY
459const MIMGG16MappingInfo *getMIMGG16MappingInfo(unsigned G);
460
461LLVM_READONLY
462int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
463 unsigned VDataDwords, unsigned VAddrDwords,
464 bool IndexedRsrc = false, bool IndexedSamp = false);
465
466LLVM_READONLY
467int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels);
468
469LLVM_READONLY
470unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
471 const MIMGDimInfo *Dim, bool IsA16,
472 bool IsG16Supported);
473
474struct MIMGInfo {
475 uint32_t Opcode;
476 uint32_t BaseOpcode;
477 uint8_t MIMGEncoding;
478 uint8_t VDataDwords;
479 uint8_t VAddrDwords;
480 uint8_t VAddrOperands;
481 bool IndexedRsrc;
482 bool IndexedSamp;
483};
484
485LLVM_READONLY
486const MIMGInfo *getMIMGInfo(unsigned Opc);
487
488LLVM_READONLY
489int getMTBUFBaseOpcode(unsigned Opc);
490
491LLVM_READONLY
492int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements);
493
494LLVM_READONLY
495int getMTBUFElements(unsigned Opc);
496
497LLVM_READONLY
498bool getMTBUFHasVAddr(unsigned Opc);
499
500LLVM_READONLY
501bool getMTBUFHasSrsrc(unsigned Opc);
502
503LLVM_READONLY
504bool getMTBUFHasSoffset(unsigned Opc);
505
506LLVM_READONLY
507int getMUBUFBaseOpcode(unsigned Opc);
508
509LLVM_READONLY
510int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements);
511
512LLVM_READONLY
513int getMUBUFElements(unsigned Opc);
514
515LLVM_READONLY
516bool getMUBUFHasVAddr(unsigned Opc);
517
518LLVM_READONLY
519bool getMUBUFHasSrsrc(unsigned Opc);
520
521LLVM_READONLY
522bool getMUBUFHasSoffset(unsigned Opc);
523
524LLVM_READONLY
525bool getMUBUFIsBufferInv(unsigned Opc);
526
527LLVM_READONLY
528bool getMUBUFTfe(unsigned Opc);
529
530LLVM_READONLY
531bool getSMEMIsBuffer(unsigned Opc);
532
533LLVM_READONLY
534bool getVOP1IsSingle(unsigned Opc);
535
536LLVM_READONLY
537bool getVOP2IsSingle(unsigned Opc);
538
539LLVM_READONLY
540bool getVOP3IsSingle(unsigned Opc);
541
542LLVM_READONLY
543bool isVOPC64DPP(unsigned Opc);
544
545LLVM_READONLY
546bool isVOPCAsmOnly(unsigned Opc);
547
548/// Returns true if MAI operation is a double precision GEMM.
549LLVM_READONLY
550bool getMAIIsDGEMM(unsigned Opc);
551
552LLVM_READONLY
553bool getMAIIsGFX940XDL(unsigned Opc);
554
555LLVM_READONLY
556bool getWMMAIsXDL(unsigned Opc);
557
558LLVM_READONLY
559bool getHasMatrixScale(unsigned Opc);
560
561// Get an equivalent BitOp3 for a binary logical \p Opc.
562// \returns BitOp3 modifier for the logical operation or zero.
563// Used in VOPD3 conversion.
564unsigned getBitOp2(unsigned Opc);
565
566struct CanBeVOPD {
567 bool X;
568 bool Y;
569};
570
571/// \returns SIEncodingFamily used for VOPD encoding on a \p ST.
572LLVM_READONLY
573unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST);
574
575LLVM_READONLY
576CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3);
577
578LLVM_READNONE
579uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal);
580
581LLVM_READONLY
582const MFMA_F8F6F4_Info *getMFMA_F8F6F4_WithFormatArgs(unsigned CBSZ,
583 unsigned BLGP,
584 unsigned F8F8Opcode);
585
586LLVM_READNONE
587uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt);
588
589LLVM_READONLY
590const MFMA_F8F6F4_Info *getWMMA_F8F6F4_WithFormatArgs(unsigned FmtA,
591 unsigned FmtB,
592 unsigned F8F8Opcode);
593
594/// \return true if this combination is listed as valid.
595LLVM_READONLY
596bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale,
597 unsigned BFmt, unsigned BScale);
598
599LLVM_READONLY
600const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
601 uint8_t NumComponents,
602 uint8_t NumFormat,
603 const MCSubtargetInfo &STI);
604LLVM_READONLY
605const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
606 const MCSubtargetInfo &STI);
607
608LLVM_READONLY
609int32_t getMCOpcode(uint32_t Opcode, unsigned Gen);
610
611LLVM_READONLY
612unsigned getVOPDOpcode(unsigned Opc, bool VOPD3);
613
614LLVM_READONLY
615int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily,
616 bool VOPD3);
617
618LLVM_READONLY
619bool isVOPD(unsigned Opc);
620
621LLVM_READNONE
622bool isMAC(unsigned Opc);
623
624LLVM_READNONE
625bool isPermlane16(unsigned Opc);
626
627LLVM_READNONE
628bool isGenericAtomic(unsigned Opc);
629
630LLVM_READNONE
631bool isCvt_F32_Fp8_Bf8_e64(unsigned Opc);
632
633namespace VOPD {
634
635enum Component : unsigned {
636 DST = 0,
637 SRC0,
638 SRC1,
639 SRC2,
640
641 DST_NUM = 1,
642 MAX_SRC_NUM = 3,
643 MAX_OPR_NUM = DST_NUM + MAX_SRC_NUM
644};
645
646// LSB mask for VGPR banks per VOPD component operand.
647// 4 banks result in a mask 3, setting 2 lower bits.
648constexpr unsigned VOPD_VGPR_BANK_MASKS[] = {1, 3, 3, 1};
649constexpr unsigned VOPD3_VGPR_BANK_MASKS[] = {1, 3, 3, 3};
650// GFX11 VOPD interlock hazard requires SRC0/SRC1 to have
651// different parities, not just on different banks. Else,
652// non-deterministic forwarding error may occur.
653constexpr unsigned VOPD_GFX11_VGPR_BANK_MASKS[] = {1, 1, 1, 1};
654
655enum ComponentIndex : unsigned { X = 0, Y = 1 };
656constexpr unsigned COMPONENTS[] = {ComponentIndex::X, ComponentIndex::Y};
657constexpr unsigned COMPONENTS_NUM = 2;
658
659// Properties of VOPD components.
660class ComponentProps {
661private:
662 unsigned SrcOperandsNum = 0;
663 unsigned MandatoryLiteralIdx = ~0u;
664 bool HasSrc2Acc = false;
665 unsigned NumVOPD3Mods = 0;
666 unsigned Opcode = 0;
667 bool IsVOP3 = false;
668
669public:
670 ComponentProps() = default;
671 ComponentProps(const MCInstrDesc &OpDesc, bool VOP3Layout = false);
672
673 // Return the total number of src operands this component has.
674 unsigned getCompSrcOperandsNum() const { return SrcOperandsNum; }
675
676 // Return the number of src operands of this component visible to the parser.
677 unsigned getCompParsedSrcOperandsNum() const {
678 return SrcOperandsNum - HasSrc2Acc;
679 }
680
681 // Return true iif this component has a mandatory literal.
682 bool hasMandatoryLiteral() const { return MandatoryLiteralIdx != ~0u; }
683
684 // If this component has a mandatory literal, return component operand
685 // index of this literal (i.e. either Component::SRC1 or Component::SRC2).
686 unsigned getMandatoryLiteralCompOperandIndex() const {
687 assert(hasMandatoryLiteral());
688 return MandatoryLiteralIdx;
689 }
690
691 // Return true iif this component has operand
692 // with component index CompSrcIdx and this operand may be a register.
693 bool hasRegSrcOperand(unsigned CompSrcIdx) const {
694 assert(CompSrcIdx < Component::MAX_SRC_NUM);
695 return SrcOperandsNum > CompSrcIdx && !hasMandatoryLiteralAt(CompSrcIdx);
696 }
697
698 // Return true iif this component has tied src2.
699 bool hasSrc2Acc() const { return HasSrc2Acc; }
700
701 // Return a number of source modifiers if instruction is used in VOPD3.
702 unsigned getCompVOPD3ModsNum() const { return NumVOPD3Mods; }
703
704 // Return opcode of the component.
705 unsigned getOpcode() const { return Opcode; }
706
707 // Returns if component opcode is in VOP3 encoding.
708 unsigned isVOP3() const { return IsVOP3; }
709
710 // Return index of BitOp3 operand or -1.
711 int getBitOp3OperandIdx() const;
712
713private:
714 bool hasMandatoryLiteralAt(unsigned CompSrcIdx) const {
715 assert(CompSrcIdx < Component::MAX_SRC_NUM);
716 return MandatoryLiteralIdx == Component::DST_NUM + CompSrcIdx;
717 }
718};
719
720enum ComponentKind : unsigned {
721 SINGLE = 0, // A single VOP1 or VOP2 instruction which may be used in VOPD.
722 COMPONENT_X, // A VOPD instruction, X component.
723 COMPONENT_Y, // A VOPD instruction, Y component.
724 MAX = COMPONENT_Y
725};
726
727// Interface functions of this class map VOPD component operand indices
728// to indices of operands in MachineInstr/MCInst or parsed operands array.
729//
730// Note that this class operates with 3 kinds of indices:
731// - VOPD component operand indices (Component::DST, Component::SRC0, etc.);
732// - MC operand indices (they refer operands in a MachineInstr/MCInst);
733// - parsed operand indices (they refer operands in parsed operands array).
734//
735// For SINGLE components mapping between these indices is trivial.
736// But things get more complicated for COMPONENT_X and
737// COMPONENT_Y because these components share the same
738// MachineInstr/MCInst and the same parsed operands array.
739// Below is an example of component operand to parsed operand
740// mapping for the following instruction:
741//
742// v_dual_add_f32 v255, v4, v5 :: v_dual_mov_b32 v6, v1
743//
744// PARSED COMPONENT PARSED
745// COMPONENT OPERANDS OPERAND INDEX OPERAND INDEX
746// -------------------------------------------------------------------
747// "v_dual_add_f32" 0
748// v_dual_add_f32 v255 0 (DST) --> 1
749// v4 1 (SRC0) --> 2
750// v5 2 (SRC1) --> 3
751// "::" 4
752// "v_dual_mov_b32" 5
753// v_dual_mov_b32 v6 0 (DST) --> 6
754// v1 1 (SRC0) --> 7
755// -------------------------------------------------------------------
756//
757class ComponentLayout {
758private:
759 // Regular MachineInstr/MCInst operands are ordered as follows:
760 // dst, src0 [, other src operands]
761 // VOPD MachineInstr/MCInst operands are ordered as follows:
762 // dstX, dstY, src0X [, other OpX operands], src0Y [, other OpY operands]
763 // Each ComponentKind has operand indices defined below.
764 static constexpr unsigned MC_DST_IDX[] = {0, 0, 1};
765
766 // VOPD3 instructions may have 2 or 3 source modifiers, src2 modifier is not
767 // used if there is tied accumulator. Indexing of this array:
768 // MC_SRC_IDX[VOPD3ModsNum][SrcNo]. This returns an index for a SINGLE
769 // instruction layout, add 1 for COMPONENT_X or COMPONENT_Y. For the second
770 // component add OpX.MCSrcNum + OpX.VOPD3ModsNum.
771 // For VOPD1/VOPD2 use column with zero modifiers.
772 static constexpr unsigned SINGLE_MC_SRC_IDX[4][3] = {
773 {1, 2, 3}, {2, 3, 4}, {2, 4, 5}, {2, 4, 6}};
774
775 // Parsed operands of regular instructions are ordered as follows:
776 // Mnemo dst src0 [vsrc1 ...]
777 // Parsed VOPD operands are ordered as follows:
778 // OpXMnemo dstX src0X [vsrc1X|imm vsrc1X|vsrc1X imm] '::'
779 // OpYMnemo dstY src0Y [vsrc1Y|imm vsrc1Y|vsrc1Y imm]
780 // Each ComponentKind has operand indices defined below.
781 static constexpr unsigned PARSED_DST_IDX[] = {1, 1,
782 4 /* + OpX.ParsedSrcNum */};
783 static constexpr unsigned FIRST_PARSED_SRC_IDX[] = {
784 2, 2, 5 /* + OpX.ParsedSrcNum */};
785
786private:
787 const ComponentKind Kind;
788 const ComponentProps PrevComp;
789 const unsigned VOPD3ModsNum;
790 const int BitOp3Idx; // Index of bitop3 operand or -1
791
792public:
793 // Create layout for COMPONENT_X or SINGLE component.
794 ComponentLayout(ComponentKind Kind, unsigned VOPD3ModsNum, int BitOp3Idx)
795 : Kind(Kind), VOPD3ModsNum(VOPD3ModsNum), BitOp3Idx(BitOp3Idx) {
796 assert(Kind == ComponentKind::SINGLE || Kind == ComponentKind::COMPONENT_X);
797 }
798
799 // Create layout for COMPONENT_Y which depends on COMPONENT_X layout.
800 ComponentLayout(const ComponentProps &OpXProps, unsigned VOPD3ModsNum,
801 int BitOp3Idx)
802 : Kind(ComponentKind::COMPONENT_Y), PrevComp(OpXProps),
803 VOPD3ModsNum(VOPD3ModsNum), BitOp3Idx(BitOp3Idx) {}
804
805public:
806 // Return the index of dst operand in MCInst operands.
807 unsigned getIndexOfDstInMCOperands() const { return MC_DST_IDX[Kind]; }
808
809 // Return the index of the specified src operand in MCInst operands.
810 unsigned getIndexOfSrcInMCOperands(unsigned CompSrcIdx, bool VOPD3) const {
811 assert(CompSrcIdx < Component::MAX_SRC_NUM);
812
813 if (Kind == SINGLE && CompSrcIdx == 2 && BitOp3Idx != -1)
814 return BitOp3Idx;
815
816 if (VOPD3) {
817 return SINGLE_MC_SRC_IDX[VOPD3ModsNum][CompSrcIdx] + getPrevCompSrcNum() +
818 getPrevCompVOPD3ModsNum() + (Kind != SINGLE ? 1 : 0);
819 }
820
821 return SINGLE_MC_SRC_IDX[0][CompSrcIdx] + getPrevCompSrcNum() +
822 (Kind != SINGLE ? 1 : 0);
823 }
824
825 // Return the index of dst operand in the parsed operands array.
826 unsigned getIndexOfDstInParsedOperands() const {
827 return PARSED_DST_IDX[Kind] + getPrevCompParsedSrcNum();
828 }
829
830 // Return the index of the specified src operand in the parsed operands array.
831 unsigned getIndexOfSrcInParsedOperands(unsigned CompSrcIdx) const {
832 assert(CompSrcIdx < Component::MAX_SRC_NUM);
833 return FIRST_PARSED_SRC_IDX[Kind] + getPrevCompParsedSrcNum() + CompSrcIdx;
834 }
835
836private:
837 unsigned getPrevCompSrcNum() const {
838 return PrevComp.getCompSrcOperandsNum();
839 }
840 unsigned getPrevCompParsedSrcNum() const {
841 return PrevComp.getCompParsedSrcOperandsNum();
842 }
843 unsigned getPrevCompVOPD3ModsNum() const {
844 return PrevComp.getCompVOPD3ModsNum();
845 }
846};
847
848// Layout and properties of VOPD components.
849class ComponentInfo : public ComponentProps, public ComponentLayout {
850public:
851 // Create ComponentInfo for COMPONENT_X or SINGLE component.
852 ComponentInfo(const MCInstrDesc &OpDesc,
853 ComponentKind Kind = ComponentKind::SINGLE,
854 bool VOP3Layout = false)
855 : ComponentProps(OpDesc, VOP3Layout),
856 ComponentLayout(Kind, getCompVOPD3ModsNum(), getBitOp3OperandIdx()) {}
857
858 // Create ComponentInfo for COMPONENT_Y which depends on COMPONENT_X layout.
859 ComponentInfo(const MCInstrDesc &OpDesc, const ComponentProps &OpXProps,
860 bool VOP3Layout = false)
861 : ComponentProps(OpDesc, VOP3Layout),
862 ComponentLayout(OpXProps, getCompVOPD3ModsNum(),
863 getBitOp3OperandIdx()) {}
864
865 // Map component operand index to parsed operand index.
866 // Return 0 if the specified operand does not exist.
867 unsigned getIndexInParsedOperands(unsigned CompOprIdx) const;
868};
869
870// Properties of VOPD instructions.
871class InstInfo {
872private:
873 const ComponentInfo CompInfo[COMPONENTS_NUM];
874
875public:
876 using RegIndices = std::array<MCRegister, Component::MAX_OPR_NUM>;
877
878 InstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY)
879 : CompInfo{OpX, OpY} {}
880
881 InstInfo(const ComponentInfo &OprInfoX, const ComponentInfo &OprInfoY)
882 : CompInfo{OprInfoX, OprInfoY} {}
883
884 const ComponentInfo &operator[](size_t ComponentIdx) const {
885 assert(ComponentIdx < COMPONENTS_NUM);
886 return CompInfo[ComponentIdx];
887 }
888
889 // Check VOPD operands constraints.
890 // GetRegIdx(Component, MCOperandIdx) must return a VGPR register index
891 // for the specified component and MC operand. The callback must return 0
892 // if the operand is not a register or not a VGPR.
893 // If \p SkipSrc is set to true then constraints for source operands are not
894 // checked.
895 // If \p AllowSameVGPR is set then same VGPRs are allowed for X and Y sources
896 // even though it violates requirement to be from different banks.
897 // If \p VOPD3 is set to true both dst registers allowed to be either odd
898 // or even and instruction may have real src2 as opposed to tied accumulator.
899 // If \p HasGFX11InterlockHazard is set then X/Y SRC0 and SRC1 VGPRs
900 // must have different register-number parity.
901 bool
902 hasInvalidOperand(std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
903 const MCRegisterInfo &MRI, bool SkipSrc = false,
904 bool AllowSameVGPR = false, bool VOPD3 = false,
905 bool HasGFX11InterlockHazard = false) const {
906 return getInvalidCompOperandIndex(GetRegIdx, MRI, SkipSrc, AllowSameVGPR,
907 VOPD3, HasGFX11InterlockHazard)
908 .has_value();
909 }
910
911 // Check VOPD operands constraints.
912 // Return the index of an invalid component operand, if any.
913 // If \p SkipSrc is set to true then constraints for source operands are not
914 // checked except for being from the same halves of VGPR file on gfx1250.
915 // If \p AllowSameVGPR is set then same VGPRs are allowed for X and Y sources
916 // even though it violates requirement to be from different banks.
917 // If \p VOPD3 is set to true both dst registers allowed to be either odd
918 // or even and instruction may have real src2 as opposed to tied accumulator.
919 // If \p HasGFX11InterlockHazard is set then X/Y SRC0 and SRC1 VGPRs
920 // must have different register-number parity.
921 std::optional<unsigned> getInvalidCompOperandIndex(
922 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
923 const MCRegisterInfo &MRI, bool SkipSrc = false,
924 bool AllowSameVGPR = false, bool VOPD3 = false,
925 bool HasGFX11InterlockHazard = false) const;
926
927private:
928 RegIndices
929 getRegIndices(unsigned ComponentIdx,
930 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
931 bool VOPD3) const;
932};
933
934} // namespace VOPD
935
936LLVM_READONLY
937std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode);
938
939LLVM_READONLY
940// Get properties of 2 single VOP1/VOP2 instructions
941// used as components to create a VOPD instruction.
942VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY);
943
944LLVM_READONLY
945// Get properties of VOPD X and Y components.
946VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,
947 const MCInstrInfo *InstrInfo);
948
949LLVM_READONLY
950bool isAsyncStore(unsigned Opc);
951LLVM_READONLY
952bool isTensorStore(unsigned Opc);
953LLVM_READONLY
954unsigned getTemporalHintType(const MCInstrDesc TID);
955
956LLVM_READONLY
957bool isTrue16Inst(unsigned Opc);
958
959LLVM_READONLY
960FPType getFPDstSelType(unsigned Opc);
961
962bool isDPMACCInstruction(unsigned Opc);
963
964LLVM_READONLY
965unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc);
966
967LLVM_READONLY
968unsigned mapWMMA3AddrTo2AddrOpcode(unsigned Opc);
969
970void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &Header,
971 const MCSubtargetInfo &STI);
972
973bool isGroupSegment(const GlobalValue *GV);
974bool isGlobalSegment(const GlobalValue *GV);
975bool isReadOnlySegment(const GlobalValue *GV);
976
977/// \returns True if constants should be emitted to .text section for given
978/// target triple \p TT, false otherwise.
979bool shouldEmitConstantsToTextSection(const Triple &TT);
980
981/// Returns a valid charcode or 0 in the first entry if this is a valid physical
982/// register name. Followed by the start register number, and the register
983/// width. Does not validate the number of registers exists in the class. Unlike
984/// parseAsmConstraintPhysReg, this does not expect the name to be wrapped in
985/// "{}".
986std::tuple<char, unsigned, unsigned> parseAsmPhysRegName(StringRef TupleString);
987
988/// Returns a valid charcode or 0 in the first entry if this is a valid physical
989/// register constraint. Followed by the start register number, and the register
990/// width. Does not validate the number of registers exists in the class.
991std::tuple<char, unsigned, unsigned>
992parseAsmConstraintPhysReg(StringRef Constraint);
993
994/// \returns A pair of integer values requested using \p F's \p Name attribute
995/// in "first[,second]" format ("second" is optional unless \p OnlyFirstRequired
996/// is false).
997///
998/// \returns \p Default if attribute is not present.
999///
1000/// \returns \p Default and emits error if one of the requested values cannot be
1001/// converted to integer, or \p OnlyFirstRequired is false and "second" value is
1002/// not present.
1003std::pair<unsigned, unsigned>
1004getIntegerPairAttribute(const Function &F, StringRef Name,
1005 std::pair<unsigned, unsigned> Default,
1006 bool OnlyFirstRequired = false);
1007
1008/// \returns A pair of integer values requested using \p F's \p Name attribute
1009/// in "first[,second]" format ("second" is optional unless \p OnlyFirstRequired
1010/// is false).
1011///
1012/// \returns \p std::nullopt if attribute is not present.
1013///
1014/// \returns \p std::nullopt and emits error if one of the requested values
1015/// cannot be converted to integer, or \p OnlyFirstRequired is false and
1016/// "second" value is not present.
1017std::optional<std::pair<unsigned, std::optional<unsigned>>>
1018getIntegerPairAttribute(const Function &F, StringRef Name,
1019 bool OnlyFirstRequired = false);
1020
1021/// \returns Generate a vector of integer values requested using \p F's \p Name
1022/// attribute.
1023/// \returns A vector of size \p Size, with all elements set to \p DefaultVal,
1024/// if any error occurs. The corresponding error will also be emitted.
1025SmallVector<unsigned> getIntegerVecAttribute(const Function &F, StringRef Name,
1026 unsigned Size,
1027 unsigned DefaultVal);
1028/// Similar to the function above, but returns std::nullopt if any error occurs.
1029std::optional<SmallVector<unsigned>>
1030getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size);
1031
1032/// \returns The maximum number of workgroups for the function.
1033SmallVector<unsigned> getMaxNumWorkGroups(const Function &F);
1034
1035inline bool isTgSplitEnabled(const Function &F) {
1036 return F.hasFnAttribute(Kind: "amdgpu-tg-split");
1037}
1038
1039/// Checks if \p Val is inside \p MD, a !range-like metadata.
1040bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val);
1041
1042// The following methods are only meaningful on targets that support
1043// S_WAITCNT.
1044
1045/// \returns Vmcnt bit mask for given isa \p Version.
1046unsigned getVmcntBitMask(const IsaVersion &Version);
1047
1048/// \returns Expcnt bit mask for given isa \p Version.
1049unsigned getExpcntBitMask(const IsaVersion &Version);
1050
1051/// \returns Lgkmcnt bit mask for given isa \p Version.
1052unsigned getLgkmcntBitMask(const IsaVersion &Version);
1053
1054/// \returns Waitcnt bit mask for given isa \p Version.
1055unsigned getWaitcntBitMask(const IsaVersion &Version);
1056
1057/// \returns Decoded Vmcnt from given \p Waitcnt for given isa \p Version.
1058unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt);
1059
1060/// \returns Decoded Expcnt from given \p Waitcnt for given isa \p Version.
1061unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt);
1062
1063/// \returns Decoded Lgkmcnt from given \p Waitcnt for given isa \p Version.
1064unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt);
1065
1066/// \returns Decoded Loadcnt from given \p Waitcnt for given isa \p Version.
1067unsigned decodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt);
1068
1069/// \returns Decoded Storecnt from given \p Waitcnt for given isa \p Version.
1070unsigned decodeStorecnt(const IsaVersion &Version, unsigned Waitcnt);
1071
1072/// \returns Decoded Dscnt from given \p Waitcnt for given isa \p Version.
1073unsigned decodeDscnt(const IsaVersion &Version, unsigned Waitcnt);
1074
1075/// Decodes Vmcnt, Expcnt and Lgkmcnt from given \p Waitcnt for given isa
1076/// \p Version, and writes decoded values into \p Vmcnt, \p Expcnt and
1077/// \p Lgkmcnt respectively. Should not be used on gfx12+, the instruction
1078/// which needs it is deprecated
1079///
1080/// \details \p Vmcnt, \p Expcnt and \p Lgkmcnt are decoded as follows:
1081/// \p Vmcnt = \p Waitcnt[3:0] (pre-gfx9)
1082/// \p Vmcnt = \p Waitcnt[15:14,3:0] (gfx9,10)
1083/// \p Vmcnt = \p Waitcnt[15:10] (gfx11)
1084/// \p Expcnt = \p Waitcnt[6:4] (pre-gfx11)
1085/// \p Expcnt = \p Waitcnt[2:0] (gfx11)
1086/// \p Lgkmcnt = \p Waitcnt[11:8] (pre-gfx10)
1087/// \p Lgkmcnt = \p Waitcnt[13:8] (gfx10)
1088/// \p Lgkmcnt = \p Waitcnt[9:4] (gfx11)
1089///
1090void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt,
1091 unsigned &Expcnt, unsigned &Lgkmcnt);
1092
1093/// \returns \p Waitcnt with encoded \p Vmcnt for given isa \p Version.
1094unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
1095 unsigned Vmcnt);
1096
1097/// \returns \p Waitcnt with encoded \p Expcnt for given isa \p Version.
1098unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
1099 unsigned Expcnt);
1100
1101/// \returns \p Waitcnt with encoded \p Lgkmcnt for given isa \p Version.
1102unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
1103 unsigned Lgkmcnt);
1104
1105/// Encodes \p Vmcnt, \p Expcnt and \p Lgkmcnt into Waitcnt for given isa
1106/// \p Version. Should not be used on gfx12+, the instruction which needs
1107/// it is deprecated
1108///
1109/// \details \p Vmcnt, \p Expcnt and \p Lgkmcnt are encoded as follows:
1110/// Waitcnt[2:0] = \p Expcnt (gfx11+)
1111/// Waitcnt[3:0] = \p Vmcnt (pre-gfx9)
1112/// Waitcnt[3:0] = \p Vmcnt[3:0] (gfx9,10)
1113/// Waitcnt[6:4] = \p Expcnt (pre-gfx11)
1114/// Waitcnt[9:4] = \p Lgkmcnt (gfx11)
1115/// Waitcnt[11:8] = \p Lgkmcnt (pre-gfx10)
1116/// Waitcnt[13:8] = \p Lgkmcnt (gfx10)
1117/// Waitcnt[15:10] = \p Vmcnt (gfx11)
1118/// Waitcnt[15:14] = \p Vmcnt[5:4] (gfx9,10)
1119///
1120/// \returns Waitcnt with encoded \p Vmcnt, \p Expcnt and \p Lgkmcnt for given
1121/// isa \p Version.
1122///
1123unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt,
1124 unsigned Expcnt, unsigned Lgkmcnt);
1125
1126/// \returns Waitcnt with encoded \p Loadcnt and \p Dscnt for given isa \p
1127/// Version.
1128unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt,
1129 unsigned Dscnt);
1130
1131/// \returns Waitcnt with encoded \p Storecnt and \p Dscnt for given isa \p
1132/// Version.
1133unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt,
1134 unsigned Dscnt);
1135
1136// The following methods are only meaningful on targets that support
1137// S_WAIT_*CNT, introduced with gfx12.
1138
1139/// \returns Loadcnt bit mask for given isa \p Version.
1140/// Returns 0 for versions that do not support LOADcnt
1141unsigned getLoadcntBitMask(const IsaVersion &Version);
1142
1143/// \returns Samplecnt bit mask for given isa \p Version.
1144/// Returns 0 for versions that do not support SAMPLEcnt
1145unsigned getSamplecntBitMask(const IsaVersion &Version);
1146
1147/// \returns Bvhcnt bit mask for given isa \p Version.
1148/// Returns 0 for versions that do not support BVHcnt
1149unsigned getBvhcntBitMask(const IsaVersion &Version);
1150
1151/// \returns Asynccnt bit mask for given isa \p Version.
1152/// Returns 0 for versions that do not support Asynccnt
1153unsigned getAsynccntBitMask(const IsaVersion &Version);
1154
1155/// \returns Dscnt bit mask for given isa \p Version.
1156/// Returns 0 for versions that do not support DScnt
1157unsigned getDscntBitMask(const IsaVersion &Version);
1158
1159/// \returns Dscnt bit mask for given isa \p Version.
1160/// Returns 0 for versions that do not support KMcnt
1161unsigned getKmcntBitMask(const IsaVersion &Version);
1162
1163/// \returns Xcnt bit mask for given isa \p Version.
1164/// Returns 0 for versions that do not support Xcnt.
1165unsigned getXcntBitMask(const IsaVersion &Version);
1166
1167/// \return STOREcnt or VScnt bit mask for given isa \p Version.
1168/// returns 0 for versions that do not support STOREcnt or VScnt.
1169/// STOREcnt and VScnt are the same counter, the name used
1170/// depends on the ISA version.
1171unsigned getStorecntBitMask(const IsaVersion &Version);
1172
1173namespace Hwreg {
1174
1175using HwregId = EncodingField<5, 0>;
1176using HwregOffset = EncodingField<10, 6>;
1177
1178struct HwregSize : EncodingField<15, 11, 32> {
1179 using EncodingField::EncodingField;
1180 constexpr uint64_t encode() const { return Value - 1; }
1181 static ValueType decode(uint64_t Encoded) { return Encoded + 1; }
1182};
1183
1184using HwregEncoding = EncodingFields<HwregId, HwregOffset, HwregSize>;
1185
1186} // namespace Hwreg
1187
1188namespace DepCtr {
1189
1190int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI);
1191int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask,
1192 const MCSubtargetInfo &STI);
1193bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal,
1194 const MCSubtargetInfo &STI);
1195bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val,
1196 bool &IsDefault, const MCSubtargetInfo &STI);
1197
1198/// \returns Maximum VaVdst value that can be encoded.
1199unsigned getVaVdstBitMask();
1200
1201/// \returns Maximum VaSdst value that can be encoded.
1202unsigned getVaSdstBitMask();
1203
1204/// \returns Maximum VaSsrc value that can be encoded.
1205unsigned getVaSsrcBitMask();
1206
1207/// \returns Maximum HoldCnt value that can be encoded.
1208unsigned getHoldCntBitMask(const IsaVersion &Version);
1209
1210/// \returns Maximum VmVsrc value that can be encoded.
1211unsigned getVmVsrcBitMask();
1212
1213/// \returns Maximum VaVcc value that can be encoded.
1214unsigned getVaVccBitMask();
1215
1216/// \returns Maximum SaSdst value that can be encoded.
1217unsigned getSaSdstBitMask();
1218
1219/// \returns Decoded VaVdst from given immediate \p Encoded.
1220unsigned decodeFieldVaVdst(unsigned Encoded);
1221
1222/// \returns Decoded VmVsrc from given immediate \p Encoded.
1223unsigned decodeFieldVmVsrc(unsigned Encoded);
1224
1225/// \returns Decoded SaSdst from given immediate \p Encoded.
1226unsigned decodeFieldSaSdst(unsigned Encoded);
1227
1228/// \returns Decoded VaSdst from given immediate \p Encoded.
1229unsigned decodeFieldVaSdst(unsigned Encoded);
1230
1231/// \returns Decoded VaVcc from given immediate \p Encoded.
1232unsigned decodeFieldVaVcc(unsigned Encoded);
1233
1234/// \returns Decoded SaSrc from given immediate \p Encoded.
1235unsigned decodeFieldVaSsrc(unsigned Encoded);
1236
1237/// \returns Decoded HoldCnt from given immediate \p Encoded.
1238unsigned decodeFieldHoldCnt(unsigned Encoded, const IsaVersion &Version);
1239
1240/// \returns \p VmVsrc as an encoded Depctr immediate.
1241unsigned encodeFieldVmVsrc(unsigned VmVsrc, const MCSubtargetInfo &STI);
1242
1243/// \returns \p Encoded combined with encoded \p VmVsrc.
1244unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc);
1245
1246/// \returns \p VaVdst as an encoded Depctr immediate.
1247unsigned encodeFieldVaVdst(unsigned VaVdst, const MCSubtargetInfo &STI);
1248
1249/// \returns \p Encoded combined with encoded \p VaVdst.
1250unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst);
1251
1252/// \returns \p SaSdst as an encoded Depctr immediate.
1253unsigned encodeFieldSaSdst(unsigned SaSdst, const MCSubtargetInfo &STI);
1254
1255/// \returns \p Encoded combined with encoded \p SaSdst.
1256unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst);
1257
1258/// \returns \p VaSdst as an encoded Depctr immediate.
1259unsigned encodeFieldVaSdst(unsigned VaSdst, const MCSubtargetInfo &STI);
1260
1261/// \returns \p Encoded combined with encoded \p VaSdst.
1262unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst);
1263
1264/// \returns \p VaVcc as an encoded Depctr immediate.
1265unsigned encodeFieldVaVcc(unsigned VaVcc, const MCSubtargetInfo &STI);
1266
1267/// \returns \p Encoded combined with encoded \p VaVcc.
1268unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc);
1269
1270/// \returns \p HoldCnt as an encoded Depctr immediate.
1271unsigned encodeFieldHoldCnt(unsigned HoldCnt, const MCSubtargetInfo &STI);
1272
1273/// \returns \p Encoded combined with encoded \p HoldCnt.
1274unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt,
1275 const IsaVersion &Version);
1276
1277/// \returns \p VaSsrc as an encoded Depctr immediate.
1278unsigned encodeFieldVaSsrc(unsigned VaSsrc, const MCSubtargetInfo &STI);
1279
1280/// \returns \p Encoded combined with encoded \p VaSsrc.
1281unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc);
1282
1283} // namespace DepCtr
1284
1285namespace Exp {
1286
1287bool getTgtName(unsigned Id, StringRef &Name, int &Index);
1288
1289LLVM_READONLY
1290unsigned getTgtId(const StringRef Name);
1291
1292LLVM_READNONE
1293bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI);
1294
1295} // namespace Exp
1296
1297namespace MTBUFFormat {
1298
1299LLVM_READNONE
1300int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt);
1301
1302void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt);
1303
1304int64_t getDfmt(const StringRef Name);
1305
1306StringRef getDfmtName(unsigned Id);
1307
1308int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI);
1309
1310StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI);
1311
1312bool isValidDfmtNfmt(unsigned Val, const MCSubtargetInfo &STI);
1313
1314bool isValidNfmt(unsigned Val, const MCSubtargetInfo &STI);
1315
1316int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI);
1317
1318StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI);
1319
1320bool isValidUnifiedFormat(unsigned Val, const MCSubtargetInfo &STI);
1321
1322int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt,
1323 const MCSubtargetInfo &STI);
1324
1325bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI);
1326
1327unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI);
1328
1329} // namespace MTBUFFormat
1330
1331namespace SendMsg {
1332
1333LLVM_READNONE
1334bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI);
1335
1336LLVM_READNONE
1337bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
1338 bool Strict = true);
1339
1340LLVM_READNONE
1341bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
1342 const MCSubtargetInfo &STI, bool Strict = true);
1343
1344LLVM_READNONE
1345bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI);
1346
1347LLVM_READNONE
1348bool msgSupportsStream(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI);
1349
1350void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,
1351 uint16_t &StreamId, const MCSubtargetInfo &STI);
1352
1353LLVM_READNONE
1354uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId);
1355
1356/// Returns true if the message does not use the m0 operand.
1357bool msgDoesNotUseM0(int64_t MsgId, const MCSubtargetInfo &STI);
1358
1359} // namespace SendMsg
1360
1361unsigned getInitialPSInputAddr(const Function &F);
1362
1363bool getHasColorExport(const Function &F);
1364
1365bool getHasDepthExport(const Function &F);
1366
1367// Returns the value of the "amdgpu-dynamic-vgpr-block-size" attribute, or 0 if
1368// the attribute is missing or its value is invalid.
1369unsigned getDynamicVGPRBlockSize(const Function &F);
1370
1371LLVM_READNONE
1372constexpr bool isShader(CallingConv::ID CC) {
1373 switch (CC) {
1374 case CallingConv::AMDGPU_VS:
1375 case CallingConv::AMDGPU_LS:
1376 case CallingConv::AMDGPU_HS:
1377 case CallingConv::AMDGPU_ES:
1378 case CallingConv::AMDGPU_GS:
1379 case CallingConv::AMDGPU_PS:
1380 case CallingConv::AMDGPU_CS_Chain:
1381 case CallingConv::AMDGPU_CS_ChainPreserve:
1382 case CallingConv::AMDGPU_CS:
1383 return true;
1384 default:
1385 return false;
1386 }
1387}
1388
1389LLVM_READNONE
1390constexpr bool isGraphics(CallingConv::ID CC) {
1391 return isShader(CC) || CC == CallingConv::AMDGPU_Gfx ||
1392 CC == CallingConv::AMDGPU_Gfx_WholeWave;
1393}
1394
1395LLVM_READNONE
1396constexpr bool isCompute(CallingConv::ID CC) {
1397 return !isGraphics(CC) || CC == CallingConv::AMDGPU_CS;
1398}
1399
1400LLVM_READNONE
1401constexpr bool isEntryFunctionCC(CallingConv::ID CC) {
1402 switch (CC) {
1403 case CallingConv::AMDGPU_KERNEL:
1404 case CallingConv::SPIR_KERNEL:
1405 case CallingConv::AMDGPU_VS:
1406 case CallingConv::AMDGPU_GS:
1407 case CallingConv::AMDGPU_PS:
1408 case CallingConv::AMDGPU_CS:
1409 case CallingConv::AMDGPU_ES:
1410 case CallingConv::AMDGPU_HS:
1411 case CallingConv::AMDGPU_LS:
1412 return true;
1413 default:
1414 return false;
1415 }
1416}
1417
1418LLVM_READNONE
1419constexpr bool isChainCC(CallingConv::ID CC) {
1420 switch (CC) {
1421 case CallingConv::AMDGPU_CS_Chain:
1422 case CallingConv::AMDGPU_CS_ChainPreserve:
1423 return true;
1424 default:
1425 return false;
1426 }
1427}
1428
1429// These functions are considered entrypoints into the current module, i.e. they
1430// are allowed to be called from outside the current module. This is different
1431// from isEntryFunctionCC, which is only true for functions that are entered by
1432// the hardware. Module entry points include all entry functions but also
1433// include functions that can be called from other functions inside or outside
1434// the current module. Module entry functions are allowed to allocate LDS.
1435//
1436// AMDGPU_CS_Chain is intended for externally callable chain functions, so it is
1437// treated as a module entrypoint. AMDGPU_CS_ChainPreserve is used for internal
1438// helper functions (e.g. retry helpers), so it is not a module entrypoint.
1439LLVM_READNONE
1440constexpr bool isModuleEntryFunctionCC(CallingConv::ID CC) {
1441 switch (CC) {
1442 case CallingConv::AMDGPU_Gfx:
1443 case CallingConv::AMDGPU_CS_Chain:
1444 return true;
1445 default:
1446 return isEntryFunctionCC(CC);
1447 }
1448}
1449
1450LLVM_READNONE
1451constexpr inline bool isKernel(CallingConv::ID CC) {
1452 switch (CC) {
1453 case CallingConv::AMDGPU_KERNEL:
1454 case CallingConv::SPIR_KERNEL:
1455 return true;
1456 default:
1457 return false;
1458 }
1459}
1460
1461inline bool isKernel(const Function &F) { return isKernel(CC: F.getCallingConv()); }
1462
1463LLVM_READNONE
1464constexpr bool canGuaranteeTCO(CallingConv::ID CC) {
1465 return CC == CallingConv::Fast;
1466}
1467
1468/// Return true if we might ever do TCO for calls with this calling convention.
1469LLVM_READNONE
1470constexpr bool mayTailCallThisCC(CallingConv::ID CC) {
1471 switch (CC) {
1472 case CallingConv::C:
1473 case CallingConv::AMDGPU_Gfx:
1474 case CallingConv::AMDGPU_Gfx_WholeWave:
1475 return true;
1476 default:
1477 return canGuaranteeTCO(CC);
1478 }
1479}
1480
1481bool hasXNACK(const MCSubtargetInfo &STI);
1482bool hasMIMG_R128(const MCSubtargetInfo &STI);
1483bool hasA16(const MCSubtargetInfo &STI);
1484bool hasG16(const MCSubtargetInfo &STI);
1485bool hasPackedD16(const MCSubtargetInfo &STI);
1486bool hasGDS(const MCSubtargetInfo &STI);
1487unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler = false);
1488unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI);
1489
1490bool isSI(const MCSubtargetInfo &STI);
1491bool isCI(const MCSubtargetInfo &STI);
1492bool isVI(const MCSubtargetInfo &STI);
1493bool isGFX9(const MCSubtargetInfo &STI);
1494bool isGFX9_GFX10(const MCSubtargetInfo &STI);
1495bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI);
1496bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI);
1497bool isGFX8Plus(const MCSubtargetInfo &STI);
1498bool isGFX9Plus(const MCSubtargetInfo &STI);
1499bool isNotGFX9Plus(const MCSubtargetInfo &STI);
1500bool isGFX10(const MCSubtargetInfo &STI);
1501bool isGFX10_GFX11(const MCSubtargetInfo &STI);
1502bool isGFX10Plus(const MCSubtargetInfo &STI);
1503bool isNotGFX10Plus(const MCSubtargetInfo &STI);
1504bool isGFX10Before1030(const MCSubtargetInfo &STI);
1505bool isGFX11(const MCSubtargetInfo &STI);
1506bool isGFX11Plus(const MCSubtargetInfo &STI);
1507bool isGFX12(const MCSubtargetInfo &STI);
1508bool isGFX12Plus(const MCSubtargetInfo &STI);
1509bool isGFX1250(const MCSubtargetInfo &STI);
1510bool isGFX1250Plus(const MCSubtargetInfo &STI);
1511bool isGFX13(const MCSubtargetInfo &STI);
1512bool isGFX13Plus(const MCSubtargetInfo &STI);
1513
1514/// \returns true if a work-group's waves run on all four SIMD32s (one
1515/// contiguous LDS) and not just on two.
1516bool isFullSIMDMode(const MCSubtargetInfo &STI);
1517
1518bool supportsWGP(const MCSubtargetInfo &STI);
1519bool isNotGFX12Plus(const MCSubtargetInfo &STI);
1520bool isNotGFX11Plus(const MCSubtargetInfo &STI);
1521bool isGCN3Encoding(const MCSubtargetInfo &STI);
1522bool isGFX10_BEncoding(const MCSubtargetInfo &STI);
1523bool hasGFX10_3Insts(const MCSubtargetInfo &STI);
1524bool isGFX10_3_GFX11(const MCSubtargetInfo &STI);
1525bool isGFX90A(const MCSubtargetInfo &STI);
1526bool isGFX940(const MCSubtargetInfo &STI);
1527bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI);
1528bool hasMAIInsts(const MCSubtargetInfo &STI);
1529bool hasPopsExitingWaveID(const MCSubtargetInfo &STI);
1530
1531/// \returns true if the src_private_base and src_private_limit aperture
1532/// registers are available on \p STI. Targets with globally addressable
1533/// scratch have no private aperture and expose src_flat_scratch_base instead.
1534bool hasPrivateApertureRegs(const MCSubtargetInfo &STI);
1535
1536bool hasVOPD(const MCSubtargetInfo &STI);
1537bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI);
1538
1539int getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR, int32_t ArgNumVGPR);
1540unsigned hasKernargPreload(const MCSubtargetInfo &STI);
1541bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST);
1542
1543/// Is Reg - scalar register
1544bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI);
1545
1546/// \returns true if \p Reg is an indexed-resource index register, i.e. either a
1547/// 32-bit SGPR (uniform-indexed form) or a Lo256 VGPR (per-lane indexed form).
1548bool isRsrcIndexReg(MCRegister Reg, const MCRegisterInfo &MRI);
1549
1550/// \returns if \p Reg occupies the high 16-bits of a 32-bit register.
1551bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI);
1552
1553/// If \p Reg is a pseudo reg, return the correct hardware register given
1554/// \p STI otherwise return \p Reg.
1555MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI);
1556
1557/// Convert hardware register \p Reg to a pseudo register
1558LLVM_READNONE
1559MCRegister mc2PseudoReg(MCRegister Reg);
1560
1561LLVM_READNONE
1562bool isInlineValue(MCRegister Reg);
1563
1564/// Is this an AMDGPU specific source operand? These include registers,
1565/// inline constants, literals and mandatory literals (KImm).
1566constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo) {
1567 return OpInfo.OperandType >= AMDGPU::OPERAND_SRC_FIRST &&
1568 OpInfo.OperandType <= AMDGPU::OPERAND_SRC_LAST;
1569}
1570
1571inline bool isSISrcOperand(const MCInstrDesc &Desc, unsigned OpNo) {
1572 return isSISrcOperand(OpInfo: Desc.operands()[OpNo]);
1573}
1574
1575/// Is this a scalar (i.e. not packed) bf16 source operand?
1576constexpr bool isBF16SrcOperand(const MCOperandInfo &OpInfo) {
1577 return OpInfo.OperandType == AMDGPU::OPERAND_REG_IMM_BF16 ||
1578 OpInfo.OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16;
1579}
1580
1581/// Is this a KImm operand?
1582bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo);
1583
1584/// Is this floating-point operand?
1585bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo);
1586
1587/// Does this operand support only inlinable literals?
1588bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo);
1589
1590/// Get the size in bits of a register from the register class \p RC.
1591unsigned getRegBitWidth(unsigned RCID);
1592
1593/// Get the size in bits of a register from the register class \p RC.
1594unsigned getRegBitWidth(const MCRegisterClass &RC);
1595
1596LLVM_READNONE
1597inline unsigned getOperandSize(const MCOperandInfo &OpInfo) {
1598 switch (OpInfo.OperandType) {
1599 case AMDGPU::OPERAND_REG_IMM_INT32:
1600 case AMDGPU::OPERAND_REG_IMM_FP32:
1601 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1602 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1603 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1604 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1605 case AMDGPU::OPERAND_REG_IMM_V2INT32:
1606 case AMDGPU::OPERAND_REG_IMM_V2FP32:
1607 case AMDGPU::OPERAND_KIMM32:
1608 case AMDGPU::OPERAND_KIMM16: // mandatory literal is always size 4
1609 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
1610 return 4;
1611
1612 case AMDGPU::OPERAND_REG_IMM_INT64:
1613 case AMDGPU::OPERAND_REG_IMM_FP64:
1614 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1615 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1616 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
1617 case AMDGPU::OPERAND_REG_IMM_V2FP64:
1618 case AMDGPU::OPERAND_REG_IMM_V2INT64:
1619 case AMDGPU::OPERAND_KIMM64:
1620 return 8;
1621
1622 case AMDGPU::OPERAND_REG_IMM_INT16:
1623 case AMDGPU::OPERAND_REG_IMM_BF16:
1624 case AMDGPU::OPERAND_REG_IMM_FP16:
1625 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
1626 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1627 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
1628 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1629 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1630 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
1631 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1632 case AMDGPU::OPERAND_REG_IMM_V2INT16:
1633 case AMDGPU::OPERAND_REG_IMM_V2BF16:
1634 case AMDGPU::OPERAND_REG_IMM_V2FP16:
1635 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
1636 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
1637 return 2;
1638
1639 default:
1640 llvm_unreachable("unhandled operand type");
1641 }
1642}
1643
1644LLVM_READNONE
1645inline unsigned getOperandSize(const MCInstrDesc &Desc, unsigned OpNo) {
1646 return getOperandSize(OpInfo: Desc.operands()[OpNo]);
1647}
1648
1649/// Is this literal inlinable, and not one of the values intended for floating
1650/// point values.
1651LLVM_READNONE
1652inline bool isInlinableIntLiteral(int64_t Literal) {
1653 return Literal >= -16 && Literal <= 64;
1654}
1655
1656/// Is this literal inlinable
1657LLVM_READNONE
1658bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi);
1659
1660LLVM_READNONE
1661bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi);
1662
1663LLVM_READNONE
1664bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi);
1665
1666LLVM_READNONE
1667bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi);
1668
1669LLVM_READNONE
1670bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi);
1671
1672LLVM_READNONE
1673std::optional<unsigned> getInlineEncodingV2I16(uint32_t Literal);
1674
1675LLVM_READNONE
1676std::optional<unsigned> getInlineEncodingV2BF16(uint32_t Literal);
1677
1678LLVM_READNONE
1679std::optional<unsigned> getInlineEncodingV2F16(uint32_t Literal);
1680
1681LLVM_READNONE
1682std::optional<unsigned> getPKFMACF16InlineEncoding(uint32_t Literal,
1683 bool IsGFX11Plus);
1684
1685LLVM_READNONE
1686bool isInlinableLiteralV216(uint32_t Literal, uint8_t OpType);
1687
1688LLVM_READNONE
1689bool isInlinableLiteralV2I16(uint32_t Literal);
1690
1691LLVM_READNONE
1692bool isInlinableLiteralV2BF16(uint32_t Literal);
1693
1694LLVM_READNONE
1695bool isInlinableLiteralV2F16(uint32_t Literal);
1696
1697LLVM_READNONE
1698bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus);
1699
1700LLVM_READNONE
1701bool isValid32BitLiteral(uint64_t Val, bool IsFP64);
1702
1703LLVM_READNONE
1704int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit);
1705
1706bool isArgPassedInSGPR(const Argument *Arg);
1707
1708bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo);
1709
1710/// The opcode is a packed fp32 instruction which only reads low 32 bits of
1711/// a scalar operand and propagates it to high channel.
1712LLVM_READONLY bool isPackedSingleSGPRFP32Inst(unsigned Opc);
1713
1714/// The opcode is a packed 64-bit instruction which only reads low 64 bits of
1715/// a scalar operand and propagates it to high channel.
1716LLVM_READONLY bool isPackedSingleSGPR64BitInst(unsigned Opc);
1717
1718/// Packed instructions that read a single SGPR for SGPR operands, except for
1719/// 64-bit elements which read two SGPRs.
1720LLVM_READONLY bool isSingleSGPRReadInst(unsigned Opc);
1721
1722LLVM_READONLY
1723bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
1724 int64_t EncodedOffset);
1725
1726LLVM_READONLY
1727bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
1728 int64_t EncodedOffset, bool IsBuffer);
1729
1730/// Convert \p ByteOffset to dwords if the subtarget uses dword SMRD immediate
1731/// offsets.
1732uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, uint64_t ByteOffset);
1733
1734/// \returns The encoding that will be used for \p ByteOffset in the
1735/// SMRD offset field, or std::nullopt if it won't fit. On GFX9 and GFX10
1736/// S_LOAD instructions have a signed offset, on other subtargets it is
1737/// unsigned. S_BUFFER has an unsigned offset for all subtargets.
1738std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
1739 int64_t ByteOffset, bool IsBuffer,
1740 bool HasSOffset = false);
1741
1742/// \return The encoding that can be used for a 32-bit literal offset in an SMRD
1743/// instruction. This is only useful on CI.s
1744std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
1745 int64_t ByteOffset);
1746
1747/// For pre-GFX12 FLAT instructions the offset must be positive;
1748/// MSB is ignored and forced to zero.
1749///
1750/// \return The number of bits available for the signed offset field in flat
1751/// instructions. Note that some forms of the instruction disallow negative
1752/// offsets.
1753unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST);
1754
1755LLVM_READNONE
1756inline bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC) {
1757 if (isGFX12(STI: ST))
1758 return DC >= DPP::ROW_SHARE_FIRST && DC <= DPP::ROW_SHARE_LAST;
1759 if (isGFX90A(STI: ST))
1760 return DC >= DPP::ROW_NEWBCAST_FIRST && DC <= DPP::ROW_NEWBCAST_LAST;
1761 return false;
1762}
1763
1764/// \returns true if an instruction may have a 64-bit VGPR operand.
1765bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
1766 const MCSubtargetInfo &ST);
1767
1768/// \returns true if an instruction is a DP ALU DPP without any 64-bit operands.
1769bool isDPALU_DPP32BitOpc(unsigned Opc);
1770
1771/// \returns true if an instruction is a DP ALU DPP.
1772bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
1773 const MCSubtargetInfo &ST);
1774
1775/// \returns true if the intrinsic is divergent
1776bool isIntrinsicSourceOfDivergence(unsigned IntrID);
1777
1778/// \returns true if the intrinsic is uniform
1779bool isIntrinsicAlwaysUniform(unsigned IntrID);
1780
1781/// \returns a register class for the physical register \p Reg if it is a VGPR
1782/// or nullptr otherwise.
1783const MCRegisterClass *getVGPRPhysRegClass(MCRegister Reg,
1784 const MCRegisterInfo &MRI);
1785
1786/// \returns the MODE bits which have to be set by the S_SET_VGPR_MSB for the
1787/// physical register \p Reg.
1788unsigned getVGPREncodingMSBs(MCRegister Reg, const MCRegisterInfo &MRI);
1789
1790/// If \p Reg is a low VGPR return a corresponding high VGPR with \p MSBs set.
1791MCRegister getVGPRWithMSBs(MCRegister Reg, unsigned MSBs,
1792 const MCRegisterInfo &MRI);
1793
1794/// \returns VGPR MSBs encoded in a S_SETREG_IMM32_B32 \p MI if it sets
1795/// it. If \p HasSetregVGPRMSBFixup is true then size of the ID_MODE mask is
1796/// ignored.
1797std::optional<unsigned> convertSetRegImmToVgprMSBs(const MachineInstr &MI,
1798 bool HasSetregVGPRMSBFixup);
1799
1800/// \returns VGPR MSBs encoded in a S_SETREG_IMM32_B32 \p MI if it sets
1801/// it. If \p HasSetregVGPRMSBFixup is true then size of the ID_MODE mask is
1802/// ignored.
1803std::optional<unsigned> convertSetRegImmToVgprMSBs(const MCInst &MI,
1804 bool HasSetregVGPRMSBFixup);
1805
1806// Returns a table for the opcode with a given \p Desc to map the VGPR MSB
1807// set by the S_SET_VGPR_MSB to one of 4 sources. In case of VOPD returns 2
1808// maps, one for X and one for Y component.
1809std::pair<const AMDGPU::OpName *, const AMDGPU::OpName *>
1810getVGPRLoweringOperandTables(const MCInstrDesc &Desc);
1811
1812/// \returns true if a memory instruction supports scale_offset modifier.
1813bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode);
1814
1815/// \returns lds block size in terms of dwords. \p
1816/// This is used to calculate the lds size encoded for PAL metadata 3.0+ which
1817/// must be defined in terms of bytes.
1818unsigned getLdsDwGranularity(const MCSubtargetInfo &ST);
1819
1820class ClusterDimsAttr {
1821public:
1822 enum class Kind { Unknown, NoCluster, VariableDims, FixedDims };
1823
1824 ClusterDimsAttr() = default;
1825
1826 Kind getKind() const { return AttrKind; }
1827
1828 bool isUnknown() const { return getKind() == Kind::Unknown; }
1829
1830 bool isNoCluster() const { return getKind() == Kind::NoCluster; }
1831
1832 bool isFixedDims() const { return getKind() == Kind::FixedDims; }
1833
1834 bool isVariableDims() const { return getKind() == Kind::VariableDims; }
1835
1836 void setUnknown() { *this = ClusterDimsAttr(Kind::Unknown); }
1837
1838 void setNoCluster() { *this = ClusterDimsAttr(Kind::NoCluster); }
1839
1840 void setVariableDims() { *this = ClusterDimsAttr(Kind::VariableDims); }
1841
1842 /// \returns the dims stored. Note that this function can only be called if
1843 /// the kind is \p Fixed.
1844 const std::array<unsigned, 3> &getDims() const;
1845
1846 bool operator==(const ClusterDimsAttr &RHS) const {
1847 return AttrKind == RHS.AttrKind && Dims == RHS.Dims;
1848 }
1849
1850 std::string to_string() const;
1851
1852 static ClusterDimsAttr get(const Function &F);
1853
1854private:
1855 enum Encoding { EncoNoCluster = 0, EncoVariableDims = 1024 };
1856
1857 ClusterDimsAttr(Kind AttrKind) : AttrKind(AttrKind) {}
1858
1859 std::array<unsigned, 3> Dims = {0, 0, 0};
1860
1861 Kind AttrKind = Kind::Unknown;
1862};
1863
1864/// Evaluate the constant-folded result of v_rcp for \p Val, accounting for
1865/// the hardware's denormal flushing on f32/f64 and its approximate rounding.
1866/// Returns std::nullopt if the hardware result is not guaranteed to match the
1867/// exact reciprocal.
1868std::optional<APFloat> evaluateRcp(const APFloat &Val);
1869
1870} // namespace AMDGPU
1871
1872raw_ostream &operator<<(raw_ostream &OS, const AMDGPU::TargetIDSetting S);
1873
1874} // end namespace llvm
1875
1876#endif // LLVM_LIB_TARGET_AMDGPU_UTILS_AMDGPUBASEINFO_H
1877