AMDGPUBaseInfo.cpp source code [llvm_projects/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp]

1	//===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "AMDGPUBaseInfo.h"
10	#include "AMDGPU.h"
11	#include "AMDGPUAsmUtils.h"
12	#include "AMDKernelCodeT.h"
13	#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14	#include "Utils/AMDKernelCodeTUtils.h"
15	#include "llvm/ADT/StringExtras.h"
16	#include "llvm/BinaryFormat/ELF.h"
17	#include "llvm/IR/Attributes.h"
18	#include "llvm/IR/Constants.h"
19	#include "llvm/IR/Function.h"
20	#include "llvm/IR/GlobalValue.h"
21	#include "llvm/IR/IntrinsicsAMDGPU.h"
22	#include "llvm/IR/IntrinsicsR600.h"
23	#include "llvm/IR/LLVMContext.h"
24	#include "llvm/IR/Metadata.h"
25	#include "llvm/MC/MCInstrInfo.h"
26	#include "llvm/MC/MCRegisterInfo.h"
27	#include "llvm/MC/MCSubtargetInfo.h"
28	#include "llvm/Support/CommandLine.h"
29	#include "llvm/TargetParser/AMDGPUTargetParser.h"
30	#include <optional>
31
32	#define GET_INSTRINFO_NAMED_OPS
33	#define GET_INSTRMAP_INFO
34	#include "AMDGPUGenInstrInfo.inc"
35
36	static llvm::cl::opt<unsigned> DefaultAMDHSACodeObjectVersion(
37	"amdhsa-code-object-version", llvm::cl::Hidden,
38	llvm::cl::init(Val: llvm::AMDGPU::AMDHSA_COV6),
39	llvm::cl::desc("Set default AMDHSA Code Object Version (module flag "
40	"or asm directive still take priority if present)"));
41
42	namespace {
43
44	/// \returns Bit mask for given bit \p Shift and bit \p Width.
45	unsigned getBitMask(unsigned Shift, unsigned Width) {
46	return ((`1` << Width) - `1`) << Shift;
47	}
48
49	/// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
50	///
51	/// \returns Packed \p Dst.
52	unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
53	unsigned Mask = getBitMask(Shift, Width);
54	return ((Src << Shift) & Mask) \| (Dst & ~Mask);
55	}
56
57	/// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
58	///
59	/// \returns Unpacked bits.
60	unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
61	return (Src & getBitMask(Shift, Width)) >> Shift;
62	}
63
64	/// \returns Vmcnt bit shift (lower bits).
65	unsigned getVmcntBitShiftLo(unsigned VersionMajor) {
66	return VersionMajor >= `11` ? `10` : `0`;
67	}
68
69	/// \returns Vmcnt bit width (lower bits).
70	unsigned getVmcntBitWidthLo(unsigned VersionMajor) {
71	return VersionMajor >= `11` ? `6` : `4`;
72	}
73
74	/// \returns Expcnt bit shift.
75	unsigned getExpcntBitShift(unsigned VersionMajor) {
76	return VersionMajor >= `11` ? `0` : `4`;
77	}
78
79	/// \returns Expcnt bit width.
80	unsigned getExpcntBitWidth(unsigned VersionMajor) { return `3`; }
81
82	/// \returns Lgkmcnt bit shift.
83	unsigned getLgkmcntBitShift(unsigned VersionMajor) {
84	return VersionMajor >= `11` ? `4` : `8`;
85	}
86
87	/// \returns Lgkmcnt bit width.
88	unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
89	return VersionMajor >= `10` ? `6` : `4`;
90	}
91
92	/// \returns Vmcnt bit shift (higher bits).
93	unsigned getVmcntBitShiftHi(unsigned VersionMajor) { return `14`; }
94
95	/// \returns Vmcnt bit width (higher bits).
96	unsigned getVmcntBitWidthHi(unsigned VersionMajor) {
97	return (VersionMajor == `9` \|\| VersionMajor == `10`) ? `2` : `0`;
98	}
99
100	/// \returns Loadcnt bit width
101	unsigned getLoadcntBitWidth(unsigned VersionMajor) {
102	return VersionMajor >= `12` ? `6` : `0`;
103	}
104
105	/// \returns Samplecnt bit width.
106	unsigned getSamplecntBitWidth(unsigned VersionMajor) {
107	return VersionMajor >= `12` ? `6` : `0`;
108	}
109
110	/// \returns Bvhcnt bit width.
111	unsigned getBvhcntBitWidth(unsigned VersionMajor) {
112	return VersionMajor >= `12` ? `3` : `0`;
113	}
114
115	/// \returns Dscnt bit width.
116	unsigned getDscntBitWidth(unsigned VersionMajor) {
117	return VersionMajor >= `12` ? `6` : `0`;
118	}
119
120	/// \returns Dscnt bit shift in combined S_WAIT instructions.
121	unsigned getDscntBitShift(unsigned VersionMajor) { return `0`; }
122
123	/// \returns Storecnt or Vscnt bit width, depending on VersionMajor.
124	unsigned getStorecntBitWidth(unsigned VersionMajor) {
125	return VersionMajor >= `10` ? `6` : `0`;
126	}
127
128	/// \returns Kmcnt bit width.
129	unsigned getKmcntBitWidth(unsigned VersionMajor) {
130	return VersionMajor >= `12` ? `5` : `0`;
131	}
132
133	/// \returns Xcnt bit width.
134	unsigned getXcntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
135	return VersionMajor == `12` && VersionMinor == `5` ? `6` : `0`;
136	}
137
138	/// \returns Asynccnt bit width.
139	unsigned getAsynccntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
140	return VersionMajor == `12` && VersionMinor == `5` ? `6` : `0`;
141	}
142
143	/// \returns shift for Loadcnt/Storecnt in combined S_WAIT instructions.
144	unsigned getLoadcntStorecntBitShift(unsigned VersionMajor) {
145	return VersionMajor >= `12` ? `8` : `0`;
146	}
147
148	/// \returns VaSdst bit width
149	inline unsigned getVaSdstBitWidth() { return `3`; }
150
151	/// \returns VaSdst bit shift
152	inline unsigned getVaSdstBitShift() { return `9`; }
153
154	/// \returns VmVsrc bit width
155	inline unsigned getVmVsrcBitWidth() { return `3`; }
156
157	/// \returns VmVsrc bit shift
158	inline unsigned getVmVsrcBitShift() { return `2`; }
159
160	/// \returns VaVdst bit width
161	inline unsigned getVaVdstBitWidth() { return `4`; }
162
163	/// \returns VaVdst bit shift
164	inline unsigned getVaVdstBitShift() { return `12`; }
165
166	/// \returns VaVcc bit width
167	inline unsigned getVaVccBitWidth() { return `1`; }
168
169	/// \returns VaVcc bit shift
170	inline unsigned getVaVccBitShift() { return `1`; }
171
172	/// \returns SaSdst bit width
173	inline unsigned getSaSdstBitWidth() { return `1`; }
174
175	/// \returns SaSdst bit shift
176	inline unsigned getSaSdstBitShift() { return `0`; }
177
178	/// \returns VaSsrc width
179	inline unsigned getVaSsrcBitWidth() { return `1`; }
180
181	/// \returns VaSsrc bit shift
182	inline unsigned getVaSsrcBitShift() { return `8`; }
183
184	/// \returns HoldCnt bit shift
185	inline unsigned getHoldCntWidth(unsigned VersionMajor, unsigned VersionMinor) {
186	static constexpr const unsigned MinMajor = `10`;
187	static constexpr const unsigned MinMinor = `3`;
188	return std::tie(args&: VersionMajor, args&: VersionMinor) >= std::tie(args: MinMajor, args: MinMinor)
189	? `1`
190	: `0`;
191	}
192
193	/// \returns HoldCnt bit shift
194	inline unsigned getHoldCntBitShift() { return `7`; }
195
196	} // end anonymous namespace
197
198	namespace llvm {
199
200	namespace AMDGPU {
201
202	/// \returns true if the target supports signed immediate offset for SMRD
203	/// instructions.
204	bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
205	return isGFX9Plus(STI: ST);
206	}
207
208	/// \returns True if \p STI is AMDHSA.
209	bool isHsaAbi(const MCSubtargetInfo &STI) {
210	return STI.getTargetTriple().getOS() == Triple::AMDHSA;
211	}
212
213	unsigned getAMDHSACodeObjectVersion(const Module &M) {
214	if (auto *Ver = mdconst::extract_or_null<ConstantInt>(
215	MD: M.getModuleFlag(Key: "amdhsa_code_object_version"))) {
216	return (unsigned)Ver->getZExtValue() / `100`;
217	}
218
219	return getDefaultAMDHSACodeObjectVersion();
220	}
221
222	unsigned getDefaultAMDHSACodeObjectVersion() {
223	return DefaultAMDHSACodeObjectVersion;
224	}
225
226	unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion) {
227	switch (ABIVersion) {
228	case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
229	return `4`;
230	case ELF::ELFABIVERSION_AMDGPU_HSA_V5:
231	return `5`;
232	case ELF::ELFABIVERSION_AMDGPU_HSA_V6:
233	return `6`;
234	default:
235	return getDefaultAMDHSACodeObjectVersion();
236	}
237	}
238
239	uint8_t getELFABIVersion(const Triple &T, unsigned CodeObjectVersion) {
240	if (T.getOS() != Triple::AMDHSA)
241	return `0`;
242
243	switch (CodeObjectVersion) {
244	case `4`:
245	return ELF::ELFABIVERSION_AMDGPU_HSA_V4;
246	case `5`:
247	return ELF::ELFABIVERSION_AMDGPU_HSA_V5;
248	case `6`:
249	return ELF::ELFABIVERSION_AMDGPU_HSA_V6;
250	default:
251	report_fatal_error(reason: "Unsupported AMDHSA Code Object Version " +
252	Twine (CodeObjectVersion));
253	}
254	}
255
256	unsigned getMultigridSyncArgImplicitArgPosition(unsigned CodeObjectVersion) {
257	switch (CodeObjectVersion) {
258	case AMDHSA_COV4:
259	return `48`;
260	case AMDHSA_COV5:
261	case AMDHSA_COV6:
262	default:
263	return AMDGPU::ImplicitArg::MULTIGRID_SYNC_ARG_OFFSET;
264	}
265	}
266
267	// FIXME: All such magic numbers about the ABI should be in a
268	// central TD file.
269	unsigned getHostcallImplicitArgPosition(unsigned CodeObjectVersion) {
270	switch (CodeObjectVersion) {
271	case AMDHSA_COV4:
272	return `24`;
273	case AMDHSA_COV5:
274	case AMDHSA_COV6:
275	default:
276	return AMDGPU::ImplicitArg::HOSTCALL_PTR_OFFSET;
277	}
278	}
279
280	unsigned getDefaultQueueImplicitArgPosition(unsigned CodeObjectVersion) {
281	switch (CodeObjectVersion) {
282	case AMDHSA_COV4:
283	return `32`;
284	case AMDHSA_COV5:
285	case AMDHSA_COV6:
286	default:
287	return AMDGPU::ImplicitArg::DEFAULT_QUEUE_OFFSET;
288	}
289	}
290
291	unsigned getCompletionActionImplicitArgPosition(unsigned CodeObjectVersion) {
292	switch (CodeObjectVersion) {
293	case AMDHSA_COV4:
294	return `40`;
295	case AMDHSA_COV5:
296	case AMDHSA_COV6:
297	default:
298	return AMDGPU::ImplicitArg::COMPLETION_ACTION_OFFSET;
299	}
300	}
301
302	#define GET_MIMGBaseOpcodesTable_IMPL
303	#define GET_MIMGDimInfoTable_IMPL
304	#define GET_MIMGInfoTable_IMPL
305	#define GET_MIMGLZMappingTable_IMPL
306	#define GET_MIMGMIPMappingTable_IMPL
307	#define GET_MIMGBiasMappingTable_IMPL
308	#define GET_MIMGOffsetMappingTable_IMPL
309	#define GET_MIMGG16MappingTable_IMPL
310	#define GET_MAIInstInfoTable_IMPL
311	#define GET_WMMAInstInfoTable_IMPL
312	#include "AMDGPUGenSearchableTables.inc"
313
314	int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
315	unsigned VDataDwords, unsigned VAddrDwords) {
316	const MIMGInfo *Info =
317	getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, VDataDwords, VAddrDwords);
318	return Info ? Info->Opcode : -`1`;
319	}
320
321	const MIMGBaseOpcodeInfo getMIMGBaseOpcode(unsigned* Opc) {
322	const MIMGInfo *Info = getMIMGInfo(Opcode: Opc);
323	return Info ? getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode) : nullptr;
324	}
325
326	int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
327	const MIMGInfo *OrigInfo = getMIMGInfo(Opcode: Opc);
328	const MIMGInfo *NewInfo =
329	getMIMGOpcodeHelper(BaseOpcode: OrigInfo->BaseOpcode, MIMGEncoding: OrigInfo->MIMGEncoding,
330	VDataDwords: NewChannels, VAddrDwords: OrigInfo->VAddrDwords);
331	return NewInfo ? NewInfo->Opcode : -`1`;
332	}
333
334	unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
335	const MIMGDimInfo Dim, bool* IsA16,
336	bool IsG16Supported) {
337	unsigned AddrWords = BaseOpcode->NumExtraArgs;
338	unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : `0`) +
339	(BaseOpcode->LodOrClampOrMip ? `1` : `0`);
340	if (IsA16)
341	AddrWords += divideCeil(Numerator: AddrComponents, Denominator: `2`);
342	else
343	AddrWords += AddrComponents;
344
345	// Note: For subtargets that support A16 but not G16, enabling A16 also
346	// enables 16 bit gradients.
347	// For subtargets that support A16 (operand) and G16 (done with a different
348	// instruction encoding), they are independent.
349
350	if (BaseOpcode->Gradients) {
351	if ((IsA16 && !IsG16Supported) \|\| BaseOpcode->G16)
352	// There are two gradients per coordinate, we pack them separately.
353	// For the 3d case,
354	// we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv)
355	AddrWords += alignTo<`2`>(Value: Dim->NumGradients / `2`);
356	else
357	AddrWords += Dim->NumGradients;
358	}
359	return AddrWords;
360	}
361
362	struct MUBUFInfo {
363	uint32_t Opcode;
364	uint32_t BaseOpcode;
365	uint8_t elements;
366	bool has_vaddr;
367	bool has_srsrc;
368	bool has_soffset;
369	bool IsBufferInv;
370	bool tfe;
371	};
372
373	struct MTBUFInfo {
374	uint32_t Opcode;
375	uint32_t BaseOpcode;
376	uint8_t elements;
377	bool has_vaddr;
378	bool has_srsrc;
379	bool has_soffset;
380	};
381
382	struct SMInfo {
383	uint32_t Opcode;
384	bool IsBuffer;
385	};
386
387	struct VOPInfo {
388	uint32_t Opcode;
389	bool IsSingle;
390	};
391
392	struct VOPC64DPPInfo {
393	uint32_t Opcode;
394	};
395
396	struct VOPCDPPAsmOnlyInfo {
397	uint32_t Opcode;
398	};
399
400	struct VOP3CDPPAsmOnlyInfo {
401	uint32_t Opcode;
402	};
403
404	struct VOPDComponentInfo {
405	uint16_t BaseVOP;
406	uint16_t VOPDOp;
407	};
408
409	struct VOPDInfo {
410	uint32_t Opcode;
411	uint16_t OpX;
412	uint16_t OpY;
413	uint16_t Subtarget;
414	bool VOPD3;
415	};
416
417	struct VOPTrue16Info {
418	uint32_t Opcode;
419	bool IsTrue16;
420	};
421
422	struct VOPDXYInfo {
423	uint16_t VOPDXYKey;
424	bool IsX;
425	bool IsY;
426	};
427
428	#define GET_FP4FP8DstByteSelTable_DECL
429	#define GET_FP4FP8DstByteSelTable_IMPL
430
431	struct DPMACCInstructionInfo {
432	uint32_t Opcode;
433	bool IsDPMACCInstruction;
434	};
435
436	struct FP4FP8DstByteSelInfo {
437	uint32_t Opcode;
438	bool HasFP8DstByteSel;
439	bool HasFP4DstByteSel;
440	};
441
442	#define GET_DPMACCInstructionTable_DECL
443	#define GET_DPMACCInstructionTable_IMPL
444	#define GET_MTBUFInfoTable_DECL
445	#define GET_MTBUFInfoTable_IMPL
446	#define GET_MUBUFInfoTable_DECL
447	#define GET_MUBUFInfoTable_IMPL
448	#define GET_SMInfoTable_DECL
449	#define GET_SMInfoTable_IMPL
450	#define GET_VOP1InfoTable_DECL
451	#define GET_VOP1InfoTable_IMPL
452	#define GET_VOP2InfoTable_DECL
453	#define GET_VOP2InfoTable_IMPL
454	#define GET_VOP3InfoTable_DECL
455	#define GET_VOP3InfoTable_IMPL
456	#define GET_VOPC64DPPTable_DECL
457	#define GET_VOPC64DPPTable_IMPL
458	#define GET_VOPC64DPP8Table_DECL
459	#define GET_VOPC64DPP8Table_IMPL
460	#define GET_VOPCAsmOnlyInfoTable_DECL
461	#define GET_VOPCAsmOnlyInfoTable_IMPL
462	#define GET_VOP3CAsmOnlyInfoTable_DECL
463	#define GET_VOP3CAsmOnlyInfoTable_IMPL
464	#define GET_VOPDComponentTable_DECL
465	#define GET_VOPDComponentTable_IMPL
466	#define GET_VOPDPairs_DECL
467	#define GET_VOPDPairs_IMPL
468	#define GET_VOPDXYTable_DECL
469	#define GET_VOPDXYTable_IMPL
470	#define GET_VOPTrue16Table_DECL
471	#define GET_VOPTrue16Table_IMPL
472	#define GET_True16D16Table_IMPL
473	#define GET_WMMAOpcode2AddrMappingTable_DECL
474	#define GET_WMMAOpcode2AddrMappingTable_IMPL
475	#define GET_WMMAOpcode3AddrMappingTable_DECL
476	#define GET_WMMAOpcode3AddrMappingTable_IMPL
477	#define GET_getMFMA_F8F6F4_WithSize_DECL
478	#define GET_getMFMA_F8F6F4_WithSize_IMPL
479	#define GET_isMFMA_F8F6F4Table_IMPL
480	#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL
481
482	#include "AMDGPUGenSearchableTables.inc"
483
484	int getMTBUFBaseOpcode(unsigned Opc) {
485	const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opcode: Opc);
486	return Info ? Info->BaseOpcode : -`1`;
487	}
488
489	int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
490	const MTBUFInfo *Info =
491	getMTBUFInfoFromBaseOpcodeAndElements(BaseOpcode: BaseOpc, elements: Elements);
492	return Info ? Info->Opcode : -`1`;
493	}
494
495	int getMTBUFElements(unsigned Opc) {
496	const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
497	return Info ? Info->elements : `0`;
498	}
499
500	bool getMTBUFHasVAddr(unsigned Opc) {
501	const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
502	return Info && Info->has_vaddr;
503	}
504
505	bool getMTBUFHasSrsrc(unsigned Opc) {
506	const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
507	return Info && Info->has_srsrc;
508	}
509
510	bool getMTBUFHasSoffset(unsigned Opc) {
511	const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
512	return Info && Info->has_soffset;
513	}
514
515	int getMUBUFBaseOpcode(unsigned Opc) {
516	const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opcode: Opc);
517	return Info ? Info->BaseOpcode : -`1`;
518	}
519
520	int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
521	const MUBUFInfo *Info =
522	getMUBUFInfoFromBaseOpcodeAndElements(BaseOpcode: BaseOpc, elements: Elements);
523	return Info ? Info->Opcode : -`1`;
524	}
525
526	int getMUBUFElements(unsigned Opc) {
527	const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
528	return Info ? Info->elements : `0`;
529	}
530
531	bool getMUBUFHasVAddr(unsigned Opc) {
532	const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
533	return Info && Info->has_vaddr;
534	}
535
536	bool getMUBUFHasSrsrc(unsigned Opc) {
537	const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
538	return Info && Info->has_srsrc;
539	}
540
541	bool getMUBUFHasSoffset(unsigned Opc) {
542	const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
543	return Info && Info->has_soffset;
544	}
545
546	bool getMUBUFIsBufferInv(unsigned Opc) {
547	const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
548	return Info && Info->IsBufferInv;
549	}
550
551	bool getMUBUFTfe(unsigned Opc) {
552	const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
553	return Info && Info->tfe;
554	}
555
556	bool getSMEMIsBuffer(unsigned Opc) {
557	const SMInfo *Info = getSMEMOpcodeHelper(Opcode: Opc);
558	return Info && Info->IsBuffer;
559	}
560
561	bool getVOP1IsSingle(unsigned Opc) {
562	const VOPInfo *Info = getVOP1OpcodeHelper(Opcode: Opc);
563	return !Info \|\| Info->IsSingle;
564	}
565
566	bool getVOP2IsSingle(unsigned Opc) {
567	const VOPInfo *Info = getVOP2OpcodeHelper(Opcode: Opc);
568	return !Info \|\| Info->IsSingle;
569	}
570
571	bool getVOP3IsSingle(unsigned Opc) {
572	const VOPInfo *Info = getVOP3OpcodeHelper(Opcode: Opc);
573	return !Info \|\| Info->IsSingle;
574	}
575
576	bool isVOPC64DPP(unsigned Opc) {
577	return isVOPC64DPPOpcodeHelper(Opcode: Opc) \|\| isVOPC64DPP8OpcodeHelper(Opcode: Opc);
578	}
579
580	bool isVOPCAsmOnly(unsigned Opc) { return isVOPCAsmOnlyOpcodeHelper(Opcode: Opc); }
581
582	bool getMAIIsDGEMM(unsigned Opc) {
583	const MAIInstInfo *Info = getMAIInstInfoHelper(Opcode: Opc);
584	return Info && Info->is_dgemm;
585	}
586
587	bool getMAIIsGFX940XDL(unsigned Opc) {
588	const MAIInstInfo *Info = getMAIInstInfoHelper(Opcode: Opc);
589	return Info && Info->is_gfx940_xdl;
590	}
591
592	bool getWMMAIsXDL(unsigned Opc) {
593	const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opcode: Opc);
594	return Info ? Info->is_wmma_xdl : false;
595	}
596
597	bool getHasMatrixScale(unsigned Opc) {
598	const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opcode: Opc);
599	return Info && Info->HasMatrixScale;
600	}
601
602	uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal) {
603	switch (EncodingVal) {
604	case MFMAScaleFormats::FP6_E2M3:
605	case MFMAScaleFormats::FP6_E3M2:
606	return `6`;
607	case MFMAScaleFormats::FP4_E2M1:
608	return `4`;
609	case MFMAScaleFormats::FP8_E4M3:
610	case MFMAScaleFormats::FP8_E5M2:
611	default:
612	return `8`;
613	}
614
615	llvm_unreachable("covered switch over mfma scale formats");
616	}
617
618	const MFMA_F8F6F4_Info getMFMA_F8F6F4_WithFormatArgs(unsigned* CBSZ,
619	unsigned BLGP,
620	unsigned F8F8Opcode) {
621	uint8_t SrcANumRegs = mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: CBSZ);
622	uint8_t SrcBNumRegs = mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: BLGP);
623	return getMFMA_F8F6F4_InstWithNumRegs(NumRegsSrcA: SrcANumRegs, NumRegsSrcB: SrcBNumRegs, F8F8Opcode);
624	}
625
626	uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt) {
627	switch (Fmt) {
628	case WMMA::MATRIX_FMT_FP8:
629	case WMMA::MATRIX_FMT_BF8:
630	return `16`;
631	case WMMA::MATRIX_FMT_FP6:
632	case WMMA::MATRIX_FMT_BF6:
633	return `12`;
634	case WMMA::MATRIX_FMT_FP4:
635	return `8`;
636	}
637
638	llvm_unreachable("covered switch over wmma scale formats");
639	}
640
641	const MFMA_F8F6F4_Info getWMMA_F8F6F4_WithFormatArgs(unsigned* FmtA,
642	unsigned FmtB,
643	unsigned F8F8Opcode) {
644	uint8_t SrcANumRegs = wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtA);
645	uint8_t SrcBNumRegs = wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtB);
646	return getMFMA_F8F6F4_InstWithNumRegs(NumRegsSrcA: SrcANumRegs, NumRegsSrcB: SrcBNumRegs, F8F8Opcode);
647	}
648
649	bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale,
650	unsigned BFmt, unsigned BScale) {
651	auto isValid = [](unsigned Fmt, unsigned Scale) -> bool {
652	switch (Fmt) {
653	case WMMA::MATRIX_FMT_FP8:
654	case WMMA::MATRIX_FMT_BF8:
655	case WMMA::MATRIX_FMT_FP6:
656	case WMMA::MATRIX_FMT_BF6:
657	if (Scale != WMMA::MATRIX_SCALE_FMT_E8)
658	return false;
659	break;
660	case WMMA::MATRIX_FMT_FP4:
661	if (Scale != WMMA::MATRIX_SCALE_FMT_E8 &&
662	Scale != WMMA::MATRIX_SCALE_FMT_E5M3 &&
663	Scale != WMMA::MATRIX_SCALE_FMT_E4M3)
664	return false;
665	break;
666	}
667	return true;
668	};
669
670	if (!isValid (AFmt, AScale) \|\| !isValid (BFmt, BScale))
671	return false;
672
673	if (AFmt == WMMA::MATRIX_FMT_FP4 && BFmt == WMMA::MATRIX_FMT_FP4 &&
674	AScale != BScale)
675	return false;
676
677	return true;
678	}
679
680	unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST) {
681	if (ST.hasFeature(Feature: AMDGPU::FeatureGFX13Insts))
682	return SIEncodingFamily::GFX13;
683	if (ST.hasFeature(Feature: AMDGPU::FeatureGFX1250Insts))
684	return SIEncodingFamily::GFX1250;
685	if (ST.hasFeature(Feature: AMDGPU::FeatureGFX12Insts))
686	return SIEncodingFamily::GFX12;
687	if (ST.hasFeature(Feature: AMDGPU::FeatureGFX11_7Insts))
688	return SIEncodingFamily::GFX1170;
689	if (ST.hasFeature(Feature: AMDGPU::FeatureGFX11Insts))
690	return SIEncodingFamily::GFX11;
691	llvm_unreachable("Subtarget generation does not support VOPD!");
692	}
693
694	CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3) {
695	bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : `0`;
696	Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
697	// Normalize through VOPDComponentTable so that e32 and e64 variants
698	// of the same logical opcode all share a single entry.
699	const VOPDComponentInfo *Info = getVOPDComponentHelper(BaseVOP: Opc);
700	if (!Info)
701	return {.X: false, .Y: false};
702	unsigned Key =
703	(Info->VOPDOp << `5`) \| (EncodingFamily << `1`) \| (VOPD3 ? `1u` : `0u`);
704	const VOPDXYInfo *XYInfo = getVOPDXYInfo(VOPDXYKey: Key);
705	if (!XYInfo)
706	return {.X: false, .Y: false};
707	return {.X: XYInfo->IsX, .Y: XYInfo->IsY};
708	}
709
710	unsigned getVOPDOpcode(unsigned Opc, bool VOPD3) {
711	bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : `0`;
712	Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
713	const VOPDComponentInfo *Info = getVOPDComponentHelper(BaseVOP: Opc);
714	return Info ? Info->VOPDOp : ~`0u`;
715	}
716
717	bool isVOPD(unsigned Opc) {
718	return AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::src0X);
719	}
720
721	bool isMAC(unsigned Opc) {
722	return Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 \|\|
723	Opc == AMDGPU::V_MAC_F32_e64_gfx10 \|\|
724	Opc == AMDGPU::V_MAC_F32_e64_vi \|\|
725	Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx6_gfx7 \|\|
726	Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx10 \|\|
727	Opc == AMDGPU::V_MAC_F16_e64_vi \|\|
728	Opc == AMDGPU::V_FMAC_F64_e64_gfx90a \|\|
729	Opc == AMDGPU::V_FMAC_F64_e64_gfx12 \|\|
730	Opc == AMDGPU::V_FMAC_F64_e64_gfx13 \|\|
731	Opc == AMDGPU::V_FMAC_F32_e64_gfx10 \|\|
732	Opc == AMDGPU::V_FMAC_F32_e64_gfx11 \|\|
733	Opc == AMDGPU::V_FMAC_F32_e64_gfx12 \|\|
734	Opc == AMDGPU::V_FMAC_F32_e64_gfx13 \|\|
735	Opc == AMDGPU::V_FMAC_F32_e64_vi \|\|
736	Opc == AMDGPU::V_FMAC_LEGACY_F32_e64_gfx10 \|\|
737	Opc == AMDGPU::V_FMAC_DX9_ZERO_F32_e64_gfx11 \|\|
738	Opc == AMDGPU::V_FMAC_F16_e64_gfx10 \|\|
739	Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx11 \|\|
740	Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx11 \|\|
741	Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx12 \|\|
742	Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx12 \|\|
743	Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx13 \|\|
744	Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx13 \|\|
745	Opc == AMDGPU::V_DOT2C_F32_F16_e64_vi \|\|
746	Opc == AMDGPU::V_DOT2C_F32_BF16_e64_vi \|\|
747	Opc == AMDGPU::V_DOT2C_I32_I16_e64_vi \|\|
748	Opc == AMDGPU::V_DOT4C_I32_I8_e64_vi \|\|
749	Opc == AMDGPU::V_DOT8C_I32_I4_e64_vi;
750	}
751
752	bool isPermlane16(unsigned Opc) {
753	return Opc == AMDGPU::V_PERMLANE16_B32_gfx10 \|\|
754	Opc == AMDGPU::V_PERMLANEX16_B32_gfx10 \|\|
755	Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx11 \|\|
756	Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx11 \|\|
757	Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx12 \|\|
758	Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx13 \|\|
759	Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx12 \|\|
760	Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx13 \|\|
761	Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx12 \|\|
762	Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx13 \|\|
763	Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx12 \|\|
764	Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx13;
765	}
766
767	bool isCvt_F32_Fp8_Bf8_e64(unsigned Opc) {
768	return Opc == AMDGPU::V_CVT_F32_BF8_e64_gfx12 \|\|
769	Opc == AMDGPU::V_CVT_F32_FP8_e64_gfx12 \|\|
770	Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp_gfx12 \|\|
771	Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp_gfx12 \|\|
772	Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp8_gfx12 \|\|
773	Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp8_gfx12 \|\|
774	Opc == AMDGPU::V_CVT_PK_F32_BF8_fake16_e64_gfx12 \|\|
775	Opc == AMDGPU::V_CVT_PK_F32_FP8_fake16_e64_gfx12 \|\|
776	Opc == AMDGPU::V_CVT_PK_F32_BF8_t16_e64_gfx12 \|\|
777	Opc == AMDGPU::V_CVT_PK_F32_FP8_t16_e64_gfx12;
778	}
779
780	bool isGenericAtomic(unsigned Opc) {
781	return Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP \|\|
782	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD \|\|
783	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB \|\|
784	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN \|\|
785	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN \|\|
786	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX \|\|
787	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX \|\|
788	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND \|\|
789	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR \|\|
790	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR \|\|
791	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC \|\|
792	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC \|\|
793	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD \|\|
794	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN \|\|
795	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX \|\|
796	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP \|\|
797	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32 \|\|
798	Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32 \|\|
799	Opc == AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG;
800	}
801
802	bool isAsyncStore(unsigned Opc) {
803	return Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_gfx1250 \|\|
804	Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_gfx1250 \|\|
805	Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_gfx1250 \|\|
806	Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_gfx1250 \|\|
807	Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR_gfx1250 \|\|
808	Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR_gfx1250 \|\|
809	Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR_gfx1250 \|\|
810	Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR_gfx1250;
811	}
812
813	bool isTensorStore(unsigned Opc) {
814	return Opc == TENSOR_STORE_FROM_LDS_d2_gfx1250 \|\|
815	Opc == TENSOR_STORE_FROM_LDS_d4_gfx1250;
816	}
817
818	unsigned getTemporalHintType(const MCInstrDesc TID) {
819	if (SIInstrFlags::isAtomic(O: TID))
820	return CPol::TH_TYPE_ATOMIC;
821	unsigned Opc = TID.getOpcode();
822	// Async and Tensor store should have the temporal hint type of TH_TYPE_STORE
823	if (TID.mayStore() &&
824	(isAsyncStore(Opc) \|\| isTensorStore(Opc) \|\| !TID.mayLoad()))
825	return CPol::TH_TYPE_STORE;
826
827	// This will default to returning TH_TYPE_LOAD when neither MayStore nor
828	// MayLoad flag is present which is the case with instructions like
829	// image_get_resinfo.
830	return CPol::TH_TYPE_LOAD;
831	}
832
833	bool isTrue16Inst(unsigned Opc) {
834	const VOPTrue16Info *Info = getTrue16OpcodeHelper(Opcode: Opc);
835	return Info && Info->IsTrue16;
836	}
837
838	FPType getFPDstSelType(unsigned Opc) {
839	const FP4FP8DstByteSelInfo *Info = getFP4FP8DstByteSelHelper(Opcode: Opc);
840	if (!Info)
841	return FPType::None;
842	if (Info->HasFP8DstByteSel)
843	return FPType::FP8;
844	if (Info->HasFP4DstByteSel)
845	return FPType::FP4;
846
847	return FPType::None;
848	}
849
850	bool isDPMACCInstruction(unsigned Opc) {
851	const DPMACCInstructionInfo *Info = getDPMACCInstructionHelper(Opcode: Opc);
852	return Info && Info->IsDPMACCInstruction;
853	}
854
855	unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc) {
856	const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom2AddrOpcode(Opcode2Addr: Opc);
857	return Info ? Info->Opcode3Addr : ~`0u`;
858	}
859
860	unsigned mapWMMA3AddrTo2AddrOpcode(unsigned Opc) {
861	const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom3AddrOpcode(Opcode3Addr: Opc);
862	return Info ? Info->Opcode2Addr : ~`0u`;
863	}
864
865	// Wrapper for Tablegen'd function. enum Subtarget is not defined in any
866	// header files, so we need to wrap it in a function that takes unsigned
867	// instead.
868	int32_t getMCOpcode(uint32_t Opcode, unsigned Gen) {
869	return getMCOpcodeGen(Opcode, inSubtarget: static_cast<Subtarget>(Gen));
870	}
871
872	unsigned getBitOp2(unsigned Opc) {
873	switch (Opc) {
874	default:
875	return `0`;
876	case AMDGPU::V_AND_B32_e32:
877	return `0x40`;
878	case AMDGPU::V_OR_B32_e32:
879	return `0x54`;
880	case AMDGPU::V_XOR_B32_e32:
881	return `0x14`;
882	case AMDGPU::V_XNOR_B32_e32:
883	return `0x41`;
884	}
885	}
886
887	int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily,
888	bool VOPD3) {
889	bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc: OpY) : `0`;
890	OpY = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : OpY;
891	const VOPDInfo *Info =
892	getVOPDInfoFromComponentOpcodes(OpX, OpY, SubTgt: EncodingFamily, VOPD3);
893	return Info ? Info->Opcode : -`1`;
894	}
895
896	std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode) {
897	const VOPDInfo *Info = getVOPDOpcodeHelper(Opcode: VOPDOpcode);
898	assert(Info);
899	const auto *OpX = getVOPDBaseFromComponent(VOPDOp: Info->OpX);
900	const auto *OpY = getVOPDBaseFromComponent(VOPDOp: Info->OpY);
901	assert(OpX && OpY);
902	return {OpX->BaseVOP, OpY->BaseVOP};
903	}
904
905	namespace VOPD {
906
907	ComponentProps::ComponentProps(const MCInstrDesc &OpDesc, bool VOP3Layout) {
908	assert(OpDesc.getNumDefs() == Component::DST_NUM);
909
910	assert(OpDesc.getOperandConstraint(Component::SRC0, MCOI::TIED_TO) == -`1`);
911	assert(OpDesc.getOperandConstraint(Component::SRC1, MCOI::TIED_TO) == -`1`);
912	auto TiedIdx = OpDesc.getOperandConstraint(OpNum: Component::SRC2, Constraint: MCOI::TIED_TO);
913	assert(TiedIdx == -`1` \|\| TiedIdx == Component::DST);
914	HasSrc2Acc = TiedIdx != -`1`;
915	Opcode = OpDesc.getOpcode();
916
917	IsVOP3 = VOP3Layout \|\| SIInstrFlags::isVOP3(O: OpDesc);
918	SrcOperandsNum = AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src2) ? `3`
919	: AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::imm) ? `3`
920	: AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src1) ? `2`
921	: `1`;
922	assert(SrcOperandsNum <= Component::MAX_SRC_NUM);
923
924	if (Opcode == AMDGPU::V_CNDMASK_B32_e32 \|\|
925	Opcode == AMDGPU::V_CNDMASK_B32_e64) {
926	// CNDMASK is an awkward exception, it has FP modifiers, but not FP
927	// operands.
928	NumVOPD3Mods = `2`;
929	if (IsVOP3)
930	SrcOperandsNum = `3`;
931	} else if (isSISrcFPOperand(Desc: OpDesc,
932	OpNo: getNamedOperandIdx(Opcode, Name: OpName::src0))) {
933	// All FP VOPD instructions have Neg modifiers for all operands except
934	// for tied src2.
935	NumVOPD3Mods = SrcOperandsNum;
936	if (HasSrc2Acc)
937	--NumVOPD3Mods;
938	}
939
940	if (SIInstrFlags::isVOP3(O: OpDesc))
941	return;
942
943	auto OperandsNum = OpDesc.getNumOperands();
944	unsigned CompOprIdx;
945	for (CompOprIdx = Component::SRC1; CompOprIdx < OperandsNum; ++CompOprIdx) {
946	if (OpDesc.operands()[CompOprIdx].OperandType == AMDGPU::OPERAND_KIMM32) {
947	MandatoryLiteralIdx = CompOprIdx;
948	break;
949	}
950	}
951	}
952
953	int ComponentProps::getBitOp3OperandIdx() const {
954	return getNamedOperandIdx(Opcode, Name: OpName::bitop3);
955	}
956
957	unsigned ComponentInfo::getIndexInParsedOperands(unsigned CompOprIdx) const {
958	assert(CompOprIdx < Component::MAX_OPR_NUM);
959
960	if (CompOprIdx == Component::DST)
961	return getIndexOfDstInParsedOperands();
962
963	auto CompSrcIdx = CompOprIdx - Component::DST_NUM;
964	if (CompSrcIdx < getCompParsedSrcOperandsNum())
965	return getIndexOfSrcInParsedOperands(CompSrcIdx);
966
967	// The specified operand does not exist.
968	return `0`;
969	}
970
971	std::optional<unsigned> InstInfo::getInvalidCompOperandIndex(
972	std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
973	const MCRegisterInfo &MRI, bool SkipSrc, bool AllowSameVGPR,
974	bool VOPD3) const {
975
976	auto OpXRegs = getRegIndices(ComponentIdx: ComponentIndex::X, GetRegIdx,
977	VOPD3: CompInfo[ComponentIndex::X].isVOP3());
978	auto OpYRegs = getRegIndices(ComponentIdx: ComponentIndex::Y, GetRegIdx,
979	VOPD3: CompInfo[ComponentIndex::Y].isVOP3());
980
981	const auto banksOverlap = [&MRI](MCRegister X, MCRegister Y,
982	unsigned BanksMask) -> bool {
983	MCRegister BaseX = MRI.getSubReg(Reg: X, Idx: AMDGPU::sub0);
984	MCRegister BaseY = MRI.getSubReg(Reg: Y, Idx: AMDGPU::sub0);
985	if (!BaseX)
986	BaseX = X;
987	if (!BaseY)
988	BaseY = Y;
989	if ((BaseX.id() & BanksMask) == (BaseY.id() & BanksMask))
990	return true;
991	if (BaseX != X / This is 64-bit register / &&
992	((BaseX.id() + `1`) & BanksMask) == (BaseY.id() & BanksMask))
993	return true;
994	if (BaseY != Y &&
995	(BaseX.id() & BanksMask) == ((BaseY.id() + `1`) & BanksMask))
996	return true;
997
998	// If both are 64-bit bank conflict will be detected yet while checking
999	// the first subreg.
1000	return false;
1001	};
1002
1003	unsigned CompOprIdx;
1004	for (CompOprIdx = `0`; CompOprIdx < Component::MAX_OPR_NUM; ++CompOprIdx) {
1005	unsigned BanksMasks = VOPD3 ? VOPD3_VGPR_BANK_MASKS[CompOprIdx]
1006	: VOPD_VGPR_BANK_MASKS[CompOprIdx];
1007	if (!OpXRegs [CompOprIdx] \|\| !OpYRegs [CompOprIdx])
1008	continue;
1009
1010	if (getVGPREncodingMSBs(Reg: OpXRegs [CompOprIdx], MRI) !=
1011	getVGPREncodingMSBs(Reg: OpYRegs [CompOprIdx], MRI))
1012	return CompOprIdx;
1013
1014	if (SkipSrc && CompOprIdx >= Component::DST_NUM)
1015	continue;
1016
1017	if (CompOprIdx < Component::DST_NUM) {
1018	// Even if we do not check vdst parity, vdst operands still shall not
1019	// overlap.
1020	if (MRI.regsOverlap(RegA: OpXRegs [CompOprIdx], RegB: OpYRegs [CompOprIdx]))
1021	return CompOprIdx;
1022	if (VOPD3) // No need to check dst parity.
1023	continue;
1024	}
1025
1026	if (banksOverlap (OpXRegs [CompOprIdx], OpYRegs [CompOprIdx], BanksMasks) &&
1027	(!AllowSameVGPR \|\| CompOprIdx < Component::DST_NUM \|\|
1028	OpXRegs [CompOprIdx] != OpYRegs [CompOprIdx]))
1029	return CompOprIdx;
1030	}
1031
1032	return {};
1033	}
1034
1035	// Return an array of VGPR registers [DST,SRC0,SRC1,SRC2] used
1036	// by the specified component. If an operand is unused
1037	// or is not a VGPR, the corresponding value is 0.
1038	//
1039	// GetRegIdx(Component, MCOperandIdx) must return a VGPR register index
1040	// for the specified component and MC operand. The callback must return 0
1041	// if the operand is not a register or not a VGPR.
1042	InstInfo::RegIndices
1043	InstInfo::getRegIndices(unsigned CompIdx,
1044	std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
1045	bool VOPD3) const {
1046	assert(CompIdx < COMPONENTS_NUM);
1047
1048	const auto &Comp = CompInfo[CompIdx];
1049	InstInfo::RegIndices RegIndices;
1050
1051	RegIndices [DST] = GetRegIdx (CompIdx, Comp.getIndexOfDstInMCOperands());
1052
1053	for (unsigned CompOprIdx : {SRC0, SRC1, SRC2}) {
1054	unsigned CompSrcIdx = CompOprIdx - DST_NUM;
1055	RegIndices [CompOprIdx] =
1056	Comp.hasRegSrcOperand(CompSrcIdx)
1057	? GetRegIdx (CompIdx,
1058	Comp.getIndexOfSrcInMCOperands(CompSrcIdx, VOPD3))
1059	: MCRegister ();
1060	}
1061	return RegIndices;
1062	}
1063
1064	} // namespace VOPD
1065
1066	VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY) {
1067	return VOPD::InstInfo(OpX, OpY);
1068	}
1069
1070	VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,
1071	const MCInstrInfo *InstrInfo) {
1072	auto [OpX, OpY] = getVOPDComponents(VOPDOpcode);
1073	const auto &OpXDesc = InstrInfo->get(Opcode: OpX);
1074	const auto &OpYDesc = InstrInfo->get(Opcode: OpY);
1075	bool VOPD3 = SIInstrFlags::isVOPD3(O: *InstrInfo, O: VOPDOpcode);
1076	VOPD::ComponentInfo OpXInfo(OpXDesc, VOPD::ComponentKind::COMPONENT_X, VOPD3);
1077	VOPD::ComponentInfo OpYInfo(OpYDesc, OpXInfo, VOPD3);
1078	return VOPD::InstInfo(OpXInfo, OpYInfo);
1079	}
1080
1081	TargetID createAMDGPUTargetID(const MCSubtargetInfo &STI,
1082	StringRef FeatureString) {
1083	TargetID TargetID(parseArchAMDGCN(CPU: STI.getCPU()), STI.getTargetTriple(),
1084	STI.getFeatureBits().test(I: FeatureXNACKOnOffModes)
1085	? TargetIDSetting::Any
1086	: TargetIDSetting::Unsupported,
1087	STI.getFeatureBits().test(I: FeatureSupportsSRAMECC)
1088	? TargetIDSetting::Any
1089	: TargetIDSetting::Unsupported);
1090
1091	// Check if xnack or sramecc is explicitly enabled or disabled. In the
1092	// absence of the target features we assume we must generate code that can run
1093	// in any environment.
1094	SubtargetFeatures Features(FeatureString);
1095	std::optional<bool> XnackRequested;
1096	std::optional<bool> SramEccRequested;
1097
1098	for (const std::string &Feature : Features.getFeatures()) {
1099	if (Feature == "+xnack")
1100	XnackRequested = true;
1101	else if (Feature == "-xnack")
1102	XnackRequested = false;
1103	else if (Feature == "+sramecc")
1104	SramEccRequested = true;
1105	else if (Feature == "-sramecc")
1106	SramEccRequested = false;
1107	}
1108
1109	// Only allow changing xnack setting if the target supports on/off modes.
1110	// Targets without on/off mode support keep their initial setting
1111	// (Unsupported).
1112
1113	bool XnackSupported = STI.getFeatureBits().test(I: FeatureXNACKOnOffModes);
1114	bool SramEccSupported = TargetID.isSramEccSupported();
1115
1116	if (XnackRequested) {
1117	if (XnackSupported) {
1118	TargetID.setXnackSetting(*XnackRequested ? TargetIDSetting::On
1119	: TargetIDSetting::Off);
1120	} else {
1121	// If a specific xnack setting was requested and this GPU does not support
1122	// xnack emit a warning. Setting will remain set to "Unsupported".
1123	if (*XnackRequested) {
1124	errs() << "warning: xnack 'On' was requested for a processor that does "
1125	"not support it!\n";
1126	} else {
1127	errs() << "warning: xnack 'Off' was requested for a processor that "
1128	"does not support it!\n";
1129	}
1130	}
1131	}
1132
1133	if (SramEccRequested) {
1134	if (SramEccSupported) {
1135	TargetID.setSramEccSetting(*SramEccRequested ? TargetIDSetting::On
1136	: TargetIDSetting::Off);
1137	} else {
1138	// If a specific sramecc setting was requested and this GPU does not
1139	// support sramecc emit a warning. Setting will remain set to
1140	// "Unsupported".
1141	if (*SramEccRequested) {
1142	errs() << "warning: sramecc 'On' was requested for a processor that "
1143	"does not support it!\n";
1144	} else {
1145	errs() << "warning: sramecc 'Off' was requested for a processor that "
1146	"does not support it!\n";
1147	}
1148	}
1149	}
1150
1151	return TargetID;
1152	}
1153
1154	namespace IsaInfo {
1155
1156	unsigned getInstCacheLineSize(const MCSubtargetInfo &STI) {
1157	if (STI.getFeatureBits().test(I: FeatureInstCacheLineSize128))
1158	return `128`;
1159	if (STI.getFeatureBits().test(I: FeatureInstCacheLineSize64))
1160	return `64`;
1161	return `64`;
1162	}
1163
1164	unsigned getWavefrontSize(const MCSubtargetInfo &STI) {
1165	if (STI.getFeatureBits().test(I: FeatureWavefrontSize16))
1166	return `16`;
1167	if (STI.getFeatureBits().test(I: FeatureWavefrontSize32))
1168	return `32`;
1169
1170	return `64`;
1171	}
1172
1173	unsigned getLocalMemorySize(const MCSubtargetInfo &STI) {
1174	unsigned BytesPerCU = getAddressableLocalMemorySize(STI);
1175
1176	// "Per CU" really means "per whatever functional block the waves of a
1177	// workgroup must share". So the effective local memory size is doubled in
1178	// WGP mode on gfx10.
1179	if (isGFX10Plus(STI) && !STI.getFeatureBits().test(I: FeatureCuMode))
1180	BytesPerCU *= `2`;
1181
1182	return BytesPerCU;
1183	}
1184
1185	unsigned getAddressableLocalMemorySize(const MCSubtargetInfo &STI) {
1186	if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize32768))
1187	return `32768`;
1188	if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize65536))
1189	return `65536`;
1190	if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize163840))
1191	return `163840`;
1192	if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize327680))
1193	return `327680`;
1194	return `32768`;
1195	}
1196
1197	unsigned getEUsPerCU(const MCSubtargetInfo &STI) {
1198	// "Per CU" really means "per whatever functional block the waves of a
1199	// workgroup must share".
1200
1201	// GFX12.5 only supports CU mode, which contains four SIMDs.
1202	if (isGFX1250(STI)) {
1203	assert(STI.getFeatureBits().test(FeatureCuMode));
1204	return `4`;
1205	}
1206
1207	// For gfx10 in CU mode the functional block is the CU, which contains
1208	// two SIMDs.
1209	if (isGFX10Plus(STI) && STI.getFeatureBits().test(I: FeatureCuMode))
1210	return `2`;
1211
1212	// Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP
1213	// contains two CUs, so a total of four SIMDs.
1214	return `4`;
1215	}
1216
1217	unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo &STI,
1218	unsigned FlatWorkGroupSize) {
1219	assert(FlatWorkGroupSize != `0`);
1220	if (!STI.getTargetTriple().isAMDGCN())
1221	return `8`;
1222	unsigned MaxWaves = getMaxWavesPerEU(STI) * getEUsPerCU(STI);
1223	unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
1224	if (N == `1`) {
1225	// Single-wave workgroups don't consume barrier resources.
1226	return MaxWaves;
1227	}
1228
1229	unsigned MaxBarriers = `16`;
1230	if (isGFX10Plus(STI) && !STI.getFeatureBits().test(I: FeatureCuMode))
1231	MaxBarriers = `32`;
1232
1233	return std::min(a: MaxWaves / N, b: MaxBarriers);
1234	}
1235
1236	unsigned getMinWavesPerEU(const MCSubtargetInfo &STI) { return `1`; }
1237
1238	unsigned getMaxWavesPerEU(const MCSubtargetInfo &STI) {
1239	// FIXME: Need to take scratch memory into account.
1240	if (isGFX90A(STI))
1241	return `8`;
1242	if (!isGFX10Plus(STI))
1243	return `10`;
1244	return hasGFX10_3Insts(STI) ? `16` : `20`;
1245	}
1246
1247	unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo &STI,
1248	unsigned FlatWorkGroupSize) {
1249	return divideCeil(Numerator: getWavesPerWorkGroup(STI, FlatWorkGroupSize),
1250	Denominator: getEUsPerCU(STI));
1251	}
1252
1253	unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo &STI) { return `1`; }
1254
1255	unsigned getWavesPerWorkGroup(const MCSubtargetInfo &STI,
1256	unsigned FlatWorkGroupSize) {
1257	return divideCeil(Numerator: FlatWorkGroupSize, Denominator: getWavefrontSize(STI));
1258	}
1259
1260	unsigned getSGPRAllocGranule(const MCSubtargetInfo &STI) {
1261	IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1262	if (Version.Major >= `10`)
1263	return getAddressableNumSGPRs(STI);
1264	if (Version.Major >= `8`)
1265	return `16`;
1266	return `8`;
1267	}
1268
1269	unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI) { return `8`; }
1270
1271	unsigned getTotalNumSGPRs(const MCSubtargetInfo &STI) {
1272	IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1273	if (Version.Major >= `8`)
1274	return `800`;
1275	return `512`;
1276	}
1277
1278	unsigned getAddressableNumSGPRs(const MCSubtargetInfo &STI) {
1279	if (STI.getFeatureBits().test(I: FeatureSGPRInitBug))
1280	return FIXED_NUM_SGPRS_FOR_INIT_BUG;
1281
1282	IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1283	if (Version.Major >= `10`)
1284	return `106`;
1285	if (Version.Major >= `8`)
1286	return `102`;
1287	return `104`;
1288	}
1289
1290	// Per-wave SGPRs reserved for the trap handler when enabled.
1291	static unsigned getSGPRTrapHandlerReserve(const MCSubtargetInfo &STI) {
1292	return STI.getFeatureBits().test(I: FeatureTrapHandler) ? TRAP_NUM_SGPRS : `0`;
1293	}
1294
1295	// Per-wave SGPR budget (before the addressable clamp): take off the trap
1296	// reserve, round down to \p Granule. Shared by getMinNumSGPRs() and
1297	// getMaxNumSGPRs(); getOccupancyWithNumSGPRs() is the closed-form algebraic
1298	// inverse of this same budget (it does not call this helper), so the two encode
1299	// one model.
1300	static unsigned getSGPRBudgetPerWave(unsigned TotalNumSGPRs,
1301	unsigned WavesPerEU, unsigned TrapReserve,
1302	unsigned Granule) {
1303	assert(WavesPerEU != `0` && Granule != `0`);
1304	unsigned Budget = TotalNumSGPRs / WavesPerEU;
1305	Budget -= std::min(a: Budget, b: TrapReserve);
1306	return alignDown(Value: Budget, Align: Granule);
1307	}
1308
1309	unsigned getMinNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU) {
1310	assert(WavesPerEU != `0`);
1311
1312	IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1313	if (Version.Major >= `10`)
1314	return `0`;
1315
1316	if (WavesPerEU >= getMaxWavesPerEU(STI))
1317	return `0`;
1318
1319	unsigned MinNumSGPRs =
1320	getSGPRBudgetPerWave(TotalNumSGPRs: getTotalNumSGPRs(STI), WavesPerEU: WavesPerEU + `1`,
1321	TrapReserve: getSGPRTrapHandlerReserve(STI),
1322	Granule: getSGPRAllocGranule(STI)) +
1323	`1`;
1324	return std::min(a: MinNumSGPRs, b: getAddressableNumSGPRs(STI));
1325	}
1326
1327	unsigned getMaxNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1328	bool Addressable) {
1329	assert(WavesPerEU != `0`);
1330
1331	unsigned AddressableNumSGPRs = getAddressableNumSGPRs(STI);
1332	IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1333	if (Version.Major >= `10`)
1334	return Addressable ? AddressableNumSGPRs : `108`;
1335	if (Version.Major >= `8` && !Addressable)
1336	AddressableNumSGPRs = `112`;
1337	unsigned MaxNumSGPRs = getSGPRBudgetPerWave(TotalNumSGPRs: getTotalNumSGPRs(STI), WavesPerEU,
1338	TrapReserve: getSGPRTrapHandlerReserve(STI),
1339	Granule: getSGPRAllocGranule(STI));
1340	return std::min(a: MaxNumSGPRs, b: AddressableNumSGPRs);
1341	}
1342
1343	bool isSGPROccupancyLimited(const MCSubtargetInfo &STI) {
1344	// From GFX10 on the SGPR file is large enough that SGPRs never limit
1345	// occupancy. Kept as one capability so callers don't each test the version.
1346	return getIsaVersion(GPU: STI.getCPU()).Major < `10`;
1347	}
1348
1349	unsigned getNumExtraSGPRs(const MCSubtargetInfo &STI, bool VCCUsed,
1350	bool FlatScrUsed, bool XNACKUsed) {
1351	unsigned ExtraSGPRs = `0`;
1352	if (VCCUsed)
1353	ExtraSGPRs = `2`;
1354
1355	IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1356	if (Version.Major >= `10`)
1357	return ExtraSGPRs;
1358
1359	if (Version.Major < `8`) {
1360	if (FlatScrUsed)
1361	ExtraSGPRs = `4`;
1362	} else {
1363	if (XNACKUsed)
1364	ExtraSGPRs = `4`;
1365
1366	if (FlatScrUsed \|\|
1367	STI.getFeatureBits().test(I: AMDGPU::FeatureArchitectedFlatScratch))
1368	ExtraSGPRs = `6`;
1369	}
1370
1371	return ExtraSGPRs;
1372	}
1373
1374	unsigned getNumExtraSGPRs(const MCSubtargetInfo &STI, bool VCCUsed,
1375	bool FlatScrUsed) {
1376	return getNumExtraSGPRs(STI, VCCUsed, FlatScrUsed,
1377	XNACKUsed: STI.getFeatureBits().test(I: AMDGPU::FeatureXNACK));
1378	}
1379
1380	static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs,
1381	unsigned Granule) {
1382	return divideCeil(Numerator: std::max(a: `1u`, b: NumRegs), Denominator: Granule);
1383	}
1384
1385	unsigned getNumSGPRBlocks(const MCSubtargetInfo &STI, unsigned NumSGPRs) {
1386	// SGPRBlocks is actual number of SGPR blocks minus 1.
1387	return getGranulatedNumRegisterBlocks(NumRegs: NumSGPRs, Granule: getSGPREncodingGranule(STI)) -
1388	`1`;
1389	}
1390
1391	unsigned getVGPRAllocGranule(const MCSubtargetInfo &STI,
1392	unsigned DynamicVGPRBlockSize,
1393	std::optional<bool> EnableWavefrontSize32) {
1394	if (STI.getFeatureBits().test(I: FeatureGFX90AInsts))
1395	return `8`;
1396
1397	if (DynamicVGPRBlockSize != `0`)
1398	return DynamicVGPRBlockSize;
1399
1400	bool IsWave32 = EnableWavefrontSize32
1401	? *EnableWavefrontSize32
1402	: STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1403
1404	if (STI.getFeatureBits().test(I: Feature1536VGPRs))
1405	return IsWave32 ? `24` : `12`;
1406
1407	if (hasGFX10_3Insts(STI))
1408	return IsWave32 ? `16` : `8`;
1409
1410	return IsWave32 ? `8` : `4`;
1411	}
1412
1413	unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI,
1414	std::optional<bool> EnableWavefrontSize32) {
1415	if (STI.getFeatureBits().test(I: FeatureGFX90AInsts))
1416	return `8`;
1417
1418	bool IsWave32 = EnableWavefrontSize32
1419	? *EnableWavefrontSize32
1420	: STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1421
1422	if (STI.getFeatureBits().test(I: Feature1024AddressableVGPRs))
1423	return IsWave32 ? `16` : `8`;
1424
1425	return IsWave32 ? `8` : `4`;
1426	}
1427
1428	unsigned getArchVGPRAllocGranule() { return `4`; }
1429
1430	unsigned getTotalNumVGPRs(const MCSubtargetInfo &STI) {
1431	if (STI.getFeatureBits().test(I: FeatureGFX90AInsts))
1432	return `512`;
1433	if (!isGFX10Plus(STI))
1434	return `256`;
1435	bool IsWave32 = STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1436	if (STI.getFeatureBits().test(I: Feature1536VGPRs))
1437	return IsWave32 ? `1536` : `768`;
1438	return IsWave32 ? `1024` : `512`;
1439	}
1440
1441	unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo &STI) {
1442	const auto &Features = STI.getFeatureBits();
1443	if (Features.test(I: Feature1024AddressableVGPRs))
1444	return Features.test(I: FeatureWavefrontSize32) ? `1024` : `512`;
1445	return `256`;
1446	}
1447
1448	unsigned getAddressableNumVGPRs(const MCSubtargetInfo &STI,
1449	unsigned DynamicVGPRBlockSize) {
1450	const auto &Features = STI.getFeatureBits();
1451	if (Features.test(I: FeatureGFX90AInsts))
1452	return `512`;
1453
1454	if (DynamicVGPRBlockSize != `0`) {
1455	// On GFX12 we can allocate at most MaxDynamicVGPRBlocks blocks of VGPRs.
1456	return MaxDynamicVGPRBlocks *
1457	getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1458	}
1459	return getAddressableNumArchVGPRs(STI);
1460	}
1461
1462	unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo &STI,
1463	unsigned NumVGPRs,
1464	unsigned DynamicVGPRBlockSize) {
1465	return getNumWavesPerEUWithNumVGPRs(
1466	NumVGPRs, Granule: getVGPRAllocGranule(STI, DynamicVGPRBlockSize),
1467	MaxWaves: getMaxWavesPerEU(STI), TotalNumVGPRs: getTotalNumVGPRs(STI));
1468	}
1469
1470	unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
1471	unsigned MaxWaves,
1472	unsigned TotalNumVGPRs) {
1473	if (NumVGPRs < Granule)
1474	return MaxWaves;
1475	unsigned RoundedRegs = alignTo(Value: NumVGPRs, Align: Granule);
1476	return std::min(a: std::max(a: TotalNumVGPRs / RoundedRegs, b: `1u`), b: MaxWaves);
1477	}
1478
1479	unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
1480	unsigned TotalNumSGPRs, unsigned Granule,
1481	unsigned TrapReserve) {
1482	// Closed-form inverse of getMaxNumSGPRs(): the budget condition
1483	// SGPRs <= alignDown(TotalNumSGPRs / W - TrapReserve, Granule)
1484	// solves to W <= TotalNumSGPRs / (alignTo(SGPRs, Granule) + TrapReserve).
1485	unsigned PerWave = alignTo(Value: SGPRs, Align: Granule) + TrapReserve;
1486	return PerWave ? std::clamp(val: TotalNumSGPRs / PerWave, lo: `1u`, hi: MaxWaves) : MaxWaves;
1487	}
1488
1489	unsigned getOccupancyWithNumSGPRs(const MCSubtargetInfo &STI, unsigned SGPRs) {
1490	unsigned MaxWaves = getMaxWavesPerEU(STI);
1491
1492	if (!isSGPROccupancyLimited(STI))
1493	return MaxWaves;
1494
1495	return getOccupancyWithNumSGPRs(SGPRs, MaxWaves, TotalNumSGPRs: getTotalNumSGPRs(STI),
1496	Granule: getSGPRAllocGranule(STI),
1497	TrapReserve: getSGPRTrapHandlerReserve(STI));
1498	}
1499
1500	unsigned getMinNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1501	unsigned DynamicVGPRBlockSize) {
1502	assert(WavesPerEU != `0`);
1503
1504	// In dynamic VGPR mode, (static) occupancy does not depend on VGPR usage,
1505	// so getMaxNumVGPRs does not depend on WavesPerEU, and thus we need to return
1506	// zero because there is no nonzero VGPR usage N where going below N
1507	// achieves higher (static) occupancy.
1508	bool DynamicVGPREnabled = (DynamicVGPRBlockSize != `0`);
1509	if (DynamicVGPREnabled)
1510	return `0`;
1511
1512	unsigned MaxWavesPerEU = getMaxWavesPerEU(STI);
1513	if (WavesPerEU >= MaxWavesPerEU)
1514	return `0`;
1515
1516	unsigned TotNumVGPRs = getTotalNumVGPRs(STI);
1517	unsigned AddrsableNumVGPRs =
1518	getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1519	unsigned Granule = getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1520	unsigned MaxNumVGPRs = alignDown(Value: TotNumVGPRs / WavesPerEU, Align: Granule);
1521
1522	if (MaxNumVGPRs == alignDown(Value: TotNumVGPRs / MaxWavesPerEU, Align: Granule))
1523	return `0`;
1524
1525	unsigned MinWavesPerEU = getNumWavesPerEUWithNumVGPRs(STI, NumVGPRs: AddrsableNumVGPRs,
1526	DynamicVGPRBlockSize);
1527	if (WavesPerEU < MinWavesPerEU)
1528	return getMinNumVGPRs(STI, WavesPerEU: MinWavesPerEU, DynamicVGPRBlockSize);
1529
1530	unsigned MaxNumVGPRsNext = alignDown(Value: TotNumVGPRs / (WavesPerEU + `1`), Align: Granule);
1531	unsigned MinNumVGPRs = `1` + std::min(a: MaxNumVGPRs - Granule, b: MaxNumVGPRsNext);
1532	return std::min(a: MinNumVGPRs, b: AddrsableNumVGPRs);
1533	}
1534
1535	unsigned getMaxNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1536	unsigned DynamicVGPRBlockSize) {
1537	assert(WavesPerEU != `0`);
1538
1539	// In dynamic VGPR mode, WavesPerEU does not imply a VGPR limit.
1540	bool DynamicVGPREnabled = (DynamicVGPRBlockSize != `0`);
1541	unsigned MaxNumVGPRs =
1542	DynamicVGPREnabled
1543	? getTotalNumVGPRs(STI)
1544	: alignDown(Value: getTotalNumVGPRs(STI) / WavesPerEU,
1545	Align: getVGPRAllocGranule(STI, DynamicVGPRBlockSize));
1546	unsigned AddressableNumVGPRs =
1547	getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1548	return std::min(a: MaxNumVGPRs, b: AddressableNumVGPRs);
1549	}
1550
1551	unsigned getEncodedNumVGPRBlocks(const MCSubtargetInfo &STI, unsigned NumVGPRs,
1552	std::optional<bool> EnableWavefrontSize32) {
1553	return getGranulatedNumRegisterBlocks(
1554	NumRegs: NumVGPRs, Granule: getVGPREncodingGranule(STI, EnableWavefrontSize32)) -
1555	`1`;
1556	}
1557
1558	unsigned getAllocatedNumVGPRBlocks(const MCSubtargetInfo &STI,
1559	unsigned NumVGPRs,
1560	unsigned DynamicVGPRBlockSize,
1561	std::optional<bool> EnableWavefrontSize32) {
1562	return getGranulatedNumRegisterBlocks(
1563	NumRegs: NumVGPRs,
1564	Granule: getVGPRAllocGranule(STI, DynamicVGPRBlockSize, EnableWavefrontSize32));
1565	}
1566	} // end namespace IsaInfo
1567
1568	void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &KernelCode,
1569	const MCSubtargetInfo &STI) {
1570	IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1571	KernelCode.amd_kernel_code_version_major = `1`;
1572	KernelCode.amd_kernel_code_version_minor = `2`;
1573	KernelCode.amd_machine_kind = `1`; // AMD_MACHINE_KIND_AMDGPU
1574	KernelCode.amd_machine_version_major = Version.Major;
1575	KernelCode.amd_machine_version_minor = Version.Minor;
1576	KernelCode.amd_machine_version_stepping = Version.Stepping;
1577	KernelCode.kernel_code_entry_byte_offset = sizeof(amd_kernel_code_t);
1578	if (STI.getFeatureBits().test(I: FeatureWavefrontSize32)) {
1579	KernelCode.wavefront_size = `5`;
1580	KernelCode.code_properties \|= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
1581	} else {
1582	KernelCode.wavefront_size = `6`;
1583	}
1584
1585	// If the code object does not support indirect functions, then the value must
1586	// be 0xffffffff.
1587	KernelCode.call_convention = -`1`;
1588
1589	// These alignment values are specified in powers of two, so alignment =
1590	// 2^n. The minimum alignment is 2^4 = 16.
1591	KernelCode.kernarg_segment_alignment = `4`;
1592	KernelCode.group_segment_alignment = `4`;
1593	KernelCode.private_segment_alignment = `4`;
1594
1595	if (Version.Major >= `10`) {
1596	KernelCode.compute_pgm_resource_registers \|=
1597	S_00B848_WGP_MODE(STI.getFeatureBits().test(FeatureCuMode) ? `0` : `1`) \|
1598	S_00B848_MEM_ORDERED(`1`) \| S_00B848_FWD_PROGRESS(`1`);
1599	}
1600	}
1601
1602	bool isGroupSegment(const GlobalValue *GV) {
1603	return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
1604	}
1605
1606	bool isGlobalSegment(const GlobalValue *GV) {
1607	return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
1608	}
1609
1610	bool isReadOnlySegment(const GlobalValue *GV) {
1611	unsigned AS = GV->getAddressSpace();
1612	return AS == AMDGPUAS::CONSTANT_ADDRESS \|\|
1613	AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
1614	}
1615
1616	bool shouldEmitConstantsToTextSection(const Triple &TT) {
1617	return TT.getArch() == Triple::r600;
1618	}
1619
1620	static bool isValidRegPrefix(char C) {
1621	return C == `'v'` \|\| C == `'s'` \|\| C == `'a'`;
1622	}
1623
1624	std::tuple<char, unsigned, unsigned> parseAsmPhysRegName(StringRef RegName) {
1625	char Kind = RegName.front();
1626	if (!isValidRegPrefix(C: Kind))
1627	return {};
1628
1629	RegName = RegName.drop_front();
1630	if (RegName.consume_front(Prefix: "[")) {
1631	unsigned Idx, End;
1632	bool Failed = RegName.consumeInteger(Radix: `10`, Result&: Idx);
1633	Failed \|= !RegName.consume_front(Prefix: ":");
1634	Failed \|= RegName.consumeInteger(Radix: `10`, Result&: End);
1635	Failed \|= !RegName.consume_back(Suffix: "]");
1636	if (!Failed) {
1637	unsigned NumRegs = End - Idx + `1`;
1638	if (NumRegs > `1`)
1639	return {Kind, Idx, NumRegs};
1640	}
1641	} else {
1642	unsigned Idx;
1643	bool Failed = RegName.getAsInteger(Radix: `10`, Result&: Idx);
1644	if (!Failed)
1645	return {Kind, Idx, `1`};
1646	}
1647
1648	return {};
1649	}
1650
1651	std::tuple<char, unsigned, unsigned>
1652	parseAsmConstraintPhysReg(StringRef Constraint) {
1653	StringRef RegName = Constraint;
1654	if (!RegName.consume_front(Prefix: "{") \|\| !RegName.consume_back(Suffix: "}"))
1655	return {};
1656	return parseAsmPhysRegName(RegName);
1657	}
1658
1659	std::pair<unsigned, unsigned>
1660	getIntegerPairAttribute(const Function &F, StringRef Name,
1661	std::pair<unsigned, unsigned> Default,
1662	bool OnlyFirstRequired) {
1663	if (auto Attr = getIntegerPairAttribute(F, Name, OnlyFirstRequired))
1664	return {Attr ->first, Attr ->second.value_or(u&: Default.second)};
1665	return Default;
1666	}
1667
1668	std::optional<std::pair<unsigned, std::optional<unsigned>>>
1669	getIntegerPairAttribute(const Function &F, StringRef Name,
1670	bool OnlyFirstRequired) {
1671	Attribute A = F.getFnAttribute(Kind: Name);
1672	if (!A.isStringAttribute())
1673	return std::nullopt;
1674
1675	LLVMContext &Ctx = F.getContext();
1676	std::pair<unsigned, std::optional<unsigned>> Ints;
1677	std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(Separator: `','`);
1678	if (Strs.first.trim().getAsInteger(Radix: `0`, Result&: Ints.first)) {
1679	Ctx.emitError(ErrorStr: "can't parse first integer attribute " + Name);
1680	return std::nullopt;
1681	}
1682	unsigned Second = `0`;
1683	if (Strs.second.trim().getAsInteger(Radix: `0`, Result&: Second)) {
1684	if (!OnlyFirstRequired \|\| !Strs.second.trim().empty()) {
1685	Ctx.emitError(ErrorStr: "can't parse second integer attribute " + Name);
1686	return std::nullopt;
1687	}
1688	} else {
1689	Ints.second = Second;
1690	}
1691
1692	return Ints;
1693	}
1694
1695	SmallVector<unsigned> getIntegerVecAttribute(const Function &F, StringRef Name,
1696	unsigned Size,
1697	unsigned DefaultVal) {
1698	std::optional<SmallVector<unsigned>> R =
1699	getIntegerVecAttribute(F, Name, Size);
1700	return R.has_value() ? R : SmallVector<unsigned*>(Size, DefaultVal);
1701	}
1702
1703	std::optional<SmallVector<unsigned>>
1704	getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) {
1705	assert(Size > `2`);
1706	LLVMContext &Ctx = F.getContext();
1707
1708	Attribute A = F.getFnAttribute(Kind: Name);
1709	if (!A.isValid())
1710	return std::nullopt;
1711	if (!A.isStringAttribute()) {
1712	Ctx.emitError(ErrorStr: Name + " is not a string attribute");
1713	return std::nullopt;
1714	}
1715
1716	SmallVector<unsigned> Vals(Size);
1717
1718	StringRef S = A.getValueAsString();
1719	unsigned i = `0`;
1720	for (; !S.empty() && i < Size; i++) {
1721	std::pair<StringRef, StringRef> Strs = S.split(Separator: `','`);
1722	unsigned IntVal;
1723	if (Strs.first.trim().getAsInteger(Radix: `0`, Result&: IntVal)) {
1724	Ctx.emitError(ErrorStr: "can't parse integer attribute " + Strs.first + " in " +
1725	Name);
1726	return std::nullopt;
1727	}
1728	Vals [i] = IntVal;
1729	S = Strs.second;
1730	}
1731
1732	if (!S.empty() \|\| i < Size) {
1733	Ctx.emitError(ErrorStr: "attribute " + Name +
1734	" has incorrect number of integers; expected " +
1735	llvm::utostr(X: Size));
1736	return std::nullopt;
1737	}
1738	return Vals;
1739	}
1740
1741	SmallVector<unsigned> getMaxNumWorkGroups(const Function &F) {
1742	return getIntegerVecAttribute(F, Name: "amdgpu-max-num-workgroups", Size: `3`,
1743	DefaultVal: std::numeric_limits<uint32_t>::max());
1744	}
1745
1746	bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) {
1747	assert((MD.getNumOperands() % `2` == `0`) && "invalid number of operands!");
1748	for (unsigned I = `0`, E = MD.getNumOperands() / `2`; I != E; ++I) {
1749	auto Low =
1750	mdconst::extract<ConstantInt>(MD: MD.getOperand(I: `2` * I + `0`))->getValue();
1751	auto High =
1752	mdconst::extract<ConstantInt>(MD: MD.getOperand(I: `2` * I + `1`))->getValue();
1753	// There are two types of [A; B) ranges:
1754	// A < B, e.g. [4; 5) which is a range that only includes 4.
1755	// A > B, e.g. [5; 4) which is a range that wraps around and includes
1756	// everything except 4.
1757	if (Low.ult(RHS: High)) {
1758	if (Low.ule(RHS: Val) && High.ugt(RHS: Val))
1759	return true;
1760	} else {
1761	if (Low.uge(RHS: Val) && High.ult(RHS: Val))
1762	return true;
1763	}
1764	}
1765
1766	return false;
1767	}
1768
1769	unsigned getVmcntBitMask(const IsaVersion &Version) {
1770	return (`1` << (getVmcntBitWidthLo(VersionMajor: Version.Major) +
1771	getVmcntBitWidthHi(VersionMajor: Version.Major))) -
1772	`1`;
1773	}
1774
1775	unsigned getLoadcntBitMask(const IsaVersion &Version) {
1776	return (`1` << getLoadcntBitWidth(VersionMajor: Version.Major)) - `1`;
1777	}
1778
1779	unsigned getSamplecntBitMask(const IsaVersion &Version) {
1780	return (`1` << getSamplecntBitWidth(VersionMajor: Version.Major)) - `1`;
1781	}
1782
1783	unsigned getBvhcntBitMask(const IsaVersion &Version) {
1784	return (`1` << getBvhcntBitWidth(VersionMajor: Version.Major)) - `1`;
1785	}
1786
1787	unsigned getExpcntBitMask(const IsaVersion &Version) {
1788	return (`1` << getExpcntBitWidth(VersionMajor: Version.Major)) - `1`;
1789	}
1790
1791	unsigned getLgkmcntBitMask(const IsaVersion &Version) {
1792	return (`1` << getLgkmcntBitWidth(VersionMajor: Version.Major)) - `1`;
1793	}
1794
1795	unsigned getDscntBitMask(const IsaVersion &Version) {
1796	return (`1` << getDscntBitWidth(VersionMajor: Version.Major)) - `1`;
1797	}
1798
1799	unsigned getKmcntBitMask(const IsaVersion &Version) {
1800	return (`1` << getKmcntBitWidth(VersionMajor: Version.Major)) - `1`;
1801	}
1802
1803	unsigned getXcntBitMask(const IsaVersion &Version) {
1804	return (`1` << getXcntBitWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - `1`;
1805	}
1806
1807	unsigned getAsynccntBitMask(const IsaVersion &Version) {
1808	return (`1` << getAsynccntBitWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - `1`;
1809	}
1810
1811	unsigned getStorecntBitMask(const IsaVersion &Version) {
1812	return (`1` << getStorecntBitWidth(VersionMajor: Version.Major)) - `1`;
1813	}
1814
1815	unsigned getWaitcntBitMask(const IsaVersion &Version) {
1816	unsigned VmcntLo = getBitMask(Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1817	Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1818	unsigned Expcnt = getBitMask(Shift: getExpcntBitShift(VersionMajor: Version.Major),
1819	Width: getExpcntBitWidth(VersionMajor: Version.Major));
1820	unsigned Lgkmcnt = getBitMask(Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1821	Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1822	unsigned VmcntHi = getBitMask(Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1823	Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1824	return VmcntLo \| Expcnt \| Lgkmcnt \| VmcntHi;
1825	}
1826
1827	unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1828	unsigned VmcntLo = unpackBits(Src: Waitcnt, Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1829	Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1830	unsigned VmcntHi = unpackBits(Src: Waitcnt, Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1831	Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1832	return VmcntLo \| VmcntHi << getVmcntBitWidthLo(VersionMajor: Version.Major);
1833	}
1834
1835	unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
1836	return unpackBits(Src: Waitcnt, Shift: getExpcntBitShift(VersionMajor: Version.Major),
1837	Width: getExpcntBitWidth(VersionMajor: Version.Major));
1838	}
1839
1840	unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1841	return unpackBits(Src: Waitcnt, Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1842	Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1843	}
1844
1845	unsigned decodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt) {
1846	return unpackBits(Src: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1847	Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1848	}
1849
1850	unsigned decodeStorecnt(const IsaVersion &Version, unsigned Waitcnt) {
1851	return unpackBits(Src: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1852	Width: getStorecntBitWidth(VersionMajor: Version.Major));
1853	}
1854
1855	unsigned decodeDscnt(const IsaVersion &Version, unsigned Waitcnt) {
1856	return unpackBits(Src: Waitcnt, Shift: getDscntBitShift(VersionMajor: Version.Major),
1857	Width: getDscntBitWidth(VersionMajor: Version.Major));
1858	}
1859
1860	void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt,
1861	unsigned &Expcnt, unsigned &Lgkmcnt) {
1862	Vmcnt = decodeVmcnt(Version, Waitcnt);
1863	Expcnt = decodeExpcnt(Version, Waitcnt);
1864	Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
1865	}
1866
1867	unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
1868	unsigned Vmcnt) {
1869	Waitcnt = packBits(Src: Vmcnt, Dst: Waitcnt, Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1870	Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1871	return packBits(Src: Vmcnt >> getVmcntBitWidthLo(VersionMajor: Version.Major), Dst: Waitcnt,
1872	Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1873	Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1874	}
1875
1876	unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
1877	unsigned Expcnt) {
1878	return packBits(Src: Expcnt, Dst: Waitcnt, Shift: getExpcntBitShift(VersionMajor: Version.Major),
1879	Width: getExpcntBitWidth(VersionMajor: Version.Major));
1880	}
1881
1882	unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
1883	unsigned Lgkmcnt) {
1884	return packBits(Src: Lgkmcnt, Dst: Waitcnt, Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1885	Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1886	}
1887
1888	unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt,
1889	unsigned Expcnt, unsigned Lgkmcnt) {
1890	unsigned Waitcnt = getWaitcntBitMask(Version);
1891	Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
1892	Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
1893	Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
1894	return Waitcnt;
1895	}
1896
1897	static unsigned getCombinedCountBitMask(const IsaVersion &Version,
1898	bool IsStore) {
1899	unsigned Dscnt = getBitMask(Shift: getDscntBitShift(VersionMajor: Version.Major),
1900	Width: getDscntBitWidth(VersionMajor: Version.Major));
1901	if (IsStore) {
1902	unsigned Storecnt = getBitMask(Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1903	Width: getStorecntBitWidth(VersionMajor: Version.Major));
1904	return Dscnt \| Storecnt;
1905	}
1906	unsigned Loadcnt = getBitMask(Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1907	Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1908	return Dscnt \| Loadcnt;
1909	}
1910
1911	static unsigned encodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt,
1912	unsigned Loadcnt) {
1913	return packBits(Src: Loadcnt, Dst: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1914	Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1915	}
1916
1917	static unsigned encodeStorecnt(const IsaVersion &Version, unsigned Waitcnt,
1918	unsigned Storecnt) {
1919	return packBits(Src: Storecnt, Dst: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1920	Width: getStorecntBitWidth(VersionMajor: Version.Major));
1921	}
1922
1923	static unsigned encodeDscnt(const IsaVersion &Version, unsigned Waitcnt,
1924	unsigned Dscnt) {
1925	return packBits(Src: Dscnt, Dst: Waitcnt, Shift: getDscntBitShift(VersionMajor: Version.Major),
1926	Width: getDscntBitWidth(VersionMajor: Version.Major));
1927	}
1928
1929	unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt,
1930	unsigned Dscnt) {
1931	unsigned Waitcnt = getCombinedCountBitMask(Version, IsStore: false);
1932	Waitcnt = encodeLoadcnt(Version, Waitcnt, Loadcnt);
1933	Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
1934	return Waitcnt;
1935	}
1936
1937	unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt,
1938	unsigned Dscnt) {
1939	unsigned Waitcnt = getCombinedCountBitMask(Version, IsStore: true);
1940	Waitcnt = encodeStorecnt(Version, Waitcnt, Storecnt);
1941	Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
1942	return Waitcnt;
1943	}
1944
1945	//===----------------------------------------------------------------------===//
1946	// Custom Operand Values
1947	//===----------------------------------------------------------------------===//
1948
1949	static unsigned getDefaultCustomOperandEncoding(const CustomOperandVal *Opr,
1950	int Size,
1951	const MCSubtargetInfo &STI) {
1952	unsigned Enc = `0`;
1953	for (int Idx = `0`; Idx < Size; ++Idx) {
1954	const auto &Op = Opr[Idx];
1955	if (Op.isSupported(STI))
1956	Enc \|= Op.encode(Val: Op.Default);
1957	}
1958	return Enc;
1959	}
1960
1961	static bool isSymbolicCustomOperandEncoding(const CustomOperandVal *Opr,
1962	int Size, unsigned Code,
1963	bool &HasNonDefaultVal,
1964	const MCSubtargetInfo &STI) {
1965	unsigned UsedOprMask = `0`;
1966	HasNonDefaultVal = false;
1967	for (int Idx = `0`; Idx < Size; ++Idx) {
1968	const auto &Op = Opr[Idx];
1969	if (!Op.isSupported(STI))
1970	continue;
1971	UsedOprMask \|= Op.getMask();
1972	unsigned Val = Op.decode(Code);
1973	if (!Op.isValid(Val))
1974	return false;
1975	HasNonDefaultVal \|= (Val != Op.Default);
1976	}
1977	return (Code & ~UsedOprMask) == `0`;
1978	}
1979
1980	static bool decodeCustomOperand(const CustomOperandVal Opr, int* Size,
1981	unsigned Code, int &Idx, StringRef &Name,
1982	unsigned &Val, bool &IsDefault,
1983	const MCSubtargetInfo &STI) {
1984	while (Idx < Size) {
1985	const auto &Op = Opr[Idx++];
1986	if (Op.isSupported(STI)) {
1987	Name = Op.Name;
1988	Val = Op.decode(Code);
1989	IsDefault = (Val == Op.Default);
1990	return true;
1991	}
1992	}
1993
1994	return false;
1995	}
1996
1997	static int encodeCustomOperandVal(const CustomOperandVal &Op,
1998	int64_t InputVal) {
1999	if (InputVal < `0` \|\| InputVal > Op.Max)
2000	return OPR_VAL_INVALID;
2001	return Op.encode(Val: InputVal);
2002	}
2003
2004	static int encodeCustomOperand(const CustomOperandVal Opr, int* Size,
2005	const StringRef Name, int64_t InputVal,
2006	unsigned &UsedOprMask,
2007	const MCSubtargetInfo &STI) {
2008	int InvalidId = OPR_ID_UNKNOWN;
2009	for (int Idx = `0`; Idx < Size; ++Idx) {
2010	const auto &Op = Opr[Idx];
2011	if (Op.Name == Name) {
2012	if (!Op.isSupported(STI)) {
2013	InvalidId = OPR_ID_UNSUPPORTED;
2014	continue;
2015	}
2016	auto OprMask = Op.getMask();
2017	if (OprMask & UsedOprMask)
2018	return OPR_ID_DUPLICATE;
2019	UsedOprMask \|= OprMask;
2020	return encodeCustomOperandVal(Op, InputVal);
2021	}
2022	}
2023	return InvalidId;
2024	}
2025
2026	//===----------------------------------------------------------------------===//
2027	// DepCtr
2028	//===----------------------------------------------------------------------===//
2029
2030	namespace DepCtr {
2031
2032	int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI) {
2033	static int Default = -`1`;
2034	if (Default == -`1`)
2035	Default = getDefaultCustomOperandEncoding(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, STI);
2036	return Default;
2037	}
2038
2039	bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal,
2040	const MCSubtargetInfo &STI) {
2041	return isSymbolicCustomOperandEncoding(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Code,
2042	HasNonDefaultVal, STI);
2043	}
2044
2045	bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val,
2046	bool &IsDefault, const MCSubtargetInfo &STI) {
2047	return decodeCustomOperand(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Code, Idx&: Id, Name, Val,
2048	IsDefault, STI);
2049	}
2050
2051	int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask,
2052	const MCSubtargetInfo &STI) {
2053	return encodeCustomOperand(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Name, InputVal: Val, UsedOprMask,
2054	STI);
2055	}
2056
2057	unsigned getVaVdstBitMask() { return (`1` << getVaVdstBitWidth()) - `1`; }
2058
2059	unsigned getVaSdstBitMask() { return (`1` << getVaSdstBitWidth()) - `1`; }
2060
2061	unsigned getVaSsrcBitMask() { return (`1` << getVaSsrcBitWidth()) - `1`; }
2062
2063	unsigned getHoldCntBitMask(const IsaVersion &Version) {
2064	return (`1` << getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - `1`;
2065	}
2066
2067	unsigned getVmVsrcBitMask() { return (`1` << getVmVsrcBitWidth()) - `1`; }
2068
2069	unsigned getVaVccBitMask() { return (`1` << getVaVccBitWidth()) - `1`; }
2070
2071	unsigned getSaSdstBitMask() { return (`1` << getSaSdstBitWidth()) - `1`; }
2072
2073	unsigned decodeFieldVmVsrc(unsigned Encoded) {
2074	return unpackBits(Src: Encoded, Shift: getVmVsrcBitShift(), Width: getVmVsrcBitWidth());
2075	}
2076
2077	unsigned decodeFieldVaVdst(unsigned Encoded) {
2078	return unpackBits(Src: Encoded, Shift: getVaVdstBitShift(), Width: getVaVdstBitWidth());
2079	}
2080
2081	unsigned decodeFieldSaSdst(unsigned Encoded) {
2082	return unpackBits(Src: Encoded, Shift: getSaSdstBitShift(), Width: getSaSdstBitWidth());
2083	}
2084
2085	unsigned decodeFieldVaSdst(unsigned Encoded) {
2086	return unpackBits(Src: Encoded, Shift: getVaSdstBitShift(), Width: getVaSdstBitWidth());
2087	}
2088
2089	unsigned decodeFieldVaVcc(unsigned Encoded) {
2090	return unpackBits(Src: Encoded, Shift: getVaVccBitShift(), Width: getVaVccBitWidth());
2091	}
2092
2093	unsigned decodeFieldVaSsrc(unsigned Encoded) {
2094	return unpackBits(Src: Encoded, Shift: getVaSsrcBitShift(), Width: getVaSsrcBitWidth());
2095	}
2096
2097	unsigned decodeFieldHoldCnt(unsigned Encoded, const IsaVersion &Version) {
2098	return unpackBits(Src: Encoded, Shift: getHoldCntBitShift(),
2099	Width: getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor));
2100	}
2101
2102	unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc) {
2103	return packBits(Src: VmVsrc, Dst: Encoded, Shift: getVmVsrcBitShift(), Width: getVmVsrcBitWidth());
2104	}
2105
2106	unsigned encodeFieldVmVsrc(unsigned VmVsrc, const MCSubtargetInfo &STI) {
2107	unsigned Encoded = getDefaultDepCtrEncoding(STI);
2108	return encodeFieldVmVsrc(Encoded, VmVsrc);
2109	}
2110
2111	unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst) {
2112	return packBits(Src: VaVdst, Dst: Encoded, Shift: getVaVdstBitShift(), Width: getVaVdstBitWidth());
2113	}
2114
2115	unsigned encodeFieldVaVdst(unsigned VaVdst, const MCSubtargetInfo &STI) {
2116	unsigned Encoded = getDefaultDepCtrEncoding(STI);
2117	return encodeFieldVaVdst(Encoded, VaVdst);
2118	}
2119
2120	unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst) {
2121	return packBits(Src: SaSdst, Dst: Encoded, Shift: getSaSdstBitShift(), Width: getSaSdstBitWidth());
2122	}
2123
2124	unsigned encodeFieldSaSdst(unsigned SaSdst, const MCSubtargetInfo &STI) {
2125	unsigned Encoded = getDefaultDepCtrEncoding(STI);
2126	return encodeFieldSaSdst(Encoded, SaSdst);
2127	}
2128
2129	unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst) {
2130	return packBits(Src: VaSdst, Dst: Encoded, Shift: getVaSdstBitShift(), Width: getVaSdstBitWidth());
2131	}
2132
2133	unsigned encodeFieldVaSdst(unsigned VaSdst, const MCSubtargetInfo &STI) {
2134	unsigned Encoded = getDefaultDepCtrEncoding(STI);
2135	return encodeFieldVaSdst(Encoded, VaSdst);
2136	}
2137
2138	unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc) {
2139	return packBits(Src: VaVcc, Dst: Encoded, Shift: getVaVccBitShift(), Width: getVaVccBitWidth());
2140	}
2141
2142	unsigned encodeFieldVaVcc(unsigned VaVcc, const MCSubtargetInfo &STI) {
2143	unsigned Encoded = getDefaultDepCtrEncoding(STI);
2144	return encodeFieldVaVcc(Encoded, VaVcc);
2145	}
2146
2147	unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc) {
2148	return packBits(Src: VaSsrc, Dst: Encoded, Shift: getVaSsrcBitShift(), Width: getVaSsrcBitWidth());
2149	}
2150
2151	unsigned encodeFieldVaSsrc(unsigned VaSsrc, const MCSubtargetInfo &STI) {
2152	unsigned Encoded = getDefaultDepCtrEncoding(STI);
2153	return encodeFieldVaSsrc(Encoded, VaSsrc);
2154	}
2155
2156	unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt,
2157	const IsaVersion &Version) {
2158	return packBits(Src: HoldCnt, Dst: Encoded, Shift: getHoldCntBitShift(),
2159	Width: getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor));
2160	}
2161
2162	unsigned encodeFieldHoldCnt(unsigned HoldCnt, const MCSubtargetInfo &STI) {
2163	unsigned Encoded = getDefaultDepCtrEncoding(STI);
2164	return encodeFieldHoldCnt(Encoded, HoldCnt, Version: getIsaVersion(GPU: STI.getCPU()));
2165	}
2166
2167	} // namespace DepCtr
2168
2169	//===----------------------------------------------------------------------===//
2170	// exp tgt
2171	//===----------------------------------------------------------------------===//
2172
2173	namespace Exp {
2174
2175	struct ExpTgt {
2176	StringLiteral Name;
2177	unsigned Tgt;
2178	unsigned MaxIndex;
2179	};
2180
2181	// clang-format off
2182	static constexpr ExpTgt ExpTgtInfo[] = {
2183	{.Name: {"null"}, .Tgt: ET_NULL, .MaxIndex: ET_NULL_MAX_IDX},
2184	{.Name: {"mrtz"}, .Tgt: ET_MRTZ, .MaxIndex: ET_MRTZ_MAX_IDX},
2185	{.Name: {"prim"}, .Tgt: ET_PRIM, .MaxIndex: ET_PRIM_MAX_IDX},
2186	{.Name: {"mrt"}, .Tgt: ET_MRT0, .MaxIndex: ET_MRT_MAX_IDX},
2187	{.Name: {"pos"}, .Tgt: ET_POS0, .MaxIndex: ET_POS_MAX_IDX},
2188	{.Name: {"dual_src_blend"},.Tgt: ET_DUAL_SRC_BLEND0, .MaxIndex: ET_DUAL_SRC_BLEND_MAX_IDX},
2189	{.Name: {"param"}, .Tgt: ET_PARAM0, .MaxIndex: ET_PARAM_MAX_IDX},
2190	};
2191	// clang-format on
2192
2193	bool getTgtName(unsigned Id, StringRef &Name, int &Index) {
2194	for (const ExpTgt &Val : ExpTgtInfo) {
2195	if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) {
2196	Index = (Val.MaxIndex == `0`) ? -`1` : (Id - Val.Tgt);
2197	Name = Val.Name;
2198	return true;
2199	}
2200	}
2201	return false;
2202	}
2203
2204	unsigned getTgtId(const StringRef Name) {
2205
2206	for (const ExpTgt &Val : ExpTgtInfo) {
2207	if (Val.MaxIndex == `0` && Name == Val.Name)
2208	return Val.Tgt;
2209
2210	if (Val.MaxIndex > `0` && Name.starts_with(Prefix: Val.Name)) {
2211	StringRef Suffix = Name.drop_front(N: Val.Name.size());
2212
2213	unsigned Id;
2214	if (Suffix.getAsInteger(Radix: `10`, Result&: Id) \|\| Id > Val.MaxIndex)
2215	return ET_INVALID;
2216
2217	// Disable leading zeroes
2218	if (Suffix.size() > `1` && Suffix [`0`] == `'0'`)
2219	return ET_INVALID;
2220
2221	return Val.Tgt + Id;
2222	}
2223	}
2224	return ET_INVALID;
2225	}
2226
2227	bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) {
2228	switch (Id) {
2229	case ET_NULL:
2230	return !isGFX11Plus(STI);
2231	case ET_POS4:
2232	case ET_PRIM:
2233	return isGFX10Plus(STI);
2234	case ET_DUAL_SRC_BLEND0:
2235	case ET_DUAL_SRC_BLEND1:
2236	return isGFX11Plus(STI);
2237	default:
2238	if (Id >= ET_PARAM0 && Id <= ET_PARAM31)
2239	return !isGFX11Plus(STI) \|\| isGFX13Plus(STI);
2240	return true;
2241	}
2242	}
2243
2244	} // namespace Exp
2245
2246	//===----------------------------------------------------------------------===//
2247	// MTBUF Format
2248	//===----------------------------------------------------------------------===//
2249
2250	namespace MTBUFFormat {
2251
2252	int64_t getDfmt(const StringRef Name) {
2253	for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
2254	if (Name == DfmtSymbolic[Id])
2255	return Id;
2256	}
2257	return DFMT_UNDEF;
2258	}
2259
2260	StringRef getDfmtName(unsigned Id) {
2261	assert(Id <= DFMT_MAX);
2262	return DfmtSymbolic[Id];
2263	}
2264
2265	static StringLiteral const getNfmtLookupTable(const* MCSubtargetInfo &STI) {
2266	if (isSI(STI) \|\| isCI(STI))
2267	return NfmtSymbolicSICI;
2268	if (isVI(STI) \|\| isGFX9(STI))
2269	return NfmtSymbolicVI;
2270	return NfmtSymbolicGFX10;
2271	}
2272
2273	int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
2274	const auto *lookupTable = getNfmtLookupTable(STI);
2275	for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
2276	if (Name == lookupTable[Id])
2277	return Id;
2278	}
2279	return NFMT_UNDEF;
2280	}
2281
2282	StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
2283	assert(Id <= NFMT_MAX);
2284	return getNfmtLookupTable(STI)[Id];
2285	}
2286
2287	bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2288	unsigned Dfmt;
2289	unsigned Nfmt;
2290	decodeDfmtNfmt(Format: Id, Dfmt, Nfmt);
2291	return isValidNfmt(Val: Nfmt, STI);
2292	}
2293
2294	bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2295	return !getNfmtName(Id, STI).empty();
2296	}
2297
2298	int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
2299	return (Dfmt << DFMT_SHIFT) \| (Nfmt << NFMT_SHIFT);
2300	}
2301
2302	void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
2303	Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;
2304	Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
2305	}
2306
2307	int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI) {
2308	if (isGFX11Plus(STI)) {
2309	for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2310	if (Name == UfmtSymbolicGFX11[Id])
2311	return Id;
2312	}
2313	} else {
2314	for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2315	if (Name == UfmtSymbolicGFX10[Id])
2316	return Id;
2317	}
2318	}
2319	return UFMT_UNDEF;
2320	}
2321
2322	StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI) {
2323	if (isValidUnifiedFormat(Val: Id, STI))
2324	return isGFX10(STI) ? UfmtSymbolicGFX10[Id] : UfmtSymbolicGFX11[Id];
2325	return "";
2326	}
2327
2328	bool isValidUnifiedFormat(unsigned Id, const MCSubtargetInfo &STI) {
2329	return isGFX10(STI) ? Id <= UfmtGFX10::UFMT_LAST : Id <= UfmtGFX11::UFMT_LAST;
2330	}
2331
2332	int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt,
2333	const MCSubtargetInfo &STI) {
2334	int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
2335	if (isGFX11Plus(STI)) {
2336	for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2337	if (Fmt == DfmtNfmt2UFmtGFX11[Id])
2338	return Id;
2339	}
2340	} else {
2341	for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2342	if (Fmt == DfmtNfmt2UFmtGFX10[Id])
2343	return Id;
2344	}
2345	}
2346	return UFMT_UNDEF;
2347	}
2348
2349	bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
2350	return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
2351	}
2352
2353	unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {
2354	if (isGFX10Plus(STI))
2355	return UFMT_DEFAULT;
2356	return DFMT_NFMT_DEFAULT;
2357	}
2358
2359	} // namespace MTBUFFormat
2360
2361	//===----------------------------------------------------------------------===//
2362	// SendMsg
2363	//===----------------------------------------------------------------------===//
2364
2365	namespace SendMsg {
2366
2367	static uint64_t getMsgIdMask(const MCSubtargetInfo &STI) {
2368	return isGFX11Plus(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_;
2369	}
2370
2371	bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) {
2372	return (MsgId & ~(getMsgIdMask(STI))) == `0`;
2373	}
2374
2375	bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
2376	bool Strict) {
2377	assert(isValidMsgId(MsgId, STI));
2378
2379	if (!Strict)
2380	return `0` <= OpId && isUInt<OP_WIDTH_>(x: OpId);
2381
2382	if (msgRequiresOp(MsgId, STI)) {
2383	if (MsgId == ID_GS_PreGFX11 && OpId == OP_GS_NOP)
2384	return false;
2385
2386	return !getMsgOpName(MsgId, Encoding: OpId, STI).empty();
2387	}
2388
2389	return OpId == OP_NONE_;
2390	}
2391
2392	bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
2393	const MCSubtargetInfo &STI, bool Strict) {
2394	assert(isValidMsgOp(MsgId, OpId, STI, Strict));
2395
2396	if (!Strict)
2397	return `0` <= StreamId && isUInt<STREAM_ID_WIDTH_>(x: StreamId);
2398
2399	if (!isGFX11Plus(STI)) {
2400	switch (MsgId) {
2401	case ID_GS_PreGFX11:
2402	return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
2403	case ID_GS_DONE_PreGFX11:
2404	return (OpId == OP_GS_NOP)
2405	? (StreamId == STREAM_ID_NONE_)
2406	: (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
2407	}
2408	}
2409	return StreamId == STREAM_ID_NONE_;
2410	}
2411
2412	bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI) {
2413	return MsgId == ID_SYSMSG \|\|
2414	(!isGFX11Plus(STI) &&
2415	(MsgId == ID_GS_PreGFX11 \|\| MsgId == ID_GS_DONE_PreGFX11));
2416	}
2417
2418	bool msgSupportsStream(int64_t MsgId, int64_t OpId,
2419	const MCSubtargetInfo &STI) {
2420	return !isGFX11Plus(STI) &&
2421	(MsgId == ID_GS_PreGFX11 \|\| MsgId == ID_GS_DONE_PreGFX11) &&
2422	OpId != OP_GS_NOP;
2423	}
2424
2425	void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,
2426	uint16_t &StreamId, const MCSubtargetInfo &STI) {
2427	MsgId = Val & getMsgIdMask(STI);
2428	if (isGFX11Plus(STI)) {
2429	OpId = `0`;
2430	StreamId = `0`;
2431	} else {
2432	OpId = (Val & OP_MASK_) >> OP_SHIFT_;
2433	StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
2434	}
2435	}
2436
2437	uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId) {
2438	return MsgId \| (OpId << OP_SHIFT_) \| (StreamId << STREAM_ID_SHIFT_);
2439	}
2440
2441	bool msgDoesNotUseM0(int64_t MsgId, const MCSubtargetInfo &STI) {
2442	// Explicitly list message types that are known to not use m0.
2443	// This is safer than excluding only GS_ALLOC_REQ, in case new message
2444	// types are added in the future that do use m0.
2445	if (isGFX11Plus(STI)) {
2446	switch (MsgId) {
2447	case ID_DEALLOC_VGPRS_GFX11Plus:
2448	return true;
2449	default:
2450	break;
2451	}
2452	}
2453	switch (MsgId) {
2454	case ID_SAVEWAVE:
2455	case ID_STALL_WAVE_GEN:
2456	case ID_HALT_WAVES:
2457	case ID_ORDERED_PS_DONE:
2458	case ID_EARLY_PRIM_DEALLOC:
2459	case ID_GET_DOORBELL:
2460	case ID_GET_DDID:
2461	case ID_SYSMSG:
2462	return true;
2463	default:
2464	return false;
2465	}
2466	}
2467
2468	} // namespace SendMsg
2469
2470	//===----------------------------------------------------------------------===//
2471	//
2472	//===----------------------------------------------------------------------===//
2473
2474	unsigned getInitialPSInputAddr(const Function &F) {
2475	return F.getFnAttributeAsParsedInteger(Kind: "InitialPSInputAddr", Default: `0`);
2476	}
2477
2478	bool getHasColorExport(const Function &F) {
2479	// As a safe default always respond as if PS has color exports.
2480	return F.getFnAttributeAsParsedInteger(
2481	Kind: "amdgpu-color-export",
2482	Default: F.getCallingConv() == CallingConv::AMDGPU_PS ? `1` : `0`) != `0`;
2483	}
2484
2485	bool getHasDepthExport(const Function &F) {
2486	return F.getFnAttributeAsParsedInteger(Kind: "amdgpu-depth-export", Default: `0`) != `0`;
2487	}
2488
2489	unsigned getDynamicVGPRBlockSize(const Function &F) {
2490	unsigned BlockSize =
2491	F.getFnAttributeAsParsedInteger(Kind: "amdgpu-dynamic-vgpr-block-size", Default: `0`);
2492
2493	if (BlockSize == `16` \|\| BlockSize == `32`)
2494	return BlockSize;
2495
2496	return `0`;
2497	}
2498
2499	bool hasXNACK(const MCSubtargetInfo &STI) {
2500	return STI.hasFeature(Feature: AMDGPU::FeatureXNACK);
2501	}
2502
2503	bool hasMIMG_R128(const MCSubtargetInfo &STI) {
2504	return STI.hasFeature(Feature: AMDGPU::FeatureMIMG_R128) &&
2505	!STI.hasFeature(Feature: AMDGPU::FeatureR128A16);
2506	}
2507
2508	bool hasA16(const MCSubtargetInfo &STI) {
2509	return STI.hasFeature(Feature: AMDGPU::FeatureA16);
2510	}
2511
2512	bool hasG16(const MCSubtargetInfo &STI) {
2513	return STI.hasFeature(Feature: AMDGPU::FeatureG16);
2514	}
2515
2516	bool hasPackedD16(const MCSubtargetInfo &STI) {
2517	return !STI.hasFeature(Feature: AMDGPU::FeatureUnpackedD16VMem) && !isCI(STI) &&
2518	!isSI(STI);
2519	}
2520
2521	bool hasGDS(const MCSubtargetInfo &STI) {
2522	return STI.hasFeature(Feature: AMDGPU::FeatureGDS);
2523	}
2524
2525	unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler) {
2526	auto Version = getIsaVersion(GPU: STI.getCPU());
2527	if (Version.Major == `10`)
2528	return Version.Minor >= `3` ? `13` : `5`;
2529	if (Version.Major == `11`)
2530	return `5`;
2531	if (Version.Major >= `12`)
2532	return HasSampler ? `4` : `5`;
2533	return `0`;
2534	}
2535
2536	unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI) {
2537	if (isGFX1250Plus(STI))
2538	return `32`;
2539	return `16`;
2540	}
2541
2542	bool isSI(const MCSubtargetInfo &STI) {
2543	return STI.hasFeature(Feature: AMDGPU::FeatureSouthernIslands);
2544	}
2545
2546	bool isCI(const MCSubtargetInfo &STI) {
2547	return STI.hasFeature(Feature: AMDGPU::FeatureSeaIslands);
2548	}
2549
2550	bool isVI(const MCSubtargetInfo &STI) {
2551	return STI.hasFeature(Feature: AMDGPU::FeatureVolcanicIslands);
2552	}
2553
2554	bool isGFX9(const MCSubtargetInfo &STI) {
2555	return STI.hasFeature(Feature: AMDGPU::FeatureGFX9);
2556	}
2557
2558	bool isGFX9_GFX10(const MCSubtargetInfo &STI) {
2559	return isGFX9(STI) \|\| isGFX10(STI);
2560	}
2561
2562	bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI) {
2563	return isGFX9(STI) \|\| isGFX10(STI) \|\| isGFX11(STI);
2564	}
2565
2566	bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI) {
2567	return isVI(STI) \|\| isGFX9(STI) \|\| isGFX10(STI);
2568	}
2569
2570	bool isGFX8Plus(const MCSubtargetInfo &STI) {
2571	return isVI(STI) \|\| isGFX9Plus(STI);
2572	}
2573
2574	bool isGFX9Plus(const MCSubtargetInfo &STI) {
2575	return isGFX9(STI) \|\| isGFX10Plus(STI);
2576	}
2577
2578	bool isNotGFX9Plus(const MCSubtargetInfo &STI) { return !isGFX9Plus(STI); }
2579
2580	bool isGFX10(const MCSubtargetInfo &STI) {
2581	return STI.hasFeature(Feature: AMDGPU::FeatureGFX10);
2582	}
2583
2584	bool isGFX10_GFX11(const MCSubtargetInfo &STI) {
2585	return isGFX10(STI) \|\| isGFX11(STI);
2586	}
2587
2588	bool isGFX10Plus(const MCSubtargetInfo &STI) {
2589	return isGFX10(STI) \|\| isGFX11Plus(STI);
2590	}
2591
2592	bool isGFX11(const MCSubtargetInfo &STI) {
2593	return STI.hasFeature(Feature: AMDGPU::FeatureGFX11);
2594	}
2595
2596	bool isGFX11Plus(const MCSubtargetInfo &STI) {
2597	return isGFX11(STI) \|\| isGFX12Plus(STI);
2598	}
2599
2600	bool isGFX12(const MCSubtargetInfo &STI) {
2601	return STI.getFeatureBits()[AMDGPU::FeatureGFX12];
2602	}
2603
2604	bool isGFX12Plus(const MCSubtargetInfo &STI) {
2605	return isGFX12(STI) \|\| isGFX13Plus(STI);
2606	}
2607
2608	bool isNotGFX12Plus(const MCSubtargetInfo &STI) { return !isGFX12Plus(STI); }
2609
2610	bool isGFX1250(const MCSubtargetInfo &STI) {
2611	return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts] && !isGFX13(STI);
2612	}
2613
2614	bool isGFX1250Plus(const MCSubtargetInfo &STI) {
2615	return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts];
2616	}
2617
2618	bool isGFX13(const MCSubtargetInfo &STI) {
2619	return STI.getFeatureBits()[AMDGPU::FeatureGFX13];
2620	}
2621
2622	bool isGFX13Plus(const MCSubtargetInfo &STI) { return isGFX13(STI); }
2623
2624	bool supportsWGP(const MCSubtargetInfo &STI) {
2625	if (isGFX1250(STI))
2626	return false;
2627	return isGFX10Plus(STI);
2628	}
2629
2630	bool isNotGFX11Plus(const MCSubtargetInfo &STI) { return !isGFX11Plus(STI); }
2631
2632	bool isNotGFX10Plus(const MCSubtargetInfo &STI) {
2633	return isSI(STI) \|\| isCI(STI) \|\| isVI(STI) \|\| isGFX9(STI);
2634	}
2635
2636	bool isGFX10Before1030(const MCSubtargetInfo &STI) {
2637	return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI);
2638	}
2639
2640	bool isGCN3Encoding(const MCSubtargetInfo &STI) {
2641	return STI.hasFeature(Feature: AMDGPU::FeatureGCN3Encoding);
2642	}
2643
2644	bool isGFX10_BEncoding(const MCSubtargetInfo &STI) {
2645	return STI.hasFeature(Feature: AMDGPU::FeatureGFX10_BEncoding);
2646	}
2647
2648	bool hasGFX10_3Insts(const MCSubtargetInfo &STI) {
2649	return STI.hasFeature(Feature: AMDGPU::FeatureGFX10_3Insts);
2650	}
2651
2652	bool isGFX10_3_GFX11(const MCSubtargetInfo &STI) {
2653	return isGFX10_BEncoding(STI) && !isGFX12Plus(STI);
2654	}
2655
2656	bool isGFX90A(const MCSubtargetInfo &STI) {
2657	return STI.hasFeature(Feature: AMDGPU::FeatureGFX90AInsts);
2658	}
2659
2660	bool isGFX940(const MCSubtargetInfo &STI) {
2661	return STI.hasFeature(Feature: AMDGPU::FeatureGFX940Insts);
2662	}
2663
2664	bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) {
2665	return STI.hasFeature(Feature: AMDGPU::FeatureArchitectedFlatScratch);
2666	}
2667
2668	bool hasMAIInsts(const MCSubtargetInfo &STI) {
2669	return STI.hasFeature(Feature: AMDGPU::FeatureMAIInsts);
2670	}
2671
2672	bool hasVOPD(const MCSubtargetInfo &STI) {
2673	return STI.hasFeature(Feature: AMDGPU::FeatureVOPDInsts);
2674	}
2675
2676	bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI) {
2677	return STI.hasFeature(Feature: AMDGPU::FeatureDPPSrc1SGPR);
2678	}
2679
2680	unsigned hasKernargPreload(const MCSubtargetInfo &STI) {
2681	return STI.hasFeature(Feature: AMDGPU::FeatureKernargPreload);
2682	}
2683
2684	int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR,
2685	int32_t ArgNumVGPR) {
2686	if (has90AInsts && ArgNumAGPR)
2687	return alignTo(Value: ArgNumVGPR, Align: `4`) + ArgNumAGPR;
2688	return std::max(a: ArgNumVGPR, b: ArgNumAGPR);
2689	}
2690
2691	bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI) {
2692	const MCRegisterClass &SGPRClass =
2693	TRI->getRegClass(i: AMDGPU::SReg_32RegClassID);
2694	const MCRegister FirstSubReg = TRI->getSubReg(Reg, Idx: AMDGPU::sub0);
2695	return SGPRClass.contains(Reg: FirstSubReg != `0` ? FirstSubReg : Reg) \|\|
2696	Reg == AMDGPU::SCC;
2697	}
2698
2699	bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI) {
2700	return MRI.getEncodingValue(Reg) & AMDGPU::HWEncoding::IS_HI16;
2701	}
2702
2703	#define MAP_REG2REG \
2704	using namespace AMDGPU; \
2705	switch (Reg.id()) { \
2706	default: \
2707	return Reg; \
2708	CASE_CI_VI(FLAT_SCR) \
2709	CASE_CI_VI(FLAT_SCR_LO) \
2710	CASE_CI_VI(FLAT_SCR_HI) \
2711	CASE_VI_GFX9PLUS(TTMP0) \
2712	CASE_VI_GFX9PLUS(TTMP1) \
2713	CASE_VI_GFX9PLUS(TTMP2) \
2714	CASE_VI_GFX9PLUS(TTMP3) \
2715	CASE_VI_GFX9PLUS(TTMP4) \
2716	CASE_VI_GFX9PLUS(TTMP5) \
2717	CASE_VI_GFX9PLUS(TTMP6) \
2718	CASE_VI_GFX9PLUS(TTMP7) \
2719	CASE_VI_GFX9PLUS(TTMP8) \
2720	CASE_VI_GFX9PLUS(TTMP9) \
2721	CASE_VI_GFX9PLUS(TTMP10) \
2722	CASE_VI_GFX9PLUS(TTMP11) \
2723	CASE_VI_GFX9PLUS(TTMP12) \
2724	CASE_VI_GFX9PLUS(TTMP13) \
2725	CASE_VI_GFX9PLUS(TTMP14) \
2726	CASE_VI_GFX9PLUS(TTMP15) \
2727	CASE_VI_GFX9PLUS(TTMP0_TTMP1) \
2728	CASE_VI_GFX9PLUS(TTMP2_TTMP3) \
2729	CASE_VI_GFX9PLUS(TTMP4_TTMP5) \
2730	CASE_VI_GFX9PLUS(TTMP6_TTMP7) \
2731	CASE_VI_GFX9PLUS(TTMP8_TTMP9) \
2732	CASE_VI_GFX9PLUS(TTMP10_TTMP11) \
2733	CASE_VI_GFX9PLUS(TTMP12_TTMP13) \
2734	CASE_VI_GFX9PLUS(TTMP14_TTMP15) \
2735	CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \
2736	CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \
2737	CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \
2738	CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \
2739	CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
2740	CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
2741	CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2742	CASE_VI_GFX9PLUS( \
2743	TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2744	CASE_GFXPRE11_GFX11PLUS(M0) \
2745	CASE_GFXPRE11_GFX11PLUS(SGPR_NULL) \
2746	CASE_GFXPRE11_GFX11PLUS_TO(SGPR_NULL64, SGPR_NULL) \
2747	}
2748
2749	#define CASE_CI_VI(node) \
2750	assert(!isSI(STI)); \
2751	case node: \
2752	return isCI(STI) ? node##_ci : node##_vi;
2753
2754	#define CASE_VI_GFX9PLUS(node) \
2755	case node: \
2756	return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi;
2757
2758	#define CASE_GFXPRE11_GFX11PLUS(node) \
2759	case node: \
2760	return isGFX11Plus(STI) ? node##_gfx11plus : node##_gfxpre11;
2761
2762	#define CASE_GFXPRE11_GFX11PLUS_TO(node, result) \
2763	case node: \
2764	return isGFX11Plus(STI) ? result##_gfx11plus : result##_gfxpre11;
2765
2766	MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI) {
2767	if (STI.getTargetTriple().getArch() == Triple::r600)
2768	return Reg;
2769	MAP_REG2REG
2770	}
2771
2772	#undef CASE_CI_VI
2773	#undef CASE_VI_GFX9PLUS
2774	#undef CASE_GFXPRE11_GFX11PLUS
2775	#undef CASE_GFXPRE11_GFX11PLUS_TO
2776
2777	#define CASE_CI_VI(node) \
2778	case node##_ci: \
2779	case node##_vi: \
2780	return node;
2781	#define CASE_VI_GFX9PLUS(node) \
2782	case node##_vi: \
2783	case node##_gfx9plus: \
2784	return node;
2785	#define CASE_GFXPRE11_GFX11PLUS(node) \
2786	case node##_gfx11plus: \
2787	case node##_gfxpre11: \
2788	return node;
2789	#define CASE_GFXPRE11_GFX11PLUS_TO(node, result)
2790
2791	MCRegister mc2PseudoReg(MCRegister Reg) { MAP_REG2REG }
2792
2793	bool isInlineValue(MCRegister Reg) {
2794	switch (Reg.id()) {
2795	case AMDGPU::SRC_SHARED_BASE_LO:
2796	case AMDGPU::SRC_SHARED_BASE:
2797	case AMDGPU::SRC_SHARED_LIMIT_LO:
2798	case AMDGPU::SRC_SHARED_LIMIT:
2799	case AMDGPU::SRC_PRIVATE_BASE_LO:
2800	case AMDGPU::SRC_PRIVATE_BASE:
2801	case AMDGPU::SRC_PRIVATE_LIMIT_LO:
2802	case AMDGPU::SRC_PRIVATE_LIMIT:
2803	case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
2804	case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
2805	case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
2806	return true;
2807	case AMDGPU::SRC_VCCZ:
2808	case AMDGPU::SRC_EXECZ:
2809	case AMDGPU::SRC_SCC:
2810	return true;
2811	case AMDGPU::SGPR_NULL:
2812	return true;
2813	default:
2814	return false;
2815	}
2816	}
2817
2818	#undef CASE_CI_VI
2819	#undef CASE_VI_GFX9PLUS
2820	#undef CASE_GFXPRE11_GFX11PLUS
2821	#undef CASE_GFXPRE11_GFX11PLUS_TO
2822	#undef MAP_REG2REG
2823
2824	bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2825	assert(OpNo < Desc.NumOperands);
2826	unsigned OpType = Desc.operands()[OpNo].OperandType;
2827	return OpType >= AMDGPU::OPERAND_KIMM_FIRST &&
2828	OpType <= AMDGPU::OPERAND_KIMM_LAST;
2829	}
2830
2831	bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2832	assert(OpNo < Desc.NumOperands);
2833	unsigned OpType = Desc.operands()[OpNo].OperandType;
2834	switch (OpType) {
2835	case AMDGPU::OPERAND_REG_IMM_FP32:
2836	case AMDGPU::OPERAND_REG_IMM_FP64:
2837	case AMDGPU::OPERAND_REG_IMM_FP16:
2838	case AMDGPU::OPERAND_REG_IMM_V2FP16:
2839	case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
2840	case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
2841	case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2842	case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2843	case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2844	case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2845	case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2846	case AMDGPU::OPERAND_REG_IMM_V2FP32:
2847	case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2848	case AMDGPU::OPERAND_REG_IMM_V2FP64:
2849	return true;
2850	default:
2851	return false;
2852	}
2853	}
2854
2855	bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2856	assert(OpNo < Desc.NumOperands);
2857	unsigned OpType = Desc.operands()[OpNo].OperandType;
2858	return (OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST &&
2859	OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST) \|\|
2860	(OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST &&
2861	OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST);
2862	}
2863
2864	// Avoid using MCRegisterClass::getSize, since that function will go away
2865	// (move from MC level to Target* level). Return size in bits.*
2866	unsigned getRegBitWidth(unsigned RCID) {
2867	switch (RCID) {
2868	case AMDGPU::VGPR_16RegClassID:
2869	case AMDGPU::VGPR_16_Lo128RegClassID:
2870	case AMDGPU::SGPR_LO16RegClassID:
2871	case AMDGPU::AGPR_LO16RegClassID:
2872	return `16`;
2873	case AMDGPU::SGPR_32RegClassID:
2874	case AMDGPU::VGPR_32RegClassID:
2875	case AMDGPU::VGPR_32_Lo256RegClassID:
2876	case AMDGPU::VRegOrLds_32RegClassID:
2877	case AMDGPU::AGPR_32RegClassID:
2878	case AMDGPU::VS_32RegClassID:
2879	case AMDGPU::AV_32RegClassID:
2880	case AMDGPU::SReg_32RegClassID:
2881	case AMDGPU::SReg_32_XM0RegClassID:
2882	case AMDGPU::SRegOrLds_32RegClassID:
2883	return `32`;
2884	case AMDGPU::SGPR_64RegClassID:
2885	case AMDGPU::VS_64RegClassID:
2886	case AMDGPU::SReg_64RegClassID:
2887	case AMDGPU::VReg_64RegClassID:
2888	case AMDGPU::AReg_64RegClassID:
2889	case AMDGPU::SReg_64_XEXECRegClassID:
2890	case AMDGPU::VReg_64_Align2RegClassID:
2891	case AMDGPU::AReg_64_Align2RegClassID:
2892	case AMDGPU::AV_64RegClassID:
2893	case AMDGPU::AV_64_Align2RegClassID:
2894	case AMDGPU::VReg_64_Lo256_Align2RegClassID:
2895	case AMDGPU::VS_64_Lo256RegClassID:
2896	return `64`;
2897	case AMDGPU::SGPR_96RegClassID:
2898	case AMDGPU::SReg_96RegClassID:
2899	case AMDGPU::VReg_96RegClassID:
2900	case AMDGPU::AReg_96RegClassID:
2901	case AMDGPU::VReg_96_Align2RegClassID:
2902	case AMDGPU::AReg_96_Align2RegClassID:
2903	case AMDGPU::AV_96RegClassID:
2904	case AMDGPU::AV_96_Align2RegClassID:
2905	case AMDGPU::VReg_96_Lo256_Align2RegClassID:
2906	return `96`;
2907	case AMDGPU::SGPR_128RegClassID:
2908	case AMDGPU::SReg_128RegClassID:
2909	case AMDGPU::VReg_128RegClassID:
2910	case AMDGPU::AReg_128RegClassID:
2911	case AMDGPU::VReg_128_Align2RegClassID:
2912	case AMDGPU::AReg_128_Align2RegClassID:
2913	case AMDGPU::AV_128RegClassID:
2914	case AMDGPU::AV_128_Align2RegClassID:
2915	case AMDGPU::SReg_128_XNULLRegClassID:
2916	case AMDGPU::VReg_128_Lo256_Align2RegClassID:
2917	return `128`;
2918	case AMDGPU::SGPR_160RegClassID:
2919	case AMDGPU::SReg_160RegClassID:
2920	case AMDGPU::VReg_160RegClassID:
2921	case AMDGPU::AReg_160RegClassID:
2922	case AMDGPU::VReg_160_Align2RegClassID:
2923	case AMDGPU::AReg_160_Align2RegClassID:
2924	case AMDGPU::AV_160RegClassID:
2925	case AMDGPU::AV_160_Align2RegClassID:
2926	case AMDGPU::VReg_160_Lo256_Align2RegClassID:
2927	return `160`;
2928	case AMDGPU::SGPR_192RegClassID:
2929	case AMDGPU::SReg_192RegClassID:
2930	case AMDGPU::VReg_192RegClassID:
2931	case AMDGPU::AReg_192RegClassID:
2932	case AMDGPU::VReg_192_Align2RegClassID:
2933	case AMDGPU::AReg_192_Align2RegClassID:
2934	case AMDGPU::AV_192RegClassID:
2935	case AMDGPU::AV_192_Align2RegClassID:
2936	case AMDGPU::VReg_192_Lo256_Align2RegClassID:
2937	return `192`;
2938	case AMDGPU::SGPR_224RegClassID:
2939	case AMDGPU::SReg_224RegClassID:
2940	case AMDGPU::VReg_224RegClassID:
2941	case AMDGPU::AReg_224RegClassID:
2942	case AMDGPU::VReg_224_Align2RegClassID:
2943	case AMDGPU::AReg_224_Align2RegClassID:
2944	case AMDGPU::AV_224RegClassID:
2945	case AMDGPU::AV_224_Align2RegClassID:
2946	case AMDGPU::VReg_224_Lo256_Align2RegClassID:
2947	return `224`;
2948	case AMDGPU::SGPR_256RegClassID:
2949	case AMDGPU::SReg_256RegClassID:
2950	case AMDGPU::VReg_256RegClassID:
2951	case AMDGPU::AReg_256RegClassID:
2952	case AMDGPU::VReg_256_Align2RegClassID:
2953	case AMDGPU::AReg_256_Align2RegClassID:
2954	case AMDGPU::AV_256RegClassID:
2955	case AMDGPU::AV_256_Align2RegClassID:
2956	case AMDGPU::SReg_256_XNULLRegClassID:
2957	case AMDGPU::VReg_256_Lo256_Align2RegClassID:
2958	return `256`;
2959	case AMDGPU::SGPR_288RegClassID:
2960	case AMDGPU::SReg_288RegClassID:
2961	case AMDGPU::VReg_288RegClassID:
2962	case AMDGPU::AReg_288RegClassID:
2963	case AMDGPU::VReg_288_Align2RegClassID:
2964	case AMDGPU::AReg_288_Align2RegClassID:
2965	case AMDGPU::AV_288RegClassID:
2966	case AMDGPU::AV_288_Align2RegClassID:
2967	case AMDGPU::VReg_288_Lo256_Align2RegClassID:
2968	return `288`;
2969	case AMDGPU::SGPR_320RegClassID:
2970	case AMDGPU::SReg_320RegClassID:
2971	case AMDGPU::VReg_320RegClassID:
2972	case AMDGPU::AReg_320RegClassID:
2973	case AMDGPU::VReg_320_Align2RegClassID:
2974	case AMDGPU::AReg_320_Align2RegClassID:
2975	case AMDGPU::AV_320RegClassID:
2976	case AMDGPU::AV_320_Align2RegClassID:
2977	case AMDGPU::VReg_320_Lo256_Align2RegClassID:
2978	return `320`;
2979	case AMDGPU::SGPR_352RegClassID:
2980	case AMDGPU::SReg_352RegClassID:
2981	case AMDGPU::VReg_352RegClassID:
2982	case AMDGPU::AReg_352RegClassID:
2983	case AMDGPU::VReg_352_Align2RegClassID:
2984	case AMDGPU::AReg_352_Align2RegClassID:
2985	case AMDGPU::AV_352RegClassID:
2986	case AMDGPU::AV_352_Align2RegClassID:
2987	case AMDGPU::VReg_352_Lo256_Align2RegClassID:
2988	return `352`;
2989	case AMDGPU::SGPR_384RegClassID:
2990	case AMDGPU::SReg_384RegClassID:
2991	case AMDGPU::VReg_384RegClassID:
2992	case AMDGPU::AReg_384RegClassID:
2993	case AMDGPU::VReg_384_Align2RegClassID:
2994	case AMDGPU::AReg_384_Align2RegClassID:
2995	case AMDGPU::AV_384RegClassID:
2996	case AMDGPU::AV_384_Align2RegClassID:
2997	case AMDGPU::VReg_384_Lo256_Align2RegClassID:
2998	return `384`;
2999	case AMDGPU::SGPR_512RegClassID:
3000	case AMDGPU::SReg_512RegClassID:
3001	case AMDGPU::VReg_512RegClassID:
3002	case AMDGPU::AReg_512RegClassID:
3003	case AMDGPU::VReg_512_Align2RegClassID:
3004	case AMDGPU::AReg_512_Align2RegClassID:
3005	case AMDGPU::AV_512RegClassID:
3006	case AMDGPU::AV_512_Align2RegClassID:
3007	case AMDGPU::VReg_512_Lo256_Align2RegClassID:
3008	return `512`;
3009	case AMDGPU::SGPR_1024RegClassID:
3010	case AMDGPU::SReg_1024RegClassID:
3011	case AMDGPU::VReg_1024RegClassID:
3012	case AMDGPU::AReg_1024RegClassID:
3013	case AMDGPU::VReg_1024_Align2RegClassID:
3014	case AMDGPU::AReg_1024_Align2RegClassID:
3015	case AMDGPU::AV_1024RegClassID:
3016	case AMDGPU::AV_1024_Align2RegClassID:
3017	case AMDGPU::VReg_1024_Lo256_Align2RegClassID:
3018	return `1024`;
3019	default:
3020	llvm_unreachable("Unexpected register class");
3021	}
3022	}
3023
3024	unsigned getRegBitWidth(const MCRegisterClass &RC) {
3025	return getRegBitWidth(RCID: RC.getID());
3026	}
3027
3028	bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
3029	if (isInlinableIntLiteral(Literal))
3030	return true;
3031
3032	uint64_t Val = static_cast<uint64_t>(Literal);
3033	return (Val == llvm::bit_cast<uint64_t>(from: `0.0`)) \|\|
3034	(Val == llvm::bit_cast<uint64_t>(from: `1.0`)) \|\|
3035	(Val == llvm::bit_cast<uint64_t>(from: -`1.0`)) \|\|
3036	(Val == llvm::bit_cast<uint64_t>(from: `0.5`)) \|\|
3037	(Val == llvm::bit_cast<uint64_t>(from: -`0.5`)) \|\|
3038	(Val == llvm::bit_cast<uint64_t>(from: `2.0`)) \|\|
3039	(Val == llvm::bit_cast<uint64_t>(from: -`2.0`)) \|\|
3040	(Val == llvm::bit_cast<uint64_t>(from: `4.0`)) \|\|
3041	(Val == llvm::bit_cast<uint64_t>(from: -`4.0`)) \|\|
3042	(Val == `0x3fc45f306dc9c882` && HasInv2Pi);
3043	}
3044
3045	bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
3046	if (isInlinableIntLiteral(Literal))
3047	return true;
3048
3049	// The actual type of the operand does not seem to matter as long
3050	// as the bits match one of the inline immediate values. For example:
3051	//
3052	// -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
3053	// so it is a legal inline immediate.
3054	//
3055	// 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
3056	// floating-point, so it is a legal inline immediate.
3057
3058	uint32_t Val = static_cast<uint32_t>(Literal);
3059	return (Val == llvm::bit_cast<uint32_t>(from: `0.0f`)) \|\|
3060	(Val == llvm::bit_cast<uint32_t>(from: `1.0f`)) \|\|
3061	(Val == llvm::bit_cast<uint32_t>(from: -`1.0f`)) \|\|
3062	(Val == llvm::bit_cast<uint32_t>(from: `0.5f`)) \|\|
3063	(Val == llvm::bit_cast<uint32_t>(from: -`0.5f`)) \|\|
3064	(Val == llvm::bit_cast<uint32_t>(from: `2.0f`)) \|\|
3065	(Val == llvm::bit_cast<uint32_t>(from: -`2.0f`)) \|\|
3066	(Val == llvm::bit_cast<uint32_t>(from: `4.0f`)) \|\|
3067	(Val == llvm::bit_cast<uint32_t>(from: -`4.0f`)) \|\|
3068	(Val == `0x3e22f983` && HasInv2Pi);
3069	}
3070
3071	bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi) {
3072	if (!HasInv2Pi)
3073	return false;
3074	if (isInlinableIntLiteral(Literal))
3075	return true;
3076	uint16_t Val = static_cast<uint16_t>(Literal);
3077	return Val == `0x3F00` \|\| // 0.5
3078	Val == `0xBF00` \|\| // -0.5
3079	Val == `0x3F80` \|\| // 1.0
3080	Val == `0xBF80` \|\| // -1.0
3081	Val == `0x4000` \|\| // 2.0
3082	Val == `0xC000` \|\| // -2.0
3083	Val == `0x4080` \|\| // 4.0
3084	Val == `0xC080` \|\| // -4.0
3085	Val == `0x3E22`; // 1.0 / (2.0 pi)*
3086	}
3087
3088	bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi) {
3089	return isInlinableLiteral32(Literal, HasInv2Pi);
3090	}
3091
3092	bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) {
3093	if (!HasInv2Pi)
3094	return false;
3095	if (isInlinableIntLiteral(Literal))
3096	return true;
3097	uint16_t Val = static_cast<uint16_t>(Literal);
3098	return Val == `0x3C00` \|\| // 1.0
3099	Val == `0xBC00` \|\| // -1.0
3100	Val == `0x3800` \|\| // 0.5
3101	Val == `0xB800` \|\| // -0.5
3102	Val == `0x4000` \|\| // 2.0
3103	Val == `0xC000` \|\| // -2.0
3104	Val == `0x4400` \|\| // 4.0
3105	Val == `0xC400` \|\| // -4.0
3106	Val == `0x3118`; // 1/2pi
3107	}
3108
3109	std::optional<unsigned> getInlineEncodingV216(bool IsFloat, uint32_t Literal) {
3110	// Unfortunately, the Instruction Set Architecture Reference Guide is
3111	// misleading about how the inline operands work for (packed) 16-bit
3112	// instructions. In a nutshell, the actual HW behavior is:
3113	//
3114	// - integer encodings (-16 .. 64) are always produced as sign-extended
3115	// 32-bit values
3116	// - float encodings are produced as:
3117	// - for F16 instructions: corresponding half-precision float values in
3118	// the LSBs, 0 in the MSBs
3119	// - for UI16 instructions: corresponding single-precision float value
3120	int32_t Signed = static_cast<int32_t>(Literal);
3121	if (Signed >= `0` && Signed <= `64`)
3122	return `128` + Signed;
3123
3124	if (Signed >= -`16` && Signed <= -`1`)
3125	return `192` + std::abs(x: Signed);
3126
3127	if (IsFloat) {
3128	// clang-format off
3129	switch (Literal) {
3130	case `0x3800`: return `240`; // 0.5
3131	case `0xB800`: return `241`; // -0.5
3132	case `0x3C00`: return `242`; // 1.0
3133	case `0xBC00`: return `243`; // -1.0
3134	case `0x4000`: return `244`; // 2.0
3135	case `0xC000`: return `245`; // -2.0
3136	case `0x4400`: return `246`; // 4.0
3137	case `0xC400`: return `247`; // -4.0
3138	case `0x3118`: return `248`; // 1.0 / (2.0 pi)*
3139	default: break;
3140	}
3141	// clang-format on
3142	} else {
3143	// clang-format off
3144	switch (Literal) {
3145	case `0x3F000000`: return `240`; // 0.5
3146	case `0xBF000000`: return `241`; // -0.5
3147	case `0x3F800000`: return `242`; // 1.0
3148	case `0xBF800000`: return `243`; // -1.0
3149	case `0x40000000`: return `244`; // 2.0
3150	case `0xC0000000`: return `245`; // -2.0
3151	case `0x40800000`: return `246`; // 4.0
3152	case `0xC0800000`: return `247`; // -4.0
3153	case `0x3E22F983`: return `248`; // 1.0 / (2.0 pi)*
3154	default: break;
3155	}
3156	// clang-format on
3157	}
3158
3159	return {};
3160	}
3161
3162	// Encoding of the literal as an inline constant for a V_PK__IU16 instruction*
3163	// or nullopt.
3164	std::optional<unsigned> getInlineEncodingV2I16(uint32_t Literal) {
3165	return getInlineEncodingV216(IsFloat: false, Literal);
3166	}
3167
3168	// Encoding of the literal as an inline constant for a V_PK__BF16 instruction*
3169	// or nullopt.
3170	std::optional<unsigned> getInlineEncodingV2BF16(uint32_t Literal) {
3171	int32_t Signed = static_cast<int32_t>(Literal);
3172	if (Signed >= `0` && Signed <= `64`)
3173	return `128` + Signed;
3174
3175	if (Signed >= -`16` && Signed <= -`1`)
3176	return `192` + std::abs(x: Signed);
3177
3178	// clang-format off
3179	switch (Literal) {
3180	case `0x3F00`: return `240`; // 0.5
3181	case `0xBF00`: return `241`; // -0.5
3182	case `0x3F80`: return `242`; // 1.0
3183	case `0xBF80`: return `243`; // -1.0
3184	case `0x4000`: return `244`; // 2.0
3185	case `0xC000`: return `245`; // -2.0
3186	case `0x4080`: return `246`; // 4.0
3187	case `0xC080`: return `247`; // -4.0
3188	case `0x3E22`: return `248`; // 1.0 / (2.0 pi)*
3189	default: break;
3190	}
3191	// clang-format on
3192
3193	return std::nullopt;
3194	}
3195
3196	// Encoding of the literal as an inline constant for a V_PK__F16 instruction*
3197	// or nullopt.
3198	std::optional<unsigned> getInlineEncodingV2F16(uint32_t Literal) {
3199	return getInlineEncodingV216(IsFloat: true, Literal);
3200	}
3201
3202	// Encoding of the literal as an inline constant for V_PK_FMAC_F16 instruction
3203	// or nullopt. This accounts for different inline constant behavior:
3204	// - Pre-GFX11: fp16 inline constants have the value in low 16 bits, 0 in high
3205	// - GFX11+: fp16 inline constants are duplicated into both halves
3206	std::optional<unsigned> getPKFMACF16InlineEncoding(uint32_t Literal,
3207	bool IsGFX11Plus) {
3208	// Pre-GFX11 behavior: f16 in low bits, 0 in high bits
3209	if (!IsGFX11Plus)
3210	return getInlineEncodingV216(/IsFloat=/true, Literal);
3211
3212	// GFX11+ behavior: f16 duplicated in both halves
3213	// First, check for sign-extended integer inline constants (-16 to 64)
3214	// These work the same across all generations
3215	int32_t Signed = static_cast<int32_t>(Literal);
3216	if (Signed >= `0` && Signed <= `64`)
3217	return `128` + Signed;
3218
3219	if (Signed >= -`16` && Signed <= -`1`)
3220	return `192` + std::abs(x: Signed);
3221
3222	// For float inline constants on GFX11+, both halves must be equal
3223	uint16_t Lo = static_cast<uint16_t>(Literal);
3224	uint16_t Hi = static_cast<uint16_t>(Literal >> `16`);
3225	if (Lo != Hi)
3226	return std::nullopt;
3227	return getInlineEncodingV216(/IsFloat=/true, Literal: Lo);
3228	}
3229
3230	// Whether the given literal can be inlined for a V_PK_ instruction.*
3231	bool isInlinableLiteralV216(uint32_t Literal, uint8_t OpType) {
3232	switch (OpType) {
3233	case AMDGPU::OPERAND_REG_IMM_V2INT16:
3234	case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
3235	return getInlineEncodingV216(IsFloat: false, Literal).has_value();
3236	case AMDGPU::OPERAND_REG_IMM_V2FP16:
3237	case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
3238	return getInlineEncodingV216(IsFloat: true, Literal).has_value();
3239	case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
3240	llvm_unreachable("OPERAND_REG_IMM_V2FP16_SPLAT is not supported");
3241	case AMDGPU::OPERAND_REG_IMM_V2BF16:
3242	case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
3243	return isInlinableLiteralV2BF16(Literal);
3244	case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
3245	return false;
3246	default:
3247	llvm_unreachable("bad packed operand type");
3248	}
3249	}
3250
3251	// Whether the given literal can be inlined for a V_PK__IU16 instruction.*
3252	bool isInlinableLiteralV2I16(uint32_t Literal) {
3253	return getInlineEncodingV2I16(Literal).has_value();
3254	}
3255
3256	// Whether the given literal can be inlined for a V_PK__BF16 instruction.*
3257	bool isInlinableLiteralV2BF16(uint32_t Literal) {
3258	return getInlineEncodingV2BF16(Literal).has_value();
3259	}
3260
3261	// Whether the given literal can be inlined for a V_PK__F16 instruction.*
3262	bool isInlinableLiteralV2F16(uint32_t Literal) {
3263	return getInlineEncodingV2F16(Literal).has_value();
3264	}
3265
3266	// Whether the given literal can be inlined for V_PK_FMAC_F16 instruction.
3267	bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus) {
3268	return getPKFMACF16InlineEncoding(Literal, IsGFX11Plus).has_value();
3269	}
3270
3271	bool isValid32BitLiteral(uint64_t Val, bool IsFP64) {
3272	if (IsFP64)
3273	return !Lo_32(Value: Val);
3274
3275	return isUInt<`32`>(x: Val) \|\| isInt<`32`>(x: Val);
3276	}
3277
3278	int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit) {
3279	switch (Type) {
3280	default:
3281	break;
3282	case OPERAND_REG_IMM_BF16:
3283	case OPERAND_REG_IMM_FP16:
3284	case OPERAND_REG_INLINE_C_BF16:
3285	case OPERAND_REG_INLINE_C_FP16:
3286	return Imm & `0xffff`;
3287	case OPERAND_INLINE_SPLIT_BARRIER_INT32:
3288	case OPERAND_REG_IMM_FP32:
3289	case OPERAND_REG_IMM_INT32:
3290	case OPERAND_REG_IMM_V2BF16:
3291	case OPERAND_REG_IMM_V2FP16:
3292	case OPERAND_REG_IMM_V2FP16_SPLAT:
3293	case OPERAND_REG_IMM_V2FP32:
3294	case OPERAND_REG_IMM_V2INT16:
3295	case OPERAND_REG_IMM_V2INT32:
3296	case OPERAND_REG_INLINE_AC_FP32:
3297	case OPERAND_REG_INLINE_AC_INT32:
3298	case OPERAND_REG_INLINE_C_FP32:
3299	case OPERAND_REG_INLINE_C_INT32:
3300	return Lo_32(Value: Imm);
3301	case OPERAND_REG_IMM_FP64:
3302	case AMDGPU::OPERAND_REG_IMM_V2FP64:
3303	return IsLit ? Imm : Hi_32(Value: Imm);
3304	}
3305	return Imm;
3306	}
3307
3308	bool isArgPassedInSGPR(const Argument *A) {
3309	const Function *F = A->getParent();
3310
3311	// Arguments to compute shaders are never a source of divergence.
3312	CallingConv::ID CC = F->getCallingConv();
3313	switch (CC) {
3314	case CallingConv::AMDGPU_KERNEL:
3315	case CallingConv::SPIR_KERNEL:
3316	return true;
3317	case CallingConv::AMDGPU_VS:
3318	case CallingConv::AMDGPU_LS:
3319	case CallingConv::AMDGPU_HS:
3320	case CallingConv::AMDGPU_ES:
3321	case CallingConv::AMDGPU_GS:
3322	case CallingConv::AMDGPU_PS:
3323	case CallingConv::AMDGPU_CS:
3324	case CallingConv::AMDGPU_Gfx:
3325	case CallingConv::AMDGPU_CS_Chain:
3326	case CallingConv::AMDGPU_CS_ChainPreserve:
3327	// For non-compute shaders, SGPR inputs are marked with either inreg or
3328	// byval. Everything else is in VGPRs.
3329	return A->hasAttribute(Kind: Attribute::InReg) \|\|
3330	A->hasAttribute(Kind: Attribute::ByVal);
3331	default:
3332	// TODO: treat i1 as divergent?
3333	return A->hasAttribute(Kind: Attribute::InReg);
3334	}
3335	}
3336
3337	bool isArgPassedInSGPR(const CallBase CB, unsigned* ArgNo) {
3338	// Arguments to compute shaders are never a source of divergence.
3339	CallingConv::ID CC = CB->getCallingConv();
3340	switch (CC) {
3341	case CallingConv::AMDGPU_KERNEL:
3342	case CallingConv::SPIR_KERNEL:
3343	return true;
3344	case CallingConv::AMDGPU_VS:
3345	case CallingConv::AMDGPU_LS:
3346	case CallingConv::AMDGPU_HS:
3347	case CallingConv::AMDGPU_ES:
3348	case CallingConv::AMDGPU_GS:
3349	case CallingConv::AMDGPU_PS:
3350	case CallingConv::AMDGPU_CS:
3351	case CallingConv::AMDGPU_Gfx:
3352	case CallingConv::AMDGPU_CS_Chain:
3353	case CallingConv::AMDGPU_CS_ChainPreserve:
3354	// For non-compute shaders, SGPR inputs are marked with either inreg or
3355	// byval. Everything else is in VGPRs.
3356	return CB->paramHasAttr(ArgNo, Kind: Attribute::InReg) \|\|
3357	CB->paramHasAttr(ArgNo, Kind: Attribute::ByVal);
3358	default:
3359	return CB->paramHasAttr(ArgNo, Kind: Attribute::InReg);
3360	}
3361	}
3362
3363	static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
3364	return isGCN3Encoding(STI: ST) \|\| isGFX10Plus(STI: ST);
3365	}
3366
3367	bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
3368	int64_t EncodedOffset) {
3369	if (isGFX12Plus(STI: ST))
3370	return isUInt<`23`>(x: EncodedOffset);
3371
3372	return hasSMEMByteOffset(ST) ? isUInt<`20`>(x: EncodedOffset)
3373	: isUInt<`8`>(x: EncodedOffset);
3374	}
3375
3376	bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
3377	int64_t EncodedOffset, bool IsBuffer) {
3378	if (isGFX12Plus(STI: ST)) {
3379	if (IsBuffer && EncodedOffset < `0`)
3380	return false;
3381	return isInt<`24`>(x: EncodedOffset);
3382	}
3383
3384	return !IsBuffer && hasSMRDSignedImmOffset(ST) && isInt<`21`>(x: EncodedOffset);
3385	}
3386
3387	static bool isDwordAligned(uint64_t ByteOffset) {
3388	return (ByteOffset & `3`) == `0`;
3389	}
3390
3391	uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
3392	uint64_t ByteOffset) {
3393	if (hasSMEMByteOffset(ST))
3394	return ByteOffset;
3395
3396	assert(isDwordAligned(ByteOffset));
3397	return ByteOffset >> `2`;
3398	}
3399
3400	std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
3401	int64_t ByteOffset, bool IsBuffer,
3402	bool HasSOffset) {
3403	// For unbuffered smem loads, it is illegal for the Immediate Offset to be
3404	// negative if the resulting (Offset + (M0 or SOffset or zero) is negative.
3405	// Handle case where SOffset is not present.
3406	if (!IsBuffer && !HasSOffset && ByteOffset < `0` && hasSMRDSignedImmOffset(ST))
3407	return std::nullopt;
3408
3409	if (isGFX12Plus(STI: ST)) // 24 bit signed offsets
3410	return isInt<`24`>(x: ByteOffset) ? std::optional<int64_t>(ByteOffset)
3411	: std::nullopt;
3412
3413	// The signed version is always a byte offset.
3414	if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
3415	assert(hasSMEMByteOffset(ST));
3416	return isInt<`20`>(x: ByteOffset) ? std::optional<int64_t>(ByteOffset)
3417	: std::nullopt;
3418	}
3419
3420	if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
3421	return std::nullopt;
3422
3423	int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3424	return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
3425	? std::optional<int64_t>(EncodedOffset)
3426	: std::nullopt;
3427	}
3428
3429	std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
3430	int64_t ByteOffset) {
3431	if (!isCI(STI: ST) \|\| !isDwordAligned(ByteOffset))
3432	return std::nullopt;
3433
3434	int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3435	return isUInt<`32`>(x: EncodedOffset) ? std::optional<int64_t>(EncodedOffset)
3436	: std::nullopt;
3437	}
3438
3439	unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST) {
3440	if (ST.getFeatureBits().test(I: FeatureFlatOffsetBits12))
3441	return `12`;
3442	if (ST.getFeatureBits().test(I: FeatureFlatOffsetBits24))
3443	return `24`;
3444	return `13`;
3445	}
3446
3447	namespace {
3448
3449	struct SourceOfDivergence {
3450	unsigned Intr;
3451	};
3452	const SourceOfDivergence lookupSourceOfDivergence(unsigned* Intr);
3453
3454	struct AlwaysUniform {
3455	unsigned Intr;
3456	};
3457	const AlwaysUniform lookupAlwaysUniform(unsigned* Intr);
3458
3459	#define GET_SourcesOfDivergence_IMPL
3460	#define GET_UniformIntrinsics_IMPL
3461	#define GET_Gfx9BufferFormat_IMPL
3462	#define GET_Gfx10BufferFormat_IMPL
3463	#define GET_Gfx11PlusBufferFormat_IMPL
3464
3465	#include "AMDGPUGenSearchableTables.inc"
3466
3467	} // end anonymous namespace
3468
3469	bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
3470	return lookupSourceOfDivergence(Intr: IntrID);
3471	}
3472
3473	bool isIntrinsicAlwaysUniform(unsigned IntrID) {
3474	return lookupAlwaysUniform(Intr: IntrID);
3475	}
3476
3477	const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
3478	uint8_t NumComponents,
3479	uint8_t NumFormat,
3480	const MCSubtargetInfo &STI) {
3481	return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(
3482	BitsPerComp, NumComponents, NumFormat)
3483	: isGFX10(STI)
3484	? getGfx10BufferFormatInfo(BitsPerComp, NumComponents, NumFormat)
3485	: getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
3486	}
3487
3488	const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
3489	const MCSubtargetInfo &STI) {
3490	return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(Format)
3491	: isGFX10(STI) ? getGfx10BufferFormatInfo(Format)
3492	: getGfx9BufferFormatInfo(Format);
3493	}
3494
3495	const MCRegisterClass *getVGPRPhysRegClass(MCRegister Reg,
3496	const MCRegisterInfo &MRI) {
3497	const unsigned VGPRClasses[] = {
3498	AMDGPU::VGPR_16RegClassID, AMDGPU::VGPR_32RegClassID,
3499	AMDGPU::VReg_64RegClassID, AMDGPU::VReg_96RegClassID,
3500	AMDGPU::VReg_128RegClassID, AMDGPU::VReg_160RegClassID,
3501	AMDGPU::VReg_192RegClassID, AMDGPU::VReg_224RegClassID,
3502	AMDGPU::VReg_256RegClassID, AMDGPU::VReg_288RegClassID,
3503	AMDGPU::VReg_320RegClassID, AMDGPU::VReg_352RegClassID,
3504	AMDGPU::VReg_384RegClassID, AMDGPU::VReg_512RegClassID,
3505	AMDGPU::VReg_1024RegClassID};
3506
3507	for (unsigned RCID : VGPRClasses) {
3508	const MCRegisterClass &RC = MRI.getRegClass(i: RCID);
3509	if (RC.contains(Reg))
3510	return &RC;
3511	}
3512
3513	return nullptr;
3514	}
3515
3516	unsigned getVGPREncodingMSBs(MCRegister Reg, const MCRegisterInfo &MRI) {
3517	unsigned Enc = MRI.getEncodingValue(Reg);
3518	unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3519	return Idx >> `8`;
3520	}
3521
3522	MCRegister getVGPRWithMSBs(MCRegister Reg, unsigned MSBs,
3523	const MCRegisterInfo &MRI) {
3524	unsigned Enc = MRI.getEncodingValue(Reg);
3525	unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3526	if (Idx >= `0x100`)
3527	return MCRegister ();
3528
3529	const MCRegisterClass *RC = getVGPRPhysRegClass(Reg, MRI);
3530	if (!RC)
3531	return MCRegister ();
3532
3533	Idx \|= MSBs << `8`;
3534	if (RC->getID() == AMDGPU::VGPR_16RegClassID) {
3535	// This class has 2048 registers with interleaved lo16 and hi16.
3536	Idx *= `2`;
3537	if (Enc & AMDGPU::HWEncoding::IS_HI16)
3538	++Idx;
3539	}
3540
3541	return RC->getRegister(i: Idx);
3542	}
3543
3544	static std::optional<unsigned>
3545	convertSetRegImmToVgprMSBs(unsigned Imm, unsigned Simm16,
3546	bool HasSetregVGPRMSBFixup) {
3547	constexpr unsigned VGPRMSBShift =
3548	llvm::countr_zero_constexpr<unsigned>(Val: AMDGPU::Hwreg::DST_VGPR_MSB);
3549
3550	auto [HwRegId, Offset, Size] = Hwreg::HwregEncoding::decode(Encoded: Simm16);
3551	if (HwRegId != Hwreg::ID_MODE \|\|
3552	(!HasSetregVGPRMSBFixup && (Offset + Size) < VGPRMSBShift))
3553	return {};
3554	// If there is SetregVGPRMSBFixup then Offset is ignored.
3555	if (!HasSetregVGPRMSBFixup)
3556	Imm <<= Offset;
3557	Imm = (Imm & Hwreg::VGPR_MSB_MASK) >> VGPRMSBShift;
3558	if (!HasSetregVGPRMSBFixup)
3559	Imm &= llvm::maskTrailingOnes<unsigned>(N: Size);
3560	return llvm::rotr<uint8_t>(V: static_cast<uint8_t>(Imm), /R=/`2`);
3561	}
3562
3563	std::optional<unsigned> convertSetRegImmToVgprMSBs(const MachineInstr &MI,
3564	bool HasSetregVGPRMSBFixup) {
3565	assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32);
3566	return convertSetRegImmToVgprMSBs(Imm: MI.getOperand(i: `0`).getImm(),
3567	Simm16: MI.getOperand(i: `1`).getImm(),
3568	HasSetregVGPRMSBFixup);
3569	}
3570
3571	std::optional<unsigned> convertSetRegImmToVgprMSBs(const MCInst &MI,
3572	bool HasSetregVGPRMSBFixup) {
3573	assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_gfx12);
3574	return convertSetRegImmToVgprMSBs(Imm: MI.getOperand(i: `0`).getImm(),
3575	Simm16: MI.getOperand(i: `1`).getImm(),
3576	HasSetregVGPRMSBFixup);
3577	}
3578
3579	std::pair<const AMDGPU::OpName , const* AMDGPU::OpName *>
3580	getVGPRLoweringOperandTables(const MCInstrDesc &Desc) {
3581	static const AMDGPU::OpName VOPOps[`4`] = {
3582	AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2,
3583	AMDGPU::OpName::vdst};
3584	static const AMDGPU::OpName VDSOps[`4`] = {
3585	AMDGPU::OpName::addr, AMDGPU::OpName::data0, AMDGPU::OpName::data1,
3586	AMDGPU::OpName::vdst};
3587	static const AMDGPU::OpName FLATOps[`4`] = {
3588	AMDGPU::OpName::vaddr, AMDGPU::OpName::vdata,
3589	AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdst};
3590	static const AMDGPU::OpName BUFOps[`4`] = {
3591	AMDGPU::OpName::vaddr, AMDGPU::OpName::NUM_OPERAND_NAMES,
3592	AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdata};
3593	static const AMDGPU::OpName VIMGOps[`4`] = {
3594	AMDGPU::OpName::vaddr0, AMDGPU::OpName::vaddr1, AMDGPU::OpName::vaddr2,
3595	AMDGPU::OpName::vdata};
3596
3597	// For VOPD instructions MSB of a corresponding Y component operand VGPR
3598	// address is supposed to match X operand, otherwise VOPD shall not be
3599	// combined.
3600	static const AMDGPU::OpName VOPDOpsX[`4`] = {
3601	AMDGPU::OpName::src0X, AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vsrc2X,
3602	AMDGPU::OpName::vdstX};
3603	static const AMDGPU::OpName VOPDOpsY[`4`] = {
3604	AMDGPU::OpName::src0Y, AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vsrc2Y,
3605	AMDGPU::OpName::vdstY};
3606
3607	// VOP2 MADMK instructions use src0, imm, src1 scheme.
3608	static const AMDGPU::OpName VOP2MADMKOps[`4`] = {
3609	AMDGPU::OpName::src0, AMDGPU::OpName::NUM_OPERAND_NAMES,
3610	AMDGPU::OpName::src1, AMDGPU::OpName::vdst};
3611	static const AMDGPU::OpName VOPDFMAMKOpsX[`4`] = {
3612	AMDGPU::OpName::src0X, AMDGPU::OpName::NUM_OPERAND_NAMES,
3613	AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vdstX};
3614	static const AMDGPU::OpName VOPDFMAMKOpsY[`4`] = {
3615	AMDGPU::OpName::src0Y, AMDGPU::OpName::NUM_OPERAND_NAMES,
3616	AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vdstY};
3617
3618	if (SIInstrFlags::isVOP1(O: Desc) \|\| SIInstrFlags::isVOP2(O: Desc) \|\|
3619	SIInstrFlags::isVOP3Like(O: Desc) \|\| SIInstrFlags::isVOPC(O: Desc) \|\|
3620	SIInstrFlags::isDPP(O: Desc)) {
3621	switch (Desc.getOpcode()) {
3622	// LD_SCALE operands ignore MSB.
3623	case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32:
3624	case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32_gfx1250:
3625	case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64:
3626	case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64_gfx1250:
3627	return {};
3628	case AMDGPU::V_FMAMK_F16:
3629	case AMDGPU::V_FMAMK_F16_t16:
3630	case AMDGPU::V_FMAMK_F16_t16_gfx12:
3631	case AMDGPU::V_FMAMK_F16_fake16:
3632	case AMDGPU::V_FMAMK_F16_fake16_gfx12:
3633	case AMDGPU::V_FMAMK_F32:
3634	case AMDGPU::V_FMAMK_F32_gfx12:
3635	case AMDGPU::V_FMAMK_F64:
3636	case AMDGPU::V_FMAMK_F64_gfx1250:
3637	return {VOP2MADMKOps, nullptr};
3638	default:
3639	break;
3640	}
3641	return {VOPOps, nullptr};
3642	}
3643
3644	if (SIInstrFlags::isDS(O: Desc))
3645	return {VDSOps, nullptr};
3646
3647	if (SIInstrFlags::isFLAT(O: Desc))
3648	return {FLATOps, nullptr};
3649
3650	if (SIInstrFlags::isBuffer(O: Desc))
3651	return {BUFOps, nullptr};
3652
3653	if (SIInstrFlags::isVIMAGE(O: Desc))
3654	return {VIMGOps, nullptr};
3655
3656	if (AMDGPU::isVOPD(Opc: Desc.getOpcode())) {
3657	auto [OpX, OpY] = getVOPDComponents(VOPDOpcode: Desc.getOpcode());
3658	return {(OpX == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsX : VOPDOpsX,
3659	(OpY == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsY : VOPDOpsY};
3660	}
3661
3662	assert(!SIInstrFlags::isMIMG(Desc));
3663
3664	if (SIInstrFlags::isVSAMPLE(O: Desc) \|\| SIInstrFlags::isEXP(O: Desc))
3665	llvm_unreachable("Sample and export VGPR lowering is not implemented and"
3666	" these instructions are not expected on gfx1250");
3667
3668	return {};
3669	}
3670
3671	bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode) {
3672	const MCInstrDesc &Desc = MII.get(Opcode);
3673	if (SIInstrFlags::isSMRD(O: Desc))
3674	return Desc.mayLoad() && !Desc.mayStore() && !getSMEMIsBuffer(Opc: Opcode);
3675	if (!SIInstrFlags::isFLAT(O: Desc))
3676	return false;
3677
3678	// Only SV and SVS modes are supported.
3679	if (SIInstrFlags::isFlatScratch(O: MII, O: Opcode))
3680	return hasNamedOperand(Opcode, NamedIdx: OpName::vaddr);
3681
3682	// Only GVS mode is supported.
3683	return hasNamedOperand(Opcode, NamedIdx: OpName::vaddr) &&
3684	hasNamedOperand(Opcode, NamedIdx: OpName::saddr);
3685
3686	return false;
3687	}
3688
3689	bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
3690	const MCSubtargetInfo &ST) {
3691	for (auto OpName : {OpName::vdst, OpName::src0, OpName::src1, OpName::src2}) {
3692	int Idx = getNamedOperandIdx(Opcode: OpDesc.getOpcode(), Name: OpName);
3693	if (Idx == -`1`)
3694	continue;
3695
3696	const MCOperandInfo &OpInfo = OpDesc.operands()[Idx];
3697	int16_t RegClass = MII.getOpRegClassID(
3698	OpInfo, HwModeId: ST.getHwMode(type: MCSubtargetInfo::HwMode_RegInfo));
3699	if (RegClass == AMDGPU::VReg_64RegClassID \|\|
3700	RegClass == AMDGPU::VReg_64_Align2RegClassID)
3701	return true;
3702	}
3703
3704	return false;
3705	}
3706
3707	bool isDPALU_DPP32BitOpc(unsigned Opc) {
3708	switch (Opc) {
3709	case AMDGPU::V_MUL_LO_U32_e64:
3710	case AMDGPU::V_MUL_LO_U32_e64_dpp:
3711	case AMDGPU::V_MUL_LO_U32_e64_dpp_gfx1250:
3712	case AMDGPU::V_MUL_HI_U32_e64:
3713	case AMDGPU::V_MUL_HI_U32_e64_dpp:
3714	case AMDGPU::V_MUL_HI_U32_e64_dpp_gfx1250:
3715	case AMDGPU::V_MUL_HI_I32_e64:
3716	case AMDGPU::V_MUL_HI_I32_e64_dpp:
3717	case AMDGPU::V_MUL_HI_I32_e64_dpp_gfx1250:
3718	case AMDGPU::V_MAD_U32_e64:
3719	case AMDGPU::V_MAD_U32_e64_dpp:
3720	case AMDGPU::V_MAD_U32_e64_dpp_gfx1250:
3721	return true;
3722	default:
3723	return false;
3724	}
3725	}
3726
3727	bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
3728	const MCSubtargetInfo &ST) {
3729	if (!ST.hasFeature(Feature: AMDGPU::FeatureDPALU_DPP))
3730	return false;
3731
3732	if (isDPALU_DPP32BitOpc(Opc: OpDesc.getOpcode()))
3733	return ST.hasFeature(Feature: AMDGPU::FeatureGFX1250Insts);
3734
3735	return hasAny64BitVGPROperands(OpDesc, MII, ST);
3736	}
3737
3738	unsigned getLdsDwGranularity(const MCSubtargetInfo &ST) {
3739	if (ST.getFeatureBits().test(I: FeatureAddressableLocalMemorySize32768))
3740	return `64`;
3741	if (ST.getFeatureBits().test(I: FeatureAddressableLocalMemorySize65536))
3742	return `128`;
3743	if (ST.getFeatureBits().test(I: FeatureAddressableLocalMemorySize163840))
3744	return `320`;
3745	if (ST.getFeatureBits().test(I: FeatureAddressableLocalMemorySize327680))
3746	return `512`;
3747	return `64`; // In sync with getAddressableLocalMemorySize
3748	}
3749
3750	bool isPackedFP32Inst(unsigned Opc) {
3751	switch (Opc) {
3752	case AMDGPU::V_PK_ADD_F32:
3753	case AMDGPU::V_PK_ADD_F32_gfx12:
3754	case AMDGPU::V_PK_MUL_F32:
3755	case AMDGPU::V_PK_MUL_F32_gfx12:
3756	case AMDGPU::V_PK_FMA_F32:
3757	case AMDGPU::V_PK_FMA_F32_gfx12:
3758	return true;
3759	default:
3760	return false;
3761	}
3762	}
3763
3764	bool isPacked64BitInst(unsigned Opc) {
3765	switch (Opc) {
3766	case AMDGPU::V_PK_ADD_F64:
3767	case AMDGPU::V_PK_ADD_F64_gfx1250:
3768	case AMDGPU::V_PK_MUL_F64:
3769	case AMDGPU::V_PK_MUL_F64_gfx1250:
3770	case AMDGPU::V_PK_FMA_F64:
3771	case AMDGPU::V_PK_FMA_F64_gfx1250:
3772	case AMDGPU::V_PK_MAX_NUM_F64:
3773	case AMDGPU::V_PK_MAX_NUM_F64_gfx1250:
3774	case AMDGPU::V_PK_MIN_NUM_F64:
3775	case AMDGPU::V_PK_MIN_NUM_F64_gfx1250:
3776	case AMDGPU::V_PK_ADD_NC_U64:
3777	case AMDGPU::V_PK_ADD_NC_U64_gfx1250:
3778	case AMDGPU::V_PK_SUB_NC_U64:
3779	case AMDGPU::V_PK_SUB_NC_U64_gfx1250:
3780	case AMDGPU::V_PK_LSHL_ADD_U64:
3781	case AMDGPU::V_PK_LSHL_ADD_U64_gfx1250:
3782	return true;
3783	default:
3784	return false;
3785	}
3786	}
3787
3788	bool isPackedFP32or64BitInst(unsigned Opc) {
3789	return isPackedFP32Inst(Opc) \|\| isPacked64BitInst(Opc);
3790	}
3791
3792	const std::array<unsigned, `3`> &ClusterDimsAttr::getDims() const {
3793	assert(isFixedDims() && "expect kind to be FixedDims");
3794	return Dims;
3795	}
3796
3797	std::string ClusterDimsAttr::to_string() const {
3798	SmallString<`10`> Buffer;
3799	raw_svector_ostream OS(Buffer);
3800
3801	switch (getKind()) {
3802	case Kind::Unknown:
3803	return "";
3804	case Kind::NoCluster: {
3805	OS << EncoNoCluster << `','` << EncoNoCluster << `','` << EncoNoCluster;
3806	return Buffer.c_str();
3807	}
3808	case Kind::VariableDims: {
3809	OS << EncoVariableDims << `','` << EncoVariableDims << `','`
3810	<< EncoVariableDims;
3811	return Buffer.c_str();
3812	}
3813	case Kind::FixedDims: {
3814	OS << Dims [`0`] << `','` << Dims [`1`] << `','` << Dims [`2`];
3815	return Buffer.c_str();
3816	}
3817	}
3818	llvm_unreachable("Unknown ClusterDimsAttr kind");
3819	}
3820
3821	ClusterDimsAttr ClusterDimsAttr::get(const Function &F) {
3822	std::optional<SmallVector<unsigned>> Attr =
3823	getIntegerVecAttribute(F, Name: "amdgpu-cluster-dims", /Size=/`3`);
3824	ClusterDimsAttr::Kind AttrKind = Kind::FixedDims;
3825
3826	if (!Attr.has_value())
3827	AttrKind = Kind::Unknown;
3828	else if (all_of(Range&: *Attr, P: equal_to(Arg: EncoNoCluster)))
3829	AttrKind = Kind::NoCluster;
3830	else if (all_of(Range&: *Attr, P: equal_to(Arg: EncoVariableDims)))
3831	AttrKind = Kind::VariableDims;
3832
3833	ClusterDimsAttr A(AttrKind);
3834	if (AttrKind == Kind::FixedDims)
3835	A.Dims = {(Attr)[`0`], (Attr)[`1`], (*Attr)[`2`]};
3836
3837	return A;
3838	}
3839
3840	} // namespace AMDGPU
3841
3842	raw_ostream &operator<<(raw_ostream &OS, const AMDGPU::TargetIDSetting S) {
3843	switch (S) {
3844	case (AMDGPU::TargetIDSetting::Unsupported):
3845	OS << "Unsupported";
3846	break;
3847	case (AMDGPU::TargetIDSetting::Any):
3848	OS << "Any";
3849	break;
3850	case (AMDGPU::TargetIDSetting::Off):
3851	OS << "Off";
3852	break;
3853	case (AMDGPU::TargetIDSetting::On):
3854	OS << "On";
3855	break;
3856	}
3857	return OS;
3858	}
3859
3860	} // namespace llvm
3861

Browse the source code of llvm_projects/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp