1//===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUBaseInfo.h"
10#include "AMDGPU.h"
11#include "AMDGPUAsmUtils.h"
12#include "AMDKernelCodeT.h"
13#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14#include "Utils/AMDKernelCodeTUtils.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/BinaryFormat/ELF.h"
17#include "llvm/IR/Attributes.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/Function.h"
20#include "llvm/IR/GlobalValue.h"
21#include "llvm/IR/IntrinsicsAMDGPU.h"
22#include "llvm/IR/IntrinsicsR600.h"
23#include "llvm/IR/LLVMContext.h"
24#include "llvm/IR/Metadata.h"
25#include "llvm/MC/MCInstrInfo.h"
26#include "llvm/MC/MCRegisterInfo.h"
27#include "llvm/MC/MCSubtargetInfo.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/TargetParser/AMDGPUTargetParser.h"
30#include <optional>
31
32#define GET_INSTRINFO_NAMED_OPS
33#define GET_INSTRMAP_INFO
34#include "AMDGPUGenInstrInfo.inc"
35
36static llvm::cl::opt<unsigned> DefaultAMDHSACodeObjectVersion(
37 "amdhsa-code-object-version", llvm::cl::Hidden,
38 llvm::cl::init(Val: llvm::AMDGPU::AMDHSA_COV6),
39 llvm::cl::desc("Set default AMDHSA Code Object Version (module flag "
40 "or asm directive still take priority if present)"));
41
42namespace {
43
44/// \returns Bit mask for given bit \p Shift and bit \p Width.
45unsigned getBitMask(unsigned Shift, unsigned Width) {
46 return ((1 << Width) - 1) << Shift;
47}
48
49/// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
50///
51/// \returns Packed \p Dst.
52unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
53 unsigned Mask = getBitMask(Shift, Width);
54 return ((Src << Shift) & Mask) | (Dst & ~Mask);
55}
56
57/// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
58///
59/// \returns Unpacked bits.
60unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
61 return (Src & getBitMask(Shift, Width)) >> Shift;
62}
63
64/// \returns Vmcnt bit shift (lower bits).
65unsigned getVmcntBitShiftLo(unsigned VersionMajor) {
66 return VersionMajor >= 11 ? 10 : 0;
67}
68
69/// \returns Vmcnt bit width (lower bits).
70unsigned getVmcntBitWidthLo(unsigned VersionMajor) {
71 return VersionMajor >= 11 ? 6 : 4;
72}
73
74/// \returns Expcnt bit shift.
75unsigned getExpcntBitShift(unsigned VersionMajor) {
76 return VersionMajor >= 11 ? 0 : 4;
77}
78
79/// \returns Expcnt bit width.
80unsigned getExpcntBitWidth(unsigned VersionMajor) { return 3; }
81
82/// \returns Lgkmcnt bit shift.
83unsigned getLgkmcntBitShift(unsigned VersionMajor) {
84 return VersionMajor >= 11 ? 4 : 8;
85}
86
87/// \returns Lgkmcnt bit width.
88unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
89 return VersionMajor >= 10 ? 6 : 4;
90}
91
92/// \returns Vmcnt bit shift (higher bits).
93unsigned getVmcntBitShiftHi(unsigned VersionMajor) { return 14; }
94
95/// \returns Vmcnt bit width (higher bits).
96unsigned getVmcntBitWidthHi(unsigned VersionMajor) {
97 return (VersionMajor == 9 || VersionMajor == 10) ? 2 : 0;
98}
99
100/// \returns Loadcnt bit width
101unsigned getLoadcntBitWidth(unsigned VersionMajor) {
102 return VersionMajor >= 12 ? 6 : 0;
103}
104
105/// \returns Samplecnt bit width.
106unsigned getSamplecntBitWidth(unsigned VersionMajor) {
107 return VersionMajor >= 12 ? 6 : 0;
108}
109
110/// \returns Bvhcnt bit width.
111unsigned getBvhcntBitWidth(unsigned VersionMajor) {
112 return VersionMajor >= 12 ? 3 : 0;
113}
114
115/// \returns Dscnt bit width.
116unsigned getDscntBitWidth(unsigned VersionMajor) {
117 return VersionMajor >= 12 ? 6 : 0;
118}
119
120/// \returns Dscnt bit shift in combined S_WAIT instructions.
121unsigned getDscntBitShift(unsigned VersionMajor) { return 0; }
122
123/// \returns Storecnt or Vscnt bit width, depending on VersionMajor.
124unsigned getStorecntBitWidth(unsigned VersionMajor) {
125 return VersionMajor >= 10 ? 6 : 0;
126}
127
128/// \returns Kmcnt bit width.
129unsigned getKmcntBitWidth(unsigned VersionMajor) {
130 return VersionMajor >= 12 ? 5 : 0;
131}
132
133/// \returns Xcnt bit width.
134unsigned getXcntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
135 return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
136}
137
138/// \returns Asynccnt bit width.
139unsigned getAsynccntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
140 return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
141}
142
143/// \returns shift for Loadcnt/Storecnt in combined S_WAIT instructions.
144unsigned getLoadcntStorecntBitShift(unsigned VersionMajor) {
145 return VersionMajor >= 12 ? 8 : 0;
146}
147
148/// \returns VaSdst bit width
149inline unsigned getVaSdstBitWidth() { return 3; }
150
151/// \returns VaSdst bit shift
152inline unsigned getVaSdstBitShift() { return 9; }
153
154/// \returns VmVsrc bit width
155inline unsigned getVmVsrcBitWidth() { return 3; }
156
157/// \returns VmVsrc bit shift
158inline unsigned getVmVsrcBitShift() { return 2; }
159
160/// \returns VaVdst bit width
161inline unsigned getVaVdstBitWidth() { return 4; }
162
163/// \returns VaVdst bit shift
164inline unsigned getVaVdstBitShift() { return 12; }
165
166/// \returns VaVcc bit width
167inline unsigned getVaVccBitWidth() { return 1; }
168
169/// \returns VaVcc bit shift
170inline unsigned getVaVccBitShift() { return 1; }
171
172/// \returns SaSdst bit width
173inline unsigned getSaSdstBitWidth() { return 1; }
174
175/// \returns SaSdst bit shift
176inline unsigned getSaSdstBitShift() { return 0; }
177
178/// \returns VaSsrc width
179inline unsigned getVaSsrcBitWidth() { return 1; }
180
181/// \returns VaSsrc bit shift
182inline unsigned getVaSsrcBitShift() { return 8; }
183
184/// \returns HoldCnt bit shift
185inline unsigned getHoldCntWidth(unsigned VersionMajor, unsigned VersionMinor) {
186 static constexpr const unsigned MinMajor = 10;
187 static constexpr const unsigned MinMinor = 3;
188 return std::tie(args&: VersionMajor, args&: VersionMinor) >= std::tie(args: MinMajor, args: MinMinor)
189 ? 1
190 : 0;
191}
192
193/// \returns HoldCnt bit shift
194inline unsigned getHoldCntBitShift() { return 7; }
195
196} // end anonymous namespace
197
198namespace llvm {
199
200namespace AMDGPU {
201
202/// \returns true if the target supports signed immediate offset for SMRD
203/// instructions.
204bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
205 return isGFX9Plus(STI: ST);
206}
207
208/// \returns True if \p STI is AMDHSA.
209bool isHsaAbi(const MCSubtargetInfo &STI) {
210 return STI.getTargetTriple().getOS() == Triple::AMDHSA;
211}
212
213unsigned getAMDHSACodeObjectVersion(const Module &M) {
214 if (auto *Ver = mdconst::extract_or_null<ConstantInt>(
215 MD: M.getModuleFlag(Key: "amdhsa_code_object_version"))) {
216 return (unsigned)Ver->getZExtValue() / 100;
217 }
218
219 return getDefaultAMDHSACodeObjectVersion();
220}
221
222unsigned getDefaultAMDHSACodeObjectVersion() {
223 return DefaultAMDHSACodeObjectVersion;
224}
225
226unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion) {
227 switch (ABIVersion) {
228 case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
229 return 4;
230 case ELF::ELFABIVERSION_AMDGPU_HSA_V5:
231 return 5;
232 case ELF::ELFABIVERSION_AMDGPU_HSA_V6:
233 return 6;
234 default:
235 return getDefaultAMDHSACodeObjectVersion();
236 }
237}
238
239uint8_t getELFABIVersion(const Triple &T, unsigned CodeObjectVersion) {
240 if (T.getOS() != Triple::AMDHSA)
241 return 0;
242
243 switch (CodeObjectVersion) {
244 case 4:
245 return ELF::ELFABIVERSION_AMDGPU_HSA_V4;
246 case 5:
247 return ELF::ELFABIVERSION_AMDGPU_HSA_V5;
248 case 6:
249 return ELF::ELFABIVERSION_AMDGPU_HSA_V6;
250 default:
251 report_fatal_error(reason: "Unsupported AMDHSA Code Object Version " +
252 Twine(CodeObjectVersion));
253 }
254}
255
256unsigned getMultigridSyncArgImplicitArgPosition(unsigned CodeObjectVersion) {
257 switch (CodeObjectVersion) {
258 case AMDHSA_COV4:
259 return 48;
260 case AMDHSA_COV5:
261 case AMDHSA_COV6:
262 default:
263 return AMDGPU::ImplicitArg::MULTIGRID_SYNC_ARG_OFFSET;
264 }
265}
266
267// FIXME: All such magic numbers about the ABI should be in a
268// central TD file.
269unsigned getHostcallImplicitArgPosition(unsigned CodeObjectVersion) {
270 switch (CodeObjectVersion) {
271 case AMDHSA_COV4:
272 return 24;
273 case AMDHSA_COV5:
274 case AMDHSA_COV6:
275 default:
276 return AMDGPU::ImplicitArg::HOSTCALL_PTR_OFFSET;
277 }
278}
279
280unsigned getDefaultQueueImplicitArgPosition(unsigned CodeObjectVersion) {
281 switch (CodeObjectVersion) {
282 case AMDHSA_COV4:
283 return 32;
284 case AMDHSA_COV5:
285 case AMDHSA_COV6:
286 default:
287 return AMDGPU::ImplicitArg::DEFAULT_QUEUE_OFFSET;
288 }
289}
290
291unsigned getCompletionActionImplicitArgPosition(unsigned CodeObjectVersion) {
292 switch (CodeObjectVersion) {
293 case AMDHSA_COV4:
294 return 40;
295 case AMDHSA_COV5:
296 case AMDHSA_COV6:
297 default:
298 return AMDGPU::ImplicitArg::COMPLETION_ACTION_OFFSET;
299 }
300}
301
302#define GET_MIMGBaseOpcodesTable_IMPL
303#define GET_MIMGDimInfoTable_IMPL
304#define GET_MIMGInfoTable_IMPL
305#define GET_MIMGLZMappingTable_IMPL
306#define GET_MIMGMIPMappingTable_IMPL
307#define GET_MIMGBiasMappingTable_IMPL
308#define GET_MIMGOffsetMappingTable_IMPL
309#define GET_MIMGG16MappingTable_IMPL
310#define GET_MAIInstInfoTable_IMPL
311#define GET_WMMAInstInfoTable_IMPL
312#include "AMDGPUGenSearchableTables.inc"
313
314int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
315 unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc,
316 bool IndexedSamp) {
317 const MIMGInfo *Info =
318 getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, VDataDwords, VAddrDwords,
319 IndexedRsrc, IndexedSamp);
320 return Info ? Info->Opcode : -1;
321}
322
323const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
324 const MIMGInfo *Info = getMIMGInfo(Opcode: Opc);
325 return Info ? getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode) : nullptr;
326}
327
328int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
329 const MIMGInfo *OrigInfo = getMIMGInfo(Opcode: Opc);
330 const MIMGInfo *NewInfo = getMIMGOpcodeHelper(
331 BaseOpcode: OrigInfo->BaseOpcode, MIMGEncoding: OrigInfo->MIMGEncoding, VDataDwords: NewChannels,
332 VAddrDwords: OrigInfo->VAddrDwords, IndexedRsrc: OrigInfo->IndexedRsrc, IndexedSamp: OrigInfo->IndexedSamp);
333 return NewInfo ? NewInfo->Opcode : -1;
334}
335
336unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
337 const MIMGDimInfo *Dim, bool IsA16,
338 bool IsG16Supported) {
339 unsigned AddrWords = BaseOpcode->NumExtraArgs;
340 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
341 (BaseOpcode->LodOrClampOrMip ? 1 : 0);
342 if (IsA16)
343 AddrWords += divideCeil(Numerator: AddrComponents, Denominator: 2);
344 else
345 AddrWords += AddrComponents;
346
347 // Note: For subtargets that support A16 but not G16, enabling A16 also
348 // enables 16 bit gradients.
349 // For subtargets that support A16 (operand) and G16 (done with a different
350 // instruction encoding), they are independent.
351
352 if (BaseOpcode->Gradients) {
353 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16)
354 // There are two gradients per coordinate, we pack them separately.
355 // For the 3d case,
356 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv)
357 AddrWords += alignTo<2>(Value: Dim->NumGradients / 2);
358 else
359 AddrWords += Dim->NumGradients;
360 }
361 return AddrWords;
362}
363
364struct MUBUFInfo {
365 uint32_t Opcode;
366 uint32_t BaseOpcode;
367 uint8_t elements;
368 bool has_vaddr;
369 bool has_srsrc;
370 bool has_soffset;
371 bool IsBufferInv;
372 bool tfe;
373};
374
375struct MTBUFInfo {
376 uint32_t Opcode;
377 uint32_t BaseOpcode;
378 uint8_t elements;
379 bool has_vaddr;
380 bool has_srsrc;
381 bool has_soffset;
382};
383
384struct SMInfo {
385 uint32_t Opcode;
386};
387
388struct VOPInfo {
389 uint32_t Opcode;
390};
391
392struct VOPC64DPPInfo {
393 uint32_t Opcode;
394};
395
396struct VOPCDPPAsmOnlyInfo {
397 uint32_t Opcode;
398};
399
400struct VOPDComponentInfo {
401 uint16_t BaseVOP;
402 uint16_t VOPDOp;
403};
404
405struct VOPDInfo {
406 uint32_t Opcode;
407 uint16_t OpX;
408 uint16_t OpY;
409 uint16_t Subtarget;
410 bool VOPD3;
411};
412
413struct VOPTrue16Info {
414 uint32_t Opcode;
415};
416
417struct VOPDXYInfo {
418 uint16_t VOPDXYKey;
419 bool IsX;
420 bool IsY;
421};
422
423#define GET_FP4FP8DstByteSelTable_DECL
424#define GET_FP4FP8DstByteSelTable_IMPL
425
426struct DPMACCInstructionInfo {
427 uint32_t Opcode;
428};
429
430struct FP4FP8DstByteSelInfo {
431 uint32_t Opcode;
432 bool HasFP8DstByteSel;
433 bool HasFP4DstByteSel;
434};
435
436#define GET_DPMACCInstructionTable_DECL
437#define GET_DPMACCInstructionTable_IMPL
438#define GET_MTBUFInfoTable_DECL
439#define GET_MTBUFInfoTable_IMPL
440#define GET_MUBUFInfoTable_DECL
441#define GET_MUBUFInfoTable_IMPL
442#define GET_SMInfoTable_DECL
443#define GET_SMInfoTable_IMPL
444#define GET_VOP1InfoTable_DECL
445#define GET_VOP1InfoTable_IMPL
446#define GET_VOP2InfoTable_DECL
447#define GET_VOP2InfoTable_IMPL
448#define GET_VOP3InfoTable_DECL
449#define GET_VOP3InfoTable_IMPL
450#define GET_VOPC64DPPTable_DECL
451#define GET_VOPC64DPPTable_IMPL
452#define GET_VOPC64DPP8Table_DECL
453#define GET_VOPC64DPP8Table_IMPL
454#define GET_VOPCAsmOnlyInfoTable_DECL
455#define GET_VOPCAsmOnlyInfoTable_IMPL
456#define GET_VOP3CAsmOnlyInfoTable_DECL
457#define GET_VOP3CAsmOnlyInfoTable_IMPL
458#define GET_VOPDComponentTable_DECL
459#define GET_VOPDComponentTable_IMPL
460#define GET_VOPDPairs_DECL
461#define GET_VOPDPairs_IMPL
462#define GET_VOPDXYTable_DECL
463#define GET_VOPDXYTable_IMPL
464#define GET_VOPTrue16Table_DECL
465#define GET_VOPTrue16Table_IMPL
466#define GET_True16D16Table_IMPL
467#define GET_WMMAOpcode2AddrMappingTable_DECL
468#define GET_WMMAOpcode2AddrMappingTable_IMPL
469#define GET_WMMAOpcode3AddrMappingTable_DECL
470#define GET_WMMAOpcode3AddrMappingTable_IMPL
471#define GET_getMFMA_F8F6F4_WithSize_DECL
472#define GET_getMFMA_F8F6F4_WithSize_IMPL
473#define GET_isMFMA_F8F6F4Table_IMPL
474#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL
475
476#include "AMDGPUGenSearchableTables.inc"
477
478int getMTBUFBaseOpcode(unsigned Opc) {
479 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opcode: Opc);
480 return Info ? Info->BaseOpcode : -1;
481}
482
483int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
484 const MTBUFInfo *Info =
485 getMTBUFInfoFromBaseOpcodeAndElements(BaseOpcode: BaseOpc, elements: Elements);
486 return Info ? Info->Opcode : -1;
487}
488
489int getMTBUFElements(unsigned Opc) {
490 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
491 return Info ? Info->elements : 0;
492}
493
494bool getMTBUFHasVAddr(unsigned Opc) {
495 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
496 return Info && Info->has_vaddr;
497}
498
499bool getMTBUFHasSrsrc(unsigned Opc) {
500 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
501 return Info && Info->has_srsrc;
502}
503
504bool getMTBUFHasSoffset(unsigned Opc) {
505 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
506 return Info && Info->has_soffset;
507}
508
509int getMUBUFBaseOpcode(unsigned Opc) {
510 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opcode: Opc);
511 return Info ? Info->BaseOpcode : -1;
512}
513
514int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
515 const MUBUFInfo *Info =
516 getMUBUFInfoFromBaseOpcodeAndElements(BaseOpcode: BaseOpc, elements: Elements);
517 return Info ? Info->Opcode : -1;
518}
519
520int getMUBUFElements(unsigned Opc) {
521 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
522 return Info ? Info->elements : 0;
523}
524
525bool getMUBUFHasVAddr(unsigned Opc) {
526 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
527 return Info && Info->has_vaddr;
528}
529
530bool getMUBUFHasSrsrc(unsigned Opc) {
531 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
532 return Info && Info->has_srsrc;
533}
534
535bool getMUBUFHasSoffset(unsigned Opc) {
536 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
537 return Info && Info->has_soffset;
538}
539
540bool getMUBUFIsBufferInv(unsigned Opc) {
541 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
542 return Info && Info->IsBufferInv;
543}
544
545bool getMUBUFTfe(unsigned Opc) {
546 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
547 return Info && Info->tfe;
548}
549
550bool getSMEMIsBuffer(unsigned Opc) {
551 return isSMEMOpcodeHelper(Opcode: Opc) != nullptr;
552}
553
554bool getVOP1IsSingle(unsigned Opc) {
555 return isVOP1SingleOpcodeHelper(Opcode: Opc) != nullptr;
556}
557
558bool getVOP2IsSingle(unsigned Opc) {
559 return isVOP2SingleOpcodeHelper(Opcode: Opc) != nullptr;
560}
561
562bool getVOP3IsSingle(unsigned Opc) {
563 return isVOP3SingleOpcodeHelper(Opcode: Opc) != nullptr;
564}
565
566bool isVOPC64DPP(unsigned Opc) {
567 return isVOPC64DPPOpcodeHelper(Opcode: Opc) || isVOPC64DPP8OpcodeHelper(Opcode: Opc);
568}
569
570bool isVOPCAsmOnly(unsigned Opc) { return isVOPCAsmOnlyOpcodeHelper(Opcode: Opc); }
571
572bool getMAIIsDGEMM(unsigned Opc) {
573 const MAIInstInfo *Info = getMAIInstInfoHelper(Opcode: Opc);
574 return Info && Info->is_dgemm;
575}
576
577bool getMAIIsGFX940XDL(unsigned Opc) {
578 const MAIInstInfo *Info = getMAIInstInfoHelper(Opcode: Opc);
579 return Info && Info->is_gfx940_xdl;
580}
581
582bool getWMMAIsXDL(unsigned Opc) {
583 const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opcode: Opc);
584 return Info ? Info->is_wmma_xdl : false;
585}
586
587bool getHasMatrixScale(unsigned Opc) {
588 const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opcode: Opc);
589 return Info && Info->HasMatrixScale;
590}
591
592uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal) {
593 switch (EncodingVal) {
594 case MFMAScaleFormats::FP6_E2M3:
595 case MFMAScaleFormats::FP6_E3M2:
596 return 6;
597 case MFMAScaleFormats::FP4_E2M1:
598 return 4;
599 case MFMAScaleFormats::FP8_E4M3:
600 case MFMAScaleFormats::FP8_E5M2:
601 default:
602 return 8;
603 }
604
605 llvm_unreachable("covered switch over mfma scale formats");
606}
607
608const MFMA_F8F6F4_Info *getMFMA_F8F6F4_WithFormatArgs(unsigned CBSZ,
609 unsigned BLGP,
610 unsigned F8F8Opcode) {
611 uint8_t SrcANumRegs = mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: CBSZ);
612 uint8_t SrcBNumRegs = mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: BLGP);
613 return getMFMA_F8F6F4_InstWithNumRegs(NumRegsSrcA: SrcANumRegs, NumRegsSrcB: SrcBNumRegs, F8F8Opcode);
614}
615
616uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt) {
617 switch (Fmt) {
618 case WMMA::MATRIX_FMT_FP8:
619 case WMMA::MATRIX_FMT_BF8:
620 return 16;
621 case WMMA::MATRIX_FMT_FP6:
622 case WMMA::MATRIX_FMT_BF6:
623 return 12;
624 case WMMA::MATRIX_FMT_FP4:
625 return 8;
626 }
627
628 llvm_unreachable("covered switch over wmma scale formats");
629}
630
631const MFMA_F8F6F4_Info *getWMMA_F8F6F4_WithFormatArgs(unsigned FmtA,
632 unsigned FmtB,
633 unsigned F8F8Opcode) {
634 uint8_t SrcANumRegs = wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtA);
635 uint8_t SrcBNumRegs = wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtB);
636 return getMFMA_F8F6F4_InstWithNumRegs(NumRegsSrcA: SrcANumRegs, NumRegsSrcB: SrcBNumRegs, F8F8Opcode);
637}
638
639bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale,
640 unsigned BFmt, unsigned BScale) {
641 auto isValid = [](unsigned Fmt, unsigned Scale) -> bool {
642 switch (Fmt) {
643 case WMMA::MATRIX_FMT_FP8:
644 case WMMA::MATRIX_FMT_BF8:
645 case WMMA::MATRIX_FMT_FP6:
646 case WMMA::MATRIX_FMT_BF6:
647 if (Scale != WMMA::MATRIX_SCALE_FMT_E8)
648 return false;
649 break;
650 case WMMA::MATRIX_FMT_FP4:
651 if (Scale != WMMA::MATRIX_SCALE_FMT_E8 &&
652 Scale != WMMA::MATRIX_SCALE_FMT_E5M3 &&
653 Scale != WMMA::MATRIX_SCALE_FMT_E4M3)
654 return false;
655 break;
656 }
657 return true;
658 };
659
660 if (!isValid(AFmt, AScale) || !isValid(BFmt, BScale))
661 return false;
662
663 if (AFmt == WMMA::MATRIX_FMT_FP4 && BFmt == WMMA::MATRIX_FMT_FP4 &&
664 AScale != BScale)
665 return false;
666
667 return true;
668}
669
670unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST) {
671 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX13Insts))
672 return SIEncodingFamily::GFX13;
673 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX1250Insts))
674 return SIEncodingFamily::GFX1250;
675 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX12Insts))
676 return SIEncodingFamily::GFX12;
677 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX11_7Insts))
678 return SIEncodingFamily::GFX1170;
679 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX11Insts))
680 return SIEncodingFamily::GFX11;
681 llvm_unreachable("Subtarget generation does not support VOPD!");
682}
683
684CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3) {
685 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
686 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
687 // Normalize through VOPDComponentTable so that e32 and e64 variants
688 // of the same logical opcode all share a single entry.
689 const VOPDComponentInfo *Info = getVOPDComponentHelper(BaseVOP: Opc);
690 if (!Info)
691 return {.X: false, .Y: false};
692 unsigned Key =
693 (Info->VOPDOp << 5) | (EncodingFamily << 1) | (VOPD3 ? 1u : 0u);
694 const VOPDXYInfo *XYInfo = getVOPDXYInfo(VOPDXYKey: static_cast<uint16_t>(Key));
695 if (!XYInfo)
696 return {.X: false, .Y: false};
697 return {.X: XYInfo->IsX, .Y: XYInfo->IsY};
698}
699
700unsigned getVOPDOpcode(unsigned Opc, bool VOPD3) {
701 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
702 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
703 const VOPDComponentInfo *Info = getVOPDComponentHelper(BaseVOP: Opc);
704 return Info ? Info->VOPDOp : ~0u;
705}
706
707bool isVOPD(unsigned Opc) {
708 return AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::src0X);
709}
710
711bool isMAC(unsigned Opc) {
712 return Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 ||
713 Opc == AMDGPU::V_MAC_F32_e64_gfx10 ||
714 Opc == AMDGPU::V_MAC_F32_e64_vi ||
715 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx6_gfx7 ||
716 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx10 ||
717 Opc == AMDGPU::V_MAC_F16_e64_vi ||
718 Opc == AMDGPU::V_FMAC_F64_e64_gfx90a ||
719 Opc == AMDGPU::V_FMAC_F64_e64_gfx12 ||
720 Opc == AMDGPU::V_FMAC_F64_e64_gfx13 ||
721 Opc == AMDGPU::V_FMAC_F32_e64_gfx10 ||
722 Opc == AMDGPU::V_FMAC_F32_e64_gfx11 ||
723 Opc == AMDGPU::V_FMAC_F32_e64_gfx12 ||
724 Opc == AMDGPU::V_FMAC_F32_e64_gfx13 ||
725 Opc == AMDGPU::V_FMAC_F32_e64_vi ||
726 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64_gfx10 ||
727 Opc == AMDGPU::V_FMAC_DX9_ZERO_F32_e64_gfx11 ||
728 Opc == AMDGPU::V_FMAC_F16_e64_gfx10 ||
729 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx11 ||
730 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx11 ||
731 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx12 ||
732 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx12 ||
733 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx13 ||
734 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx13 ||
735 Opc == AMDGPU::V_DOT2C_F32_F16_e64_vi ||
736 Opc == AMDGPU::V_DOT2C_F32_BF16_e64_vi ||
737 Opc == AMDGPU::V_DOT2C_I32_I16_e64_vi ||
738 Opc == AMDGPU::V_DOT4C_I32_I8_e64_vi ||
739 Opc == AMDGPU::V_DOT8C_I32_I4_e64_vi;
740}
741
742bool isPermlane16(unsigned Opc) {
743 return Opc == AMDGPU::V_PERMLANE16_B32_gfx10 ||
744 Opc == AMDGPU::V_PERMLANEX16_B32_gfx10 ||
745 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx11 ||
746 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx11 ||
747 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx12 ||
748 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx13 ||
749 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx12 ||
750 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx13 ||
751 Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx12 ||
752 Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx13 ||
753 Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx12 ||
754 Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx13;
755}
756
757bool isCvt_F32_Fp8_Bf8_e64(unsigned Opc) {
758 return Opc == AMDGPU::V_CVT_F32_BF8_e64_gfx12 ||
759 Opc == AMDGPU::V_CVT_F32_FP8_e64_gfx12 ||
760 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp_gfx12 ||
761 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp_gfx12 ||
762 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp8_gfx12 ||
763 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp8_gfx12 ||
764 Opc == AMDGPU::V_CVT_PK_F32_BF8_fake16_e64_gfx12 ||
765 Opc == AMDGPU::V_CVT_PK_F32_FP8_fake16_e64_gfx12 ||
766 Opc == AMDGPU::V_CVT_PK_F32_BF8_t16_e64_gfx12 ||
767 Opc == AMDGPU::V_CVT_PK_F32_FP8_t16_e64_gfx12;
768}
769
770bool isGenericAtomic(unsigned Opc) {
771 return Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP ||
772 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD ||
773 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB ||
774 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN ||
775 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN ||
776 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX ||
777 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX ||
778 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND ||
779 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR ||
780 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR ||
781 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC ||
782 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC ||
783 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD ||
784 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN ||
785 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX ||
786 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP ||
787 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32 ||
788 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32 ||
789 Opc == AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG;
790}
791
792bool isAsyncStore(unsigned Opc) {
793 return Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_gfx1250 ||
794 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_gfx1250 ||
795 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_gfx1250 ||
796 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_gfx1250 ||
797 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR_gfx1250 ||
798 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR_gfx1250 ||
799 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR_gfx1250 ||
800 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR_gfx1250;
801}
802
803bool isTensorStore(unsigned Opc) {
804 return Opc == TENSOR_STORE_FROM_LDS_d2_gfx1250 ||
805 Opc == TENSOR_STORE_FROM_LDS_d4_gfx1250;
806}
807
808unsigned getTemporalHintType(const MCInstrDesc TID) {
809 if (SIInstrFlags::isAtomic(O: TID))
810 return CPol::TH_TYPE_ATOMIC;
811 unsigned Opc = TID.getOpcode();
812 // Async and Tensor store should have the temporal hint type of TH_TYPE_STORE
813 if (TID.mayStore() &&
814 (isAsyncStore(Opc) || isTensorStore(Opc) || !TID.mayLoad()))
815 return CPol::TH_TYPE_STORE;
816
817 // This will default to returning TH_TYPE_LOAD when neither MayStore nor
818 // MayLoad flag is present which is the case with instructions like
819 // image_get_resinfo.
820 return CPol::TH_TYPE_LOAD;
821}
822
823bool isTrue16Inst(unsigned Opc) { return isTrue16Opcode(Opcode: Opc) != nullptr; }
824
825FPType getFPDstSelType(unsigned Opc) {
826 const FP4FP8DstByteSelInfo *Info = getFP4FP8DstByteSelHelper(Opcode: Opc);
827 if (!Info)
828 return FPType::None;
829 if (Info->HasFP8DstByteSel)
830 return FPType::FP8;
831 if (Info->HasFP4DstByteSel)
832 return FPType::FP4;
833
834 return FPType::None;
835}
836
837bool isDPMACCInstruction(unsigned Opc) {
838 return isDPMACCInstructionHelper(Opcode: Opc) != nullptr;
839}
840
841unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc) {
842 const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom2AddrOpcode(Opcode2Addr: Opc);
843 return Info ? Info->Opcode3Addr : ~0u;
844}
845
846// Wrapper for Tablegen'd function. enum Subtarget is not defined in any
847// header files, so we need to wrap it in a function that takes unsigned
848// instead.
849int32_t getMCOpcode(uint32_t Opcode, unsigned Gen) {
850 return getMCOpcodeGen(Opcode, inSubtarget: static_cast<Subtarget>(Gen));
851}
852
853unsigned getBitOp2(unsigned Opc) {
854 switch (Opc) {
855 default:
856 return 0;
857 case AMDGPU::V_AND_B32_e32:
858 return 0x40;
859 case AMDGPU::V_OR_B32_e32:
860 return 0x54;
861 case AMDGPU::V_XOR_B32_e32:
862 return 0x14;
863 case AMDGPU::V_XNOR_B32_e32:
864 return 0x41;
865 }
866}
867
868int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily,
869 bool VOPD3) {
870 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc: OpY) : 0;
871 OpY = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : OpY;
872 const VOPDInfo *Info = getVOPDInfoFromComponentOpcodes(
873 OpX: static_cast<uint8_t>(OpX), OpY: static_cast<uint8_t>(OpY),
874 SubTgt: static_cast<uint8_t>(EncodingFamily), VOPD3);
875 return Info ? Info->Opcode : -1;
876}
877
878std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode) {
879 const VOPDInfo *Info = getVOPDOpcodeHelper(Opcode: VOPDOpcode);
880 assert(Info);
881 const auto *OpX = getVOPDBaseFromComponent(VOPDOp: Info->OpX);
882 const auto *OpY = getVOPDBaseFromComponent(VOPDOp: Info->OpY);
883 assert(OpX && OpY);
884 return {OpX->BaseVOP, OpY->BaseVOP};
885}
886
887namespace VOPD {
888
889ComponentProps::ComponentProps(const MCInstrDesc &OpDesc, bool VOP3Layout) {
890 assert(OpDesc.getNumDefs() == Component::DST_NUM);
891
892 assert(OpDesc.getOperandConstraint(Component::SRC0, MCOI::TIED_TO) == -1);
893 assert(OpDesc.getOperandConstraint(Component::SRC1, MCOI::TIED_TO) == -1);
894 auto TiedIdx = OpDesc.getOperandConstraint(OpNum: Component::SRC2, Constraint: MCOI::TIED_TO);
895 assert(TiedIdx == -1 || TiedIdx == Component::DST);
896 HasSrc2Acc = TiedIdx != -1;
897 Opcode = OpDesc.getOpcode();
898
899 IsVOP3 = VOP3Layout || SIInstrFlags::isVOP3(O: OpDesc);
900 SrcOperandsNum = AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src2) ? 3
901 : AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::imm) ? 3
902 : AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src1) ? 2
903 : 1;
904 assert(SrcOperandsNum <= Component::MAX_SRC_NUM);
905
906 if (Opcode == AMDGPU::V_CNDMASK_B32_e32 ||
907 Opcode == AMDGPU::V_CNDMASK_B32_e64) {
908 // CNDMASK is an awkward exception, it has FP modifiers, but not FP
909 // operands.
910 NumVOPD3Mods = 2;
911 if (IsVOP3)
912 SrcOperandsNum = 3;
913 } else if (Opcode == AMDGPU::V_DOT2_F32_F16 ||
914 Opcode == AMDGPU::V_DOT2_F32_BF16) {
915 // VOP3P opcodes that have VOPD but don't have VOP2 version. Using VOPD3
916 // path in getIndexOfSrcInMCOperands to get correct src operand indexes,
917 // but generating VOPD, not VOPD3.
918 NumVOPD3Mods = SrcOperandsNum;
919 } else if (isSISrcFPOperand(Desc: OpDesc,
920 OpNo: getNamedOperandIdx(Opcode, Name: OpName::src0))) {
921 // All FP VOPD instructions have Neg modifiers for all operands except
922 // for tied src2.
923 NumVOPD3Mods = SrcOperandsNum;
924 if (HasSrc2Acc)
925 --NumVOPD3Mods;
926 }
927
928 if (SIInstrFlags::isVOP3(O: OpDesc))
929 return;
930
931 auto OperandsNum = OpDesc.getNumOperands();
932 unsigned CompOprIdx;
933 for (CompOprIdx = Component::SRC1; CompOprIdx < OperandsNum; ++CompOprIdx) {
934 if (OpDesc.operands()[CompOprIdx].OperandType == AMDGPU::OPERAND_KIMM32) {
935 MandatoryLiteralIdx = CompOprIdx;
936 break;
937 }
938 }
939}
940
941int ComponentProps::getBitOp3OperandIdx() const {
942 return getNamedOperandIdx(Opcode, Name: OpName::bitop3);
943}
944
945unsigned ComponentInfo::getIndexInParsedOperands(unsigned CompOprIdx) const {
946 assert(CompOprIdx < Component::MAX_OPR_NUM);
947
948 if (CompOprIdx == Component::DST)
949 return getIndexOfDstInParsedOperands();
950
951 auto CompSrcIdx = CompOprIdx - Component::DST_NUM;
952 if (CompSrcIdx < getCompParsedSrcOperandsNum())
953 return getIndexOfSrcInParsedOperands(CompSrcIdx);
954
955 // The specified operand does not exist.
956 return 0;
957}
958
959std::optional<unsigned> InstInfo::getInvalidCompOperandIndex(
960 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
961 const MCRegisterInfo &MRI, bool SkipSrc, bool AllowSameVGPR, bool VOPD3,
962 bool HasGFX11InterlockHazard) const {
963
964 auto OpXRegs = getRegIndices(ComponentIdx: ComponentIndex::X, GetRegIdx,
965 VOPD3: CompInfo[ComponentIndex::X].isVOP3());
966 auto OpYRegs = getRegIndices(ComponentIdx: ComponentIndex::Y, GetRegIdx,
967 VOPD3: CompInfo[ComponentIndex::Y].isVOP3());
968
969 const auto banksOverlap = [&MRI](MCRegister X, MCRegister Y,
970 unsigned BanksMask) -> bool {
971 MCRegister BaseX = MRI.getSubReg(Reg: X, Idx: AMDGPU::sub0);
972 MCRegister BaseY = MRI.getSubReg(Reg: Y, Idx: AMDGPU::sub0);
973 if (!BaseX)
974 BaseX = X;
975 if (!BaseY)
976 BaseY = Y;
977 if ((BaseX.id() & BanksMask) == (BaseY.id() & BanksMask))
978 return true;
979 if (BaseX != X /* This is 64-bit register */ &&
980 ((BaseX.id() + 1) & BanksMask) == (BaseY.id() & BanksMask))
981 return true;
982 if (BaseY != Y &&
983 (BaseX.id() & BanksMask) == ((BaseY.id() + 1) & BanksMask))
984 return true;
985
986 // If both are 64-bit bank conflict will be detected yet while checking
987 // the first subreg.
988 return false;
989 };
990
991 unsigned CompOprIdx;
992 for (CompOprIdx = 0; CompOprIdx < Component::MAX_OPR_NUM; ++CompOprIdx) {
993 unsigned BanksMasks = VOPD3 ? VOPD3_VGPR_BANK_MASKS[CompOprIdx]
994 : HasGFX11InterlockHazard
995 ? VOPD_GFX11_VGPR_BANK_MASKS[CompOprIdx]
996 : VOPD_VGPR_BANK_MASKS[CompOprIdx];
997 if (!OpXRegs[CompOprIdx] || !OpYRegs[CompOprIdx])
998 continue;
999
1000 if (getVGPREncodingMSBs(Reg: OpXRegs[CompOprIdx], MRI) !=
1001 getVGPREncodingMSBs(Reg: OpYRegs[CompOprIdx], MRI))
1002 return CompOprIdx;
1003
1004 if (SkipSrc && CompOprIdx >= Component::DST_NUM)
1005 continue;
1006
1007 if (CompOprIdx < Component::DST_NUM) {
1008 // Even if we do not check vdst parity, vdst operands still shall not
1009 // overlap.
1010 if (MRI.regsOverlap(RegA: OpXRegs[CompOprIdx], RegB: OpYRegs[CompOprIdx]))
1011 return CompOprIdx;
1012 if (VOPD3) // No need to check dst parity.
1013 continue;
1014 }
1015
1016 if (banksOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx], BanksMasks) &&
1017 (!AllowSameVGPR || CompOprIdx < Component::DST_NUM ||
1018 OpXRegs[CompOprIdx] != OpYRegs[CompOprIdx]))
1019 return CompOprIdx;
1020 }
1021
1022 return {};
1023}
1024
1025// Return an array of VGPR registers [DST,SRC0,SRC1,SRC2] used
1026// by the specified component. If an operand is unused
1027// or is not a VGPR, the corresponding value is 0.
1028//
1029// GetRegIdx(Component, MCOperandIdx) must return a VGPR register index
1030// for the specified component and MC operand. The callback must return 0
1031// if the operand is not a register or not a VGPR.
1032InstInfo::RegIndices
1033InstInfo::getRegIndices(unsigned CompIdx,
1034 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
1035 bool VOPD3) const {
1036 assert(CompIdx < COMPONENTS_NUM);
1037
1038 const auto &Comp = CompInfo[CompIdx];
1039 InstInfo::RegIndices RegIndices;
1040
1041 RegIndices[DST] = GetRegIdx(CompIdx, Comp.getIndexOfDstInMCOperands());
1042
1043 for (unsigned CompOprIdx : {SRC0, SRC1, SRC2}) {
1044 unsigned CompSrcIdx = CompOprIdx - DST_NUM;
1045 RegIndices[CompOprIdx] =
1046 Comp.hasRegSrcOperand(CompSrcIdx)
1047 ? GetRegIdx(CompIdx,
1048 Comp.getIndexOfSrcInMCOperands(CompSrcIdx, VOPD3))
1049 : MCRegister();
1050 }
1051 return RegIndices;
1052}
1053
1054} // namespace VOPD
1055
1056VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY) {
1057 return VOPD::InstInfo(OpX, OpY);
1058}
1059
1060VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,
1061 const MCInstrInfo *InstrInfo) {
1062 auto [OpX, OpY] = getVOPDComponents(VOPDOpcode);
1063 const auto &OpXDesc = InstrInfo->get(Opcode: OpX);
1064 const auto &OpYDesc = InstrInfo->get(Opcode: OpY);
1065 bool VOPD3 = SIInstrFlags::isVOPD3(O: *InstrInfo, O: VOPDOpcode);
1066 VOPD::ComponentInfo OpXInfo(OpXDesc, VOPD::ComponentKind::COMPONENT_X, VOPD3);
1067 VOPD::ComponentInfo OpYInfo(OpYDesc, OpXInfo, VOPD3);
1068 return VOPD::InstInfo(OpXInfo, OpYInfo);
1069}
1070
1071TargetID createAMDGPUTargetID(const MCSubtargetInfo &STI,
1072 StringRef FeatureString) {
1073 // In codegen the mode comes from module flags and FeatureString is empty, so
1074 // the processor defaults apply. The assembler has no target directive, so it
1075 // pins the mode via the +xnack/-xnack/+sramecc/-sramecc feature string.
1076 return TargetID::createFromSubtargetFeatures(TT: STI.getTargetTriple(),
1077 CPU: STI.getCPU(), FeatureString);
1078}
1079
1080namespace IsaInfo {
1081
1082unsigned getInstCacheLineSize(const MCSubtargetInfo &STI) {
1083 if (STI.getFeatureBits().test(I: FeatureInstCacheLineSize128))
1084 return 128;
1085 if (STI.getFeatureBits().test(I: FeatureInstCacheLineSize64))
1086 return 64;
1087 return 64;
1088}
1089
1090unsigned getWavefrontSize(const MCSubtargetInfo &STI) {
1091 if (STI.getFeatureBits().test(I: FeatureWavefrontSize16))
1092 return 16;
1093 if (STI.getFeatureBits().test(I: FeatureWavefrontSize32))
1094 return 32;
1095
1096 return 64;
1097}
1098
1099// Maximum LDS a single work-group can address. This is a fixed HW cap. It does
1100// not depend on how many SIMDs a work-group runs on.
1101static unsigned getMaxHWAddressableLocalMemorySize(const MCSubtargetInfo &STI) {
1102 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize32768))
1103 return 32768;
1104 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize65536))
1105 return 65536;
1106 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize163840))
1107 return 163840;
1108 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize196608))
1109 return 196608;
1110 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize327680))
1111 return 327680;
1112 return 32768;
1113}
1114
1115// Total physical size of LDS on the block, in bytes. On targets with
1116// FeatureHalfAddressablePhysicalLocalMemory the physical block is twice the
1117// addressable size (gfx6: 64 KiB physical and 32 KiB addressable;
1118// gfx10/11/12: 128 KiB physical and 64 KiB addressable). On other targets it is
1119// equal to the addressable size.
1120static unsigned getPhysicalLocalMemorySize(const MCSubtargetInfo &STI) {
1121 unsigned Addressable = getMaxHWAddressableLocalMemorySize(STI);
1122 if (STI.getFeatureBits().test(I: FeatureHalfAddressablePhysicalLocalMemory))
1123 return 2 * Addressable;
1124 return Addressable;
1125}
1126
1127// Sizes in use, by generation (addressable / physical block):
1128// gfx6 : 32 KiB addressable, 64 KiB physical block
1129// gfx7 / gfx8 / gfx9: 64 KiB
1130// gfx9.5 (gfx950) : 160 KiB
1131// gfx10 / 11 / 12 : 64 KiB addressable, 128 KiB physical block
1132// gfx12.5 (gfx1250) : 320 KiB (always runs on four SIMDs)
1133// gfx13 : 192 KiB on four SIMDs, 96 KiB on two
1134// Total available in the current mode. The physical size is halved when a
1135// work-group runs on two SIMDs.
1136unsigned getLocalMemorySize(const MCSubtargetInfo &STI) {
1137 unsigned Size = getPhysicalLocalMemorySize(STI);
1138 if (!isFullSIMDMode(STI))
1139 Size /= 2;
1140 return Size;
1141}
1142
1143// What one work-group can allocate in the current mode. This is the HW
1144// addressable cap, but never more than the total available in the current mode.
1145unsigned getAddressableLocalMemorySize(const MCSubtargetInfo &STI) {
1146 return std::min(a: getMaxHWAddressableLocalMemorySize(STI),
1147 b: getLocalMemorySize(STI));
1148}
1149
1150unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo &STI,
1151 unsigned FlatWorkGroupSize) {
1152 assert(FlatWorkGroupSize != 0);
1153 if (!STI.getTargetTriple().isAMDGCN())
1154 return 8;
1155 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1156 unsigned MaxWaves =
1157 getMaxWavesPerEU(AK: Kind) * getNumWorkGroupSIMDs(FullSIMDMode: isFullSIMDMode(STI));
1158 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
1159 if (N == 1) {
1160 // Single-wave workgroups don't consume barrier resources.
1161 return MaxWaves;
1162 }
1163
1164 unsigned MaxBarriers = 16;
1165 if (isGFX10Plus(STI) && !STI.getFeatureBits().test(I: FeatureCuMode))
1166 MaxBarriers = 32;
1167
1168 return std::min(a: MaxWaves / N, b: MaxBarriers);
1169}
1170
1171unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo &STI,
1172 unsigned FlatWorkGroupSize) {
1173 return divideCeil(Numerator: getWavesPerWorkGroup(STI, FlatWorkGroupSize),
1174 Denominator: getNumWorkGroupSIMDs(FullSIMDMode: isFullSIMDMode(STI)));
1175}
1176
1177unsigned getWavesPerWorkGroup(const MCSubtargetInfo &STI,
1178 unsigned FlatWorkGroupSize) {
1179 return divideCeil(Numerator: FlatWorkGroupSize, Denominator: getWavefrontSize(STI));
1180}
1181
1182unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI) { return 8; }
1183
1184// Per-wave SGPRs reserved for the trap handler when enabled.
1185static unsigned getSGPRTrapHandlerReserve(const MCSubtargetInfo &STI) {
1186 return STI.getFeatureBits().test(I: FeatureTrapHandler) ? TRAP_NUM_SGPRS : 0;
1187}
1188
1189// Per-wave SGPR budget (before the addressable clamp): take off the trap
1190// reserve, round down to \p Granule. Shared by getMinNumSGPRs() and
1191// getMaxNumSGPRs(); getOccupancyWithNumSGPRs() is the closed-form algebraic
1192// inverse of this same budget (it does not call this helper), so the two encode
1193// one model.
1194static unsigned getSGPRBudgetPerWave(unsigned TotalNumSGPRs,
1195 unsigned WavesPerEU, unsigned TrapReserve,
1196 unsigned Granule) {
1197 assert(WavesPerEU != 0 && Granule != 0);
1198 unsigned Budget = TotalNumSGPRs / WavesPerEU;
1199 Budget -= std::min(a: Budget, b: TrapReserve);
1200 return alignDown(Value: Budget, Align: Granule);
1201}
1202
1203unsigned getMinNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU) {
1204 assert(WavesPerEU != 0);
1205
1206 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1207 if (Version.Major >= 10)
1208 return 0;
1209
1210 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1211 if (WavesPerEU >= getMaxWavesPerEU(AK: Kind))
1212 return 0;
1213
1214 unsigned MinNumSGPRs =
1215 getSGPRBudgetPerWave(TotalNumSGPRs: getTotalNumSGPRs(AK: Kind), WavesPerEU: WavesPerEU + 1,
1216 TrapReserve: getSGPRTrapHandlerReserve(STI),
1217 Granule: getSGPRAllocGranule(AK: Kind)) +
1218 1;
1219 return std::min(a: MinNumSGPRs, b: getAddressableNumSGPRs(AK: Kind));
1220}
1221
1222unsigned getMaxNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1223 bool Addressable) {
1224 assert(WavesPerEU != 0);
1225
1226 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1227 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(AK: Kind);
1228 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1229 if (Version.Major >= 10)
1230 return Addressable ? AddressableNumSGPRs : 108;
1231 if (Version.Major >= 8 && !Addressable)
1232 AddressableNumSGPRs = 112;
1233 unsigned MaxNumSGPRs = getSGPRBudgetPerWave(
1234 TotalNumSGPRs: getTotalNumSGPRs(AK: Kind), WavesPerEU, TrapReserve: getSGPRTrapHandlerReserve(STI),
1235 Granule: getSGPRAllocGranule(AK: Kind));
1236 return std::min(a: MaxNumSGPRs, b: AddressableNumSGPRs);
1237}
1238
1239bool isSGPROccupancyLimited(const MCSubtargetInfo &STI) {
1240 // From GFX10 on the SGPR file is large enough that SGPRs never limit
1241 // occupancy. Kept as one capability so callers don't each test the version.
1242 return getIsaVersion(GPU: STI.getCPU()).Major < 10;
1243}
1244
1245unsigned getNumExtraSGPRs(const MCSubtargetInfo &STI, bool VCCUsed,
1246 bool FlatScrUsed, bool XNACKUsed) {
1247 unsigned ExtraSGPRs = 0;
1248 if (VCCUsed)
1249 ExtraSGPRs = 2;
1250
1251 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1252 if (Version.Major >= 10)
1253 return ExtraSGPRs;
1254
1255 if (Version.Major < 8) {
1256 if (FlatScrUsed)
1257 ExtraSGPRs = 4;
1258 } else {
1259 if (XNACKUsed)
1260 ExtraSGPRs = 4;
1261
1262 if (FlatScrUsed ||
1263 STI.getFeatureBits().test(I: AMDGPU::FeatureArchitectedFlatScratch))
1264 ExtraSGPRs = 6;
1265 }
1266
1267 return ExtraSGPRs;
1268}
1269
1270static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs,
1271 unsigned Granule) {
1272 return divideCeil(Numerator: std::max(a: 1u, b: NumRegs), Denominator: Granule);
1273}
1274
1275unsigned getNumSGPRBlocks(const MCSubtargetInfo &STI, unsigned NumSGPRs) {
1276 // SGPRBlocks is actual number of SGPR blocks minus 1.
1277 return getGranulatedNumRegisterBlocks(NumRegs: NumSGPRs, Granule: getSGPREncodingGranule(STI)) -
1278 1;
1279}
1280
1281unsigned getVGPRAllocGranule(const MCSubtargetInfo &STI,
1282 unsigned DynamicVGPRBlockSize,
1283 std::optional<bool> EnableWavefrontSize32) {
1284 if (STI.getFeatureBits().test(I: FeatureGFX90AInsts))
1285 return 8;
1286
1287 if (DynamicVGPRBlockSize != 0)
1288 return DynamicVGPRBlockSize;
1289
1290 bool IsWave32 = EnableWavefrontSize32
1291 ? *EnableWavefrontSize32
1292 : STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1293
1294 if (STI.getFeatureBits().test(I: Feature1536VGPRs))
1295 return IsWave32 ? 24 : 12;
1296
1297 if (hasGFX10_3Insts(STI))
1298 return IsWave32 ? 16 : 8;
1299
1300 return IsWave32 ? 8 : 4;
1301}
1302
1303unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI,
1304 std::optional<bool> EnableWavefrontSize32) {
1305 if (STI.getFeatureBits().test(I: FeatureGFX90AInsts))
1306 return 8;
1307
1308 bool IsWave32 = EnableWavefrontSize32
1309 ? *EnableWavefrontSize32
1310 : STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1311
1312 if (STI.getFeatureBits().test(I: Feature1024AddressableVGPRs))
1313 return IsWave32 ? 16 : 8;
1314
1315 return IsWave32 ? 8 : 4;
1316}
1317
1318unsigned getArchVGPRAllocGranule() { return 4; }
1319
1320unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo &STI) {
1321 const auto &Features = STI.getFeatureBits();
1322 if (Features.test(I: Feature1024AddressableVGPRs))
1323 return Features.test(I: FeatureWavefrontSize32) ? 1024 : 512;
1324 return 256;
1325}
1326
1327unsigned getAddressableNumVGPRs(const MCSubtargetInfo &STI,
1328 unsigned DynamicVGPRBlockSize) {
1329 const auto &Features = STI.getFeatureBits();
1330 if (Features.test(I: FeatureGFX90AInsts))
1331 return 512;
1332
1333 if (DynamicVGPRBlockSize != 0) {
1334 // On GFX12 we can allocate at most MaxDynamicVGPRBlocks blocks of VGPRs.
1335 return MaxDynamicVGPRBlocks *
1336 getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1337 }
1338 return getAddressableNumArchVGPRs(STI);
1339}
1340
1341unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo &STI,
1342 unsigned NumVGPRs,
1343 unsigned DynamicVGPRBlockSize) {
1344 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1345 bool IsWave32 = STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1346 return getNumWavesPerEUWithNumVGPRs(
1347 NumVGPRs, Granule: getVGPRAllocGranule(STI, DynamicVGPRBlockSize),
1348 MaxWaves: getMaxWavesPerEU(AK: Kind), TotalNumVGPRs: AMDGPU::getTotalNumVGPRs(AK: Kind, IsWave32));
1349}
1350
1351unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
1352 unsigned MaxWaves,
1353 unsigned TotalNumVGPRs) {
1354 if (NumVGPRs < Granule)
1355 return MaxWaves;
1356 unsigned RoundedRegs = alignTo(Value: NumVGPRs, Align: Granule);
1357 return std::min(a: std::max(a: TotalNumVGPRs / RoundedRegs, b: 1u), b: MaxWaves);
1358}
1359
1360unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
1361 unsigned TotalNumSGPRs, unsigned Granule,
1362 unsigned TrapReserve) {
1363 // Closed-form inverse of getMaxNumSGPRs(): the budget condition
1364 // SGPRs <= alignDown(TotalNumSGPRs / W - TrapReserve, Granule)
1365 // solves to W <= TotalNumSGPRs / (alignTo(SGPRs, Granule) + TrapReserve).
1366 unsigned PerWave = alignTo(Value: SGPRs, Align: Granule) + TrapReserve;
1367 return PerWave ? std::clamp(val: TotalNumSGPRs / PerWave, lo: 1u, hi: MaxWaves) : MaxWaves;
1368}
1369
1370unsigned getOccupancyWithNumSGPRs(const MCSubtargetInfo &STI, unsigned SGPRs) {
1371 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1372 unsigned MaxWaves = getMaxWavesPerEU(AK: Kind);
1373
1374 if (!isSGPROccupancyLimited(STI))
1375 return MaxWaves;
1376
1377 return getOccupancyWithNumSGPRs(SGPRs, MaxWaves, TotalNumSGPRs: getTotalNumSGPRs(AK: Kind),
1378 Granule: getSGPRAllocGranule(AK: Kind),
1379 TrapReserve: getSGPRTrapHandlerReserve(STI));
1380}
1381
1382unsigned getMinNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1383 unsigned DynamicVGPRBlockSize) {
1384 assert(WavesPerEU != 0);
1385
1386 // In dynamic VGPR mode, (static) occupancy does not depend on VGPR usage,
1387 // so getMaxNumVGPRs does not depend on WavesPerEU, and thus we need to return
1388 // zero because there is no nonzero VGPR usage N where going below N
1389 // achieves higher (static) occupancy.
1390 bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
1391 if (DynamicVGPREnabled)
1392 return 0;
1393
1394 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1395 unsigned MaxWavesPerEU = getMaxWavesPerEU(AK: Kind);
1396 if (WavesPerEU >= MaxWavesPerEU)
1397 return 0;
1398
1399 unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
1400 AK: Kind, IsWave32: STI.getFeatureBits().test(I: FeatureWavefrontSize32));
1401 unsigned AddrsableNumVGPRs =
1402 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1403 unsigned Granule = getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1404 unsigned MaxNumVGPRs = alignDown(Value: TotNumVGPRs / WavesPerEU, Align: Granule);
1405
1406 if (MaxNumVGPRs == alignDown(Value: TotNumVGPRs / MaxWavesPerEU, Align: Granule))
1407 return 0;
1408
1409 unsigned MinWavesPerEU = getNumWavesPerEUWithNumVGPRs(STI, NumVGPRs: AddrsableNumVGPRs,
1410 DynamicVGPRBlockSize);
1411 if (WavesPerEU < MinWavesPerEU)
1412 return getMinNumVGPRs(STI, WavesPerEU: MinWavesPerEU, DynamicVGPRBlockSize);
1413
1414 unsigned MaxNumVGPRsNext = alignDown(Value: TotNumVGPRs / (WavesPerEU + 1), Align: Granule);
1415 unsigned MinNumVGPRs = 1 + std::min(a: MaxNumVGPRs - Granule, b: MaxNumVGPRsNext);
1416 return std::min(a: MinNumVGPRs, b: AddrsableNumVGPRs);
1417}
1418
1419unsigned getMaxNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1420 unsigned DynamicVGPRBlockSize) {
1421 assert(WavesPerEU != 0);
1422
1423 unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
1424 AK: parseArchAMDGCN(CPU: STI.getCPU()),
1425 IsWave32: STI.getFeatureBits().test(I: FeatureWavefrontSize32));
1426
1427 // In dynamic VGPR mode, WavesPerEU does not imply a VGPR limit.
1428 bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
1429 unsigned MaxNumVGPRs =
1430 DynamicVGPREnabled
1431 ? TotNumVGPRs
1432 : alignDown(Value: TotNumVGPRs / WavesPerEU,
1433 Align: getVGPRAllocGranule(STI, DynamicVGPRBlockSize));
1434 unsigned AddressableNumVGPRs =
1435 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1436 return std::min(a: MaxNumVGPRs, b: AddressableNumVGPRs);
1437}
1438
1439unsigned getAllocatedNumVGPRBlocks(const MCSubtargetInfo &STI,
1440 unsigned NumVGPRs,
1441 unsigned DynamicVGPRBlockSize,
1442 std::optional<bool> EnableWavefrontSize32) {
1443 return getGranulatedNumRegisterBlocks(
1444 NumRegs: NumVGPRs,
1445 Granule: getVGPRAllocGranule(STI, DynamicVGPRBlockSize, EnableWavefrontSize32));
1446}
1447} // end namespace IsaInfo
1448
1449void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &KernelCode,
1450 const MCSubtargetInfo &STI) {
1451 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1452 KernelCode.amd_kernel_code_version_major = 1;
1453 KernelCode.amd_kernel_code_version_minor = 2;
1454 KernelCode.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
1455 KernelCode.amd_machine_version_major = Version.Major;
1456 KernelCode.amd_machine_version_minor = Version.Minor;
1457 KernelCode.amd_machine_version_stepping = Version.Stepping;
1458 KernelCode.kernel_code_entry_byte_offset = sizeof(amd_kernel_code_t);
1459 if (STI.getFeatureBits().test(I: FeatureWavefrontSize32)) {
1460 KernelCode.wavefront_size = 5;
1461 KernelCode.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
1462 } else {
1463 KernelCode.wavefront_size = 6;
1464 }
1465
1466 // If the code object does not support indirect functions, then the value must
1467 // be 0xffffffff.
1468 KernelCode.call_convention = -1;
1469
1470 // These alignment values are specified in powers of two, so alignment =
1471 // 2^n. The minimum alignment is 2^4 = 16.
1472 KernelCode.kernarg_segment_alignment = 4;
1473 KernelCode.group_segment_alignment = 4;
1474 KernelCode.private_segment_alignment = 4;
1475
1476 if (Version.Major >= 10) {
1477 KernelCode.compute_pgm_resource_registers |=
1478 S_00B848_WGP_MODE(STI.getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
1479 S_00B848_MEM_ORDERED(1) | S_00B848_FWD_PROGRESS(1);
1480 }
1481}
1482
1483bool isReadOnlySegment(const GlobalValue *GV) {
1484 unsigned AS = GV->getAddressSpace();
1485 return AS == AMDGPUAS::CONSTANT_ADDRESS ||
1486 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
1487}
1488
1489bool shouldEmitConstantsToTextSection(const Triple &TT) {
1490 return TT.getArch() == Triple::r600;
1491}
1492
1493static bool isValidRegPrefix(char C) {
1494 return C == 'v' || C == 's' || C == 'a';
1495}
1496
1497std::tuple<char, unsigned, unsigned> parseAsmPhysRegName(StringRef RegName) {
1498 if (RegName.empty())
1499 return {};
1500
1501 char Kind = RegName.front();
1502 if (!isValidRegPrefix(C: Kind))
1503 return {};
1504
1505 RegName = RegName.drop_front();
1506 if (RegName.consume_front(Prefix: "[")) {
1507 unsigned Idx, End;
1508 bool Failed = RegName.consumeInteger(Radix: 10, Result&: Idx);
1509 Failed |= !RegName.consume_front(Prefix: ":");
1510 Failed |= RegName.consumeInteger(Radix: 10, Result&: End);
1511 Failed |= !RegName.consume_back(Suffix: "]");
1512 if (!Failed) {
1513 unsigned NumRegs = End - Idx + 1;
1514 if (NumRegs > 1)
1515 return {Kind, Idx, NumRegs};
1516 }
1517 } else {
1518 unsigned Idx;
1519 bool Failed = RegName.getAsInteger(Radix: 10, Result&: Idx);
1520 if (!Failed)
1521 return {Kind, Idx, 1};
1522 }
1523
1524 return {};
1525}
1526
1527std::tuple<char, unsigned, unsigned>
1528parseAsmConstraintPhysReg(StringRef Constraint) {
1529 StringRef RegName = Constraint;
1530 if (!RegName.consume_front(Prefix: "{") || !RegName.consume_back(Suffix: "}"))
1531 return {};
1532 return parseAsmPhysRegName(RegName);
1533}
1534
1535std::pair<unsigned, unsigned>
1536getIntegerPairAttribute(const Function &F, StringRef Name,
1537 std::pair<unsigned, unsigned> Default,
1538 bool OnlyFirstRequired) {
1539 if (auto Attr = getIntegerPairAttribute(F, Name, OnlyFirstRequired))
1540 return {Attr->first, Attr->second.value_or(u&: Default.second)};
1541 return Default;
1542}
1543
1544std::optional<std::pair<unsigned, std::optional<unsigned>>>
1545getIntegerPairAttribute(const Function &F, StringRef Name,
1546 bool OnlyFirstRequired) {
1547 Attribute A = F.getFnAttribute(Kind: Name);
1548 if (!A.isStringAttribute())
1549 return std::nullopt;
1550
1551 LLVMContext &Ctx = F.getContext();
1552 std::pair<unsigned, std::optional<unsigned>> Ints;
1553 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(Separator: ',');
1554 if (Strs.first.trim().getAsInteger(Radix: 0, Result&: Ints.first)) {
1555 Ctx.emitError(ErrorStr: "can't parse first integer attribute " + Name);
1556 return std::nullopt;
1557 }
1558 unsigned Second = 0;
1559 if (Strs.second.trim().getAsInteger(Radix: 0, Result&: Second)) {
1560 if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
1561 Ctx.emitError(ErrorStr: "can't parse second integer attribute " + Name);
1562 return std::nullopt;
1563 }
1564 } else {
1565 Ints.second = Second;
1566 }
1567
1568 return Ints;
1569}
1570
1571SmallVector<unsigned> getIntegerVecAttribute(const Function &F, StringRef Name,
1572 unsigned Size,
1573 unsigned DefaultVal) {
1574 std::optional<SmallVector<unsigned>> R =
1575 getIntegerVecAttribute(F, Name, Size);
1576 return R.has_value() ? *R : SmallVector<unsigned>(Size, DefaultVal);
1577}
1578
1579std::optional<SmallVector<unsigned>>
1580getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) {
1581 assert(Size > 2);
1582 LLVMContext &Ctx = F.getContext();
1583
1584 Attribute A = F.getFnAttribute(Kind: Name);
1585 if (!A.isValid())
1586 return std::nullopt;
1587 if (!A.isStringAttribute()) {
1588 Ctx.emitError(ErrorStr: Name + " is not a string attribute");
1589 return std::nullopt;
1590 }
1591
1592 SmallVector<unsigned> Vals(Size);
1593
1594 StringRef S = A.getValueAsString();
1595 unsigned i = 0;
1596 for (; !S.empty() && i < Size; i++) {
1597 std::pair<StringRef, StringRef> Strs = S.split(Separator: ',');
1598 unsigned IntVal;
1599 if (Strs.first.trim().getAsInteger(Radix: 0, Result&: IntVal)) {
1600 Ctx.emitError(ErrorStr: "can't parse integer attribute " + Strs.first + " in " +
1601 Name);
1602 return std::nullopt;
1603 }
1604 Vals[i] = IntVal;
1605 S = Strs.second;
1606 }
1607
1608 if (!S.empty() || i < Size) {
1609 Ctx.emitError(ErrorStr: "attribute " + Name +
1610 " has incorrect number of integers; expected " +
1611 llvm::utostr(X: Size));
1612 return std::nullopt;
1613 }
1614 return Vals;
1615}
1616
1617SmallVector<unsigned> getMaxNumWorkGroups(const Function &F) {
1618 return getIntegerVecAttribute(F, Name: "amdgpu-max-num-workgroups", Size: 3,
1619 DefaultVal: std::numeric_limits<uint32_t>::max());
1620}
1621
1622bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) {
1623 assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!");
1624 for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) {
1625 auto Low =
1626 mdconst::extract<ConstantInt>(MD: MD.getOperand(I: 2 * I + 0))->getValue();
1627 auto High =
1628 mdconst::extract<ConstantInt>(MD: MD.getOperand(I: 2 * I + 1))->getValue();
1629 // There are two types of [A; B) ranges:
1630 // A < B, e.g. [4; 5) which is a range that only includes 4.
1631 // A > B, e.g. [5; 4) which is a range that wraps around and includes
1632 // everything except 4.
1633 if (Low.ult(RHS: High)) {
1634 if (Low.ule(RHS: Val) && High.ugt(RHS: Val))
1635 return true;
1636 } else {
1637 if (Low.uge(RHS: Val) && High.ult(RHS: Val))
1638 return true;
1639 }
1640 }
1641
1642 return false;
1643}
1644
1645unsigned getVmcntBitMask(const IsaVersion &Version) {
1646 return (1 << (getVmcntBitWidthLo(VersionMajor: Version.Major) +
1647 getVmcntBitWidthHi(VersionMajor: Version.Major))) -
1648 1;
1649}
1650
1651unsigned getLoadcntBitMask(const IsaVersion &Version) {
1652 return (1 << getLoadcntBitWidth(VersionMajor: Version.Major)) - 1;
1653}
1654
1655unsigned getSamplecntBitMask(const IsaVersion &Version) {
1656 return (1 << getSamplecntBitWidth(VersionMajor: Version.Major)) - 1;
1657}
1658
1659unsigned getBvhcntBitMask(const IsaVersion &Version) {
1660 return (1 << getBvhcntBitWidth(VersionMajor: Version.Major)) - 1;
1661}
1662
1663unsigned getExpcntBitMask(const IsaVersion &Version) {
1664 return (1 << getExpcntBitWidth(VersionMajor: Version.Major)) - 1;
1665}
1666
1667unsigned getLgkmcntBitMask(const IsaVersion &Version) {
1668 return (1 << getLgkmcntBitWidth(VersionMajor: Version.Major)) - 1;
1669}
1670
1671unsigned getDscntBitMask(const IsaVersion &Version) {
1672 return (1 << getDscntBitWidth(VersionMajor: Version.Major)) - 1;
1673}
1674
1675unsigned getKmcntBitMask(const IsaVersion &Version) {
1676 return (1 << getKmcntBitWidth(VersionMajor: Version.Major)) - 1;
1677}
1678
1679unsigned getXcntBitMask(const IsaVersion &Version) {
1680 return (1 << getXcntBitWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1681}
1682
1683unsigned getAsynccntBitMask(const IsaVersion &Version) {
1684 return (1 << getAsynccntBitWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1685}
1686
1687unsigned getStorecntBitMask(const IsaVersion &Version) {
1688 return (1 << getStorecntBitWidth(VersionMajor: Version.Major)) - 1;
1689}
1690
1691unsigned getWaitcntBitMask(const IsaVersion &Version) {
1692 unsigned VmcntLo = getBitMask(Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1693 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1694 unsigned Expcnt = getBitMask(Shift: getExpcntBitShift(VersionMajor: Version.Major),
1695 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1696 unsigned Lgkmcnt = getBitMask(Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1697 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1698 unsigned VmcntHi = getBitMask(Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1699 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1700 return VmcntLo | Expcnt | Lgkmcnt | VmcntHi;
1701}
1702
1703unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1704 unsigned VmcntLo = unpackBits(Src: Waitcnt, Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1705 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1706 unsigned VmcntHi = unpackBits(Src: Waitcnt, Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1707 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1708 return VmcntLo | VmcntHi << getVmcntBitWidthLo(VersionMajor: Version.Major);
1709}
1710
1711unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
1712 return unpackBits(Src: Waitcnt, Shift: getExpcntBitShift(VersionMajor: Version.Major),
1713 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1714}
1715
1716unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1717 return unpackBits(Src: Waitcnt, Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1718 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1719}
1720
1721unsigned decodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt) {
1722 return unpackBits(Src: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1723 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1724}
1725
1726unsigned decodeStorecnt(const IsaVersion &Version, unsigned Waitcnt) {
1727 return unpackBits(Src: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1728 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1729}
1730
1731unsigned decodeDscnt(const IsaVersion &Version, unsigned Waitcnt) {
1732 return unpackBits(Src: Waitcnt, Shift: getDscntBitShift(VersionMajor: Version.Major),
1733 Width: getDscntBitWidth(VersionMajor: Version.Major));
1734}
1735
1736void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt,
1737 unsigned &Expcnt, unsigned &Lgkmcnt) {
1738 Vmcnt = decodeVmcnt(Version, Waitcnt);
1739 Expcnt = decodeExpcnt(Version, Waitcnt);
1740 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
1741}
1742
1743unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
1744 unsigned Vmcnt) {
1745 Waitcnt = packBits(Src: Vmcnt, Dst: Waitcnt, Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1746 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1747 return packBits(Src: Vmcnt >> getVmcntBitWidthLo(VersionMajor: Version.Major), Dst: Waitcnt,
1748 Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1749 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1750}
1751
1752unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
1753 unsigned Expcnt) {
1754 return packBits(Src: Expcnt, Dst: Waitcnt, Shift: getExpcntBitShift(VersionMajor: Version.Major),
1755 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1756}
1757
1758unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
1759 unsigned Lgkmcnt) {
1760 return packBits(Src: Lgkmcnt, Dst: Waitcnt, Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1761 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1762}
1763
1764unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt,
1765 unsigned Expcnt, unsigned Lgkmcnt) {
1766 unsigned Waitcnt = getWaitcntBitMask(Version);
1767 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
1768 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
1769 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
1770 return Waitcnt;
1771}
1772
1773static unsigned getCombinedCountBitMask(const IsaVersion &Version,
1774 bool IsStore) {
1775 unsigned Dscnt = getBitMask(Shift: getDscntBitShift(VersionMajor: Version.Major),
1776 Width: getDscntBitWidth(VersionMajor: Version.Major));
1777 if (IsStore) {
1778 unsigned Storecnt = getBitMask(Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1779 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1780 return Dscnt | Storecnt;
1781 }
1782 unsigned Loadcnt = getBitMask(Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1783 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1784 return Dscnt | Loadcnt;
1785}
1786
1787static unsigned encodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt,
1788 unsigned Loadcnt) {
1789 return packBits(Src: Loadcnt, Dst: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1790 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1791}
1792
1793static unsigned encodeStorecnt(const IsaVersion &Version, unsigned Waitcnt,
1794 unsigned Storecnt) {
1795 return packBits(Src: Storecnt, Dst: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1796 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1797}
1798
1799static unsigned encodeDscnt(const IsaVersion &Version, unsigned Waitcnt,
1800 unsigned Dscnt) {
1801 return packBits(Src: Dscnt, Dst: Waitcnt, Shift: getDscntBitShift(VersionMajor: Version.Major),
1802 Width: getDscntBitWidth(VersionMajor: Version.Major));
1803}
1804
1805unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt,
1806 unsigned Dscnt) {
1807 unsigned Waitcnt = getCombinedCountBitMask(Version, IsStore: false);
1808 Waitcnt = encodeLoadcnt(Version, Waitcnt, Loadcnt);
1809 Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
1810 return Waitcnt;
1811}
1812
1813unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt,
1814 unsigned Dscnt) {
1815 unsigned Waitcnt = getCombinedCountBitMask(Version, IsStore: true);
1816 Waitcnt = encodeStorecnt(Version, Waitcnt, Storecnt);
1817 Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
1818 return Waitcnt;
1819}
1820
1821//===----------------------------------------------------------------------===//
1822// Custom Operand Values
1823//===----------------------------------------------------------------------===//
1824
1825static unsigned getDefaultCustomOperandEncoding(const CustomOperandVal *Opr,
1826 int Size,
1827 const MCSubtargetInfo &STI) {
1828 unsigned Enc = 0;
1829 for (int Idx = 0; Idx < Size; ++Idx) {
1830 const auto &Op = Opr[Idx];
1831 if (Op.isSupported(STI))
1832 Enc |= Op.encode(Val: Op.Default);
1833 }
1834 return Enc;
1835}
1836
1837static bool isSymbolicCustomOperandEncoding(const CustomOperandVal *Opr,
1838 int Size, unsigned Code,
1839 bool &HasNonDefaultVal,
1840 const MCSubtargetInfo &STI) {
1841 unsigned UsedOprMask = 0;
1842 HasNonDefaultVal = false;
1843 for (int Idx = 0; Idx < Size; ++Idx) {
1844 const auto &Op = Opr[Idx];
1845 if (!Op.isSupported(STI))
1846 continue;
1847 UsedOprMask |= Op.getMask();
1848 unsigned Val = Op.decode(Code);
1849 if (!Op.isValid(Val))
1850 return false;
1851 HasNonDefaultVal |= (Val != Op.Default);
1852 }
1853 return (Code & ~UsedOprMask) == 0;
1854}
1855
1856static bool decodeCustomOperand(const CustomOperandVal *Opr, int Size,
1857 unsigned Code, int &Idx, StringRef &Name,
1858 unsigned &Val, bool &IsDefault,
1859 const MCSubtargetInfo &STI) {
1860 while (Idx < Size) {
1861 const auto &Op = Opr[Idx++];
1862 if (Op.isSupported(STI)) {
1863 Name = Op.Name;
1864 Val = Op.decode(Code);
1865 IsDefault = (Val == Op.Default);
1866 return true;
1867 }
1868 }
1869
1870 return false;
1871}
1872
1873static int encodeCustomOperandVal(const CustomOperandVal &Op,
1874 int64_t InputVal) {
1875 if (InputVal < 0 || InputVal > Op.Max)
1876 return OPR_VAL_INVALID;
1877 return Op.encode(Val: static_cast<unsigned>(InputVal));
1878}
1879
1880static int encodeCustomOperand(const CustomOperandVal *Opr, int Size,
1881 const StringRef Name, int64_t InputVal,
1882 unsigned &UsedOprMask,
1883 const MCSubtargetInfo &STI) {
1884 int InvalidId = OPR_ID_UNKNOWN;
1885 for (int Idx = 0; Idx < Size; ++Idx) {
1886 const auto &Op = Opr[Idx];
1887 if (Op.Name == Name) {
1888 if (!Op.isSupported(STI)) {
1889 InvalidId = OPR_ID_UNSUPPORTED;
1890 continue;
1891 }
1892 auto OprMask = Op.getMask();
1893 if (OprMask & UsedOprMask)
1894 return OPR_ID_DUPLICATE;
1895 UsedOprMask |= OprMask;
1896 return encodeCustomOperandVal(Op, InputVal);
1897 }
1898 }
1899 return InvalidId;
1900}
1901
1902//===----------------------------------------------------------------------===//
1903// DepCtr
1904//===----------------------------------------------------------------------===//
1905
1906namespace DepCtr {
1907
1908int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI) {
1909 static int Default = -1;
1910 if (Default == -1)
1911 Default = getDefaultCustomOperandEncoding(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, STI);
1912 return Default;
1913}
1914
1915bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal,
1916 const MCSubtargetInfo &STI) {
1917 return isSymbolicCustomOperandEncoding(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Code,
1918 HasNonDefaultVal, STI);
1919}
1920
1921bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val,
1922 bool &IsDefault, const MCSubtargetInfo &STI) {
1923 return decodeCustomOperand(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Code, Idx&: Id, Name, Val,
1924 IsDefault, STI);
1925}
1926
1927int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask,
1928 const MCSubtargetInfo &STI) {
1929 return encodeCustomOperand(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Name, InputVal: Val, UsedOprMask,
1930 STI);
1931}
1932
1933unsigned getVaVdstBitMask() { return (1 << getVaVdstBitWidth()) - 1; }
1934
1935unsigned getVaSdstBitMask() { return (1 << getVaSdstBitWidth()) - 1; }
1936
1937unsigned getVaSsrcBitMask() { return (1 << getVaSsrcBitWidth()) - 1; }
1938
1939unsigned getHoldCntBitMask(const IsaVersion &Version) {
1940 return (1 << getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1941}
1942
1943unsigned getVmVsrcBitMask() { return (1 << getVmVsrcBitWidth()) - 1; }
1944
1945unsigned getVaVccBitMask() { return (1 << getVaVccBitWidth()) - 1; }
1946
1947unsigned getSaSdstBitMask() { return (1 << getSaSdstBitWidth()) - 1; }
1948
1949unsigned decodeFieldVmVsrc(unsigned Encoded) {
1950 return unpackBits(Src: Encoded, Shift: getVmVsrcBitShift(), Width: getVmVsrcBitWidth());
1951}
1952
1953unsigned decodeFieldVaVdst(unsigned Encoded) {
1954 return unpackBits(Src: Encoded, Shift: getVaVdstBitShift(), Width: getVaVdstBitWidth());
1955}
1956
1957unsigned decodeFieldSaSdst(unsigned Encoded) {
1958 return unpackBits(Src: Encoded, Shift: getSaSdstBitShift(), Width: getSaSdstBitWidth());
1959}
1960
1961unsigned decodeFieldVaSdst(unsigned Encoded) {
1962 return unpackBits(Src: Encoded, Shift: getVaSdstBitShift(), Width: getVaSdstBitWidth());
1963}
1964
1965unsigned decodeFieldVaVcc(unsigned Encoded) {
1966 return unpackBits(Src: Encoded, Shift: getVaVccBitShift(), Width: getVaVccBitWidth());
1967}
1968
1969unsigned decodeFieldVaSsrc(unsigned Encoded) {
1970 return unpackBits(Src: Encoded, Shift: getVaSsrcBitShift(), Width: getVaSsrcBitWidth());
1971}
1972
1973unsigned decodeFieldHoldCnt(unsigned Encoded, const IsaVersion &Version) {
1974 return unpackBits(Src: Encoded, Shift: getHoldCntBitShift(),
1975 Width: getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor));
1976}
1977
1978unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc) {
1979 return packBits(Src: VmVsrc, Dst: Encoded, Shift: getVmVsrcBitShift(), Width: getVmVsrcBitWidth());
1980}
1981
1982unsigned encodeFieldVmVsrc(unsigned VmVsrc, const MCSubtargetInfo &STI) {
1983 unsigned Encoded = getDefaultDepCtrEncoding(STI);
1984 return encodeFieldVmVsrc(Encoded, VmVsrc);
1985}
1986
1987unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst) {
1988 return packBits(Src: VaVdst, Dst: Encoded, Shift: getVaVdstBitShift(), Width: getVaVdstBitWidth());
1989}
1990
1991unsigned encodeFieldVaVdst(unsigned VaVdst, const MCSubtargetInfo &STI) {
1992 unsigned Encoded = getDefaultDepCtrEncoding(STI);
1993 return encodeFieldVaVdst(Encoded, VaVdst);
1994}
1995
1996unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst) {
1997 return packBits(Src: SaSdst, Dst: Encoded, Shift: getSaSdstBitShift(), Width: getSaSdstBitWidth());
1998}
1999
2000unsigned encodeFieldSaSdst(unsigned SaSdst, const MCSubtargetInfo &STI) {
2001 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2002 return encodeFieldSaSdst(Encoded, SaSdst);
2003}
2004
2005unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst) {
2006 return packBits(Src: VaSdst, Dst: Encoded, Shift: getVaSdstBitShift(), Width: getVaSdstBitWidth());
2007}
2008
2009unsigned encodeFieldVaSdst(unsigned VaSdst, const MCSubtargetInfo &STI) {
2010 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2011 return encodeFieldVaSdst(Encoded, VaSdst);
2012}
2013
2014unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc) {
2015 return packBits(Src: VaVcc, Dst: Encoded, Shift: getVaVccBitShift(), Width: getVaVccBitWidth());
2016}
2017
2018unsigned encodeFieldVaVcc(unsigned VaVcc, const MCSubtargetInfo &STI) {
2019 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2020 return encodeFieldVaVcc(Encoded, VaVcc);
2021}
2022
2023unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc) {
2024 return packBits(Src: VaSsrc, Dst: Encoded, Shift: getVaSsrcBitShift(), Width: getVaSsrcBitWidth());
2025}
2026
2027unsigned encodeFieldVaSsrc(unsigned VaSsrc, const MCSubtargetInfo &STI) {
2028 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2029 return encodeFieldVaSsrc(Encoded, VaSsrc);
2030}
2031
2032unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt,
2033 const IsaVersion &Version) {
2034 return packBits(Src: HoldCnt, Dst: Encoded, Shift: getHoldCntBitShift(),
2035 Width: getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor));
2036}
2037
2038unsigned encodeFieldHoldCnt(unsigned HoldCnt, const MCSubtargetInfo &STI) {
2039 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2040 return encodeFieldHoldCnt(Encoded, HoldCnt, Version: getIsaVersion(GPU: STI.getCPU()));
2041}
2042
2043} // namespace DepCtr
2044
2045//===----------------------------------------------------------------------===//
2046// exp tgt
2047//===----------------------------------------------------------------------===//
2048
2049namespace Exp {
2050
2051struct ExpTgt {
2052 StringLiteral Name;
2053 unsigned Tgt;
2054 unsigned MaxIndex;
2055};
2056
2057// clang-format off
2058static constexpr ExpTgt ExpTgtInfo[] = {
2059 {.Name: {"null"}, .Tgt: ET_NULL, .MaxIndex: ET_NULL_MAX_IDX},
2060 {.Name: {"mrtz"}, .Tgt: ET_MRTZ, .MaxIndex: ET_MRTZ_MAX_IDX},
2061 {.Name: {"prim"}, .Tgt: ET_PRIM, .MaxIndex: ET_PRIM_MAX_IDX},
2062 {.Name: {"mrt"}, .Tgt: ET_MRT0, .MaxIndex: ET_MRT_MAX_IDX},
2063 {.Name: {"pos"}, .Tgt: ET_POS0, .MaxIndex: ET_POS_MAX_IDX},
2064 {.Name: {"dual_src_blend"},.Tgt: ET_DUAL_SRC_BLEND0, .MaxIndex: ET_DUAL_SRC_BLEND_MAX_IDX},
2065 {.Name: {"param"}, .Tgt: ET_PARAM0, .MaxIndex: ET_PARAM_MAX_IDX},
2066};
2067// clang-format on
2068
2069bool getTgtName(unsigned Id, StringRef &Name, int &Index) {
2070 for (const ExpTgt &Val : ExpTgtInfo) {
2071 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) {
2072 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt);
2073 Name = Val.Name;
2074 return true;
2075 }
2076 }
2077 return false;
2078}
2079
2080unsigned getTgtId(const StringRef Name) {
2081
2082 for (const ExpTgt &Val : ExpTgtInfo) {
2083 if (Val.MaxIndex == 0 && Name == Val.Name)
2084 return Val.Tgt;
2085
2086 if (Val.MaxIndex > 0 && Name.starts_with(Prefix: Val.Name)) {
2087 StringRef Suffix = Name.drop_front(N: Val.Name.size());
2088
2089 unsigned Id;
2090 if (Suffix.getAsInteger(Radix: 10, Result&: Id) || Id > Val.MaxIndex)
2091 return ET_INVALID;
2092
2093 // Disable leading zeroes
2094 if (Suffix.size() > 1 && Suffix[0] == '0')
2095 return ET_INVALID;
2096
2097 return Val.Tgt + Id;
2098 }
2099 }
2100 return ET_INVALID;
2101}
2102
2103bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) {
2104 switch (Id) {
2105 case ET_NULL:
2106 return !isGFX11Plus(STI);
2107 case ET_POS4:
2108 case ET_PRIM:
2109 return isGFX10Plus(STI);
2110 case ET_DUAL_SRC_BLEND0:
2111 case ET_DUAL_SRC_BLEND1:
2112 return isGFX11Plus(STI);
2113 default:
2114 if (Id >= ET_PARAM0 && Id <= ET_PARAM31)
2115 return !isGFX11Plus(STI) || isGFX13Plus(STI);
2116 return true;
2117 }
2118}
2119
2120} // namespace Exp
2121
2122//===----------------------------------------------------------------------===//
2123// MTBUF Format
2124//===----------------------------------------------------------------------===//
2125
2126namespace MTBUFFormat {
2127
2128int64_t getDfmt(const StringRef Name) {
2129 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
2130 if (Name == DfmtSymbolic[Id])
2131 return Id;
2132 }
2133 return DFMT_UNDEF;
2134}
2135
2136StringRef getDfmtName(unsigned Id) {
2137 assert(Id <= DFMT_MAX);
2138 return DfmtSymbolic[Id];
2139}
2140
2141static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {
2142 if (isSI(STI) || isCI(STI))
2143 return NfmtSymbolicSICI;
2144 if (isVI(STI) || isGFX9(STI))
2145 return NfmtSymbolicVI;
2146 return NfmtSymbolicGFX10;
2147}
2148
2149int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
2150 const auto *lookupTable = getNfmtLookupTable(STI);
2151 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
2152 if (Name == lookupTable[Id])
2153 return Id;
2154 }
2155 return NFMT_UNDEF;
2156}
2157
2158StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
2159 assert(Id <= NFMT_MAX);
2160 return getNfmtLookupTable(STI)[Id];
2161}
2162
2163bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2164 unsigned Dfmt;
2165 unsigned Nfmt;
2166 decodeDfmtNfmt(Format: Id, Dfmt, Nfmt);
2167 return isValidNfmt(Val: Nfmt, STI);
2168}
2169
2170bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2171 return !getNfmtName(Id, STI).empty();
2172}
2173
2174int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
2175 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);
2176}
2177
2178void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
2179 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;
2180 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
2181}
2182
2183int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI) {
2184 if (isGFX11Plus(STI)) {
2185 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2186 if (Name == UfmtSymbolicGFX11[Id])
2187 return Id;
2188 }
2189 } else {
2190 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2191 if (Name == UfmtSymbolicGFX10[Id])
2192 return Id;
2193 }
2194 }
2195 return UFMT_UNDEF;
2196}
2197
2198StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI) {
2199 if (isValidUnifiedFormat(Val: Id, STI))
2200 return isGFX10(STI) ? UfmtSymbolicGFX10[Id] : UfmtSymbolicGFX11[Id];
2201 return "";
2202}
2203
2204bool isValidUnifiedFormat(unsigned Id, const MCSubtargetInfo &STI) {
2205 return isGFX10(STI) ? Id <= UfmtGFX10::UFMT_LAST : Id <= UfmtGFX11::UFMT_LAST;
2206}
2207
2208int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt,
2209 const MCSubtargetInfo &STI) {
2210 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
2211 if (isGFX11Plus(STI)) {
2212 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2213 if (Fmt == DfmtNfmt2UFmtGFX11[Id])
2214 return Id;
2215 }
2216 } else {
2217 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2218 if (Fmt == DfmtNfmt2UFmtGFX10[Id])
2219 return Id;
2220 }
2221 }
2222 return UFMT_UNDEF;
2223}
2224
2225bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
2226 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
2227}
2228
2229unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {
2230 if (isGFX10Plus(STI))
2231 return UFMT_DEFAULT;
2232 return DFMT_NFMT_DEFAULT;
2233}
2234
2235} // namespace MTBUFFormat
2236
2237//===----------------------------------------------------------------------===//
2238// SendMsg
2239//===----------------------------------------------------------------------===//
2240
2241namespace SendMsg {
2242
2243static uint64_t getMsgIdMask(const MCSubtargetInfo &STI) {
2244 return isGFX11Plus(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_;
2245}
2246
2247bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) {
2248 return (MsgId & ~(getMsgIdMask(STI))) == 0;
2249}
2250
2251bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
2252 bool Strict) {
2253 assert(isValidMsgId(MsgId, STI));
2254
2255 if (!Strict)
2256 return 0 <= OpId && isUInt<OP_WIDTH_>(x: OpId);
2257
2258 if (msgRequiresOp(MsgId, STI)) {
2259 if (MsgId == ID_GS_PreGFX11 && OpId == OP_GS_NOP)
2260 return false;
2261
2262 return !getMsgOpName(MsgId, Encoding: OpId, STI).empty();
2263 }
2264
2265 return OpId == OP_NONE_;
2266}
2267
2268bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
2269 const MCSubtargetInfo &STI, bool Strict) {
2270 assert(isValidMsgOp(MsgId, OpId, STI, Strict));
2271
2272 if (!Strict)
2273 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(x: StreamId);
2274
2275 if (!isGFX11Plus(STI)) {
2276 switch (MsgId) {
2277 case ID_GS_PreGFX11:
2278 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
2279 case ID_GS_DONE_PreGFX11:
2280 return (OpId == OP_GS_NOP)
2281 ? (StreamId == STREAM_ID_NONE_)
2282 : (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
2283 }
2284 }
2285 return StreamId == STREAM_ID_NONE_;
2286}
2287
2288bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI) {
2289 return MsgId == ID_SYSMSG ||
2290 (!isGFX11Plus(STI) &&
2291 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11));
2292}
2293
2294bool msgSupportsStream(int64_t MsgId, int64_t OpId,
2295 const MCSubtargetInfo &STI) {
2296 return !isGFX11Plus(STI) &&
2297 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11) &&
2298 OpId != OP_GS_NOP;
2299}
2300
2301void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,
2302 uint16_t &StreamId, const MCSubtargetInfo &STI) {
2303 MsgId = static_cast<uint16_t>(Val & getMsgIdMask(STI));
2304 if (isGFX11Plus(STI)) {
2305 OpId = 0;
2306 StreamId = 0;
2307 } else {
2308 OpId = (Val & OP_MASK_) >> OP_SHIFT_;
2309 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
2310 }
2311}
2312
2313uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId) {
2314 return MsgId | (OpId << OP_SHIFT_) | (StreamId << STREAM_ID_SHIFT_);
2315}
2316
2317bool msgDoesNotUseM0(int64_t MsgId, const MCSubtargetInfo &STI) {
2318 // Explicitly list message types that are known to not use m0.
2319 // This is safer than excluding only GS_ALLOC_REQ, in case new message
2320 // types are added in the future that do use m0.
2321 if (isGFX11Plus(STI)) {
2322 switch (MsgId) {
2323 case ID_DEALLOC_VGPRS_GFX11Plus:
2324 return true;
2325 default:
2326 break;
2327 }
2328 }
2329 switch (MsgId) {
2330 case ID_SAVEWAVE:
2331 case ID_STALL_WAVE_GEN:
2332 case ID_HALT_WAVES:
2333 case ID_ORDERED_PS_DONE:
2334 case ID_EARLY_PRIM_DEALLOC:
2335 case ID_GET_DOORBELL:
2336 case ID_GET_DDID:
2337 case ID_SYSMSG:
2338 return true;
2339 default:
2340 return false;
2341 }
2342}
2343
2344} // namespace SendMsg
2345
2346//===----------------------------------------------------------------------===//
2347//
2348//===----------------------------------------------------------------------===//
2349
2350unsigned getInitialPSInputAddr(const Function &F) {
2351 return static_cast<unsigned>(
2352 F.getFnAttributeAsParsedInteger(Kind: "InitialPSInputAddr", Default: 0));
2353}
2354
2355bool getHasColorExport(const Function &F) {
2356 // As a safe default always respond as if PS has color exports.
2357 return F.getFnAttributeAsParsedInteger(
2358 Kind: "amdgpu-color-export",
2359 Default: F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0;
2360}
2361
2362bool getHasDepthExport(const Function &F) {
2363 return F.getFnAttributeAsParsedInteger(Kind: "amdgpu-depth-export", Default: 0) != 0;
2364}
2365
2366unsigned getDynamicVGPRBlockSize(const Function &F) {
2367 unsigned BlockSize = static_cast<unsigned>(
2368 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-dynamic-vgpr-block-size", Default: 0));
2369
2370 if (BlockSize == 16 || BlockSize == 32)
2371 return BlockSize;
2372
2373 return 0;
2374}
2375
2376bool hasMIMG_R128(const MCSubtargetInfo &STI) {
2377 return STI.hasFeature(Feature: AMDGPU::FeatureMIMG_R128) &&
2378 !STI.hasFeature(Feature: AMDGPU::FeatureR128A16);
2379}
2380
2381bool hasA16(const MCSubtargetInfo &STI) {
2382 return STI.hasFeature(Feature: AMDGPU::FeatureA16);
2383}
2384
2385bool hasG16(const MCSubtargetInfo &STI) {
2386 return STI.hasFeature(Feature: AMDGPU::FeatureG16);
2387}
2388
2389bool hasPackedD16(const MCSubtargetInfo &STI) {
2390 return !STI.hasFeature(Feature: AMDGPU::FeatureUnpackedD16VMem) && !isCI(STI) &&
2391 !isSI(STI);
2392}
2393
2394bool hasGDS(const MCSubtargetInfo &STI) {
2395 return STI.hasFeature(Feature: AMDGPU::FeatureGDS);
2396}
2397
2398unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler) {
2399 auto Version = getIsaVersion(GPU: STI.getCPU());
2400 if (Version.Major == 10)
2401 return Version.Minor >= 3 ? 13 : 5;
2402 if (Version.Major == 11)
2403 return 5;
2404 if (Version.Major >= 12)
2405 return HasSampler ? 4 : 5;
2406 return 0;
2407}
2408
2409unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI) {
2410 if (isGFX1250Plus(STI))
2411 return 32;
2412 return 16;
2413}
2414
2415bool isSI(const MCSubtargetInfo &STI) {
2416 return STI.hasFeature(Feature: AMDGPU::FeatureSouthernIslands);
2417}
2418
2419bool isCI(const MCSubtargetInfo &STI) {
2420 return STI.hasFeature(Feature: AMDGPU::FeatureSeaIslands);
2421}
2422
2423bool isVI(const MCSubtargetInfo &STI) {
2424 return STI.hasFeature(Feature: AMDGPU::FeatureVolcanicIslands);
2425}
2426
2427bool isGFX9(const MCSubtargetInfo &STI) {
2428 return STI.hasFeature(Feature: AMDGPU::FeatureGFX9);
2429}
2430
2431bool isGFX9_GFX10(const MCSubtargetInfo &STI) {
2432 return isGFX9(STI) || isGFX10(STI);
2433}
2434
2435bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI) {
2436 return isGFX9(STI) || isGFX10(STI) || isGFX11(STI);
2437}
2438
2439bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI) {
2440 return isVI(STI) || isGFX9(STI) || isGFX10(STI);
2441}
2442
2443bool isGFX8Plus(const MCSubtargetInfo &STI) {
2444 return isVI(STI) || isGFX9Plus(STI);
2445}
2446
2447bool isGFX9Plus(const MCSubtargetInfo &STI) {
2448 return isGFX9(STI) || isGFX10Plus(STI);
2449}
2450
2451bool isNotGFX9Plus(const MCSubtargetInfo &STI) { return !isGFX9Plus(STI); }
2452
2453bool hasPopsExitingWaveID(const MCSubtargetInfo &STI) {
2454 return STI.hasFeature(Feature: AMDGPU::FeaturePopsExitingWaveID);
2455}
2456
2457bool hasPrivateApertureRegs(const MCSubtargetInfo &STI) {
2458 return STI.hasFeature(Feature: AMDGPU::FeatureApertureRegs) &&
2459 !STI.hasFeature(Feature: AMDGPU::FeatureGloballyAddressableScratch);
2460}
2461
2462bool isGFX10(const MCSubtargetInfo &STI) {
2463 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10);
2464}
2465
2466bool isGFX10_GFX11(const MCSubtargetInfo &STI) {
2467 return isGFX10(STI) || isGFX11(STI);
2468}
2469
2470bool isGFX10Plus(const MCSubtargetInfo &STI) {
2471 return isGFX10(STI) || isGFX11Plus(STI);
2472}
2473
2474bool isGFX11(const MCSubtargetInfo &STI) {
2475 return STI.hasFeature(Feature: AMDGPU::FeatureGFX11);
2476}
2477
2478bool isGFX11Plus(const MCSubtargetInfo &STI) {
2479 return isGFX11(STI) || isGFX12Plus(STI);
2480}
2481
2482bool isGFX12(const MCSubtargetInfo &STI) {
2483 return STI.getFeatureBits()[AMDGPU::FeatureGFX12];
2484}
2485
2486bool isGFX12Plus(const MCSubtargetInfo &STI) {
2487 return isGFX12(STI) || isGFX13Plus(STI);
2488}
2489
2490bool isNotGFX12Plus(const MCSubtargetInfo &STI) { return !isGFX12Plus(STI); }
2491
2492bool isGFX1250(const MCSubtargetInfo &STI) {
2493 return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts] && !isGFX13(STI);
2494}
2495
2496bool isFullSIMDMode(const MCSubtargetInfo &STI) {
2497 return isGFX1250(STI) || !STI.getFeatureBits().test(I: FeatureCuMode);
2498}
2499
2500bool isGFX1250Plus(const MCSubtargetInfo &STI) {
2501 return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts];
2502}
2503
2504bool isGFX13(const MCSubtargetInfo &STI) {
2505 return STI.getFeatureBits()[AMDGPU::FeatureGFX13];
2506}
2507
2508bool isGFX13Plus(const MCSubtargetInfo &STI) { return isGFX13(STI); }
2509
2510bool supportsWGP(const MCSubtargetInfo &STI) {
2511 if (isGFX1250(STI))
2512 return false;
2513 return isGFX10Plus(STI);
2514}
2515
2516bool isNotGFX11Plus(const MCSubtargetInfo &STI) { return !isGFX11Plus(STI); }
2517
2518bool isNotGFX10Plus(const MCSubtargetInfo &STI) {
2519 return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI);
2520}
2521
2522bool isGFX10Before1030(const MCSubtargetInfo &STI) {
2523 return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI);
2524}
2525
2526bool isGCN3Encoding(const MCSubtargetInfo &STI) {
2527 return STI.hasFeature(Feature: AMDGPU::FeatureGCN3Encoding);
2528}
2529
2530bool isGFX10_BEncoding(const MCSubtargetInfo &STI) {
2531 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10_BEncoding);
2532}
2533
2534bool hasGFX10_3Insts(const MCSubtargetInfo &STI) {
2535 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10_3Insts);
2536}
2537
2538bool isGFX10_3_GFX11(const MCSubtargetInfo &STI) {
2539 return isGFX10_BEncoding(STI) && !isGFX12Plus(STI);
2540}
2541
2542bool isGFX90A(const MCSubtargetInfo &STI) {
2543 return STI.hasFeature(Feature: AMDGPU::FeatureGFX90AInsts);
2544}
2545
2546bool isGFX940(const MCSubtargetInfo &STI) {
2547 return STI.hasFeature(Feature: AMDGPU::FeatureGFX940Insts);
2548}
2549
2550bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) {
2551 return STI.hasFeature(Feature: AMDGPU::FeatureArchitectedFlatScratch);
2552}
2553
2554bool hasMAIInsts(const MCSubtargetInfo &STI) {
2555 return STI.hasFeature(Feature: AMDGPU::FeatureMAIInsts);
2556}
2557
2558bool hasVOPD(const MCSubtargetInfo &STI) {
2559 return STI.hasFeature(Feature: AMDGPU::FeatureVOPDInsts);
2560}
2561
2562bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI) {
2563 return STI.hasFeature(Feature: AMDGPU::FeatureDPPSrc1SGPR);
2564}
2565
2566unsigned hasKernargPreload(const MCSubtargetInfo &STI) {
2567 return STI.hasFeature(Feature: AMDGPU::FeatureKernargPreload);
2568}
2569
2570int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR,
2571 int32_t ArgNumVGPR) {
2572 if (has90AInsts && ArgNumAGPR)
2573 return alignTo(Value: ArgNumVGPR, Align: 4) + ArgNumAGPR;
2574 return std::max(a: ArgNumVGPR, b: ArgNumAGPR);
2575}
2576
2577bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI) {
2578 const MCRegisterClass &SGPRClass =
2579 TRI->getRegClass(i: AMDGPU::SReg_32RegClassID);
2580 const MCRegister FirstSubReg = TRI->getSubReg(Reg, Idx: AMDGPU::sub0);
2581 return SGPRClass.contains(Reg: FirstSubReg != 0 ? FirstSubReg : Reg) ||
2582 Reg == AMDGPU::SCC;
2583}
2584
2585bool isRsrcIndexReg(MCRegister Reg, const MCRegisterInfo &MRI) {
2586 return MRI.getRegClass(i: AMDGPU::RsrcReg32RegClassID).contains(Reg);
2587}
2588
2589bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI) {
2590 return MRI.getEncodingValue(Reg) & AMDGPU::HWEncoding::IS_HI16;
2591}
2592
2593#define MAP_REG2REG \
2594 using namespace AMDGPU; \
2595 switch (Reg.id()) { \
2596 default: \
2597 return Reg; \
2598 CASE_CI_VI(FLAT_SCR) \
2599 CASE_CI_VI(FLAT_SCR_LO) \
2600 CASE_CI_VI(FLAT_SCR_HI) \
2601 CASE_VI_GFX9PLUS(TTMP0) \
2602 CASE_VI_GFX9PLUS(TTMP1) \
2603 CASE_VI_GFX9PLUS(TTMP2) \
2604 CASE_VI_GFX9PLUS(TTMP3) \
2605 CASE_VI_GFX9PLUS(TTMP4) \
2606 CASE_VI_GFX9PLUS(TTMP5) \
2607 CASE_VI_GFX9PLUS(TTMP6) \
2608 CASE_VI_GFX9PLUS(TTMP7) \
2609 CASE_VI_GFX9PLUS(TTMP8) \
2610 CASE_VI_GFX9PLUS(TTMP9) \
2611 CASE_VI_GFX9PLUS(TTMP10) \
2612 CASE_VI_GFX9PLUS(TTMP11) \
2613 CASE_VI_GFX9PLUS(TTMP12) \
2614 CASE_VI_GFX9PLUS(TTMP13) \
2615 CASE_VI_GFX9PLUS(TTMP14) \
2616 CASE_VI_GFX9PLUS(TTMP15) \
2617 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \
2618 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \
2619 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \
2620 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \
2621 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \
2622 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \
2623 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \
2624 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \
2625 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \
2626 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \
2627 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \
2628 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \
2629 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
2630 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
2631 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2632 CASE_VI_GFX9PLUS( \
2633 TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2634 CASE_GFXPRE11_GFX11PLUS(M0) \
2635 CASE_GFXPRE11_GFX11PLUS(SGPR_NULL) \
2636 CASE_GFXPRE11_GFX11PLUS_TO(SGPR_NULL64, SGPR_NULL) \
2637 }
2638
2639#define CASE_CI_VI(node) \
2640 assert(!isSI(STI)); \
2641 case node: \
2642 return isCI(STI) ? node##_ci : node##_vi;
2643
2644#define CASE_VI_GFX9PLUS(node) \
2645 case node: \
2646 return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi;
2647
2648#define CASE_GFXPRE11_GFX11PLUS(node) \
2649 case node: \
2650 return isGFX11Plus(STI) ? node##_gfx11plus : node##_gfxpre11;
2651
2652#define CASE_GFXPRE11_GFX11PLUS_TO(node, result) \
2653 case node: \
2654 return isGFX11Plus(STI) ? result##_gfx11plus : result##_gfxpre11;
2655
2656MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI) {
2657 if (STI.getTargetTriple().getArch() == Triple::r600)
2658 return Reg;
2659 MAP_REG2REG
2660}
2661
2662#undef CASE_CI_VI
2663#undef CASE_VI_GFX9PLUS
2664#undef CASE_GFXPRE11_GFX11PLUS
2665#undef CASE_GFXPRE11_GFX11PLUS_TO
2666
2667#define CASE_CI_VI(node) \
2668 case node##_ci: \
2669 case node##_vi: \
2670 return node;
2671#define CASE_VI_GFX9PLUS(node) \
2672 case node##_vi: \
2673 case node##_gfx9plus: \
2674 return node;
2675#define CASE_GFXPRE11_GFX11PLUS(node) \
2676 case node##_gfx11plus: \
2677 case node##_gfxpre11: \
2678 return node;
2679#define CASE_GFXPRE11_GFX11PLUS_TO(node, result)
2680
2681MCRegister mc2PseudoReg(MCRegister Reg) { MAP_REG2REG }
2682
2683bool isInlineValue(MCRegister Reg) {
2684 switch (Reg.id()) {
2685 case AMDGPU::SRC_SHARED_BASE_LO:
2686 case AMDGPU::SRC_SHARED_BASE:
2687 case AMDGPU::SRC_SHARED_LIMIT_LO:
2688 case AMDGPU::SRC_SHARED_LIMIT:
2689 case AMDGPU::SRC_PRIVATE_BASE_LO:
2690 case AMDGPU::SRC_PRIVATE_BASE:
2691 case AMDGPU::SRC_PRIVATE_LIMIT_LO:
2692 case AMDGPU::SRC_PRIVATE_LIMIT:
2693 case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
2694 case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
2695 case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
2696 return true;
2697 case AMDGPU::SRC_VCCZ:
2698 case AMDGPU::SRC_EXECZ:
2699 case AMDGPU::SRC_SCC:
2700 return true;
2701 case AMDGPU::SGPR_NULL:
2702 return true;
2703 default:
2704 return false;
2705 }
2706}
2707
2708#undef CASE_CI_VI
2709#undef CASE_VI_GFX9PLUS
2710#undef CASE_GFXPRE11_GFX11PLUS
2711#undef CASE_GFXPRE11_GFX11PLUS_TO
2712#undef MAP_REG2REG
2713
2714bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2715 assert(OpNo < Desc.NumOperands);
2716 unsigned OpType = Desc.operands()[OpNo].OperandType;
2717 return OpType >= AMDGPU::OPERAND_KIMM_FIRST &&
2718 OpType <= AMDGPU::OPERAND_KIMM_LAST;
2719}
2720
2721bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2722 assert(OpNo < Desc.NumOperands);
2723 unsigned OpType = Desc.operands()[OpNo].OperandType;
2724 switch (OpType) {
2725 case AMDGPU::OPERAND_REG_IMM_FP32:
2726 case AMDGPU::OPERAND_REG_IMM_FP64:
2727 case AMDGPU::OPERAND_REG_IMM_FP16:
2728 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
2729 case AMDGPU::OPERAND_REG_IMM_V2FP16:
2730 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
2731 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
2732 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2733 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2734 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2735 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2736 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2737 case AMDGPU::OPERAND_REG_IMM_V2FP32:
2738 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2739 case AMDGPU::OPERAND_REG_IMM_V2FP64:
2740 return true;
2741 default:
2742 return false;
2743 }
2744}
2745
2746// Avoid using MCRegisterClass::getSize, since that function will go away
2747// (move from MC* level to Target* level). Return size in bits.
2748unsigned getRegBitWidth(unsigned RCID) {
2749 switch (RCID) {
2750 case AMDGPU::VGPR_16RegClassID:
2751 case AMDGPU::VGPR_16_Lo128RegClassID:
2752 case AMDGPU::SGPR_LO16RegClassID:
2753 case AMDGPU::AGPR_LO16RegClassID:
2754 return 16;
2755 case AMDGPU::SGPR_32RegClassID:
2756 case AMDGPU::VGPR_32RegClassID:
2757 case AMDGPU::VGPR_32_Lo256RegClassID:
2758 case AMDGPU::VRegOrLds_32RegClassID:
2759 case AMDGPU::AGPR_32RegClassID:
2760 case AMDGPU::VS_32RegClassID:
2761 case AMDGPU::AV_32RegClassID:
2762 case AMDGPU::SReg_32RegClassID:
2763 case AMDGPU::SReg_32_XM0RegClassID:
2764 case AMDGPU::SRegOrLds_32RegClassID:
2765 return 32;
2766 case AMDGPU::SGPR_64RegClassID:
2767 case AMDGPU::VS_64RegClassID:
2768 case AMDGPU::SReg_64RegClassID:
2769 case AMDGPU::VReg_64RegClassID:
2770 case AMDGPU::AReg_64RegClassID:
2771 case AMDGPU::SReg_64_XEXECRegClassID:
2772 case AMDGPU::VReg_64_Align2RegClassID:
2773 case AMDGPU::AReg_64_Align2RegClassID:
2774 case AMDGPU::AV_64RegClassID:
2775 case AMDGPU::AV_64_Align2RegClassID:
2776 case AMDGPU::VReg_64_Lo256_Align2RegClassID:
2777 case AMDGPU::VS_64_Lo256RegClassID:
2778 return 64;
2779 case AMDGPU::SGPR_96RegClassID:
2780 case AMDGPU::SReg_96RegClassID:
2781 case AMDGPU::VReg_96RegClassID:
2782 case AMDGPU::AReg_96RegClassID:
2783 case AMDGPU::VReg_96_Align2RegClassID:
2784 case AMDGPU::AReg_96_Align2RegClassID:
2785 case AMDGPU::AV_96RegClassID:
2786 case AMDGPU::AV_96_Align2RegClassID:
2787 case AMDGPU::VReg_96_Lo256_Align2RegClassID:
2788 return 96;
2789 case AMDGPU::SGPR_128RegClassID:
2790 case AMDGPU::SReg_128RegClassID:
2791 case AMDGPU::VReg_128RegClassID:
2792 case AMDGPU::AReg_128RegClassID:
2793 case AMDGPU::VReg_128_Align2RegClassID:
2794 case AMDGPU::AReg_128_Align2RegClassID:
2795 case AMDGPU::AV_128RegClassID:
2796 case AMDGPU::AV_128_Align2RegClassID:
2797 case AMDGPU::SReg_128_XNULLRegClassID:
2798 case AMDGPU::VReg_128_Lo256_Align2RegClassID:
2799 return 128;
2800 case AMDGPU::SGPR_160RegClassID:
2801 case AMDGPU::SReg_160RegClassID:
2802 case AMDGPU::VReg_160RegClassID:
2803 case AMDGPU::AReg_160RegClassID:
2804 case AMDGPU::VReg_160_Align2RegClassID:
2805 case AMDGPU::AReg_160_Align2RegClassID:
2806 case AMDGPU::AV_160RegClassID:
2807 case AMDGPU::AV_160_Align2RegClassID:
2808 case AMDGPU::VReg_160_Lo256_Align2RegClassID:
2809 return 160;
2810 case AMDGPU::SGPR_192RegClassID:
2811 case AMDGPU::SReg_192RegClassID:
2812 case AMDGPU::VReg_192RegClassID:
2813 case AMDGPU::AReg_192RegClassID:
2814 case AMDGPU::VReg_192_Align2RegClassID:
2815 case AMDGPU::AReg_192_Align2RegClassID:
2816 case AMDGPU::AV_192RegClassID:
2817 case AMDGPU::AV_192_Align2RegClassID:
2818 case AMDGPU::VReg_192_Lo256_Align2RegClassID:
2819 return 192;
2820 case AMDGPU::SGPR_224RegClassID:
2821 case AMDGPU::SReg_224RegClassID:
2822 case AMDGPU::VReg_224RegClassID:
2823 case AMDGPU::AReg_224RegClassID:
2824 case AMDGPU::VReg_224_Align2RegClassID:
2825 case AMDGPU::AReg_224_Align2RegClassID:
2826 case AMDGPU::AV_224RegClassID:
2827 case AMDGPU::AV_224_Align2RegClassID:
2828 case AMDGPU::VReg_224_Lo256_Align2RegClassID:
2829 return 224;
2830 case AMDGPU::SGPR_256RegClassID:
2831 case AMDGPU::SReg_256RegClassID:
2832 case AMDGPU::VReg_256RegClassID:
2833 case AMDGPU::AReg_256RegClassID:
2834 case AMDGPU::VReg_256_Align2RegClassID:
2835 case AMDGPU::AReg_256_Align2RegClassID:
2836 case AMDGPU::AV_256RegClassID:
2837 case AMDGPU::AV_256_Align2RegClassID:
2838 case AMDGPU::SReg_256_XNULLRegClassID:
2839 case AMDGPU::VReg_256_Lo256_Align2RegClassID:
2840 return 256;
2841 case AMDGPU::SGPR_288RegClassID:
2842 case AMDGPU::SReg_288RegClassID:
2843 case AMDGPU::VReg_288RegClassID:
2844 case AMDGPU::AReg_288RegClassID:
2845 case AMDGPU::VReg_288_Align2RegClassID:
2846 case AMDGPU::AReg_288_Align2RegClassID:
2847 case AMDGPU::AV_288RegClassID:
2848 case AMDGPU::AV_288_Align2RegClassID:
2849 case AMDGPU::VReg_288_Lo256_Align2RegClassID:
2850 return 288;
2851 case AMDGPU::SGPR_320RegClassID:
2852 case AMDGPU::SReg_320RegClassID:
2853 case AMDGPU::VReg_320RegClassID:
2854 case AMDGPU::AReg_320RegClassID:
2855 case AMDGPU::VReg_320_Align2RegClassID:
2856 case AMDGPU::AReg_320_Align2RegClassID:
2857 case AMDGPU::AV_320RegClassID:
2858 case AMDGPU::AV_320_Align2RegClassID:
2859 case AMDGPU::VReg_320_Lo256_Align2RegClassID:
2860 return 320;
2861 case AMDGPU::SGPR_352RegClassID:
2862 case AMDGPU::SReg_352RegClassID:
2863 case AMDGPU::VReg_352RegClassID:
2864 case AMDGPU::AReg_352RegClassID:
2865 case AMDGPU::VReg_352_Align2RegClassID:
2866 case AMDGPU::AReg_352_Align2RegClassID:
2867 case AMDGPU::AV_352RegClassID:
2868 case AMDGPU::AV_352_Align2RegClassID:
2869 case AMDGPU::VReg_352_Lo256_Align2RegClassID:
2870 return 352;
2871 case AMDGPU::SGPR_384RegClassID:
2872 case AMDGPU::SReg_384RegClassID:
2873 case AMDGPU::VReg_384RegClassID:
2874 case AMDGPU::AReg_384RegClassID:
2875 case AMDGPU::VReg_384_Align2RegClassID:
2876 case AMDGPU::AReg_384_Align2RegClassID:
2877 case AMDGPU::AV_384RegClassID:
2878 case AMDGPU::AV_384_Align2RegClassID:
2879 case AMDGPU::VReg_384_Lo256_Align2RegClassID:
2880 return 384;
2881 case AMDGPU::SGPR_512RegClassID:
2882 case AMDGPU::SReg_512RegClassID:
2883 case AMDGPU::VReg_512RegClassID:
2884 case AMDGPU::AReg_512RegClassID:
2885 case AMDGPU::VReg_512_Align2RegClassID:
2886 case AMDGPU::AReg_512_Align2RegClassID:
2887 case AMDGPU::AV_512RegClassID:
2888 case AMDGPU::AV_512_Align2RegClassID:
2889 case AMDGPU::VReg_512_Lo256_Align2RegClassID:
2890 return 512;
2891 case AMDGPU::SGPR_1024RegClassID:
2892 case AMDGPU::SReg_1024RegClassID:
2893 case AMDGPU::VReg_1024RegClassID:
2894 case AMDGPU::AReg_1024RegClassID:
2895 case AMDGPU::VReg_1024_Align2RegClassID:
2896 case AMDGPU::AReg_1024_Align2RegClassID:
2897 case AMDGPU::AV_1024RegClassID:
2898 case AMDGPU::AV_1024_Align2RegClassID:
2899 case AMDGPU::VReg_1024_Lo256_Align2RegClassID:
2900 return 1024;
2901 default:
2902 llvm_unreachable("Unexpected register class");
2903 }
2904}
2905
2906unsigned getRegBitWidth(const MCRegisterClass &RC) {
2907 return getRegBitWidth(RCID: RC.getID());
2908}
2909
2910bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
2911 if (isInlinableIntLiteral(Literal))
2912 return true;
2913
2914 uint64_t Val = static_cast<uint64_t>(Literal);
2915 return (Val == llvm::bit_cast<uint64_t>(from: 0.0)) ||
2916 (Val == llvm::bit_cast<uint64_t>(from: 1.0)) ||
2917 (Val == llvm::bit_cast<uint64_t>(from: -1.0)) ||
2918 (Val == llvm::bit_cast<uint64_t>(from: 0.5)) ||
2919 (Val == llvm::bit_cast<uint64_t>(from: -0.5)) ||
2920 (Val == llvm::bit_cast<uint64_t>(from: 2.0)) ||
2921 (Val == llvm::bit_cast<uint64_t>(from: -2.0)) ||
2922 (Val == llvm::bit_cast<uint64_t>(from: 4.0)) ||
2923 (Val == llvm::bit_cast<uint64_t>(from: -4.0)) ||
2924 (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
2925}
2926
2927bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
2928 if (isInlinableIntLiteral(Literal))
2929 return true;
2930
2931 // The actual type of the operand does not seem to matter as long
2932 // as the bits match one of the inline immediate values. For example:
2933 //
2934 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
2935 // so it is a legal inline immediate.
2936 //
2937 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
2938 // floating-point, so it is a legal inline immediate.
2939
2940 uint32_t Val = static_cast<uint32_t>(Literal);
2941 return (Val == llvm::bit_cast<uint32_t>(from: 0.0f)) ||
2942 (Val == llvm::bit_cast<uint32_t>(from: 1.0f)) ||
2943 (Val == llvm::bit_cast<uint32_t>(from: -1.0f)) ||
2944 (Val == llvm::bit_cast<uint32_t>(from: 0.5f)) ||
2945 (Val == llvm::bit_cast<uint32_t>(from: -0.5f)) ||
2946 (Val == llvm::bit_cast<uint32_t>(from: 2.0f)) ||
2947 (Val == llvm::bit_cast<uint32_t>(from: -2.0f)) ||
2948 (Val == llvm::bit_cast<uint32_t>(from: 4.0f)) ||
2949 (Val == llvm::bit_cast<uint32_t>(from: -4.0f)) ||
2950 (Val == 0x3e22f983 && HasInv2Pi);
2951}
2952
2953bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi) {
2954 if (!HasInv2Pi)
2955 return false;
2956 if (isInlinableIntLiteral(Literal))
2957 return true;
2958 uint16_t Val = static_cast<uint16_t>(Literal);
2959 return Val == 0x3F00 || // 0.5
2960 Val == 0xBF00 || // -0.5
2961 Val == 0x3F80 || // 1.0
2962 Val == 0xBF80 || // -1.0
2963 Val == 0x4000 || // 2.0
2964 Val == 0xC000 || // -2.0
2965 Val == 0x4080 || // 4.0
2966 Val == 0xC080 || // -4.0
2967 Val == 0x3E22; // 1.0 / (2.0 * pi)
2968}
2969
2970bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi) {
2971 return isInlinableLiteral32(Literal, HasInv2Pi);
2972}
2973
2974bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) {
2975 if (!HasInv2Pi)
2976 return false;
2977 if (isInlinableIntLiteral(Literal))
2978 return true;
2979 uint16_t Val = static_cast<uint16_t>(Literal);
2980 return Val == 0x3C00 || // 1.0
2981 Val == 0xBC00 || // -1.0
2982 Val == 0x3800 || // 0.5
2983 Val == 0xB800 || // -0.5
2984 Val == 0x4000 || // 2.0
2985 Val == 0xC000 || // -2.0
2986 Val == 0x4400 || // 4.0
2987 Val == 0xC400 || // -4.0
2988 Val == 0x3118; // 1/2pi
2989}
2990
2991std::optional<unsigned> getInlineEncodingV216(bool IsFloat, uint32_t Literal) {
2992 // Unfortunately, the Instruction Set Architecture Reference Guide is
2993 // misleading about how the inline operands work for (packed) 16-bit
2994 // instructions. In a nutshell, the actual HW behavior is:
2995 //
2996 // - integer encodings (-16 .. 64) are always produced as sign-extended
2997 // 32-bit values
2998 // - float encodings are produced as:
2999 // - for F16 instructions: corresponding half-precision float values in
3000 // the LSBs, 0 in the MSBs
3001 // - for UI16 instructions: corresponding single-precision float value
3002 int32_t Signed = static_cast<int32_t>(Literal);
3003 if (Signed >= 0 && Signed <= 64)
3004 return 128 + Signed;
3005
3006 if (Signed >= -16 && Signed <= -1)
3007 return 192 + std::abs(x: Signed);
3008
3009 if (IsFloat) {
3010 // clang-format off
3011 switch (Literal) {
3012 case 0x3800: return 240; // 0.5
3013 case 0xB800: return 241; // -0.5
3014 case 0x3C00: return 242; // 1.0
3015 case 0xBC00: return 243; // -1.0
3016 case 0x4000: return 244; // 2.0
3017 case 0xC000: return 245; // -2.0
3018 case 0x4400: return 246; // 4.0
3019 case 0xC400: return 247; // -4.0
3020 case 0x3118: return 248; // 1.0 / (2.0 * pi)
3021 default: break;
3022 }
3023 // clang-format on
3024 } else {
3025 // clang-format off
3026 switch (Literal) {
3027 case 0x3F000000: return 240; // 0.5
3028 case 0xBF000000: return 241; // -0.5
3029 case 0x3F800000: return 242; // 1.0
3030 case 0xBF800000: return 243; // -1.0
3031 case 0x40000000: return 244; // 2.0
3032 case 0xC0000000: return 245; // -2.0
3033 case 0x40800000: return 246; // 4.0
3034 case 0xC0800000: return 247; // -4.0
3035 case 0x3E22F983: return 248; // 1.0 / (2.0 * pi)
3036 default: break;
3037 }
3038 // clang-format on
3039 }
3040
3041 return {};
3042}
3043
3044// Encoding of the literal as an inline constant for a V_PK_*_IU16 instruction
3045// or nullopt.
3046std::optional<unsigned> getInlineEncodingV2I16(uint32_t Literal) {
3047 return getInlineEncodingV216(IsFloat: false, Literal);
3048}
3049
3050// Encoding of the literal as an inline constant for a V_PK_*_BF16 instruction
3051// or nullopt.
3052std::optional<unsigned> getInlineEncodingV2BF16(uint32_t Literal) {
3053 int32_t Signed = static_cast<int32_t>(Literal);
3054 if (Signed >= 0 && Signed <= 64)
3055 return 128 + Signed;
3056
3057 if (Signed >= -16 && Signed <= -1)
3058 return 192 + std::abs(x: Signed);
3059
3060 // clang-format off
3061 switch (Literal) {
3062 case 0x3F00: return 240; // 0.5
3063 case 0xBF00: return 241; // -0.5
3064 case 0x3F80: return 242; // 1.0
3065 case 0xBF80: return 243; // -1.0
3066 case 0x4000: return 244; // 2.0
3067 case 0xC000: return 245; // -2.0
3068 case 0x4080: return 246; // 4.0
3069 case 0xC080: return 247; // -4.0
3070 case 0x3E22: return 248; // 1.0 / (2.0 * pi)
3071 default: break;
3072 }
3073 // clang-format on
3074
3075 return std::nullopt;
3076}
3077
3078// Encoding of the literal as an inline constant for a V_PK_*_F16 instruction
3079// or nullopt.
3080std::optional<unsigned> getInlineEncodingV2F16(uint32_t Literal) {
3081 return getInlineEncodingV216(IsFloat: true, Literal);
3082}
3083
3084// Encoding of the literal as an inline constant for V_PK_FMAC_F16 instruction
3085// or nullopt. This accounts for different inline constant behavior:
3086// - Pre-GFX11: fp16 inline constants have the value in low 16 bits, 0 in high
3087// - GFX11+: fp16 inline constants are duplicated into both halves
3088std::optional<unsigned> getPKFMACF16InlineEncoding(uint32_t Literal,
3089 bool IsGFX11Plus) {
3090 // Pre-GFX11 behavior: f16 in low bits, 0 in high bits
3091 if (!IsGFX11Plus)
3092 return getInlineEncodingV216(/*IsFloat=*/true, Literal);
3093
3094 // GFX11+ behavior: f16 duplicated in both halves
3095 // First, check for sign-extended integer inline constants (-16 to 64)
3096 // These work the same across all generations
3097 int32_t Signed = static_cast<int32_t>(Literal);
3098 if (Signed >= 0 && Signed <= 64)
3099 return 128 + Signed;
3100
3101 if (Signed >= -16 && Signed <= -1)
3102 return 192 + std::abs(x: Signed);
3103
3104 // For float inline constants on GFX11+, both halves must be equal
3105 uint16_t Lo = static_cast<uint16_t>(Literal);
3106 uint16_t Hi = static_cast<uint16_t>(Literal >> 16);
3107 if (Lo != Hi)
3108 return std::nullopt;
3109 return getInlineEncodingV216(/*IsFloat=*/true, Literal: Lo);
3110}
3111
3112// Whether the given literal can be inlined for a V_PK_* instruction.
3113bool isInlinableLiteralV216(uint32_t Literal, uint8_t OpType) {
3114 switch (OpType) {
3115 case AMDGPU::OPERAND_REG_IMM_V2INT16:
3116 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
3117 return getInlineEncodingV216(IsFloat: false, Literal).has_value();
3118 case AMDGPU::OPERAND_REG_IMM_V2FP16:
3119 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
3120 return getInlineEncodingV216(IsFloat: true, Literal).has_value();
3121 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
3122 llvm_unreachable("OPERAND_REG_IMM_V2FP16_SPLAT is not supported");
3123 case AMDGPU::OPERAND_REG_IMM_V2BF16:
3124 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
3125 return isInlinableLiteralV2BF16(Literal);
3126 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
3127 return false;
3128 default:
3129 llvm_unreachable("bad packed operand type");
3130 }
3131}
3132
3133// Whether the given literal can be inlined for a V_PK_*_IU16 instruction.
3134bool isInlinableLiteralV2I16(uint32_t Literal) {
3135 return getInlineEncodingV2I16(Literal).has_value();
3136}
3137
3138// Whether the given literal can be inlined for a V_PK_*_BF16 instruction.
3139bool isInlinableLiteralV2BF16(uint32_t Literal) {
3140 return getInlineEncodingV2BF16(Literal).has_value();
3141}
3142
3143// Whether the given literal can be inlined for a V_PK_*_F16 instruction.
3144bool isInlinableLiteralV2F16(uint32_t Literal) {
3145 return getInlineEncodingV2F16(Literal).has_value();
3146}
3147
3148// Whether the given literal can be inlined for V_PK_FMAC_F16 instruction.
3149bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus) {
3150 return getPKFMACF16InlineEncoding(Literal, IsGFX11Plus).has_value();
3151}
3152
3153bool isValid32BitLiteral(uint64_t Val, bool IsFP64) {
3154 if (IsFP64)
3155 return !Lo_32(Value: Val);
3156
3157 return isUInt<32>(x: Val) || isInt<32>(x: Val);
3158}
3159
3160int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit) {
3161 switch (Type) {
3162 default:
3163 break;
3164 case OPERAND_REG_IMM_BF16:
3165 case OPERAND_REG_IMM_FP16:
3166 case OPERAND_REG_IMM_NOINLINE_FP16:
3167 case OPERAND_REG_INLINE_C_BF16:
3168 case OPERAND_REG_INLINE_C_FP16:
3169 return Imm & 0xffff;
3170 case OPERAND_INLINE_SPLIT_BARRIER_INT32:
3171 case OPERAND_REG_IMM_FP32:
3172 case OPERAND_REG_IMM_INT32:
3173 case OPERAND_REG_IMM_V2BF16:
3174 case OPERAND_REG_IMM_V2FP16:
3175 case OPERAND_REG_IMM_V2FP16_SPLAT:
3176 case OPERAND_REG_IMM_V2FP32:
3177 case OPERAND_REG_IMM_V2INT16:
3178 case OPERAND_REG_IMM_V2INT32:
3179 case OPERAND_REG_INLINE_AC_FP32:
3180 case OPERAND_REG_INLINE_AC_INT32:
3181 case OPERAND_REG_INLINE_C_FP32:
3182 case OPERAND_REG_INLINE_C_INT32:
3183 return Lo_32(Value: Imm);
3184 case OPERAND_REG_IMM_FP64:
3185 case AMDGPU::OPERAND_REG_IMM_V2FP64:
3186 return IsLit ? Imm : Hi_32(Value: Imm);
3187 }
3188 return Imm;
3189}
3190
3191bool isArgPassedInSGPR(const Argument *A) {
3192 const Function *F = A->getParent();
3193
3194 // Arguments to compute shaders are never a source of divergence.
3195 CallingConv::ID CC = F->getCallingConv();
3196 switch (CC) {
3197 case CallingConv::AMDGPU_KERNEL:
3198 case CallingConv::SPIR_KERNEL:
3199 return true;
3200 case CallingConv::AMDGPU_VS:
3201 case CallingConv::AMDGPU_LS:
3202 case CallingConv::AMDGPU_HS:
3203 case CallingConv::AMDGPU_ES:
3204 case CallingConv::AMDGPU_GS:
3205 case CallingConv::AMDGPU_PS:
3206 case CallingConv::AMDGPU_CS:
3207 case CallingConv::AMDGPU_Gfx:
3208 case CallingConv::AMDGPU_CS_Chain:
3209 case CallingConv::AMDGPU_CS_ChainPreserve:
3210 // For non-compute shaders, SGPR inputs are marked with either inreg or
3211 // byval. Everything else is in VGPRs.
3212 return A->hasAttribute(Kind: Attribute::InReg) ||
3213 A->hasAttribute(Kind: Attribute::ByVal);
3214 default:
3215 // TODO: treat i1 as divergent?
3216 return A->hasAttribute(Kind: Attribute::InReg);
3217 }
3218}
3219
3220bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo) {
3221 // Arguments to compute shaders are never a source of divergence.
3222 CallingConv::ID CC = CB->getCallingConv();
3223 switch (CC) {
3224 case CallingConv::AMDGPU_KERNEL:
3225 case CallingConv::SPIR_KERNEL:
3226 return true;
3227 case CallingConv::AMDGPU_VS:
3228 case CallingConv::AMDGPU_LS:
3229 case CallingConv::AMDGPU_HS:
3230 case CallingConv::AMDGPU_ES:
3231 case CallingConv::AMDGPU_GS:
3232 case CallingConv::AMDGPU_PS:
3233 case CallingConv::AMDGPU_CS:
3234 case CallingConv::AMDGPU_Gfx:
3235 case CallingConv::AMDGPU_CS_Chain:
3236 case CallingConv::AMDGPU_CS_ChainPreserve:
3237 // For non-compute shaders, SGPR inputs are marked with either inreg or
3238 // byval. Everything else is in VGPRs.
3239 return CB->paramHasAttr(ArgNo, Kind: Attribute::InReg) ||
3240 CB->isByValArgument(ArgNo);
3241 default:
3242 return CB->paramHasAttr(ArgNo, Kind: Attribute::InReg);
3243 }
3244}
3245
3246static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
3247 return isGCN3Encoding(STI: ST) || isGFX10Plus(STI: ST);
3248}
3249
3250bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
3251 int64_t EncodedOffset) {
3252 if (isGFX12Plus(STI: ST))
3253 return isUInt<23>(x: EncodedOffset);
3254
3255 return hasSMEMByteOffset(ST) ? isUInt<20>(x: EncodedOffset)
3256 : isUInt<8>(x: EncodedOffset);
3257}
3258
3259bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
3260 int64_t EncodedOffset, bool IsBuffer) {
3261 if (isGFX12Plus(STI: ST)) {
3262 if (IsBuffer && EncodedOffset < 0)
3263 return false;
3264 return isInt<24>(x: EncodedOffset);
3265 }
3266
3267 return !IsBuffer && hasSMRDSignedImmOffset(ST) && isInt<21>(x: EncodedOffset);
3268}
3269
3270static bool isDwordAligned(uint64_t ByteOffset) {
3271 return (ByteOffset & 3) == 0;
3272}
3273
3274uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
3275 uint64_t ByteOffset) {
3276 if (hasSMEMByteOffset(ST))
3277 return ByteOffset;
3278
3279 assert(isDwordAligned(ByteOffset));
3280 return ByteOffset >> 2;
3281}
3282
3283std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
3284 int64_t ByteOffset, bool IsBuffer,
3285 bool HasSOffset) {
3286 // For unbuffered smem loads, it is illegal for the Immediate Offset to be
3287 // negative if the resulting (Offset + (M0 or SOffset or zero) is negative.
3288 // Handle case where SOffset is not present.
3289 if (!IsBuffer && !HasSOffset && ByteOffset < 0 && hasSMRDSignedImmOffset(ST))
3290 return std::nullopt;
3291
3292 if (isGFX12Plus(STI: ST)) // 24 bit signed offsets
3293 return isInt<24>(x: ByteOffset) ? std::optional<int64_t>(ByteOffset)
3294 : std::nullopt;
3295
3296 // The signed version is always a byte offset.
3297 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
3298 assert(hasSMEMByteOffset(ST));
3299 return isInt<20>(x: ByteOffset) ? std::optional<int64_t>(ByteOffset)
3300 : std::nullopt;
3301 }
3302
3303 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
3304 return std::nullopt;
3305
3306 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3307 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
3308 ? std::optional<int64_t>(EncodedOffset)
3309 : std::nullopt;
3310}
3311
3312std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
3313 int64_t ByteOffset) {
3314 if (!isCI(STI: ST) || !isDwordAligned(ByteOffset))
3315 return std::nullopt;
3316
3317 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3318 return isUInt<32>(x: EncodedOffset) ? std::optional<int64_t>(EncodedOffset)
3319 : std::nullopt;
3320}
3321
3322unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST) {
3323 if (ST.getFeatureBits().test(I: FeatureFlatOffsetBits12))
3324 return 12;
3325 if (ST.getFeatureBits().test(I: FeatureFlatOffsetBits24))
3326 return 24;
3327 return 13;
3328}
3329
3330namespace {
3331
3332struct SourceOfDivergence {
3333 unsigned Intr;
3334};
3335const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
3336
3337struct AlwaysUniform {
3338 unsigned Intr;
3339};
3340const AlwaysUniform *lookupAlwaysUniform(unsigned Intr);
3341
3342#define GET_SourcesOfDivergence_IMPL
3343#define GET_UniformIntrinsics_IMPL
3344#define GET_Gfx9BufferFormat_IMPL
3345#define GET_Gfx10BufferFormat_IMPL
3346#define GET_Gfx11PlusBufferFormat_IMPL
3347
3348#include "AMDGPUGenSearchableTables.inc"
3349
3350} // end anonymous namespace
3351
3352bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
3353 return lookupSourceOfDivergence(Intr: IntrID);
3354}
3355
3356bool isIntrinsicAlwaysUniform(unsigned IntrID) {
3357 return lookupAlwaysUniform(Intr: IntrID);
3358}
3359
3360const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
3361 uint8_t NumComponents,
3362 uint8_t NumFormat,
3363 const MCSubtargetInfo &STI) {
3364 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(
3365 BitsPerComp, NumComponents, NumFormat)
3366 : isGFX10(STI)
3367 ? getGfx10BufferFormatInfo(BitsPerComp, NumComponents, NumFormat)
3368 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
3369}
3370
3371const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
3372 const MCSubtargetInfo &STI) {
3373 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(Format)
3374 : isGFX10(STI) ? getGfx10BufferFormatInfo(Format)
3375 : getGfx9BufferFormatInfo(Format);
3376}
3377
3378const MCRegisterClass *getVGPRPhysRegClass(MCRegister Reg,
3379 const MCRegisterInfo &MRI) {
3380 const unsigned VGPRClasses[] = {
3381 AMDGPU::VGPR_16RegClassID, AMDGPU::VGPR_32RegClassID,
3382 AMDGPU::VReg_64RegClassID, AMDGPU::VReg_96RegClassID,
3383 AMDGPU::VReg_128RegClassID, AMDGPU::VReg_160RegClassID,
3384 AMDGPU::VReg_192RegClassID, AMDGPU::VReg_224RegClassID,
3385 AMDGPU::VReg_256RegClassID, AMDGPU::VReg_288RegClassID,
3386 AMDGPU::VReg_320RegClassID, AMDGPU::VReg_352RegClassID,
3387 AMDGPU::VReg_384RegClassID, AMDGPU::VReg_512RegClassID,
3388 AMDGPU::VReg_1024RegClassID};
3389
3390 for (unsigned RCID : VGPRClasses) {
3391 const MCRegisterClass &RC = MRI.getRegClass(i: RCID);
3392 if (RC.contains(Reg))
3393 return &RC;
3394 }
3395
3396 return nullptr;
3397}
3398
3399unsigned getVGPREncodingMSBs(MCRegister Reg, const MCRegisterInfo &MRI) {
3400 unsigned Enc = MRI.getEncodingValue(Reg);
3401 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3402 return Idx >> 8;
3403}
3404
3405MCRegister getVGPRWithMSBs(MCRegister Reg, unsigned MSBs,
3406 const MCRegisterInfo &MRI) {
3407 unsigned Enc = MRI.getEncodingValue(Reg);
3408 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3409 if (Idx >= 0x100)
3410 return MCRegister();
3411
3412 const MCRegisterClass *RC = getVGPRPhysRegClass(Reg, MRI);
3413 if (!RC)
3414 return MCRegister();
3415
3416 Idx |= MSBs << 8;
3417 if (RC->getID() == AMDGPU::VGPR_16RegClassID) {
3418 // This class has 2048 registers with interleaved lo16 and hi16.
3419 Idx *= 2;
3420 if (Enc & AMDGPU::HWEncoding::IS_HI16)
3421 ++Idx;
3422 }
3423
3424 return RC->getRegister(i: Idx);
3425}
3426
3427static std::optional<unsigned>
3428convertSetRegImmToVgprMSBs(uint64_t Imm, uint64_t Simm16,
3429 bool HasSetregVGPRMSBFixup) {
3430 constexpr unsigned VGPRMSBShift =
3431 llvm::countr_zero_constexpr<unsigned>(Val: AMDGPU::Hwreg::DST_VGPR_MSB);
3432
3433 auto [HwRegId, Offset, Size] = Hwreg::HwregEncoding::decode(Encoded: Simm16);
3434 if (HwRegId != Hwreg::ID_MODE ||
3435 (!HasSetregVGPRMSBFixup && (Offset + Size) < VGPRMSBShift))
3436 return {};
3437 // If there is SetregVGPRMSBFixup then Offset is ignored.
3438 if (!HasSetregVGPRMSBFixup)
3439 Imm <<= Offset;
3440 Imm = (Imm & Hwreg::VGPR_MSB_MASK) >> VGPRMSBShift;
3441 if (!HasSetregVGPRMSBFixup)
3442 Imm &= llvm::maskTrailingOnes<unsigned>(N: Size);
3443 return llvm::rotr<uint8_t>(V: static_cast<uint8_t>(Imm), /*R=*/2);
3444}
3445
3446std::optional<unsigned> convertSetRegImmToVgprMSBs(const MachineInstr &MI,
3447 bool HasSetregVGPRMSBFixup) {
3448 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32);
3449 return convertSetRegImmToVgprMSBs(Imm: MI.getOperand(i: 0).getImm(),
3450 Simm16: MI.getOperand(i: 1).getImm(),
3451 HasSetregVGPRMSBFixup);
3452}
3453
3454std::optional<unsigned> convertSetRegImmToVgprMSBs(const MCInst &MI,
3455 bool HasSetregVGPRMSBFixup) {
3456 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_gfx12);
3457 return convertSetRegImmToVgprMSBs(Imm: MI.getOperand(i: 0).getImm(),
3458 Simm16: MI.getOperand(i: 1).getImm(),
3459 HasSetregVGPRMSBFixup);
3460}
3461
3462std::pair<const AMDGPU::OpName *, const AMDGPU::OpName *>
3463getVGPRLoweringOperandTables(const MCInstrDesc &Desc) {
3464 static const AMDGPU::OpName VOPOps[4] = {
3465 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2,
3466 AMDGPU::OpName::vdst};
3467 static const AMDGPU::OpName VDSOps[4] = {
3468 AMDGPU::OpName::addr, AMDGPU::OpName::data0, AMDGPU::OpName::data1,
3469 AMDGPU::OpName::vdst};
3470 static const AMDGPU::OpName FLATOps[4] = {
3471 AMDGPU::OpName::vaddr, AMDGPU::OpName::vdata,
3472 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdst};
3473 static const AMDGPU::OpName BUFOps[4] = {
3474 AMDGPU::OpName::vaddr, AMDGPU::OpName::NUM_OPERAND_NAMES,
3475 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdata};
3476 static const AMDGPU::OpName VIMGOps[4] = {
3477 AMDGPU::OpName::vaddr0, AMDGPU::OpName::vaddr1, AMDGPU::OpName::vaddr2,
3478 AMDGPU::OpName::vdata};
3479
3480 // For VOPD instructions MSB of a corresponding Y component operand VGPR
3481 // address is supposed to match X operand, otherwise VOPD shall not be
3482 // combined.
3483 static const AMDGPU::OpName VOPDOpsX[4] = {
3484 AMDGPU::OpName::src0X, AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vsrc2X,
3485 AMDGPU::OpName::vdstX};
3486 static const AMDGPU::OpName VOPDOpsY[4] = {
3487 AMDGPU::OpName::src0Y, AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vsrc2Y,
3488 AMDGPU::OpName::vdstY};
3489
3490 // VOP2 MADMK instructions use src0, imm, src1 scheme.
3491 static const AMDGPU::OpName VOP2MADMKOps[4] = {
3492 AMDGPU::OpName::src0, AMDGPU::OpName::NUM_OPERAND_NAMES,
3493 AMDGPU::OpName::src1, AMDGPU::OpName::vdst};
3494 static const AMDGPU::OpName VOPDFMAMKOpsX[4] = {
3495 AMDGPU::OpName::src0X, AMDGPU::OpName::NUM_OPERAND_NAMES,
3496 AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vdstX};
3497 static const AMDGPU::OpName VOPDFMAMKOpsY[4] = {
3498 AMDGPU::OpName::src0Y, AMDGPU::OpName::NUM_OPERAND_NAMES,
3499 AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vdstY};
3500
3501 if (SIInstrFlags::isVOP1(O: Desc) || SIInstrFlags::isVOP2(O: Desc) ||
3502 SIInstrFlags::isVOP3Like(O: Desc) || SIInstrFlags::isVOPC(O: Desc) ||
3503 SIInstrFlags::isDPP(O: Desc)) {
3504 switch (Desc.getOpcode()) {
3505 // LD_SCALE operands ignore MSB.
3506 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32:
3507 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32_gfx1250:
3508 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64:
3509 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64_gfx1250:
3510 return {};
3511 case AMDGPU::V_FMAMK_F16:
3512 case AMDGPU::V_FMAMK_F16_t16:
3513 case AMDGPU::V_FMAMK_F16_t16_gfx12:
3514 case AMDGPU::V_FMAMK_F16_fake16:
3515 case AMDGPU::V_FMAMK_F16_fake16_gfx12:
3516 case AMDGPU::V_FMAMK_F32:
3517 case AMDGPU::V_FMAMK_F32_gfx12:
3518 case AMDGPU::V_FMAMK_F64:
3519 case AMDGPU::V_FMAMK_F64_gfx1250:
3520 return {VOP2MADMKOps, nullptr};
3521 default:
3522 break;
3523 }
3524 return {VOPOps, nullptr};
3525 }
3526
3527 if (SIInstrFlags::isDS(O: Desc))
3528 return {VDSOps, nullptr};
3529
3530 if (SIInstrFlags::isFLAT(O: Desc))
3531 return {FLATOps, nullptr};
3532
3533 if (SIInstrFlags::isBuffer(O: Desc))
3534 return {BUFOps, nullptr};
3535
3536 if (SIInstrFlags::isVIMAGE(O: Desc))
3537 return {VIMGOps, nullptr};
3538
3539 if (AMDGPU::isVOPD(Opc: Desc.getOpcode())) {
3540 auto [OpX, OpY] = getVOPDComponents(VOPDOpcode: Desc.getOpcode());
3541 return {(OpX == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsX : VOPDOpsX,
3542 (OpY == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsY : VOPDOpsY};
3543 }
3544
3545 assert(!SIInstrFlags::isMIMG(Desc));
3546
3547 if (SIInstrFlags::isVSAMPLE(O: Desc) || SIInstrFlags::isEXP(O: Desc))
3548 llvm_unreachable("Sample and export VGPR lowering is not implemented and"
3549 " these instructions are not expected on gfx1250");
3550
3551 return {};
3552}
3553
3554bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode) {
3555 const MCInstrDesc &Desc = MII.get(Opcode);
3556 if (SIInstrFlags::isSMRD(O: Desc))
3557 return Desc.mayLoad() && !Desc.mayStore() && !getSMEMIsBuffer(Opc: Opcode);
3558 if (!SIInstrFlags::isFLAT(O: Desc))
3559 return false;
3560
3561 // Only SV and SVS modes are supported.
3562 if (SIInstrFlags::isFlatScratch(O: MII, O: Opcode))
3563 return hasNamedOperand(Opcode, NamedIdx: OpName::vaddr);
3564
3565 // Only GVS mode is supported.
3566 return hasNamedOperand(Opcode, NamedIdx: OpName::vaddr) &&
3567 hasNamedOperand(Opcode, NamedIdx: OpName::saddr);
3568
3569 return false;
3570}
3571
3572static bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc,
3573 const MCInstrInfo &MII,
3574 const MCSubtargetInfo &ST) {
3575 for (auto OpName : {OpName::vdst, OpName::src0, OpName::src1, OpName::src2}) {
3576 int Idx = getNamedOperandIdx(Opcode: OpDesc.getOpcode(), Name: OpName);
3577 if (Idx == -1)
3578 continue;
3579
3580 const MCOperandInfo &OpInfo = OpDesc.operands()[Idx];
3581 int16_t RegClass = MII.getOpRegClassID(
3582 OpInfo, HwModeId: ST.getHwMode(type: MCSubtargetInfo::HwMode_RegInfo));
3583 if (RegClass == AMDGPU::VReg_64RegClassID ||
3584 RegClass == AMDGPU::VReg_64_Align2RegClassID)
3585 return true;
3586 }
3587
3588 return false;
3589}
3590
3591bool isDPALU_DPP32BitOpc(unsigned Opc) {
3592 switch (Opc) {
3593 case AMDGPU::V_MUL_LO_U32_e64:
3594 case AMDGPU::V_MUL_LO_U32_e64_dpp:
3595 case AMDGPU::V_MUL_LO_U32_e64_dpp_gfx1250:
3596 case AMDGPU::V_MUL_HI_U32_e64:
3597 case AMDGPU::V_MUL_HI_U32_e64_dpp:
3598 case AMDGPU::V_MUL_HI_U32_e64_dpp_gfx1250:
3599 case AMDGPU::V_MUL_HI_I32_e64:
3600 case AMDGPU::V_MUL_HI_I32_e64_dpp:
3601 case AMDGPU::V_MUL_HI_I32_e64_dpp_gfx1250:
3602 case AMDGPU::V_MAD_U32_e64:
3603 case AMDGPU::V_MAD_U32_e64_dpp:
3604 case AMDGPU::V_MAD_U32_e64_dpp_gfx1250:
3605 return true;
3606 default:
3607 return false;
3608 }
3609}
3610
3611bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
3612 const MCSubtargetInfo &ST) {
3613 if (isDPALU_DPP32BitOpc(Opc: OpDesc.getOpcode()))
3614 return true;
3615
3616 return hasAny64BitVGPROperands(OpDesc, MII, ST);
3617}
3618
3619bool isPackedSingleSGPRFP32Inst(unsigned Opc) {
3620 switch (Opc) {
3621 case AMDGPU::V_PK_ADD_F32_gfx1250:
3622 case AMDGPU::V_PK_ADD_F32_gfx1250_gfx12:
3623 case AMDGPU::V_PK_MUL_F32_gfx1250:
3624 case AMDGPU::V_PK_MUL_F32_gfx1250_gfx12:
3625 case AMDGPU::V_PK_FMA_F32_gfx1250:
3626 case AMDGPU::V_PK_FMA_F32_gfx1250_gfx12:
3627 return true;
3628 default:
3629 return false;
3630 }
3631}
3632
3633// NOTE: This function is currently only used before pseudo-expansion.
3634bool isPackedSingleSGPR64BitInst(unsigned Opc) {
3635 switch (Opc) {
3636 case AMDGPU::V_PK_ADD_F64:
3637 case AMDGPU::V_PK_MUL_F64:
3638 case AMDGPU::V_PK_FMA_F64:
3639 case AMDGPU::V_PK_MAX_NUM_F64:
3640 case AMDGPU::V_PK_MIN_NUM_F64:
3641 case AMDGPU::V_PK_ADD_NC_U64:
3642 case AMDGPU::V_PK_SUB_NC_U64:
3643 case AMDGPU::V_PK_LSHL_ADD_U64:
3644 return true;
3645 default:
3646 return false;
3647 }
3648}
3649
3650bool isSingleSGPRReadInst(unsigned Opc) {
3651 return isPackedSingleSGPRFP32Inst(Opc) || isPackedSingleSGPR64BitInst(Opc);
3652}
3653
3654const std::array<unsigned, 3> &ClusterDimsAttr::getDims() const {
3655 assert(isFixedDims() && "expect kind to be FixedDims");
3656 return Dims;
3657}
3658
3659std::string ClusterDimsAttr::to_string() const {
3660 SmallString<10> Buffer;
3661 raw_svector_ostream OS(Buffer);
3662
3663 switch (getKind()) {
3664 case Kind::Unknown:
3665 return "";
3666 case Kind::NoCluster: {
3667 OS << EncoNoCluster << ',' << EncoNoCluster << ',' << EncoNoCluster;
3668 return Buffer.c_str();
3669 }
3670 case Kind::VariableDims: {
3671 OS << EncoVariableDims << ',' << EncoVariableDims << ','
3672 << EncoVariableDims;
3673 return Buffer.c_str();
3674 }
3675 case Kind::FixedDims: {
3676 OS << Dims[0] << ',' << Dims[1] << ',' << Dims[2];
3677 return Buffer.c_str();
3678 }
3679 }
3680 llvm_unreachable("Unknown ClusterDimsAttr kind");
3681}
3682
3683ClusterDimsAttr ClusterDimsAttr::get(const Function &F) {
3684 std::optional<SmallVector<unsigned>> Attr =
3685 getIntegerVecAttribute(F, Name: "amdgpu-cluster-dims", /*Size=*/3);
3686 ClusterDimsAttr::Kind AttrKind = Kind::FixedDims;
3687
3688 if (!Attr.has_value())
3689 AttrKind = Kind::Unknown;
3690 else if (all_of(Range&: *Attr, P: equal_to(Arg: EncoNoCluster)))
3691 AttrKind = Kind::NoCluster;
3692 else if (all_of(Range&: *Attr, P: equal_to(Arg: EncoVariableDims)))
3693 AttrKind = Kind::VariableDims;
3694
3695 ClusterDimsAttr A(AttrKind);
3696 if (AttrKind == Kind::FixedDims)
3697 A.Dims = {(*Attr)[0], (*Attr)[1], (*Attr)[2]};
3698
3699 return A;
3700}
3701
3702std::optional<APFloat> evaluateRcp(const APFloat &Val) {
3703 const fltSemantics &Sem = Val.getSemantics();
3704
3705 // v_rcp_f16/bf16 are correctly rounded.
3706 if (&Sem == &APFloat::IEEEhalf() || &Sem == &APFloat::BFloat())
3707 return APFloat::getOne(Sem) / Val;
3708
3709 // v_rcp_f32/f64 always flush a denormal input to zero (preserving sign)
3710 // before reciprocating.
3711 APFloat Arg = Val;
3712 if (Arg.isDenormal())
3713 Arg = APFloat::getZero(Sem, Negative: Arg.isNegative());
3714
3715 APFloat Result = APFloat::getOne(Sem) / Arg;
3716
3717 // v_rcp_f32/f64 always flush a denormal result to zero (preserving sign).
3718 if (Result.isDenormal())
3719 Result = APFloat::getZero(Sem, Negative: Result.isNegative());
3720
3721 // v_rcp_f32/f64 only approximate the reciprocal, except for these special
3722 // cases where the result is exact.
3723 if (!Result.isZero() && !Result.isInfinity() && !Result.isNaN() &&
3724 !Result.isOne() && !Result.isMinusOne())
3725 return std::nullopt;
3726
3727 return Result;
3728}
3729
3730} // namespace AMDGPU
3731
3732raw_ostream &operator<<(raw_ostream &OS, const AMDGPU::TargetIDSetting S) {
3733 switch (S) {
3734 case (AMDGPU::TargetIDSetting::Unsupported):
3735 OS << "Unsupported";
3736 break;
3737 case (AMDGPU::TargetIDSetting::Any):
3738 OS << "Any";
3739 break;
3740 case (AMDGPU::TargetIDSetting::Off):
3741 OS << "Off";
3742 break;
3743 case (AMDGPU::TargetIDSetting::On):
3744 OS << "On";
3745 break;
3746 }
3747 return OS;
3748}
3749
3750} // namespace llvm
3751