1//===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUBaseInfo.h"
10#include "AMDGPU.h"
11#include "AMDGPUAsmUtils.h"
12#include "AMDKernelCodeT.h"
13#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14#include "Utils/AMDKernelCodeTUtils.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/BinaryFormat/ELF.h"
17#include "llvm/IR/Attributes.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/Function.h"
20#include "llvm/IR/GlobalValue.h"
21#include "llvm/IR/IntrinsicsAMDGPU.h"
22#include "llvm/IR/IntrinsicsR600.h"
23#include "llvm/IR/LLVMContext.h"
24#include "llvm/IR/Metadata.h"
25#include "llvm/MC/MCInstrInfo.h"
26#include "llvm/MC/MCRegisterInfo.h"
27#include "llvm/MC/MCSubtargetInfo.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/TargetParser/AMDGPUTargetParser.h"
30#include <optional>
31
32#define GET_INSTRINFO_NAMED_OPS
33#define GET_INSTRMAP_INFO
34#include "AMDGPUGenInstrInfo.inc"
35
36static llvm::cl::opt<unsigned> DefaultAMDHSACodeObjectVersion(
37 "amdhsa-code-object-version", llvm::cl::Hidden,
38 llvm::cl::init(Val: llvm::AMDGPU::AMDHSA_COV6),
39 llvm::cl::desc("Set default AMDHSA Code Object Version (module flag "
40 "or asm directive still take priority if present)"));
41
42namespace {
43
44/// \returns Bit mask for given bit \p Shift and bit \p Width.
45unsigned getBitMask(unsigned Shift, unsigned Width) {
46 return ((1 << Width) - 1) << Shift;
47}
48
49/// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
50///
51/// \returns Packed \p Dst.
52unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
53 unsigned Mask = getBitMask(Shift, Width);
54 return ((Src << Shift) & Mask) | (Dst & ~Mask);
55}
56
57/// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
58///
59/// \returns Unpacked bits.
60unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
61 return (Src & getBitMask(Shift, Width)) >> Shift;
62}
63
64/// \returns Vmcnt bit shift (lower bits).
65unsigned getVmcntBitShiftLo(unsigned VersionMajor) {
66 return VersionMajor >= 11 ? 10 : 0;
67}
68
69/// \returns Vmcnt bit width (lower bits).
70unsigned getVmcntBitWidthLo(unsigned VersionMajor) {
71 return VersionMajor >= 11 ? 6 : 4;
72}
73
74/// \returns Expcnt bit shift.
75unsigned getExpcntBitShift(unsigned VersionMajor) {
76 return VersionMajor >= 11 ? 0 : 4;
77}
78
79/// \returns Expcnt bit width.
80unsigned getExpcntBitWidth(unsigned VersionMajor) { return 3; }
81
82/// \returns Lgkmcnt bit shift.
83unsigned getLgkmcntBitShift(unsigned VersionMajor) {
84 return VersionMajor >= 11 ? 4 : 8;
85}
86
87/// \returns Lgkmcnt bit width.
88unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
89 return VersionMajor >= 10 ? 6 : 4;
90}
91
92/// \returns Vmcnt bit shift (higher bits).
93unsigned getVmcntBitShiftHi(unsigned VersionMajor) { return 14; }
94
95/// \returns Vmcnt bit width (higher bits).
96unsigned getVmcntBitWidthHi(unsigned VersionMajor) {
97 return (VersionMajor == 9 || VersionMajor == 10) ? 2 : 0;
98}
99
100/// \returns Loadcnt bit width
101unsigned getLoadcntBitWidth(unsigned VersionMajor) {
102 return VersionMajor >= 12 ? 6 : 0;
103}
104
105/// \returns Samplecnt bit width.
106unsigned getSamplecntBitWidth(unsigned VersionMajor) {
107 return VersionMajor >= 12 ? 6 : 0;
108}
109
110/// \returns Bvhcnt bit width.
111unsigned getBvhcntBitWidth(unsigned VersionMajor) {
112 return VersionMajor >= 12 ? 3 : 0;
113}
114
115/// \returns Dscnt bit width.
116unsigned getDscntBitWidth(unsigned VersionMajor) {
117 return VersionMajor >= 12 ? 6 : 0;
118}
119
120/// \returns Dscnt bit shift in combined S_WAIT instructions.
121unsigned getDscntBitShift(unsigned VersionMajor) { return 0; }
122
123/// \returns Storecnt or Vscnt bit width, depending on VersionMajor.
124unsigned getStorecntBitWidth(unsigned VersionMajor) {
125 return VersionMajor >= 10 ? 6 : 0;
126}
127
128/// \returns Kmcnt bit width.
129unsigned getKmcntBitWidth(unsigned VersionMajor) {
130 return VersionMajor >= 12 ? 5 : 0;
131}
132
133/// \returns Xcnt bit width.
134unsigned getXcntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
135 return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
136}
137
138/// \returns Asynccnt bit width.
139unsigned getAsynccntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
140 return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
141}
142
143/// \returns shift for Loadcnt/Storecnt in combined S_WAIT instructions.
144unsigned getLoadcntStorecntBitShift(unsigned VersionMajor) {
145 return VersionMajor >= 12 ? 8 : 0;
146}
147
148/// \returns VaSdst bit width
149inline unsigned getVaSdstBitWidth() { return 3; }
150
151/// \returns VaSdst bit shift
152inline unsigned getVaSdstBitShift() { return 9; }
153
154/// \returns VmVsrc bit width
155inline unsigned getVmVsrcBitWidth() { return 3; }
156
157/// \returns VmVsrc bit shift
158inline unsigned getVmVsrcBitShift() { return 2; }
159
160/// \returns VaVdst bit width
161inline unsigned getVaVdstBitWidth() { return 4; }
162
163/// \returns VaVdst bit shift
164inline unsigned getVaVdstBitShift() { return 12; }
165
166/// \returns VaVcc bit width
167inline unsigned getVaVccBitWidth() { return 1; }
168
169/// \returns VaVcc bit shift
170inline unsigned getVaVccBitShift() { return 1; }
171
172/// \returns SaSdst bit width
173inline unsigned getSaSdstBitWidth() { return 1; }
174
175/// \returns SaSdst bit shift
176inline unsigned getSaSdstBitShift() { return 0; }
177
178/// \returns VaSsrc width
179inline unsigned getVaSsrcBitWidth() { return 1; }
180
181/// \returns VaSsrc bit shift
182inline unsigned getVaSsrcBitShift() { return 8; }
183
184/// \returns HoldCnt bit shift
185inline unsigned getHoldCntWidth(unsigned VersionMajor, unsigned VersionMinor) {
186 static constexpr const unsigned MinMajor = 10;
187 static constexpr const unsigned MinMinor = 3;
188 return std::tie(args&: VersionMajor, args&: VersionMinor) >= std::tie(args: MinMajor, args: MinMinor)
189 ? 1
190 : 0;
191}
192
193/// \returns HoldCnt bit shift
194inline unsigned getHoldCntBitShift() { return 7; }
195
196} // end anonymous namespace
197
198namespace llvm {
199
200namespace AMDGPU {
201
202/// \returns true if the target supports signed immediate offset for SMRD
203/// instructions.
204bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
205 return isGFX9Plus(STI: ST);
206}
207
208/// \returns True if \p STI is AMDHSA.
209bool isHsaAbi(const MCSubtargetInfo &STI) {
210 return STI.getTargetTriple().getOS() == Triple::AMDHSA;
211}
212
213unsigned getAMDHSACodeObjectVersion(const Module &M) {
214 if (auto *Ver = mdconst::extract_or_null<ConstantInt>(
215 MD: M.getModuleFlag(Key: "amdhsa_code_object_version"))) {
216 return (unsigned)Ver->getZExtValue() / 100;
217 }
218
219 return getDefaultAMDHSACodeObjectVersion();
220}
221
222unsigned getDefaultAMDHSACodeObjectVersion() {
223 return DefaultAMDHSACodeObjectVersion;
224}
225
226unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion) {
227 switch (ABIVersion) {
228 case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
229 return 4;
230 case ELF::ELFABIVERSION_AMDGPU_HSA_V5:
231 return 5;
232 case ELF::ELFABIVERSION_AMDGPU_HSA_V6:
233 return 6;
234 default:
235 return getDefaultAMDHSACodeObjectVersion();
236 }
237}
238
239uint8_t getELFABIVersion(const Triple &T, unsigned CodeObjectVersion) {
240 if (T.getOS() != Triple::AMDHSA)
241 return 0;
242
243 switch (CodeObjectVersion) {
244 case 4:
245 return ELF::ELFABIVERSION_AMDGPU_HSA_V4;
246 case 5:
247 return ELF::ELFABIVERSION_AMDGPU_HSA_V5;
248 case 6:
249 return ELF::ELFABIVERSION_AMDGPU_HSA_V6;
250 default:
251 report_fatal_error(reason: "Unsupported AMDHSA Code Object Version " +
252 Twine(CodeObjectVersion));
253 }
254}
255
256unsigned getMultigridSyncArgImplicitArgPosition(unsigned CodeObjectVersion) {
257 switch (CodeObjectVersion) {
258 case AMDHSA_COV4:
259 return 48;
260 case AMDHSA_COV5:
261 case AMDHSA_COV6:
262 default:
263 return AMDGPU::ImplicitArg::MULTIGRID_SYNC_ARG_OFFSET;
264 }
265}
266
267// FIXME: All such magic numbers about the ABI should be in a
268// central TD file.
269unsigned getHostcallImplicitArgPosition(unsigned CodeObjectVersion) {
270 switch (CodeObjectVersion) {
271 case AMDHSA_COV4:
272 return 24;
273 case AMDHSA_COV5:
274 case AMDHSA_COV6:
275 default:
276 return AMDGPU::ImplicitArg::HOSTCALL_PTR_OFFSET;
277 }
278}
279
280unsigned getDefaultQueueImplicitArgPosition(unsigned CodeObjectVersion) {
281 switch (CodeObjectVersion) {
282 case AMDHSA_COV4:
283 return 32;
284 case AMDHSA_COV5:
285 case AMDHSA_COV6:
286 default:
287 return AMDGPU::ImplicitArg::DEFAULT_QUEUE_OFFSET;
288 }
289}
290
291unsigned getCompletionActionImplicitArgPosition(unsigned CodeObjectVersion) {
292 switch (CodeObjectVersion) {
293 case AMDHSA_COV4:
294 return 40;
295 case AMDHSA_COV5:
296 case AMDHSA_COV6:
297 default:
298 return AMDGPU::ImplicitArg::COMPLETION_ACTION_OFFSET;
299 }
300}
301
302#define GET_MIMGBaseOpcodesTable_IMPL
303#define GET_MIMGDimInfoTable_IMPL
304#define GET_MIMGInfoTable_IMPL
305#define GET_MIMGLZMappingTable_IMPL
306#define GET_MIMGMIPMappingTable_IMPL
307#define GET_MIMGBiasMappingTable_IMPL
308#define GET_MIMGOffsetMappingTable_IMPL
309#define GET_MIMGG16MappingTable_IMPL
310#define GET_MAIInstInfoTable_IMPL
311#define GET_WMMAInstInfoTable_IMPL
312#include "AMDGPUGenSearchableTables.inc"
313
314int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
315 unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc,
316 bool IndexedSamp) {
317 const MIMGInfo *Info =
318 getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, VDataDwords, VAddrDwords,
319 IndexedRsrc, IndexedSamp);
320 return Info ? Info->Opcode : -1;
321}
322
323const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
324 const MIMGInfo *Info = getMIMGInfo(Opcode: Opc);
325 return Info ? getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode) : nullptr;
326}
327
328int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
329 const MIMGInfo *OrigInfo = getMIMGInfo(Opcode: Opc);
330 const MIMGInfo *NewInfo = getMIMGOpcodeHelper(
331 BaseOpcode: OrigInfo->BaseOpcode, MIMGEncoding: OrigInfo->MIMGEncoding, VDataDwords: NewChannels,
332 VAddrDwords: OrigInfo->VAddrDwords, IndexedRsrc: OrigInfo->IndexedRsrc, IndexedSamp: OrigInfo->IndexedSamp);
333 return NewInfo ? NewInfo->Opcode : -1;
334}
335
336unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
337 const MIMGDimInfo *Dim, bool IsA16,
338 bool IsG16Supported) {
339 unsigned AddrWords = BaseOpcode->NumExtraArgs;
340 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
341 (BaseOpcode->LodOrClampOrMip ? 1 : 0);
342 if (IsA16)
343 AddrWords += divideCeil(Numerator: AddrComponents, Denominator: 2);
344 else
345 AddrWords += AddrComponents;
346
347 // Note: For subtargets that support A16 but not G16, enabling A16 also
348 // enables 16 bit gradients.
349 // For subtargets that support A16 (operand) and G16 (done with a different
350 // instruction encoding), they are independent.
351
352 if (BaseOpcode->Gradients) {
353 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16)
354 // There are two gradients per coordinate, we pack them separately.
355 // For the 3d case,
356 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv)
357 AddrWords += alignTo<2>(Value: Dim->NumGradients / 2);
358 else
359 AddrWords += Dim->NumGradients;
360 }
361 return AddrWords;
362}
363
364struct MUBUFInfo {
365 uint32_t Opcode;
366 uint32_t BaseOpcode;
367 uint8_t elements;
368 bool has_vaddr;
369 bool has_srsrc;
370 bool has_soffset;
371 bool IsBufferInv;
372 bool tfe;
373};
374
375struct MTBUFInfo {
376 uint32_t Opcode;
377 uint32_t BaseOpcode;
378 uint8_t elements;
379 bool has_vaddr;
380 bool has_srsrc;
381 bool has_soffset;
382};
383
384struct SMInfo {
385 uint32_t Opcode;
386 bool IsBuffer;
387};
388
389struct VOPInfo {
390 uint32_t Opcode;
391 bool IsSingle;
392};
393
394struct VOPC64DPPInfo {
395 uint32_t Opcode;
396};
397
398struct VOPCDPPAsmOnlyInfo {
399 uint32_t Opcode;
400};
401
402struct VOP3CDPPAsmOnlyInfo {
403 uint32_t Opcode;
404};
405
406struct VOPDComponentInfo {
407 uint16_t BaseVOP;
408 uint16_t VOPDOp;
409};
410
411struct VOPDInfo {
412 uint32_t Opcode;
413 uint16_t OpX;
414 uint16_t OpY;
415 uint16_t Subtarget;
416 bool VOPD3;
417};
418
419struct VOPTrue16Info {
420 uint32_t Opcode;
421 bool IsTrue16;
422};
423
424struct VOPDXYInfo {
425 uint16_t VOPDXYKey;
426 bool IsX;
427 bool IsY;
428};
429
430#define GET_FP4FP8DstByteSelTable_DECL
431#define GET_FP4FP8DstByteSelTable_IMPL
432
433struct DPMACCInstructionInfo {
434 uint32_t Opcode;
435 bool IsDPMACCInstruction;
436};
437
438struct FP4FP8DstByteSelInfo {
439 uint32_t Opcode;
440 bool HasFP8DstByteSel;
441 bool HasFP4DstByteSel;
442};
443
444#define GET_DPMACCInstructionTable_DECL
445#define GET_DPMACCInstructionTable_IMPL
446#define GET_MTBUFInfoTable_DECL
447#define GET_MTBUFInfoTable_IMPL
448#define GET_MUBUFInfoTable_DECL
449#define GET_MUBUFInfoTable_IMPL
450#define GET_SMInfoTable_DECL
451#define GET_SMInfoTable_IMPL
452#define GET_VOP1InfoTable_DECL
453#define GET_VOP1InfoTable_IMPL
454#define GET_VOP2InfoTable_DECL
455#define GET_VOP2InfoTable_IMPL
456#define GET_VOP3InfoTable_DECL
457#define GET_VOP3InfoTable_IMPL
458#define GET_VOPC64DPPTable_DECL
459#define GET_VOPC64DPPTable_IMPL
460#define GET_VOPC64DPP8Table_DECL
461#define GET_VOPC64DPP8Table_IMPL
462#define GET_VOPCAsmOnlyInfoTable_DECL
463#define GET_VOPCAsmOnlyInfoTable_IMPL
464#define GET_VOP3CAsmOnlyInfoTable_DECL
465#define GET_VOP3CAsmOnlyInfoTable_IMPL
466#define GET_VOPDComponentTable_DECL
467#define GET_VOPDComponentTable_IMPL
468#define GET_VOPDPairs_DECL
469#define GET_VOPDPairs_IMPL
470#define GET_VOPDXYTable_DECL
471#define GET_VOPDXYTable_IMPL
472#define GET_VOPTrue16Table_DECL
473#define GET_VOPTrue16Table_IMPL
474#define GET_True16D16Table_IMPL
475#define GET_WMMAOpcode2AddrMappingTable_DECL
476#define GET_WMMAOpcode2AddrMappingTable_IMPL
477#define GET_WMMAOpcode3AddrMappingTable_DECL
478#define GET_WMMAOpcode3AddrMappingTable_IMPL
479#define GET_getMFMA_F8F6F4_WithSize_DECL
480#define GET_getMFMA_F8F6F4_WithSize_IMPL
481#define GET_isMFMA_F8F6F4Table_IMPL
482#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL
483
484#include "AMDGPUGenSearchableTables.inc"
485
486int getMTBUFBaseOpcode(unsigned Opc) {
487 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opcode: Opc);
488 return Info ? Info->BaseOpcode : -1;
489}
490
491int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
492 const MTBUFInfo *Info =
493 getMTBUFInfoFromBaseOpcodeAndElements(BaseOpcode: BaseOpc, elements: Elements);
494 return Info ? Info->Opcode : -1;
495}
496
497int getMTBUFElements(unsigned Opc) {
498 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
499 return Info ? Info->elements : 0;
500}
501
502bool getMTBUFHasVAddr(unsigned Opc) {
503 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
504 return Info && Info->has_vaddr;
505}
506
507bool getMTBUFHasSrsrc(unsigned Opc) {
508 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
509 return Info && Info->has_srsrc;
510}
511
512bool getMTBUFHasSoffset(unsigned Opc) {
513 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
514 return Info && Info->has_soffset;
515}
516
517int getMUBUFBaseOpcode(unsigned Opc) {
518 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opcode: Opc);
519 return Info ? Info->BaseOpcode : -1;
520}
521
522int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
523 const MUBUFInfo *Info =
524 getMUBUFInfoFromBaseOpcodeAndElements(BaseOpcode: BaseOpc, elements: Elements);
525 return Info ? Info->Opcode : -1;
526}
527
528int getMUBUFElements(unsigned Opc) {
529 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
530 return Info ? Info->elements : 0;
531}
532
533bool getMUBUFHasVAddr(unsigned Opc) {
534 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
535 return Info && Info->has_vaddr;
536}
537
538bool getMUBUFHasSrsrc(unsigned Opc) {
539 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
540 return Info && Info->has_srsrc;
541}
542
543bool getMUBUFHasSoffset(unsigned Opc) {
544 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
545 return Info && Info->has_soffset;
546}
547
548bool getMUBUFIsBufferInv(unsigned Opc) {
549 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
550 return Info && Info->IsBufferInv;
551}
552
553bool getMUBUFTfe(unsigned Opc) {
554 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
555 return Info && Info->tfe;
556}
557
558bool getSMEMIsBuffer(unsigned Opc) {
559 const SMInfo *Info = getSMEMOpcodeHelper(Opcode: Opc);
560 return Info && Info->IsBuffer;
561}
562
563bool getVOP1IsSingle(unsigned Opc) {
564 const VOPInfo *Info = getVOP1OpcodeHelper(Opcode: Opc);
565 return !Info || Info->IsSingle;
566}
567
568bool getVOP2IsSingle(unsigned Opc) {
569 const VOPInfo *Info = getVOP2OpcodeHelper(Opcode: Opc);
570 return !Info || Info->IsSingle;
571}
572
573bool getVOP3IsSingle(unsigned Opc) {
574 const VOPInfo *Info = getVOP3OpcodeHelper(Opcode: Opc);
575 return !Info || Info->IsSingle;
576}
577
578bool isVOPC64DPP(unsigned Opc) {
579 return isVOPC64DPPOpcodeHelper(Opcode: Opc) || isVOPC64DPP8OpcodeHelper(Opcode: Opc);
580}
581
582bool isVOPCAsmOnly(unsigned Opc) { return isVOPCAsmOnlyOpcodeHelper(Opcode: Opc); }
583
584bool getMAIIsDGEMM(unsigned Opc) {
585 const MAIInstInfo *Info = getMAIInstInfoHelper(Opcode: Opc);
586 return Info && Info->is_dgemm;
587}
588
589bool getMAIIsGFX940XDL(unsigned Opc) {
590 const MAIInstInfo *Info = getMAIInstInfoHelper(Opcode: Opc);
591 return Info && Info->is_gfx940_xdl;
592}
593
594bool getWMMAIsXDL(unsigned Opc) {
595 const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opcode: Opc);
596 return Info ? Info->is_wmma_xdl : false;
597}
598
599bool getHasMatrixScale(unsigned Opc) {
600 const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opcode: Opc);
601 return Info && Info->HasMatrixScale;
602}
603
604uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal) {
605 switch (EncodingVal) {
606 case MFMAScaleFormats::FP6_E2M3:
607 case MFMAScaleFormats::FP6_E3M2:
608 return 6;
609 case MFMAScaleFormats::FP4_E2M1:
610 return 4;
611 case MFMAScaleFormats::FP8_E4M3:
612 case MFMAScaleFormats::FP8_E5M2:
613 default:
614 return 8;
615 }
616
617 llvm_unreachable("covered switch over mfma scale formats");
618}
619
620const MFMA_F8F6F4_Info *getMFMA_F8F6F4_WithFormatArgs(unsigned CBSZ,
621 unsigned BLGP,
622 unsigned F8F8Opcode) {
623 uint8_t SrcANumRegs = mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: CBSZ);
624 uint8_t SrcBNumRegs = mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: BLGP);
625 return getMFMA_F8F6F4_InstWithNumRegs(NumRegsSrcA: SrcANumRegs, NumRegsSrcB: SrcBNumRegs, F8F8Opcode);
626}
627
628uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt) {
629 switch (Fmt) {
630 case WMMA::MATRIX_FMT_FP8:
631 case WMMA::MATRIX_FMT_BF8:
632 return 16;
633 case WMMA::MATRIX_FMT_FP6:
634 case WMMA::MATRIX_FMT_BF6:
635 return 12;
636 case WMMA::MATRIX_FMT_FP4:
637 return 8;
638 }
639
640 llvm_unreachable("covered switch over wmma scale formats");
641}
642
643const MFMA_F8F6F4_Info *getWMMA_F8F6F4_WithFormatArgs(unsigned FmtA,
644 unsigned FmtB,
645 unsigned F8F8Opcode) {
646 uint8_t SrcANumRegs = wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtA);
647 uint8_t SrcBNumRegs = wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtB);
648 return getMFMA_F8F6F4_InstWithNumRegs(NumRegsSrcA: SrcANumRegs, NumRegsSrcB: SrcBNumRegs, F8F8Opcode);
649}
650
651bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale,
652 unsigned BFmt, unsigned BScale) {
653 auto isValid = [](unsigned Fmt, unsigned Scale) -> bool {
654 switch (Fmt) {
655 case WMMA::MATRIX_FMT_FP8:
656 case WMMA::MATRIX_FMT_BF8:
657 case WMMA::MATRIX_FMT_FP6:
658 case WMMA::MATRIX_FMT_BF6:
659 if (Scale != WMMA::MATRIX_SCALE_FMT_E8)
660 return false;
661 break;
662 case WMMA::MATRIX_FMT_FP4:
663 if (Scale != WMMA::MATRIX_SCALE_FMT_E8 &&
664 Scale != WMMA::MATRIX_SCALE_FMT_E5M3 &&
665 Scale != WMMA::MATRIX_SCALE_FMT_E4M3)
666 return false;
667 break;
668 }
669 return true;
670 };
671
672 if (!isValid(AFmt, AScale) || !isValid(BFmt, BScale))
673 return false;
674
675 if (AFmt == WMMA::MATRIX_FMT_FP4 && BFmt == WMMA::MATRIX_FMT_FP4 &&
676 AScale != BScale)
677 return false;
678
679 return true;
680}
681
682unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST) {
683 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX13Insts))
684 return SIEncodingFamily::GFX13;
685 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX1250Insts))
686 return SIEncodingFamily::GFX1250;
687 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX12Insts))
688 return SIEncodingFamily::GFX12;
689 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX11_7Insts))
690 return SIEncodingFamily::GFX1170;
691 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX11Insts))
692 return SIEncodingFamily::GFX11;
693 llvm_unreachable("Subtarget generation does not support VOPD!");
694}
695
696CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3) {
697 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
698 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
699 // Normalize through VOPDComponentTable so that e32 and e64 variants
700 // of the same logical opcode all share a single entry.
701 const VOPDComponentInfo *Info = getVOPDComponentHelper(BaseVOP: Opc);
702 if (!Info)
703 return {.X: false, .Y: false};
704 unsigned Key =
705 (Info->VOPDOp << 5) | (EncodingFamily << 1) | (VOPD3 ? 1u : 0u);
706 const VOPDXYInfo *XYInfo = getVOPDXYInfo(VOPDXYKey: Key);
707 if (!XYInfo)
708 return {.X: false, .Y: false};
709 return {.X: XYInfo->IsX, .Y: XYInfo->IsY};
710}
711
712unsigned getVOPDOpcode(unsigned Opc, bool VOPD3) {
713 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
714 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
715 const VOPDComponentInfo *Info = getVOPDComponentHelper(BaseVOP: Opc);
716 return Info ? Info->VOPDOp : ~0u;
717}
718
719bool isVOPD(unsigned Opc) {
720 return AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::src0X);
721}
722
723bool isMAC(unsigned Opc) {
724 return Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 ||
725 Opc == AMDGPU::V_MAC_F32_e64_gfx10 ||
726 Opc == AMDGPU::V_MAC_F32_e64_vi ||
727 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx6_gfx7 ||
728 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx10 ||
729 Opc == AMDGPU::V_MAC_F16_e64_vi ||
730 Opc == AMDGPU::V_FMAC_F64_e64_gfx90a ||
731 Opc == AMDGPU::V_FMAC_F64_e64_gfx12 ||
732 Opc == AMDGPU::V_FMAC_F64_e64_gfx13 ||
733 Opc == AMDGPU::V_FMAC_F32_e64_gfx10 ||
734 Opc == AMDGPU::V_FMAC_F32_e64_gfx11 ||
735 Opc == AMDGPU::V_FMAC_F32_e64_gfx12 ||
736 Opc == AMDGPU::V_FMAC_F32_e64_gfx13 ||
737 Opc == AMDGPU::V_FMAC_F32_e64_vi ||
738 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64_gfx10 ||
739 Opc == AMDGPU::V_FMAC_DX9_ZERO_F32_e64_gfx11 ||
740 Opc == AMDGPU::V_FMAC_F16_e64_gfx10 ||
741 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx11 ||
742 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx11 ||
743 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx12 ||
744 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx12 ||
745 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx13 ||
746 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx13 ||
747 Opc == AMDGPU::V_DOT2C_F32_F16_e64_vi ||
748 Opc == AMDGPU::V_DOT2C_F32_BF16_e64_vi ||
749 Opc == AMDGPU::V_DOT2C_I32_I16_e64_vi ||
750 Opc == AMDGPU::V_DOT4C_I32_I8_e64_vi ||
751 Opc == AMDGPU::V_DOT8C_I32_I4_e64_vi;
752}
753
754bool isPermlane16(unsigned Opc) {
755 return Opc == AMDGPU::V_PERMLANE16_B32_gfx10 ||
756 Opc == AMDGPU::V_PERMLANEX16_B32_gfx10 ||
757 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx11 ||
758 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx11 ||
759 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx12 ||
760 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx13 ||
761 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx12 ||
762 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx13 ||
763 Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx12 ||
764 Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx13 ||
765 Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx12 ||
766 Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx13;
767}
768
769bool isCvt_F32_Fp8_Bf8_e64(unsigned Opc) {
770 return Opc == AMDGPU::V_CVT_F32_BF8_e64_gfx12 ||
771 Opc == AMDGPU::V_CVT_F32_FP8_e64_gfx12 ||
772 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp_gfx12 ||
773 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp_gfx12 ||
774 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp8_gfx12 ||
775 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp8_gfx12 ||
776 Opc == AMDGPU::V_CVT_PK_F32_BF8_fake16_e64_gfx12 ||
777 Opc == AMDGPU::V_CVT_PK_F32_FP8_fake16_e64_gfx12 ||
778 Opc == AMDGPU::V_CVT_PK_F32_BF8_t16_e64_gfx12 ||
779 Opc == AMDGPU::V_CVT_PK_F32_FP8_t16_e64_gfx12;
780}
781
782bool isGenericAtomic(unsigned Opc) {
783 return Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP ||
784 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD ||
785 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB ||
786 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN ||
787 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN ||
788 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX ||
789 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX ||
790 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND ||
791 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR ||
792 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR ||
793 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC ||
794 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC ||
795 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD ||
796 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN ||
797 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX ||
798 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP ||
799 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32 ||
800 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32 ||
801 Opc == AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG;
802}
803
804bool isAsyncStore(unsigned Opc) {
805 return Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_gfx1250 ||
806 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_gfx1250 ||
807 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_gfx1250 ||
808 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_gfx1250 ||
809 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR_gfx1250 ||
810 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR_gfx1250 ||
811 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR_gfx1250 ||
812 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR_gfx1250;
813}
814
815bool isTensorStore(unsigned Opc) {
816 return Opc == TENSOR_STORE_FROM_LDS_d2_gfx1250 ||
817 Opc == TENSOR_STORE_FROM_LDS_d4_gfx1250;
818}
819
820unsigned getTemporalHintType(const MCInstrDesc TID) {
821 if (SIInstrFlags::isAtomic(O: TID))
822 return CPol::TH_TYPE_ATOMIC;
823 unsigned Opc = TID.getOpcode();
824 // Async and Tensor store should have the temporal hint type of TH_TYPE_STORE
825 if (TID.mayStore() &&
826 (isAsyncStore(Opc) || isTensorStore(Opc) || !TID.mayLoad()))
827 return CPol::TH_TYPE_STORE;
828
829 // This will default to returning TH_TYPE_LOAD when neither MayStore nor
830 // MayLoad flag is present which is the case with instructions like
831 // image_get_resinfo.
832 return CPol::TH_TYPE_LOAD;
833}
834
835bool isTrue16Inst(unsigned Opc) {
836 const VOPTrue16Info *Info = getTrue16OpcodeHelper(Opcode: Opc);
837 return Info && Info->IsTrue16;
838}
839
840FPType getFPDstSelType(unsigned Opc) {
841 const FP4FP8DstByteSelInfo *Info = getFP4FP8DstByteSelHelper(Opcode: Opc);
842 if (!Info)
843 return FPType::None;
844 if (Info->HasFP8DstByteSel)
845 return FPType::FP8;
846 if (Info->HasFP4DstByteSel)
847 return FPType::FP4;
848
849 return FPType::None;
850}
851
852bool isDPMACCInstruction(unsigned Opc) {
853 const DPMACCInstructionInfo *Info = getDPMACCInstructionHelper(Opcode: Opc);
854 return Info && Info->IsDPMACCInstruction;
855}
856
857unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc) {
858 const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom2AddrOpcode(Opcode2Addr: Opc);
859 return Info ? Info->Opcode3Addr : ~0u;
860}
861
862unsigned mapWMMA3AddrTo2AddrOpcode(unsigned Opc) {
863 const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom3AddrOpcode(Opcode3Addr: Opc);
864 return Info ? Info->Opcode2Addr : ~0u;
865}
866
867// Wrapper for Tablegen'd function. enum Subtarget is not defined in any
868// header files, so we need to wrap it in a function that takes unsigned
869// instead.
870int32_t getMCOpcode(uint32_t Opcode, unsigned Gen) {
871 return getMCOpcodeGen(Opcode, inSubtarget: static_cast<Subtarget>(Gen));
872}
873
874unsigned getBitOp2(unsigned Opc) {
875 switch (Opc) {
876 default:
877 return 0;
878 case AMDGPU::V_AND_B32_e32:
879 return 0x40;
880 case AMDGPU::V_OR_B32_e32:
881 return 0x54;
882 case AMDGPU::V_XOR_B32_e32:
883 return 0x14;
884 case AMDGPU::V_XNOR_B32_e32:
885 return 0x41;
886 }
887}
888
889int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily,
890 bool VOPD3) {
891 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc: OpY) : 0;
892 OpY = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : OpY;
893 const VOPDInfo *Info =
894 getVOPDInfoFromComponentOpcodes(OpX, OpY, SubTgt: EncodingFamily, VOPD3);
895 return Info ? Info->Opcode : -1;
896}
897
898std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode) {
899 const VOPDInfo *Info = getVOPDOpcodeHelper(Opcode: VOPDOpcode);
900 assert(Info);
901 const auto *OpX = getVOPDBaseFromComponent(VOPDOp: Info->OpX);
902 const auto *OpY = getVOPDBaseFromComponent(VOPDOp: Info->OpY);
903 assert(OpX && OpY);
904 return {OpX->BaseVOP, OpY->BaseVOP};
905}
906
907namespace VOPD {
908
909ComponentProps::ComponentProps(const MCInstrDesc &OpDesc, bool VOP3Layout) {
910 assert(OpDesc.getNumDefs() == Component::DST_NUM);
911
912 assert(OpDesc.getOperandConstraint(Component::SRC0, MCOI::TIED_TO) == -1);
913 assert(OpDesc.getOperandConstraint(Component::SRC1, MCOI::TIED_TO) == -1);
914 auto TiedIdx = OpDesc.getOperandConstraint(OpNum: Component::SRC2, Constraint: MCOI::TIED_TO);
915 assert(TiedIdx == -1 || TiedIdx == Component::DST);
916 HasSrc2Acc = TiedIdx != -1;
917 Opcode = OpDesc.getOpcode();
918
919 IsVOP3 = VOP3Layout || SIInstrFlags::isVOP3(O: OpDesc);
920 SrcOperandsNum = AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src2) ? 3
921 : AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::imm) ? 3
922 : AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src1) ? 2
923 : 1;
924 assert(SrcOperandsNum <= Component::MAX_SRC_NUM);
925
926 if (Opcode == AMDGPU::V_CNDMASK_B32_e32 ||
927 Opcode == AMDGPU::V_CNDMASK_B32_e64) {
928 // CNDMASK is an awkward exception, it has FP modifiers, but not FP
929 // operands.
930 NumVOPD3Mods = 2;
931 if (IsVOP3)
932 SrcOperandsNum = 3;
933 } else if (Opcode == AMDGPU::V_DOT2_F32_F16 ||
934 Opcode == AMDGPU::V_DOT2_F32_BF16) {
935 // VOP3P opcodes that have VOPD but don't have VOP2 version. Using VOPD3
936 // path in getIndexOfSrcInMCOperands to get correct src operand indexes,
937 // but generating VOPD, not VOPD3.
938 NumVOPD3Mods = SrcOperandsNum;
939 } else if (isSISrcFPOperand(Desc: OpDesc,
940 OpNo: getNamedOperandIdx(Opcode, Name: OpName::src0))) {
941 // All FP VOPD instructions have Neg modifiers for all operands except
942 // for tied src2.
943 NumVOPD3Mods = SrcOperandsNum;
944 if (HasSrc2Acc)
945 --NumVOPD3Mods;
946 }
947
948 if (SIInstrFlags::isVOP3(O: OpDesc))
949 return;
950
951 auto OperandsNum = OpDesc.getNumOperands();
952 unsigned CompOprIdx;
953 for (CompOprIdx = Component::SRC1; CompOprIdx < OperandsNum; ++CompOprIdx) {
954 if (OpDesc.operands()[CompOprIdx].OperandType == AMDGPU::OPERAND_KIMM32) {
955 MandatoryLiteralIdx = CompOprIdx;
956 break;
957 }
958 }
959}
960
961int ComponentProps::getBitOp3OperandIdx() const {
962 return getNamedOperandIdx(Opcode, Name: OpName::bitop3);
963}
964
965unsigned ComponentInfo::getIndexInParsedOperands(unsigned CompOprIdx) const {
966 assert(CompOprIdx < Component::MAX_OPR_NUM);
967
968 if (CompOprIdx == Component::DST)
969 return getIndexOfDstInParsedOperands();
970
971 auto CompSrcIdx = CompOprIdx - Component::DST_NUM;
972 if (CompSrcIdx < getCompParsedSrcOperandsNum())
973 return getIndexOfSrcInParsedOperands(CompSrcIdx);
974
975 // The specified operand does not exist.
976 return 0;
977}
978
979std::optional<unsigned> InstInfo::getInvalidCompOperandIndex(
980 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
981 const MCRegisterInfo &MRI, bool SkipSrc, bool AllowSameVGPR, bool VOPD3,
982 bool HasGFX11InterlockHazard) const {
983
984 auto OpXRegs = getRegIndices(ComponentIdx: ComponentIndex::X, GetRegIdx,
985 VOPD3: CompInfo[ComponentIndex::X].isVOP3());
986 auto OpYRegs = getRegIndices(ComponentIdx: ComponentIndex::Y, GetRegIdx,
987 VOPD3: CompInfo[ComponentIndex::Y].isVOP3());
988
989 const auto banksOverlap = [&MRI](MCRegister X, MCRegister Y,
990 unsigned BanksMask) -> bool {
991 MCRegister BaseX = MRI.getSubReg(Reg: X, Idx: AMDGPU::sub0);
992 MCRegister BaseY = MRI.getSubReg(Reg: Y, Idx: AMDGPU::sub0);
993 if (!BaseX)
994 BaseX = X;
995 if (!BaseY)
996 BaseY = Y;
997 if ((BaseX.id() & BanksMask) == (BaseY.id() & BanksMask))
998 return true;
999 if (BaseX != X /* This is 64-bit register */ &&
1000 ((BaseX.id() + 1) & BanksMask) == (BaseY.id() & BanksMask))
1001 return true;
1002 if (BaseY != Y &&
1003 (BaseX.id() & BanksMask) == ((BaseY.id() + 1) & BanksMask))
1004 return true;
1005
1006 // If both are 64-bit bank conflict will be detected yet while checking
1007 // the first subreg.
1008 return false;
1009 };
1010
1011 unsigned CompOprIdx;
1012 for (CompOprIdx = 0; CompOprIdx < Component::MAX_OPR_NUM; ++CompOprIdx) {
1013 unsigned BanksMasks = VOPD3 ? VOPD3_VGPR_BANK_MASKS[CompOprIdx]
1014 : HasGFX11InterlockHazard
1015 ? VOPD_GFX11_VGPR_BANK_MASKS[CompOprIdx]
1016 : VOPD_VGPR_BANK_MASKS[CompOprIdx];
1017 if (!OpXRegs[CompOprIdx] || !OpYRegs[CompOprIdx])
1018 continue;
1019
1020 if (getVGPREncodingMSBs(Reg: OpXRegs[CompOprIdx], MRI) !=
1021 getVGPREncodingMSBs(Reg: OpYRegs[CompOprIdx], MRI))
1022 return CompOprIdx;
1023
1024 if (SkipSrc && CompOprIdx >= Component::DST_NUM)
1025 continue;
1026
1027 if (CompOprIdx < Component::DST_NUM) {
1028 // Even if we do not check vdst parity, vdst operands still shall not
1029 // overlap.
1030 if (MRI.regsOverlap(RegA: OpXRegs[CompOprIdx], RegB: OpYRegs[CompOprIdx]))
1031 return CompOprIdx;
1032 if (VOPD3) // No need to check dst parity.
1033 continue;
1034 }
1035
1036 if (banksOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx], BanksMasks) &&
1037 (!AllowSameVGPR || CompOprIdx < Component::DST_NUM ||
1038 OpXRegs[CompOprIdx] != OpYRegs[CompOprIdx]))
1039 return CompOprIdx;
1040 }
1041
1042 return {};
1043}
1044
1045// Return an array of VGPR registers [DST,SRC0,SRC1,SRC2] used
1046// by the specified component. If an operand is unused
1047// or is not a VGPR, the corresponding value is 0.
1048//
1049// GetRegIdx(Component, MCOperandIdx) must return a VGPR register index
1050// for the specified component and MC operand. The callback must return 0
1051// if the operand is not a register or not a VGPR.
1052InstInfo::RegIndices
1053InstInfo::getRegIndices(unsigned CompIdx,
1054 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
1055 bool VOPD3) const {
1056 assert(CompIdx < COMPONENTS_NUM);
1057
1058 const auto &Comp = CompInfo[CompIdx];
1059 InstInfo::RegIndices RegIndices;
1060
1061 RegIndices[DST] = GetRegIdx(CompIdx, Comp.getIndexOfDstInMCOperands());
1062
1063 for (unsigned CompOprIdx : {SRC0, SRC1, SRC2}) {
1064 unsigned CompSrcIdx = CompOprIdx - DST_NUM;
1065 RegIndices[CompOprIdx] =
1066 Comp.hasRegSrcOperand(CompSrcIdx)
1067 ? GetRegIdx(CompIdx,
1068 Comp.getIndexOfSrcInMCOperands(CompSrcIdx, VOPD3))
1069 : MCRegister();
1070 }
1071 return RegIndices;
1072}
1073
1074} // namespace VOPD
1075
1076VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY) {
1077 return VOPD::InstInfo(OpX, OpY);
1078}
1079
1080VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,
1081 const MCInstrInfo *InstrInfo) {
1082 auto [OpX, OpY] = getVOPDComponents(VOPDOpcode);
1083 const auto &OpXDesc = InstrInfo->get(Opcode: OpX);
1084 const auto &OpYDesc = InstrInfo->get(Opcode: OpY);
1085 bool VOPD3 = SIInstrFlags::isVOPD3(O: *InstrInfo, O: VOPDOpcode);
1086 VOPD::ComponentInfo OpXInfo(OpXDesc, VOPD::ComponentKind::COMPONENT_X, VOPD3);
1087 VOPD::ComponentInfo OpYInfo(OpYDesc, OpXInfo, VOPD3);
1088 return VOPD::InstInfo(OpXInfo, OpYInfo);
1089}
1090
1091TargetID createAMDGPUTargetID(const MCSubtargetInfo &STI,
1092 StringRef FeatureString) {
1093 // In codegen the mode comes from module flags and FeatureString is empty, so
1094 // the processor defaults apply. The assembler has no target directive, so it
1095 // pins the mode via the +xnack/-xnack/+sramecc/-sramecc feature string.
1096 return TargetID::createFromSubtargetFeatures(TT: STI.getTargetTriple(),
1097 CPU: STI.getCPU(), FeatureString);
1098}
1099
1100namespace IsaInfo {
1101
1102unsigned getInstCacheLineSize(const MCSubtargetInfo &STI) {
1103 if (STI.getFeatureBits().test(I: FeatureInstCacheLineSize128))
1104 return 128;
1105 if (STI.getFeatureBits().test(I: FeatureInstCacheLineSize64))
1106 return 64;
1107 return 64;
1108}
1109
1110unsigned getWavefrontSize(const MCSubtargetInfo &STI) {
1111 if (STI.getFeatureBits().test(I: FeatureWavefrontSize16))
1112 return 16;
1113 if (STI.getFeatureBits().test(I: FeatureWavefrontSize32))
1114 return 32;
1115
1116 return 64;
1117}
1118
1119// Maximum LDS a single work-group can address. This is a fixed HW cap. It does
1120// not depend on how many SIMDs a work-group runs on.
1121static unsigned getMaxHWAddressableLocalMemorySize(const MCSubtargetInfo &STI) {
1122 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize32768))
1123 return 32768;
1124 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize65536))
1125 return 65536;
1126 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize163840))
1127 return 163840;
1128 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize196608))
1129 return 196608;
1130 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize327680))
1131 return 327680;
1132 return 32768;
1133}
1134
1135// Total physical size of LDS on the block, in bytes. On targets with
1136// FeatureHalfAddressablePhysicalLocalMemory the physical block is twice the
1137// addressable size (gfx6: 64 KiB physical and 32 KiB addressable;
1138// gfx10/11/12: 128 KiB physical and 64 KiB addressable). On other targets it is
1139// equal to the addressable size.
1140static unsigned getPhysicalLocalMemorySize(const MCSubtargetInfo &STI) {
1141 unsigned Addressable = getMaxHWAddressableLocalMemorySize(STI);
1142 if (STI.getFeatureBits().test(I: FeatureHalfAddressablePhysicalLocalMemory))
1143 return 2 * Addressable;
1144 return Addressable;
1145}
1146
1147// Sizes in use, by generation (addressable / physical block):
1148// gfx6 : 32 KiB addressable, 64 KiB physical block
1149// gfx7 / gfx8 / gfx9: 64 KiB
1150// gfx9.5 (gfx950) : 160 KiB
1151// gfx10 / 11 / 12 : 64 KiB addressable, 128 KiB physical block
1152// gfx12.5 (gfx1250) : 320 KiB (always runs on four SIMDs)
1153// gfx13 : 192 KiB on four SIMDs, 96 KiB on two
1154// Total available in the current mode. The physical size is halved when a
1155// work-group runs on two SIMDs.
1156unsigned getLocalMemorySize(const MCSubtargetInfo &STI) {
1157 unsigned Size = getPhysicalLocalMemorySize(STI);
1158 if (!isFullSIMDMode(STI))
1159 Size /= 2;
1160 return Size;
1161}
1162
1163// What one work-group can allocate in the current mode. This is the HW
1164// addressable cap, but never more than the total available in the current mode.
1165unsigned getAddressableLocalMemorySize(const MCSubtargetInfo &STI) {
1166 return std::min(a: getMaxHWAddressableLocalMemorySize(STI),
1167 b: getLocalMemorySize(STI));
1168}
1169
1170unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo &STI,
1171 unsigned FlatWorkGroupSize) {
1172 assert(FlatWorkGroupSize != 0);
1173 if (!STI.getTargetTriple().isAMDGCN())
1174 return 8;
1175 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1176 unsigned MaxWaves =
1177 getMaxWavesPerEU(AK: Kind) * getNumWorkGroupSIMDs(FullSIMDMode: isFullSIMDMode(STI));
1178 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
1179 if (N == 1) {
1180 // Single-wave workgroups don't consume barrier resources.
1181 return MaxWaves;
1182 }
1183
1184 unsigned MaxBarriers = 16;
1185 if (isGFX10Plus(STI) && !STI.getFeatureBits().test(I: FeatureCuMode))
1186 MaxBarriers = 32;
1187
1188 return std::min(a: MaxWaves / N, b: MaxBarriers);
1189}
1190
1191unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo &STI,
1192 unsigned FlatWorkGroupSize) {
1193 return divideCeil(Numerator: getWavesPerWorkGroup(STI, FlatWorkGroupSize),
1194 Denominator: getNumWorkGroupSIMDs(FullSIMDMode: isFullSIMDMode(STI)));
1195}
1196
1197unsigned getWavesPerWorkGroup(const MCSubtargetInfo &STI,
1198 unsigned FlatWorkGroupSize) {
1199 return divideCeil(Numerator: FlatWorkGroupSize, Denominator: getWavefrontSize(STI));
1200}
1201
1202unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI) { return 8; }
1203
1204// Per-wave SGPRs reserved for the trap handler when enabled.
1205static unsigned getSGPRTrapHandlerReserve(const MCSubtargetInfo &STI) {
1206 return STI.getFeatureBits().test(I: FeatureTrapHandler) ? TRAP_NUM_SGPRS : 0;
1207}
1208
1209// Per-wave SGPR budget (before the addressable clamp): take off the trap
1210// reserve, round down to \p Granule. Shared by getMinNumSGPRs() and
1211// getMaxNumSGPRs(); getOccupancyWithNumSGPRs() is the closed-form algebraic
1212// inverse of this same budget (it does not call this helper), so the two encode
1213// one model.
1214static unsigned getSGPRBudgetPerWave(unsigned TotalNumSGPRs,
1215 unsigned WavesPerEU, unsigned TrapReserve,
1216 unsigned Granule) {
1217 assert(WavesPerEU != 0 && Granule != 0);
1218 unsigned Budget = TotalNumSGPRs / WavesPerEU;
1219 Budget -= std::min(a: Budget, b: TrapReserve);
1220 return alignDown(Value: Budget, Align: Granule);
1221}
1222
1223unsigned getMinNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU) {
1224 assert(WavesPerEU != 0);
1225
1226 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1227 if (Version.Major >= 10)
1228 return 0;
1229
1230 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1231 if (WavesPerEU >= getMaxWavesPerEU(AK: Kind))
1232 return 0;
1233
1234 unsigned MinNumSGPRs =
1235 getSGPRBudgetPerWave(TotalNumSGPRs: getTotalNumSGPRs(AK: Kind), WavesPerEU: WavesPerEU + 1,
1236 TrapReserve: getSGPRTrapHandlerReserve(STI),
1237 Granule: getSGPRAllocGranule(AK: Kind)) +
1238 1;
1239 return std::min(a: MinNumSGPRs, b: getAddressableNumSGPRs(AK: Kind));
1240}
1241
1242unsigned getMaxNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1243 bool Addressable) {
1244 assert(WavesPerEU != 0);
1245
1246 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1247 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(AK: Kind);
1248 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1249 if (Version.Major >= 10)
1250 return Addressable ? AddressableNumSGPRs : 108;
1251 if (Version.Major >= 8 && !Addressable)
1252 AddressableNumSGPRs = 112;
1253 unsigned MaxNumSGPRs = getSGPRBudgetPerWave(
1254 TotalNumSGPRs: getTotalNumSGPRs(AK: Kind), WavesPerEU, TrapReserve: getSGPRTrapHandlerReserve(STI),
1255 Granule: getSGPRAllocGranule(AK: Kind));
1256 return std::min(a: MaxNumSGPRs, b: AddressableNumSGPRs);
1257}
1258
1259bool isSGPROccupancyLimited(const MCSubtargetInfo &STI) {
1260 // From GFX10 on the SGPR file is large enough that SGPRs never limit
1261 // occupancy. Kept as one capability so callers don't each test the version.
1262 return getIsaVersion(GPU: STI.getCPU()).Major < 10;
1263}
1264
1265unsigned getNumExtraSGPRs(const MCSubtargetInfo &STI, bool VCCUsed,
1266 bool FlatScrUsed, bool XNACKUsed) {
1267 unsigned ExtraSGPRs = 0;
1268 if (VCCUsed)
1269 ExtraSGPRs = 2;
1270
1271 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1272 if (Version.Major >= 10)
1273 return ExtraSGPRs;
1274
1275 if (Version.Major < 8) {
1276 if (FlatScrUsed)
1277 ExtraSGPRs = 4;
1278 } else {
1279 if (XNACKUsed)
1280 ExtraSGPRs = 4;
1281
1282 if (FlatScrUsed ||
1283 STI.getFeatureBits().test(I: AMDGPU::FeatureArchitectedFlatScratch))
1284 ExtraSGPRs = 6;
1285 }
1286
1287 return ExtraSGPRs;
1288}
1289
1290static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs,
1291 unsigned Granule) {
1292 return divideCeil(Numerator: std::max(a: 1u, b: NumRegs), Denominator: Granule);
1293}
1294
1295unsigned getNumSGPRBlocks(const MCSubtargetInfo &STI, unsigned NumSGPRs) {
1296 // SGPRBlocks is actual number of SGPR blocks minus 1.
1297 return getGranulatedNumRegisterBlocks(NumRegs: NumSGPRs, Granule: getSGPREncodingGranule(STI)) -
1298 1;
1299}
1300
1301unsigned getVGPRAllocGranule(const MCSubtargetInfo &STI,
1302 unsigned DynamicVGPRBlockSize,
1303 std::optional<bool> EnableWavefrontSize32) {
1304 if (STI.getFeatureBits().test(I: FeatureGFX90AInsts))
1305 return 8;
1306
1307 if (DynamicVGPRBlockSize != 0)
1308 return DynamicVGPRBlockSize;
1309
1310 bool IsWave32 = EnableWavefrontSize32
1311 ? *EnableWavefrontSize32
1312 : STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1313
1314 if (STI.getFeatureBits().test(I: Feature1536VGPRs))
1315 return IsWave32 ? 24 : 12;
1316
1317 if (hasGFX10_3Insts(STI))
1318 return IsWave32 ? 16 : 8;
1319
1320 return IsWave32 ? 8 : 4;
1321}
1322
1323unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI,
1324 std::optional<bool> EnableWavefrontSize32) {
1325 if (STI.getFeatureBits().test(I: FeatureGFX90AInsts))
1326 return 8;
1327
1328 bool IsWave32 = EnableWavefrontSize32
1329 ? *EnableWavefrontSize32
1330 : STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1331
1332 if (STI.getFeatureBits().test(I: Feature1024AddressableVGPRs))
1333 return IsWave32 ? 16 : 8;
1334
1335 return IsWave32 ? 8 : 4;
1336}
1337
1338unsigned getArchVGPRAllocGranule() { return 4; }
1339
1340unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo &STI) {
1341 const auto &Features = STI.getFeatureBits();
1342 if (Features.test(I: Feature1024AddressableVGPRs))
1343 return Features.test(I: FeatureWavefrontSize32) ? 1024 : 512;
1344 return 256;
1345}
1346
1347unsigned getAddressableNumVGPRs(const MCSubtargetInfo &STI,
1348 unsigned DynamicVGPRBlockSize) {
1349 const auto &Features = STI.getFeatureBits();
1350 if (Features.test(I: FeatureGFX90AInsts))
1351 return 512;
1352
1353 if (DynamicVGPRBlockSize != 0) {
1354 // On GFX12 we can allocate at most MaxDynamicVGPRBlocks blocks of VGPRs.
1355 return MaxDynamicVGPRBlocks *
1356 getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1357 }
1358 return getAddressableNumArchVGPRs(STI);
1359}
1360
1361unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo &STI,
1362 unsigned NumVGPRs,
1363 unsigned DynamicVGPRBlockSize) {
1364 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1365 bool IsWave32 = STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1366 return getNumWavesPerEUWithNumVGPRs(
1367 NumVGPRs, Granule: getVGPRAllocGranule(STI, DynamicVGPRBlockSize),
1368 MaxWaves: getMaxWavesPerEU(AK: Kind), TotalNumVGPRs: AMDGPU::getTotalNumVGPRs(AK: Kind, IsWave32));
1369}
1370
1371unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
1372 unsigned MaxWaves,
1373 unsigned TotalNumVGPRs) {
1374 if (NumVGPRs < Granule)
1375 return MaxWaves;
1376 unsigned RoundedRegs = alignTo(Value: NumVGPRs, Align: Granule);
1377 return std::min(a: std::max(a: TotalNumVGPRs / RoundedRegs, b: 1u), b: MaxWaves);
1378}
1379
1380unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
1381 unsigned TotalNumSGPRs, unsigned Granule,
1382 unsigned TrapReserve) {
1383 // Closed-form inverse of getMaxNumSGPRs(): the budget condition
1384 // SGPRs <= alignDown(TotalNumSGPRs / W - TrapReserve, Granule)
1385 // solves to W <= TotalNumSGPRs / (alignTo(SGPRs, Granule) + TrapReserve).
1386 unsigned PerWave = alignTo(Value: SGPRs, Align: Granule) + TrapReserve;
1387 return PerWave ? std::clamp(val: TotalNumSGPRs / PerWave, lo: 1u, hi: MaxWaves) : MaxWaves;
1388}
1389
1390unsigned getOccupancyWithNumSGPRs(const MCSubtargetInfo &STI, unsigned SGPRs) {
1391 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1392 unsigned MaxWaves = getMaxWavesPerEU(AK: Kind);
1393
1394 if (!isSGPROccupancyLimited(STI))
1395 return MaxWaves;
1396
1397 return getOccupancyWithNumSGPRs(SGPRs, MaxWaves, TotalNumSGPRs: getTotalNumSGPRs(AK: Kind),
1398 Granule: getSGPRAllocGranule(AK: Kind),
1399 TrapReserve: getSGPRTrapHandlerReserve(STI));
1400}
1401
1402unsigned getMinNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1403 unsigned DynamicVGPRBlockSize) {
1404 assert(WavesPerEU != 0);
1405
1406 // In dynamic VGPR mode, (static) occupancy does not depend on VGPR usage,
1407 // so getMaxNumVGPRs does not depend on WavesPerEU, and thus we need to return
1408 // zero because there is no nonzero VGPR usage N where going below N
1409 // achieves higher (static) occupancy.
1410 bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
1411 if (DynamicVGPREnabled)
1412 return 0;
1413
1414 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1415 unsigned MaxWavesPerEU = getMaxWavesPerEU(AK: Kind);
1416 if (WavesPerEU >= MaxWavesPerEU)
1417 return 0;
1418
1419 unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
1420 AK: Kind, IsWave32: STI.getFeatureBits().test(I: FeatureWavefrontSize32));
1421 unsigned AddrsableNumVGPRs =
1422 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1423 unsigned Granule = getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1424 unsigned MaxNumVGPRs = alignDown(Value: TotNumVGPRs / WavesPerEU, Align: Granule);
1425
1426 if (MaxNumVGPRs == alignDown(Value: TotNumVGPRs / MaxWavesPerEU, Align: Granule))
1427 return 0;
1428
1429 unsigned MinWavesPerEU = getNumWavesPerEUWithNumVGPRs(STI, NumVGPRs: AddrsableNumVGPRs,
1430 DynamicVGPRBlockSize);
1431 if (WavesPerEU < MinWavesPerEU)
1432 return getMinNumVGPRs(STI, WavesPerEU: MinWavesPerEU, DynamicVGPRBlockSize);
1433
1434 unsigned MaxNumVGPRsNext = alignDown(Value: TotNumVGPRs / (WavesPerEU + 1), Align: Granule);
1435 unsigned MinNumVGPRs = 1 + std::min(a: MaxNumVGPRs - Granule, b: MaxNumVGPRsNext);
1436 return std::min(a: MinNumVGPRs, b: AddrsableNumVGPRs);
1437}
1438
1439unsigned getMaxNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1440 unsigned DynamicVGPRBlockSize) {
1441 assert(WavesPerEU != 0);
1442
1443 unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
1444 AK: parseArchAMDGCN(CPU: STI.getCPU()),
1445 IsWave32: STI.getFeatureBits().test(I: FeatureWavefrontSize32));
1446
1447 // In dynamic VGPR mode, WavesPerEU does not imply a VGPR limit.
1448 bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
1449 unsigned MaxNumVGPRs =
1450 DynamicVGPREnabled
1451 ? TotNumVGPRs
1452 : alignDown(Value: TotNumVGPRs / WavesPerEU,
1453 Align: getVGPRAllocGranule(STI, DynamicVGPRBlockSize));
1454 unsigned AddressableNumVGPRs =
1455 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1456 return std::min(a: MaxNumVGPRs, b: AddressableNumVGPRs);
1457}
1458
1459unsigned getEncodedNumVGPRBlocks(const MCSubtargetInfo &STI, unsigned NumVGPRs,
1460 std::optional<bool> EnableWavefrontSize32) {
1461 return getGranulatedNumRegisterBlocks(
1462 NumRegs: NumVGPRs, Granule: getVGPREncodingGranule(STI, EnableWavefrontSize32)) -
1463 1;
1464}
1465
1466unsigned getAllocatedNumVGPRBlocks(const MCSubtargetInfo &STI,
1467 unsigned NumVGPRs,
1468 unsigned DynamicVGPRBlockSize,
1469 std::optional<bool> EnableWavefrontSize32) {
1470 return getGranulatedNumRegisterBlocks(
1471 NumRegs: NumVGPRs,
1472 Granule: getVGPRAllocGranule(STI, DynamicVGPRBlockSize, EnableWavefrontSize32));
1473}
1474} // end namespace IsaInfo
1475
1476void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &KernelCode,
1477 const MCSubtargetInfo &STI) {
1478 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1479 KernelCode.amd_kernel_code_version_major = 1;
1480 KernelCode.amd_kernel_code_version_minor = 2;
1481 KernelCode.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
1482 KernelCode.amd_machine_version_major = Version.Major;
1483 KernelCode.amd_machine_version_minor = Version.Minor;
1484 KernelCode.amd_machine_version_stepping = Version.Stepping;
1485 KernelCode.kernel_code_entry_byte_offset = sizeof(amd_kernel_code_t);
1486 if (STI.getFeatureBits().test(I: FeatureWavefrontSize32)) {
1487 KernelCode.wavefront_size = 5;
1488 KernelCode.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
1489 } else {
1490 KernelCode.wavefront_size = 6;
1491 }
1492
1493 // If the code object does not support indirect functions, then the value must
1494 // be 0xffffffff.
1495 KernelCode.call_convention = -1;
1496
1497 // These alignment values are specified in powers of two, so alignment =
1498 // 2^n. The minimum alignment is 2^4 = 16.
1499 KernelCode.kernarg_segment_alignment = 4;
1500 KernelCode.group_segment_alignment = 4;
1501 KernelCode.private_segment_alignment = 4;
1502
1503 if (Version.Major >= 10) {
1504 KernelCode.compute_pgm_resource_registers |=
1505 S_00B848_WGP_MODE(STI.getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
1506 S_00B848_MEM_ORDERED(1) | S_00B848_FWD_PROGRESS(1);
1507 }
1508}
1509
1510bool isGroupSegment(const GlobalValue *GV) {
1511 return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
1512}
1513
1514bool isGlobalSegment(const GlobalValue *GV) {
1515 return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;
1516}
1517
1518bool isReadOnlySegment(const GlobalValue *GV) {
1519 unsigned AS = GV->getAddressSpace();
1520 return AS == AMDGPUAS::CONSTANT_ADDRESS ||
1521 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
1522}
1523
1524bool shouldEmitConstantsToTextSection(const Triple &TT) {
1525 return TT.getArch() == Triple::r600;
1526}
1527
1528static bool isValidRegPrefix(char C) {
1529 return C == 'v' || C == 's' || C == 'a';
1530}
1531
1532std::tuple<char, unsigned, unsigned> parseAsmPhysRegName(StringRef RegName) {
1533 char Kind = RegName.front();
1534 if (!isValidRegPrefix(C: Kind))
1535 return {};
1536
1537 RegName = RegName.drop_front();
1538 if (RegName.consume_front(Prefix: "[")) {
1539 unsigned Idx, End;
1540 bool Failed = RegName.consumeInteger(Radix: 10, Result&: Idx);
1541 Failed |= !RegName.consume_front(Prefix: ":");
1542 Failed |= RegName.consumeInteger(Radix: 10, Result&: End);
1543 Failed |= !RegName.consume_back(Suffix: "]");
1544 if (!Failed) {
1545 unsigned NumRegs = End - Idx + 1;
1546 if (NumRegs > 1)
1547 return {Kind, Idx, NumRegs};
1548 }
1549 } else {
1550 unsigned Idx;
1551 bool Failed = RegName.getAsInteger(Radix: 10, Result&: Idx);
1552 if (!Failed)
1553 return {Kind, Idx, 1};
1554 }
1555
1556 return {};
1557}
1558
1559std::tuple<char, unsigned, unsigned>
1560parseAsmConstraintPhysReg(StringRef Constraint) {
1561 StringRef RegName = Constraint;
1562 if (!RegName.consume_front(Prefix: "{") || !RegName.consume_back(Suffix: "}"))
1563 return {};
1564 return parseAsmPhysRegName(RegName);
1565}
1566
1567std::pair<unsigned, unsigned>
1568getIntegerPairAttribute(const Function &F, StringRef Name,
1569 std::pair<unsigned, unsigned> Default,
1570 bool OnlyFirstRequired) {
1571 if (auto Attr = getIntegerPairAttribute(F, Name, OnlyFirstRequired))
1572 return {Attr->first, Attr->second.value_or(u&: Default.second)};
1573 return Default;
1574}
1575
1576std::optional<std::pair<unsigned, std::optional<unsigned>>>
1577getIntegerPairAttribute(const Function &F, StringRef Name,
1578 bool OnlyFirstRequired) {
1579 Attribute A = F.getFnAttribute(Kind: Name);
1580 if (!A.isStringAttribute())
1581 return std::nullopt;
1582
1583 LLVMContext &Ctx = F.getContext();
1584 std::pair<unsigned, std::optional<unsigned>> Ints;
1585 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(Separator: ',');
1586 if (Strs.first.trim().getAsInteger(Radix: 0, Result&: Ints.first)) {
1587 Ctx.emitError(ErrorStr: "can't parse first integer attribute " + Name);
1588 return std::nullopt;
1589 }
1590 unsigned Second = 0;
1591 if (Strs.second.trim().getAsInteger(Radix: 0, Result&: Second)) {
1592 if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
1593 Ctx.emitError(ErrorStr: "can't parse second integer attribute " + Name);
1594 return std::nullopt;
1595 }
1596 } else {
1597 Ints.second = Second;
1598 }
1599
1600 return Ints;
1601}
1602
1603SmallVector<unsigned> getIntegerVecAttribute(const Function &F, StringRef Name,
1604 unsigned Size,
1605 unsigned DefaultVal) {
1606 std::optional<SmallVector<unsigned>> R =
1607 getIntegerVecAttribute(F, Name, Size);
1608 return R.has_value() ? *R : SmallVector<unsigned>(Size, DefaultVal);
1609}
1610
1611std::optional<SmallVector<unsigned>>
1612getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) {
1613 assert(Size > 2);
1614 LLVMContext &Ctx = F.getContext();
1615
1616 Attribute A = F.getFnAttribute(Kind: Name);
1617 if (!A.isValid())
1618 return std::nullopt;
1619 if (!A.isStringAttribute()) {
1620 Ctx.emitError(ErrorStr: Name + " is not a string attribute");
1621 return std::nullopt;
1622 }
1623
1624 SmallVector<unsigned> Vals(Size);
1625
1626 StringRef S = A.getValueAsString();
1627 unsigned i = 0;
1628 for (; !S.empty() && i < Size; i++) {
1629 std::pair<StringRef, StringRef> Strs = S.split(Separator: ',');
1630 unsigned IntVal;
1631 if (Strs.first.trim().getAsInteger(Radix: 0, Result&: IntVal)) {
1632 Ctx.emitError(ErrorStr: "can't parse integer attribute " + Strs.first + " in " +
1633 Name);
1634 return std::nullopt;
1635 }
1636 Vals[i] = IntVal;
1637 S = Strs.second;
1638 }
1639
1640 if (!S.empty() || i < Size) {
1641 Ctx.emitError(ErrorStr: "attribute " + Name +
1642 " has incorrect number of integers; expected " +
1643 llvm::utostr(X: Size));
1644 return std::nullopt;
1645 }
1646 return Vals;
1647}
1648
1649SmallVector<unsigned> getMaxNumWorkGroups(const Function &F) {
1650 return getIntegerVecAttribute(F, Name: "amdgpu-max-num-workgroups", Size: 3,
1651 DefaultVal: std::numeric_limits<uint32_t>::max());
1652}
1653
1654bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) {
1655 assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!");
1656 for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) {
1657 auto Low =
1658 mdconst::extract<ConstantInt>(MD: MD.getOperand(I: 2 * I + 0))->getValue();
1659 auto High =
1660 mdconst::extract<ConstantInt>(MD: MD.getOperand(I: 2 * I + 1))->getValue();
1661 // There are two types of [A; B) ranges:
1662 // A < B, e.g. [4; 5) which is a range that only includes 4.
1663 // A > B, e.g. [5; 4) which is a range that wraps around and includes
1664 // everything except 4.
1665 if (Low.ult(RHS: High)) {
1666 if (Low.ule(RHS: Val) && High.ugt(RHS: Val))
1667 return true;
1668 } else {
1669 if (Low.uge(RHS: Val) && High.ult(RHS: Val))
1670 return true;
1671 }
1672 }
1673
1674 return false;
1675}
1676
1677unsigned getVmcntBitMask(const IsaVersion &Version) {
1678 return (1 << (getVmcntBitWidthLo(VersionMajor: Version.Major) +
1679 getVmcntBitWidthHi(VersionMajor: Version.Major))) -
1680 1;
1681}
1682
1683unsigned getLoadcntBitMask(const IsaVersion &Version) {
1684 return (1 << getLoadcntBitWidth(VersionMajor: Version.Major)) - 1;
1685}
1686
1687unsigned getSamplecntBitMask(const IsaVersion &Version) {
1688 return (1 << getSamplecntBitWidth(VersionMajor: Version.Major)) - 1;
1689}
1690
1691unsigned getBvhcntBitMask(const IsaVersion &Version) {
1692 return (1 << getBvhcntBitWidth(VersionMajor: Version.Major)) - 1;
1693}
1694
1695unsigned getExpcntBitMask(const IsaVersion &Version) {
1696 return (1 << getExpcntBitWidth(VersionMajor: Version.Major)) - 1;
1697}
1698
1699unsigned getLgkmcntBitMask(const IsaVersion &Version) {
1700 return (1 << getLgkmcntBitWidth(VersionMajor: Version.Major)) - 1;
1701}
1702
1703unsigned getDscntBitMask(const IsaVersion &Version) {
1704 return (1 << getDscntBitWidth(VersionMajor: Version.Major)) - 1;
1705}
1706
1707unsigned getKmcntBitMask(const IsaVersion &Version) {
1708 return (1 << getKmcntBitWidth(VersionMajor: Version.Major)) - 1;
1709}
1710
1711unsigned getXcntBitMask(const IsaVersion &Version) {
1712 return (1 << getXcntBitWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1713}
1714
1715unsigned getAsynccntBitMask(const IsaVersion &Version) {
1716 return (1 << getAsynccntBitWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1717}
1718
1719unsigned getStorecntBitMask(const IsaVersion &Version) {
1720 return (1 << getStorecntBitWidth(VersionMajor: Version.Major)) - 1;
1721}
1722
1723unsigned getWaitcntBitMask(const IsaVersion &Version) {
1724 unsigned VmcntLo = getBitMask(Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1725 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1726 unsigned Expcnt = getBitMask(Shift: getExpcntBitShift(VersionMajor: Version.Major),
1727 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1728 unsigned Lgkmcnt = getBitMask(Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1729 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1730 unsigned VmcntHi = getBitMask(Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1731 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1732 return VmcntLo | Expcnt | Lgkmcnt | VmcntHi;
1733}
1734
1735unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1736 unsigned VmcntLo = unpackBits(Src: Waitcnt, Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1737 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1738 unsigned VmcntHi = unpackBits(Src: Waitcnt, Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1739 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1740 return VmcntLo | VmcntHi << getVmcntBitWidthLo(VersionMajor: Version.Major);
1741}
1742
1743unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
1744 return unpackBits(Src: Waitcnt, Shift: getExpcntBitShift(VersionMajor: Version.Major),
1745 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1746}
1747
1748unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1749 return unpackBits(Src: Waitcnt, Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1750 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1751}
1752
1753unsigned decodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt) {
1754 return unpackBits(Src: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1755 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1756}
1757
1758unsigned decodeStorecnt(const IsaVersion &Version, unsigned Waitcnt) {
1759 return unpackBits(Src: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1760 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1761}
1762
1763unsigned decodeDscnt(const IsaVersion &Version, unsigned Waitcnt) {
1764 return unpackBits(Src: Waitcnt, Shift: getDscntBitShift(VersionMajor: Version.Major),
1765 Width: getDscntBitWidth(VersionMajor: Version.Major));
1766}
1767
1768void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt,
1769 unsigned &Expcnt, unsigned &Lgkmcnt) {
1770 Vmcnt = decodeVmcnt(Version, Waitcnt);
1771 Expcnt = decodeExpcnt(Version, Waitcnt);
1772 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
1773}
1774
1775unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
1776 unsigned Vmcnt) {
1777 Waitcnt = packBits(Src: Vmcnt, Dst: Waitcnt, Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1778 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1779 return packBits(Src: Vmcnt >> getVmcntBitWidthLo(VersionMajor: Version.Major), Dst: Waitcnt,
1780 Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1781 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1782}
1783
1784unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
1785 unsigned Expcnt) {
1786 return packBits(Src: Expcnt, Dst: Waitcnt, Shift: getExpcntBitShift(VersionMajor: Version.Major),
1787 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1788}
1789
1790unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
1791 unsigned Lgkmcnt) {
1792 return packBits(Src: Lgkmcnt, Dst: Waitcnt, Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1793 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1794}
1795
1796unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt,
1797 unsigned Expcnt, unsigned Lgkmcnt) {
1798 unsigned Waitcnt = getWaitcntBitMask(Version);
1799 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
1800 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
1801 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
1802 return Waitcnt;
1803}
1804
1805static unsigned getCombinedCountBitMask(const IsaVersion &Version,
1806 bool IsStore) {
1807 unsigned Dscnt = getBitMask(Shift: getDscntBitShift(VersionMajor: Version.Major),
1808 Width: getDscntBitWidth(VersionMajor: Version.Major));
1809 if (IsStore) {
1810 unsigned Storecnt = getBitMask(Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1811 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1812 return Dscnt | Storecnt;
1813 }
1814 unsigned Loadcnt = getBitMask(Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1815 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1816 return Dscnt | Loadcnt;
1817}
1818
1819static unsigned encodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt,
1820 unsigned Loadcnt) {
1821 return packBits(Src: Loadcnt, Dst: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1822 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1823}
1824
1825static unsigned encodeStorecnt(const IsaVersion &Version, unsigned Waitcnt,
1826 unsigned Storecnt) {
1827 return packBits(Src: Storecnt, Dst: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1828 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1829}
1830
1831static unsigned encodeDscnt(const IsaVersion &Version, unsigned Waitcnt,
1832 unsigned Dscnt) {
1833 return packBits(Src: Dscnt, Dst: Waitcnt, Shift: getDscntBitShift(VersionMajor: Version.Major),
1834 Width: getDscntBitWidth(VersionMajor: Version.Major));
1835}
1836
1837unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt,
1838 unsigned Dscnt) {
1839 unsigned Waitcnt = getCombinedCountBitMask(Version, IsStore: false);
1840 Waitcnt = encodeLoadcnt(Version, Waitcnt, Loadcnt);
1841 Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
1842 return Waitcnt;
1843}
1844
1845unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt,
1846 unsigned Dscnt) {
1847 unsigned Waitcnt = getCombinedCountBitMask(Version, IsStore: true);
1848 Waitcnt = encodeStorecnt(Version, Waitcnt, Storecnt);
1849 Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
1850 return Waitcnt;
1851}
1852
1853//===----------------------------------------------------------------------===//
1854// Custom Operand Values
1855//===----------------------------------------------------------------------===//
1856
1857static unsigned getDefaultCustomOperandEncoding(const CustomOperandVal *Opr,
1858 int Size,
1859 const MCSubtargetInfo &STI) {
1860 unsigned Enc = 0;
1861 for (int Idx = 0; Idx < Size; ++Idx) {
1862 const auto &Op = Opr[Idx];
1863 if (Op.isSupported(STI))
1864 Enc |= Op.encode(Val: Op.Default);
1865 }
1866 return Enc;
1867}
1868
1869static bool isSymbolicCustomOperandEncoding(const CustomOperandVal *Opr,
1870 int Size, unsigned Code,
1871 bool &HasNonDefaultVal,
1872 const MCSubtargetInfo &STI) {
1873 unsigned UsedOprMask = 0;
1874 HasNonDefaultVal = false;
1875 for (int Idx = 0; Idx < Size; ++Idx) {
1876 const auto &Op = Opr[Idx];
1877 if (!Op.isSupported(STI))
1878 continue;
1879 UsedOprMask |= Op.getMask();
1880 unsigned Val = Op.decode(Code);
1881 if (!Op.isValid(Val))
1882 return false;
1883 HasNonDefaultVal |= (Val != Op.Default);
1884 }
1885 return (Code & ~UsedOprMask) == 0;
1886}
1887
1888static bool decodeCustomOperand(const CustomOperandVal *Opr, int Size,
1889 unsigned Code, int &Idx, StringRef &Name,
1890 unsigned &Val, bool &IsDefault,
1891 const MCSubtargetInfo &STI) {
1892 while (Idx < Size) {
1893 const auto &Op = Opr[Idx++];
1894 if (Op.isSupported(STI)) {
1895 Name = Op.Name;
1896 Val = Op.decode(Code);
1897 IsDefault = (Val == Op.Default);
1898 return true;
1899 }
1900 }
1901
1902 return false;
1903}
1904
1905static int encodeCustomOperandVal(const CustomOperandVal &Op,
1906 int64_t InputVal) {
1907 if (InputVal < 0 || InputVal > Op.Max)
1908 return OPR_VAL_INVALID;
1909 return Op.encode(Val: InputVal);
1910}
1911
1912static int encodeCustomOperand(const CustomOperandVal *Opr, int Size,
1913 const StringRef Name, int64_t InputVal,
1914 unsigned &UsedOprMask,
1915 const MCSubtargetInfo &STI) {
1916 int InvalidId = OPR_ID_UNKNOWN;
1917 for (int Idx = 0; Idx < Size; ++Idx) {
1918 const auto &Op = Opr[Idx];
1919 if (Op.Name == Name) {
1920 if (!Op.isSupported(STI)) {
1921 InvalidId = OPR_ID_UNSUPPORTED;
1922 continue;
1923 }
1924 auto OprMask = Op.getMask();
1925 if (OprMask & UsedOprMask)
1926 return OPR_ID_DUPLICATE;
1927 UsedOprMask |= OprMask;
1928 return encodeCustomOperandVal(Op, InputVal);
1929 }
1930 }
1931 return InvalidId;
1932}
1933
1934//===----------------------------------------------------------------------===//
1935// DepCtr
1936//===----------------------------------------------------------------------===//
1937
1938namespace DepCtr {
1939
1940int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI) {
1941 static int Default = -1;
1942 if (Default == -1)
1943 Default = getDefaultCustomOperandEncoding(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, STI);
1944 return Default;
1945}
1946
1947bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal,
1948 const MCSubtargetInfo &STI) {
1949 return isSymbolicCustomOperandEncoding(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Code,
1950 HasNonDefaultVal, STI);
1951}
1952
1953bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val,
1954 bool &IsDefault, const MCSubtargetInfo &STI) {
1955 return decodeCustomOperand(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Code, Idx&: Id, Name, Val,
1956 IsDefault, STI);
1957}
1958
1959int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask,
1960 const MCSubtargetInfo &STI) {
1961 return encodeCustomOperand(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Name, InputVal: Val, UsedOprMask,
1962 STI);
1963}
1964
1965unsigned getVaVdstBitMask() { return (1 << getVaVdstBitWidth()) - 1; }
1966
1967unsigned getVaSdstBitMask() { return (1 << getVaSdstBitWidth()) - 1; }
1968
1969unsigned getVaSsrcBitMask() { return (1 << getVaSsrcBitWidth()) - 1; }
1970
1971unsigned getHoldCntBitMask(const IsaVersion &Version) {
1972 return (1 << getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1973}
1974
1975unsigned getVmVsrcBitMask() { return (1 << getVmVsrcBitWidth()) - 1; }
1976
1977unsigned getVaVccBitMask() { return (1 << getVaVccBitWidth()) - 1; }
1978
1979unsigned getSaSdstBitMask() { return (1 << getSaSdstBitWidth()) - 1; }
1980
1981unsigned decodeFieldVmVsrc(unsigned Encoded) {
1982 return unpackBits(Src: Encoded, Shift: getVmVsrcBitShift(), Width: getVmVsrcBitWidth());
1983}
1984
1985unsigned decodeFieldVaVdst(unsigned Encoded) {
1986 return unpackBits(Src: Encoded, Shift: getVaVdstBitShift(), Width: getVaVdstBitWidth());
1987}
1988
1989unsigned decodeFieldSaSdst(unsigned Encoded) {
1990 return unpackBits(Src: Encoded, Shift: getSaSdstBitShift(), Width: getSaSdstBitWidth());
1991}
1992
1993unsigned decodeFieldVaSdst(unsigned Encoded) {
1994 return unpackBits(Src: Encoded, Shift: getVaSdstBitShift(), Width: getVaSdstBitWidth());
1995}
1996
1997unsigned decodeFieldVaVcc(unsigned Encoded) {
1998 return unpackBits(Src: Encoded, Shift: getVaVccBitShift(), Width: getVaVccBitWidth());
1999}
2000
2001unsigned decodeFieldVaSsrc(unsigned Encoded) {
2002 return unpackBits(Src: Encoded, Shift: getVaSsrcBitShift(), Width: getVaSsrcBitWidth());
2003}
2004
2005unsigned decodeFieldHoldCnt(unsigned Encoded, const IsaVersion &Version) {
2006 return unpackBits(Src: Encoded, Shift: getHoldCntBitShift(),
2007 Width: getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor));
2008}
2009
2010unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc) {
2011 return packBits(Src: VmVsrc, Dst: Encoded, Shift: getVmVsrcBitShift(), Width: getVmVsrcBitWidth());
2012}
2013
2014unsigned encodeFieldVmVsrc(unsigned VmVsrc, const MCSubtargetInfo &STI) {
2015 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2016 return encodeFieldVmVsrc(Encoded, VmVsrc);
2017}
2018
2019unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst) {
2020 return packBits(Src: VaVdst, Dst: Encoded, Shift: getVaVdstBitShift(), Width: getVaVdstBitWidth());
2021}
2022
2023unsigned encodeFieldVaVdst(unsigned VaVdst, const MCSubtargetInfo &STI) {
2024 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2025 return encodeFieldVaVdst(Encoded, VaVdst);
2026}
2027
2028unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst) {
2029 return packBits(Src: SaSdst, Dst: Encoded, Shift: getSaSdstBitShift(), Width: getSaSdstBitWidth());
2030}
2031
2032unsigned encodeFieldSaSdst(unsigned SaSdst, const MCSubtargetInfo &STI) {
2033 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2034 return encodeFieldSaSdst(Encoded, SaSdst);
2035}
2036
2037unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst) {
2038 return packBits(Src: VaSdst, Dst: Encoded, Shift: getVaSdstBitShift(), Width: getVaSdstBitWidth());
2039}
2040
2041unsigned encodeFieldVaSdst(unsigned VaSdst, const MCSubtargetInfo &STI) {
2042 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2043 return encodeFieldVaSdst(Encoded, VaSdst);
2044}
2045
2046unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc) {
2047 return packBits(Src: VaVcc, Dst: Encoded, Shift: getVaVccBitShift(), Width: getVaVccBitWidth());
2048}
2049
2050unsigned encodeFieldVaVcc(unsigned VaVcc, const MCSubtargetInfo &STI) {
2051 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2052 return encodeFieldVaVcc(Encoded, VaVcc);
2053}
2054
2055unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc) {
2056 return packBits(Src: VaSsrc, Dst: Encoded, Shift: getVaSsrcBitShift(), Width: getVaSsrcBitWidth());
2057}
2058
2059unsigned encodeFieldVaSsrc(unsigned VaSsrc, const MCSubtargetInfo &STI) {
2060 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2061 return encodeFieldVaSsrc(Encoded, VaSsrc);
2062}
2063
2064unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt,
2065 const IsaVersion &Version) {
2066 return packBits(Src: HoldCnt, Dst: Encoded, Shift: getHoldCntBitShift(),
2067 Width: getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor));
2068}
2069
2070unsigned encodeFieldHoldCnt(unsigned HoldCnt, const MCSubtargetInfo &STI) {
2071 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2072 return encodeFieldHoldCnt(Encoded, HoldCnt, Version: getIsaVersion(GPU: STI.getCPU()));
2073}
2074
2075} // namespace DepCtr
2076
2077//===----------------------------------------------------------------------===//
2078// exp tgt
2079//===----------------------------------------------------------------------===//
2080
2081namespace Exp {
2082
2083struct ExpTgt {
2084 StringLiteral Name;
2085 unsigned Tgt;
2086 unsigned MaxIndex;
2087};
2088
2089// clang-format off
2090static constexpr ExpTgt ExpTgtInfo[] = {
2091 {.Name: {"null"}, .Tgt: ET_NULL, .MaxIndex: ET_NULL_MAX_IDX},
2092 {.Name: {"mrtz"}, .Tgt: ET_MRTZ, .MaxIndex: ET_MRTZ_MAX_IDX},
2093 {.Name: {"prim"}, .Tgt: ET_PRIM, .MaxIndex: ET_PRIM_MAX_IDX},
2094 {.Name: {"mrt"}, .Tgt: ET_MRT0, .MaxIndex: ET_MRT_MAX_IDX},
2095 {.Name: {"pos"}, .Tgt: ET_POS0, .MaxIndex: ET_POS_MAX_IDX},
2096 {.Name: {"dual_src_blend"},.Tgt: ET_DUAL_SRC_BLEND0, .MaxIndex: ET_DUAL_SRC_BLEND_MAX_IDX},
2097 {.Name: {"param"}, .Tgt: ET_PARAM0, .MaxIndex: ET_PARAM_MAX_IDX},
2098};
2099// clang-format on
2100
2101bool getTgtName(unsigned Id, StringRef &Name, int &Index) {
2102 for (const ExpTgt &Val : ExpTgtInfo) {
2103 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) {
2104 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt);
2105 Name = Val.Name;
2106 return true;
2107 }
2108 }
2109 return false;
2110}
2111
2112unsigned getTgtId(const StringRef Name) {
2113
2114 for (const ExpTgt &Val : ExpTgtInfo) {
2115 if (Val.MaxIndex == 0 && Name == Val.Name)
2116 return Val.Tgt;
2117
2118 if (Val.MaxIndex > 0 && Name.starts_with(Prefix: Val.Name)) {
2119 StringRef Suffix = Name.drop_front(N: Val.Name.size());
2120
2121 unsigned Id;
2122 if (Suffix.getAsInteger(Radix: 10, Result&: Id) || Id > Val.MaxIndex)
2123 return ET_INVALID;
2124
2125 // Disable leading zeroes
2126 if (Suffix.size() > 1 && Suffix[0] == '0')
2127 return ET_INVALID;
2128
2129 return Val.Tgt + Id;
2130 }
2131 }
2132 return ET_INVALID;
2133}
2134
2135bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) {
2136 switch (Id) {
2137 case ET_NULL:
2138 return !isGFX11Plus(STI);
2139 case ET_POS4:
2140 case ET_PRIM:
2141 return isGFX10Plus(STI);
2142 case ET_DUAL_SRC_BLEND0:
2143 case ET_DUAL_SRC_BLEND1:
2144 return isGFX11Plus(STI);
2145 default:
2146 if (Id >= ET_PARAM0 && Id <= ET_PARAM31)
2147 return !isGFX11Plus(STI) || isGFX13Plus(STI);
2148 return true;
2149 }
2150}
2151
2152} // namespace Exp
2153
2154//===----------------------------------------------------------------------===//
2155// MTBUF Format
2156//===----------------------------------------------------------------------===//
2157
2158namespace MTBUFFormat {
2159
2160int64_t getDfmt(const StringRef Name) {
2161 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
2162 if (Name == DfmtSymbolic[Id])
2163 return Id;
2164 }
2165 return DFMT_UNDEF;
2166}
2167
2168StringRef getDfmtName(unsigned Id) {
2169 assert(Id <= DFMT_MAX);
2170 return DfmtSymbolic[Id];
2171}
2172
2173static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {
2174 if (isSI(STI) || isCI(STI))
2175 return NfmtSymbolicSICI;
2176 if (isVI(STI) || isGFX9(STI))
2177 return NfmtSymbolicVI;
2178 return NfmtSymbolicGFX10;
2179}
2180
2181int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
2182 const auto *lookupTable = getNfmtLookupTable(STI);
2183 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
2184 if (Name == lookupTable[Id])
2185 return Id;
2186 }
2187 return NFMT_UNDEF;
2188}
2189
2190StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
2191 assert(Id <= NFMT_MAX);
2192 return getNfmtLookupTable(STI)[Id];
2193}
2194
2195bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2196 unsigned Dfmt;
2197 unsigned Nfmt;
2198 decodeDfmtNfmt(Format: Id, Dfmt, Nfmt);
2199 return isValidNfmt(Val: Nfmt, STI);
2200}
2201
2202bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2203 return !getNfmtName(Id, STI).empty();
2204}
2205
2206int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
2207 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);
2208}
2209
2210void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
2211 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;
2212 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
2213}
2214
2215int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI) {
2216 if (isGFX11Plus(STI)) {
2217 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2218 if (Name == UfmtSymbolicGFX11[Id])
2219 return Id;
2220 }
2221 } else {
2222 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2223 if (Name == UfmtSymbolicGFX10[Id])
2224 return Id;
2225 }
2226 }
2227 return UFMT_UNDEF;
2228}
2229
2230StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI) {
2231 if (isValidUnifiedFormat(Val: Id, STI))
2232 return isGFX10(STI) ? UfmtSymbolicGFX10[Id] : UfmtSymbolicGFX11[Id];
2233 return "";
2234}
2235
2236bool isValidUnifiedFormat(unsigned Id, const MCSubtargetInfo &STI) {
2237 return isGFX10(STI) ? Id <= UfmtGFX10::UFMT_LAST : Id <= UfmtGFX11::UFMT_LAST;
2238}
2239
2240int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt,
2241 const MCSubtargetInfo &STI) {
2242 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
2243 if (isGFX11Plus(STI)) {
2244 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2245 if (Fmt == DfmtNfmt2UFmtGFX11[Id])
2246 return Id;
2247 }
2248 } else {
2249 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2250 if (Fmt == DfmtNfmt2UFmtGFX10[Id])
2251 return Id;
2252 }
2253 }
2254 return UFMT_UNDEF;
2255}
2256
2257bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
2258 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
2259}
2260
2261unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {
2262 if (isGFX10Plus(STI))
2263 return UFMT_DEFAULT;
2264 return DFMT_NFMT_DEFAULT;
2265}
2266
2267} // namespace MTBUFFormat
2268
2269//===----------------------------------------------------------------------===//
2270// SendMsg
2271//===----------------------------------------------------------------------===//
2272
2273namespace SendMsg {
2274
2275static uint64_t getMsgIdMask(const MCSubtargetInfo &STI) {
2276 return isGFX11Plus(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_;
2277}
2278
2279bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) {
2280 return (MsgId & ~(getMsgIdMask(STI))) == 0;
2281}
2282
2283bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
2284 bool Strict) {
2285 assert(isValidMsgId(MsgId, STI));
2286
2287 if (!Strict)
2288 return 0 <= OpId && isUInt<OP_WIDTH_>(x: OpId);
2289
2290 if (msgRequiresOp(MsgId, STI)) {
2291 if (MsgId == ID_GS_PreGFX11 && OpId == OP_GS_NOP)
2292 return false;
2293
2294 return !getMsgOpName(MsgId, Encoding: OpId, STI).empty();
2295 }
2296
2297 return OpId == OP_NONE_;
2298}
2299
2300bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
2301 const MCSubtargetInfo &STI, bool Strict) {
2302 assert(isValidMsgOp(MsgId, OpId, STI, Strict));
2303
2304 if (!Strict)
2305 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(x: StreamId);
2306
2307 if (!isGFX11Plus(STI)) {
2308 switch (MsgId) {
2309 case ID_GS_PreGFX11:
2310 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
2311 case ID_GS_DONE_PreGFX11:
2312 return (OpId == OP_GS_NOP)
2313 ? (StreamId == STREAM_ID_NONE_)
2314 : (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
2315 }
2316 }
2317 return StreamId == STREAM_ID_NONE_;
2318}
2319
2320bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI) {
2321 return MsgId == ID_SYSMSG ||
2322 (!isGFX11Plus(STI) &&
2323 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11));
2324}
2325
2326bool msgSupportsStream(int64_t MsgId, int64_t OpId,
2327 const MCSubtargetInfo &STI) {
2328 return !isGFX11Plus(STI) &&
2329 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11) &&
2330 OpId != OP_GS_NOP;
2331}
2332
2333void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,
2334 uint16_t &StreamId, const MCSubtargetInfo &STI) {
2335 MsgId = Val & getMsgIdMask(STI);
2336 if (isGFX11Plus(STI)) {
2337 OpId = 0;
2338 StreamId = 0;
2339 } else {
2340 OpId = (Val & OP_MASK_) >> OP_SHIFT_;
2341 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
2342 }
2343}
2344
2345uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId) {
2346 return MsgId | (OpId << OP_SHIFT_) | (StreamId << STREAM_ID_SHIFT_);
2347}
2348
2349bool msgDoesNotUseM0(int64_t MsgId, const MCSubtargetInfo &STI) {
2350 // Explicitly list message types that are known to not use m0.
2351 // This is safer than excluding only GS_ALLOC_REQ, in case new message
2352 // types are added in the future that do use m0.
2353 if (isGFX11Plus(STI)) {
2354 switch (MsgId) {
2355 case ID_DEALLOC_VGPRS_GFX11Plus:
2356 return true;
2357 default:
2358 break;
2359 }
2360 }
2361 switch (MsgId) {
2362 case ID_SAVEWAVE:
2363 case ID_STALL_WAVE_GEN:
2364 case ID_HALT_WAVES:
2365 case ID_ORDERED_PS_DONE:
2366 case ID_EARLY_PRIM_DEALLOC:
2367 case ID_GET_DOORBELL:
2368 case ID_GET_DDID:
2369 case ID_SYSMSG:
2370 return true;
2371 default:
2372 return false;
2373 }
2374}
2375
2376} // namespace SendMsg
2377
2378//===----------------------------------------------------------------------===//
2379//
2380//===----------------------------------------------------------------------===//
2381
2382unsigned getInitialPSInputAddr(const Function &F) {
2383 return F.getFnAttributeAsParsedInteger(Kind: "InitialPSInputAddr", Default: 0);
2384}
2385
2386bool getHasColorExport(const Function &F) {
2387 // As a safe default always respond as if PS has color exports.
2388 return F.getFnAttributeAsParsedInteger(
2389 Kind: "amdgpu-color-export",
2390 Default: F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0;
2391}
2392
2393bool getHasDepthExport(const Function &F) {
2394 return F.getFnAttributeAsParsedInteger(Kind: "amdgpu-depth-export", Default: 0) != 0;
2395}
2396
2397unsigned getDynamicVGPRBlockSize(const Function &F) {
2398 unsigned BlockSize =
2399 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-dynamic-vgpr-block-size", Default: 0);
2400
2401 if (BlockSize == 16 || BlockSize == 32)
2402 return BlockSize;
2403
2404 return 0;
2405}
2406
2407bool hasXNACK(const MCSubtargetInfo &STI) {
2408 // Only hardwired-on xnack (gfx1250) is knowable from the subtarget alone;
2409 // toggleable targets take their mode from the TargetID.
2410 return STI.hasFeature(Feature: AMDGPU::FeatureSupportsXNACK) &&
2411 !STI.hasFeature(Feature: AMDGPU::FeatureXNACKOnOffModes);
2412}
2413
2414bool hasMIMG_R128(const MCSubtargetInfo &STI) {
2415 return STI.hasFeature(Feature: AMDGPU::FeatureMIMG_R128) &&
2416 !STI.hasFeature(Feature: AMDGPU::FeatureR128A16);
2417}
2418
2419bool hasA16(const MCSubtargetInfo &STI) {
2420 return STI.hasFeature(Feature: AMDGPU::FeatureA16);
2421}
2422
2423bool hasG16(const MCSubtargetInfo &STI) {
2424 return STI.hasFeature(Feature: AMDGPU::FeatureG16);
2425}
2426
2427bool hasPackedD16(const MCSubtargetInfo &STI) {
2428 return !STI.hasFeature(Feature: AMDGPU::FeatureUnpackedD16VMem) && !isCI(STI) &&
2429 !isSI(STI);
2430}
2431
2432bool hasGDS(const MCSubtargetInfo &STI) {
2433 return STI.hasFeature(Feature: AMDGPU::FeatureGDS);
2434}
2435
2436unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler) {
2437 auto Version = getIsaVersion(GPU: STI.getCPU());
2438 if (Version.Major == 10)
2439 return Version.Minor >= 3 ? 13 : 5;
2440 if (Version.Major == 11)
2441 return 5;
2442 if (Version.Major >= 12)
2443 return HasSampler ? 4 : 5;
2444 return 0;
2445}
2446
2447unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI) {
2448 if (isGFX1250Plus(STI))
2449 return 32;
2450 return 16;
2451}
2452
2453bool isSI(const MCSubtargetInfo &STI) {
2454 return STI.hasFeature(Feature: AMDGPU::FeatureSouthernIslands);
2455}
2456
2457bool isCI(const MCSubtargetInfo &STI) {
2458 return STI.hasFeature(Feature: AMDGPU::FeatureSeaIslands);
2459}
2460
2461bool isVI(const MCSubtargetInfo &STI) {
2462 return STI.hasFeature(Feature: AMDGPU::FeatureVolcanicIslands);
2463}
2464
2465bool isGFX9(const MCSubtargetInfo &STI) {
2466 return STI.hasFeature(Feature: AMDGPU::FeatureGFX9);
2467}
2468
2469bool isGFX9_GFX10(const MCSubtargetInfo &STI) {
2470 return isGFX9(STI) || isGFX10(STI);
2471}
2472
2473bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI) {
2474 return isGFX9(STI) || isGFX10(STI) || isGFX11(STI);
2475}
2476
2477bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI) {
2478 return isVI(STI) || isGFX9(STI) || isGFX10(STI);
2479}
2480
2481bool isGFX8Plus(const MCSubtargetInfo &STI) {
2482 return isVI(STI) || isGFX9Plus(STI);
2483}
2484
2485bool isGFX9Plus(const MCSubtargetInfo &STI) {
2486 return isGFX9(STI) || isGFX10Plus(STI);
2487}
2488
2489bool isNotGFX9Plus(const MCSubtargetInfo &STI) { return !isGFX9Plus(STI); }
2490
2491bool hasPopsExitingWaveID(const MCSubtargetInfo &STI) {
2492 return STI.hasFeature(Feature: AMDGPU::FeaturePopsExitingWaveID);
2493}
2494
2495bool hasPrivateApertureRegs(const MCSubtargetInfo &STI) {
2496 return STI.hasFeature(Feature: AMDGPU::FeatureApertureRegs) &&
2497 !STI.hasFeature(Feature: AMDGPU::FeatureGloballyAddressableScratch);
2498}
2499
2500bool isGFX10(const MCSubtargetInfo &STI) {
2501 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10);
2502}
2503
2504bool isGFX10_GFX11(const MCSubtargetInfo &STI) {
2505 return isGFX10(STI) || isGFX11(STI);
2506}
2507
2508bool isGFX10Plus(const MCSubtargetInfo &STI) {
2509 return isGFX10(STI) || isGFX11Plus(STI);
2510}
2511
2512bool isGFX11(const MCSubtargetInfo &STI) {
2513 return STI.hasFeature(Feature: AMDGPU::FeatureGFX11);
2514}
2515
2516bool isGFX11Plus(const MCSubtargetInfo &STI) {
2517 return isGFX11(STI) || isGFX12Plus(STI);
2518}
2519
2520bool isGFX12(const MCSubtargetInfo &STI) {
2521 return STI.getFeatureBits()[AMDGPU::FeatureGFX12];
2522}
2523
2524bool isGFX12Plus(const MCSubtargetInfo &STI) {
2525 return isGFX12(STI) || isGFX13Plus(STI);
2526}
2527
2528bool isNotGFX12Plus(const MCSubtargetInfo &STI) { return !isGFX12Plus(STI); }
2529
2530bool isGFX1250(const MCSubtargetInfo &STI) {
2531 return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts] && !isGFX13(STI);
2532}
2533
2534bool isFullSIMDMode(const MCSubtargetInfo &STI) {
2535 return isGFX1250(STI) || !STI.getFeatureBits().test(I: FeatureCuMode);
2536}
2537
2538bool isGFX1250Plus(const MCSubtargetInfo &STI) {
2539 return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts];
2540}
2541
2542bool isGFX13(const MCSubtargetInfo &STI) {
2543 return STI.getFeatureBits()[AMDGPU::FeatureGFX13];
2544}
2545
2546bool isGFX13Plus(const MCSubtargetInfo &STI) { return isGFX13(STI); }
2547
2548bool supportsWGP(const MCSubtargetInfo &STI) {
2549 if (isGFX1250(STI))
2550 return false;
2551 return isGFX10Plus(STI);
2552}
2553
2554bool isNotGFX11Plus(const MCSubtargetInfo &STI) { return !isGFX11Plus(STI); }
2555
2556bool isNotGFX10Plus(const MCSubtargetInfo &STI) {
2557 return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI);
2558}
2559
2560bool isGFX10Before1030(const MCSubtargetInfo &STI) {
2561 return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI);
2562}
2563
2564bool isGCN3Encoding(const MCSubtargetInfo &STI) {
2565 return STI.hasFeature(Feature: AMDGPU::FeatureGCN3Encoding);
2566}
2567
2568bool isGFX10_BEncoding(const MCSubtargetInfo &STI) {
2569 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10_BEncoding);
2570}
2571
2572bool hasGFX10_3Insts(const MCSubtargetInfo &STI) {
2573 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10_3Insts);
2574}
2575
2576bool isGFX10_3_GFX11(const MCSubtargetInfo &STI) {
2577 return isGFX10_BEncoding(STI) && !isGFX12Plus(STI);
2578}
2579
2580bool isGFX90A(const MCSubtargetInfo &STI) {
2581 return STI.hasFeature(Feature: AMDGPU::FeatureGFX90AInsts);
2582}
2583
2584bool isGFX940(const MCSubtargetInfo &STI) {
2585 return STI.hasFeature(Feature: AMDGPU::FeatureGFX940Insts);
2586}
2587
2588bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) {
2589 return STI.hasFeature(Feature: AMDGPU::FeatureArchitectedFlatScratch);
2590}
2591
2592bool hasMAIInsts(const MCSubtargetInfo &STI) {
2593 return STI.hasFeature(Feature: AMDGPU::FeatureMAIInsts);
2594}
2595
2596bool hasVOPD(const MCSubtargetInfo &STI) {
2597 return STI.hasFeature(Feature: AMDGPU::FeatureVOPDInsts);
2598}
2599
2600bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI) {
2601 return STI.hasFeature(Feature: AMDGPU::FeatureDPPSrc1SGPR);
2602}
2603
2604unsigned hasKernargPreload(const MCSubtargetInfo &STI) {
2605 return STI.hasFeature(Feature: AMDGPU::FeatureKernargPreload);
2606}
2607
2608int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR,
2609 int32_t ArgNumVGPR) {
2610 if (has90AInsts && ArgNumAGPR)
2611 return alignTo(Value: ArgNumVGPR, Align: 4) + ArgNumAGPR;
2612 return std::max(a: ArgNumVGPR, b: ArgNumAGPR);
2613}
2614
2615bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI) {
2616 const MCRegisterClass &SGPRClass =
2617 TRI->getRegClass(i: AMDGPU::SReg_32RegClassID);
2618 const MCRegister FirstSubReg = TRI->getSubReg(Reg, Idx: AMDGPU::sub0);
2619 return SGPRClass.contains(Reg: FirstSubReg != 0 ? FirstSubReg : Reg) ||
2620 Reg == AMDGPU::SCC;
2621}
2622
2623bool isRsrcIndexReg(MCRegister Reg, const MCRegisterInfo &MRI) {
2624 return MRI.getRegClass(i: AMDGPU::RsrcReg32RegClassID).contains(Reg);
2625}
2626
2627bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI) {
2628 return MRI.getEncodingValue(Reg) & AMDGPU::HWEncoding::IS_HI16;
2629}
2630
2631#define MAP_REG2REG \
2632 using namespace AMDGPU; \
2633 switch (Reg.id()) { \
2634 default: \
2635 return Reg; \
2636 CASE_CI_VI(FLAT_SCR) \
2637 CASE_CI_VI(FLAT_SCR_LO) \
2638 CASE_CI_VI(FLAT_SCR_HI) \
2639 CASE_VI_GFX9PLUS(TTMP0) \
2640 CASE_VI_GFX9PLUS(TTMP1) \
2641 CASE_VI_GFX9PLUS(TTMP2) \
2642 CASE_VI_GFX9PLUS(TTMP3) \
2643 CASE_VI_GFX9PLUS(TTMP4) \
2644 CASE_VI_GFX9PLUS(TTMP5) \
2645 CASE_VI_GFX9PLUS(TTMP6) \
2646 CASE_VI_GFX9PLUS(TTMP7) \
2647 CASE_VI_GFX9PLUS(TTMP8) \
2648 CASE_VI_GFX9PLUS(TTMP9) \
2649 CASE_VI_GFX9PLUS(TTMP10) \
2650 CASE_VI_GFX9PLUS(TTMP11) \
2651 CASE_VI_GFX9PLUS(TTMP12) \
2652 CASE_VI_GFX9PLUS(TTMP13) \
2653 CASE_VI_GFX9PLUS(TTMP14) \
2654 CASE_VI_GFX9PLUS(TTMP15) \
2655 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \
2656 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \
2657 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \
2658 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \
2659 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \
2660 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \
2661 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \
2662 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \
2663 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \
2664 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \
2665 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \
2666 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \
2667 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
2668 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
2669 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2670 CASE_VI_GFX9PLUS( \
2671 TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2672 CASE_GFXPRE11_GFX11PLUS(M0) \
2673 CASE_GFXPRE11_GFX11PLUS(SGPR_NULL) \
2674 CASE_GFXPRE11_GFX11PLUS_TO(SGPR_NULL64, SGPR_NULL) \
2675 }
2676
2677#define CASE_CI_VI(node) \
2678 assert(!isSI(STI)); \
2679 case node: \
2680 return isCI(STI) ? node##_ci : node##_vi;
2681
2682#define CASE_VI_GFX9PLUS(node) \
2683 case node: \
2684 return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi;
2685
2686#define CASE_GFXPRE11_GFX11PLUS(node) \
2687 case node: \
2688 return isGFX11Plus(STI) ? node##_gfx11plus : node##_gfxpre11;
2689
2690#define CASE_GFXPRE11_GFX11PLUS_TO(node, result) \
2691 case node: \
2692 return isGFX11Plus(STI) ? result##_gfx11plus : result##_gfxpre11;
2693
2694MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI) {
2695 if (STI.getTargetTriple().getArch() == Triple::r600)
2696 return Reg;
2697 MAP_REG2REG
2698}
2699
2700#undef CASE_CI_VI
2701#undef CASE_VI_GFX9PLUS
2702#undef CASE_GFXPRE11_GFX11PLUS
2703#undef CASE_GFXPRE11_GFX11PLUS_TO
2704
2705#define CASE_CI_VI(node) \
2706 case node##_ci: \
2707 case node##_vi: \
2708 return node;
2709#define CASE_VI_GFX9PLUS(node) \
2710 case node##_vi: \
2711 case node##_gfx9plus: \
2712 return node;
2713#define CASE_GFXPRE11_GFX11PLUS(node) \
2714 case node##_gfx11plus: \
2715 case node##_gfxpre11: \
2716 return node;
2717#define CASE_GFXPRE11_GFX11PLUS_TO(node, result)
2718
2719MCRegister mc2PseudoReg(MCRegister Reg) { MAP_REG2REG }
2720
2721bool isInlineValue(MCRegister Reg) {
2722 switch (Reg.id()) {
2723 case AMDGPU::SRC_SHARED_BASE_LO:
2724 case AMDGPU::SRC_SHARED_BASE:
2725 case AMDGPU::SRC_SHARED_LIMIT_LO:
2726 case AMDGPU::SRC_SHARED_LIMIT:
2727 case AMDGPU::SRC_PRIVATE_BASE_LO:
2728 case AMDGPU::SRC_PRIVATE_BASE:
2729 case AMDGPU::SRC_PRIVATE_LIMIT_LO:
2730 case AMDGPU::SRC_PRIVATE_LIMIT:
2731 case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
2732 case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
2733 case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
2734 return true;
2735 case AMDGPU::SRC_VCCZ:
2736 case AMDGPU::SRC_EXECZ:
2737 case AMDGPU::SRC_SCC:
2738 return true;
2739 case AMDGPU::SGPR_NULL:
2740 return true;
2741 default:
2742 return false;
2743 }
2744}
2745
2746#undef CASE_CI_VI
2747#undef CASE_VI_GFX9PLUS
2748#undef CASE_GFXPRE11_GFX11PLUS
2749#undef CASE_GFXPRE11_GFX11PLUS_TO
2750#undef MAP_REG2REG
2751
2752bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2753 assert(OpNo < Desc.NumOperands);
2754 unsigned OpType = Desc.operands()[OpNo].OperandType;
2755 return OpType >= AMDGPU::OPERAND_KIMM_FIRST &&
2756 OpType <= AMDGPU::OPERAND_KIMM_LAST;
2757}
2758
2759bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2760 assert(OpNo < Desc.NumOperands);
2761 unsigned OpType = Desc.operands()[OpNo].OperandType;
2762 switch (OpType) {
2763 case AMDGPU::OPERAND_REG_IMM_FP32:
2764 case AMDGPU::OPERAND_REG_IMM_FP64:
2765 case AMDGPU::OPERAND_REG_IMM_FP16:
2766 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
2767 case AMDGPU::OPERAND_REG_IMM_V2FP16:
2768 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
2769 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
2770 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2771 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2772 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2773 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2774 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2775 case AMDGPU::OPERAND_REG_IMM_V2FP32:
2776 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2777 case AMDGPU::OPERAND_REG_IMM_V2FP64:
2778 return true;
2779 default:
2780 return false;
2781 }
2782}
2783
2784bool isSISrcInlinableOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2785 assert(OpNo < Desc.NumOperands);
2786 unsigned OpType = Desc.operands()[OpNo].OperandType;
2787 return (OpType >= AMDGPU::OPERAND_REG_INLINE_C_FIRST &&
2788 OpType <= AMDGPU::OPERAND_REG_INLINE_C_LAST) ||
2789 (OpType >= AMDGPU::OPERAND_REG_INLINE_AC_FIRST &&
2790 OpType <= AMDGPU::OPERAND_REG_INLINE_AC_LAST);
2791}
2792
2793// Avoid using MCRegisterClass::getSize, since that function will go away
2794// (move from MC* level to Target* level). Return size in bits.
2795unsigned getRegBitWidth(unsigned RCID) {
2796 switch (RCID) {
2797 case AMDGPU::VGPR_16RegClassID:
2798 case AMDGPU::VGPR_16_Lo128RegClassID:
2799 case AMDGPU::SGPR_LO16RegClassID:
2800 case AMDGPU::AGPR_LO16RegClassID:
2801 return 16;
2802 case AMDGPU::SGPR_32RegClassID:
2803 case AMDGPU::VGPR_32RegClassID:
2804 case AMDGPU::VGPR_32_Lo256RegClassID:
2805 case AMDGPU::VRegOrLds_32RegClassID:
2806 case AMDGPU::AGPR_32RegClassID:
2807 case AMDGPU::VS_32RegClassID:
2808 case AMDGPU::AV_32RegClassID:
2809 case AMDGPU::SReg_32RegClassID:
2810 case AMDGPU::SReg_32_XM0RegClassID:
2811 case AMDGPU::SRegOrLds_32RegClassID:
2812 return 32;
2813 case AMDGPU::SGPR_64RegClassID:
2814 case AMDGPU::VS_64RegClassID:
2815 case AMDGPU::SReg_64RegClassID:
2816 case AMDGPU::VReg_64RegClassID:
2817 case AMDGPU::AReg_64RegClassID:
2818 case AMDGPU::SReg_64_XEXECRegClassID:
2819 case AMDGPU::VReg_64_Align2RegClassID:
2820 case AMDGPU::AReg_64_Align2RegClassID:
2821 case AMDGPU::AV_64RegClassID:
2822 case AMDGPU::AV_64_Align2RegClassID:
2823 case AMDGPU::VReg_64_Lo256_Align2RegClassID:
2824 case AMDGPU::VS_64_Lo256RegClassID:
2825 return 64;
2826 case AMDGPU::SGPR_96RegClassID:
2827 case AMDGPU::SReg_96RegClassID:
2828 case AMDGPU::VReg_96RegClassID:
2829 case AMDGPU::AReg_96RegClassID:
2830 case AMDGPU::VReg_96_Align2RegClassID:
2831 case AMDGPU::AReg_96_Align2RegClassID:
2832 case AMDGPU::AV_96RegClassID:
2833 case AMDGPU::AV_96_Align2RegClassID:
2834 case AMDGPU::VReg_96_Lo256_Align2RegClassID:
2835 return 96;
2836 case AMDGPU::SGPR_128RegClassID:
2837 case AMDGPU::SReg_128RegClassID:
2838 case AMDGPU::VReg_128RegClassID:
2839 case AMDGPU::AReg_128RegClassID:
2840 case AMDGPU::VReg_128_Align2RegClassID:
2841 case AMDGPU::AReg_128_Align2RegClassID:
2842 case AMDGPU::AV_128RegClassID:
2843 case AMDGPU::AV_128_Align2RegClassID:
2844 case AMDGPU::SReg_128_XNULLRegClassID:
2845 case AMDGPU::VReg_128_Lo256_Align2RegClassID:
2846 return 128;
2847 case AMDGPU::SGPR_160RegClassID:
2848 case AMDGPU::SReg_160RegClassID:
2849 case AMDGPU::VReg_160RegClassID:
2850 case AMDGPU::AReg_160RegClassID:
2851 case AMDGPU::VReg_160_Align2RegClassID:
2852 case AMDGPU::AReg_160_Align2RegClassID:
2853 case AMDGPU::AV_160RegClassID:
2854 case AMDGPU::AV_160_Align2RegClassID:
2855 case AMDGPU::VReg_160_Lo256_Align2RegClassID:
2856 return 160;
2857 case AMDGPU::SGPR_192RegClassID:
2858 case AMDGPU::SReg_192RegClassID:
2859 case AMDGPU::VReg_192RegClassID:
2860 case AMDGPU::AReg_192RegClassID:
2861 case AMDGPU::VReg_192_Align2RegClassID:
2862 case AMDGPU::AReg_192_Align2RegClassID:
2863 case AMDGPU::AV_192RegClassID:
2864 case AMDGPU::AV_192_Align2RegClassID:
2865 case AMDGPU::VReg_192_Lo256_Align2RegClassID:
2866 return 192;
2867 case AMDGPU::SGPR_224RegClassID:
2868 case AMDGPU::SReg_224RegClassID:
2869 case AMDGPU::VReg_224RegClassID:
2870 case AMDGPU::AReg_224RegClassID:
2871 case AMDGPU::VReg_224_Align2RegClassID:
2872 case AMDGPU::AReg_224_Align2RegClassID:
2873 case AMDGPU::AV_224RegClassID:
2874 case AMDGPU::AV_224_Align2RegClassID:
2875 case AMDGPU::VReg_224_Lo256_Align2RegClassID:
2876 return 224;
2877 case AMDGPU::SGPR_256RegClassID:
2878 case AMDGPU::SReg_256RegClassID:
2879 case AMDGPU::VReg_256RegClassID:
2880 case AMDGPU::AReg_256RegClassID:
2881 case AMDGPU::VReg_256_Align2RegClassID:
2882 case AMDGPU::AReg_256_Align2RegClassID:
2883 case AMDGPU::AV_256RegClassID:
2884 case AMDGPU::AV_256_Align2RegClassID:
2885 case AMDGPU::SReg_256_XNULLRegClassID:
2886 case AMDGPU::VReg_256_Lo256_Align2RegClassID:
2887 return 256;
2888 case AMDGPU::SGPR_288RegClassID:
2889 case AMDGPU::SReg_288RegClassID:
2890 case AMDGPU::VReg_288RegClassID:
2891 case AMDGPU::AReg_288RegClassID:
2892 case AMDGPU::VReg_288_Align2RegClassID:
2893 case AMDGPU::AReg_288_Align2RegClassID:
2894 case AMDGPU::AV_288RegClassID:
2895 case AMDGPU::AV_288_Align2RegClassID:
2896 case AMDGPU::VReg_288_Lo256_Align2RegClassID:
2897 return 288;
2898 case AMDGPU::SGPR_320RegClassID:
2899 case AMDGPU::SReg_320RegClassID:
2900 case AMDGPU::VReg_320RegClassID:
2901 case AMDGPU::AReg_320RegClassID:
2902 case AMDGPU::VReg_320_Align2RegClassID:
2903 case AMDGPU::AReg_320_Align2RegClassID:
2904 case AMDGPU::AV_320RegClassID:
2905 case AMDGPU::AV_320_Align2RegClassID:
2906 case AMDGPU::VReg_320_Lo256_Align2RegClassID:
2907 return 320;
2908 case AMDGPU::SGPR_352RegClassID:
2909 case AMDGPU::SReg_352RegClassID:
2910 case AMDGPU::VReg_352RegClassID:
2911 case AMDGPU::AReg_352RegClassID:
2912 case AMDGPU::VReg_352_Align2RegClassID:
2913 case AMDGPU::AReg_352_Align2RegClassID:
2914 case AMDGPU::AV_352RegClassID:
2915 case AMDGPU::AV_352_Align2RegClassID:
2916 case AMDGPU::VReg_352_Lo256_Align2RegClassID:
2917 return 352;
2918 case AMDGPU::SGPR_384RegClassID:
2919 case AMDGPU::SReg_384RegClassID:
2920 case AMDGPU::VReg_384RegClassID:
2921 case AMDGPU::AReg_384RegClassID:
2922 case AMDGPU::VReg_384_Align2RegClassID:
2923 case AMDGPU::AReg_384_Align2RegClassID:
2924 case AMDGPU::AV_384RegClassID:
2925 case AMDGPU::AV_384_Align2RegClassID:
2926 case AMDGPU::VReg_384_Lo256_Align2RegClassID:
2927 return 384;
2928 case AMDGPU::SGPR_512RegClassID:
2929 case AMDGPU::SReg_512RegClassID:
2930 case AMDGPU::VReg_512RegClassID:
2931 case AMDGPU::AReg_512RegClassID:
2932 case AMDGPU::VReg_512_Align2RegClassID:
2933 case AMDGPU::AReg_512_Align2RegClassID:
2934 case AMDGPU::AV_512RegClassID:
2935 case AMDGPU::AV_512_Align2RegClassID:
2936 case AMDGPU::VReg_512_Lo256_Align2RegClassID:
2937 return 512;
2938 case AMDGPU::SGPR_1024RegClassID:
2939 case AMDGPU::SReg_1024RegClassID:
2940 case AMDGPU::VReg_1024RegClassID:
2941 case AMDGPU::AReg_1024RegClassID:
2942 case AMDGPU::VReg_1024_Align2RegClassID:
2943 case AMDGPU::AReg_1024_Align2RegClassID:
2944 case AMDGPU::AV_1024RegClassID:
2945 case AMDGPU::AV_1024_Align2RegClassID:
2946 case AMDGPU::VReg_1024_Lo256_Align2RegClassID:
2947 return 1024;
2948 default:
2949 llvm_unreachable("Unexpected register class");
2950 }
2951}
2952
2953unsigned getRegBitWidth(const MCRegisterClass &RC) {
2954 return getRegBitWidth(RCID: RC.getID());
2955}
2956
2957bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
2958 if (isInlinableIntLiteral(Literal))
2959 return true;
2960
2961 uint64_t Val = static_cast<uint64_t>(Literal);
2962 return (Val == llvm::bit_cast<uint64_t>(from: 0.0)) ||
2963 (Val == llvm::bit_cast<uint64_t>(from: 1.0)) ||
2964 (Val == llvm::bit_cast<uint64_t>(from: -1.0)) ||
2965 (Val == llvm::bit_cast<uint64_t>(from: 0.5)) ||
2966 (Val == llvm::bit_cast<uint64_t>(from: -0.5)) ||
2967 (Val == llvm::bit_cast<uint64_t>(from: 2.0)) ||
2968 (Val == llvm::bit_cast<uint64_t>(from: -2.0)) ||
2969 (Val == llvm::bit_cast<uint64_t>(from: 4.0)) ||
2970 (Val == llvm::bit_cast<uint64_t>(from: -4.0)) ||
2971 (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
2972}
2973
2974bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
2975 if (isInlinableIntLiteral(Literal))
2976 return true;
2977
2978 // The actual type of the operand does not seem to matter as long
2979 // as the bits match one of the inline immediate values. For example:
2980 //
2981 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
2982 // so it is a legal inline immediate.
2983 //
2984 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
2985 // floating-point, so it is a legal inline immediate.
2986
2987 uint32_t Val = static_cast<uint32_t>(Literal);
2988 return (Val == llvm::bit_cast<uint32_t>(from: 0.0f)) ||
2989 (Val == llvm::bit_cast<uint32_t>(from: 1.0f)) ||
2990 (Val == llvm::bit_cast<uint32_t>(from: -1.0f)) ||
2991 (Val == llvm::bit_cast<uint32_t>(from: 0.5f)) ||
2992 (Val == llvm::bit_cast<uint32_t>(from: -0.5f)) ||
2993 (Val == llvm::bit_cast<uint32_t>(from: 2.0f)) ||
2994 (Val == llvm::bit_cast<uint32_t>(from: -2.0f)) ||
2995 (Val == llvm::bit_cast<uint32_t>(from: 4.0f)) ||
2996 (Val == llvm::bit_cast<uint32_t>(from: -4.0f)) ||
2997 (Val == 0x3e22f983 && HasInv2Pi);
2998}
2999
3000bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi) {
3001 if (!HasInv2Pi)
3002 return false;
3003 if (isInlinableIntLiteral(Literal))
3004 return true;
3005 uint16_t Val = static_cast<uint16_t>(Literal);
3006 return Val == 0x3F00 || // 0.5
3007 Val == 0xBF00 || // -0.5
3008 Val == 0x3F80 || // 1.0
3009 Val == 0xBF80 || // -1.0
3010 Val == 0x4000 || // 2.0
3011 Val == 0xC000 || // -2.0
3012 Val == 0x4080 || // 4.0
3013 Val == 0xC080 || // -4.0
3014 Val == 0x3E22; // 1.0 / (2.0 * pi)
3015}
3016
3017bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi) {
3018 return isInlinableLiteral32(Literal, HasInv2Pi);
3019}
3020
3021bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) {
3022 if (!HasInv2Pi)
3023 return false;
3024 if (isInlinableIntLiteral(Literal))
3025 return true;
3026 uint16_t Val = static_cast<uint16_t>(Literal);
3027 return Val == 0x3C00 || // 1.0
3028 Val == 0xBC00 || // -1.0
3029 Val == 0x3800 || // 0.5
3030 Val == 0xB800 || // -0.5
3031 Val == 0x4000 || // 2.0
3032 Val == 0xC000 || // -2.0
3033 Val == 0x4400 || // 4.0
3034 Val == 0xC400 || // -4.0
3035 Val == 0x3118; // 1/2pi
3036}
3037
3038std::optional<unsigned> getInlineEncodingV216(bool IsFloat, uint32_t Literal) {
3039 // Unfortunately, the Instruction Set Architecture Reference Guide is
3040 // misleading about how the inline operands work for (packed) 16-bit
3041 // instructions. In a nutshell, the actual HW behavior is:
3042 //
3043 // - integer encodings (-16 .. 64) are always produced as sign-extended
3044 // 32-bit values
3045 // - float encodings are produced as:
3046 // - for F16 instructions: corresponding half-precision float values in
3047 // the LSBs, 0 in the MSBs
3048 // - for UI16 instructions: corresponding single-precision float value
3049 int32_t Signed = static_cast<int32_t>(Literal);
3050 if (Signed >= 0 && Signed <= 64)
3051 return 128 + Signed;
3052
3053 if (Signed >= -16 && Signed <= -1)
3054 return 192 + std::abs(x: Signed);
3055
3056 if (IsFloat) {
3057 // clang-format off
3058 switch (Literal) {
3059 case 0x3800: return 240; // 0.5
3060 case 0xB800: return 241; // -0.5
3061 case 0x3C00: return 242; // 1.0
3062 case 0xBC00: return 243; // -1.0
3063 case 0x4000: return 244; // 2.0
3064 case 0xC000: return 245; // -2.0
3065 case 0x4400: return 246; // 4.0
3066 case 0xC400: return 247; // -4.0
3067 case 0x3118: return 248; // 1.0 / (2.0 * pi)
3068 default: break;
3069 }
3070 // clang-format on
3071 } else {
3072 // clang-format off
3073 switch (Literal) {
3074 case 0x3F000000: return 240; // 0.5
3075 case 0xBF000000: return 241; // -0.5
3076 case 0x3F800000: return 242; // 1.0
3077 case 0xBF800000: return 243; // -1.0
3078 case 0x40000000: return 244; // 2.0
3079 case 0xC0000000: return 245; // -2.0
3080 case 0x40800000: return 246; // 4.0
3081 case 0xC0800000: return 247; // -4.0
3082 case 0x3E22F983: return 248; // 1.0 / (2.0 * pi)
3083 default: break;
3084 }
3085 // clang-format on
3086 }
3087
3088 return {};
3089}
3090
3091// Encoding of the literal as an inline constant for a V_PK_*_IU16 instruction
3092// or nullopt.
3093std::optional<unsigned> getInlineEncodingV2I16(uint32_t Literal) {
3094 return getInlineEncodingV216(IsFloat: false, Literal);
3095}
3096
3097// Encoding of the literal as an inline constant for a V_PK_*_BF16 instruction
3098// or nullopt.
3099std::optional<unsigned> getInlineEncodingV2BF16(uint32_t Literal) {
3100 int32_t Signed = static_cast<int32_t>(Literal);
3101 if (Signed >= 0 && Signed <= 64)
3102 return 128 + Signed;
3103
3104 if (Signed >= -16 && Signed <= -1)
3105 return 192 + std::abs(x: Signed);
3106
3107 // clang-format off
3108 switch (Literal) {
3109 case 0x3F00: return 240; // 0.5
3110 case 0xBF00: return 241; // -0.5
3111 case 0x3F80: return 242; // 1.0
3112 case 0xBF80: return 243; // -1.0
3113 case 0x4000: return 244; // 2.0
3114 case 0xC000: return 245; // -2.0
3115 case 0x4080: return 246; // 4.0
3116 case 0xC080: return 247; // -4.0
3117 case 0x3E22: return 248; // 1.0 / (2.0 * pi)
3118 default: break;
3119 }
3120 // clang-format on
3121
3122 return std::nullopt;
3123}
3124
3125// Encoding of the literal as an inline constant for a V_PK_*_F16 instruction
3126// or nullopt.
3127std::optional<unsigned> getInlineEncodingV2F16(uint32_t Literal) {
3128 return getInlineEncodingV216(IsFloat: true, Literal);
3129}
3130
3131// Encoding of the literal as an inline constant for V_PK_FMAC_F16 instruction
3132// or nullopt. This accounts for different inline constant behavior:
3133// - Pre-GFX11: fp16 inline constants have the value in low 16 bits, 0 in high
3134// - GFX11+: fp16 inline constants are duplicated into both halves
3135std::optional<unsigned> getPKFMACF16InlineEncoding(uint32_t Literal,
3136 bool IsGFX11Plus) {
3137 // Pre-GFX11 behavior: f16 in low bits, 0 in high bits
3138 if (!IsGFX11Plus)
3139 return getInlineEncodingV216(/*IsFloat=*/true, Literal);
3140
3141 // GFX11+ behavior: f16 duplicated in both halves
3142 // First, check for sign-extended integer inline constants (-16 to 64)
3143 // These work the same across all generations
3144 int32_t Signed = static_cast<int32_t>(Literal);
3145 if (Signed >= 0 && Signed <= 64)
3146 return 128 + Signed;
3147
3148 if (Signed >= -16 && Signed <= -1)
3149 return 192 + std::abs(x: Signed);
3150
3151 // For float inline constants on GFX11+, both halves must be equal
3152 uint16_t Lo = static_cast<uint16_t>(Literal);
3153 uint16_t Hi = static_cast<uint16_t>(Literal >> 16);
3154 if (Lo != Hi)
3155 return std::nullopt;
3156 return getInlineEncodingV216(/*IsFloat=*/true, Literal: Lo);
3157}
3158
3159// Whether the given literal can be inlined for a V_PK_* instruction.
3160bool isInlinableLiteralV216(uint32_t Literal, uint8_t OpType) {
3161 switch (OpType) {
3162 case AMDGPU::OPERAND_REG_IMM_V2INT16:
3163 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
3164 return getInlineEncodingV216(IsFloat: false, Literal).has_value();
3165 case AMDGPU::OPERAND_REG_IMM_V2FP16:
3166 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
3167 return getInlineEncodingV216(IsFloat: true, Literal).has_value();
3168 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
3169 llvm_unreachable("OPERAND_REG_IMM_V2FP16_SPLAT is not supported");
3170 case AMDGPU::OPERAND_REG_IMM_V2BF16:
3171 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
3172 return isInlinableLiteralV2BF16(Literal);
3173 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
3174 return false;
3175 default:
3176 llvm_unreachable("bad packed operand type");
3177 }
3178}
3179
3180// Whether the given literal can be inlined for a V_PK_*_IU16 instruction.
3181bool isInlinableLiteralV2I16(uint32_t Literal) {
3182 return getInlineEncodingV2I16(Literal).has_value();
3183}
3184
3185// Whether the given literal can be inlined for a V_PK_*_BF16 instruction.
3186bool isInlinableLiteralV2BF16(uint32_t Literal) {
3187 return getInlineEncodingV2BF16(Literal).has_value();
3188}
3189
3190// Whether the given literal can be inlined for a V_PK_*_F16 instruction.
3191bool isInlinableLiteralV2F16(uint32_t Literal) {
3192 return getInlineEncodingV2F16(Literal).has_value();
3193}
3194
3195// Whether the given literal can be inlined for V_PK_FMAC_F16 instruction.
3196bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus) {
3197 return getPKFMACF16InlineEncoding(Literal, IsGFX11Plus).has_value();
3198}
3199
3200bool isValid32BitLiteral(uint64_t Val, bool IsFP64) {
3201 if (IsFP64)
3202 return !Lo_32(Value: Val);
3203
3204 return isUInt<32>(x: Val) || isInt<32>(x: Val);
3205}
3206
3207int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit) {
3208 switch (Type) {
3209 default:
3210 break;
3211 case OPERAND_REG_IMM_BF16:
3212 case OPERAND_REG_IMM_FP16:
3213 case OPERAND_REG_IMM_NOINLINE_FP16:
3214 case OPERAND_REG_INLINE_C_BF16:
3215 case OPERAND_REG_INLINE_C_FP16:
3216 return Imm & 0xffff;
3217 case OPERAND_INLINE_SPLIT_BARRIER_INT32:
3218 case OPERAND_REG_IMM_FP32:
3219 case OPERAND_REG_IMM_INT32:
3220 case OPERAND_REG_IMM_V2BF16:
3221 case OPERAND_REG_IMM_V2FP16:
3222 case OPERAND_REG_IMM_V2FP16_SPLAT:
3223 case OPERAND_REG_IMM_V2FP32:
3224 case OPERAND_REG_IMM_V2INT16:
3225 case OPERAND_REG_IMM_V2INT32:
3226 case OPERAND_REG_INLINE_AC_FP32:
3227 case OPERAND_REG_INLINE_AC_INT32:
3228 case OPERAND_REG_INLINE_C_FP32:
3229 case OPERAND_REG_INLINE_C_INT32:
3230 return Lo_32(Value: Imm);
3231 case OPERAND_REG_IMM_FP64:
3232 case AMDGPU::OPERAND_REG_IMM_V2FP64:
3233 return IsLit ? Imm : Hi_32(Value: Imm);
3234 }
3235 return Imm;
3236}
3237
3238bool isArgPassedInSGPR(const Argument *A) {
3239 const Function *F = A->getParent();
3240
3241 // Arguments to compute shaders are never a source of divergence.
3242 CallingConv::ID CC = F->getCallingConv();
3243 switch (CC) {
3244 case CallingConv::AMDGPU_KERNEL:
3245 case CallingConv::SPIR_KERNEL:
3246 return true;
3247 case CallingConv::AMDGPU_VS:
3248 case CallingConv::AMDGPU_LS:
3249 case CallingConv::AMDGPU_HS:
3250 case CallingConv::AMDGPU_ES:
3251 case CallingConv::AMDGPU_GS:
3252 case CallingConv::AMDGPU_PS:
3253 case CallingConv::AMDGPU_CS:
3254 case CallingConv::AMDGPU_Gfx:
3255 case CallingConv::AMDGPU_CS_Chain:
3256 case CallingConv::AMDGPU_CS_ChainPreserve:
3257 // For non-compute shaders, SGPR inputs are marked with either inreg or
3258 // byval. Everything else is in VGPRs.
3259 return A->hasAttribute(Kind: Attribute::InReg) ||
3260 A->hasAttribute(Kind: Attribute::ByVal);
3261 default:
3262 // TODO: treat i1 as divergent?
3263 return A->hasAttribute(Kind: Attribute::InReg);
3264 }
3265}
3266
3267bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo) {
3268 // Arguments to compute shaders are never a source of divergence.
3269 CallingConv::ID CC = CB->getCallingConv();
3270 switch (CC) {
3271 case CallingConv::AMDGPU_KERNEL:
3272 case CallingConv::SPIR_KERNEL:
3273 return true;
3274 case CallingConv::AMDGPU_VS:
3275 case CallingConv::AMDGPU_LS:
3276 case CallingConv::AMDGPU_HS:
3277 case CallingConv::AMDGPU_ES:
3278 case CallingConv::AMDGPU_GS:
3279 case CallingConv::AMDGPU_PS:
3280 case CallingConv::AMDGPU_CS:
3281 case CallingConv::AMDGPU_Gfx:
3282 case CallingConv::AMDGPU_CS_Chain:
3283 case CallingConv::AMDGPU_CS_ChainPreserve:
3284 // For non-compute shaders, SGPR inputs are marked with either inreg or
3285 // byval. Everything else is in VGPRs.
3286 return CB->paramHasAttr(ArgNo, Kind: Attribute::InReg) ||
3287 CB->isByValArgument(ArgNo);
3288 default:
3289 return CB->paramHasAttr(ArgNo, Kind: Attribute::InReg);
3290 }
3291}
3292
3293static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
3294 return isGCN3Encoding(STI: ST) || isGFX10Plus(STI: ST);
3295}
3296
3297bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
3298 int64_t EncodedOffset) {
3299 if (isGFX12Plus(STI: ST))
3300 return isUInt<23>(x: EncodedOffset);
3301
3302 return hasSMEMByteOffset(ST) ? isUInt<20>(x: EncodedOffset)
3303 : isUInt<8>(x: EncodedOffset);
3304}
3305
3306bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
3307 int64_t EncodedOffset, bool IsBuffer) {
3308 if (isGFX12Plus(STI: ST)) {
3309 if (IsBuffer && EncodedOffset < 0)
3310 return false;
3311 return isInt<24>(x: EncodedOffset);
3312 }
3313
3314 return !IsBuffer && hasSMRDSignedImmOffset(ST) && isInt<21>(x: EncodedOffset);
3315}
3316
3317static bool isDwordAligned(uint64_t ByteOffset) {
3318 return (ByteOffset & 3) == 0;
3319}
3320
3321uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
3322 uint64_t ByteOffset) {
3323 if (hasSMEMByteOffset(ST))
3324 return ByteOffset;
3325
3326 assert(isDwordAligned(ByteOffset));
3327 return ByteOffset >> 2;
3328}
3329
3330std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
3331 int64_t ByteOffset, bool IsBuffer,
3332 bool HasSOffset) {
3333 // For unbuffered smem loads, it is illegal for the Immediate Offset to be
3334 // negative if the resulting (Offset + (M0 or SOffset or zero) is negative.
3335 // Handle case where SOffset is not present.
3336 if (!IsBuffer && !HasSOffset && ByteOffset < 0 && hasSMRDSignedImmOffset(ST))
3337 return std::nullopt;
3338
3339 if (isGFX12Plus(STI: ST)) // 24 bit signed offsets
3340 return isInt<24>(x: ByteOffset) ? std::optional<int64_t>(ByteOffset)
3341 : std::nullopt;
3342
3343 // The signed version is always a byte offset.
3344 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
3345 assert(hasSMEMByteOffset(ST));
3346 return isInt<20>(x: ByteOffset) ? std::optional<int64_t>(ByteOffset)
3347 : std::nullopt;
3348 }
3349
3350 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
3351 return std::nullopt;
3352
3353 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3354 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
3355 ? std::optional<int64_t>(EncodedOffset)
3356 : std::nullopt;
3357}
3358
3359std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
3360 int64_t ByteOffset) {
3361 if (!isCI(STI: ST) || !isDwordAligned(ByteOffset))
3362 return std::nullopt;
3363
3364 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3365 return isUInt<32>(x: EncodedOffset) ? std::optional<int64_t>(EncodedOffset)
3366 : std::nullopt;
3367}
3368
3369unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST) {
3370 if (ST.getFeatureBits().test(I: FeatureFlatOffsetBits12))
3371 return 12;
3372 if (ST.getFeatureBits().test(I: FeatureFlatOffsetBits24))
3373 return 24;
3374 return 13;
3375}
3376
3377namespace {
3378
3379struct SourceOfDivergence {
3380 unsigned Intr;
3381};
3382const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
3383
3384struct AlwaysUniform {
3385 unsigned Intr;
3386};
3387const AlwaysUniform *lookupAlwaysUniform(unsigned Intr);
3388
3389#define GET_SourcesOfDivergence_IMPL
3390#define GET_UniformIntrinsics_IMPL
3391#define GET_Gfx9BufferFormat_IMPL
3392#define GET_Gfx10BufferFormat_IMPL
3393#define GET_Gfx11PlusBufferFormat_IMPL
3394
3395#include "AMDGPUGenSearchableTables.inc"
3396
3397} // end anonymous namespace
3398
3399bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
3400 return lookupSourceOfDivergence(Intr: IntrID);
3401}
3402
3403bool isIntrinsicAlwaysUniform(unsigned IntrID) {
3404 return lookupAlwaysUniform(Intr: IntrID);
3405}
3406
3407const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
3408 uint8_t NumComponents,
3409 uint8_t NumFormat,
3410 const MCSubtargetInfo &STI) {
3411 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(
3412 BitsPerComp, NumComponents, NumFormat)
3413 : isGFX10(STI)
3414 ? getGfx10BufferFormatInfo(BitsPerComp, NumComponents, NumFormat)
3415 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
3416}
3417
3418const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
3419 const MCSubtargetInfo &STI) {
3420 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(Format)
3421 : isGFX10(STI) ? getGfx10BufferFormatInfo(Format)
3422 : getGfx9BufferFormatInfo(Format);
3423}
3424
3425const MCRegisterClass *getVGPRPhysRegClass(MCRegister Reg,
3426 const MCRegisterInfo &MRI) {
3427 const unsigned VGPRClasses[] = {
3428 AMDGPU::VGPR_16RegClassID, AMDGPU::VGPR_32RegClassID,
3429 AMDGPU::VReg_64RegClassID, AMDGPU::VReg_96RegClassID,
3430 AMDGPU::VReg_128RegClassID, AMDGPU::VReg_160RegClassID,
3431 AMDGPU::VReg_192RegClassID, AMDGPU::VReg_224RegClassID,
3432 AMDGPU::VReg_256RegClassID, AMDGPU::VReg_288RegClassID,
3433 AMDGPU::VReg_320RegClassID, AMDGPU::VReg_352RegClassID,
3434 AMDGPU::VReg_384RegClassID, AMDGPU::VReg_512RegClassID,
3435 AMDGPU::VReg_1024RegClassID};
3436
3437 for (unsigned RCID : VGPRClasses) {
3438 const MCRegisterClass &RC = MRI.getRegClass(i: RCID);
3439 if (RC.contains(Reg))
3440 return &RC;
3441 }
3442
3443 return nullptr;
3444}
3445
3446unsigned getVGPREncodingMSBs(MCRegister Reg, const MCRegisterInfo &MRI) {
3447 unsigned Enc = MRI.getEncodingValue(Reg);
3448 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3449 return Idx >> 8;
3450}
3451
3452MCRegister getVGPRWithMSBs(MCRegister Reg, unsigned MSBs,
3453 const MCRegisterInfo &MRI) {
3454 unsigned Enc = MRI.getEncodingValue(Reg);
3455 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3456 if (Idx >= 0x100)
3457 return MCRegister();
3458
3459 const MCRegisterClass *RC = getVGPRPhysRegClass(Reg, MRI);
3460 if (!RC)
3461 return MCRegister();
3462
3463 Idx |= MSBs << 8;
3464 if (RC->getID() == AMDGPU::VGPR_16RegClassID) {
3465 // This class has 2048 registers with interleaved lo16 and hi16.
3466 Idx *= 2;
3467 if (Enc & AMDGPU::HWEncoding::IS_HI16)
3468 ++Idx;
3469 }
3470
3471 return RC->getRegister(i: Idx);
3472}
3473
3474static std::optional<unsigned>
3475convertSetRegImmToVgprMSBs(unsigned Imm, unsigned Simm16,
3476 bool HasSetregVGPRMSBFixup) {
3477 constexpr unsigned VGPRMSBShift =
3478 llvm::countr_zero_constexpr<unsigned>(Val: AMDGPU::Hwreg::DST_VGPR_MSB);
3479
3480 auto [HwRegId, Offset, Size] = Hwreg::HwregEncoding::decode(Encoded: Simm16);
3481 if (HwRegId != Hwreg::ID_MODE ||
3482 (!HasSetregVGPRMSBFixup && (Offset + Size) < VGPRMSBShift))
3483 return {};
3484 // If there is SetregVGPRMSBFixup then Offset is ignored.
3485 if (!HasSetregVGPRMSBFixup)
3486 Imm <<= Offset;
3487 Imm = (Imm & Hwreg::VGPR_MSB_MASK) >> VGPRMSBShift;
3488 if (!HasSetregVGPRMSBFixup)
3489 Imm &= llvm::maskTrailingOnes<unsigned>(N: Size);
3490 return llvm::rotr<uint8_t>(V: static_cast<uint8_t>(Imm), /*R=*/2);
3491}
3492
3493std::optional<unsigned> convertSetRegImmToVgprMSBs(const MachineInstr &MI,
3494 bool HasSetregVGPRMSBFixup) {
3495 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32);
3496 return convertSetRegImmToVgprMSBs(Imm: MI.getOperand(i: 0).getImm(),
3497 Simm16: MI.getOperand(i: 1).getImm(),
3498 HasSetregVGPRMSBFixup);
3499}
3500
3501std::optional<unsigned> convertSetRegImmToVgprMSBs(const MCInst &MI,
3502 bool HasSetregVGPRMSBFixup) {
3503 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_gfx12);
3504 return convertSetRegImmToVgprMSBs(Imm: MI.getOperand(i: 0).getImm(),
3505 Simm16: MI.getOperand(i: 1).getImm(),
3506 HasSetregVGPRMSBFixup);
3507}
3508
3509std::pair<const AMDGPU::OpName *, const AMDGPU::OpName *>
3510getVGPRLoweringOperandTables(const MCInstrDesc &Desc) {
3511 static const AMDGPU::OpName VOPOps[4] = {
3512 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2,
3513 AMDGPU::OpName::vdst};
3514 static const AMDGPU::OpName VDSOps[4] = {
3515 AMDGPU::OpName::addr, AMDGPU::OpName::data0, AMDGPU::OpName::data1,
3516 AMDGPU::OpName::vdst};
3517 static const AMDGPU::OpName FLATOps[4] = {
3518 AMDGPU::OpName::vaddr, AMDGPU::OpName::vdata,
3519 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdst};
3520 static const AMDGPU::OpName BUFOps[4] = {
3521 AMDGPU::OpName::vaddr, AMDGPU::OpName::NUM_OPERAND_NAMES,
3522 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdata};
3523 static const AMDGPU::OpName VIMGOps[4] = {
3524 AMDGPU::OpName::vaddr0, AMDGPU::OpName::vaddr1, AMDGPU::OpName::vaddr2,
3525 AMDGPU::OpName::vdata};
3526
3527 // For VOPD instructions MSB of a corresponding Y component operand VGPR
3528 // address is supposed to match X operand, otherwise VOPD shall not be
3529 // combined.
3530 static const AMDGPU::OpName VOPDOpsX[4] = {
3531 AMDGPU::OpName::src0X, AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vsrc2X,
3532 AMDGPU::OpName::vdstX};
3533 static const AMDGPU::OpName VOPDOpsY[4] = {
3534 AMDGPU::OpName::src0Y, AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vsrc2Y,
3535 AMDGPU::OpName::vdstY};
3536
3537 // VOP2 MADMK instructions use src0, imm, src1 scheme.
3538 static const AMDGPU::OpName VOP2MADMKOps[4] = {
3539 AMDGPU::OpName::src0, AMDGPU::OpName::NUM_OPERAND_NAMES,
3540 AMDGPU::OpName::src1, AMDGPU::OpName::vdst};
3541 static const AMDGPU::OpName VOPDFMAMKOpsX[4] = {
3542 AMDGPU::OpName::src0X, AMDGPU::OpName::NUM_OPERAND_NAMES,
3543 AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vdstX};
3544 static const AMDGPU::OpName VOPDFMAMKOpsY[4] = {
3545 AMDGPU::OpName::src0Y, AMDGPU::OpName::NUM_OPERAND_NAMES,
3546 AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vdstY};
3547
3548 if (SIInstrFlags::isVOP1(O: Desc) || SIInstrFlags::isVOP2(O: Desc) ||
3549 SIInstrFlags::isVOP3Like(O: Desc) || SIInstrFlags::isVOPC(O: Desc) ||
3550 SIInstrFlags::isDPP(O: Desc)) {
3551 switch (Desc.getOpcode()) {
3552 // LD_SCALE operands ignore MSB.
3553 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32:
3554 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32_gfx1250:
3555 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64:
3556 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64_gfx1250:
3557 return {};
3558 case AMDGPU::V_FMAMK_F16:
3559 case AMDGPU::V_FMAMK_F16_t16:
3560 case AMDGPU::V_FMAMK_F16_t16_gfx12:
3561 case AMDGPU::V_FMAMK_F16_fake16:
3562 case AMDGPU::V_FMAMK_F16_fake16_gfx12:
3563 case AMDGPU::V_FMAMK_F32:
3564 case AMDGPU::V_FMAMK_F32_gfx12:
3565 case AMDGPU::V_FMAMK_F64:
3566 case AMDGPU::V_FMAMK_F64_gfx1250:
3567 return {VOP2MADMKOps, nullptr};
3568 default:
3569 break;
3570 }
3571 return {VOPOps, nullptr};
3572 }
3573
3574 if (SIInstrFlags::isDS(O: Desc))
3575 return {VDSOps, nullptr};
3576
3577 if (SIInstrFlags::isFLAT(O: Desc))
3578 return {FLATOps, nullptr};
3579
3580 if (SIInstrFlags::isBuffer(O: Desc))
3581 return {BUFOps, nullptr};
3582
3583 if (SIInstrFlags::isVIMAGE(O: Desc))
3584 return {VIMGOps, nullptr};
3585
3586 if (AMDGPU::isVOPD(Opc: Desc.getOpcode())) {
3587 auto [OpX, OpY] = getVOPDComponents(VOPDOpcode: Desc.getOpcode());
3588 return {(OpX == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsX : VOPDOpsX,
3589 (OpY == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsY : VOPDOpsY};
3590 }
3591
3592 assert(!SIInstrFlags::isMIMG(Desc));
3593
3594 if (SIInstrFlags::isVSAMPLE(O: Desc) || SIInstrFlags::isEXP(O: Desc))
3595 llvm_unreachable("Sample and export VGPR lowering is not implemented and"
3596 " these instructions are not expected on gfx1250");
3597
3598 return {};
3599}
3600
3601bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode) {
3602 const MCInstrDesc &Desc = MII.get(Opcode);
3603 if (SIInstrFlags::isSMRD(O: Desc))
3604 return Desc.mayLoad() && !Desc.mayStore() && !getSMEMIsBuffer(Opc: Opcode);
3605 if (!SIInstrFlags::isFLAT(O: Desc))
3606 return false;
3607
3608 // Only SV and SVS modes are supported.
3609 if (SIInstrFlags::isFlatScratch(O: MII, O: Opcode))
3610 return hasNamedOperand(Opcode, NamedIdx: OpName::vaddr);
3611
3612 // Only GVS mode is supported.
3613 return hasNamedOperand(Opcode, NamedIdx: OpName::vaddr) &&
3614 hasNamedOperand(Opcode, NamedIdx: OpName::saddr);
3615
3616 return false;
3617}
3618
3619bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
3620 const MCSubtargetInfo &ST) {
3621 for (auto OpName : {OpName::vdst, OpName::src0, OpName::src1, OpName::src2}) {
3622 int Idx = getNamedOperandIdx(Opcode: OpDesc.getOpcode(), Name: OpName);
3623 if (Idx == -1)
3624 continue;
3625
3626 const MCOperandInfo &OpInfo = OpDesc.operands()[Idx];
3627 int16_t RegClass = MII.getOpRegClassID(
3628 OpInfo, HwModeId: ST.getHwMode(type: MCSubtargetInfo::HwMode_RegInfo));
3629 if (RegClass == AMDGPU::VReg_64RegClassID ||
3630 RegClass == AMDGPU::VReg_64_Align2RegClassID)
3631 return true;
3632 }
3633
3634 return false;
3635}
3636
3637bool isDPALU_DPP32BitOpc(unsigned Opc) {
3638 switch (Opc) {
3639 case AMDGPU::V_MUL_LO_U32_e64:
3640 case AMDGPU::V_MUL_LO_U32_e64_dpp:
3641 case AMDGPU::V_MUL_LO_U32_e64_dpp_gfx1250:
3642 case AMDGPU::V_MUL_HI_U32_e64:
3643 case AMDGPU::V_MUL_HI_U32_e64_dpp:
3644 case AMDGPU::V_MUL_HI_U32_e64_dpp_gfx1250:
3645 case AMDGPU::V_MUL_HI_I32_e64:
3646 case AMDGPU::V_MUL_HI_I32_e64_dpp:
3647 case AMDGPU::V_MUL_HI_I32_e64_dpp_gfx1250:
3648 case AMDGPU::V_MAD_U32_e64:
3649 case AMDGPU::V_MAD_U32_e64_dpp:
3650 case AMDGPU::V_MAD_U32_e64_dpp_gfx1250:
3651 return true;
3652 default:
3653 return false;
3654 }
3655}
3656
3657bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
3658 const MCSubtargetInfo &ST) {
3659 if (!ST.hasFeature(Feature: AMDGPU::FeatureDPALU_DPP))
3660 return false;
3661
3662 if (isDPALU_DPP32BitOpc(Opc: OpDesc.getOpcode()))
3663 return ST.hasFeature(Feature: AMDGPU::FeatureGFX1250Insts);
3664
3665 return hasAny64BitVGPROperands(OpDesc, MII, ST);
3666}
3667
3668unsigned getLdsDwGranularity(const MCSubtargetInfo &ST) {
3669 if (ST.getFeatureBits().test(I: FeatureAddressableLocalMemorySize32768))
3670 return 64;
3671 if (ST.getFeatureBits().test(I: FeatureAddressableLocalMemorySize65536))
3672 return 128;
3673 if (ST.getFeatureBits().test(I: FeatureAddressableLocalMemorySize196608))
3674 return 256;
3675 if (ST.getFeatureBits().test(I: FeatureAddressableLocalMemorySize163840))
3676 return 320;
3677 if (ST.getFeatureBits().test(I: FeatureAddressableLocalMemorySize327680))
3678 return 512;
3679 return 64; // In sync with getAddressableLocalMemorySize
3680}
3681
3682bool isPackedSingleSGPRFP32Inst(unsigned Opc) {
3683 switch (Opc) {
3684 case AMDGPU::V_PK_ADD_F32_gfx1250:
3685 case AMDGPU::V_PK_ADD_F32_gfx1250_gfx12:
3686 case AMDGPU::V_PK_MUL_F32_gfx1250:
3687 case AMDGPU::V_PK_MUL_F32_gfx1250_gfx12:
3688 case AMDGPU::V_PK_FMA_F32_gfx1250:
3689 case AMDGPU::V_PK_FMA_F32_gfx1250_gfx12:
3690 return true;
3691 default:
3692 return false;
3693 }
3694}
3695
3696// NOTE: This function is currently only used before pseudo-expansion.
3697bool isPackedSingleSGPR64BitInst(unsigned Opc) {
3698 switch (Opc) {
3699 case AMDGPU::V_PK_ADD_F64:
3700 case AMDGPU::V_PK_MUL_F64:
3701 case AMDGPU::V_PK_FMA_F64:
3702 case AMDGPU::V_PK_MAX_NUM_F64:
3703 case AMDGPU::V_PK_MIN_NUM_F64:
3704 case AMDGPU::V_PK_ADD_NC_U64:
3705 case AMDGPU::V_PK_SUB_NC_U64:
3706 case AMDGPU::V_PK_LSHL_ADD_U64:
3707 return true;
3708 default:
3709 return false;
3710 }
3711}
3712
3713bool isSingleSGPRReadInst(unsigned Opc) {
3714 return isPackedSingleSGPRFP32Inst(Opc) || isPackedSingleSGPR64BitInst(Opc);
3715}
3716
3717const std::array<unsigned, 3> &ClusterDimsAttr::getDims() const {
3718 assert(isFixedDims() && "expect kind to be FixedDims");
3719 return Dims;
3720}
3721
3722std::string ClusterDimsAttr::to_string() const {
3723 SmallString<10> Buffer;
3724 raw_svector_ostream OS(Buffer);
3725
3726 switch (getKind()) {
3727 case Kind::Unknown:
3728 return "";
3729 case Kind::NoCluster: {
3730 OS << EncoNoCluster << ',' << EncoNoCluster << ',' << EncoNoCluster;
3731 return Buffer.c_str();
3732 }
3733 case Kind::VariableDims: {
3734 OS << EncoVariableDims << ',' << EncoVariableDims << ','
3735 << EncoVariableDims;
3736 return Buffer.c_str();
3737 }
3738 case Kind::FixedDims: {
3739 OS << Dims[0] << ',' << Dims[1] << ',' << Dims[2];
3740 return Buffer.c_str();
3741 }
3742 }
3743 llvm_unreachable("Unknown ClusterDimsAttr kind");
3744}
3745
3746ClusterDimsAttr ClusterDimsAttr::get(const Function &F) {
3747 std::optional<SmallVector<unsigned>> Attr =
3748 getIntegerVecAttribute(F, Name: "amdgpu-cluster-dims", /*Size=*/3);
3749 ClusterDimsAttr::Kind AttrKind = Kind::FixedDims;
3750
3751 if (!Attr.has_value())
3752 AttrKind = Kind::Unknown;
3753 else if (all_of(Range&: *Attr, P: equal_to(Arg: EncoNoCluster)))
3754 AttrKind = Kind::NoCluster;
3755 else if (all_of(Range&: *Attr, P: equal_to(Arg: EncoVariableDims)))
3756 AttrKind = Kind::VariableDims;
3757
3758 ClusterDimsAttr A(AttrKind);
3759 if (AttrKind == Kind::FixedDims)
3760 A.Dims = {(*Attr)[0], (*Attr)[1], (*Attr)[2]};
3761
3762 return A;
3763}
3764
3765std::optional<APFloat> evaluateRcp(const APFloat &Val) {
3766 const fltSemantics &Sem = Val.getSemantics();
3767
3768 // v_rcp_f16/bf16 are correctly rounded.
3769 if (&Sem == &APFloat::IEEEhalf() || &Sem == &APFloat::BFloat())
3770 return APFloat::getOne(Sem) / Val;
3771
3772 // v_rcp_f32/f64 always flush a denormal input to zero (preserving sign)
3773 // before reciprocating.
3774 APFloat Arg = Val;
3775 if (Arg.isDenormal())
3776 Arg = APFloat::getZero(Sem, Negative: Arg.isNegative());
3777
3778 APFloat Result = APFloat::getOne(Sem) / Arg;
3779
3780 // v_rcp_f32/f64 always flush a denormal result to zero (preserving sign).
3781 if (Result.isDenormal())
3782 Result = APFloat::getZero(Sem, Negative: Result.isNegative());
3783
3784 // v_rcp_f32/f64 only approximate the reciprocal, except for these special
3785 // cases where the result is exact.
3786 if (!Result.isZero() && !Result.isInfinity() && !Result.isNaN() &&
3787 !Result.isOne() && !Result.isMinusOne())
3788 return std::nullopt;
3789
3790 return Result;
3791}
3792
3793} // namespace AMDGPU
3794
3795raw_ostream &operator<<(raw_ostream &OS, const AMDGPU::TargetIDSetting S) {
3796 switch (S) {
3797 case (AMDGPU::TargetIDSetting::Unsupported):
3798 OS << "Unsupported";
3799 break;
3800 case (AMDGPU::TargetIDSetting::Any):
3801 OS << "Any";
3802 break;
3803 case (AMDGPU::TargetIDSetting::Off):
3804 OS << "Off";
3805 break;
3806 case (AMDGPU::TargetIDSetting::On):
3807 OS << "On";
3808 break;
3809 }
3810 return OS;
3811}
3812
3813} // namespace llvm
3814