1//===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUBaseInfo.h"
10#include "AMDGPU.h"
11#include "AMDGPUAsmUtils.h"
12#include "AMDKernelCodeT.h"
13#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14#include "Utils/AMDKernelCodeTUtils.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/BinaryFormat/ELF.h"
17#include "llvm/IR/Attributes.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/Function.h"
20#include "llvm/IR/GlobalValue.h"
21#include "llvm/IR/IntrinsicInst.h"
22#include "llvm/IR/IntrinsicsAMDGPU.h"
23#include "llvm/IR/IntrinsicsR600.h"
24#include "llvm/IR/LLVMContext.h"
25#include "llvm/IR/Metadata.h"
26#include "llvm/MC/MCInstrInfo.h"
27#include "llvm/MC/MCRegisterInfo.h"
28#include "llvm/MC/MCSubtargetInfo.h"
29#include "llvm/Support/CommandLine.h"
30#include "llvm/TargetParser/AMDGPUTargetParser.h"
31#include <optional>
32
33#define GET_INSTRINFO_NAMED_OPS
34#define GET_INSTRMAP_INFO
35#include "AMDGPUGenInstrInfo.inc"
36
37static llvm::cl::opt<unsigned> DefaultAMDHSACodeObjectVersion(
38 "amdhsa-code-object-version", llvm::cl::Hidden,
39 llvm::cl::init(Val: llvm::AMDGPU::AMDHSA_COV6),
40 llvm::cl::desc("Set default AMDHSA Code Object Version (module flag "
41 "or asm directive still take priority if present)"));
42
43namespace {
44
45/// \returns Bit mask for given bit \p Shift and bit \p Width.
46unsigned getBitMask(unsigned Shift, unsigned Width) {
47 return ((1 << Width) - 1) << Shift;
48}
49
50/// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
51///
52/// \returns Packed \p Dst.
53unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
54 unsigned Mask = getBitMask(Shift, Width);
55 return ((Src << Shift) & Mask) | (Dst & ~Mask);
56}
57
58/// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
59///
60/// \returns Unpacked bits.
61unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
62 return (Src & getBitMask(Shift, Width)) >> Shift;
63}
64
65/// \returns Vmcnt bit shift (lower bits).
66unsigned getVmcntBitShiftLo(unsigned VersionMajor) {
67 return VersionMajor >= 11 ? 10 : 0;
68}
69
70/// \returns Vmcnt bit width (lower bits).
71unsigned getVmcntBitWidthLo(unsigned VersionMajor) {
72 return VersionMajor >= 11 ? 6 : 4;
73}
74
75/// \returns Expcnt bit shift.
76unsigned getExpcntBitShift(unsigned VersionMajor) {
77 return VersionMajor >= 11 ? 0 : 4;
78}
79
80/// \returns Expcnt bit width.
81unsigned getExpcntBitWidth(unsigned VersionMajor) { return 3; }
82
83/// \returns Lgkmcnt bit shift.
84unsigned getLgkmcntBitShift(unsigned VersionMajor) {
85 return VersionMajor >= 11 ? 4 : 8;
86}
87
88/// \returns Lgkmcnt bit width.
89unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
90 return VersionMajor >= 10 ? 6 : 4;
91}
92
93/// \returns Vmcnt bit shift (higher bits).
94unsigned getVmcntBitShiftHi(unsigned VersionMajor) { return 14; }
95
96/// \returns Vmcnt bit width (higher bits).
97unsigned getVmcntBitWidthHi(unsigned VersionMajor) {
98 return (VersionMajor == 9 || VersionMajor == 10) ? 2 : 0;
99}
100
101/// \returns Loadcnt bit width
102unsigned getLoadcntBitWidth(unsigned VersionMajor) {
103 return VersionMajor >= 12 ? 6 : 0;
104}
105
106/// \returns Samplecnt bit width.
107unsigned getSamplecntBitWidth(unsigned VersionMajor) {
108 return VersionMajor >= 12 ? 6 : 0;
109}
110
111/// \returns Bvhcnt bit width.
112unsigned getBvhcntBitWidth(unsigned VersionMajor) {
113 return VersionMajor >= 12 ? 3 : 0;
114}
115
116/// \returns Dscnt bit width.
117unsigned getDscntBitWidth(unsigned VersionMajor) {
118 return VersionMajor >= 12 ? 6 : 0;
119}
120
121/// \returns Dscnt bit shift in combined S_WAIT instructions.
122unsigned getDscntBitShift(unsigned VersionMajor) { return 0; }
123
124/// \returns Storecnt or Vscnt bit width, depending on VersionMajor.
125unsigned getStorecntBitWidth(unsigned VersionMajor) {
126 return VersionMajor >= 10 ? 6 : 0;
127}
128
129/// \returns Kmcnt bit width.
130unsigned getKmcntBitWidth(unsigned VersionMajor) {
131 return VersionMajor >= 12 ? 5 : 0;
132}
133
134/// \returns Xcnt bit width.
135unsigned getXcntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
136 return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
137}
138
139/// \returns Asynccnt bit width.
140unsigned getAsynccntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
141 return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
142}
143
144/// \returns shift for Loadcnt/Storecnt in combined S_WAIT instructions.
145unsigned getLoadcntStorecntBitShift(unsigned VersionMajor) {
146 return VersionMajor >= 12 ? 8 : 0;
147}
148
149/// \returns barrier member count bit width in M0.
150unsigned getBarrierMemberCountBitWidth(unsigned VersionMajor) {
151 return VersionMajor >= 13 ? 8 : 6;
152}
153
154/// \returns VaSdst bit width
155inline unsigned getVaSdstBitWidth() { return 3; }
156
157/// \returns VaSdst bit shift
158inline unsigned getVaSdstBitShift() { return 9; }
159
160/// \returns VmVsrc bit width
161inline unsigned getVmVsrcBitWidth() { return 3; }
162
163/// \returns VmVsrc bit shift
164inline unsigned getVmVsrcBitShift() { return 2; }
165
166/// \returns VaVdst bit width
167inline unsigned getVaVdstBitWidth() { return 4; }
168
169/// \returns VaVdst bit shift
170inline unsigned getVaVdstBitShift() { return 12; }
171
172/// \returns VaVcc bit width
173inline unsigned getVaVccBitWidth() { return 1; }
174
175/// \returns VaVcc bit shift
176inline unsigned getVaVccBitShift() { return 1; }
177
178/// \returns SaSdst bit width
179inline unsigned getSaSdstBitWidth() { return 1; }
180
181/// \returns SaSdst bit shift
182inline unsigned getSaSdstBitShift() { return 0; }
183
184/// \returns VaSsrc width
185inline unsigned getVaSsrcBitWidth() { return 1; }
186
187/// \returns VaSsrc bit shift
188inline unsigned getVaSsrcBitShift() { return 8; }
189
190/// \returns HoldCnt bit shift
191inline unsigned getHoldCntWidth(unsigned VersionMajor, unsigned VersionMinor) {
192 static constexpr const unsigned MinMajor = 10;
193 static constexpr const unsigned MinMinor = 3;
194 return std::tie(args&: VersionMajor, args&: VersionMinor) >= std::tie(args: MinMajor, args: MinMinor)
195 ? 1
196 : 0;
197}
198
199/// \returns HoldCnt bit shift
200inline unsigned getHoldCntBitShift() { return 7; }
201
202} // end anonymous namespace
203
204namespace llvm {
205
206namespace AMDGPU {
207
208/// \returns true if the target supports signed immediate offset for SMRD
209/// instructions.
210bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
211 return isGFX9Plus(STI: ST);
212}
213
214/// \returns True if \p STI is AMDHSA.
215bool isHsaAbi(const MCSubtargetInfo &STI) {
216 return STI.getTargetTriple().getOS() == Triple::AMDHSA;
217}
218
219unsigned getAMDHSACodeObjectVersion(const Module &M) {
220 if (auto *Ver = mdconst::extract_or_null<ConstantInt>(
221 MD: M.getModuleFlag(Key: "amdhsa_code_object_version"))) {
222 return (unsigned)Ver->getZExtValue() / 100;
223 }
224
225 return getDefaultAMDHSACodeObjectVersion();
226}
227
228unsigned getDefaultAMDHSACodeObjectVersion() {
229 return DefaultAMDHSACodeObjectVersion;
230}
231
232unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion) {
233 switch (ABIVersion) {
234 case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
235 return 4;
236 case ELF::ELFABIVERSION_AMDGPU_HSA_V5:
237 return 5;
238 case ELF::ELFABIVERSION_AMDGPU_HSA_V6:
239 return 6;
240 default:
241 return getDefaultAMDHSACodeObjectVersion();
242 }
243}
244
245uint8_t getELFABIVersion(const Triple &T, unsigned CodeObjectVersion) {
246 if (T.getOS() != Triple::AMDHSA)
247 return 0;
248
249 switch (CodeObjectVersion) {
250 case 4:
251 return ELF::ELFABIVERSION_AMDGPU_HSA_V4;
252 case 5:
253 return ELF::ELFABIVERSION_AMDGPU_HSA_V5;
254 case 6:
255 return ELF::ELFABIVERSION_AMDGPU_HSA_V6;
256 default:
257 report_fatal_error(reason: "Unsupported AMDHSA Code Object Version " +
258 Twine(CodeObjectVersion));
259 }
260}
261
262unsigned getMultigridSyncArgImplicitArgPosition(unsigned CodeObjectVersion) {
263 switch (CodeObjectVersion) {
264 case AMDHSA_COV4:
265 return 48;
266 case AMDHSA_COV5:
267 case AMDHSA_COV6:
268 default:
269 return AMDGPU::ImplicitArg::MULTIGRID_SYNC_ARG_OFFSET;
270 }
271}
272
273// FIXME: All such magic numbers about the ABI should be in a
274// central TD file.
275unsigned getHostcallImplicitArgPosition(unsigned CodeObjectVersion) {
276 switch (CodeObjectVersion) {
277 case AMDHSA_COV4:
278 return 24;
279 case AMDHSA_COV5:
280 case AMDHSA_COV6:
281 default:
282 return AMDGPU::ImplicitArg::HOSTCALL_PTR_OFFSET;
283 }
284}
285
286unsigned getDefaultQueueImplicitArgPosition(unsigned CodeObjectVersion) {
287 switch (CodeObjectVersion) {
288 case AMDHSA_COV4:
289 return 32;
290 case AMDHSA_COV5:
291 case AMDHSA_COV6:
292 default:
293 return AMDGPU::ImplicitArg::DEFAULT_QUEUE_OFFSET;
294 }
295}
296
297unsigned getCompletionActionImplicitArgPosition(unsigned CodeObjectVersion) {
298 switch (CodeObjectVersion) {
299 case AMDHSA_COV4:
300 return 40;
301 case AMDHSA_COV5:
302 case AMDHSA_COV6:
303 default:
304 return AMDGPU::ImplicitArg::COMPLETION_ACTION_OFFSET;
305 }
306}
307
308#define GET_MIMGBaseOpcodesTable_IMPL
309#define GET_MIMGDimInfoTable_IMPL
310#define GET_MIMGInfoTable_IMPL
311#define GET_MIMGLZMappingTable_IMPL
312#define GET_MIMGMIPMappingTable_IMPL
313#define GET_MIMGBiasMappingTable_IMPL
314#define GET_MIMGOffsetMappingTable_IMPL
315#define GET_MIMGG16MappingTable_IMPL
316#define GET_MAIInstInfoTable_IMPL
317#define GET_WMMAInstInfoTable_IMPL
318#include "AMDGPUGenSearchableTables.inc"
319
320int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
321 unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc,
322 bool IndexedSamp) {
323 const MIMGInfo *Info =
324 getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, VDataDwords, VAddrDwords,
325 IndexedRsrc, IndexedSamp);
326 return Info ? Info->Opcode : -1;
327}
328
329const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
330 const MIMGInfo *Info = getMIMGInfo(Opcode: Opc);
331 return Info ? getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode) : nullptr;
332}
333
334int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
335 const MIMGInfo *OrigInfo = getMIMGInfo(Opcode: Opc);
336 const MIMGInfo *NewInfo = getMIMGOpcodeHelper(
337 BaseOpcode: OrigInfo->BaseOpcode, MIMGEncoding: OrigInfo->MIMGEncoding, VDataDwords: NewChannels,
338 VAddrDwords: OrigInfo->VAddrDwords, IndexedRsrc: OrigInfo->IndexedRsrc, IndexedSamp: OrigInfo->IndexedSamp);
339 return NewInfo ? NewInfo->Opcode : -1;
340}
341
342unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
343 const MIMGDimInfo *Dim, bool IsA16,
344 bool IsG16Supported) {
345 unsigned AddrWords = BaseOpcode->NumExtraArgs;
346 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
347 (BaseOpcode->LodOrClampOrMip ? 1 : 0);
348 if (IsA16)
349 AddrWords += divideCeil(Numerator: AddrComponents, Denominator: 2);
350 else
351 AddrWords += AddrComponents;
352
353 // Note: For subtargets that support A16 but not G16, enabling A16 also
354 // enables 16 bit gradients.
355 // For subtargets that support A16 (operand) and G16 (done with a different
356 // instruction encoding), they are independent.
357
358 if (BaseOpcode->Gradients) {
359 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16)
360 // There are two gradients per coordinate, we pack them separately.
361 // For the 3d case,
362 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv)
363 AddrWords += alignTo<2>(Value: Dim->NumGradients / 2);
364 else
365 AddrWords += Dim->NumGradients;
366 }
367 return AddrWords;
368}
369
370struct MUBUFInfo {
371 uint32_t Opcode;
372 uint32_t BaseOpcode;
373 uint8_t elements;
374 bool has_vaddr;
375 bool has_srsrc;
376 bool has_soffset;
377 bool IsBufferInv;
378 bool tfe;
379};
380
381struct MTBUFInfo {
382 uint32_t Opcode;
383 uint32_t BaseOpcode;
384 uint8_t elements;
385 bool has_vaddr;
386 bool has_srsrc;
387 bool has_soffset;
388};
389
390struct SMInfo {
391 uint32_t Opcode;
392};
393
394struct VOPInfo {
395 uint32_t Opcode;
396};
397
398struct VOPC64DPPInfo {
399 uint32_t Opcode;
400};
401
402struct VOPCDPPAsmOnlyInfo {
403 uint32_t Opcode;
404};
405
406struct VOPDComponentInfo {
407 uint16_t BaseVOP;
408 uint16_t VOPDOp;
409};
410
411struct VOPDInfo {
412 uint32_t Opcode;
413 uint16_t OpX;
414 uint16_t OpY;
415 uint16_t Subtarget;
416 bool VOPD3;
417};
418
419struct VOPTrue16Info {
420 uint32_t Opcode;
421};
422
423struct VOPDXYInfo {
424 uint16_t VOPDXYKey;
425 bool IsX;
426 bool IsY;
427};
428
429#define GET_FP4FP8DstByteSelTable_DECL
430#define GET_FP4FP8DstByteSelTable_IMPL
431
432struct DPMACCInstructionInfo {
433 uint32_t Opcode;
434};
435
436struct FP4FP8DstByteSelInfo {
437 uint32_t Opcode;
438 bool HasFP8DstByteSel;
439 bool HasFP4DstByteSel;
440};
441
442struct VOP3PHasNoOpSelHiInfo {
443 uint32_t Opcode;
444};
445
446#define GET_DPMACCInstructionTable_DECL
447#define GET_DPMACCInstructionTable_IMPL
448#define GET_MTBUFInfoTable_DECL
449#define GET_MTBUFInfoTable_IMPL
450#define GET_MUBUFInfoTable_DECL
451#define GET_MUBUFInfoTable_IMPL
452#define GET_SMInfoTable_DECL
453#define GET_SMInfoTable_IMPL
454#define GET_VOP1InfoTable_DECL
455#define GET_VOP1InfoTable_IMPL
456#define GET_VOP2InfoTable_DECL
457#define GET_VOP2InfoTable_IMPL
458#define GET_VOP3InfoTable_DECL
459#define GET_VOP3InfoTable_IMPL
460#define GET_VOPC64DPPTable_DECL
461#define GET_VOPC64DPPTable_IMPL
462#define GET_VOPC64DPP8Table_DECL
463#define GET_VOPC64DPP8Table_IMPL
464#define GET_VOPCAsmOnlyInfoTable_DECL
465#define GET_VOPCAsmOnlyInfoTable_IMPL
466#define GET_VOP3CAsmOnlyInfoTable_DECL
467#define GET_VOP3CAsmOnlyInfoTable_IMPL
468#define GET_VOPDComponentTable_DECL
469#define GET_VOPDComponentTable_IMPL
470#define GET_VOPDPairs_DECL
471#define GET_VOPDPairs_IMPL
472#define GET_VOPDXYTable_DECL
473#define GET_VOPDXYTable_IMPL
474#define GET_VOPTrue16Table_DECL
475#define GET_VOPTrue16Table_IMPL
476#define GET_VOP3PHasNoOpSelHiTable_DECL
477#define GET_VOP3PHasNoOpSelHiTable_IMPL
478#define GET_True16D16Table_IMPL
479#define GET_WMMAOpcode2AddrMappingTable_DECL
480#define GET_WMMAOpcode2AddrMappingTable_IMPL
481#define GET_WMMAOpcode3AddrMappingTable_DECL
482#define GET_WMMAOpcode3AddrMappingTable_IMPL
483#define GET_getMFMA_F8F6F4_WithSize_DECL
484#define GET_getMFMA_F8F6F4_WithSize_IMPL
485#define GET_isMFMA_F8F6F4Table_IMPL
486#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL
487#define GET_VLdStIdxOpcodeInfoTable_IMPL
488
489#include "AMDGPUGenSearchableTables.inc"
490
491const VLdStIdxOpcodeInfo *
492getVLdStIdxOpcodeInfoByKey(unsigned BitWidth, bool IsStore, bool IsGPRIdx) {
493 assert(isUInt<16>(BitWidth) && "the table's key is 16 bits");
494 return getVLdStIdxOpcodeInfoByKeyImpl(BitWidth, IsStore, IsGPRIdx);
495}
496
497int getMTBUFBaseOpcode(unsigned Opc) {
498 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opcode: Opc);
499 return Info ? Info->BaseOpcode : -1;
500}
501
502int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
503 const MTBUFInfo *Info =
504 getMTBUFInfoFromBaseOpcodeAndElements(BaseOpcode: BaseOpc, elements: Elements);
505 return Info ? Info->Opcode : -1;
506}
507
508int getMTBUFElements(unsigned Opc) {
509 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
510 return Info ? Info->elements : 0;
511}
512
513bool getMTBUFHasVAddr(unsigned Opc) {
514 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
515 return Info && Info->has_vaddr;
516}
517
518bool getMTBUFHasSrsrc(unsigned Opc) {
519 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
520 return Info && Info->has_srsrc;
521}
522
523bool getMTBUFHasSoffset(unsigned Opc) {
524 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
525 return Info && Info->has_soffset;
526}
527
528int getMUBUFBaseOpcode(unsigned Opc) {
529 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opcode: Opc);
530 return Info ? Info->BaseOpcode : -1;
531}
532
533int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
534 const MUBUFInfo *Info =
535 getMUBUFInfoFromBaseOpcodeAndElements(BaseOpcode: BaseOpc, elements: Elements);
536 return Info ? Info->Opcode : -1;
537}
538
539int getMUBUFElements(unsigned Opc) {
540 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
541 return Info ? Info->elements : 0;
542}
543
544bool getMUBUFHasVAddr(unsigned Opc) {
545 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
546 return Info && Info->has_vaddr;
547}
548
549bool getMUBUFHasSrsrc(unsigned Opc) {
550 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
551 return Info && Info->has_srsrc;
552}
553
554bool getMUBUFHasSoffset(unsigned Opc) {
555 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
556 return Info && Info->has_soffset;
557}
558
559bool getMUBUFIsBufferInv(unsigned Opc) {
560 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
561 return Info && Info->IsBufferInv;
562}
563
564bool getMUBUFTfe(unsigned Opc) {
565 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
566 return Info && Info->tfe;
567}
568
569bool getSMEMIsBuffer(unsigned Opc) {
570 return isSMEMOpcodeHelper(Opcode: Opc) != nullptr;
571}
572
573bool getVOP1IsSingle(unsigned Opc) {
574 return isVOP1SingleOpcodeHelper(Opcode: Opc) != nullptr;
575}
576
577bool getVOP2IsSingle(unsigned Opc) {
578 return isVOP2SingleOpcodeHelper(Opcode: Opc) != nullptr;
579}
580
581bool getVOP3IsSingle(unsigned Opc) {
582 return isVOP3SingleOpcodeHelper(Opcode: Opc) != nullptr;
583}
584
585bool isVOPC64DPP(unsigned Opc) {
586 return isVOPC64DPPOpcodeHelper(Opcode: Opc) || isVOPC64DPP8OpcodeHelper(Opcode: Opc);
587}
588
589bool isVOPCAsmOnly(unsigned Opc) { return isVOPCAsmOnlyOpcodeHelper(Opcode: Opc); }
590
591bool getMAIIsDGEMM(unsigned Opc) {
592 const MAIInstInfo *Info = getMAIInstInfoHelper(Opcode: Opc);
593 return Info && Info->is_dgemm;
594}
595
596bool getMAIIsGFX940XDL(unsigned Opc) {
597 const MAIInstInfo *Info = getMAIInstInfoHelper(Opcode: Opc);
598 return Info && Info->is_gfx940_xdl;
599}
600
601bool getWMMAIsXDL(unsigned Opc) {
602 const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opcode: Opc);
603 return Info ? Info->is_wmma_xdl : false;
604}
605
606bool getHasMatrixScale(unsigned Opc) {
607 const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opcode: Opc);
608 return Info && Info->HasMatrixScale;
609}
610
611uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal) {
612 switch (EncodingVal) {
613 case MFMAScaleFormats::FP6_E2M3:
614 case MFMAScaleFormats::FP6_E3M2:
615 return 6;
616 case MFMAScaleFormats::FP4_E2M1:
617 return 4;
618 case MFMAScaleFormats::FP8_E4M3:
619 case MFMAScaleFormats::FP8_E5M2:
620 default:
621 return 8;
622 }
623
624 llvm_unreachable("covered switch over mfma scale formats");
625}
626
627const MFMA_F8F6F4_Info *getMFMA_F8F6F4_WithFormatArgs(unsigned CBSZ,
628 unsigned BLGP,
629 unsigned F8F8Opcode) {
630 uint8_t SrcANumRegs = mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: CBSZ);
631 uint8_t SrcBNumRegs = mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: BLGP);
632 return getMFMA_F8F6F4_InstWithNumRegs(NumRegsSrcA: SrcANumRegs, NumRegsSrcB: SrcBNumRegs, F8F8Opcode);
633}
634
635uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt) {
636 switch (Fmt) {
637 case WMMA::MATRIX_FMT_FP8:
638 case WMMA::MATRIX_FMT_BF8:
639 return 16;
640 case WMMA::MATRIX_FMT_FP6:
641 case WMMA::MATRIX_FMT_BF6:
642 return 12;
643 case WMMA::MATRIX_FMT_FP4:
644 return 8;
645 }
646
647 llvm_unreachable("covered switch over wmma scale formats");
648}
649
650const MFMA_F8F6F4_Info *getWMMA_F8F6F4_WithFormatArgs(unsigned FmtA,
651 unsigned FmtB,
652 unsigned F8F8Opcode) {
653 uint8_t SrcANumRegs = wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtA);
654 uint8_t SrcBNumRegs = wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtB);
655 return getMFMA_F8F6F4_InstWithNumRegs(NumRegsSrcA: SrcANumRegs, NumRegsSrcB: SrcBNumRegs, F8F8Opcode);
656}
657
658bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale,
659 unsigned BFmt, unsigned BScale) {
660 auto isValid = [](unsigned Fmt, unsigned Scale) -> bool {
661 switch (Fmt) {
662 case WMMA::MATRIX_FMT_FP8:
663 case WMMA::MATRIX_FMT_BF8:
664 case WMMA::MATRIX_FMT_FP6:
665 case WMMA::MATRIX_FMT_BF6:
666 if (Scale != WMMA::MATRIX_SCALE_FMT_E8)
667 return false;
668 break;
669 case WMMA::MATRIX_FMT_FP4:
670 if (Scale != WMMA::MATRIX_SCALE_FMT_E8 &&
671 Scale != WMMA::MATRIX_SCALE_FMT_E5M3 &&
672 Scale != WMMA::MATRIX_SCALE_FMT_E4M3)
673 return false;
674 break;
675 }
676 return true;
677 };
678
679 if (!isValid(AFmt, AScale) || !isValid(BFmt, BScale))
680 return false;
681
682 if (AFmt == WMMA::MATRIX_FMT_FP4 && BFmt == WMMA::MATRIX_FMT_FP4 &&
683 AScale != BScale)
684 return false;
685
686 return true;
687}
688
689unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST) {
690 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX13Insts))
691 return SIEncodingFamily::GFX13;
692 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX1250Insts))
693 return SIEncodingFamily::GFX1250;
694 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX12Insts))
695 return SIEncodingFamily::GFX12;
696 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX11_7Insts))
697 return SIEncodingFamily::GFX1170;
698 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX11Insts))
699 return SIEncodingFamily::GFX11;
700 llvm_unreachable("Subtarget generation does not support VOPD!");
701}
702
703CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3) {
704 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
705 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
706 // Normalize through VOPDComponentTable so that e32 and e64 variants
707 // of the same logical opcode all share a single entry.
708 const VOPDComponentInfo *Info = getVOPDComponentHelper(BaseVOP: Opc);
709 if (!Info)
710 return {.X: false, .Y: false};
711 unsigned Key =
712 (Info->VOPDOp << 5) | (EncodingFamily << 1) | (VOPD3 ? 1u : 0u);
713 const VOPDXYInfo *XYInfo = getVOPDXYInfo(VOPDXYKey: static_cast<uint16_t>(Key));
714 if (!XYInfo)
715 return {.X: false, .Y: false};
716 return {.X: XYInfo->IsX, .Y: XYInfo->IsY};
717}
718
719unsigned getVOPDOpcode(unsigned Opc, bool VOPD3) {
720 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
721 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
722 const VOPDComponentInfo *Info = getVOPDComponentHelper(BaseVOP: Opc);
723 return Info ? Info->VOPDOp : ~0u;
724}
725
726bool isVOPD(unsigned Opc) {
727 return AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::src0X);
728}
729
730bool isMAC(unsigned Opc) {
731 return Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 ||
732 Opc == AMDGPU::V_MAC_F32_e64_gfx10 ||
733 Opc == AMDGPU::V_MAC_F32_e64_vi ||
734 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx6_gfx7 ||
735 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx10 ||
736 Opc == AMDGPU::V_MAC_F16_e64_vi ||
737 Opc == AMDGPU::V_FMAC_F64_e64_gfx90a ||
738 Opc == AMDGPU::V_FMAC_F64_e64_gfx12 ||
739 Opc == AMDGPU::V_FMAC_F64_e64_gfx13 ||
740 Opc == AMDGPU::V_FMAC_F32_e64_gfx10 ||
741 Opc == AMDGPU::V_FMAC_F32_e64_gfx11 ||
742 Opc == AMDGPU::V_FMAC_F32_e64_gfx12 ||
743 Opc == AMDGPU::V_FMAC_F32_e64_gfx13 ||
744 Opc == AMDGPU::V_FMAC_F32_e64_vi ||
745 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64_gfx10 ||
746 Opc == AMDGPU::V_FMAC_DX9_ZERO_F32_e64_gfx11 ||
747 Opc == AMDGPU::V_FMAC_F16_e64_gfx10 ||
748 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx11 ||
749 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx11 ||
750 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx12 ||
751 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx12 ||
752 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx13 ||
753 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx13 ||
754 Opc == AMDGPU::V_DOT2C_F32_F16_e64_vi ||
755 Opc == AMDGPU::V_DOT2C_F32_BF16_e64_vi ||
756 Opc == AMDGPU::V_DOT2C_I32_I16_e64_vi ||
757 Opc == AMDGPU::V_DOT4C_I32_I8_e64_vi ||
758 Opc == AMDGPU::V_DOT8C_I32_I4_e64_vi;
759}
760
761bool isPermlane16(unsigned Opc) {
762 return Opc == AMDGPU::V_PERMLANE16_B32_gfx10 ||
763 Opc == AMDGPU::V_PERMLANEX16_B32_gfx10 ||
764 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx11 ||
765 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx11 ||
766 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx12 ||
767 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx13 ||
768 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx12 ||
769 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx13 ||
770 Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx12 ||
771 Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx13 ||
772 Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx12 ||
773 Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx13;
774}
775
776bool isCvt_F32_Fp8_Bf8_e64(unsigned Opc) {
777 return Opc == AMDGPU::V_CVT_F32_BF8_e64_gfx12 ||
778 Opc == AMDGPU::V_CVT_F32_FP8_e64_gfx12 ||
779 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp_gfx12 ||
780 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp_gfx12 ||
781 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp8_gfx12 ||
782 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp8_gfx12 ||
783 Opc == AMDGPU::V_CVT_PK_F32_BF8_fake16_e64_gfx12 ||
784 Opc == AMDGPU::V_CVT_PK_F32_FP8_fake16_e64_gfx12 ||
785 Opc == AMDGPU::V_CVT_PK_F32_BF8_t16_e64_gfx12 ||
786 Opc == AMDGPU::V_CVT_PK_F32_FP8_t16_e64_gfx12;
787}
788
789bool isGenericAtomic(unsigned Opc) {
790 return Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP ||
791 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD ||
792 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB ||
793 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN ||
794 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN ||
795 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX ||
796 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX ||
797 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND ||
798 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR ||
799 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR ||
800 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC ||
801 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC ||
802 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD ||
803 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN ||
804 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX ||
805 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP ||
806 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32 ||
807 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32 ||
808 Opc == AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG;
809}
810
811bool isAsyncStore(unsigned Opc) {
812 return Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_gfx1250 ||
813 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_gfx1250 ||
814 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_gfx1250 ||
815 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_gfx1250 ||
816 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR_gfx1250 ||
817 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR_gfx1250 ||
818 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR_gfx1250 ||
819 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR_gfx1250;
820}
821
822bool isTensorStore(unsigned Opc) {
823 return Opc == TENSOR_STORE_FROM_LDS_d2_gfx1250 ||
824 Opc == TENSOR_STORE_FROM_LDS_d4_gfx1250;
825}
826
827unsigned getTemporalHintType(const MCInstrDesc TID) {
828 if (SIInstrFlags::isAtomic(O: TID))
829 return CPol::TH_TYPE_ATOMIC;
830 unsigned Opc = TID.getOpcode();
831 // Async and Tensor store should have the temporal hint type of TH_TYPE_STORE
832 if (TID.mayStore() &&
833 (isAsyncStore(Opc) || isTensorStore(Opc) || !TID.mayLoad()))
834 return CPol::TH_TYPE_STORE;
835
836 // This will default to returning TH_TYPE_LOAD when neither MayStore nor
837 // MayLoad flag is present which is the case with instructions like
838 // image_get_resinfo.
839 return CPol::TH_TYPE_LOAD;
840}
841
842bool isTrue16Inst(unsigned Opc) { return isTrue16Opcode(Opcode: Opc) != nullptr; }
843
844bool hasNoOpSelHi(unsigned Opc) {
845 return getVOP3PHasNoOpSelHiHelper(Opcode: Opc) != nullptr;
846}
847
848FPType getFPDstSelType(unsigned Opc) {
849 const FP4FP8DstByteSelInfo *Info = getFP4FP8DstByteSelHelper(Opcode: Opc);
850 if (!Info)
851 return FPType::None;
852 if (Info->HasFP8DstByteSel)
853 return FPType::FP8;
854 if (Info->HasFP4DstByteSel)
855 return FPType::FP4;
856
857 return FPType::None;
858}
859
860bool isDPMACCInstruction(unsigned Opc) {
861 return isDPMACCInstructionHelper(Opcode: Opc) != nullptr;
862}
863
864unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc) {
865 const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom2AddrOpcode(Opcode2Addr: Opc);
866 return Info ? Info->Opcode3Addr : ~0u;
867}
868
869// Wrapper for Tablegen'd function. enum Subtarget is not defined in any
870// header files, so we need to wrap it in a function that takes unsigned
871// instead.
872int32_t getMCOpcode(uint32_t Opcode, unsigned Gen) {
873 return getMCOpcodeGen(Opcode, inSubtarget: static_cast<Subtarget>(Gen));
874}
875
876unsigned getBitOp2(unsigned Opc) {
877 switch (Opc) {
878 default:
879 return 0;
880 case AMDGPU::V_AND_B32_e32:
881 return 0x40;
882 case AMDGPU::V_OR_B32_e32:
883 return 0x54;
884 case AMDGPU::V_XOR_B32_e32:
885 return 0x14;
886 case AMDGPU::V_XNOR_B32_e32:
887 return 0x41;
888 }
889}
890
891int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily,
892 bool VOPD3) {
893 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc: OpY) : 0;
894 OpY = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : OpY;
895 const VOPDInfo *Info = getVOPDInfoFromComponentOpcodes(
896 OpX: static_cast<uint8_t>(OpX), OpY: static_cast<uint8_t>(OpY),
897 SubTgt: static_cast<uint8_t>(EncodingFamily), VOPD3);
898 return Info ? Info->Opcode : -1;
899}
900
901std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode) {
902 const VOPDInfo *Info = getVOPDOpcodeHelper(Opcode: VOPDOpcode);
903 assert(Info);
904 const auto *OpX = getVOPDBaseFromComponent(VOPDOp: Info->OpX);
905 const auto *OpY = getVOPDBaseFromComponent(VOPDOp: Info->OpY);
906 assert(OpX && OpY);
907 return {OpX->BaseVOP, OpY->BaseVOP};
908}
909
910namespace VOPD {
911
912ComponentProps::ComponentProps(const MCInstrDesc &OpDesc, bool VOP3Layout) {
913 assert(OpDesc.getNumDefs() == Component::DST_NUM);
914
915 assert(OpDesc.getOperandConstraint(Component::SRC0, MCOI::TIED_TO) == -1);
916 assert(OpDesc.getOperandConstraint(Component::SRC1, MCOI::TIED_TO) == -1);
917 auto TiedIdx = OpDesc.getOperandConstraint(OpNum: Component::SRC2, Constraint: MCOI::TIED_TO);
918 assert(TiedIdx == -1 || TiedIdx == Component::DST);
919 HasSrc2Acc = TiedIdx != -1;
920 Opcode = OpDesc.getOpcode();
921
922 IsVOP3 = VOP3Layout || SIInstrFlags::isVOP3(O: OpDesc);
923 SrcOperandsNum = AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src2) ? 3
924 : AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::imm) ? 3
925 : AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src1) ? 2
926 : 1;
927 assert(SrcOperandsNum <= Component::MAX_SRC_NUM);
928
929 if (Opcode == AMDGPU::V_CNDMASK_B32_e32 ||
930 Opcode == AMDGPU::V_CNDMASK_B32_e64) {
931 // CNDMASK is an awkward exception, it has FP modifiers, but not FP
932 // operands.
933 NumVOPD3Mods = 2;
934 if (IsVOP3)
935 SrcOperandsNum = 3;
936 } else if (Opcode == AMDGPU::V_DOT2_F32_F16 ||
937 Opcode == AMDGPU::V_DOT2_F32_BF16) {
938 // VOP3P opcodes that have VOPD but don't have VOP2 version. Using VOPD3
939 // path in getIndexOfSrcInMCOperands to get correct src operand indexes,
940 // but generating VOPD, not VOPD3.
941 NumVOPD3Mods = SrcOperandsNum;
942 } else if (isSISrcFPOperand(Desc: OpDesc,
943 OpNo: getNamedOperandIdx(Opcode, Name: OpName::src0))) {
944 // All FP VOPD instructions have Neg modifiers for all operands except
945 // for tied src2.
946 NumVOPD3Mods = SrcOperandsNum;
947 if (HasSrc2Acc)
948 --NumVOPD3Mods;
949 }
950
951 if (SIInstrFlags::isVOP3(O: OpDesc))
952 return;
953
954 auto OperandsNum = OpDesc.getNumOperands();
955 unsigned CompOprIdx;
956 for (CompOprIdx = Component::SRC1; CompOprIdx < OperandsNum; ++CompOprIdx) {
957 if (OpDesc.operands()[CompOprIdx].OperandType == AMDGPU::OPERAND_KIMM32) {
958 MandatoryLiteralIdx = CompOprIdx;
959 break;
960 }
961 }
962}
963
964int ComponentProps::getBitOp3OperandIdx() const {
965 return getNamedOperandIdx(Opcode, Name: OpName::bitop3);
966}
967
968unsigned ComponentInfo::getIndexInParsedOperands(unsigned CompOprIdx) const {
969 assert(CompOprIdx < Component::MAX_OPR_NUM);
970
971 if (CompOprIdx == Component::DST)
972 return getIndexOfDstInParsedOperands();
973
974 auto CompSrcIdx = CompOprIdx - Component::DST_NUM;
975 if (CompSrcIdx < getCompParsedSrcOperandsNum())
976 return getIndexOfSrcInParsedOperands(CompSrcIdx);
977
978 // The specified operand does not exist.
979 return 0;
980}
981
982std::optional<unsigned> InstInfo::getInvalidCompOperandIndex(
983 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
984 const MCRegisterInfo &MRI, bool SkipSrc, bool AllowSameVGPR, bool VOPD3,
985 bool HasGFX11InterlockHazard) const {
986
987 auto OpXRegs = getRegIndices(ComponentIdx: ComponentIndex::X, GetRegIdx,
988 VOPD3: CompInfo[ComponentIndex::X].isVOP3());
989 auto OpYRegs = getRegIndices(ComponentIdx: ComponentIndex::Y, GetRegIdx,
990 VOPD3: CompInfo[ComponentIndex::Y].isVOP3());
991
992 const auto banksOverlap = [&MRI](MCRegister X, MCRegister Y,
993 unsigned BanksMask) -> bool {
994 MCRegister BaseX = MRI.getSubReg(Reg: X, Idx: AMDGPU::sub0);
995 MCRegister BaseY = MRI.getSubReg(Reg: Y, Idx: AMDGPU::sub0);
996 if (!BaseX)
997 BaseX = X;
998 if (!BaseY)
999 BaseY = Y;
1000 if ((BaseX.id() & BanksMask) == (BaseY.id() & BanksMask))
1001 return true;
1002 if (BaseX != X /* This is 64-bit register */ &&
1003 ((BaseX.id() + 1) & BanksMask) == (BaseY.id() & BanksMask))
1004 return true;
1005 if (BaseY != Y &&
1006 (BaseX.id() & BanksMask) == ((BaseY.id() + 1) & BanksMask))
1007 return true;
1008
1009 // If both are 64-bit bank conflict will be detected yet while checking
1010 // the first subreg.
1011 return false;
1012 };
1013
1014 unsigned CompOprIdx;
1015 for (CompOprIdx = 0; CompOprIdx < Component::MAX_OPR_NUM; ++CompOprIdx) {
1016 unsigned BanksMasks = VOPD3 ? VOPD3_VGPR_BANK_MASKS[CompOprIdx]
1017 : HasGFX11InterlockHazard
1018 ? VOPD_GFX11_VGPR_BANK_MASKS[CompOprIdx]
1019 : VOPD_VGPR_BANK_MASKS[CompOprIdx];
1020 if (!OpXRegs[CompOprIdx] || !OpYRegs[CompOprIdx])
1021 continue;
1022
1023 if (getVGPREncodingMSBs(Reg: OpXRegs[CompOprIdx], MRI) !=
1024 getVGPREncodingMSBs(Reg: OpYRegs[CompOprIdx], MRI))
1025 return CompOprIdx;
1026
1027 if (SkipSrc && CompOprIdx >= Component::DST_NUM)
1028 continue;
1029
1030 if (CompOprIdx < Component::DST_NUM) {
1031 // Even if we do not check vdst parity, vdst operands still shall not
1032 // overlap.
1033 if (MRI.regsOverlap(RegA: OpXRegs[CompOprIdx], RegB: OpYRegs[CompOprIdx]))
1034 return CompOprIdx;
1035 if (VOPD3) // No need to check dst parity.
1036 continue;
1037 }
1038
1039 if (banksOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx], BanksMasks) &&
1040 (!AllowSameVGPR || CompOprIdx < Component::DST_NUM ||
1041 OpXRegs[CompOprIdx] != OpYRegs[CompOprIdx]))
1042 return CompOprIdx;
1043 }
1044
1045 return {};
1046}
1047
1048// Return an array of VGPR registers [DST,SRC0,SRC1,SRC2] used
1049// by the specified component. If an operand is unused
1050// or is not a VGPR, the corresponding value is 0.
1051//
1052// GetRegIdx(Component, MCOperandIdx) must return a VGPR register index
1053// for the specified component and MC operand. The callback must return 0
1054// if the operand is not a register or not a VGPR.
1055InstInfo::RegIndices
1056InstInfo::getRegIndices(unsigned CompIdx,
1057 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
1058 bool VOPD3) const {
1059 assert(CompIdx < COMPONENTS_NUM);
1060
1061 const auto &Comp = CompInfo[CompIdx];
1062 InstInfo::RegIndices RegIndices;
1063
1064 RegIndices[DST] = GetRegIdx(CompIdx, Comp.getIndexOfDstInMCOperands());
1065
1066 for (unsigned CompOprIdx : {SRC0, SRC1, SRC2}) {
1067 unsigned CompSrcIdx = CompOprIdx - DST_NUM;
1068 RegIndices[CompOprIdx] =
1069 Comp.hasRegSrcOperand(CompSrcIdx)
1070 ? GetRegIdx(CompIdx,
1071 Comp.getIndexOfSrcInMCOperands(CompSrcIdx, VOPD3))
1072 : MCRegister();
1073 }
1074 return RegIndices;
1075}
1076
1077} // namespace VOPD
1078
1079VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY) {
1080 return VOPD::InstInfo(OpX, OpY);
1081}
1082
1083VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,
1084 const MCInstrInfo *InstrInfo) {
1085 auto [OpX, OpY] = getVOPDComponents(VOPDOpcode);
1086 const auto &OpXDesc = InstrInfo->get(Opcode: OpX);
1087 const auto &OpYDesc = InstrInfo->get(Opcode: OpY);
1088 bool VOPD3 = SIInstrFlags::isVOPD3(O: *InstrInfo, O: VOPDOpcode);
1089 VOPD::ComponentInfo OpXInfo(OpXDesc, VOPD::ComponentKind::COMPONENT_X, VOPD3);
1090 VOPD::ComponentInfo OpYInfo(OpYDesc, OpXInfo, VOPD3);
1091 return VOPD::InstInfo(OpXInfo, OpYInfo);
1092}
1093
1094TargetID createAMDGPUTargetID(const MCSubtargetInfo &STI,
1095 StringRef FeatureString) {
1096 // In codegen the mode comes from module flags and FeatureString is empty, so
1097 // the processor defaults apply. The assembler has no target directive, so it
1098 // pins the mode via the +xnack/-xnack/+sramecc/-sramecc feature string.
1099 return TargetID::createFromSubtargetFeatures(TT: STI.getTargetTriple(),
1100 CPU: STI.getCPU(), FeatureString);
1101}
1102
1103namespace IsaInfo {
1104
1105unsigned getInstCacheLineSize(const MCSubtargetInfo &STI) {
1106 if (STI.getFeatureBits().test(I: FeatureInstCacheLineSize128))
1107 return 128;
1108 if (STI.getFeatureBits().test(I: FeatureInstCacheLineSize64))
1109 return 64;
1110 return 64;
1111}
1112
1113unsigned getWavefrontSize(const MCSubtargetInfo &STI) {
1114 if (STI.getFeatureBits().test(I: FeatureWavefrontSize16))
1115 return 16;
1116 if (STI.getFeatureBits().test(I: FeatureWavefrontSize32))
1117 return 32;
1118
1119 return 64;
1120}
1121
1122// Maximum LDS a single work-group can address. This is a fixed HW cap. It does
1123// not depend on how many SIMDs a work-group runs on.
1124static unsigned getMaxHWAddressableLocalMemorySize(const MCSubtargetInfo &STI) {
1125 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize32768))
1126 return 32768;
1127 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize65536))
1128 return 65536;
1129 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize163840))
1130 return 163840;
1131 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize196608))
1132 return 196608;
1133 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize327680))
1134 return 327680;
1135 return 32768;
1136}
1137
1138// Total physical size of LDS on the block, in bytes. On targets with
1139// FeatureHalfAddressablePhysicalLocalMemory the physical block is twice the
1140// addressable size (gfx6: 64 KiB physical and 32 KiB addressable;
1141// gfx10/11/12: 128 KiB physical and 64 KiB addressable). On other targets it is
1142// equal to the addressable size.
1143static unsigned getPhysicalLocalMemorySize(const MCSubtargetInfo &STI) {
1144 unsigned Addressable = getMaxHWAddressableLocalMemorySize(STI);
1145 if (STI.getFeatureBits().test(I: FeatureHalfAddressablePhysicalLocalMemory))
1146 return 2 * Addressable;
1147 return Addressable;
1148}
1149
1150// Sizes in use, by generation (addressable / physical block):
1151// gfx6 : 32 KiB addressable, 64 KiB physical block
1152// gfx7 / gfx8 / gfx9: 64 KiB
1153// gfx9.5 (gfx950) : 160 KiB
1154// gfx10 / 11 / 12 : 64 KiB addressable, 128 KiB physical block
1155// gfx12.5 (gfx1250) : 320 KiB (always runs on four SIMDs)
1156// gfx13 : 192 KiB on four SIMDs, 96 KiB on two
1157// Total available in the current mode. The physical size is halved when a
1158// work-group runs on two SIMDs.
1159unsigned getLocalMemorySize(const MCSubtargetInfo &STI) {
1160 unsigned Size = getPhysicalLocalMemorySize(STI);
1161 if (!isFullSIMDMode(STI))
1162 Size /= 2;
1163 return Size;
1164}
1165
1166// What one work-group can allocate in the current mode. This is the HW
1167// addressable cap, but never more than the total available in the current mode.
1168unsigned getAddressableLocalMemorySize(const MCSubtargetInfo &STI) {
1169 return std::min(a: getMaxHWAddressableLocalMemorySize(STI),
1170 b: getLocalMemorySize(STI));
1171}
1172
1173unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo &STI,
1174 unsigned FlatWorkGroupSize) {
1175 assert(FlatWorkGroupSize != 0);
1176 if (!STI.getTargetTriple().isAMDGCN())
1177 return 8;
1178 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1179 unsigned MaxWaves =
1180 getMaxWavesPerEU(AK: Kind) * getNumWorkGroupSIMDs(FullSIMDMode: isFullSIMDMode(STI));
1181 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
1182 if (N == 1) {
1183 // Single-wave workgroups don't consume barrier resources.
1184 return MaxWaves;
1185 }
1186
1187 unsigned MaxBarriers = 16;
1188 if (isGFX10Plus(STI) && !STI.getFeatureBits().test(I: FeatureCuMode))
1189 MaxBarriers = 32;
1190
1191 return std::min(a: MaxWaves / N, b: MaxBarriers);
1192}
1193
1194unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo &STI,
1195 unsigned FlatWorkGroupSize) {
1196 return divideCeil(Numerator: getWavesPerWorkGroup(STI, FlatWorkGroupSize),
1197 Denominator: getNumWorkGroupSIMDs(FullSIMDMode: isFullSIMDMode(STI)));
1198}
1199
1200unsigned getWavesPerWorkGroup(const MCSubtargetInfo &STI,
1201 unsigned FlatWorkGroupSize) {
1202 return divideCeil(Numerator: FlatWorkGroupSize, Denominator: getWavefrontSize(STI));
1203}
1204
1205unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI) { return 8; }
1206
1207// Per-wave SGPRs reserved for the trap handler when enabled.
1208static unsigned getSGPRTrapHandlerReserve(const MCSubtargetInfo &STI) {
1209 return STI.getFeatureBits().test(I: FeatureTrapHandler) ? TRAP_NUM_SGPRS : 0;
1210}
1211
1212// Per-wave SGPR budget (before the addressable clamp): take off the trap
1213// reserve, round down to \p Granule. Shared by getMinNumSGPRs() and
1214// getMaxNumSGPRs(); getOccupancyWithNumSGPRs() is the closed-form algebraic
1215// inverse of this same budget (it does not call this helper), so the two encode
1216// one model.
1217static unsigned getSGPRBudgetPerWave(unsigned TotalNumSGPRs,
1218 unsigned WavesPerEU, unsigned TrapReserve,
1219 unsigned Granule) {
1220 assert(WavesPerEU != 0 && Granule != 0);
1221 unsigned Budget = TotalNumSGPRs / WavesPerEU;
1222 Budget -= std::min(a: Budget, b: TrapReserve);
1223 return alignDown(Value: Budget, Align: Granule);
1224}
1225
1226unsigned getMinNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU) {
1227 assert(WavesPerEU != 0);
1228
1229 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1230 if (Version.Major >= 10)
1231 return 0;
1232
1233 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1234 if (WavesPerEU >= getMaxWavesPerEU(AK: Kind))
1235 return 0;
1236
1237 unsigned MinNumSGPRs =
1238 getSGPRBudgetPerWave(TotalNumSGPRs: getTotalNumSGPRs(AK: Kind), WavesPerEU: WavesPerEU + 1,
1239 TrapReserve: getSGPRTrapHandlerReserve(STI),
1240 Granule: getSGPRAllocGranule(AK: Kind)) +
1241 1;
1242 return std::min(a: MinNumSGPRs, b: getAddressableNumSGPRs(AK: Kind));
1243}
1244
1245unsigned getMaxNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1246 bool Addressable) {
1247 assert(WavesPerEU != 0);
1248
1249 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1250 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(AK: Kind);
1251 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1252 if (Version.Major >= 10)
1253 return Addressable ? AddressableNumSGPRs : 108;
1254 if (Version.Major >= 8 && !Addressable)
1255 AddressableNumSGPRs = 112;
1256 unsigned MaxNumSGPRs = getSGPRBudgetPerWave(
1257 TotalNumSGPRs: getTotalNumSGPRs(AK: Kind), WavesPerEU, TrapReserve: getSGPRTrapHandlerReserve(STI),
1258 Granule: getSGPRAllocGranule(AK: Kind));
1259 return std::min(a: MaxNumSGPRs, b: AddressableNumSGPRs);
1260}
1261
1262bool isSGPROccupancyLimited(const MCSubtargetInfo &STI) {
1263 // From GFX10 on the SGPR file is large enough that SGPRs never limit
1264 // occupancy. Kept as one capability so callers don't each test the version.
1265 return getIsaVersion(GPU: STI.getCPU()).Major < 10;
1266}
1267
1268unsigned getNumExtraSGPRs(const MCSubtargetInfo &STI, bool VCCUsed,
1269 bool FlatScrUsed, bool XNACKUsed) {
1270 unsigned ExtraSGPRs = 0;
1271 if (VCCUsed)
1272 ExtraSGPRs = 2;
1273
1274 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1275 if (Version.Major >= 10)
1276 return ExtraSGPRs;
1277
1278 if (Version.Major < 8) {
1279 if (FlatScrUsed)
1280 ExtraSGPRs = 4;
1281 } else {
1282 if (XNACKUsed)
1283 ExtraSGPRs = 4;
1284
1285 if (FlatScrUsed ||
1286 STI.getFeatureBits().test(I: AMDGPU::FeatureArchitectedFlatScratch))
1287 ExtraSGPRs = 6;
1288 }
1289
1290 return ExtraSGPRs;
1291}
1292
1293static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs,
1294 unsigned Granule) {
1295 return divideCeil(Numerator: std::max(a: 1u, b: NumRegs), Denominator: Granule);
1296}
1297
1298unsigned getNumSGPRBlocks(const MCSubtargetInfo &STI, unsigned NumSGPRs) {
1299 // SGPRBlocks is actual number of SGPR blocks minus 1.
1300 return getGranulatedNumRegisterBlocks(NumRegs: NumSGPRs, Granule: getSGPREncodingGranule(STI)) -
1301 1;
1302}
1303
1304unsigned getVGPRAllocGranule(const MCSubtargetInfo &STI,
1305 unsigned DynamicVGPRBlockSize,
1306 std::optional<bool> EnableWavefrontSize32) {
1307 if (STI.getFeatureBits().test(I: FeatureGFX90AInsts))
1308 return 8;
1309
1310 if (DynamicVGPRBlockSize != 0)
1311 return DynamicVGPRBlockSize;
1312
1313 bool IsWave32 = EnableWavefrontSize32
1314 ? *EnableWavefrontSize32
1315 : STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1316
1317 if (STI.getFeatureBits().test(I: Feature1536VGPRs))
1318 return IsWave32 ? 24 : 12;
1319
1320 if (hasGFX10_3Insts(STI))
1321 return IsWave32 ? 16 : 8;
1322
1323 return IsWave32 ? 8 : 4;
1324}
1325
1326unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI,
1327 std::optional<bool> EnableWavefrontSize32) {
1328 if (STI.getFeatureBits().test(I: FeatureGFX90AInsts))
1329 return 8;
1330
1331 bool IsWave32 = EnableWavefrontSize32
1332 ? *EnableWavefrontSize32
1333 : STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1334
1335 if (STI.getFeatureBits().test(I: FeatureVGPRMSBs))
1336 return IsWave32 ? 16 : 8;
1337
1338 return IsWave32 ? 8 : 4;
1339}
1340
1341unsigned getArchVGPRAllocGranule() { return 4; }
1342
1343unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo &STI) {
1344 const auto &Features = STI.getFeatureBits();
1345 if (Features.test(I: FeatureVGPRMSBs))
1346 return Features.test(I: FeatureWavefrontSize32) ? 1024 : 512;
1347 return 256;
1348}
1349
1350unsigned getAddressableNumVGPRs(const MCSubtargetInfo &STI,
1351 unsigned DynamicVGPRBlockSize) {
1352 const auto &Features = STI.getFeatureBits();
1353 if (Features.test(I: FeatureGFX90AInsts))
1354 return 512;
1355
1356 if (DynamicVGPRBlockSize != 0) {
1357 // On GFX12 we can allocate at most MaxDynamicVGPRBlocks blocks of VGPRs.
1358 return MaxDynamicVGPRBlocks *
1359 getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1360 }
1361 return getAddressableNumArchVGPRs(STI);
1362}
1363
1364unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo &STI,
1365 unsigned NumVGPRs,
1366 unsigned DynamicVGPRBlockSize) {
1367 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1368 bool IsWave32 = STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1369 return getNumWavesPerEUWithNumVGPRs(
1370 NumVGPRs, Granule: getVGPRAllocGranule(STI, DynamicVGPRBlockSize),
1371 MaxWaves: getMaxWavesPerEU(AK: Kind), TotalNumVGPRs: AMDGPU::getTotalNumVGPRs(AK: Kind, IsWave32));
1372}
1373
1374unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
1375 unsigned MaxWaves,
1376 unsigned TotalNumVGPRs) {
1377 if (NumVGPRs < Granule)
1378 return MaxWaves;
1379 unsigned RoundedRegs = alignTo(Value: NumVGPRs, Align: Granule);
1380 return std::min(a: std::max(a: TotalNumVGPRs / RoundedRegs, b: 1u), b: MaxWaves);
1381}
1382
1383unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
1384 unsigned TotalNumSGPRs, unsigned Granule,
1385 unsigned TrapReserve) {
1386 // Closed-form inverse of getMaxNumSGPRs(): the budget condition
1387 // SGPRs <= alignDown(TotalNumSGPRs / W - TrapReserve, Granule)
1388 // solves to W <= TotalNumSGPRs / (alignTo(SGPRs, Granule) + TrapReserve).
1389 unsigned PerWave = alignTo(Value: SGPRs, Align: Granule) + TrapReserve;
1390 return PerWave ? std::clamp(val: TotalNumSGPRs / PerWave, lo: 1u, hi: MaxWaves) : MaxWaves;
1391}
1392
1393unsigned getOccupancyWithNumSGPRs(const MCSubtargetInfo &STI, unsigned SGPRs) {
1394 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1395 unsigned MaxWaves = getMaxWavesPerEU(AK: Kind);
1396
1397 if (!isSGPROccupancyLimited(STI))
1398 return MaxWaves;
1399
1400 return getOccupancyWithNumSGPRs(SGPRs, MaxWaves, TotalNumSGPRs: getTotalNumSGPRs(AK: Kind),
1401 Granule: getSGPRAllocGranule(AK: Kind),
1402 TrapReserve: getSGPRTrapHandlerReserve(STI));
1403}
1404
1405unsigned getMinNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1406 unsigned DynamicVGPRBlockSize) {
1407 assert(WavesPerEU != 0);
1408
1409 // In dynamic VGPR mode, (static) occupancy does not depend on VGPR usage,
1410 // so getMaxNumVGPRs does not depend on WavesPerEU, and thus we need to return
1411 // zero because there is no nonzero VGPR usage N where going below N
1412 // achieves higher (static) occupancy.
1413 bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
1414 if (DynamicVGPREnabled)
1415 return 0;
1416
1417 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1418 unsigned MaxWavesPerEU = getMaxWavesPerEU(AK: Kind);
1419 if (WavesPerEU >= MaxWavesPerEU)
1420 return 0;
1421
1422 unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
1423 AK: Kind, IsWave32: STI.getFeatureBits().test(I: FeatureWavefrontSize32));
1424 unsigned AddrsableNumVGPRs =
1425 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1426 unsigned Granule = getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1427 unsigned MaxNumVGPRs = alignDown(Value: TotNumVGPRs / WavesPerEU, Align: Granule);
1428
1429 if (MaxNumVGPRs == alignDown(Value: TotNumVGPRs / MaxWavesPerEU, Align: Granule))
1430 return 0;
1431
1432 unsigned MinWavesPerEU = getNumWavesPerEUWithNumVGPRs(STI, NumVGPRs: AddrsableNumVGPRs,
1433 DynamicVGPRBlockSize);
1434 if (WavesPerEU < MinWavesPerEU)
1435 return getMinNumVGPRs(STI, WavesPerEU: MinWavesPerEU, DynamicVGPRBlockSize);
1436
1437 unsigned MaxNumVGPRsNext = alignDown(Value: TotNumVGPRs / (WavesPerEU + 1), Align: Granule);
1438 unsigned MinNumVGPRs = 1 + std::min(a: MaxNumVGPRs - Granule, b: MaxNumVGPRsNext);
1439 return std::min(a: MinNumVGPRs, b: AddrsableNumVGPRs);
1440}
1441
1442unsigned getMaxNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1443 unsigned DynamicVGPRBlockSize) {
1444 assert(WavesPerEU != 0);
1445
1446 unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
1447 AK: parseArchAMDGCN(CPU: STI.getCPU()),
1448 IsWave32: STI.getFeatureBits().test(I: FeatureWavefrontSize32));
1449
1450 // In dynamic VGPR mode, WavesPerEU does not imply a VGPR limit.
1451 bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
1452 unsigned MaxNumVGPRs =
1453 DynamicVGPREnabled
1454 ? TotNumVGPRs
1455 : alignDown(Value: TotNumVGPRs / WavesPerEU,
1456 Align: getVGPRAllocGranule(STI, DynamicVGPRBlockSize));
1457 unsigned AddressableNumVGPRs =
1458 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1459 return std::min(a: MaxNumVGPRs, b: AddressableNumVGPRs);
1460}
1461
1462unsigned getAllocatedNumVGPRBlocks(const MCSubtargetInfo &STI,
1463 unsigned NumVGPRs,
1464 unsigned DynamicVGPRBlockSize,
1465 std::optional<bool> EnableWavefrontSize32) {
1466 return getGranulatedNumRegisterBlocks(
1467 NumRegs: NumVGPRs,
1468 Granule: getVGPRAllocGranule(STI, DynamicVGPRBlockSize, EnableWavefrontSize32));
1469}
1470} // end namespace IsaInfo
1471
1472void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &KernelCode,
1473 const MCSubtargetInfo &STI) {
1474 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1475 KernelCode.amd_kernel_code_version_major = 1;
1476 KernelCode.amd_kernel_code_version_minor = 2;
1477 KernelCode.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
1478 KernelCode.amd_machine_version_major = Version.Major;
1479 KernelCode.amd_machine_version_minor = Version.Minor;
1480 KernelCode.amd_machine_version_stepping = Version.Stepping;
1481 KernelCode.kernel_code_entry_byte_offset = sizeof(amd_kernel_code_t);
1482 if (STI.getFeatureBits().test(I: FeatureWavefrontSize32)) {
1483 KernelCode.wavefront_size = 5;
1484 KernelCode.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
1485 } else {
1486 KernelCode.wavefront_size = 6;
1487 }
1488
1489 // If the code object does not support indirect functions, then the value must
1490 // be 0xffffffff.
1491 KernelCode.call_convention = -1;
1492
1493 // These alignment values are specified in powers of two, so alignment =
1494 // 2^n. The minimum alignment is 2^4 = 16.
1495 KernelCode.kernarg_segment_alignment = 4;
1496 KernelCode.group_segment_alignment = 4;
1497 KernelCode.private_segment_alignment = 4;
1498
1499 if (Version.Major >= 10) {
1500 KernelCode.compute_pgm_resource_registers |=
1501 S_00B848_WGP_MODE(STI.getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
1502 S_00B848_MEM_ORDERED(1) | S_00B848_FWD_PROGRESS(1);
1503 }
1504}
1505
1506bool isReadOnlySegment(const GlobalValue *GV) {
1507 unsigned AS = GV->getAddressSpace();
1508 return AS == AMDGPUAS::CONSTANT_ADDRESS ||
1509 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
1510}
1511
1512bool shouldEmitConstantsToTextSection(const Triple &TT) {
1513 return TT.getArch() == Triple::r600;
1514}
1515
1516static bool isValidRegPrefix(char C) {
1517 return C == 'v' || C == 's' || C == 'a';
1518}
1519
1520std::tuple<char, unsigned, unsigned> parseAsmPhysRegName(StringRef RegName) {
1521 if (RegName.empty())
1522 return {};
1523
1524 char Kind = RegName.front();
1525 if (!isValidRegPrefix(C: Kind))
1526 return {};
1527
1528 RegName = RegName.drop_front();
1529 if (RegName.consume_front(Prefix: "[")) {
1530 unsigned Idx, End;
1531 bool Failed = RegName.consumeInteger(Radix: 10, Result&: Idx);
1532 Failed |= !RegName.consume_front(Prefix: ":");
1533 Failed |= RegName.consumeInteger(Radix: 10, Result&: End);
1534 Failed |= !RegName.consume_back(Suffix: "]");
1535 if (!Failed) {
1536 unsigned NumRegs = End - Idx + 1;
1537 if (NumRegs > 1)
1538 return {Kind, Idx, NumRegs};
1539 }
1540 } else {
1541 unsigned Idx;
1542 bool Failed = RegName.getAsInteger(Radix: 10, Result&: Idx);
1543 if (!Failed)
1544 return {Kind, Idx, 1};
1545 }
1546
1547 return {};
1548}
1549
1550std::tuple<char, unsigned, unsigned>
1551parseAsmConstraintPhysReg(StringRef Constraint) {
1552 StringRef RegName = Constraint;
1553 if (!RegName.consume_front(Prefix: "{") || !RegName.consume_back(Suffix: "}"))
1554 return {};
1555 return parseAsmPhysRegName(RegName);
1556}
1557
1558std::pair<unsigned, unsigned>
1559getIntegerPairAttribute(const Function &F, StringRef Name,
1560 std::pair<unsigned, unsigned> Default,
1561 bool OnlyFirstRequired) {
1562 if (auto Attr = getIntegerPairAttribute(F, Name, OnlyFirstRequired))
1563 return {Attr->first, Attr->second.value_or(u&: Default.second)};
1564 return Default;
1565}
1566
1567std::optional<std::pair<unsigned, std::optional<unsigned>>>
1568getIntegerPairAttribute(const Function &F, StringRef Name,
1569 bool OnlyFirstRequired) {
1570 Attribute A = F.getFnAttribute(Kind: Name);
1571 if (!A.isStringAttribute())
1572 return std::nullopt;
1573
1574 LLVMContext &Ctx = F.getContext();
1575 std::pair<unsigned, std::optional<unsigned>> Ints;
1576 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(Separator: ',');
1577 if (Strs.first.trim().getAsInteger(Radix: 0, Result&: Ints.first)) {
1578 Ctx.emitError(ErrorStr: "can't parse first integer attribute " + Name);
1579 return std::nullopt;
1580 }
1581 unsigned Second = 0;
1582 if (Strs.second.trim().getAsInteger(Radix: 0, Result&: Second)) {
1583 if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
1584 Ctx.emitError(ErrorStr: "can't parse second integer attribute " + Name);
1585 return std::nullopt;
1586 }
1587 } else {
1588 Ints.second = Second;
1589 }
1590
1591 return Ints;
1592}
1593
1594SmallVector<unsigned> getIntegerVecAttribute(const Function &F, StringRef Name,
1595 unsigned Size,
1596 unsigned DefaultVal) {
1597 std::optional<SmallVector<unsigned>> R =
1598 getIntegerVecAttribute(F, Name, Size);
1599 return R.has_value() ? *R : SmallVector<unsigned>(Size, DefaultVal);
1600}
1601
1602std::optional<SmallVector<unsigned>>
1603getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) {
1604 assert(Size > 2);
1605 LLVMContext &Ctx = F.getContext();
1606
1607 Attribute A = F.getFnAttribute(Kind: Name);
1608 if (!A.isValid())
1609 return std::nullopt;
1610 if (!A.isStringAttribute()) {
1611 Ctx.emitError(ErrorStr: Name + " is not a string attribute");
1612 return std::nullopt;
1613 }
1614
1615 SmallVector<unsigned> Vals(Size);
1616
1617 StringRef S = A.getValueAsString();
1618 unsigned i = 0;
1619 for (; !S.empty() && i < Size; i++) {
1620 std::pair<StringRef, StringRef> Strs = S.split(Separator: ',');
1621 unsigned IntVal;
1622 if (Strs.first.trim().getAsInteger(Radix: 0, Result&: IntVal)) {
1623 Ctx.emitError(ErrorStr: "can't parse integer attribute " + Strs.first + " in " +
1624 Name);
1625 return std::nullopt;
1626 }
1627 Vals[i] = IntVal;
1628 S = Strs.second;
1629 }
1630
1631 if (!S.empty() || i < Size) {
1632 Ctx.emitError(ErrorStr: "attribute " + Name +
1633 " has incorrect number of integers; expected " +
1634 llvm::utostr(X: Size));
1635 return std::nullopt;
1636 }
1637 return Vals;
1638}
1639
1640SmallVector<unsigned> getMaxNumWorkGroups(const Function &F) {
1641 return getIntegerVecAttribute(F, Name: "amdgpu-max-num-workgroups", Size: 3,
1642 DefaultVal: std::numeric_limits<uint32_t>::max());
1643}
1644
1645bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) {
1646 assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!");
1647 for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) {
1648 auto Low =
1649 mdconst::extract<ConstantInt>(MD: MD.getOperand(I: 2 * I + 0))->getValue();
1650 auto High =
1651 mdconst::extract<ConstantInt>(MD: MD.getOperand(I: 2 * I + 1))->getValue();
1652 // There are two types of [A; B) ranges:
1653 // A < B, e.g. [4; 5) which is a range that only includes 4.
1654 // A > B, e.g. [5; 4) which is a range that wraps around and includes
1655 // everything except 4.
1656 if (Low.ult(RHS: High)) {
1657 if (Low.ule(RHS: Val) && High.ugt(RHS: Val))
1658 return true;
1659 } else {
1660 if (Low.uge(RHS: Val) && High.ult(RHS: Val))
1661 return true;
1662 }
1663 }
1664
1665 return false;
1666}
1667
1668unsigned getVmcntBitMask(const IsaVersion &Version) {
1669 return (1 << (getVmcntBitWidthLo(VersionMajor: Version.Major) +
1670 getVmcntBitWidthHi(VersionMajor: Version.Major))) -
1671 1;
1672}
1673
1674unsigned getLoadcntBitMask(const IsaVersion &Version) {
1675 return (1 << getLoadcntBitWidth(VersionMajor: Version.Major)) - 1;
1676}
1677
1678unsigned getSamplecntBitMask(const IsaVersion &Version) {
1679 return (1 << getSamplecntBitWidth(VersionMajor: Version.Major)) - 1;
1680}
1681
1682unsigned getBvhcntBitMask(const IsaVersion &Version) {
1683 return (1 << getBvhcntBitWidth(VersionMajor: Version.Major)) - 1;
1684}
1685
1686unsigned getExpcntBitMask(const IsaVersion &Version) {
1687 return (1 << getExpcntBitWidth(VersionMajor: Version.Major)) - 1;
1688}
1689
1690unsigned getLgkmcntBitMask(const IsaVersion &Version) {
1691 return (1 << getLgkmcntBitWidth(VersionMajor: Version.Major)) - 1;
1692}
1693
1694unsigned getDscntBitMask(const IsaVersion &Version) {
1695 return (1 << getDscntBitWidth(VersionMajor: Version.Major)) - 1;
1696}
1697
1698unsigned getKmcntBitMask(const IsaVersion &Version) {
1699 return (1 << getKmcntBitWidth(VersionMajor: Version.Major)) - 1;
1700}
1701
1702unsigned getXcntBitMask(const IsaVersion &Version) {
1703 return (1 << getXcntBitWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1704}
1705
1706unsigned getAsynccntBitMask(const IsaVersion &Version) {
1707 return (1 << getAsynccntBitWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1708}
1709
1710unsigned getStorecntBitMask(const IsaVersion &Version) {
1711 return (1 << getStorecntBitWidth(VersionMajor: Version.Major)) - 1;
1712}
1713
1714unsigned getBarrierMemberCountBitMask(const IsaVersion &Version) {
1715 return (1 << getBarrierMemberCountBitWidth(VersionMajor: Version.Major)) - 1;
1716}
1717
1718unsigned getWaitcntBitMask(const IsaVersion &Version) {
1719 unsigned VmcntLo = getBitMask(Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1720 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1721 unsigned Expcnt = getBitMask(Shift: getExpcntBitShift(VersionMajor: Version.Major),
1722 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1723 unsigned Lgkmcnt = getBitMask(Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1724 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1725 unsigned VmcntHi = getBitMask(Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1726 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1727 return VmcntLo | Expcnt | Lgkmcnt | VmcntHi;
1728}
1729
1730unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1731 unsigned VmcntLo = unpackBits(Src: Waitcnt, Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1732 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1733 unsigned VmcntHi = unpackBits(Src: Waitcnt, Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1734 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1735 return VmcntLo | VmcntHi << getVmcntBitWidthLo(VersionMajor: Version.Major);
1736}
1737
1738unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
1739 return unpackBits(Src: Waitcnt, Shift: getExpcntBitShift(VersionMajor: Version.Major),
1740 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1741}
1742
1743unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1744 return unpackBits(Src: Waitcnt, Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1745 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1746}
1747
1748unsigned decodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt) {
1749 return unpackBits(Src: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1750 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1751}
1752
1753unsigned decodeStorecnt(const IsaVersion &Version, unsigned Waitcnt) {
1754 return unpackBits(Src: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1755 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1756}
1757
1758unsigned decodeDscnt(const IsaVersion &Version, unsigned Waitcnt) {
1759 return unpackBits(Src: Waitcnt, Shift: getDscntBitShift(VersionMajor: Version.Major),
1760 Width: getDscntBitWidth(VersionMajor: Version.Major));
1761}
1762
1763void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt,
1764 unsigned &Expcnt, unsigned &Lgkmcnt) {
1765 Vmcnt = decodeVmcnt(Version, Waitcnt);
1766 Expcnt = decodeExpcnt(Version, Waitcnt);
1767 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
1768}
1769
1770unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
1771 unsigned Vmcnt) {
1772 Waitcnt = packBits(Src: Vmcnt, Dst: Waitcnt, Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1773 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1774 return packBits(Src: Vmcnt >> getVmcntBitWidthLo(VersionMajor: Version.Major), Dst: Waitcnt,
1775 Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1776 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1777}
1778
1779unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
1780 unsigned Expcnt) {
1781 return packBits(Src: Expcnt, Dst: Waitcnt, Shift: getExpcntBitShift(VersionMajor: Version.Major),
1782 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1783}
1784
1785unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
1786 unsigned Lgkmcnt) {
1787 return packBits(Src: Lgkmcnt, Dst: Waitcnt, Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1788 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1789}
1790
1791unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt,
1792 unsigned Expcnt, unsigned Lgkmcnt) {
1793 unsigned Waitcnt = getWaitcntBitMask(Version);
1794 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
1795 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
1796 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
1797 return Waitcnt;
1798}
1799
1800static unsigned getCombinedCountBitMask(const IsaVersion &Version,
1801 bool IsStore) {
1802 unsigned Dscnt = getBitMask(Shift: getDscntBitShift(VersionMajor: Version.Major),
1803 Width: getDscntBitWidth(VersionMajor: Version.Major));
1804 if (IsStore) {
1805 unsigned Storecnt = getBitMask(Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1806 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1807 return Dscnt | Storecnt;
1808 }
1809 unsigned Loadcnt = getBitMask(Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1810 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1811 return Dscnt | Loadcnt;
1812}
1813
1814static unsigned encodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt,
1815 unsigned Loadcnt) {
1816 return packBits(Src: Loadcnt, Dst: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1817 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1818}
1819
1820static unsigned encodeStorecnt(const IsaVersion &Version, unsigned Waitcnt,
1821 unsigned Storecnt) {
1822 return packBits(Src: Storecnt, Dst: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1823 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1824}
1825
1826static unsigned encodeDscnt(const IsaVersion &Version, unsigned Waitcnt,
1827 unsigned Dscnt) {
1828 return packBits(Src: Dscnt, Dst: Waitcnt, Shift: getDscntBitShift(VersionMajor: Version.Major),
1829 Width: getDscntBitWidth(VersionMajor: Version.Major));
1830}
1831
1832unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt,
1833 unsigned Dscnt) {
1834 unsigned Waitcnt = getCombinedCountBitMask(Version, IsStore: false);
1835 Waitcnt = encodeLoadcnt(Version, Waitcnt, Loadcnt);
1836 Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
1837 return Waitcnt;
1838}
1839
1840unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt,
1841 unsigned Dscnt) {
1842 unsigned Waitcnt = getCombinedCountBitMask(Version, IsStore: true);
1843 Waitcnt = encodeStorecnt(Version, Waitcnt, Storecnt);
1844 Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
1845 return Waitcnt;
1846}
1847
1848//===----------------------------------------------------------------------===//
1849// Custom Operand Values
1850//===----------------------------------------------------------------------===//
1851
1852static bool isSymbolicCustomOperandEncoding(const CustomOperandVal *Opr,
1853 int Size, unsigned Code,
1854 bool &HasNonDefaultVal,
1855 const MCSubtargetInfo &STI) {
1856 unsigned UsedOprMask = 0;
1857 HasNonDefaultVal = false;
1858 for (int Idx = 0; Idx < Size; ++Idx) {
1859 const auto &Op = Opr[Idx];
1860 if (!Op.isSupported(STI))
1861 continue;
1862 UsedOprMask |= Op.getMask();
1863 unsigned Val = Op.decode(Code);
1864 if (!Op.isValid(Val))
1865 return false;
1866 HasNonDefaultVal |= (Val != Op.Default);
1867 }
1868 return (Code & ~UsedOprMask) == 0;
1869}
1870
1871static bool decodeCustomOperand(const CustomOperandVal *Opr, int Size,
1872 unsigned Code, int &Idx, StringRef &Name,
1873 unsigned &Val, bool &IsDefault,
1874 const MCSubtargetInfo &STI) {
1875 while (Idx < Size) {
1876 const auto &Op = Opr[Idx++];
1877 if (Op.isSupported(STI)) {
1878 Name = Op.Name;
1879 Val = Op.decode(Code);
1880 IsDefault = (Val == Op.Default);
1881 return true;
1882 }
1883 }
1884
1885 return false;
1886}
1887
1888static int encodeCustomOperandVal(const CustomOperandVal &Op,
1889 int64_t InputVal) {
1890 if (InputVal < 0 || InputVal > Op.Max)
1891 return OPR_VAL_INVALID;
1892 return Op.encode(Val: static_cast<unsigned>(InputVal));
1893}
1894
1895static int encodeCustomOperand(const CustomOperandVal *Opr, int Size,
1896 const StringRef Name, int64_t InputVal,
1897 unsigned &UsedOprMask,
1898 const MCSubtargetInfo &STI) {
1899 int InvalidId = OPR_ID_UNKNOWN;
1900 for (int Idx = 0; Idx < Size; ++Idx) {
1901 const auto &Op = Opr[Idx];
1902 if (Op.Name == Name) {
1903 if (!Op.isSupported(STI)) {
1904 InvalidId = OPR_ID_UNSUPPORTED;
1905 continue;
1906 }
1907 auto OprMask = Op.getMask();
1908 if (OprMask & UsedOprMask)
1909 return OPR_ID_DUPLICATE;
1910 UsedOprMask |= OprMask;
1911 return encodeCustomOperandVal(Op, InputVal);
1912 }
1913 }
1914 return InvalidId;
1915}
1916
1917//===----------------------------------------------------------------------===//
1918// DepCtr
1919//===----------------------------------------------------------------------===//
1920
1921namespace DepCtr {
1922
1923int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI) {
1924 unsigned Enc = 0;
1925 for (int Idx = 0; Idx < DEP_CTR_SIZE; ++Idx) {
1926 const auto &Op = DepCtrInfo[Idx];
1927 if (Op.isSupported(STI))
1928 Enc |= Op.encode(Val: Op.Default);
1929 }
1930 return Enc;
1931}
1932
1933bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal,
1934 const MCSubtargetInfo &STI) {
1935 return isSymbolicCustomOperandEncoding(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Code,
1936 HasNonDefaultVal, STI);
1937}
1938
1939bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val,
1940 bool &IsDefault, const MCSubtargetInfo &STI) {
1941 return decodeCustomOperand(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Code, Idx&: Id, Name, Val,
1942 IsDefault, STI);
1943}
1944
1945int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask,
1946 const MCSubtargetInfo &STI) {
1947 return encodeCustomOperand(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Name, InputVal: Val, UsedOprMask,
1948 STI);
1949}
1950
1951unsigned getVaVdstBitMask() { return (1 << getVaVdstBitWidth()) - 1; }
1952
1953unsigned getVaSdstBitMask() { return (1 << getVaSdstBitWidth()) - 1; }
1954
1955unsigned getVaSsrcBitMask() { return (1 << getVaSsrcBitWidth()) - 1; }
1956
1957unsigned getHoldCntBitMask(const IsaVersion &Version) {
1958 return (1 << getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1959}
1960
1961unsigned getVmVsrcBitMask() { return (1 << getVmVsrcBitWidth()) - 1; }
1962
1963unsigned getVaVccBitMask() { return (1 << getVaVccBitWidth()) - 1; }
1964
1965unsigned getSaSdstBitMask() { return (1 << getSaSdstBitWidth()) - 1; }
1966
1967unsigned decodeFieldVmVsrc(unsigned Encoded) {
1968 return unpackBits(Src: Encoded, Shift: getVmVsrcBitShift(), Width: getVmVsrcBitWidth());
1969}
1970
1971unsigned decodeFieldVaVdst(unsigned Encoded) {
1972 return unpackBits(Src: Encoded, Shift: getVaVdstBitShift(), Width: getVaVdstBitWidth());
1973}
1974
1975unsigned decodeFieldSaSdst(unsigned Encoded) {
1976 return unpackBits(Src: Encoded, Shift: getSaSdstBitShift(), Width: getSaSdstBitWidth());
1977}
1978
1979unsigned decodeFieldVaSdst(unsigned Encoded) {
1980 return unpackBits(Src: Encoded, Shift: getVaSdstBitShift(), Width: getVaSdstBitWidth());
1981}
1982
1983unsigned decodeFieldVaVcc(unsigned Encoded) {
1984 return unpackBits(Src: Encoded, Shift: getVaVccBitShift(), Width: getVaVccBitWidth());
1985}
1986
1987unsigned decodeFieldVaSsrc(unsigned Encoded) {
1988 return unpackBits(Src: Encoded, Shift: getVaSsrcBitShift(), Width: getVaSsrcBitWidth());
1989}
1990
1991unsigned decodeFieldHoldCnt(unsigned Encoded, const IsaVersion &Version) {
1992 return unpackBits(Src: Encoded, Shift: getHoldCntBitShift(),
1993 Width: getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor));
1994}
1995
1996unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc) {
1997 return packBits(Src: VmVsrc, Dst: Encoded, Shift: getVmVsrcBitShift(), Width: getVmVsrcBitWidth());
1998}
1999
2000unsigned encodeFieldVmVsrc(unsigned VmVsrc, const MCSubtargetInfo &STI) {
2001 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2002 return encodeFieldVmVsrc(Encoded, VmVsrc);
2003}
2004
2005unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst) {
2006 return packBits(Src: VaVdst, Dst: Encoded, Shift: getVaVdstBitShift(), Width: getVaVdstBitWidth());
2007}
2008
2009unsigned encodeFieldVaVdst(unsigned VaVdst, const MCSubtargetInfo &STI) {
2010 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2011 return encodeFieldVaVdst(Encoded, VaVdst);
2012}
2013
2014unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst) {
2015 return packBits(Src: SaSdst, Dst: Encoded, Shift: getSaSdstBitShift(), Width: getSaSdstBitWidth());
2016}
2017
2018unsigned encodeFieldSaSdst(unsigned SaSdst, const MCSubtargetInfo &STI) {
2019 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2020 return encodeFieldSaSdst(Encoded, SaSdst);
2021}
2022
2023unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst) {
2024 return packBits(Src: VaSdst, Dst: Encoded, Shift: getVaSdstBitShift(), Width: getVaSdstBitWidth());
2025}
2026
2027unsigned encodeFieldVaSdst(unsigned VaSdst, const MCSubtargetInfo &STI) {
2028 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2029 return encodeFieldVaSdst(Encoded, VaSdst);
2030}
2031
2032unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc) {
2033 return packBits(Src: VaVcc, Dst: Encoded, Shift: getVaVccBitShift(), Width: getVaVccBitWidth());
2034}
2035
2036unsigned encodeFieldVaVcc(unsigned VaVcc, const MCSubtargetInfo &STI) {
2037 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2038 return encodeFieldVaVcc(Encoded, VaVcc);
2039}
2040
2041unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc) {
2042 return packBits(Src: VaSsrc, Dst: Encoded, Shift: getVaSsrcBitShift(), Width: getVaSsrcBitWidth());
2043}
2044
2045unsigned encodeFieldVaSsrc(unsigned VaSsrc, const MCSubtargetInfo &STI) {
2046 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2047 return encodeFieldVaSsrc(Encoded, VaSsrc);
2048}
2049
2050unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt,
2051 const IsaVersion &Version) {
2052 return packBits(Src: HoldCnt, Dst: Encoded, Shift: getHoldCntBitShift(),
2053 Width: getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor));
2054}
2055
2056unsigned encodeFieldHoldCnt(unsigned HoldCnt, const MCSubtargetInfo &STI) {
2057 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2058 return encodeFieldHoldCnt(Encoded, HoldCnt, Version: getIsaVersion(GPU: STI.getCPU()));
2059}
2060
2061} // namespace DepCtr
2062
2063//===----------------------------------------------------------------------===//
2064// exp tgt
2065//===----------------------------------------------------------------------===//
2066
2067namespace Exp {
2068
2069struct ExpTgt {
2070 StringLiteral Name;
2071 unsigned Tgt;
2072 unsigned MaxIndex;
2073};
2074
2075// clang-format off
2076static constexpr ExpTgt ExpTgtInfo[] = {
2077 {.Name: {"null"}, .Tgt: ET_NULL, .MaxIndex: ET_NULL_MAX_IDX},
2078 {.Name: {"mrtz"}, .Tgt: ET_MRTZ, .MaxIndex: ET_MRTZ_MAX_IDX},
2079 {.Name: {"prim"}, .Tgt: ET_PRIM, .MaxIndex: ET_PRIM_MAX_IDX},
2080 {.Name: {"mrt"}, .Tgt: ET_MRT0, .MaxIndex: ET_MRT_MAX_IDX},
2081 {.Name: {"pos"}, .Tgt: ET_POS0, .MaxIndex: ET_POS_MAX_IDX},
2082 {.Name: {"dual_src_blend"},.Tgt: ET_DUAL_SRC_BLEND0, .MaxIndex: ET_DUAL_SRC_BLEND_MAX_IDX},
2083 {.Name: {"param"}, .Tgt: ET_PARAM0, .MaxIndex: ET_PARAM_MAX_IDX},
2084};
2085// clang-format on
2086
2087bool getTgtName(unsigned Id, StringRef &Name, int &Index) {
2088 for (const ExpTgt &Val : ExpTgtInfo) {
2089 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) {
2090 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt);
2091 Name = Val.Name;
2092 return true;
2093 }
2094 }
2095 return false;
2096}
2097
2098unsigned getTgtId(const StringRef Name) {
2099
2100 for (const ExpTgt &Val : ExpTgtInfo) {
2101 if (Val.MaxIndex == 0 && Name == Val.Name)
2102 return Val.Tgt;
2103
2104 if (Val.MaxIndex > 0 && Name.starts_with(Prefix: Val.Name)) {
2105 StringRef Suffix = Name.drop_front(N: Val.Name.size());
2106
2107 unsigned Id;
2108 if (Suffix.getAsInteger(Radix: 10, Result&: Id) || Id > Val.MaxIndex)
2109 return ET_INVALID;
2110
2111 // Disable leading zeroes
2112 if (Suffix.size() > 1 && Suffix[0] == '0')
2113 return ET_INVALID;
2114
2115 return Val.Tgt + Id;
2116 }
2117 }
2118 return ET_INVALID;
2119}
2120
2121bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) {
2122 switch (Id) {
2123 case ET_NULL:
2124 return !isGFX11Plus(STI);
2125 case ET_POS4:
2126 case ET_PRIM:
2127 return isGFX10Plus(STI);
2128 case ET_DUAL_SRC_BLEND0:
2129 case ET_DUAL_SRC_BLEND1:
2130 return isGFX11Plus(STI);
2131 default:
2132 if (Id >= ET_PARAM0 && Id <= ET_PARAM31)
2133 return !isGFX11Plus(STI) || isGFX13Plus(STI);
2134 return true;
2135 }
2136}
2137
2138} // namespace Exp
2139
2140//===----------------------------------------------------------------------===//
2141// MTBUF Format
2142//===----------------------------------------------------------------------===//
2143
2144namespace MTBUFFormat {
2145
2146int64_t getDfmt(const StringRef Name) {
2147 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
2148 if (Name == DfmtSymbolic[Id])
2149 return Id;
2150 }
2151 return DFMT_UNDEF;
2152}
2153
2154StringRef getDfmtName(unsigned Id) {
2155 assert(Id <= DFMT_MAX);
2156 return DfmtSymbolic[Id];
2157}
2158
2159static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {
2160 if (isSI(STI) || isCI(STI))
2161 return NfmtSymbolicSICI;
2162 if (isVI(STI) || isGFX9(STI))
2163 return NfmtSymbolicVI;
2164 return NfmtSymbolicGFX10;
2165}
2166
2167int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
2168 const auto *lookupTable = getNfmtLookupTable(STI);
2169 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
2170 if (Name == lookupTable[Id])
2171 return Id;
2172 }
2173 return NFMT_UNDEF;
2174}
2175
2176StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
2177 assert(Id <= NFMT_MAX);
2178 return getNfmtLookupTable(STI)[Id];
2179}
2180
2181bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2182 unsigned Dfmt;
2183 unsigned Nfmt;
2184 decodeDfmtNfmt(Format: Id, Dfmt, Nfmt);
2185 return isValidNfmt(Val: Nfmt, STI);
2186}
2187
2188bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2189 return !getNfmtName(Id, STI).empty();
2190}
2191
2192int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
2193 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);
2194}
2195
2196void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
2197 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;
2198 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
2199}
2200
2201int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI) {
2202 if (isGFX11Plus(STI)) {
2203 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2204 if (Name == UfmtSymbolicGFX11[Id])
2205 return Id;
2206 }
2207 } else {
2208 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2209 if (Name == UfmtSymbolicGFX10[Id])
2210 return Id;
2211 }
2212 }
2213 return UFMT_UNDEF;
2214}
2215
2216StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI) {
2217 if (isValidUnifiedFormat(Val: Id, STI))
2218 return isGFX10(STI) ? UfmtSymbolicGFX10[Id] : UfmtSymbolicGFX11[Id];
2219 return "";
2220}
2221
2222bool isValidUnifiedFormat(unsigned Id, const MCSubtargetInfo &STI) {
2223 return isGFX10(STI) ? Id <= UfmtGFX10::UFMT_LAST : Id <= UfmtGFX11::UFMT_LAST;
2224}
2225
2226int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt,
2227 const MCSubtargetInfo &STI) {
2228 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
2229 if (isGFX11Plus(STI)) {
2230 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2231 if (Fmt == DfmtNfmt2UFmtGFX11[Id])
2232 return Id;
2233 }
2234 } else {
2235 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2236 if (Fmt == DfmtNfmt2UFmtGFX10[Id])
2237 return Id;
2238 }
2239 }
2240 return UFMT_UNDEF;
2241}
2242
2243bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
2244 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
2245}
2246
2247unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {
2248 if (isGFX10Plus(STI))
2249 return UFMT_DEFAULT;
2250 return DFMT_NFMT_DEFAULT;
2251}
2252
2253} // namespace MTBUFFormat
2254
2255//===----------------------------------------------------------------------===//
2256// SendMsg
2257//===----------------------------------------------------------------------===//
2258
2259namespace SendMsg {
2260
2261static uint64_t getMsgIdMask(const MCSubtargetInfo &STI) {
2262 return isGFX11Plus(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_;
2263}
2264
2265bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) {
2266 return (MsgId & ~(getMsgIdMask(STI))) == 0;
2267}
2268
2269bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
2270 bool Strict) {
2271 assert(isValidMsgId(MsgId, STI));
2272
2273 if (!Strict)
2274 return 0 <= OpId && isUInt<OP_WIDTH_>(x: OpId);
2275
2276 if (msgRequiresOp(MsgId, STI)) {
2277 if (MsgId == ID_GS_PreGFX11 && OpId == OP_GS_NOP)
2278 return false;
2279
2280 return !getMsgOpName(MsgId, Encoding: OpId, STI).empty();
2281 }
2282
2283 return OpId == OP_NONE_;
2284}
2285
2286bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
2287 const MCSubtargetInfo &STI, bool Strict) {
2288 assert(isValidMsgOp(MsgId, OpId, STI, Strict));
2289
2290 if (!Strict)
2291 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(x: StreamId);
2292
2293 if (!isGFX11Plus(STI)) {
2294 switch (MsgId) {
2295 case ID_GS_PreGFX11:
2296 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
2297 case ID_GS_DONE_PreGFX11:
2298 return (OpId == OP_GS_NOP)
2299 ? (StreamId == STREAM_ID_NONE_)
2300 : (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
2301 }
2302 }
2303 return StreamId == STREAM_ID_NONE_;
2304}
2305
2306bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI) {
2307 return MsgId == ID_SYSMSG ||
2308 (!isGFX11Plus(STI) &&
2309 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11));
2310}
2311
2312bool msgSupportsStream(int64_t MsgId, int64_t OpId,
2313 const MCSubtargetInfo &STI) {
2314 return !isGFX11Plus(STI) &&
2315 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11) &&
2316 OpId != OP_GS_NOP;
2317}
2318
2319void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,
2320 uint16_t &StreamId, const MCSubtargetInfo &STI) {
2321 MsgId = static_cast<uint16_t>(Val & getMsgIdMask(STI));
2322 if (isGFX11Plus(STI)) {
2323 OpId = 0;
2324 StreamId = 0;
2325 } else {
2326 OpId = (Val & OP_MASK_) >> OP_SHIFT_;
2327 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
2328 }
2329}
2330
2331uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId) {
2332 return MsgId | (OpId << OP_SHIFT_) | (StreamId << STREAM_ID_SHIFT_);
2333}
2334
2335bool msgDoesNotUseM0(int64_t MsgId, const MCSubtargetInfo &STI) {
2336 // Explicitly list message types that are known to not use m0.
2337 // This is safer than excluding only GS_ALLOC_REQ, in case new message
2338 // types are added in the future that do use m0.
2339 if (isGFX11Plus(STI)) {
2340 switch (MsgId) {
2341 case ID_DEALLOC_VGPRS_GFX11Plus:
2342 return true;
2343 default:
2344 break;
2345 }
2346 }
2347 switch (MsgId) {
2348 case ID_SAVEWAVE:
2349 case ID_STALL_WAVE_GEN:
2350 case ID_HALT_WAVES:
2351 case ID_ORDERED_PS_DONE:
2352 case ID_EARLY_PRIM_DEALLOC:
2353 case ID_GET_DOORBELL:
2354 case ID_GET_DDID:
2355 case ID_SYSMSG:
2356 return true;
2357 default:
2358 return false;
2359 }
2360}
2361
2362} // namespace SendMsg
2363
2364//===----------------------------------------------------------------------===//
2365//
2366//===----------------------------------------------------------------------===//
2367
2368unsigned getInitialPSInputAddr(const Function &F) {
2369 return static_cast<unsigned>(
2370 F.getFnAttributeAsParsedInteger(Kind: "InitialPSInputAddr", Default: 0));
2371}
2372
2373bool getHasColorExport(const Function &F) {
2374 // As a safe default always respond as if PS has color exports.
2375 return F.getFnAttributeAsParsedInteger(
2376 Kind: "amdgpu-color-export",
2377 Default: F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0;
2378}
2379
2380bool getHasDepthExport(const Function &F) {
2381 return F.getFnAttributeAsParsedInteger(Kind: "amdgpu-depth-export", Default: 0) != 0;
2382}
2383
2384unsigned getDynamicVGPRBlockSize(const Function &F) {
2385 unsigned BlockSize = static_cast<unsigned>(
2386 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-dynamic-vgpr-block-size", Default: 0));
2387
2388 if (BlockSize == 16 || BlockSize == 32)
2389 return BlockSize;
2390
2391 return 0;
2392}
2393
2394bool hasMIMG_R128(const MCSubtargetInfo &STI) {
2395 return STI.hasFeature(Feature: AMDGPU::FeatureMIMG_R128) &&
2396 !STI.hasFeature(Feature: AMDGPU::FeatureR128A16);
2397}
2398
2399bool hasA16(const MCSubtargetInfo &STI) {
2400 return STI.hasFeature(Feature: AMDGPU::FeatureA16);
2401}
2402
2403bool hasG16(const MCSubtargetInfo &STI) {
2404 return STI.hasFeature(Feature: AMDGPU::FeatureG16);
2405}
2406
2407bool hasPackedD16(const MCSubtargetInfo &STI) {
2408 return !STI.hasFeature(Feature: AMDGPU::FeatureUnpackedD16VMem) && !isCI(STI) &&
2409 !isSI(STI);
2410}
2411
2412bool hasGDS(const MCSubtargetInfo &STI) {
2413 return STI.hasFeature(Feature: AMDGPU::FeatureGDS);
2414}
2415
2416unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler) {
2417 auto Version = getIsaVersion(GPU: STI.getCPU());
2418 if (Version.Major == 10)
2419 return Version.Minor >= 3 ? 13 : 5;
2420 if (Version.Major == 11)
2421 return 5;
2422 if (Version.Major >= 12)
2423 return HasSampler ? 4 : 5;
2424 return 0;
2425}
2426
2427unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI) {
2428 if (isGFX1250Plus(STI))
2429 return 32;
2430 return 16;
2431}
2432
2433bool isSI(const MCSubtargetInfo &STI) {
2434 return STI.hasFeature(Feature: AMDGPU::FeatureSouthernIslands);
2435}
2436
2437bool isCI(const MCSubtargetInfo &STI) {
2438 return STI.hasFeature(Feature: AMDGPU::FeatureSeaIslands);
2439}
2440
2441bool isVI(const MCSubtargetInfo &STI) {
2442 return STI.hasFeature(Feature: AMDGPU::FeatureVolcanicIslands);
2443}
2444
2445bool isGFX9(const MCSubtargetInfo &STI) {
2446 return STI.hasFeature(Feature: AMDGPU::FeatureGFX9);
2447}
2448
2449bool isGFX9_GFX10(const MCSubtargetInfo &STI) {
2450 return isGFX9(STI) || isGFX10(STI);
2451}
2452
2453bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI) {
2454 return isGFX9(STI) || isGFX10(STI) || isGFX11(STI);
2455}
2456
2457bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI) {
2458 return isVI(STI) || isGFX9(STI) || isGFX10(STI);
2459}
2460
2461bool isGFX8Plus(const MCSubtargetInfo &STI) {
2462 return isVI(STI) || isGFX9Plus(STI);
2463}
2464
2465bool isGFX9Plus(const MCSubtargetInfo &STI) {
2466 return isGFX9(STI) || isGFX10Plus(STI);
2467}
2468
2469bool isNotGFX9Plus(const MCSubtargetInfo &STI) { return !isGFX9Plus(STI); }
2470
2471bool hasPopsExitingWaveID(const MCSubtargetInfo &STI) {
2472 return STI.hasFeature(Feature: AMDGPU::FeaturePopsExitingWaveID);
2473}
2474
2475bool hasPrivateApertureRegs(const MCSubtargetInfo &STI) {
2476 return STI.hasFeature(Feature: AMDGPU::FeatureApertureRegs) &&
2477 !STI.hasFeature(Feature: AMDGPU::FeatureGloballyAddressableScratch);
2478}
2479
2480bool isGFX10(const MCSubtargetInfo &STI) {
2481 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10);
2482}
2483
2484bool isGFX10_GFX11(const MCSubtargetInfo &STI) {
2485 return isGFX10(STI) || isGFX11(STI);
2486}
2487
2488bool isGFX10Plus(const MCSubtargetInfo &STI) {
2489 return isGFX10(STI) || isGFX11Plus(STI);
2490}
2491
2492bool isGFX11(const MCSubtargetInfo &STI) {
2493 return STI.hasFeature(Feature: AMDGPU::FeatureGFX11);
2494}
2495
2496bool isGFX11Plus(const MCSubtargetInfo &STI) {
2497 return isGFX11(STI) || isGFX12Plus(STI);
2498}
2499
2500bool isGFX12(const MCSubtargetInfo &STI) {
2501 return STI.getFeatureBits()[AMDGPU::FeatureGFX12];
2502}
2503
2504bool isGFX12Plus(const MCSubtargetInfo &STI) {
2505 return isGFX12(STI) || isGFX13Plus(STI);
2506}
2507
2508bool isNotGFX12Plus(const MCSubtargetInfo &STI) { return !isGFX12Plus(STI); }
2509
2510bool isGFX1250(const MCSubtargetInfo &STI) {
2511 return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts] && !isGFX13(STI);
2512}
2513
2514bool isFullSIMDMode(const MCSubtargetInfo &STI) {
2515 return isGFX1250(STI) || !STI.getFeatureBits().test(I: FeatureCuMode);
2516}
2517
2518bool isGFX1250Plus(const MCSubtargetInfo &STI) {
2519 return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts];
2520}
2521
2522bool isGFX13(const MCSubtargetInfo &STI) {
2523 return STI.getFeatureBits()[AMDGPU::FeatureGFX13];
2524}
2525
2526bool isGFX13Plus(const MCSubtargetInfo &STI) { return isGFX13(STI); }
2527
2528bool supportsWGP(const MCSubtargetInfo &STI) {
2529 if (isGFX1250(STI))
2530 return false;
2531 return isGFX10Plus(STI);
2532}
2533
2534bool isNotGFX11Plus(const MCSubtargetInfo &STI) { return !isGFX11Plus(STI); }
2535
2536bool isNotGFX10Plus(const MCSubtargetInfo &STI) {
2537 return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI);
2538}
2539
2540bool isGFX10Before1030(const MCSubtargetInfo &STI) {
2541 return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI);
2542}
2543
2544bool isGCN3Encoding(const MCSubtargetInfo &STI) {
2545 return STI.hasFeature(Feature: AMDGPU::FeatureGCN3Encoding);
2546}
2547
2548bool isGFX10_BEncoding(const MCSubtargetInfo &STI) {
2549 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10_BEncoding);
2550}
2551
2552bool hasGFX10_3Insts(const MCSubtargetInfo &STI) {
2553 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10_3Insts);
2554}
2555
2556bool isGFX10_3_GFX11(const MCSubtargetInfo &STI) {
2557 return isGFX10_BEncoding(STI) && !isGFX12Plus(STI);
2558}
2559
2560bool isGFX90A(const MCSubtargetInfo &STI) {
2561 return STI.hasFeature(Feature: AMDGPU::FeatureGFX90AInsts);
2562}
2563
2564bool isGFX940(const MCSubtargetInfo &STI) {
2565 return STI.hasFeature(Feature: AMDGPU::FeatureGFX940Insts);
2566}
2567
2568bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) {
2569 return STI.hasFeature(Feature: AMDGPU::FeatureArchitectedFlatScratch);
2570}
2571
2572bool hasMAIInsts(const MCSubtargetInfo &STI) {
2573 return STI.hasFeature(Feature: AMDGPU::FeatureMAIInsts);
2574}
2575
2576bool hasVOPD(const MCSubtargetInfo &STI) {
2577 return STI.hasFeature(Feature: AMDGPU::FeatureVOPDInsts);
2578}
2579
2580bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI) {
2581 return STI.hasFeature(Feature: AMDGPU::FeatureDPPSrc1SGPR);
2582}
2583
2584unsigned hasKernargPreload(const MCSubtargetInfo &STI) {
2585 return STI.hasFeature(Feature: AMDGPU::FeatureKernargPreload);
2586}
2587
2588uint32_t getTotalNumVGPRs(bool has90AInsts, uint32_t ArgNumAGPR,
2589 uint32_t ArgNumVGPR) {
2590 if (has90AInsts && ArgNumAGPR)
2591 return alignTo(Value: ArgNumVGPR, Align: 4) + ArgNumAGPR;
2592 return std::max(a: ArgNumVGPR, b: ArgNumAGPR);
2593}
2594
2595bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI) {
2596 const MCRegisterClass &SGPRClass =
2597 TRI->getRegClass(i: AMDGPU::SReg_32RegClassID);
2598 const MCRegister FirstSubReg = TRI->getSubReg(Reg, Idx: AMDGPU::sub0);
2599 return SGPRClass.contains(Reg: FirstSubReg != 0 ? FirstSubReg : Reg) ||
2600 Reg == AMDGPU::SCC;
2601}
2602
2603bool isRsrcIndexReg(MCRegister Reg, const MCRegisterInfo &MRI) {
2604 return MRI.getRegClass(i: AMDGPU::RsrcReg32RegClassID).contains(Reg);
2605}
2606
2607bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI) {
2608 return MRI.getEncodingValue(Reg) & AMDGPU::HWEncoding::IS_HI16;
2609}
2610
2611#define MAP_REG2REG \
2612 using namespace AMDGPU; \
2613 switch (Reg.id()) { \
2614 default: \
2615 return Reg; \
2616 CASE_CI_VI(FLAT_SCR) \
2617 CASE_CI_VI(FLAT_SCR_LO) \
2618 CASE_CI_VI(FLAT_SCR_HI) \
2619 CASE_VI_GFX9PLUS(TTMP0) \
2620 CASE_VI_GFX9PLUS(TTMP1) \
2621 CASE_VI_GFX9PLUS(TTMP2) \
2622 CASE_VI_GFX9PLUS(TTMP3) \
2623 CASE_VI_GFX9PLUS(TTMP4) \
2624 CASE_VI_GFX9PLUS(TTMP5) \
2625 CASE_VI_GFX9PLUS(TTMP6) \
2626 CASE_VI_GFX9PLUS(TTMP7) \
2627 CASE_VI_GFX9PLUS(TTMP8) \
2628 CASE_VI_GFX9PLUS(TTMP9) \
2629 CASE_VI_GFX9PLUS(TTMP10) \
2630 CASE_VI_GFX9PLUS(TTMP11) \
2631 CASE_VI_GFX9PLUS(TTMP12) \
2632 CASE_VI_GFX9PLUS(TTMP13) \
2633 CASE_VI_GFX9PLUS(TTMP14) \
2634 CASE_VI_GFX9PLUS(TTMP15) \
2635 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \
2636 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \
2637 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \
2638 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \
2639 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \
2640 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \
2641 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \
2642 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \
2643 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \
2644 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \
2645 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \
2646 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \
2647 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
2648 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
2649 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2650 CASE_VI_GFX9PLUS( \
2651 TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2652 CASE_GFXPRE11_GFX11PLUS(M0) \
2653 CASE_GFXPRE11_GFX11PLUS(SGPR_NULL) \
2654 CASE_GFXPRE11_GFX11PLUS_TO(SGPR_NULL64, SGPR_NULL) \
2655 }
2656
2657#define CASE_CI_VI(node) \
2658 assert(!isSI(STI)); \
2659 case node: \
2660 return isCI(STI) ? node##_ci : node##_vi;
2661
2662#define CASE_VI_GFX9PLUS(node) \
2663 case node: \
2664 return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi;
2665
2666#define CASE_GFXPRE11_GFX11PLUS(node) \
2667 case node: \
2668 return isGFX11Plus(STI) ? node##_gfx11plus : node##_gfxpre11;
2669
2670#define CASE_GFXPRE11_GFX11PLUS_TO(node, result) \
2671 case node: \
2672 return isGFX11Plus(STI) ? result##_gfx11plus : result##_gfxpre11;
2673
2674MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI) {
2675 if (STI.getTargetTriple().getArch() == Triple::r600)
2676 return Reg;
2677 MAP_REG2REG
2678}
2679
2680#undef CASE_CI_VI
2681#undef CASE_VI_GFX9PLUS
2682#undef CASE_GFXPRE11_GFX11PLUS
2683#undef CASE_GFXPRE11_GFX11PLUS_TO
2684
2685#define CASE_CI_VI(node) \
2686 case node##_ci: \
2687 case node##_vi: \
2688 return node;
2689#define CASE_VI_GFX9PLUS(node) \
2690 case node##_vi: \
2691 case node##_gfx9plus: \
2692 return node;
2693#define CASE_GFXPRE11_GFX11PLUS(node) \
2694 case node##_gfx11plus: \
2695 case node##_gfxpre11: \
2696 return node;
2697#define CASE_GFXPRE11_GFX11PLUS_TO(node, result)
2698
2699MCRegister mc2PseudoReg(MCRegister Reg) { MAP_REG2REG }
2700
2701bool isInlineValue(MCRegister Reg) {
2702 switch (Reg.id()) {
2703 case AMDGPU::SRC_SHARED_BASE_LO:
2704 case AMDGPU::SRC_SHARED_BASE:
2705 case AMDGPU::SRC_SHARED_LIMIT_LO:
2706 case AMDGPU::SRC_SHARED_LIMIT:
2707 case AMDGPU::SRC_PRIVATE_BASE_LO:
2708 case AMDGPU::SRC_PRIVATE_BASE:
2709 case AMDGPU::SRC_PRIVATE_LIMIT_LO:
2710 case AMDGPU::SRC_PRIVATE_LIMIT:
2711 case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
2712 case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
2713 case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
2714 return true;
2715 case AMDGPU::SRC_VCCZ:
2716 case AMDGPU::SRC_EXECZ:
2717 case AMDGPU::SRC_SCC:
2718 return true;
2719 case AMDGPU::SGPR_NULL:
2720 return true;
2721 default:
2722 return false;
2723 }
2724}
2725
2726#undef CASE_CI_VI
2727#undef CASE_VI_GFX9PLUS
2728#undef CASE_GFXPRE11_GFX11PLUS
2729#undef CASE_GFXPRE11_GFX11PLUS_TO
2730#undef MAP_REG2REG
2731
2732bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2733 assert(OpNo < Desc.NumOperands);
2734 unsigned OpType = Desc.operands()[OpNo].OperandType;
2735 return OpType >= AMDGPU::OPERAND_KIMM_FIRST &&
2736 OpType <= AMDGPU::OPERAND_KIMM_LAST;
2737}
2738
2739bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2740 assert(OpNo < Desc.NumOperands);
2741 unsigned OpType = Desc.operands()[OpNo].OperandType;
2742 switch (OpType) {
2743 case AMDGPU::OPERAND_REG_IMM_FP32:
2744 case AMDGPU::OPERAND_REG_IMM_FP64:
2745 case AMDGPU::OPERAND_REG_IMM_FP16:
2746 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
2747 case AMDGPU::OPERAND_REG_IMM_V2FP16:
2748 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
2749 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
2750 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2751 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2752 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2753 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2754 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2755 case AMDGPU::OPERAND_REG_IMM_V2FP32:
2756 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2757 case AMDGPU::OPERAND_REG_IMM_V2FP64:
2758 return true;
2759 default:
2760 return false;
2761 }
2762}
2763
2764// Avoid using MCRegisterClass::getSize, since that function will go away
2765// (move from MC* level to Target* level). Return size in bits.
2766unsigned getRegBitWidth(unsigned RCID) {
2767 switch (RCID) {
2768 case AMDGPU::VGPR_16RegClassID:
2769 case AMDGPU::VGPR_16_Lo128RegClassID:
2770 case AMDGPU::SGPR_LO16RegClassID:
2771 case AMDGPU::AGPR_LO16RegClassID:
2772 return 16;
2773 case AMDGPU::SGPR_32RegClassID:
2774 case AMDGPU::VGPR_32RegClassID:
2775 case AMDGPU::VGPR_32_Lo256RegClassID:
2776 case AMDGPU::VRegOrLds_32RegClassID:
2777 case AMDGPU::AGPR_32RegClassID:
2778 case AMDGPU::VS_32RegClassID:
2779 case AMDGPU::AV_32RegClassID:
2780 case AMDGPU::SReg_32RegClassID:
2781 case AMDGPU::SReg_32_XM0RegClassID:
2782 case AMDGPU::SRegOrLds_32RegClassID:
2783 return 32;
2784 case AMDGPU::SGPR_64RegClassID:
2785 case AMDGPU::VS_64RegClassID:
2786 case AMDGPU::SReg_64RegClassID:
2787 case AMDGPU::VReg_64RegClassID:
2788 case AMDGPU::AReg_64RegClassID:
2789 case AMDGPU::SReg_64_XEXECRegClassID:
2790 case AMDGPU::VReg_64_Align2RegClassID:
2791 case AMDGPU::AReg_64_Align2RegClassID:
2792 case AMDGPU::AV_64RegClassID:
2793 case AMDGPU::AV_64_Align2RegClassID:
2794 case AMDGPU::VReg_64_Lo256_Align2RegClassID:
2795 case AMDGPU::VS_64_Lo256RegClassID:
2796 return 64;
2797 case AMDGPU::SGPR_96RegClassID:
2798 case AMDGPU::SReg_96RegClassID:
2799 case AMDGPU::VReg_96RegClassID:
2800 case AMDGPU::AReg_96RegClassID:
2801 case AMDGPU::VReg_96_Align2RegClassID:
2802 case AMDGPU::AReg_96_Align2RegClassID:
2803 case AMDGPU::AV_96RegClassID:
2804 case AMDGPU::AV_96_Align2RegClassID:
2805 case AMDGPU::VReg_96_Lo256_Align2RegClassID:
2806 return 96;
2807 case AMDGPU::SGPR_128RegClassID:
2808 case AMDGPU::SReg_128RegClassID:
2809 case AMDGPU::VReg_128RegClassID:
2810 case AMDGPU::AReg_128RegClassID:
2811 case AMDGPU::VReg_128_Align2RegClassID:
2812 case AMDGPU::AReg_128_Align2RegClassID:
2813 case AMDGPU::AV_128RegClassID:
2814 case AMDGPU::AV_128_Align2RegClassID:
2815 case AMDGPU::SReg_128_XNULLRegClassID:
2816 case AMDGPU::VReg_128_Lo256_Align2RegClassID:
2817 return 128;
2818 case AMDGPU::SGPR_160RegClassID:
2819 case AMDGPU::SReg_160RegClassID:
2820 case AMDGPU::VReg_160RegClassID:
2821 case AMDGPU::AReg_160RegClassID:
2822 case AMDGPU::VReg_160_Align2RegClassID:
2823 case AMDGPU::AReg_160_Align2RegClassID:
2824 case AMDGPU::AV_160RegClassID:
2825 case AMDGPU::AV_160_Align2RegClassID:
2826 case AMDGPU::VReg_160_Lo256_Align2RegClassID:
2827 return 160;
2828 case AMDGPU::SGPR_192RegClassID:
2829 case AMDGPU::SReg_192RegClassID:
2830 case AMDGPU::VReg_192RegClassID:
2831 case AMDGPU::AReg_192RegClassID:
2832 case AMDGPU::VReg_192_Align2RegClassID:
2833 case AMDGPU::AReg_192_Align2RegClassID:
2834 case AMDGPU::AV_192RegClassID:
2835 case AMDGPU::AV_192_Align2RegClassID:
2836 case AMDGPU::VReg_192_Lo256_Align2RegClassID:
2837 return 192;
2838 case AMDGPU::SGPR_224RegClassID:
2839 case AMDGPU::SReg_224RegClassID:
2840 case AMDGPU::VReg_224RegClassID:
2841 case AMDGPU::AReg_224RegClassID:
2842 case AMDGPU::VReg_224_Align2RegClassID:
2843 case AMDGPU::AReg_224_Align2RegClassID:
2844 case AMDGPU::AV_224RegClassID:
2845 case AMDGPU::AV_224_Align2RegClassID:
2846 case AMDGPU::VReg_224_Lo256_Align2RegClassID:
2847 return 224;
2848 case AMDGPU::SGPR_256RegClassID:
2849 case AMDGPU::SReg_256RegClassID:
2850 case AMDGPU::VReg_256RegClassID:
2851 case AMDGPU::AReg_256RegClassID:
2852 case AMDGPU::VReg_256_Align2RegClassID:
2853 case AMDGPU::AReg_256_Align2RegClassID:
2854 case AMDGPU::AV_256RegClassID:
2855 case AMDGPU::AV_256_Align2RegClassID:
2856 case AMDGPU::SReg_256_XNULLRegClassID:
2857 case AMDGPU::VReg_256_Lo256_Align2RegClassID:
2858 return 256;
2859 case AMDGPU::SGPR_288RegClassID:
2860 case AMDGPU::SReg_288RegClassID:
2861 case AMDGPU::VReg_288RegClassID:
2862 case AMDGPU::AReg_288RegClassID:
2863 case AMDGPU::VReg_288_Align2RegClassID:
2864 case AMDGPU::AReg_288_Align2RegClassID:
2865 case AMDGPU::AV_288RegClassID:
2866 case AMDGPU::AV_288_Align2RegClassID:
2867 case AMDGPU::VReg_288_Lo256_Align2RegClassID:
2868 return 288;
2869 case AMDGPU::SGPR_320RegClassID:
2870 case AMDGPU::SReg_320RegClassID:
2871 case AMDGPU::VReg_320RegClassID:
2872 case AMDGPU::AReg_320RegClassID:
2873 case AMDGPU::VReg_320_Align2RegClassID:
2874 case AMDGPU::AReg_320_Align2RegClassID:
2875 case AMDGPU::AV_320RegClassID:
2876 case AMDGPU::AV_320_Align2RegClassID:
2877 case AMDGPU::VReg_320_Lo256_Align2RegClassID:
2878 return 320;
2879 case AMDGPU::SGPR_352RegClassID:
2880 case AMDGPU::SReg_352RegClassID:
2881 case AMDGPU::VReg_352RegClassID:
2882 case AMDGPU::AReg_352RegClassID:
2883 case AMDGPU::VReg_352_Align2RegClassID:
2884 case AMDGPU::AReg_352_Align2RegClassID:
2885 case AMDGPU::AV_352RegClassID:
2886 case AMDGPU::AV_352_Align2RegClassID:
2887 case AMDGPU::VReg_352_Lo256_Align2RegClassID:
2888 return 352;
2889 case AMDGPU::SGPR_384RegClassID:
2890 case AMDGPU::SReg_384RegClassID:
2891 case AMDGPU::VReg_384RegClassID:
2892 case AMDGPU::AReg_384RegClassID:
2893 case AMDGPU::VReg_384_Align2RegClassID:
2894 case AMDGPU::AReg_384_Align2RegClassID:
2895 case AMDGPU::AV_384RegClassID:
2896 case AMDGPU::AV_384_Align2RegClassID:
2897 case AMDGPU::VReg_384_Lo256_Align2RegClassID:
2898 return 384;
2899 case AMDGPU::SGPR_512RegClassID:
2900 case AMDGPU::SReg_512RegClassID:
2901 case AMDGPU::VReg_512RegClassID:
2902 case AMDGPU::AReg_512RegClassID:
2903 case AMDGPU::VReg_512_Align2RegClassID:
2904 case AMDGPU::AReg_512_Align2RegClassID:
2905 case AMDGPU::AV_512RegClassID:
2906 case AMDGPU::AV_512_Align2RegClassID:
2907 case AMDGPU::VReg_512_Lo256_Align2RegClassID:
2908 return 512;
2909 case AMDGPU::SGPR_1024RegClassID:
2910 case AMDGPU::SReg_1024RegClassID:
2911 case AMDGPU::VReg_1024RegClassID:
2912 case AMDGPU::AReg_1024RegClassID:
2913 case AMDGPU::VReg_1024_Align2RegClassID:
2914 case AMDGPU::AReg_1024_Align2RegClassID:
2915 case AMDGPU::AV_1024RegClassID:
2916 case AMDGPU::AV_1024_Align2RegClassID:
2917 case AMDGPU::VReg_1024_Lo256_Align2RegClassID:
2918 return 1024;
2919 default:
2920 llvm_unreachable("Unexpected register class");
2921 }
2922}
2923
2924unsigned getRegBitWidth(const MCRegisterClass &RC) {
2925 return getRegBitWidth(RCID: RC.getID());
2926}
2927
2928bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
2929 if (isInlinableIntLiteral(Literal))
2930 return true;
2931
2932 uint64_t Val = static_cast<uint64_t>(Literal);
2933 return (Val == llvm::bit_cast<uint64_t>(from: 0.0)) ||
2934 (Val == llvm::bit_cast<uint64_t>(from: 1.0)) ||
2935 (Val == llvm::bit_cast<uint64_t>(from: -1.0)) ||
2936 (Val == llvm::bit_cast<uint64_t>(from: 0.5)) ||
2937 (Val == llvm::bit_cast<uint64_t>(from: -0.5)) ||
2938 (Val == llvm::bit_cast<uint64_t>(from: 2.0)) ||
2939 (Val == llvm::bit_cast<uint64_t>(from: -2.0)) ||
2940 (Val == llvm::bit_cast<uint64_t>(from: 4.0)) ||
2941 (Val == llvm::bit_cast<uint64_t>(from: -4.0)) ||
2942 (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
2943}
2944
2945bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
2946 if (isInlinableIntLiteral(Literal))
2947 return true;
2948
2949 // The actual type of the operand does not seem to matter as long
2950 // as the bits match one of the inline immediate values. For example:
2951 //
2952 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
2953 // so it is a legal inline immediate.
2954 //
2955 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
2956 // floating-point, so it is a legal inline immediate.
2957
2958 uint32_t Val = static_cast<uint32_t>(Literal);
2959 return (Val == llvm::bit_cast<uint32_t>(from: 0.0f)) ||
2960 (Val == llvm::bit_cast<uint32_t>(from: 1.0f)) ||
2961 (Val == llvm::bit_cast<uint32_t>(from: -1.0f)) ||
2962 (Val == llvm::bit_cast<uint32_t>(from: 0.5f)) ||
2963 (Val == llvm::bit_cast<uint32_t>(from: -0.5f)) ||
2964 (Val == llvm::bit_cast<uint32_t>(from: 2.0f)) ||
2965 (Val == llvm::bit_cast<uint32_t>(from: -2.0f)) ||
2966 (Val == llvm::bit_cast<uint32_t>(from: 4.0f)) ||
2967 (Val == llvm::bit_cast<uint32_t>(from: -4.0f)) ||
2968 (Val == 0x3e22f983 && HasInv2Pi);
2969}
2970
2971bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi, bool *IsBF16) {
2972 if (!HasInv2Pi)
2973 return false;
2974 if (IsBF16)
2975 *IsBF16 = false;
2976 if (isInlinableIntLiteral(Literal))
2977 return true;
2978 uint16_t Val = static_cast<uint16_t>(Literal);
2979 if (IsBF16)
2980 *IsBF16 = true;
2981 return Val == 0x3F00 || // 0.5
2982 Val == 0xBF00 || // -0.5
2983 Val == 0x3F80 || // 1.0
2984 Val == 0xBF80 || // -1.0
2985 Val == 0x4000 || // 2.0
2986 Val == 0xC000 || // -2.0
2987 Val == 0x4080 || // 4.0
2988 Val == 0xC080 || // -4.0
2989 Val == 0x3E22; // 1.0 / (2.0 * pi)
2990}
2991
2992bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi) {
2993 return isInlinableLiteral32(Literal, HasInv2Pi);
2994}
2995
2996bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) {
2997 if (!HasInv2Pi)
2998 return false;
2999 if (isInlinableIntLiteral(Literal))
3000 return true;
3001 uint16_t Val = static_cast<uint16_t>(Literal);
3002 return Val == 0x3C00 || // 1.0
3003 Val == 0xBC00 || // -1.0
3004 Val == 0x3800 || // 0.5
3005 Val == 0xB800 || // -0.5
3006 Val == 0x4000 || // 2.0
3007 Val == 0xC000 || // -2.0
3008 Val == 0x4400 || // 4.0
3009 Val == 0xC400 || // -4.0
3010 Val == 0x3118; // 1/2pi
3011}
3012
3013std::optional<unsigned> getInlineEncodingV216(bool IsFloat, uint32_t Literal) {
3014 // Unfortunately, the Instruction Set Architecture Reference Guide is
3015 // misleading about how the inline operands work for (packed) 16-bit
3016 // instructions. In a nutshell, the actual HW behavior is:
3017 //
3018 // - integer encodings (-16 .. 64) are always produced as sign-extended
3019 // 32-bit values
3020 // - float encodings are produced as:
3021 // - for F16 instructions: corresponding half-precision float values in
3022 // the LSBs, 0 in the MSBs
3023 // - for UI16 instructions: corresponding single-precision float value
3024 int32_t Signed = static_cast<int32_t>(Literal);
3025 if (Signed >= 0 && Signed <= 64)
3026 return 128 + Signed;
3027
3028 if (Signed >= -16 && Signed <= -1)
3029 return 192 + std::abs(x: Signed);
3030
3031 if (IsFloat) {
3032 // clang-format off
3033 switch (Literal) {
3034 case 0x3800: return 240; // 0.5
3035 case 0xB800: return 241; // -0.5
3036 case 0x3C00: return 242; // 1.0
3037 case 0xBC00: return 243; // -1.0
3038 case 0x4000: return 244; // 2.0
3039 case 0xC000: return 245; // -2.0
3040 case 0x4400: return 246; // 4.0
3041 case 0xC400: return 247; // -4.0
3042 case 0x3118: return 248; // 1.0 / (2.0 * pi)
3043 default: break;
3044 }
3045 // clang-format on
3046 } else {
3047 // clang-format off
3048 switch (Literal) {
3049 case 0x3F000000: return 240; // 0.5
3050 case 0xBF000000: return 241; // -0.5
3051 case 0x3F800000: return 242; // 1.0
3052 case 0xBF800000: return 243; // -1.0
3053 case 0x40000000: return 244; // 2.0
3054 case 0xC0000000: return 245; // -2.0
3055 case 0x40800000: return 246; // 4.0
3056 case 0xC0800000: return 247; // -4.0
3057 case 0x3E22F983: return 248; // 1.0 / (2.0 * pi)
3058 default: break;
3059 }
3060 // clang-format on
3061 }
3062
3063 return {};
3064}
3065
3066// Encoding of the literal as an inline constant for a V_PK_*_IU16 instruction
3067// or nullopt.
3068std::optional<unsigned> getInlineEncodingV2I16(uint32_t Literal) {
3069 return getInlineEncodingV216(IsFloat: false, Literal);
3070}
3071
3072// Encoding of the literal as an inline constant for a V_PK_*_BF16 instruction
3073// or nullopt.
3074std::optional<unsigned> getInlineEncodingV2BF16(uint32_t Literal) {
3075 int32_t Signed = static_cast<int32_t>(Literal);
3076 if (Signed >= 0 && Signed <= 64)
3077 return 128 + Signed;
3078
3079 if (Signed >= -16 && Signed <= -1)
3080 return 192 + std::abs(x: Signed);
3081
3082 // clang-format off
3083 switch (Literal) {
3084 case 0x3F00: return 240; // 0.5
3085 case 0xBF00: return 241; // -0.5
3086 case 0x3F80: return 242; // 1.0
3087 case 0xBF80: return 243; // -1.0
3088 case 0x4000: return 244; // 2.0
3089 case 0xC000: return 245; // -2.0
3090 case 0x4080: return 246; // 4.0
3091 case 0xC080: return 247; // -4.0
3092 case 0x3E22: return 248; // 1.0 / (2.0 * pi)
3093 default: break;
3094 }
3095 // clang-format on
3096
3097 return std::nullopt;
3098}
3099
3100// Encoding of the literal as an inline constant for a V_PK_*_F16 instruction
3101// or nullopt.
3102std::optional<unsigned> getInlineEncodingV2F16(uint32_t Literal) {
3103 return getInlineEncodingV216(IsFloat: true, Literal);
3104}
3105
3106// Encoding of the literal as an inline constant for V_PK_FMAC_F16 instruction
3107// or nullopt. This accounts for different inline constant behavior:
3108// - Pre-GFX11: fp16 inline constants have the value in low 16 bits, 0 in high
3109// - GFX11+: fp16 inline constants are duplicated into both halves
3110std::optional<unsigned> getPKFMACF16InlineEncoding(uint32_t Literal,
3111 bool IsGFX11Plus) {
3112 // Pre-GFX11 behavior: f16 in low bits, 0 in high bits
3113 if (!IsGFX11Plus)
3114 return getInlineEncodingV216(/*IsFloat=*/true, Literal);
3115
3116 // GFX11+ behavior: f16 duplicated in both halves
3117 // First, check for sign-extended integer inline constants (-16 to 64)
3118 // These work the same across all generations
3119 int32_t Signed = static_cast<int32_t>(Literal);
3120 if (Signed >= 0 && Signed <= 64)
3121 return 128 + Signed;
3122
3123 if (Signed >= -16 && Signed <= -1)
3124 return 192 + std::abs(x: Signed);
3125
3126 // For float inline constants on GFX11+, both halves must be equal
3127 uint16_t Lo = static_cast<uint16_t>(Literal);
3128 uint16_t Hi = static_cast<uint16_t>(Literal >> 16);
3129 if (Lo != Hi)
3130 return std::nullopt;
3131 return getInlineEncodingV216(/*IsFloat=*/true, Literal: Lo);
3132}
3133
3134// Whether the given literal can be inlined for a V_PK_* instruction.
3135bool isInlinableLiteralV216(uint32_t Literal, uint8_t OpType) {
3136 switch (OpType) {
3137 case AMDGPU::OPERAND_REG_IMM_V2INT16:
3138 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
3139 return getInlineEncodingV216(IsFloat: false, Literal).has_value();
3140 case AMDGPU::OPERAND_REG_IMM_V2FP16:
3141 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
3142 return getInlineEncodingV216(IsFloat: true, Literal).has_value();
3143 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
3144 llvm_unreachable("OPERAND_REG_IMM_V2FP16_SPLAT is not supported");
3145 case AMDGPU::OPERAND_REG_IMM_V2BF16:
3146 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
3147 return isInlinableLiteralV2BF16(Literal);
3148 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
3149 return false;
3150 default:
3151 llvm_unreachable("bad packed operand type");
3152 }
3153}
3154
3155// Whether the given literal can be inlined for a V_PK_*_IU16 instruction.
3156bool isInlinableLiteralV2I16(uint32_t Literal) {
3157 return getInlineEncodingV2I16(Literal).has_value();
3158}
3159
3160// Whether the given literal can be inlined for a V_PK_*_BF16 instruction.
3161bool isInlinableLiteralV2BF16(uint32_t Literal) {
3162 return getInlineEncodingV2BF16(Literal).has_value();
3163}
3164
3165// Whether the given literal can be inlined for a V_PK_*_F16 instruction.
3166bool isInlinableLiteralV2F16(uint32_t Literal) {
3167 return getInlineEncodingV2F16(Literal).has_value();
3168}
3169
3170// Whether the given literal can be inlined for V_PK_FMAC_F16 instruction.
3171bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus) {
3172 return getPKFMACF16InlineEncoding(Literal, IsGFX11Plus).has_value();
3173}
3174
3175bool isValid32BitLiteral(uint64_t Val, bool IsFP64) {
3176 if (IsFP64)
3177 return !Lo_32(Value: Val);
3178
3179 return isUInt<32>(x: Val) || isInt<32>(x: Val);
3180}
3181
3182int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit) {
3183 switch (Type) {
3184 default:
3185 break;
3186 case OPERAND_REG_IMM_BF16:
3187 case OPERAND_REG_IMM_FP16:
3188 case OPERAND_REG_IMM_NOINLINE_FP16:
3189 case OPERAND_REG_INLINE_C_BF16:
3190 case OPERAND_REG_INLINE_C_FP16:
3191 return Imm & 0xffff;
3192 case OPERAND_INLINE_SPLIT_BARRIER_INT32:
3193 case OPERAND_REG_IMM_FP32:
3194 case OPERAND_REG_IMM_INT32:
3195 case OPERAND_REG_IMM_V2BF16:
3196 case OPERAND_REG_IMM_V2FP16:
3197 case OPERAND_REG_IMM_V2FP16_SPLAT:
3198 case OPERAND_REG_IMM_V2FP32:
3199 case OPERAND_REG_IMM_V2INT16:
3200 case OPERAND_REG_IMM_V2INT32:
3201 case OPERAND_REG_INLINE_AC_FP32:
3202 case OPERAND_REG_INLINE_AC_INT32:
3203 case OPERAND_REG_INLINE_C_FP32:
3204 case OPERAND_REG_INLINE_C_INT32:
3205 return Lo_32(Value: Imm);
3206 case OPERAND_REG_IMM_FP64:
3207 case AMDGPU::OPERAND_REG_IMM_V2FP64:
3208 return IsLit ? Imm : Hi_32(Value: Imm);
3209 }
3210 return Imm;
3211}
3212
3213bool isArgPassedInSGPR(const Argument *A) {
3214 const Function *F = A->getParent();
3215
3216 // Arguments to compute shaders are never a source of divergence.
3217 CallingConv::ID CC = F->getCallingConv();
3218 switch (CC) {
3219 case CallingConv::AMDGPU_KERNEL:
3220 case CallingConv::SPIR_KERNEL:
3221 return true;
3222 case CallingConv::AMDGPU_VS:
3223 case CallingConv::AMDGPU_LS:
3224 case CallingConv::AMDGPU_HS:
3225 case CallingConv::AMDGPU_ES:
3226 case CallingConv::AMDGPU_GS:
3227 case CallingConv::AMDGPU_PS:
3228 case CallingConv::AMDGPU_CS:
3229 case CallingConv::AMDGPU_Gfx:
3230 case CallingConv::AMDGPU_CS_Chain:
3231 case CallingConv::AMDGPU_CS_ChainPreserve:
3232 // For non-compute shaders, SGPR inputs are marked with either inreg or
3233 // byval. Everything else is in VGPRs.
3234 return A->hasAttribute(Kind: Attribute::InReg) ||
3235 A->hasAttribute(Kind: Attribute::ByVal);
3236 default:
3237 // TODO: treat i1 as divergent?
3238 return A->hasAttribute(Kind: Attribute::InReg);
3239 }
3240}
3241
3242bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo) {
3243 // Arguments to compute shaders are never a source of divergence.
3244 CallingConv::ID CC = CB->getCallingConv();
3245 switch (CC) {
3246 case CallingConv::AMDGPU_KERNEL:
3247 case CallingConv::SPIR_KERNEL:
3248 return true;
3249 case CallingConv::AMDGPU_VS:
3250 case CallingConv::AMDGPU_LS:
3251 case CallingConv::AMDGPU_HS:
3252 case CallingConv::AMDGPU_ES:
3253 case CallingConv::AMDGPU_GS:
3254 case CallingConv::AMDGPU_PS:
3255 case CallingConv::AMDGPU_CS:
3256 case CallingConv::AMDGPU_Gfx:
3257 case CallingConv::AMDGPU_CS_Chain:
3258 case CallingConv::AMDGPU_CS_ChainPreserve:
3259 // For non-compute shaders, SGPR inputs are marked with either inreg or
3260 // byval. Everything else is in VGPRs.
3261 return CB->paramHasAttr(ArgNo, Kind: Attribute::InReg) ||
3262 CB->isByValArgument(ArgNo);
3263 default:
3264 return CB->paramHasAttr(ArgNo, Kind: Attribute::InReg);
3265 }
3266}
3267
3268static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
3269 return isGCN3Encoding(STI: ST) || isGFX10Plus(STI: ST);
3270}
3271
3272bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
3273 int64_t EncodedOffset) {
3274 if (isGFX12Plus(STI: ST))
3275 return isUInt<23>(x: EncodedOffset);
3276
3277 return hasSMEMByteOffset(ST) ? isUInt<20>(x: EncodedOffset)
3278 : isUInt<8>(x: EncodedOffset);
3279}
3280
3281bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
3282 int64_t EncodedOffset, bool IsBuffer) {
3283 if (isGFX12Plus(STI: ST)) {
3284 if (IsBuffer && EncodedOffset < 0)
3285 return false;
3286 return isInt<24>(x: EncodedOffset);
3287 }
3288
3289 return !IsBuffer && hasSMRDSignedImmOffset(ST) && isInt<21>(x: EncodedOffset);
3290}
3291
3292static bool isDwordAligned(uint64_t ByteOffset) {
3293 return (ByteOffset & 3) == 0;
3294}
3295
3296uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
3297 uint64_t ByteOffset) {
3298 if (hasSMEMByteOffset(ST))
3299 return ByteOffset;
3300
3301 assert(isDwordAligned(ByteOffset));
3302 return ByteOffset >> 2;
3303}
3304
3305std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
3306 int64_t ByteOffset, bool IsBuffer,
3307 bool HasSOffset) {
3308 // For unbuffered smem loads, it is illegal for the Immediate Offset to be
3309 // negative if the resulting (Offset + (M0 or SOffset or zero) is negative.
3310 // Handle case where SOffset is not present.
3311 if (!IsBuffer && !HasSOffset && ByteOffset < 0 && hasSMRDSignedImmOffset(ST))
3312 return std::nullopt;
3313
3314 if (isGFX12Plus(STI: ST)) // 24 bit signed offsets
3315 return isInt<24>(x: ByteOffset) ? std::optional<int64_t>(ByteOffset)
3316 : std::nullopt;
3317
3318 // The signed version is always a byte offset.
3319 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
3320 assert(hasSMEMByteOffset(ST));
3321 return isInt<20>(x: ByteOffset) ? std::optional<int64_t>(ByteOffset)
3322 : std::nullopt;
3323 }
3324
3325 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
3326 return std::nullopt;
3327
3328 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3329 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
3330 ? std::optional<int64_t>(EncodedOffset)
3331 : std::nullopt;
3332}
3333
3334std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
3335 int64_t ByteOffset) {
3336 if (!isCI(STI: ST) || !isDwordAligned(ByteOffset))
3337 return std::nullopt;
3338
3339 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3340 return isUInt<32>(x: EncodedOffset) ? std::optional<int64_t>(EncodedOffset)
3341 : std::nullopt;
3342}
3343
3344unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST) {
3345 if (ST.getFeatureBits().test(I: FeatureFlatOffsetBits12))
3346 return 12;
3347 if (ST.getFeatureBits().test(I: FeatureFlatOffsetBits24))
3348 return 24;
3349 return 13;
3350}
3351
3352namespace {
3353
3354struct SourceOfDivergence {
3355 unsigned Intr;
3356};
3357const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
3358
3359struct AlwaysUniform {
3360 unsigned Intr;
3361};
3362const AlwaysUniform *lookupAlwaysUniform(unsigned Intr);
3363
3364struct LDSDMAIntrinsic {
3365 unsigned Intr;
3366};
3367const LDSDMAIntrinsic *lookupLDSDMAIntrinsic(unsigned Intr);
3368
3369struct ExponentOnlyScaleIntrinsic {
3370 unsigned Intr;
3371 uint8_t ScaleArg;
3372};
3373const ExponentOnlyScaleIntrinsic *
3374lookupExponentOnlyScaleIntrinsic(unsigned Intr);
3375
3376#define GET_SourcesOfDivergence_IMPL
3377#define GET_UniformIntrinsics_IMPL
3378#define GET_LDSDMAIntrinsicTable_IMPL
3379#define GET_ExponentOnlyScaleIntrinsics_IMPL
3380#define GET_Gfx9BufferFormat_IMPL
3381#define GET_Gfx10BufferFormat_IMPL
3382#define GET_Gfx11PlusBufferFormat_IMPL
3383
3384#include "AMDGPUGenSearchableTables.inc"
3385
3386} // end anonymous namespace
3387
3388bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
3389 return lookupSourceOfDivergence(Intr: IntrID);
3390}
3391
3392bool isIntrinsicAlwaysUniform(unsigned IntrID) {
3393 return lookupAlwaysUniform(Intr: IntrID);
3394}
3395
3396bool isTriviallyUniform(const Value *V, const Instruction *CtxI) {
3397 if (isa<Constant>(Val: V))
3398 return true;
3399 if (const auto *A = dyn_cast<Argument>(Val: V))
3400 return isArgPassedInSGPR(A);
3401 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V)) {
3402 if (!isIntrinsicAlwaysUniform(IntrID: II->getIntrinsicID()))
3403 return false;
3404 // If II and CtxI are in different blocks then there is a possibility of a
3405 // temporal divergence.
3406 return II->getParent() == CtxI->getParent();
3407 }
3408 return false;
3409}
3410
3411bool isTriviallyUniform(const Use &U) {
3412 return isTriviallyUniform(V: U.get(), CtxI: cast<Instruction>(Val: U.getUser()));
3413}
3414
3415bool isLDSDMAIntrinsic(unsigned IntrID) {
3416 return lookupLDSDMAIntrinsic(Intr: IntrID);
3417}
3418
3419std::optional<unsigned> getExponentOnlyScaleArgIdx(unsigned IntrID) {
3420 if (const ExponentOnlyScaleIntrinsic *Info =
3421 lookupExponentOnlyScaleIntrinsic(Intr: IntrID))
3422 return Info->ScaleArg;
3423 return std::nullopt;
3424}
3425
3426const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
3427 uint8_t NumComponents,
3428 uint8_t NumFormat,
3429 const MCSubtargetInfo &STI) {
3430 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(
3431 BitsPerComp, NumComponents, NumFormat)
3432 : isGFX10(STI)
3433 ? getGfx10BufferFormatInfo(BitsPerComp, NumComponents, NumFormat)
3434 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
3435}
3436
3437const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
3438 const MCSubtargetInfo &STI) {
3439 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(Format)
3440 : isGFX10(STI) ? getGfx10BufferFormatInfo(Format)
3441 : getGfx9BufferFormatInfo(Format);
3442}
3443
3444const MCRegisterClass *getVGPRPhysRegClass(MCRegister Reg,
3445 const MCRegisterInfo &MRI) {
3446 const unsigned VGPRClasses[] = {
3447 AMDGPU::VGPR_16RegClassID, AMDGPU::VGPR_32RegClassID,
3448 AMDGPU::VReg_64RegClassID, AMDGPU::VReg_96RegClassID,
3449 AMDGPU::VReg_128RegClassID, AMDGPU::VReg_160RegClassID,
3450 AMDGPU::VReg_192RegClassID, AMDGPU::VReg_224RegClassID,
3451 AMDGPU::VReg_256RegClassID, AMDGPU::VReg_288RegClassID,
3452 AMDGPU::VReg_320RegClassID, AMDGPU::VReg_352RegClassID,
3453 AMDGPU::VReg_384RegClassID, AMDGPU::VReg_512RegClassID,
3454 AMDGPU::VReg_1024RegClassID};
3455
3456 for (unsigned RCID : VGPRClasses) {
3457 const MCRegisterClass &RC = MRI.getRegClass(i: RCID);
3458 if (RC.contains(Reg))
3459 return &RC;
3460 }
3461
3462 return nullptr;
3463}
3464
3465unsigned getVGPREncodingMSBs(MCRegister Reg, const MCRegisterInfo &MRI) {
3466 unsigned Enc = MRI.getEncodingValue(Reg);
3467 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3468 return Idx >> 8;
3469}
3470
3471MCRegister getVGPRWithMSBs(MCRegister Reg, unsigned MSBs,
3472 const MCRegisterInfo &MRI) {
3473 unsigned Enc = MRI.getEncodingValue(Reg);
3474 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3475 if (Idx >= 0x100)
3476 return MCRegister();
3477
3478 const MCRegisterClass *RC = getVGPRPhysRegClass(Reg, MRI);
3479 if (!RC)
3480 return MCRegister();
3481
3482 Idx |= MSBs << 8;
3483 if (RC->getID() == AMDGPU::VGPR_16RegClassID) {
3484 // This class has 2048 registers with interleaved lo16 and hi16.
3485 Idx *= 2;
3486 if (Enc & AMDGPU::HWEncoding::IS_HI16)
3487 ++Idx;
3488 }
3489
3490 return RC->getRegister(i: Idx);
3491}
3492
3493static std::optional<unsigned>
3494convertSetRegImmToVgprMSBs(uint64_t Imm, uint64_t Simm16,
3495 bool HasSetregVGPRMSBFixup) {
3496 constexpr unsigned VGPRMSBShift =
3497 llvm::countr_zero_constexpr<unsigned>(Val: AMDGPU::Hwreg::DST_VGPR_MSB);
3498
3499 auto [HwRegId, Offset, Size] = Hwreg::HwregEncoding::decode(Encoded: Simm16);
3500 if (HwRegId != Hwreg::ID_MODE ||
3501 (!HasSetregVGPRMSBFixup && (Offset + Size) < VGPRMSBShift))
3502 return {};
3503 // If there is SetregVGPRMSBFixup then Offset is ignored.
3504 if (!HasSetregVGPRMSBFixup)
3505 Imm <<= Offset;
3506 Imm = (Imm & Hwreg::VGPR_MSB_MASK) >> VGPRMSBShift;
3507 if (!HasSetregVGPRMSBFixup)
3508 Imm &= llvm::maskTrailingOnes<unsigned>(N: Size);
3509 return llvm::rotr<uint8_t>(V: static_cast<uint8_t>(Imm), /*R=*/2);
3510}
3511
3512std::optional<unsigned> convertSetRegImmToVgprMSBs(const MachineInstr &MI,
3513 bool HasSetregVGPRMSBFixup) {
3514 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32);
3515 return convertSetRegImmToVgprMSBs(Imm: MI.getOperand(i: 0).getImm(),
3516 Simm16: MI.getOperand(i: 1).getImm(),
3517 HasSetregVGPRMSBFixup);
3518}
3519
3520std::optional<unsigned> convertSetRegImmToVgprMSBs(const MCInst &MI,
3521 bool HasSetregVGPRMSBFixup) {
3522 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_gfx12);
3523 return convertSetRegImmToVgprMSBs(Imm: MI.getOperand(i: 0).getImm(),
3524 Simm16: MI.getOperand(i: 1).getImm(),
3525 HasSetregVGPRMSBFixup);
3526}
3527
3528std::pair<const AMDGPU::OpName *, const AMDGPU::OpName *>
3529getVGPRLoweringOperandTables(const MCInstrDesc &Desc) {
3530 static const AMDGPU::OpName VOPOps[4] = {
3531 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2,
3532 AMDGPU::OpName::vdst};
3533 static const AMDGPU::OpName VDSOps[4] = {
3534 AMDGPU::OpName::addr, AMDGPU::OpName::data0, AMDGPU::OpName::data1,
3535 AMDGPU::OpName::vdst};
3536 static const AMDGPU::OpName FLATOps[4] = {
3537 AMDGPU::OpName::vaddr, AMDGPU::OpName::vdata,
3538 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdst};
3539 static const AMDGPU::OpName BUFOps[4] = {
3540 AMDGPU::OpName::vaddr, AMDGPU::OpName::NUM_OPERAND_NAMES,
3541 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdata};
3542 static const AMDGPU::OpName VIMGOps[4] = {
3543 AMDGPU::OpName::vaddr0, AMDGPU::OpName::vaddr1, AMDGPU::OpName::vaddr2,
3544 AMDGPU::OpName::vdata};
3545
3546 // For VOPD instructions MSB of a corresponding Y component operand VGPR
3547 // address is supposed to match X operand, otherwise VOPD shall not be
3548 // combined.
3549 static const AMDGPU::OpName VOPDOpsX[4] = {
3550 AMDGPU::OpName::src0X, AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vsrc2X,
3551 AMDGPU::OpName::vdstX};
3552 static const AMDGPU::OpName VOPDOpsY[4] = {
3553 AMDGPU::OpName::src0Y, AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vsrc2Y,
3554 AMDGPU::OpName::vdstY};
3555
3556 // VOP2 MADMK instructions use src0, imm, src1 scheme.
3557 static const AMDGPU::OpName VOP2MADMKOps[4] = {
3558 AMDGPU::OpName::src0, AMDGPU::OpName::NUM_OPERAND_NAMES,
3559 AMDGPU::OpName::src1, AMDGPU::OpName::vdst};
3560 static const AMDGPU::OpName VOPDFMAMKOpsX[4] = {
3561 AMDGPU::OpName::src0X, AMDGPU::OpName::NUM_OPERAND_NAMES,
3562 AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vdstX};
3563 static const AMDGPU::OpName VOPDFMAMKOpsY[4] = {
3564 AMDGPU::OpName::src0Y, AMDGPU::OpName::NUM_OPERAND_NAMES,
3565 AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vdstY};
3566
3567 if (SIInstrFlags::isVOP1(O: Desc) || SIInstrFlags::isVOP2(O: Desc) ||
3568 SIInstrFlags::isVOP3Like(O: Desc) || SIInstrFlags::isVOPC(O: Desc) ||
3569 SIInstrFlags::isDPP(O: Desc)) {
3570 switch (Desc.getOpcode()) {
3571 // LD_SCALE operands ignore MSB.
3572 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32:
3573 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32_gfx1250:
3574 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64:
3575 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64_gfx1250:
3576 return {};
3577 case AMDGPU::V_FMAMK_F16:
3578 case AMDGPU::V_FMAMK_F16_t16:
3579 case AMDGPU::V_FMAMK_F16_t16_gfx12:
3580 case AMDGPU::V_FMAMK_F16_fake16:
3581 case AMDGPU::V_FMAMK_F16_fake16_gfx12:
3582 case AMDGPU::V_FMAMK_F32:
3583 case AMDGPU::V_FMAMK_F32_gfx12:
3584 case AMDGPU::V_FMAMK_F64:
3585 case AMDGPU::V_FMAMK_F64_gfx1250:
3586 return {VOP2MADMKOps, nullptr};
3587 default:
3588 break;
3589 }
3590 return {VOPOps, nullptr};
3591 }
3592
3593 if (SIInstrFlags::isDS(O: Desc))
3594 return {VDSOps, nullptr};
3595
3596 if (SIInstrFlags::isFLAT(O: Desc))
3597 return {FLATOps, nullptr};
3598
3599 if (SIInstrFlags::isBuffer(O: Desc))
3600 return {BUFOps, nullptr};
3601
3602 if (SIInstrFlags::isVIMAGE(O: Desc))
3603 return {VIMGOps, nullptr};
3604
3605 if (AMDGPU::isVOPD(Opc: Desc.getOpcode())) {
3606 auto [OpX, OpY] = getVOPDComponents(VOPDOpcode: Desc.getOpcode());
3607 return {(OpX == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsX : VOPDOpsX,
3608 (OpY == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsY : VOPDOpsY};
3609 }
3610
3611 assert(!SIInstrFlags::isMIMG(Desc));
3612
3613 if (SIInstrFlags::isVSAMPLE(O: Desc) || SIInstrFlags::isEXP(O: Desc))
3614 llvm_unreachable("Sample and export VGPR lowering is not implemented and"
3615 " these instructions are not expected on gfx1250");
3616
3617 return {};
3618}
3619
3620bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode) {
3621 const MCInstrDesc &Desc = MII.get(Opcode);
3622 if (SIInstrFlags::isSMRD(O: Desc))
3623 return Desc.mayLoad() && !Desc.mayStore() && !getSMEMIsBuffer(Opc: Opcode);
3624 if (!SIInstrFlags::isFLAT(O: Desc))
3625 return false;
3626
3627 // Only SV and SVS modes are supported.
3628 if (SIInstrFlags::isFlatScratch(O: MII, O: Opcode))
3629 return hasNamedOperand(Opcode, NamedIdx: OpName::vaddr);
3630
3631 // Only GVS mode is supported.
3632 return hasNamedOperand(Opcode, NamedIdx: OpName::vaddr) &&
3633 hasNamedOperand(Opcode, NamedIdx: OpName::saddr);
3634
3635 return false;
3636}
3637
3638static bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc,
3639 const MCInstrInfo &MII,
3640 const MCSubtargetInfo &ST) {
3641 for (auto OpName : {OpName::vdst, OpName::src0, OpName::src1, OpName::src2}) {
3642 int Idx = getNamedOperandIdx(Opcode: OpDesc.getOpcode(), Name: OpName);
3643 if (Idx == -1)
3644 continue;
3645
3646 const MCOperandInfo &OpInfo = OpDesc.operands()[Idx];
3647 int16_t RegClass = MII.getOpRegClassID(
3648 OpInfo, HwModeId: ST.getHwMode(type: MCSubtargetInfo::HwMode_RegInfo));
3649 if (RegClass == AMDGPU::VReg_64RegClassID ||
3650 RegClass == AMDGPU::VReg_64_Align2RegClassID)
3651 return true;
3652 }
3653
3654 return false;
3655}
3656
3657bool isDPALU_DPP32BitOpc(unsigned Opc) {
3658 switch (Opc) {
3659 case AMDGPU::V_MUL_LO_U32_e64:
3660 case AMDGPU::V_MUL_LO_U32_e64_dpp:
3661 case AMDGPU::V_MUL_LO_U32_e64_dpp_gfx1250:
3662 case AMDGPU::V_MUL_HI_U32_e64:
3663 case AMDGPU::V_MUL_HI_U32_e64_dpp:
3664 case AMDGPU::V_MUL_HI_U32_e64_dpp_gfx1250:
3665 case AMDGPU::V_MUL_HI_I32_e64:
3666 case AMDGPU::V_MUL_HI_I32_e64_dpp:
3667 case AMDGPU::V_MUL_HI_I32_e64_dpp_gfx1250:
3668 case AMDGPU::V_MAD_U32_e64:
3669 case AMDGPU::V_MAD_U32_e64_dpp:
3670 case AMDGPU::V_MAD_U32_e64_dpp_gfx1250:
3671 return true;
3672 default:
3673 return false;
3674 }
3675}
3676
3677bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
3678 const MCSubtargetInfo &ST) {
3679 if (isDPALU_DPP32BitOpc(Opc: OpDesc.getOpcode()))
3680 return true;
3681
3682 return hasAny64BitVGPROperands(OpDesc, MII, ST);
3683}
3684
3685bool isPackedSingleSGPRFP32Inst(unsigned Opc) {
3686 switch (Opc) {
3687 case AMDGPU::V_PK_ADD_F32_gfx1250:
3688 case AMDGPU::V_PK_ADD_F32_gfx1250_gfx12:
3689 case AMDGPU::V_PK_MUL_F32_gfx1250:
3690 case AMDGPU::V_PK_MUL_F32_gfx1250_gfx12:
3691 case AMDGPU::V_PK_FMA_F32_gfx1250:
3692 case AMDGPU::V_PK_FMA_F32_gfx1250_gfx12:
3693 return true;
3694 default:
3695 return false;
3696 }
3697}
3698
3699// NOTE: This function is currently only used before pseudo-expansion.
3700bool isPackedSingleSGPR64BitInst(unsigned Opc) {
3701 switch (Opc) {
3702 case AMDGPU::V_PK_ADD_F64:
3703 case AMDGPU::V_PK_MUL_F64:
3704 case AMDGPU::V_PK_FMA_F64:
3705 case AMDGPU::V_PK_MAX_NUM_F64:
3706 case AMDGPU::V_PK_MIN_NUM_F64:
3707 case AMDGPU::V_PK_ADD_NC_U64:
3708 case AMDGPU::V_PK_SUB_NC_U64:
3709 case AMDGPU::V_PK_LSHL_ADD_U64:
3710 return true;
3711 default:
3712 return false;
3713 }
3714}
3715
3716bool isSingleSGPRReadInst(unsigned Opc) {
3717 return isPackedSingleSGPRFP32Inst(Opc) || isPackedSingleSGPR64BitInst(Opc);
3718}
3719
3720const std::array<unsigned, 3> &ClusterDimsAttr::getDims() const {
3721 assert(isFixedDims() && "expect kind to be FixedDims");
3722 return Dims;
3723}
3724
3725std::string ClusterDimsAttr::to_string() const {
3726 SmallString<10> Buffer;
3727 raw_svector_ostream OS(Buffer);
3728
3729 switch (getKind()) {
3730 case Kind::Unknown:
3731 return "";
3732 case Kind::NoCluster: {
3733 OS << EncoNoCluster << ',' << EncoNoCluster << ',' << EncoNoCluster;
3734 return Buffer.c_str();
3735 }
3736 case Kind::VariableDims: {
3737 OS << EncoVariableDims << ',' << EncoVariableDims << ','
3738 << EncoVariableDims;
3739 return Buffer.c_str();
3740 }
3741 case Kind::FixedDims: {
3742 OS << Dims[0] << ',' << Dims[1] << ',' << Dims[2];
3743 return Buffer.c_str();
3744 }
3745 }
3746 llvm_unreachable("Unknown ClusterDimsAttr kind");
3747}
3748
3749ClusterDimsAttr ClusterDimsAttr::get(const Function &F) {
3750 std::optional<SmallVector<unsigned>> Attr =
3751 getIntegerVecAttribute(F, Name: "amdgpu-cluster-dims", /*Size=*/3);
3752 ClusterDimsAttr::Kind AttrKind = Kind::FixedDims;
3753
3754 if (!Attr.has_value())
3755 AttrKind = Kind::Unknown;
3756 else if (all_of(Range&: *Attr, P: equal_to(Arg: EncoNoCluster)))
3757 AttrKind = Kind::NoCluster;
3758 else if (all_of(Range&: *Attr, P: equal_to(Arg: EncoVariableDims)))
3759 AttrKind = Kind::VariableDims;
3760
3761 ClusterDimsAttr A(AttrKind);
3762 if (AttrKind == Kind::FixedDims)
3763 A.Dims = {(*Attr)[0], (*Attr)[1], (*Attr)[2]};
3764
3765 return A;
3766}
3767
3768std::optional<APFloat> evaluateRcp(const APFloat &Val) {
3769 const fltSemantics &Sem = Val.getSemantics();
3770
3771 // v_rcp_f16/bf16 are correctly rounded.
3772 if (&Sem == &APFloat::IEEEhalf() || &Sem == &APFloat::BFloat())
3773 return APFloat::getOne(Sem) / Val;
3774
3775 // v_rcp_f32/f64 always flush a denormal input to zero (preserving sign)
3776 // before reciprocating.
3777 APFloat Arg = Val;
3778 if (Arg.isDenormal())
3779 Arg = APFloat::getZero(Sem, Negative: Arg.isNegative());
3780
3781 APFloat Result = APFloat::getOne(Sem) / Arg;
3782
3783 // v_rcp_f32/f64 always flush a denormal result to zero (preserving sign).
3784 if (Result.isDenormal())
3785 Result = APFloat::getZero(Sem, Negative: Result.isNegative());
3786
3787 // v_rcp_f32/f64 only approximate the reciprocal, except for these special
3788 // cases where the result is exact.
3789 if (!Result.isZero() && !Result.isInfinity() && !Result.isNaN() &&
3790 !Result.isOne() && !Result.isMinusOne())
3791 return std::nullopt;
3792
3793 return Result;
3794}
3795
3796} // namespace AMDGPU
3797
3798raw_ostream &operator<<(raw_ostream &OS, const AMDGPU::TargetIDSetting S) {
3799 switch (S) {
3800 case (AMDGPU::TargetIDSetting::Unsupported):
3801 OS << "Unsupported";
3802 break;
3803 case (AMDGPU::TargetIDSetting::Any):
3804 OS << "Any";
3805 break;
3806 case (AMDGPU::TargetIDSetting::Off):
3807 OS << "Off";
3808 break;
3809 case (AMDGPU::TargetIDSetting::On):
3810 OS << "On";
3811 break;
3812 }
3813 return OS;
3814}
3815
3816} // namespace llvm
3817