1//===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUBaseInfo.h"
10#include "AMDGPU.h"
11#include "AMDGPUAsmUtils.h"
12#include "AMDKernelCodeT.h"
13#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14#include "Utils/AMDKernelCodeTUtils.h"
15#include "llvm/ADT/StringExtras.h"
16#include "llvm/BinaryFormat/ELF.h"
17#include "llvm/IR/Attributes.h"
18#include "llvm/IR/Constants.h"
19#include "llvm/IR/Function.h"
20#include "llvm/IR/GlobalValue.h"
21#include "llvm/IR/IntrinsicsAMDGPU.h"
22#include "llvm/IR/IntrinsicsR600.h"
23#include "llvm/IR/LLVMContext.h"
24#include "llvm/IR/Metadata.h"
25#include "llvm/MC/MCInstrInfo.h"
26#include "llvm/MC/MCRegisterInfo.h"
27#include "llvm/MC/MCSubtargetInfo.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/TargetParser/AMDGPUTargetParser.h"
30#include <optional>
31
32#define GET_INSTRINFO_NAMED_OPS
33#define GET_INSTRMAP_INFO
34#include "AMDGPUGenInstrInfo.inc"
35
36static llvm::cl::opt<unsigned> DefaultAMDHSACodeObjectVersion(
37 "amdhsa-code-object-version", llvm::cl::Hidden,
38 llvm::cl::init(Val: llvm::AMDGPU::AMDHSA_COV6),
39 llvm::cl::desc("Set default AMDHSA Code Object Version (module flag "
40 "or asm directive still take priority if present)"));
41
42namespace {
43
44/// \returns Bit mask for given bit \p Shift and bit \p Width.
45unsigned getBitMask(unsigned Shift, unsigned Width) {
46 return ((1 << Width) - 1) << Shift;
47}
48
49/// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
50///
51/// \returns Packed \p Dst.
52unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
53 unsigned Mask = getBitMask(Shift, Width);
54 return ((Src << Shift) & Mask) | (Dst & ~Mask);
55}
56
57/// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
58///
59/// \returns Unpacked bits.
60unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
61 return (Src & getBitMask(Shift, Width)) >> Shift;
62}
63
64/// \returns Vmcnt bit shift (lower bits).
65unsigned getVmcntBitShiftLo(unsigned VersionMajor) {
66 return VersionMajor >= 11 ? 10 : 0;
67}
68
69/// \returns Vmcnt bit width (lower bits).
70unsigned getVmcntBitWidthLo(unsigned VersionMajor) {
71 return VersionMajor >= 11 ? 6 : 4;
72}
73
74/// \returns Expcnt bit shift.
75unsigned getExpcntBitShift(unsigned VersionMajor) {
76 return VersionMajor >= 11 ? 0 : 4;
77}
78
79/// \returns Expcnt bit width.
80unsigned getExpcntBitWidth(unsigned VersionMajor) { return 3; }
81
82/// \returns Lgkmcnt bit shift.
83unsigned getLgkmcntBitShift(unsigned VersionMajor) {
84 return VersionMajor >= 11 ? 4 : 8;
85}
86
87/// \returns Lgkmcnt bit width.
88unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
89 return VersionMajor >= 10 ? 6 : 4;
90}
91
92/// \returns Vmcnt bit shift (higher bits).
93unsigned getVmcntBitShiftHi(unsigned VersionMajor) { return 14; }
94
95/// \returns Vmcnt bit width (higher bits).
96unsigned getVmcntBitWidthHi(unsigned VersionMajor) {
97 return (VersionMajor == 9 || VersionMajor == 10) ? 2 : 0;
98}
99
100/// \returns Loadcnt bit width
101unsigned getLoadcntBitWidth(unsigned VersionMajor) {
102 return VersionMajor >= 12 ? 6 : 0;
103}
104
105/// \returns Samplecnt bit width.
106unsigned getSamplecntBitWidth(unsigned VersionMajor) {
107 return VersionMajor >= 12 ? 6 : 0;
108}
109
110/// \returns Bvhcnt bit width.
111unsigned getBvhcntBitWidth(unsigned VersionMajor) {
112 return VersionMajor >= 12 ? 3 : 0;
113}
114
115/// \returns Dscnt bit width.
116unsigned getDscntBitWidth(unsigned VersionMajor) {
117 return VersionMajor >= 12 ? 6 : 0;
118}
119
120/// \returns Dscnt bit shift in combined S_WAIT instructions.
121unsigned getDscntBitShift(unsigned VersionMajor) { return 0; }
122
123/// \returns Storecnt or Vscnt bit width, depending on VersionMajor.
124unsigned getStorecntBitWidth(unsigned VersionMajor) {
125 return VersionMajor >= 10 ? 6 : 0;
126}
127
128/// \returns Kmcnt bit width.
129unsigned getKmcntBitWidth(unsigned VersionMajor) {
130 return VersionMajor >= 12 ? 5 : 0;
131}
132
133/// \returns Xcnt bit width.
134unsigned getXcntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
135 return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
136}
137
138/// \returns Asynccnt bit width.
139unsigned getAsynccntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
140 return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
141}
142
143/// \returns shift for Loadcnt/Storecnt in combined S_WAIT instructions.
144unsigned getLoadcntStorecntBitShift(unsigned VersionMajor) {
145 return VersionMajor >= 12 ? 8 : 0;
146}
147
148/// \returns barrier member count bit width in M0.
149unsigned getBarrierMemberCountBitWidth(unsigned VersionMajor) {
150 return VersionMajor >= 13 ? 8 : 6;
151}
152
153/// \returns VaSdst bit width
154inline unsigned getVaSdstBitWidth() { return 3; }
155
156/// \returns VaSdst bit shift
157inline unsigned getVaSdstBitShift() { return 9; }
158
159/// \returns VmVsrc bit width
160inline unsigned getVmVsrcBitWidth() { return 3; }
161
162/// \returns VmVsrc bit shift
163inline unsigned getVmVsrcBitShift() { return 2; }
164
165/// \returns VaVdst bit width
166inline unsigned getVaVdstBitWidth() { return 4; }
167
168/// \returns VaVdst bit shift
169inline unsigned getVaVdstBitShift() { return 12; }
170
171/// \returns VaVcc bit width
172inline unsigned getVaVccBitWidth() { return 1; }
173
174/// \returns VaVcc bit shift
175inline unsigned getVaVccBitShift() { return 1; }
176
177/// \returns SaSdst bit width
178inline unsigned getSaSdstBitWidth() { return 1; }
179
180/// \returns SaSdst bit shift
181inline unsigned getSaSdstBitShift() { return 0; }
182
183/// \returns VaSsrc width
184inline unsigned getVaSsrcBitWidth() { return 1; }
185
186/// \returns VaSsrc bit shift
187inline unsigned getVaSsrcBitShift() { return 8; }
188
189/// \returns HoldCnt bit shift
190inline unsigned getHoldCntWidth(unsigned VersionMajor, unsigned VersionMinor) {
191 static constexpr const unsigned MinMajor = 10;
192 static constexpr const unsigned MinMinor = 3;
193 return std::tie(args&: VersionMajor, args&: VersionMinor) >= std::tie(args: MinMajor, args: MinMinor)
194 ? 1
195 : 0;
196}
197
198/// \returns HoldCnt bit shift
199inline unsigned getHoldCntBitShift() { return 7; }
200
201} // end anonymous namespace
202
203namespace llvm {
204
205namespace AMDGPU {
206
207/// \returns true if the target supports signed immediate offset for SMRD
208/// instructions.
209bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
210 return isGFX9Plus(STI: ST);
211}
212
213/// \returns True if \p STI is AMDHSA.
214bool isHsaAbi(const MCSubtargetInfo &STI) {
215 return STI.getTargetTriple().getOS() == Triple::AMDHSA;
216}
217
218unsigned getAMDHSACodeObjectVersion(const Module &M) {
219 if (auto *Ver = mdconst::extract_or_null<ConstantInt>(
220 MD: M.getModuleFlag(Key: "amdhsa_code_object_version"))) {
221 return (unsigned)Ver->getZExtValue() / 100;
222 }
223
224 return getDefaultAMDHSACodeObjectVersion();
225}
226
227unsigned getDefaultAMDHSACodeObjectVersion() {
228 return DefaultAMDHSACodeObjectVersion;
229}
230
231unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion) {
232 switch (ABIVersion) {
233 case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
234 return 4;
235 case ELF::ELFABIVERSION_AMDGPU_HSA_V5:
236 return 5;
237 case ELF::ELFABIVERSION_AMDGPU_HSA_V6:
238 return 6;
239 default:
240 return getDefaultAMDHSACodeObjectVersion();
241 }
242}
243
244uint8_t getELFABIVersion(const Triple &T, unsigned CodeObjectVersion) {
245 if (T.getOS() != Triple::AMDHSA)
246 return 0;
247
248 switch (CodeObjectVersion) {
249 case 4:
250 return ELF::ELFABIVERSION_AMDGPU_HSA_V4;
251 case 5:
252 return ELF::ELFABIVERSION_AMDGPU_HSA_V5;
253 case 6:
254 return ELF::ELFABIVERSION_AMDGPU_HSA_V6;
255 default:
256 report_fatal_error(reason: "Unsupported AMDHSA Code Object Version " +
257 Twine(CodeObjectVersion));
258 }
259}
260
261unsigned getMultigridSyncArgImplicitArgPosition(unsigned CodeObjectVersion) {
262 switch (CodeObjectVersion) {
263 case AMDHSA_COV4:
264 return 48;
265 case AMDHSA_COV5:
266 case AMDHSA_COV6:
267 default:
268 return AMDGPU::ImplicitArg::MULTIGRID_SYNC_ARG_OFFSET;
269 }
270}
271
272// FIXME: All such magic numbers about the ABI should be in a
273// central TD file.
274unsigned getHostcallImplicitArgPosition(unsigned CodeObjectVersion) {
275 switch (CodeObjectVersion) {
276 case AMDHSA_COV4:
277 return 24;
278 case AMDHSA_COV5:
279 case AMDHSA_COV6:
280 default:
281 return AMDGPU::ImplicitArg::HOSTCALL_PTR_OFFSET;
282 }
283}
284
285unsigned getDefaultQueueImplicitArgPosition(unsigned CodeObjectVersion) {
286 switch (CodeObjectVersion) {
287 case AMDHSA_COV4:
288 return 32;
289 case AMDHSA_COV5:
290 case AMDHSA_COV6:
291 default:
292 return AMDGPU::ImplicitArg::DEFAULT_QUEUE_OFFSET;
293 }
294}
295
296unsigned getCompletionActionImplicitArgPosition(unsigned CodeObjectVersion) {
297 switch (CodeObjectVersion) {
298 case AMDHSA_COV4:
299 return 40;
300 case AMDHSA_COV5:
301 case AMDHSA_COV6:
302 default:
303 return AMDGPU::ImplicitArg::COMPLETION_ACTION_OFFSET;
304 }
305}
306
307#define GET_MIMGBaseOpcodesTable_IMPL
308#define GET_MIMGDimInfoTable_IMPL
309#define GET_MIMGInfoTable_IMPL
310#define GET_MIMGLZMappingTable_IMPL
311#define GET_MIMGMIPMappingTable_IMPL
312#define GET_MIMGBiasMappingTable_IMPL
313#define GET_MIMGOffsetMappingTable_IMPL
314#define GET_MIMGG16MappingTable_IMPL
315#define GET_MAIInstInfoTable_IMPL
316#define GET_WMMAInstInfoTable_IMPL
317#include "AMDGPUGenSearchableTables.inc"
318
319int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
320 unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc,
321 bool IndexedSamp) {
322 const MIMGInfo *Info =
323 getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, VDataDwords, VAddrDwords,
324 IndexedRsrc, IndexedSamp);
325 return Info ? Info->Opcode : -1;
326}
327
328const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
329 const MIMGInfo *Info = getMIMGInfo(Opcode: Opc);
330 return Info ? getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode) : nullptr;
331}
332
333int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
334 const MIMGInfo *OrigInfo = getMIMGInfo(Opcode: Opc);
335 const MIMGInfo *NewInfo = getMIMGOpcodeHelper(
336 BaseOpcode: OrigInfo->BaseOpcode, MIMGEncoding: OrigInfo->MIMGEncoding, VDataDwords: NewChannels,
337 VAddrDwords: OrigInfo->VAddrDwords, IndexedRsrc: OrigInfo->IndexedRsrc, IndexedSamp: OrigInfo->IndexedSamp);
338 return NewInfo ? NewInfo->Opcode : -1;
339}
340
341unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
342 const MIMGDimInfo *Dim, bool IsA16,
343 bool IsG16Supported) {
344 unsigned AddrWords = BaseOpcode->NumExtraArgs;
345 unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
346 (BaseOpcode->LodOrClampOrMip ? 1 : 0);
347 if (IsA16)
348 AddrWords += divideCeil(Numerator: AddrComponents, Denominator: 2);
349 else
350 AddrWords += AddrComponents;
351
352 // Note: For subtargets that support A16 but not G16, enabling A16 also
353 // enables 16 bit gradients.
354 // For subtargets that support A16 (operand) and G16 (done with a different
355 // instruction encoding), they are independent.
356
357 if (BaseOpcode->Gradients) {
358 if ((IsA16 && !IsG16Supported) || BaseOpcode->G16)
359 // There are two gradients per coordinate, we pack them separately.
360 // For the 3d case,
361 // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv)
362 AddrWords += alignTo<2>(Value: Dim->NumGradients / 2);
363 else
364 AddrWords += Dim->NumGradients;
365 }
366 return AddrWords;
367}
368
369struct MUBUFInfo {
370 uint32_t Opcode;
371 uint32_t BaseOpcode;
372 uint8_t elements;
373 bool has_vaddr;
374 bool has_srsrc;
375 bool has_soffset;
376 bool IsBufferInv;
377 bool tfe;
378};
379
380struct MTBUFInfo {
381 uint32_t Opcode;
382 uint32_t BaseOpcode;
383 uint8_t elements;
384 bool has_vaddr;
385 bool has_srsrc;
386 bool has_soffset;
387};
388
389struct SMInfo {
390 uint32_t Opcode;
391};
392
393struct VOPInfo {
394 uint32_t Opcode;
395};
396
397struct VOPC64DPPInfo {
398 uint32_t Opcode;
399};
400
401struct VOPCDPPAsmOnlyInfo {
402 uint32_t Opcode;
403};
404
405struct VOPDComponentInfo {
406 uint16_t BaseVOP;
407 uint16_t VOPDOp;
408};
409
410struct VOPDInfo {
411 uint32_t Opcode;
412 uint16_t OpX;
413 uint16_t OpY;
414 uint16_t Subtarget;
415 bool VOPD3;
416};
417
418struct VOPTrue16Info {
419 uint32_t Opcode;
420};
421
422struct VOPDXYInfo {
423 uint16_t VOPDXYKey;
424 bool IsX;
425 bool IsY;
426};
427
428#define GET_FP4FP8DstByteSelTable_DECL
429#define GET_FP4FP8DstByteSelTable_IMPL
430
431struct DPMACCInstructionInfo {
432 uint32_t Opcode;
433};
434
435struct FP4FP8DstByteSelInfo {
436 uint32_t Opcode;
437 bool HasFP8DstByteSel;
438 bool HasFP4DstByteSel;
439};
440
441struct VOP3PHasNoOpSelHiInfo {
442 uint32_t Opcode;
443};
444
445#define GET_DPMACCInstructionTable_DECL
446#define GET_DPMACCInstructionTable_IMPL
447#define GET_MTBUFInfoTable_DECL
448#define GET_MTBUFInfoTable_IMPL
449#define GET_MUBUFInfoTable_DECL
450#define GET_MUBUFInfoTable_IMPL
451#define GET_SMInfoTable_DECL
452#define GET_SMInfoTable_IMPL
453#define GET_VOP1InfoTable_DECL
454#define GET_VOP1InfoTable_IMPL
455#define GET_VOP2InfoTable_DECL
456#define GET_VOP2InfoTable_IMPL
457#define GET_VOP3InfoTable_DECL
458#define GET_VOP3InfoTable_IMPL
459#define GET_VOPC64DPPTable_DECL
460#define GET_VOPC64DPPTable_IMPL
461#define GET_VOPC64DPP8Table_DECL
462#define GET_VOPC64DPP8Table_IMPL
463#define GET_VOPCAsmOnlyInfoTable_DECL
464#define GET_VOPCAsmOnlyInfoTable_IMPL
465#define GET_VOP3CAsmOnlyInfoTable_DECL
466#define GET_VOP3CAsmOnlyInfoTable_IMPL
467#define GET_VOPDComponentTable_DECL
468#define GET_VOPDComponentTable_IMPL
469#define GET_VOPDPairs_DECL
470#define GET_VOPDPairs_IMPL
471#define GET_VOPDXYTable_DECL
472#define GET_VOPDXYTable_IMPL
473#define GET_VOPTrue16Table_DECL
474#define GET_VOPTrue16Table_IMPL
475#define GET_VOP3PHasNoOpSelHiTable_DECL
476#define GET_VOP3PHasNoOpSelHiTable_IMPL
477#define GET_True16D16Table_IMPL
478#define GET_WMMAOpcode2AddrMappingTable_DECL
479#define GET_WMMAOpcode2AddrMappingTable_IMPL
480#define GET_WMMAOpcode3AddrMappingTable_DECL
481#define GET_WMMAOpcode3AddrMappingTable_IMPL
482#define GET_getMFMA_F8F6F4_WithSize_DECL
483#define GET_getMFMA_F8F6F4_WithSize_IMPL
484#define GET_isMFMA_F8F6F4Table_IMPL
485#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL
486
487#include "AMDGPUGenSearchableTables.inc"
488
489int getMTBUFBaseOpcode(unsigned Opc) {
490 const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opcode: Opc);
491 return Info ? Info->BaseOpcode : -1;
492}
493
494int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
495 const MTBUFInfo *Info =
496 getMTBUFInfoFromBaseOpcodeAndElements(BaseOpcode: BaseOpc, elements: Elements);
497 return Info ? Info->Opcode : -1;
498}
499
500int getMTBUFElements(unsigned Opc) {
501 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
502 return Info ? Info->elements : 0;
503}
504
505bool getMTBUFHasVAddr(unsigned Opc) {
506 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
507 return Info && Info->has_vaddr;
508}
509
510bool getMTBUFHasSrsrc(unsigned Opc) {
511 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
512 return Info && Info->has_srsrc;
513}
514
515bool getMTBUFHasSoffset(unsigned Opc) {
516 const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opcode: Opc);
517 return Info && Info->has_soffset;
518}
519
520int getMUBUFBaseOpcode(unsigned Opc) {
521 const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opcode: Opc);
522 return Info ? Info->BaseOpcode : -1;
523}
524
525int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
526 const MUBUFInfo *Info =
527 getMUBUFInfoFromBaseOpcodeAndElements(BaseOpcode: BaseOpc, elements: Elements);
528 return Info ? Info->Opcode : -1;
529}
530
531int getMUBUFElements(unsigned Opc) {
532 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
533 return Info ? Info->elements : 0;
534}
535
536bool getMUBUFHasVAddr(unsigned Opc) {
537 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
538 return Info && Info->has_vaddr;
539}
540
541bool getMUBUFHasSrsrc(unsigned Opc) {
542 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
543 return Info && Info->has_srsrc;
544}
545
546bool getMUBUFHasSoffset(unsigned Opc) {
547 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
548 return Info && Info->has_soffset;
549}
550
551bool getMUBUFIsBufferInv(unsigned Opc) {
552 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
553 return Info && Info->IsBufferInv;
554}
555
556bool getMUBUFTfe(unsigned Opc) {
557 const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opcode: Opc);
558 return Info && Info->tfe;
559}
560
561bool getSMEMIsBuffer(unsigned Opc) {
562 return isSMEMOpcodeHelper(Opcode: Opc) != nullptr;
563}
564
565bool getVOP1IsSingle(unsigned Opc) {
566 return isVOP1SingleOpcodeHelper(Opcode: Opc) != nullptr;
567}
568
569bool getVOP2IsSingle(unsigned Opc) {
570 return isVOP2SingleOpcodeHelper(Opcode: Opc) != nullptr;
571}
572
573bool getVOP3IsSingle(unsigned Opc) {
574 return isVOP3SingleOpcodeHelper(Opcode: Opc) != nullptr;
575}
576
577bool isVOPC64DPP(unsigned Opc) {
578 return isVOPC64DPPOpcodeHelper(Opcode: Opc) || isVOPC64DPP8OpcodeHelper(Opcode: Opc);
579}
580
581bool isVOPCAsmOnly(unsigned Opc) { return isVOPCAsmOnlyOpcodeHelper(Opcode: Opc); }
582
583bool getMAIIsDGEMM(unsigned Opc) {
584 const MAIInstInfo *Info = getMAIInstInfoHelper(Opcode: Opc);
585 return Info && Info->is_dgemm;
586}
587
588bool getMAIIsGFX940XDL(unsigned Opc) {
589 const MAIInstInfo *Info = getMAIInstInfoHelper(Opcode: Opc);
590 return Info && Info->is_gfx940_xdl;
591}
592
593bool getWMMAIsXDL(unsigned Opc) {
594 const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opcode: Opc);
595 return Info ? Info->is_wmma_xdl : false;
596}
597
598bool getHasMatrixScale(unsigned Opc) {
599 const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opcode: Opc);
600 return Info && Info->HasMatrixScale;
601}
602
603uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal) {
604 switch (EncodingVal) {
605 case MFMAScaleFormats::FP6_E2M3:
606 case MFMAScaleFormats::FP6_E3M2:
607 return 6;
608 case MFMAScaleFormats::FP4_E2M1:
609 return 4;
610 case MFMAScaleFormats::FP8_E4M3:
611 case MFMAScaleFormats::FP8_E5M2:
612 default:
613 return 8;
614 }
615
616 llvm_unreachable("covered switch over mfma scale formats");
617}
618
619const MFMA_F8F6F4_Info *getMFMA_F8F6F4_WithFormatArgs(unsigned CBSZ,
620 unsigned BLGP,
621 unsigned F8F8Opcode) {
622 uint8_t SrcANumRegs = mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: CBSZ);
623 uint8_t SrcBNumRegs = mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: BLGP);
624 return getMFMA_F8F6F4_InstWithNumRegs(NumRegsSrcA: SrcANumRegs, NumRegsSrcB: SrcBNumRegs, F8F8Opcode);
625}
626
627uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt) {
628 switch (Fmt) {
629 case WMMA::MATRIX_FMT_FP8:
630 case WMMA::MATRIX_FMT_BF8:
631 return 16;
632 case WMMA::MATRIX_FMT_FP6:
633 case WMMA::MATRIX_FMT_BF6:
634 return 12;
635 case WMMA::MATRIX_FMT_FP4:
636 return 8;
637 }
638
639 llvm_unreachable("covered switch over wmma scale formats");
640}
641
642const MFMA_F8F6F4_Info *getWMMA_F8F6F4_WithFormatArgs(unsigned FmtA,
643 unsigned FmtB,
644 unsigned F8F8Opcode) {
645 uint8_t SrcANumRegs = wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtA);
646 uint8_t SrcBNumRegs = wmmaScaleF8F6F4FormatToNumRegs(Fmt: FmtB);
647 return getMFMA_F8F6F4_InstWithNumRegs(NumRegsSrcA: SrcANumRegs, NumRegsSrcB: SrcBNumRegs, F8F8Opcode);
648}
649
650bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale,
651 unsigned BFmt, unsigned BScale) {
652 auto isValid = [](unsigned Fmt, unsigned Scale) -> bool {
653 switch (Fmt) {
654 case WMMA::MATRIX_FMT_FP8:
655 case WMMA::MATRIX_FMT_BF8:
656 case WMMA::MATRIX_FMT_FP6:
657 case WMMA::MATRIX_FMT_BF6:
658 if (Scale != WMMA::MATRIX_SCALE_FMT_E8)
659 return false;
660 break;
661 case WMMA::MATRIX_FMT_FP4:
662 if (Scale != WMMA::MATRIX_SCALE_FMT_E8 &&
663 Scale != WMMA::MATRIX_SCALE_FMT_E5M3 &&
664 Scale != WMMA::MATRIX_SCALE_FMT_E4M3)
665 return false;
666 break;
667 }
668 return true;
669 };
670
671 if (!isValid(AFmt, AScale) || !isValid(BFmt, BScale))
672 return false;
673
674 if (AFmt == WMMA::MATRIX_FMT_FP4 && BFmt == WMMA::MATRIX_FMT_FP4 &&
675 AScale != BScale)
676 return false;
677
678 return true;
679}
680
681unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST) {
682 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX13Insts))
683 return SIEncodingFamily::GFX13;
684 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX1250Insts))
685 return SIEncodingFamily::GFX1250;
686 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX12Insts))
687 return SIEncodingFamily::GFX12;
688 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX11_7Insts))
689 return SIEncodingFamily::GFX1170;
690 if (ST.hasFeature(Feature: AMDGPU::FeatureGFX11Insts))
691 return SIEncodingFamily::GFX11;
692 llvm_unreachable("Subtarget generation does not support VOPD!");
693}
694
695CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3) {
696 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
697 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
698 // Normalize through VOPDComponentTable so that e32 and e64 variants
699 // of the same logical opcode all share a single entry.
700 const VOPDComponentInfo *Info = getVOPDComponentHelper(BaseVOP: Opc);
701 if (!Info)
702 return {.X: false, .Y: false};
703 unsigned Key =
704 (Info->VOPDOp << 5) | (EncodingFamily << 1) | (VOPD3 ? 1u : 0u);
705 const VOPDXYInfo *XYInfo = getVOPDXYInfo(VOPDXYKey: static_cast<uint16_t>(Key));
706 if (!XYInfo)
707 return {.X: false, .Y: false};
708 return {.X: XYInfo->IsX, .Y: XYInfo->IsY};
709}
710
711unsigned getVOPDOpcode(unsigned Opc, bool VOPD3) {
712 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
713 Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
714 const VOPDComponentInfo *Info = getVOPDComponentHelper(BaseVOP: Opc);
715 return Info ? Info->VOPDOp : ~0u;
716}
717
718bool isVOPD(unsigned Opc) {
719 return AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::src0X);
720}
721
722bool isMAC(unsigned Opc) {
723 return Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 ||
724 Opc == AMDGPU::V_MAC_F32_e64_gfx10 ||
725 Opc == AMDGPU::V_MAC_F32_e64_vi ||
726 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx6_gfx7 ||
727 Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx10 ||
728 Opc == AMDGPU::V_MAC_F16_e64_vi ||
729 Opc == AMDGPU::V_FMAC_F64_e64_gfx90a ||
730 Opc == AMDGPU::V_FMAC_F64_e64_gfx12 ||
731 Opc == AMDGPU::V_FMAC_F64_e64_gfx13 ||
732 Opc == AMDGPU::V_FMAC_F32_e64_gfx10 ||
733 Opc == AMDGPU::V_FMAC_F32_e64_gfx11 ||
734 Opc == AMDGPU::V_FMAC_F32_e64_gfx12 ||
735 Opc == AMDGPU::V_FMAC_F32_e64_gfx13 ||
736 Opc == AMDGPU::V_FMAC_F32_e64_vi ||
737 Opc == AMDGPU::V_FMAC_LEGACY_F32_e64_gfx10 ||
738 Opc == AMDGPU::V_FMAC_DX9_ZERO_F32_e64_gfx11 ||
739 Opc == AMDGPU::V_FMAC_F16_e64_gfx10 ||
740 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx11 ||
741 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx11 ||
742 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx12 ||
743 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx12 ||
744 Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx13 ||
745 Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx13 ||
746 Opc == AMDGPU::V_DOT2C_F32_F16_e64_vi ||
747 Opc == AMDGPU::V_DOT2C_F32_BF16_e64_vi ||
748 Opc == AMDGPU::V_DOT2C_I32_I16_e64_vi ||
749 Opc == AMDGPU::V_DOT4C_I32_I8_e64_vi ||
750 Opc == AMDGPU::V_DOT8C_I32_I4_e64_vi;
751}
752
753bool isPermlane16(unsigned Opc) {
754 return Opc == AMDGPU::V_PERMLANE16_B32_gfx10 ||
755 Opc == AMDGPU::V_PERMLANEX16_B32_gfx10 ||
756 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx11 ||
757 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx11 ||
758 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx12 ||
759 Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx13 ||
760 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx12 ||
761 Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx13 ||
762 Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx12 ||
763 Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx13 ||
764 Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx12 ||
765 Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx13;
766}
767
768bool isCvt_F32_Fp8_Bf8_e64(unsigned Opc) {
769 return Opc == AMDGPU::V_CVT_F32_BF8_e64_gfx12 ||
770 Opc == AMDGPU::V_CVT_F32_FP8_e64_gfx12 ||
771 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp_gfx12 ||
772 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp_gfx12 ||
773 Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp8_gfx12 ||
774 Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp8_gfx12 ||
775 Opc == AMDGPU::V_CVT_PK_F32_BF8_fake16_e64_gfx12 ||
776 Opc == AMDGPU::V_CVT_PK_F32_FP8_fake16_e64_gfx12 ||
777 Opc == AMDGPU::V_CVT_PK_F32_BF8_t16_e64_gfx12 ||
778 Opc == AMDGPU::V_CVT_PK_F32_FP8_t16_e64_gfx12;
779}
780
781bool isGenericAtomic(unsigned Opc) {
782 return Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP ||
783 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD ||
784 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB ||
785 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN ||
786 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN ||
787 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX ||
788 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX ||
789 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND ||
790 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR ||
791 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR ||
792 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC ||
793 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC ||
794 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD ||
795 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN ||
796 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX ||
797 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP ||
798 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32 ||
799 Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32 ||
800 Opc == AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG;
801}
802
803bool isAsyncStore(unsigned Opc) {
804 return Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_gfx1250 ||
805 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_gfx1250 ||
806 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_gfx1250 ||
807 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_gfx1250 ||
808 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR_gfx1250 ||
809 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR_gfx1250 ||
810 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR_gfx1250 ||
811 Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR_gfx1250;
812}
813
814bool isTensorStore(unsigned Opc) {
815 return Opc == TENSOR_STORE_FROM_LDS_d2_gfx1250 ||
816 Opc == TENSOR_STORE_FROM_LDS_d4_gfx1250;
817}
818
819unsigned getTemporalHintType(const MCInstrDesc TID) {
820 if (SIInstrFlags::isAtomic(O: TID))
821 return CPol::TH_TYPE_ATOMIC;
822 unsigned Opc = TID.getOpcode();
823 // Async and Tensor store should have the temporal hint type of TH_TYPE_STORE
824 if (TID.mayStore() &&
825 (isAsyncStore(Opc) || isTensorStore(Opc) || !TID.mayLoad()))
826 return CPol::TH_TYPE_STORE;
827
828 // This will default to returning TH_TYPE_LOAD when neither MayStore nor
829 // MayLoad flag is present which is the case with instructions like
830 // image_get_resinfo.
831 return CPol::TH_TYPE_LOAD;
832}
833
834bool isTrue16Inst(unsigned Opc) { return isTrue16Opcode(Opcode: Opc) != nullptr; }
835
836bool hasNoOpSelHi(unsigned Opc) {
837 return getVOP3PHasNoOpSelHiHelper(Opcode: Opc) != nullptr;
838}
839
840FPType getFPDstSelType(unsigned Opc) {
841 const FP4FP8DstByteSelInfo *Info = getFP4FP8DstByteSelHelper(Opcode: Opc);
842 if (!Info)
843 return FPType::None;
844 if (Info->HasFP8DstByteSel)
845 return FPType::FP8;
846 if (Info->HasFP4DstByteSel)
847 return FPType::FP4;
848
849 return FPType::None;
850}
851
852bool isDPMACCInstruction(unsigned Opc) {
853 return isDPMACCInstructionHelper(Opcode: Opc) != nullptr;
854}
855
856unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc) {
857 const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom2AddrOpcode(Opcode2Addr: Opc);
858 return Info ? Info->Opcode3Addr : ~0u;
859}
860
861// Wrapper for Tablegen'd function. enum Subtarget is not defined in any
862// header files, so we need to wrap it in a function that takes unsigned
863// instead.
864int32_t getMCOpcode(uint32_t Opcode, unsigned Gen) {
865 return getMCOpcodeGen(Opcode, inSubtarget: static_cast<Subtarget>(Gen));
866}
867
868unsigned getBitOp2(unsigned Opc) {
869 switch (Opc) {
870 default:
871 return 0;
872 case AMDGPU::V_AND_B32_e32:
873 return 0x40;
874 case AMDGPU::V_OR_B32_e32:
875 return 0x54;
876 case AMDGPU::V_XOR_B32_e32:
877 return 0x14;
878 case AMDGPU::V_XNOR_B32_e32:
879 return 0x41;
880 }
881}
882
883int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily,
884 bool VOPD3) {
885 bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc: OpY) : 0;
886 OpY = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : OpY;
887 const VOPDInfo *Info = getVOPDInfoFromComponentOpcodes(
888 OpX: static_cast<uint8_t>(OpX), OpY: static_cast<uint8_t>(OpY),
889 SubTgt: static_cast<uint8_t>(EncodingFamily), VOPD3);
890 return Info ? Info->Opcode : -1;
891}
892
893std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode) {
894 const VOPDInfo *Info = getVOPDOpcodeHelper(Opcode: VOPDOpcode);
895 assert(Info);
896 const auto *OpX = getVOPDBaseFromComponent(VOPDOp: Info->OpX);
897 const auto *OpY = getVOPDBaseFromComponent(VOPDOp: Info->OpY);
898 assert(OpX && OpY);
899 return {OpX->BaseVOP, OpY->BaseVOP};
900}
901
902namespace VOPD {
903
904ComponentProps::ComponentProps(const MCInstrDesc &OpDesc, bool VOP3Layout) {
905 assert(OpDesc.getNumDefs() == Component::DST_NUM);
906
907 assert(OpDesc.getOperandConstraint(Component::SRC0, MCOI::TIED_TO) == -1);
908 assert(OpDesc.getOperandConstraint(Component::SRC1, MCOI::TIED_TO) == -1);
909 auto TiedIdx = OpDesc.getOperandConstraint(OpNum: Component::SRC2, Constraint: MCOI::TIED_TO);
910 assert(TiedIdx == -1 || TiedIdx == Component::DST);
911 HasSrc2Acc = TiedIdx != -1;
912 Opcode = OpDesc.getOpcode();
913
914 IsVOP3 = VOP3Layout || SIInstrFlags::isVOP3(O: OpDesc);
915 SrcOperandsNum = AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src2) ? 3
916 : AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::imm) ? 3
917 : AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src1) ? 2
918 : 1;
919 assert(SrcOperandsNum <= Component::MAX_SRC_NUM);
920
921 if (Opcode == AMDGPU::V_CNDMASK_B32_e32 ||
922 Opcode == AMDGPU::V_CNDMASK_B32_e64) {
923 // CNDMASK is an awkward exception, it has FP modifiers, but not FP
924 // operands.
925 NumVOPD3Mods = 2;
926 if (IsVOP3)
927 SrcOperandsNum = 3;
928 } else if (Opcode == AMDGPU::V_DOT2_F32_F16 ||
929 Opcode == AMDGPU::V_DOT2_F32_BF16) {
930 // VOP3P opcodes that have VOPD but don't have VOP2 version. Using VOPD3
931 // path in getIndexOfSrcInMCOperands to get correct src operand indexes,
932 // but generating VOPD, not VOPD3.
933 NumVOPD3Mods = SrcOperandsNum;
934 } else if (isSISrcFPOperand(Desc: OpDesc,
935 OpNo: getNamedOperandIdx(Opcode, Name: OpName::src0))) {
936 // All FP VOPD instructions have Neg modifiers for all operands except
937 // for tied src2.
938 NumVOPD3Mods = SrcOperandsNum;
939 if (HasSrc2Acc)
940 --NumVOPD3Mods;
941 }
942
943 if (SIInstrFlags::isVOP3(O: OpDesc))
944 return;
945
946 auto OperandsNum = OpDesc.getNumOperands();
947 unsigned CompOprIdx;
948 for (CompOprIdx = Component::SRC1; CompOprIdx < OperandsNum; ++CompOprIdx) {
949 if (OpDesc.operands()[CompOprIdx].OperandType == AMDGPU::OPERAND_KIMM32) {
950 MandatoryLiteralIdx = CompOprIdx;
951 break;
952 }
953 }
954}
955
956int ComponentProps::getBitOp3OperandIdx() const {
957 return getNamedOperandIdx(Opcode, Name: OpName::bitop3);
958}
959
960unsigned ComponentInfo::getIndexInParsedOperands(unsigned CompOprIdx) const {
961 assert(CompOprIdx < Component::MAX_OPR_NUM);
962
963 if (CompOprIdx == Component::DST)
964 return getIndexOfDstInParsedOperands();
965
966 auto CompSrcIdx = CompOprIdx - Component::DST_NUM;
967 if (CompSrcIdx < getCompParsedSrcOperandsNum())
968 return getIndexOfSrcInParsedOperands(CompSrcIdx);
969
970 // The specified operand does not exist.
971 return 0;
972}
973
974std::optional<unsigned> InstInfo::getInvalidCompOperandIndex(
975 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
976 const MCRegisterInfo &MRI, bool SkipSrc, bool AllowSameVGPR, bool VOPD3,
977 bool HasGFX11InterlockHazard) const {
978
979 auto OpXRegs = getRegIndices(ComponentIdx: ComponentIndex::X, GetRegIdx,
980 VOPD3: CompInfo[ComponentIndex::X].isVOP3());
981 auto OpYRegs = getRegIndices(ComponentIdx: ComponentIndex::Y, GetRegIdx,
982 VOPD3: CompInfo[ComponentIndex::Y].isVOP3());
983
984 const auto banksOverlap = [&MRI](MCRegister X, MCRegister Y,
985 unsigned BanksMask) -> bool {
986 MCRegister BaseX = MRI.getSubReg(Reg: X, Idx: AMDGPU::sub0);
987 MCRegister BaseY = MRI.getSubReg(Reg: Y, Idx: AMDGPU::sub0);
988 if (!BaseX)
989 BaseX = X;
990 if (!BaseY)
991 BaseY = Y;
992 if ((BaseX.id() & BanksMask) == (BaseY.id() & BanksMask))
993 return true;
994 if (BaseX != X /* This is 64-bit register */ &&
995 ((BaseX.id() + 1) & BanksMask) == (BaseY.id() & BanksMask))
996 return true;
997 if (BaseY != Y &&
998 (BaseX.id() & BanksMask) == ((BaseY.id() + 1) & BanksMask))
999 return true;
1000
1001 // If both are 64-bit bank conflict will be detected yet while checking
1002 // the first subreg.
1003 return false;
1004 };
1005
1006 unsigned CompOprIdx;
1007 for (CompOprIdx = 0; CompOprIdx < Component::MAX_OPR_NUM; ++CompOprIdx) {
1008 unsigned BanksMasks = VOPD3 ? VOPD3_VGPR_BANK_MASKS[CompOprIdx]
1009 : HasGFX11InterlockHazard
1010 ? VOPD_GFX11_VGPR_BANK_MASKS[CompOprIdx]
1011 : VOPD_VGPR_BANK_MASKS[CompOprIdx];
1012 if (!OpXRegs[CompOprIdx] || !OpYRegs[CompOprIdx])
1013 continue;
1014
1015 if (getVGPREncodingMSBs(Reg: OpXRegs[CompOprIdx], MRI) !=
1016 getVGPREncodingMSBs(Reg: OpYRegs[CompOprIdx], MRI))
1017 return CompOprIdx;
1018
1019 if (SkipSrc && CompOprIdx >= Component::DST_NUM)
1020 continue;
1021
1022 if (CompOprIdx < Component::DST_NUM) {
1023 // Even if we do not check vdst parity, vdst operands still shall not
1024 // overlap.
1025 if (MRI.regsOverlap(RegA: OpXRegs[CompOprIdx], RegB: OpYRegs[CompOprIdx]))
1026 return CompOprIdx;
1027 if (VOPD3) // No need to check dst parity.
1028 continue;
1029 }
1030
1031 if (banksOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx], BanksMasks) &&
1032 (!AllowSameVGPR || CompOprIdx < Component::DST_NUM ||
1033 OpXRegs[CompOprIdx] != OpYRegs[CompOprIdx]))
1034 return CompOprIdx;
1035 }
1036
1037 return {};
1038}
1039
1040// Return an array of VGPR registers [DST,SRC0,SRC1,SRC2] used
1041// by the specified component. If an operand is unused
1042// or is not a VGPR, the corresponding value is 0.
1043//
1044// GetRegIdx(Component, MCOperandIdx) must return a VGPR register index
1045// for the specified component and MC operand. The callback must return 0
1046// if the operand is not a register or not a VGPR.
1047InstInfo::RegIndices
1048InstInfo::getRegIndices(unsigned CompIdx,
1049 std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
1050 bool VOPD3) const {
1051 assert(CompIdx < COMPONENTS_NUM);
1052
1053 const auto &Comp = CompInfo[CompIdx];
1054 InstInfo::RegIndices RegIndices;
1055
1056 RegIndices[DST] = GetRegIdx(CompIdx, Comp.getIndexOfDstInMCOperands());
1057
1058 for (unsigned CompOprIdx : {SRC0, SRC1, SRC2}) {
1059 unsigned CompSrcIdx = CompOprIdx - DST_NUM;
1060 RegIndices[CompOprIdx] =
1061 Comp.hasRegSrcOperand(CompSrcIdx)
1062 ? GetRegIdx(CompIdx,
1063 Comp.getIndexOfSrcInMCOperands(CompSrcIdx, VOPD3))
1064 : MCRegister();
1065 }
1066 return RegIndices;
1067}
1068
1069} // namespace VOPD
1070
1071VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY) {
1072 return VOPD::InstInfo(OpX, OpY);
1073}
1074
1075VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,
1076 const MCInstrInfo *InstrInfo) {
1077 auto [OpX, OpY] = getVOPDComponents(VOPDOpcode);
1078 const auto &OpXDesc = InstrInfo->get(Opcode: OpX);
1079 const auto &OpYDesc = InstrInfo->get(Opcode: OpY);
1080 bool VOPD3 = SIInstrFlags::isVOPD3(O: *InstrInfo, O: VOPDOpcode);
1081 VOPD::ComponentInfo OpXInfo(OpXDesc, VOPD::ComponentKind::COMPONENT_X, VOPD3);
1082 VOPD::ComponentInfo OpYInfo(OpYDesc, OpXInfo, VOPD3);
1083 return VOPD::InstInfo(OpXInfo, OpYInfo);
1084}
1085
1086TargetID createAMDGPUTargetID(const MCSubtargetInfo &STI,
1087 StringRef FeatureString) {
1088 // In codegen the mode comes from module flags and FeatureString is empty, so
1089 // the processor defaults apply. The assembler has no target directive, so it
1090 // pins the mode via the +xnack/-xnack/+sramecc/-sramecc feature string.
1091 return TargetID::createFromSubtargetFeatures(TT: STI.getTargetTriple(),
1092 CPU: STI.getCPU(), FeatureString);
1093}
1094
1095namespace IsaInfo {
1096
1097unsigned getInstCacheLineSize(const MCSubtargetInfo &STI) {
1098 if (STI.getFeatureBits().test(I: FeatureInstCacheLineSize128))
1099 return 128;
1100 if (STI.getFeatureBits().test(I: FeatureInstCacheLineSize64))
1101 return 64;
1102 return 64;
1103}
1104
1105unsigned getWavefrontSize(const MCSubtargetInfo &STI) {
1106 if (STI.getFeatureBits().test(I: FeatureWavefrontSize16))
1107 return 16;
1108 if (STI.getFeatureBits().test(I: FeatureWavefrontSize32))
1109 return 32;
1110
1111 return 64;
1112}
1113
1114// Maximum LDS a single work-group can address. This is a fixed HW cap. It does
1115// not depend on how many SIMDs a work-group runs on.
1116static unsigned getMaxHWAddressableLocalMemorySize(const MCSubtargetInfo &STI) {
1117 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize32768))
1118 return 32768;
1119 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize65536))
1120 return 65536;
1121 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize163840))
1122 return 163840;
1123 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize196608))
1124 return 196608;
1125 if (STI.getFeatureBits().test(I: FeatureAddressableLocalMemorySize327680))
1126 return 327680;
1127 return 32768;
1128}
1129
1130// Total physical size of LDS on the block, in bytes. On targets with
1131// FeatureHalfAddressablePhysicalLocalMemory the physical block is twice the
1132// addressable size (gfx6: 64 KiB physical and 32 KiB addressable;
1133// gfx10/11/12: 128 KiB physical and 64 KiB addressable). On other targets it is
1134// equal to the addressable size.
1135static unsigned getPhysicalLocalMemorySize(const MCSubtargetInfo &STI) {
1136 unsigned Addressable = getMaxHWAddressableLocalMemorySize(STI);
1137 if (STI.getFeatureBits().test(I: FeatureHalfAddressablePhysicalLocalMemory))
1138 return 2 * Addressable;
1139 return Addressable;
1140}
1141
1142// Sizes in use, by generation (addressable / physical block):
1143// gfx6 : 32 KiB addressable, 64 KiB physical block
1144// gfx7 / gfx8 / gfx9: 64 KiB
1145// gfx9.5 (gfx950) : 160 KiB
1146// gfx10 / 11 / 12 : 64 KiB addressable, 128 KiB physical block
1147// gfx12.5 (gfx1250) : 320 KiB (always runs on four SIMDs)
1148// gfx13 : 192 KiB on four SIMDs, 96 KiB on two
1149// Total available in the current mode. The physical size is halved when a
1150// work-group runs on two SIMDs.
1151unsigned getLocalMemorySize(const MCSubtargetInfo &STI) {
1152 unsigned Size = getPhysicalLocalMemorySize(STI);
1153 if (!isFullSIMDMode(STI))
1154 Size /= 2;
1155 return Size;
1156}
1157
1158// What one work-group can allocate in the current mode. This is the HW
1159// addressable cap, but never more than the total available in the current mode.
1160unsigned getAddressableLocalMemorySize(const MCSubtargetInfo &STI) {
1161 return std::min(a: getMaxHWAddressableLocalMemorySize(STI),
1162 b: getLocalMemorySize(STI));
1163}
1164
1165unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo &STI,
1166 unsigned FlatWorkGroupSize) {
1167 assert(FlatWorkGroupSize != 0);
1168 if (!STI.getTargetTriple().isAMDGCN())
1169 return 8;
1170 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1171 unsigned MaxWaves =
1172 getMaxWavesPerEU(AK: Kind) * getNumWorkGroupSIMDs(FullSIMDMode: isFullSIMDMode(STI));
1173 unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
1174 if (N == 1) {
1175 // Single-wave workgroups don't consume barrier resources.
1176 return MaxWaves;
1177 }
1178
1179 unsigned MaxBarriers = 16;
1180 if (isGFX10Plus(STI) && !STI.getFeatureBits().test(I: FeatureCuMode))
1181 MaxBarriers = 32;
1182
1183 return std::min(a: MaxWaves / N, b: MaxBarriers);
1184}
1185
1186unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo &STI,
1187 unsigned FlatWorkGroupSize) {
1188 return divideCeil(Numerator: getWavesPerWorkGroup(STI, FlatWorkGroupSize),
1189 Denominator: getNumWorkGroupSIMDs(FullSIMDMode: isFullSIMDMode(STI)));
1190}
1191
1192unsigned getWavesPerWorkGroup(const MCSubtargetInfo &STI,
1193 unsigned FlatWorkGroupSize) {
1194 return divideCeil(Numerator: FlatWorkGroupSize, Denominator: getWavefrontSize(STI));
1195}
1196
1197unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI) { return 8; }
1198
1199// Per-wave SGPRs reserved for the trap handler when enabled.
1200static unsigned getSGPRTrapHandlerReserve(const MCSubtargetInfo &STI) {
1201 return STI.getFeatureBits().test(I: FeatureTrapHandler) ? TRAP_NUM_SGPRS : 0;
1202}
1203
1204// Per-wave SGPR budget (before the addressable clamp): take off the trap
1205// reserve, round down to \p Granule. Shared by getMinNumSGPRs() and
1206// getMaxNumSGPRs(); getOccupancyWithNumSGPRs() is the closed-form algebraic
1207// inverse of this same budget (it does not call this helper), so the two encode
1208// one model.
1209static unsigned getSGPRBudgetPerWave(unsigned TotalNumSGPRs,
1210 unsigned WavesPerEU, unsigned TrapReserve,
1211 unsigned Granule) {
1212 assert(WavesPerEU != 0 && Granule != 0);
1213 unsigned Budget = TotalNumSGPRs / WavesPerEU;
1214 Budget -= std::min(a: Budget, b: TrapReserve);
1215 return alignDown(Value: Budget, Align: Granule);
1216}
1217
1218unsigned getMinNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU) {
1219 assert(WavesPerEU != 0);
1220
1221 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1222 if (Version.Major >= 10)
1223 return 0;
1224
1225 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1226 if (WavesPerEU >= getMaxWavesPerEU(AK: Kind))
1227 return 0;
1228
1229 unsigned MinNumSGPRs =
1230 getSGPRBudgetPerWave(TotalNumSGPRs: getTotalNumSGPRs(AK: Kind), WavesPerEU: WavesPerEU + 1,
1231 TrapReserve: getSGPRTrapHandlerReserve(STI),
1232 Granule: getSGPRAllocGranule(AK: Kind)) +
1233 1;
1234 return std::min(a: MinNumSGPRs, b: getAddressableNumSGPRs(AK: Kind));
1235}
1236
1237unsigned getMaxNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1238 bool Addressable) {
1239 assert(WavesPerEU != 0);
1240
1241 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1242 unsigned AddressableNumSGPRs = getAddressableNumSGPRs(AK: Kind);
1243 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1244 if (Version.Major >= 10)
1245 return Addressable ? AddressableNumSGPRs : 108;
1246 if (Version.Major >= 8 && !Addressable)
1247 AddressableNumSGPRs = 112;
1248 unsigned MaxNumSGPRs = getSGPRBudgetPerWave(
1249 TotalNumSGPRs: getTotalNumSGPRs(AK: Kind), WavesPerEU, TrapReserve: getSGPRTrapHandlerReserve(STI),
1250 Granule: getSGPRAllocGranule(AK: Kind));
1251 return std::min(a: MaxNumSGPRs, b: AddressableNumSGPRs);
1252}
1253
1254bool isSGPROccupancyLimited(const MCSubtargetInfo &STI) {
1255 // From GFX10 on the SGPR file is large enough that SGPRs never limit
1256 // occupancy. Kept as one capability so callers don't each test the version.
1257 return getIsaVersion(GPU: STI.getCPU()).Major < 10;
1258}
1259
1260unsigned getNumExtraSGPRs(const MCSubtargetInfo &STI, bool VCCUsed,
1261 bool FlatScrUsed, bool XNACKUsed) {
1262 unsigned ExtraSGPRs = 0;
1263 if (VCCUsed)
1264 ExtraSGPRs = 2;
1265
1266 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1267 if (Version.Major >= 10)
1268 return ExtraSGPRs;
1269
1270 if (Version.Major < 8) {
1271 if (FlatScrUsed)
1272 ExtraSGPRs = 4;
1273 } else {
1274 if (XNACKUsed)
1275 ExtraSGPRs = 4;
1276
1277 if (FlatScrUsed ||
1278 STI.getFeatureBits().test(I: AMDGPU::FeatureArchitectedFlatScratch))
1279 ExtraSGPRs = 6;
1280 }
1281
1282 return ExtraSGPRs;
1283}
1284
1285static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs,
1286 unsigned Granule) {
1287 return divideCeil(Numerator: std::max(a: 1u, b: NumRegs), Denominator: Granule);
1288}
1289
1290unsigned getNumSGPRBlocks(const MCSubtargetInfo &STI, unsigned NumSGPRs) {
1291 // SGPRBlocks is actual number of SGPR blocks minus 1.
1292 return getGranulatedNumRegisterBlocks(NumRegs: NumSGPRs, Granule: getSGPREncodingGranule(STI)) -
1293 1;
1294}
1295
1296unsigned getVGPRAllocGranule(const MCSubtargetInfo &STI,
1297 unsigned DynamicVGPRBlockSize,
1298 std::optional<bool> EnableWavefrontSize32) {
1299 if (STI.getFeatureBits().test(I: FeatureGFX90AInsts))
1300 return 8;
1301
1302 if (DynamicVGPRBlockSize != 0)
1303 return DynamicVGPRBlockSize;
1304
1305 bool IsWave32 = EnableWavefrontSize32
1306 ? *EnableWavefrontSize32
1307 : STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1308
1309 if (STI.getFeatureBits().test(I: Feature1536VGPRs))
1310 return IsWave32 ? 24 : 12;
1311
1312 if (hasGFX10_3Insts(STI))
1313 return IsWave32 ? 16 : 8;
1314
1315 return IsWave32 ? 8 : 4;
1316}
1317
1318unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI,
1319 std::optional<bool> EnableWavefrontSize32) {
1320 if (STI.getFeatureBits().test(I: FeatureGFX90AInsts))
1321 return 8;
1322
1323 bool IsWave32 = EnableWavefrontSize32
1324 ? *EnableWavefrontSize32
1325 : STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1326
1327 if (STI.getFeatureBits().test(I: Feature1024AddressableVGPRs))
1328 return IsWave32 ? 16 : 8;
1329
1330 return IsWave32 ? 8 : 4;
1331}
1332
1333unsigned getArchVGPRAllocGranule() { return 4; }
1334
1335unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo &STI) {
1336 const auto &Features = STI.getFeatureBits();
1337 if (Features.test(I: Feature1024AddressableVGPRs))
1338 return Features.test(I: FeatureWavefrontSize32) ? 1024 : 512;
1339 return 256;
1340}
1341
1342unsigned getAddressableNumVGPRs(const MCSubtargetInfo &STI,
1343 unsigned DynamicVGPRBlockSize) {
1344 const auto &Features = STI.getFeatureBits();
1345 if (Features.test(I: FeatureGFX90AInsts))
1346 return 512;
1347
1348 if (DynamicVGPRBlockSize != 0) {
1349 // On GFX12 we can allocate at most MaxDynamicVGPRBlocks blocks of VGPRs.
1350 return MaxDynamicVGPRBlocks *
1351 getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1352 }
1353 return getAddressableNumArchVGPRs(STI);
1354}
1355
1356unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo &STI,
1357 unsigned NumVGPRs,
1358 unsigned DynamicVGPRBlockSize) {
1359 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1360 bool IsWave32 = STI.getFeatureBits().test(I: FeatureWavefrontSize32);
1361 return getNumWavesPerEUWithNumVGPRs(
1362 NumVGPRs, Granule: getVGPRAllocGranule(STI, DynamicVGPRBlockSize),
1363 MaxWaves: getMaxWavesPerEU(AK: Kind), TotalNumVGPRs: AMDGPU::getTotalNumVGPRs(AK: Kind, IsWave32));
1364}
1365
1366unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
1367 unsigned MaxWaves,
1368 unsigned TotalNumVGPRs) {
1369 if (NumVGPRs < Granule)
1370 return MaxWaves;
1371 unsigned RoundedRegs = alignTo(Value: NumVGPRs, Align: Granule);
1372 return std::min(a: std::max(a: TotalNumVGPRs / RoundedRegs, b: 1u), b: MaxWaves);
1373}
1374
1375unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
1376 unsigned TotalNumSGPRs, unsigned Granule,
1377 unsigned TrapReserve) {
1378 // Closed-form inverse of getMaxNumSGPRs(): the budget condition
1379 // SGPRs <= alignDown(TotalNumSGPRs / W - TrapReserve, Granule)
1380 // solves to W <= TotalNumSGPRs / (alignTo(SGPRs, Granule) + TrapReserve).
1381 unsigned PerWave = alignTo(Value: SGPRs, Align: Granule) + TrapReserve;
1382 return PerWave ? std::clamp(val: TotalNumSGPRs / PerWave, lo: 1u, hi: MaxWaves) : MaxWaves;
1383}
1384
1385unsigned getOccupancyWithNumSGPRs(const MCSubtargetInfo &STI, unsigned SGPRs) {
1386 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1387 unsigned MaxWaves = getMaxWavesPerEU(AK: Kind);
1388
1389 if (!isSGPROccupancyLimited(STI))
1390 return MaxWaves;
1391
1392 return getOccupancyWithNumSGPRs(SGPRs, MaxWaves, TotalNumSGPRs: getTotalNumSGPRs(AK: Kind),
1393 Granule: getSGPRAllocGranule(AK: Kind),
1394 TrapReserve: getSGPRTrapHandlerReserve(STI));
1395}
1396
1397unsigned getMinNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1398 unsigned DynamicVGPRBlockSize) {
1399 assert(WavesPerEU != 0);
1400
1401 // In dynamic VGPR mode, (static) occupancy does not depend on VGPR usage,
1402 // so getMaxNumVGPRs does not depend on WavesPerEU, and thus we need to return
1403 // zero because there is no nonzero VGPR usage N where going below N
1404 // achieves higher (static) occupancy.
1405 bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
1406 if (DynamicVGPREnabled)
1407 return 0;
1408
1409 GPUKind Kind = parseArchAMDGCN(CPU: STI.getCPU());
1410 unsigned MaxWavesPerEU = getMaxWavesPerEU(AK: Kind);
1411 if (WavesPerEU >= MaxWavesPerEU)
1412 return 0;
1413
1414 unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
1415 AK: Kind, IsWave32: STI.getFeatureBits().test(I: FeatureWavefrontSize32));
1416 unsigned AddrsableNumVGPRs =
1417 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1418 unsigned Granule = getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
1419 unsigned MaxNumVGPRs = alignDown(Value: TotNumVGPRs / WavesPerEU, Align: Granule);
1420
1421 if (MaxNumVGPRs == alignDown(Value: TotNumVGPRs / MaxWavesPerEU, Align: Granule))
1422 return 0;
1423
1424 unsigned MinWavesPerEU = getNumWavesPerEUWithNumVGPRs(STI, NumVGPRs: AddrsableNumVGPRs,
1425 DynamicVGPRBlockSize);
1426 if (WavesPerEU < MinWavesPerEU)
1427 return getMinNumVGPRs(STI, WavesPerEU: MinWavesPerEU, DynamicVGPRBlockSize);
1428
1429 unsigned MaxNumVGPRsNext = alignDown(Value: TotNumVGPRs / (WavesPerEU + 1), Align: Granule);
1430 unsigned MinNumVGPRs = 1 + std::min(a: MaxNumVGPRs - Granule, b: MaxNumVGPRsNext);
1431 return std::min(a: MinNumVGPRs, b: AddrsableNumVGPRs);
1432}
1433
1434unsigned getMaxNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
1435 unsigned DynamicVGPRBlockSize) {
1436 assert(WavesPerEU != 0);
1437
1438 unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
1439 AK: parseArchAMDGCN(CPU: STI.getCPU()),
1440 IsWave32: STI.getFeatureBits().test(I: FeatureWavefrontSize32));
1441
1442 // In dynamic VGPR mode, WavesPerEU does not imply a VGPR limit.
1443 bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
1444 unsigned MaxNumVGPRs =
1445 DynamicVGPREnabled
1446 ? TotNumVGPRs
1447 : alignDown(Value: TotNumVGPRs / WavesPerEU,
1448 Align: getVGPRAllocGranule(STI, DynamicVGPRBlockSize));
1449 unsigned AddressableNumVGPRs =
1450 getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
1451 return std::min(a: MaxNumVGPRs, b: AddressableNumVGPRs);
1452}
1453
1454unsigned getAllocatedNumVGPRBlocks(const MCSubtargetInfo &STI,
1455 unsigned NumVGPRs,
1456 unsigned DynamicVGPRBlockSize,
1457 std::optional<bool> EnableWavefrontSize32) {
1458 return getGranulatedNumRegisterBlocks(
1459 NumRegs: NumVGPRs,
1460 Granule: getVGPRAllocGranule(STI, DynamicVGPRBlockSize, EnableWavefrontSize32));
1461}
1462} // end namespace IsaInfo
1463
1464void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &KernelCode,
1465 const MCSubtargetInfo &STI) {
1466 IsaVersion Version = getIsaVersion(GPU: STI.getCPU());
1467 KernelCode.amd_kernel_code_version_major = 1;
1468 KernelCode.amd_kernel_code_version_minor = 2;
1469 KernelCode.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
1470 KernelCode.amd_machine_version_major = Version.Major;
1471 KernelCode.amd_machine_version_minor = Version.Minor;
1472 KernelCode.amd_machine_version_stepping = Version.Stepping;
1473 KernelCode.kernel_code_entry_byte_offset = sizeof(amd_kernel_code_t);
1474 if (STI.getFeatureBits().test(I: FeatureWavefrontSize32)) {
1475 KernelCode.wavefront_size = 5;
1476 KernelCode.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
1477 } else {
1478 KernelCode.wavefront_size = 6;
1479 }
1480
1481 // If the code object does not support indirect functions, then the value must
1482 // be 0xffffffff.
1483 KernelCode.call_convention = -1;
1484
1485 // These alignment values are specified in powers of two, so alignment =
1486 // 2^n. The minimum alignment is 2^4 = 16.
1487 KernelCode.kernarg_segment_alignment = 4;
1488 KernelCode.group_segment_alignment = 4;
1489 KernelCode.private_segment_alignment = 4;
1490
1491 if (Version.Major >= 10) {
1492 KernelCode.compute_pgm_resource_registers |=
1493 S_00B848_WGP_MODE(STI.getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
1494 S_00B848_MEM_ORDERED(1) | S_00B848_FWD_PROGRESS(1);
1495 }
1496}
1497
1498bool isReadOnlySegment(const GlobalValue *GV) {
1499 unsigned AS = GV->getAddressSpace();
1500 return AS == AMDGPUAS::CONSTANT_ADDRESS ||
1501 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
1502}
1503
1504bool shouldEmitConstantsToTextSection(const Triple &TT) {
1505 return TT.getArch() == Triple::r600;
1506}
1507
1508static bool isValidRegPrefix(char C) {
1509 return C == 'v' || C == 's' || C == 'a';
1510}
1511
1512std::tuple<char, unsigned, unsigned> parseAsmPhysRegName(StringRef RegName) {
1513 if (RegName.empty())
1514 return {};
1515
1516 char Kind = RegName.front();
1517 if (!isValidRegPrefix(C: Kind))
1518 return {};
1519
1520 RegName = RegName.drop_front();
1521 if (RegName.consume_front(Prefix: "[")) {
1522 unsigned Idx, End;
1523 bool Failed = RegName.consumeInteger(Radix: 10, Result&: Idx);
1524 Failed |= !RegName.consume_front(Prefix: ":");
1525 Failed |= RegName.consumeInteger(Radix: 10, Result&: End);
1526 Failed |= !RegName.consume_back(Suffix: "]");
1527 if (!Failed) {
1528 unsigned NumRegs = End - Idx + 1;
1529 if (NumRegs > 1)
1530 return {Kind, Idx, NumRegs};
1531 }
1532 } else {
1533 unsigned Idx;
1534 bool Failed = RegName.getAsInteger(Radix: 10, Result&: Idx);
1535 if (!Failed)
1536 return {Kind, Idx, 1};
1537 }
1538
1539 return {};
1540}
1541
1542std::tuple<char, unsigned, unsigned>
1543parseAsmConstraintPhysReg(StringRef Constraint) {
1544 StringRef RegName = Constraint;
1545 if (!RegName.consume_front(Prefix: "{") || !RegName.consume_back(Suffix: "}"))
1546 return {};
1547 return parseAsmPhysRegName(RegName);
1548}
1549
1550std::pair<unsigned, unsigned>
1551getIntegerPairAttribute(const Function &F, StringRef Name,
1552 std::pair<unsigned, unsigned> Default,
1553 bool OnlyFirstRequired) {
1554 if (auto Attr = getIntegerPairAttribute(F, Name, OnlyFirstRequired))
1555 return {Attr->first, Attr->second.value_or(u&: Default.second)};
1556 return Default;
1557}
1558
1559std::optional<std::pair<unsigned, std::optional<unsigned>>>
1560getIntegerPairAttribute(const Function &F, StringRef Name,
1561 bool OnlyFirstRequired) {
1562 Attribute A = F.getFnAttribute(Kind: Name);
1563 if (!A.isStringAttribute())
1564 return std::nullopt;
1565
1566 LLVMContext &Ctx = F.getContext();
1567 std::pair<unsigned, std::optional<unsigned>> Ints;
1568 std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(Separator: ',');
1569 if (Strs.first.trim().getAsInteger(Radix: 0, Result&: Ints.first)) {
1570 Ctx.emitError(ErrorStr: "can't parse first integer attribute " + Name);
1571 return std::nullopt;
1572 }
1573 unsigned Second = 0;
1574 if (Strs.second.trim().getAsInteger(Radix: 0, Result&: Second)) {
1575 if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
1576 Ctx.emitError(ErrorStr: "can't parse second integer attribute " + Name);
1577 return std::nullopt;
1578 }
1579 } else {
1580 Ints.second = Second;
1581 }
1582
1583 return Ints;
1584}
1585
1586SmallVector<unsigned> getIntegerVecAttribute(const Function &F, StringRef Name,
1587 unsigned Size,
1588 unsigned DefaultVal) {
1589 std::optional<SmallVector<unsigned>> R =
1590 getIntegerVecAttribute(F, Name, Size);
1591 return R.has_value() ? *R : SmallVector<unsigned>(Size, DefaultVal);
1592}
1593
1594std::optional<SmallVector<unsigned>>
1595getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) {
1596 assert(Size > 2);
1597 LLVMContext &Ctx = F.getContext();
1598
1599 Attribute A = F.getFnAttribute(Kind: Name);
1600 if (!A.isValid())
1601 return std::nullopt;
1602 if (!A.isStringAttribute()) {
1603 Ctx.emitError(ErrorStr: Name + " is not a string attribute");
1604 return std::nullopt;
1605 }
1606
1607 SmallVector<unsigned> Vals(Size);
1608
1609 StringRef S = A.getValueAsString();
1610 unsigned i = 0;
1611 for (; !S.empty() && i < Size; i++) {
1612 std::pair<StringRef, StringRef> Strs = S.split(Separator: ',');
1613 unsigned IntVal;
1614 if (Strs.first.trim().getAsInteger(Radix: 0, Result&: IntVal)) {
1615 Ctx.emitError(ErrorStr: "can't parse integer attribute " + Strs.first + " in " +
1616 Name);
1617 return std::nullopt;
1618 }
1619 Vals[i] = IntVal;
1620 S = Strs.second;
1621 }
1622
1623 if (!S.empty() || i < Size) {
1624 Ctx.emitError(ErrorStr: "attribute " + Name +
1625 " has incorrect number of integers; expected " +
1626 llvm::utostr(X: Size));
1627 return std::nullopt;
1628 }
1629 return Vals;
1630}
1631
1632SmallVector<unsigned> getMaxNumWorkGroups(const Function &F) {
1633 return getIntegerVecAttribute(F, Name: "amdgpu-max-num-workgroups", Size: 3,
1634 DefaultVal: std::numeric_limits<uint32_t>::max());
1635}
1636
1637bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) {
1638 assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!");
1639 for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) {
1640 auto Low =
1641 mdconst::extract<ConstantInt>(MD: MD.getOperand(I: 2 * I + 0))->getValue();
1642 auto High =
1643 mdconst::extract<ConstantInt>(MD: MD.getOperand(I: 2 * I + 1))->getValue();
1644 // There are two types of [A; B) ranges:
1645 // A < B, e.g. [4; 5) which is a range that only includes 4.
1646 // A > B, e.g. [5; 4) which is a range that wraps around and includes
1647 // everything except 4.
1648 if (Low.ult(RHS: High)) {
1649 if (Low.ule(RHS: Val) && High.ugt(RHS: Val))
1650 return true;
1651 } else {
1652 if (Low.uge(RHS: Val) && High.ult(RHS: Val))
1653 return true;
1654 }
1655 }
1656
1657 return false;
1658}
1659
1660unsigned getVmcntBitMask(const IsaVersion &Version) {
1661 return (1 << (getVmcntBitWidthLo(VersionMajor: Version.Major) +
1662 getVmcntBitWidthHi(VersionMajor: Version.Major))) -
1663 1;
1664}
1665
1666unsigned getLoadcntBitMask(const IsaVersion &Version) {
1667 return (1 << getLoadcntBitWidth(VersionMajor: Version.Major)) - 1;
1668}
1669
1670unsigned getSamplecntBitMask(const IsaVersion &Version) {
1671 return (1 << getSamplecntBitWidth(VersionMajor: Version.Major)) - 1;
1672}
1673
1674unsigned getBvhcntBitMask(const IsaVersion &Version) {
1675 return (1 << getBvhcntBitWidth(VersionMajor: Version.Major)) - 1;
1676}
1677
1678unsigned getExpcntBitMask(const IsaVersion &Version) {
1679 return (1 << getExpcntBitWidth(VersionMajor: Version.Major)) - 1;
1680}
1681
1682unsigned getLgkmcntBitMask(const IsaVersion &Version) {
1683 return (1 << getLgkmcntBitWidth(VersionMajor: Version.Major)) - 1;
1684}
1685
1686unsigned getDscntBitMask(const IsaVersion &Version) {
1687 return (1 << getDscntBitWidth(VersionMajor: Version.Major)) - 1;
1688}
1689
1690unsigned getKmcntBitMask(const IsaVersion &Version) {
1691 return (1 << getKmcntBitWidth(VersionMajor: Version.Major)) - 1;
1692}
1693
1694unsigned getXcntBitMask(const IsaVersion &Version) {
1695 return (1 << getXcntBitWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1696}
1697
1698unsigned getAsynccntBitMask(const IsaVersion &Version) {
1699 return (1 << getAsynccntBitWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1700}
1701
1702unsigned getStorecntBitMask(const IsaVersion &Version) {
1703 return (1 << getStorecntBitWidth(VersionMajor: Version.Major)) - 1;
1704}
1705
1706unsigned getBarrierMemberCountBitMask(const IsaVersion &Version) {
1707 return (1 << getBarrierMemberCountBitWidth(VersionMajor: Version.Major)) - 1;
1708}
1709
1710unsigned getWaitcntBitMask(const IsaVersion &Version) {
1711 unsigned VmcntLo = getBitMask(Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1712 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1713 unsigned Expcnt = getBitMask(Shift: getExpcntBitShift(VersionMajor: Version.Major),
1714 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1715 unsigned Lgkmcnt = getBitMask(Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1716 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1717 unsigned VmcntHi = getBitMask(Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1718 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1719 return VmcntLo | Expcnt | Lgkmcnt | VmcntHi;
1720}
1721
1722unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1723 unsigned VmcntLo = unpackBits(Src: Waitcnt, Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1724 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1725 unsigned VmcntHi = unpackBits(Src: Waitcnt, Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1726 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1727 return VmcntLo | VmcntHi << getVmcntBitWidthLo(VersionMajor: Version.Major);
1728}
1729
1730unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
1731 return unpackBits(Src: Waitcnt, Shift: getExpcntBitShift(VersionMajor: Version.Major),
1732 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1733}
1734
1735unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
1736 return unpackBits(Src: Waitcnt, Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1737 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1738}
1739
1740unsigned decodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt) {
1741 return unpackBits(Src: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1742 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1743}
1744
1745unsigned decodeStorecnt(const IsaVersion &Version, unsigned Waitcnt) {
1746 return unpackBits(Src: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1747 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1748}
1749
1750unsigned decodeDscnt(const IsaVersion &Version, unsigned Waitcnt) {
1751 return unpackBits(Src: Waitcnt, Shift: getDscntBitShift(VersionMajor: Version.Major),
1752 Width: getDscntBitWidth(VersionMajor: Version.Major));
1753}
1754
1755void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt,
1756 unsigned &Expcnt, unsigned &Lgkmcnt) {
1757 Vmcnt = decodeVmcnt(Version, Waitcnt);
1758 Expcnt = decodeExpcnt(Version, Waitcnt);
1759 Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
1760}
1761
1762unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
1763 unsigned Vmcnt) {
1764 Waitcnt = packBits(Src: Vmcnt, Dst: Waitcnt, Shift: getVmcntBitShiftLo(VersionMajor: Version.Major),
1765 Width: getVmcntBitWidthLo(VersionMajor: Version.Major));
1766 return packBits(Src: Vmcnt >> getVmcntBitWidthLo(VersionMajor: Version.Major), Dst: Waitcnt,
1767 Shift: getVmcntBitShiftHi(VersionMajor: Version.Major),
1768 Width: getVmcntBitWidthHi(VersionMajor: Version.Major));
1769}
1770
1771unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
1772 unsigned Expcnt) {
1773 return packBits(Src: Expcnt, Dst: Waitcnt, Shift: getExpcntBitShift(VersionMajor: Version.Major),
1774 Width: getExpcntBitWidth(VersionMajor: Version.Major));
1775}
1776
1777unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
1778 unsigned Lgkmcnt) {
1779 return packBits(Src: Lgkmcnt, Dst: Waitcnt, Shift: getLgkmcntBitShift(VersionMajor: Version.Major),
1780 Width: getLgkmcntBitWidth(VersionMajor: Version.Major));
1781}
1782
1783unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt,
1784 unsigned Expcnt, unsigned Lgkmcnt) {
1785 unsigned Waitcnt = getWaitcntBitMask(Version);
1786 Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
1787 Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
1788 Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
1789 return Waitcnt;
1790}
1791
1792static unsigned getCombinedCountBitMask(const IsaVersion &Version,
1793 bool IsStore) {
1794 unsigned Dscnt = getBitMask(Shift: getDscntBitShift(VersionMajor: Version.Major),
1795 Width: getDscntBitWidth(VersionMajor: Version.Major));
1796 if (IsStore) {
1797 unsigned Storecnt = getBitMask(Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1798 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1799 return Dscnt | Storecnt;
1800 }
1801 unsigned Loadcnt = getBitMask(Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1802 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1803 return Dscnt | Loadcnt;
1804}
1805
1806static unsigned encodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt,
1807 unsigned Loadcnt) {
1808 return packBits(Src: Loadcnt, Dst: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1809 Width: getLoadcntBitWidth(VersionMajor: Version.Major));
1810}
1811
1812static unsigned encodeStorecnt(const IsaVersion &Version, unsigned Waitcnt,
1813 unsigned Storecnt) {
1814 return packBits(Src: Storecnt, Dst: Waitcnt, Shift: getLoadcntStorecntBitShift(VersionMajor: Version.Major),
1815 Width: getStorecntBitWidth(VersionMajor: Version.Major));
1816}
1817
1818static unsigned encodeDscnt(const IsaVersion &Version, unsigned Waitcnt,
1819 unsigned Dscnt) {
1820 return packBits(Src: Dscnt, Dst: Waitcnt, Shift: getDscntBitShift(VersionMajor: Version.Major),
1821 Width: getDscntBitWidth(VersionMajor: Version.Major));
1822}
1823
1824unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt,
1825 unsigned Dscnt) {
1826 unsigned Waitcnt = getCombinedCountBitMask(Version, IsStore: false);
1827 Waitcnt = encodeLoadcnt(Version, Waitcnt, Loadcnt);
1828 Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
1829 return Waitcnt;
1830}
1831
1832unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt,
1833 unsigned Dscnt) {
1834 unsigned Waitcnt = getCombinedCountBitMask(Version, IsStore: true);
1835 Waitcnt = encodeStorecnt(Version, Waitcnt, Storecnt);
1836 Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
1837 return Waitcnt;
1838}
1839
1840//===----------------------------------------------------------------------===//
1841// Custom Operand Values
1842//===----------------------------------------------------------------------===//
1843
1844static bool isSymbolicCustomOperandEncoding(const CustomOperandVal *Opr,
1845 int Size, unsigned Code,
1846 bool &HasNonDefaultVal,
1847 const MCSubtargetInfo &STI) {
1848 unsigned UsedOprMask = 0;
1849 HasNonDefaultVal = false;
1850 for (int Idx = 0; Idx < Size; ++Idx) {
1851 const auto &Op = Opr[Idx];
1852 if (!Op.isSupported(STI))
1853 continue;
1854 UsedOprMask |= Op.getMask();
1855 unsigned Val = Op.decode(Code);
1856 if (!Op.isValid(Val))
1857 return false;
1858 HasNonDefaultVal |= (Val != Op.Default);
1859 }
1860 return (Code & ~UsedOprMask) == 0;
1861}
1862
1863static bool decodeCustomOperand(const CustomOperandVal *Opr, int Size,
1864 unsigned Code, int &Idx, StringRef &Name,
1865 unsigned &Val, bool &IsDefault,
1866 const MCSubtargetInfo &STI) {
1867 while (Idx < Size) {
1868 const auto &Op = Opr[Idx++];
1869 if (Op.isSupported(STI)) {
1870 Name = Op.Name;
1871 Val = Op.decode(Code);
1872 IsDefault = (Val == Op.Default);
1873 return true;
1874 }
1875 }
1876
1877 return false;
1878}
1879
1880static int encodeCustomOperandVal(const CustomOperandVal &Op,
1881 int64_t InputVal) {
1882 if (InputVal < 0 || InputVal > Op.Max)
1883 return OPR_VAL_INVALID;
1884 return Op.encode(Val: static_cast<unsigned>(InputVal));
1885}
1886
1887static int encodeCustomOperand(const CustomOperandVal *Opr, int Size,
1888 const StringRef Name, int64_t InputVal,
1889 unsigned &UsedOprMask,
1890 const MCSubtargetInfo &STI) {
1891 int InvalidId = OPR_ID_UNKNOWN;
1892 for (int Idx = 0; Idx < Size; ++Idx) {
1893 const auto &Op = Opr[Idx];
1894 if (Op.Name == Name) {
1895 if (!Op.isSupported(STI)) {
1896 InvalidId = OPR_ID_UNSUPPORTED;
1897 continue;
1898 }
1899 auto OprMask = Op.getMask();
1900 if (OprMask & UsedOprMask)
1901 return OPR_ID_DUPLICATE;
1902 UsedOprMask |= OprMask;
1903 return encodeCustomOperandVal(Op, InputVal);
1904 }
1905 }
1906 return InvalidId;
1907}
1908
1909//===----------------------------------------------------------------------===//
1910// DepCtr
1911//===----------------------------------------------------------------------===//
1912
1913namespace DepCtr {
1914
1915int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI) {
1916 unsigned Enc = 0;
1917 for (int Idx = 0; Idx < DEP_CTR_SIZE; ++Idx) {
1918 const auto &Op = DepCtrInfo[Idx];
1919 if (Op.isSupported(STI))
1920 Enc |= Op.encode(Val: Op.Default);
1921 }
1922 return Enc;
1923}
1924
1925bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal,
1926 const MCSubtargetInfo &STI) {
1927 return isSymbolicCustomOperandEncoding(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Code,
1928 HasNonDefaultVal, STI);
1929}
1930
1931bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val,
1932 bool &IsDefault, const MCSubtargetInfo &STI) {
1933 return decodeCustomOperand(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Code, Idx&: Id, Name, Val,
1934 IsDefault, STI);
1935}
1936
1937int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask,
1938 const MCSubtargetInfo &STI) {
1939 return encodeCustomOperand(Opr: DepCtrInfo, Size: DEP_CTR_SIZE, Name, InputVal: Val, UsedOprMask,
1940 STI);
1941}
1942
1943unsigned getVaVdstBitMask() { return (1 << getVaVdstBitWidth()) - 1; }
1944
1945unsigned getVaSdstBitMask() { return (1 << getVaSdstBitWidth()) - 1; }
1946
1947unsigned getVaSsrcBitMask() { return (1 << getVaSsrcBitWidth()) - 1; }
1948
1949unsigned getHoldCntBitMask(const IsaVersion &Version) {
1950 return (1 << getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor)) - 1;
1951}
1952
1953unsigned getVmVsrcBitMask() { return (1 << getVmVsrcBitWidth()) - 1; }
1954
1955unsigned getVaVccBitMask() { return (1 << getVaVccBitWidth()) - 1; }
1956
1957unsigned getSaSdstBitMask() { return (1 << getSaSdstBitWidth()) - 1; }
1958
1959unsigned decodeFieldVmVsrc(unsigned Encoded) {
1960 return unpackBits(Src: Encoded, Shift: getVmVsrcBitShift(), Width: getVmVsrcBitWidth());
1961}
1962
1963unsigned decodeFieldVaVdst(unsigned Encoded) {
1964 return unpackBits(Src: Encoded, Shift: getVaVdstBitShift(), Width: getVaVdstBitWidth());
1965}
1966
1967unsigned decodeFieldSaSdst(unsigned Encoded) {
1968 return unpackBits(Src: Encoded, Shift: getSaSdstBitShift(), Width: getSaSdstBitWidth());
1969}
1970
1971unsigned decodeFieldVaSdst(unsigned Encoded) {
1972 return unpackBits(Src: Encoded, Shift: getVaSdstBitShift(), Width: getVaSdstBitWidth());
1973}
1974
1975unsigned decodeFieldVaVcc(unsigned Encoded) {
1976 return unpackBits(Src: Encoded, Shift: getVaVccBitShift(), Width: getVaVccBitWidth());
1977}
1978
1979unsigned decodeFieldVaSsrc(unsigned Encoded) {
1980 return unpackBits(Src: Encoded, Shift: getVaSsrcBitShift(), Width: getVaSsrcBitWidth());
1981}
1982
1983unsigned decodeFieldHoldCnt(unsigned Encoded, const IsaVersion &Version) {
1984 return unpackBits(Src: Encoded, Shift: getHoldCntBitShift(),
1985 Width: getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor));
1986}
1987
1988unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc) {
1989 return packBits(Src: VmVsrc, Dst: Encoded, Shift: getVmVsrcBitShift(), Width: getVmVsrcBitWidth());
1990}
1991
1992unsigned encodeFieldVmVsrc(unsigned VmVsrc, const MCSubtargetInfo &STI) {
1993 unsigned Encoded = getDefaultDepCtrEncoding(STI);
1994 return encodeFieldVmVsrc(Encoded, VmVsrc);
1995}
1996
1997unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst) {
1998 return packBits(Src: VaVdst, Dst: Encoded, Shift: getVaVdstBitShift(), Width: getVaVdstBitWidth());
1999}
2000
2001unsigned encodeFieldVaVdst(unsigned VaVdst, const MCSubtargetInfo &STI) {
2002 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2003 return encodeFieldVaVdst(Encoded, VaVdst);
2004}
2005
2006unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst) {
2007 return packBits(Src: SaSdst, Dst: Encoded, Shift: getSaSdstBitShift(), Width: getSaSdstBitWidth());
2008}
2009
2010unsigned encodeFieldSaSdst(unsigned SaSdst, const MCSubtargetInfo &STI) {
2011 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2012 return encodeFieldSaSdst(Encoded, SaSdst);
2013}
2014
2015unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst) {
2016 return packBits(Src: VaSdst, Dst: Encoded, Shift: getVaSdstBitShift(), Width: getVaSdstBitWidth());
2017}
2018
2019unsigned encodeFieldVaSdst(unsigned VaSdst, const MCSubtargetInfo &STI) {
2020 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2021 return encodeFieldVaSdst(Encoded, VaSdst);
2022}
2023
2024unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc) {
2025 return packBits(Src: VaVcc, Dst: Encoded, Shift: getVaVccBitShift(), Width: getVaVccBitWidth());
2026}
2027
2028unsigned encodeFieldVaVcc(unsigned VaVcc, const MCSubtargetInfo &STI) {
2029 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2030 return encodeFieldVaVcc(Encoded, VaVcc);
2031}
2032
2033unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc) {
2034 return packBits(Src: VaSsrc, Dst: Encoded, Shift: getVaSsrcBitShift(), Width: getVaSsrcBitWidth());
2035}
2036
2037unsigned encodeFieldVaSsrc(unsigned VaSsrc, const MCSubtargetInfo &STI) {
2038 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2039 return encodeFieldVaSsrc(Encoded, VaSsrc);
2040}
2041
2042unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt,
2043 const IsaVersion &Version) {
2044 return packBits(Src: HoldCnt, Dst: Encoded, Shift: getHoldCntBitShift(),
2045 Width: getHoldCntWidth(VersionMajor: Version.Major, VersionMinor: Version.Minor));
2046}
2047
2048unsigned encodeFieldHoldCnt(unsigned HoldCnt, const MCSubtargetInfo &STI) {
2049 unsigned Encoded = getDefaultDepCtrEncoding(STI);
2050 return encodeFieldHoldCnt(Encoded, HoldCnt, Version: getIsaVersion(GPU: STI.getCPU()));
2051}
2052
2053} // namespace DepCtr
2054
2055//===----------------------------------------------------------------------===//
2056// exp tgt
2057//===----------------------------------------------------------------------===//
2058
2059namespace Exp {
2060
2061struct ExpTgt {
2062 StringLiteral Name;
2063 unsigned Tgt;
2064 unsigned MaxIndex;
2065};
2066
2067// clang-format off
2068static constexpr ExpTgt ExpTgtInfo[] = {
2069 {.Name: {"null"}, .Tgt: ET_NULL, .MaxIndex: ET_NULL_MAX_IDX},
2070 {.Name: {"mrtz"}, .Tgt: ET_MRTZ, .MaxIndex: ET_MRTZ_MAX_IDX},
2071 {.Name: {"prim"}, .Tgt: ET_PRIM, .MaxIndex: ET_PRIM_MAX_IDX},
2072 {.Name: {"mrt"}, .Tgt: ET_MRT0, .MaxIndex: ET_MRT_MAX_IDX},
2073 {.Name: {"pos"}, .Tgt: ET_POS0, .MaxIndex: ET_POS_MAX_IDX},
2074 {.Name: {"dual_src_blend"},.Tgt: ET_DUAL_SRC_BLEND0, .MaxIndex: ET_DUAL_SRC_BLEND_MAX_IDX},
2075 {.Name: {"param"}, .Tgt: ET_PARAM0, .MaxIndex: ET_PARAM_MAX_IDX},
2076};
2077// clang-format on
2078
2079bool getTgtName(unsigned Id, StringRef &Name, int &Index) {
2080 for (const ExpTgt &Val : ExpTgtInfo) {
2081 if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) {
2082 Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt);
2083 Name = Val.Name;
2084 return true;
2085 }
2086 }
2087 return false;
2088}
2089
2090unsigned getTgtId(const StringRef Name) {
2091
2092 for (const ExpTgt &Val : ExpTgtInfo) {
2093 if (Val.MaxIndex == 0 && Name == Val.Name)
2094 return Val.Tgt;
2095
2096 if (Val.MaxIndex > 0 && Name.starts_with(Prefix: Val.Name)) {
2097 StringRef Suffix = Name.drop_front(N: Val.Name.size());
2098
2099 unsigned Id;
2100 if (Suffix.getAsInteger(Radix: 10, Result&: Id) || Id > Val.MaxIndex)
2101 return ET_INVALID;
2102
2103 // Disable leading zeroes
2104 if (Suffix.size() > 1 && Suffix[0] == '0')
2105 return ET_INVALID;
2106
2107 return Val.Tgt + Id;
2108 }
2109 }
2110 return ET_INVALID;
2111}
2112
2113bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) {
2114 switch (Id) {
2115 case ET_NULL:
2116 return !isGFX11Plus(STI);
2117 case ET_POS4:
2118 case ET_PRIM:
2119 return isGFX10Plus(STI);
2120 case ET_DUAL_SRC_BLEND0:
2121 case ET_DUAL_SRC_BLEND1:
2122 return isGFX11Plus(STI);
2123 default:
2124 if (Id >= ET_PARAM0 && Id <= ET_PARAM31)
2125 return !isGFX11Plus(STI) || isGFX13Plus(STI);
2126 return true;
2127 }
2128}
2129
2130} // namespace Exp
2131
2132//===----------------------------------------------------------------------===//
2133// MTBUF Format
2134//===----------------------------------------------------------------------===//
2135
2136namespace MTBUFFormat {
2137
2138int64_t getDfmt(const StringRef Name) {
2139 for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
2140 if (Name == DfmtSymbolic[Id])
2141 return Id;
2142 }
2143 return DFMT_UNDEF;
2144}
2145
2146StringRef getDfmtName(unsigned Id) {
2147 assert(Id <= DFMT_MAX);
2148 return DfmtSymbolic[Id];
2149}
2150
2151static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {
2152 if (isSI(STI) || isCI(STI))
2153 return NfmtSymbolicSICI;
2154 if (isVI(STI) || isGFX9(STI))
2155 return NfmtSymbolicVI;
2156 return NfmtSymbolicGFX10;
2157}
2158
2159int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
2160 const auto *lookupTable = getNfmtLookupTable(STI);
2161 for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
2162 if (Name == lookupTable[Id])
2163 return Id;
2164 }
2165 return NFMT_UNDEF;
2166}
2167
2168StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
2169 assert(Id <= NFMT_MAX);
2170 return getNfmtLookupTable(STI)[Id];
2171}
2172
2173bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2174 unsigned Dfmt;
2175 unsigned Nfmt;
2176 decodeDfmtNfmt(Format: Id, Dfmt, Nfmt);
2177 return isValidNfmt(Val: Nfmt, STI);
2178}
2179
2180bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
2181 return !getNfmtName(Id, STI).empty();
2182}
2183
2184int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
2185 return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);
2186}
2187
2188void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
2189 Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;
2190 Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
2191}
2192
2193int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI) {
2194 if (isGFX11Plus(STI)) {
2195 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2196 if (Name == UfmtSymbolicGFX11[Id])
2197 return Id;
2198 }
2199 } else {
2200 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2201 if (Name == UfmtSymbolicGFX10[Id])
2202 return Id;
2203 }
2204 }
2205 return UFMT_UNDEF;
2206}
2207
2208StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI) {
2209 if (isValidUnifiedFormat(Val: Id, STI))
2210 return isGFX10(STI) ? UfmtSymbolicGFX10[Id] : UfmtSymbolicGFX11[Id];
2211 return "";
2212}
2213
2214bool isValidUnifiedFormat(unsigned Id, const MCSubtargetInfo &STI) {
2215 return isGFX10(STI) ? Id <= UfmtGFX10::UFMT_LAST : Id <= UfmtGFX11::UFMT_LAST;
2216}
2217
2218int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt,
2219 const MCSubtargetInfo &STI) {
2220 int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
2221 if (isGFX11Plus(STI)) {
2222 for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
2223 if (Fmt == DfmtNfmt2UFmtGFX11[Id])
2224 return Id;
2225 }
2226 } else {
2227 for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
2228 if (Fmt == DfmtNfmt2UFmtGFX10[Id])
2229 return Id;
2230 }
2231 }
2232 return UFMT_UNDEF;
2233}
2234
2235bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
2236 return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
2237}
2238
2239unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {
2240 if (isGFX10Plus(STI))
2241 return UFMT_DEFAULT;
2242 return DFMT_NFMT_DEFAULT;
2243}
2244
2245} // namespace MTBUFFormat
2246
2247//===----------------------------------------------------------------------===//
2248// SendMsg
2249//===----------------------------------------------------------------------===//
2250
2251namespace SendMsg {
2252
2253static uint64_t getMsgIdMask(const MCSubtargetInfo &STI) {
2254 return isGFX11Plus(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_;
2255}
2256
2257bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) {
2258 return (MsgId & ~(getMsgIdMask(STI))) == 0;
2259}
2260
2261bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
2262 bool Strict) {
2263 assert(isValidMsgId(MsgId, STI));
2264
2265 if (!Strict)
2266 return 0 <= OpId && isUInt<OP_WIDTH_>(x: OpId);
2267
2268 if (msgRequiresOp(MsgId, STI)) {
2269 if (MsgId == ID_GS_PreGFX11 && OpId == OP_GS_NOP)
2270 return false;
2271
2272 return !getMsgOpName(MsgId, Encoding: OpId, STI).empty();
2273 }
2274
2275 return OpId == OP_NONE_;
2276}
2277
2278bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
2279 const MCSubtargetInfo &STI, bool Strict) {
2280 assert(isValidMsgOp(MsgId, OpId, STI, Strict));
2281
2282 if (!Strict)
2283 return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(x: StreamId);
2284
2285 if (!isGFX11Plus(STI)) {
2286 switch (MsgId) {
2287 case ID_GS_PreGFX11:
2288 return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
2289 case ID_GS_DONE_PreGFX11:
2290 return (OpId == OP_GS_NOP)
2291 ? (StreamId == STREAM_ID_NONE_)
2292 : (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
2293 }
2294 }
2295 return StreamId == STREAM_ID_NONE_;
2296}
2297
2298bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI) {
2299 return MsgId == ID_SYSMSG ||
2300 (!isGFX11Plus(STI) &&
2301 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11));
2302}
2303
2304bool msgSupportsStream(int64_t MsgId, int64_t OpId,
2305 const MCSubtargetInfo &STI) {
2306 return !isGFX11Plus(STI) &&
2307 (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11) &&
2308 OpId != OP_GS_NOP;
2309}
2310
2311void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,
2312 uint16_t &StreamId, const MCSubtargetInfo &STI) {
2313 MsgId = static_cast<uint16_t>(Val & getMsgIdMask(STI));
2314 if (isGFX11Plus(STI)) {
2315 OpId = 0;
2316 StreamId = 0;
2317 } else {
2318 OpId = (Val & OP_MASK_) >> OP_SHIFT_;
2319 StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
2320 }
2321}
2322
2323uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId) {
2324 return MsgId | (OpId << OP_SHIFT_) | (StreamId << STREAM_ID_SHIFT_);
2325}
2326
2327bool msgDoesNotUseM0(int64_t MsgId, const MCSubtargetInfo &STI) {
2328 // Explicitly list message types that are known to not use m0.
2329 // This is safer than excluding only GS_ALLOC_REQ, in case new message
2330 // types are added in the future that do use m0.
2331 if (isGFX11Plus(STI)) {
2332 switch (MsgId) {
2333 case ID_DEALLOC_VGPRS_GFX11Plus:
2334 return true;
2335 default:
2336 break;
2337 }
2338 }
2339 switch (MsgId) {
2340 case ID_SAVEWAVE:
2341 case ID_STALL_WAVE_GEN:
2342 case ID_HALT_WAVES:
2343 case ID_ORDERED_PS_DONE:
2344 case ID_EARLY_PRIM_DEALLOC:
2345 case ID_GET_DOORBELL:
2346 case ID_GET_DDID:
2347 case ID_SYSMSG:
2348 return true;
2349 default:
2350 return false;
2351 }
2352}
2353
2354} // namespace SendMsg
2355
2356//===----------------------------------------------------------------------===//
2357//
2358//===----------------------------------------------------------------------===//
2359
2360unsigned getInitialPSInputAddr(const Function &F) {
2361 return static_cast<unsigned>(
2362 F.getFnAttributeAsParsedInteger(Kind: "InitialPSInputAddr", Default: 0));
2363}
2364
2365bool getHasColorExport(const Function &F) {
2366 // As a safe default always respond as if PS has color exports.
2367 return F.getFnAttributeAsParsedInteger(
2368 Kind: "amdgpu-color-export",
2369 Default: F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0;
2370}
2371
2372bool getHasDepthExport(const Function &F) {
2373 return F.getFnAttributeAsParsedInteger(Kind: "amdgpu-depth-export", Default: 0) != 0;
2374}
2375
2376unsigned getDynamicVGPRBlockSize(const Function &F) {
2377 unsigned BlockSize = static_cast<unsigned>(
2378 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-dynamic-vgpr-block-size", Default: 0));
2379
2380 if (BlockSize == 16 || BlockSize == 32)
2381 return BlockSize;
2382
2383 return 0;
2384}
2385
2386bool hasMIMG_R128(const MCSubtargetInfo &STI) {
2387 return STI.hasFeature(Feature: AMDGPU::FeatureMIMG_R128) &&
2388 !STI.hasFeature(Feature: AMDGPU::FeatureR128A16);
2389}
2390
2391bool hasA16(const MCSubtargetInfo &STI) {
2392 return STI.hasFeature(Feature: AMDGPU::FeatureA16);
2393}
2394
2395bool hasG16(const MCSubtargetInfo &STI) {
2396 return STI.hasFeature(Feature: AMDGPU::FeatureG16);
2397}
2398
2399bool hasPackedD16(const MCSubtargetInfo &STI) {
2400 return !STI.hasFeature(Feature: AMDGPU::FeatureUnpackedD16VMem) && !isCI(STI) &&
2401 !isSI(STI);
2402}
2403
2404bool hasGDS(const MCSubtargetInfo &STI) {
2405 return STI.hasFeature(Feature: AMDGPU::FeatureGDS);
2406}
2407
2408unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler) {
2409 auto Version = getIsaVersion(GPU: STI.getCPU());
2410 if (Version.Major == 10)
2411 return Version.Minor >= 3 ? 13 : 5;
2412 if (Version.Major == 11)
2413 return 5;
2414 if (Version.Major >= 12)
2415 return HasSampler ? 4 : 5;
2416 return 0;
2417}
2418
2419unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI) {
2420 if (isGFX1250Plus(STI))
2421 return 32;
2422 return 16;
2423}
2424
2425bool isSI(const MCSubtargetInfo &STI) {
2426 return STI.hasFeature(Feature: AMDGPU::FeatureSouthernIslands);
2427}
2428
2429bool isCI(const MCSubtargetInfo &STI) {
2430 return STI.hasFeature(Feature: AMDGPU::FeatureSeaIslands);
2431}
2432
2433bool isVI(const MCSubtargetInfo &STI) {
2434 return STI.hasFeature(Feature: AMDGPU::FeatureVolcanicIslands);
2435}
2436
2437bool isGFX9(const MCSubtargetInfo &STI) {
2438 return STI.hasFeature(Feature: AMDGPU::FeatureGFX9);
2439}
2440
2441bool isGFX9_GFX10(const MCSubtargetInfo &STI) {
2442 return isGFX9(STI) || isGFX10(STI);
2443}
2444
2445bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI) {
2446 return isGFX9(STI) || isGFX10(STI) || isGFX11(STI);
2447}
2448
2449bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI) {
2450 return isVI(STI) || isGFX9(STI) || isGFX10(STI);
2451}
2452
2453bool isGFX8Plus(const MCSubtargetInfo &STI) {
2454 return isVI(STI) || isGFX9Plus(STI);
2455}
2456
2457bool isGFX9Plus(const MCSubtargetInfo &STI) {
2458 return isGFX9(STI) || isGFX10Plus(STI);
2459}
2460
2461bool isNotGFX9Plus(const MCSubtargetInfo &STI) { return !isGFX9Plus(STI); }
2462
2463bool hasPopsExitingWaveID(const MCSubtargetInfo &STI) {
2464 return STI.hasFeature(Feature: AMDGPU::FeaturePopsExitingWaveID);
2465}
2466
2467bool hasPrivateApertureRegs(const MCSubtargetInfo &STI) {
2468 return STI.hasFeature(Feature: AMDGPU::FeatureApertureRegs) &&
2469 !STI.hasFeature(Feature: AMDGPU::FeatureGloballyAddressableScratch);
2470}
2471
2472bool isGFX10(const MCSubtargetInfo &STI) {
2473 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10);
2474}
2475
2476bool isGFX10_GFX11(const MCSubtargetInfo &STI) {
2477 return isGFX10(STI) || isGFX11(STI);
2478}
2479
2480bool isGFX10Plus(const MCSubtargetInfo &STI) {
2481 return isGFX10(STI) || isGFX11Plus(STI);
2482}
2483
2484bool isGFX11(const MCSubtargetInfo &STI) {
2485 return STI.hasFeature(Feature: AMDGPU::FeatureGFX11);
2486}
2487
2488bool isGFX11Plus(const MCSubtargetInfo &STI) {
2489 return isGFX11(STI) || isGFX12Plus(STI);
2490}
2491
2492bool isGFX12(const MCSubtargetInfo &STI) {
2493 return STI.getFeatureBits()[AMDGPU::FeatureGFX12];
2494}
2495
2496bool isGFX12Plus(const MCSubtargetInfo &STI) {
2497 return isGFX12(STI) || isGFX13Plus(STI);
2498}
2499
2500bool isNotGFX12Plus(const MCSubtargetInfo &STI) { return !isGFX12Plus(STI); }
2501
2502bool isGFX1250(const MCSubtargetInfo &STI) {
2503 return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts] && !isGFX13(STI);
2504}
2505
2506bool isFullSIMDMode(const MCSubtargetInfo &STI) {
2507 return isGFX1250(STI) || !STI.getFeatureBits().test(I: FeatureCuMode);
2508}
2509
2510bool isGFX1250Plus(const MCSubtargetInfo &STI) {
2511 return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts];
2512}
2513
2514bool isGFX13(const MCSubtargetInfo &STI) {
2515 return STI.getFeatureBits()[AMDGPU::FeatureGFX13];
2516}
2517
2518bool isGFX13Plus(const MCSubtargetInfo &STI) { return isGFX13(STI); }
2519
2520bool supportsWGP(const MCSubtargetInfo &STI) {
2521 if (isGFX1250(STI))
2522 return false;
2523 return isGFX10Plus(STI);
2524}
2525
2526bool isNotGFX11Plus(const MCSubtargetInfo &STI) { return !isGFX11Plus(STI); }
2527
2528bool isNotGFX10Plus(const MCSubtargetInfo &STI) {
2529 return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI);
2530}
2531
2532bool isGFX10Before1030(const MCSubtargetInfo &STI) {
2533 return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI);
2534}
2535
2536bool isGCN3Encoding(const MCSubtargetInfo &STI) {
2537 return STI.hasFeature(Feature: AMDGPU::FeatureGCN3Encoding);
2538}
2539
2540bool isGFX10_BEncoding(const MCSubtargetInfo &STI) {
2541 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10_BEncoding);
2542}
2543
2544bool hasGFX10_3Insts(const MCSubtargetInfo &STI) {
2545 return STI.hasFeature(Feature: AMDGPU::FeatureGFX10_3Insts);
2546}
2547
2548bool isGFX10_3_GFX11(const MCSubtargetInfo &STI) {
2549 return isGFX10_BEncoding(STI) && !isGFX12Plus(STI);
2550}
2551
2552bool isGFX90A(const MCSubtargetInfo &STI) {
2553 return STI.hasFeature(Feature: AMDGPU::FeatureGFX90AInsts);
2554}
2555
2556bool isGFX940(const MCSubtargetInfo &STI) {
2557 return STI.hasFeature(Feature: AMDGPU::FeatureGFX940Insts);
2558}
2559
2560bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) {
2561 return STI.hasFeature(Feature: AMDGPU::FeatureArchitectedFlatScratch);
2562}
2563
2564bool hasMAIInsts(const MCSubtargetInfo &STI) {
2565 return STI.hasFeature(Feature: AMDGPU::FeatureMAIInsts);
2566}
2567
2568bool hasVOPD(const MCSubtargetInfo &STI) {
2569 return STI.hasFeature(Feature: AMDGPU::FeatureVOPDInsts);
2570}
2571
2572bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI) {
2573 return STI.hasFeature(Feature: AMDGPU::FeatureDPPSrc1SGPR);
2574}
2575
2576unsigned hasKernargPreload(const MCSubtargetInfo &STI) {
2577 return STI.hasFeature(Feature: AMDGPU::FeatureKernargPreload);
2578}
2579
2580int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR,
2581 int32_t ArgNumVGPR) {
2582 if (has90AInsts && ArgNumAGPR)
2583 return alignTo(Value: ArgNumVGPR, Align: 4) + ArgNumAGPR;
2584 return std::max(a: ArgNumVGPR, b: ArgNumAGPR);
2585}
2586
2587bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI) {
2588 const MCRegisterClass &SGPRClass =
2589 TRI->getRegClass(i: AMDGPU::SReg_32RegClassID);
2590 const MCRegister FirstSubReg = TRI->getSubReg(Reg, Idx: AMDGPU::sub0);
2591 return SGPRClass.contains(Reg: FirstSubReg != 0 ? FirstSubReg : Reg) ||
2592 Reg == AMDGPU::SCC;
2593}
2594
2595bool isRsrcIndexReg(MCRegister Reg, const MCRegisterInfo &MRI) {
2596 return MRI.getRegClass(i: AMDGPU::RsrcReg32RegClassID).contains(Reg);
2597}
2598
2599bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI) {
2600 return MRI.getEncodingValue(Reg) & AMDGPU::HWEncoding::IS_HI16;
2601}
2602
2603#define MAP_REG2REG \
2604 using namespace AMDGPU; \
2605 switch (Reg.id()) { \
2606 default: \
2607 return Reg; \
2608 CASE_CI_VI(FLAT_SCR) \
2609 CASE_CI_VI(FLAT_SCR_LO) \
2610 CASE_CI_VI(FLAT_SCR_HI) \
2611 CASE_VI_GFX9PLUS(TTMP0) \
2612 CASE_VI_GFX9PLUS(TTMP1) \
2613 CASE_VI_GFX9PLUS(TTMP2) \
2614 CASE_VI_GFX9PLUS(TTMP3) \
2615 CASE_VI_GFX9PLUS(TTMP4) \
2616 CASE_VI_GFX9PLUS(TTMP5) \
2617 CASE_VI_GFX9PLUS(TTMP6) \
2618 CASE_VI_GFX9PLUS(TTMP7) \
2619 CASE_VI_GFX9PLUS(TTMP8) \
2620 CASE_VI_GFX9PLUS(TTMP9) \
2621 CASE_VI_GFX9PLUS(TTMP10) \
2622 CASE_VI_GFX9PLUS(TTMP11) \
2623 CASE_VI_GFX9PLUS(TTMP12) \
2624 CASE_VI_GFX9PLUS(TTMP13) \
2625 CASE_VI_GFX9PLUS(TTMP14) \
2626 CASE_VI_GFX9PLUS(TTMP15) \
2627 CASE_VI_GFX9PLUS(TTMP0_TTMP1) \
2628 CASE_VI_GFX9PLUS(TTMP2_TTMP3) \
2629 CASE_VI_GFX9PLUS(TTMP4_TTMP5) \
2630 CASE_VI_GFX9PLUS(TTMP6_TTMP7) \
2631 CASE_VI_GFX9PLUS(TTMP8_TTMP9) \
2632 CASE_VI_GFX9PLUS(TTMP10_TTMP11) \
2633 CASE_VI_GFX9PLUS(TTMP12_TTMP13) \
2634 CASE_VI_GFX9PLUS(TTMP14_TTMP15) \
2635 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3) \
2636 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7) \
2637 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11) \
2638 CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15) \
2639 CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7) \
2640 CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11) \
2641 CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2642 CASE_VI_GFX9PLUS( \
2643 TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
2644 CASE_GFXPRE11_GFX11PLUS(M0) \
2645 CASE_GFXPRE11_GFX11PLUS(SGPR_NULL) \
2646 CASE_GFXPRE11_GFX11PLUS_TO(SGPR_NULL64, SGPR_NULL) \
2647 }
2648
2649#define CASE_CI_VI(node) \
2650 assert(!isSI(STI)); \
2651 case node: \
2652 return isCI(STI) ? node##_ci : node##_vi;
2653
2654#define CASE_VI_GFX9PLUS(node) \
2655 case node: \
2656 return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi;
2657
2658#define CASE_GFXPRE11_GFX11PLUS(node) \
2659 case node: \
2660 return isGFX11Plus(STI) ? node##_gfx11plus : node##_gfxpre11;
2661
2662#define CASE_GFXPRE11_GFX11PLUS_TO(node, result) \
2663 case node: \
2664 return isGFX11Plus(STI) ? result##_gfx11plus : result##_gfxpre11;
2665
2666MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI) {
2667 if (STI.getTargetTriple().getArch() == Triple::r600)
2668 return Reg;
2669 MAP_REG2REG
2670}
2671
2672#undef CASE_CI_VI
2673#undef CASE_VI_GFX9PLUS
2674#undef CASE_GFXPRE11_GFX11PLUS
2675#undef CASE_GFXPRE11_GFX11PLUS_TO
2676
2677#define CASE_CI_VI(node) \
2678 case node##_ci: \
2679 case node##_vi: \
2680 return node;
2681#define CASE_VI_GFX9PLUS(node) \
2682 case node##_vi: \
2683 case node##_gfx9plus: \
2684 return node;
2685#define CASE_GFXPRE11_GFX11PLUS(node) \
2686 case node##_gfx11plus: \
2687 case node##_gfxpre11: \
2688 return node;
2689#define CASE_GFXPRE11_GFX11PLUS_TO(node, result)
2690
2691MCRegister mc2PseudoReg(MCRegister Reg) { MAP_REG2REG }
2692
2693bool isInlineValue(MCRegister Reg) {
2694 switch (Reg.id()) {
2695 case AMDGPU::SRC_SHARED_BASE_LO:
2696 case AMDGPU::SRC_SHARED_BASE:
2697 case AMDGPU::SRC_SHARED_LIMIT_LO:
2698 case AMDGPU::SRC_SHARED_LIMIT:
2699 case AMDGPU::SRC_PRIVATE_BASE_LO:
2700 case AMDGPU::SRC_PRIVATE_BASE:
2701 case AMDGPU::SRC_PRIVATE_LIMIT_LO:
2702 case AMDGPU::SRC_PRIVATE_LIMIT:
2703 case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
2704 case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
2705 case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
2706 return true;
2707 case AMDGPU::SRC_VCCZ:
2708 case AMDGPU::SRC_EXECZ:
2709 case AMDGPU::SRC_SCC:
2710 return true;
2711 case AMDGPU::SGPR_NULL:
2712 return true;
2713 default:
2714 return false;
2715 }
2716}
2717
2718#undef CASE_CI_VI
2719#undef CASE_VI_GFX9PLUS
2720#undef CASE_GFXPRE11_GFX11PLUS
2721#undef CASE_GFXPRE11_GFX11PLUS_TO
2722#undef MAP_REG2REG
2723
2724bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2725 assert(OpNo < Desc.NumOperands);
2726 unsigned OpType = Desc.operands()[OpNo].OperandType;
2727 return OpType >= AMDGPU::OPERAND_KIMM_FIRST &&
2728 OpType <= AMDGPU::OPERAND_KIMM_LAST;
2729}
2730
2731bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
2732 assert(OpNo < Desc.NumOperands);
2733 unsigned OpType = Desc.operands()[OpNo].OperandType;
2734 switch (OpType) {
2735 case AMDGPU::OPERAND_REG_IMM_FP32:
2736 case AMDGPU::OPERAND_REG_IMM_FP64:
2737 case AMDGPU::OPERAND_REG_IMM_FP16:
2738 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
2739 case AMDGPU::OPERAND_REG_IMM_V2FP16:
2740 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
2741 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
2742 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2743 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2744 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2745 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2746 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2747 case AMDGPU::OPERAND_REG_IMM_V2FP32:
2748 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2749 case AMDGPU::OPERAND_REG_IMM_V2FP64:
2750 return true;
2751 default:
2752 return false;
2753 }
2754}
2755
2756// Avoid using MCRegisterClass::getSize, since that function will go away
2757// (move from MC* level to Target* level). Return size in bits.
2758unsigned getRegBitWidth(unsigned RCID) {
2759 switch (RCID) {
2760 case AMDGPU::VGPR_16RegClassID:
2761 case AMDGPU::VGPR_16_Lo128RegClassID:
2762 case AMDGPU::SGPR_LO16RegClassID:
2763 case AMDGPU::AGPR_LO16RegClassID:
2764 return 16;
2765 case AMDGPU::SGPR_32RegClassID:
2766 case AMDGPU::VGPR_32RegClassID:
2767 case AMDGPU::VGPR_32_Lo256RegClassID:
2768 case AMDGPU::VRegOrLds_32RegClassID:
2769 case AMDGPU::AGPR_32RegClassID:
2770 case AMDGPU::VS_32RegClassID:
2771 case AMDGPU::AV_32RegClassID:
2772 case AMDGPU::SReg_32RegClassID:
2773 case AMDGPU::SReg_32_XM0RegClassID:
2774 case AMDGPU::SRegOrLds_32RegClassID:
2775 return 32;
2776 case AMDGPU::SGPR_64RegClassID:
2777 case AMDGPU::VS_64RegClassID:
2778 case AMDGPU::SReg_64RegClassID:
2779 case AMDGPU::VReg_64RegClassID:
2780 case AMDGPU::AReg_64RegClassID:
2781 case AMDGPU::SReg_64_XEXECRegClassID:
2782 case AMDGPU::VReg_64_Align2RegClassID:
2783 case AMDGPU::AReg_64_Align2RegClassID:
2784 case AMDGPU::AV_64RegClassID:
2785 case AMDGPU::AV_64_Align2RegClassID:
2786 case AMDGPU::VReg_64_Lo256_Align2RegClassID:
2787 case AMDGPU::VS_64_Lo256RegClassID:
2788 return 64;
2789 case AMDGPU::SGPR_96RegClassID:
2790 case AMDGPU::SReg_96RegClassID:
2791 case AMDGPU::VReg_96RegClassID:
2792 case AMDGPU::AReg_96RegClassID:
2793 case AMDGPU::VReg_96_Align2RegClassID:
2794 case AMDGPU::AReg_96_Align2RegClassID:
2795 case AMDGPU::AV_96RegClassID:
2796 case AMDGPU::AV_96_Align2RegClassID:
2797 case AMDGPU::VReg_96_Lo256_Align2RegClassID:
2798 return 96;
2799 case AMDGPU::SGPR_128RegClassID:
2800 case AMDGPU::SReg_128RegClassID:
2801 case AMDGPU::VReg_128RegClassID:
2802 case AMDGPU::AReg_128RegClassID:
2803 case AMDGPU::VReg_128_Align2RegClassID:
2804 case AMDGPU::AReg_128_Align2RegClassID:
2805 case AMDGPU::AV_128RegClassID:
2806 case AMDGPU::AV_128_Align2RegClassID:
2807 case AMDGPU::SReg_128_XNULLRegClassID:
2808 case AMDGPU::VReg_128_Lo256_Align2RegClassID:
2809 return 128;
2810 case AMDGPU::SGPR_160RegClassID:
2811 case AMDGPU::SReg_160RegClassID:
2812 case AMDGPU::VReg_160RegClassID:
2813 case AMDGPU::AReg_160RegClassID:
2814 case AMDGPU::VReg_160_Align2RegClassID:
2815 case AMDGPU::AReg_160_Align2RegClassID:
2816 case AMDGPU::AV_160RegClassID:
2817 case AMDGPU::AV_160_Align2RegClassID:
2818 case AMDGPU::VReg_160_Lo256_Align2RegClassID:
2819 return 160;
2820 case AMDGPU::SGPR_192RegClassID:
2821 case AMDGPU::SReg_192RegClassID:
2822 case AMDGPU::VReg_192RegClassID:
2823 case AMDGPU::AReg_192RegClassID:
2824 case AMDGPU::VReg_192_Align2RegClassID:
2825 case AMDGPU::AReg_192_Align2RegClassID:
2826 case AMDGPU::AV_192RegClassID:
2827 case AMDGPU::AV_192_Align2RegClassID:
2828 case AMDGPU::VReg_192_Lo256_Align2RegClassID:
2829 return 192;
2830 case AMDGPU::SGPR_224RegClassID:
2831 case AMDGPU::SReg_224RegClassID:
2832 case AMDGPU::VReg_224RegClassID:
2833 case AMDGPU::AReg_224RegClassID:
2834 case AMDGPU::VReg_224_Align2RegClassID:
2835 case AMDGPU::AReg_224_Align2RegClassID:
2836 case AMDGPU::AV_224RegClassID:
2837 case AMDGPU::AV_224_Align2RegClassID:
2838 case AMDGPU::VReg_224_Lo256_Align2RegClassID:
2839 return 224;
2840 case AMDGPU::SGPR_256RegClassID:
2841 case AMDGPU::SReg_256RegClassID:
2842 case AMDGPU::VReg_256RegClassID:
2843 case AMDGPU::AReg_256RegClassID:
2844 case AMDGPU::VReg_256_Align2RegClassID:
2845 case AMDGPU::AReg_256_Align2RegClassID:
2846 case AMDGPU::AV_256RegClassID:
2847 case AMDGPU::AV_256_Align2RegClassID:
2848 case AMDGPU::SReg_256_XNULLRegClassID:
2849 case AMDGPU::VReg_256_Lo256_Align2RegClassID:
2850 return 256;
2851 case AMDGPU::SGPR_288RegClassID:
2852 case AMDGPU::SReg_288RegClassID:
2853 case AMDGPU::VReg_288RegClassID:
2854 case AMDGPU::AReg_288RegClassID:
2855 case AMDGPU::VReg_288_Align2RegClassID:
2856 case AMDGPU::AReg_288_Align2RegClassID:
2857 case AMDGPU::AV_288RegClassID:
2858 case AMDGPU::AV_288_Align2RegClassID:
2859 case AMDGPU::VReg_288_Lo256_Align2RegClassID:
2860 return 288;
2861 case AMDGPU::SGPR_320RegClassID:
2862 case AMDGPU::SReg_320RegClassID:
2863 case AMDGPU::VReg_320RegClassID:
2864 case AMDGPU::AReg_320RegClassID:
2865 case AMDGPU::VReg_320_Align2RegClassID:
2866 case AMDGPU::AReg_320_Align2RegClassID:
2867 case AMDGPU::AV_320RegClassID:
2868 case AMDGPU::AV_320_Align2RegClassID:
2869 case AMDGPU::VReg_320_Lo256_Align2RegClassID:
2870 return 320;
2871 case AMDGPU::SGPR_352RegClassID:
2872 case AMDGPU::SReg_352RegClassID:
2873 case AMDGPU::VReg_352RegClassID:
2874 case AMDGPU::AReg_352RegClassID:
2875 case AMDGPU::VReg_352_Align2RegClassID:
2876 case AMDGPU::AReg_352_Align2RegClassID:
2877 case AMDGPU::AV_352RegClassID:
2878 case AMDGPU::AV_352_Align2RegClassID:
2879 case AMDGPU::VReg_352_Lo256_Align2RegClassID:
2880 return 352;
2881 case AMDGPU::SGPR_384RegClassID:
2882 case AMDGPU::SReg_384RegClassID:
2883 case AMDGPU::VReg_384RegClassID:
2884 case AMDGPU::AReg_384RegClassID:
2885 case AMDGPU::VReg_384_Align2RegClassID:
2886 case AMDGPU::AReg_384_Align2RegClassID:
2887 case AMDGPU::AV_384RegClassID:
2888 case AMDGPU::AV_384_Align2RegClassID:
2889 case AMDGPU::VReg_384_Lo256_Align2RegClassID:
2890 return 384;
2891 case AMDGPU::SGPR_512RegClassID:
2892 case AMDGPU::SReg_512RegClassID:
2893 case AMDGPU::VReg_512RegClassID:
2894 case AMDGPU::AReg_512RegClassID:
2895 case AMDGPU::VReg_512_Align2RegClassID:
2896 case AMDGPU::AReg_512_Align2RegClassID:
2897 case AMDGPU::AV_512RegClassID:
2898 case AMDGPU::AV_512_Align2RegClassID:
2899 case AMDGPU::VReg_512_Lo256_Align2RegClassID:
2900 return 512;
2901 case AMDGPU::SGPR_1024RegClassID:
2902 case AMDGPU::SReg_1024RegClassID:
2903 case AMDGPU::VReg_1024RegClassID:
2904 case AMDGPU::AReg_1024RegClassID:
2905 case AMDGPU::VReg_1024_Align2RegClassID:
2906 case AMDGPU::AReg_1024_Align2RegClassID:
2907 case AMDGPU::AV_1024RegClassID:
2908 case AMDGPU::AV_1024_Align2RegClassID:
2909 case AMDGPU::VReg_1024_Lo256_Align2RegClassID:
2910 return 1024;
2911 default:
2912 llvm_unreachable("Unexpected register class");
2913 }
2914}
2915
2916unsigned getRegBitWidth(const MCRegisterClass &RC) {
2917 return getRegBitWidth(RCID: RC.getID());
2918}
2919
2920bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
2921 if (isInlinableIntLiteral(Literal))
2922 return true;
2923
2924 uint64_t Val = static_cast<uint64_t>(Literal);
2925 return (Val == llvm::bit_cast<uint64_t>(from: 0.0)) ||
2926 (Val == llvm::bit_cast<uint64_t>(from: 1.0)) ||
2927 (Val == llvm::bit_cast<uint64_t>(from: -1.0)) ||
2928 (Val == llvm::bit_cast<uint64_t>(from: 0.5)) ||
2929 (Val == llvm::bit_cast<uint64_t>(from: -0.5)) ||
2930 (Val == llvm::bit_cast<uint64_t>(from: 2.0)) ||
2931 (Val == llvm::bit_cast<uint64_t>(from: -2.0)) ||
2932 (Val == llvm::bit_cast<uint64_t>(from: 4.0)) ||
2933 (Val == llvm::bit_cast<uint64_t>(from: -4.0)) ||
2934 (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
2935}
2936
2937bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
2938 if (isInlinableIntLiteral(Literal))
2939 return true;
2940
2941 // The actual type of the operand does not seem to matter as long
2942 // as the bits match one of the inline immediate values. For example:
2943 //
2944 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
2945 // so it is a legal inline immediate.
2946 //
2947 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
2948 // floating-point, so it is a legal inline immediate.
2949
2950 uint32_t Val = static_cast<uint32_t>(Literal);
2951 return (Val == llvm::bit_cast<uint32_t>(from: 0.0f)) ||
2952 (Val == llvm::bit_cast<uint32_t>(from: 1.0f)) ||
2953 (Val == llvm::bit_cast<uint32_t>(from: -1.0f)) ||
2954 (Val == llvm::bit_cast<uint32_t>(from: 0.5f)) ||
2955 (Val == llvm::bit_cast<uint32_t>(from: -0.5f)) ||
2956 (Val == llvm::bit_cast<uint32_t>(from: 2.0f)) ||
2957 (Val == llvm::bit_cast<uint32_t>(from: -2.0f)) ||
2958 (Val == llvm::bit_cast<uint32_t>(from: 4.0f)) ||
2959 (Val == llvm::bit_cast<uint32_t>(from: -4.0f)) ||
2960 (Val == 0x3e22f983 && HasInv2Pi);
2961}
2962
2963bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi) {
2964 if (!HasInv2Pi)
2965 return false;
2966 if (isInlinableIntLiteral(Literal))
2967 return true;
2968 uint16_t Val = static_cast<uint16_t>(Literal);
2969 return Val == 0x3F00 || // 0.5
2970 Val == 0xBF00 || // -0.5
2971 Val == 0x3F80 || // 1.0
2972 Val == 0xBF80 || // -1.0
2973 Val == 0x4000 || // 2.0
2974 Val == 0xC000 || // -2.0
2975 Val == 0x4080 || // 4.0
2976 Val == 0xC080 || // -4.0
2977 Val == 0x3E22; // 1.0 / (2.0 * pi)
2978}
2979
2980bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi) {
2981 return isInlinableLiteral32(Literal, HasInv2Pi);
2982}
2983
2984bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) {
2985 if (!HasInv2Pi)
2986 return false;
2987 if (isInlinableIntLiteral(Literal))
2988 return true;
2989 uint16_t Val = static_cast<uint16_t>(Literal);
2990 return Val == 0x3C00 || // 1.0
2991 Val == 0xBC00 || // -1.0
2992 Val == 0x3800 || // 0.5
2993 Val == 0xB800 || // -0.5
2994 Val == 0x4000 || // 2.0
2995 Val == 0xC000 || // -2.0
2996 Val == 0x4400 || // 4.0
2997 Val == 0xC400 || // -4.0
2998 Val == 0x3118; // 1/2pi
2999}
3000
3001std::optional<unsigned> getInlineEncodingV216(bool IsFloat, uint32_t Literal) {
3002 // Unfortunately, the Instruction Set Architecture Reference Guide is
3003 // misleading about how the inline operands work for (packed) 16-bit
3004 // instructions. In a nutshell, the actual HW behavior is:
3005 //
3006 // - integer encodings (-16 .. 64) are always produced as sign-extended
3007 // 32-bit values
3008 // - float encodings are produced as:
3009 // - for F16 instructions: corresponding half-precision float values in
3010 // the LSBs, 0 in the MSBs
3011 // - for UI16 instructions: corresponding single-precision float value
3012 int32_t Signed = static_cast<int32_t>(Literal);
3013 if (Signed >= 0 && Signed <= 64)
3014 return 128 + Signed;
3015
3016 if (Signed >= -16 && Signed <= -1)
3017 return 192 + std::abs(x: Signed);
3018
3019 if (IsFloat) {
3020 // clang-format off
3021 switch (Literal) {
3022 case 0x3800: return 240; // 0.5
3023 case 0xB800: return 241; // -0.5
3024 case 0x3C00: return 242; // 1.0
3025 case 0xBC00: return 243; // -1.0
3026 case 0x4000: return 244; // 2.0
3027 case 0xC000: return 245; // -2.0
3028 case 0x4400: return 246; // 4.0
3029 case 0xC400: return 247; // -4.0
3030 case 0x3118: return 248; // 1.0 / (2.0 * pi)
3031 default: break;
3032 }
3033 // clang-format on
3034 } else {
3035 // clang-format off
3036 switch (Literal) {
3037 case 0x3F000000: return 240; // 0.5
3038 case 0xBF000000: return 241; // -0.5
3039 case 0x3F800000: return 242; // 1.0
3040 case 0xBF800000: return 243; // -1.0
3041 case 0x40000000: return 244; // 2.0
3042 case 0xC0000000: return 245; // -2.0
3043 case 0x40800000: return 246; // 4.0
3044 case 0xC0800000: return 247; // -4.0
3045 case 0x3E22F983: return 248; // 1.0 / (2.0 * pi)
3046 default: break;
3047 }
3048 // clang-format on
3049 }
3050
3051 return {};
3052}
3053
3054// Encoding of the literal as an inline constant for a V_PK_*_IU16 instruction
3055// or nullopt.
3056std::optional<unsigned> getInlineEncodingV2I16(uint32_t Literal) {
3057 return getInlineEncodingV216(IsFloat: false, Literal);
3058}
3059
3060// Encoding of the literal as an inline constant for a V_PK_*_BF16 instruction
3061// or nullopt.
3062std::optional<unsigned> getInlineEncodingV2BF16(uint32_t Literal) {
3063 int32_t Signed = static_cast<int32_t>(Literal);
3064 if (Signed >= 0 && Signed <= 64)
3065 return 128 + Signed;
3066
3067 if (Signed >= -16 && Signed <= -1)
3068 return 192 + std::abs(x: Signed);
3069
3070 // clang-format off
3071 switch (Literal) {
3072 case 0x3F00: return 240; // 0.5
3073 case 0xBF00: return 241; // -0.5
3074 case 0x3F80: return 242; // 1.0
3075 case 0xBF80: return 243; // -1.0
3076 case 0x4000: return 244; // 2.0
3077 case 0xC000: return 245; // -2.0
3078 case 0x4080: return 246; // 4.0
3079 case 0xC080: return 247; // -4.0
3080 case 0x3E22: return 248; // 1.0 / (2.0 * pi)
3081 default: break;
3082 }
3083 // clang-format on
3084
3085 return std::nullopt;
3086}
3087
3088// Encoding of the literal as an inline constant for a V_PK_*_F16 instruction
3089// or nullopt.
3090std::optional<unsigned> getInlineEncodingV2F16(uint32_t Literal) {
3091 return getInlineEncodingV216(IsFloat: true, Literal);
3092}
3093
3094// Encoding of the literal as an inline constant for V_PK_FMAC_F16 instruction
3095// or nullopt. This accounts for different inline constant behavior:
3096// - Pre-GFX11: fp16 inline constants have the value in low 16 bits, 0 in high
3097// - GFX11+: fp16 inline constants are duplicated into both halves
3098std::optional<unsigned> getPKFMACF16InlineEncoding(uint32_t Literal,
3099 bool IsGFX11Plus) {
3100 // Pre-GFX11 behavior: f16 in low bits, 0 in high bits
3101 if (!IsGFX11Plus)
3102 return getInlineEncodingV216(/*IsFloat=*/true, Literal);
3103
3104 // GFX11+ behavior: f16 duplicated in both halves
3105 // First, check for sign-extended integer inline constants (-16 to 64)
3106 // These work the same across all generations
3107 int32_t Signed = static_cast<int32_t>(Literal);
3108 if (Signed >= 0 && Signed <= 64)
3109 return 128 + Signed;
3110
3111 if (Signed >= -16 && Signed <= -1)
3112 return 192 + std::abs(x: Signed);
3113
3114 // For float inline constants on GFX11+, both halves must be equal
3115 uint16_t Lo = static_cast<uint16_t>(Literal);
3116 uint16_t Hi = static_cast<uint16_t>(Literal >> 16);
3117 if (Lo != Hi)
3118 return std::nullopt;
3119 return getInlineEncodingV216(/*IsFloat=*/true, Literal: Lo);
3120}
3121
3122// Whether the given literal can be inlined for a V_PK_* instruction.
3123bool isInlinableLiteralV216(uint32_t Literal, uint8_t OpType) {
3124 switch (OpType) {
3125 case AMDGPU::OPERAND_REG_IMM_V2INT16:
3126 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
3127 return getInlineEncodingV216(IsFloat: false, Literal).has_value();
3128 case AMDGPU::OPERAND_REG_IMM_V2FP16:
3129 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
3130 return getInlineEncodingV216(IsFloat: true, Literal).has_value();
3131 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
3132 llvm_unreachable("OPERAND_REG_IMM_V2FP16_SPLAT is not supported");
3133 case AMDGPU::OPERAND_REG_IMM_V2BF16:
3134 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
3135 return isInlinableLiteralV2BF16(Literal);
3136 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
3137 return false;
3138 default:
3139 llvm_unreachable("bad packed operand type");
3140 }
3141}
3142
3143// Whether the given literal can be inlined for a V_PK_*_IU16 instruction.
3144bool isInlinableLiteralV2I16(uint32_t Literal) {
3145 return getInlineEncodingV2I16(Literal).has_value();
3146}
3147
3148// Whether the given literal can be inlined for a V_PK_*_BF16 instruction.
3149bool isInlinableLiteralV2BF16(uint32_t Literal) {
3150 return getInlineEncodingV2BF16(Literal).has_value();
3151}
3152
3153// Whether the given literal can be inlined for a V_PK_*_F16 instruction.
3154bool isInlinableLiteralV2F16(uint32_t Literal) {
3155 return getInlineEncodingV2F16(Literal).has_value();
3156}
3157
3158// Whether the given literal can be inlined for V_PK_FMAC_F16 instruction.
3159bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus) {
3160 return getPKFMACF16InlineEncoding(Literal, IsGFX11Plus).has_value();
3161}
3162
3163bool isValid32BitLiteral(uint64_t Val, bool IsFP64) {
3164 if (IsFP64)
3165 return !Lo_32(Value: Val);
3166
3167 return isUInt<32>(x: Val) || isInt<32>(x: Val);
3168}
3169
3170int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit) {
3171 switch (Type) {
3172 default:
3173 break;
3174 case OPERAND_REG_IMM_BF16:
3175 case OPERAND_REG_IMM_FP16:
3176 case OPERAND_REG_IMM_NOINLINE_FP16:
3177 case OPERAND_REG_INLINE_C_BF16:
3178 case OPERAND_REG_INLINE_C_FP16:
3179 return Imm & 0xffff;
3180 case OPERAND_INLINE_SPLIT_BARRIER_INT32:
3181 case OPERAND_REG_IMM_FP32:
3182 case OPERAND_REG_IMM_INT32:
3183 case OPERAND_REG_IMM_V2BF16:
3184 case OPERAND_REG_IMM_V2FP16:
3185 case OPERAND_REG_IMM_V2FP16_SPLAT:
3186 case OPERAND_REG_IMM_V2FP32:
3187 case OPERAND_REG_IMM_V2INT16:
3188 case OPERAND_REG_IMM_V2INT32:
3189 case OPERAND_REG_INLINE_AC_FP32:
3190 case OPERAND_REG_INLINE_AC_INT32:
3191 case OPERAND_REG_INLINE_C_FP32:
3192 case OPERAND_REG_INLINE_C_INT32:
3193 return Lo_32(Value: Imm);
3194 case OPERAND_REG_IMM_FP64:
3195 case AMDGPU::OPERAND_REG_IMM_V2FP64:
3196 return IsLit ? Imm : Hi_32(Value: Imm);
3197 }
3198 return Imm;
3199}
3200
3201bool isArgPassedInSGPR(const Argument *A) {
3202 const Function *F = A->getParent();
3203
3204 // Arguments to compute shaders are never a source of divergence.
3205 CallingConv::ID CC = F->getCallingConv();
3206 switch (CC) {
3207 case CallingConv::AMDGPU_KERNEL:
3208 case CallingConv::SPIR_KERNEL:
3209 return true;
3210 case CallingConv::AMDGPU_VS:
3211 case CallingConv::AMDGPU_LS:
3212 case CallingConv::AMDGPU_HS:
3213 case CallingConv::AMDGPU_ES:
3214 case CallingConv::AMDGPU_GS:
3215 case CallingConv::AMDGPU_PS:
3216 case CallingConv::AMDGPU_CS:
3217 case CallingConv::AMDGPU_Gfx:
3218 case CallingConv::AMDGPU_CS_Chain:
3219 case CallingConv::AMDGPU_CS_ChainPreserve:
3220 // For non-compute shaders, SGPR inputs are marked with either inreg or
3221 // byval. Everything else is in VGPRs.
3222 return A->hasAttribute(Kind: Attribute::InReg) ||
3223 A->hasAttribute(Kind: Attribute::ByVal);
3224 default:
3225 // TODO: treat i1 as divergent?
3226 return A->hasAttribute(Kind: Attribute::InReg);
3227 }
3228}
3229
3230bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo) {
3231 // Arguments to compute shaders are never a source of divergence.
3232 CallingConv::ID CC = CB->getCallingConv();
3233 switch (CC) {
3234 case CallingConv::AMDGPU_KERNEL:
3235 case CallingConv::SPIR_KERNEL:
3236 return true;
3237 case CallingConv::AMDGPU_VS:
3238 case CallingConv::AMDGPU_LS:
3239 case CallingConv::AMDGPU_HS:
3240 case CallingConv::AMDGPU_ES:
3241 case CallingConv::AMDGPU_GS:
3242 case CallingConv::AMDGPU_PS:
3243 case CallingConv::AMDGPU_CS:
3244 case CallingConv::AMDGPU_Gfx:
3245 case CallingConv::AMDGPU_CS_Chain:
3246 case CallingConv::AMDGPU_CS_ChainPreserve:
3247 // For non-compute shaders, SGPR inputs are marked with either inreg or
3248 // byval. Everything else is in VGPRs.
3249 return CB->paramHasAttr(ArgNo, Kind: Attribute::InReg) ||
3250 CB->isByValArgument(ArgNo);
3251 default:
3252 return CB->paramHasAttr(ArgNo, Kind: Attribute::InReg);
3253 }
3254}
3255
3256static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
3257 return isGCN3Encoding(STI: ST) || isGFX10Plus(STI: ST);
3258}
3259
3260bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
3261 int64_t EncodedOffset) {
3262 if (isGFX12Plus(STI: ST))
3263 return isUInt<23>(x: EncodedOffset);
3264
3265 return hasSMEMByteOffset(ST) ? isUInt<20>(x: EncodedOffset)
3266 : isUInt<8>(x: EncodedOffset);
3267}
3268
3269bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
3270 int64_t EncodedOffset, bool IsBuffer) {
3271 if (isGFX12Plus(STI: ST)) {
3272 if (IsBuffer && EncodedOffset < 0)
3273 return false;
3274 return isInt<24>(x: EncodedOffset);
3275 }
3276
3277 return !IsBuffer && hasSMRDSignedImmOffset(ST) && isInt<21>(x: EncodedOffset);
3278}
3279
3280static bool isDwordAligned(uint64_t ByteOffset) {
3281 return (ByteOffset & 3) == 0;
3282}
3283
3284uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
3285 uint64_t ByteOffset) {
3286 if (hasSMEMByteOffset(ST))
3287 return ByteOffset;
3288
3289 assert(isDwordAligned(ByteOffset));
3290 return ByteOffset >> 2;
3291}
3292
3293std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
3294 int64_t ByteOffset, bool IsBuffer,
3295 bool HasSOffset) {
3296 // For unbuffered smem loads, it is illegal for the Immediate Offset to be
3297 // negative if the resulting (Offset + (M0 or SOffset or zero) is negative.
3298 // Handle case where SOffset is not present.
3299 if (!IsBuffer && !HasSOffset && ByteOffset < 0 && hasSMRDSignedImmOffset(ST))
3300 return std::nullopt;
3301
3302 if (isGFX12Plus(STI: ST)) // 24 bit signed offsets
3303 return isInt<24>(x: ByteOffset) ? std::optional<int64_t>(ByteOffset)
3304 : std::nullopt;
3305
3306 // The signed version is always a byte offset.
3307 if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
3308 assert(hasSMEMByteOffset(ST));
3309 return isInt<20>(x: ByteOffset) ? std::optional<int64_t>(ByteOffset)
3310 : std::nullopt;
3311 }
3312
3313 if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
3314 return std::nullopt;
3315
3316 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3317 return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
3318 ? std::optional<int64_t>(EncodedOffset)
3319 : std::nullopt;
3320}
3321
3322std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
3323 int64_t ByteOffset) {
3324 if (!isCI(STI: ST) || !isDwordAligned(ByteOffset))
3325 return std::nullopt;
3326
3327 int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
3328 return isUInt<32>(x: EncodedOffset) ? std::optional<int64_t>(EncodedOffset)
3329 : std::nullopt;
3330}
3331
3332unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST) {
3333 if (ST.getFeatureBits().test(I: FeatureFlatOffsetBits12))
3334 return 12;
3335 if (ST.getFeatureBits().test(I: FeatureFlatOffsetBits24))
3336 return 24;
3337 return 13;
3338}
3339
3340namespace {
3341
3342struct SourceOfDivergence {
3343 unsigned Intr;
3344};
3345const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);
3346
3347struct AlwaysUniform {
3348 unsigned Intr;
3349};
3350const AlwaysUniform *lookupAlwaysUniform(unsigned Intr);
3351
3352struct LDSDMAIntrinsic {
3353 unsigned Intr;
3354};
3355const LDSDMAIntrinsic *lookupLDSDMAIntrinsic(unsigned Intr);
3356
3357#define GET_SourcesOfDivergence_IMPL
3358#define GET_UniformIntrinsics_IMPL
3359#define GET_LDSDMAIntrinsicTable_IMPL
3360#define GET_Gfx9BufferFormat_IMPL
3361#define GET_Gfx10BufferFormat_IMPL
3362#define GET_Gfx11PlusBufferFormat_IMPL
3363
3364#include "AMDGPUGenSearchableTables.inc"
3365
3366} // end anonymous namespace
3367
3368bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
3369 return lookupSourceOfDivergence(Intr: IntrID);
3370}
3371
3372bool isIntrinsicAlwaysUniform(unsigned IntrID) {
3373 return lookupAlwaysUniform(Intr: IntrID);
3374}
3375
3376bool isLDSDMAIntrinsic(unsigned IntrID) {
3377 return lookupLDSDMAIntrinsic(Intr: IntrID);
3378}
3379
3380const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
3381 uint8_t NumComponents,
3382 uint8_t NumFormat,
3383 const MCSubtargetInfo &STI) {
3384 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(
3385 BitsPerComp, NumComponents, NumFormat)
3386 : isGFX10(STI)
3387 ? getGfx10BufferFormatInfo(BitsPerComp, NumComponents, NumFormat)
3388 : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
3389}
3390
3391const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
3392 const MCSubtargetInfo &STI) {
3393 return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(Format)
3394 : isGFX10(STI) ? getGfx10BufferFormatInfo(Format)
3395 : getGfx9BufferFormatInfo(Format);
3396}
3397
3398const MCRegisterClass *getVGPRPhysRegClass(MCRegister Reg,
3399 const MCRegisterInfo &MRI) {
3400 const unsigned VGPRClasses[] = {
3401 AMDGPU::VGPR_16RegClassID, AMDGPU::VGPR_32RegClassID,
3402 AMDGPU::VReg_64RegClassID, AMDGPU::VReg_96RegClassID,
3403 AMDGPU::VReg_128RegClassID, AMDGPU::VReg_160RegClassID,
3404 AMDGPU::VReg_192RegClassID, AMDGPU::VReg_224RegClassID,
3405 AMDGPU::VReg_256RegClassID, AMDGPU::VReg_288RegClassID,
3406 AMDGPU::VReg_320RegClassID, AMDGPU::VReg_352RegClassID,
3407 AMDGPU::VReg_384RegClassID, AMDGPU::VReg_512RegClassID,
3408 AMDGPU::VReg_1024RegClassID};
3409
3410 for (unsigned RCID : VGPRClasses) {
3411 const MCRegisterClass &RC = MRI.getRegClass(i: RCID);
3412 if (RC.contains(Reg))
3413 return &RC;
3414 }
3415
3416 return nullptr;
3417}
3418
3419unsigned getVGPREncodingMSBs(MCRegister Reg, const MCRegisterInfo &MRI) {
3420 unsigned Enc = MRI.getEncodingValue(Reg);
3421 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3422 return Idx >> 8;
3423}
3424
3425MCRegister getVGPRWithMSBs(MCRegister Reg, unsigned MSBs,
3426 const MCRegisterInfo &MRI) {
3427 unsigned Enc = MRI.getEncodingValue(Reg);
3428 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
3429 if (Idx >= 0x100)
3430 return MCRegister();
3431
3432 const MCRegisterClass *RC = getVGPRPhysRegClass(Reg, MRI);
3433 if (!RC)
3434 return MCRegister();
3435
3436 Idx |= MSBs << 8;
3437 if (RC->getID() == AMDGPU::VGPR_16RegClassID) {
3438 // This class has 2048 registers with interleaved lo16 and hi16.
3439 Idx *= 2;
3440 if (Enc & AMDGPU::HWEncoding::IS_HI16)
3441 ++Idx;
3442 }
3443
3444 return RC->getRegister(i: Idx);
3445}
3446
3447static std::optional<unsigned>
3448convertSetRegImmToVgprMSBs(uint64_t Imm, uint64_t Simm16,
3449 bool HasSetregVGPRMSBFixup) {
3450 constexpr unsigned VGPRMSBShift =
3451 llvm::countr_zero_constexpr<unsigned>(Val: AMDGPU::Hwreg::DST_VGPR_MSB);
3452
3453 auto [HwRegId, Offset, Size] = Hwreg::HwregEncoding::decode(Encoded: Simm16);
3454 if (HwRegId != Hwreg::ID_MODE ||
3455 (!HasSetregVGPRMSBFixup && (Offset + Size) < VGPRMSBShift))
3456 return {};
3457 // If there is SetregVGPRMSBFixup then Offset is ignored.
3458 if (!HasSetregVGPRMSBFixup)
3459 Imm <<= Offset;
3460 Imm = (Imm & Hwreg::VGPR_MSB_MASK) >> VGPRMSBShift;
3461 if (!HasSetregVGPRMSBFixup)
3462 Imm &= llvm::maskTrailingOnes<unsigned>(N: Size);
3463 return llvm::rotr<uint8_t>(V: static_cast<uint8_t>(Imm), /*R=*/2);
3464}
3465
3466std::optional<unsigned> convertSetRegImmToVgprMSBs(const MachineInstr &MI,
3467 bool HasSetregVGPRMSBFixup) {
3468 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32);
3469 return convertSetRegImmToVgprMSBs(Imm: MI.getOperand(i: 0).getImm(),
3470 Simm16: MI.getOperand(i: 1).getImm(),
3471 HasSetregVGPRMSBFixup);
3472}
3473
3474std::optional<unsigned> convertSetRegImmToVgprMSBs(const MCInst &MI,
3475 bool HasSetregVGPRMSBFixup) {
3476 assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_gfx12);
3477 return convertSetRegImmToVgprMSBs(Imm: MI.getOperand(i: 0).getImm(),
3478 Simm16: MI.getOperand(i: 1).getImm(),
3479 HasSetregVGPRMSBFixup);
3480}
3481
3482std::pair<const AMDGPU::OpName *, const AMDGPU::OpName *>
3483getVGPRLoweringOperandTables(const MCInstrDesc &Desc) {
3484 static const AMDGPU::OpName VOPOps[4] = {
3485 AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2,
3486 AMDGPU::OpName::vdst};
3487 static const AMDGPU::OpName VDSOps[4] = {
3488 AMDGPU::OpName::addr, AMDGPU::OpName::data0, AMDGPU::OpName::data1,
3489 AMDGPU::OpName::vdst};
3490 static const AMDGPU::OpName FLATOps[4] = {
3491 AMDGPU::OpName::vaddr, AMDGPU::OpName::vdata,
3492 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdst};
3493 static const AMDGPU::OpName BUFOps[4] = {
3494 AMDGPU::OpName::vaddr, AMDGPU::OpName::NUM_OPERAND_NAMES,
3495 AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdata};
3496 static const AMDGPU::OpName VIMGOps[4] = {
3497 AMDGPU::OpName::vaddr0, AMDGPU::OpName::vaddr1, AMDGPU::OpName::vaddr2,
3498 AMDGPU::OpName::vdata};
3499
3500 // For VOPD instructions MSB of a corresponding Y component operand VGPR
3501 // address is supposed to match X operand, otherwise VOPD shall not be
3502 // combined.
3503 static const AMDGPU::OpName VOPDOpsX[4] = {
3504 AMDGPU::OpName::src0X, AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vsrc2X,
3505 AMDGPU::OpName::vdstX};
3506 static const AMDGPU::OpName VOPDOpsY[4] = {
3507 AMDGPU::OpName::src0Y, AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vsrc2Y,
3508 AMDGPU::OpName::vdstY};
3509
3510 // VOP2 MADMK instructions use src0, imm, src1 scheme.
3511 static const AMDGPU::OpName VOP2MADMKOps[4] = {
3512 AMDGPU::OpName::src0, AMDGPU::OpName::NUM_OPERAND_NAMES,
3513 AMDGPU::OpName::src1, AMDGPU::OpName::vdst};
3514 static const AMDGPU::OpName VOPDFMAMKOpsX[4] = {
3515 AMDGPU::OpName::src0X, AMDGPU::OpName::NUM_OPERAND_NAMES,
3516 AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vdstX};
3517 static const AMDGPU::OpName VOPDFMAMKOpsY[4] = {
3518 AMDGPU::OpName::src0Y, AMDGPU::OpName::NUM_OPERAND_NAMES,
3519 AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vdstY};
3520
3521 if (SIInstrFlags::isVOP1(O: Desc) || SIInstrFlags::isVOP2(O: Desc) ||
3522 SIInstrFlags::isVOP3Like(O: Desc) || SIInstrFlags::isVOPC(O: Desc) ||
3523 SIInstrFlags::isDPP(O: Desc)) {
3524 switch (Desc.getOpcode()) {
3525 // LD_SCALE operands ignore MSB.
3526 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32:
3527 case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32_gfx1250:
3528 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64:
3529 case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64_gfx1250:
3530 return {};
3531 case AMDGPU::V_FMAMK_F16:
3532 case AMDGPU::V_FMAMK_F16_t16:
3533 case AMDGPU::V_FMAMK_F16_t16_gfx12:
3534 case AMDGPU::V_FMAMK_F16_fake16:
3535 case AMDGPU::V_FMAMK_F16_fake16_gfx12:
3536 case AMDGPU::V_FMAMK_F32:
3537 case AMDGPU::V_FMAMK_F32_gfx12:
3538 case AMDGPU::V_FMAMK_F64:
3539 case AMDGPU::V_FMAMK_F64_gfx1250:
3540 return {VOP2MADMKOps, nullptr};
3541 default:
3542 break;
3543 }
3544 return {VOPOps, nullptr};
3545 }
3546
3547 if (SIInstrFlags::isDS(O: Desc))
3548 return {VDSOps, nullptr};
3549
3550 if (SIInstrFlags::isFLAT(O: Desc))
3551 return {FLATOps, nullptr};
3552
3553 if (SIInstrFlags::isBuffer(O: Desc))
3554 return {BUFOps, nullptr};
3555
3556 if (SIInstrFlags::isVIMAGE(O: Desc))
3557 return {VIMGOps, nullptr};
3558
3559 if (AMDGPU::isVOPD(Opc: Desc.getOpcode())) {
3560 auto [OpX, OpY] = getVOPDComponents(VOPDOpcode: Desc.getOpcode());
3561 return {(OpX == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsX : VOPDOpsX,
3562 (OpY == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsY : VOPDOpsY};
3563 }
3564
3565 assert(!SIInstrFlags::isMIMG(Desc));
3566
3567 if (SIInstrFlags::isVSAMPLE(O: Desc) || SIInstrFlags::isEXP(O: Desc))
3568 llvm_unreachable("Sample and export VGPR lowering is not implemented and"
3569 " these instructions are not expected on gfx1250");
3570
3571 return {};
3572}
3573
3574bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode) {
3575 const MCInstrDesc &Desc = MII.get(Opcode);
3576 if (SIInstrFlags::isSMRD(O: Desc))
3577 return Desc.mayLoad() && !Desc.mayStore() && !getSMEMIsBuffer(Opc: Opcode);
3578 if (!SIInstrFlags::isFLAT(O: Desc))
3579 return false;
3580
3581 // Only SV and SVS modes are supported.
3582 if (SIInstrFlags::isFlatScratch(O: MII, O: Opcode))
3583 return hasNamedOperand(Opcode, NamedIdx: OpName::vaddr);
3584
3585 // Only GVS mode is supported.
3586 return hasNamedOperand(Opcode, NamedIdx: OpName::vaddr) &&
3587 hasNamedOperand(Opcode, NamedIdx: OpName::saddr);
3588
3589 return false;
3590}
3591
3592static bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc,
3593 const MCInstrInfo &MII,
3594 const MCSubtargetInfo &ST) {
3595 for (auto OpName : {OpName::vdst, OpName::src0, OpName::src1, OpName::src2}) {
3596 int Idx = getNamedOperandIdx(Opcode: OpDesc.getOpcode(), Name: OpName);
3597 if (Idx == -1)
3598 continue;
3599
3600 const MCOperandInfo &OpInfo = OpDesc.operands()[Idx];
3601 int16_t RegClass = MII.getOpRegClassID(
3602 OpInfo, HwModeId: ST.getHwMode(type: MCSubtargetInfo::HwMode_RegInfo));
3603 if (RegClass == AMDGPU::VReg_64RegClassID ||
3604 RegClass == AMDGPU::VReg_64_Align2RegClassID)
3605 return true;
3606 }
3607
3608 return false;
3609}
3610
3611bool isDPALU_DPP32BitOpc(unsigned Opc) {
3612 switch (Opc) {
3613 case AMDGPU::V_MUL_LO_U32_e64:
3614 case AMDGPU::V_MUL_LO_U32_e64_dpp:
3615 case AMDGPU::V_MUL_LO_U32_e64_dpp_gfx1250:
3616 case AMDGPU::V_MUL_HI_U32_e64:
3617 case AMDGPU::V_MUL_HI_U32_e64_dpp:
3618 case AMDGPU::V_MUL_HI_U32_e64_dpp_gfx1250:
3619 case AMDGPU::V_MUL_HI_I32_e64:
3620 case AMDGPU::V_MUL_HI_I32_e64_dpp:
3621 case AMDGPU::V_MUL_HI_I32_e64_dpp_gfx1250:
3622 case AMDGPU::V_MAD_U32_e64:
3623 case AMDGPU::V_MAD_U32_e64_dpp:
3624 case AMDGPU::V_MAD_U32_e64_dpp_gfx1250:
3625 return true;
3626 default:
3627 return false;
3628 }
3629}
3630
3631bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
3632 const MCSubtargetInfo &ST) {
3633 if (isDPALU_DPP32BitOpc(Opc: OpDesc.getOpcode()))
3634 return true;
3635
3636 return hasAny64BitVGPROperands(OpDesc, MII, ST);
3637}
3638
3639bool isPackedSingleSGPRFP32Inst(unsigned Opc) {
3640 switch (Opc) {
3641 case AMDGPU::V_PK_ADD_F32_gfx1250:
3642 case AMDGPU::V_PK_ADD_F32_gfx1250_gfx12:
3643 case AMDGPU::V_PK_MUL_F32_gfx1250:
3644 case AMDGPU::V_PK_MUL_F32_gfx1250_gfx12:
3645 case AMDGPU::V_PK_FMA_F32_gfx1250:
3646 case AMDGPU::V_PK_FMA_F32_gfx1250_gfx12:
3647 return true;
3648 default:
3649 return false;
3650 }
3651}
3652
3653// NOTE: This function is currently only used before pseudo-expansion.
3654bool isPackedSingleSGPR64BitInst(unsigned Opc) {
3655 switch (Opc) {
3656 case AMDGPU::V_PK_ADD_F64:
3657 case AMDGPU::V_PK_MUL_F64:
3658 case AMDGPU::V_PK_FMA_F64:
3659 case AMDGPU::V_PK_MAX_NUM_F64:
3660 case AMDGPU::V_PK_MIN_NUM_F64:
3661 case AMDGPU::V_PK_ADD_NC_U64:
3662 case AMDGPU::V_PK_SUB_NC_U64:
3663 case AMDGPU::V_PK_LSHL_ADD_U64:
3664 return true;
3665 default:
3666 return false;
3667 }
3668}
3669
3670bool isSingleSGPRReadInst(unsigned Opc) {
3671 return isPackedSingleSGPRFP32Inst(Opc) || isPackedSingleSGPR64BitInst(Opc);
3672}
3673
3674const std::array<unsigned, 3> &ClusterDimsAttr::getDims() const {
3675 assert(isFixedDims() && "expect kind to be FixedDims");
3676 return Dims;
3677}
3678
3679std::string ClusterDimsAttr::to_string() const {
3680 SmallString<10> Buffer;
3681 raw_svector_ostream OS(Buffer);
3682
3683 switch (getKind()) {
3684 case Kind::Unknown:
3685 return "";
3686 case Kind::NoCluster: {
3687 OS << EncoNoCluster << ',' << EncoNoCluster << ',' << EncoNoCluster;
3688 return Buffer.c_str();
3689 }
3690 case Kind::VariableDims: {
3691 OS << EncoVariableDims << ',' << EncoVariableDims << ','
3692 << EncoVariableDims;
3693 return Buffer.c_str();
3694 }
3695 case Kind::FixedDims: {
3696 OS << Dims[0] << ',' << Dims[1] << ',' << Dims[2];
3697 return Buffer.c_str();
3698 }
3699 }
3700 llvm_unreachable("Unknown ClusterDimsAttr kind");
3701}
3702
3703ClusterDimsAttr ClusterDimsAttr::get(const Function &F) {
3704 std::optional<SmallVector<unsigned>> Attr =
3705 getIntegerVecAttribute(F, Name: "amdgpu-cluster-dims", /*Size=*/3);
3706 ClusterDimsAttr::Kind AttrKind = Kind::FixedDims;
3707
3708 if (!Attr.has_value())
3709 AttrKind = Kind::Unknown;
3710 else if (all_of(Range&: *Attr, P: equal_to(Arg: EncoNoCluster)))
3711 AttrKind = Kind::NoCluster;
3712 else if (all_of(Range&: *Attr, P: equal_to(Arg: EncoVariableDims)))
3713 AttrKind = Kind::VariableDims;
3714
3715 ClusterDimsAttr A(AttrKind);
3716 if (AttrKind == Kind::FixedDims)
3717 A.Dims = {(*Attr)[0], (*Attr)[1], (*Attr)[2]};
3718
3719 return A;
3720}
3721
3722std::optional<APFloat> evaluateRcp(const APFloat &Val) {
3723 const fltSemantics &Sem = Val.getSemantics();
3724
3725 // v_rcp_f16/bf16 are correctly rounded.
3726 if (&Sem == &APFloat::IEEEhalf() || &Sem == &APFloat::BFloat())
3727 return APFloat::getOne(Sem) / Val;
3728
3729 // v_rcp_f32/f64 always flush a denormal input to zero (preserving sign)
3730 // before reciprocating.
3731 APFloat Arg = Val;
3732 if (Arg.isDenormal())
3733 Arg = APFloat::getZero(Sem, Negative: Arg.isNegative());
3734
3735 APFloat Result = APFloat::getOne(Sem) / Arg;
3736
3737 // v_rcp_f32/f64 always flush a denormal result to zero (preserving sign).
3738 if (Result.isDenormal())
3739 Result = APFloat::getZero(Sem, Negative: Result.isNegative());
3740
3741 // v_rcp_f32/f64 only approximate the reciprocal, except for these special
3742 // cases where the result is exact.
3743 if (!Result.isZero() && !Result.isInfinity() && !Result.isNaN() &&
3744 !Result.isOne() && !Result.isMinusOne())
3745 return std::nullopt;
3746
3747 return Result;
3748}
3749
3750} // namespace AMDGPU
3751
3752raw_ostream &operator<<(raw_ostream &OS, const AMDGPU::TargetIDSetting S) {
3753 switch (S) {
3754 case (AMDGPU::TargetIDSetting::Unsupported):
3755 OS << "Unsupported";
3756 break;
3757 case (AMDGPU::TargetIDSetting::Any):
3758 OS << "Any";
3759 break;
3760 case (AMDGPU::TargetIDSetting::Off):
3761 OS << "Off";
3762 break;
3763 case (AMDGPU::TargetIDSetting::On):
3764 OS << "On";
3765 break;
3766 }
3767 return OS;
3768}
3769
3770} // namespace llvm
3771