1//===-- SIDefines.h - SI Helper Macros ----------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7/// \file
8//===----------------------------------------------------------------------===//
9
10#ifndef LLVM_LIB_TARGET_AMDGPU_SIDEFINES_H
11#define LLVM_LIB_TARGET_AMDGPU_SIDEFINES_H
12
13#include "llvm/MC/MCInst.h"
14#include "llvm/MC/MCInstrDesc.h"
15#include "llvm/MC/MCInstrInfo.h"
16#include "llvm/Support/AMDGPUAddrSpace.h"
17
18namespace llvm {
19
20// This needs to be kept in sync with the field bits in SIRegisterClass.
21enum SIRCFlags : uint8_t {
22 RegTupleAlignUnitsWidth = 2,
23 HasVGPRBit = RegTupleAlignUnitsWidth,
24 HasAGPRBit,
25 HasSGPRbit,
26
27 HasVGPR = 1 << HasVGPRBit,
28 HasAGPR = 1 << HasAGPRBit,
29 HasSGPR = 1 << HasSGPRbit,
30
31 RegTupleAlignUnitsMask = (1 << RegTupleAlignUnitsWidth) - 1,
32 RegKindMask = (HasVGPR | HasAGPR | HasSGPR)
33}; // enum SIRCFlagsr
34
35namespace SIEncodingFamily {
36// This must be kept in sync with the SIEncodingFamily class in SIInstrInfo.td
37// and the columns of the getMCOpcodeGen table.
38enum {
39 SI = 0,
40 VI = 1,
41 SDWA = 2,
42 SDWA9 = 3,
43 GFX80 = 4,
44 GFX9 = 5,
45 GFX10 = 6,
46 SDWA10 = 7,
47 GFX90A = 8,
48 GFX940 = 9,
49 GFX11 = 10,
50 GFX1170 = 11,
51 GFX12 = 12,
52 GFX1250 = 13,
53 GFX13 = 14,
54};
55}
56
57namespace SIInstrFlags {
58
59// Raw TSFlags bit constants — use the predicate functions below instead of
60// testing these directly. This namespace makes accidental direct use a
61// compile error outside of SIDefines.h itself.
62namespace DontUseRawTSFlags {
63// This needs to be kept in sync with the field bits in InstSI.
64enum : uint64_t {
65 // Low bits - basic encoding information.
66 SALU = 1 << 0,
67 VALU = 1 << 1,
68
69 // SALU instruction formats.
70 SOP1 = 1 << 2,
71 SOP2 = 1 << 3,
72 SOPC = 1 << 4,
73 SOPK = 1 << 5,
74 SOPP = 1 << 6,
75
76 // VALU instruction formats.
77 VOP1 = 1 << 7,
78 VOP2 = 1 << 8,
79 VOPC = 1 << 9,
80
81 // TODO: Should this be spilt into VOP3 a and b?
82 VOP3 = 1 << 10,
83 VOP3P = 1 << 12,
84
85 VINTRP = 1 << 13,
86 SDWA = 1 << 14,
87 DPP = 1 << 15,
88 TRANS = 1 << 16,
89
90 // Memory instruction formats.
91 MUBUF = 1 << 17,
92 MTBUF = 1 << 18,
93 SMRD = 1 << 19,
94 MIMG = 1 << 20,
95 VIMAGE = 1 << 21,
96 VSAMPLE = 1 << 22,
97 EXP = 1 << 23,
98 FLAT = 1 << 24,
99 DS = 1 << 25,
100
101 // Combined SGPR/VGPR Spill bit
102 // Logic to separate them out is done in isSGPRSpill and isVGPRSpill
103 Spill = 1 << 26,
104
105 // LDSDIR instruction format.
106 LDSDIR = 1 << 28,
107
108 // VINTERP instruction format.
109 VINTERP = 1 << 29,
110
111 VOPD3 = 1 << 30,
112
113 // High bits - other information.
114 VM_CNT = UINT64_C(1) << 32,
115 EXP_CNT = UINT64_C(1) << 33,
116 LGKM_CNT = UINT64_C(1) << 34,
117
118 WQM = UINT64_C(1) << 35,
119 DisableWQM = UINT64_C(1) << 36,
120 Gather4 = UINT64_C(1) << 37,
121
122 TENSOR_CNT = UINT64_C(1) << 38,
123
124 SCALAR_STORE = UINT64_C(1) << 39,
125 FIXED_SIZE = UINT64_C(1) << 40,
126
127 ASYNC_CNT = UINT64_C(1) << 41,
128
129 VOP3_OPSEL = UINT64_C(1) << 42,
130 maybeAtomic = UINT64_C(1) << 43,
131
132 // A load or store pseudo of the VGPR "as memory" address space (13).
133 VGPRAsMem = UINT64_C(1) << 44,
134
135 // Is a clamp on FP type.
136 FPClamp = UINT64_C(1) << 45,
137
138 // Is an integer clamp
139 IntClamp = UINT64_C(1) << 46,
140
141 // Clamps lo component of register.
142 ClampLo = UINT64_C(1) << 47,
143
144 // Clamps hi component of register.
145 // ClampLo and ClampHi set for packed clamp.
146 ClampHi = UINT64_C(1) << 48,
147
148 // Is a packed VOP3P instruction.
149 IsPacked = UINT64_C(1) << 49,
150
151 // FLAT instruction accesses FLAT_GLBL segment.
152 FlatGlobal = UINT64_C(1) << 51,
153
154 // Uses floating point double precision rounding mode
155 FPDPRounding = UINT64_C(1) << 52,
156
157 // Instruction is FP atomic.
158 FPAtomic = UINT64_C(1) << 53,
159
160 // Is a MFMA instruction.
161 IsMAI = UINT64_C(1) << 54,
162
163 // Is a DOT instruction.
164 IsDOT = UINT64_C(1) << 55,
165
166 // FLAT instruction accesses FLAT_SCRATCH segment.
167 FlatScratch = UINT64_C(1) << 56,
168
169 // Atomic without return.
170 IsAtomicNoRet = UINT64_C(1) << 57,
171
172 // Atomic with return.
173 IsAtomicRet = UINT64_C(1) << 58,
174
175 // Is a WMMA instruction.
176 IsWMMA = UINT64_C(1) << 59,
177
178 // Whether tied sources will be read.
179 TiedSourceNotRead = UINT64_C(1) << 60,
180
181 // Is never uniform.
182 IsNeverUniform = UINT64_C(1) << 61,
183
184 // ds_gws_* instructions.
185 GWS = UINT64_C(1) << 62,
186
187 // Is a SWMMAC instruction.
188 IsSWMMAC = UINT64_C(1) << 63,
189};
190} // namespace DontUseRawTSFlags
191
192// Predicate functions over TSFlags — the single place where raw TSFlags bit
193// tests are written. All callers (SIInstrInfo methods, MC-layer code) go
194// through these so that bit-layout changes require updating only this file.
195//
196// getTSFlags is overloaded for MCInstrDesc, (MCInstrInfo, Opcode), and
197// (MCInstrInfo, MCInst) here; SIInstrInfo.h adds a MachineInstr overload in
198// namespace llvm so ADL finds it when predicates are instantiated with
199// MachineInstr.
200
201constexpr uint64_t getTSFlags(const MCInstrDesc &Desc) { return Desc.TSFlags; }
202inline uint64_t getTSFlags(const MCInstrInfo &MII, unsigned Opcode) {
203 return MII.get(Opcode).TSFlags;
204}
205inline uint64_t getTSFlags(const MCInstrInfo &MII, const MCInst &Inst) {
206 return MII.get(Opcode: Inst.getOpcode()).TSFlags;
207}
208
209template <typename... T> constexpr bool isSALU(const T &...O) {
210 return getTSFlags(O...) & DontUseRawTSFlags::SALU;
211}
212template <typename... T> constexpr bool isVALU(const T &...O) {
213 return getTSFlags(O...) & DontUseRawTSFlags::VALU;
214}
215template <typename... T> constexpr bool isSOP1(const T &...O) {
216 return getTSFlags(O...) & DontUseRawTSFlags::SOP1;
217}
218template <typename... T> constexpr bool isSOP2(const T &...O) {
219 return getTSFlags(O...) & DontUseRawTSFlags::SOP2;
220}
221template <typename... T> constexpr bool isSOPC(const T &...O) {
222 return getTSFlags(O...) & DontUseRawTSFlags::SOPC;
223}
224template <typename... T> constexpr bool isSOPK(const T &...O) {
225 return getTSFlags(O...) & DontUseRawTSFlags::SOPK;
226}
227template <typename... T> constexpr bool isSOPP(const T &...O) {
228 return getTSFlags(O...) & DontUseRawTSFlags::SOPP;
229}
230template <typename... T> constexpr bool isVOP1(const T &...O) {
231 return getTSFlags(O...) & DontUseRawTSFlags::VOP1;
232}
233template <typename... T> constexpr bool isVOP2(const T &...O) {
234 return getTSFlags(O...) & DontUseRawTSFlags::VOP2;
235}
236template <typename... T> constexpr bool isVOPC(const T &...O) {
237 return getTSFlags(O...) & DontUseRawTSFlags::VOPC;
238}
239template <typename... T> constexpr bool isVOP3(const T &...O) {
240 return getTSFlags(O...) & DontUseRawTSFlags::VOP3;
241}
242template <typename... T> constexpr bool isVOP3P(const T &...O) {
243 return getTSFlags(O...) & DontUseRawTSFlags::VOP3P;
244}
245template <typename... T> constexpr bool isVOP3Like(const T &...O) {
246 return getTSFlags(O...) &
247 (DontUseRawTSFlags::VOP3 | DontUseRawTSFlags::VOP3P);
248}
249template <typename... T> constexpr bool isVINTRP(const T &...O) {
250 return getTSFlags(O...) & DontUseRawTSFlags::VINTRP;
251}
252template <typename... T> constexpr bool isSDWA(const T &...O) {
253 return getTSFlags(O...) & DontUseRawTSFlags::SDWA;
254}
255template <typename... T> constexpr bool isDPP(const T &...O) {
256 return getTSFlags(O...) & DontUseRawTSFlags::DPP;
257}
258template <typename... T> constexpr bool isTRANS(const T &...O) {
259 return getTSFlags(O...) & DontUseRawTSFlags::TRANS;
260}
261template <typename... T> constexpr bool isMUBUF(const T &...O) {
262 return getTSFlags(O...) & DontUseRawTSFlags::MUBUF;
263}
264template <typename... T> constexpr bool isMTBUF(const T &...O) {
265 return getTSFlags(O...) & DontUseRawTSFlags::MTBUF;
266}
267template <typename... T> constexpr bool isBuffer(const T &...O) {
268 return getTSFlags(O...) &
269 (DontUseRawTSFlags::MUBUF | DontUseRawTSFlags::MTBUF);
270}
271template <typename... T> constexpr bool isSMRD(const T &...O) {
272 return getTSFlags(O...) & DontUseRawTSFlags::SMRD;
273}
274template <typename... T> constexpr bool isMIMG(const T &...O) {
275 return getTSFlags(O...) & DontUseRawTSFlags::MIMG;
276}
277template <typename... T> constexpr bool isVIMAGE(const T &...O) {
278 return getTSFlags(O...) & DontUseRawTSFlags::VIMAGE;
279}
280template <typename... T> constexpr bool isVSAMPLE(const T &...O) {
281 return getTSFlags(O...) & DontUseRawTSFlags::VSAMPLE;
282}
283template <typename... T> constexpr bool isEXP(const T &...O) {
284 return getTSFlags(O...) & DontUseRawTSFlags::EXP;
285}
286template <typename... T> constexpr bool isFLAT(const T &...O) {
287 return getTSFlags(O...) & DontUseRawTSFlags::FLAT;
288}
289template <typename... T> constexpr bool isDS(const T &...O) {
290 return getTSFlags(O...) & DontUseRawTSFlags::DS;
291}
292template <typename... T> constexpr bool isSpill(const T &...O) {
293 return getTSFlags(O...) & DontUseRawTSFlags::Spill;
294}
295template <typename... T> constexpr bool isLDSDIR(const T &...O) {
296 return getTSFlags(O...) & DontUseRawTSFlags::LDSDIR;
297}
298template <typename... T> constexpr bool isVINTERP(const T &...O) {
299 return getTSFlags(O...) & DontUseRawTSFlags::VINTERP;
300}
301template <typename... T> constexpr bool isWQM(const T &...O) {
302 return getTSFlags(O...) & DontUseRawTSFlags::WQM;
303}
304template <typename... T> constexpr bool isDisableWQM(const T &...O) {
305 return getTSFlags(O...) & DontUseRawTSFlags::DisableWQM;
306}
307template <typename... T> constexpr bool isGather4(const T &...O) {
308 return getTSFlags(O...) & DontUseRawTSFlags::Gather4;
309}
310template <typename... T> constexpr bool usesTENSOR_CNT(const T &...O) {
311 return getTSFlags(O...) & DontUseRawTSFlags::TENSOR_CNT;
312}
313template <typename... T> constexpr bool isScalarStore(const T &...O) {
314 return getTSFlags(O...) & DontUseRawTSFlags::SCALAR_STORE;
315}
316template <typename... T> constexpr bool isFixedSize(const T &...O) {
317 return getTSFlags(O...) & DontUseRawTSFlags::FIXED_SIZE;
318}
319template <typename... T> constexpr bool usesASYNC_CNT(const T &...O) {
320 return getTSFlags(O...) & DontUseRawTSFlags::ASYNC_CNT;
321}
322template <typename... T> constexpr bool hasVOP3OpSel(const T &...O) {
323 return getTSFlags(O...) & DontUseRawTSFlags::VOP3_OPSEL;
324}
325template <typename... T> constexpr bool isMaybeAtomic(const T &...O) {
326 return getTSFlags(O...) & DontUseRawTSFlags::maybeAtomic;
327}
328template <typename... T> constexpr bool isVGPRAsMem(const T &...O) {
329 return getTSFlags(O...) & DontUseRawTSFlags::VGPRAsMem;
330}
331template <typename... T> constexpr bool hasFPClamp(const T &...O) {
332 return getTSFlags(O...) & DontUseRawTSFlags::FPClamp;
333}
334template <typename... T> constexpr bool hasIntClamp(const T &...O) {
335 return getTSFlags(O...) & DontUseRawTSFlags::IntClamp;
336}
337template <typename... T> constexpr bool hasClampLo(const T &...O) {
338 return getTSFlags(O...) & DontUseRawTSFlags::ClampLo;
339}
340template <typename... T> constexpr bool hasClampHi(const T &...O) {
341 return getTSFlags(O...) & DontUseRawTSFlags::ClampHi;
342}
343template <typename... T> constexpr bool isPacked(const T &...O) {
344 return getTSFlags(O...) & DontUseRawTSFlags::IsPacked;
345}
346template <typename... T> constexpr bool isFlatGlobal(const T &...O) {
347 return getTSFlags(O...) & DontUseRawTSFlags::FlatGlobal;
348}
349template <typename... T> constexpr bool usesFPDPRounding(const T &...O) {
350 return getTSFlags(O...) & DontUseRawTSFlags::FPDPRounding;
351}
352template <typename... T> constexpr bool isFPAtomic(const T &...O) {
353 return getTSFlags(O...) & DontUseRawTSFlags::FPAtomic;
354}
355template <typename... T> constexpr bool isMAI(const T &...O) {
356 return getTSFlags(O...) & DontUseRawTSFlags::IsMAI;
357}
358template <typename... T> constexpr bool isDOT(const T &...O) {
359 return getTSFlags(O...) & DontUseRawTSFlags::IsDOT;
360}
361template <typename... T> constexpr bool isFlatScratch(const T &...O) {
362 return getTSFlags(O...) & DontUseRawTSFlags::FlatScratch;
363}
364template <typename... T> constexpr bool isAtomicNoRet(const T &...O) {
365 return getTSFlags(O...) & DontUseRawTSFlags::IsAtomicNoRet;
366}
367template <typename... T> constexpr bool isAtomicRet(const T &...O) {
368 return getTSFlags(O...) & DontUseRawTSFlags::IsAtomicRet;
369}
370template <typename... T> constexpr bool isWMMA(const T &...O) {
371 return getTSFlags(O...) & DontUseRawTSFlags::IsWMMA;
372}
373template <typename... T> constexpr bool isTiedSourceNotRead(const T &...O) {
374 return getTSFlags(O...) & DontUseRawTSFlags::TiedSourceNotRead;
375}
376template <typename... T> constexpr bool isNeverUniform(const T &...O) {
377 return getTSFlags(O...) & DontUseRawTSFlags::IsNeverUniform;
378}
379template <typename... T> constexpr bool isGWS(const T &...O) {
380 return getTSFlags(O...) & DontUseRawTSFlags::GWS;
381}
382template <typename... T> constexpr bool isSWMMAC(const T &...O) {
383 return getTSFlags(O...) & DontUseRawTSFlags::IsSWMMAC;
384}
385template <typename... T> constexpr bool isVOPD3(const T &...O) {
386 return getTSFlags(O...) & DontUseRawTSFlags::VOPD3;
387}
388template <typename... T> constexpr bool usesVM_CNT(const T &...O) {
389 return getTSFlags(O...) & DontUseRawTSFlags::VM_CNT;
390}
391template <typename... T> constexpr bool usesLGKM_CNT(const T &...O) {
392 return getTSFlags(O...) & DontUseRawTSFlags::LGKM_CNT;
393}
394
395// Compound predicates.
396template <typename... T> constexpr bool isAtomic(const T &...O) {
397 return isAtomicNoRet(O...) || isAtomicRet(O...);
398}
399template <typename... T> constexpr bool isSegmentSpecificFLAT(const T &...O) {
400 return isFlatGlobal(O...) || isFlatScratch(O...);
401}
402// Any image-family instruction: pre-gfx11 MIMG, gfx11+ VIMAGE or VSAMPLE.
403template <typename... T> constexpr bool isImage(const T &...O) {
404 return isMIMG(O...) || isVIMAGE(O...) || isVSAMPLE(O...);
405}
406// Vector memory: buffer + image + flat.
407template <typename... T> constexpr bool isVMEM(const T &...O) {
408 return isMUBUF(O...) || isMTBUF(O...) || isImage(O...) || isFLAT(O...);
409}
410
411// v_cmp_class_* etc. use a 10-bit mask for what operation is checked.
412// The result is true if any of these tests are true.
413enum ClassFlags : unsigned {
414 S_NAN = 1 << 0, // Signaling NaN
415 Q_NAN = 1 << 1, // Quiet NaN
416 N_INFINITY = 1 << 2, // Negative infinity
417 N_NORMAL = 1 << 3, // Negative normal
418 N_SUBNORMAL = 1 << 4, // Negative subnormal
419 N_ZERO = 1 << 5, // Negative zero
420 P_ZERO = 1 << 6, // Positive zero
421 P_SUBNORMAL = 1 << 7, // Positive subnormal
422 P_NORMAL = 1 << 8, // Positive normal
423 P_INFINITY = 1 << 9 // Positive infinity
424};
425}
426
427namespace AMDGPU {
428
429enum OperandType : unsigned {
430 /// Operands with register, 32-bit, or 64-bit immediate
431 OPERAND_REG_IMM_INT32 = MCOI::OPERAND_FIRST_TARGET,
432 OPERAND_REG_IMM_INT64,
433 OPERAND_REG_IMM_INT16,
434 OPERAND_REG_IMM_FP32,
435 OPERAND_REG_IMM_FP64,
436 OPERAND_REG_IMM_BF16,
437 OPERAND_REG_IMM_FP16,
438 OPERAND_REG_IMM_NOINLINE_FP16,
439 OPERAND_REG_IMM_V2BF16,
440 OPERAND_REG_IMM_V2FP16,
441 OPERAND_REG_IMM_V2FP16_SPLAT,
442 OPERAND_REG_IMM_V2INT16,
443 OPERAND_REG_IMM_V2INT64,
444 OPERAND_REG_IMM_NOINLINE_V2FP16,
445 OPERAND_REG_IMM_V2INT32,
446 OPERAND_REG_IMM_V2FP32,
447 OPERAND_REG_IMM_V2FP64,
448
449 /// Operands with register or inline constant
450 OPERAND_REG_INLINE_C_INT16,
451 OPERAND_REG_INLINE_C_INT32,
452 OPERAND_REG_INLINE_C_INT64,
453 OPERAND_REG_INLINE_C_BF16,
454 OPERAND_REG_INLINE_C_FP16,
455 OPERAND_REG_INLINE_C_FP32,
456 OPERAND_REG_INLINE_C_FP64,
457 OPERAND_REG_INLINE_C_V2INT16,
458 OPERAND_REG_INLINE_C_V2BF16,
459 OPERAND_REG_INLINE_C_V2FP16,
460
461 // Operand for split barrier inline constant
462 OPERAND_INLINE_SPLIT_BARRIER_INT32,
463
464 /// Operand with 32-bit immediate that uses the constant bus.
465 OPERAND_KIMM32,
466 OPERAND_KIMM16,
467 OPERAND_KIMM64,
468
469 /// Operands with an AccVGPR register or inline constant
470 OPERAND_REG_INLINE_AC_INT32,
471 OPERAND_REG_INLINE_AC_FP32,
472 OPERAND_REG_INLINE_AC_FP64,
473
474 // Operand for AV_MOV_B64_IMM_PSEUDO, which is a pair of 32-bit inline
475 // constants. Does not accept registers.
476 OPERAND_INLINE_C_AV64_PSEUDO,
477
478 // Operand for source modifiers for VOP instructions
479 OPERAND_INPUT_MODS,
480
481 // Operand for SDWA instructions
482 OPERAND_SDWA_VOPC_DST,
483
484 OPERAND_REG_IMM_FIRST = OPERAND_REG_IMM_INT32,
485 OPERAND_REG_IMM_LAST = OPERAND_REG_IMM_V2FP64,
486
487 OPERAND_REG_INLINE_C_FIRST = OPERAND_REG_INLINE_C_INT16,
488 OPERAND_REG_INLINE_C_LAST = OPERAND_REG_INLINE_AC_FP64,
489
490 OPERAND_REG_INLINE_AC_FIRST = OPERAND_REG_INLINE_AC_INT32,
491 OPERAND_REG_INLINE_AC_LAST = OPERAND_INLINE_C_AV64_PSEUDO,
492
493 OPERAND_SRC_FIRST = OPERAND_REG_IMM_INT32,
494 OPERAND_SRC_LAST = OPERAND_REG_INLINE_C_LAST,
495
496 OPERAND_KIMM_FIRST = OPERAND_KIMM32,
497 OPERAND_KIMM_LAST = OPERAND_KIMM64
498
499};
500}
501
502// Input operand modifiers bit-masks
503// NEG and SEXT share same bit-mask because they can't be set simultaneously.
504namespace SISrcMods {
505enum : unsigned {
506 NONE = 0,
507 NEG = 1 << 0, // Floating-point negate modifier
508 ABS = 1 << 1, // Floating-point absolute modifier
509 SEXT = 1 << 4, // Integer sign-extend modifier
510 NEG_HI = ABS, // Floating-point negate high packed component modifier.
511 OP_SEL_0 = 1 << 2,
512 OP_SEL_1 = 1 << 3,
513 DST_OP_SEL = 1 << 3 // VOP3 dst op_sel (share mask with OP_SEL_1)
514};
515}
516
517namespace SIOutMods {
518 enum : unsigned {
519 NONE = 0,
520 MUL2 = 1,
521 MUL4 = 2,
522 DIV2 = 3
523 };
524}
525
526namespace AMDGPU {
527namespace VGPRIndexMode {
528
529enum Id : unsigned { // id of symbolic names
530 ID_SRC0 = 0,
531 ID_SRC1,
532 ID_SRC2,
533 ID_DST,
534
535 ID_MIN = ID_SRC0,
536 ID_MAX = ID_DST
537};
538
539enum EncBits : unsigned {
540 OFF = 0,
541 SRC0_ENABLE = 1 << ID_SRC0,
542 SRC1_ENABLE = 1 << ID_SRC1,
543 SRC2_ENABLE = 1 << ID_SRC2,
544 DST_ENABLE = 1 << ID_DST,
545 ENABLE_MASK = SRC0_ENABLE | SRC1_ENABLE | SRC2_ENABLE | DST_ENABLE,
546 UNDEF = 0xFFFF
547};
548
549} // namespace VGPRIndexMode
550} // namespace AMDGPU
551
552namespace AMDGPUAsmVariants {
553 enum : unsigned {
554 DEFAULT = 0,
555 VOP3 = 1,
556 SDWA = 2,
557 SDWA9 = 3,
558 DPP = 4,
559 VOP3_DPP = 5
560 };
561} // namespace AMDGPUAsmVariants
562
563namespace AMDGPU {
564namespace EncValues { // Encoding values of enum9/8/7 operands
565
566enum : unsigned {
567 SGPR_MIN = 0,
568 SGPR_MAX_SI = 101,
569 SGPR_MAX_GFX10 = 105,
570 TTMP_VI_MIN = 112,
571 TTMP_VI_MAX = 123,
572 TTMP_GFX9PLUS_MIN = 108,
573 TTMP_GFX9PLUS_MAX = 123,
574 INLINE_INTEGER_C_MIN = 128,
575 INLINE_INTEGER_C_POSITIVE_MAX = 192, // 64
576 INLINE_INTEGER_C_MAX = 208,
577 INLINE_FLOATING_C_MIN = 240,
578 INLINE_FLOATING_C_MAX = 248,
579 LITERAL64_CONST = 254,
580 LITERAL_CONST = 255,
581 VGPR_MIN = 256,
582 VGPR_MAX = 511,
583 IS_VGPR = 256, // Indicates VGPR or AGPR
584};
585
586} // namespace EncValues
587
588// Register codes as defined in the TableGen's HWEncoding field.
589namespace HWEncoding {
590enum : unsigned {
591 REG_IDX_MASK = 0x3ff,
592 LO256_REG_IDX_MASK = 0xff,
593 IS_VGPR = 1 << 10,
594 IS_AGPR = 1 << 11,
595 IS_HI16 = 1 << 12,
596};
597} // namespace HWEncoding
598
599namespace CPol {
600
601enum CPol {
602 GLC = 1,
603 SLC = 2,
604 DLC = 4,
605 SCC = 16,
606 SC0 = GLC,
607 SC1 = SCC,
608 NT = SLC,
609 ALL_pregfx12 = GLC | SLC | DLC | SCC,
610 SWZ_pregfx12 = 8,
611
612 // Below are GFX12+ cache policy bits
613
614 // Temporal hint
615 TH = 0x7, // All TH bits
616 TH_RT = 0, // regular
617 TH_NT = 1, // non-temporal
618 TH_HT = 2, // high-temporal
619 TH_LU = 3, // last use
620 TH_WB = 3, // regular (CU, SE), high-temporal with write-back (MALL)
621 TH_NT_RT = 4, // non-temporal (CU, SE), regular (MALL)
622 TH_RT_NT = 5, // regular (CU, SE), non-temporal (MALL)
623 TH_NT_HT = 6, // non-temporal (CU, SE), high-temporal (MALL)
624 TH_NT_WB = 7, // non-temporal (CU, SE), high-temporal with write-back (MALL)
625 TH_BYPASS = 3, // only to be used with scope = 3
626
627 TH_RESERVED = 7, // unused value for load insts
628
629 // Bits of TH for atomics
630 TH_ATOMIC_RETURN = GLC, // Returning vs non-returning
631 TH_ATOMIC_NT = SLC, // Non-temporal vs regular
632 TH_ATOMIC_CASCADE = 4, // Cascading vs regular
633
634 // Scope
635 SCOPE_SHIFT = 3,
636 SCOPE_MASK = 0x3,
637 SCOPE = SCOPE_MASK << SCOPE_SHIFT, // All Scope bits
638 SCOPE_CU = 0 << SCOPE_SHIFT,
639 SCOPE_SE = 1 << SCOPE_SHIFT,
640 SCOPE_DEV = 2 << SCOPE_SHIFT,
641 SCOPE_SYS = 3 << SCOPE_SHIFT,
642
643 NV = 1 << 5, // Non-volatile bit
644
645 SWZ = 1 << 6, // Swizzle bit
646
647 // Cache fill size
648 CFS_SHIFT = 7,
649 CFS_MASK = 0x3,
650 CFS = CFS_MASK << CFS_SHIFT,
651 CFS_256B = 0 << CFS_SHIFT,
652 CFS_128B = 1 << CFS_SHIFT,
653 CFS_64B = 2 << CFS_SHIFT,
654 CFS_32B = 3 << CFS_SHIFT,
655
656 ALL = TH | SCOPE | NV | CFS,
657
658 SCAL = 1 << 9, // Scale offset bit
659
660 // Helper bits
661 TH_TYPE_LOAD = 1 << 27, // TH_LOAD policy
662 TH_TYPE_STORE = 1 << 28, // TH_STORE policy
663 TH_TYPE_ATOMIC = 1 << 29, // TH_ATOMIC policy
664 TH_REAL_BYPASS = 1 << 30, // is TH=3 bypass policy or not
665
666 // Volatile (used to preserve/signal operation volatility for buffer
667 // operations not a real instruction bit)
668 VOLATILE = 1 << 31,
669 // The set of "cache policy" bits used for compiler features that
670 // do not correspond to handware features.
671 VIRTUAL_BITS = VOLATILE,
672};
673
674} // namespace CPol
675
676namespace SendMsg { // Encoding of SIMM16 used in s_sendmsg* insns.
677
678enum Id { // Message ID, width(4) [3:0].
679 ID_INTERRUPT = 1,
680
681 ID_GS_PreGFX11 = 2, // replaced in GFX11
682 ID_GS_DONE_PreGFX11 = 3, // replaced in GFX11
683
684 ID_HS_TESSFACTOR_GFX11Plus = 2, // reused in GFX11
685 ID_DEALLOC_VGPRS_GFX11Plus = 3, // reused in GFX11
686
687 ID_SAVEWAVE = 4, // added in GFX8, removed in GFX11
688 ID_STALL_WAVE_GEN = 5, // added in GFX9, removed in GFX12
689 ID_HALT_WAVES = 6, // added in GFX9, removed in GFX12
690 ID_ORDERED_PS_DONE = 7, // added in GFX9, removed in GFX11
691 ID_EARLY_PRIM_DEALLOC = 8, // added in GFX9, removed in GFX10
692 ID_GS_ALLOC_REQ = 9, // added in GFX9
693 ID_GET_DOORBELL = 10, // added in GFX9, removed in GFX11
694 ID_GET_DDID = 11, // added in GFX10, removed in GFX11
695 ID_SYSMSG = 15,
696
697 ID_RTN_GET_DOORBELL = 128,
698 ID_RTN_GET_DDID = 129,
699 ID_RTN_GET_TMA = 130,
700 ID_RTN_GET_REALTIME = 131,
701 ID_RTN_SAVE_WAVE = 132,
702 ID_RTN_GET_TBA = 133,
703 ID_RTN_GET_TBA_TO_PC = 134,
704 ID_RTN_GET_SE_AID_ID = 135,
705
706 ID_RTN_GET_CLUSTER_BARRIER_STATE = 136, // added in GFX1250
707 ID_RTN_SAVE_WAVE_HAS_TDM = 152, // added in GFX1250
708
709 ID_MASK_PreGFX11_ = 0xF,
710 ID_MASK_GFX11Plus_ = 0xFF
711};
712
713enum Op { // Both GS and SYS operation IDs.
714 OP_SHIFT_ = 4,
715 OP_NONE_ = 0,
716 // Bits used for operation encoding
717 OP_WIDTH_ = 3,
718 OP_MASK_ = (((1 << OP_WIDTH_) - 1) << OP_SHIFT_),
719 // GS operations are encoded in bits 5:4
720 OP_GS_NOP = 0,
721 OP_GS_CUT = 1,
722 OP_GS_EMIT = 2,
723 OP_GS_EMIT_CUT = 3,
724 OP_GS_FIRST_ = OP_GS_NOP,
725 // SYS operations are encoded in bits 6:4
726 OP_SYS_ECC_ERR_INTERRUPT = 1,
727 OP_SYS_REG_RD = 2,
728 OP_SYS_HOST_TRAP_ACK = 3,
729 OP_SYS_TTRACE_PC = 4,
730 OP_SYS_FIRST_ = OP_SYS_ECC_ERR_INTERRUPT,
731};
732
733enum StreamId : unsigned { // Stream ID, (2) [9:8].
734 STREAM_ID_NONE_ = 0,
735 STREAM_ID_DEFAULT_ = 0,
736 STREAM_ID_LAST_ = 4,
737 STREAM_ID_FIRST_ = STREAM_ID_DEFAULT_,
738 STREAM_ID_SHIFT_ = 8,
739 STREAM_ID_WIDTH_= 2,
740 STREAM_ID_MASK_ = (((1 << STREAM_ID_WIDTH_) - 1) << STREAM_ID_SHIFT_)
741};
742
743} // namespace SendMsg
744
745namespace WaitEvent { // Encoding of SIMM16 used in s_wait_event
746enum Id {
747 DONT_WAIT_EXPORT_READY = 1 << 0, // Only used in gfx11
748 EXPORT_READY = 1 << 1, // gfx12+
749};
750
751} // namespace WaitEvent
752
753namespace Hwreg { // Encoding of SIMM16 used in s_setreg/getreg* insns.
754
755enum Id { // HwRegCode, (6) [5:0]
756 ID_MODE = 1,
757 ID_STATUS = 2,
758 ID_TRAPSTS = 3,
759 ID_HW_ID = 4,
760 ID_GPR_ALLOC = 5,
761 ID_LDS_ALLOC = 6,
762 ID_IB_STS = 7,
763 ID_PERF_SNAPSHOT_DATA_gfx12 = 10,
764 ID_PERF_SNAPSHOT_PC_LO_gfx12 = 11,
765 ID_PERF_SNAPSHOT_PC_HI_gfx12 = 12,
766 ID_MEM_BASES = 15,
767 ID_TBA_LO = 16,
768 ID_TBA_HI = 17,
769 ID_TMA_LO = 18,
770 ID_TMA_HI = 19,
771 ID_FLAT_SCR_LO = 20,
772 ID_FLAT_SCR_HI = 21,
773 ID_XNACK_MASK = 22,
774 ID_HW_ID1 = 23,
775 ID_HW_ID2 = 24,
776 ID_POPS_PACKER = 25,
777 ID_SCHED_MODE = 26,
778 ID_PERF_SNAPSHOT_DATA_gfx11 = 27,
779 ID_IB_STS2 = 28,
780 ID_SHADER_CYCLES = 29,
781 ID_SHADER_CYCLES_HI = 30,
782 ID_DVGPR_ALLOC_LO = 31,
783 ID_DVGPR_ALLOC_HI = 32,
784
785 // Register numbers reused in GFX11
786 ID_PERF_SNAPSHOT_PC_LO_gfx11 = 18,
787 ID_PERF_SNAPSHOT_PC_HI_gfx11 = 19,
788
789 // Register numbers reused in GFX12+
790 ID_STATE_PRIV = 4,
791 ID_PERF_SNAPSHOT_DATA1 = 15,
792 ID_PERF_SNAPSHOT_DATA2 = 16,
793 ID_EXCP_FLAG_PRIV = 17,
794 ID_EXCP_FLAG_USER = 18,
795 ID_TRAP_CTRL = 19,
796
797 // GFX94* specific registers
798 ID_XCC_ID = 20,
799 ID_SQ_PERF_SNAPSHOT_DATA = 21,
800 ID_SQ_PERF_SNAPSHOT_DATA1 = 22,
801 ID_SQ_PERF_SNAPSHOT_PC_LO = 23,
802 ID_SQ_PERF_SNAPSHOT_PC_HI = 24,
803
804 // GFX1250
805 ID_XNACK_STATE_PRIV = 33,
806 ID_XNACK_MASK_gfx1250 = 34,
807};
808
809enum Offset : unsigned { // Offset, (5) [10:6]
810 OFFSET_MEM_VIOL = 8,
811 OFFSET_ME_ID = 8, // in HW_ID2
812};
813
814enum ModeRegisterMasks : uint32_t {
815 FP_ROUND_MASK = 0xf << 0, // Bits 0..3
816 FP_DENORM_MASK = 0xf << 4, // Bits 4..7
817 DX10_CLAMP_MASK = 1 << 8,
818 IEEE_MODE_MASK = 1 << 9,
819 LOD_CLAMP_MASK = 1 << 10,
820 DEBUG_MASK = 1 << 11,
821
822 // EXCP_EN fields.
823 EXCP_EN_INVALID_MASK = 1 << 12,
824 EXCP_EN_INPUT_DENORMAL_MASK = 1 << 13,
825 EXCP_EN_FLOAT_DIV0_MASK = 1 << 14,
826 EXCP_EN_OVERFLOW_MASK = 1 << 15,
827 EXCP_EN_UNDERFLOW_MASK = 1 << 16,
828 EXCP_EN_INEXACT_MASK = 1 << 17,
829 EXCP_EN_INT_DIV0_MASK = 1 << 18,
830
831 GPR_IDX_EN_MASK = 1 << 27,
832 VSKIP_MASK = 1 << 28,
833 CSP_MASK = 0x7u << 29, // Bits 29..31
834
835 // GFX1250
836 DST_VGPR_MSB = 0x3 << 12,
837 SRC0_VGPR_MSB = 0x3 << 14,
838 SRC1_VGPR_MSB = 0x3 << 16,
839 SRC2_VGPR_MSB = 0x3 << 18,
840 VGPR_MSB_MASK = 0xff << 12, // Bits 12..19
841
842 REPLAY_MODE = 1 << 25,
843 FLAT_SCRATCH_IS_NV = 1 << 26,
844};
845
846} // namespace Hwreg
847
848namespace MTBUFFormat {
849
850enum DataFormat : int64_t {
851 DFMT_INVALID = 0,
852 DFMT_8,
853 DFMT_16,
854 DFMT_8_8,
855 DFMT_32,
856 DFMT_16_16,
857 DFMT_10_11_11,
858 DFMT_11_11_10,
859 DFMT_10_10_10_2,
860 DFMT_2_10_10_10,
861 DFMT_8_8_8_8,
862 DFMT_32_32,
863 DFMT_16_16_16_16,
864 DFMT_32_32_32,
865 DFMT_32_32_32_32,
866 DFMT_RESERVED_15,
867
868 DFMT_MIN = DFMT_INVALID,
869 DFMT_MAX = DFMT_RESERVED_15,
870
871 DFMT_UNDEF = -1,
872 DFMT_DEFAULT = DFMT_8,
873
874 DFMT_SHIFT = 0,
875 DFMT_MASK = 0xF
876};
877
878enum NumFormat : int64_t {
879 NFMT_UNORM = 0,
880 NFMT_SNORM,
881 NFMT_USCALED,
882 NFMT_SSCALED,
883 NFMT_UINT,
884 NFMT_SINT,
885 NFMT_RESERVED_6, // VI and GFX9
886 NFMT_SNORM_OGL = NFMT_RESERVED_6, // SI and CI only
887 NFMT_FLOAT,
888
889 NFMT_MIN = NFMT_UNORM,
890 NFMT_MAX = NFMT_FLOAT,
891
892 NFMT_UNDEF = -1,
893 NFMT_DEFAULT = NFMT_UNORM,
894
895 NFMT_SHIFT = 4,
896 NFMT_MASK = 7
897};
898
899enum MergedFormat : int64_t {
900 DFMT_NFMT_UNDEF = -1,
901 DFMT_NFMT_DEFAULT = ((DFMT_DEFAULT & DFMT_MASK) << DFMT_SHIFT) |
902 ((NFMT_DEFAULT & NFMT_MASK) << NFMT_SHIFT),
903
904
905 DFMT_NFMT_MASK = (DFMT_MASK << DFMT_SHIFT) | (NFMT_MASK << NFMT_SHIFT),
906
907 DFMT_NFMT_MAX = DFMT_NFMT_MASK
908};
909
910enum UnifiedFormatCommon : int64_t {
911 UFMT_MAX = 127,
912 UFMT_UNDEF = -1,
913 UFMT_DEFAULT = 1
914};
915
916} // namespace MTBUFFormat
917
918namespace UfmtGFX10 {
919enum UnifiedFormat : int64_t {
920 UFMT_INVALID = 0,
921
922 UFMT_8_UNORM,
923 UFMT_8_SNORM,
924 UFMT_8_USCALED,
925 UFMT_8_SSCALED,
926 UFMT_8_UINT,
927 UFMT_8_SINT,
928
929 UFMT_16_UNORM,
930 UFMT_16_SNORM,
931 UFMT_16_USCALED,
932 UFMT_16_SSCALED,
933 UFMT_16_UINT,
934 UFMT_16_SINT,
935 UFMT_16_FLOAT,
936
937 UFMT_8_8_UNORM,
938 UFMT_8_8_SNORM,
939 UFMT_8_8_USCALED,
940 UFMT_8_8_SSCALED,
941 UFMT_8_8_UINT,
942 UFMT_8_8_SINT,
943
944 UFMT_32_UINT,
945 UFMT_32_SINT,
946 UFMT_32_FLOAT,
947
948 UFMT_16_16_UNORM,
949 UFMT_16_16_SNORM,
950 UFMT_16_16_USCALED,
951 UFMT_16_16_SSCALED,
952 UFMT_16_16_UINT,
953 UFMT_16_16_SINT,
954 UFMT_16_16_FLOAT,
955
956 UFMT_10_11_11_UNORM,
957 UFMT_10_11_11_SNORM,
958 UFMT_10_11_11_USCALED,
959 UFMT_10_11_11_SSCALED,
960 UFMT_10_11_11_UINT,
961 UFMT_10_11_11_SINT,
962 UFMT_10_11_11_FLOAT,
963
964 UFMT_11_11_10_UNORM,
965 UFMT_11_11_10_SNORM,
966 UFMT_11_11_10_USCALED,
967 UFMT_11_11_10_SSCALED,
968 UFMT_11_11_10_UINT,
969 UFMT_11_11_10_SINT,
970 UFMT_11_11_10_FLOAT,
971
972 UFMT_10_10_10_2_UNORM,
973 UFMT_10_10_10_2_SNORM,
974 UFMT_10_10_10_2_USCALED,
975 UFMT_10_10_10_2_SSCALED,
976 UFMT_10_10_10_2_UINT,
977 UFMT_10_10_10_2_SINT,
978
979 UFMT_2_10_10_10_UNORM,
980 UFMT_2_10_10_10_SNORM,
981 UFMT_2_10_10_10_USCALED,
982 UFMT_2_10_10_10_SSCALED,
983 UFMT_2_10_10_10_UINT,
984 UFMT_2_10_10_10_SINT,
985
986 UFMT_8_8_8_8_UNORM,
987 UFMT_8_8_8_8_SNORM,
988 UFMT_8_8_8_8_USCALED,
989 UFMT_8_8_8_8_SSCALED,
990 UFMT_8_8_8_8_UINT,
991 UFMT_8_8_8_8_SINT,
992
993 UFMT_32_32_UINT,
994 UFMT_32_32_SINT,
995 UFMT_32_32_FLOAT,
996
997 UFMT_16_16_16_16_UNORM,
998 UFMT_16_16_16_16_SNORM,
999 UFMT_16_16_16_16_USCALED,
1000 UFMT_16_16_16_16_SSCALED,
1001 UFMT_16_16_16_16_UINT,
1002 UFMT_16_16_16_16_SINT,
1003 UFMT_16_16_16_16_FLOAT,
1004
1005 UFMT_32_32_32_UINT,
1006 UFMT_32_32_32_SINT,
1007 UFMT_32_32_32_FLOAT,
1008 UFMT_32_32_32_32_UINT,
1009 UFMT_32_32_32_32_SINT,
1010 UFMT_32_32_32_32_FLOAT,
1011
1012 UFMT_FIRST = UFMT_INVALID,
1013 UFMT_LAST = UFMT_32_32_32_32_FLOAT,
1014};
1015
1016} // namespace UfmtGFX10
1017
1018namespace UfmtGFX11 {
1019enum UnifiedFormat : int64_t {
1020 UFMT_INVALID = 0,
1021
1022 UFMT_8_UNORM,
1023 UFMT_8_SNORM,
1024 UFMT_8_USCALED,
1025 UFMT_8_SSCALED,
1026 UFMT_8_UINT,
1027 UFMT_8_SINT,
1028
1029 UFMT_16_UNORM,
1030 UFMT_16_SNORM,
1031 UFMT_16_USCALED,
1032 UFMT_16_SSCALED,
1033 UFMT_16_UINT,
1034 UFMT_16_SINT,
1035 UFMT_16_FLOAT,
1036
1037 UFMT_8_8_UNORM,
1038 UFMT_8_8_SNORM,
1039 UFMT_8_8_USCALED,
1040 UFMT_8_8_SSCALED,
1041 UFMT_8_8_UINT,
1042 UFMT_8_8_SINT,
1043
1044 UFMT_32_UINT,
1045 UFMT_32_SINT,
1046 UFMT_32_FLOAT,
1047
1048 UFMT_16_16_UNORM,
1049 UFMT_16_16_SNORM,
1050 UFMT_16_16_USCALED,
1051 UFMT_16_16_SSCALED,
1052 UFMT_16_16_UINT,
1053 UFMT_16_16_SINT,
1054 UFMT_16_16_FLOAT,
1055
1056 UFMT_10_11_11_FLOAT,
1057
1058 UFMT_11_11_10_FLOAT,
1059
1060 UFMT_10_10_10_2_UNORM,
1061 UFMT_10_10_10_2_SNORM,
1062 UFMT_10_10_10_2_UINT,
1063 UFMT_10_10_10_2_SINT,
1064
1065 UFMT_2_10_10_10_UNORM,
1066 UFMT_2_10_10_10_SNORM,
1067 UFMT_2_10_10_10_USCALED,
1068 UFMT_2_10_10_10_SSCALED,
1069 UFMT_2_10_10_10_UINT,
1070 UFMT_2_10_10_10_SINT,
1071
1072 UFMT_8_8_8_8_UNORM,
1073 UFMT_8_8_8_8_SNORM,
1074 UFMT_8_8_8_8_USCALED,
1075 UFMT_8_8_8_8_SSCALED,
1076 UFMT_8_8_8_8_UINT,
1077 UFMT_8_8_8_8_SINT,
1078
1079 UFMT_32_32_UINT,
1080 UFMT_32_32_SINT,
1081 UFMT_32_32_FLOAT,
1082
1083 UFMT_16_16_16_16_UNORM,
1084 UFMT_16_16_16_16_SNORM,
1085 UFMT_16_16_16_16_USCALED,
1086 UFMT_16_16_16_16_SSCALED,
1087 UFMT_16_16_16_16_UINT,
1088 UFMT_16_16_16_16_SINT,
1089 UFMT_16_16_16_16_FLOAT,
1090
1091 UFMT_32_32_32_UINT,
1092 UFMT_32_32_32_SINT,
1093 UFMT_32_32_32_FLOAT,
1094 UFMT_32_32_32_32_UINT,
1095 UFMT_32_32_32_32_SINT,
1096 UFMT_32_32_32_32_FLOAT,
1097
1098 UFMT_FIRST = UFMT_INVALID,
1099 UFMT_LAST = UFMT_32_32_32_32_FLOAT,
1100};
1101
1102} // namespace UfmtGFX11
1103
1104namespace Swizzle { // Encoding of swizzle macro used in ds_swizzle_b32.
1105
1106enum Id : unsigned { // id of symbolic names
1107 ID_QUAD_PERM = 0,
1108 ID_BITMASK_PERM,
1109 ID_SWAP,
1110 ID_REVERSE,
1111 ID_BROADCAST,
1112 ID_FFT,
1113 ID_ROTATE
1114};
1115
1116// clang-format off
1117enum EncBits : unsigned {
1118
1119 // swizzle mode encodings
1120
1121 QUAD_PERM_ENC = 0x8000,
1122 QUAD_PERM_ENC_MASK = 0xFF00,
1123
1124 BITMASK_PERM_ENC = 0x0000,
1125 BITMASK_PERM_ENC_MASK = 0x8000,
1126
1127 FFT_MODE_ENC = 0xE000,
1128
1129 ROTATE_MODE_ENC = 0xC000,
1130 FFT_ROTATE_MODE_MASK = 0xF000,
1131
1132 ROTATE_MODE_LO = 0xC000,
1133 FFT_MODE_LO = 0xE000,
1134
1135 // QUAD_PERM encodings
1136
1137 LANE_MASK = 0x3,
1138 LANE_MAX = LANE_MASK,
1139 LANE_SHIFT = 2,
1140 LANE_NUM = 4,
1141
1142 // BITMASK_PERM encodings
1143
1144 BITMASK_MASK = 0x1F,
1145 BITMASK_MAX = BITMASK_MASK,
1146 BITMASK_WIDTH = 5,
1147
1148 BITMASK_AND_SHIFT = 0,
1149 BITMASK_OR_SHIFT = 5,
1150 BITMASK_XOR_SHIFT = 10,
1151
1152 // FFT encodings
1153
1154 FFT_SWIZZLE_MASK = 0x1F,
1155 FFT_SWIZZLE_MAX = 0x1F,
1156
1157 // ROTATE encodings
1158 ROTATE_MAX_SIZE = 0x1F,
1159 ROTATE_DIR_SHIFT = 10, // bit position of rotate direction
1160 ROTATE_DIR_MASK = 0x1,
1161 ROTATE_SIZE_SHIFT = 5, // bit position of rotate size
1162 ROTATE_SIZE_MASK = ROTATE_MAX_SIZE,
1163};
1164// clang-format on
1165
1166} // namespace Swizzle
1167
1168namespace SDWA {
1169
1170enum SdwaSel : unsigned {
1171 BYTE_0 = 0,
1172 BYTE_1 = 1,
1173 BYTE_2 = 2,
1174 BYTE_3 = 3,
1175 WORD_0 = 4,
1176 WORD_1 = 5,
1177 DWORD = 6,
1178};
1179
1180enum DstUnused : unsigned {
1181 UNUSED_PAD = 0,
1182 UNUSED_SEXT = 1,
1183 UNUSED_PRESERVE = 2,
1184};
1185
1186enum SDWA9EncValues : unsigned {
1187 SRC_SGPR_MASK = 0x100,
1188 SRC_VGPR_MASK = 0xFF,
1189 VOPC_DST_VCC_MASK = 0x80,
1190 VOPC_DST_SGPR_MASK = 0x7F,
1191
1192 SRC_VGPR_MIN = 0,
1193 SRC_VGPR_MAX = 255,
1194 SRC_SGPR_MIN = 256,
1195 SRC_SGPR_MAX_SI = 357,
1196 SRC_SGPR_MAX_GFX10 = 361,
1197 SRC_TTMP_MIN = 364,
1198 SRC_TTMP_MAX = 379,
1199};
1200
1201} // namespace SDWA
1202
1203namespace DPP {
1204
1205// clang-format off
1206enum DppCtrl : unsigned {
1207 QUAD_PERM_FIRST = 0,
1208 QUAD_PERM_ID = 0xE4, // identity permutation
1209 QUAD_PERM_LAST = 0xFF,
1210 DPP_UNUSED1 = 0x100,
1211 ROW_SHL0 = 0x100,
1212 ROW_SHL_FIRST = 0x101,
1213 ROW_SHL_LAST = 0x10F,
1214 DPP_UNUSED2 = 0x110,
1215 ROW_SHR0 = 0x110,
1216 ROW_SHR_FIRST = 0x111,
1217 ROW_SHR_LAST = 0x11F,
1218 DPP_UNUSED3 = 0x120,
1219 ROW_ROR0 = 0x120,
1220 ROW_ROR_FIRST = 0x121,
1221 ROW_ROR_LAST = 0x12F,
1222 WAVE_SHL1 = 0x130,
1223 DPP_UNUSED4_FIRST = 0x131,
1224 DPP_UNUSED4_LAST = 0x133,
1225 WAVE_ROL1 = 0x134,
1226 DPP_UNUSED5_FIRST = 0x135,
1227 DPP_UNUSED5_LAST = 0x137,
1228 WAVE_SHR1 = 0x138,
1229 DPP_UNUSED6_FIRST = 0x139,
1230 DPP_UNUSED6_LAST = 0x13B,
1231 WAVE_ROR1 = 0x13C,
1232 DPP_UNUSED7_FIRST = 0x13D,
1233 DPP_UNUSED7_LAST = 0x13F,
1234 ROW_MIRROR = 0x140,
1235 ROW_HALF_MIRROR = 0x141,
1236 BCAST15 = 0x142,
1237 BCAST31 = 0x143,
1238 DPP_UNUSED8_FIRST = 0x144,
1239 DPP_UNUSED8_LAST = 0x14F,
1240 ROW_NEWBCAST_FIRST= 0x150,
1241 ROW_NEWBCAST_LAST = 0x15F,
1242 ROW_SHARE0 = 0x150,
1243 ROW_SHARE_FIRST = 0x150,
1244 ROW_SHARE_LAST = 0x15F,
1245 ROW_XMASK0 = 0x160,
1246 ROW_XMASK_FIRST = 0x160,
1247 ROW_XMASK_LAST = 0x16F,
1248 DPP_LAST = ROW_XMASK_LAST
1249};
1250// clang-format on
1251
1252enum DppFiMode {
1253 DPP_FI_0 = 0,
1254 DPP_FI_1 = 1,
1255 DPP8_FI_0 = 0xE9,
1256 DPP8_FI_1 = 0xEA,
1257};
1258
1259} // namespace DPP
1260
1261namespace Exp {
1262
1263enum Target : unsigned {
1264 ET_MRT0 = 0,
1265 ET_MRT7 = 7,
1266 ET_MRTZ = 8,
1267 ET_NULL = 9, // Pre-GFX11
1268 ET_POS0 = 12,
1269 ET_POS3 = 15,
1270 ET_POS4 = 16, // GFX10+
1271 ET_POS_LAST = ET_POS4, // Highest pos used on any subtarget
1272 ET_PRIM = 20, // GFX10+
1273 ET_DUAL_SRC_BLEND0 = 21, // GFX11+
1274 ET_DUAL_SRC_BLEND1 = 22, // GFX11+
1275 ET_PARAM0 = 32, // Pre-GFX11
1276 ET_PARAM31 = 63, // Pre-GFX11
1277
1278 ET_NULL_MAX_IDX = 0,
1279 ET_MRTZ_MAX_IDX = 0,
1280 ET_PRIM_MAX_IDX = 0,
1281 ET_MRT_MAX_IDX = 7,
1282 ET_POS_MAX_IDX = 4,
1283 ET_DUAL_SRC_BLEND_MAX_IDX = 1,
1284 ET_PARAM_MAX_IDX = 31,
1285
1286 ET_INVALID = 255,
1287};
1288
1289} // namespace Exp
1290
1291namespace WMMA {
1292enum MatrixFMT : unsigned {
1293 MATRIX_FMT_FP8 = 0,
1294 MATRIX_FMT_BF8 = 1,
1295 MATRIX_FMT_FP6 = 2,
1296 MATRIX_FMT_BF6 = 3,
1297 MATRIX_FMT_FP4 = 4
1298};
1299
1300enum MatrixScale : unsigned {
1301 MATRIX_SCALE_ROW0 = 0,
1302 MATRIX_SCALE_ROW1 = 1,
1303};
1304
1305enum MatrixScaleFmt : unsigned {
1306 MATRIX_SCALE_FMT_E8 = 0,
1307 MATRIX_SCALE_FMT_E5M3 = 1,
1308 MATRIX_SCALE_FMT_E4M3 = 2
1309};
1310} // namespace WMMA
1311
1312namespace VOP3PEncoding {
1313
1314enum OpSel : uint64_t {
1315 OP_SEL_HI_0 = UINT64_C(1) << 59,
1316 OP_SEL_HI_1 = UINT64_C(1) << 60,
1317 OP_SEL_HI_2 = UINT64_C(1) << 14,
1318};
1319
1320} // namespace VOP3PEncoding
1321
1322namespace ImplicitArg {
1323// Implicit kernel argument offset for code object version 5.
1324enum Offset_COV5 : unsigned {
1325 HOSTCALL_PTR_OFFSET = 80,
1326 MULTIGRID_SYNC_ARG_OFFSET = 88,
1327 HEAP_PTR_OFFSET = 96,
1328
1329 DEFAULT_QUEUE_OFFSET = 104,
1330 COMPLETION_ACTION_OFFSET = 112,
1331
1332 PRIVATE_BASE_OFFSET = 192,
1333 SHARED_BASE_OFFSET = 196,
1334 QUEUE_PTR_OFFSET = 200,
1335};
1336
1337} // namespace ImplicitArg
1338
1339namespace MFMAScaleFormats {
1340// Enum value used in cbsz/blgp for F8F6F4 MFMA operations to select the matrix
1341// format.
1342enum MFMAScaleFormats {
1343 FP8_E4M3 = 0,
1344 FP8_E5M2 = 1,
1345 FP6_E2M3 = 2,
1346 FP6_E3M2 = 3,
1347 FP4_E2M1 = 4
1348};
1349} // namespace MFMAScaleFormats
1350
1351namespace VirtRegFlag {
1352// Virtual register flags used for various target specific handlings during
1353// codegen.
1354enum Register_Flag : uint8_t {
1355 // Register operand in a whole-wave mode operation.
1356 WWM_REG = 1 << 0,
1357};
1358
1359} // namespace VirtRegFlag
1360
1361} // namespace AMDGPU
1362
1363namespace AMDGPU {
1364namespace Barrier {
1365
1366enum Type {
1367 CLUSTER_TRAP = -4,
1368 CLUSTER = -3,
1369 TRAP = -2,
1370 WORKGROUP = -1,
1371 NAMED_BARRIER_FIRST = 1,
1372 NAMED_BARRIER_LAST = 16,
1373};
1374
1375} // namespace Barrier
1376
1377namespace SyntheticAperture {
1378/// Synthetic aperture numbers. This is OR'd with the high
1379/// bits of the LDS aperture pointer.
1380///
1381/// NOTE: This is also documented in AMDGPUUsage.
1382enum SyntheticAperture {
1383 None = 0x00000000,
1384
1385 BARRIER = 0x00000001,
1386 VGPR = 0x00000003,
1387};
1388} // namespace SyntheticAperture
1389
1390} // namespace AMDGPU
1391
1392// clang-format off
1393
1394#define R_00B028_SPI_SHADER_PGM_RSRC1_PS 0x00B028
1395#define S_00B028_VGPRS(x) (((x) & 0x3F) << 0)
1396#define S_00B028_SGPRS(x) (((x) & 0x0F) << 6)
1397#define S_00B028_MEM_ORDERED(x) (((x) & 0x1) << 25)
1398#define G_00B028_MEM_ORDERED(x) (((x) >> 25) & 0x1)
1399#define C_00B028_MEM_ORDERED 0xFDFFFFFF
1400
1401#define R_00B02C_SPI_SHADER_PGM_RSRC2_PS 0x00B02C
1402#define S_00B02C_EXTRA_LDS_SIZE(x) (((x) & 0xFF) << 8)
1403#define R_00B128_SPI_SHADER_PGM_RSRC1_VS 0x00B128
1404#define S_00B128_MEM_ORDERED(x) (((x) & 0x1) << 27)
1405#define G_00B128_MEM_ORDERED(x) (((x) >> 27) & 0x1)
1406#define C_00B128_MEM_ORDERED 0xF7FFFFFF
1407
1408#define R_00B228_SPI_SHADER_PGM_RSRC1_GS 0x00B228
1409#define S_00B228_WGP_MODE(x) (((x) & 0x1) << 27)
1410#define G_00B228_WGP_MODE(x) (((x) >> 27) & 0x1)
1411#define C_00B228_WGP_MODE 0xF7FFFFFF
1412#define S_00B228_MEM_ORDERED(x) (((x) & 0x1) << 25)
1413#define G_00B228_MEM_ORDERED(x) (((x) >> 25) & 0x1)
1414#define C_00B228_MEM_ORDERED 0xFDFFFFFF
1415
1416#define R_00B328_SPI_SHADER_PGM_RSRC1_ES 0x00B328
1417#define R_00B428_SPI_SHADER_PGM_RSRC1_HS 0x00B428
1418#define S_00B428_WGP_MODE(x) (((x) & 0x1) << 26)
1419#define G_00B428_WGP_MODE(x) (((x) >> 26) & 0x1)
1420#define C_00B428_WGP_MODE 0xFBFFFFFF
1421#define S_00B428_MEM_ORDERED(x) (((x) & 0x1) << 24)
1422#define G_00B428_MEM_ORDERED(x) (((x) >> 24) & 0x1)
1423#define C_00B428_MEM_ORDERED 0xFEFFFFFF
1424
1425#define R_00B528_SPI_SHADER_PGM_RSRC1_LS 0x00B528
1426
1427#define R_00B84C_COMPUTE_PGM_RSRC2 0x00B84C
1428#define S_00B84C_SCRATCH_EN(x) (((x) & 0x1) << 0)
1429#define G_00B84C_SCRATCH_EN(x) (((x) >> 0) & 0x1)
1430#define C_00B84C_SCRATCH_EN 0xFFFFFFFE
1431#define S_00B84C_USER_SGPR(x) (((x) & 0x1F) << 1)
1432#define G_00B84C_USER_SGPR(x) (((x) >> 1) & 0x1F)
1433#define C_00B84C_USER_SGPR 0xFFFFFFC1
1434#define S_00B84C_TRAP_HANDLER(x) (((x) & 0x1) << 6)
1435#define G_00B84C_TRAP_HANDLER(x) (((x) >> 6) & 0x1)
1436#define C_00B84C_TRAP_HANDLER 0xFFFFFFBF
1437#define S_00B84C_TGID_X_EN(x) (((x) & 0x1) << 7)
1438#define G_00B84C_TGID_X_EN(x) (((x) >> 7) & 0x1)
1439#define C_00B84C_TGID_X_EN 0xFFFFFF7F
1440#define S_00B84C_TGID_Y_EN(x) (((x) & 0x1) << 8)
1441#define G_00B84C_TGID_Y_EN(x) (((x) >> 8) & 0x1)
1442#define C_00B84C_TGID_Y_EN 0xFFFFFEFF
1443#define S_00B84C_TGID_Z_EN(x) (((x) & 0x1) << 9)
1444#define G_00B84C_TGID_Z_EN(x) (((x) >> 9) & 0x1)
1445#define C_00B84C_TGID_Z_EN 0xFFFFFDFF
1446#define S_00B84C_TG_SIZE_EN(x) (((x) & 0x1) << 10)
1447#define G_00B84C_TG_SIZE_EN(x) (((x) >> 10) & 0x1)
1448#define C_00B84C_TG_SIZE_EN 0xFFFFFBFF
1449#define S_00B84C_TIDIG_COMP_CNT(x) (((x) & 0x03) << 11)
1450#define G_00B84C_TIDIG_COMP_CNT(x) (((x) >> 11) & 0x03)
1451#define C_00B84C_TIDIG_COMP_CNT 0xFFFFE7FF
1452/* CIK */
1453#define S_00B84C_EXCP_EN_MSB(x) (((x) & 0x03) << 13)
1454#define G_00B84C_EXCP_EN_MSB(x) (((x) >> 13) & 0x03)
1455#define C_00B84C_EXCP_EN_MSB 0xFFFF9FFF
1456/* */
1457#define S_00B84C_LDS_SIZE(x) (((x) & 0x1FF) << 15)
1458#define G_00B84C_LDS_SIZE(x) (((x) >> 15) & 0x1FF)
1459#define C_00B84C_LDS_SIZE 0xFF007FFF
1460#define S_00B84C_EXCP_EN(x) (((x) & 0x7F) << 24)
1461#define G_00B84C_EXCP_EN(x) (((x) >> 24) & 0x7F)
1462#define C_00B84C_EXCP_EN 0x80FFFFFF
1463
1464#define S_00B84C_USER_SGPR_GFX1250(x) (((x) & 0x3F) << 1)
1465#define G_00B84C_USER_SGPR_GFX1250(x) (((x) >> 1) & 0x3F)
1466#define C_00B84C_USER_SGPR_GFX1250 0xFFFFFF81
1467
1468#define R_0286CC_SPI_PS_INPUT_ENA 0x0286CC
1469#define R_0286D0_SPI_PS_INPUT_ADDR 0x0286D0
1470
1471#define R_00B848_COMPUTE_PGM_RSRC1 0x00B848
1472#define S_00B848_VGPRS(x) (((x) & 0x3F) << 0)
1473#define G_00B848_VGPRS(x) (((x) >> 0) & 0x3F)
1474#define C_00B848_VGPRS 0xFFFFFFC0
1475#define S_00B848_SGPRS(x) (((x) & 0x0F) << 6)
1476#define G_00B848_SGPRS(x) (((x) >> 6) & 0x0F)
1477#define C_00B848_SGPRS 0xFFFFFC3F
1478#define S_00B848_PRIORITY(x) (((x) & 0x03) << 10)
1479#define G_00B848_PRIORITY(x) (((x) >> 10) & 0x03)
1480#define C_00B848_PRIORITY 0xFFFFF3FF
1481#define S_00B848_FLOAT_MODE(x) (((x) & 0xFF) << 12)
1482#define G_00B848_FLOAT_MODE(x) (((x) >> 12) & 0xFF)
1483#define C_00B848_FLOAT_MODE 0xFFF00FFF
1484#define S_00B848_PRIV(x) (((x) & 0x1) << 20)
1485#define G_00B848_PRIV(x) (((x) >> 20) & 0x1)
1486#define C_00B848_PRIV 0xFFEFFFFF
1487#define S_00B848_DX10_CLAMP(x) (((x) & 0x1) << 21)
1488#define G_00B848_DX10_CLAMP(x) (((x) >> 21) & 0x1)
1489#define C_00B848_DX10_CLAMP 0xFFDFFFFF
1490#define S_00B848_RR_WG_MODE(x) (((x) & 0x1) << 21)
1491#define G_00B848_RR_WG_MODE(x) (((x) >> 21) & 0x1)
1492#define C_00B848_RR_WG_MODE 0xFFDFFFFF
1493#define S_00B848_DEBUG_MODE(x) (((x) & 0x1) << 22)
1494#define G_00B848_DEBUG_MODE(x) (((x) >> 22) & 0x1)
1495#define C_00B848_DEBUG_MODE 0xFFBFFFFF
1496#define S_00B848_IEEE_MODE(x) (((x) & 0x1) << 23)
1497#define G_00B848_IEEE_MODE(x) (((x) >> 23) & 0x1)
1498#define C_00B848_IEEE_MODE 0xFF7FFFFF
1499#define S_00B848_WGP_MODE(x) (((x) & 0x1) << 29)
1500#define G_00B848_WGP_MODE(x) (((x) >> 29) & 0x1)
1501#define C_00B848_WGP_MODE 0xDFFFFFFF
1502#define S_00B848_MEM_ORDERED(x) (((x) & 0x1) << 30)
1503#define G_00B848_MEM_ORDERED(x) (((x) >> 30) & 0x1)
1504#define C_00B848_MEM_ORDERED 0xBFFFFFFF
1505#define S_00B848_FWD_PROGRESS(x) (((x) & 0x1) << 31)
1506#define G_00B848_FWD_PROGRESS(x) (((x) >> 31) & 0x1)
1507#define C_00B848_FWD_PROGRESS 0x7FFFFFFF
1508
1509// Helpers for setting FLOAT_MODE
1510#define FP_ROUND_ROUND_TO_NEAREST 0
1511#define FP_ROUND_ROUND_TO_INF 1
1512#define FP_ROUND_ROUND_TO_NEGINF 2
1513#define FP_ROUND_ROUND_TO_ZERO 3
1514
1515// Bits 3:0 control rounding mode. 1:0 control single precision, 3:2 double
1516// precision.
1517#define FP_ROUND_MODE_SP(x) ((x) & 0x3)
1518#define FP_ROUND_MODE_DP(x) (((x) & 0x3) << 2)
1519
1520#define FP_DENORM_FLUSH_IN_FLUSH_OUT 0
1521#define FP_DENORM_FLUSH_OUT 1
1522#define FP_DENORM_FLUSH_IN 2
1523#define FP_DENORM_FLUSH_NONE 3
1524
1525
1526// Bits 7:4 control denormal handling. 5:4 control single precision, 6:7 double
1527// precision.
1528#define FP_DENORM_MODE_SP(x) (((x) & 0x3) << 4)
1529#define FP_DENORM_MODE_DP(x) (((x) & 0x3) << 6)
1530
1531#define R_00B860_COMPUTE_TMPRING_SIZE 0x00B860
1532#define S_00B860_WAVESIZE_PreGFX11(x) (((x) & 0x1FFF) << 12)
1533#define S_00B860_WAVESIZE_GFX11(x) (((x) & 0x7FFF) << 12)
1534#define S_00B860_WAVESIZE_GFX12Plus(x) (((x) & 0x3FFFF) << 12)
1535
1536#define R_0286E8_SPI_TMPRING_SIZE 0x0286E8
1537#define S_0286E8_WAVESIZE_PreGFX11(x) (((x) & 0x1FFF) << 12)
1538#define S_0286E8_WAVESIZE_GFX11(x) (((x) & 0x7FFF) << 12)
1539#define S_0286E8_WAVESIZE_GFX12Plus(x) (((x) & 0x3FFFF) << 12)
1540
1541#define R_028B54_VGT_SHADER_STAGES_EN 0x028B54
1542#define S_028B54_HS_W32_EN(x) (((x) & 0x1) << 21)
1543#define S_028B54_GS_W32_EN(x) (((x) & 0x1) << 22)
1544#define S_028B54_VS_W32_EN(x) (((x) & 0x1) << 23)
1545#define R_0286D8_SPI_PS_IN_CONTROL 0x0286D8
1546#define S_0286D8_PS_W32_EN(x) (((x) & 0x1) << 15)
1547#define R_00B800_COMPUTE_DISPATCH_INITIATOR 0x00B800
1548#define S_00B800_CS_W32_EN(x) (((x) & 0x1) << 15)
1549
1550#define R_SPILLED_SGPRS 0x4
1551#define R_SPILLED_VGPRS 0x8
1552
1553// clang-format on
1554
1555} // End namespace llvm
1556
1557#endif
1558