1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file contains NVVM-specific IR verification logic. These checks are
11/// always compiled and linked as part of LLVMCore.
12///
13//===----------------------------------------------------------------------===//
14
15#include "VerifierInternal.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/DerivedTypes.h"
18#include "llvm/IR/IntrinsicsNVPTX.h"
19#include "llvm/IR/NVVMIntrinsicUtils.h"
20#include "llvm/Support/FormatVariadic.h"
21#include "llvm/Support/MathExtras.h"
22#include <optional>
23
24using namespace llvm;
25
26#define Check(C, ...) \
27 do { \
28 if (!(C)) { \
29 VS.CheckFailed(__VA_ARGS__); \
30 return; \
31 } \
32 } while (false)
33
34namespace {
35
36struct SPVectorInfo {
37 unsigned ElemSize;
38 unsigned NumElements;
39 unsigned NumRegisters;
40};
41
42// Register layout of the sparse intrinsic operands, used by the IR verifier.
43struct SPOperandLayout {
44 unsigned MetadataSize;
45 unsigned CompressedDataSize;
46 unsigned DataSize;
47};
48
49} // namespace
50
51// PTX limits the combined vector size of the mdata, cdata, and data operands
52// of spcompress and spdecompress to 253 32-bit registers.
53constexpr unsigned MaxSPOperandRegisters = 253;
54
55static bool isValidSPElemSize(unsigned ElemSize) {
56 return ElemSize == 8 || ElemSize == 16;
57}
58
59static bool isValidSPIdxSize(unsigned IdxSize) {
60 return IdxSize == 2 || IdxSize == 4;
61}
62
63static bool isValidSPRepeatFactor(unsigned RepeatFactor) {
64 return isPowerOf2_32(Value: RepeatFactor) && RepeatFactor <= 64;
65}
66
67static bool isValidSPDecompressFactor(unsigned NumSrc, unsigned NumTgt) {
68 switch (NumSrc) {
69 case 1:
70 return NumTgt == 2 || NumTgt == 4 || NumTgt == 8 || NumTgt == 16;
71 case 2:
72 return NumTgt == 4 || NumTgt == 8 || NumTgt == 16;
73 case 4:
74 return NumTgt == 8 || NumTgt == 16;
75 default:
76 return false;
77 }
78}
79
80static std::optional<SPOperandLayout>
81getSPCompressLayout(unsigned ElemSize, unsigned IdxSize,
82 unsigned RepeatFactor) {
83 if (!isValidSPElemSize(ElemSize) || !isValidSPIdxSize(IdxSize) ||
84 !isValidSPRepeatFactor(RepeatFactor))
85 return std::nullopt;
86
87 SPOperandLayout Layout = {.MetadataSize: divideCeil(Numerator: RepeatFactor * IdxSize, Denominator: ElemSize),
88 .CompressedDataSize: RepeatFactor, .DataSize: RepeatFactor * 2};
89 if (Layout.MetadataSize + Layout.CompressedDataSize + Layout.DataSize >
90 MaxSPOperandRegisters)
91 return std::nullopt;
92 return Layout;
93}
94
95static std::optional<SPOperandLayout>
96getSPDecompressLayout(unsigned NumSrc, unsigned NumTgt, unsigned ElemSize,
97 unsigned IdxSize, unsigned RepeatFactor) {
98 if (!isValidSPDecompressFactor(NumSrc, NumTgt) ||
99 !isValidSPElemSize(ElemSize) || !isValidSPIdxSize(IdxSize) ||
100 !isValidSPRepeatFactor(RepeatFactor) || NumSrc * ElemSize > 32 ||
101 (IdxSize == 2 && NumTgt > 4))
102 return std::nullopt;
103
104 unsigned DataBits = NumTgt * ElemSize * RepeatFactor;
105 if (DataBits < 32 || DataBits > 4096)
106 return std::nullopt;
107
108 SPOperandLayout Layout = {.MetadataSize: divideCeil(Numerator: NumSrc * IdxSize * RepeatFactor, Denominator: 32),
109 .CompressedDataSize: divideCeil(Numerator: NumSrc * ElemSize * RepeatFactor, Denominator: 32),
110 .DataSize: divideCeil(Numerator: DataBits, Denominator: 32)};
111 if (Layout.MetadataSize + Layout.CompressedDataSize + Layout.DataSize >
112 MaxSPOperandRegisters)
113 return std::nullopt;
114 return Layout;
115}
116
117static std::optional<SPVectorInfo> getSPVectorInfo(Type *Ty) {
118 auto *VT = dyn_cast<FixedVectorType>(Val: Ty);
119 if (!VT)
120 return std::nullopt;
121 auto *ElemTy = dyn_cast<IntegerType>(Val: VT->getElementType());
122 if (!ElemTy)
123 return std::nullopt;
124 unsigned ElemSize = ElemTy->getBitWidth();
125 if (ElemSize != 8 && ElemSize != 16)
126 return std::nullopt;
127 unsigned NumElements = VT->getNumElements();
128 return SPVectorInfo{.ElemSize: ElemSize, .NumElements: NumElements,
129 .NumRegisters: divideCeil(Numerator: NumElements, Denominator: 32 / ElemSize)};
130}
131
132static std::optional<unsigned> getSPMetadataRegisters(Type *Ty) {
133 if (Ty->isIntegerTy(BitWidth: 32))
134 return 1;
135 auto *VT = dyn_cast<FixedVectorType>(Val: Ty);
136 if (!VT || !VT->getElementType()->isIntegerTy(BitWidth: 32) || VT->getNumElements() < 2)
137 return std::nullopt;
138 return VT->getNumElements();
139}
140
141static void verifySPCompress(VerifierSupport &VS, CallBase &Call) {
142 auto *ResultTy = dyn_cast<StructType>(Val: Call.getType());
143 Check(ResultTy && ResultTy->getNumElements() == 2,
144 "invalid llvm.nvvm.spcompress result type", &Call);
145
146 auto MDataRegs = getSPMetadataRegisters(Ty: ResultTy->getElementType(N: 0));
147 auto CData = getSPVectorInfo(Ty: ResultTy->getElementType(N: 1));
148 auto Data = getSPVectorInfo(Ty: Call.getArgOperand(i: 0)->getType());
149 Check(MDataRegs && CData && Data && CData->ElemSize == Data->ElemSize,
150 "invalid llvm.nvvm.spcompress operand or result type", &Call);
151
152 unsigned IdxSize = cast<ConstantInt>(Val: Call.getArgOperand(i: 2))->getZExtValue();
153 unsigned NumTgt = cast<ConstantInt>(Val: Call.getArgOperand(i: 3))->getZExtValue();
154 // spcompress only implements the 2:4 pattern: each group of num_tgt = 4
155 // data elements keeps 2 of them in cdata. The repeat factor counts pairs of
156 // data registers, so data must occupy an even number of them.
157 Check(NumTgt == 4 && Data->NumElements % NumTgt == 0 &&
158 CData->NumElements == 2 * (Data->NumElements / NumTgt) &&
159 Data->NumRegisters % 2 == 0,
160 "invalid llvm.nvvm.spcompress layout", &Call);
161
162 unsigned RepeatFactor = Data->NumRegisters / 2;
163 auto Layout = getSPCompressLayout(ElemSize: Data->ElemSize, IdxSize, RepeatFactor);
164 // The declared types must match the register layout PTX gives these
165 // qualifiers.
166 Check(Layout && *MDataRegs == Layout->MetadataSize &&
167 CData->NumRegisters == Layout->CompressedDataSize &&
168 Data->NumRegisters == Layout->DataSize,
169 "invalid llvm.nvvm.spcompress layout", &Call);
170}
171
172static void verifySPDecompress(VerifierSupport &VS, CallBase &Call) {
173 auto Data = getSPVectorInfo(Ty: Call.getType());
174 auto MDataRegs = getSPMetadataRegisters(Ty: Call.getArgOperand(i: 0)->getType());
175 auto CData = getSPVectorInfo(Ty: Call.getArgOperand(i: 1)->getType());
176 Check(Data && MDataRegs && CData && Data->ElemSize == CData->ElemSize,
177 "invalid llvm.nvvm.spdecompress operand or result type", &Call);
178
179 unsigned IdxSize = cast<ConstantInt>(Val: Call.getArgOperand(i: 2))->getZExtValue();
180 unsigned NumTgt = cast<ConstantInt>(Val: Call.getArgOperand(i: 3))->getZExtValue();
181 // data holds repeat_factor groups of num_tgt elements.
182 Check(NumTgt != 0 && Data->NumElements % NumTgt == 0,
183 "invalid llvm.nvvm.spdecompress layout", &Call);
184
185 unsigned RepeatFactor = Data->NumElements / NumTgt;
186 // cdata holds num_src elements for each of those groups.
187 Check(RepeatFactor != 0 && CData->NumElements % RepeatFactor == 0,
188 "invalid llvm.nvvm.spdecompress layout", &Call);
189
190 unsigned NumSrc = CData->NumElements / RepeatFactor;
191 auto Layout = getSPDecompressLayout(NumSrc, NumTgt, ElemSize: Data->ElemSize, IdxSize,
192 RepeatFactor);
193 // The declared types must match the register layout PTX gives these
194 // qualifiers.
195 Check(Layout && *MDataRegs == Layout->MetadataSize &&
196 CData->NumRegisters == Layout->CompressedDataSize &&
197 Data->NumRegisters == Layout->DataSize,
198 "invalid llvm.nvvm.spdecompress layout", &Call);
199}
200
201// Returns true if the NVVM fadd/fmul/fma intrinsic with the given operand type
202// and rounding mode corresponds to a PTX instruction.
203//
204// TODO: Replace this with explicit type constraints on the intrinsic
205// definitions once https://github.com/llvm/llvm-project/pull/172442 lands.
206static bool isSupportedFPArith(Intrinsic::ID ID, Type *Ty,
207 APFloat::roundingMode RoundingMode) {
208 const bool IsVector = Ty->isVectorTy();
209 const auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
210 if (IsVector && (!VTy || VTy->getNumElements() != 2))
211 return false;
212
213 const bool IsRN = RoundingMode == APFloat::rmNearestTiesToEven;
214 const bool IsFTZ = nvvm::FPArithShouldFTZ(IntrinsicID: ID);
215 const bool IsSat = nvvm::FPArithIsSaturating(IntrinsicID: ID);
216 const bool IsRelu = nvvm::FPArithIsRelu(IntrinsicID: ID);
217 switch (Ty->getScalarType()->getTypeID()) {
218 case Type::HalfTyID:
219 return IsRN;
220 case Type::BFloatTyID:
221 return IsRN && !IsSat && !IsFTZ;
222 case Type::FloatTyID:
223 return !IsRelu && !(IsVector && IsSat);
224 case Type::DoubleTyID:
225 return !IsVector && !IsSat && !IsFTZ && !IsRelu;
226 default:
227 return false;
228 }
229}
230
231static void verifyFPArith(VerifierSupport &VS, Intrinsic::ID ID,
232 CallBase &Call) {
233 // The rounding mode is the trailing operand of all these intrinsics.
234 const APFloat::roundingMode RoundingMode =
235 nvvm::GetRoundingModeFromImmArg(ImmArgVal: Call.getArgOperand(i: Call.arg_size() - 1));
236 Type *Ty = Call.getType();
237 Check(isSupportedFPArith(ID, Ty, RoundingMode),
238 formatv("{} with rounding mode {} and operand type {} is not supported",
239 Intrinsic::getBaseName(ID),
240 nvvm::GetRoundingModeName(RoundingMode), *Ty),
241 &Call);
242}
243
244void llvm::verifyNVVMIntrinsicCall(VerifierSupport &VS, Intrinsic::ID ID,
245 CallBase &Call) {
246 switch (ID) {
247 default:
248 return;
249 case Intrinsic::nvvm_spcompress:
250 verifySPCompress(VS, Call);
251 return;
252 case Intrinsic::nvvm_spdecompress:
253 verifySPDecompress(VS, Call);
254 return;
255 case Intrinsic::nvvm_fadd:
256 case Intrinsic::nvvm_fadd_ftz:
257 case Intrinsic::nvvm_fadd_sat:
258 case Intrinsic::nvvm_fadd_ftz_sat:
259 case Intrinsic::nvvm_fma:
260 case Intrinsic::nvvm_fma_ftz:
261 case Intrinsic::nvvm_fma_sat:
262 case Intrinsic::nvvm_fma_ftz_sat:
263 case Intrinsic::nvvm_fma_relu:
264 case Intrinsic::nvvm_fma_ftz_relu:
265 case Intrinsic::nvvm_fmul:
266 case Intrinsic::nvvm_fmul_ftz:
267 case Intrinsic::nvvm_fmul_sat:
268 case Intrinsic::nvvm_fmul_ftz_sat:
269 verifyFPArith(VS, ID, Call);
270 return;
271 }
272}
273
274#undef Check
275