1//===-- VerifierAMDGPU.cpp - AMDGPU-specific IR verification ---------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains AMDGPU-specific IR verification logic that was extracted
10// from Verifier.cpp for code organization purposes only. These checks are
11// always compiled and linked as part of LLVMCore — this is not a target-
12// dependent IR verifier, which would require a different design.
13//
14// This file should only contain checks for AMDGPU-specific IR constructs
15// (e.g. amdgcn intrinsics, AMDGPU address spaces). It must not contain
16// checks for generic IR that might behave differently under AMDGPU.
17//
18//===----------------------------------------------------------------------===//
19
20#include "VerifierInternal.h"
21#include "llvm/ADT/StringExtras.h"
22#include "llvm/IR/CallingConv.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/Function.h"
26#include "llvm/IR/GlobalVariable.h"
27#include "llvm/IR/IntrinsicInst.h"
28#include "llvm/IR/IntrinsicsAMDGPU.h"
29#include "llvm/Support/AMDGPUAddrSpace.h"
30
31using namespace llvm;
32
33#define Check(C, ...) \
34 do { \
35 if (!(C)) { \
36 VS.CheckFailed(__VA_ARGS__); \
37 return; \
38 } \
39 } while (false)
40
41void llvm::verifyAMDGPUModuleFlag(VerifierSupport &VS, const MDString *ID,
42 Module::ModFlagBehavior MFB,
43 const MDNode *Op) {
44 StringRef FlagName = ID->getString();
45 if (!FlagName.consume_front(Prefix: "amdgpu."))
46 return;
47
48 if (FlagName == "buffer.oob.mode" || FlagName == "tbuffer.oob.mode") {
49 Check(MFB == Module::Max,
50 "'" + ID->getString() +
51 "' module flag must use 'max' merge behaviour");
52 ConstantInt *Value =
53 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 2));
54 Check(Value, "'" + ID->getString() +
55 "' module flag must have a constant integer value");
56 Check(Value->getZExtValue() <= 2,
57 "'" + ID->getString() + "' module flag must be 0, 1, or 2");
58 return;
59 }
60
61 if (FlagName == "xnack" || FlagName == "sramecc") {
62 Check(MFB == Module::Error,
63 "'" + ID->getString() +
64 "' module flag must use 'error' merge behaviour");
65 ConstantInt *Value =
66 mdconst::dyn_extract_or_null<ConstantInt>(MD: Op->getOperand(I: 2));
67 Check(Value, "'" + ID->getString() +
68 "' module flag must have a constant integer value");
69 Check(Value->getZExtValue() <= 1,
70 "'" + ID->getString() + "' module flag must be 0 or 1");
71 return;
72 }
73}
74
75// Verify that when a function has !reqd_work_group_size metadata, it also has
76// an amdgpu-flat-work-group-size attribute that matches the product of the
77// reqd_work_group_size operands.
78static void verifyAMDGPUReqdWorkGroupSize(VerifierSupport &VS,
79 const Function &F) {
80 // This is not required for other targets so we only check for AMDGPU.
81 if (!VS.TT.isAMDGPU())
82 return;
83
84 MDNode *ReqdWorkGroupSize = F.getMetadata(Kind: "reqd_work_group_size");
85 if (!ReqdWorkGroupSize || ReqdWorkGroupSize->getNumOperands() != 3)
86 return;
87
88 uint64_t Product = 1;
89 for (const MDOperand &Op : ReqdWorkGroupSize->operands()) {
90 ConstantInt *C = mdconst::dyn_extract<ConstantInt>(MD: Op);
91 if (!C || C->getValue().getActiveBits() > 64)
92 return;
93 uint64_t Dim = C->getZExtValue();
94 if (Dim != 0 && Product > std::numeric_limits<uint64_t>::max() / Dim)
95 return;
96 Product *= Dim;
97 }
98
99 Attribute FlatWorkGroupSize = F.getFnAttribute(Kind: "amdgpu-flat-work-group-size");
100 if (!FlatWorkGroupSize.isValid()) {
101 VS.CheckFailed(Message: "reqd_work_group_size requires amdgpu-flat-work-group-size",
102 V1: &F, Vs: ReqdWorkGroupSize);
103 return;
104 }
105
106 if (!FlatWorkGroupSize.isStringAttribute()) {
107 VS.CheckFailed(Message: "amdgpu-flat-work-group-size must be a string attribute",
108 V1: &F);
109 return;
110 }
111
112 StringRef AttrValue = FlatWorkGroupSize.getValueAsString();
113 std::pair<StringRef, StringRef> Values = AttrValue.split(Separator: ',');
114 uint64_t Min = 0;
115 uint64_t Max = 0;
116 bool Parsed = !Values.second.contains(C: ',') &&
117 llvm::to_integer(S: Values.first.trim(), Num&: Min) &&
118 llvm::to_integer(S: Values.second.trim(), Num&: Max);
119 if (!Parsed) {
120 VS.CheckFailed(Message: "amdgpu-flat-work-group-size must be a pair of unsigned "
121 "integers",
122 V1: &F);
123 return;
124 }
125
126 if (Min != Product || Max != Product) {
127 VS.CheckFailed(Message: "amdgpu-flat-work-group-size must equal the product of "
128 "reqd_work_group_size operands",
129 V1: &F, Vs: ReqdWorkGroupSize);
130 }
131}
132
133void llvm::verifyAMDGPUFunctionMetadata(VerifierSupport &VS,
134 const Function &F) {
135 verifyAMDGPUReqdWorkGroupSize(VS, F);
136}
137
138void llvm::verifyAMDGPUGlobalVariable(VerifierSupport &VS,
139 const GlobalVariable &GV) {
140 // This is not required for other targets so we only check for AMDGPU.
141 if (!VS.TT.isAMDGPU())
142 return;
143
144 // The VGPR address space is a view of one wave's own vector registers, which
145 // exist only while that wave runs. A global variable needs storage that
146 // outlives any particular wave, so there is nothing here for it to name.
147 if (GV.getAddressSpace() == AMDGPUAS::VGPR)
148 VS.CheckFailed(Message: "global variable on amdgpu must not be in addrspace(13)",
149 V1: &GV);
150}
151
152void llvm::verifyAMDGPUAlloca(VerifierSupport &VS, const AllocaInst &AI) {
153 // This is not required for other targets so we only check for AMDGPU.
154 if (!VS.TT.isAMDGPU())
155 return;
156
157 if (AI.getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS &&
158 AI.getAddressSpace() != AMDGPUAS::VGPR)
159 VS.CheckFailed(Message: "alloca on amdgpu must be in addrspace(5) or addrspace(13)",
160 V1: &AI);
161
162 // Only static allocas can live in VGPRs; a dynamically sized one has no
163 // register-file representation. (Other address spaces are already rejected
164 // above, so this only adds the more specific diagnostic for addrspace(13).)
165 if (!AI.isStaticAlloca() && AI.getAddressSpace() == AMDGPUAS::VGPR)
166 VS.CheckFailed(Message: "dynamic alloca on amdgpu must be in addrspace(5)", V1: &AI);
167}
168
169bool llvm::isAMDGPUCallBrIntrinsic(Intrinsic::ID ID) {
170 switch (ID) {
171 default:
172 return false;
173 case Intrinsic::amdgcn_kill:
174 return true;
175 }
176}
177
178void llvm::verifyAMDGPUIntrinsicCall(VerifierSupport &VS, Intrinsic::ID ID,
179 CallBase &Call) {
180 switch (ID) {
181 default:
182 return;
183 case Intrinsic::amdgcn_kill: {
184 if (auto *CBI = dyn_cast<CallBrInst>(Val: &Call)) {
185 Check(CBI->getNumIndirectDests() == 1,
186 "callbr amdgcn_kill only supports one indirect dest");
187 // We assume that amdgcn_unreachable is only introduced by
188 // AMDGPUUnifyDivergentExitNodes, which replaces the block's original
189 // unreachable terminator by a call to amdgcn_unreachable + a return.
190 const Instruction *Term = CBI->getIndirectDest(i: 0)->getTerminator();
191 const CallInst *CI =
192 Term ? dyn_cast_if_present<CallInst>(Val: Term->getPrevNode()) : nullptr;
193 Check(isa_and_nonnull<UnreachableInst>(Term) ||
194 (CI && CI->getIntrinsicID() == Intrinsic::amdgcn_unreachable),
195 "callbr amdgcn_kill indirect dest needs to be unreachable");
196 }
197 break;
198 }
199 case Intrinsic::amdgcn_cs_chain: {
200 CallingConv::ID CallerCC = Call.getCaller()->getCallingConv();
201 switch (CallerCC) {
202 case CallingConv::AMDGPU_CS:
203 case CallingConv::AMDGPU_CS_Chain:
204 case CallingConv::AMDGPU_CS_ChainPreserve:
205 case CallingConv::AMDGPU_ES:
206 case CallingConv::AMDGPU_GS:
207 case CallingConv::AMDGPU_HS:
208 case CallingConv::AMDGPU_LS:
209 case CallingConv::AMDGPU_VS:
210 break;
211 default:
212 VS.CheckFailed(Message: "Intrinsic cannot be called from functions with this "
213 "calling convention",
214 V1: &Call);
215 break;
216 }
217
218 Check(Call.paramHasAttr(2, Attribute::InReg),
219 "SGPR arguments must have the `inreg` attribute", &Call);
220 Check(!Call.paramHasAttr(3, Attribute::InReg),
221 "VGPR arguments must not have the `inreg` attribute", &Call);
222
223 ConstantInt *FlagsArg = cast<ConstantInt>(Val: Call.getArgOperand(i: 4));
224 Check(FlagsArg->getValue().ult(2),
225 "flags must be 0 or 1 for llvm.amdgcn.cs.chain", &Call);
226
227 Instruction *Next = Call.getNextNode();
228 bool IsAMDUnreachable = isa_and_nonnull<IntrinsicInst>(Val: Next) &&
229 cast<IntrinsicInst>(Val: Next)->getIntrinsicID() ==
230 Intrinsic::amdgcn_unreachable;
231 Check(Next && (isa<UnreachableInst>(Next) || IsAMDUnreachable),
232 "llvm.amdgcn.cs.chain must be followed by unreachable", &Call);
233 break;
234 }
235 case Intrinsic::amdgcn_init_exec_from_input: {
236 const Argument *Arg = dyn_cast<Argument>(Val: Call.getOperand(i_nocapture: 0));
237 Check(Arg && Arg->hasInRegAttr(),
238 "only inreg arguments to the parent function are valid as inputs to "
239 "this intrinsic",
240 &Call);
241 break;
242 }
243 case Intrinsic::amdgcn_set_inactive_chain_arg: {
244 CallingConv::ID CallerCC = Call.getCaller()->getCallingConv();
245 switch (CallerCC) {
246 case CallingConv::AMDGPU_CS_Chain:
247 case CallingConv::AMDGPU_CS_ChainPreserve:
248 break;
249 default:
250 VS.CheckFailed(Message: "Intrinsic can only be used from functions with the "
251 "amdgpu_cs_chain or amdgpu_cs_chain_preserve "
252 "calling conventions",
253 V1: &Call);
254 break;
255 }
256
257 unsigned InactiveIdx = 1;
258 Check(!Call.paramHasAttr(InactiveIdx, Attribute::InReg),
259 "Value for inactive lanes must not have the `inreg` attribute",
260 &Call);
261 Check(isa<Argument>(Call.getArgOperand(InactiveIdx)),
262 "Value for inactive lanes must be a function argument", &Call);
263 Check(!cast<Argument>(Call.getArgOperand(InactiveIdx))->hasInRegAttr(),
264 "Value for inactive lanes must be a VGPR function argument", &Call);
265 break;
266 }
267 case Intrinsic::amdgcn_call_whole_wave: {
268 Function *F = dyn_cast<Function>(Val: Call.getArgOperand(i: 0));
269 Check(F, "Indirect whole wave calls are not allowed", &Call);
270
271 CallingConv::ID CC = F->getCallingConv();
272 Check(CC == CallingConv::AMDGPU_Gfx_WholeWave,
273 "Callee must have the amdgpu_gfx_whole_wave calling convention",
274 &Call);
275
276 Check(!F->isVarArg(), "Variadic whole wave calls are not allowed", &Call);
277
278 Check(Call.arg_size() == F->arg_size(),
279 "Call argument count must match callee argument count", &Call);
280
281 Check(F->arg_begin()->getType()->isIntegerTy(1),
282 "Callee must have i1 as its first argument", &Call);
283 for (auto [CallArg, FuncArg] :
284 drop_begin(RangeOrContainer: zip_equal(t: Call.args(), u: F->args()))) {
285 Check(CallArg->getType() == FuncArg.getType(),
286 "Argument types must match", &Call);
287
288 Check(Call.paramHasAttr(FuncArg.getArgNo(), Attribute::InReg) ==
289 FuncArg.hasInRegAttr(),
290 "Argument inreg attributes must match", &Call);
291 }
292 break;
293 }
294 case Intrinsic::amdgcn_s_prefetch_data: {
295 Check(
296 AMDGPU::isFlatGlobalAddrSpace(
297 Call.getArgOperand(0)->getType()->getPointerAddressSpace()),
298 "llvm.amdgcn.s.prefetch.data only supports global or constant memory");
299 break;
300 }
301 case Intrinsic::amdgcn_load_to_lds:
302 case Intrinsic::amdgcn_load_async_to_lds:
303 case Intrinsic::amdgcn_global_load_lds:
304 case Intrinsic::amdgcn_global_load_async_lds:
305 case Intrinsic::amdgcn_raw_buffer_load_lds:
306 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
307 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
308 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
309 case Intrinsic::amdgcn_struct_buffer_load_lds:
310 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
311 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
312 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
313 uint64_t Size = cast<ConstantInt>(Val: Call.getArgOperand(i: 2))->getZExtValue();
314 Check(Size == 1 || Size == 2 || Size == 4 || Size == 12 || Size == 16,
315 "invalid data size for load-to-LDS intrinsic; must be 1, 2, 4, 12, "
316 "or 16",
317 &Call);
318 break;
319 }
320 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
321 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
322 Value *Src0 = Call.getArgOperand(i: 0);
323 Value *Src1 = Call.getArgOperand(i: 1);
324
325 uint64_t CBSZ = cast<ConstantInt>(Val: Call.getArgOperand(i: 3))->getZExtValue();
326 uint64_t BLGP = cast<ConstantInt>(Val: Call.getArgOperand(i: 4))->getZExtValue();
327 Check(CBSZ <= 4, "invalid value for cbsz format", Call,
328 Call.getArgOperand(3));
329 Check(BLGP <= 4, "invalid value for blgp format", Call,
330 Call.getArgOperand(4));
331
332 auto GetFormatNumRegs = [](unsigned FormatVal) {
333 switch (FormatVal) {
334 case 0:
335 case 1:
336 return 8u;
337 case 2:
338 case 3:
339 return 6u;
340 case 4:
341 return 4u;
342 default:
343 llvm_unreachable("invalid format value");
344 }
345 };
346
347 auto IsValidSrcASrcBVector = [](FixedVectorType *Ty) {
348 if (!Ty || !Ty->getElementType()->isIntegerTy(BitWidth: 32))
349 return false;
350 unsigned NumElts = Ty->getNumElements();
351 return NumElts == 4 || NumElts == 6 || NumElts == 8;
352 };
353
354 FixedVectorType *Src0Ty = dyn_cast<FixedVectorType>(Val: Src0->getType());
355 FixedVectorType *Src1Ty = dyn_cast<FixedVectorType>(Val: Src1->getType());
356 Check(IsValidSrcASrcBVector(Src0Ty),
357 "operand 0 must be 4, 6 or 8 element i32 vector", &Call, Src0);
358 Check(IsValidSrcASrcBVector(Src1Ty),
359 "operand 1 must be 4, 6 or 8 element i32 vector", &Call, Src1);
360
361 Check(Src0Ty->getNumElements() >= GetFormatNumRegs(CBSZ),
362 "invalid vector type for format", &Call, Src0, Call.getArgOperand(3));
363 Check(Src1Ty->getNumElements() >= GetFormatNumRegs(BLGP),
364 "invalid vector type for format", &Call, Src1, Call.getArgOperand(5));
365 break;
366 }
367 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
368 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
369 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
370 Value *Src0 = Call.getArgOperand(i: 1);
371 Value *Src1 = Call.getArgOperand(i: 3);
372
373 unsigned FmtA = cast<ConstantInt>(Val: Call.getArgOperand(i: 0))->getZExtValue();
374 unsigned FmtB = cast<ConstantInt>(Val: Call.getArgOperand(i: 2))->getZExtValue();
375 Check(FmtA <= 4, "invalid value for matrix format", Call,
376 Call.getArgOperand(0));
377 Check(FmtB <= 4, "invalid value for matrix format", Call,
378 Call.getArgOperand(2));
379
380 auto GetFormatNumRegs = [](unsigned FormatVal) {
381 switch (FormatVal) {
382 case 0:
383 case 1:
384 return 16u;
385 case 2:
386 case 3:
387 return 12u;
388 case 4:
389 return 8u;
390 default:
391 llvm_unreachable("invalid format value");
392 }
393 };
394
395 auto IsValidSrcASrcBVector = [](FixedVectorType *Ty) {
396 if (!Ty || !Ty->getElementType()->isIntegerTy(BitWidth: 32))
397 return false;
398 unsigned NumElts = Ty->getNumElements();
399 return NumElts == 16 || NumElts == 12 || NumElts == 8;
400 };
401
402 FixedVectorType *Src0Ty = dyn_cast<FixedVectorType>(Val: Src0->getType());
403 FixedVectorType *Src1Ty = dyn_cast<FixedVectorType>(Val: Src1->getType());
404 Check(IsValidSrcASrcBVector(Src0Ty),
405 "operand 1 must be 8, 12 or 16 element i32 vector", &Call, Src0);
406 Check(IsValidSrcASrcBVector(Src1Ty),
407 "operand 3 must be 8, 12 or 16 element i32 vector", &Call, Src1);
408
409 Check(Src0Ty->getNumElements() >= GetFormatNumRegs(FmtA),
410 "invalid vector type for format", &Call, Src0, Call.getArgOperand(0));
411 Check(Src1Ty->getNumElements() >= GetFormatNumRegs(FmtB),
412 "invalid vector type for format", &Call, Src1, Call.getArgOperand(2));
413 break;
414 }
415 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
416 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
417 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
418 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
419 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
420 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
421 Value *PtrArg = Call.getArgOperand(i: 0);
422 const unsigned AS = PtrArg->getType()->getPointerAddressSpace();
423 Check(AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS,
424 "cooperative atomic intrinsics require a generic or global pointer",
425 &Call, PtrArg);
426
427 MetadataAsValue *Op =
428 cast<MetadataAsValue>(Val: Call.getArgOperand(i: Call.arg_size() - 1));
429 MDNode *MD = cast<MDNode>(Val: Op->getMetadata());
430 Check((MD->getNumOperands() == 1) && isa<MDString>(MD->getOperand(0)),
431 "cooperative atomic intrinsics require that the last argument is a "
432 "metadata string",
433 &Call, Op);
434 break;
435 }
436 case Intrinsic::amdgcn_av_load_b128:
437 case Intrinsic::amdgcn_av_store_b128: {
438 MetadataAsValue *Op =
439 cast<MetadataAsValue>(Val: Call.getArgOperand(i: Call.arg_size() - 1));
440 MDNode *MD = dyn_cast<MDNode>(Val: Op->getMetadata());
441 Check(MD && (MD->getNumOperands() == 1) && isa<MDString>(MD->getOperand(0)),
442 "the last argument to av load/store intrinsics must be a "
443 "metadata string",
444 &Call, Op);
445 break;
446 }
447 }
448}
449
450#undef Check
451