1//===- SPIR.cpp -----------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "HLSLBufferLayoutBuilder.h"
11#include "TargetInfo.h"
12#include "clang/AST/DeclCXX.h"
13#include "clang/Basic/LangOptions.h"
14#include "llvm/IR/DerivedTypes.h"
15#include "llvm/IR/LLVMContext.h"
16
17#include <stdint.h>
18#include <utility>
19
20using namespace clang;
21using namespace clang::CodeGen;
22
23//===----------------------------------------------------------------------===//
24// Base ABI and target codegen info implementation common between SPIR and
25// SPIR-V.
26//===----------------------------------------------------------------------===//
27
28namespace {
29class CommonSPIRABIInfo : public DefaultABIInfo {
30public:
31 CommonSPIRABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) { setCCs(); }
32
33private:
34 void setCCs();
35};
36
37class SPIRVABIInfo : public CommonSPIRABIInfo {
38public:
39 SPIRVABIInfo(CodeGenTypes &CGT) : CommonSPIRABIInfo(CGT) {}
40 void computeInfo(CGFunctionInfo &FI) const override;
41 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
42 AggValueSlot Slot) const override;
43
44 llvm::FixedVectorType *
45 getOptimalVectorMemoryType(llvm::FixedVectorType *Ty,
46 const LangOptions &LangOpt) const override;
47
48private:
49 ABIArgInfo classifyKernelArgumentType(QualType Ty) const;
50};
51
52class AMDGCNSPIRVABIInfo : public SPIRVABIInfo {
53 // TODO: this should be unified / shared with AMDGPU, ideally we'd like to
54 // re-use AMDGPUABIInfo eventually, rather than duplicate.
55 static constexpr unsigned MaxNumRegsForArgsRet = 16; // 16 32-bit registers
56 mutable unsigned NumRegsLeft = 0;
57
58 uint64_t numRegsForType(QualType Ty) const;
59
60 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
61 return true;
62 }
63 bool isHomogeneousAggregateSmallEnough(const Type *Base,
64 uint64_t Members) const override {
65 uint32_t NumRegs = (getContext().getTypeSize(T: Base) + 31) / 32;
66
67 // Homogeneous Aggregates may occupy at most 16 registers.
68 return Members * NumRegs <= MaxNumRegsForArgsRet;
69 }
70
71 // Coerce HIP scalar pointer arguments from generic pointers to global ones.
72 llvm::Type *coerceKernelArgumentType(llvm::Type *Ty, unsigned FromAS,
73 unsigned ToAS) const;
74
75 ABIArgInfo classifyReturnType(QualType RetTy) const;
76 ABIArgInfo classifyKernelArgumentType(QualType Ty) const;
77 ABIArgInfo classifyArgumentType(QualType Ty, bool Variadic) const;
78
79public:
80 AMDGCNSPIRVABIInfo(CodeGenTypes &CGT) : SPIRVABIInfo(CGT) {}
81 void computeInfo(CGFunctionInfo &FI) const override;
82
83 llvm::FixedVectorType *
84 getOptimalVectorMemoryType(llvm::FixedVectorType *Ty,
85 const LangOptions &LangOpt) const override;
86};
87} // end anonymous namespace
88namespace {
89class CommonSPIRTargetCodeGenInfo : public TargetCodeGenInfo {
90public:
91 CommonSPIRTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
92 : TargetCodeGenInfo(std::make_unique<CommonSPIRABIInfo>(args&: CGT)) {}
93 CommonSPIRTargetCodeGenInfo(std::unique_ptr<ABIInfo> ABIInfo)
94 : TargetCodeGenInfo(std::move(ABIInfo)) {}
95
96 unsigned getDeviceKernelCallingConv() const override;
97 llvm::Type *getOpenCLType(CodeGenModule &CGM, const Type *T) const override;
98 llvm::Type *getHLSLType(CodeGenModule &CGM, const Type *Ty,
99 const CGHLSLOffsetInfo &OffsetInfo) const override;
100
101 llvm::Type *getHLSLPadding(CodeGenModule &CGM,
102 CharUnits NumBytes) const override {
103 unsigned Size = NumBytes.getQuantity();
104 return llvm::TargetExtType::get(Context&: CGM.getLLVMContext(), Name: "spirv.Padding", Types: {},
105 Ints: {Size});
106 }
107
108 bool isHLSLPadding(llvm::Type *Ty) const override {
109 if (auto *TET = dyn_cast<llvm::TargetExtType>(Val: Ty))
110 return TET->getName() == "spirv.Padding";
111 return false;
112 }
113
114 llvm::Type *getSPIRVImageTypeFromHLSLResource(
115 const HLSLAttributedResourceType::Attributes &attributes,
116 QualType SampledType, CodeGenModule &CGM) const;
117 void
118 setOCLKernelStubCallingConvention(const FunctionType *&FT) const override;
119 llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM,
120 llvm::PointerType *T,
121 QualType QT) const override;
122};
123class SPIRVTargetCodeGenInfo : public CommonSPIRTargetCodeGenInfo {
124public:
125 SPIRVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
126 : CommonSPIRTargetCodeGenInfo(
127 (CGT.getTarget().getTriple().getVendor() == llvm::Triple::AMD)
128 ? std::make_unique<AMDGCNSPIRVABIInfo>(args&: CGT)
129 : std::make_unique<SPIRVABIInfo>(args&: CGT)) {}
130 void setCUDAKernelCallingConvention(const FunctionType *&FT) const override;
131 LangAS getGlobalVarAddressSpace(CodeGenModule &CGM,
132 const VarDecl *D) const override;
133 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
134 CodeGen::CodeGenModule &M) const override;
135 StringRef getLLVMSyncScopeStr(const LangOptions &LangOpts, SyncScope Scope,
136 llvm::AtomicOrdering Ordering) const override;
137 void setTargetAtomicMetadata(CodeGenFunction &CGF,
138 llvm::Instruction &AtomicInst,
139 const AtomicExpr *Expr = nullptr) const override;
140 bool supportsLibCall() const override {
141 return getABIInfo().getTarget().getTriple().getVendor() !=
142 llvm::Triple::AMD;
143 }
144
145 LangAS getSRetAddrSpace(const CXXRecordDecl *RD) const override;
146};
147} // End anonymous namespace.
148
149void CommonSPIRABIInfo::setCCs() {
150 assert(getRuntimeCC() == llvm::CallingConv::C);
151 RuntimeCC = llvm::CallingConv::SPIR_FUNC;
152}
153
154ABIArgInfo SPIRVABIInfo::classifyKernelArgumentType(QualType Ty) const {
155 // Coerce pointer arguments with default address space to CrossWorkGroup
156 // pointers as default address space kernel
157 // arguments are not allowed. We use the opencl_global language address
158 // space which always maps to CrossWorkGroup.
159 llvm::Type *LTy = CGT.ConvertType(T: Ty);
160 auto DefaultAS = getContext().getTargetAddressSpace(AS: LangAS::Default);
161 auto GlobalAS = getContext().getTargetAddressSpace(AS: LangAS::opencl_global);
162 auto *PtrTy = llvm::dyn_cast<llvm::PointerType>(Val: LTy);
163 if (PtrTy && PtrTy->getAddressSpace() == DefaultAS) {
164 LTy = llvm::PointerType::get(C&: PtrTy->getContext(), AddressSpace: GlobalAS);
165 return ABIArgInfo::getDirect(T: LTy, Offset: 0, Padding: nullptr, CanBeFlattened: false);
166 }
167
168 if (getContext().getLangOpts().isTargetDevice() &&
169 isAggregateTypeForABI(T: Ty)) {
170 // Force copying aggregate type in kernel arguments by value when
171 // compiling CUDA targeting SPIR-V. This is required for the object
172 // copied to be valid on the device.
173 // This behavior follows the CUDA spec
174 // https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#global-function-argument-processing,
175 // and matches the NVPTX implementation. TODO: hardcoding to 0 should be
176 // revisited if HIPSPV / byval starts making use of the AS of an indirect
177 // arg.
178 return getNaturalAlignIndirect(Ty, /*AddrSpace=*/0, /*byval=*/ByVal: true);
179 }
180 return classifyArgumentType(RetTy: Ty);
181}
182
183void SPIRVABIInfo::computeInfo(CGFunctionInfo &FI) const {
184 // The logic is same as in DefaultABIInfo with an exception on the kernel
185 // arguments handling.
186 llvm::CallingConv::ID CC = FI.getCallingConvention();
187
188 for (auto &&[ArgumentsCount, I] : llvm::enumerate(First: FI.arguments()))
189 I.info = ArgumentsCount < FI.getNumRequiredArgs()
190 ? classifyArgumentType(RetTy: I.type)
191 : ABIArgInfo::getDirect();
192
193 if (!getCXXABI().classifyReturnType(FI))
194 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType());
195
196 for (auto &I : FI.arguments()) {
197 if (CC == llvm::CallingConv::SPIR_KERNEL) {
198 I.info = classifyKernelArgumentType(Ty: I.type);
199 } else {
200 I.info = classifyArgumentType(RetTy: I.type);
201 }
202 }
203}
204
205RValue SPIRVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
206 QualType Ty, AggValueSlot Slot) const {
207 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, /*IsIndirect=*/false,
208 ValueInfo: getContext().getTypeInfoInChars(T: Ty),
209 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 1),
210 /*AllowHigherAlign=*/true, Slot);
211}
212
213uint64_t AMDGCNSPIRVABIInfo::numRegsForType(QualType Ty) const {
214 // This duplicates the AMDGPUABI computation.
215 uint64_t NumRegs = 0;
216
217 if (const VectorType *VT = Ty->getAs<VectorType>()) {
218 // Compute from the number of elements. The reported size is based on the
219 // in-memory size, which includes the padding 4th element for 3-vectors.
220 QualType EltTy = VT->getElementType();
221 uint64_t EltSize = getContext().getTypeSize(T: EltTy);
222
223 // 16-bit element vectors should be passed as packed.
224 if (EltSize == 16)
225 return (VT->getNumElements() + 1) / 2;
226
227 uint64_t EltNumRegs = (EltSize + 31) / 32;
228 return EltNumRegs * VT->getNumElements();
229 }
230
231 if (const auto *RD = Ty->getAsRecordDecl()) {
232 assert(!RD->hasFlexibleArrayMember());
233
234 for (const FieldDecl *Field : RD->fields()) {
235 QualType FieldTy = Field->getType();
236 NumRegs += numRegsForType(Ty: FieldTy);
237 }
238
239 return NumRegs;
240 }
241
242 return (getContext().getTypeSize(T: Ty) + 31) / 32;
243}
244
245llvm::Type *AMDGCNSPIRVABIInfo::coerceKernelArgumentType(llvm::Type *Ty,
246 unsigned FromAS,
247 unsigned ToAS) const {
248 // Single value types.
249 auto *PtrTy = llvm::dyn_cast<llvm::PointerType>(Val: Ty);
250 if (PtrTy && PtrTy->getAddressSpace() == FromAS)
251 return llvm::PointerType::get(C&: Ty->getContext(), AddressSpace: ToAS);
252 return Ty;
253}
254
255ABIArgInfo AMDGCNSPIRVABIInfo::classifyReturnType(QualType RetTy) const {
256 if (!isAggregateTypeForABI(T: RetTy) || getRecordArgABI(T: RetTy, CXXABI&: getCXXABI()))
257 return DefaultABIInfo::classifyReturnType(RetTy);
258
259 // Ignore empty structs/unions.
260 if (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true))
261 return ABIArgInfo::getIgnore();
262
263 // Lower single-element structs to just return a regular value.
264 if (const Type *SeltTy = isSingleElementStruct(T: RetTy, Context&: getContext()))
265 return ABIArgInfo::getDirect(T: CGT.ConvertType(T: QualType(SeltTy, 0)));
266
267 if (const auto *RD = RetTy->getAsRecordDecl();
268 RD && RD->hasFlexibleArrayMember())
269 return DefaultABIInfo::classifyReturnType(RetTy);
270
271 // Pack aggregates <= 4 bytes into single VGPR or pair.
272 uint64_t Size = getContext().getTypeSize(T: RetTy);
273 if (Size <= 16)
274 return ABIArgInfo::getDirect(T: llvm::Type::getInt16Ty(C&: getVMContext()));
275
276 if (Size <= 32)
277 return ABIArgInfo::getDirect(T: llvm::Type::getInt32Ty(C&: getVMContext()));
278
279 // TODO: This carried over from AMDGPU oddity, we retain it to
280 // ensure consistency, but it might be reasonable to return Int64.
281 if (Size <= 64) {
282 llvm::Type *I32Ty = llvm::Type::getInt32Ty(C&: getVMContext());
283 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: I32Ty, NumElements: 2));
284 }
285
286 if (numRegsForType(Ty: RetTy) <= MaxNumRegsForArgsRet)
287 return ABIArgInfo::getDirect();
288 return DefaultABIInfo::classifyReturnType(RetTy);
289}
290
291/// For kernels all parameters are really passed in a special buffer. It doesn't
292/// make sense to pass anything byval, so everything must be direct.
293ABIArgInfo AMDGCNSPIRVABIInfo::classifyKernelArgumentType(QualType Ty) const {
294 Ty = useFirstFieldIfTransparentUnion(Ty);
295
296 // TODO: Can we omit empty structs?
297
298 if (const Type *SeltTy = isSingleElementStruct(T: Ty, Context&: getContext()))
299 Ty = QualType(SeltTy, 0);
300
301 llvm::Type *OrigLTy = CGT.ConvertType(T: Ty);
302 llvm::Type *LTy = OrigLTy;
303 if (getContext().getLangOpts().isTargetDevice()) {
304 LTy = coerceKernelArgumentType(
305 Ty: OrigLTy, /*FromAS=*/getContext().getTargetAddressSpace(AS: LangAS::Default),
306 /*ToAS=*/getContext().getTargetAddressSpace(AS: LangAS::opencl_global));
307 }
308
309 // FIXME: This doesn't apply the optimization of coercing pointers in structs
310 // to global address space when using byref. This would require implementing a
311 // new kind of coercion of the in-memory type when for indirect arguments.
312 if (LTy == OrigLTy && isAggregateTypeForABI(T: Ty)) {
313 return ABIArgInfo::getIndirectAliased(
314 Alignment: getContext().getTypeAlignInChars(T: Ty),
315 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_constant),
316 Realign: false /*Realign*/, Padding: nullptr /*Padding*/);
317 }
318
319 // TODO: inhibiting flattening is an AMDGPU workaround for Clover, which might
320 // be vestigial and should be revisited.
321 return ABIArgInfo::getDirect(T: LTy, Offset: 0, Padding: nullptr, CanBeFlattened: false);
322}
323
324ABIArgInfo AMDGCNSPIRVABIInfo::classifyArgumentType(QualType Ty,
325 bool Variadic) const {
326 assert(NumRegsLeft <= MaxNumRegsForArgsRet && "register estimate underflow");
327
328 Ty = useFirstFieldIfTransparentUnion(Ty);
329
330 if (Variadic) {
331 return ABIArgInfo::getDirect(/*T=*/nullptr,
332 /*Offset=*/0,
333 /*Padding=*/nullptr,
334 /*CanBeFlattened=*/false,
335 /*Align=*/0);
336 }
337
338 if (!isAggregateTypeForABI(T: Ty)) {
339 ABIArgInfo ArgInfo = DefaultABIInfo::classifyArgumentType(RetTy: Ty);
340 if (!ArgInfo.isIndirect()) {
341 uint64_t NumRegs = numRegsForType(Ty);
342 NumRegsLeft -= std::min(a: NumRegs, b: uint64_t{NumRegsLeft});
343 }
344
345 return ArgInfo;
346 }
347
348 // Records with non-trivial destructors/copy-constructors should not be
349 // passed by value.
350 if (auto RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI()))
351 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
352 ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
353
354 // Ignore empty structs/unions.
355 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
356 return ABIArgInfo::getIgnore();
357
358 // Lower single-element structs to just pass a regular value. TODO: We
359 // could do reasonable-size multiple-element structs too, using getExpand(),
360 // though watch out for things like bitfields.
361 if (const Type *SeltTy = isSingleElementStruct(T: Ty, Context&: getContext()))
362 return ABIArgInfo::getDirect(T: CGT.ConvertType(T: QualType(SeltTy, 0)));
363
364 if (const auto *RD = Ty->getAsRecordDecl();
365 RD && RD->hasFlexibleArrayMember())
366 return DefaultABIInfo::classifyArgumentType(RetTy: Ty);
367
368 uint64_t Size = getContext().getTypeSize(T: Ty);
369 if (Size <= 64) {
370 // Pack aggregates <= 8 bytes into single VGPR or pair.
371 unsigned NumRegs = (Size + 31) / 32;
372 NumRegsLeft -= std::min(a: NumRegsLeft, b: NumRegs);
373
374 if (Size <= 16)
375 return ABIArgInfo::getDirect(T: llvm::Type::getInt16Ty(C&: getVMContext()));
376
377 if (Size <= 32)
378 return ABIArgInfo::getDirect(T: llvm::Type::getInt32Ty(C&: getVMContext()));
379
380 // TODO: This is an AMDGPU oddity, and might be vestigial, we retain it to
381 // ensure consistency, but it should be revisited.
382 llvm::Type *I32Ty = llvm::Type::getInt32Ty(C&: getVMContext());
383 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: I32Ty, NumElements: 2));
384 }
385
386 if (NumRegsLeft > 0) {
387 uint64_t NumRegs = numRegsForType(Ty);
388 if (NumRegsLeft >= NumRegs) {
389 NumRegsLeft -= NumRegs;
390 return ABIArgInfo::getDirect();
391 }
392 }
393
394 // Use pass-by-reference in stead of pass-by-value for struct arguments in
395 // function ABI.
396 return ABIArgInfo::getIndirectAliased(
397 Alignment: getContext().getTypeAlignInChars(T: Ty),
398 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_private));
399}
400
401void AMDGCNSPIRVABIInfo::computeInfo(CGFunctionInfo &FI) const {
402 llvm::CallingConv::ID CC = FI.getCallingConvention();
403
404 if (!getCXXABI().classifyReturnType(FI))
405 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType());
406
407 unsigned ArgumentIndex = 0;
408 const unsigned NumRequiredArgs = FI.getNumRequiredArgs();
409
410 NumRegsLeft = MaxNumRegsForArgsRet;
411 for (auto &I : FI.arguments()) {
412 if (CC == llvm::CallingConv::SPIR_KERNEL) {
413 I.info = classifyKernelArgumentType(Ty: I.type);
414 } else {
415 bool FixedArgument = ArgumentIndex++ < NumRequiredArgs;
416 I.info = classifyArgumentType(Ty: I.type, Variadic: !FixedArgument);
417 }
418 }
419}
420
421llvm::FixedVectorType *
422SPIRVABIInfo::getOptimalVectorMemoryType(llvm::FixedVectorType *Ty,
423 const LangOptions &LangOpt) const {
424 // For Logical SPIR-V, we don't know the underlying hardware or layout.
425 // This means we don't know which vector size is better, and also cannot
426 // assume a smaller vector size is stored in a larger vector size.
427 if (getTarget().getTriple().isSPIRVLogical())
428 return Ty;
429 return DefaultABIInfo::getOptimalVectorMemoryType(T: Ty, Opt: LangOpt);
430}
431
432llvm::FixedVectorType *AMDGCNSPIRVABIInfo::getOptimalVectorMemoryType(
433 llvm::FixedVectorType *Ty, const LangOptions &LangOpt) const {
434 // AMDGPU has legal instructions for 96-bit so 3x32 can be supported.
435 if (Ty->getNumElements() == 3 && getDataLayout().getTypeSizeInBits(Ty) == 96)
436 return Ty;
437 return DefaultABIInfo::getOptimalVectorMemoryType(T: Ty, Opt: LangOpt);
438}
439
440namespace clang {
441namespace CodeGen {
442void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI) {
443 if (CGM.getTarget().getTriple().isSPIRV()) {
444 if (CGM.getTarget().getTriple().getVendor() == llvm::Triple::AMD)
445 AMDGCNSPIRVABIInfo(CGM.getTypes()).computeInfo(FI);
446 else
447 SPIRVABIInfo(CGM.getTypes()).computeInfo(FI);
448 } else {
449 CommonSPIRABIInfo(CGM.getTypes()).computeInfo(FI);
450 }
451}
452}
453}
454
455unsigned CommonSPIRTargetCodeGenInfo::getDeviceKernelCallingConv() const {
456 return llvm::CallingConv::SPIR_KERNEL;
457}
458
459LangAS SPIRVTargetCodeGenInfo::getSRetAddrSpace(const CXXRecordDecl *RD) const {
460 // Types with no viable copy/move must be constructed in-place, use the
461 // default AS so the sret pointer matches the "this" convention.
462 if (RD && !RD->canPassInRegisters())
463 return LangAS::Default;
464 return getLangASFromTargetAS(
465 TargetAS: getABIInfo().getDataLayout().getAllocaAddrSpace());
466}
467
468void SPIRVTargetCodeGenInfo::setCUDAKernelCallingConvention(
469 const FunctionType *&FT) const {
470 // Convert HIP kernels to SPIR-V kernels.
471 if (getABIInfo().getContext().getLangOpts().HIP) {
472 FT = getABIInfo().getContext().adjustFunctionType(
473 Fn: FT, EInfo: FT->getExtInfo().withCallingConv(cc: CC_DeviceKernel));
474 return;
475 }
476}
477
478void CommonSPIRTargetCodeGenInfo::setOCLKernelStubCallingConvention(
479 const FunctionType *&FT) const {
480 FT = getABIInfo().getContext().adjustFunctionType(
481 Fn: FT, EInfo: FT->getExtInfo().withCallingConv(cc: CC_C));
482}
483
484// LLVM currently assumes a null pointer has the bit pattern 0, but some GPU
485// targets use a non-zero encoding for null in certain address spaces.
486// Because SPIR(-V) is a generic target and the bit pattern of null in
487// non-generic AS is unspecified, materialize null in non-generic AS via an
488// addrspacecast from null in generic AS. This allows later lowering to
489// substitute the target's real sentinel value.
490llvm::Constant *
491CommonSPIRTargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM,
492 llvm::PointerType *PT,
493 QualType QT) const {
494 LangAS AS = QT->getUnqualifiedDesugaredType()->isNullPtrType()
495 ? LangAS::Default
496 : QT->getPointeeType().getAddressSpace();
497 unsigned ASAsInt = static_cast<unsigned>(AS);
498 unsigned FirstTargetASAsInt =
499 static_cast<unsigned>(LangAS::FirstTargetAddressSpace);
500 unsigned CodeSectionINTELAS = FirstTargetASAsInt + 9;
501 // As per SPV_INTEL_function_pointers, it is illegal to addrspacecast
502 // function pointers to/from the generic AS.
503 bool IsFunctionPtrAS =
504 CGM.getTriple().isSPIRV() && ASAsInt == CodeSectionINTELAS;
505 if (AS == LangAS::Default || AS == LangAS::opencl_generic ||
506 AS == LangAS::opencl_constant || IsFunctionPtrAS)
507 return llvm::ConstantPointerNull::get(T: PT);
508
509 auto &Ctx = CGM.getContext();
510 auto NPT = llvm::PointerType::get(
511 C&: PT->getContext(), AddressSpace: Ctx.getTargetAddressSpace(AS: LangAS::opencl_generic));
512 return llvm::ConstantExpr::getAddrSpaceCast(
513 C: llvm::ConstantPointerNull::get(T: NPT), Ty: PT);
514}
515
516LangAS
517SPIRVTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
518 const VarDecl *D) const {
519 assert(!CGM.getLangOpts().OpenCL &&
520 !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
521 "Address space agnostic languages only");
522 // If we're here it means that we're using the SPIRDefIsGen ASMap, hence for
523 // the global AS we can rely on either cuda_device or sycl_global to be
524 // correct; however, since this is not a CUDA Device context, we use
525 // sycl_global to prevent confusion with the assertion.
526 LangAS DefaultGlobalAS = getLangASFromTargetAS(
527 TargetAS: CGM.getContext().getTargetAddressSpace(AS: LangAS::sycl_global));
528 if (!D)
529 return DefaultGlobalAS;
530
531 LangAS AddrSpace = D->getType().getAddressSpace();
532 if (AddrSpace != LangAS::Default)
533 return AddrSpace;
534
535 return DefaultGlobalAS;
536}
537
538void SPIRVTargetCodeGenInfo::setTargetAttributes(
539 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
540 if (GV->isDeclaration())
541 return;
542
543 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
544 if (!FD)
545 return;
546
547 llvm::Function *F = dyn_cast<llvm::Function>(Val: GV);
548 assert(F && "Expected GlobalValue to be a Function");
549
550 if (!M.getLangOpts().HIP ||
551 M.getTarget().getTriple().getVendor() != llvm::Triple::AMD)
552 return;
553
554 if (!FD->hasAttr<CUDAGlobalAttr>())
555 return;
556
557 unsigned N = M.getLangOpts().GPUMaxThreadsPerBlock;
558 if (auto FlatWGS = FD->getAttr<AMDGPUFlatWorkGroupSizeAttr>()) {
559 N = FlatWGS->getMax()->EvaluateKnownConstInt(Ctx: M.getContext()).getExtValue();
560 } else if (auto LB = FD->getAttr<CUDALaunchBoundsAttr>()) {
561 if (uint64_t MaxThreads = LB->getMaxThreads()
562 ->EvaluateKnownConstInt(Ctx: M.getContext())
563 .getExtValue())
564 N = MaxThreads;
565 }
566
567 // We encode the maximum flat WG size in the first component of the 3D
568 // max_work_group_size attribute, which will get reverse translated into the
569 // original AMDGPU attribute when targeting AMDGPU.
570 auto Int32Ty = llvm::IntegerType::getInt32Ty(C&: M.getLLVMContext());
571 llvm::Metadata *AttrMDArgs[] = {
572 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: Int32Ty, V: N)),
573 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: Int32Ty, V: 1)),
574 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: Int32Ty, V: 1))};
575
576 F->setMetadata(Kind: "max_work_group_size",
577 Node: llvm::MDNode::get(Context&: M.getLLVMContext(), MDs: AttrMDArgs));
578}
579
580StringRef SPIRVTargetCodeGenInfo::getLLVMSyncScopeStr(
581 const LangOptions &, SyncScope Scope, llvm::AtomicOrdering) const {
582 return *llvm::getAtomicScopeIRString(T: getABIInfo().getTarget().getTriple(),
583 S: getAtomicScope(S: Scope));
584}
585
586void SPIRVTargetCodeGenInfo::setTargetAtomicMetadata(
587 CodeGenFunction &CGF, llvm::Instruction &AtomicInst,
588 const AtomicExpr *AE) const {
589 if (CGF.CGM.getTriple().getVendor() != llvm::Triple::VendorType::AMD)
590 return;
591
592 auto *RMW = dyn_cast<llvm::AtomicRMWInst>(Val: &AtomicInst);
593 if (!RMW)
594 return;
595
596 AtomicOptions AO = CGF.CGM.getAtomicOpts();
597 llvm::MDNode *Empty = llvm::MDNode::get(Context&: CGF.getLLVMContext(), MDs: {});
598 if (!AO.getOption(Kind: clang::AtomicOptionKind::FineGrainedMemory))
599 RMW->setMetadata(Kind: "amdgpu.no.fine.grained.memory", Node: Empty);
600 if (!AO.getOption(Kind: clang::AtomicOptionKind::RemoteMemory))
601 RMW->setMetadata(Kind: "amdgpu.no.remote.memory", Node: Empty);
602 if (AO.getOption(Kind: clang::AtomicOptionKind::IgnoreDenormalMode) &&
603 RMW->getOperation() == llvm::AtomicRMWInst::FAdd &&
604 RMW->getType()->isFloatTy())
605 RMW->setMetadata(KindID: llvm::LLVMContext::MD_atomic_ignore_denormal_mode, Node: Empty);
606}
607
608/// Construct a SPIR-V target extension type for the given OpenCL image type.
609static llvm::Type *getSPIRVImageType(llvm::LLVMContext &Ctx, StringRef BaseType,
610 StringRef OpenCLName,
611 unsigned AccessQualifier) {
612 // These parameters compare to the operands of OpTypeImage (see
613 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpTypeImage
614 // for more details). The first 6 integer parameters all default to 0, and
615 // will be changed to 1 only for the image type(s) that set the parameter to
616 // one. The 7th integer parameter is the access qualifier, which is tacked on
617 // at the end.
618 SmallVector<unsigned, 7> IntParams = {0, 0, 0, 0, 0, 0};
619
620 // Choose the dimension of the image--this corresponds to the Dim enum in
621 // SPIR-V (first integer parameter of OpTypeImage).
622 if (OpenCLName.starts_with(Prefix: "image2d"))
623 IntParams[0] = 1;
624 else if (OpenCLName.starts_with(Prefix: "image3d"))
625 IntParams[0] = 2;
626 else if (OpenCLName == "image1d_buffer")
627 IntParams[0] = 5; // Buffer
628 else
629 assert(OpenCLName.starts_with("image1d") && "Unknown image type");
630
631 // Set the other integer parameters of OpTypeImage if necessary. Note that the
632 // OpenCL image types don't provide any information for the Sampled or
633 // Image Format parameters.
634 if (OpenCLName.contains(Other: "_depth"))
635 IntParams[1] = 1;
636 if (OpenCLName.contains(Other: "_array"))
637 IntParams[2] = 1;
638 if (OpenCLName.contains(Other: "_msaa"))
639 IntParams[3] = 1;
640
641 // Access qualifier
642 IntParams.push_back(Elt: AccessQualifier);
643
644 return llvm::TargetExtType::get(Context&: Ctx, Name: BaseType, Types: {llvm::Type::getVoidTy(C&: Ctx)},
645 Ints: IntParams);
646}
647
648llvm::Type *CommonSPIRTargetCodeGenInfo::getOpenCLType(CodeGenModule &CGM,
649 const Type *Ty) const {
650 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
651 if (auto *PipeTy = dyn_cast<PipeType>(Val: Ty))
652 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Pipe", Types: {},
653 Ints: {!PipeTy->isReadOnly()});
654 if (auto *BuiltinTy = dyn_cast<BuiltinType>(Val: Ty)) {
655 enum AccessQualifier : unsigned { AQ_ro = 0, AQ_wo = 1, AQ_rw = 2 };
656 switch (BuiltinTy->getKind()) {
657#define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
658 case BuiltinType::Id: \
659 return getSPIRVImageType(Ctx, "spirv.Image", #ImgType, AQ_##Suffix);
660#include "clang/Basic/OpenCLImageTypes.def"
661 case BuiltinType::OCLSampler:
662 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Sampler");
663 case BuiltinType::OCLEvent:
664 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Event");
665 case BuiltinType::OCLClkEvent:
666 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.DeviceEvent");
667 case BuiltinType::OCLQueue:
668 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Queue");
669 case BuiltinType::OCLReserveID:
670 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.ReserveId");
671#define INTEL_SUBGROUP_AVC_TYPE(Name, Id) \
672 case BuiltinType::OCLIntelSubgroupAVC##Id: \
673 return llvm::TargetExtType::get(Ctx, "spirv.Avc" #Id "INTEL");
674#include "clang/Basic/OpenCLExtensionTypes.def"
675 default:
676 return nullptr;
677 }
678 }
679
680 return nullptr;
681}
682
683// Gets a spirv.IntegralConstant or spirv.Literal. If IntegralType is present,
684// returns an IntegralConstant, otherwise returns a Literal.
685static llvm::Type *getInlineSpirvConstant(CodeGenModule &CGM,
686 llvm::Type *IntegralType,
687 llvm::APInt Value) {
688 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
689
690 // Convert the APInt value to an array of uint32_t words
691 llvm::SmallVector<uint32_t> Words;
692
693 while (Value.ugt(RHS: 0)) {
694 uint32_t Word = Value.trunc(width: 32).getZExtValue();
695 Value.lshrInPlace(ShiftAmt: 32);
696
697 Words.push_back(Elt: Word);
698 }
699 if (Words.size() == 0)
700 Words.push_back(Elt: 0);
701
702 if (IntegralType)
703 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.IntegralConstant",
704 Types: {IntegralType}, Ints: Words);
705 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Literal", Types: {}, Ints: Words);
706}
707
708static llvm::Type *getInlineSpirvType(CodeGenModule &CGM,
709 const HLSLInlineSpirvType *SpirvType) {
710 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
711
712 llvm::SmallVector<llvm::Type *> Operands;
713
714 for (auto &Operand : SpirvType->getOperands()) {
715 using SpirvOperandKind = SpirvOperand::SpirvOperandKind;
716
717 llvm::Type *Result = nullptr;
718 switch (Operand.getKind()) {
719 case SpirvOperandKind::ConstantId: {
720 llvm::Type *IntegralType =
721 CGM.getTypes().ConvertType(T: Operand.getResultType());
722
723 Result = getInlineSpirvConstant(CGM, IntegralType, Value: Operand.getValue());
724 break;
725 }
726 case SpirvOperandKind::Literal: {
727 Result = getInlineSpirvConstant(CGM, IntegralType: nullptr, Value: Operand.getValue());
728 break;
729 }
730 case SpirvOperandKind::TypeId: {
731 QualType TypeOperand = Operand.getResultType();
732 if (const auto *RD = TypeOperand->getAsRecordDecl()) {
733 assert(RD->isCompleteDefinition() &&
734 "Type completion should have been required in Sema");
735
736 const FieldDecl *HandleField = RD->findFirstNamedDataMember();
737 if (HandleField) {
738 QualType ResourceType = HandleField->getType();
739 if (ResourceType->getAs<HLSLAttributedResourceType>()) {
740 TypeOperand = ResourceType;
741 }
742 }
743 }
744 Result = CGM.getTypes().ConvertType(T: TypeOperand);
745 break;
746 }
747 default:
748 llvm_unreachable("HLSLInlineSpirvType had invalid operand!");
749 break;
750 }
751
752 assert(Result);
753 Operands.push_back(Elt: Result);
754 }
755
756 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Type", Types: Operands,
757 Ints: {SpirvType->getOpcode(), SpirvType->getSize(),
758 SpirvType->getAlignment()});
759}
760
761llvm::Type *CommonSPIRTargetCodeGenInfo::getHLSLType(
762 CodeGenModule &CGM, const Type *Ty,
763 const CGHLSLOffsetInfo &OffsetInfo) const {
764 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
765
766 if (auto *SpirvType = dyn_cast<HLSLInlineSpirvType>(Val: Ty))
767 return getInlineSpirvType(CGM, SpirvType);
768
769 auto *ResType = dyn_cast<HLSLAttributedResourceType>(Val: Ty);
770 if (!ResType)
771 return nullptr;
772
773 const HLSLAttributedResourceType::Attributes &ResAttrs = ResType->getAttrs();
774 switch (ResAttrs.ResourceClass) {
775 case llvm::dxil::ResourceClass::UAV:
776 case llvm::dxil::ResourceClass::SRV: {
777 // TypedBuffer and RawBuffer both need element type
778 QualType ContainedTy = ResType->getContainedType();
779 if (ContainedTy.isNull())
780 return nullptr;
781
782 assert(!ResAttrs.IsROV &&
783 "Rasterizer order views not implemented for SPIR-V yet");
784
785 if (!ResAttrs.RawBuffer) {
786 // convert element type
787 return getSPIRVImageTypeFromHLSLResource(attributes: ResAttrs, SampledType: ContainedTy, CGM);
788 }
789
790 if (ResAttrs.IsCounter) {
791 llvm::Type *ElemType = llvm::Type::getInt32Ty(C&: Ctx);
792 uint32_t StorageClass = /* StorageBuffer storage class */ 12;
793 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.VulkanBuffer", Types: {ElemType},
794 Ints: {StorageClass, true});
795 }
796 llvm::Type *ElemType = CGM.getTypes().ConvertTypeForMem(T: ContainedTy);
797 llvm::ArrayType *RuntimeArrayType = llvm::ArrayType::get(ElementType: ElemType, NumElements: 0);
798 uint32_t StorageClass = /* StorageBuffer storage class */ 12;
799 bool IsWritable = ResAttrs.ResourceClass == llvm::dxil::ResourceClass::UAV;
800 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.VulkanBuffer",
801 Types: {RuntimeArrayType},
802 Ints: {StorageClass, IsWritable});
803 }
804 case llvm::dxil::ResourceClass::CBuffer: {
805 QualType ContainedTy = ResType->getContainedType();
806 if (ContainedTy.isNull() || !ContainedTy->isStructureType())
807 return nullptr;
808
809 llvm::StructType *BufferLayoutTy =
810 HLSLBufferLayoutBuilder(CGM).layOutStruct(
811 StructType: ContainedTy->getAsCanonical<RecordType>(), OffsetInfo);
812 uint32_t StorageClass = /* Uniform storage class */ 2;
813 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.VulkanBuffer", Types: {BufferLayoutTy},
814 Ints: {StorageClass, false});
815 break;
816 }
817 case llvm::dxil::ResourceClass::Sampler:
818 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Sampler");
819 }
820 return nullptr;
821}
822
823static unsigned
824getImageFormat(const LangOptions &LangOpts,
825 const HLSLAttributedResourceType::Attributes &attributes,
826 llvm::Type *SampledType, QualType Ty, unsigned NumChannels) {
827 // For images with `Sampled` operand equal to 2, there are restrictions on
828 // using the Unknown image format. To avoid these restrictions in common
829 // cases, we guess an image format for them based on the sampled type and the
830 // number of channels. This is intended to match the behaviour of DXC.
831 if (LangOpts.HLSLSpvUseUnknownImageFormat ||
832 attributes.ResourceClass != llvm::dxil::ResourceClass::UAV) {
833 return 0; // Unknown
834 }
835
836 if (SampledType->isIntegerTy(BitWidth: 32)) {
837 if (Ty->isSignedIntegerType()) {
838 if (NumChannels == 1)
839 return 24; // R32i
840 if (NumChannels == 2)
841 return 25; // Rg32i
842 if (NumChannels == 4)
843 return 21; // Rgba32i
844 } else {
845 if (NumChannels == 1)
846 return 33; // R32ui
847 if (NumChannels == 2)
848 return 35; // Rg32ui
849 if (NumChannels == 4)
850 return 30; // Rgba32ui
851 }
852 } else if (SampledType->isIntegerTy(BitWidth: 64)) {
853 if (NumChannels == 1) {
854 if (Ty->isSignedIntegerType()) {
855 return 41; // R64i
856 }
857 return 40; // R64ui
858 }
859 } else if (SampledType->isFloatTy()) {
860 if (NumChannels == 1)
861 return 3; // R32f
862 if (NumChannels == 2)
863 return 6; // Rg32f
864 if (NumChannels == 4)
865 return 1; // Rgba32f
866 }
867
868 return 0; // Unknown
869}
870
871llvm::Type *CommonSPIRTargetCodeGenInfo::getSPIRVImageTypeFromHLSLResource(
872 const HLSLAttributedResourceType::Attributes &attributes, QualType Ty,
873 CodeGenModule &CGM) const {
874 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
875
876 unsigned NumChannels = 1;
877 Ty = Ty->getCanonicalTypeUnqualified();
878 if (const VectorType *V = dyn_cast<VectorType>(Val&: Ty)) {
879 NumChannels = V->getNumElements();
880 Ty = V->getElementType();
881 }
882 assert(!Ty->isVectorType() && "We still have a vector type.");
883
884 llvm::Type *SampledType = CGM.getTypes().ConvertTypeForMem(T: Ty);
885
886 assert((SampledType->isIntegerTy() || SampledType->isFloatingPointTy()) &&
887 "The element type for a SPIR-V resource must be a scalar integer or "
888 "floating point type.");
889
890 assert((!SampledType->isIntegerTy(64) || NumChannels <= 2) &&
891 "A 64-bit SPIR-V resource element can have at most 2 components.");
892
893 // SPIR-V has no 64-bit multi-component image format, so pack a 2-component
894 // 64-bit typed buffer into a 4-component 32-bit image. The backend
895 // reinterprets it with OpBitcast on load and store.
896 if (SampledType->isIntegerTy(BitWidth: 64) && NumChannels == 2) {
897 SampledType = llvm::Type::getInt32Ty(C&: Ctx);
898 NumChannels = 4;
899 }
900
901 // These parameters correspond to the operands to the OpTypeImage SPIR-V
902 // instruction. See
903 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpTypeImage.
904 SmallVector<unsigned, 6> IntParams(6, 0);
905
906 const char *Name =
907 Ty->isSignedIntegerType() ? "spirv.SignedImage" : "spirv.Image";
908
909 // Dim
910 switch (attributes.ResourceDimension) {
911 case llvm::dxil::ResourceDimension::Dim1D:
912 IntParams[0] = 0;
913 break;
914 case llvm::dxil::ResourceDimension::Dim2D:
915 IntParams[0] = 1;
916 break;
917 case llvm::dxil::ResourceDimension::Dim3D:
918 IntParams[0] = 2;
919 break;
920 case llvm::dxil::ResourceDimension::Cube:
921 IntParams[0] = 3;
922 break;
923 case llvm::dxil::ResourceDimension::Unknown:
924 IntParams[0] = 5;
925 break;
926 }
927
928 // Depth
929 // HLSL does not indicate if it is a depth texture or not, so we use unknown.
930 IntParams[1] = 2;
931
932 // Arrayed
933 IntParams[2] = static_cast<unsigned>(attributes.IsArray);
934
935 // MS
936 IntParams[3] = static_cast<unsigned>(attributes.isMultiSampled());
937
938 // Sampled
939 IntParams[4] =
940 attributes.ResourceClass == llvm::dxil::ResourceClass::UAV ? 2 : 1;
941
942 // Image format.
943 IntParams[5] = getImageFormat(LangOpts: CGM.getLangOpts(), attributes, SampledType, Ty,
944 NumChannels);
945
946 llvm::TargetExtType *ImageType =
947 llvm::TargetExtType::get(Context&: Ctx, Name, Types: {SampledType}, Ints: IntParams);
948 return ImageType;
949}
950
951std::unique_ptr<TargetCodeGenInfo>
952CodeGen::createCommonSPIRTargetCodeGenInfo(CodeGenModule &CGM) {
953 return std::make_unique<CommonSPIRTargetCodeGenInfo>(args&: CGM.getTypes());
954}
955
956std::unique_ptr<TargetCodeGenInfo>
957CodeGen::createSPIRVTargetCodeGenInfo(CodeGenModule &CGM) {
958 return std::make_unique<SPIRVTargetCodeGenInfo>(args&: CGM.getTypes());
959}
960