1//===- SPIR.cpp -----------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "HLSLBufferLayoutBuilder.h"
11#include "TargetInfo.h"
12#include "clang/Basic/LangOptions.h"
13#include "llvm/IR/DerivedTypes.h"
14
15#include <stdint.h>
16#include <utility>
17
18using namespace clang;
19using namespace clang::CodeGen;
20
21//===----------------------------------------------------------------------===//
22// Base ABI and target codegen info implementation common between SPIR and
23// SPIR-V.
24//===----------------------------------------------------------------------===//
25
26namespace {
27class CommonSPIRABIInfo : public DefaultABIInfo {
28public:
29 CommonSPIRABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) { setCCs(); }
30
31private:
32 void setCCs();
33};
34
35class SPIRVABIInfo : public CommonSPIRABIInfo {
36public:
37 SPIRVABIInfo(CodeGenTypes &CGT) : CommonSPIRABIInfo(CGT) {}
38 void computeInfo(CGFunctionInfo &FI) const override;
39 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
40 AggValueSlot Slot) const override;
41
42private:
43 ABIArgInfo classifyKernelArgumentType(QualType Ty) const;
44};
45
46class AMDGCNSPIRVABIInfo : public SPIRVABIInfo {
47 // TODO: this should be unified / shared with AMDGPU, ideally we'd like to
48 // re-use AMDGPUABIInfo eventually, rather than duplicate.
49 static constexpr unsigned MaxNumRegsForArgsRet = 16; // 16 32-bit registers
50 mutable unsigned NumRegsLeft = 0;
51
52 uint64_t numRegsForType(QualType Ty) const;
53
54 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
55 return true;
56 }
57 bool isHomogeneousAggregateSmallEnough(const Type *Base,
58 uint64_t Members) const override {
59 uint32_t NumRegs = (getContext().getTypeSize(T: Base) + 31) / 32;
60
61 // Homogeneous Aggregates may occupy at most 16 registers.
62 return Members * NumRegs <= MaxNumRegsForArgsRet;
63 }
64
65 // Coerce HIP scalar pointer arguments from generic pointers to global ones.
66 llvm::Type *coerceKernelArgumentType(llvm::Type *Ty, unsigned FromAS,
67 unsigned ToAS) const;
68
69 ABIArgInfo classifyReturnType(QualType RetTy) const;
70 ABIArgInfo classifyKernelArgumentType(QualType Ty) const;
71 ABIArgInfo classifyArgumentType(QualType Ty) const;
72
73public:
74 AMDGCNSPIRVABIInfo(CodeGenTypes &CGT) : SPIRVABIInfo(CGT) {}
75 void computeInfo(CGFunctionInfo &FI) const override;
76
77 llvm::FixedVectorType *
78 getOptimalVectorMemoryType(llvm::FixedVectorType *Ty,
79 const LangOptions &LangOpt) const override;
80};
81} // end anonymous namespace
82namespace {
83class CommonSPIRTargetCodeGenInfo : public TargetCodeGenInfo {
84public:
85 CommonSPIRTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
86 : TargetCodeGenInfo(std::make_unique<CommonSPIRABIInfo>(args&: CGT)) {}
87 CommonSPIRTargetCodeGenInfo(std::unique_ptr<ABIInfo> ABIInfo)
88 : TargetCodeGenInfo(std::move(ABIInfo)) {}
89
90 LangAS getASTAllocaAddressSpace() const override {
91 return getLangASFromTargetAS(
92 TargetAS: getABIInfo().getDataLayout().getAllocaAddrSpace());
93 }
94
95 unsigned getDeviceKernelCallingConv() const override;
96 llvm::Type *getOpenCLType(CodeGenModule &CGM, const Type *T) const override;
97 llvm::Type *getHLSLType(CodeGenModule &CGM, const Type *Ty,
98 const CGHLSLOffsetInfo &OffsetInfo) const override;
99
100 llvm::Type *getHLSLPadding(CodeGenModule &CGM,
101 CharUnits NumBytes) const override {
102 unsigned Size = NumBytes.getQuantity();
103 return llvm::TargetExtType::get(Context&: CGM.getLLVMContext(), Name: "spirv.Padding", Types: {},
104 Ints: {Size});
105 }
106
107 bool isHLSLPadding(llvm::Type *Ty) const override {
108 if (auto *TET = dyn_cast<llvm::TargetExtType>(Val: Ty))
109 return TET->getName() == "spirv.Padding";
110 return false;
111 }
112
113 llvm::Type *getSPIRVImageTypeFromHLSLResource(
114 const HLSLAttributedResourceType::Attributes &attributes,
115 QualType SampledType, CodeGenModule &CGM) const;
116 void
117 setOCLKernelStubCallingConvention(const FunctionType *&FT) const override;
118 llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM,
119 llvm::PointerType *T,
120 QualType QT) const override;
121};
122class SPIRVTargetCodeGenInfo : public CommonSPIRTargetCodeGenInfo {
123public:
124 SPIRVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
125 : CommonSPIRTargetCodeGenInfo(
126 (CGT.getTarget().getTriple().getVendor() == llvm::Triple::AMD)
127 ? std::make_unique<AMDGCNSPIRVABIInfo>(args&: CGT)
128 : std::make_unique<SPIRVABIInfo>(args&: CGT)) {}
129 void setCUDAKernelCallingConvention(const FunctionType *&FT) const override;
130 LangAS getGlobalVarAddressSpace(CodeGenModule &CGM,
131 const VarDecl *D) const override;
132 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
133 CodeGen::CodeGenModule &M) const override;
134 llvm::SyncScope::ID getLLVMSyncScopeID(const LangOptions &LangOpts,
135 SyncScope Scope,
136 llvm::AtomicOrdering Ordering,
137 llvm::LLVMContext &Ctx) const override;
138 bool supportsLibCall() const override {
139 return getABIInfo().getTarget().getTriple().getVendor() !=
140 llvm::Triple::AMD;
141 }
142};
143
144inline StringRef mapClangSyncScopeToLLVM(SyncScope Scope) {
145 switch (Scope) {
146 case SyncScope::HIPSingleThread:
147 case SyncScope::SingleScope:
148 return "singlethread";
149 case SyncScope::HIPWavefront:
150 case SyncScope::OpenCLSubGroup:
151 case SyncScope::WavefrontScope:
152 return "subgroup";
153 case SyncScope::HIPCluster:
154 case SyncScope::ClusterScope:
155 case SyncScope::HIPWorkgroup:
156 case SyncScope::OpenCLWorkGroup:
157 case SyncScope::WorkgroupScope:
158 return "workgroup";
159 case SyncScope::HIPAgent:
160 case SyncScope::OpenCLDevice:
161 case SyncScope::DeviceScope:
162 return "device";
163 case SyncScope::SystemScope:
164 case SyncScope::HIPSystem:
165 case SyncScope::OpenCLAllSVMDevices:
166 return "";
167 }
168 return "";
169}
170} // End anonymous namespace.
171
172void CommonSPIRABIInfo::setCCs() {
173 assert(getRuntimeCC() == llvm::CallingConv::C);
174 RuntimeCC = llvm::CallingConv::SPIR_FUNC;
175}
176
177ABIArgInfo SPIRVABIInfo::classifyKernelArgumentType(QualType Ty) const {
178 if (getContext().getLangOpts().isTargetDevice()) {
179 // Coerce pointer arguments with default address space to CrossWorkGroup
180 // pointers for target devices as default address space kernel arguments
181 // are not allowed. We use the opencl_global language address space which
182 // always maps to CrossWorkGroup.
183 llvm::Type *LTy = CGT.ConvertType(T: Ty);
184 auto DefaultAS = getContext().getTargetAddressSpace(AS: LangAS::Default);
185 auto GlobalAS = getContext().getTargetAddressSpace(AS: LangAS::opencl_global);
186 auto *PtrTy = llvm::dyn_cast<llvm::PointerType>(Val: LTy);
187 if (PtrTy && PtrTy->getAddressSpace() == DefaultAS) {
188 LTy = llvm::PointerType::get(C&: PtrTy->getContext(), AddressSpace: GlobalAS);
189 return ABIArgInfo::getDirect(T: LTy, Offset: 0, Padding: nullptr, CanBeFlattened: false);
190 }
191
192 if (isAggregateTypeForABI(T: Ty)) {
193 // Force copying aggregate type in kernel arguments by value when
194 // compiling CUDA targeting SPIR-V. This is required for the object
195 // copied to be valid on the device.
196 // This behavior follows the CUDA spec
197 // https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#global-function-argument-processing,
198 // and matches the NVPTX implementation. TODO: hardcoding to 0 should be
199 // revisited if HIPSPV / byval starts making use of the AS of an indirect
200 // arg.
201 return getNaturalAlignIndirect(Ty, /*AddrSpace=*/0, /*byval=*/ByVal: true);
202 }
203 }
204 return classifyArgumentType(RetTy: Ty);
205}
206
207void SPIRVABIInfo::computeInfo(CGFunctionInfo &FI) const {
208 // The logic is same as in DefaultABIInfo with an exception on the kernel
209 // arguments handling.
210 llvm::CallingConv::ID CC = FI.getCallingConvention();
211
212 for (auto &&[ArgumentsCount, I] : llvm::enumerate(First: FI.arguments()))
213 I.info = ArgumentsCount < FI.getNumRequiredArgs()
214 ? classifyArgumentType(RetTy: I.type)
215 : ABIArgInfo::getDirect();
216
217 if (!getCXXABI().classifyReturnType(FI))
218 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType());
219
220 for (auto &I : FI.arguments()) {
221 if (CC == llvm::CallingConv::SPIR_KERNEL) {
222 I.info = classifyKernelArgumentType(Ty: I.type);
223 } else {
224 I.info = classifyArgumentType(RetTy: I.type);
225 }
226 }
227}
228
229RValue SPIRVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
230 QualType Ty, AggValueSlot Slot) const {
231 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, /*IsIndirect=*/false,
232 ValueInfo: getContext().getTypeInfoInChars(T: Ty),
233 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 1),
234 /*AllowHigherAlign=*/true, Slot);
235}
236
237uint64_t AMDGCNSPIRVABIInfo::numRegsForType(QualType Ty) const {
238 // This duplicates the AMDGPUABI computation.
239 uint64_t NumRegs = 0;
240
241 if (const VectorType *VT = Ty->getAs<VectorType>()) {
242 // Compute from the number of elements. The reported size is based on the
243 // in-memory size, which includes the padding 4th element for 3-vectors.
244 QualType EltTy = VT->getElementType();
245 uint64_t EltSize = getContext().getTypeSize(T: EltTy);
246
247 // 16-bit element vectors should be passed as packed.
248 if (EltSize == 16)
249 return (VT->getNumElements() + 1) / 2;
250
251 uint64_t EltNumRegs = (EltSize + 31) / 32;
252 return EltNumRegs * VT->getNumElements();
253 }
254
255 if (const auto *RD = Ty->getAsRecordDecl()) {
256 assert(!RD->hasFlexibleArrayMember());
257
258 for (const FieldDecl *Field : RD->fields()) {
259 QualType FieldTy = Field->getType();
260 NumRegs += numRegsForType(Ty: FieldTy);
261 }
262
263 return NumRegs;
264 }
265
266 return (getContext().getTypeSize(T: Ty) + 31) / 32;
267}
268
269llvm::Type *AMDGCNSPIRVABIInfo::coerceKernelArgumentType(llvm::Type *Ty,
270 unsigned FromAS,
271 unsigned ToAS) const {
272 // Single value types.
273 auto *PtrTy = llvm::dyn_cast<llvm::PointerType>(Val: Ty);
274 if (PtrTy && PtrTy->getAddressSpace() == FromAS)
275 return llvm::PointerType::get(C&: Ty->getContext(), AddressSpace: ToAS);
276 return Ty;
277}
278
279ABIArgInfo AMDGCNSPIRVABIInfo::classifyReturnType(QualType RetTy) const {
280 if (!isAggregateTypeForABI(T: RetTy) || getRecordArgABI(T: RetTy, CXXABI&: getCXXABI()))
281 return DefaultABIInfo::classifyReturnType(RetTy);
282
283 // Ignore empty structs/unions.
284 if (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true))
285 return ABIArgInfo::getIgnore();
286
287 // Lower single-element structs to just return a regular value.
288 if (const Type *SeltTy = isSingleElementStruct(T: RetTy, Context&: getContext()))
289 return ABIArgInfo::getDirect(T: CGT.ConvertType(T: QualType(SeltTy, 0)));
290
291 if (const auto *RD = RetTy->getAsRecordDecl();
292 RD && RD->hasFlexibleArrayMember())
293 return DefaultABIInfo::classifyReturnType(RetTy);
294
295 // Pack aggregates <= 4 bytes into single VGPR or pair.
296 uint64_t Size = getContext().getTypeSize(T: RetTy);
297 if (Size <= 16)
298 return ABIArgInfo::getDirect(T: llvm::Type::getInt16Ty(C&: getVMContext()));
299
300 if (Size <= 32)
301 return ABIArgInfo::getDirect(T: llvm::Type::getInt32Ty(C&: getVMContext()));
302
303 // TODO: This carried over from AMDGPU oddity, we retain it to
304 // ensure consistency, but it might be reasonable to return Int64.
305 if (Size <= 64) {
306 llvm::Type *I32Ty = llvm::Type::getInt32Ty(C&: getVMContext());
307 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: I32Ty, NumElements: 2));
308 }
309
310 if (numRegsForType(Ty: RetTy) <= MaxNumRegsForArgsRet)
311 return ABIArgInfo::getDirect();
312 return DefaultABIInfo::classifyReturnType(RetTy);
313}
314
315/// For kernels all parameters are really passed in a special buffer. It doesn't
316/// make sense to pass anything byval, so everything must be direct.
317ABIArgInfo AMDGCNSPIRVABIInfo::classifyKernelArgumentType(QualType Ty) const {
318 Ty = useFirstFieldIfTransparentUnion(Ty);
319
320 // TODO: Can we omit empty structs?
321
322 if (const Type *SeltTy = isSingleElementStruct(T: Ty, Context&: getContext()))
323 Ty = QualType(SeltTy, 0);
324
325 llvm::Type *OrigLTy = CGT.ConvertType(T: Ty);
326 llvm::Type *LTy = OrigLTy;
327 if (getContext().getLangOpts().isTargetDevice()) {
328 LTy = coerceKernelArgumentType(
329 Ty: OrigLTy, /*FromAS=*/getContext().getTargetAddressSpace(AS: LangAS::Default),
330 /*ToAS=*/getContext().getTargetAddressSpace(AS: LangAS::opencl_global));
331 }
332
333 // FIXME: This doesn't apply the optimization of coercing pointers in structs
334 // to global address space when using byref. This would require implementing a
335 // new kind of coercion of the in-memory type when for indirect arguments.
336 if (LTy == OrigLTy && isAggregateTypeForABI(T: Ty)) {
337 return ABIArgInfo::getIndirectAliased(
338 Alignment: getContext().getTypeAlignInChars(T: Ty),
339 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_constant),
340 Realign: false /*Realign*/, Padding: nullptr /*Padding*/);
341 }
342
343 // TODO: inhibiting flattening is an AMDGPU workaround for Clover, which might
344 // be vestigial and should be revisited.
345 return ABIArgInfo::getDirect(T: LTy, Offset: 0, Padding: nullptr, CanBeFlattened: false);
346}
347
348ABIArgInfo AMDGCNSPIRVABIInfo::classifyArgumentType(QualType Ty) const {
349 assert(NumRegsLeft <= MaxNumRegsForArgsRet && "register estimate underflow");
350
351 Ty = useFirstFieldIfTransparentUnion(Ty);
352
353 // TODO: support for variadics.
354
355 if (!isAggregateTypeForABI(T: Ty)) {
356 ABIArgInfo ArgInfo = DefaultABIInfo::classifyArgumentType(RetTy: Ty);
357 if (!ArgInfo.isIndirect()) {
358 uint64_t NumRegs = numRegsForType(Ty);
359 NumRegsLeft -= std::min(a: NumRegs, b: uint64_t{NumRegsLeft});
360 }
361
362 return ArgInfo;
363 }
364
365 // Records with non-trivial destructors/copy-constructors should not be
366 // passed by value.
367 if (auto RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI()))
368 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
369 ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
370
371 // Ignore empty structs/unions.
372 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
373 return ABIArgInfo::getIgnore();
374
375 // Lower single-element structs to just pass a regular value. TODO: We
376 // could do reasonable-size multiple-element structs too, using getExpand(),
377 // though watch out for things like bitfields.
378 if (const Type *SeltTy = isSingleElementStruct(T: Ty, Context&: getContext()))
379 return ABIArgInfo::getDirect(T: CGT.ConvertType(T: QualType(SeltTy, 0)));
380
381 if (const auto *RD = Ty->getAsRecordDecl();
382 RD && RD->hasFlexibleArrayMember())
383 return DefaultABIInfo::classifyArgumentType(RetTy: Ty);
384
385 uint64_t Size = getContext().getTypeSize(T: Ty);
386 if (Size <= 64) {
387 // Pack aggregates <= 8 bytes into single VGPR or pair.
388 unsigned NumRegs = (Size + 31) / 32;
389 NumRegsLeft -= std::min(a: NumRegsLeft, b: NumRegs);
390
391 if (Size <= 16)
392 return ABIArgInfo::getDirect(T: llvm::Type::getInt16Ty(C&: getVMContext()));
393
394 if (Size <= 32)
395 return ABIArgInfo::getDirect(T: llvm::Type::getInt32Ty(C&: getVMContext()));
396
397 // TODO: This is an AMDGPU oddity, and might be vestigial, we retain it to
398 // ensure consistency, but it should be revisited.
399 llvm::Type *I32Ty = llvm::Type::getInt32Ty(C&: getVMContext());
400 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: I32Ty, NumElements: 2));
401 }
402
403 if (NumRegsLeft > 0) {
404 uint64_t NumRegs = numRegsForType(Ty);
405 if (NumRegsLeft >= NumRegs) {
406 NumRegsLeft -= NumRegs;
407 return ABIArgInfo::getDirect();
408 }
409 }
410
411 // Use pass-by-reference in stead of pass-by-value for struct arguments in
412 // function ABI.
413 return ABIArgInfo::getIndirectAliased(
414 Alignment: getContext().getTypeAlignInChars(T: Ty),
415 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_private));
416}
417
418void AMDGCNSPIRVABIInfo::computeInfo(CGFunctionInfo &FI) const {
419 llvm::CallingConv::ID CC = FI.getCallingConvention();
420
421 if (!getCXXABI().classifyReturnType(FI))
422 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType());
423
424 NumRegsLeft = MaxNumRegsForArgsRet;
425 for (auto &I : FI.arguments()) {
426 if (CC == llvm::CallingConv::SPIR_KERNEL)
427 I.info = classifyKernelArgumentType(Ty: I.type);
428 else
429 I.info = classifyArgumentType(Ty: I.type);
430 }
431}
432
433llvm::FixedVectorType *AMDGCNSPIRVABIInfo::getOptimalVectorMemoryType(
434 llvm::FixedVectorType *Ty, const LangOptions &LangOpt) const {
435 // AMDGPU has legal instructions for 96-bit so 3x32 can be supported.
436 if (Ty->getNumElements() == 3 && getDataLayout().getTypeSizeInBits(Ty) == 96)
437 return Ty;
438 return DefaultABIInfo::getOptimalVectorMemoryType(T: Ty, Opt: LangOpt);
439}
440
441namespace clang {
442namespace CodeGen {
443void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI) {
444 if (CGM.getTarget().getTriple().isSPIRV()) {
445 if (CGM.getTarget().getTriple().getVendor() == llvm::Triple::AMD)
446 AMDGCNSPIRVABIInfo(CGM.getTypes()).computeInfo(FI);
447 else
448 SPIRVABIInfo(CGM.getTypes()).computeInfo(FI);
449 } else {
450 CommonSPIRABIInfo(CGM.getTypes()).computeInfo(FI);
451 }
452}
453}
454}
455
456unsigned CommonSPIRTargetCodeGenInfo::getDeviceKernelCallingConv() const {
457 return llvm::CallingConv::SPIR_KERNEL;
458}
459
460void SPIRVTargetCodeGenInfo::setCUDAKernelCallingConvention(
461 const FunctionType *&FT) const {
462 // Convert HIP kernels to SPIR-V kernels.
463 if (getABIInfo().getContext().getLangOpts().HIP) {
464 FT = getABIInfo().getContext().adjustFunctionType(
465 Fn: FT, EInfo: FT->getExtInfo().withCallingConv(cc: CC_DeviceKernel));
466 return;
467 }
468}
469
470void CommonSPIRTargetCodeGenInfo::setOCLKernelStubCallingConvention(
471 const FunctionType *&FT) const {
472 FT = getABIInfo().getContext().adjustFunctionType(
473 Fn: FT, EInfo: FT->getExtInfo().withCallingConv(cc: CC_SpirFunction));
474}
475
476// LLVM currently assumes a null pointer has the bit pattern 0, but some GPU
477// targets use a non-zero encoding for null in certain address spaces.
478// Because SPIR(-V) is a generic target and the bit pattern of null in
479// non-generic AS is unspecified, materialize null in non-generic AS via an
480// addrspacecast from null in generic AS. This allows later lowering to
481// substitute the target's real sentinel value.
482llvm::Constant *
483CommonSPIRTargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM,
484 llvm::PointerType *PT,
485 QualType QT) const {
486 LangAS AS = QT->getUnqualifiedDesugaredType()->isNullPtrType()
487 ? LangAS::Default
488 : QT->getPointeeType().getAddressSpace();
489 unsigned ASAsInt = static_cast<unsigned>(AS);
490 unsigned FirstTargetASAsInt =
491 static_cast<unsigned>(LangAS::FirstTargetAddressSpace);
492 unsigned CodeSectionINTELAS = FirstTargetASAsInt + 9;
493 // As per SPV_INTEL_function_pointers, it is illegal to addrspacecast
494 // function pointers to/from the generic AS.
495 bool IsFunctionPtrAS =
496 CGM.getTriple().isSPIRV() && ASAsInt == CodeSectionINTELAS;
497 if (AS == LangAS::Default || AS == LangAS::opencl_generic ||
498 AS == LangAS::opencl_constant || IsFunctionPtrAS)
499 return llvm::ConstantPointerNull::get(T: PT);
500
501 auto &Ctx = CGM.getContext();
502 auto NPT = llvm::PointerType::get(
503 C&: PT->getContext(), AddressSpace: Ctx.getTargetAddressSpace(AS: LangAS::opencl_generic));
504 return llvm::ConstantExpr::getAddrSpaceCast(
505 C: llvm::ConstantPointerNull::get(T: NPT), Ty: PT);
506}
507
508LangAS
509SPIRVTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
510 const VarDecl *D) const {
511 assert(!CGM.getLangOpts().OpenCL &&
512 !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
513 "Address space agnostic languages only");
514 // If we're here it means that we're using the SPIRDefIsGen ASMap, hence for
515 // the global AS we can rely on either cuda_device or sycl_global to be
516 // correct; however, since this is not a CUDA Device context, we use
517 // sycl_global to prevent confusion with the assertion.
518 LangAS DefaultGlobalAS = getLangASFromTargetAS(
519 TargetAS: CGM.getContext().getTargetAddressSpace(AS: LangAS::sycl_global));
520 if (!D)
521 return DefaultGlobalAS;
522
523 LangAS AddrSpace = D->getType().getAddressSpace();
524 if (AddrSpace != LangAS::Default)
525 return AddrSpace;
526
527 return DefaultGlobalAS;
528}
529
530void SPIRVTargetCodeGenInfo::setTargetAttributes(
531 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
532 if (GV->isDeclaration())
533 return;
534
535 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
536 if (!FD)
537 return;
538
539 llvm::Function *F = dyn_cast<llvm::Function>(Val: GV);
540 assert(F && "Expected GlobalValue to be a Function");
541
542 if (!M.getLangOpts().HIP ||
543 M.getTarget().getTriple().getVendor() != llvm::Triple::AMD)
544 return;
545
546 if (!FD->hasAttr<CUDAGlobalAttr>())
547 return;
548
549 unsigned N = M.getLangOpts().GPUMaxThreadsPerBlock;
550 if (auto FlatWGS = FD->getAttr<AMDGPUFlatWorkGroupSizeAttr>())
551 N = FlatWGS->getMax()->EvaluateKnownConstInt(Ctx: M.getContext()).getExtValue();
552
553 // We encode the maximum flat WG size in the first component of the 3D
554 // max_work_group_size attribute, which will get reverse translated into the
555 // original AMDGPU attribute when targeting AMDGPU.
556 auto Int32Ty = llvm::IntegerType::getInt32Ty(C&: M.getLLVMContext());
557 llvm::Metadata *AttrMDArgs[] = {
558 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: Int32Ty, V: N)),
559 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: Int32Ty, V: 1)),
560 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: Int32Ty, V: 1))};
561
562 F->setMetadata(Kind: "max_work_group_size",
563 Node: llvm::MDNode::get(Context&: M.getLLVMContext(), MDs: AttrMDArgs));
564}
565
566llvm::SyncScope::ID
567SPIRVTargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &, SyncScope Scope,
568 llvm::AtomicOrdering,
569 llvm::LLVMContext &Ctx) const {
570 return Ctx.getOrInsertSyncScopeID(SSN: mapClangSyncScopeToLLVM(Scope));
571}
572
573/// Construct a SPIR-V target extension type for the given OpenCL image type.
574static llvm::Type *getSPIRVImageType(llvm::LLVMContext &Ctx, StringRef BaseType,
575 StringRef OpenCLName,
576 unsigned AccessQualifier) {
577 // These parameters compare to the operands of OpTypeImage (see
578 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpTypeImage
579 // for more details). The first 6 integer parameters all default to 0, and
580 // will be changed to 1 only for the image type(s) that set the parameter to
581 // one. The 7th integer parameter is the access qualifier, which is tacked on
582 // at the end.
583 SmallVector<unsigned, 7> IntParams = {0, 0, 0, 0, 0, 0};
584
585 // Choose the dimension of the image--this corresponds to the Dim enum in
586 // SPIR-V (first integer parameter of OpTypeImage).
587 if (OpenCLName.starts_with(Prefix: "image2d"))
588 IntParams[0] = 1;
589 else if (OpenCLName.starts_with(Prefix: "image3d"))
590 IntParams[0] = 2;
591 else if (OpenCLName == "image1d_buffer")
592 IntParams[0] = 5; // Buffer
593 else
594 assert(OpenCLName.starts_with("image1d") && "Unknown image type");
595
596 // Set the other integer parameters of OpTypeImage if necessary. Note that the
597 // OpenCL image types don't provide any information for the Sampled or
598 // Image Format parameters.
599 if (OpenCLName.contains(Other: "_depth"))
600 IntParams[1] = 1;
601 if (OpenCLName.contains(Other: "_array"))
602 IntParams[2] = 1;
603 if (OpenCLName.contains(Other: "_msaa"))
604 IntParams[3] = 1;
605
606 // Access qualifier
607 IntParams.push_back(Elt: AccessQualifier);
608
609 return llvm::TargetExtType::get(Context&: Ctx, Name: BaseType, Types: {llvm::Type::getVoidTy(C&: Ctx)},
610 Ints: IntParams);
611}
612
613llvm::Type *CommonSPIRTargetCodeGenInfo::getOpenCLType(CodeGenModule &CGM,
614 const Type *Ty) const {
615 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
616 if (auto *PipeTy = dyn_cast<PipeType>(Val: Ty))
617 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Pipe", Types: {},
618 Ints: {!PipeTy->isReadOnly()});
619 if (auto *BuiltinTy = dyn_cast<BuiltinType>(Val: Ty)) {
620 enum AccessQualifier : unsigned { AQ_ro = 0, AQ_wo = 1, AQ_rw = 2 };
621 switch (BuiltinTy->getKind()) {
622#define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
623 case BuiltinType::Id: \
624 return getSPIRVImageType(Ctx, "spirv.Image", #ImgType, AQ_##Suffix);
625#include "clang/Basic/OpenCLImageTypes.def"
626 case BuiltinType::OCLSampler:
627 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Sampler");
628 case BuiltinType::OCLEvent:
629 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Event");
630 case BuiltinType::OCLClkEvent:
631 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.DeviceEvent");
632 case BuiltinType::OCLQueue:
633 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Queue");
634 case BuiltinType::OCLReserveID:
635 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.ReserveId");
636#define INTEL_SUBGROUP_AVC_TYPE(Name, Id) \
637 case BuiltinType::OCLIntelSubgroupAVC##Id: \
638 return llvm::TargetExtType::get(Ctx, "spirv.Avc" #Id "INTEL");
639#include "clang/Basic/OpenCLExtensionTypes.def"
640 default:
641 return nullptr;
642 }
643 }
644
645 return nullptr;
646}
647
648// Gets a spirv.IntegralConstant or spirv.Literal. If IntegralType is present,
649// returns an IntegralConstant, otherwise returns a Literal.
650static llvm::Type *getInlineSpirvConstant(CodeGenModule &CGM,
651 llvm::Type *IntegralType,
652 llvm::APInt Value) {
653 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
654
655 // Convert the APInt value to an array of uint32_t words
656 llvm::SmallVector<uint32_t> Words;
657
658 while (Value.ugt(RHS: 0)) {
659 uint32_t Word = Value.trunc(width: 32).getZExtValue();
660 Value.lshrInPlace(ShiftAmt: 32);
661
662 Words.push_back(Elt: Word);
663 }
664 if (Words.size() == 0)
665 Words.push_back(Elt: 0);
666
667 if (IntegralType)
668 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.IntegralConstant",
669 Types: {IntegralType}, Ints: Words);
670 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Literal", Types: {}, Ints: Words);
671}
672
673static llvm::Type *getInlineSpirvType(CodeGenModule &CGM,
674 const HLSLInlineSpirvType *SpirvType) {
675 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
676
677 llvm::SmallVector<llvm::Type *> Operands;
678
679 for (auto &Operand : SpirvType->getOperands()) {
680 using SpirvOperandKind = SpirvOperand::SpirvOperandKind;
681
682 llvm::Type *Result = nullptr;
683 switch (Operand.getKind()) {
684 case SpirvOperandKind::ConstantId: {
685 llvm::Type *IntegralType =
686 CGM.getTypes().ConvertType(T: Operand.getResultType());
687
688 Result = getInlineSpirvConstant(CGM, IntegralType, Value: Operand.getValue());
689 break;
690 }
691 case SpirvOperandKind::Literal: {
692 Result = getInlineSpirvConstant(CGM, IntegralType: nullptr, Value: Operand.getValue());
693 break;
694 }
695 case SpirvOperandKind::TypeId: {
696 QualType TypeOperand = Operand.getResultType();
697 if (const auto *RD = TypeOperand->getAsRecordDecl()) {
698 assert(RD->isCompleteDefinition() &&
699 "Type completion should have been required in Sema");
700
701 const FieldDecl *HandleField = RD->findFirstNamedDataMember();
702 if (HandleField) {
703 QualType ResourceType = HandleField->getType();
704 if (ResourceType->getAs<HLSLAttributedResourceType>()) {
705 TypeOperand = ResourceType;
706 }
707 }
708 }
709 Result = CGM.getTypes().ConvertType(T: TypeOperand);
710 break;
711 }
712 default:
713 llvm_unreachable("HLSLInlineSpirvType had invalid operand!");
714 break;
715 }
716
717 assert(Result);
718 Operands.push_back(Elt: Result);
719 }
720
721 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Type", Types: Operands,
722 Ints: {SpirvType->getOpcode(), SpirvType->getSize(),
723 SpirvType->getAlignment()});
724}
725
726llvm::Type *CommonSPIRTargetCodeGenInfo::getHLSLType(
727 CodeGenModule &CGM, const Type *Ty,
728 const CGHLSLOffsetInfo &OffsetInfo) const {
729 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
730
731 if (auto *SpirvType = dyn_cast<HLSLInlineSpirvType>(Val: Ty))
732 return getInlineSpirvType(CGM, SpirvType);
733
734 auto *ResType = dyn_cast<HLSLAttributedResourceType>(Val: Ty);
735 if (!ResType)
736 return nullptr;
737
738 const HLSLAttributedResourceType::Attributes &ResAttrs = ResType->getAttrs();
739 switch (ResAttrs.ResourceClass) {
740 case llvm::dxil::ResourceClass::UAV:
741 case llvm::dxil::ResourceClass::SRV: {
742 // TypedBuffer and RawBuffer both need element type
743 QualType ContainedTy = ResType->getContainedType();
744 if (ContainedTy.isNull())
745 return nullptr;
746
747 assert(!ResAttrs.IsROV &&
748 "Rasterizer order views not implemented for SPIR-V yet");
749
750 if (!ResAttrs.RawBuffer) {
751 // convert element type
752 return getSPIRVImageTypeFromHLSLResource(attributes: ResAttrs, SampledType: ContainedTy, CGM);
753 }
754
755 if (ResAttrs.IsCounter) {
756 llvm::Type *ElemType = llvm::Type::getInt32Ty(C&: Ctx);
757 uint32_t StorageClass = /* StorageBuffer storage class */ 12;
758 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.VulkanBuffer", Types: {ElemType},
759 Ints: {StorageClass, true});
760 }
761 llvm::Type *ElemType = CGM.getTypes().ConvertTypeForMem(T: ContainedTy);
762 llvm::ArrayType *RuntimeArrayType = llvm::ArrayType::get(ElementType: ElemType, NumElements: 0);
763 uint32_t StorageClass = /* StorageBuffer storage class */ 12;
764 bool IsWritable = ResAttrs.ResourceClass == llvm::dxil::ResourceClass::UAV;
765 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.VulkanBuffer",
766 Types: {RuntimeArrayType},
767 Ints: {StorageClass, IsWritable});
768 }
769 case llvm::dxil::ResourceClass::CBuffer: {
770 QualType ContainedTy = ResType->getContainedType();
771 if (ContainedTy.isNull() || !ContainedTy->isStructureType())
772 return nullptr;
773
774 llvm::StructType *BufferLayoutTy =
775 HLSLBufferLayoutBuilder(CGM).layOutStruct(
776 StructType: ContainedTy->getAsCanonical<RecordType>(), OffsetInfo);
777 uint32_t StorageClass = /* Uniform storage class */ 2;
778 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.VulkanBuffer", Types: {BufferLayoutTy},
779 Ints: {StorageClass, false});
780 break;
781 }
782 case llvm::dxil::ResourceClass::Sampler:
783 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Sampler");
784 }
785 return nullptr;
786}
787
788static unsigned
789getImageFormat(const LangOptions &LangOpts,
790 const HLSLAttributedResourceType::Attributes &attributes,
791 llvm::Type *SampledType, QualType Ty, unsigned NumChannels) {
792 // For images with `Sampled` operand equal to 2, there are restrictions on
793 // using the Unknown image format. To avoid these restrictions in common
794 // cases, we guess an image format for them based on the sampled type and the
795 // number of channels. This is intended to match the behaviour of DXC.
796 if (LangOpts.HLSLSpvUseUnknownImageFormat ||
797 attributes.ResourceClass != llvm::dxil::ResourceClass::UAV) {
798 return 0; // Unknown
799 }
800
801 if (SampledType->isIntegerTy(Bitwidth: 32)) {
802 if (Ty->isSignedIntegerType()) {
803 if (NumChannels == 1)
804 return 24; // R32i
805 if (NumChannels == 2)
806 return 25; // Rg32i
807 if (NumChannels == 4)
808 return 21; // Rgba32i
809 } else {
810 if (NumChannels == 1)
811 return 33; // R32ui
812 if (NumChannels == 2)
813 return 35; // Rg32ui
814 if (NumChannels == 4)
815 return 30; // Rgba32ui
816 }
817 } else if (SampledType->isIntegerTy(Bitwidth: 64)) {
818 if (NumChannels == 1) {
819 if (Ty->isSignedIntegerType()) {
820 return 41; // R64i
821 }
822 return 40; // R64ui
823 }
824 } else if (SampledType->isFloatTy()) {
825 if (NumChannels == 1)
826 return 3; // R32f
827 if (NumChannels == 2)
828 return 6; // Rg32f
829 if (NumChannels == 4)
830 return 1; // Rgba32f
831 }
832
833 return 0; // Unknown
834}
835
836llvm::Type *CommonSPIRTargetCodeGenInfo::getSPIRVImageTypeFromHLSLResource(
837 const HLSLAttributedResourceType::Attributes &attributes, QualType Ty,
838 CodeGenModule &CGM) const {
839 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
840
841 unsigned NumChannels = 1;
842 Ty = Ty->getCanonicalTypeUnqualified();
843 if (const VectorType *V = dyn_cast<VectorType>(Val&: Ty)) {
844 NumChannels = V->getNumElements();
845 Ty = V->getElementType();
846 }
847 assert(!Ty->isVectorType() && "We still have a vector type.");
848
849 llvm::Type *SampledType = CGM.getTypes().ConvertTypeForMem(T: Ty);
850
851 assert((SampledType->isIntegerTy() || SampledType->isFloatingPointTy()) &&
852 "The element type for a SPIR-V resource must be a scalar integer or "
853 "floating point type.");
854
855 // These parameters correspond to the operands to the OpTypeImage SPIR-V
856 // instruction. See
857 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpTypeImage.
858 SmallVector<unsigned, 6> IntParams(6, 0);
859
860 const char *Name =
861 Ty->isSignedIntegerType() ? "spirv.SignedImage" : "spirv.Image";
862
863 // Dim
864 switch (attributes.ResourceDimension) {
865 case llvm::dxil::ResourceDimension::Dim1D:
866 IntParams[0] = 0;
867 break;
868 case llvm::dxil::ResourceDimension::Dim2D:
869 IntParams[0] = 1;
870 break;
871 case llvm::dxil::ResourceDimension::Dim3D:
872 IntParams[0] = 2;
873 break;
874 case llvm::dxil::ResourceDimension::Cube:
875 IntParams[0] = 3;
876 break;
877 case llvm::dxil::ResourceDimension::Unknown:
878 IntParams[0] = 5;
879 break;
880 }
881
882 // Depth
883 // HLSL does not indicate if it is a depth texture or not, so we use unknown.
884 IntParams[1] = 2;
885
886 // Arrayed
887 IntParams[2] = 0;
888
889 // MS
890 IntParams[3] = 0;
891
892 // Sampled
893 IntParams[4] =
894 attributes.ResourceClass == llvm::dxil::ResourceClass::UAV ? 2 : 1;
895
896 // Image format.
897 IntParams[5] = getImageFormat(LangOpts: CGM.getLangOpts(), attributes, SampledType, Ty,
898 NumChannels);
899
900 llvm::TargetExtType *ImageType =
901 llvm::TargetExtType::get(Context&: Ctx, Name, Types: {SampledType}, Ints: IntParams);
902 return ImageType;
903}
904
905std::unique_ptr<TargetCodeGenInfo>
906CodeGen::createCommonSPIRTargetCodeGenInfo(CodeGenModule &CGM) {
907 return std::make_unique<CommonSPIRTargetCodeGenInfo>(args&: CGM.getTypes());
908}
909
910std::unique_ptr<TargetCodeGenInfo>
911CodeGen::createSPIRVTargetCodeGenInfo(CodeGenModule &CGM) {
912 return std::make_unique<SPIRVTargetCodeGenInfo>(args&: CGM.getTypes());
913}
914