1//===- SPIR.cpp -----------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ABIInfoImpl.h"
10#include "HLSLBufferLayoutBuilder.h"
11#include "TargetInfo.h"
12#include "clang/AST/DeclCXX.h"
13#include "clang/Basic/LangOptions.h"
14#include "clang/CodeGenUtils/TargetUtils.h"
15#include "llvm/IR/DerivedTypes.h"
16#include "llvm/IR/LLVMContext.h"
17
18#include <stdint.h>
19#include <utility>
20
21using namespace clang;
22using namespace clang::CodeGen;
23
24//===----------------------------------------------------------------------===//
25// Base ABI and target codegen info implementation common between SPIR and
26// SPIR-V.
27//===----------------------------------------------------------------------===//
28
29namespace {
30class CommonSPIRABIInfo : public DefaultABIInfo {
31public:
32 CommonSPIRABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) { setCCs(); }
33
34private:
35 void setCCs();
36};
37
38class SPIRVABIInfo : public CommonSPIRABIInfo {
39public:
40 SPIRVABIInfo(CodeGenTypes &CGT) : CommonSPIRABIInfo(CGT) {}
41 void computeInfo(CGFunctionInfo &FI) const override;
42 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
43 AggValueSlot Slot) const override;
44
45 llvm::FixedVectorType *
46 getOptimalVectorMemoryType(llvm::FixedVectorType *Ty,
47 const LangOptions &LangOpt) const override;
48
49private:
50 ABIArgInfo classifyKernelArgumentType(QualType Ty) const;
51};
52
53class AMDGCNSPIRVABIInfo : public SPIRVABIInfo {
54 // TODO: this should be unified / shared with AMDGPU, ideally we'd like to
55 // re-use AMDGPUABIInfo eventually, rather than duplicate.
56 static constexpr unsigned MaxNumRegsForArgsRet = 16; // 16 32-bit registers
57 mutable unsigned NumRegsLeft = 0;
58
59 uint64_t numRegsForType(QualType Ty) const;
60
61 bool isHomogeneousAggregateBaseType(QualType Ty) const override {
62 return true;
63 }
64 bool isHomogeneousAggregateSmallEnough(const Type *Base,
65 uint64_t Members) const override {
66 uint32_t NumRegs = (getContext().getTypeSize(T: Base) + 31) / 32;
67
68 // Homogeneous Aggregates may occupy at most 16 registers.
69 return Members * NumRegs <= MaxNumRegsForArgsRet;
70 }
71
72 // Coerce HIP scalar pointer arguments from generic pointers to global ones.
73 llvm::Type *coerceKernelArgumentType(llvm::Type *Ty, unsigned FromAS,
74 unsigned ToAS) const;
75
76 ABIArgInfo classifyReturnType(QualType RetTy) const;
77 ABIArgInfo classifyKernelArgumentType(QualType Ty) const;
78 ABIArgInfo classifyArgumentType(QualType Ty, bool Variadic) const;
79
80public:
81 AMDGCNSPIRVABIInfo(CodeGenTypes &CGT) : SPIRVABIInfo(CGT) {}
82 void computeInfo(CGFunctionInfo &FI) const override;
83
84 llvm::FixedVectorType *
85 getOptimalVectorMemoryType(llvm::FixedVectorType *Ty,
86 const LangOptions &LangOpt) const override;
87};
88} // end anonymous namespace
89namespace {
90class CommonSPIRTargetCodeGenInfo : public TargetCodeGenInfo {
91public:
92 CommonSPIRTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
93 : TargetCodeGenInfo(std::make_unique<CommonSPIRABIInfo>(args&: CGT)) {}
94 CommonSPIRTargetCodeGenInfo(std::unique_ptr<ABIInfo> ABIInfo)
95 : TargetCodeGenInfo(std::move(ABIInfo)) {}
96
97 unsigned getDeviceKernelCallingConv() const override;
98 llvm::Type *getOpenCLType(CodeGenModule &CGM, const Type *T) const override;
99 llvm::Type *getHLSLType(CodeGenModule &CGM, const Type *Ty,
100 const CGHLSLOffsetInfo &OffsetInfo) const override;
101
102 llvm::Type *getHLSLPadding(CodeGenModule &CGM,
103 CharUnits NumBytes) const override {
104 unsigned Size = NumBytes.getQuantity();
105 return llvm::TargetExtType::get(Context&: CGM.getLLVMContext(), Name: "spirv.Padding", Types: {},
106 Ints: {Size});
107 }
108
109 bool isHLSLPadding(llvm::Type *Ty) const override {
110 if (auto *TET = dyn_cast<llvm::TargetExtType>(Val: Ty))
111 return TET->getName() == "spirv.Padding";
112 return false;
113 }
114
115 llvm::Type *getSPIRVImageTypeFromHLSLResource(
116 const HLSLAttributedResourceType::Attributes &attributes,
117 QualType SampledType, CodeGenModule &CGM) const;
118 void
119 setOCLKernelStubCallingConvention(const FunctionType *&FT) const override;
120 llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM,
121 llvm::PointerType *T,
122 QualType QT) const override;
123};
124class SPIRVTargetCodeGenInfo : public CommonSPIRTargetCodeGenInfo {
125public:
126 SPIRVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
127 : CommonSPIRTargetCodeGenInfo(
128 (CGT.getTarget().getTriple().getVendor() == llvm::Triple::AMD)
129 ? std::make_unique<AMDGCNSPIRVABIInfo>(args&: CGT)
130 : std::make_unique<SPIRVABIInfo>(args&: CGT)) {}
131 void setCUDAKernelCallingConvention(const FunctionType *&FT) const override;
132 LangAS getGlobalVarAddressSpace(CodeGenModule &CGM,
133 const VarDecl *D) const override;
134 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
135 CodeGen::CodeGenModule &M) const override;
136 StringRef getLLVMSyncScopeStr(const LangOptions &LangOpts, SyncScope Scope,
137 llvm::AtomicOrdering Ordering) const override;
138 void setTargetAtomicMetadata(CodeGenFunction &CGF,
139 llvm::Instruction &AtomicInst,
140 const AtomicExpr *Expr = nullptr) const override;
141 bool supportsLibCall() const override {
142 return getABIInfo().getTarget().getTriple().getVendor() !=
143 llvm::Triple::AMD;
144 }
145
146 LangAS getSRetAddrSpace(const CXXRecordDecl *RD) const override;
147};
148} // End anonymous namespace.
149
150void CommonSPIRABIInfo::setCCs() {
151 assert(getRuntimeCC() == llvm::CallingConv::C);
152 RuntimeCC = llvm::CallingConv::SPIR_FUNC;
153}
154
155ABIArgInfo SPIRVABIInfo::classifyKernelArgumentType(QualType Ty) const {
156 // Coerce pointer arguments with default address space to CrossWorkGroup
157 // pointers as default address space kernel
158 // arguments are not allowed. We use the opencl_global language address
159 // space which always maps to CrossWorkGroup.
160 llvm::Type *LTy = CGT.ConvertType(T: Ty);
161 auto DefaultAS = getContext().getTargetAddressSpace(AS: LangAS::Default);
162 auto GlobalAS = getContext().getTargetAddressSpace(AS: LangAS::opencl_global);
163 auto *PtrTy = llvm::dyn_cast<llvm::PointerType>(Val: LTy);
164 if (PtrTy && PtrTy->getAddressSpace() == DefaultAS) {
165 LTy = llvm::PointerType::get(C&: PtrTy->getContext(), AddressSpace: GlobalAS);
166 return ABIArgInfo::getDirect(T: LTy, Offset: 0, Padding: nullptr, CanBeFlattened: false);
167 }
168
169 if (getContext().getLangOpts().isTargetDevice() &&
170 isAggregateTypeForABI(T: Ty)) {
171 // Force copying aggregate type in kernel arguments by value when
172 // compiling CUDA targeting SPIR-V. This is required for the object
173 // copied to be valid on the device.
174 // This behavior follows the CUDA spec
175 // https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#global-function-argument-processing,
176 // and matches the NVPTX implementation. TODO: hardcoding to 0 should be
177 // revisited if HIPSPV / byval starts making use of the AS of an indirect
178 // arg.
179 return getNaturalAlignIndirect(Ty, /*AddrSpace=*/0, /*byval=*/ByVal: true);
180 }
181 return classifyArgumentType(RetTy: Ty);
182}
183
184void SPIRVABIInfo::computeInfo(CGFunctionInfo &FI) const {
185 // The logic is same as in DefaultABIInfo with an exception on the kernel
186 // arguments handling.
187 llvm::CallingConv::ID CC = FI.getCallingConvention();
188
189 if (!getCXXABI().classifyReturnType(FI))
190 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType());
191
192 for (auto &I : FI.arguments()) {
193 if (CC == llvm::CallingConv::SPIR_KERNEL) {
194 I.info = classifyKernelArgumentType(Ty: I.type);
195 } else {
196 I.info = classifyArgumentType(RetTy: I.type);
197 }
198 }
199}
200
201RValue SPIRVABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
202 QualType Ty, AggValueSlot Slot) const {
203 return emitVoidPtrVAArg(CGF, VAListAddr, ValueTy: Ty, /*IsIndirect=*/false,
204 ValueInfo: getContext().getTypeInfoInChars(T: Ty),
205 SlotSizeAndAlign: CharUnits::fromQuantity(Quantity: 1),
206 /*AllowHigherAlign=*/true, Slot);
207}
208
209uint64_t AMDGCNSPIRVABIInfo::numRegsForType(QualType Ty) const {
210 // This duplicates the AMDGPUABI computation.
211 uint64_t NumRegs = 0;
212
213 if (const VectorType *VT = Ty->getAs<VectorType>()) {
214 // Compute from the number of elements. The reported size is based on the
215 // in-memory size, which includes the padding 4th element for 3-vectors.
216 QualType EltTy = VT->getElementType();
217 uint64_t EltSize = getContext().getTypeSize(T: EltTy);
218
219 // 16-bit element vectors should be passed as packed.
220 if (EltSize == 16)
221 return (VT->getNumElements() + 1) / 2;
222
223 uint64_t EltNumRegs = (EltSize + 31) / 32;
224 return EltNumRegs * VT->getNumElements();
225 }
226
227 if (const auto *RD = Ty->getAsRecordDecl()) {
228 assert(!RD->hasFlexibleArrayMember());
229
230 for (const FieldDecl *Field : RD->fields()) {
231 QualType FieldTy = Field->getType();
232 NumRegs += numRegsForType(Ty: FieldTy);
233 }
234
235 return NumRegs;
236 }
237
238 return (getContext().getTypeSize(T: Ty) + 31) / 32;
239}
240
241llvm::Type *AMDGCNSPIRVABIInfo::coerceKernelArgumentType(llvm::Type *Ty,
242 unsigned FromAS,
243 unsigned ToAS) const {
244 // Single value types.
245 auto *PtrTy = llvm::dyn_cast<llvm::PointerType>(Val: Ty);
246 if (PtrTy && PtrTy->getAddressSpace() == FromAS)
247 return llvm::PointerType::get(C&: Ty->getContext(), AddressSpace: ToAS);
248 return Ty;
249}
250
251ABIArgInfo AMDGCNSPIRVABIInfo::classifyReturnType(QualType RetTy) const {
252 if (!isAggregateTypeForABI(T: RetTy) || getRecordArgABI(T: RetTy, CXXABI&: getCXXABI()))
253 return DefaultABIInfo::classifyReturnType(RetTy);
254
255 // Ignore empty structs/unions.
256 if (isEmptyRecord(Context&: getContext(), T: RetTy, AllowArrays: true))
257 return ABIArgInfo::getIgnore();
258
259 // Lower single-element structs to just return a regular value.
260 if (const Type *SeltTy = isSingleElementRecord(T: RetTy, Context&: getContext()))
261 return ABIArgInfo::getDirect(T: CGT.ConvertType(T: QualType(SeltTy, 0)));
262
263 if (const auto *RD = RetTy->getAsRecordDecl();
264 RD && RD->hasFlexibleArrayMember())
265 return DefaultABIInfo::classifyReturnType(RetTy);
266
267 // Pack aggregates <= 4 bytes into single VGPR or pair.
268 uint64_t Size = getContext().getTypeSize(T: RetTy);
269 if (Size <= 16)
270 return ABIArgInfo::getDirect(T: llvm::Type::getInt16Ty(C&: getVMContext()));
271
272 if (Size <= 32)
273 return ABIArgInfo::getDirect(T: llvm::Type::getInt32Ty(C&: getVMContext()));
274
275 // TODO: This carried over from AMDGPU oddity, we retain it to
276 // ensure consistency, but it might be reasonable to return Int64.
277 if (Size <= 64) {
278 llvm::Type *I32Ty = llvm::Type::getInt32Ty(C&: getVMContext());
279 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: I32Ty, NumElements: 2));
280 }
281
282 if (numRegsForType(Ty: RetTy) <= MaxNumRegsForArgsRet)
283 return ABIArgInfo::getDirect();
284 return DefaultABIInfo::classifyReturnType(RetTy);
285}
286
287/// For kernels all parameters are really passed in a special buffer. It doesn't
288/// make sense to pass anything byval, so everything must be direct.
289ABIArgInfo AMDGCNSPIRVABIInfo::classifyKernelArgumentType(QualType Ty) const {
290 Ty = useFirstFieldIfTransparentUnion(Ty);
291
292 // TODO: Can we omit empty structs?
293
294 if (const Type *SeltTy = isSingleElementRecord(T: Ty, Context&: getContext()))
295 Ty = QualType(SeltTy, 0);
296
297 llvm::Type *OrigLTy = CGT.ConvertType(T: Ty);
298 llvm::Type *LTy = OrigLTy;
299 if (getContext().getLangOpts().isTargetDevice()) {
300 LTy = coerceKernelArgumentType(
301 Ty: OrigLTy, /*FromAS=*/getContext().getTargetAddressSpace(AS: LangAS::Default),
302 /*ToAS=*/getContext().getTargetAddressSpace(AS: LangAS::opencl_global));
303 }
304
305 // FIXME: This doesn't apply the optimization of coercing pointers in structs
306 // to global address space when using byref. This would require implementing a
307 // new kind of coercion of the in-memory type when for indirect arguments.
308 if (LTy == OrigLTy && isAggregateTypeForABI(T: Ty)) {
309 return ABIArgInfo::getIndirectAliased(
310 Alignment: getContext().getTypeAlignInChars(T: Ty),
311 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_constant),
312 Realign: false /*Realign*/, Padding: nullptr /*Padding*/);
313 }
314
315 // TODO: inhibiting flattening is an AMDGPU workaround for Clover, which might
316 // be vestigial and should be revisited.
317 return ABIArgInfo::getDirect(T: LTy, Offset: 0, Padding: nullptr, CanBeFlattened: false);
318}
319
320ABIArgInfo AMDGCNSPIRVABIInfo::classifyArgumentType(QualType Ty,
321 bool Variadic) const {
322 assert(NumRegsLeft <= MaxNumRegsForArgsRet && "register estimate underflow");
323
324 Ty = useFirstFieldIfTransparentUnion(Ty);
325
326 if (Variadic) {
327 return ABIArgInfo::getDirect(/*T=*/nullptr,
328 /*Offset=*/0,
329 /*Padding=*/nullptr,
330 /*CanBeFlattened=*/false,
331 /*Align=*/0);
332 }
333
334 if (!isAggregateTypeForABI(T: Ty)) {
335 ABIArgInfo ArgInfo = DefaultABIInfo::classifyArgumentType(RetTy: Ty);
336 if (!ArgInfo.isIndirect()) {
337 uint64_t NumRegs = numRegsForType(Ty);
338 NumRegsLeft -= std::min(a: NumRegs, b: uint64_t{NumRegsLeft});
339 }
340
341 return ArgInfo;
342 }
343
344 // Records with non-trivial destructors/copy-constructors should not be
345 // passed by value.
346 if (auto RAA = getRecordArgABI(T: Ty, CXXABI&: getCXXABI()))
347 return getNaturalAlignIndirect(Ty, AddrSpace: getDataLayout().getAllocaAddrSpace(),
348 ByVal: RAA == CGCXXABI::RAA_DirectInMemory);
349
350 // Ignore empty structs/unions.
351 if (isEmptyRecord(Context&: getContext(), T: Ty, AllowArrays: true))
352 return ABIArgInfo::getIgnore();
353
354 // Lower single-element structs to just pass a regular value. TODO: We
355 // could do reasonable-size multiple-element structs too, using getExpand(),
356 // though watch out for things like bitfields.
357 if (const Type *SeltTy = isSingleElementRecord(T: Ty, Context&: getContext()))
358 return ABIArgInfo::getDirect(T: CGT.ConvertType(T: QualType(SeltTy, 0)));
359
360 if (const auto *RD = Ty->getAsRecordDecl();
361 RD && RD->hasFlexibleArrayMember())
362 return DefaultABIInfo::classifyArgumentType(RetTy: Ty);
363
364 uint64_t Size = getContext().getTypeSize(T: Ty);
365 if (Size <= 64) {
366 // Pack aggregates <= 8 bytes into single VGPR or pair.
367 unsigned NumRegs = (Size + 31) / 32;
368 NumRegsLeft -= std::min(a: NumRegsLeft, b: NumRegs);
369
370 if (Size <= 16)
371 return ABIArgInfo::getDirect(T: llvm::Type::getInt16Ty(C&: getVMContext()));
372
373 if (Size <= 32)
374 return ABIArgInfo::getDirect(T: llvm::Type::getInt32Ty(C&: getVMContext()));
375
376 // TODO: This is an AMDGPU oddity, and might be vestigial, we retain it to
377 // ensure consistency, but it should be revisited.
378 llvm::Type *I32Ty = llvm::Type::getInt32Ty(C&: getVMContext());
379 return ABIArgInfo::getDirect(T: llvm::ArrayType::get(ElementType: I32Ty, NumElements: 2));
380 }
381
382 if (NumRegsLeft > 0) {
383 uint64_t NumRegs = numRegsForType(Ty);
384 if (NumRegsLeft >= NumRegs) {
385 NumRegsLeft -= NumRegs;
386 return ABIArgInfo::getDirect();
387 }
388 }
389
390 // Use pass-by-reference in stead of pass-by-value for struct arguments in
391 // function ABI.
392 return ABIArgInfo::getIndirectAliased(
393 Alignment: getContext().getTypeAlignInChars(T: Ty),
394 AddrSpace: getContext().getTargetAddressSpace(AS: LangAS::opencl_private));
395}
396
397void AMDGCNSPIRVABIInfo::computeInfo(CGFunctionInfo &FI) const {
398 llvm::CallingConv::ID CC = FI.getCallingConvention();
399
400 if (!getCXXABI().classifyReturnType(FI))
401 FI.getReturnInfo() = classifyReturnType(RetTy: FI.getReturnType());
402
403 unsigned ArgumentIndex = 0;
404 const unsigned NumRequiredArgs = FI.getNumRequiredArgs();
405
406 NumRegsLeft = MaxNumRegsForArgsRet;
407 for (auto &I : FI.arguments()) {
408 if (CC == llvm::CallingConv::SPIR_KERNEL) {
409 I.info = classifyKernelArgumentType(Ty: I.type);
410 } else {
411 bool FixedArgument = ArgumentIndex++ < NumRequiredArgs;
412 I.info = classifyArgumentType(Ty: I.type, Variadic: !FixedArgument);
413 }
414 }
415}
416
417llvm::FixedVectorType *
418SPIRVABIInfo::getOptimalVectorMemoryType(llvm::FixedVectorType *Ty,
419 const LangOptions &LangOpt) const {
420 // For Logical SPIR-V, we don't know the underlying hardware or layout.
421 // This means we don't know which vector size is better, and also cannot
422 // assume a smaller vector size is stored in a larger vector size.
423 if (getTarget().getTriple().isSPIRVLogical())
424 return Ty;
425 return DefaultABIInfo::getOptimalVectorMemoryType(T: Ty, Opt: LangOpt);
426}
427
428llvm::FixedVectorType *AMDGCNSPIRVABIInfo::getOptimalVectorMemoryType(
429 llvm::FixedVectorType *Ty, const LangOptions &LangOpt) const {
430 // AMDGPU has legal instructions for 96-bit so 3x32 can be supported.
431 if (Ty->getNumElements() == 3 && getDataLayout().getTypeSizeInBits(Ty) == 96)
432 return Ty;
433 return DefaultABIInfo::getOptimalVectorMemoryType(T: Ty, Opt: LangOpt);
434}
435
436namespace clang {
437namespace CodeGen {
438void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI) {
439 if (CGM.getTarget().getTriple().isSPIRV()) {
440 if (CGM.getTarget().getTriple().getVendor() == llvm::Triple::AMD)
441 AMDGCNSPIRVABIInfo(CGM.getTypes()).computeInfo(FI);
442 else
443 SPIRVABIInfo(CGM.getTypes()).computeInfo(FI);
444 } else {
445 CommonSPIRABIInfo(CGM.getTypes()).computeInfo(FI);
446 }
447}
448}
449}
450
451unsigned CommonSPIRTargetCodeGenInfo::getDeviceKernelCallingConv() const {
452 return llvm::CallingConv::SPIR_KERNEL;
453}
454
455LangAS SPIRVTargetCodeGenInfo::getSRetAddrSpace(const CXXRecordDecl *RD) const {
456 // Types with no viable copy/move must be constructed in-place, use the
457 // default AS so the sret pointer matches the "this" convention.
458 if (RD && !RD->canPassInRegisters())
459 return LangAS::Default;
460 return getLangASFromTargetAS(
461 TargetAS: getABIInfo().getDataLayout().getAllocaAddrSpace());
462}
463
464void SPIRVTargetCodeGenInfo::setCUDAKernelCallingConvention(
465 const FunctionType *&FT) const {
466 // Convert HIP kernels to SPIR-V kernels.
467 if (getABIInfo().getContext().getLangOpts().HIP) {
468 FT = getABIInfo().getContext().adjustFunctionType(
469 Fn: FT, EInfo: FT->getExtInfo().withCallingConv(cc: CC_DeviceKernel));
470 return;
471 }
472}
473
474void CommonSPIRTargetCodeGenInfo::setOCLKernelStubCallingConvention(
475 const FunctionType *&FT) const {
476 FT = getABIInfo().getContext().adjustFunctionType(
477 Fn: FT, EInfo: FT->getExtInfo().withCallingConv(cc: CC_C));
478}
479
480// LLVM currently assumes a null pointer has the bit pattern 0, but some GPU
481// targets use a non-zero encoding for null in certain address spaces.
482// Because SPIR(-V) is a generic target and the bit pattern of null in
483// non-generic AS is unspecified, materialize null in non-generic AS via an
484// addrspacecast from null in generic AS. This allows later lowering to
485// substitute the target's real sentinel value.
486llvm::Constant *
487CommonSPIRTargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM,
488 llvm::PointerType *PT,
489 QualType QT) const {
490 if (!CodeGenUtils::spirNullPointerNeedsGenericCast(QT, Triple: CGM.getTriple()))
491 return llvm::ConstantPointerNull::get(T: PT);
492
493 auto &Ctx = CGM.getContext();
494 auto NPT = llvm::PointerType::get(
495 C&: PT->getContext(), AddressSpace: Ctx.getTargetAddressSpace(AS: LangAS::opencl_generic));
496 return llvm::ConstantExpr::getAddrSpaceCast(
497 C: llvm::ConstantPointerNull::get(T: NPT), Ty: PT);
498}
499
500LangAS
501SPIRVTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
502 const VarDecl *D) const {
503 assert(!CGM.getLangOpts().OpenCL &&
504 !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
505 "Address space agnostic languages only");
506 // If we're here it means that we're using the SPIRDefIsGen ASMap, hence for
507 // the global AS we can rely on either cuda_device or sycl_global to be
508 // correct; however, since this is not a CUDA Device context, we use
509 // sycl_global to prevent confusion with the assertion.
510 LangAS DefaultGlobalAS = getLangASFromTargetAS(
511 TargetAS: CGM.getContext().getTargetAddressSpace(AS: LangAS::sycl_global));
512 if (!D)
513 return DefaultGlobalAS;
514
515 LangAS AddrSpace = D->getType().getAddressSpace();
516 if (AddrSpace != LangAS::Default)
517 return AddrSpace;
518
519 return DefaultGlobalAS;
520}
521
522void SPIRVTargetCodeGenInfo::setTargetAttributes(
523 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &M) const {
524 if (GV->isDeclaration())
525 return;
526
527 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(Val: D);
528 if (!FD)
529 return;
530
531 llvm::Function *F = dyn_cast<llvm::Function>(Val: GV);
532 assert(F && "Expected GlobalValue to be a Function");
533
534 if (!M.getLangOpts().HIP ||
535 M.getTarget().getTriple().getVendor() != llvm::Triple::AMD)
536 return;
537
538 if (!FD->hasAttr<CUDAGlobalAttr>())
539 return;
540
541 unsigned N = M.getLangOpts().GPUMaxThreadsPerBlock;
542 if (auto FlatWGS = FD->getAttr<AMDGPUFlatWorkGroupSizeAttr>()) {
543 N = FlatWGS->getMax()->EvaluateKnownConstInt(Ctx: M.getContext()).getExtValue();
544 } else if (auto LB = FD->getAttr<CUDALaunchBoundsAttr>()) {
545 if (uint64_t MaxThreads = LB->getMaxThreads()
546 ->EvaluateKnownConstInt(Ctx: M.getContext())
547 .getExtValue())
548 N = MaxThreads;
549 }
550
551 // We encode the maximum flat WG size in the first component of the 3D
552 // max_work_group_size attribute, which will get reverse translated into the
553 // original AMDGPU attribute when targeting AMDGPU.
554 auto Int32Ty = llvm::IntegerType::getInt32Ty(C&: M.getLLVMContext());
555 llvm::Metadata *AttrMDArgs[] = {
556 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: Int32Ty, V: N)),
557 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: Int32Ty, V: 1)),
558 llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(Ty: Int32Ty, V: 1))};
559
560 F->setMetadata(Kind: "max_work_group_size",
561 Node: llvm::MDNode::get(Context&: M.getLLVMContext(), MDs: AttrMDArgs));
562}
563
564StringRef SPIRVTargetCodeGenInfo::getLLVMSyncScopeStr(
565 const LangOptions &, SyncScope Scope, llvm::AtomicOrdering) const {
566 return *llvm::getAtomicScopeIRString(T: getABIInfo().getTarget().getTriple(),
567 S: getAtomicScope(S: Scope));
568}
569
570void SPIRVTargetCodeGenInfo::setTargetAtomicMetadata(
571 CodeGenFunction &CGF, llvm::Instruction &AtomicInst,
572 const AtomicExpr *AE) const {
573 if (CGF.CGM.getTriple().getVendor() != llvm::Triple::VendorType::AMD)
574 return;
575
576 auto *RMW = dyn_cast<llvm::AtomicRMWInst>(Val: &AtomicInst);
577 if (!RMW)
578 return;
579
580 AtomicOptions AO = CGF.CGM.getAtomicOpts();
581 llvm::MDNode *Empty = llvm::MDNode::get(Context&: CGF.getLLVMContext(), MDs: {});
582 if (!AO.getOption(Kind: clang::AtomicOptionKind::FineGrainedMemory))
583 RMW->setMetadata(Kind: "amdgpu.no.fine.grained.memory", Node: Empty);
584 if (!AO.getOption(Kind: clang::AtomicOptionKind::RemoteMemory))
585 RMW->setMetadata(Kind: "amdgpu.no.remote.memory", Node: Empty);
586 if (AO.getOption(Kind: clang::AtomicOptionKind::IgnoreDenormalMode))
587 addAtomicIgnoreDenormalModeMetadata(CGF, AtomicInst&: *RMW);
588}
589
590/// Construct a SPIR-V target extension type for the given OpenCL image type.
591static llvm::Type *getSPIRVImageType(llvm::LLVMContext &Ctx, StringRef BaseType,
592 StringRef OpenCLName,
593 unsigned AccessQualifier) {
594 // These parameters compare to the operands of OpTypeImage (see
595 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpTypeImage
596 // for more details). The first 6 integer parameters all default to 0, and
597 // will be changed to 1 only for the image type(s) that set the parameter to
598 // one. The 7th integer parameter is the access qualifier, which is tacked on
599 // at the end.
600 SmallVector<unsigned, 7> IntParams = {0, 0, 0, 0, 0, 0};
601
602 // Choose the dimension of the image--this corresponds to the Dim enum in
603 // SPIR-V (first integer parameter of OpTypeImage).
604 if (OpenCLName.starts_with(Prefix: "image2d"))
605 IntParams[0] = 1;
606 else if (OpenCLName.starts_with(Prefix: "image3d"))
607 IntParams[0] = 2;
608 else if (OpenCLName == "image1d_buffer")
609 IntParams[0] = 5; // Buffer
610 else
611 assert(OpenCLName.starts_with("image1d") && "Unknown image type");
612
613 // Set the other integer parameters of OpTypeImage if necessary. Note that the
614 // OpenCL image types don't provide any information for the Sampled or
615 // Image Format parameters.
616 if (OpenCLName.contains(Other: "_depth"))
617 IntParams[1] = 1;
618 if (OpenCLName.contains(Other: "_array"))
619 IntParams[2] = 1;
620 if (OpenCLName.contains(Other: "_msaa"))
621 IntParams[3] = 1;
622
623 // Access qualifier
624 IntParams.push_back(Elt: AccessQualifier);
625
626 return llvm::TargetExtType::get(Context&: Ctx, Name: BaseType, Types: {llvm::Type::getVoidTy(C&: Ctx)},
627 Ints: IntParams);
628}
629
630llvm::Type *CommonSPIRTargetCodeGenInfo::getOpenCLType(CodeGenModule &CGM,
631 const Type *Ty) const {
632 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
633 if (auto *PipeTy = dyn_cast<PipeType>(Val: Ty))
634 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Pipe", Types: {},
635 Ints: {!PipeTy->isReadOnly()});
636 if (auto *BuiltinTy = dyn_cast<BuiltinType>(Val: Ty)) {
637 enum AccessQualifier : unsigned { AQ_ro = 0, AQ_wo = 1, AQ_rw = 2 };
638 switch (BuiltinTy->getKind()) {
639#define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
640 case BuiltinType::Id: \
641 return getSPIRVImageType(Ctx, "spirv.Image", #ImgType, AQ_##Suffix);
642#include "clang/Basic/OpenCLImageTypes.def"
643 case BuiltinType::OCLSampler:
644 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Sampler");
645 case BuiltinType::OCLEvent:
646 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Event");
647 case BuiltinType::OCLClkEvent:
648 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.DeviceEvent");
649 case BuiltinType::OCLQueue:
650 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Queue");
651 case BuiltinType::OCLReserveID:
652 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.ReserveId");
653#define INTEL_SUBGROUP_AVC_TYPE(Name, Id) \
654 case BuiltinType::OCLIntelSubgroupAVC##Id: \
655 return llvm::TargetExtType::get(Ctx, "spirv.Avc" #Id "INTEL");
656#include "clang/Basic/OpenCLExtensionTypes.def"
657 default:
658 return nullptr;
659 }
660 }
661
662 return nullptr;
663}
664
665// Gets a spirv.IntegralConstant or spirv.Literal. If IntegralType is present,
666// returns an IntegralConstant, otherwise returns a Literal.
667static llvm::Type *getInlineSpirvConstant(CodeGenModule &CGM,
668 llvm::Type *IntegralType,
669 llvm::APInt Value) {
670 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
671
672 // Convert the APInt value to an array of uint32_t words
673 llvm::SmallVector<uint32_t> Words;
674
675 while (Value.ugt(RHS: 0)) {
676 uint32_t Word = Value.trunc(width: 32).getZExtValue();
677 Value.lshrInPlace(ShiftAmt: 32);
678
679 Words.push_back(Elt: Word);
680 }
681 if (Words.size() == 0)
682 Words.push_back(Elt: 0);
683
684 if (IntegralType)
685 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.IntegralConstant",
686 Types: {IntegralType}, Ints: Words);
687 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Literal", Types: {}, Ints: Words);
688}
689
690static llvm::Type *getInlineSpirvType(CodeGenModule &CGM,
691 const HLSLInlineSpirvType *SpirvType) {
692 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
693
694 llvm::SmallVector<llvm::Type *> Operands;
695
696 for (auto &Operand : SpirvType->getOperands()) {
697 using SpirvOperandKind = SpirvOperand::SpirvOperandKind;
698
699 llvm::Type *Result = nullptr;
700 switch (Operand.getKind()) {
701 case SpirvOperandKind::ConstantId: {
702 llvm::Type *IntegralType =
703 CGM.getTypes().ConvertType(T: Operand.getResultType());
704
705 Result = getInlineSpirvConstant(CGM, IntegralType, Value: Operand.getValue());
706 break;
707 }
708 case SpirvOperandKind::Literal: {
709 Result = getInlineSpirvConstant(CGM, IntegralType: nullptr, Value: Operand.getValue());
710 break;
711 }
712 case SpirvOperandKind::TypeId: {
713 QualType TypeOperand = Operand.getResultType();
714 if (const auto *RD = TypeOperand->getAsRecordDecl()) {
715 assert(RD->isCompleteDefinition() &&
716 "Type completion should have been required in Sema");
717
718 const FieldDecl *HandleField = RD->findFirstNamedDataMember();
719 if (HandleField) {
720 QualType ResourceType = HandleField->getType();
721 if (ResourceType->getAs<HLSLAttributedResourceType>()) {
722 TypeOperand = ResourceType;
723 }
724 }
725 }
726 Result = CGM.getTypes().ConvertType(T: TypeOperand);
727 break;
728 }
729 default:
730 llvm_unreachable("HLSLInlineSpirvType had invalid operand!");
731 break;
732 }
733
734 assert(Result);
735 Operands.push_back(Elt: Result);
736 }
737
738 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Type", Types: Operands,
739 Ints: {SpirvType->getOpcode(), SpirvType->getSize(),
740 SpirvType->getAlignment()});
741}
742
743llvm::Type *CommonSPIRTargetCodeGenInfo::getHLSLType(
744 CodeGenModule &CGM, const Type *Ty,
745 const CGHLSLOffsetInfo &OffsetInfo) const {
746 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
747
748 if (auto *SpirvType = dyn_cast<HLSLInlineSpirvType>(Val: Ty))
749 return getInlineSpirvType(CGM, SpirvType);
750
751 auto *ResType = dyn_cast<HLSLAttributedResourceType>(Val: Ty);
752 if (!ResType)
753 return nullptr;
754
755 const HLSLAttributedResourceType::Attributes &ResAttrs = ResType->getAttrs();
756 switch (ResAttrs.ResourceClass) {
757 case llvm::dxil::ResourceClass::UAV:
758 case llvm::dxil::ResourceClass::SRV: {
759 // TypedBuffer and RawBuffer both need element type
760 QualType ContainedTy = ResType->getContainedType();
761 if (ContainedTy.isNull())
762 return nullptr;
763
764 assert(!ResAttrs.IsROV &&
765 "Rasterizer order views not implemented for SPIR-V yet");
766
767 if (!ResAttrs.RawBuffer) {
768 // convert element type
769 return getSPIRVImageTypeFromHLSLResource(attributes: ResAttrs, SampledType: ContainedTy, CGM);
770 }
771
772 if (ResAttrs.IsCounter) {
773 llvm::Type *ElemType = llvm::Type::getInt32Ty(C&: Ctx);
774 uint32_t StorageClass = /* StorageBuffer storage class */ 12;
775 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.VulkanBuffer", Types: {ElemType},
776 Ints: {StorageClass, true});
777 }
778 llvm::Type *ElemType = CGM.getTypes().ConvertTypeForMem(T: ContainedTy);
779 llvm::ArrayType *RuntimeArrayType = llvm::ArrayType::get(ElementType: ElemType, NumElements: 0);
780 uint32_t StorageClass = /* StorageBuffer storage class */ 12;
781 bool IsWritable = ResAttrs.ResourceClass == llvm::dxil::ResourceClass::UAV;
782 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.VulkanBuffer",
783 Types: {RuntimeArrayType},
784 Ints: {StorageClass, IsWritable});
785 }
786 case llvm::dxil::ResourceClass::CBuffer: {
787 QualType ContainedTy = ResType->getContainedType();
788 if (ContainedTy.isNull() || !ContainedTy->isStructureType())
789 return nullptr;
790
791 llvm::StructType *BufferLayoutTy =
792 HLSLBufferLayoutBuilder(CGM).layOutStruct(
793 StructType: ContainedTy->getAsCanonical<RecordType>(), OffsetInfo);
794 uint32_t StorageClass = /* Uniform storage class */ 2;
795 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.VulkanBuffer", Types: {BufferLayoutTy},
796 Ints: {StorageClass, false});
797 break;
798 }
799 case llvm::dxil::ResourceClass::Sampler:
800 return llvm::TargetExtType::get(Context&: Ctx, Name: "spirv.Sampler");
801 }
802 return nullptr;
803}
804
805static unsigned
806getImageFormat(const LangOptions &LangOpts,
807 const HLSLAttributedResourceType::Attributes &attributes,
808 llvm::Type *SampledType, QualType Ty, unsigned NumChannels) {
809 // For images with `Sampled` operand equal to 2, there are restrictions on
810 // using the Unknown image format. To avoid these restrictions in common
811 // cases, we guess an image format for them based on the sampled type and the
812 // number of channels. This is intended to match the behaviour of DXC.
813 if (LangOpts.HLSLSpvUseUnknownImageFormat ||
814 attributes.ResourceClass != llvm::dxil::ResourceClass::UAV) {
815 return 0; // Unknown
816 }
817
818 if (SampledType->isIntegerTy(BitWidth: 32)) {
819 if (Ty->isSignedIntegerType()) {
820 if (NumChannels == 1)
821 return 24; // R32i
822 if (NumChannels == 2)
823 return 25; // Rg32i
824 if (NumChannels == 4)
825 return 21; // Rgba32i
826 } else {
827 if (NumChannels == 1)
828 return 33; // R32ui
829 if (NumChannels == 2)
830 return 35; // Rg32ui
831 if (NumChannels == 4)
832 return 30; // Rgba32ui
833 }
834 } else if (SampledType->isIntegerTy(BitWidth: 64)) {
835 if (NumChannels == 1) {
836 if (Ty->isSignedIntegerType()) {
837 return 41; // R64i
838 }
839 return 40; // R64ui
840 }
841 } else if (SampledType->isFloatTy()) {
842 if (NumChannels == 1)
843 return 3; // R32f
844 if (NumChannels == 2)
845 return 6; // Rg32f
846 if (NumChannels == 4)
847 return 1; // Rgba32f
848 }
849
850 return 0; // Unknown
851}
852
853llvm::Type *CommonSPIRTargetCodeGenInfo::getSPIRVImageTypeFromHLSLResource(
854 const HLSLAttributedResourceType::Attributes &attributes, QualType Ty,
855 CodeGenModule &CGM) const {
856 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
857
858 unsigned NumChannels = 1;
859 Ty = Ty->getCanonicalTypeUnqualified();
860 if (const VectorType *V = dyn_cast<VectorType>(Val&: Ty)) {
861 NumChannels = V->getNumElements();
862 Ty = V->getElementType();
863 }
864 assert(!Ty->isVectorType() && "We still have a vector type.");
865
866 llvm::Type *SampledType = CGM.getTypes().ConvertTypeForMem(T: Ty);
867
868 assert((SampledType->isIntegerTy() || SampledType->isFloatingPointTy()) &&
869 "The element type for a SPIR-V resource must be a scalar integer or "
870 "floating point type.");
871
872 assert((!SampledType->isIntegerTy(64) || NumChannels <= 2) &&
873 "A 64-bit SPIR-V resource element can have at most 2 components.");
874
875 // SPIR-V has no 64-bit multi-component image format, so pack a 2-component
876 // 64-bit typed buffer into a 4-component 32-bit image. The backend
877 // reinterprets it with OpBitcast on load and store.
878 if (SampledType->isIntegerTy(BitWidth: 64) && NumChannels == 2) {
879 SampledType = llvm::Type::getInt32Ty(C&: Ctx);
880 NumChannels = 4;
881 }
882
883 // These parameters correspond to the operands to the OpTypeImage SPIR-V
884 // instruction. See
885 // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpTypeImage.
886 SmallVector<unsigned, 6> IntParams(6, 0);
887
888 const char *Name =
889 Ty->isSignedIntegerType() ? "spirv.SignedImage" : "spirv.Image";
890
891 // Dim
892 switch (attributes.ResourceDimension) {
893 case llvm::dxil::ResourceDimension::Dim1D:
894 IntParams[0] = 0;
895 break;
896 case llvm::dxil::ResourceDimension::Dim2D:
897 IntParams[0] = 1;
898 break;
899 case llvm::dxil::ResourceDimension::Dim3D:
900 IntParams[0] = 2;
901 break;
902 case llvm::dxil::ResourceDimension::Cube:
903 IntParams[0] = 3;
904 break;
905 case llvm::dxil::ResourceDimension::Unknown:
906 IntParams[0] = 5;
907 break;
908 }
909
910 // Depth
911 // HLSL does not indicate if it is a depth texture or not, so we use unknown.
912 IntParams[1] = 2;
913
914 // Arrayed
915 IntParams[2] = static_cast<unsigned>(attributes.IsArray);
916
917 // MS
918 IntParams[3] = static_cast<unsigned>(attributes.isMultiSampled());
919
920 // Sampled
921 IntParams[4] =
922 attributes.ResourceClass == llvm::dxil::ResourceClass::UAV ? 2 : 1;
923
924 // Image format.
925 IntParams[5] = getImageFormat(LangOpts: CGM.getLangOpts(), attributes, SampledType, Ty,
926 NumChannels);
927
928 llvm::TargetExtType *ImageType =
929 llvm::TargetExtType::get(Context&: Ctx, Name, Types: {SampledType}, Ints: IntParams);
930 return ImageType;
931}
932
933std::unique_ptr<TargetCodeGenInfo>
934CodeGen::createCommonSPIRTargetCodeGenInfo(CodeGenModule &CGM) {
935 return std::make_unique<CommonSPIRTargetCodeGenInfo>(args&: CGM.getTypes());
936}
937
938std::unique_ptr<TargetCodeGenInfo>
939CodeGen::createSPIRVTargetCodeGenInfo(CodeGenModule &CGM) {
940 return std::make_unique<SPIRVTargetCodeGenInfo>(args&: CGM.getTypes());
941}
942