1//===- SPIRVTypes.cpp - MLIR SPIR-V Types ---------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the types in the SPIR-V dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
14#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
15#include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
16#include "mlir/IR/BuiltinTypes.h"
17#include "mlir/Support/LLVM.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/TypeSwitch.h"
20#include "llvm/Support/ErrorHandling.h"
21
22#include <cstdint>
23#include <optional>
24
25using namespace mlir;
26using namespace mlir::spirv;
27
28namespace {
29// Helper function to collect extensions implied by a type by visiting all its
30// subtypes. Maintains a set of `seen` types to avoid recursion in structs.
31//
32// Serves as the source-of-truth for type extension information. All extension
33// logic should be added to this class, while the
34// `SPIRVType::getExtensions` function should not handle extension-related logic
35// directly and only invoke `TypeExtensionVisitor::add(Type *)`.
36class TypeExtensionVisitor {
37public:
38 TypeExtensionVisitor(SPIRVType::ExtensionArrayRefVector &extensions,
39 std::optional<StorageClass> storage)
40 : extensions(extensions), storage(storage) {}
41
42 // Main visitor entry point. Adds all extensions to the vector. Saves `type`
43 // as seen and dispatches to the right concrete `.add` function.
44 void add(SPIRVType type) {
45 if (auto [_it, inserted] = seen.insert(V: {type, storage}); !inserted)
46 return;
47
48 TypeSwitch<SPIRVType>(type)
49 .Case<CooperativeMatrixType, ImageType, PointerType, ScalarType,
50 TensorArmType>(
51 caseFn: [this](auto concreteType) { addConcrete(concreteType); })
52 .Case<ArrayType, MatrixType, RuntimeArrayType, VectorType>(
53 caseFn: [this](auto concreteType) { add(concreteType.getElementType()); })
54 .Case(caseFn: [this](SampledImageType concreteType) {
55 add(type: concreteType.getImageType());
56 })
57 .Case(caseFn: [this](StructType concreteType) {
58 for (Type elementType : concreteType.getElementTypes())
59 add(type: elementType);
60 })
61 .Case<SamplerType, NamedBarrierType>(caseFn: [](auto) { /* no extensions */ })
62 .DefaultUnreachable(message: "Unhandled type");
63 }
64
65 void add(Type type) { add(type: cast<SPIRVType>(Val&: type)); }
66
67private:
68 // Types that add unique extensions.
69 void addConcrete(CooperativeMatrixType type);
70 void addConcrete(ImageType type);
71 void addConcrete(PointerType type);
72 void addConcrete(ScalarType type);
73 void addConcrete(TensorArmType type);
74
75 template <Extension... Es>
76 void pushExts() {
77 static constexpr Extension exts[] = {Es...};
78 extensions.push_back(Elt: exts);
79 }
80
81 SPIRVType::ExtensionArrayRefVector &extensions;
82 std::optional<StorageClass> storage;
83 llvm::SmallDenseSet<std::pair<Type, std::optional<StorageClass>>> seen;
84};
85
86// Helper function to collect capabilities implied by a type by visiting all its
87// subtypes. Maintains a set of `seen` types to avoid recursion in structs.
88//
89// Serves as the source-of-truth for type capability information. All capability
90// logic should be added to this class, while the
91// `SPIRVType::getCapabilities` function should not handle capability-related
92// logic directly and only invoke `TypeCapabilityVisitor::add(Type *)`.
93class TypeCapabilityVisitor {
94public:
95 TypeCapabilityVisitor(SPIRVType::CapabilityArrayRefVector &capabilities,
96 std::optional<StorageClass> storage)
97 : capabilities(capabilities), storage(storage) {}
98
99 // Main visitor entry point. Adds all extensions to the vector. Saves `type`
100 // as seen and dispatches to the right concrete `.add` function.
101 void add(SPIRVType type) {
102 if (auto [_it, inserted] = seen.insert(V: {type, storage}); !inserted)
103 return;
104
105 TypeSwitch<SPIRVType>(type)
106 .Case<CooperativeMatrixType, ImageType, MatrixType, PointerType,
107 RuntimeArrayType, ScalarType, TensorArmType, VectorType>(
108 caseFn: [this](auto concreteType) { addConcrete(concreteType); })
109 .Case(caseFn: [this](ArrayType concreteType) {
110 add(type: concreteType.getElementType());
111 })
112 .Case(caseFn: [this](SampledImageType concreteType) {
113 add(type: concreteType.getImageType());
114 })
115 .Case(caseFn: [this](StructType concreteType) {
116 for (Type elementType : concreteType.getElementTypes())
117 add(type: elementType);
118 })
119 .Case(caseFn: [](SamplerType) { /* no capabilities */ })
120 .Case(
121 caseFn: [this](NamedBarrierType) { pushCaps<Capability::NamedBarrier>(); })
122 .DefaultUnreachable(message: "Unhandled type");
123 }
124
125 void add(Type type) { add(type: cast<SPIRVType>(Val&: type)); }
126
127private:
128 // Types that add unique extensions.
129 void addConcrete(CooperativeMatrixType type);
130 void addConcrete(ImageType type);
131 void addConcrete(MatrixType type);
132 void addConcrete(PointerType type);
133 void addConcrete(RuntimeArrayType type);
134 void addConcrete(ScalarType type);
135 void addConcrete(TensorArmType type);
136 void addConcrete(VectorType type);
137
138 template <Capability... Cs>
139 void pushCaps() {
140 static constexpr Capability caps[] = {Cs...};
141 capabilities.push_back(Elt: caps);
142 }
143
144 SPIRVType::CapabilityArrayRefVector &capabilities;
145 std::optional<StorageClass> storage;
146 llvm::SmallDenseSet<std::pair<Type, std::optional<StorageClass>>> seen;
147};
148
149} // namespace
150
151//===----------------------------------------------------------------------===//
152// ArrayType
153//===----------------------------------------------------------------------===//
154
155struct spirv::detail::ArrayTypeStorage : public TypeStorage {
156 using KeyTy = std::tuple<Type, unsigned, unsigned>;
157
158 static ArrayTypeStorage *construct(TypeStorageAllocator &allocator,
159 const KeyTy &key) {
160 return new (allocator.allocate<ArrayTypeStorage>()) ArrayTypeStorage(key);
161 }
162
163 bool operator==(const KeyTy &key) const {
164 return key == KeyTy(elementType, elementCount, stride);
165 }
166
167 ArrayTypeStorage(const KeyTy &key)
168 : elementType(std::get<0>(t: key)), elementCount(std::get<1>(t: key)),
169 stride(std::get<2>(t: key)) {}
170
171 Type elementType;
172 unsigned elementCount;
173 unsigned stride;
174};
175
176ArrayType ArrayType::get(Type elementType, unsigned elementCount) {
177 assert(elementCount && "ArrayType needs at least one element");
178 return Base::get(ctx: elementType.getContext(), args&: elementType, args&: elementCount,
179 /*stride=*/args: 0);
180}
181
182ArrayType ArrayType::get(Type elementType, unsigned elementCount,
183 unsigned stride) {
184 assert(elementCount && "ArrayType needs at least one element");
185 return Base::get(ctx: elementType.getContext(), args&: elementType, args&: elementCount, args&: stride);
186}
187
188unsigned ArrayType::getNumElements() const { return getImpl()->elementCount; }
189
190Type ArrayType::getElementType() const { return getImpl()->elementType; }
191
192unsigned ArrayType::getArrayStride() const { return getImpl()->stride; }
193
194//===----------------------------------------------------------------------===//
195// CompositeType
196//===----------------------------------------------------------------------===//
197
198bool CompositeType::classof(Type type) {
199 if (auto vectorType = dyn_cast<VectorType>(Val&: type))
200 return isValid(vectorType);
201 return isa<spirv::ArrayType, spirv::CooperativeMatrixType, spirv::MatrixType,
202 spirv::RuntimeArrayType, spirv::StructType, spirv::TensorArmType>(
203 Val: type);
204}
205
206bool CompositeType::isValid(VectorType type) {
207 return type.getRank() == 1 &&
208 llvm::is_contained(Set: {2, 3, 4, 8, 16}, Element: type.getNumElements()) &&
209 (isa<ScalarType>(Val: type.getElementType()) ||
210 isa<PointerType>(Val: type.getElementType()));
211}
212
213Type CompositeType::getElementType(unsigned index) const {
214 return TypeSwitch<Type, Type>(*this)
215 .Case<ArrayType, CooperativeMatrixType, RuntimeArrayType, VectorType,
216 TensorArmType>(caseFn: [](auto type) { return type.getElementType(); })
217 .Case(caseFn: [](MatrixType type) { return type.getColumnType(); })
218 .Case(caseFn: [index](StructType type) { return type.getElementType(index); })
219 .DefaultUnreachable(message: "Invalid composite type");
220}
221
222unsigned CompositeType::getNumElements() const {
223 return TypeSwitch<SPIRVType, unsigned>(*this)
224 .Case<ArrayType, StructType, TensorArmType, VectorType>(
225 caseFn: [](auto type) { return type.getNumElements(); })
226 .Case(caseFn: [](MatrixType type) { return type.getNumColumns(); })
227 .DefaultUnreachable(message: "Invalid type for number of elements query");
228}
229
230bool CompositeType::hasCompileTimeKnownNumElements() const {
231 return !isa<CooperativeMatrixType, RuntimeArrayType>(Val: *this);
232}
233
234void TypeCapabilityVisitor::addConcrete(VectorType type) {
235 add(type: type.getElementType());
236
237 int64_t vecSize = type.getNumElements();
238 if (vecSize == 8 || vecSize == 16)
239 pushCaps<Capability::Vector16>();
240}
241
242//===----------------------------------------------------------------------===//
243// CooperativeMatrixType
244//===----------------------------------------------------------------------===//
245
246struct spirv::detail::CooperativeMatrixTypeStorage final : TypeStorage {
247 // In the specification dimensions of the Cooperative Matrix are 32-bit
248 // integers --- the initial implementation kept those values as such. However,
249 // the `ShapedType` expects the shape to be `int64_t`. We could keep the shape
250 // as 32-bits and expose it as int64_t through `getShape`, however, this
251 // method returns an `ArrayRef`, so returning `ArrayRef<int64_t>` having two
252 // 32-bits integers would require an extra logic and storage. So, we diverge
253 // from the spec and internally represent the dimensions as 64-bit integers,
254 // so we can easily return an `ArrayRef` from `getShape` without any extra
255 // logic. Alternatively, we could store both rows and columns (both 32-bits)
256 // and shape (64-bits), assigning rows and columns to shape whenever
257 // `getShape` is called. This would be at the cost of extra logic and storage.
258 // Note: Because `ArrayRef` is returned we cannot construct an object in
259 // `getShape` on the fly.
260 using KeyTy =
261 std::tuple<Type, int64_t, int64_t, Scope, CooperativeMatrixUseKHR>;
262
263 static CooperativeMatrixTypeStorage *
264 construct(TypeStorageAllocator &allocator, const KeyTy &key) {
265 return new (allocator.allocate<CooperativeMatrixTypeStorage>())
266 CooperativeMatrixTypeStorage(key);
267 }
268
269 bool operator==(const KeyTy &key) const {
270 return key == KeyTy(elementType, shape[0], shape[1], scope, use);
271 }
272
273 CooperativeMatrixTypeStorage(const KeyTy &key)
274 : elementType(std::get<0>(t: key)),
275 shape({std::get<1>(t: key), std::get<2>(t: key)}), scope(std::get<3>(t: key)),
276 use(std::get<4>(t: key)) {}
277
278 Type elementType;
279 // [#rows, #columns]
280 std::array<int64_t, 2> shape;
281 Scope scope;
282 CooperativeMatrixUseKHR use;
283};
284
285CooperativeMatrixType CooperativeMatrixType::get(Type elementType,
286 uint32_t rows,
287 uint32_t columns, Scope scope,
288 CooperativeMatrixUseKHR use) {
289 return Base::get(ctx: elementType.getContext(), args&: elementType, args&: rows, args&: columns, args&: scope,
290 args&: use);
291}
292
293Type CooperativeMatrixType::getElementType() const {
294 return getImpl()->elementType;
295}
296
297uint32_t CooperativeMatrixType::getRows() const {
298 assert(getImpl()->shape[0] != ShapedType::kDynamic);
299 return static_cast<uint32_t>(getImpl()->shape[0]);
300}
301
302uint32_t CooperativeMatrixType::getColumns() const {
303 assert(getImpl()->shape[1] != ShapedType::kDynamic);
304 return static_cast<uint32_t>(getImpl()->shape[1]);
305}
306
307ArrayRef<int64_t> CooperativeMatrixType::getShape() const {
308 return getImpl()->shape;
309}
310
311Scope CooperativeMatrixType::getScope() const { return getImpl()->scope; }
312
313CooperativeMatrixUseKHR CooperativeMatrixType::getUse() const {
314 return getImpl()->use;
315}
316
317void TypeExtensionVisitor::addConcrete(CooperativeMatrixType type) {
318 add(type: type.getElementType());
319 pushExts<Extension::SPV_KHR_cooperative_matrix>();
320}
321
322void TypeCapabilityVisitor::addConcrete(CooperativeMatrixType type) {
323 Type elementType = type.getElementType();
324 add(type: elementType);
325 pushCaps<Capability::CooperativeMatrixKHR>();
326 if (elementType.isBF16())
327 pushCaps<Capability::BFloat16CooperativeMatrixKHR>();
328 if (elementType.isF8E4M3FN() || elementType.isF8E5M2())
329 pushCaps<Capability::Float8CooperativeMatrixEXT>();
330}
331
332//===----------------------------------------------------------------------===//
333// ImageType
334//===----------------------------------------------------------------------===//
335
336template <typename T>
337static constexpr unsigned getNumBits() {
338 return 0;
339}
340template <>
341constexpr unsigned getNumBits<Dim>() {
342 static_assert((1 << 3) > getMaxEnumValForDim(),
343 "Not enough bits to encode Dim value");
344 return 3;
345}
346template <>
347constexpr unsigned getNumBits<ImageDepthInfo>() {
348 static_assert((1 << 2) > getMaxEnumValForImageDepthInfo(),
349 "Not enough bits to encode ImageDepthInfo value");
350 return 2;
351}
352template <>
353constexpr unsigned getNumBits<ImageArrayedInfo>() {
354 static_assert((1 << 1) > getMaxEnumValForImageArrayedInfo(),
355 "Not enough bits to encode ImageArrayedInfo value");
356 return 1;
357}
358template <>
359constexpr unsigned getNumBits<ImageSamplingInfo>() {
360 static_assert((1 << 1) > getMaxEnumValForImageSamplingInfo(),
361 "Not enough bits to encode ImageSamplingInfo value");
362 return 1;
363}
364template <>
365constexpr unsigned getNumBits<ImageSamplerUseInfo>() {
366 static_assert((1 << 2) > getMaxEnumValForImageSamplerUseInfo(),
367 "Not enough bits to encode ImageSamplerUseInfo value");
368 return 2;
369}
370template <>
371constexpr unsigned getNumBits<ImageFormat>() {
372 static_assert((1 << 6) > getMaxEnumValForImageFormat(),
373 "Not enough bits to encode ImageFormat value");
374 return 6;
375}
376
377struct spirv::detail::ImageTypeStorage : public TypeStorage {
378public:
379 using KeyTy = std::tuple<Type, Dim, ImageDepthInfo, ImageArrayedInfo,
380 ImageSamplingInfo, ImageSamplerUseInfo, ImageFormat>;
381
382 static ImageTypeStorage *construct(TypeStorageAllocator &allocator,
383 const KeyTy &key) {
384 return new (allocator.allocate<ImageTypeStorage>()) ImageTypeStorage(key);
385 }
386
387 bool operator==(const KeyTy &key) const {
388 return key == KeyTy(elementType, dim, depthInfo, arrayedInfo, samplingInfo,
389 samplerUseInfo, format);
390 }
391
392 ImageTypeStorage(const KeyTy &key)
393 : elementType(std::get<0>(t: key)), dim(std::get<1>(t: key)),
394 depthInfo(std::get<2>(t: key)), arrayedInfo(std::get<3>(t: key)),
395 samplingInfo(std::get<4>(t: key)), samplerUseInfo(std::get<5>(t: key)),
396 format(std::get<6>(t: key)) {}
397
398 Type elementType;
399 Dim dim : getNumBits<Dim>();
400 ImageDepthInfo depthInfo : getNumBits<ImageDepthInfo>();
401 ImageArrayedInfo arrayedInfo : getNumBits<ImageArrayedInfo>();
402 ImageSamplingInfo samplingInfo : getNumBits<ImageSamplingInfo>();
403 ImageSamplerUseInfo samplerUseInfo : getNumBits<ImageSamplerUseInfo>();
404 ImageFormat format : getNumBits<ImageFormat>();
405};
406
407ImageType
408ImageType::get(std::tuple<Type, Dim, ImageDepthInfo, ImageArrayedInfo,
409 ImageSamplingInfo, ImageSamplerUseInfo, ImageFormat>
410 value) {
411 return Base::get(ctx: std::get<0>(t&: value).getContext(), args&: value);
412}
413
414Type ImageType::getElementType() const { return getImpl()->elementType; }
415
416Dim ImageType::getDim() const { return getImpl()->dim; }
417
418ImageDepthInfo ImageType::getDepthInfo() const { return getImpl()->depthInfo; }
419
420ImageArrayedInfo ImageType::getArrayedInfo() const {
421 return getImpl()->arrayedInfo;
422}
423
424ImageSamplingInfo ImageType::getSamplingInfo() const {
425 return getImpl()->samplingInfo;
426}
427
428ImageSamplerUseInfo ImageType::getSamplerUseInfo() const {
429 return getImpl()->samplerUseInfo;
430}
431
432ImageFormat ImageType::getImageFormat() const { return getImpl()->format; }
433
434void TypeExtensionVisitor::addConcrete(ImageType type) {
435 // OpTypeImage with a 64-bit integer Sampled Type requires the
436 // SPV_EXT_shader_image_int64 extension (companion to Int64ImageEXT).
437 if (auto intTy = dyn_cast<IntegerType>(Val: type.getElementType());
438 intTy && intTy.getWidth() == 64)
439 pushExts<Extension::SPV_EXT_shader_image_int64>();
440 add(type: type.getElementType());
441}
442
443void TypeCapabilityVisitor::addConcrete(ImageType type) {
444 // Capability requirements for OpTypeImage are determined jointly by Dim,
445 // Sampled, MS, and Arrayed - see the SPIR-V spec's "Capabilities" column on
446 // OpTypeImage.
447 Dim dim = type.getDim();
448 bool isMultisampled =
449 type.getSamplingInfo() == ImageSamplingInfo::MultiSampled;
450 bool isArrayed = type.getArrayedInfo() == ImageArrayedInfo::Arrayed;
451 ImageSamplerUseInfo sampler = type.getSamplerUseInfo();
452 bool noSampler = sampler == ImageSamplerUseInfo::NoSampler;
453 bool needSampler = sampler == ImageSamplerUseInfo::NeedSampler;
454
455 switch (dim) {
456 case Dim::Dim1D:
457 if (needSampler)
458 pushCaps<Capability::Sampled1D>();
459 else if (noSampler)
460 pushCaps<Capability::Image1D>();
461 else
462 pushCaps<Capability::Image1D, Capability::Sampled1D>();
463 break;
464 case Dim::Dim2D:
465 if (isMultisampled && noSampler)
466 pushCaps<Capability::StorageImageMultisample>();
467 if (isMultisampled && isArrayed)
468 pushCaps<Capability::ImageMSArray>();
469 break;
470 case Dim::Dim3D:
471 break;
472 case Dim::Cube:
473 pushCaps<Capability::Shader>();
474 if (isArrayed)
475 pushCaps<Capability::ImageCubeArray>();
476 break;
477 case Dim::Rect:
478 pushCaps<Capability::ImageRect, Capability::SampledRect>();
479 break;
480 case Dim::Buffer:
481 if (needSampler)
482 pushCaps<Capability::SampledBuffer>();
483 else if (noSampler)
484 pushCaps<Capability::ImageBuffer>();
485 else
486 pushCaps<Capability::ImageBuffer, Capability::SampledBuffer>();
487 break;
488 case Dim::SubpassData:
489 pushCaps<Capability::InputAttachment>();
490 break;
491 }
492
493 if (auto fmtCaps = spirv::getCapabilities(value: type.getImageFormat()))
494 capabilities.push_back(Elt: *fmtCaps);
495
496 // OpTypeImage with a 64-bit integer Sampled Type requires Int64ImageEXT.
497 if (auto intTy = dyn_cast<IntegerType>(Val: type.getElementType());
498 intTy && intTy.getWidth() == 64)
499 pushCaps<Capability::Int64ImageEXT>();
500
501 add(type: type.getElementType());
502}
503
504//===----------------------------------------------------------------------===//
505// PointerType
506//===----------------------------------------------------------------------===//
507
508struct spirv::detail::PointerTypeStorage : public TypeStorage {
509 // (Type, StorageClass) as the key: Type stored in this struct, and
510 // StorageClass stored as TypeStorage's subclass data.
511 using KeyTy = std::pair<Type, StorageClass>;
512
513 static PointerTypeStorage *construct(TypeStorageAllocator &allocator,
514 const KeyTy &key) {
515 return new (allocator.allocate<PointerTypeStorage>())
516 PointerTypeStorage(key);
517 }
518
519 bool operator==(const KeyTy &key) const {
520 return key == KeyTy(pointeeType, storageClass);
521 }
522
523 PointerTypeStorage(const KeyTy &key)
524 : pointeeType(key.first), storageClass(key.second) {}
525
526 Type pointeeType;
527 StorageClass storageClass;
528};
529
530PointerType PointerType::get(Type pointeeType, StorageClass storageClass) {
531 return Base::get(ctx: pointeeType.getContext(), args&: pointeeType, args&: storageClass);
532}
533
534Type PointerType::getPointeeType() const { return getImpl()->pointeeType; }
535
536StorageClass PointerType::getStorageClass() const {
537 return getImpl()->storageClass;
538}
539
540void TypeExtensionVisitor::addConcrete(PointerType type) {
541 // Use this pointer type's storage class because this pointer indicates we are
542 // using the pointee type in that specific storage class.
543 std::optional<StorageClass> oldStorageClass = storage;
544 storage = type.getStorageClass();
545 add(type: type.getPointeeType());
546 storage = oldStorageClass;
547
548 if (auto scExts = spirv::getExtensions(value: type.getStorageClass()))
549 extensions.push_back(Elt: *scExts);
550}
551
552void TypeCapabilityVisitor::addConcrete(PointerType type) {
553 // Use this pointer type's storage class because this pointer indicates we are
554 // using the pointee type in that specific storage class.
555 std::optional<StorageClass> oldStorageClass = storage;
556 storage = type.getStorageClass();
557 add(type: type.getPointeeType());
558 storage = oldStorageClass;
559
560 if (auto scCaps = spirv::getCapabilities(value: type.getStorageClass()))
561 capabilities.push_back(Elt: *scCaps);
562}
563
564//===----------------------------------------------------------------------===//
565// RuntimeArrayType
566//===----------------------------------------------------------------------===//
567
568struct spirv::detail::RuntimeArrayTypeStorage : public TypeStorage {
569 using KeyTy = std::pair<Type, unsigned>;
570
571 static RuntimeArrayTypeStorage *construct(TypeStorageAllocator &allocator,
572 const KeyTy &key) {
573 return new (allocator.allocate<RuntimeArrayTypeStorage>())
574 RuntimeArrayTypeStorage(key);
575 }
576
577 bool operator==(const KeyTy &key) const {
578 return key == KeyTy(elementType, stride);
579 }
580
581 RuntimeArrayTypeStorage(const KeyTy &key)
582 : elementType(key.first), stride(key.second) {}
583
584 Type elementType;
585 unsigned stride;
586};
587
588RuntimeArrayType RuntimeArrayType::get(Type elementType) {
589 return Base::get(ctx: elementType.getContext(), args&: elementType, /*stride=*/args: 0);
590}
591
592RuntimeArrayType RuntimeArrayType::get(Type elementType, unsigned stride) {
593 return Base::get(ctx: elementType.getContext(), args&: elementType, args&: stride);
594}
595
596Type RuntimeArrayType::getElementType() const { return getImpl()->elementType; }
597
598unsigned RuntimeArrayType::getArrayStride() const { return getImpl()->stride; }
599
600void TypeCapabilityVisitor::addConcrete(RuntimeArrayType type) {
601 add(type: type.getElementType());
602 pushCaps<Capability::Shader>();
603}
604
605//===----------------------------------------------------------------------===//
606// ScalarType
607//===----------------------------------------------------------------------===//
608
609bool ScalarType::classof(Type type) {
610 if (auto floatType = dyn_cast<FloatType>(Val&: type)) {
611 return isValid(floatType);
612 }
613 if (auto intType = dyn_cast<IntegerType>(Val&: type)) {
614 return isValid(intType);
615 }
616 return false;
617}
618
619bool ScalarType::isValid(FloatType type) {
620 if (type.isF8E4M3FN() || type.isF8E5M2())
621 return true;
622 return llvm::is_contained(Set: {16u, 32u, 64u}, Element: type.getWidth());
623}
624
625bool ScalarType::isValid(IntegerType type) {
626 return llvm::is_contained(Set: {1u, 8u, 16u, 32u, 64u}, Element: type.getWidth());
627}
628
629void TypeExtensionVisitor::addConcrete(ScalarType type) {
630 if (type.isBF16())
631 pushExts<Extension::SPV_KHR_bfloat16>();
632
633 if (type.isF8E4M3FN() || type.isF8E5M2())
634 pushExts<Extension::SPV_EXT_float8>();
635
636 // 8- or 16-bit integer/floating-point numbers will require extra extensions
637 // to appear in interface storage classes. See SPV_KHR_16bit_storage and
638 // SPV_KHR_8bit_storage for more details.
639 if (!storage)
640 return;
641
642 switch (*storage) {
643 case StorageClass::PushConstant:
644 case StorageClass::StorageBuffer:
645 case StorageClass::Uniform:
646 if (type.getIntOrFloatBitWidth() == 8)
647 pushExts<Extension::SPV_KHR_8bit_storage>();
648 [[fallthrough]];
649 case StorageClass::Input:
650 case StorageClass::Output:
651 if (type.getIntOrFloatBitWidth() == 16)
652 pushExts<Extension::SPV_KHR_16bit_storage>();
653 break;
654 default:
655 break;
656 }
657}
658
659void TypeCapabilityVisitor::addConcrete(ScalarType type) {
660 unsigned bitwidth = type.getIntOrFloatBitWidth();
661
662 // 8- or 16-bit integer/floating-point numbers will require extra capabilities
663 // to appear in interface storage classes. See SPV_KHR_16bit_storage and
664 // SPV_KHR_8bit_storage for more details.
665
666#define STORAGE_CASE(storage, cap8, cap16) \
667 case StorageClass::storage: { \
668 if (bitwidth == 8) { \
669 pushCaps<Capability::cap8>(); \
670 return; \
671 } \
672 if (bitwidth == 16) { \
673 pushCaps<Capability::cap16>(); \
674 return; \
675 } \
676 /* For 64-bit integers/floats, Int64/Float64 enables support for all */ \
677 /* storage classes. Fall through to the next section. */ \
678 } break
679
680 // This part only handles the cases where special bitwidths appearing in
681 // interface storage classes.
682 if (storage) {
683 switch (*storage) {
684 STORAGE_CASE(PushConstant, StoragePushConstant8, StoragePushConstant16);
685 STORAGE_CASE(StorageBuffer, StorageBuffer8BitAccess,
686 StorageBuffer16BitAccess);
687 STORAGE_CASE(Uniform, UniformAndStorageBuffer8BitAccess,
688 StorageUniform16);
689 case StorageClass::Input:
690 case StorageClass::Output: {
691 if (bitwidth == 16) {
692 pushCaps<Capability::StorageInputOutput16>();
693 return;
694 }
695 break;
696 }
697 default:
698 break;
699 }
700 }
701#undef STORAGE_CASE
702
703 // For other non-interface storage classes, require a different set of
704 // capabilities for special bitwidths.
705
706#define WIDTH_CASE(type, width) \
707 case width: \
708 pushCaps<Capability::type##width>(); \
709 break
710
711 if (auto intType = dyn_cast<IntegerType>(Val&: type)) {
712 switch (bitwidth) {
713 WIDTH_CASE(Int, 8);
714 WIDTH_CASE(Int, 16);
715 WIDTH_CASE(Int, 64);
716 case 1:
717 case 32:
718 break;
719 default:
720 llvm_unreachable("invalid bitwidth to getCapabilities");
721 }
722 } else {
723 assert(isa<FloatType>(type));
724 switch (bitwidth) {
725 case 8: {
726 if (type.isF8E4M3FN() || type.isF8E5M2())
727 pushCaps<Capability::Float8EXT>();
728 else
729 llvm_unreachable("invalid 8-bit float type to getCapabilities");
730 break;
731 }
732 case 16: {
733 if (type.isBF16())
734 pushCaps<Capability::BFloat16TypeKHR>();
735 else
736 pushCaps<Capability::Float16>();
737 break;
738 }
739 WIDTH_CASE(Float, 64);
740 case 32:
741 break;
742 default:
743 llvm_unreachable("invalid bitwidth to getCapabilities");
744 }
745 }
746
747#undef WIDTH_CASE
748}
749
750//===----------------------------------------------------------------------===//
751// SPIRVType
752//===----------------------------------------------------------------------===//
753
754bool SPIRVType::classof(Type type) {
755 // Allow SPIR-V dialect types
756 if (isa<SPIRVDialect>(Val: type.getDialect()))
757 return true;
758 if (isa<ScalarType>(Val: type))
759 return true;
760 if (auto vectorType = dyn_cast<VectorType>(Val&: type))
761 return CompositeType::isValid(type: vectorType);
762 if (auto tensorArmType = dyn_cast<TensorArmType>(Val&: type))
763 return isa<ScalarType>(Val: tensorArmType.getElementType());
764 return false;
765}
766
767bool SPIRVType::isScalarOrVector() {
768 return isIntOrFloat() || isa<VectorType>(Val: *this);
769}
770
771void SPIRVType::getExtensions(SPIRVType::ExtensionArrayRefVector &extensions,
772 std::optional<StorageClass> storage) {
773 TypeExtensionVisitor{extensions, storage}.add(type: *this);
774}
775
776void SPIRVType::getCapabilities(
777 SPIRVType::CapabilityArrayRefVector &capabilities,
778 std::optional<StorageClass> storage) {
779 TypeCapabilityVisitor{capabilities, storage}.add(type: *this);
780}
781
782std::optional<int64_t> SPIRVType::getSizeInBytes() {
783 return TypeSwitch<SPIRVType, std::optional<int64_t>>(*this)
784 .Case(caseFn: [](ScalarType type) -> std::optional<int64_t> {
785 // According to the SPIR-V spec:
786 // "There is no physical size or bit pattern defined for values with
787 // boolean type. If they are stored (in conjunction with OpVariable),
788 // they can only be used with logical addressing operations, not
789 // physical, and only with non-externally visible shader Storage
790 // Classes: Workgroup, CrossWorkgroup, Private, Function, Input, and
791 // Output."
792 int64_t bitWidth = type.getIntOrFloatBitWidth();
793 if (bitWidth == 1)
794 return std::nullopt;
795 return bitWidth / 8;
796 })
797 .Case(caseFn: [](ArrayType type) -> std::optional<int64_t> {
798 // The stride, if set, is the per-element byte distance and already
799 // includes the element size; otherwise the array is tightly packed.
800 auto elementType = cast<SPIRVType>(Val: type.getElementType());
801 if (unsigned stride = type.getArrayStride())
802 return stride * type.getNumElements();
803 if (std::optional<int64_t> size = elementType.getSizeInBytes())
804 return *size * type.getNumElements();
805 return std::nullopt;
806 })
807 .Case<VectorType, TensorArmType>(caseFn: [](auto type) -> std::optional<int64_t> {
808 if (std::optional<int64_t> elementSize =
809 cast<ScalarType>(type.getElementType()).getSizeInBytes())
810 return *elementSize * type.getNumElements();
811 return std::nullopt;
812 })
813 .Default(std::nullopt);
814}
815
816//===----------------------------------------------------------------------===//
817// SampledImageType
818//===----------------------------------------------------------------------===//
819struct spirv::detail::SampledImageTypeStorage : public TypeStorage {
820 using KeyTy = Type;
821
822 SampledImageTypeStorage(const KeyTy &key) : imageType{key} {}
823
824 bool operator==(const KeyTy &key) const { return key == KeyTy(imageType); }
825
826 static SampledImageTypeStorage *construct(TypeStorageAllocator &allocator,
827 const KeyTy &key) {
828 return new (allocator.allocate<SampledImageTypeStorage>())
829 SampledImageTypeStorage(key);
830 }
831
832 Type imageType;
833};
834
835SampledImageType SampledImageType::get(Type imageType) {
836 return Base::get(ctx: imageType.getContext(), args&: imageType);
837}
838
839SampledImageType
840SampledImageType::getChecked(function_ref<InFlightDiagnostic()> emitError,
841 Type imageType) {
842 return Base::getChecked(emitErrorFn: emitError, ctx: imageType.getContext(), args: imageType);
843}
844
845Type SampledImageType::getImageType() const { return getImpl()->imageType; }
846
847LogicalResult
848SampledImageType::verifyInvariants(function_ref<InFlightDiagnostic()> emitError,
849 Type imageType) {
850 auto image = dyn_cast<ImageType>(Val&: imageType);
851 if (!image)
852 return emitError() << "expected image type";
853
854 // As per SPIR-V spec: "It [ImageType] must not have a Dim of SubpassData.
855 // Additionally, starting with version 1.6, it must not have a Dim of Buffer.
856 // ("3.3.6. Type-Declaration Instructions")
857 if (llvm::is_contained(Set: {Dim::SubpassData, Dim::Buffer}, Element: image.getDim()))
858 return emitError() << "Dim must not be SubpassData or Buffer";
859
860 return success();
861}
862
863//===----------------------------------------------------------------------===//
864// SamplerType
865//===----------------------------------------------------------------------===//
866
867SamplerType SamplerType::get(MLIRContext *context) {
868 return Base::get(ctx: context);
869}
870
871//===----------------------------------------------------------------------===//
872// NamedBarrierType
873//===----------------------------------------------------------------------===//
874
875NamedBarrierType NamedBarrierType::get(MLIRContext *context) {
876 return Base::get(ctx: context);
877}
878
879//===----------------------------------------------------------------------===//
880// StructType
881//===----------------------------------------------------------------------===//
882
883/// Type storage for SPIR-V structure types:
884///
885/// Structures are uniqued using:
886/// - for identified structs:
887/// - a string identifier;
888/// - for literal structs:
889/// - a list of member types;
890/// - a list of member offset info;
891/// - a list of member decoration info;
892/// - a list of struct decoration info.
893///
894/// Identified structures only have a mutable component consisting of:
895/// - a list of member types;
896/// - a list of member offset info;
897/// - a list of member decoration info;
898/// - a list of struct decoration info.
899struct spirv::detail::StructTypeStorage : public TypeStorage {
900 /// Construct a storage object for an identified struct type. A struct type
901 /// associated with such storage must call StructType::trySetBody(...) later
902 /// in order to mutate the storage object providing the actual content.
903 StructTypeStorage(StringRef identifier)
904 : memberTypesAndIsBodySet(nullptr, false), offsetInfo(nullptr),
905 numMembers(0), numMemberDecorations(0), memberDecorationsInfo(nullptr),
906 numStructDecorations(0), structDecorationsInfo(nullptr),
907 identifier(identifier) {}
908
909 /// Construct a storage object for a literal struct type. A struct type
910 /// associated with such storage is immutable.
911 StructTypeStorage(
912 unsigned numMembers, Type const *memberTypes,
913 StructType::OffsetInfo const *layoutInfo, unsigned numMemberDecorations,
914 StructType::MemberDecorationInfo const *memberDecorationsInfo,
915 unsigned numStructDecorations,
916 StructType::StructDecorationInfo const *structDecorationsInfo)
917 : memberTypesAndIsBodySet(memberTypes, false), offsetInfo(layoutInfo),
918 numMembers(numMembers), numMemberDecorations(numMemberDecorations),
919 memberDecorationsInfo(memberDecorationsInfo),
920 numStructDecorations(numStructDecorations),
921 structDecorationsInfo(structDecorationsInfo) {}
922
923 /// A storage key is divided into 2 parts:
924 /// - for identified structs:
925 /// - a StringRef representing the struct identifier;
926 /// - for literal structs:
927 /// - an ArrayRef<Type> for member types;
928 /// - an ArrayRef<StructType::OffsetInfo> for member offset info;
929 /// - an ArrayRef<StructType::MemberDecorationInfo> for member decoration
930 /// info;
931 /// - an ArrayRef<StructType::StructDecorationInfo> for struct decoration
932 /// info.
933 ///
934 /// An identified struct type is uniqued only by the first part (field 0)
935 /// of the key.
936 ///
937 /// A literal struct type is uniqued only by the second part (fields 1, 2, 3
938 /// and 4) of the key. The identifier field (field 0) must be empty.
939 using KeyTy =
940 std::tuple<StringRef, ArrayRef<Type>, ArrayRef<StructType::OffsetInfo>,
941 ArrayRef<StructType::MemberDecorationInfo>,
942 ArrayRef<StructType::StructDecorationInfo>>;
943
944 /// For identified structs, return true if the given key contains the same
945 /// identifier.
946 ///
947 /// For literal structs, return true if the given key contains a matching list
948 /// of member types + offset info + decoration info.
949 bool operator==(const KeyTy &key) const {
950 if (isIdentified()) {
951 // Identified types are uniqued by their identifier.
952 return getIdentifier() == std::get<0>(t: key);
953 }
954
955 return key == KeyTy(StringRef(), getMemberTypes(), getOffsetInfo(),
956 getMemberDecorationsInfo(), getStructDecorationsInfo());
957 }
958
959 /// If the given key contains a non-empty identifier, this method constructs
960 /// an identified struct and leaves the rest of the struct type data to be set
961 /// through a later call to StructType::trySetBody(...).
962 ///
963 /// If, on the other hand, the key contains an empty identifier, a literal
964 /// struct is constructed using the other fields of the key.
965 static StructTypeStorage *construct(TypeStorageAllocator &allocator,
966 const KeyTy &key) {
967 StringRef keyIdentifier = std::get<0>(t: key);
968
969 if (!keyIdentifier.empty()) {
970 StringRef identifier = allocator.copyInto(str: keyIdentifier);
971
972 // Identified StructType body/members will be set through trySetBody(...)
973 // later.
974 return new (allocator.allocate<StructTypeStorage>())
975 StructTypeStorage(identifier);
976 }
977
978 ArrayRef<Type> keyTypes = std::get<1>(t: key);
979
980 // Copy the member type and layout information into the bump pointer
981 const Type *typesList = nullptr;
982 if (!keyTypes.empty()) {
983 typesList = allocator.copyInto(elements: keyTypes).data();
984 }
985
986 const StructType::OffsetInfo *offsetInfoList = nullptr;
987 if (!std::get<2>(t: key).empty()) {
988 ArrayRef<StructType::OffsetInfo> keyOffsetInfo = std::get<2>(t: key);
989 assert(keyOffsetInfo.size() == keyTypes.size() &&
990 "size of offset information must be same as the size of number of "
991 "elements");
992 offsetInfoList = allocator.copyInto(elements: keyOffsetInfo).data();
993 }
994
995 const StructType::MemberDecorationInfo *memberDecorationList = nullptr;
996 unsigned numMemberDecorations = 0;
997 if (!std::get<3>(t: key).empty()) {
998 auto keyMemberDecorations = std::get<3>(t: key);
999 numMemberDecorations = keyMemberDecorations.size();
1000 memberDecorationList = allocator.copyInto(elements: keyMemberDecorations).data();
1001 }
1002
1003 const StructType::StructDecorationInfo *structDecorationList = nullptr;
1004 unsigned numStructDecorations = 0;
1005 if (!std::get<4>(t: key).empty()) {
1006 auto keyStructDecorations = std::get<4>(t: key);
1007 numStructDecorations = keyStructDecorations.size();
1008 structDecorationList = allocator.copyInto(elements: keyStructDecorations).data();
1009 }
1010
1011 return new (allocator.allocate<StructTypeStorage>()) StructTypeStorage(
1012 keyTypes.size(), typesList, offsetInfoList, numMemberDecorations,
1013 memberDecorationList, numStructDecorations, structDecorationList);
1014 }
1015
1016 ArrayRef<Type> getMemberTypes() const {
1017 return ArrayRef<Type>(memberTypesAndIsBodySet.getPointer(), numMembers);
1018 }
1019
1020 ArrayRef<StructType::OffsetInfo> getOffsetInfo() const {
1021 if (offsetInfo) {
1022 return ArrayRef<StructType::OffsetInfo>(offsetInfo, numMembers);
1023 }
1024 return {};
1025 }
1026
1027 ArrayRef<StructType::MemberDecorationInfo> getMemberDecorationsInfo() const {
1028 if (memberDecorationsInfo) {
1029 return ArrayRef<StructType::MemberDecorationInfo>(memberDecorationsInfo,
1030 numMemberDecorations);
1031 }
1032 return {};
1033 }
1034
1035 ArrayRef<StructType::StructDecorationInfo> getStructDecorationsInfo() const {
1036 if (structDecorationsInfo)
1037 return ArrayRef<StructType::StructDecorationInfo>(structDecorationsInfo,
1038 numStructDecorations);
1039 return {};
1040 }
1041
1042 StringRef getIdentifier() const { return identifier; }
1043
1044 bool isIdentified() const { return !identifier.empty(); }
1045
1046 /// Sets the struct type content for identified structs. Calling this method
1047 /// is only valid for identified structs.
1048 ///
1049 /// Fails under the following conditions:
1050 /// - If called for a literal struct;
1051 /// - If called for an identified struct whose body was set before (through a
1052 /// call to this method) but with different contents from the passed
1053 /// arguments.
1054 LogicalResult
1055 mutate(TypeStorageAllocator &allocator, ArrayRef<Type> structMemberTypes,
1056 ArrayRef<StructType::OffsetInfo> structOffsetInfo,
1057 ArrayRef<StructType::MemberDecorationInfo> structMemberDecorationInfo,
1058 ArrayRef<StructType::StructDecorationInfo> structDecorationInfo) {
1059 if (!isIdentified())
1060 return failure();
1061
1062 if (memberTypesAndIsBodySet.getInt() &&
1063 (getMemberTypes() != structMemberTypes ||
1064 getOffsetInfo() != structOffsetInfo ||
1065 getMemberDecorationsInfo() != structMemberDecorationInfo ||
1066 getStructDecorationsInfo() != structDecorationInfo))
1067 return failure();
1068
1069 memberTypesAndIsBodySet.setInt(true);
1070 numMembers = structMemberTypes.size();
1071
1072 // Copy the member type and layout information into the bump pointer.
1073 if (!structMemberTypes.empty())
1074 memberTypesAndIsBodySet.setPointer(
1075 allocator.copyInto(elements: structMemberTypes).data());
1076
1077 if (!structOffsetInfo.empty()) {
1078 assert(structOffsetInfo.size() == structMemberTypes.size() &&
1079 "size of offset information must be same as the size of number of "
1080 "elements");
1081 offsetInfo = allocator.copyInto(elements: structOffsetInfo).data();
1082 }
1083
1084 if (!structMemberDecorationInfo.empty()) {
1085 numMemberDecorations = structMemberDecorationInfo.size();
1086 memberDecorationsInfo =
1087 allocator.copyInto(elements: structMemberDecorationInfo).data();
1088 }
1089
1090 if (!structDecorationInfo.empty()) {
1091 numStructDecorations = structDecorationInfo.size();
1092 structDecorationsInfo = allocator.copyInto(elements: structDecorationInfo).data();
1093 }
1094
1095 return success();
1096 }
1097
1098 llvm::PointerIntPair<Type const *, 1, bool> memberTypesAndIsBodySet;
1099 StructType::OffsetInfo const *offsetInfo;
1100 unsigned numMembers;
1101 unsigned numMemberDecorations;
1102 StructType::MemberDecorationInfo const *memberDecorationsInfo;
1103 unsigned numStructDecorations;
1104 StructType::StructDecorationInfo const *structDecorationsInfo;
1105 StringRef identifier;
1106};
1107
1108StructType
1109StructType::get(ArrayRef<Type> memberTypes,
1110 ArrayRef<StructType::OffsetInfo> offsetInfo,
1111 ArrayRef<StructType::MemberDecorationInfo> memberDecorations,
1112 ArrayRef<StructType::StructDecorationInfo> structDecorations) {
1113 assert(!memberTypes.empty() && "Struct needs at least one member type");
1114 // Sort the decorations.
1115 SmallVector<StructType::MemberDecorationInfo, 4> sortedMemberDecorations(
1116 memberDecorations);
1117 llvm::array_pod_sort(Start: sortedMemberDecorations.begin(),
1118 End: sortedMemberDecorations.end());
1119 SmallVector<StructType::StructDecorationInfo, 1> sortedStructDecorations(
1120 structDecorations);
1121 llvm::array_pod_sort(Start: sortedStructDecorations.begin(),
1122 End: sortedStructDecorations.end());
1123
1124 return Base::get(ctx: memberTypes.vec().front().getContext(),
1125 /*identifier=*/args: StringRef(), args&: memberTypes, args&: offsetInfo,
1126 args&: sortedMemberDecorations, args&: sortedStructDecorations);
1127}
1128
1129StructType StructType::getIdentified(MLIRContext *context,
1130 StringRef identifier) {
1131 assert(!identifier.empty() &&
1132 "StructType identifier must be non-empty string");
1133
1134 return Base::get(ctx: context, args&: identifier, args: ArrayRef<Type>(),
1135 args: ArrayRef<StructType::OffsetInfo>(),
1136 args: ArrayRef<StructType::MemberDecorationInfo>(),
1137 args: ArrayRef<StructType::StructDecorationInfo>());
1138}
1139
1140StructType StructType::getEmpty(MLIRContext *context, StringRef identifier) {
1141 StructType newStructType = Base::get(
1142 ctx: context, args&: identifier, args: ArrayRef<Type>(), args: ArrayRef<StructType::OffsetInfo>(),
1143 args: ArrayRef<StructType::MemberDecorationInfo>(),
1144 args: ArrayRef<StructType::StructDecorationInfo>());
1145 // Set an empty body in case this is a identified struct.
1146 if (newStructType.isIdentified() &&
1147 failed(Result: newStructType.trySetBody(
1148 memberTypes: ArrayRef<Type>(), offsetInfo: ArrayRef<StructType::OffsetInfo>(),
1149 memberDecorations: ArrayRef<StructType::MemberDecorationInfo>(),
1150 structDecorations: ArrayRef<StructType::StructDecorationInfo>())))
1151 return StructType();
1152
1153 return newStructType;
1154}
1155
1156StringRef StructType::getIdentifier() const { return getImpl()->identifier; }
1157
1158bool StructType::isIdentified() const { return getImpl()->isIdentified(); }
1159
1160unsigned StructType::getNumElements() const { return getImpl()->numMembers; }
1161
1162Type StructType::getElementType(unsigned index) const {
1163 assert(getNumElements() > index && "member index out of range");
1164 return getImpl()->memberTypesAndIsBodySet.getPointer()[index];
1165}
1166
1167TypeRange StructType::getElementTypes() const {
1168 return TypeRange(getImpl()->memberTypesAndIsBodySet.getPointer(),
1169 getNumElements());
1170}
1171
1172bool StructType::hasOffset() const { return getImpl()->offsetInfo; }
1173
1174bool StructType::hasDecoration(spirv::Decoration decoration) const {
1175 for (StructType::StructDecorationInfo info :
1176 getImpl()->getStructDecorationsInfo())
1177 if (info.decoration == decoration)
1178 return true;
1179
1180 return false;
1181}
1182
1183uint64_t StructType::getMemberOffset(unsigned index) const {
1184 assert(getNumElements() > index && "member index out of range");
1185 return getImpl()->offsetInfo[index];
1186}
1187
1188void StructType::getMemberDecorations(
1189 SmallVectorImpl<StructType::MemberDecorationInfo> &memberDecorations)
1190 const {
1191 memberDecorations.clear();
1192 auto implMemberDecorations = getImpl()->getMemberDecorationsInfo();
1193 memberDecorations.append(in_start: implMemberDecorations.begin(),
1194 in_end: implMemberDecorations.end());
1195}
1196
1197void StructType::getMemberDecorations(
1198 unsigned index,
1199 SmallVectorImpl<StructType::MemberDecorationInfo> &decorationsInfo) const {
1200 assert(getNumElements() > index && "member index out of range");
1201 auto memberDecorations = getImpl()->getMemberDecorationsInfo();
1202 decorationsInfo.clear();
1203 for (const auto &memberDecoration : memberDecorations) {
1204 if (memberDecoration.memberIndex == index) {
1205 decorationsInfo.push_back(Elt: memberDecoration);
1206 }
1207 if (memberDecoration.memberIndex > index) {
1208 // Early exit since the decorations are stored sorted.
1209 return;
1210 }
1211 }
1212}
1213
1214void StructType::getStructDecorations(
1215 SmallVectorImpl<StructType::StructDecorationInfo> &structDecorations)
1216 const {
1217 structDecorations.clear();
1218 auto implDecorations = getImpl()->getStructDecorationsInfo();
1219 structDecorations.append(in_start: implDecorations.begin(), in_end: implDecorations.end());
1220}
1221
1222LogicalResult
1223StructType::trySetBody(ArrayRef<Type> memberTypes,
1224 ArrayRef<OffsetInfo> offsetInfo,
1225 ArrayRef<MemberDecorationInfo> memberDecorations,
1226 ArrayRef<StructDecorationInfo> structDecorations) {
1227 return Base::mutate(args&: memberTypes, args&: offsetInfo, args&: memberDecorations,
1228 args&: structDecorations);
1229}
1230
1231llvm::hash_code spirv::hash_value(
1232 const StructType::MemberDecorationInfo &memberDecorationInfo) {
1233 return llvm::hash_combine(args: memberDecorationInfo.memberIndex,
1234 args: memberDecorationInfo.decoration);
1235}
1236
1237llvm::hash_code spirv::hash_value(
1238 const StructType::StructDecorationInfo &structDecorationInfo) {
1239 return llvm::hash_value(value: structDecorationInfo.decoration);
1240}
1241
1242//===----------------------------------------------------------------------===//
1243// MatrixType
1244//===----------------------------------------------------------------------===//
1245
1246struct spirv::detail::MatrixTypeStorage : public TypeStorage {
1247 // Use a 64-bit integer as a column count internally to better support a
1248 // `ShapedType` interface. See comment in `CooperativeMatrixType` for more
1249 // context.
1250 using KeyTy = std::tuple<Type, int64_t>;
1251
1252 MatrixTypeStorage(const KeyTy &key)
1253 : columnType(std::get<0>(t: key)),
1254 shape({cast<VectorType>(Val: std::get<0>(t: key)).getShape()[0],
1255 std::get<1>(t: key)}) {}
1256
1257 static MatrixTypeStorage *construct(TypeStorageAllocator &allocator,
1258 const KeyTy &key) {
1259
1260 // Initialize the memory using placement new.
1261 return new (allocator.allocate<MatrixTypeStorage>()) MatrixTypeStorage(key);
1262 }
1263
1264 bool operator==(const KeyTy &key) const {
1265 return key == KeyTy(columnType, shape[1]);
1266 }
1267
1268 Type columnType;
1269 // [#rows, #columns]
1270 std::array<int64_t, 2> shape;
1271};
1272
1273MatrixType MatrixType::get(Type columnType, uint32_t columnCount) {
1274 return Base::get(ctx: columnType.getContext(), args&: columnType, args&: columnCount);
1275}
1276
1277MatrixType MatrixType::getChecked(function_ref<InFlightDiagnostic()> emitError,
1278 Type columnType, uint32_t columnCount) {
1279 return Base::getChecked(emitErrorFn: emitError, ctx: columnType.getContext(), args: columnType,
1280 args: columnCount);
1281}
1282
1283LogicalResult
1284MatrixType::verifyInvariants(function_ref<InFlightDiagnostic()> emitError,
1285 Type columnType, uint32_t columnCount) {
1286 if (columnCount < 2 || columnCount > 4)
1287 return emitError() << "matrix can have 2, 3, or 4 columns only";
1288
1289 if (!isValidColumnType(columnType))
1290 return emitError() << "matrix columns must be vectors of floats";
1291
1292 /// The underlying vectors (columns) must be of size 2, 3, or 4
1293 ArrayRef<int64_t> columnShape = cast<VectorType>(Val&: columnType).getShape();
1294 if (columnShape.size() != 1)
1295 return emitError() << "matrix columns must be 1D vectors";
1296
1297 if (columnShape[0] < 2 || columnShape[0] > 4)
1298 return emitError() << "matrix columns must be of size 2, 3, or 4";
1299
1300 return success();
1301}
1302
1303/// Returns true if the matrix elements are vectors of float elements
1304bool MatrixType::isValidColumnType(Type columnType) {
1305 if (auto vectorType = dyn_cast<VectorType>(Val&: columnType)) {
1306 if (isa<FloatType>(Val: vectorType.getElementType()))
1307 return true;
1308 }
1309 return false;
1310}
1311
1312Type MatrixType::getColumnType() const { return getImpl()->columnType; }
1313
1314Type MatrixType::getElementType() const {
1315 return cast<VectorType>(Val&: getImpl()->columnType).getElementType();
1316}
1317
1318unsigned MatrixType::getNumColumns() const {
1319 assert(getImpl()->shape[1] >= 0); // Also includes ShapedType::kDynamic.
1320 assert(getImpl()->shape[1] <= std::numeric_limits<unsigned>::max());
1321 return static_cast<uint32_t>(getImpl()->shape[1]);
1322}
1323
1324unsigned MatrixType::getNumRows() const {
1325 assert(getImpl()->shape[0] >= 0); // Also includes ShapedType::kDynamic.
1326 assert(getImpl()->shape[0] <= std::numeric_limits<unsigned>::max());
1327 return static_cast<uint32_t>(getImpl()->shape[0]);
1328}
1329
1330unsigned MatrixType::getNumElements() const {
1331 return getNumColumns() * getNumRows();
1332}
1333
1334ArrayRef<int64_t> MatrixType::getShape() const { return getImpl()->shape; }
1335
1336void TypeCapabilityVisitor::addConcrete(MatrixType type) {
1337 add(type: type.getColumnType());
1338 pushCaps<Capability::Matrix>();
1339}
1340
1341//===----------------------------------------------------------------------===//
1342// TensorArmType
1343//===----------------------------------------------------------------------===//
1344
1345struct spirv::detail::TensorArmTypeStorage final : TypeStorage {
1346 using KeyTy = std::tuple<ArrayRef<int64_t>, Type>;
1347
1348 static TensorArmTypeStorage *construct(TypeStorageAllocator &allocator,
1349 const KeyTy &key) {
1350 auto [shape, elementType] = key;
1351 shape = allocator.copyInto(elements: shape);
1352 return new (allocator.allocate<TensorArmTypeStorage>())
1353 TensorArmTypeStorage(shape, elementType);
1354 }
1355
1356 static llvm::hash_code hashKey(const KeyTy &key) {
1357 auto [shape, elementType] = key;
1358 return llvm::hash_combine(args: shape, args: elementType);
1359 }
1360
1361 bool operator==(const KeyTy &key) const {
1362 return key == KeyTy(shape, elementType);
1363 }
1364
1365 TensorArmTypeStorage(ArrayRef<int64_t> shape, Type elementType)
1366 : shape(shape), elementType(elementType) {}
1367
1368 ArrayRef<int64_t> shape;
1369 Type elementType;
1370};
1371
1372TensorArmType TensorArmType::get(ArrayRef<int64_t> shape, Type elementType) {
1373 return Base::get(ctx: elementType.getContext(), args&: shape, args&: elementType);
1374}
1375
1376TensorArmType TensorArmType::cloneWith(std::optional<ArrayRef<int64_t>> shape,
1377 Type elementType) const {
1378 return TensorArmType::get(shape: shape.value_or(u: getShape()), elementType);
1379}
1380
1381Type TensorArmType::getElementType() const { return getImpl()->elementType; }
1382ArrayRef<int64_t> TensorArmType::getShape() const { return getImpl()->shape; }
1383
1384void TypeExtensionVisitor::addConcrete(TensorArmType type) {
1385 add(type: type.getElementType());
1386 pushExts<Extension::SPV_ARM_tensors>();
1387}
1388
1389void TypeCapabilityVisitor::addConcrete(TensorArmType type) {
1390 add(type: type.getElementType());
1391 pushCaps<Capability::TensorsARM>();
1392}
1393
1394LogicalResult
1395TensorArmType::verifyInvariants(function_ref<InFlightDiagnostic()> emitError,
1396 ArrayRef<int64_t> shape, Type elementType) {
1397 if (llvm::is_contained(Range&: shape, Element: 0))
1398 return emitError() << "arm.tensor do not support dimensions = 0";
1399 if (llvm::any_of(Range&: shape, P: [](int64_t dim) { return dim < 0; }) &&
1400 llvm::any_of(Range&: shape, P: [](int64_t dim) { return dim > 0; }))
1401 return emitError()
1402 << "arm.tensor shape dimensions must be either fully dynamic or "
1403 "completed shaped";
1404 return success();
1405}
1406
1407//===----------------------------------------------------------------------===//
1408// SPIR-V Dialect
1409//===----------------------------------------------------------------------===//
1410
1411void SPIRVDialect::registerTypes() {
1412 addTypes<ArrayType, CooperativeMatrixType, ImageType, MatrixType,
1413 NamedBarrierType, PointerType, RuntimeArrayType, SampledImageType,
1414 SamplerType, StructType, TensorArmType>();
1415}
1416