1//===- SPIRVConversion.cpp - SPIR-V Conversion Utilities ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements utilities used to lower to SPIR-V dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
14#include "mlir/Dialect/Arith/IR/Arith.h"
15#include "mlir/Dialect/Func/IR/FuncOps.h"
16#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
17#include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
18#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
19#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
20#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
21#include "mlir/Dialect/Utils/IndexingUtils.h"
22#include "mlir/Dialect/Vector/IR/VectorOps.h"
23#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
24#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
25#include "mlir/IR/BuiltinTypes.h"
26#include "mlir/IR/Operation.h"
27#include "mlir/IR/PatternMatch.h"
28#include "mlir/Support/LLVM.h"
29#include "mlir/Transforms/DialectConversion.h"
30#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
31#include "llvm/ADT/APInt.h"
32#include "llvm/ADT/STLExtras.h"
33#include "llvm/ADT/SmallVector.h"
34#include "llvm/ADT/StringExtras.h"
35#include "llvm/Support/CheckedArithmetic.h"
36#include "llvm/Support/Debug.h"
37#include "llvm/Support/MathExtras.h"
38
39#include <optional>
40
41#define DEBUG_TYPE "mlir-spirv-conversion"
42
43using namespace mlir;
44
45namespace {
46
47//===----------------------------------------------------------------------===//
48// Utility functions
49//===----------------------------------------------------------------------===//
50
51static std::optional<SmallVector<int64_t>> getTargetShape(VectorType vecType) {
52 LLVM_DEBUG(llvm::dbgs() << "Get target shape\n");
53 if (vecType.isScalable()) {
54 LLVM_DEBUG(llvm::dbgs()
55 << "--scalable vectors are not supported -> BAIL\n");
56 return std::nullopt;
57 }
58 if (vecType.getRank() == 0) {
59 LLVM_DEBUG(llvm::dbgs() << "--0-D vectors are not supported -> BAIL\n");
60 return std::nullopt;
61 }
62 SmallVector<int64_t> unrollShape = llvm::to_vector<4>(Range: vecType.getShape());
63 std::optional<SmallVector<int64_t>> targetShape = SmallVector<int64_t>(
64 1, mlir::spirv::getComputeVectorSize(size: vecType.getShape().back()));
65 if (!targetShape) {
66 LLVM_DEBUG(llvm::dbgs() << "--no unrolling target shape defined\n");
67 return std::nullopt;
68 }
69 auto maybeShapeRatio = computeShapeRatio(shape: unrollShape, subShape: *targetShape);
70 if (!maybeShapeRatio) {
71 LLVM_DEBUG(llvm::dbgs()
72 << "--could not compute integral shape ratio -> BAIL\n");
73 return std::nullopt;
74 }
75 if (llvm::all_of(Range&: *maybeShapeRatio, P: [](int64_t v) { return v == 1; })) {
76 LLVM_DEBUG(llvm::dbgs() << "--no unrolling needed -> SKIP\n");
77 return std::nullopt;
78 }
79 LLVM_DEBUG(llvm::dbgs()
80 << "--found an integral shape ratio to unroll to -> SUCCESS\n");
81 return targetShape;
82}
83
84/// Checks that `candidates` extension requirements are possible to be satisfied
85/// with the given `targetEnv`.
86///
87/// `candidates` is a vector of vector for extension requirements following
88/// ((Extension::A OR Extension::B) AND (Extension::C OR Extension::D))
89/// convention.
90template <typename LabelT>
91static LogicalResult checkExtensionRequirements(
92 LabelT label, const spirv::TargetEnv &targetEnv,
93 const spirv::SPIRVType::ExtensionArrayRefVector &candidates) {
94 for (const auto &ors : candidates) {
95 if (targetEnv.allows(ors))
96 continue;
97
98 LLVM_DEBUG({
99 SmallVector<StringRef> extStrings;
100 for (spirv::Extension ext : ors)
101 extStrings.push_back(spirv::stringifyExtension(ext));
102
103 llvm::dbgs() << label << " illegal: requires at least one extension in ["
104 << llvm::join(extStrings, ", ")
105 << "] but none allowed in target environment\n";
106 });
107 return failure();
108 }
109 return success();
110}
111
112/// Checks that `candidates`capability requirements are possible to be satisfied
113/// with the given `isAllowedFn`.
114///
115/// `candidates` is a vector of vector for capability requirements following
116/// ((Capability::A OR Capability::B) AND (Capability::C OR Capability::D))
117/// convention.
118template <typename LabelT>
119static LogicalResult checkCapabilityRequirements(
120 LabelT label, const spirv::TargetEnv &targetEnv,
121 const spirv::SPIRVType::CapabilityArrayRefVector &candidates) {
122 for (const auto &ors : candidates) {
123 if (targetEnv.allows(ors))
124 continue;
125
126 LLVM_DEBUG({
127 SmallVector<StringRef> capStrings;
128 for (spirv::Capability cap : ors)
129 capStrings.push_back(spirv::stringifyCapability(cap));
130
131 llvm::dbgs() << label << " illegal: requires at least one capability in ["
132 << llvm::join(capStrings, ", ")
133 << "] but none allowed in target environment\n";
134 });
135 return failure();
136 }
137 return success();
138}
139
140/// Returns true if the given `storageClass` needs explicit layout when used in
141/// Shader environments.
142static bool needsExplicitLayout(spirv::StorageClass storageClass) {
143 switch (storageClass) {
144 case spirv::StorageClass::PhysicalStorageBuffer:
145 case spirv::StorageClass::PushConstant:
146 case spirv::StorageClass::StorageBuffer:
147 case spirv::StorageClass::Uniform:
148 return true;
149 default:
150 return false;
151 }
152}
153
154/// Wraps the given `elementType` in a struct and gets the pointer to the
155/// struct. This is used to satisfy Vulkan interface requirements.
156static spirv::PointerType
157wrapInStructAndGetPointer(Type elementType, spirv::StorageClass storageClass) {
158 auto structType = needsExplicitLayout(storageClass)
159 ? spirv::StructType::get(memberTypes: elementType, /*offsetInfo=*/0)
160 : spirv::StructType::get(memberTypes: elementType);
161 return spirv::PointerType::get(pointeeType: structType, storageClass);
162}
163
164//===----------------------------------------------------------------------===//
165// Type Conversion
166//===----------------------------------------------------------------------===//
167
168static spirv::ScalarType getIndexType(MLIRContext *ctx,
169 const SPIRVConversionOptions &options) {
170 return cast<spirv::ScalarType>(
171 Val: IntegerType::get(context: ctx, width: options.use64bitIndex ? 64 : 32));
172}
173
174// TODO: This is a utility function that should probably be exposed by the
175// SPIR-V dialect. Keeping it local till the use case arises.
176static std::optional<int64_t>
177getTypeNumBytes(const SPIRVConversionOptions &options, Type type) {
178 if (isa<spirv::ScalarType>(Val: type)) {
179 auto bitWidth = type.getIntOrFloatBitWidth();
180 // According to the SPIR-V spec:
181 // "There is no physical size or bit pattern defined for values with boolean
182 // type. If they are stored (in conjunction with OpVariable), they can only
183 // be used with logical addressing operations, not physical, and only with
184 // non-externally visible shader Storage Classes: Workgroup, CrossWorkgroup,
185 // Private, Function, Input, and Output."
186 if (bitWidth == 1)
187 return std::nullopt;
188 return bitWidth / 8;
189 }
190
191 // Handle 8-bit floats.
192 if (options.emulateUnsupportedFloatTypes && isa<FloatType>(Val: type)) {
193 auto bitWidth = type.getIntOrFloatBitWidth();
194 if (bitWidth == 8)
195 return bitWidth / 8;
196 return std::nullopt;
197 }
198
199 if (auto complexType = dyn_cast<ComplexType>(Val&: type)) {
200 auto elementSize = getTypeNumBytes(options, type: complexType.getElementType());
201 if (!elementSize)
202 return std::nullopt;
203 return 2 * *elementSize;
204 }
205
206 if (auto vecType = dyn_cast<VectorType>(Val&: type)) {
207 auto elementSize = getTypeNumBytes(options, type: vecType.getElementType());
208 if (!elementSize)
209 return std::nullopt;
210 return vecType.getNumElements() * *elementSize;
211 }
212
213 if (auto memRefType = dyn_cast<MemRefType>(Val&: type)) {
214 // TODO: Layout should also be controlled by the ABI attributes. For now
215 // using the layout from MemRef.
216 int64_t offset;
217 SmallVector<int64_t, 4> strides;
218 if (!memRefType.hasStaticShape() ||
219 failed(Result: memRefType.getStridesAndOffset(strides, offset)))
220 return std::nullopt;
221
222 // To get the size of the memref object in memory, the total size is the
223 // max(stride * dimension-size) computed for all dimensions times the size
224 // of the element.
225 auto elementSize = getTypeNumBytes(options, type: memRefType.getElementType());
226 if (!elementSize)
227 return std::nullopt;
228
229 if (memRefType.getRank() == 0)
230 return elementSize;
231
232 auto dims = memRefType.getShape();
233 if (llvm::is_contained(Range&: dims, Element: ShapedType::kDynamic) ||
234 ShapedType::isDynamic(dValue: offset) ||
235 llvm::is_contained(Range&: strides, Element: ShapedType::kDynamic))
236 return std::nullopt;
237
238 int64_t memrefSize = -1;
239 for (const auto &shape : enumerate(First&: dims))
240 memrefSize = std::max(a: memrefSize, b: shape.value() * strides[shape.index()]);
241
242 return (offset + memrefSize) * *elementSize;
243 }
244
245 if (auto tensorType = dyn_cast<TensorType>(Val&: type)) {
246 if (!tensorType.hasStaticShape())
247 return std::nullopt;
248
249 auto elementSize = getTypeNumBytes(options, type: tensorType.getElementType());
250 if (!elementSize)
251 return std::nullopt;
252
253 int64_t size = *elementSize;
254 for (auto shape : tensorType.getShape())
255 size *= shape;
256
257 return size;
258 }
259
260 // TODO: Add size computation for other types.
261 return std::nullopt;
262}
263
264/// Converts a scalar `type` to a suitable type under the given `targetEnv`.
265static Type
266convertScalarType(const spirv::TargetEnv &targetEnv,
267 const SPIRVConversionOptions &options, spirv::ScalarType type,
268 std::optional<spirv::StorageClass> storageClass = {}) {
269 // Get extension and capability requirements for the given type.
270 SmallVector<ArrayRef<spirv::Extension>, 1> extensions;
271 SmallVector<ArrayRef<spirv::Capability>, 2> capabilities;
272 type.getExtensions(extensions, storage: storageClass);
273 type.getCapabilities(capabilities, storage: storageClass);
274
275 // If all requirements are met, then we can accept this type as-is.
276 if (succeeded(Result: checkCapabilityRequirements(label: type, targetEnv, candidates: capabilities)) &&
277 succeeded(Result: checkExtensionRequirements(label: type, targetEnv, candidates: extensions)))
278 return type;
279
280 // Otherwise we need to adjust the type, which really means adjusting the
281 // bitwidth given this is a scalar type.
282 if (!options.emulateLT32BitScalarTypes)
283 return nullptr;
284
285 // We only emulate narrower scalar types here and do not truncate results.
286 if (type.getIntOrFloatBitWidth() > 32) {
287 LLVM_DEBUG(llvm::dbgs()
288 << type
289 << " not converted to 32-bit for SPIR-V to avoid truncation\n");
290 return nullptr;
291 }
292
293 if (auto floatType = dyn_cast<FloatType>(Val&: type)) {
294 LLVM_DEBUG(llvm::dbgs() << type << " converted to 32-bit for SPIR-V\n");
295 return Builder(targetEnv.getContext()).getF32Type();
296 }
297
298 auto intType = cast<IntegerType>(Val&: type);
299 LLVM_DEBUG(llvm::dbgs() << type << " converted to 32-bit for SPIR-V\n");
300 return IntegerType::get(context: targetEnv.getContext(), /*width=*/32,
301 signedness: intType.getSignedness());
302}
303
304/// Converts a sub-byte integer `type` to i32 regardless of target environment.
305/// Returns a nullptr for unsupported integer types, including non sub-byte
306/// types.
307///
308/// Note that we don't recognize sub-byte types in `spirv::ScalarType` and use
309/// the above given that these sub-byte types are not supported at all in
310/// SPIR-V; there are no compute/storage capability for them like other
311/// supported integer types.
312static Type convertSubByteIntegerType(const SPIRVConversionOptions &options,
313 IntegerType type) {
314 if (type.getWidth() > 8) {
315 LLVM_DEBUG(llvm::dbgs() << "not a subbyte type\n");
316 return nullptr;
317 }
318 if (options.subByteTypeStorage != SPIRVSubByteTypeStorage::Packed) {
319 LLVM_DEBUG(llvm::dbgs() << "unsupported sub-byte storage kind\n");
320 return nullptr;
321 }
322
323 if (!llvm::isPowerOf2_32(Value: type.getWidth())) {
324 LLVM_DEBUG(llvm::dbgs()
325 << "unsupported non-power-of-two bitwidth in sub-byte" << type
326 << "\n");
327 return nullptr;
328 }
329
330 LLVM_DEBUG(llvm::dbgs() << type << " converted to 32-bit for SPIR-V\n");
331 return IntegerType::get(context: type.getContext(), /*width=*/32,
332 signedness: type.getSignedness());
333}
334
335/// Converts 8-bit float types to integer types with the same bit width.
336/// Returns a nullptr for unsupported 8-bit float types.
337static Type convert8BitFloatType(const SPIRVConversionOptions &options,
338 FloatType type) {
339 if (!options.emulateUnsupportedFloatTypes)
340 return nullptr;
341 // F8 types are converted to integer types with the same bit width.
342 if (isa<Float8E5M2Type, Float8E4M3Type, Float8E4M3FNType, Float8E5M2FNUZType,
343 Float8E4M3FNUZType, Float8E4M3B11FNUZType, Float8E3M4Type,
344 Float8E8M0FNUType>(Val: type))
345 return IntegerType::get(context: type.getContext(), width: type.getWidth());
346 LLVM_DEBUG(llvm::dbgs() << "unsupported 8-bit float type: " << type << "\n");
347 return nullptr;
348}
349
350/// Returns a type with the same shape but with any 8-bit float element type
351/// converted to the same bit width integer type. This is a noop when the
352/// element type is not the 8-bit float type or emulation flag is set to false.
353static ShapedType
354convertShaped8BitFloatType(ShapedType type,
355 const SPIRVConversionOptions &options) {
356 if (!options.emulateUnsupportedFloatTypes)
357 return type;
358 Type srcElementType = type.getElementType();
359 Type convertedElementType = nullptr;
360 // F8 types are converted to integer types with the same bit width.
361 if (isa<Float8E5M2Type, Float8E4M3Type, Float8E4M3FNType, Float8E5M2FNUZType,
362 Float8E4M3FNUZType, Float8E4M3B11FNUZType, Float8E3M4Type,
363 Float8E8M0FNUType>(Val: srcElementType))
364 convertedElementType = IntegerType::get(
365 context: type.getContext(), width: srcElementType.getIntOrFloatBitWidth());
366
367 if (!convertedElementType)
368 return type;
369
370 return type.clone(elementType: convertedElementType);
371}
372
373/// Returns a type with the same shape but with any index element type converted
374/// to the matching integer type. This is a noop when the element type is not
375/// the index type.
376static ShapedType
377convertIndexElementType(ShapedType type,
378 const SPIRVConversionOptions &options) {
379 Type indexType = dyn_cast<IndexType>(Val: type.getElementType());
380 if (!indexType)
381 return type;
382
383 return type.clone(elementType: getIndexType(ctx: type.getContext(), options));
384}
385
386/// Converts a vector `type` to a suitable type under the given `targetEnv`.
387static Type
388convertVectorType(const spirv::TargetEnv &targetEnv,
389 const SPIRVConversionOptions &options, VectorType type,
390 std::optional<spirv::StorageClass> storageClass = {}) {
391 type = cast<VectorType>(Val: convertIndexElementType(type, options));
392 type = cast<VectorType>(Val: convertShaped8BitFloatType(type, options));
393 auto scalarType = dyn_cast_or_null<spirv::ScalarType>(Val: type.getElementType());
394 if (!scalarType) {
395 // If this is not a spec allowed scalar type, try to handle sub-byte integer
396 // types.
397 auto intType = dyn_cast<IntegerType>(Val: type.getElementType());
398 if (!intType) {
399 LLVM_DEBUG(llvm::dbgs()
400 << type
401 << " illegal: cannot convert non-scalar element type\n");
402 return nullptr;
403 }
404
405 Type elementType = convertSubByteIntegerType(options, type: intType);
406 if (!elementType)
407 return nullptr;
408
409 if (type.getRank() <= 1 && type.getNumElements() == 1)
410 return elementType;
411
412 if (type.getNumElements() > 4) {
413 LLVM_DEBUG(llvm::dbgs()
414 << type << " illegal: > 4-element unimplemented\n");
415 return nullptr;
416 }
417
418 return VectorType::get(shape: type.getShape(), elementType);
419 }
420
421 if (type.getRank() <= 1 && type.getNumElements() == 1)
422 return convertScalarType(targetEnv, options, type: scalarType, storageClass);
423
424 if (!spirv::CompositeType::isValid(type)) {
425 LLVM_DEBUG(llvm::dbgs()
426 << type << " illegal: not a valid composite type\n");
427 return nullptr;
428 }
429
430 // Get extension and capability requirements for the given type.
431 SmallVector<ArrayRef<spirv::Extension>, 1> extensions;
432 SmallVector<ArrayRef<spirv::Capability>, 2> capabilities;
433 cast<spirv::CompositeType>(Val&: type).getExtensions(extensions, storage: storageClass);
434 cast<spirv::CompositeType>(Val&: type).getCapabilities(capabilities, storage: storageClass);
435
436 // If all requirements are met, then we can accept this type as-is.
437 if (succeeded(Result: checkCapabilityRequirements(label: type, targetEnv, candidates: capabilities)) &&
438 succeeded(Result: checkExtensionRequirements(label: type, targetEnv, candidates: extensions)))
439 return type;
440
441 auto elementType =
442 convertScalarType(targetEnv, options, type: scalarType, storageClass);
443 if (elementType)
444 return VectorType::get(shape: type.getShape(), elementType);
445 return nullptr;
446}
447
448static Type
449convertComplexType(const spirv::TargetEnv &targetEnv,
450 const SPIRVConversionOptions &options, ComplexType type,
451 std::optional<spirv::StorageClass> storageClass = {}) {
452 auto scalarType = dyn_cast_or_null<spirv::ScalarType>(Val: type.getElementType());
453 if (!scalarType) {
454 LLVM_DEBUG(llvm::dbgs()
455 << type << " illegal: cannot convert non-scalar element type\n");
456 return nullptr;
457 }
458
459 auto elementType =
460 convertScalarType(targetEnv, options, type: scalarType, storageClass);
461 if (!elementType)
462 return nullptr;
463 if (elementType != type.getElementType()) {
464 LLVM_DEBUG(llvm::dbgs()
465 << type << " illegal: complex type emulation unsupported\n");
466 return nullptr;
467 }
468
469 return VectorType::get(shape: 2, elementType);
470}
471
472/// Converts a tensor `type` to a suitable type under the given `targetEnv`.
473///
474/// Note that this is mainly for lowering constant tensors. In SPIR-V one can
475/// create composite constants with OpConstantComposite to embed relative large
476/// constant values and use OpCompositeExtract and OpCompositeInsert to
477/// manipulate, like what we do for vectors.
478static Type convertTensorType(const spirv::TargetEnv &targetEnv,
479 const SPIRVConversionOptions &options,
480 TensorType type) {
481 // TODO: Handle dynamic shapes.
482 if (!type.hasStaticShape()) {
483 LLVM_DEBUG(llvm::dbgs()
484 << type << " illegal: dynamic shape unimplemented\n");
485 return nullptr;
486 }
487
488 type = cast<TensorType>(Val: convertIndexElementType(type, options));
489 type = cast<TensorType>(Val: convertShaped8BitFloatType(type, options));
490 auto scalarType = dyn_cast_or_null<spirv::ScalarType>(Val: type.getElementType());
491 if (!scalarType) {
492 LLVM_DEBUG(llvm::dbgs()
493 << type << " illegal: cannot convert non-scalar element type\n");
494 return nullptr;
495 }
496
497 std::optional<int64_t> scalarSize = getTypeNumBytes(options, type: scalarType);
498 std::optional<int64_t> tensorSize = getTypeNumBytes(options, type);
499 if (!scalarSize || !tensorSize) {
500 LLVM_DEBUG(llvm::dbgs()
501 << type << " illegal: cannot deduce element count\n");
502 return nullptr;
503 }
504
505 int64_t arrayElemCount = *tensorSize / *scalarSize;
506 if (arrayElemCount == 0) {
507 LLVM_DEBUG(llvm::dbgs()
508 << type << " illegal: cannot handle zero-element tensors\n");
509 return nullptr;
510 }
511 if (arrayElemCount > std::numeric_limits<unsigned>::max()) {
512 LLVM_DEBUG(llvm::dbgs()
513 << type << " illegal: cannot fit tensor into target type\n");
514 return nullptr;
515 }
516
517 Type arrayElemType = convertScalarType(targetEnv, options, type: scalarType);
518 if (!arrayElemType)
519 return nullptr;
520 std::optional<int64_t> arrayElemSize =
521 getTypeNumBytes(options, type: arrayElemType);
522 if (!arrayElemSize) {
523 LLVM_DEBUG(llvm::dbgs()
524 << type << " illegal: cannot deduce converted element size\n");
525 return nullptr;
526 }
527
528 return spirv::ArrayType::get(elementType: arrayElemType, elementCount: arrayElemCount);
529}
530
531static Type convertBoolMemrefType(const spirv::TargetEnv &targetEnv,
532 const SPIRVConversionOptions &options,
533 MemRefType type,
534 spirv::StorageClass storageClass) {
535 unsigned numBoolBits = options.boolNumBits;
536 if (numBoolBits != 8) {
537 LLVM_DEBUG(llvm::dbgs()
538 << "using non-8-bit storage for bool types unimplemented");
539 return nullptr;
540 }
541 auto elementType = dyn_cast<spirv::ScalarType>(
542 Val: IntegerType::get(context: type.getContext(), width: numBoolBits));
543 if (!elementType)
544 return nullptr;
545 Type arrayElemType =
546 convertScalarType(targetEnv, options, type: elementType, storageClass);
547 if (!arrayElemType)
548 return nullptr;
549 std::optional<int64_t> arrayElemSize =
550 getTypeNumBytes(options, type: arrayElemType);
551 if (!arrayElemSize) {
552 LLVM_DEBUG(llvm::dbgs()
553 << type << " illegal: cannot deduce converted element size\n");
554 return nullptr;
555 }
556
557 if (!type.hasStaticShape()) {
558 // For OpenCL Kernel, dynamic shaped memrefs convert into a pointer pointing
559 // to the element.
560 if (targetEnv.allows(spirv::Capability::Kernel))
561 return spirv::PointerType::get(pointeeType: arrayElemType, storageClass);
562 int64_t stride = needsExplicitLayout(storageClass) ? *arrayElemSize : 0;
563 auto arrayType = spirv::RuntimeArrayType::get(elementType: arrayElemType, stride);
564 // For Vulkan we need extra wrapping struct and array to satisfy interface
565 // needs.
566 return wrapInStructAndGetPointer(elementType: arrayType, storageClass);
567 }
568
569 if (type.getNumElements() == 0) {
570 LLVM_DEBUG(llvm::dbgs()
571 << type << " illegal: zero-element memrefs are not supported\n");
572 return nullptr;
573 }
574
575 int64_t memrefSize = llvm::divideCeil(Numerator: type.getNumElements() * numBoolBits, Denominator: 8);
576 int64_t arrayElemCount = llvm::divideCeil(Numerator: memrefSize, Denominator: *arrayElemSize);
577 int64_t stride = needsExplicitLayout(storageClass) ? *arrayElemSize : 0;
578 auto arrayType = spirv::ArrayType::get(elementType: arrayElemType, elementCount: arrayElemCount, stride);
579 if (targetEnv.allows(spirv::Capability::Kernel))
580 return spirv::PointerType::get(pointeeType: arrayType, storageClass);
581 return wrapInStructAndGetPointer(elementType: arrayType, storageClass);
582}
583
584static Type convertSubByteMemrefType(const spirv::TargetEnv &targetEnv,
585 const SPIRVConversionOptions &options,
586 MemRefType type,
587 spirv::StorageClass storageClass) {
588 IntegerType elementType = cast<IntegerType>(Val: type.getElementType());
589 Type arrayElemType = convertSubByteIntegerType(options, type: elementType);
590 if (!arrayElemType)
591 return nullptr;
592 int64_t arrayElemSize = *getTypeNumBytes(options, type: arrayElemType);
593
594 if (!type.hasStaticShape()) {
595 // For OpenCL Kernel, dynamic shaped memrefs convert into a pointer pointing
596 // to the element.
597 if (targetEnv.allows(spirv::Capability::Kernel))
598 return spirv::PointerType::get(pointeeType: arrayElemType, storageClass);
599 int64_t stride = needsExplicitLayout(storageClass) ? arrayElemSize : 0;
600 auto arrayType = spirv::RuntimeArrayType::get(elementType: arrayElemType, stride);
601 // For Vulkan we need extra wrapping struct and array to satisfy interface
602 // needs.
603 return wrapInStructAndGetPointer(elementType: arrayType, storageClass);
604 }
605
606 if (type.getNumElements() == 0) {
607 LLVM_DEBUG(llvm::dbgs()
608 << type << " illegal: zero-element memrefs are not supported\n");
609 return nullptr;
610 }
611
612 int64_t memrefSize =
613 llvm::divideCeil(Numerator: type.getNumElements() * elementType.getWidth(), Denominator: 8);
614 int64_t arrayElemCount = llvm::divideCeil(Numerator: memrefSize, Denominator: arrayElemSize);
615 int64_t stride = needsExplicitLayout(storageClass) ? arrayElemSize : 0;
616 auto arrayType = spirv::ArrayType::get(elementType: arrayElemType, elementCount: arrayElemCount, stride);
617 if (targetEnv.allows(spirv::Capability::Kernel))
618 return spirv::PointerType::get(pointeeType: arrayType, storageClass);
619 return wrapInStructAndGetPointer(elementType: arrayType, storageClass);
620}
621
622static spirv::Dim convertRank(int64_t rank) {
623 switch (rank) {
624 case 1:
625 return spirv::Dim::Dim1D;
626 case 2:
627 return spirv::Dim::Dim2D;
628 case 3:
629 return spirv::Dim::Dim3D;
630 default:
631 llvm_unreachable("Invalid memref rank!");
632 }
633}
634
635static spirv::ImageFormat getImageFormat(Type elementType) {
636 return TypeSwitch<Type, spirv::ImageFormat>(elementType)
637 .Case(caseFn: [](Float16Type) { return spirv::ImageFormat::R16f; })
638 .Case(caseFn: [](Float32Type) { return spirv::ImageFormat::R32f; })
639 .Case(caseFn: [](IntegerType intType) {
640 auto const isSigned = intType.isSigned() || intType.isSignless();
641#define BIT_WIDTH_CASE(BIT_WIDTH) \
642 case BIT_WIDTH: \
643 return isSigned ? spirv::ImageFormat::R##BIT_WIDTH##i \
644 : spirv::ImageFormat::R##BIT_WIDTH##ui
645
646 switch (intType.getWidth()) {
647 BIT_WIDTH_CASE(16);
648 BIT_WIDTH_CASE(32);
649 default:
650 llvm_unreachable("Unhandled integer type!");
651 }
652 })
653 .DefaultUnreachable(message: "Unhandled element type!");
654#undef BIT_WIDTH_CASE
655}
656
657static Type convertMemrefType(const spirv::TargetEnv &targetEnv,
658 const SPIRVConversionOptions &options,
659 MemRefType type) {
660 auto attr = dyn_cast_or_null<spirv::StorageClassAttr>(Val: type.getMemorySpace());
661 if (!attr) {
662 LLVM_DEBUG(
663 llvm::dbgs()
664 << type
665 << " illegal: expected memory space to be a SPIR-V storage class "
666 "attribute; please use MemorySpaceToStorageClassConverter to map "
667 "numeric memory spaces beforehand\n");
668 return nullptr;
669 }
670 spirv::StorageClass storageClass = attr.getValue();
671
672 // Images are a special case since they are an opaque type from which elements
673 // may be accessed via image specific ops or directly through a texture
674 // pointer.
675 if (storageClass == spirv::StorageClass::Image) {
676 const int64_t rank = type.getRank();
677 if (rank < 1 || rank > 3) {
678 LLVM_DEBUG(llvm::dbgs()
679 << type << " illegal: cannot lower memref of rank " << rank
680 << " to a SPIR-V Image\n");
681 return nullptr;
682 }
683
684 // Note that we currently only support lowering to single element texels
685 // e.g. R32f.
686 auto elementType = type.getElementType();
687 if (!isa<spirv::ScalarType>(Val: elementType)) {
688 LLVM_DEBUG(llvm::dbgs() << type << " illegal: cannot lower memref of "
689 << elementType << " to a SPIR-V Image\n");
690 return nullptr;
691 }
692
693 // Currently every memref in the image storage class is converted to a
694 // sampled image so we can hardcode the NeedSampler field. Future work
695 // will generalize this to support regular non-sampled images.
696 auto spvImageType = spirv::ImageType::get(
697 elementType, dim: convertRank(rank), depth: spirv::ImageDepthInfo::DepthUnknown,
698 arrayed: spirv::ImageArrayedInfo::NonArrayed,
699 samplingInfo: spirv::ImageSamplingInfo::SingleSampled,
700 samplerUse: spirv::ImageSamplerUseInfo::NeedSampler, format: getImageFormat(elementType));
701 auto spvSampledImageType = spirv::SampledImageType::get(imageType: spvImageType);
702 auto imagePtrType = spirv::PointerType::get(
703 pointeeType: spvSampledImageType, storageClass: spirv::StorageClass::UniformConstant);
704 return imagePtrType;
705 }
706
707 if (isa<IntegerType>(Val: type.getElementType())) {
708 if (type.getElementTypeBitWidth() == 1)
709 return convertBoolMemrefType(targetEnv, options, type, storageClass);
710 if (type.getElementTypeBitWidth() < 8)
711 return convertSubByteMemrefType(targetEnv, options, type, storageClass);
712 }
713
714 Type arrayElemType;
715 Type elementType = type.getElementType();
716 if (auto vecType = dyn_cast<VectorType>(Val&: elementType)) {
717 arrayElemType =
718 convertVectorType(targetEnv, options, type: vecType, storageClass);
719 } else if (auto complexType = dyn_cast<ComplexType>(Val&: elementType)) {
720 arrayElemType =
721 convertComplexType(targetEnv, options, type: complexType, storageClass);
722 } else if (auto scalarType = dyn_cast<spirv::ScalarType>(Val&: elementType)) {
723 arrayElemType =
724 convertScalarType(targetEnv, options, type: scalarType, storageClass);
725 } else if (auto indexType = dyn_cast<IndexType>(Val&: elementType)) {
726 type = cast<MemRefType>(Val: convertIndexElementType(type, options));
727 arrayElemType = type.getElementType();
728 } else if (auto floatType = dyn_cast<FloatType>(Val&: elementType)) {
729 // Hnadle 8 bit float types.
730 type = cast<MemRefType>(Val: convertShaped8BitFloatType(type, options));
731 arrayElemType = type.getElementType();
732 } else {
733 LLVM_DEBUG(
734 llvm::dbgs()
735 << type
736 << " unhandled: can only convert scalar or vector element type\n");
737 return nullptr;
738 }
739 if (!arrayElemType)
740 return nullptr;
741
742 std::optional<int64_t> arrayElemSize =
743 getTypeNumBytes(options, type: arrayElemType);
744 if (!arrayElemSize) {
745 LLVM_DEBUG(llvm::dbgs()
746 << type << " illegal: cannot deduce converted element size\n");
747 return nullptr;
748 }
749
750 if (!type.hasStaticShape()) {
751 // For OpenCL Kernel, dynamic shaped memrefs convert into a pointer pointing
752 // to the element.
753 if (targetEnv.allows(spirv::Capability::Kernel))
754 return spirv::PointerType::get(pointeeType: arrayElemType, storageClass);
755 int64_t stride = needsExplicitLayout(storageClass) ? *arrayElemSize : 0;
756 auto arrayType = spirv::RuntimeArrayType::get(elementType: arrayElemType, stride);
757 // For Vulkan we need extra wrapping struct and array to satisfy interface
758 // needs.
759 return wrapInStructAndGetPointer(elementType: arrayType, storageClass);
760 }
761
762 std::optional<int64_t> memrefSize = getTypeNumBytes(options, type);
763 if (!memrefSize) {
764 LLVM_DEBUG(llvm::dbgs()
765 << type << " illegal: cannot deduce element count\n");
766 return nullptr;
767 }
768
769 if (*memrefSize == 0) {
770 LLVM_DEBUG(llvm::dbgs()
771 << type << " illegal: zero-element memrefs are not supported\n");
772 return nullptr;
773 }
774
775 int64_t arrayElemCount = llvm::divideCeil(Numerator: *memrefSize, Denominator: *arrayElemSize);
776 int64_t stride = needsExplicitLayout(storageClass) ? *arrayElemSize : 0;
777 auto arrayType = spirv::ArrayType::get(elementType: arrayElemType, elementCount: arrayElemCount, stride);
778 if (targetEnv.allows(spirv::Capability::Kernel))
779 return spirv::PointerType::get(pointeeType: arrayType, storageClass);
780 return wrapInStructAndGetPointer(elementType: arrayType, storageClass);
781}
782
783//===----------------------------------------------------------------------===//
784// Type casting materialization
785//===----------------------------------------------------------------------===//
786
787/// Converts the given `inputs` to the original source `type` considering the
788/// `targetEnv`'s capabilities.
789///
790/// This function is meant to be used for source materialization in type
791/// converters. When the type converter needs to materialize a cast op back
792/// to some original source type, we need to check whether the original source
793/// type is supported in the target environment. If so, we can insert legal
794/// SPIR-V cast ops accordingly.
795///
796/// Note that in SPIR-V the capabilities for storage and compute are separate.
797/// This function is meant to handle the **compute** side; so it does not
798/// involve storage classes in its logic. The storage side is expected to be
799/// handled by MemRef conversion logic.
800static Value castToSourceType(const spirv::TargetEnv &targetEnv,
801 OpBuilder &builder, Type type, ValueRange inputs,
802 Location loc) {
803 // We can only cast one value in SPIR-V.
804 if (inputs.size() != 1) {
805 auto castOp =
806 UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs);
807 return castOp.getResult(i: 0);
808 }
809 Value input = inputs.front();
810
811 // Only support integer types for now. Floating point types to be implemented.
812 if (!isa<IntegerType>(Val: type)) {
813 auto castOp =
814 UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs);
815 return castOp.getResult(i: 0);
816 }
817 auto inputType = cast<IntegerType>(Val: input.getType());
818
819 auto scalarType = dyn_cast<spirv::ScalarType>(Val&: type);
820 if (!scalarType) {
821 auto castOp =
822 UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs);
823 return castOp.getResult(i: 0);
824 }
825
826 // Only support source type with a smaller bitwidth. This would mean we are
827 // truncating to go back so we don't need to worry about the signedness.
828 // For extension, we cannot have enough signal here to decide which op to use.
829 if (inputType.getIntOrFloatBitWidth() < scalarType.getIntOrFloatBitWidth()) {
830 auto castOp =
831 UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs);
832 return castOp.getResult(i: 0);
833 }
834
835 // Boolean values would need to use different ops than normal integer values.
836 if (type.isInteger(width: 1)) {
837 Value one = spirv::ConstantOp::getOne(type: inputType, loc, builder);
838 return spirv::IEqualOp::create(builder, location: loc, operand1: input, operand2: one);
839 }
840
841 // Check that the source integer type is supported by the environment.
842 SmallVector<ArrayRef<spirv::Extension>, 1> exts;
843 SmallVector<ArrayRef<spirv::Capability>, 2> caps;
844 scalarType.getExtensions(extensions&: exts);
845 scalarType.getCapabilities(capabilities&: caps);
846 if (failed(Result: checkCapabilityRequirements(label: type, targetEnv, candidates: caps)) ||
847 failed(Result: checkExtensionRequirements(label: type, targetEnv, candidates: exts))) {
848 auto castOp =
849 UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs);
850 return castOp.getResult(i: 0);
851 }
852
853 // We've already made sure this is truncating previously, so we don't need to
854 // care about signedness here. Still try to use a corresponding op for better
855 // consistency though.
856 if (type.isSignedInteger()) {
857 return spirv::SConvertOp::create(builder, location: loc, result: type, operand: input);
858 }
859 return spirv::UConvertOp::create(builder, location: loc, result: type, operand: input);
860}
861
862//===----------------------------------------------------------------------===//
863// Builtin Variables
864//===----------------------------------------------------------------------===//
865
866static spirv::GlobalVariableOp getBuiltinVariable(Block &body,
867 spirv::BuiltIn builtin) {
868 // Look through all global variables in the given `body` block and check if
869 // there is a spirv.GlobalVariable that has the same `builtin` attribute.
870 for (auto varOp : body.getOps<spirv::GlobalVariableOp>()) {
871 if (StringAttr builtinAttr = varOp.getBuiltInAttr()) {
872 auto varBuiltIn = spirv::symbolizeBuiltIn(builtinAttr.getValue());
873 if (varBuiltIn == builtin) {
874 return varOp;
875 }
876 }
877 }
878 return nullptr;
879}
880
881/// Gets name of global variable for a builtin.
882std::string getBuiltinVarName(spirv::BuiltIn builtin, StringRef prefix,
883 StringRef suffix) {
884 return Twine(prefix).concat(Suffix: stringifyBuiltIn(builtin)).concat(Suffix: suffix).str();
885}
886
887/// Gets or inserts a global variable for a builtin within `body` block.
888static spirv::GlobalVariableOp
889getOrInsertBuiltinVariable(Block &body, Location loc, spirv::BuiltIn builtin,
890 Type integerType, OpBuilder &builder,
891 StringRef prefix, StringRef suffix) {
892 if (auto varOp = getBuiltinVariable(body, builtin))
893 return varOp;
894
895 OpBuilder::InsertionGuard guard(builder);
896 builder.setInsertionPointToStart(&body);
897
898 spirv::GlobalVariableOp newVarOp;
899 switch (builtin) {
900 case spirv::BuiltIn::NumWorkgroups:
901 case spirv::BuiltIn::WorkgroupSize:
902 case spirv::BuiltIn::WorkgroupId:
903 case spirv::BuiltIn::LocalInvocationId:
904 case spirv::BuiltIn::GlobalInvocationId: {
905 auto ptrType = spirv::PointerType::get(pointeeType: VectorType::get(shape: {3}, elementType: integerType),
906 storageClass: spirv::StorageClass::Input);
907 std::string name = getBuiltinVarName(builtin, prefix, suffix);
908 newVarOp =
909 spirv::GlobalVariableOp::create(builder, location: loc, type: ptrType, name, builtin);
910 break;
911 }
912 case spirv::BuiltIn::SubgroupId:
913 case spirv::BuiltIn::NumSubgroups:
914 case spirv::BuiltIn::SubgroupSize:
915 case spirv::BuiltIn::SubgroupLocalInvocationId: {
916 auto ptrType =
917 spirv::PointerType::get(pointeeType: integerType, storageClass: spirv::StorageClass::Input);
918 std::string name = getBuiltinVarName(builtin, prefix, suffix);
919 newVarOp =
920 spirv::GlobalVariableOp::create(builder, location: loc, type: ptrType, name, builtin);
921 break;
922 }
923 default:
924 emitError(loc, message: "unimplemented builtin variable generation for ")
925 << stringifyBuiltIn(builtin);
926 }
927 return newVarOp;
928}
929
930//===----------------------------------------------------------------------===//
931// Push constant storage
932//===----------------------------------------------------------------------===//
933
934/// Returns the pointer type for the push constant storage containing
935/// `elementCount` 32-bit integer values.
936static spirv::PointerType getPushConstantStorageType(unsigned elementCount,
937 Builder &builder,
938 Type indexType) {
939 auto arrayType = spirv::ArrayType::get(elementType: indexType, elementCount,
940 /*stride=*/4);
941 auto structType = spirv::StructType::get(memberTypes: {arrayType}, /*offsetInfo=*/0);
942 return spirv::PointerType::get(pointeeType: structType, storageClass: spirv::StorageClass::PushConstant);
943}
944
945/// Returns the push constant varible containing `elementCount` 32-bit integer
946/// values in `body`. Returns null op if such an op does not exit.
947static spirv::GlobalVariableOp getPushConstantVariable(Block &body,
948 unsigned elementCount) {
949 for (auto varOp : body.getOps<spirv::GlobalVariableOp>()) {
950 auto ptrType = dyn_cast<spirv::PointerType>(Val: varOp.getType());
951 if (!ptrType)
952 continue;
953
954 // Note that Vulkan requires "There must be no more than one push constant
955 // block statically used per shader entry point." So we should always reuse
956 // the existing one.
957 if (ptrType.getStorageClass() == spirv::StorageClass::PushConstant) {
958 auto numElements = cast<spirv::ArrayType>(
959 Val: cast<spirv::StructType>(Val: ptrType.getPointeeType())
960 .getElementType(0))
961 .getNumElements();
962 if (numElements == elementCount)
963 return varOp;
964 }
965 }
966 return nullptr;
967}
968
969/// Gets or inserts a global variable for push constant storage containing
970/// `elementCount` 32-bit integer values in `block`.
971static spirv::GlobalVariableOp
972getOrInsertPushConstantVariable(Location loc, Block &block,
973 unsigned elementCount, OpBuilder &b,
974 Type indexType) {
975 if (auto varOp = getPushConstantVariable(body&: block, elementCount))
976 return varOp;
977
978 auto builder = OpBuilder::atBlockBegin(block: &block, listener: b.getListener());
979 auto type = getPushConstantStorageType(elementCount, builder, indexType);
980 const char *name = "__push_constant_var__";
981 return spirv::GlobalVariableOp::create(builder, location: loc, type, sym_name: name,
982 /*initializer=*/nullptr);
983}
984
985//===----------------------------------------------------------------------===//
986// func::FuncOp Conversion Patterns
987//===----------------------------------------------------------------------===//
988
989/// A pattern for rewriting function signature to convert arguments of functions
990/// to be of valid SPIR-V types.
991struct FuncOpConversion final : OpConversionPattern<func::FuncOp> {
992 using Base::Base;
993
994 LogicalResult
995 matchAndRewrite(func::FuncOp funcOp, OpAdaptor adaptor,
996 ConversionPatternRewriter &rewriter) const override {
997 FunctionType fnType = funcOp.getFunctionType();
998 if (fnType.getNumResults() > 1)
999 return failure();
1000
1001 TypeConverter::SignatureConversion signatureConverter(
1002 fnType.getNumInputs());
1003 for (const auto &argType : enumerate(First: fnType.getInputs())) {
1004 auto convertedType = getTypeConverter()->convertType(t: argType.value());
1005 if (!convertedType)
1006 return failure();
1007 signatureConverter.addInputs(origInputNo: argType.index(), types: convertedType);
1008 }
1009
1010 Type resultType;
1011 if (fnType.getNumResults() == 1) {
1012 resultType = getTypeConverter()->convertType(t: fnType.getResult(i: 0));
1013 if (!resultType)
1014 return failure();
1015 }
1016
1017 // Create the converted spirv.func op.
1018 auto newFuncOp = spirv::FuncOp::create(
1019 builder&: rewriter, location: funcOp.getLoc(), name: funcOp.getName(),
1020 type: rewriter.getFunctionType(inputs: signatureConverter.getConvertedTypes(),
1021 results: resultType ? TypeRange(resultType)
1022 : TypeRange()));
1023
1024 newFuncOp.setArgAttrsAttr(funcOp.getArgAttrsAttr());
1025 newFuncOp.setResAttrsAttr(funcOp.getResAttrsAttr());
1026 cast<SymbolOpInterface>(Val: newFuncOp.getOperation())
1027 .setVisibility(
1028 cast<SymbolOpInterface>(Val: funcOp.getOperation()).getVisibility());
1029
1030 // Copy over the discardable attributes.
1031 for (NamedAttribute namedAttr :
1032 funcOp->getDiscardableAttrDictionary().getValue())
1033 newFuncOp->setDiscardableAttr(name: namedAttr.getName(), value: namedAttr.getValue());
1034
1035 rewriter.inlineRegionBefore(region&: funcOp.getBody(), parent&: newFuncOp.getBody(),
1036 before: newFuncOp.end());
1037 if (failed(Result: rewriter.convertRegionTypes(
1038 region: &newFuncOp.getBody(), converter: *getTypeConverter(), entryConversion: &signatureConverter)))
1039 return failure();
1040 rewriter.eraseOp(op: funcOp);
1041 return success();
1042 }
1043};
1044
1045/// A pattern for rewriting function signature to convert vector arguments of
1046/// functions to be of valid types
1047struct FuncOpVectorUnroll final : OpRewritePattern<func::FuncOp> {
1048 using Base::Base;
1049
1050 LogicalResult matchAndRewrite(func::FuncOp funcOp,
1051 PatternRewriter &rewriter) const override {
1052 FunctionType fnType = funcOp.getFunctionType();
1053
1054 // TODO: Handle declarations.
1055 if (funcOp.isDeclaration()) {
1056 LLVM_DEBUG(llvm::dbgs()
1057 << fnType << " illegal: declarations are unsupported\n");
1058 return failure();
1059 }
1060
1061 // Bail out early for dynamically-shaped argument types: getZeroAttr
1062 // requires a statically-shaped type. VectorType is always statically
1063 // shaped, so this correctly skips it without a special-case guard.
1064 if (llvm::any_of(Range: fnType.getInputs(), P: [](Type argType) {
1065 auto shapedType = dyn_cast<ShapedType>(Val&: argType);
1066 return shapedType && !shapedType.hasStaticShape();
1067 }))
1068 return failure();
1069
1070 // Create a new func op with the original type and copy the function body.
1071 auto newFuncOp = func::FuncOp::create(builder&: rewriter, location: funcOp.getLoc(),
1072 name: funcOp.getName(), type: fnType);
1073 rewriter.inlineRegionBefore(region&: funcOp.getBody(), parent&: newFuncOp.getBody(),
1074 before: newFuncOp.end());
1075
1076 Location loc = newFuncOp.getBody().getLoc();
1077
1078 Block &entryBlock = newFuncOp.getBlocks().front();
1079 OpBuilder::InsertionGuard guard(rewriter);
1080 rewriter.setInsertionPointToStart(&entryBlock);
1081
1082 TypeConverter::SignatureConversion oneToNTypeMapping(
1083 fnType.getInputs().size());
1084
1085 // For arguments that are of illegal types and require unrolling.
1086 // `unrolledInputNums` stores the indices of arguments that result from
1087 // unrolling in the new function signature. `newInputNo` is a counter.
1088 SmallVector<size_t> unrolledInputNums;
1089 size_t newInputNo = 0;
1090
1091 // For arguments that are of legal types and do not require unrolling.
1092 // `tmpOps` stores a mapping from temporary operations that serve as
1093 // placeholders for new arguments that will be added later. These operations
1094 // will be erased once the entry block's argument list is updated.
1095 llvm::SmallDenseMap<Operation *, size_t> tmpOps;
1096
1097 // This counts the number of new operations created.
1098 size_t newOpCount = 0;
1099
1100 // Enumerate through the arguments.
1101 for (auto [origInputNo, origType] : enumerate(First: fnType.getInputs())) {
1102 // Check whether the argument is of vector type.
1103 auto origVecType = dyn_cast<VectorType>(Val: origType);
1104 if (!origVecType) {
1105 // We need a placeholder for the old argument that will be erased later.
1106 auto castOp = UnrealizedConversionCastOp::create(
1107 builder&: rewriter, location: loc, resultTypes: origType, operands: ValueRange());
1108 Value result = castOp.getResult(i: 0);
1109 rewriter.replaceAllUsesWith(from: newFuncOp.getArgument(idx: origInputNo), to: result);
1110 tmpOps.insert(KV: {result.getDefiningOp(), newInputNo});
1111 oneToNTypeMapping.addInputs(origInputNo, types: origType);
1112 ++newInputNo;
1113 ++newOpCount;
1114 continue;
1115 }
1116 // Check whether the vector needs unrolling.
1117 auto targetShape = getTargetShape(vecType: origVecType);
1118 if (!targetShape) {
1119 // We need a placeholder for the old argument that will be erased later.
1120 auto castOp = UnrealizedConversionCastOp::create(
1121 builder&: rewriter, location: loc, resultTypes: origType, operands: ValueRange());
1122 Value result = castOp.getResult(i: 0);
1123 rewriter.replaceAllUsesWith(from: newFuncOp.getArgument(idx: origInputNo), to: result);
1124 tmpOps.insert(KV: {result.getDefiningOp(), newInputNo});
1125 oneToNTypeMapping.addInputs(origInputNo, types: origType);
1126 ++newInputNo;
1127 ++newOpCount;
1128 continue;
1129 }
1130 VectorType unrolledType =
1131 VectorType::get(shape: *targetShape, elementType: origVecType.getElementType());
1132 auto originalShape =
1133 llvm::to_vector_of<int64_t, 4>(Range: origVecType.getShape());
1134
1135 // Prepare the result vector.
1136 Value result = arith::ConstantOp::create(
1137 builder&: rewriter, location: loc, result: origVecType, value: rewriter.getZeroAttr(type: origVecType));
1138 ++newOpCount;
1139 // Prepare the placeholder for the new arguments that will be added later.
1140 Value dummy = arith::ConstantOp::create(
1141 builder&: rewriter, location: loc, result: unrolledType, value: rewriter.getZeroAttr(type: unrolledType));
1142 ++newOpCount;
1143
1144 // Create the `vector.insert_strided_slice` ops.
1145 SmallVector<int64_t> strides(targetShape->size(), 1);
1146 SmallVector<Type> newTypes;
1147 for (SmallVector<int64_t> offsets :
1148 StaticTileOffsetRange(originalShape, *targetShape)) {
1149 result = vector::InsertStridedSliceOp::create(builder&: rewriter, location: loc, valueToStore: dummy,
1150 dest: result, offsets, strides);
1151 newTypes.push_back(Elt: unrolledType);
1152 unrolledInputNums.push_back(Elt: newInputNo);
1153 ++newInputNo;
1154 ++newOpCount;
1155 }
1156 rewriter.replaceAllUsesWith(from: newFuncOp.getArgument(idx: origInputNo), to: result);
1157 oneToNTypeMapping.addInputs(origInputNo, types: newTypes);
1158 }
1159
1160 // Change the function signature.
1161 auto convertedTypes = oneToNTypeMapping.getConvertedTypes();
1162 auto newFnType = fnType.clone(inputs: convertedTypes, results: fnType.getResults());
1163 rewriter.modifyOpInPlace(root: newFuncOp,
1164 callable: [&] { newFuncOp.setFunctionType(newFnType); });
1165
1166 // Update the arguments in the entry block.
1167 entryBlock.eraseArguments(start: 0, num: fnType.getNumInputs());
1168 SmallVector<Location> locs(convertedTypes.size(), newFuncOp.getLoc());
1169 entryBlock.addArguments(types: convertedTypes, locs);
1170
1171 // Replace all uses of placeholders for initially legal arguments with their
1172 // original function arguments (that were added to `newFuncOp`).
1173 for (auto &[placeholderOp, argIdx] : tmpOps) {
1174 if (!placeholderOp)
1175 continue;
1176 Value replacement = newFuncOp.getArgument(idx: argIdx);
1177 rewriter.replaceAllUsesWith(from: placeholderOp->getResult(idx: 0), to: replacement);
1178 }
1179
1180 // Replace dummy operands of new `vector.insert_strided_slice` ops with
1181 // their corresponding new function arguments. The new
1182 // `vector.insert_strided_slice` ops are inserted only into the entry block,
1183 // so iterating over that block is sufficient.
1184 size_t unrolledInputIdx = 0;
1185 for (auto [count, op] : enumerate(First&: entryBlock.getOperations())) {
1186 Operation &curOp = op;
1187 // Since all newly created operations are in the beginning, reaching the
1188 // end of them means that any later `vector.insert_strided_slice` should
1189 // not be touched.
1190 if (count >= newOpCount)
1191 continue;
1192 if (auto vecOp = dyn_cast<vector::InsertStridedSliceOp>(Val&: op)) {
1193 size_t unrolledInputNo = unrolledInputNums[unrolledInputIdx];
1194 rewriter.modifyOpInPlace(root: &curOp, callable: [&] {
1195 curOp.setOperand(idx: 0, value: newFuncOp.getArgument(idx: unrolledInputNo));
1196 });
1197 ++unrolledInputIdx;
1198 }
1199 }
1200
1201 // Erase the original funcOp. The `tmpOps` do not need to be erased since
1202 // they have no uses and will be handled by dead-code elimination.
1203 rewriter.eraseOp(op: funcOp);
1204 return success();
1205 }
1206};
1207
1208//===----------------------------------------------------------------------===//
1209// func::ReturnOp Conversion Patterns
1210//===----------------------------------------------------------------------===//
1211
1212/// A pattern for rewriting function signature and the return op to convert
1213/// vectors to be of valid types.
1214struct ReturnOpVectorUnroll final : OpRewritePattern<func::ReturnOp> {
1215 using Base::Base;
1216
1217 LogicalResult matchAndRewrite(func::ReturnOp returnOp,
1218 PatternRewriter &rewriter) const override {
1219 // Check whether the parent funcOp is valid.
1220 auto funcOp = dyn_cast<func::FuncOp>(Val: returnOp->getParentOp());
1221 if (!funcOp)
1222 return failure();
1223
1224 FunctionType fnType = funcOp.getFunctionType();
1225 TypeConverter::SignatureConversion oneToNTypeMapping(
1226 fnType.getResults().size());
1227 Location loc = returnOp.getLoc();
1228
1229 // For the new return op.
1230 SmallVector<Value> newOperands;
1231
1232 // Enumerate through the results.
1233 for (auto [origResultNo, origType] : enumerate(First: fnType.getResults())) {
1234 // Check whether the argument is of vector type.
1235 auto origVecType = dyn_cast<VectorType>(Val: origType);
1236 if (!origVecType) {
1237 oneToNTypeMapping.addInputs(origInputNo: origResultNo, types: origType);
1238 newOperands.push_back(Elt: returnOp.getOperand(i: origResultNo));
1239 continue;
1240 }
1241 // Check whether the vector needs unrolling.
1242 auto targetShape = getTargetShape(vecType: origVecType);
1243 if (!targetShape) {
1244 // The original argument can be used.
1245 oneToNTypeMapping.addInputs(origInputNo: origResultNo, types: origType);
1246 newOperands.push_back(Elt: returnOp.getOperand(i: origResultNo));
1247 continue;
1248 }
1249 VectorType unrolledType =
1250 VectorType::get(shape: *targetShape, elementType: origVecType.getElementType());
1251
1252 // Create `vector.extract_strided_slice` ops to form legal vectors from
1253 // the original operand of illegal type.
1254 auto originalShape =
1255 llvm::to_vector_of<int64_t, 4>(Range: origVecType.getShape());
1256 SmallVector<int64_t> strides(originalShape.size(), 1);
1257 SmallVector<int64_t> extractShape(originalShape.size(), 1);
1258 extractShape.back() = targetShape->back();
1259 SmallVector<Type> newTypes;
1260 Value returnValue = returnOp.getOperand(i: origResultNo);
1261 for (SmallVector<int64_t> offsets :
1262 StaticTileOffsetRange(originalShape, *targetShape)) {
1263 Value result = vector::ExtractStridedSliceOp::create(
1264 builder&: rewriter, location: loc, source: returnValue, offsets, sizes: extractShape, strides);
1265 if (originalShape.size() > 1) {
1266 SmallVector<int64_t> extractIndices(originalShape.size() - 1, 0);
1267 result =
1268 vector::ExtractOp::create(builder&: rewriter, location: loc, source: result, position: extractIndices);
1269 }
1270 newOperands.push_back(Elt: result);
1271 newTypes.push_back(Elt: unrolledType);
1272 }
1273 oneToNTypeMapping.addInputs(origInputNo: origResultNo, types: newTypes);
1274 }
1275
1276 // Change the function signature.
1277 auto newFnType =
1278 FunctionType::get(context: rewriter.getContext(), inputs: TypeRange(fnType.getInputs()),
1279 results: TypeRange(oneToNTypeMapping.getConvertedTypes()));
1280 rewriter.modifyOpInPlace(root: funcOp,
1281 callable: [&] { funcOp.setFunctionType(newFnType); });
1282
1283 // Replace the return op using the new operands. This will automatically
1284 // update the entry block as well.
1285 rewriter.replaceOp(op: returnOp,
1286 newOp: func::ReturnOp::create(builder&: rewriter, location: loc, operands: newOperands));
1287
1288 return success();
1289 }
1290};
1291
1292static void addNoWrapDecorations(Operation *op,
1293 spirv::LinearizedIndexNoWrapFlags flags,
1294 OpBuilder &builder) {
1295 if (flags.noSignedWrap)
1296 op->setDiscardableAttr(
1297 name: spirv::getDecorationString(decoration: spirv::Decoration::NoSignedWrap),
1298 value: builder.getUnitAttr());
1299 if (flags.noUnsignedWrap)
1300 op->setDiscardableAttr(
1301 name: spirv::getDecorationString(decoration: spirv::Decoration::NoUnsignedWrap),
1302 value: builder.getUnitAttr());
1303}
1304
1305static std::optional<uint64_t> getMaxLinearizedIndex(ArrayRef<int64_t> shape,
1306 ArrayRef<int64_t> strides,
1307 int64_t offset) {
1308 if (shape.size() != strides.size() || offset < 0)
1309 return std::nullopt;
1310
1311 uint64_t maxLinearIndex = offset;
1312 for (auto [dimension, stride] : llvm::zip(t&: shape, u&: strides)) {
1313 if (dimension <= 0 || stride < 0)
1314 return std::nullopt;
1315 std::optional<uint64_t> nextMaxLinearIndex = llvm::checkedMulAddUnsigned(
1316 A: static_cast<uint64_t>(dimension - 1), B: static_cast<uint64_t>(stride),
1317 C: maxLinearIndex);
1318 if (!nextMaxLinearIndex)
1319 return std::nullopt;
1320 maxLinearIndex = *nextMaxLinearIndex;
1321 }
1322 return maxLinearIndex;
1323}
1324
1325static spirv::ArrayType getStorageBufferArrayType(Value basePtr) {
1326 auto pointerType = dyn_cast<spirv::PointerType>(Val: basePtr.getType());
1327 if (!pointerType ||
1328 pointerType.getStorageClass() != spirv::StorageClass::StorageBuffer)
1329 return {};
1330
1331 Type pointeeType = pointerType.getPointeeType();
1332 if (auto structType = dyn_cast<spirv::StructType>(Val&: pointeeType)) {
1333 if (structType.getNumElements() != 1)
1334 return {};
1335 pointeeType = structType.getElementType(0);
1336 }
1337 return dyn_cast<spirv::ArrayType>(Val&: pointeeType);
1338}
1339
1340static bool shouldEmitInBoundsAccessChain(MemRefType baseType, Value basePtr,
1341 ArrayRef<int64_t> strides,
1342 int64_t offset,
1343 uint64_t accessElementCount) {
1344 std::optional<uint64_t> maxSourceElementIndex =
1345 getMaxLinearizedIndex(shape: baseType.getShape(), strides, offset);
1346 spirv::ArrayType storageArrayType = getStorageBufferArrayType(basePtr);
1347 if (!maxSourceElementIndex || !storageArrayType)
1348 return false;
1349
1350 // Source indices and storage element counts use the same units only when
1351 // each source element maps to one SPIR-V array element. An i16 or a narrower
1352 // memref source may be stored using a wider SPIR-V array element than that
1353 // of the source. Keep a plain access chain so later bitwidth emulation can
1354 // adjust the final index in storage-element units.
1355 if (baseType.getElementType() != storageArrayType.getElementType())
1356 return false;
1357
1358 uint64_t storageElementCount = storageArrayType.getNumElements();
1359 if (accessElementCount == 0 || accessElementCount > storageElementCount)
1360 return false;
1361
1362 // `InBoundsAccessChain` requires the computed pointer to stay within the
1363 // SPIR-V base object. Dynamic index validity is assumed from the source
1364 // operation/caller contract; for vector accesses, `accessElementCount` only
1365 // rejects widths that cannot fit in the fixed StorageBuffer object at all.
1366 // The static proof here is that the memref layout's linear index space maps
1367 // into that same object.
1368 return *maxSourceElementIndex < storageElementCount;
1369}
1370
1371} // namespace
1372
1373//===----------------------------------------------------------------------===//
1374// Public function for builtin variables
1375//===----------------------------------------------------------------------===//
1376
1377Value mlir::spirv::getBuiltinVariableValue(Operation *op,
1378 spirv::BuiltIn builtin,
1379 Type integerType, OpBuilder &builder,
1380 StringRef prefix, StringRef suffix) {
1381 Operation *parent = SymbolTable::getNearestSymbolTable(from: op->getParentOp());
1382 if (!parent) {
1383 op->emitError(message: "expected operation to be within a module-like op");
1384 return nullptr;
1385 }
1386
1387 spirv::GlobalVariableOp varOp =
1388 getOrInsertBuiltinVariable(body&: *parent->getRegion(index: 0).begin(), loc: op->getLoc(),
1389 builtin, integerType, builder, prefix, suffix);
1390 Value ptr = spirv::AddressOfOp::create(builder, location: op->getLoc(), var: varOp);
1391 return spirv::LoadOp::create(builder, location: op->getLoc(), basePtr: ptr);
1392}
1393
1394//===----------------------------------------------------------------------===//
1395// Public function for pushing constant storage
1396//===----------------------------------------------------------------------===//
1397
1398Value spirv::getPushConstantValue(Operation *op, unsigned elementCount,
1399 unsigned offset, Type integerType,
1400 OpBuilder &builder) {
1401 Location loc = op->getLoc();
1402 Operation *parent = SymbolTable::getNearestSymbolTable(from: op->getParentOp());
1403 if (!parent) {
1404 op->emitError(message: "expected operation to be within a module-like op");
1405 return nullptr;
1406 }
1407
1408 spirv::GlobalVariableOp varOp = getOrInsertPushConstantVariable(
1409 loc, block&: parent->getRegion(index: 0).front(), elementCount, b&: builder, indexType: integerType);
1410
1411 Value zeroOp = spirv::ConstantOp::getZero(type: integerType, loc, builder);
1412 Value offsetOp = spirv::ConstantOp::create(builder, location: loc, constant: integerType,
1413 value: builder.getI32IntegerAttr(value: offset));
1414 auto addrOp = spirv::AddressOfOp::create(builder, location: loc, var: varOp);
1415 auto acOp = spirv::AccessChainOp::create(builder, location: loc, basePtr: addrOp,
1416 indices: llvm::ArrayRef({zeroOp, offsetOp}));
1417 return spirv::LoadOp::create(builder, location: loc, basePtr: acOp);
1418}
1419
1420//===----------------------------------------------------------------------===//
1421// Public functions for index calculation
1422//===----------------------------------------------------------------------===//
1423
1424mlir::spirv::LinearizedIndexNoWrapFlags
1425mlir::spirv::getLinearizedIndexNoWrapFlags(const TargetEnv &targetEnv,
1426 ArrayRef<int64_t> shape,
1427 ArrayRef<int64_t> strides,
1428 int64_t offset, Type integerType) {
1429 LinearizedIndexNoWrapFlags flags;
1430 if (!targetEnv.allows(Extension::SPV_KHR_no_integer_wrap_decoration))
1431 return flags;
1432
1433 auto integer = dyn_cast<IntegerType>(Val&: integerType);
1434 if (!integer)
1435 return flags;
1436
1437 std::optional<uint64_t> maxLinearIndex =
1438 getMaxLinearizedIndex(shape, strides, offset);
1439 if (!maxLinearIndex)
1440 return flags;
1441
1442 flags.noSignedWrap =
1443 *maxLinearIndex <=
1444 APInt::getSignedMaxValue(numBits: integer.getWidth()).getZExtValue();
1445 flags.noUnsignedWrap =
1446 *maxLinearIndex <= APInt::getMaxValue(numBits: integer.getWidth()).getZExtValue();
1447 return flags;
1448}
1449
1450Value mlir::spirv::linearizeIndex(ValueRange indices, ArrayRef<int64_t> strides,
1451 int64_t offset, Type integerType,
1452 Location loc, OpBuilder &builder,
1453 LinearizedIndexNoWrapFlags noWrapFlags) {
1454 assert(indices.size() == strides.size() &&
1455 "must provide indices for all dimensions");
1456
1457 // TODO: Consider moving to use affine.apply and patterns converting
1458 // affine.apply to standard ops. This needs converting to SPIR-V passes to be
1459 // broken down into progressive small steps so we can have intermediate steps
1460 // using other dialects. At the moment SPIR-V is the final sink.
1461
1462 Value linearizedIndex = builder.createOrFold<spirv::ConstantOp>(
1463 location: loc, args&: integerType, args: IntegerAttr::get(type: integerType, value: offset));
1464 for (const auto &index : llvm::enumerate(First&: indices)) {
1465 Value strideVal = builder.createOrFold<spirv::ConstantOp>(
1466 location: loc, args&: integerType,
1467 args: IntegerAttr::get(type: integerType, value: strides[index.index()]));
1468 Value update =
1469 builder.createOrFold<spirv::IMulOp>(location: loc, args&: index.value(), args&: strideVal);
1470 if (noWrapFlags.noSignedWrap || noWrapFlags.noUnsignedWrap)
1471 if (auto mul = update.getDefiningOp<spirv::IMulOp>())
1472 addNoWrapDecorations(op: mul, flags: noWrapFlags, builder);
1473
1474 linearizedIndex =
1475 builder.createOrFold<spirv::IAddOp>(location: loc, args&: update, args&: linearizedIndex);
1476 if (noWrapFlags.noSignedWrap || noWrapFlags.noUnsignedWrap)
1477 if (auto add = linearizedIndex.getDefiningOp<spirv::IAddOp>())
1478 addNoWrapDecorations(op: add, flags: noWrapFlags, builder);
1479 }
1480 return linearizedIndex;
1481}
1482
1483Value mlir::spirv::getVulkanElementPtr(const SPIRVTypeConverter &typeConverter,
1484 MemRefType baseType, Value basePtr,
1485 ValueRange indices, Location loc,
1486 OpBuilder &builder,
1487 uint64_t accessElementCount) {
1488 // Get base and offset of the MemRefType and verify they are static.
1489
1490 int64_t offset;
1491 SmallVector<int64_t, 4> strides;
1492 if (failed(Result: baseType.getStridesAndOffset(strides, offset)) ||
1493 llvm::is_contained(Range&: strides, Element: ShapedType::kDynamic) ||
1494 ShapedType::isDynamic(dValue: offset)) {
1495 return nullptr;
1496 }
1497
1498 auto indexType = typeConverter.getIndexType();
1499 LinearizedIndexNoWrapFlags noWrapFlags = getLinearizedIndexNoWrapFlags(
1500 targetEnv: typeConverter.getTargetEnv(), shape: baseType.getShape(), strides, offset,
1501 integerType: indexType);
1502
1503 SmallVector<Value, 2> linearizedIndices;
1504 auto zero = spirv::ConstantOp::getZero(type: indexType, loc, builder);
1505
1506 if (baseType.getRank() == 0) {
1507 linearizedIndices.push_back(Elt: zero);
1508 } else {
1509 linearizedIndices.push_back(Elt: linearizeIndex(
1510 indices, strides, offset, integerType: indexType, loc, builder, noWrapFlags));
1511 }
1512
1513 const Type pointeeType =
1514 cast<spirv::PointerType>(Val: basePtr.getType()).getPointeeType();
1515 // Interface memrefs are wrapped in a struct: index to its first elem.
1516 if (isa<spirv::StructType>(Val: pointeeType))
1517 linearizedIndices.insert(I: linearizedIndices.begin(), Elt: zero);
1518 if (shouldEmitInBoundsAccessChain(baseType, basePtr, strides, offset,
1519 accessElementCount))
1520 return spirv::InBoundsAccessChainOp::create(builder, location: loc, basePtr,
1521 indices: linearizedIndices);
1522 return spirv::AccessChainOp::create(builder, location: loc, basePtr, indices: linearizedIndices);
1523}
1524
1525Value mlir::spirv::getVulkanElementPtr(const SPIRVTypeConverter &typeConverter,
1526 MemRefType baseType, Value basePtr,
1527 ValueRange indices, Location loc,
1528 OpBuilder &builder) {
1529 return getVulkanElementPtr(typeConverter, baseType, basePtr, indices, loc,
1530 builder, /*accessElementCount=*/1);
1531}
1532
1533Value mlir::spirv::getOpenCLElementPtr(const SPIRVTypeConverter &typeConverter,
1534 MemRefType baseType, Value basePtr,
1535 ValueRange indices, Location loc,
1536 OpBuilder &builder) {
1537 // Get base and offset of the MemRefType and verify they are static.
1538
1539 int64_t offset;
1540 SmallVector<int64_t, 4> strides;
1541 if (failed(Result: baseType.getStridesAndOffset(strides, offset)) ||
1542 llvm::is_contained(Range&: strides, Element: ShapedType::kDynamic) ||
1543 ShapedType::isDynamic(dValue: offset)) {
1544 return nullptr;
1545 }
1546
1547 auto indexType = typeConverter.getIndexType();
1548 LinearizedIndexNoWrapFlags noWrapFlags = getLinearizedIndexNoWrapFlags(
1549 targetEnv: typeConverter.getTargetEnv(), shape: baseType.getShape(), strides, offset,
1550 integerType: indexType);
1551
1552 SmallVector<Value, 2> linearizedIndices;
1553 Value linearIndex;
1554 if (baseType.getRank() == 0) {
1555 linearIndex = spirv::ConstantOp::getZero(type: indexType, loc, builder);
1556 } else {
1557 linearIndex = linearizeIndex(indices, strides, offset, integerType: indexType, loc,
1558 builder, noWrapFlags);
1559 }
1560 Type pointeeType =
1561 cast<spirv::PointerType>(Val: basePtr.getType()).getPointeeType();
1562 if (isa<spirv::ArrayType>(Val: pointeeType)) {
1563 linearizedIndices.push_back(Elt: linearIndex);
1564 return spirv::AccessChainOp::create(builder, location: loc, basePtr,
1565 indices: linearizedIndices);
1566 }
1567 return spirv::PtrAccessChainOp::create(builder, location: loc, basePtr, element: linearIndex,
1568 indices: linearizedIndices);
1569}
1570
1571Value mlir::spirv::getElementPtr(const SPIRVTypeConverter &typeConverter,
1572 MemRefType baseType, Value basePtr,
1573 ValueRange indices, Location loc,
1574 OpBuilder &builder,
1575 uint64_t accessElementCount) {
1576
1577 if (typeConverter.allows(capability: spirv::Capability::Kernel)) {
1578 return getOpenCLElementPtr(typeConverter, baseType, basePtr, indices, loc,
1579 builder);
1580 }
1581
1582 return getVulkanElementPtr(typeConverter, baseType, basePtr, indices, loc,
1583 builder, accessElementCount);
1584}
1585
1586Value mlir::spirv::getElementPtr(const SPIRVTypeConverter &typeConverter,
1587 MemRefType baseType, Value basePtr,
1588 ValueRange indices, Location loc,
1589 OpBuilder &builder) {
1590 return getElementPtr(typeConverter, baseType, basePtr, indices, loc, builder,
1591 /*accessElementCount=*/1);
1592}
1593
1594//===----------------------------------------------------------------------===//
1595// Public functions for vector unrolling
1596//===----------------------------------------------------------------------===//
1597
1598int mlir::spirv::getComputeVectorSize(int64_t size) {
1599 for (int i : {4, 3, 2}) {
1600 if (size % i == 0)
1601 return i;
1602 }
1603 return 1;
1604}
1605
1606SmallVector<int64_t>
1607mlir::spirv::getNativeVectorShapeImpl(vector::ReductionOp op) {
1608 VectorType srcVectorType = op.getSourceVectorType();
1609 assert(srcVectorType.getRank() == 1); // Guaranteed by semantics
1610 int64_t vectorSize =
1611 mlir::spirv::getComputeVectorSize(size: srcVectorType.getDimSize(idx: 0));
1612 return {vectorSize};
1613}
1614
1615SmallVector<int64_t>
1616mlir::spirv::getNativeVectorShapeImpl(vector::TransposeOp op) {
1617 VectorType vectorType = op.getResultVectorType();
1618 SmallVector<int64_t> nativeSize(vectorType.getRank(), 1);
1619 nativeSize.back() =
1620 mlir::spirv::getComputeVectorSize(size: vectorType.getShape().back());
1621 return nativeSize;
1622}
1623
1624std::optional<SmallVector<int64_t>>
1625mlir::spirv::getNativeVectorShape(Operation *op) {
1626 if (OpTrait::hasElementwiseMappableTraits(op) && op->getNumResults() == 1) {
1627 if (auto vecType = dyn_cast<VectorType>(Val: op->getResultTypes()[0])) {
1628 if (vecType.getRank() == 0)
1629 return std::nullopt;
1630 SmallVector<int64_t> nativeSize(vecType.getRank(), 1);
1631 nativeSize.back() =
1632 mlir::spirv::getComputeVectorSize(size: vecType.getShape().back());
1633 return nativeSize;
1634 }
1635 }
1636
1637 return TypeSwitch<Operation *, std::optional<SmallVector<int64_t>>>(op)
1638 .Case<vector::ReductionOp, vector::TransposeOp>(
1639 caseFn: [](auto typedOp) { return getNativeVectorShapeImpl(typedOp); })
1640 .Default(std::nullopt);
1641}
1642
1643LogicalResult mlir::spirv::unrollVectorsInSignatures(Operation *op) {
1644 MLIRContext *context = op->getContext();
1645 RewritePatternSet patterns(context);
1646 populateFuncOpVectorRewritePatterns(patterns);
1647 populateReturnOpVectorRewritePatterns(patterns);
1648 // We only want to apply signature conversion once to the existing func ops.
1649 // Without specifying strictMode, the greedy pattern rewriter will keep
1650 // looking for newly created func ops.
1651 return applyPatternsGreedily(op, patterns: std::move(patterns),
1652 config: GreedyRewriteConfig().setStrictness(
1653 GreedyRewriteStrictness::ExistingOps));
1654}
1655
1656LogicalResult mlir::spirv::unrollVectorsInFuncBodies(Operation *op) {
1657 MLIRContext *context = op->getContext();
1658
1659 // Unroll vectors in function bodies to native vector size.
1660 {
1661 RewritePatternSet patterns(context);
1662 auto options = vector::UnrollVectorOptions().setNativeShapeFn(
1663 [](auto op) { return mlir::spirv::getNativeVectorShape(op); });
1664 populateVectorUnrollPatterns(patterns, options);
1665 if (failed(Result: applyPatternsGreedily(op, patterns: std::move(patterns))))
1666 return failure();
1667 }
1668
1669 // Convert transpose ops into extract and insert pairs, in preparation of
1670 // further transformations to canonicalize/cancel.
1671 {
1672 RewritePatternSet patterns(context);
1673 vector::populateVectorTransposeLoweringPatterns(
1674 patterns, vectorTransposeLowering: vector::VectorTransposeLowering::EltWise);
1675 vector::populateVectorShapeCastLoweringPatterns(patterns);
1676 if (failed(Result: applyPatternsGreedily(op, patterns: std::move(patterns))))
1677 return failure();
1678 }
1679
1680 // Run canonicalization to cast away leading size-1 dimensions.
1681 {
1682 RewritePatternSet patterns(context);
1683
1684 // We need to pull in casting way leading one dims.
1685 vector::populateCastAwayVectorLeadingOneDimPatterns(patterns);
1686 vector::ReductionOp::getCanonicalizationPatterns(results&: patterns, context);
1687 vector::TransposeOp::getCanonicalizationPatterns(results&: patterns, context);
1688
1689 // Decompose different rank insert_strided_slice and n-D
1690 // extract_slided_slice.
1691 vector::populateVectorInsertExtractStridedSliceDecompositionPatterns(
1692 patterns);
1693 vector::InsertOp::getCanonicalizationPatterns(results&: patterns, context);
1694 vector::ExtractOp::getCanonicalizationPatterns(results&: patterns, context);
1695
1696 // Trimming leading unit dims may generate broadcast/shape_cast ops. Clean
1697 // them up.
1698 vector::BroadcastOp::getCanonicalizationPatterns(results&: patterns, context);
1699 vector::ShapeCastOp::getCanonicalizationPatterns(results&: patterns, context);
1700
1701 if (failed(Result: applyPatternsGreedily(op, patterns: std::move(patterns))))
1702 return failure();
1703 }
1704 return success();
1705}
1706
1707//===----------------------------------------------------------------------===//
1708// SPIR-V TypeConverter
1709//===----------------------------------------------------------------------===//
1710
1711SPIRVTypeConverter::SPIRVTypeConverter(spirv::TargetEnvAttr targetAttr,
1712 const SPIRVConversionOptions &options)
1713 : targetEnv(targetAttr), options(options) {
1714 // Add conversions. The order matters here: later ones will be tried earlier.
1715
1716 // Allow all SPIR-V dialect specific types. This assumes all builtin types
1717 // adopted in the SPIR-V dialect (i.e., IntegerType, FloatType, VectorType)
1718 // were tried before.
1719 //
1720 // TODO: This assumes that the SPIR-V types are valid to use in the given
1721 // target environment, which should be the case if the whole pipeline is
1722 // driven by the same target environment. Still, we probably still want to
1723 // validate and convert to be safe.
1724 addConversion(callback: [](spirv::SPIRVType type) { return type; });
1725
1726 addConversion(callback: [this](IndexType /*indexType*/) { return getIndexType(); });
1727
1728 addConversion(callback: [this](IntegerType intType) -> std::optional<Type> {
1729 if (auto scalarType = dyn_cast<spirv::ScalarType>(Val&: intType))
1730 return convertScalarType(targetEnv: this->targetEnv, options: this->options, type: scalarType);
1731 if (intType.getWidth() < 8)
1732 return convertSubByteIntegerType(options: this->options, type: intType);
1733 return Type();
1734 });
1735
1736 addConversion(callback: [this](FloatType floatType) -> std::optional<Type> {
1737 if (auto scalarType = dyn_cast<spirv::ScalarType>(Val&: floatType))
1738 return convertScalarType(targetEnv: this->targetEnv, options: this->options, type: scalarType);
1739 if (floatType.getWidth() == 8)
1740 return convert8BitFloatType(options: this->options, type: floatType);
1741 return Type();
1742 });
1743
1744 addConversion(callback: [this](ComplexType complexType) {
1745 return convertComplexType(targetEnv: this->targetEnv, options: this->options, type: complexType);
1746 });
1747
1748 addConversion(callback: [this](VectorType vectorType) {
1749 return convertVectorType(targetEnv: this->targetEnv, options: this->options, type: vectorType);
1750 });
1751
1752 addConversion(callback: [this](TensorType tensorType) {
1753 return convertTensorType(targetEnv: this->targetEnv, options: this->options, type: tensorType);
1754 });
1755
1756 addConversion(callback: [this](MemRefType memRefType) {
1757 return convertMemrefType(targetEnv: this->targetEnv, options: this->options, type: memRefType);
1758 });
1759
1760 // Register some last line of defense casting logic.
1761 addSourceMaterialization(
1762 callback: [this](OpBuilder &builder, Type type, ValueRange inputs, Location loc) {
1763 return castToSourceType(targetEnv: this->targetEnv, builder, type, inputs, loc);
1764 });
1765 addTargetMaterialization(callback: [](OpBuilder &builder, Type type, ValueRange inputs,
1766 Location loc) {
1767 auto cast = UnrealizedConversionCastOp::create(builder, location: loc, resultTypes: type, operands: inputs);
1768 return cast.getResult(i: 0);
1769 });
1770}
1771
1772Type SPIRVTypeConverter::getIndexType() const {
1773 return ::getIndexType(ctx: getContext(), options);
1774}
1775
1776MLIRContext *SPIRVTypeConverter::getContext() const {
1777 return targetEnv.getAttr().getContext();
1778}
1779
1780bool SPIRVTypeConverter::allows(spirv::Capability capability) const {
1781 return targetEnv.allows(capability);
1782}
1783
1784//===----------------------------------------------------------------------===//
1785// SPIR-V ConversionTarget
1786//===----------------------------------------------------------------------===//
1787
1788std::unique_ptr<SPIRVConversionTarget>
1789SPIRVConversionTarget::get(spirv::TargetEnvAttr targetAttr) {
1790 std::unique_ptr<SPIRVConversionTarget> target(
1791 // std::make_unique does not work here because the constructor is private.
1792 new SPIRVConversionTarget(targetAttr));
1793 SPIRVConversionTarget *targetPtr = target.get();
1794 target->addDynamicallyLegalDialect<spirv::SPIRVDialect>(
1795 // We need to capture the raw pointer here because it is stable:
1796 // target will be destroyed once this function is returned.
1797 callback: [targetPtr](Operation *op) { return targetPtr->isLegalOp(op); });
1798 return target;
1799}
1800
1801SPIRVConversionTarget::SPIRVConversionTarget(spirv::TargetEnvAttr targetAttr)
1802 : ConversionTarget(*targetAttr.getContext()), targetEnv(targetAttr) {}
1803
1804bool SPIRVConversionTarget::isLegalOp(Operation *op) {
1805 // Make sure this op is available at the given version. Ops not implementing
1806 // QueryMinVersionInterface/QueryMaxVersionInterface are available to all
1807 // SPIR-V versions.
1808 if (auto minVersionIfx = dyn_cast<spirv::QueryMinVersionInterface>(Val: op)) {
1809 std::optional<spirv::Version> minVersion = minVersionIfx.getMinVersion();
1810 if (minVersion && *minVersion > this->targetEnv.getVersion()) {
1811 LLVM_DEBUG(llvm::dbgs()
1812 << op->getName() << " illegal: requiring min version "
1813 << spirv::stringifyVersion(*minVersion) << "\n");
1814 return false;
1815 }
1816 }
1817 if (auto maxVersionIfx = dyn_cast<spirv::QueryMaxVersionInterface>(Val: op)) {
1818 std::optional<spirv::Version> maxVersion = maxVersionIfx.getMaxVersion();
1819 if (maxVersion && *maxVersion < this->targetEnv.getVersion()) {
1820 LLVM_DEBUG(llvm::dbgs()
1821 << op->getName() << " illegal: requiring max version "
1822 << spirv::stringifyVersion(*maxVersion) << "\n");
1823 return false;
1824 }
1825 }
1826
1827 // Make sure this op's required extensions are allowed to use. Ops not
1828 // implementing QueryExtensionInterface do not require extensions to be
1829 // available.
1830 if (auto extensions = dyn_cast<spirv::QueryExtensionInterface>(Val: op))
1831 if (failed(Result: checkExtensionRequirements(label: op->getName(), targetEnv: this->targetEnv,
1832 candidates: extensions.getExtensions())))
1833 return false;
1834
1835 // Make sure this op's required extensions are allowed to use. Ops not
1836 // implementing QueryCapabilityInterface do not require capabilities to be
1837 // available.
1838 if (auto capabilities = dyn_cast<spirv::QueryCapabilityInterface>(Val: op))
1839 if (failed(Result: checkCapabilityRequirements(label: op->getName(), targetEnv: this->targetEnv,
1840 candidates: capabilities.getCapabilities())))
1841 return false;
1842
1843 SmallVector<Type, 4> valueTypes;
1844 valueTypes.append(in_start: op->operand_type_begin(), in_end: op->operand_type_end());
1845 valueTypes.append(in_start: op->result_type_begin(), in_end: op->result_type_end());
1846
1847 // Ensure that all types have been converted to SPIRV types.
1848 if (llvm::any_of(Range&: valueTypes,
1849 P: [](Type t) { return !isa<spirv::SPIRVType>(Val: t); }))
1850 return false;
1851
1852 // Special treatment for global variables, whose type requirements are
1853 // conveyed by type attributes.
1854 if (auto globalVar = dyn_cast<spirv::GlobalVariableOp>(Val: op))
1855 valueTypes.push_back(Elt: globalVar.getType());
1856
1857 // Make sure the op's operands/results use types that are allowed by the
1858 // target environment.
1859 SmallVector<ArrayRef<spirv::Extension>, 4> typeExtensions;
1860 SmallVector<ArrayRef<spirv::Capability>, 8> typeCapabilities;
1861 for (Type valueType : valueTypes) {
1862 typeExtensions.clear();
1863 cast<spirv::SPIRVType>(Val&: valueType).getExtensions(extensions&: typeExtensions);
1864 if (failed(Result: checkExtensionRequirements(label: op->getName(), targetEnv: this->targetEnv,
1865 candidates: typeExtensions)))
1866 return false;
1867
1868 typeCapabilities.clear();
1869 cast<spirv::SPIRVType>(Val&: valueType).getCapabilities(capabilities&: typeCapabilities);
1870 if (failed(Result: checkCapabilityRequirements(label: op->getName(), targetEnv: this->targetEnv,
1871 candidates: typeCapabilities)))
1872 return false;
1873 }
1874
1875 return true;
1876}
1877
1878//===----------------------------------------------------------------------===//
1879// Public functions for populating patterns
1880//===----------------------------------------------------------------------===//
1881
1882void mlir::populateBuiltinFuncToSPIRVPatterns(
1883 const SPIRVTypeConverter &typeConverter, RewritePatternSet &patterns) {
1884 patterns.add<FuncOpConversion>(arg: typeConverter, args: patterns.getContext());
1885}
1886
1887void mlir::populateFuncOpVectorRewritePatterns(RewritePatternSet &patterns) {
1888 patterns.add<FuncOpVectorUnroll>(arg: patterns.getContext());
1889}
1890
1891void mlir::populateReturnOpVectorRewritePatterns(RewritePatternSet &patterns) {
1892 patterns.add<ReturnOpVectorUnroll>(arg: patterns.getContext());
1893}
1894