1//===- BuiltinTypes.cpp - MLIR Builtin Type Classes -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/IR/BuiltinTypes.h"
10#include "TypeDetail.h"
11#include "mlir/Dialect/Utils/StaticValueUtils.h"
12#include "mlir/IR/AffineExpr.h"
13#include "mlir/IR/AffineMap.h"
14#include "mlir/IR/BuiltinAttributes.h"
15#include "mlir/IR/BuiltinDialect.h"
16#include "mlir/IR/BuiltinTypeInterfaces.h"
17#include "mlir/IR/Diagnostics.h"
18#include "mlir/IR/Dialect.h"
19#include "mlir/IR/TensorEncoding.h"
20#include "mlir/IR/TypeUtilities.h"
21#include "llvm/ADT/APFixedPoint.h"
22#include "llvm/ADT/APFloat.h"
23#include "llvm/ADT/APInt.h"
24#include "llvm/ADT/Sequence.h"
25#include "llvm/ADT/TypeSwitch.h"
26#include "llvm/Support/CheckedArithmetic.h"
27#include <cstring>
28
29using namespace mlir;
30using namespace mlir::detail;
31
32//===----------------------------------------------------------------------===//
33/// Tablegen Type Definitions
34//===----------------------------------------------------------------------===//
35
36#define GET_TYPEDEF_CLASSES
37#include "mlir/IR/BuiltinTypes.cpp.inc"
38
39namespace mlir {
40#include "mlir/IR/BuiltinTypeConstraints.cpp.inc"
41} // namespace mlir
42
43//===----------------------------------------------------------------------===//
44// BuiltinDialect
45//===----------------------------------------------------------------------===//
46
47void BuiltinDialect::registerTypes() {
48 addTypes<
49#define GET_TYPEDEF_LIST
50#include "mlir/IR/BuiltinTypes.cpp.inc"
51 >();
52}
53
54//===----------------------------------------------------------------------===//
55/// ComplexType
56//===----------------------------------------------------------------------===//
57
58/// Verify the construction of an integer type.
59LogicalResult ComplexType::verify(function_ref<InFlightDiagnostic()> emitError,
60 Type elementType) {
61 if (!elementType.isIntOrFloat())
62 return emitError() << "invalid element type for complex";
63 return success();
64}
65
66size_t ComplexType::getDenseElementBitSize() const {
67 auto elemTy = cast<DenseElementType>(Val: getElementType());
68 return llvm::alignTo<8>(Value: elemTy.getDenseElementBitSize()) * 2;
69}
70
71Attribute ComplexType::convertToAttribute(ArrayRef<char> rawData) const {
72 auto elemTy = cast<DenseElementType>(Val: getElementType());
73 size_t singleElementBytes =
74 llvm::alignTo<8>(Value: elemTy.getDenseElementBitSize()) / 8;
75 Attribute real =
76 elemTy.convertToAttribute(rawData: rawData.take_front(N: singleElementBytes));
77 Attribute imag =
78 elemTy.convertToAttribute(rawData: rawData.take_back(N: singleElementBytes));
79 return ArrayAttr::get(context: getContext(), value: {real, imag});
80}
81
82LogicalResult
83ComplexType::convertFromAttribute(Attribute attr,
84 SmallVectorImpl<char> &result) const {
85 auto arrayAttr = dyn_cast<ArrayAttr>(Val&: attr);
86 if (!arrayAttr || arrayAttr.size() != 2)
87 return failure();
88 auto elemTy = cast<DenseElementType>(Val: getElementType());
89 SmallVector<char> realData, imagData;
90 if (failed(Result: elemTy.convertFromAttribute(attr: arrayAttr[0], result&: realData)))
91 return failure();
92 if (failed(Result: elemTy.convertFromAttribute(attr: arrayAttr[1], result&: imagData)))
93 return failure();
94 result.append(RHS: realData);
95 result.append(RHS: imagData);
96 return success();
97}
98
99//===----------------------------------------------------------------------===//
100// Integer Type
101//===----------------------------------------------------------------------===//
102
103/// Verify the construction of an integer type.
104LogicalResult IntegerType::verify(function_ref<InFlightDiagnostic()> emitError,
105 unsigned width,
106 SignednessSemantics signedness) {
107 if (width > IntegerType::kMaxWidth) {
108 return emitError() << "integer bitwidth is limited to "
109 << IntegerType::kMaxWidth << " bits";
110 }
111 return success();
112}
113
114unsigned IntegerType::getWidth() const { return getImpl()->width; }
115
116IntegerType::SignednessSemantics IntegerType::getSignedness() const {
117 return getImpl()->signedness;
118}
119
120IntegerType IntegerType::scaleElementBitwidth(unsigned scale) {
121 if (!scale)
122 return IntegerType();
123 return IntegerType::get(context: getContext(), width: scale * getWidth(), signedness: getSignedness());
124}
125
126size_t IntegerType::getDenseElementBitSize() const {
127 // Return the actual bit width. Storage alignment is handled separately.
128 return getWidth();
129}
130
131Attribute IntegerType::convertToAttribute(ArrayRef<char> rawData) const {
132 APInt value = detail::readBits(rawData: rawData.data(), /*bitPos=*/0, bitWidth: getWidth());
133 return IntegerAttr::get(type: *this, value);
134}
135
136static void writeAPIntToVector(APInt apInt, SmallVectorImpl<char> &result) {
137 size_t byteSize = llvm::divideCeil(Numerator: apInt.getBitWidth(), CHAR_BIT);
138 size_t bitPos = result.size() * CHAR_BIT;
139 result.resize(N: result.size() + byteSize);
140 detail::writeBits(rawData: result.data(), bitPos, value: apInt);
141}
142
143LogicalResult
144IntegerType::convertFromAttribute(Attribute attr,
145 SmallVectorImpl<char> &result) const {
146 auto intAttr = dyn_cast<IntegerAttr>(Val&: attr);
147 if (!intAttr || intAttr.getType() != *this)
148 return failure();
149 writeAPIntToVector(apInt: intAttr.getValue(), result);
150 return success();
151}
152
153//===----------------------------------------------------------------------===//
154// MXI8 Type
155//===----------------------------------------------------------------------===//
156
157llvm::FixedPointSemantics MXI8Type::getFixedPointSemantics() {
158 // Saturation is selected by a conversion operation, not by the value type.
159 return {/*Width=*/8, /*Scale=*/6, /*IsSigned=*/true,
160 /*IsSaturated=*/false, /*HasUnsignedPadding=*/false};
161}
162
163size_t MXI8Type::getDenseElementBitSize() const { return 8; }
164
165Attribute MXI8Type::convertToAttribute(ArrayRef<char> rawData) const {
166 return IntegerType::get(context: getContext(), width: 8).convertToAttribute(rawData);
167}
168
169LogicalResult
170MXI8Type::convertFromAttribute(Attribute attr,
171 SmallVectorImpl<char> &result) const {
172 auto intAttr = dyn_cast<IntegerAttr>(Val&: attr);
173 if (!intAttr || !intAttr.getType().isSignlessInteger(width: 8))
174 return failure();
175 return cast<IntegerType>(Val: intAttr.getType())
176 .convertFromAttribute(attr, result);
177}
178
179//===----------------------------------------------------------------------===//
180// Index Type
181//===----------------------------------------------------------------------===//
182
183size_t IndexType::getDenseElementBitSize() const {
184 return kInternalStorageBitWidth;
185}
186
187Attribute IndexType::convertToAttribute(ArrayRef<char> rawData) const {
188 APInt value =
189 detail::readBits(rawData: rawData.data(), /*bitPos=*/0, bitWidth: kInternalStorageBitWidth);
190 return IntegerAttr::get(type: *this, value);
191}
192
193LogicalResult
194IndexType::convertFromAttribute(Attribute attr,
195 SmallVectorImpl<char> &result) const {
196 auto intAttr = dyn_cast<IntegerAttr>(Val&: attr);
197 if (!intAttr || intAttr.getType() != *this)
198 return failure();
199 writeAPIntToVector(apInt: intAttr.getValue(), result);
200 return success();
201}
202
203//===----------------------------------------------------------------------===//
204// Float Types
205//===----------------------------------------------------------------------===//
206
207// Mapping from MLIR FloatType to APFloat semantics.
208#define FLOAT_TYPE_SEMANTICS(TYPE, SEM) \
209 const llvm::fltSemantics &TYPE::getFloatSemantics() const { \
210 return APFloat::SEM(); \
211 }
212FLOAT_TYPE_SEMANTICS(Float4E2M1FNType, Float4E2M1FN)
213FLOAT_TYPE_SEMANTICS(Float6E2M3FNType, Float6E2M3FN)
214FLOAT_TYPE_SEMANTICS(Float6E3M2FNType, Float6E3M2FN)
215FLOAT_TYPE_SEMANTICS(Float8E5M2Type, Float8E5M2)
216FLOAT_TYPE_SEMANTICS(Float8E4M3Type, Float8E4M3)
217FLOAT_TYPE_SEMANTICS(Float8E4M3FNType, Float8E4M3FN)
218FLOAT_TYPE_SEMANTICS(Float8E5M2FNUZType, Float8E5M2FNUZ)
219FLOAT_TYPE_SEMANTICS(Float8E4M3FNUZType, Float8E4M3FNUZ)
220FLOAT_TYPE_SEMANTICS(Float8E4M3B11FNUZType, Float8E4M3B11FNUZ)
221FLOAT_TYPE_SEMANTICS(Float8E3M4Type, Float8E3M4)
222FLOAT_TYPE_SEMANTICS(Float8E8M0FNUType, Float8E8M0FNU)
223FLOAT_TYPE_SEMANTICS(Float8E5M3FNUType, Float8E5M3FNU)
224FLOAT_TYPE_SEMANTICS(BFloat16Type, BFloat)
225FLOAT_TYPE_SEMANTICS(Float16Type, IEEEhalf)
226FLOAT_TYPE_SEMANTICS(FloatTF32Type, FloatTF32)
227FLOAT_TYPE_SEMANTICS(Float32Type, IEEEsingle)
228FLOAT_TYPE_SEMANTICS(Float64Type, IEEEdouble)
229FLOAT_TYPE_SEMANTICS(Float80Type, x87DoubleExtended)
230FLOAT_TYPE_SEMANTICS(Float128Type, IEEEquad)
231#undef FLOAT_TYPE_SEMANTICS
232
233FloatType Float16Type::scaleElementBitwidth(unsigned scale) const {
234 if (scale == 2)
235 return Float32Type::get(context: getContext());
236 if (scale == 4)
237 return Float64Type::get(context: getContext());
238 return FloatType();
239}
240
241FloatType BFloat16Type::scaleElementBitwidth(unsigned scale) const {
242 if (scale == 2)
243 return Float32Type::get(context: getContext());
244 if (scale == 4)
245 return Float64Type::get(context: getContext());
246 return FloatType();
247}
248
249FloatType Float32Type::scaleElementBitwidth(unsigned scale) const {
250 if (scale == 2)
251 return Float64Type::get(context: getContext());
252 return FloatType();
253}
254
255//===----------------------------------------------------------------------===//
256// FunctionType
257//===----------------------------------------------------------------------===//
258
259unsigned FunctionType::getNumInputs() const { return getImpl()->numInputs; }
260
261ArrayRef<Type> FunctionType::getInputs() const {
262 return getImpl()->getInputs();
263}
264
265unsigned FunctionType::getNumResults() const { return getImpl()->numResults; }
266
267ArrayRef<Type> FunctionType::getResults() const {
268 return getImpl()->getResults();
269}
270
271FunctionType FunctionType::clone(TypeRange inputs, TypeRange results) const {
272 return get(context: getContext(), inputs, results);
273}
274
275/// Returns a new function type with the specified arguments and results
276/// inserted.
277FunctionType FunctionType::getWithArgsAndResults(
278 ArrayRef<unsigned> argIndices, TypeRange argTypes,
279 ArrayRef<unsigned> resultIndices, TypeRange resultTypes) {
280 SmallVector<Type> argStorage, resultStorage;
281 TypeRange newArgTypes =
282 insertTypesInto(oldTypes: getInputs(), indices: argIndices, newTypes: argTypes, storage&: argStorage);
283 TypeRange newResultTypes =
284 insertTypesInto(oldTypes: getResults(), indices: resultIndices, newTypes: resultTypes, storage&: resultStorage);
285 return clone(inputs: newArgTypes, results: newResultTypes);
286}
287
288/// Returns a new function type without the specified arguments and results.
289FunctionType
290FunctionType::getWithoutArgsAndResults(const BitVector &argIndices,
291 const BitVector &resultIndices) {
292 SmallVector<Type> argStorage, resultStorage;
293 TypeRange newArgTypes = filterTypesOut(types: getInputs(), indices: argIndices, storage&: argStorage);
294 TypeRange newResultTypes =
295 filterTypesOut(types: getResults(), indices: resultIndices, storage&: resultStorage);
296 return clone(inputs: newArgTypes, results: newResultTypes);
297}
298
299//===----------------------------------------------------------------------===//
300// GraphType
301//===----------------------------------------------------------------------===//
302
303unsigned GraphType::getNumInputs() const { return getImpl()->numInputs; }
304
305ArrayRef<Type> GraphType::getInputs() const { return getImpl()->getInputs(); }
306
307unsigned GraphType::getNumResults() const { return getImpl()->numResults; }
308
309ArrayRef<Type> GraphType::getResults() const { return getImpl()->getResults(); }
310
311GraphType GraphType::clone(TypeRange inputs, TypeRange results) const {
312 return get(context: getContext(), inputs, results);
313}
314
315/// Returns a new function type with the specified arguments and results
316/// inserted.
317GraphType GraphType::getWithArgsAndResults(ArrayRef<unsigned> argIndices,
318 TypeRange argTypes,
319 ArrayRef<unsigned> resultIndices,
320 TypeRange resultTypes) {
321 SmallVector<Type> argStorage, resultStorage;
322 TypeRange newArgTypes =
323 insertTypesInto(oldTypes: getInputs(), indices: argIndices, newTypes: argTypes, storage&: argStorage);
324 TypeRange newResultTypes =
325 insertTypesInto(oldTypes: getResults(), indices: resultIndices, newTypes: resultTypes, storage&: resultStorage);
326 return clone(inputs: newArgTypes, results: newResultTypes);
327}
328
329/// Returns a new function type without the specified arguments and results.
330GraphType GraphType::getWithoutArgsAndResults(const BitVector &argIndices,
331 const BitVector &resultIndices) {
332 SmallVector<Type> argStorage, resultStorage;
333 TypeRange newArgTypes = filterTypesOut(types: getInputs(), indices: argIndices, storage&: argStorage);
334 TypeRange newResultTypes =
335 filterTypesOut(types: getResults(), indices: resultIndices, storage&: resultStorage);
336 return clone(inputs: newArgTypes, results: newResultTypes);
337}
338//===----------------------------------------------------------------------===//
339// OpaqueType
340//===----------------------------------------------------------------------===//
341
342/// Verify the construction of an opaque type.
343LogicalResult OpaqueType::verify(function_ref<InFlightDiagnostic()> emitError,
344 StringAttr dialect, StringRef typeData) {
345 if (!Dialect::isValidNamespace(str: dialect.strref()))
346 return emitError() << "invalid dialect namespace '" << dialect << "'";
347
348 // Check that the dialect is actually registered.
349 MLIRContext *context = dialect.getContext();
350 if (!context->allowsUnregisteredDialects() &&
351 !context->getLoadedDialect(name: dialect.strref())) {
352 return emitError()
353 << "`!" << dialect << "<\"" << typeData << "\">"
354 << "` type created with unregistered dialect. If this is "
355 "intended, please call allowUnregisteredDialects() on the "
356 "MLIRContext, or use -allow-unregistered-dialect with "
357 "the MLIR opt tool used";
358 }
359
360 return success();
361}
362
363//===----------------------------------------------------------------------===//
364// VectorType
365//===----------------------------------------------------------------------===//
366
367bool VectorType::isValidElementType(Type t) {
368 return isValidVectorTypeElementType(type: t);
369}
370
371LogicalResult VectorType::verify(function_ref<InFlightDiagnostic()> emitError,
372 ArrayRef<int64_t> shape, Type elementType,
373 ArrayRef<bool> scalableDims) {
374 if (!isValidElementType(t: elementType))
375 return emitError()
376 << "vector elements must be int/index/float type but got "
377 << elementType;
378
379 if (any_of(Range&: shape, P: [](int64_t i) { return i <= 0; }))
380 return emitError()
381 << "vector types must have positive constant sizes but got "
382 << shape;
383
384 if (scalableDims.size() != shape.size())
385 return emitError() << "number of dims must match, got "
386 << scalableDims.size() << " and " << shape.size();
387
388 return success();
389}
390
391VectorType VectorType::scaleElementBitwidth(unsigned scale) {
392 if (!scale)
393 return VectorType();
394 if (auto et = llvm::dyn_cast<IntegerType>(Val: getElementType()))
395 if (auto scaledEt = et.scaleElementBitwidth(scale))
396 return VectorType::get(shape: getShape(), elementType: scaledEt, scalableDims: getScalableDims());
397 if (auto et = llvm::dyn_cast<FloatType>(Val: getElementType()))
398 if (auto scaledEt = et.scaleElementBitwidth(scale))
399 return VectorType::get(shape: getShape(), elementType: scaledEt, scalableDims: getScalableDims());
400 return VectorType();
401}
402
403VectorType VectorType::cloneWith(std::optional<ArrayRef<int64_t>> shape,
404 Type elementType) const {
405 return VectorType::get(shape: shape.value_or(u: getShape()), elementType,
406 scalableDims: getScalableDims());
407}
408
409//===----------------------------------------------------------------------===//
410// TensorType
411//===----------------------------------------------------------------------===//
412
413Type TensorType::getElementType() const {
414 return llvm::TypeSwitch<TensorType, Type>(*this)
415 .Case<RankedTensorType, UnrankedTensorType>(
416 caseFn: [](auto type) { return type.getElementType(); });
417}
418
419bool TensorType::hasRank() const {
420 return !llvm::isa<UnrankedTensorType>(Val: *this);
421}
422
423ArrayRef<int64_t> TensorType::getShape() const {
424 return llvm::cast<RankedTensorType>(Val: *this).getShape();
425}
426
427TensorType TensorType::cloneWith(std::optional<ArrayRef<int64_t>> shape,
428 Type elementType) const {
429 if (llvm::dyn_cast<UnrankedTensorType>(Val: *this)) {
430 if (shape)
431 return RankedTensorType::get(shape: *shape, elementType);
432 return UnrankedTensorType::get(elementType);
433 }
434
435 auto rankedTy = llvm::cast<RankedTensorType>(Val: *this);
436 if (!shape)
437 return RankedTensorType::get(shape: rankedTy.getShape(), elementType,
438 encoding: rankedTy.getEncoding());
439 return RankedTensorType::get(shape: shape.value_or(u: rankedTy.getShape()), elementType,
440 encoding: rankedTy.getEncoding());
441}
442
443RankedTensorType TensorType::clone(::llvm::ArrayRef<int64_t> shape,
444 Type elementType) const {
445 return ::llvm::cast<RankedTensorType>(Val: cloneWith(shape, elementType));
446}
447
448RankedTensorType TensorType::clone(::llvm::ArrayRef<int64_t> shape) const {
449 return ::llvm::cast<RankedTensorType>(Val: cloneWith(shape, elementType: getElementType()));
450}
451
452// Check if "elementType" can be an element type of a tensor.
453static LogicalResult
454checkTensorElementType(function_ref<InFlightDiagnostic()> emitError,
455 Type elementType) {
456 if (!TensorType::isValidElementType(type: elementType))
457 return emitError() << "invalid tensor element type: " << elementType;
458 return success();
459}
460
461/// Return true if the specified element type is ok in a tensor.
462bool TensorType::isValidElementType(Type type) {
463 // Note: Non standard/builtin types are allowed to exist within tensor
464 // types. Dialects are expected to verify that tensor types have a valid
465 // element type within that dialect.
466 return llvm::isa<ComplexType, FloatType, IntegerType, OpaqueType, VectorType,
467 IndexType, MXI8Type>(Val: type) ||
468 !llvm::isa<BuiltinDialect>(Val: type.getDialect());
469}
470
471//===----------------------------------------------------------------------===//
472// RankedTensorType
473//===----------------------------------------------------------------------===//
474
475LogicalResult
476RankedTensorType::verify(function_ref<InFlightDiagnostic()> emitError,
477 ArrayRef<int64_t> shape, Type elementType,
478 Attribute encoding) {
479 for (int64_t s : shape)
480 if (s < 0 && ShapedType::isStatic(dValue: s))
481 return emitError() << "invalid tensor dimension size";
482 if (auto v = llvm::dyn_cast_or_null<VerifiableTensorEncoding>(Val&: encoding))
483 if (failed(Result: v.verifyEncoding(shape, elementType, emitError)))
484 return failure();
485 return checkTensorElementType(emitError, elementType);
486}
487
488//===----------------------------------------------------------------------===//
489// UnrankedTensorType
490//===----------------------------------------------------------------------===//
491
492LogicalResult
493UnrankedTensorType::verify(function_ref<InFlightDiagnostic()> emitError,
494 Type elementType) {
495 return checkTensorElementType(emitError, elementType);
496}
497
498//===----------------------------------------------------------------------===//
499// BaseMemRefType
500//===----------------------------------------------------------------------===//
501
502Type BaseMemRefType::getElementType() const {
503 return llvm::TypeSwitch<BaseMemRefType, Type>(*this)
504 .Case<MemRefType, UnrankedMemRefType>(
505 caseFn: [](auto type) { return type.getElementType(); });
506}
507
508bool BaseMemRefType::hasRank() const {
509 return !llvm::isa<UnrankedMemRefType>(Val: *this);
510}
511
512ArrayRef<int64_t> BaseMemRefType::getShape() const {
513 return llvm::cast<MemRefType>(Val: *this).getShape();
514}
515
516BaseMemRefType BaseMemRefType::cloneWith(std::optional<ArrayRef<int64_t>> shape,
517 Type elementType) const {
518 if (llvm::dyn_cast<UnrankedMemRefType>(Val: *this)) {
519 if (!shape)
520 return UnrankedMemRefType::get(elementType, memorySpace: getMemorySpace());
521 MemRefType::Builder builder(*shape, elementType);
522 builder.setMemorySpace(getMemorySpace());
523 return builder;
524 }
525
526 MemRefType::Builder builder(llvm::cast<MemRefType>(Val: *this));
527 if (shape)
528 builder.setShape(*shape);
529 builder.setElementType(elementType);
530 return builder;
531}
532
533FailureOr<PtrLikeTypeInterface>
534BaseMemRefType::clonePtrWith(Attribute memorySpace,
535 std::optional<Type> elementType) const {
536 Type eTy = elementType ? *elementType : getElementType();
537 if (llvm::dyn_cast<UnrankedMemRefType>(Val: *this))
538 return cast<PtrLikeTypeInterface>(
539 Val: UnrankedMemRefType::get(elementType: eTy, memorySpace));
540
541 MemRefType::Builder builder(llvm::cast<MemRefType>(Val: *this));
542 builder.setElementType(eTy);
543 builder.setMemorySpace(memorySpace);
544 return cast<PtrLikeTypeInterface>(Val: static_cast<MemRefType>(builder));
545}
546
547MemRefType BaseMemRefType::clone(::llvm::ArrayRef<int64_t> shape,
548 Type elementType) const {
549 return ::llvm::cast<MemRefType>(Val: cloneWith(shape, elementType));
550}
551
552MemRefType BaseMemRefType::clone(::llvm::ArrayRef<int64_t> shape) const {
553 return ::llvm::cast<MemRefType>(Val: cloneWith(shape, elementType: getElementType()));
554}
555
556Attribute BaseMemRefType::getMemorySpace() const {
557 if (auto rankedMemRefTy = llvm::dyn_cast<MemRefType>(Val: *this))
558 return rankedMemRefTy.getMemorySpace();
559 return llvm::cast<UnrankedMemRefType>(Val: *this).getMemorySpace();
560}
561
562unsigned BaseMemRefType::getMemorySpaceAsInt() const {
563 if (auto rankedMemRefTy = llvm::dyn_cast<MemRefType>(Val: *this))
564 return rankedMemRefTy.getMemorySpaceAsInt();
565 return llvm::cast<UnrankedMemRefType>(Val: *this).getMemorySpaceAsInt();
566}
567
568//===----------------------------------------------------------------------===//
569// MemRefType
570//===----------------------------------------------------------------------===//
571
572std::optional<llvm::SmallDenseSet<unsigned>>
573mlir::computeRankReductionMask(ArrayRef<int64_t> originalShape,
574 ArrayRef<int64_t> reducedShape,
575 bool matchDynamic) {
576 size_t originalRank = originalShape.size(), reducedRank = reducedShape.size();
577 llvm::SmallDenseSet<unsigned> unusedDims;
578 unsigned reducedIdx = 0;
579 for (unsigned originalIdx = 0; originalIdx < originalRank; ++originalIdx) {
580 // Greedily insert `originalIdx` if match.
581 int64_t origSize = originalShape[originalIdx];
582 // if `matchDynamic`, count dynamic dims as a match, unless `origSize` is 1.
583 if (matchDynamic && reducedIdx < reducedRank && origSize != 1 &&
584 (ShapedType::isDynamic(dValue: reducedShape[reducedIdx]) ||
585 ShapedType::isDynamic(dValue: origSize))) {
586 reducedIdx++;
587 continue;
588 }
589 if (reducedIdx < reducedRank && origSize == reducedShape[reducedIdx]) {
590 reducedIdx++;
591 continue;
592 }
593
594 unusedDims.insert(V: originalIdx);
595 // If no match on `originalIdx`, the `originalShape` at this dimension
596 // must be 1, otherwise we bail.
597 if (origSize != 1)
598 return std::nullopt;
599 }
600 // The whole reducedShape must be scanned, otherwise we bail.
601 if (reducedIdx != reducedRank)
602 return std::nullopt;
603 return unusedDims;
604}
605
606SliceVerificationResult
607mlir::isRankReducedType(ShapedType originalType,
608 ShapedType candidateReducedType) {
609 if (originalType == candidateReducedType)
610 return SliceVerificationResult::Success;
611
612 ShapedType originalShapedType = llvm::cast<ShapedType>(Val&: originalType);
613 ShapedType candidateReducedShapedType =
614 llvm::cast<ShapedType>(Val&: candidateReducedType);
615
616 // Rank and size logic is valid for all ShapedTypes.
617 ArrayRef<int64_t> originalShape = originalShapedType.getShape();
618 ArrayRef<int64_t> candidateReducedShape =
619 candidateReducedShapedType.getShape();
620 unsigned originalRank = originalShape.size(),
621 candidateReducedRank = candidateReducedShape.size();
622 if (candidateReducedRank > originalRank)
623 return SliceVerificationResult::RankTooLarge;
624
625 auto optionalUnusedDimsMask =
626 computeRankReductionMask(originalShape, reducedShape: candidateReducedShape);
627
628 // Sizes cannot be matched in case empty vector is returned.
629 if (!optionalUnusedDimsMask)
630 return SliceVerificationResult::SizeMismatch;
631
632 if (originalShapedType.getElementType() !=
633 candidateReducedShapedType.getElementType())
634 return SliceVerificationResult::ElemTypeMismatch;
635
636 return SliceVerificationResult::Success;
637}
638
639Attribute mlir::detail::wrapIntegerMemorySpace(unsigned memorySpace,
640 MLIRContext *ctx) {
641 if (memorySpace == 0)
642 return nullptr;
643
644 return IntegerAttr::get(type: IntegerType::get(context: ctx, width: 64), value: memorySpace);
645}
646
647Attribute mlir::detail::skipDefaultMemorySpace(Attribute memorySpace) {
648 IntegerAttr intMemorySpace = llvm::dyn_cast_or_null<IntegerAttr>(Val&: memorySpace);
649 if (intMemorySpace && intMemorySpace.getValue() == 0)
650 return nullptr;
651
652 return memorySpace;
653}
654
655unsigned mlir::detail::getMemorySpaceAsInt(Attribute memorySpace) {
656 if (!memorySpace)
657 return 0;
658
659 assert(llvm::isa<IntegerAttr>(memorySpace) &&
660 "Using `getMemorySpaceInteger` with non-Integer attribute");
661
662 return static_cast<unsigned>(llvm::cast<IntegerAttr>(Val&: memorySpace).getInt());
663}
664
665unsigned MemRefType::getMemorySpaceAsInt() const {
666 return detail::getMemorySpaceAsInt(memorySpace: getMemorySpace());
667}
668
669MemRefType MemRefType::get(ArrayRef<int64_t> shape, Type elementType,
670 MemRefLayoutAttrInterface layout,
671 Attribute memorySpace) {
672 // Use default layout for empty attribute.
673 if (!layout)
674 layout = AffineMapAttr::get(value: AffineMap::getMultiDimIdentityMap(
675 numDims: shape.size(), context: elementType.getContext()));
676
677 // Drop default memory space value and replace it with empty attribute.
678 memorySpace = skipDefaultMemorySpace(memorySpace);
679
680 return Base::get(ctx: elementType.getContext(), args&: shape, args&: elementType, args&: layout,
681 args&: memorySpace);
682}
683
684MemRefType MemRefType::getChecked(
685 function_ref<InFlightDiagnostic()> emitErrorFn, ArrayRef<int64_t> shape,
686 Type elementType, MemRefLayoutAttrInterface layout, Attribute memorySpace) {
687
688 // Use default layout for empty attribute.
689 if (!layout)
690 layout = AffineMapAttr::get(value: AffineMap::getMultiDimIdentityMap(
691 numDims: shape.size(), context: elementType.getContext()));
692
693 // Drop default memory space value and replace it with empty attribute.
694 memorySpace = skipDefaultMemorySpace(memorySpace);
695
696 return Base::getChecked(emitErrorFn, ctx: elementType.getContext(), args: shape,
697 args: elementType, args: layout, args: memorySpace);
698}
699
700MemRefType MemRefType::get(ArrayRef<int64_t> shape, Type elementType,
701 AffineMap map, Attribute memorySpace) {
702
703 // Use default layout for empty map.
704 if (!map)
705 map = AffineMap::getMultiDimIdentityMap(numDims: shape.size(),
706 context: elementType.getContext());
707
708 // Wrap AffineMap into Attribute.
709 auto layout = AffineMapAttr::get(value: map);
710
711 // Drop default memory space value and replace it with empty attribute.
712 memorySpace = skipDefaultMemorySpace(memorySpace);
713
714 return Base::get(ctx: elementType.getContext(), args&: shape, args&: elementType, args&: layout,
715 args&: memorySpace);
716}
717
718MemRefType
719MemRefType::getChecked(function_ref<InFlightDiagnostic()> emitErrorFn,
720 ArrayRef<int64_t> shape, Type elementType, AffineMap map,
721 Attribute memorySpace) {
722
723 // Use default layout for empty map.
724 if (!map)
725 map = AffineMap::getMultiDimIdentityMap(numDims: shape.size(),
726 context: elementType.getContext());
727
728 // Wrap AffineMap into Attribute.
729 auto layout = AffineMapAttr::get(value: map);
730
731 // Drop default memory space value and replace it with empty attribute.
732 memorySpace = skipDefaultMemorySpace(memorySpace);
733
734 return Base::getChecked(emitErrorFn, ctx: elementType.getContext(), args: shape,
735 args: elementType, args: layout, args: memorySpace);
736}
737
738MemRefType MemRefType::get(ArrayRef<int64_t> shape, Type elementType,
739 AffineMap map, unsigned memorySpaceInd) {
740
741 // Use default layout for empty map.
742 if (!map)
743 map = AffineMap::getMultiDimIdentityMap(numDims: shape.size(),
744 context: elementType.getContext());
745
746 // Wrap AffineMap into Attribute.
747 auto layout = AffineMapAttr::get(value: map);
748
749 // Convert deprecated integer-like memory space to Attribute.
750 Attribute memorySpace =
751 wrapIntegerMemorySpace(memorySpace: memorySpaceInd, ctx: elementType.getContext());
752
753 return Base::get(ctx: elementType.getContext(), args&: shape, args&: elementType, args&: layout,
754 args&: memorySpace);
755}
756
757MemRefType
758MemRefType::getChecked(function_ref<InFlightDiagnostic()> emitErrorFn,
759 ArrayRef<int64_t> shape, Type elementType, AffineMap map,
760 unsigned memorySpaceInd) {
761
762 // Use default layout for empty map.
763 if (!map)
764 map = AffineMap::getMultiDimIdentityMap(numDims: shape.size(),
765 context: elementType.getContext());
766
767 // Wrap AffineMap into Attribute.
768 auto layout = AffineMapAttr::get(value: map);
769
770 // Convert deprecated integer-like memory space to Attribute.
771 Attribute memorySpace =
772 wrapIntegerMemorySpace(memorySpace: memorySpaceInd, ctx: elementType.getContext());
773
774 return Base::getChecked(emitErrorFn, ctx: elementType.getContext(), args: shape,
775 args: elementType, args: layout, args: memorySpace);
776}
777
778LogicalResult MemRefType::verify(function_ref<InFlightDiagnostic()> emitError,
779 ArrayRef<int64_t> shape, Type elementType,
780 MemRefLayoutAttrInterface layout,
781 Attribute memorySpace) {
782 if (!BaseMemRefType::isValidElementType(type: elementType))
783 return emitError() << "invalid memref element type";
784
785 // Negative sizes are not allowed except for `kDynamic`.
786 for (int64_t s : shape)
787 if (s < 0 && ShapedType::isStatic(dValue: s))
788 return emitError() << "invalid memref size";
789
790 assert(layout && "missing layout specification");
791 if (failed(Result: layout.verifyLayout(shape, emitError)))
792 return failure();
793
794 return success();
795}
796
797bool MemRefType::areTrailingDimsContiguous(int64_t n) {
798 assert(n <= getRank() &&
799 "number of dimensions to check must not exceed rank");
800 return n <= getNumContiguousTrailingDims();
801}
802
803int64_t MemRefType::getNumContiguousTrailingDims() {
804 const int64_t n = getRank();
805
806 // memrefs with identity layout are entirely contiguous.
807 if (getLayout().isIdentity())
808 return n;
809
810 // Get the strides (if any). Failing to do that, conservatively assume a
811 // non-contiguous layout.
812 int64_t offset;
813 SmallVector<int64_t> strides;
814 if (!succeeded(Result: getStridesAndOffset(strides, offset)))
815 return 0;
816
817 ArrayRef<int64_t> shape = getShape();
818
819 // A memref with dimensions `d0, d1, ..., dn-1` and strides
820 // `s0, s1, ..., sn-1` is contiguous up to dimension `k`
821 // if each stride `si` is the product of the dimensions `di+1, ..., dn-1`,
822 // for `i` in `[k, n-1]`.
823 // Ignore stride elements if the corresponding dimension is 1, as they are
824 // of no consequence.
825 int64_t dimProduct = 1;
826 for (int64_t i = n - 1; i >= 0; --i) {
827 if (shape[i] == 1)
828 continue;
829 if (strides[i] != dimProduct)
830 return n - i - 1;
831 if (shape[i] == ShapedType::kDynamic)
832 return n - i;
833 dimProduct *= shape[i];
834 }
835
836 return n;
837}
838
839MemRefType MemRefType::canonicalizeStridedLayout() {
840 AffineMap m = getLayout().getAffineMap();
841
842 // Already in canonical form.
843 if (m.isIdentity())
844 return *this;
845
846 // Can't reduce to canonical identity form, return in canonical form.
847 if (m.getNumResults() > 1)
848 return *this;
849
850 // Corner-case for 0-D affine maps.
851 if (m.getNumDims() == 0 && m.getNumSymbols() == 0) {
852 if (auto cst = llvm::dyn_cast<AffineConstantExpr>(Val: m.getResult(idx: 0)))
853 if (cst.getValue() == 0)
854 return MemRefType::Builder(*this).setLayout({});
855 return *this;
856 }
857
858 // 0-D corner case for empty shape that still have an affine map. Example:
859 // `memref<f32, affine_map<()[s0] -> (s0)>>`. This is a 1 element memref whose
860 // offset needs to remain, just return t.
861 if (getShape().empty())
862 return *this;
863
864 // If the canonical strided layout for the sizes of `t` is equal to the
865 // simplified layout of `t` we can just return an empty layout. Otherwise,
866 // just simplify the existing layout.
867 AffineExpr expr = makeCanonicalStridedLayoutExpr(sizes: getShape(), context: getContext());
868 auto simplifiedLayoutExpr =
869 simplifyAffineExpr(expr: m.getResult(idx: 0), numDims: m.getNumDims(), numSymbols: m.getNumSymbols());
870 if (expr != simplifiedLayoutExpr)
871 return MemRefType::Builder(*this).setLayout(
872 AffineMapAttr::get(value: AffineMap::get(dimCount: m.getNumDims(), symbolCount: m.getNumSymbols(),
873 result: simplifiedLayoutExpr)));
874 return MemRefType::Builder(*this).setLayout({});
875}
876
877LogicalResult MemRefType::getStridesAndOffset(SmallVectorImpl<int64_t> &strides,
878 int64_t &offset) const {
879 return getLayout().getStridesAndOffset(shape: getShape(), strides, offset);
880}
881
882std::pair<SmallVector<int64_t>, int64_t>
883MemRefType::getStridesAndOffset() const {
884 SmallVector<int64_t> strides;
885 int64_t offset;
886 LogicalResult status = getStridesAndOffset(strides, offset);
887 (void)status;
888 assert(succeeded(status) && "Invalid use of check-free getStridesAndOffset");
889 return {strides, offset};
890}
891
892bool MemRefType::isStrided() {
893 int64_t offset;
894 SmallVector<int64_t, 4> strides;
895 auto res = getStridesAndOffset(strides, offset);
896 return succeeded(Result: res);
897}
898
899bool MemRefType::isLastDimUnitStride() {
900 int64_t offset;
901 SmallVector<int64_t> strides;
902 auto successStrides = getStridesAndOffset(strides, offset);
903 return succeeded(Result: successStrides) && (strides.empty() || strides.back() == 1);
904}
905
906//===----------------------------------------------------------------------===//
907// UnrankedMemRefType
908//===----------------------------------------------------------------------===//
909
910unsigned UnrankedMemRefType::getMemorySpaceAsInt() const {
911 return detail::getMemorySpaceAsInt(memorySpace: getMemorySpace());
912}
913
914LogicalResult
915UnrankedMemRefType::verify(function_ref<InFlightDiagnostic()> emitError,
916 Type elementType, Attribute memorySpace) {
917 if (!BaseMemRefType::isValidElementType(type: elementType))
918 return emitError() << "invalid memref element type";
919
920 return success();
921}
922
923//===----------------------------------------------------------------------===//
924/// TupleType
925//===----------------------------------------------------------------------===//
926
927/// Return the elements types for this tuple.
928ArrayRef<Type> TupleType::getTypes() const { return getImpl()->getTypes(); }
929
930/// Accumulate the types contained in this tuple and tuples nested within it.
931/// Note that this only flattens nested tuples, not any other container type,
932/// e.g. a tuple<i32, tensor<i32>, tuple<f32, tuple<i64>>> is flattened to
933/// (i32, tensor<i32>, f32, i64)
934void TupleType::getFlattenedTypes(SmallVectorImpl<Type> &types) {
935 for (Type type : getTypes()) {
936 if (auto nestedTuple = llvm::dyn_cast<TupleType>(Val&: type))
937 nestedTuple.getFlattenedTypes(types);
938 else
939 types.push_back(Elt: type);
940 }
941}
942
943/// Return the number of element types.
944size_t TupleType::size() const { return getImpl()->size(); }
945
946//===----------------------------------------------------------------------===//
947// Type Utilities
948//===----------------------------------------------------------------------===//
949
950AffineExpr mlir::makeCanonicalStridedLayoutExpr(ArrayRef<int64_t> sizes,
951 ArrayRef<AffineExpr> exprs,
952 MLIRContext *context) {
953 // Size 0 corner case is useful for canonicalizations.
954 if (sizes.empty())
955 return getAffineConstantExpr(constant: 0, context);
956
957 assert(!exprs.empty() && "expected exprs");
958 auto maps = AffineMap::inferFromExprList(exprsList: exprs, context);
959 assert(!maps.empty() && "Expected one non-empty map");
960 unsigned numDims = maps[0].getNumDims(), nSymbols = maps[0].getNumSymbols();
961
962 AffineExpr expr;
963 bool dynamicPoisonBit = false;
964 int64_t runningSize = 1;
965 for (auto en : llvm::zip(t: llvm::reverse(C&: exprs), u: llvm::reverse(C&: sizes))) {
966 int64_t size = std::get<1>(t&: en);
967 AffineExpr dimExpr = std::get<0>(t&: en);
968 AffineExpr stride = dynamicPoisonBit
969 ? getAffineSymbolExpr(position: nSymbols++, context)
970 : getAffineConstantExpr(constant: runningSize, context);
971 expr = expr ? expr + dimExpr * stride : dimExpr * stride;
972 if (size > 0) {
973 auto result = llvm::checkedMul(LHS: runningSize, RHS: size);
974 if (!result) {
975 // Overflow occurred, treat as dynamic
976 dynamicPoisonBit = true;
977 } else {
978 runningSize = *result;
979 }
980 } else {
981 dynamicPoisonBit = true;
982 }
983 }
984 return simplifyAffineExpr(expr, numDims, numSymbols: nSymbols);
985}
986
987AffineExpr mlir::makeCanonicalStridedLayoutExpr(ArrayRef<int64_t> sizes,
988 MLIRContext *context) {
989 SmallVector<AffineExpr, 4> exprs;
990 exprs.reserve(N: sizes.size());
991 for (auto dim : llvm::seq<unsigned>(Begin: 0, End: sizes.size()))
992 exprs.push_back(Elt: getAffineDimExpr(position: dim, context));
993 return makeCanonicalStridedLayoutExpr(sizes, exprs, context);
994}
995