1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Affine/IR/AffineOps.h"
10#include "mlir/Dialect/Arith/IR/Arith.h"
11#include "mlir/Dialect/Arith/Utils/Utils.h"
12#include "mlir/Dialect/Complex/IR/Complex.h"
13#include "mlir/Dialect/Linalg/IR/RelayoutOpInterface.h"
14#include "mlir/Dialect/Tensor/IR/Tensor.h"
15#include "mlir/Dialect/Utils/IndexingUtils.h"
16#include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
17#include "mlir/Dialect/Utils/StaticValueUtils.h"
18#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
19#include "mlir/Dialect/Utils/VerificationUtils.h"
20#include "mlir/IR/Builders.h"
21#include "mlir/IR/BuiltinAttributeInterfaces.h"
22#include "mlir/IR/BuiltinTypeInterfaces.h"
23#include "mlir/IR/BuiltinTypes.h"
24#include "mlir/IR/IRMapping.h"
25#include "mlir/IR/Matchers.h"
26#include "mlir/IR/OpDefinition.h"
27#include "mlir/IR/PatternMatch.h"
28#include "mlir/IR/TensorEncoding.h"
29#include "mlir/IR/TypeUtilities.h"
30#include "mlir/Interfaces/DestinationStyleOpInterface.h"
31#include "mlir/Interfaces/InferIntRangeInterface.h"
32#include "mlir/Interfaces/LoopLikeInterface.h"
33#include "mlir/Interfaces/Utils/InferIntRangeCommon.h"
34#include "mlir/Interfaces/ViewLikeInterface.h"
35#include "mlir/Support/LLVM.h"
36#include "llvm/ADT/DenseSet.h"
37#include "llvm/ADT/Repeated.h"
38#include "llvm/ADT/STLExtras.h"
39#include "llvm/ADT/SmallBitVector.h"
40#include "llvm/ADT/SmallVectorExtras.h"
41#include "llvm/ADT/StringRef.h"
42#include "llvm/Support/Casting.h"
43#include "llvm/Support/MathExtras.h"
44#include <optional>
45
46using namespace mlir;
47using namespace mlir::tensor;
48
49/// Implements the `VerifiableTensorEncoding` contract documented in
50/// TensorEncoding.td for patterns that refine a tensor's shape to be more
51/// static: verifiable encodings are re-checked against the refined shape and
52/// dropped if they no longer hold; opaque encodings (not implementing the
53/// interface) are propagated unconditionally.
54static Attribute propagateEncoding(Attribute encoding, ArrayRef<int64_t> shape,
55 Type elementType) {
56 auto verifiable = dyn_cast_or_null<VerifiableTensorEncoding>(Val&: encoding);
57 if (!verifiable)
58 return encoding;
59
60 MLIRContext *ctx = encoding.getContext();
61 // to avoid user's error stream
62 ScopedDiagnosticHandler swallow(ctx, [](Diagnostic &) { return success(); });
63 auto emit = [ctx]() { return mlir::emitError(loc: UnknownLoc::get(context: ctx)); };
64 return succeeded(Result: verifiable.verifyEncoding(shape, elementType, emitError: emit))
65 ? encoding
66 : Attribute{};
67}
68
69/// Materialize a single constant operation from a given attribute value with
70/// the desired resultant type.
71Operation *TensorDialect::materializeConstant(OpBuilder &builder,
72 Attribute value, Type type,
73 Location loc) {
74 if (auto op = arith::ConstantOp::materialize(builder, value, type, loc))
75 return op;
76 if (complex::ConstantOp::isBuildableWith(value, type))
77 return complex::ConstantOp::create(builder, location: loc, complex: type,
78 value: llvm::cast<ArrayAttr>(Val&: value));
79 return nullptr;
80}
81
82OpFoldResult tensor::getMixedSize(OpBuilder &builder, Location loc, Value value,
83 int64_t dim) {
84 auto tensorType = llvm::cast<RankedTensorType>(Val: value.getType());
85 if (tensorType.isDynamicDim(idx: dim))
86 return builder.createOrFold<tensor::DimOp>(location: loc, args&: value, args&: dim);
87
88 return builder.getIndexAttr(value: tensorType.getDimSize(idx: dim));
89}
90
91SmallVector<OpFoldResult> tensor::getMixedSizes(OpBuilder &builder,
92 Location loc, Value value) {
93 auto tensorType = llvm::cast<RankedTensorType>(Val: value.getType());
94 SmallVector<OpFoldResult> result;
95 for (int64_t i = 0; i < tensorType.getRank(); ++i)
96 result.push_back(Elt: getMixedSize(builder, loc, value, dim: i));
97 return result;
98}
99
100FailureOr<Value> tensor::getOrCreateDestination(OpBuilder &b, Location loc,
101 OpResult opResult) {
102 auto tensorType = llvm::dyn_cast<TensorType>(Val: opResult.getType());
103 assert(tensorType && "expected tensor type");
104
105 // If the op has a destination, it implements DestinationStyleOpInterface and
106 // we can query the destination operand from that interface.
107 auto destOp = opResult.getDefiningOp<DestinationStyleOpInterface>();
108 if (destOp)
109 return destOp.getTiedOpOperand(opResult)->get();
110
111 // Otherwise, create a new destination tensor with the same shape.
112 OpBuilder::InsertionGuard g(b);
113 b.setInsertionPoint(opResult.getDefiningOp());
114
115 // Compute sizes.
116 SmallVector<OpFoldResult> mixedSizes;
117 if (!tensorType.hasStaticShape()) {
118 // Dynamic shape: Query ReifyRankedShapedTypeOpInterface.
119 ReifiedRankedShapedTypeDims reifiedShapes;
120 if (failed(Result: reifyResultShapes(b, op: opResult.getDefiningOp(), reifiedReturnShapes&: reifiedShapes)))
121 return failure();
122 mixedSizes = reifiedShapes[opResult.getResultNumber()];
123 } else {
124 // Static shape: Take static sizes directly.
125 for (int64_t sz : tensorType.getShape())
126 mixedSizes.push_back(Elt: b.getIndexAttr(value: sz));
127 }
128
129 // Create empty tensor with the same encoding as the result type.
130 Attribute encoding;
131 if (auto rankedTensorType = dyn_cast<RankedTensorType>(Val&: tensorType))
132 encoding = rankedTensorType.getEncoding();
133 Value emptyTensor = tensor::EmptyOp::create(
134 builder&: b, location: loc, sizes: mixedSizes, elementType: tensorType.getElementType(), encoding);
135 return emptyTensor;
136}
137
138LogicalResult tensor::getOrCreateDestinations(OpBuilder &b, Location loc,
139 Operation *op,
140 SmallVector<Value> &result) {
141 for (OpResult opResult : op->getResults()) {
142 if (llvm::isa<TensorType>(Val: opResult.getType())) {
143 FailureOr<Value> destination = getOrCreateDestination(b, loc, opResult);
144 if (failed(Result: destination))
145 return failure();
146 result.push_back(Elt: *destination);
147 }
148 }
149 return success();
150}
151
152bool tensor::isSameTypeWithoutEncoding(Type tp1, Type tp2) {
153 if (auto rtp1 = llvm::dyn_cast<RankedTensorType>(Val&: tp1)) {
154 if (auto rtp2 = llvm::dyn_cast<RankedTensorType>(Val&: tp2))
155 return rtp1.getShape() == rtp2.getShape() &&
156 rtp1.getElementType() == rtp2.getElementType();
157 return false;
158 }
159 return tp1 == tp2; // default implementation
160}
161
162/// Compute the dropped dimensions of a rank-reducing tensor.extract_slice op or
163/// rank-extending tensor.insert_slice op.
164static llvm::SmallBitVector getDroppedDims(ArrayRef<int64_t> reducedShape,
165 ArrayRef<OpFoldResult> mixedSizes) {
166 llvm::SmallBitVector droppedDims(mixedSizes.size());
167 int64_t shapePos = static_cast<int64_t>(reducedShape.size()) - 1;
168
169 for (const auto &size : enumerate(First: llvm::reverse(C&: mixedSizes))) {
170 size_t idx = mixedSizes.size() - size.index() - 1;
171 // Rank-reduced dims must have a static unit dimension.
172 bool isStaticUnitSize =
173 isa<Attribute>(Val: size.value()) &&
174 llvm::cast<IntegerAttr>(Val: cast<Attribute>(Val: size.value())).getInt() == 1;
175
176 if (shapePos < 0) {
177 // There are no more dims in the reduced shape. All remaining sizes must
178 // be rank-reduced dims.
179 assert(isStaticUnitSize && "expected unit dim");
180 droppedDims.set(idx);
181 continue;
182 }
183
184 // Dim is preserved if the size is not a static 1.
185 if (!isStaticUnitSize) {
186 --shapePos;
187 continue;
188 }
189
190 // Dim is preserved if the reduced shape dim is also 1.
191 if (reducedShape[shapePos] == 1) {
192 --shapePos;
193 continue;
194 }
195
196 // Otherwise: Dim is dropped.
197 droppedDims.set(idx);
198 }
199
200 assert(shapePos < 0 && "dimension mismatch");
201 return droppedDims;
202}
203
204/// Given a ranked tensor type and a range of values that defines its dynamic
205/// dimension sizes, turn all dynamic sizes that have a constant value into
206/// static dimension sizes.
207static RankedTensorType
208foldDynamicToStaticDimSizes(RankedTensorType type, ValueRange dynamicSizes,
209 SmallVector<Value> &foldedDynamicSizes) {
210 SmallVector<int64_t> staticShape(type.getShape());
211 assert(type.getNumDynamicDims() == dynamicSizes.size() &&
212 "incorrect number of dynamic sizes");
213
214 // Compute new static and dynamic sizes.
215 unsigned ctr = 0;
216 for (int64_t i = 0, e = type.getRank(); i < e; ++i) {
217 if (type.isDynamicDim(idx: i)) {
218 Value dynamicSize = dynamicSizes[ctr++];
219 std::optional<int64_t> cst = getConstantIntValue(ofr: dynamicSize);
220 if (cst.has_value()) {
221 // Dynamic size must be non-negative.
222 if (cst.value() < 0) {
223 foldedDynamicSizes.push_back(Elt: dynamicSize);
224 continue;
225 }
226 staticShape[i] = *cst;
227 } else {
228 foldedDynamicSizes.push_back(Elt: dynamicSize);
229 }
230 }
231 }
232
233 return RankedTensorType::get(shape: staticShape, elementType: type.getElementType(),
234 encoding: type.getEncoding());
235}
236
237//===----------------------------------------------------------------------===//
238// BitcastOp
239//===----------------------------------------------------------------------===//
240
241bool BitcastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
242 if (inputs.size() != 1 || outputs.size() != 1)
243 return false;
244 Type a = inputs.front(), b = outputs.front();
245 auto aT = dyn_cast<TensorType>(Val&: a);
246 auto bT = dyn_cast<TensorType>(Val&: b);
247 if (!aT || !bT)
248 return false;
249
250 if (aT.getElementTypeBitWidth() != bT.getElementTypeBitWidth())
251 return false;
252
253 return succeeded(Result: verifyCompatibleShape(type1: aT, type2: bT));
254}
255
256namespace {
257
258/// Replaces chains of two tensor.bitcast operations by a single tensor.bitcast
259/// operation.
260struct ChainedTensorBitcast : public OpRewritePattern<BitcastOp> {
261 using OpRewritePattern<BitcastOp>::OpRewritePattern;
262
263 LogicalResult matchAndRewrite(BitcastOp tensorBitcast,
264 PatternRewriter &rewriter) const final {
265 auto tensorBitcastOperand =
266 tensorBitcast.getOperand().getDefiningOp<BitcastOp>();
267 if (!tensorBitcastOperand)
268 return failure();
269
270 auto resultType = cast<TensorType>(Val: tensorBitcast.getType());
271 rewriter.replaceOpWithNewOp<BitcastOp>(op: tensorBitcast, args&: resultType,
272 args: tensorBitcastOperand.getOperand());
273 return success();
274 }
275};
276
277} // namespace
278
279void BitcastOp::getCanonicalizationPatterns(RewritePatternSet &results,
280 MLIRContext *context) {
281 results.add<ChainedTensorBitcast>(arg&: context);
282}
283
284//===----------------------------------------------------------------------===//
285// CastOp
286//===----------------------------------------------------------------------===//
287
288void CastOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
289 setNameFn(getResult(), "cast");
290}
291
292/// Returns true if `target` is a ranked tensor type that preserves static
293/// information available in the `source` ranked tensor type.
294bool mlir::tensor::preservesStaticInformation(Type source, Type target) {
295 auto sourceType = llvm::dyn_cast<RankedTensorType>(Val&: source);
296 auto targetType = llvm::dyn_cast<RankedTensorType>(Val&: target);
297
298 // Requires RankedTensorType.
299 if (!sourceType || !targetType)
300 return false;
301
302 // Requires same elemental type.
303 if (sourceType.getElementType() != targetType.getElementType())
304 return false;
305
306 // Requires same rank.
307 if (sourceType.getRank() != targetType.getRank())
308 return false;
309
310 // Requires same encoding.
311 if (sourceType.getEncoding() != targetType.getEncoding())
312 return false;
313
314 // If cast is towards more static sizes along any dimension, don't fold.
315 for (auto t : llvm::zip(t: sourceType.getShape(), u: targetType.getShape())) {
316 if (ShapedType::isStatic(dValue: std::get<0>(t&: t)) &&
317 ShapedType::isDynamic(dValue: std::get<1>(t&: t)))
318 return false;
319 }
320
321 return true;
322}
323
324/// Determines whether tensor::CastOp casts to a more dynamic version of the
325/// source tensor. This is useful to fold a tensor.cast into a consuming op and
326/// implement canonicalization patterns for ops in different dialects that may
327/// consume the results of tensor.cast operations. Such foldable tensor.cast
328/// operations are typically inserted as `slice` ops and are canonicalized,
329/// to preserve the type compatibility of their uses.
330///
331/// Returns true when all conditions are met:
332/// 1. source and result are ranked tensors with same element type and rank.
333/// 2. the tensor type has more static information than the result
334///
335/// Example:
336/// ```mlir
337/// %1 = tensor.cast %0 : tensor<8x16xf32> to tensor<?x?xf32>
338/// %2 = consumer %1 ... : tensor<?x?xf32> ...
339/// ```
340///
341/// folds into:
342///
343/// ```mlir
344/// %2 = consumer %0 ... : tensor<8x16xf32> ...
345/// ```
346bool mlir::tensor::canFoldIntoConsumerOp(CastOp castOp) {
347 if (!castOp)
348 return false;
349
350 // Can fold if the source of cast has at least as much static information as
351 // its results.
352 return preservesStaticInformation(source: castOp.getType(),
353 target: castOp.getSource().getType());
354}
355
356/// Determines whether the tensor::CastOp casts to a more static version of the
357/// source tensor. This is useful to fold into a producing op and implement
358/// canonicalization patterns with the `tensor.cast` op as the root, but
359/// producer being from different dialects. Returns true when all conditions are
360/// met:
361/// 1. source and result and ranked tensors with same element type and rank.
362/// 2. the result type has more static information than the source.
363///
364/// Example:
365/// ```mlir
366/// %1 = producer ... : tensor<?x?xf32>
367/// %2 = tensor.cast %1 : tensor<?x?xf32> to tensor<8x16xf32>
368/// ```
369///
370/// can be canonicalized to :
371///
372/// ```mlir
373/// %2 = producer ... : tensor<8x16xf32>
374/// ```
375/// Not all ops might be canonicalizable this way, but for those that can be,
376/// this method provides a check that it is worth doing the canonicalization.
377bool mlir::tensor::canFoldIntoProducerOp(CastOp castOp) {
378 if (!castOp)
379 return false;
380 return preservesStaticInformation(source: castOp.getSource().getType(),
381 target: castOp.getType());
382}
383
384bool mlir::tensor::hasFoldableTensorCastOperand(Operation *op) {
385 return llvm::any_of(Range: op->getOpOperands(), P: [&](OpOperand &opOperand) {
386 if (llvm::isa<BlockArgument>(Val: opOperand.get()))
387 return false;
388 auto castOp = opOperand.get().getDefiningOp<tensor::CastOp>();
389 return castOp && canFoldIntoConsumerOp(castOp);
390 });
391}
392
393SmallVector<Value> mlir::tensor::getUpdatedOperandsAfterCastOpFolding(
394 DestinationStyleOpInterface op, SmallVector<Type> &newResTy) {
395 SmallVector<Value> newOperands;
396 newOperands.reserve(N: op->getNumOperands());
397
398 assert(hasFoldableTensorCastOperand(op) && "No foldable CastOp operands!");
399
400 // Assumes that the result has dpsInits followed by nonDpsInits.
401 int64_t dpsInitIdx = 0;
402 for (OpOperand &opOperand : op->getOpOperands()) {
403 auto tensorCastOp = opOperand.get().getDefiningOp<tensor::CastOp>();
404 bool fold = canFoldIntoConsumerOp(castOp: tensorCastOp);
405 newOperands.push_back(Elt: fold ? tensorCastOp.getOperand() : opOperand.get());
406 if (op.isDpsInit(opOperand: &opOperand) &&
407 !llvm::isa<MemRefType>(Val: newOperands.back().getType()))
408 newResTy[dpsInitIdx++] = newOperands.back().getType();
409 }
410 return newOperands;
411}
412
413/// Performs folding of any operand of `op` if it comes from a tensor::CastOp
414/// that can be folded.
415LogicalResult mlir::tensor::foldTensorCast(Operation *op) {
416 bool folded = false;
417 for (OpOperand &operand : op->getOpOperands()) {
418 auto castOp = operand.get().getDefiningOp<tensor::CastOp>();
419 if (castOp && tensor::canFoldIntoConsumerOp(castOp)) {
420 operand.set(castOp.getOperand());
421 folded = true;
422 }
423 }
424 return success(IsSuccess: folded);
425}
426
427bool CastOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
428 if (inputs.size() != 1 || outputs.size() != 1)
429 return false;
430 Type a = inputs.front(), b = outputs.front();
431 auto aT = llvm::dyn_cast<TensorType>(Val&: a);
432 auto bT = llvm::dyn_cast<TensorType>(Val&: b);
433 if (!aT || !bT)
434 return false;
435
436 if (aT.getElementType() != bT.getElementType())
437 return false;
438
439 return succeeded(Result: verifyCompatibleShape(type1: aT, type2: bT));
440}
441
442/// Compute a TensorType that has the joined shape knowledge of the two
443/// given TensorTypes. The element types need to match.
444static TensorType joinShapes(TensorType one, TensorType two) {
445 assert(one.getElementType() == two.getElementType());
446
447 if (!one.hasRank())
448 return two;
449 if (!two.hasRank())
450 return one;
451
452 int64_t rank = one.getRank();
453 if (rank != two.getRank())
454 return {};
455
456 SmallVector<int64_t, 4> join;
457 join.reserve(N: rank);
458 for (int64_t i = 0; i < rank; ++i) {
459 if (one.isDynamicDim(idx: i)) {
460 join.push_back(Elt: two.getDimSize(idx: i));
461 continue;
462 }
463 if (two.isDynamicDim(idx: i)) {
464 join.push_back(Elt: one.getDimSize(idx: i));
465 continue;
466 }
467 if (one.getDimSize(idx: i) != two.getDimSize(idx: i))
468 return {};
469 join.push_back(Elt: one.getDimSize(idx: i));
470 }
471 return RankedTensorType::get(shape: join, elementType: one.getElementType());
472}
473
474namespace {
475
476/// Replaces chains of two tensor.cast operations by a single tensor.cast
477/// operation if doing so does not remove runtime constraints.
478struct ChainedTensorCast : public OpRewritePattern<CastOp> {
479 using OpRewritePattern<CastOp>::OpRewritePattern;
480
481 LogicalResult matchAndRewrite(CastOp tensorCast,
482 PatternRewriter &rewriter) const final {
483 auto tensorCastOperand = tensorCast.getOperand().getDefiningOp<CastOp>();
484
485 if (!tensorCastOperand)
486 return failure();
487
488 auto sourceType =
489 llvm::cast<TensorType>(Val: tensorCastOperand.getOperand().getType());
490 auto intermediateType = llvm::cast<TensorType>(Val: tensorCastOperand.getType());
491 auto resultType = llvm::cast<TensorType>(Val: tensorCast.getType());
492
493 // We can remove the intermediate cast if joining all three produces the
494 // same result as just joining the source and result shapes.
495 auto firstJoin =
496 joinShapes(one: joinShapes(one: sourceType, two: intermediateType), two: resultType);
497
498 // The join might not exist if the cast sequence would fail at runtime.
499 if (!firstJoin)
500 return failure();
501
502 // The newJoin always exists if the above join exists, it might just contain
503 // less information. If so, we cannot drop the intermediate cast, as doing
504 // so would remove runtime checks.
505 auto newJoin = joinShapes(one: sourceType, two: resultType);
506 if (firstJoin != newJoin)
507 return failure();
508
509 rewriter.replaceOpWithNewOp<CastOp>(op: tensorCast, args&: resultType,
510 args: tensorCastOperand.getOperand());
511 return success();
512 }
513};
514
515/// Fold tensor.cast into tesor.extract_slice producer.
516/// Example:
517/// ```
518/// %0 = tensor.extract_slice %arg0[%o, 0] [%s, 512] [1, 1] :
519/// tensor<128x512xf32> to tensor<?x512xf32>
520/// %1 = tensor.cast %0 : tensor<?x512xf32> to tensor<16x512xf32>
521/// ```
522/// ->
523/// ```
524/// %1 = tensor.extract_slice %arg0[%o, 0] [16, 512] [1, 1] :
525/// tensor<128x512xf32> to tensor<16x512xf32>
526/// ```
527struct TensorCastExtractSlice : public OpRewritePattern<CastOp> {
528 using OpRewritePattern<CastOp>::OpRewritePattern;
529
530 LogicalResult matchAndRewrite(CastOp tensorCast,
531 PatternRewriter &rewriter) const final {
532 auto extractOperand =
533 tensorCast.getOperand().getDefiningOp<ExtractSliceOp>();
534
535 // Cannot fold cast to unranked tensor.
536 auto rankedResultType =
537 llvm::dyn_cast<RankedTensorType>(Val: tensorCast.getType());
538 if (!rankedResultType)
539 return failure();
540
541 if (!extractOperand || !canFoldIntoProducerOp(castOp: tensorCast) ||
542 rankedResultType.getShape() ==
543 llvm::cast<RankedTensorType>(Val: tensorCast.getSource().getType())
544 .getShape())
545 return failure();
546
547 SmallVector<OpFoldResult, 4> sizes = extractOperand.getMixedSizes();
548 auto dimMask = computeRankReductionMask(
549 originalShape: extractOperand.getStaticSizes(), reducedShape: extractOperand.getType().getShape());
550 size_t dimIndex = 0;
551 for (size_t i = 0, e = sizes.size(); i < e; i++) {
552 if (dimMask && dimMask->count(V: i))
553 continue;
554 int64_t dim = rankedResultType.getShape()[dimIndex++];
555 if (ShapedType::isDynamic(dValue: dim))
556 continue;
557 sizes[i] = rewriter.getIndexAttr(value: dim);
558 }
559
560 rewriter.replaceOpWithNewOp<ExtractSliceOp>(
561 op: tensorCast, args&: rankedResultType, args: extractOperand.getSource(),
562 args: extractOperand.getMixedOffsets(), args&: sizes,
563 args: extractOperand.getMixedStrides());
564 return success();
565 }
566};
567
568} // namespace
569
570void CastOp::getCanonicalizationPatterns(RewritePatternSet &results,
571 MLIRContext *context) {
572 results.add<ChainedTensorCast, TensorCastExtractSlice>(arg&: context);
573}
574
575//===----------------------------------------------------------------------===//
576// ConcatOp
577//===----------------------------------------------------------------------===//
578
579RankedTensorType ConcatOp::inferResultType(int64_t dim, TypeRange inputTypes) {
580 assert(!inputTypes.empty() && "cannot concatenate 0 tensors");
581 auto tensorTypes =
582 llvm::map_to_vector<4>(C&: inputTypes, F: llvm::CastTo<RankedTensorType>);
583 int64_t concatRank = tensorTypes[0].getRank();
584
585 // The concatenation dim must be in the range [0, rank).
586 assert(dim >= 0 && dim < concatRank && "Invalid concatenation dim");
587
588 SmallVector<int64_t> sizes(concatRank);
589 for (int64_t i = 0, e = concatRank; i < e; ++i) {
590 if (i == dim)
591 continue;
592 SaturatedInteger size;
593 for (auto tensorType : tensorTypes)
594 size = *size.desaturate(other: SaturatedInteger::wrap(v: tensorType.getDimSize(idx: i)));
595 sizes[i] = size.asInteger();
596 }
597 auto concatSize = SaturatedInteger::wrap(v: 0);
598 for (auto tensorType : tensorTypes)
599 concatSize =
600 concatSize + SaturatedInteger::wrap(v: tensorType.getDimSize(idx: dim));
601 sizes[dim] = concatSize.asInteger();
602 return RankedTensorType::get(shape: sizes, elementType: tensorTypes[0].getElementType());
603}
604
605void ConcatOp::build(OpBuilder &builder, OperationState &result, int64_t dim,
606 ValueRange inputs) {
607 FailureOr<RankedTensorType> resultType =
608 inferResultType(dim, inputTypes: inputs.getTypes());
609 assert(succeeded(resultType) && "failed to infer concatenation result type");
610 build(odsBuilder&: builder, odsState&: result, result: *resultType, dim, inputs);
611}
612
613LogicalResult ConcatOp::verify() {
614 if (getInputs().size() < 1)
615 return emitOpError(message: "requires at least one input");
616
617 SmallVector<RankedTensorType> inputTypes;
618 for (auto input : getInputs())
619 inputTypes.push_back(Elt: cast<RankedTensorType>(Val: input.getType()));
620
621 RankedTensorType resultType = getResultType();
622 int64_t resultRank = getRank();
623 if (llvm::any_of(Range&: inputTypes, P: [resultRank](RankedTensorType type) {
624 return type.getRank() != resultRank;
625 }))
626 return emitOpError(message: "rank of concatenated inputs must match result rank");
627
628 Type resultElementType = resultType.getElementType();
629 if (llvm::any_of(Range&: inputTypes, P: [&](RankedTensorType type) {
630 return type.getElementType() != resultElementType;
631 }))
632 return emitOpError(message: "inputs and result element type must match");
633
634 int64_t dim = getDim();
635 if (dim >= resultRank)
636 return emitOpError(message: "concatenation dim must be less than the tensor rank");
637
638 SmallVector<int64_t> sizes(resultRank);
639 for (int64_t i = 0, e = resultRank; i < e; ++i) {
640 if (i == dim)
641 continue;
642 SaturatedInteger size;
643 for (auto tensorType : inputTypes) {
644 FailureOr<SaturatedInteger> maybeSize =
645 size.desaturate(other: SaturatedInteger::wrap(v: tensorType.getDimSize(idx: i)));
646 if (failed(Result: maybeSize))
647 return emitOpError(message: "static concatenation size mismatch along ")
648 << "non-concatenated dimension " << i;
649 size = *maybeSize;
650 }
651 sizes[i] = size.asInteger();
652 }
653 auto concatSize = SaturatedInteger::wrap(v: 0);
654 for (auto tensorType : inputTypes)
655 concatSize =
656 concatSize + SaturatedInteger::wrap(v: tensorType.getDimSize(idx: dim));
657 sizes[dim] = concatSize.asInteger();
658 auto inferredResultType =
659 RankedTensorType::get(shape: sizes, elementType: inputTypes[0].getElementType());
660
661 for (auto [inferredSize, actualSize] :
662 llvm::zip_equal(t: inferredResultType.getShape(), u: resultType.getShape())) {
663 bool hasDynamic = ShapedType::isDynamic(dValue: inferredSize) ||
664 ShapedType::isDynamic(dValue: actualSize);
665 if (!hasDynamic && inferredSize != actualSize)
666 return emitOpError(message: "result type ")
667 << resultType << "does not match inferred shape "
668 << inferredResultType << " static sizes";
669 }
670
671 return success();
672}
673
674FailureOr<SmallVector<Value>> ConcatOp::decomposeOperation(OpBuilder &builder) {
675 size_t numInputs = getInputs().size();
676 uint64_t concatDim = getDim();
677
678 SmallVector<SmallVector<OpFoldResult>> inputShapes;
679 inputShapes.reserve(N: numInputs);
680 SmallVector<OpFoldResult> concatOffsets;
681 concatOffsets.reserve(N: numInputs);
682 SmallVector<OpFoldResult> outputShape;
683
684 AffineExpr addExpr =
685 builder.getAffineSymbolExpr(position: 0) + builder.getAffineSymbolExpr(position: 1);
686 OpFoldResult zero = builder.getIndexAttr(value: 0);
687 Location loc = getLoc();
688 for (auto [index, input] : llvm::enumerate(First: getInputs())) {
689 SmallVector<OpFoldResult> inputShape =
690 tensor::getMixedSizes(builder, loc: input.getLoc(), value: input);
691 if (index == 0) {
692 outputShape = inputShape;
693 concatOffsets.push_back(Elt: zero);
694 } else {
695 concatOffsets.push_back(Elt: outputShape[concatDim]);
696 outputShape[concatDim] = affine::makeComposedFoldedAffineApply(
697 b&: builder, loc, expr: addExpr,
698 operands: {outputShape[concatDim], inputShape[concatDim]});
699 }
700 inputShapes.emplace_back(Args: std::move(inputShape));
701 }
702
703 Value replacement = tensor::EmptyOp::create(builder, location: loc, sizes: outputShape,
704 elementType: getType().getElementType());
705
706 int64_t rank = getType().getRank();
707 OpFoldResult one = builder.getIndexAttr(value: 1);
708 SmallVector<OpFoldResult> strides(rank, one);
709 SmallVector<OpFoldResult> offsets(rank, zero);
710 for (auto [index, input] : llvm::enumerate(First: getInputs())) {
711 offsets[concatDim] = concatOffsets[index];
712 auto insertSlice = tensor::InsertSliceOp::create(
713 builder, location: loc, source: input, dest: replacement, offsets, sizes: inputShapes[index], strides);
714 replacement = insertSlice.getResult();
715 }
716 if (replacement.getType() != getType()) {
717 replacement = tensor::CastOp::create(builder, location: loc, dest: getType(), source: replacement);
718 }
719 return SmallVector<Value>{replacement};
720}
721
722LogicalResult
723ConcatOp::reifyResultShapes(OpBuilder &builder,
724 ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
725 ValueRange inputs = getInputs();
726 int64_t dim = getDim();
727 RankedTensorType inferredResultType = inferResultType(dim, inputTypes: inputs.getTypes());
728
729 Value init = inputs[0];
730 int64_t rank = getType().getRank();
731
732 reifiedReturnShapes.resize(N: 1, NV: SmallVector<OpFoldResult>(rank));
733
734 // Pre-populate the result sizes with as much static information as possible
735 // from the given result type, as well as the inferred result type, otherwise
736 // use the dim sizes from the first input.
737 for (int64_t i = 0; i < rank; ++i) {
738 if (i == dim)
739 continue;
740 if (!getType().isDynamicDim(idx: i)) {
741 reifiedReturnShapes[0][i] = builder.getIndexAttr(value: getType().getDimSize(idx: i));
742 } else if (!inferredResultType.isDynamicDim(idx: i)) {
743 reifiedReturnShapes[0][i] = getValueOrCreateConstantIndexOp(
744 b&: builder, loc: getLoc(),
745 ofr: builder.getIndexAttr(value: inferredResultType.getDimSize(idx: i)));
746 } else {
747 reifiedReturnShapes[0][i] =
748 tensor::DimOp::create(builder, location: init.getLoc(), source: init, index: i).getResult();
749 }
750 }
751
752 if (getType().isDynamicDim(idx: dim)) {
753 // Take the sum of the input sizes along the concatenated dim.
754 AffineExpr sum = builder.getAffineDimExpr(position: 0);
755 SmallVector<OpFoldResult> sizes = {
756 builder.createOrFold<tensor::DimOp>(location: init.getLoc(), args&: init, args&: dim)};
757 for (auto [idx, input] : llvm::enumerate(First: inputs.drop_front())) {
758 sum = sum + builder.getAffineDimExpr(position: idx + 1);
759 sizes.push_back(
760 Elt: builder.createOrFold<tensor::DimOp>(location: input.getLoc(), args&: input, args&: dim));
761 }
762 reifiedReturnShapes[0][dim] = getValueOrCreateConstantIndexOp(
763 b&: builder, loc: getLoc(),
764 ofr: affine::makeComposedFoldedAffineApply(b&: builder, loc: getLoc(), expr: sum, operands: sizes));
765 } else {
766 // If the result shape is static along the concatenated dim, use the static
767 // shape.
768 reifiedReturnShapes[0][dim] =
769 builder.getIndexAttr(value: getType().getDimSize(idx: dim));
770 }
771 return success();
772}
773
774void ConcatOp::getAsmResultNames(
775 function_ref<void(Value, StringRef)> setNameFn) {
776 setNameFn(getResult(), "concat");
777}
778
779OpFoldResult ConcatOp::fold(FoldAdaptor) {
780 ValueRange inputs = getInputs();
781 if (inputs.size() == 1 && inputs[0].getType() == getResultType())
782 return inputs[0];
783 return {};
784}
785
786namespace {
787/// Fold a concat op with a single input to a cast.
788struct SingleInputConcatOp : public OpRewritePattern<ConcatOp> {
789 using OpRewritePattern<ConcatOp>::OpRewritePattern;
790
791 LogicalResult matchAndRewrite(ConcatOp concatOp,
792 PatternRewriter &rewriter) const override {
793 if (concatOp.getInputs().size() != 1)
794 return failure();
795 rewriter.replaceOpWithNewOp<CastOp>(op: concatOp, args: concatOp.getResultType(),
796 args: concatOp.getInputs()[0]);
797 return success();
798 }
799};
800
801/// Propagate static shapes into the operands of a `tensor.concat`.
802///
803/// `tensor.concat` requires every operand to match on all dimensions except the
804/// concatenation dimension. If one operand is already static in those
805/// dimensions, the other operands may safely be refined to that same static
806/// shape.
807///
808/// Example:
809///
810/// ```mlir
811/// %2 = tensor.concat dim(0) %0, %1: (tensor<?x12xi32>, tensor<?x?xi32>) ->
812/// tensor<?x12xi32>
813/// ```
814/// ->
815/// ```mlir
816/// %cast = tensor.cast %1 : tensor<?x?xi32> to tensor<?x12xi32>
817/// %2 = tensor.concat dim(0) %0, %cast :
818/// (tensor<?x12xi32>, tensor<?x12xi32>) -> tensor<?x12xi32>
819/// ```
820struct InferConcatOperandTypes : public OpRewritePattern<ConcatOp> {
821 using OpRewritePattern<ConcatOp>::OpRewritePattern;
822
823 LogicalResult matchAndRewrite(ConcatOp concatOp,
824 PatternRewriter &rewriter) const override {
825 int64_t dim = concatOp.getDim();
826 RankedTensorType inferredResultType =
827 ConcatOp::inferResultType(dim, inputTypes: concatOp->getOperandTypes());
828
829 // Find operands for which a more static shape can be inferred.
830 LogicalResult matched = failure();
831 // Inferred operand shapes are identical in every dimension except the
832 // concatenation dimension.
833 SmallVector<int64_t> inferredOperandShape(inferredResultType.getShape());
834 for (auto [operandIdx, operandType] :
835 llvm::enumerate(First: concatOp->getOperandTypes())) {
836 // Compute inferred type for operand.
837 inferredOperandShape[dim] =
838 cast<RankedTensorType>(Val&: operandType).getDimSize(idx: dim);
839 auto inferredOperandType = RankedTensorType::get(
840 shape: inferredOperandShape, elementType: inferredResultType.getElementType());
841
842 // Check if inferred type is more static.
843 if (!preservesStaticInformation(source: inferredOperandType, target: operandType)) {
844 matched = success();
845
846 // Use refined operand type and create cast from original operand.
847 auto castOp =
848 CastOp::create(builder&: rewriter, location: concatOp->getLoc(), dest: inferredOperandType,
849 source: concatOp.getOperand(i: operandIdx));
850 rewriter.modifyOpInPlace(root: concatOp, callable: [=, operandIdx = operandIdx] {
851 concatOp->setOperand(idx: operandIdx, value: castOp->getResult(idx: 0));
852 });
853 }
854 }
855
856 return matched;
857 }
858};
859
860// Ensure `tensor.concat`'s result type is at least as static as can be inferred
861// from its operand types.
862///
863/// Example:
864/// ```mlir
865/// %2 = tensor.concat dim(0) %0, %1: (tensor<?x12xi32>, tensor<?x12xi32>) ->
866/// tensor<?x?xi32>
867/// ```
868/// ->
869/// ```mlir
870/// %2 = tensor.concat dim(0) %0, %cast : (tensor<?x12xi32>, tensor<?x12xi32>)
871/// -> tensor<?x12xi32> %cast = tensor.cast %2 : tensor<?x12xi32> to
872/// tensor<?x?xi32>
873/// ```
874struct InferConcatResultType : public OpRewritePattern<ConcatOp> {
875 using OpRewritePattern<ConcatOp>::OpRewritePattern;
876
877 LogicalResult matchAndRewrite(ConcatOp concatOp,
878 PatternRewriter &rewriter) const override {
879 int64_t dim = concatOp.getDim();
880 RankedTensorType inferredResultType =
881 ConcatOp::inferResultType(dim, inputTypes: concatOp->getOperandTypes());
882
883 // The result type should be at least as static as inferred result type.
884 if (preservesStaticInformation(source: inferredResultType,
885 target: concatOp.getResultType())) {
886 return failure();
887 }
888
889 auto newConcatOp =
890 ConcatOp::create(builder&: rewriter, location: concatOp->getLoc(), result: inferredResultType, dim,
891 inputs: concatOp->getOperands());
892 rewriter.replaceOpWithNewOp<CastOp>(op: concatOp, args: concatOp.getResultType(),
893 args&: newConcatOp);
894
895 return success();
896 }
897};
898} // namespace
899
900void ConcatOp::getCanonicalizationPatterns(RewritePatternSet &results,
901 MLIRContext *context) {
902 results
903 .add<SingleInputConcatOp, InferConcatOperandTypes, InferConcatResultType>(
904 arg&: context);
905}
906
907//===----------------------------------------------------------------------===//
908// DimOp
909//===----------------------------------------------------------------------===//
910
911void DimOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
912 setNameFn(getResult(), "dim");
913}
914
915void DimOp::build(OpBuilder &builder, OperationState &result, Value source,
916 int64_t index) {
917 auto loc = result.location;
918 Value indexValue = arith::ConstantIndexOp::create(builder, location: loc, value: index);
919 build(odsBuilder&: builder, odsState&: result, source, index: indexValue);
920}
921
922std::optional<int64_t> DimOp::getConstantIndex() {
923 return getConstantIntValue(ofr: getIndex());
924}
925
926Speculation::Speculatability DimOp::getSpeculatability() {
927 auto constantIndex = getConstantIndex();
928 if (!constantIndex)
929 return Speculation::NotSpeculatable;
930
931 auto rankedSourceType = dyn_cast<RankedTensorType>(Val: getSource().getType());
932 if (!rankedSourceType)
933 return Speculation::NotSpeculatable;
934
935 if (rankedSourceType.getRank() <= constantIndex)
936 return Speculation::NotSpeculatable;
937
938 return Speculation::Speculatable;
939}
940
941void DimOp::inferResultRangesFromOptional(ArrayRef<IntegerValueRange> argRanges,
942 SetIntLatticeFn setResultRange) {
943 setResultRange(getResult(),
944 intrange::inferShapedDimOpInterface(op: *this, maybeDim: argRanges[1]));
945}
946
947OpFoldResult DimOp::fold(FoldAdaptor adaptor) {
948 // All forms of folding require a known index.
949 std::optional<int64_t> index = getConstantIndex();
950 if (!index)
951 return {};
952
953 // Folding for unranked types (UnrankedTensorType) is not supported.
954 auto tensorType = llvm::dyn_cast<RankedTensorType>(Val: getSource().getType());
955 if (!tensorType)
956 return {};
957
958 // Out of bound indices produce undefined behavior but are still valid IR.
959 // Don't choke on them.
960 int64_t indexVal = index.value();
961 if (indexVal < 0 || indexVal >= tensorType.getRank())
962 return {};
963
964 // Fold if the shape extent along the given index is known.
965 if (!tensorType.isDynamicDim(idx: indexVal)) {
966 Builder builder(getContext());
967 return builder.getIndexAttr(value: tensorType.getShape()[indexVal]);
968 }
969
970 Operation *definingOp = getSource().getDefiningOp();
971
972 // Fold dim to the operand of tensor.generate.
973 if (auto fromElements = dyn_cast_or_null<tensor::GenerateOp>(Val: definingOp)) {
974 auto resultType =
975 llvm::cast<RankedTensorType>(Val: fromElements.getResult().getType());
976 // The case where the type encodes the size of the dimension is handled
977 // above.
978 assert(ShapedType::isDynamic(resultType.getShape()[indexVal]));
979
980 // Find the operand of the fromElements that corresponds to this index.
981 auto dynExtents = fromElements.getDynamicExtents().begin();
982 for (auto dim : resultType.getShape().take_front(N: indexVal))
983 if (ShapedType::isDynamic(dValue: dim))
984 dynExtents++;
985
986 return Value{*dynExtents};
987 }
988
989 // The size at the given index is now known to be a dynamic size.
990 if (auto sliceOp = dyn_cast_or_null<tensor::ExtractSliceOp>(Val: definingOp)) {
991 // Fold only for non-rank reduced ops. For the rank-reduced version, rely on
992 // `resolve-shaped-type-result-dims` pass.
993 if (sliceOp.getType().getRank() == sliceOp.getSourceType().getRank() &&
994 sliceOp.isDynamicSize(idx: indexVal)) {
995 return {sliceOp.getDynamicSize(idx: indexVal)};
996 }
997 }
998
999 // dim(cast) -> dim
1000 if (succeeded(Result: foldTensorCast(op: *this)))
1001 return getResult();
1002
1003 return {};
1004}
1005
1006namespace {
1007/// Fold dim of a cast into the dim of the source of the tensor cast.
1008struct DimOfCastOp : public OpRewritePattern<DimOp> {
1009 using OpRewritePattern<DimOp>::OpRewritePattern;
1010
1011 LogicalResult matchAndRewrite(DimOp dimOp,
1012 PatternRewriter &rewriter) const override {
1013 auto castOp = dimOp.getSource().getDefiningOp<CastOp>();
1014 if (!castOp)
1015 return failure();
1016 Value newSource = castOp.getOperand();
1017 rewriter.replaceOpWithNewOp<DimOp>(op: dimOp, args&: newSource, args: dimOp.getIndex());
1018 return success();
1019 }
1020};
1021
1022/// Fold dim of a destination passing style op into the dim of the corresponding
1023/// init.
1024struct DimOfDestStyleOp : public OpRewritePattern<DimOp> {
1025 using OpRewritePattern<DimOp>::OpRewritePattern;
1026
1027 LogicalResult matchAndRewrite(DimOp dimOp,
1028 PatternRewriter &rewriter) const override {
1029 auto source = dimOp.getSource();
1030 auto destOp = source.getDefiningOp<DestinationStyleOpInterface>();
1031 if (!destOp)
1032 return failure();
1033
1034 auto resultIndex = cast<OpResult>(Val&: source).getResultNumber();
1035 auto *initOperand = destOp.getDpsInitOperand(i: resultIndex);
1036
1037 rewriter.modifyOpInPlace(
1038 root: dimOp, callable: [&]() { dimOp.getSourceMutable().assign(value: initOperand->get()); });
1039 return success();
1040 }
1041};
1042
1043/// Fold dim of a tensor reshape operation to a extract into the reshape's shape
1044/// operand.
1045struct DimOfReshapeOp : public OpRewritePattern<DimOp> {
1046 using OpRewritePattern<DimOp>::OpRewritePattern;
1047
1048 LogicalResult matchAndRewrite(DimOp dim,
1049 PatternRewriter &rewriter) const override {
1050 auto reshape = dim.getSource().getDefiningOp<ReshapeOp>();
1051
1052 if (!reshape)
1053 return failure();
1054
1055 // Since tensors are immutable we don't need to worry about where to place
1056 // the extract call
1057 rewriter.setInsertionPointAfter(dim);
1058 Location loc = dim.getLoc();
1059 Value extract =
1060 ExtractOp::create(builder&: rewriter, location: loc, tensor: reshape.getShape(), indices: dim.getIndex());
1061 if (extract.getType() != dim.getType())
1062 extract =
1063 arith::IndexCastOp::create(builder&: rewriter, location: loc, out: dim.getType(), in: extract);
1064 rewriter.replaceOp(op: dim, newValues: extract);
1065 return success();
1066 }
1067};
1068} // namespace
1069
1070void DimOp::getCanonicalizationPatterns(RewritePatternSet &results,
1071 MLIRContext *context) {
1072 results.add<DimOfCastOp, DimOfDestStyleOp, DimOfReshapeOp>(arg&: context);
1073}
1074
1075//===----------------------------------------------------------------------===//
1076// EmptyOp
1077//===----------------------------------------------------------------------===//
1078
1079void EmptyOp::build(OpBuilder &builder, OperationState &result,
1080 ArrayRef<int64_t> staticShape, Type elementType,
1081 Attribute encoding) {
1082 assert(none_of(staticShape, ShapedType::isDynamic) &&
1083 "expected only static sizes");
1084 build(odsBuilder&: builder, odsState&: result, staticShape, elementType, dynamicSizes: ValueRange{}, encoding);
1085}
1086
1087void EmptyOp::build(OpBuilder &builder, OperationState &result,
1088 ArrayRef<int64_t> staticShape, Type elementType,
1089 ValueRange dynamicSizes, Attribute encoding) {
1090 auto tensorType = RankedTensorType::get(shape: staticShape, elementType, encoding);
1091 build(odsBuilder&: builder, odsState&: result, result: tensorType, dynamicSizes);
1092}
1093
1094void EmptyOp::build(OpBuilder &builder, OperationState &result,
1095 ArrayRef<OpFoldResult> sizes, Type elementType,
1096 Attribute encoding) {
1097 SmallVector<int64_t> staticShape;
1098 SmallVector<Value> dynamicSizes;
1099 dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticShape);
1100 build(builder, result, staticShape, elementType, dynamicSizes, encoding);
1101}
1102
1103LogicalResult EmptyOp::verify() {
1104 return verifyDynamicDimensionCount(op: getOperation(), type: getType(),
1105 dynamicSizes: getDynamicSizes());
1106}
1107
1108LogicalResult
1109EmptyOp::reifyResultShapes(OpBuilder &builder,
1110 ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
1111 reifiedReturnShapes.resize(N: 1, NV: SmallVector<OpFoldResult>(getType().getRank()));
1112 unsigned ctr = 0;
1113 for (int64_t i = 0; i < getType().getRank(); ++i) {
1114 if (getType().isDynamicDim(idx: i)) {
1115 reifiedReturnShapes[0][i] = getDynamicSizes()[ctr++];
1116 } else {
1117 reifiedReturnShapes[0][i] = builder.getIndexAttr(value: getType().getDimSize(idx: i));
1118 }
1119 }
1120 return success();
1121}
1122
1123Value EmptyOp::getDynamicSize(unsigned idx) {
1124 assert(getType().isDynamicDim(idx) && "expected dynamic dim");
1125 unsigned ctr = 0;
1126 for (int64_t i = 0; i < static_cast<int64_t>(idx); ++i)
1127 if (getType().isDynamicDim(idx: i))
1128 ++ctr;
1129 return getDynamicSizes()[ctr];
1130}
1131
1132SmallVector<OpFoldResult> EmptyOp::getMixedSizes() {
1133 SmallVector<OpFoldResult> result;
1134 unsigned ctr = 0;
1135 Builder b(getContext());
1136 for (int64_t dim : getType().getShape()) {
1137 if (ShapedType::isDynamic(dValue: dim)) {
1138 result.push_back(Elt: getDynamicSizes()[ctr++]);
1139 } else {
1140 result.push_back(Elt: b.getIndexAttr(value: dim));
1141 }
1142 }
1143 return result;
1144}
1145
1146namespace {
1147/// Change the type of the result of a `tensor.empty` by making the result
1148/// type statically sized along dimensions that in the original operation were
1149/// defined as dynamic, but the size was defined using a `constant` op. For
1150/// example
1151///
1152/// %c5 = arith.constant 5: index
1153/// %0 = tensor.empty(%arg0, %c5) : tensor<?x?xf32>
1154///
1155/// to
1156///
1157/// %0 = tensor.empty(%arg0) : tensor<?x5xf32>
1158struct ReplaceEmptyTensorStaticShapeDims : OpRewritePattern<EmptyOp> {
1159 using OpRewritePattern<EmptyOp>::OpRewritePattern;
1160
1161 LogicalResult matchAndRewrite(EmptyOp op,
1162 PatternRewriter &rewriter) const override {
1163 SmallVector<Value> foldedDynamicSizes;
1164 RankedTensorType foldedTensorType = foldDynamicToStaticDimSizes(
1165 type: op.getType(), dynamicSizes: op.getDynamicSizes(), foldedDynamicSizes);
1166
1167 // Stop here if no dynamic size was promoted to static.
1168 if (foldedTensorType == op.getType())
1169 return failure();
1170
1171 auto newOp = EmptyOp::create(builder&: rewriter, location: op.getLoc(), result: foldedTensorType,
1172 dynamicSizes: foldedDynamicSizes);
1173 rewriter.replaceOpWithNewOp<tensor::CastOp>(op, args: op.getType(), args&: newOp);
1174 return success();
1175 }
1176};
1177
1178struct FoldEmptyTensorWithDimOp : public OpRewritePattern<DimOp> {
1179 using OpRewritePattern<DimOp>::OpRewritePattern;
1180
1181 LogicalResult matchAndRewrite(tensor::DimOp dimOp,
1182 PatternRewriter &rewriter) const override {
1183 std::optional<int64_t> maybeConstantIndex = dimOp.getConstantIndex();
1184 auto emptyTensorOp = dimOp.getSource().getDefiningOp<EmptyOp>();
1185 if (!emptyTensorOp || !maybeConstantIndex)
1186 return failure();
1187 auto emptyTensorType = emptyTensorOp.getType();
1188 if (*maybeConstantIndex < 0 ||
1189 *maybeConstantIndex >= emptyTensorType.getRank() ||
1190 !emptyTensorType.isDynamicDim(idx: *maybeConstantIndex))
1191 return failure();
1192 rewriter.replaceOp(op: dimOp,
1193 newValues: emptyTensorOp.getDynamicSize(idx: *maybeConstantIndex));
1194 return success();
1195 }
1196};
1197
1198/// Canonicalize
1199///
1200/// ```mlir
1201/// %0 = tensor.empty(%d0, %d1) : tensor<?x?xf32>
1202/// %1 = tensor.cast %0 : tensor<?x?xf32> to tensor<4x?xf32>
1203/// ```
1204///
1205/// into
1206///
1207/// ```mlir
1208/// %0 = tensor.empty(%d1) : tensor<4x?xf32>
1209/// ```
1210///
1211/// This assumes the input program is correct in terms of its shape. So it is
1212/// safe to assume that `%d0` is in fact 4.
1213struct FoldEmptyTensorWithCastOp : public OpRewritePattern<CastOp> {
1214 using OpRewritePattern<CastOp>::OpRewritePattern;
1215
1216 LogicalResult matchAndRewrite(CastOp castOp,
1217 PatternRewriter &rewriter) const override {
1218 if (!canFoldIntoProducerOp(castOp))
1219 return failure();
1220 auto producer = castOp.getSource().getDefiningOp<EmptyOp>();
1221 if (!producer)
1222 return failure();
1223
1224 auto resultType =
1225 llvm::cast<RankedTensorType>(Val: castOp->getResult(idx: 0).getType());
1226 ArrayRef<int64_t> resultShape = resultType.getShape();
1227 SmallVector<OpFoldResult> currMixedSizes = producer.getMixedSizes();
1228 SmallVector<OpFoldResult> newMixedSizes;
1229 newMixedSizes.reserve(N: currMixedSizes.size());
1230 assert(resultShape.size() == currMixedSizes.size() &&
1231 "mismatch in result shape and sizes of empty op");
1232 for (auto [newDim, currDim] : llvm::zip(t&: resultShape, u&: currMixedSizes)) {
1233 // Case 1: The empty tensor dim is static. Check that the tensor cast
1234 // result dim matches.
1235 if (auto attr = llvm::dyn_cast_if_present<Attribute>(Val&: currDim)) {
1236 if (ShapedType::isDynamic(dValue: newDim) ||
1237 newDim != llvm::cast<IntegerAttr>(Val&: attr).getInt()) {
1238 // Something is off, the cast result shape cannot be more dynamic
1239 // than the empty tensor result shape (enforced by
1240 // `canFoldIntoProducer`). Abort for now.
1241 return rewriter.notifyMatchFailure(
1242 arg&: producer, msg: "mismatch in static value of shape of empty tensor "
1243 "result and cast result");
1244 }
1245 newMixedSizes.push_back(Elt: attr);
1246 continue;
1247 }
1248
1249 // Case 2 : The tensor cast shape is static, but empty tensor result
1250 // shape is dynamic.
1251 if (ShapedType::isStatic(dValue: newDim)) {
1252 newMixedSizes.push_back(Elt: rewriter.getIndexAttr(value: newDim));
1253 continue;
1254 }
1255
1256 // Case 3 : The tensor cast shape is dynamic and empty tensor result
1257 // shape is dynamic. Use the dynamic value from the empty tensor op.
1258 newMixedSizes.push_back(Elt: currDim);
1259 }
1260
1261 rewriter.replaceOpWithNewOp<EmptyOp>(op: castOp, args&: newMixedSizes,
1262 args: resultType.getElementType(),
1263 args: resultType.getEncoding());
1264 return success();
1265 }
1266};
1267
1268} // namespace
1269
1270void EmptyOp::getCanonicalizationPatterns(RewritePatternSet &results,
1271 MLIRContext *context) {
1272 results.add<FoldEmptyTensorWithCastOp, FoldEmptyTensorWithDimOp,
1273 ReplaceEmptyTensorStaticShapeDims>(arg&: context);
1274}
1275
1276//===----------------------------------------------------------------------===//
1277// ExtractOp
1278//===----------------------------------------------------------------------===//
1279
1280namespace {
1281
1282/// Canonicalizes the pattern of the form
1283///
1284/// %val = tensor.cast %source : : tensor<?xi32> to tensor<2xi32>
1285/// %extracted_element = tensor.extract %val[%c0] : tensor<2xi32>
1286///
1287/// to
1288///
1289/// %extracted_element = tensor.extract %source[%c0] : tensor<?xi32>
1290struct ExtractFromTensorCast : public OpRewritePattern<tensor::ExtractOp> {
1291 using OpRewritePattern<tensor::ExtractOp>::OpRewritePattern;
1292
1293 LogicalResult matchAndRewrite(tensor::ExtractOp extract,
1294 PatternRewriter &rewriter) const final {
1295 auto tensorCast = extract.getTensor().getDefiningOp<tensor::CastOp>();
1296 if (!tensorCast)
1297 return failure();
1298 if (!llvm::isa<RankedTensorType>(Val: tensorCast.getSource().getType()))
1299 return failure();
1300 rewriter.replaceOpWithNewOp<tensor::ExtractOp>(
1301 op: extract, args: tensorCast.getSource(), args: extract.getIndices());
1302 return success();
1303 }
1304};
1305
1306/// Canonicalizes the pattern of the form
1307///
1308/// %val = tensor.collapse_shape %src[[0, 1]] : tensor<3x4xf64> into
1309/// tensor<12xf64>
1310/// %extracted_element = tensor.extract %val[%c10] :
1311/// tensor<12xf64>
1312///
1313/// to
1314///
1315/// %extracted_element = tensor.extract %src[%c2, %c2] : tensor<3x4xf64>
1316struct ExtractFromCollapseShape : public OpRewritePattern<tensor::ExtractOp> {
1317 using OpRewritePattern<tensor::ExtractOp>::OpRewritePattern;
1318
1319 LogicalResult matchAndRewrite(tensor::ExtractOp extractOp,
1320 PatternRewriter &rewriter) const final {
1321 auto collapseOp =
1322 extractOp.getTensor().getDefiningOp<tensor::CollapseShapeOp>();
1323 if (!collapseOp)
1324 return failure();
1325 if (!collapseOp.getSrcType().hasStaticShape())
1326 return failure();
1327
1328 auto sourceSizes = collapseOp.getSrcType().getShape();
1329
1330 SmallVector<Value> indices(extractOp.getIndices().begin(),
1331 extractOp.getIndices().end());
1332 SmallVector<Value> sourceIndices;
1333 for (auto [index, group] :
1334 llvm::zip(t&: indices, u: collapseOp.getReassociationIndices())) {
1335 assert(!group.empty() && "association indices groups cannot be empty");
1336 auto groupSize = group.size();
1337
1338 if (groupSize == 1) {
1339 sourceIndices.push_back(Elt: index);
1340 continue;
1341 }
1342
1343 SmallVector<int64_t> basis =
1344 llvm::map_to_vector(C&: group, F: [&](int64_t d) { return sourceSizes[d]; });
1345 auto delinearize = affine::AffineDelinearizeIndexOp::create(
1346 builder&: rewriter, location: extractOp.getLoc(), linear_index: index, basis, /*hasOuterBound=*/true);
1347 llvm::append_range(C&: sourceIndices, R: delinearize.getResults());
1348 }
1349 if (collapseOp.getReassociationIndices().empty()) {
1350 auto zeroAffineMap = rewriter.getConstantAffineMap(val: 0);
1351 int64_t srcRank =
1352 cast<RankedTensorType>(Val: collapseOp.getSrcType()).getRank();
1353 OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
1354 b&: rewriter, loc: extractOp.getLoc(), map: zeroAffineMap,
1355 operands: ArrayRef<OpFoldResult>{});
1356 for (int64_t i = 0; i < srcRank; i++) {
1357 sourceIndices.push_back(
1358 Elt: getValueOrCreateConstantIndexOp(b&: rewriter, loc: extractOp.getLoc(), ofr));
1359 }
1360 }
1361
1362 rewriter.replaceOpWithNewOp<tensor::ExtractOp>(
1363 op: extractOp, args: collapseOp.getSrc(), args&: sourceIndices);
1364 return success();
1365 }
1366};
1367
1368/// Canonicalizes the pattern of the form
1369///
1370/// %tensor = tensor.generate %x {
1371/// ^bb0(%arg0: index):
1372/// <computation>
1373/// yield %1 : index
1374/// } : tensor<?xindex>
1375/// %extracted_element = tensor.extract %tensor[%c0] : tensor<?xi32>
1376///
1377/// to just <computation> with %arg0 replaced by %c0. We only do this if the
1378/// tensor.generate operation has no side-effects.
1379struct ExtractFromTensorGenerate : public OpRewritePattern<tensor::ExtractOp> {
1380 using OpRewritePattern<tensor::ExtractOp>::OpRewritePattern;
1381
1382 LogicalResult matchAndRewrite(tensor::ExtractOp extract,
1383 PatternRewriter &rewriter) const final {
1384 auto tensorFromElements = extract.getTensor().getDefiningOp<GenerateOp>();
1385 if (!tensorFromElements || !wouldOpBeTriviallyDead(op: tensorFromElements))
1386 return failure();
1387
1388 IRMapping mapping;
1389 Block *body = &tensorFromElements.getBody().front();
1390 mapping.map(from: body->getArguments(), to: extract.getIndices());
1391 for (auto &op : body->without_terminator())
1392 rewriter.clone(op, mapper&: mapping);
1393
1394 auto yield = cast<YieldOp>(Val: body->getTerminator());
1395
1396 rewriter.replaceOp(op: extract, newValues: mapping.lookupOrDefault(from: yield.getValue()));
1397 return success();
1398 }
1399};
1400
1401} // namespace
1402
1403void ExtractOp::getAsmResultNames(
1404 function_ref<void(Value, StringRef)> setNameFn) {
1405 setNameFn(getResult(), "extracted");
1406}
1407
1408LogicalResult ExtractOp::verify() {
1409 // Verify the # indices match if we have a ranked type.
1410 auto tensorType = llvm::cast<RankedTensorType>(Val: getTensor().getType());
1411 if (tensorType.getRank() != static_cast<int64_t>(getIndices().size()))
1412 return emitOpError(message: "incorrect number of indices for extract_element");
1413 return success();
1414}
1415
1416/// If we have an ExtractOp consuming an InsertOp with the same
1417/// indices, we can return the InsertOp's scalar directly.
1418// TODO: This only checks the immediate producer; extend to go up the
1419// insert/extract chain if the slices are disjoint.
1420static Value foldExtractAfterInsert(ExtractOp extractOp) {
1421 auto insertOp = extractOp.getTensor().getDefiningOp<InsertOp>();
1422
1423 auto isSame = [](Value a, Value b) {
1424 return getAsOpFoldResult(val: a) == getAsOpFoldResult(val: b);
1425 };
1426 if (insertOp && insertOp.getScalar().getType() == extractOp.getType() &&
1427 llvm::equal(LRange: insertOp.getIndices(), RRange: extractOp.getIndices(), P: isSame))
1428 return insertOp.getScalar();
1429
1430 return {};
1431}
1432
1433OpFoldResult ExtractOp::fold(FoldAdaptor adaptor) {
1434 if (Attribute tensor = adaptor.getTensor()) {
1435 // If this is a splat elements attribute, simply return the value.
1436 // All of the elements of a splat attribute are the same.
1437 if (auto splatTensor = llvm::dyn_cast<SplatElementsAttr>(Val&: tensor))
1438 return splatTensor.getSplatValue<Attribute>();
1439
1440 // If this is a dense resource elements attribute, return.
1441 if (isa<DenseResourceElementsAttr>(Val: tensor))
1442 return {};
1443 }
1444
1445 if (Value result = foldExtractAfterInsert(extractOp: *this))
1446 return result;
1447
1448 // Collect the constant indices into the tensor.
1449 SmallVector<uint64_t, 8> indices;
1450 for (Attribute indice : adaptor.getIndices()) {
1451 if (!indice || !llvm::isa<IntegerAttr>(Val: indice))
1452 return {};
1453 indices.push_back(Elt: llvm::cast<IntegerAttr>(Val&: indice).getInt());
1454 }
1455
1456 // Fold extract(from_elements(...)).
1457 if (auto fromElementsOp = getTensor().getDefiningOp<FromElementsOp>()) {
1458 auto tensorType = llvm::cast<RankedTensorType>(Val: fromElementsOp.getType());
1459 auto rank = tensorType.getRank();
1460 assert(static_cast<int64_t>(indices.size()) == tensorType.getRank() &&
1461 "rank mismatch");
1462 int flatIndex = 0;
1463 int stride = 1;
1464 for (int i = rank - 1; i >= 0; --i) {
1465 flatIndex += indices[i] * stride;
1466 stride *= tensorType.getDimSize(idx: i);
1467 }
1468 // Prevent out of bounds accesses. This can happen in invalid code that
1469 // will never execute.
1470 if (static_cast<int>(fromElementsOp.getElements().size()) <= flatIndex ||
1471 flatIndex < 0)
1472 return {};
1473 return fromElementsOp.getElements()[flatIndex];
1474 }
1475
1476 // If this is an elements attribute, query the value at the given indices.
1477 if (Attribute tensor = adaptor.getTensor()) {
1478 auto elementsAttr = llvm::dyn_cast<ElementsAttr>(Val&: tensor);
1479 if (elementsAttr && elementsAttr.isValidIndex(index: indices))
1480 return elementsAttr.getValues<Attribute>()[indices];
1481 }
1482
1483 return {};
1484}
1485
1486void ExtractOp::getCanonicalizationPatterns(RewritePatternSet &results,
1487 MLIRContext *context) {
1488 results.add<ExtractFromTensorCast, ExtractFromTensorGenerate>(arg&: context);
1489}
1490
1491void mlir::tensor::populateFoldCollapseExtractPatterns(
1492 RewritePatternSet &patterns) {
1493 patterns.add<ExtractFromCollapseShape>(arg: patterns.getContext());
1494}
1495
1496//===----------------------------------------------------------------------===//
1497// FromElementsOp
1498//===----------------------------------------------------------------------===//
1499
1500void FromElementsOp::getAsmResultNames(
1501 function_ref<void(Value, StringRef)> setNameFn) {
1502 setNameFn(getResult(), "from_elements");
1503}
1504
1505void FromElementsOp::build(OpBuilder &builder, OperationState &result,
1506 ValueRange elements) {
1507 assert(!elements.empty() && "expected at least one element");
1508 Type resultType = RankedTensorType::get(
1509 shape: {static_cast<int64_t>(elements.size())}, elementType: elements.front().getType());
1510 build(odsBuilder&: builder, odsState&: result, result: resultType, elements);
1511}
1512
1513OpFoldResult FromElementsOp::fold(FoldAdaptor adaptor) {
1514 // DenseElementsAttr::get requires StringAttr for element types that are not
1515 // integer, index, float, or complex (e.g. vector types), but folded constants
1516 // won't be StringAttr instances. Only fold for element types directly
1517 // supported by DenseElementsAttr.
1518 Type eltType = getType().getElementType();
1519 if (!eltType.isIntOrIndexOrFloat() && !isa<ComplexType>(Val: eltType))
1520 return {};
1521 if (!llvm::is_contained(Range: adaptor.getElements(), Element: nullptr))
1522 return DenseElementsAttr::get(type: getType(), values: adaptor.getElements());
1523 return {};
1524}
1525
1526namespace {
1527
1528// Pushes the index_casts that occur before extractions to after the extract.
1529// This minimizes type conversion in some cases and enables the extract
1530// canonicalizer. This changes:
1531//
1532// %cast = arith.index_cast %tensor : tensor<1xi32> to tensor<1xindex>
1533// %extract = tensor.extract %cast[%index] : tensor<1xindex>
1534//
1535// to the following:
1536//
1537// %extract = tensor.extract %tensor[%index] : tensor<1xindex>
1538// %cast = arith.index_cast %extract : i32 to index
1539//
1540// to just %element.
1541//
1542// Consider expanding this to a template and handle all tensor cast
1543// operations.
1544struct ExtractElementFromIndexCast
1545 : public OpRewritePattern<tensor::ExtractOp> {
1546 using OpRewritePattern<tensor::ExtractOp>::OpRewritePattern;
1547
1548 LogicalResult matchAndRewrite(tensor::ExtractOp extract,
1549 PatternRewriter &rewriter) const final {
1550 Location loc = extract.getLoc();
1551 auto indexCast = extract.getTensor().getDefiningOp<arith::IndexCastOp>();
1552 if (!indexCast)
1553 return failure();
1554
1555 Type elementTy = getElementTypeOrSelf(val: indexCast.getIn());
1556
1557 auto newExtract = tensor::ExtractOp::create(
1558 builder&: rewriter, location: loc, result: elementTy, tensor: indexCast.getIn(), indices: extract.getIndices());
1559
1560 rewriter.replaceOpWithNewOp<arith::IndexCastOp>(op: extract, args: extract.getType(),
1561 args&: newExtract);
1562
1563 return success();
1564 }
1565};
1566
1567} // namespace
1568
1569void FromElementsOp::getCanonicalizationPatterns(RewritePatternSet &results,
1570 MLIRContext *context) {
1571 results.add<ExtractElementFromIndexCast>(arg&: context);
1572}
1573
1574//===----------------------------------------------------------------------===//
1575// GatherOp
1576//===----------------------------------------------------------------------===//
1577
1578void GatherOp::getAsmResultNames(
1579 function_ref<void(Value, StringRef)> setNameFn) {
1580 setNameFn(getResult(), "gather");
1581}
1582
1583/// Return the inferred result type for a gatherOp where:
1584/// - sourceType is the type of the source tensor gathered from
1585/// - indicesType is the type of the indices used to gather
1586/// - gatherDims are the dims along which the gather occurs.
1587/// Return a full rank or ranked-reduced variant of the type depending on
1588/// the value of rankReduced.
1589///
1590/// The leading dimensions of the index tensor give the result tensor its
1591/// leading dimensions.
1592/// The trailing dimensions of the result tensor are obtained from the source
1593/// tensor by setting the dimensions specified in gather_dims to `1` (if
1594/// rankedReduced is false), or skipping them (otherwise).
1595RankedTensorType GatherOp::inferResultType(RankedTensorType sourceType,
1596 RankedTensorType indicesType,
1597 ArrayRef<int64_t> gatherDims,
1598 bool rankReduced) {
1599 SmallVector<int64_t> resultShape(indicesType.getShape().drop_back());
1600 resultShape.reserve(N: resultShape.size() + sourceType.getRank());
1601 for (int64_t idx : llvm::seq<int64_t>(Begin: 0, End: sourceType.getRank())) {
1602 if (llvm::binary_search(Range&: gatherDims, Value&: idx)) {
1603 if (!rankReduced)
1604 resultShape.push_back(Elt: 1);
1605 continue;
1606 }
1607 resultShape.push_back(Elt: sourceType.getDimSize(idx));
1608 }
1609 return RankedTensorType::Builder(sourceType).setShape(resultShape);
1610}
1611
1612static LogicalResult
1613verifyGatherOrScatterDims(Operation *op, ArrayRef<int64_t> dims,
1614 ArrayRef<int64_t> indices, int64_t rank,
1615 StringRef gatherOrScatter, StringRef sourceOrDest) {
1616 if (dims.empty())
1617 return op->emitOpError(message: gatherOrScatter) << "_dims must be non-empty";
1618
1619 int64_t numGatherDims = dims.size();
1620 if (numGatherDims > rank)
1621 return op->emitOpError(message: gatherOrScatter)
1622 << "_dims overflow " << sourceOrDest << " rank";
1623 if (indices.empty() || indices.back() != numGatherDims)
1624 return op->emitOpError(message: gatherOrScatter)
1625 << "_dims length must match the size of last dimension of indices";
1626 for (int64_t val : dims) {
1627 if (val < 0)
1628 return op->emitOpError(message: gatherOrScatter)
1629 << "_dims value must be non-negative";
1630 if (val >= rank)
1631 return op->emitOpError(message: gatherOrScatter)
1632 << "_dims value must be smaller than " << sourceOrDest << " rank";
1633 }
1634 for (int64_t i = 1; i < numGatherDims; ++i) {
1635 if (dims[i - 1] >= dims[i])
1636 return op->emitOpError(message: gatherOrScatter)
1637 << "_dims values must be strictly increasing";
1638 }
1639 return success();
1640}
1641
1642LogicalResult GatherOp::verify() {
1643 int64_t sourceRank = getSourceType().getRank();
1644 ArrayRef<int64_t> gatherDims = getGatherDims();
1645 if (failed(Result: verifyGatherOrScatterDims(op: getOperation(), dims: gatherDims,
1646 indices: getIndicesType().getShape(), rank: sourceRank,
1647 gatherOrScatter: "gather", sourceOrDest: "source")))
1648 return failure();
1649
1650 RankedTensorType expectedResultType = GatherOp::inferResultType(
1651 sourceType: getSourceType(), indicesType: getIndicesType(), gatherDims, /*rankReduced=*/false);
1652 RankedTensorType expectedRankReducedResultType = GatherOp::inferResultType(
1653 sourceType: getSourceType(), indicesType: getIndicesType(), gatherDims, /*rankReduced=*/true);
1654 if (getResultType() != expectedResultType &&
1655 getResultType() != expectedRankReducedResultType) {
1656 return emitOpError(message: "result type "
1657 "mismatch: "
1658 "expected ")
1659 << expectedResultType << " or its rank-reduced variant "
1660 << expectedRankReducedResultType << " (got: " << getResultType()
1661 << ")";
1662 }
1663
1664 return success();
1665}
1666
1667OpFoldResult GatherOp::fold(FoldAdaptor adaptor) {
1668 if (OpFoldResult reshapedSource = reshapeConstantSource(
1669 source: llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getSource()),
1670 result: getResult().getType()))
1671 return reshapedSource;
1672 return {};
1673}
1674
1675//===----------------------------------------------------------------------===//
1676// InsertOp
1677//===----------------------------------------------------------------------===//
1678
1679void InsertOp::getAsmResultNames(
1680 function_ref<void(Value, StringRef)> setNameFn) {
1681 setNameFn(getResult(), "inserted");
1682}
1683
1684LogicalResult InsertOp::verify() {
1685 // Verify the # indices match if we have a ranked type.
1686 auto destType = llvm::cast<RankedTensorType>(Val: getDest().getType());
1687 if (destType.getRank() != static_cast<int64_t>(getIndices().size()))
1688 return emitOpError(message: "incorrect number of indices");
1689 return success();
1690}
1691
1692OpFoldResult InsertOp::fold(FoldAdaptor adaptor) {
1693 Attribute scalar = adaptor.getScalar();
1694 Attribute dest = adaptor.getDest();
1695 if (scalar && dest)
1696 if (auto splatDest = llvm::dyn_cast<SplatElementsAttr>(Val&: dest))
1697 if (scalar == splatDest.getSplatValue<Attribute>())
1698 return dest;
1699 return {};
1700}
1701
1702//===----------------------------------------------------------------------===//
1703// GenerateOp
1704//===----------------------------------------------------------------------===//
1705
1706void GenerateOp::getAsmResultNames(
1707 function_ref<void(Value, StringRef)> setNameFn) {
1708 setNameFn(getResult(), "generated");
1709}
1710
1711LogicalResult GenerateOp::reifyResultShapes(
1712 OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
1713 reifiedReturnShapes.resize(N: 1, NV: SmallVector<OpFoldResult>(getType().getRank()));
1714 int idx = 0;
1715 for (auto dim : llvm::seq<int64_t>(Begin: 0, End: getType().getRank())) {
1716 if (getType().isDynamicDim(idx: dim)) {
1717 reifiedReturnShapes[0][dim] = getOperand(i: idx++);
1718 } else {
1719 reifiedReturnShapes[0][dim] =
1720 builder.getIndexAttr(value: getType().getDimSize(idx: dim));
1721 }
1722 }
1723 return success();
1724}
1725
1726LogicalResult GenerateOp::verify() {
1727 // Ensure that the tensor type has as many dynamic dimensions as are
1728 // specified by the operands.
1729 RankedTensorType resultType = llvm::cast<RankedTensorType>(Val: getType());
1730 if (failed(Result: verifyDynamicDimensionCount(op: getOperation(), type: resultType,
1731 dynamicSizes: getOperands())))
1732 return failure();
1733 return success();
1734}
1735
1736LogicalResult GenerateOp::verifyRegions() {
1737 RankedTensorType resultTy = llvm::cast<RankedTensorType>(Val: getType());
1738 // Ensure that region arguments span the index space.
1739 if (!llvm::all_of(Range: getBody().getArgumentTypes(),
1740 P: [](Type ty) { return ty.isIndex(); }))
1741 return emitError(message: "all body arguments must be index");
1742 if (getBody().getNumArguments() != resultTy.getRank())
1743 return emitError(message: "must have one body argument per input dimension");
1744
1745 // Ensure that the region yields an element of the right type.
1746 auto yieldOp = cast<YieldOp>(Val: getBody().getBlocks().front().getTerminator());
1747
1748 if (yieldOp.getValue().getType() != resultTy.getElementType())
1749 return emitOpError(
1750 message: "body must be terminated with a `yield` operation of the tensor "
1751 "element type");
1752
1753 return success();
1754}
1755
1756void GenerateOp::build(
1757 OpBuilder &b, OperationState &result, Type resultTy,
1758 ValueRange dynamicExtents,
1759 function_ref<void(OpBuilder &, Location, ValueRange)> bodyBuilder) {
1760 build(odsBuilder&: b, odsState&: result, result: resultTy, dynamicExtents);
1761
1762 // Build and populate body.
1763 OpBuilder::InsertionGuard guard(b);
1764 Region *bodyRegion = result.regions.front().get();
1765 auto rank = llvm::cast<RankedTensorType>(Val&: resultTy).getRank();
1766 SmallVector<Type, 2> argumentTypes(rank, b.getIndexType());
1767 SmallVector<Location, 2> argumentLocs(rank, result.location);
1768 Block *bodyBlock =
1769 b.createBlock(parent: bodyRegion, insertPt: bodyRegion->end(), argTypes: argumentTypes, locs: argumentLocs);
1770 bodyBuilder(b, result.location, bodyBlock->getArguments());
1771}
1772
1773namespace {
1774
1775/// Canonicalizes tensor.generate operations with a constant
1776/// operand into the equivalent operation with the operand expressed in the
1777/// result type, instead. We also insert a type cast to make sure that the
1778/// resulting IR is still well-typed.
1779struct StaticTensorGenerate : public OpRewritePattern<GenerateOp> {
1780 using OpRewritePattern<GenerateOp>::OpRewritePattern;
1781
1782 LogicalResult matchAndRewrite(GenerateOp generateOp,
1783 PatternRewriter &rewriter) const final {
1784 SmallVector<Value> foldedDynamicSizes;
1785 RankedTensorType foldedTensorType = foldDynamicToStaticDimSizes(
1786 type: generateOp.getType(), dynamicSizes: generateOp.getDynamicExtents(),
1787 foldedDynamicSizes);
1788
1789 // Stop here if no dynamic size was promoted to static.
1790 if (foldedTensorType == generateOp.getType())
1791 return failure();
1792
1793 auto loc = generateOp.getLoc();
1794 auto newOp =
1795 GenerateOp::create(builder&: rewriter, location: loc, result: foldedTensorType, dynamicExtents: foldedDynamicSizes);
1796 rewriter.inlineRegionBefore(region&: generateOp.getBody(), parent&: newOp.getBody(),
1797 before: newOp.getBody().begin());
1798 rewriter.replaceOpWithNewOp<tensor::CastOp>(op: generateOp,
1799 args: generateOp.getType(), args&: newOp);
1800 return success();
1801 }
1802};
1803} // namespace
1804
1805void GenerateOp::getCanonicalizationPatterns(RewritePatternSet &results,
1806 MLIRContext *context) {
1807 results.add<StaticTensorGenerate>(arg&: context);
1808}
1809
1810//===----------------------------------------------------------------------===//
1811// RankOp
1812//===----------------------------------------------------------------------===//
1813
1814void RankOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
1815 setNameFn(getResult(), "rank");
1816}
1817
1818OpFoldResult RankOp::fold(FoldAdaptor adaptor) {
1819 // Constant fold rank when the rank of the operand is known.
1820 auto type = getOperand().getType();
1821 auto shapedType = llvm::dyn_cast<ShapedType>(Val&: type);
1822 if (shapedType && shapedType.hasRank())
1823 return IntegerAttr::get(type: IndexType::get(context: getContext()), value: shapedType.getRank());
1824 return IntegerAttr();
1825}
1826
1827//===----------------------------------------------------------------------===//
1828// ReshapeOp
1829//===----------------------------------------------------------------------===//
1830
1831void ReshapeOp::getAsmResultNames(
1832 function_ref<void(Value, StringRef)> setNameFn) {
1833 setNameFn(getResult(), "reshape");
1834}
1835
1836static int64_t getNumElements(ShapedType type) {
1837 int64_t numElements = 1;
1838 for (auto dim : type.getShape())
1839 numElements *= dim;
1840 return numElements;
1841}
1842
1843LogicalResult ReshapeOp::verify() {
1844 TensorType operandType = llvm::cast<TensorType>(Val: getSource().getType());
1845 TensorType resultType = llvm::cast<TensorType>(Val: getResult().getType());
1846
1847 if (operandType.getElementType() != resultType.getElementType())
1848 return emitOpError(message: "element types of source and destination tensor "
1849 "types should be the same");
1850
1851 int64_t shapeSize =
1852 llvm::cast<RankedTensorType>(Val: getShape().getType()).getDimSize(idx: 0);
1853 auto resultRankedType = llvm::dyn_cast<RankedTensorType>(Val&: resultType);
1854 auto operandRankedType = llvm::dyn_cast<RankedTensorType>(Val&: operandType);
1855
1856 if (resultRankedType) {
1857 if (operandRankedType && resultRankedType.hasStaticShape() &&
1858 operandRankedType.hasStaticShape()) {
1859 if (getNumElements(type: operandRankedType) != getNumElements(type: resultRankedType))
1860 return emitOpError(message: "source and destination tensor should have the "
1861 "same number of elements");
1862 }
1863 if (ShapedType::isDynamic(dValue: shapeSize))
1864 return emitOpError(message: "cannot use shape operand with dynamic length to "
1865 "reshape to statically-ranked tensor type");
1866 if (shapeSize != resultRankedType.getRank())
1867 return emitOpError(
1868 message: "length of shape operand differs from the result's tensor rank");
1869 }
1870 return success();
1871}
1872
1873OpFoldResult ReshapeOp::fold(FoldAdaptor adaptor) {
1874 if (OpFoldResult reshapedSource = reshapeConstantSource(
1875 source: llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getSource()),
1876 result: getResult().getType()))
1877 return reshapedSource;
1878
1879 // If the producer of operand 'source' is another 'tensor.reshape' op, use the
1880 // producer's input instead as the original tensor to reshape. This could
1881 // render such producer dead code.
1882 if (auto reshapeOpProducer = getSource().getDefiningOp<ReshapeOp>()) {
1883 getSourceMutable().assign(value: reshapeOpProducer.getSource());
1884 return getResult();
1885 }
1886
1887 auto source = getSource();
1888 auto sourceTy = dyn_cast<RankedTensorType>(Val: source.getType());
1889 auto resultTy = dyn_cast<RankedTensorType>(Val: getType());
1890 if (!sourceTy || !resultTy || sourceTy != resultTy)
1891 return {};
1892
1893 // If the source and result are both 0D or 1D tensors and have the same type,
1894 // the reshape has no effect, even if the tensor is dynamically shaped.
1895 if (sourceTy.getRank() <= 1)
1896 return source;
1897
1898 if (auto fromElements = getShape().getDefiningOp<tensor::FromElementsOp>()) {
1899 auto elements = fromElements.getElements();
1900 bool dynamicNoop =
1901 sourceTy.getRank() == static_cast<int64_t>(elements.size());
1902 for (int id = 0, s = elements.size(); id < s && dynamicNoop; ++id) {
1903 auto element = elements[id];
1904
1905 if (auto cst = getConstantIntValue(ofr: element)) {
1906 dynamicNoop &= cst.value() == sourceTy.getDimSize(idx: id);
1907 continue;
1908 }
1909
1910 if (auto dimOp = element.getDefiningOp<tensor::DimOp>()) {
1911 dynamicNoop &= dimOp.getSource() == source;
1912
1913 auto cst = getConstantIntValue(ofr: dimOp.getIndex());
1914 dynamicNoop &=
1915 cst.has_value() && cst.value() == static_cast<int64_t>(id);
1916 continue;
1917 }
1918
1919 dynamicNoop = false;
1920 break;
1921 }
1922
1923 if (dynamicNoop)
1924 return source;
1925 }
1926
1927 return {};
1928}
1929
1930//===----------------------------------------------------------------------===//
1931// Reassociative reshape ops
1932//===----------------------------------------------------------------------===//
1933
1934void CollapseShapeOp::getAsmResultNames(
1935 function_ref<void(Value, StringRef)> setNameFn) {
1936 setNameFn(getResult(), "collapsed");
1937}
1938
1939void ExpandShapeOp::getAsmResultNames(
1940 function_ref<void(Value, StringRef)> setNameFn) {
1941 setNameFn(getResult(), "expanded");
1942}
1943
1944int64_t ExpandShapeOp::getCorrespondingSourceDim(int64_t resultDim) {
1945 assert(resultDim >= 0 && resultDim < getResultType().getRank() &&
1946 "invalid resultDim");
1947 for (const auto &it : llvm::enumerate(First: getReassociationIndices()))
1948 if (llvm::is_contained(Range&: it.value(), Element: resultDim))
1949 return it.index();
1950 llvm_unreachable("could not find reassociation group");
1951}
1952
1953FailureOr<SmallVector<OpFoldResult>>
1954ExpandShapeOp::inferOutputShape(OpBuilder &b, Location loc,
1955 RankedTensorType expandedType,
1956 ArrayRef<ReassociationIndices> reassociation,
1957 ArrayRef<OpFoldResult> inputShape) {
1958 std::optional<SmallVector<OpFoldResult>> outputShape =
1959 inferExpandShapeOutputShape(b, loc, expandedType, reassociation,
1960 inputShape);
1961 if (!outputShape)
1962 return failure();
1963 return *outputShape;
1964}
1965
1966SmallVector<OpFoldResult> ExpandShapeOp::getMixedOutputShape() {
1967 return getMixedValues(staticValues: getStaticOutputShape(), dynamicValues: getOutputShape(), context: getContext());
1968}
1969
1970void ExpandShapeOp::build(OpBuilder &builder, OperationState &result,
1971 Type resultType, Value src,
1972 ArrayRef<ReassociationIndices> reassociation,
1973 ArrayRef<OpFoldResult> outputShape) {
1974 auto [staticOutputShape, dynamicOutputShape] =
1975 decomposeMixedValues(mixedValues: SmallVector<OpFoldResult>(outputShape));
1976 build(odsBuilder&: builder, odsState&: result, result: cast<RankedTensorType>(Val&: resultType), src,
1977 reassociation: getReassociationIndicesAttribute(b&: builder, reassociation),
1978 output_shape: dynamicOutputShape, static_output_shape: staticOutputShape);
1979}
1980
1981void ExpandShapeOp::build(OpBuilder &builder, OperationState &result,
1982 Type resultType, Value src,
1983 ArrayRef<ReassociationIndices> reassociation) {
1984 SmallVector<OpFoldResult> inputShape =
1985 getMixedSizes(builder, loc: result.location, value: src);
1986 auto tensorResultTy = cast<RankedTensorType>(Val&: resultType);
1987 FailureOr<SmallVector<OpFoldResult>> outputShape = inferOutputShape(
1988 b&: builder, loc: result.location, expandedType: tensorResultTy, reassociation, inputShape);
1989 SmallVector<OpFoldResult> outputShapeOrEmpty;
1990 if (succeeded(Result: outputShape)) {
1991 outputShapeOrEmpty = *outputShape;
1992 }
1993 build(builder, result, resultType: tensorResultTy, src, reassociation,
1994 outputShape: outputShapeOrEmpty);
1995}
1996
1997SmallVector<AffineMap, 4> CollapseShapeOp::getReassociationMaps() {
1998 return getSymbolLessAffineMaps(reassociation: getReassociationExprs());
1999}
2000SmallVector<ReassociationExprs, 4> CollapseShapeOp::getReassociationExprs() {
2001 return convertReassociationIndicesToExprs(context: getContext(),
2002 reassociationIndices: getReassociationIndices());
2003}
2004
2005SmallVector<AffineMap, 4> ExpandShapeOp::getReassociationMaps() {
2006 return getSymbolLessAffineMaps(reassociation: getReassociationExprs());
2007}
2008SmallVector<ReassociationExprs, 4> ExpandShapeOp::getReassociationExprs() {
2009 return convertReassociationIndicesToExprs(context: getContext(),
2010 reassociationIndices: getReassociationIndices());
2011}
2012
2013RankedTensorType CollapseShapeOp::inferCollapsedType(
2014 RankedTensorType type, ArrayRef<ReassociationIndices> reassociation) {
2015 return inferCollapsedType(
2016 type, reassociation: getSymbolLessAffineMaps(reassociation: convertReassociationIndicesToExprs(
2017 context: type.getContext(), reassociationIndices: reassociation)));
2018}
2019
2020/// Compute the RankedTensorType obtained by applying `reassociation` to
2021/// `type`.
2022RankedTensorType
2023CollapseShapeOp::inferCollapsedType(RankedTensorType type,
2024 ArrayRef<AffineMap> reassociation) {
2025 auto shape = type.getShape();
2026 SmallVector<int64_t, 4> newShape;
2027 newShape.reserve(N: reassociation.size());
2028
2029 // Use the fact that reassociation is valid to simplify the logic: only use
2030 // each map's rank.
2031 assert(isReassociationValid(reassociation) && "invalid reassociation");
2032 unsigned currentDim = 0;
2033 for (AffineMap m : reassociation) {
2034 unsigned dim = m.getNumResults();
2035 auto band = shape.slice(N: currentDim, M: dim);
2036 int64_t size = 1;
2037 if (llvm::is_contained(Range&: band, Element: ShapedType::kDynamic))
2038 size = ShapedType::kDynamic;
2039 else
2040 for (unsigned d = 0; d < dim; ++d)
2041 size *= shape[currentDim + d];
2042 newShape.push_back(Elt: size);
2043 currentDim += dim;
2044 }
2045
2046 return RankedTensorType::get(shape: newShape, elementType: type.getElementType());
2047}
2048
2049void CollapseShapeOp::build(OpBuilder &b, OperationState &result, Value src,
2050 ArrayRef<ReassociationIndices> reassociation,
2051 ArrayRef<NamedAttribute> attrs) {
2052 auto srcType = llvm::cast<RankedTensorType>(Val: src.getType());
2053 RankedTensorType collapsedType = inferCollapsedType(type: srcType, reassociation);
2054 auto resultType =
2055 RankedTensorType::get(shape: collapsedType.getShape(), elementType: srcType.getElementType(),
2056 encoding: srcType.getEncoding());
2057 buildPropertiesAndDiscardableAttributes(odsState&: result, attributes: attrs);
2058 result.getOrAddProperties<Properties>().reassociation =
2059 getReassociationIndicesAttribute(b, reassociation);
2060 result.addOperands(newOperands: src);
2061 result.addTypes(newTypes: resultType);
2062}
2063
2064template <typename TensorReshapeOp, bool isExpansion = std::is_same<
2065 TensorReshapeOp, ExpandShapeOp>::value>
2066static LogicalResult verifyTensorReshapeOp(TensorReshapeOp op,
2067 RankedTensorType expandedType,
2068 RankedTensorType collapsedType) {
2069 if (failed(
2070 verifyReshapeLikeTypes(op, expandedType, collapsedType, isExpansion)))
2071 return failure();
2072
2073 // Reshape must preserve the number of elements when statically known.
2074 if (expandedType.hasStaticShape() && collapsedType.hasStaticShape()) {
2075 std::optional<int64_t> expandedNumElements =
2076 ShapedType::tryGetNumElements(shape: expandedType.getShape());
2077 std::optional<int64_t> collapsedNumElements =
2078 ShapedType::tryGetNumElements(shape: collapsedType.getShape());
2079 if (!expandedNumElements || !collapsedNumElements)
2080 return op.emitOpError("number of elements exceeds the int64 range");
2081 if (*expandedNumElements != *collapsedNumElements)
2082 return op.emitOpError("number of elements must be preserved: ")
2083 << *expandedNumElements << " != " << *collapsedNumElements;
2084 }
2085
2086 auto maps = op.getReassociationMaps();
2087 RankedTensorType expectedType =
2088 CollapseShapeOp::inferCollapsedType(expandedType, maps);
2089 if (!isSameTypeWithoutEncoding(tp1: collapsedType, tp2: expectedType))
2090 return op.emitOpError("expected collapsed type to be ")
2091 << expectedType << ", but got " << collapsedType;
2092 return success();
2093}
2094
2095LogicalResult ExpandShapeOp::verify() {
2096 if (failed(Result: verifyReassociationIndicesNotEmpty(op: *this)))
2097 return failure();
2098
2099 RankedTensorType srcType = getSrc().getType();
2100 RankedTensorType resultType = getResult().getType();
2101
2102 if ((int64_t)getStaticOutputShape().size() != resultType.getRank())
2103 return emitOpError(message: "expected number of static shape dims to be equal to "
2104 "the output rank (")
2105 << resultType.getRank() << ") but found "
2106 << getStaticOutputShape().size() << " inputs instead";
2107
2108 if ((int64_t)getOutputShape().size() !=
2109 llvm::count(Range: getStaticOutputShape(), Element: ShapedType::kDynamic))
2110 return emitOpError(message: "mismatch in dynamic dims in output_shape and "
2111 "static_output_shape: static_output_shape has ")
2112 << llvm::count(Range: getStaticOutputShape(), Element: ShapedType::kDynamic)
2113 << " dynamic dims while output_shape has " << getOutputShape().size()
2114 << " values";
2115
2116 // Verify that the number of dynamic dims in output_shape matches the number
2117 // of dynamic dims in the result type.
2118 if (failed(Result: verifyDynamicDimensionCount(op: getOperation(), type: resultType,
2119 dynamicSizes: getOutputShape())))
2120 return failure();
2121
2122 // Verify if provided output shapes are in agreement with output type.
2123 DenseI64ArrayAttr staticOutputShapes = getStaticOutputShapeAttr();
2124 ArrayRef<int64_t> resShape = getResult().getType().getShape();
2125 for (auto [pos, shape] : llvm::enumerate(First&: resShape))
2126 if (ShapedType::isStatic(dValue: shape) && shape != staticOutputShapes[pos])
2127 return emitOpError(message: "invalid output shape provided at pos ") << pos;
2128
2129 return verifyTensorReshapeOp(op: *this, expandedType: resultType, collapsedType: srcType);
2130}
2131
2132LogicalResult CollapseShapeOp::verify() {
2133 CollapseShapeOp op = *this;
2134 if (failed(Result: verifyReassociationIndicesNotEmpty(op)))
2135 return failure();
2136
2137 RankedTensorType srcType = op.getSrc().getType();
2138 RankedTensorType resultType = op.getResult().getType();
2139
2140 return verifyTensorReshapeOp(op, expandedType: srcType, collapsedType: resultType);
2141}
2142
2143namespace {
2144/// Reshape of a splat constant can be replaced with a constant of the result
2145/// type.
2146template <typename TensorReshapeOp>
2147struct FoldReshapeWithConstant : OpRewritePattern<TensorReshapeOp> {
2148 using OpRewritePattern<TensorReshapeOp>::OpRewritePattern;
2149 LogicalResult matchAndRewrite(TensorReshapeOp reshapeOp,
2150 PatternRewriter &rewriter) const override {
2151 DenseElementsAttr attr;
2152 if (!matchPattern(reshapeOp.getSrc(), m_Constant(bind_value: &attr)))
2153 return failure();
2154 if (!attr || !attr.isSplat())
2155 return failure();
2156 // DenseElementsAttr requires a static shape; skip folding for dynamic
2157 // result types.
2158 if (!reshapeOp.getResultType().hasStaticShape())
2159 return failure();
2160 DenseElementsAttr newAttr = DenseElementsAttr::getFromRawBuffer(
2161 type: reshapeOp.getResultType(), rawBuffer: attr.getRawData());
2162 rewriter.replaceOpWithNewOp<arith::ConstantOp>(reshapeOp, newAttr);
2163 return success();
2164 }
2165};
2166
2167// Folds TensorReshapeOp(splat x : src_type) : res_type into splat x : res_type.
2168template <typename TensorReshapeOp>
2169class FoldReshapeWithSplat : public OpRewritePattern<TensorReshapeOp> {
2170public:
2171 using OpRewritePattern<TensorReshapeOp>::OpRewritePattern;
2172
2173 LogicalResult matchAndRewrite(TensorReshapeOp reshapeOp,
2174 PatternRewriter &rewriter) const override {
2175 auto splatOp = reshapeOp.getSrc().template getDefiningOp<tensor::SplatOp>();
2176 if (!splatOp || !splatOp.getAggregate().getType().hasStaticShape())
2177 return failure();
2178
2179 rewriter.replaceOpWithNewOp<tensor::SplatOp>(
2180 reshapeOp, reshapeOp.getResultType(), splatOp.getInput());
2181 return success();
2182 }
2183};
2184
2185/// Reshape of a FromElements can be replaced with a FromElements of the
2186/// result type
2187template <typename TensorReshapeOp>
2188struct FoldReshapeWithFromElements : OpRewritePattern<TensorReshapeOp> {
2189 using OpRewritePattern<TensorReshapeOp>::OpRewritePattern;
2190 LogicalResult matchAndRewrite(TensorReshapeOp reshapeOp,
2191 PatternRewriter &rewriter) const override {
2192 auto fromElements =
2193 reshapeOp.getSrc().template getDefiningOp<FromElementsOp>();
2194 if (!fromElements)
2195 return failure();
2196
2197 auto shapedTy = llvm::cast<ShapedType>(reshapeOp.getType());
2198
2199 if (!shapedTy.hasStaticShape())
2200 return failure();
2201
2202 rewriter.replaceOpWithNewOp<FromElementsOp>(reshapeOp, reshapeOp.getType(),
2203 fromElements.getElements());
2204 return success();
2205 }
2206};
2207
2208// Fold CastOp into CollapseShapeOp when adding static information.
2209struct FoldCollapseOfCastOp : public OpRewritePattern<CollapseShapeOp> {
2210 using OpRewritePattern<CollapseShapeOp>::OpRewritePattern;
2211
2212 LogicalResult matchAndRewrite(CollapseShapeOp collapseShapeOp,
2213 PatternRewriter &rewriter) const override {
2214 auto castOp = collapseShapeOp.getSrc().getDefiningOp<tensor::CastOp>();
2215 if (!tensor::canFoldIntoConsumerOp(castOp))
2216 return failure();
2217
2218 RankedTensorType srcType =
2219 llvm::cast<RankedTensorType>(Val: castOp.getSource().getType());
2220 RankedTensorType newResultType = CollapseShapeOp::inferCollapsedType(
2221 type: srcType, reassociation: collapseShapeOp.getReassociationMaps());
2222
2223 if (newResultType == collapseShapeOp.getResultType()) {
2224 rewriter.modifyOpInPlace(root: collapseShapeOp, callable: [&]() {
2225 collapseShapeOp.getSrcMutable().assign(value: castOp.getSource());
2226 });
2227 } else {
2228 auto newOp = CollapseShapeOp::create(builder&: rewriter, location: collapseShapeOp.getLoc(),
2229 result: newResultType, src: castOp.getSource(),
2230 reassociation: collapseShapeOp.getReassociation());
2231 rewriter.replaceOpWithNewOp<tensor::CastOp>(
2232 op: collapseShapeOp, args: collapseShapeOp.getResultType(), args&: newOp);
2233 }
2234 return success();
2235 }
2236};
2237
2238/// Fold/sink a producer `tensor.cast` with a consumer `tensor.expand_shape` by
2239/// matching constant output_shape operands of the expand. This makes the
2240/// `tensor.expand_shape` more static and creates a consumer cast that can be
2241/// propagated further.
2242struct ConvertToStaticExpandShape : public OpRewritePattern<ExpandShapeOp> {
2243 using OpRewritePattern<ExpandShapeOp>::OpRewritePattern;
2244
2245 LogicalResult matchAndRewrite(ExpandShapeOp expandOp,
2246 PatternRewriter &rewriter) const override {
2247 auto castOp = expandOp.getSrc().getDefiningOp<CastOp>();
2248 if (!canFoldIntoConsumerOp(castOp))
2249 return failure();
2250
2251 ArrayRef<int64_t> castSrcShape = castOp.getSource().getType().getShape();
2252 SmallVector<ReassociationIndices, 4> reassoc =
2253 expandOp.getReassociationIndices();
2254
2255 SmallVector<int64_t> newOutputShape(expandOp.getResultType().getShape());
2256 SmallVector<Value> dynamicOutputShape;
2257 auto outputIt = expandOp.getOutputShape().begin();
2258
2259 for (const auto &[inputDim, innerReassoc] : llvm::enumerate(First&: reassoc)) {
2260 for (uint64_t outDim : innerReassoc) {
2261 if (ShapedType::isStatic(dValue: newOutputShape[outDim]))
2262 continue;
2263
2264 // If the cast's src type is dynamic, don't infer any of the
2265 // corresponding expanded dimensions. `tensor.expand_shape` requires at
2266 // least one of the expanded dimensions to be dynamic if the input is
2267 // dynamic.
2268 Value val = *outputIt;
2269 ++outputIt;
2270 if (ShapedType::isDynamic(dValue: castSrcShape[inputDim])) {
2271 dynamicOutputShape.push_back(Elt: val);
2272 continue;
2273 }
2274
2275 APInt cst;
2276 if (matchPattern(value: val, pattern: m_ConstantInt(bind_value: &cst))) {
2277 int64_t dim = cst.getSExtValue();
2278 if (dim < 0)
2279 return failure();
2280 newOutputShape[outDim] = dim;
2281 } else {
2282 dynamicOutputShape.push_back(Elt: val);
2283 }
2284 }
2285 }
2286
2287 // Couldn't match any values, nothing to change
2288 if (expandOp.getOutputShape().size() == dynamicOutputShape.size())
2289 return failure();
2290
2291 // Calculate the input shape from the output
2292 SmallVector<int64_t> newInputShape(expandOp.getSrcType().getRank(), 1l);
2293 for (auto inDim : llvm::seq<int>(Begin: 0, End: newInputShape.size())) {
2294 for (auto outDim : reassoc[inDim]) {
2295 auto ofr = newOutputShape[outDim];
2296 if (ShapedType::isDynamic(dValue: ofr)) {
2297 newInputShape[inDim] = ShapedType::kDynamic;
2298 break;
2299 }
2300 newInputShape[inDim] *= ofr;
2301 }
2302 }
2303
2304 SmallVector<OpFoldResult> outputOfr =
2305 getMixedValues(staticValues: newOutputShape, dynamicValues: dynamicOutputShape, b&: rewriter);
2306 // The refined types are still applied to the same src/result values, so
2307 // propagate their encodings, letting each encoding self-decide whether it
2308 // still holds on the more-static shape.
2309 Type elementType = expandOp.getSrcType().getElementType();
2310 auto inputType = RankedTensorType::get(
2311 shape: newInputShape, elementType,
2312 encoding: propagateEncoding(encoding: expandOp.getSrcType().getEncoding(), shape: newInputShape,
2313 elementType));
2314 auto outputType = RankedTensorType::get(
2315 shape: newOutputShape, elementType,
2316 encoding: propagateEncoding(encoding: expandOp.getResultType().getEncoding(),
2317 shape: newOutputShape, elementType));
2318 auto inputCast = CastOp::create(builder&: rewriter, location: expandOp.getLoc(), dest: inputType,
2319 source: expandOp.getSrc());
2320 auto newExpand = ExpandShapeOp::create(
2321 builder&: rewriter, location: expandOp.getLoc(), resultType: outputType, src: inputCast.getResult(),
2322 reassociation: expandOp.getReassociationIndices(), outputShape: outputOfr);
2323 rewriter.replaceOpWithNewOp<CastOp>(op: expandOp, args: expandOp.getType(),
2324 args: newExpand.getResult());
2325 return success();
2326 }
2327};
2328} // namespace
2329
2330void ExpandShapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
2331 MLIRContext *context) {
2332 results.add<
2333 ComposeReassociativeReshapeOps<ExpandShapeOp, ReshapeOpKind::kExpand>,
2334 ComposeExpandOfCollapseOp<ExpandShapeOp, CollapseShapeOp, CastOp>,
2335 ConvertToStaticExpandShape, FoldReshapeWithConstant<ExpandShapeOp>,
2336 FoldReshapeWithSplat<ExpandShapeOp>,
2337 FoldReshapeWithFromElements<ExpandShapeOp>>(arg&: context);
2338}
2339
2340void CollapseShapeOp::getCanonicalizationPatterns(RewritePatternSet &results,
2341 MLIRContext *context) {
2342 results.add<
2343 ComposeReassociativeReshapeOps<CollapseShapeOp, ReshapeOpKind::kCollapse>,
2344 ComposeCollapseOfExpandOp<CollapseShapeOp, ExpandShapeOp, CastOp,
2345 tensor::DimOp, RankedTensorType>,
2346 FoldReshapeWithConstant<CollapseShapeOp>,
2347 FoldReshapeWithSplat<CollapseShapeOp>,
2348 FoldReshapeWithFromElements<CollapseShapeOp>, FoldCollapseOfCastOp>(
2349 arg&: context);
2350}
2351
2352OpFoldResult ExpandShapeOp::fold(FoldAdaptor adaptor) {
2353 return foldReshapeOp<ExpandShapeOp, CollapseShapeOp>(reshapeOp: *this,
2354 operands: adaptor.getOperands());
2355}
2356
2357OpFoldResult CollapseShapeOp::fold(FoldAdaptor adaptor) {
2358 return foldReshapeOp<CollapseShapeOp, ExpandShapeOp>(reshapeOp: *this,
2359 operands: adaptor.getOperands());
2360}
2361
2362//===----------------------------------------------------------------------===//
2363// ExtractSliceOp
2364//===----------------------------------------------------------------------===//
2365
2366void ExtractSliceOp::getAsmResultNames(
2367 function_ref<void(Value, StringRef)> setNameFn) {
2368 setNameFn(getResult(), "extracted_slice");
2369}
2370
2371/// An extract_slice result type can be inferred, when it is not
2372/// rank-reduced, from the source type and the static representation of
2373/// offsets, sizes and strides. Special sentinels encode the dynamic case.
2374RankedTensorType
2375ExtractSliceOp::inferResultType(RankedTensorType sourceTensorType,
2376 ArrayRef<int64_t> staticSizes) {
2377 // An extract_slice op may specify only a leading subset of offset/sizes/
2378 // strides in which case we complete with offset=0, sizes from memref type
2379 // and strides=1.
2380 assert(static_cast<int64_t>(staticSizes.size()) ==
2381 sourceTensorType.getRank() &&
2382 "unexpected staticSizes not equal to rank of source");
2383 return RankedTensorType::get(shape: staticSizes, elementType: sourceTensorType.getElementType(),
2384 encoding: sourceTensorType.getEncoding());
2385}
2386
2387RankedTensorType
2388ExtractSliceOp::inferResultType(RankedTensorType sourceTensorType,
2389 ArrayRef<OpFoldResult> sizes) {
2390 SmallVector<int64_t> staticSizes;
2391 std::tie(args&: staticSizes, args: std::ignore) = decomposeMixedValues(mixedValues: sizes);
2392
2393 assert(static_cast<int64_t>(staticSizes.size()) ==
2394 sourceTensorType.getRank() &&
2395 "unexpected staticSizes not equal to rank of source");
2396 return RankedTensorType::get(shape: staticSizes, elementType: sourceTensorType.getElementType(),
2397 encoding: sourceTensorType.getEncoding());
2398}
2399
2400RankedTensorType
2401mlir::tensor::inferSliceType(RankedTensorType sourceTensorType,
2402 ArrayRef<int64_t> staticSizes,
2403 const llvm::SmallBitVector &droppedDims) {
2404 assert(staticSizes.size() == droppedDims.size() &&
2405 "expected one dropped-dimension bit per size");
2406
2407 SmallVector<int64_t> resultShape;
2408 resultShape.reserve(N: staticSizes.size() - droppedDims.count());
2409 for (auto [idx, size] : llvm::enumerate(First&: staticSizes))
2410 if (!droppedDims.test(Idx: idx))
2411 resultShape.push_back(Elt: size);
2412
2413 Type elementType = sourceTensorType.getElementType();
2414 return RankedTensorType::get(shape: resultShape, elementType,
2415 encoding: propagateEncoding(encoding: sourceTensorType.getEncoding(),
2416 shape: resultShape, elementType));
2417}
2418
2419RankedTensorType
2420mlir::tensor::inferSliceType(RankedTensorType sourceTensorType,
2421 ArrayRef<OpFoldResult> sizes,
2422 const llvm::SmallBitVector &droppedDims) {
2423 SmallVector<int64_t> staticSizes;
2424 std::tie(args&: staticSizes, args: std::ignore) = decomposeMixedValues(mixedValues: sizes);
2425 return inferSliceType(sourceTensorType, staticSizes, droppedDims);
2426}
2427
2428/// Build an ExtractSliceOp with mixed static and dynamic entries and custom
2429/// result type. If the type passed is nullptr, it is inferred.
2430void ExtractSliceOp::build(OpBuilder &b, OperationState &result,
2431 RankedTensorType resultType, Value source,
2432 ArrayRef<OpFoldResult> offsets,
2433 ArrayRef<OpFoldResult> sizes,
2434 ArrayRef<OpFoldResult> strides,
2435 ArrayRef<NamedAttribute> attrs) {
2436 SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
2437 SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
2438 dispatchIndexOpFoldResults(ofrs: offsets, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets);
2439 dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes);
2440 dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides);
2441 auto sourceRankedTensorType = llvm::cast<RankedTensorType>(Val: source.getType());
2442 // Structuring implementation this way avoids duplication between builders.
2443 if (!resultType) {
2444 resultType = llvm::cast<RankedTensorType>(
2445 Val: ExtractSliceOp::inferResultType(sourceTensorType: sourceRankedTensorType, staticSizes));
2446 }
2447 result.addAttributes(newAttributes: attrs);
2448 build(odsBuilder&: b, odsState&: result, result: resultType, source, offsets: dynamicOffsets, sizes: dynamicSizes,
2449 strides: dynamicStrides, static_offsets: b.getDenseI64ArrayAttr(values: staticOffsets),
2450 static_sizes: b.getDenseI64ArrayAttr(values: staticSizes),
2451 static_strides: b.getDenseI64ArrayAttr(values: staticStrides));
2452}
2453
2454/// Build an ExtractSliceOp with mixed static and dynamic entries and inferred
2455/// result type.
2456void ExtractSliceOp::build(OpBuilder &b, OperationState &result, Value source,
2457 ArrayRef<OpFoldResult> offsets,
2458 ArrayRef<OpFoldResult> sizes,
2459 ArrayRef<OpFoldResult> strides,
2460 ArrayRef<NamedAttribute> attrs) {
2461 build(b, result, resultType: RankedTensorType(), source, offsets, sizes, strides, attrs);
2462}
2463
2464/// Build an ExtractSliceOp with mixed static and dynamic entries packed into
2465/// a Range vector.
2466void ExtractSliceOp::build(OpBuilder &b, OperationState &result, Value source,
2467 ArrayRef<Range> ranges,
2468 ArrayRef<NamedAttribute> attrs) {
2469 auto [offsets, sizes, strides] = getOffsetsSizesAndStrides(ranges);
2470 build(b, result, resultType: RankedTensorType(), source, offsets, sizes, strides, attrs);
2471}
2472
2473/// Build an ExtractSliceOp with dynamic entries and custom result type. If
2474/// the type passed is nullptr, it is inferred.
2475void ExtractSliceOp::build(OpBuilder &b, OperationState &result,
2476 RankedTensorType resultType, Value source,
2477 ValueRange offsets, ValueRange sizes,
2478 ValueRange strides, ArrayRef<NamedAttribute> attrs) {
2479 SmallVector<OpFoldResult> offsetValues = llvm::map_to_vector<4>(
2480 C&: offsets, F: [](Value v) -> OpFoldResult { return v; });
2481 SmallVector<OpFoldResult> sizeValues =
2482 llvm::map_to_vector<4>(C&: sizes, F: [](Value v) -> OpFoldResult { return v; });
2483 SmallVector<OpFoldResult> strideValues = llvm::map_to_vector<4>(
2484 C&: strides, F: [](Value v) -> OpFoldResult { return v; });
2485 build(b, result, resultType, source, offsets: offsetValues, sizes: sizeValues, strides: strideValues);
2486}
2487
2488/// Build an ExtractSliceOp with dynamic entries and inferred result type.
2489void ExtractSliceOp::build(OpBuilder &b, OperationState &result, Value source,
2490 ValueRange offsets, ValueRange sizes,
2491 ValueRange strides, ArrayRef<NamedAttribute> attrs) {
2492 build(b, result, resultType: RankedTensorType(), source, offsets, sizes, strides, attrs);
2493}
2494
2495static LogicalResult produceSliceErrorMsg(SliceVerificationResult result,
2496 Operation *op,
2497 RankedTensorType expectedType) {
2498 switch (result) {
2499 case SliceVerificationResult::Success:
2500 return success();
2501 case SliceVerificationResult::RankTooLarge:
2502 return op->emitError(message: "expected rank to be smaller or equal to ")
2503 << "the other rank. ";
2504 case SliceVerificationResult::SizeMismatch:
2505 return op->emitError(message: "expected type to be ")
2506 << expectedType << " or a rank-reduced version. (size mismatch) ";
2507 case SliceVerificationResult::ElemTypeMismatch:
2508 return op->emitError(message: "expected element type to be ")
2509 << expectedType.getElementType();
2510 default:
2511 llvm_unreachable("unexpected extract_slice op verification result");
2512 }
2513}
2514
2515/// Build an ExtractSliceOp with mixed static and dynamic sizes, inferred
2516/// result type, offsets set to 0 and strides set to 1.
2517void ExtractSliceOp::build(OpBuilder &b, OperationState &result,
2518 RankedTensorType resultType, Value source,
2519 ArrayRef<OpFoldResult> sizes,
2520 ArrayRef<NamedAttribute> attrs) {
2521 Attribute zeroIdxAttr = b.getIndexAttr(value: 0);
2522 Attribute oneIdxAttr = b.getIndexAttr(value: 1);
2523 SmallVector<OpFoldResult> readStrides(sizes.size(), oneIdxAttr);
2524 SmallVector<OpFoldResult> readOffsets(sizes.size(), zeroIdxAttr);
2525 build(b, result, resultType, source, offsets: readOffsets, sizes, strides: readStrides, attrs);
2526}
2527
2528/// Verifier for ExtractSliceOp.
2529LogicalResult ExtractSliceOp::verify() {
2530 RankedTensorType sourceType = getSourceType();
2531
2532 // Verify result type against inferred type.
2533 RankedTensorType expectedType =
2534 ExtractSliceOp::inferResultType(sourceTensorType: sourceType, sizes: getMixedSizes());
2535 SliceVerificationResult result = isRankReducedType(originalType: expectedType, candidateReducedType: getType());
2536 if (result != SliceVerificationResult::Success)
2537 return produceSliceErrorMsg(result, op: *this, expectedType);
2538
2539 // Verify that offsets, sizes, strides do not run out-of-bounds with respect
2540 // to the source tensor.
2541 SliceBoundsVerificationResult boundsResult = verifyInBoundsSlice(
2542 shape: sourceType.getShape(), staticOffsets: getStaticOffsets(), staticSizes: getStaticSizes(),
2543 staticStrides: getStaticStrides(), /*generateErrorMessage=*/true);
2544 if (!boundsResult.isValid)
2545 return getOperation()->emitError(message: boundsResult.errorMessage);
2546
2547 return success();
2548}
2549
2550llvm::SmallBitVector ExtractSliceOp::getDroppedDims() {
2551 return ::getDroppedDims(reducedShape: getType().getShape(), mixedSizes: getMixedSizes());
2552}
2553
2554FailureOr<Value>
2555ExtractSliceOp::rankReduceIfNeeded(OpBuilder &b, Location loc, Value value,
2556 ArrayRef<int64_t> desiredShape) {
2557 auto sourceTensorType = llvm::dyn_cast<RankedTensorType>(Val: value.getType());
2558 assert(sourceTensorType && "not a ranked tensor type");
2559 auto sourceShape = sourceTensorType.getShape();
2560 if (sourceShape.equals(RHS: desiredShape))
2561 return value;
2562 auto maybeRankReductionMask =
2563 mlir::computeRankReductionMask(originalShape: sourceShape, reducedShape: desiredShape);
2564 if (!maybeRankReductionMask)
2565 return failure();
2566 return createCanonicalRankReducingExtractSliceOp(
2567 b, loc, tensor: value,
2568 targetType: RankedTensorType::Builder(sourceTensorType).setShape(desiredShape));
2569}
2570
2571LogicalResult ExtractSliceOp::reifyResultShapes(
2572 OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
2573 reifiedReturnShapes.resize(N: 1);
2574 reifiedReturnShapes[0].reserve(N: getType().getRank());
2575 SmallVector<OpFoldResult> mixedSizes = getMixedSizes();
2576 llvm::SmallBitVector droppedDims = getDroppedDims();
2577 for (const auto &size : enumerate(First&: mixedSizes)) {
2578 if (droppedDims.test(Idx: size.index()))
2579 continue;
2580 reifiedReturnShapes[0].push_back(Elt: size.value());
2581 }
2582 return success();
2583}
2584
2585namespace {
2586/// Pattern to rewrite an extract_slice op with tensor::Cast arguments.
2587/// This essentially pushes memref_cast past its consuming slice when
2588/// `canFoldIntoConsumerOp` is true.
2589///
2590/// Example:
2591/// ```
2592/// %0 = tensor.cast %V : tensor<16x16xf32> to tensor<?x?xf32>
2593/// %1 = tensor.extract_slice %0[0, 0][3, 4][1, 1] : tensor<?x?xf32> to
2594/// tensor<3x4xf32>
2595/// ```
2596/// is rewritten into:
2597/// ```
2598/// %0 = tensor.extract_slice %V[0, 0][3, 4][1, 1] : tensor<16x16xf32> to
2599/// tensor<3x4xf32> %1 = tensor.cast %0: tensor<3x4xf32> to tensor<3x4xf32>
2600/// ```
2601class ExtractSliceOpCastFolder final : public OpRewritePattern<ExtractSliceOp> {
2602public:
2603 using OpRewritePattern<ExtractSliceOp>::OpRewritePattern;
2604
2605 LogicalResult matchAndRewrite(ExtractSliceOp sliceOp,
2606 PatternRewriter &rewriter) const override {
2607 // Any constant operand, just return to let the constant folder kick in.
2608 if (llvm::any_of(Range: sliceOp.getOperands(), P: [](Value operand) {
2609 return matchPattern(value: operand, pattern: matchConstantIndex());
2610 }))
2611 return failure();
2612
2613 auto castOp = sliceOp.getSource().getDefiningOp<CastOp>();
2614 if (!castOp)
2615 return failure();
2616
2617 if (!canFoldIntoConsumerOp(castOp))
2618 return failure();
2619
2620 // Pattern does not apply if the produced op would not verify.
2621 SliceBoundsVerificationResult sliceResult = verifyInBoundsSlice(
2622 shape: cast<RankedTensorType>(Val: castOp.getSource().getType()).getShape(),
2623 staticOffsets: sliceOp.getStaticOffsets(), staticSizes: sliceOp.getStaticSizes(),
2624 staticStrides: sliceOp.getStaticStrides());
2625 if (!sliceResult.isValid)
2626 return failure();
2627
2628 // Create folded extract.
2629 Location loc = sliceOp.getLoc();
2630 Value newResult = ExtractSliceOp::create(
2631 builder&: rewriter, location: loc, result: sliceOp.getType(), source: castOp.getSource(),
2632 offsets: sliceOp.getOffsets(), sizes: sliceOp.getSizes(), strides: sliceOp.getStrides(),
2633 static_offsets: sliceOp.getStaticOffsets(), static_sizes: sliceOp.getStaticSizes(),
2634 static_strides: sliceOp.getStaticStrides());
2635 rewriter.replaceOp(op: sliceOp, newValues: newResult);
2636 return success();
2637 }
2638};
2639
2640/// Slice elements from `values` into `outValues`. `counts` represents the
2641/// numbers of elements to stride in the original values for each dimension.
2642/// The output values can be used to construct a DenseElementsAttr.
2643template <typename IterTy, typename ElemTy>
2644static void sliceElements(IterTy values, ArrayRef<int64_t> counts,
2645 ArrayRef<int64_t> offsets, ArrayRef<int64_t> sizes,
2646 ArrayRef<int64_t> strides,
2647 llvm::SmallVectorImpl<ElemTy> *outValues) {
2648 assert(offsets.size() == sizes.size());
2649 assert(offsets.size() == strides.size());
2650 if (offsets.empty())
2651 return;
2652
2653 int64_t offset = offsets.front();
2654 int64_t size = sizes.front();
2655 int64_t stride = strides.front();
2656 if (offsets.size() == 1) {
2657 for (int64_t i = 0; i < size; ++i, offset += stride)
2658 outValues->push_back(*(values + offset));
2659
2660 return;
2661 }
2662
2663 for (int64_t i = 0; i < size; ++i, offset += stride) {
2664 auto begin = values + offset * counts.front();
2665 sliceElements<IterTy, ElemTy>(begin, counts.drop_front(),
2666 offsets.drop_front(), sizes.drop_front(),
2667 strides.drop_front(), outValues);
2668 }
2669}
2670
2671/// Fold arith.constant and tensor.extract_slice into arith.constant. The
2672/// folded operation might introduce more constant data; Users can control
2673/// their heuristics by the control function.
2674class ConstantOpExtractSliceFolder final
2675 : public OpRewritePattern<ExtractSliceOp> {
2676public:
2677 using OpRewritePattern<ExtractSliceOp>::OpRewritePattern;
2678
2679 ConstantOpExtractSliceFolder(MLIRContext *context,
2680 ControlConstantExtractSliceFusionFn controlFn)
2681 : OpRewritePattern<ExtractSliceOp>(context),
2682 controlFn(std::move(controlFn)) {}
2683
2684 LogicalResult matchAndRewrite(ExtractSliceOp op,
2685 PatternRewriter &rewriter) const override {
2686 DenseElementsAttr attr;
2687 if (!matchPattern(value: op.getSource(), pattern: m_Constant(bind_value: &attr)))
2688 return failure();
2689
2690 // A constant splat is handled by fold().
2691 if (attr.isSplat())
2692 return failure();
2693
2694 // Dynamic result shape is not supported.
2695 auto sourceType = llvm::cast<ShapedType>(Val: op.getSource().getType());
2696 auto resultType = llvm::cast<ShapedType>(Val: op.getResult().getType());
2697 if (!sourceType.hasStaticShape() || !resultType.hasStaticShape())
2698 return failure();
2699
2700 // Customized control over the folding.
2701 if (!controlFn(op))
2702 return failure();
2703
2704 int64_t count = sourceType.getNumElements();
2705 if (count == 0)
2706 return failure();
2707
2708 // Check if there are any dynamic parts, which are not supported.
2709 auto offsets = op.getStaticOffsets();
2710 if (llvm::is_contained(Range&: offsets, Element: ShapedType::kDynamic))
2711 return failure();
2712 auto sizes = op.getStaticSizes();
2713 if (llvm::is_contained(Range&: sizes, Element: ShapedType::kDynamic))
2714 return failure();
2715 auto strides = op.getStaticStrides();
2716 if (llvm::is_contained(Range&: strides, Element: ShapedType::kDynamic))
2717 return failure();
2718
2719 // Compute the stride for each dimension.
2720 SmallVector<int64_t> counts;
2721 ArrayRef<int64_t> shape = sourceType.getShape();
2722 counts.reserve(N: shape.size());
2723 for (int64_t v : shape) {
2724 count = count / v;
2725 counts.push_back(Elt: count);
2726 }
2727
2728 // Slice the elements and construct a new attribute.
2729 SmallVector<Attribute> outValues;
2730 outValues.reserve(N: resultType.getNumElements());
2731 sliceElements(values: attr.value_begin<Attribute>(), counts, offsets, sizes,
2732 strides, outValues: &outValues);
2733 auto newAttr = DenseElementsAttr::get(type: resultType, values: outValues);
2734 rewriter.replaceOpWithNewOp<arith::ConstantOp>(op, args&: resultType, args&: newAttr);
2735 return success();
2736 }
2737
2738private:
2739 /// This additionally controls whether the fold happens or not. Users can
2740 /// impose their heuristics in the function.
2741 ControlConstantExtractSliceFusionFn controlFn;
2742};
2743
2744} // namespace
2745
2746void mlir::tensor::populateFoldConstantExtractSlicePatterns(
2747 RewritePatternSet &patterns,
2748 const ControlConstantExtractSliceFusionFn &controlFn) {
2749 patterns.add<ConstantOpExtractSliceFolder>(arg: patterns.getContext(), args: controlFn);
2750}
2751
2752/// Return the canonical type of the result of an extract_slice op.
2753/// Note: offsets and strides are not needed to determine the result type of
2754/// an extract_slice. The operator arguments are just there for interface
2755/// compatibility.
2756struct SliceReturnTypeCanonicalizer {
2757 RankedTensorType operator()(ExtractSliceOp op,
2758 ArrayRef<OpFoldResult> mixedOffsets,
2759 ArrayRef<OpFoldResult> mixedSizes,
2760 ArrayRef<OpFoldResult> mixedStrides) {
2761 return inferSliceType(sourceTensorType: op.getSourceType(), sizes: mixedSizes, droppedDims: op.getDroppedDims());
2762 }
2763};
2764
2765/// A canonicalizer wrapper to replace ExtractSliceOps.
2766struct SliceCanonicalizer {
2767 void operator()(PatternRewriter &rewriter, ExtractSliceOp op,
2768 ExtractSliceOp newOp) {
2769 Value replacement = newOp.getResult();
2770 if (replacement.getType() != op.getType())
2771 replacement = tensor::CastOp::create(builder&: rewriter, location: op.getLoc(), dest: op.getType(),
2772 source: replacement);
2773 rewriter.replaceOp(op, newValues: replacement);
2774 }
2775};
2776
2777void ExtractSliceOp::getCanonicalizationPatterns(RewritePatternSet &results,
2778 MLIRContext *context) {
2779 results.add<
2780 OpWithOffsetSizesAndStridesConstantArgumentFolder<
2781 ExtractSliceOp, SliceReturnTypeCanonicalizer, SliceCanonicalizer>,
2782 ExtractSliceOpCastFolder>(arg&: context);
2783}
2784
2785//
2786static LogicalResult
2787foldIdentityOffsetSizeAndStrideOpInterface(OffsetSizeAndStrideOpInterface op,
2788 ShapedType shapedType) {
2789 OpBuilder b(op.getContext());
2790 for (OpFoldResult ofr : op.getMixedOffsets())
2791 if (getConstantIntValue(ofr) != static_cast<int64_t>(0))
2792 return failure();
2793 // Rank-reducing noops only need to inspect the leading dimensions:
2794 // llvm::zip is appropriate.
2795 auto shape = shapedType.getShape();
2796 for (auto it : llvm::zip(t: op.getMixedSizes(), u&: shape))
2797 if (getConstantIntValue(ofr: std::get<0>(t&: it)) != std::get<1>(t&: it))
2798 return failure();
2799 for (OpFoldResult ofr : op.getMixedStrides())
2800 if (getConstantIntValue(ofr) != static_cast<int64_t>(1))
2801 return failure();
2802 return success();
2803}
2804
2805/// If we have an ExtractSliceOp consuming an InsertSliceOp with the same
2806/// slice, we can return the InsertSliceOp's source directly.
2807// TODO: This only checks the immediate producer; extend to go up the
2808// insert/extract chain if the slices are disjoint.
2809static Value foldExtractAfterInsertSlice(ExtractSliceOp extractOp) {
2810 auto insertOp = extractOp.getSource().getDefiningOp<InsertSliceOp>();
2811
2812 auto isSame = [](OpFoldResult a, OpFoldResult b) { return a == b; };
2813 if (insertOp && insertOp.getSource().getType() == extractOp.getType() &&
2814 insertOp.isSameAs(other: extractOp, cmp: isSame))
2815 return insertOp.getSource();
2816
2817 return {};
2818}
2819
2820OpFoldResult ExtractSliceOp::fold(FoldAdaptor adaptor) {
2821 if (OpFoldResult reshapedSource = reshapeConstantSource(
2822 source: llvm::dyn_cast_if_present<SplatElementsAttr>(Val: adaptor.getSource()),
2823 result: getResult().getType()))
2824 return reshapedSource;
2825 if (getSourceType() == getType() &&
2826 succeeded(Result: foldIdentityOffsetSizeAndStrideOpInterface(op: *this, shapedType: getType())))
2827 return this->getSource();
2828 if (Value slice = foldExtractAfterInsertSlice(extractOp: *this))
2829 return slice;
2830
2831 return OpFoldResult();
2832}
2833
2834Value mlir::tensor::createCanonicalRankReducingExtractSliceOp(
2835 OpBuilder &b, Location loc, Value tensor, RankedTensorType targetType) {
2836 auto rankedTensorType = llvm::cast<RankedTensorType>(Val: tensor.getType());
2837 unsigned rank = rankedTensorType.getRank();
2838 SmallVector<OpFoldResult> offsets(rank, b.getIndexAttr(value: 0));
2839 SmallVector<OpFoldResult> sizes = getMixedSizes(builder&: b, loc, value: tensor);
2840 SmallVector<OpFoldResult> strides(rank, b.getIndexAttr(value: 1));
2841 return b.createOrFold<tensor::ExtractSliceOp>(location: loc, args&: targetType, args&: tensor,
2842 args&: offsets, args&: sizes, args&: strides);
2843}
2844
2845//===----------------------------------------------------------------------===//
2846// InsertSliceOp
2847//===----------------------------------------------------------------------===//
2848
2849void InsertSliceOp::getAsmResultNames(
2850 function_ref<void(Value, StringRef)> setNameFn) {
2851 setNameFn(getResult(), "inserted_slice");
2852}
2853
2854// Build a InsertSliceOp with mixed static and dynamic entries.
2855void InsertSliceOp::build(OpBuilder &b, OperationState &result, Value source,
2856 Value dest, ArrayRef<OpFoldResult> offsets,
2857 ArrayRef<OpFoldResult> sizes,
2858 ArrayRef<OpFoldResult> strides,
2859 ArrayRef<NamedAttribute> attrs) {
2860 SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
2861 SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
2862 dispatchIndexOpFoldResults(ofrs: offsets, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets);
2863 dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes);
2864 dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides);
2865 result.addAttributes(newAttributes: attrs);
2866 build(odsBuilder&: b, odsState&: result, result: dest.getType(), source, dest, offsets: dynamicOffsets, sizes: dynamicSizes,
2867 strides: dynamicStrides, static_offsets: b.getDenseI64ArrayAttr(values: staticOffsets),
2868 static_sizes: b.getDenseI64ArrayAttr(values: staticSizes),
2869 static_strides: b.getDenseI64ArrayAttr(values: staticStrides));
2870}
2871
2872/// Build an InsertSliceOp with mixed static and dynamic entries packed into a
2873/// Range vector.
2874void InsertSliceOp::build(OpBuilder &b, OperationState &result, Value source,
2875 Value dest, ArrayRef<Range> ranges,
2876 ArrayRef<NamedAttribute> attrs) {
2877 auto [offsets, sizes, strides] = getOffsetsSizesAndStrides(ranges);
2878 build(b, result, source, dest, offsets, sizes, strides, attrs);
2879}
2880
2881// Build a InsertSliceOp with dynamic entries.
2882void InsertSliceOp::build(OpBuilder &b, OperationState &result, Value source,
2883 Value dest, ValueRange offsets, ValueRange sizes,
2884 ValueRange strides, ArrayRef<NamedAttribute> attrs) {
2885 SmallVector<OpFoldResult> offsetValues = llvm::map_to_vector<4>(
2886 C&: offsets, F: [](Value v) -> OpFoldResult { return v; });
2887 SmallVector<OpFoldResult> sizeValues =
2888 llvm::map_to_vector<4>(C&: sizes, F: [](Value v) -> OpFoldResult { return v; });
2889 SmallVector<OpFoldResult> strideValues = llvm::map_to_vector<4>(
2890 C&: strides, F: [](Value v) -> OpFoldResult { return v; });
2891 build(b, result, source, dest, offsets: offsetValues, sizes: sizeValues, strides: strideValues);
2892}
2893
2894/// Rank-reducing type verification for both InsertSliceOp and
2895/// ParallelInsertSliceOp.
2896static SliceVerificationResult verifyInsertSliceOp(
2897 RankedTensorType srcType, RankedTensorType dstType,
2898 ArrayRef<int64_t> staticOffsets, ArrayRef<int64_t> staticSizes,
2899 ArrayRef<int64_t> staticStrides, RankedTensorType *expectedType = nullptr) {
2900 // insert_slice is the inverse of extract_slice, use the same type
2901 // inference.
2902 RankedTensorType expected =
2903 ExtractSliceOp::inferResultType(sourceTensorType: dstType, staticSizes);
2904 if (expectedType)
2905 *expectedType = expected;
2906 return isRankReducedType(originalType: expected, candidateReducedType: srcType);
2907}
2908
2909/// Verifier for InsertSliceOp.
2910LogicalResult InsertSliceOp::verify() {
2911 // Verify result type against inferred type.
2912 RankedTensorType expectedType;
2913 SliceVerificationResult result =
2914 verifyInsertSliceOp(srcType: getSourceType(), dstType: getType(), staticOffsets: getStaticOffsets(),
2915 staticSizes: getStaticSizes(), staticStrides: getStaticStrides(), expectedType: &expectedType);
2916 if (result != SliceVerificationResult::Success)
2917 return produceSliceErrorMsg(result, op: *this, expectedType);
2918
2919 // Verify that offsets, sizes, strides do not run out-of-bounds with respect
2920 // to the destination tensor.
2921 SliceBoundsVerificationResult boundsResult = verifyInBoundsSlice(
2922 shape: getDestType().getShape(), staticOffsets: getStaticOffsets(), staticSizes: getStaticSizes(),
2923 staticStrides: getStaticStrides(), /*generateErrorMessage=*/true);
2924 if (!boundsResult.isValid)
2925 return getOperation()->emitError(message: boundsResult.errorMessage);
2926
2927 return success();
2928}
2929
2930/// If we have two consecutive InsertSliceOp writing to the same slice, we
2931/// can mutate the second InsertSliceOp's destination to the first one's.
2932///
2933/// Example:
2934///
2935/// ```mlir
2936/// %0 = tensor.insert_slice %slice0 into %input[0, 0] [64, 64] [1, 1]
2937/// %1 = tensor.insert_slice %slice1 into %0[0, 0] [64, 64] [1, 1]
2938/// ```
2939///
2940/// folds into:
2941///
2942/// ```mlir
2943/// %1 = tensor.insert_slice %slice1 into %input[0, 0] [64, 64] [1, 1]
2944/// ```
2945///
2946/// This pattern works with both InsertSliceOp and ParallelInsertSliceOp.
2947static LogicalResult foldInsertAfterInsertSlice(InsertSliceOp insertOp) {
2948 auto prevInsertOp = insertOp.getDest().getDefiningOp<InsertSliceOp>();
2949
2950 auto isSame = [](OpFoldResult a, OpFoldResult b) { return a == b; };
2951 if (!prevInsertOp ||
2952 prevInsertOp.getSource().getType() != insertOp.getSource().getType() ||
2953 !prevInsertOp.isSameAs(other: insertOp, cmp: isSame))
2954 return failure();
2955
2956 insertOp.getDestMutable().assign(value: prevInsertOp.getDest());
2957 return success();
2958}
2959
2960/// Folds round-trip extract/insert slice op pairs.
2961/// Example:
2962/// ```mlir
2963/// %0 = tensor.extract_slice %val[0, 0, 0, 0] [1, 1, 2, 4] [1, 1, 1, 1]
2964/// %1 = tensor.insert_slice %0 into %val[0, 0, 0, 0] [1, 1, 2, 4] [1, 1, 1, 1]
2965/// ```
2966/// can be folded into %val.
2967static Value foldInsertAfterExtractSlice(InsertSliceOp insertOp) {
2968 auto extractOp = insertOp.getSource().getDefiningOp<ExtractSliceOp>();
2969
2970 auto isSame = [](OpFoldResult a, OpFoldResult b) { return a == b; };
2971 if (!extractOp || extractOp.getSource() != insertOp.getDest() ||
2972 !extractOp.isSameAs(other: insertOp, cmp: isSame))
2973 return nullptr;
2974
2975 return extractOp.getSource();
2976}
2977
2978OpFoldResult InsertSliceOp::fold(FoldAdaptor) {
2979 if (getSourceType().hasStaticShape() && getType().hasStaticShape() &&
2980 getSourceType() == getType() &&
2981 succeeded(Result: foldIdentityOffsetSizeAndStrideOpInterface(op: *this, shapedType: getType())))
2982 return this->getSource();
2983 if (succeeded(Result: foldInsertAfterInsertSlice(insertOp: *this)))
2984 return getResult();
2985 if (auto result = foldInsertAfterExtractSlice(insertOp: *this))
2986 return result;
2987 if (llvm::any_of(Range: getMixedSizes(), P: isZeroInteger))
2988 return getDest();
2989 return OpFoldResult();
2990}
2991
2992LogicalResult InsertSliceOp::reifyResultShapes(
2993 OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
2994 reifiedReturnShapes.resize(N: 1, NV: SmallVector<OpFoldResult>(getType().getRank()));
2995 reifiedReturnShapes[0] = tensor::getMixedSizes(builder, loc: getLoc(), value: getDest());
2996 return success();
2997}
2998
2999namespace {
3000/// Pattern to rewrite a insert_slice op with constant arguments.
3001///
3002/// This pattern works with both InsertSliceOp and ParallelInsertSliceOp.
3003template <typename InsertOpTy>
3004class InsertSliceOpConstantArgumentFolder final
3005 : public OpRewritePattern<InsertOpTy> {
3006public:
3007 using OpRewritePattern<InsertOpTy>::OpRewritePattern;
3008
3009 LogicalResult matchAndRewrite(InsertOpTy insertSliceOp,
3010 PatternRewriter &rewriter) const override {
3011 SmallVector<OpFoldResult> mixedOffsets(insertSliceOp.getMixedOffsets());
3012 SmallVector<OpFoldResult> mixedSizes(insertSliceOp.getMixedSizes());
3013 SmallVector<OpFoldResult> mixedStrides(insertSliceOp.getMixedStrides());
3014
3015 // No constant operands were folded, just return;
3016 if (failed(Result: foldDynamicOffsetSizeList(offsetsOrSizes&: mixedOffsets)) &&
3017 failed(Result: foldDynamicOffsetSizeList(offsetsOrSizes&: mixedSizes)) &&
3018 failed(Result: foldDynamicStrideList(strides&: mixedStrides)))
3019 return failure();
3020
3021 // Pattern does not apply if the produced op would not verify.
3022 SliceBoundsVerificationResult sliceResult =
3023 verifyInBoundsSlice(insertSliceOp.getDest().getType().getShape(),
3024 mixedOffsets, mixedSizes, mixedStrides);
3025 if (!sliceResult.isValid)
3026 return failure();
3027
3028 auto sourceType = inferSliceType(insertSliceOp.getSourceType(), mixedSizes,
3029 insertSliceOp.getDroppedDims());
3030 Value toInsert = insertSliceOp.getSource();
3031 if (sourceType != insertSliceOp.getSourceType()) {
3032 OpBuilder::InsertionGuard g(rewriter);
3033 // The only difference between InsertSliceOp and ParallelInsertSliceOp
3034 // is that the insertion point is just before the InParallelOp in
3035 // the parallel case.
3036 if (isa<InParallelOpInterface>(insertSliceOp->getParentOp()))
3037 rewriter.setInsertionPoint(insertSliceOp->getParentOp());
3038 toInsert = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
3039 sourceType, toInsert);
3040 }
3041 rewriter.replaceOpWithNewOp<InsertOpTy>(
3042 insertSliceOp, toInsert, insertSliceOp.getDest(), mixedOffsets,
3043 mixedSizes, mixedStrides);
3044 return success();
3045 }
3046};
3047
3048/// Fold tensor_casts with insert_slice operations. If the source or
3049/// destination tensor is a tensor_cast that removes static type information,
3050/// the cast is folded into the insert_slice operation. E.g.:
3051///
3052/// ```mlir
3053/// %1 = tensor.cast %0 : tensor<8x16xf32> to tensor<?x?xf32>
3054/// %2 = tensor.insert_slice %1 into ... : tensor<?x?xf32> into ...
3055/// ```
3056///
3057/// folds into:
3058///
3059/// ```mlir
3060/// %2 = tensor.insert_slice %0 into ... : tensor<8x16xf32> into ...
3061/// ```
3062///
3063/// Note: When folding a cast on the destination tensor, the result of the
3064/// insert_slice operation is casted to ensure that the type of the result did
3065/// not change.
3066///
3067/// This pattern works with both InsertSliceOp and ParallelInsertSliceOp.
3068template <typename InsertOpTy>
3069struct InsertSliceOpCastFolder final : public OpRewritePattern<InsertOpTy> {
3070 using OpRewritePattern<InsertOpTy>::OpRewritePattern;
3071
3072 LogicalResult matchAndRewrite(InsertOpTy insertSliceOp,
3073 PatternRewriter &rewriter) const override {
3074 if (llvm::any_of(insertSliceOp.getOperands(), [](Value operand) {
3075 return matchPattern(value: operand, pattern: matchConstantIndex());
3076 }))
3077 return failure();
3078
3079 auto getSourceOfCastOp = [](Value v) -> std::optional<Value> {
3080 auto castOp = v.getDefiningOp<tensor::CastOp>();
3081 if (!castOp || !canFoldIntoConsumerOp(castOp))
3082 return std::nullopt;
3083 return castOp.getSource();
3084 };
3085 std::optional<Value> sourceCastSource =
3086 getSourceOfCastOp(insertSliceOp.getSource());
3087 std::optional<Value> destCastSource =
3088 getSourceOfCastOp(insertSliceOp.getDest());
3089 if (!sourceCastSource && !destCastSource)
3090 return failure();
3091
3092 auto src =
3093 (sourceCastSource ? *sourceCastSource : insertSliceOp.getSource());
3094 auto dst = (destCastSource ? *destCastSource : insertSliceOp.getDest());
3095 auto srcType = llvm::dyn_cast<RankedTensorType>(src.getType());
3096 auto dstType = llvm::dyn_cast<RankedTensorType>(dst.getType());
3097 if (!srcType || !dstType)
3098 return failure();
3099
3100 // The tensor.cast source could have additional static information not seen
3101 // in the insert slice op static sizes, so we ignore dynamic dims when
3102 // computing the rank reduction mask.
3103 SmallVector<int64_t> staticSizes(insertSliceOp.getStaticSizes());
3104 auto rankReductionMask = computeRankReductionMask(
3105 staticSizes, srcType.getShape(), /*matchDynamic=*/true);
3106 if (!rankReductionMask.has_value())
3107 return failure();
3108 // Replace dimensions in the insert slice op with corresponding static dims
3109 // from the cast source type. If the insert slice sizes have static dims
3110 // that are not static in the tensor.cast source (i.e., when the cast op
3111 // casts a dynamic dim to static), the dim should not be replaced, and the
3112 // pattern will fail later in `verifyInsertSliceOp`.
3113 SmallVector<OpFoldResult> mixedSizes(insertSliceOp.getMixedSizes());
3114 int64_t rankReducedIdx = 0;
3115 for (auto [idx, size] : enumerate(First&: staticSizes)) {
3116 if (!rankReductionMask.value().contains(idx) &&
3117 !srcType.isDynamicDim(rankReducedIdx)) {
3118 mixedSizes[idx] = getAsIndexOpFoldResult(
3119 rewriter.getContext(), srcType.getDimSize(rankReducedIdx));
3120 size = srcType.getDimSize(rankReducedIdx++);
3121 }
3122 }
3123
3124 // Pattern does not apply if the produced op would not verify.
3125 if (verifyInsertSliceOp(srcType, dstType, insertSliceOp.getStaticOffsets(),
3126 staticSizes, insertSliceOp.getStaticStrides()) !=
3127 SliceVerificationResult::Success)
3128 return failure();
3129 SliceBoundsVerificationResult sliceResult =
3130 verifyInBoundsSlice(dstType.getShape(), insertSliceOp.getMixedOffsets(),
3131 mixedSizes, insertSliceOp.getMixedStrides());
3132 if (!sliceResult.isValid)
3133 return failure();
3134
3135 Operation *replacement =
3136 InsertOpTy::create(rewriter, insertSliceOp.getLoc(), src, dst,
3137 insertSliceOp.getMixedOffsets(), mixedSizes,
3138 insertSliceOp.getMixedStrides());
3139
3140 // In the parallel case there is no result and so nothing to cast.
3141 bool isParallelInsert =
3142 std::is_same<InsertOpTy, ParallelInsertSliceOp>::value;
3143 if (!isParallelInsert && dst.getType() != insertSliceOp.getDestType()) {
3144 replacement = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
3145 insertSliceOp.getDestType(),
3146 replacement->getResult(idx: 0));
3147 }
3148 rewriter.replaceOp(insertSliceOp, replacement->getResults());
3149 return success();
3150 }
3151};
3152
3153/// If additional static type information can be deduced from a insert_slice's
3154/// size operands, insert an explicit cast of the op's source operand. This
3155/// enables other canonicalization patterns that are matching for tensor_cast
3156/// ops such as `ForOpTensorCastFolder` in SCF.
3157///
3158/// Example:
3159///
3160/// ```mlir
3161/// %r = tensor.insert_slice %0 into %1[...] [64, 64] [1, 1]
3162/// : tensor<?x?xf32> into ...
3163/// ```
3164///
3165/// folds into:
3166///
3167/// ```mlir
3168/// %tmp = tensor.cast %0 : tensor<?x?xf32> to tensor<64x64xf32>
3169/// %r = tensor.insert_slice %tmp into %1[...] [64, 64] [1, 1]
3170/// : tensor<64x64xf32> into ...
3171/// ```
3172///
3173/// This patterns works with both InsertSliceOp and ParallelInsertSliceOp.
3174template <typename InsertOpTy>
3175struct InsertSliceOpSourceCastInserter final
3176 : public OpRewritePattern<InsertOpTy> {
3177 using OpRewritePattern<InsertOpTy>::OpRewritePattern;
3178
3179 LogicalResult matchAndRewrite(InsertOpTy insertSliceOp,
3180 PatternRewriter &rewriter) const override {
3181 RankedTensorType srcType = insertSliceOp.getSourceType();
3182 if (srcType.getRank() != insertSliceOp.getDestType().getRank())
3183 return failure();
3184 SmallVector<int64_t> newSrcShape(srcType.getShape());
3185 for (int64_t i = 0; i < srcType.getRank(); ++i) {
3186 if (std::optional<int64_t> constInt =
3187 getConstantIntValue(insertSliceOp.getMixedSizes()[i])) {
3188 // Bail on invalid IR.
3189 if (*constInt < 0)
3190 return failure();
3191 newSrcShape[i] = *constInt;
3192 }
3193 }
3194 if (!hasValidSizesOffsets(sizesOrOffsets: newSrcShape))
3195 return failure();
3196
3197 RankedTensorType newSrcType = RankedTensorType::get(
3198 shape: newSrcShape, elementType: srcType.getElementType(), encoding: srcType.getEncoding());
3199 if (srcType == newSrcType ||
3200 !preservesStaticInformation(source: srcType, target: newSrcType) ||
3201 !tensor::CastOp::areCastCompatible(inputs: srcType, outputs: newSrcType))
3202 return failure();
3203
3204 // newSrcType is:
3205 // 1) Different from srcType.
3206 // 2) "More static" than srcType.
3207 // 3) Cast-compatible with srcType.
3208 // Insert the cast.
3209 OpBuilder::InsertionGuard g(rewriter);
3210 // The only difference between InsertSliceOp and ParallelInsertSliceOp is
3211 // that the insertion point is just before the InParallelOp in the
3212 // parallel case.
3213 if (isa<ParallelCombiningOpInterface>(insertSliceOp->getParentOp()))
3214 rewriter.setInsertionPoint(insertSliceOp->getParentOp());
3215 Value cast = tensor::CastOp::create(rewriter, insertSliceOp.getLoc(),
3216 newSrcType, insertSliceOp.getSource());
3217 rewriter.replaceOpWithNewOp<InsertOpTy>(
3218 insertSliceOp, cast, insertSliceOp.getDest(),
3219 insertSliceOp.getMixedOffsets(), insertSliceOp.getMixedSizes(),
3220 insertSliceOp.getMixedStrides());
3221 return success();
3222 }
3223};
3224} // namespace
3225
3226llvm::SmallBitVector InsertSliceOp::getDroppedDims() {
3227 return ::getDroppedDims(reducedShape: getSourceType().getShape(), mixedSizes: getMixedSizes());
3228}
3229
3230void InsertSliceOp::getCanonicalizationPatterns(RewritePatternSet &results,
3231 MLIRContext *context) {
3232 results.add<InsertSliceOpConstantArgumentFolder<InsertSliceOp>,
3233 InsertSliceOpCastFolder<InsertSliceOp>,
3234 InsertSliceOpSourceCastInserter<InsertSliceOp>>(arg&: context);
3235}
3236
3237Value mlir::tensor::createCanonicalRankReducingInsertSliceOp(OpBuilder &b,
3238 Location loc,
3239 Value tensor,
3240 Value dest) {
3241 auto rankedTensorType = llvm::cast<RankedTensorType>(Val: dest.getType());
3242 unsigned rank = rankedTensorType.getRank();
3243 SmallVector<OpFoldResult> offsets(rank, b.getIndexAttr(value: 0));
3244 SmallVector<OpFoldResult> sizes = getMixedSizes(builder&: b, loc, value: dest);
3245 SmallVector<OpFoldResult> strides(rank, b.getIndexAttr(value: 1));
3246 return b.createOrFold<tensor::InsertSliceOp>(location: loc, args&: tensor, args&: dest, args&: offsets,
3247 args&: sizes, args&: strides);
3248}
3249
3250//===----------------------------------------------------------------------===//
3251// PadOp
3252//===----------------------------------------------------------------------===//
3253
3254void PadOp::getAsmResultNames(function_ref<void(Value, StringRef)> setNameFn) {
3255 setNameFn(getResult(), "padded");
3256}
3257
3258LogicalResult PadOp::verify() {
3259 for (ArrayRef<int64_t> padding : {getStaticLow(), getStaticHigh()})
3260 for (int64_t size : padding)
3261 if (!ShapedType::isDynamic(dValue: size) && size < 0)
3262 return emitOpError(message: "padding must be non-negative, got ") << size;
3263
3264 auto sourceType = llvm::cast<RankedTensorType>(Val: getSource().getType());
3265 auto resultType = llvm::cast<RankedTensorType>(Val: getResult().getType());
3266 auto expectedType =
3267 PadOp::inferResultType(sourceType, staticLow: getStaticLow(), staticHigh: getStaticHigh());
3268 if (!expectedType) {
3269 return emitError(message: "failed to infer expectedType from sourceType ")
3270 << sourceType << ", specified resultType is " << resultType;
3271 }
3272 if (resultType.getRank() != expectedType.getRank()) {
3273 return emitError(message: "specified type ")
3274 << resultType << " does not match the inferred type "
3275 << expectedType;
3276 }
3277 for (int i = 0, e = sourceType.getRank(); i < e; ++i) {
3278 if (resultType.getDimSize(idx: i) == expectedType.getDimSize(idx: i))
3279 continue;
3280 if (expectedType.isDynamicDim(idx: i))
3281 continue;
3282 return emitError(message: "specified type ")
3283 << resultType << " does not match the inferred type "
3284 << expectedType;
3285 }
3286
3287 return success();
3288}
3289
3290LogicalResult PadOp::verifyRegions() {
3291 auto &region = getRegion();
3292 unsigned rank = llvm::cast<RankedTensorType>(Val: getResult().getType()).getRank();
3293 Block &block = region.front();
3294 if (block.getNumArguments() != rank)
3295 return emitError(message: "expected the block to have ") << rank << " arguments";
3296
3297 // Note: the number and type of yield values are checked in the YieldOp.
3298 for (const auto &en : llvm::enumerate(First: block.getArgumentTypes())) {
3299 if (!en.value().isIndex())
3300 return emitOpError(message: "expected block argument ")
3301 << (en.index() + 1) << " to be an index";
3302 }
3303
3304 // Ensure that the region yields an element of the right type.
3305 auto yieldOp = llvm::cast<YieldOp>(Val: block.getTerminator());
3306 if (yieldOp.getValue().getType() !=
3307 llvm::cast<ShapedType>(Val: getType()).getElementType())
3308 return emitOpError(message: "expected yield type to match shape element type");
3309
3310 return success();
3311}
3312
3313RankedTensorType PadOp::inferResultType(RankedTensorType sourceType,
3314 ArrayRef<int64_t> staticLow,
3315 ArrayRef<int64_t> staticHigh,
3316 ArrayRef<int64_t> resultShape) {
3317 unsigned rank = sourceType.getRank();
3318 if (staticLow.size() != rank)
3319 return RankedTensorType();
3320 if (staticHigh.size() != rank)
3321 return RankedTensorType();
3322 if (!resultShape.empty() && resultShape.size() != rank)
3323 return RankedTensorType();
3324
3325 SmallVector<int64_t, 4> inferredShape;
3326 for (auto i : llvm::seq<unsigned>(Begin: 0, End: rank)) {
3327 if (sourceType.isDynamicDim(idx: i) || staticLow[i] == ShapedType::kDynamic ||
3328 staticHigh[i] == ShapedType::kDynamic) {
3329 inferredShape.push_back(Elt: resultShape.empty() ? ShapedType::kDynamic
3330 : resultShape[i]);
3331 } else {
3332 int64_t size = sourceType.getDimSize(idx: i) + staticLow[i] + staticHigh[i];
3333 assert((resultShape.empty() || size == resultShape[i] ||
3334 resultShape[i] == ShapedType::kDynamic) &&
3335 "mismatch between inferred shape and result shape");
3336 inferredShape.push_back(Elt: size);
3337 }
3338 }
3339
3340 Type elementType = sourceType.getElementType();
3341 return RankedTensorType::get(
3342 shape: inferredShape, elementType,
3343 encoding: propagateEncoding(encoding: sourceType.getEncoding(), shape: inferredShape, elementType));
3344}
3345
3346void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
3347 Value source, ArrayRef<int64_t> staticLow,
3348 ArrayRef<int64_t> staticHigh, ValueRange low, ValueRange high,
3349 bool nofold, ArrayRef<NamedAttribute> attrs) {
3350 auto sourceType = llvm::cast<RankedTensorType>(Val: source.getType());
3351 if (!resultType)
3352 resultType = inferResultType(sourceType, staticLow, staticHigh);
3353 result.addAttributes(newAttributes: attrs);
3354 build(odsBuilder&: b, odsState&: result, result: resultType, source, low, high,
3355 static_low: b.getDenseI64ArrayAttr(values: staticLow), static_high: b.getDenseI64ArrayAttr(values: staticHigh),
3356 nofold: nofold ? b.getUnitAttr() : UnitAttr());
3357}
3358
3359void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
3360 Value source, ValueRange low, ValueRange high, bool nofold,
3361 ArrayRef<NamedAttribute> attrs) {
3362 auto sourceType = llvm::cast<RankedTensorType>(Val: source.getType());
3363 unsigned rank = sourceType.getRank();
3364 SmallVector<int64_t, 4> staticVector(rank, ShapedType::kDynamic);
3365 build(b, result, resultType, source, staticLow: staticVector, staticHigh: staticVector, low, high,
3366 nofold, attrs);
3367}
3368
3369void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
3370 Value source, ArrayRef<OpFoldResult> low,
3371 ArrayRef<OpFoldResult> high, bool nofold,
3372 ArrayRef<NamedAttribute> attrs) {
3373 auto sourceType = llvm::cast<RankedTensorType>(Val: source.getType());
3374 SmallVector<Value, 4> dynamicLow, dynamicHigh;
3375 SmallVector<int64_t, 4> staticLow, staticHigh;
3376 // staticLow and staticHigh have full information of the padding config.
3377 // This will grow staticLow and staticHigh with 1 value. If the config is
3378 // dynamic (ie not a constant), dynamicLow and dynamicHigh will grow with 1
3379 // value as well.
3380 dispatchIndexOpFoldResults(ofrs: low, dynamicVec&: dynamicLow, staticVec&: staticLow);
3381 dispatchIndexOpFoldResults(ofrs: high, dynamicVec&: dynamicHigh, staticVec&: staticHigh);
3382 if (!resultType) {
3383 resultType = PadOp::inferResultType(sourceType, staticLow, staticHigh);
3384 }
3385 assert(llvm::isa<RankedTensorType>(resultType));
3386 result.addAttributes(newAttributes: attrs);
3387 build(odsBuilder&: b, odsState&: result, result: resultType, source, low: dynamicLow, high: dynamicHigh,
3388 static_low: b.getDenseI64ArrayAttr(values: staticLow), static_high: b.getDenseI64ArrayAttr(values: staticHigh),
3389 nofold: nofold ? b.getUnitAttr() : UnitAttr());
3390}
3391
3392void PadOp::build(OpBuilder &b, OperationState &result, Type resultType,
3393 Value source, ArrayRef<OpFoldResult> low,
3394 ArrayRef<OpFoldResult> high, Value constantPadValue,
3395 bool nofold, ArrayRef<NamedAttribute> attrs) {
3396 build(b, result, resultType, source, low, high, nofold, attrs);
3397
3398 // Add a region and a block to yield the pad value.
3399 Region *region = result.regions[0].get();
3400 int sourceRank = llvm::cast<RankedTensorType>(Val: source.getType()).getRank();
3401 Repeated<Type> blockArgTypes(sourceRank, b.getIndexType());
3402 SmallVector<Location> blockArgLocs(sourceRank, result.location);
3403
3404 // `builder.createBlock` changes the insertion point within the block. Create
3405 // a guard to reset the insertion point of the builder after it is destroyed.
3406 OpBuilder::InsertionGuard guard(b);
3407 b.createBlock(parent: region, insertPt: region->end(), argTypes: blockArgTypes, locs: blockArgLocs);
3408 tensor::YieldOp::create(builder&: b, location: result.location, value: constantPadValue);
3409}
3410
3411llvm::SmallBitVector PadOp::getPaddedDims() {
3412 llvm::SmallBitVector paddedDims(getSourceType().getRank());
3413 auto extractPaddedDims = [&](ArrayRef<OpFoldResult> paddingWidths) {
3414 for (const auto &en : enumerate(First&: paddingWidths))
3415 if (getConstantIntValue(ofr: en.value()) != static_cast<int64_t>(0))
3416 paddedDims.set(en.index());
3417 };
3418 extractPaddedDims(getMixedLowPad());
3419 extractPaddedDims(getMixedHighPad());
3420 return paddedDims;
3421}
3422
3423namespace {
3424// Folds tensor.pad when padding is static zeros and the attribute
3425// doesn't request otherwise.
3426struct FoldStaticZeroPadding : public OpRewritePattern<PadOp> {
3427 using OpRewritePattern<PadOp>::OpRewritePattern;
3428
3429 LogicalResult matchAndRewrite(PadOp padTensorOp,
3430 PatternRewriter &rewriter) const override {
3431 if (!padTensorOp.hasZeroLowPad() || !padTensorOp.hasZeroHighPad())
3432 return failure();
3433 if (padTensorOp.getNofold())
3434 return failure();
3435 rewriter.replaceOpWithNewOp<tensor::CastOp>(
3436 op: padTensorOp, args: padTensorOp.getResult().getType(),
3437 args: padTensorOp.getSource());
3438 return success();
3439 }
3440};
3441
3442// Fold CastOp into PadOp when adding static information.
3443struct FoldSourceTensorCast : public OpRewritePattern<PadOp> {
3444 using OpRewritePattern<PadOp>::OpRewritePattern;
3445
3446 LogicalResult matchAndRewrite(PadOp padTensorOp,
3447 PatternRewriter &rewriter) const override {
3448 auto castOp = padTensorOp.getSource().getDefiningOp<tensor::CastOp>();
3449 if (!tensor::canFoldIntoConsumerOp(castOp))
3450 return failure();
3451
3452 auto newResultType = PadOp::inferResultType(
3453 sourceType: llvm::cast<RankedTensorType>(Val: castOp.getSource().getType()),
3454 staticLow: padTensorOp.getStaticLow(), staticHigh: padTensorOp.getStaticHigh(),
3455 resultShape: padTensorOp.getResultType().getShape());
3456
3457 if (newResultType == padTensorOp.getResultType()) {
3458 rewriter.modifyOpInPlace(root: padTensorOp, callable: [&]() {
3459 padTensorOp.getSourceMutable().assign(value: castOp.getSource());
3460 });
3461 } else {
3462 auto newOp = PadOp::create(
3463 builder&: rewriter, location: padTensorOp->getLoc(), resultType: newResultType,
3464 source: padTensorOp.getSource(), staticLow: padTensorOp.getStaticLow(),
3465 staticHigh: padTensorOp.getStaticHigh(), low: padTensorOp.getLow(),
3466 high: padTensorOp.getHigh(), nofold: padTensorOp.getNofold(),
3467 attrs: getPrunedAttributeList(op: padTensorOp, elidedAttrs: PadOp::getAttributeNames()));
3468 IRMapping mapper;
3469 padTensorOp.getRegion().cloneInto(dest: &newOp.getRegion(), mapper);
3470
3471 rewriter.replaceOpWithNewOp<tensor::CastOp>(
3472 op: padTensorOp, args: padTensorOp.getResultType(), args&: newOp);
3473 }
3474 return success();
3475 }
3476};
3477
3478// Fold CastOp using the result of PadOp back into the latter if it adds
3479// static information.
3480struct FoldTargetTensorCast : public OpRewritePattern<PadOp> {
3481 using OpRewritePattern<PadOp>::OpRewritePattern;
3482
3483 LogicalResult matchAndRewrite(PadOp padTensorOp,
3484 PatternRewriter &rewriter) const override {
3485 if (!padTensorOp.getResult().hasOneUse())
3486 return failure();
3487 auto tensorCastOp =
3488 dyn_cast<tensor::CastOp>(Val: *padTensorOp->getUsers().begin());
3489 if (!tensorCastOp)
3490 return failure();
3491 if (!tensor::preservesStaticInformation(source: padTensorOp.getResult().getType(),
3492 target: tensorCastOp.getDest().getType()))
3493 return failure();
3494
3495 auto replacementOp = PadOp::create(
3496 builder&: rewriter, location: padTensorOp.getLoc(), resultType: tensorCastOp.getDest().getType(),
3497 source: padTensorOp.getSource(), staticLow: padTensorOp.getStaticLow(),
3498 staticHigh: padTensorOp.getStaticHigh(), low: padTensorOp.getLow(),
3499 high: padTensorOp.getHigh(), nofold: padTensorOp.getNofold(),
3500 attrs: getPrunedAttributeList(op: padTensorOp, elidedAttrs: PadOp::getAttributeNames()));
3501 replacementOp.getRegion().takeBody(other&: padTensorOp.getRegion());
3502
3503 rewriter.replaceOp(op: padTensorOp, newValues: replacementOp.getResult());
3504 rewriter.replaceOp(op: tensorCastOp, newValues: replacementOp.getResult());
3505 return success();
3506 }
3507};
3508
3509/// Fold chains of tensor::ExtractSliceOp, tensor::PadOp pairs that pad
3510/// different dimensions. The pattern applies if the following preconditions
3511/// hold:
3512/// 1) the tensor::ExtractSliceOps are not rank-reducing,
3513/// 2) the tensor::ExtractSliceOps have only unit-strides,
3514/// 3) the tensor::PadOps perform only high-padding,
3515/// 4) the tensor::PadOps have the same constant padding value,
3516/// 5) the tensor::PadOps do not have common padding dimensions,
3517/// 6) one tensor::ExtractSliceOp, tensor::PadOp pair has zero-padding and
3518/// zero-offset for every dimension.
3519/// 7) the tensor::ExtractSliceOp sizes match the source tensor sizes for
3520/// the
3521/// padded source dimensions.
3522///
3523/// Example:
3524///
3525/// ```mlir
3526/// %0 = tensor.extract_slice %input[16, 0] [%sz0, 64] [1, 1]
3527/// : tensor<64x64xf32> to tensor<?x64xf32>
3528/// %1 = tensor.pad %0 low[0, 0] high[%pw0, 0] { ...
3529/// } : tensor<?x64xf32> to tensor<8x64xf32>
3530/// %2 = tensor.extract_slice %1[0, 4] [8, %sz1] [1, 1]
3531/// : tensor<8x64xf32> to tensor<8x?xf32>
3532/// %res = tensor.pad %2 nofold low[0, 0] high[0, %pw1] { ...
3533/// } : tensor<8x?xf32> to tensor<8x4xf32>
3534/// ```
3535///
3536/// folds into:
3537///
3538/// ```mlir
3539/// %0 = tensor.extract_slice %input[16, 4] [%sz0, %sz1] [1, 1]
3540/// : tensor<64x64xf32> to tensor<?x?xf32>
3541/// %res = tensor.pad %0 nofold low[0, 0] high[%pw0, %pw1] { ...
3542/// } : tensor<?x?xf32> to tensor<8x4xf32>
3543/// ```
3544struct FoldOrthogonalPaddings : public OpRewritePattern<PadOp> {
3545 using OpRewritePattern<PadOp>::OpRewritePattern;
3546
3547 LogicalResult matchAndRewrite(PadOp padOp,
3548 PatternRewriter &rewriter) const override {
3549 auto innerSliceOp = padOp.getSource().getDefiningOp<ExtractSliceOp>();
3550 if (!innerSliceOp)
3551 return failure();
3552 auto outerPadOp = innerSliceOp.getSource().getDefiningOp<PadOp>();
3553 if (!outerPadOp || outerPadOp.getNofold())
3554 return failure();
3555 auto outerSliceOp = outerPadOp.getSource().getDefiningOp<ExtractSliceOp>();
3556 if (!outerSliceOp)
3557 return failure();
3558
3559 // 1) Fail if the chain is rank-reducing.
3560 int64_t rank = padOp.getSourceType().getRank();
3561 if (outerSliceOp.getSourceType().getRank() != rank) {
3562 return rewriter.notifyMatchFailure(arg&: padOp,
3563 msg: "cannot fold rank-reducing chain");
3564 }
3565
3566 // 2) Fail if the tensor::ExtractSliceOps have non-unit strides.
3567 if (!innerSliceOp.hasUnitStride() || !outerSliceOp.hasUnitStride()) {
3568 return rewriter.notifyMatchFailure(
3569 arg&: padOp, msg: "cannot fold non-unit stride ExtractSliceOps");
3570 }
3571
3572 // 3) Fail if the tensor::PadOps have non-zero low padding.
3573 if (!padOp.hasZeroLowPad() || !outerPadOp.hasZeroLowPad()) {
3574 return rewriter.notifyMatchFailure(arg&: padOp,
3575 msg: "cannot fold PadOps with low padding");
3576 }
3577
3578 // 4) Fail if the tensor::PadOps padding values do not match.
3579 Attribute innerAttr, outerAttr;
3580 Value innerValue = padOp.getConstantPaddingValue();
3581 Value outerValue = outerPadOp.getConstantPaddingValue();
3582 if (!innerValue || !outerValue ||
3583 !matchPattern(value: innerValue, pattern: m_Constant(bind_value: &innerAttr)) ||
3584 !matchPattern(value: outerValue, pattern: m_Constant(bind_value: &outerAttr)) ||
3585 innerAttr != outerAttr) {
3586 return rewriter.notifyMatchFailure(
3587 arg&: padOp, msg: "cannot fold PadOps with different padding values");
3588 }
3589
3590 // 5) Fail if a dimension is padded by both tensor::PadOps.
3591 llvm::SmallBitVector innerDims = padOp.getPaddedDims();
3592 llvm::SmallBitVector outerDims = outerPadOp.getPaddedDims();
3593 if (innerDims.anyCommon(RHS: outerDims)) {
3594 return rewriter.notifyMatchFailure(
3595 arg&: padOp, msg: "cannot fold PadOps with common padding dimensions");
3596 }
3597
3598 // 6) Combine the offsets of the two tensor::ExtractSliceOps. Find the
3599 // zero-offset and zero-padding tensor::ExtractSliceOp, tensor::PadOp pair
3600 // for every dimension, and use the offset the other pair. Fail if no
3601 // zero-offset and zero-padding tensor::ExtractSliceOp, tensor::PadOp pair
3602 // exists.
3603 SmallVector<OpFoldResult> newOffsets(rank, rewriter.getIndexAttr(value: 0));
3604 for (auto en : enumerate(First&: newOffsets)) {
3605 OpFoldResult innerOffset = innerSliceOp.getMixedOffsets()[en.index()];
3606 OpFoldResult outerOffset = outerSliceOp.getMixedOffsets()[en.index()];
3607 if (!innerDims.test(Idx: en.index()) &&
3608 (getConstantIntValue(ofr: innerOffset) == static_cast<int64_t>(0))) {
3609 en.value() = outerOffset;
3610 continue;
3611 }
3612 if (!outerDims.test(Idx: en.index()) &&
3613 (getConstantIntValue(ofr: outerOffset) == static_cast<int64_t>(0))) {
3614 en.value() = innerOffset;
3615 continue;
3616 }
3617 return rewriter.notifyMatchFailure(
3618 arg&: padOp, msg: "cannot find zero-offset and zero-padding pair");
3619 }
3620
3621 // 7) Combine the sizes of the two tensor::ExtractSliceOps. Take the size
3622 // of the outer tensor::ExtractSliceOp for the dimensions padded by the
3623 // outer tensor::PadOp and fail if the size of the inner
3624 // tensor::ExtractSliceOp does not match the size of the padded dimension.
3625 // Otherwise, take the size of the inner tensor::ExtractSliceOp.
3626 SmallVector<OpFoldResult> newSizes = innerSliceOp.getMixedSizes();
3627 for (auto en : enumerate(First&: newSizes)) {
3628 if (!outerDims.test(Idx: en.index()))
3629 continue;
3630 OpFoldResult sliceSize = innerSliceOp.getMixedSizes()[en.index()];
3631 int64_t sourceSize = innerSliceOp.getSourceType().getShape()[en.index()];
3632 assert(ShapedType::isStatic(sourceSize) &&
3633 "expected padded dimension to have a static size");
3634 if (getConstantIntValue(ofr: sliceSize) != sourceSize) {
3635 return rewriter.notifyMatchFailure(
3636 arg&: padOp, msg: "cannot fold since the inner ExtractSliceOp size does not "
3637 "match the size of the outer padding");
3638 }
3639 en.value() = outerSliceOp.getMixedSizes()[en.index()];
3640 }
3641
3642 // Combine the high paddings of the two tensor::PadOps.
3643 SmallVector<OpFoldResult> newHighPad(rank, rewriter.getIndexAttr(value: 0));
3644 for (auto en : enumerate(First&: newHighPad)) {
3645 if (innerDims.test(Idx: en.index()))
3646 newHighPad[en.index()] = padOp.getMixedHighPad()[en.index()];
3647 if (outerDims.test(Idx: en.index()))
3648 newHighPad[en.index()] = outerPadOp.getMixedHighPad()[en.index()];
3649 }
3650
3651 // Create a new tensor::ExtractSliceOp, tensor::PadOp pair that performs
3652 // the two paddings in one step.
3653 auto newSliceOp = ExtractSliceOp::create(
3654 builder&: rewriter, location: padOp.getLoc(), source: outerSliceOp.getSource(), offsets: newOffsets,
3655 sizes: newSizes, strides: innerSliceOp.getMixedStrides());
3656 auto newPadOp = PadOp::create(
3657 builder&: rewriter, location: padOp.getLoc(), resultType: padOp.getResultType(), source: newSliceOp.getResult(),
3658 low: padOp.getMixedLowPad(), high: newHighPad, nofold: padOp.getNofold(),
3659 attrs: getPrunedAttributeList(op: padOp, elidedAttrs: PadOp::getAttributeNames()));
3660 rewriter.inlineRegionBefore(region&: padOp.getRegion(), parent&: newPadOp.getRegion(),
3661 before: newPadOp.getRegion().begin());
3662 rewriter.replaceOp(op: padOp, newValues: newPadOp.getResult());
3663 return success();
3664 }
3665};
3666
3667struct FoldStaticPadding : public OpRewritePattern<PadOp> {
3668 using OpRewritePattern<PadOp>::OpRewritePattern;
3669
3670 LogicalResult matchAndRewrite(PadOp padTensorOp,
3671 PatternRewriter &rewriter) const override {
3672 Value input = padTensorOp.getSource();
3673 if (!llvm::isa<RankedTensorType>(Val: input.getType()))
3674 return failure();
3675 auto inputDims = llvm::cast<RankedTensorType>(Val: input.getType()).getShape();
3676 auto inputRank = inputDims.size();
3677
3678 auto oldResultType =
3679 dyn_cast<RankedTensorType>(Val: padTensorOp.getResult().getType());
3680 if (!oldResultType)
3681 return failure();
3682
3683 auto outputDims = oldResultType.getShape();
3684
3685 // Extract the static info from the high and low operands. A negative
3686 // constant is not folded, since the attribute must be non-negative.
3687 // TODO: we should consider folding to a poison value in this case.
3688 SmallVector<int64_t> constOperandsLow;
3689 SmallVector<Value> newLows;
3690 for (auto operand : padTensorOp.getLow()) {
3691 APSInt intOp;
3692 if (!matchPattern(value: operand, pattern: m_ConstantInt(bind_value: &intOp)) ||
3693 intOp.getExtValue() < 0) {
3694 constOperandsLow.push_back(Elt: ShapedType::kDynamic);
3695 newLows.push_back(Elt: operand);
3696 continue;
3697 }
3698 constOperandsLow.push_back(Elt: intOp.getExtValue());
3699 }
3700 SmallVector<int64_t> constOperandsHigh;
3701 SmallVector<Value> newHighs;
3702 for (auto operand : padTensorOp.getHigh()) {
3703 APSInt intOp;
3704 if (!matchPattern(value: operand, pattern: m_ConstantInt(bind_value: &intOp)) ||
3705 intOp.getExtValue() < 0) {
3706 constOperandsHigh.push_back(Elt: ShapedType::kDynamic);
3707 newHighs.push_back(Elt: operand);
3708 continue;
3709 }
3710 constOperandsHigh.push_back(Elt: intOp.getExtValue());
3711 }
3712
3713 SmallVector<int64_t> constLow(padTensorOp.getStaticLow());
3714 SmallVector<int64_t> constHigh(padTensorOp.getStaticHigh());
3715
3716 // Verify the op is well-formed.
3717 if (inputDims.size() != outputDims.size() ||
3718 inputDims.size() != constLow.size() ||
3719 inputDims.size() != constHigh.size())
3720 return failure();
3721
3722 auto lowCount = 0;
3723 auto highCount = 0;
3724 for (size_t i = 0; i < inputRank; i++) {
3725 if (constLow[i] == ShapedType::kDynamic)
3726 constLow[i] = constOperandsLow[lowCount++];
3727 if (constHigh[i] == ShapedType::kDynamic)
3728 constHigh[i] = constOperandsHigh[highCount++];
3729 }
3730
3731 auto staticLow = ArrayRef<int64_t>(constLow);
3732 auto staticHigh = ArrayRef<int64_t>(constHigh);
3733
3734 // Calculate the output sizes with the static information.
3735 SmallVector<int64_t> newOutDims;
3736 for (size_t i = 0; i < inputRank; i++) {
3737 if (outputDims[i] == ShapedType::kDynamic) {
3738 newOutDims.push_back(
3739 Elt: (staticLow[i] == ShapedType::kDynamic ||
3740 staticHigh[i] == ShapedType::kDynamic ||
3741 inputDims[i] == ShapedType::kDynamic
3742 ? ShapedType::kDynamic
3743 : inputDims[i] + staticLow[i] + staticHigh[i]));
3744 } else {
3745 newOutDims.push_back(Elt: outputDims[i]);
3746 }
3747 }
3748
3749 if (SmallVector<int64_t>(outputDims) == newOutDims ||
3750 llvm::all_of(Range&: newOutDims,
3751 P: [&](int64_t x) { return x == ShapedType::kDynamic; }))
3752 return failure();
3753
3754 Type elementType = padTensorOp.getType().getElementType();
3755 auto newResultType = RankedTensorType::get(
3756 shape: newOutDims, elementType,
3757 encoding: propagateEncoding(encoding: padTensorOp.getType().getEncoding(), shape: newOutDims,
3758 elementType));
3759 auto newOp = PadOp::create(
3760 builder&: rewriter, location: padTensorOp->getLoc(), resultType: newResultType, source: input, staticLow,
3761 staticHigh, low: newLows, high: newHighs, nofold: padTensorOp.getNofold(),
3762 attrs: getPrunedAttributeList(op: padTensorOp, elidedAttrs: PadOp::getAttributeNames()));
3763
3764 IRMapping mapper;
3765 padTensorOp.getRegion().cloneInto(dest: &newOp.getRegion(), mapper);
3766 rewriter.replaceOpWithNewOp<tensor::CastOp>(op: padTensorOp, args&: oldResultType,
3767 args&: newOp);
3768
3769 return success();
3770 }
3771};
3772
3773/// Folds a chain of `tensor.pad` ops with the same constant padding value.
3774///
3775/// Example:
3776///
3777/// ```mlir
3778/// %1 = tensor.pad %0 low[0, 1] high[0, 2] {
3779/// tensor.yield %val
3780/// } : tensor<1x2xf32> to tensor<2x5xf32>
3781/// %res = tensor.pad %1 low[0, 2] high[3, 0] {
3782/// tensor.yield %val
3783/// } : tensor<1x5xf32> to tensor<5x7xf32>
3784/// ```
3785///
3786/// folds into:
3787///
3788/// ```mlir
3789/// %res = tensor.pad %0 low[0, 3] high[3, 2] {
3790/// tensor.yield %val
3791/// } : tensor<1x2xf32> to tensor<5x7xf32>
3792/// ```
3793struct FoldConsecutiveConstantPadding : public OpRewritePattern<tensor::PadOp> {
3794 using OpRewritePattern<tensor::PadOp>::OpRewritePattern;
3795
3796 LogicalResult matchAndRewrite(tensor::PadOp padOp,
3797 PatternRewriter &rewriter) const override {
3798 if (padOp.getNofold()) {
3799 return rewriter.notifyMatchFailure(arg&: padOp, msg: "skipping unfoldable pad");
3800 }
3801
3802 auto producerPad = padOp.getSource().getDefiningOp<tensor::PadOp>();
3803 if (!producerPad || producerPad.getNofold()) {
3804 return rewriter.notifyMatchFailure(
3805 arg&: padOp, msg: "producer is not a foldable tensor.pad op");
3806 }
3807
3808 // Fail if the tensor::PadOps padding values do not match.
3809 Value consumerPadValue = padOp.getConstantPaddingValue();
3810 Value producerPadValue = producerPad.getConstantPaddingValue();
3811 if (!consumerPadValue || !producerPadValue ||
3812 consumerPadValue != producerPadValue) {
3813 return rewriter.notifyMatchFailure(
3814 arg&: padOp,
3815 msg: "cannot fold PadOps with different or non-constant padding values");
3816 }
3817
3818 Location loc = padOp.getLoc();
3819 AffineExpr d0, d1;
3820 bindDims(ctx: rewriter.getContext(), exprs&: d0, exprs&: d1);
3821
3822 // Combine the low/high paddings of the two tensor::PadOps.
3823 auto addPaddings = [&](ArrayRef<OpFoldResult> consumerPaddings,
3824 ArrayRef<OpFoldResult> producerPaddings) {
3825 SmallVector<OpFoldResult> sumPaddings;
3826 for (auto [consumerIndex, producerIndex] :
3827 llvm::zip_equal(t&: consumerPaddings, u&: producerPaddings)) {
3828 sumPaddings.push_back(Elt: affine::makeComposedFoldedAffineApply(
3829 b&: rewriter, loc, expr: d0 + d1, operands: {consumerIndex, producerIndex}));
3830 }
3831 return sumPaddings;
3832 };
3833
3834 SmallVector<OpFoldResult> newHighPad =
3835 addPaddings(padOp.getMixedHighPad(), producerPad.getMixedHighPad());
3836 SmallVector<OpFoldResult> newLowPad =
3837 addPaddings(padOp.getMixedLowPad(), producerPad.getMixedLowPad());
3838
3839 auto newPadOp = tensor::PadOp::create(
3840 builder&: rewriter, location: padOp.getLoc(), resultType: padOp.getResultType(),
3841 source: producerPad.getSource(), low: newLowPad, high: newHighPad, nofold: padOp.getNofold(),
3842 attrs: getPrunedAttributeList(op: padOp, elidedAttrs: tensor::PadOp::getAttributeNames()));
3843 rewriter.inlineRegionBefore(region&: padOp.getRegion(), parent&: newPadOp.getRegion(),
3844 before: newPadOp.getRegion().begin());
3845 rewriter.replaceOp(op: padOp, newValues: newPadOp.getResult());
3846 return success();
3847 }
3848};
3849
3850} // namespace
3851
3852LogicalResult
3853PadOp::reifyResultShapes(OpBuilder &b,
3854 ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
3855 reifiedReturnShapes.resize(N: 1, NV: SmallVector<OpFoldResult>(getType().getRank()));
3856 SmallVector<OpFoldResult> lp = getMixedLowPad();
3857 SmallVector<OpFoldResult> hp = getMixedHighPad();
3858 for (int64_t i = 0; i < getResultType().getRank(); ++i) {
3859 if (!getType().isDynamicDim(idx: i)) {
3860 reifiedReturnShapes[0][i] = b.getIndexAttr(value: getType().getDimSize(idx: i));
3861 continue;
3862 }
3863 Location loc = getLoc();
3864 Value dim = b.createOrFold<tensor::DimOp>(
3865 location: loc, args: getSource(), args: arith::ConstantIndexOp::create(builder&: b, location: loc, value: i));
3866
3867 AffineExpr d0, d1, d2;
3868 bindDims(ctx: b.getContext(), exprs&: d0, exprs&: d1, exprs&: d2);
3869 reifiedReturnShapes[0][i] = affine::makeComposedFoldedAffineApply(
3870 b, loc, expr: {d0 + d1 + d2}, operands: {dim, lp[i], hp[i]});
3871 }
3872 return success();
3873}
3874
3875void PadOp::getCanonicalizationPatterns(RewritePatternSet &results,
3876 MLIRContext *context) {
3877 results.add<FoldStaticZeroPadding, FoldSourceTensorCast, FoldTargetTensorCast,
3878 FoldOrthogonalPaddings, FoldStaticPadding,
3879 FoldConsecutiveConstantPadding>(arg&: context);
3880}
3881
3882/// Return the padding value of the PadOp if it constant. In this context,
3883/// "constant" means an actual constant or "defined outside of the block".
3884///
3885/// Values are considered constant in three cases:
3886/// - A ConstantLike value.
3887/// - A basic block argument from a different block.
3888/// - A value defined outside of the block.
3889///
3890/// If the padding value is not constant, an empty Value is returned.
3891Value PadOp::getConstantPaddingValue() {
3892 auto yieldOp = dyn_cast<YieldOp>(Val: getRegion().front().getTerminator());
3893 if (!yieldOp)
3894 return {};
3895 Value padValue = yieldOp.getValue();
3896 // Check if yield value is a constant.
3897 if (matchPattern(value: padValue, pattern: m_Constant()))
3898 return padValue;
3899 // Check if yield value is defined inside the PadOp block.
3900 if (padValue.getParentBlock() == &getRegion().front())
3901 return {};
3902 // Else: Yield value defined outside of the PadOp block.
3903 return padValue;
3904}
3905
3906OpFoldResult PadOp::fold(FoldAdaptor) {
3907 if (getResultType().hasStaticShape() && getResultType() == getSourceType() &&
3908 !getNofold())
3909 return getSource();
3910 return {};
3911}
3912
3913//===----------------------------------------------------------------------===//
3914// ParallelInsertSliceOp
3915//===----------------------------------------------------------------------===//
3916
3917OpResult ParallelInsertSliceOp::getTiedOpResult() {
3918 InParallelOpInterface parallelCombiningParent = getParallelCombiningParent();
3919 for (const auto &it :
3920 llvm::enumerate(First: parallelCombiningParent.getYieldingOps())) {
3921 Operation &nextOp = it.value();
3922 if (&nextOp == getOperation())
3923 return parallelCombiningParent.getParentResult(idx: it.index());
3924 }
3925 llvm_unreachable("ParallelInsertSliceOp no tied OpResult found");
3926}
3927
3928// Build a ParallelInsertSliceOp with mixed static and dynamic entries.
3929void ParallelInsertSliceOp::build(OpBuilder &b, OperationState &result,
3930 Value source, Value dest,
3931 ArrayRef<OpFoldResult> offsets,
3932 ArrayRef<OpFoldResult> sizes,
3933 ArrayRef<OpFoldResult> strides,
3934 ArrayRef<NamedAttribute> attrs) {
3935 SmallVector<int64_t> staticOffsets, staticSizes, staticStrides;
3936 SmallVector<Value> dynamicOffsets, dynamicSizes, dynamicStrides;
3937 dispatchIndexOpFoldResults(ofrs: offsets, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets);
3938 dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticSizes);
3939 dispatchIndexOpFoldResults(ofrs: strides, dynamicVec&: dynamicStrides, staticVec&: staticStrides);
3940 result.addAttributes(newAttributes: attrs);
3941 build(odsBuilder&: b, odsState&: result, resultTypes: {}, source, dest, offsets: dynamicOffsets, sizes: dynamicSizes,
3942 strides: dynamicStrides, static_offsets: b.getDenseI64ArrayAttr(values: staticOffsets),
3943 static_sizes: b.getDenseI64ArrayAttr(values: staticSizes),
3944 static_strides: b.getDenseI64ArrayAttr(values: staticStrides));
3945}
3946
3947/// Build an ParallelInsertSliceOp with mixed static and dynamic entries
3948/// packed into a Range vector.
3949void ParallelInsertSliceOp::build(OpBuilder &b, OperationState &result,
3950 Value source, Value dest,
3951 ArrayRef<Range> ranges,
3952 ArrayRef<NamedAttribute> attrs) {
3953 auto [offsets, sizes, strides] = getOffsetsSizesAndStrides(ranges);
3954 build(b, result, source, dest, offsets, sizes, strides, attrs);
3955}
3956
3957// Build a ParallelInsertSliceOp with dynamic entries.
3958void ParallelInsertSliceOp::build(OpBuilder &b, OperationState &result,
3959 Value source, Value dest, ValueRange offsets,
3960 ValueRange sizes, ValueRange strides,
3961 ArrayRef<NamedAttribute> attrs) {
3962 SmallVector<OpFoldResult> offsetValues = llvm::map_to_vector<4>(
3963 C&: offsets, F: [](Value v) -> OpFoldResult { return v; });
3964 SmallVector<OpFoldResult> sizeValues =
3965 llvm::map_to_vector<4>(C&: sizes, F: [](Value v) -> OpFoldResult { return v; });
3966 SmallVector<OpFoldResult> strideValues = llvm::map_to_vector<4>(
3967 C&: strides, F: [](Value v) -> OpFoldResult { return v; });
3968 build(b, result, source, dest, offsets: offsetValues, sizes: sizeValues, strides: strideValues);
3969}
3970
3971// Build an InsertSliceOp with mixed static and dynamic sizes, offsets set
3972// to 0, strides set to 1 and inferred result type.
3973void InsertSliceOp::build(OpBuilder &b, OperationState &result, Value source,
3974 Value dest, ArrayRef<OpFoldResult> sizes,
3975 ArrayRef<NamedAttribute> attrs) {
3976 Attribute zeroIdxAttr = b.getIndexAttr(value: 0);
3977 Attribute oneIdxAttr = b.getIndexAttr(value: 1);
3978 SmallVector<OpFoldResult> writeStrides(sizes.size(), oneIdxAttr);
3979 SmallVector<OpFoldResult> writeOffsets(sizes.size(), zeroIdxAttr);
3980 build(b, result, source, dest, offsets: writeOffsets, sizes, strides: writeStrides, attrs);
3981}
3982
3983LogicalResult ParallelInsertSliceOp::verify() {
3984 if (!isa<InParallelOpInterface>(Val: getOperation()->getParentOp()))
3985 return this->emitError(message: "expected InParallelOpInterface parent, got:")
3986 << *(getOperation()->getParentOp());
3987
3988 // Verify result type against inferred type.
3989 RankedTensorType expectedType;
3990 SliceVerificationResult result =
3991 verifyInsertSliceOp(srcType: getSourceType(), dstType: getDestType(), staticOffsets: getStaticOffsets(),
3992 staticSizes: getStaticSizes(), staticStrides: getStaticStrides(), expectedType: &expectedType);
3993 if (result != SliceVerificationResult::Success)
3994 return produceSliceErrorMsg(result, op: *this, expectedType);
3995
3996 // Verify that offsets, sizes, strides do not run out-of-bounds with respect
3997 // to the destination tensor.
3998 SliceBoundsVerificationResult boundsResult = verifyInBoundsSlice(
3999 shape: getDestType().getShape(), staticOffsets: getStaticOffsets(), staticSizes: getStaticSizes(),
4000 staticStrides: getStaticStrides(), /*generateErrorMessage=*/true);
4001 if (!boundsResult.isValid)
4002 return getOperation()->emitError(message: boundsResult.errorMessage);
4003
4004 return success();
4005}
4006
4007void ParallelInsertSliceOp::getCanonicalizationPatterns(
4008 RewritePatternSet &results, MLIRContext *context) {
4009 results.add<InsertSliceOpConstantArgumentFolder<ParallelInsertSliceOp>,
4010 InsertSliceOpCastFolder<ParallelInsertSliceOp>,
4011 InsertSliceOpSourceCastInserter<ParallelInsertSliceOp>>(arg&: context);
4012}
4013
4014llvm::SmallBitVector ParallelInsertSliceOp::getDroppedDims() {
4015 return ::getDroppedDims(reducedShape: getSourceType().getShape(), mixedSizes: getMixedSizes());
4016}
4017
4018// ParallelCombiningOpInterface implementation.
4019MutableOperandRange ParallelInsertSliceOp::getUpdatedDestinations() {
4020 return getDestMutable();
4021}
4022
4023Operation *ParallelInsertSliceOp::getIteratingParent() {
4024 // Return the parent InParallelOpInterface's parent.
4025 if (auto combiningOp =
4026 dyn_cast<InParallelOpInterface>(Val: getOperation()->getParentOp()))
4027 return combiningOp->getParentOp();
4028 return nullptr;
4029}
4030
4031//===----------------------------------------------------------------------===//
4032// ScatterOp
4033//===----------------------------------------------------------------------===//
4034
4035void ScatterOp::getAsmResultNames(
4036 function_ref<void(Value, StringRef)> setNameFn) {
4037 setNameFn(getResult(), "scatter");
4038}
4039
4040LogicalResult ScatterOp::verify() {
4041 int64_t destRank = getDestType().getRank();
4042 ArrayRef<int64_t> scatterDims = getScatterDims();
4043 if (failed(Result: verifyGatherOrScatterDims(op: getOperation(), dims: scatterDims,
4044 indices: getIndicesType().getShape(), rank: destRank,
4045 gatherOrScatter: "scatter", sourceOrDest: "dest")))
4046 return failure();
4047
4048 if (!getUnique())
4049 return emitOpError(message: "requires 'unique' attribute to be set");
4050 // TODO: we could also check statically that there are fewer leading index
4051 // tensor dims than the dest dims. If this is not the case, the unique
4052 // attribute cannot be true.
4053
4054 // Use the GatherOp::inferResultType on the `dest` type and verify the
4055 // expected type matches the source type.
4056 RankedTensorType expectedSourceType = GatherOp::inferResultType(
4057 sourceType: getDestType(), indicesType: getIndicesType(), gatherDims: scatterDims, /*rankReduced=*/false);
4058 RankedTensorType expectedRankReducedSourceType = GatherOp::inferResultType(
4059 sourceType: getDestType(), indicesType: getIndicesType(), gatherDims: scatterDims, /*rankReduced=*/true);
4060 if (getSourceType() != expectedSourceType &&
4061 getSourceType() != expectedRankReducedSourceType) {
4062 return emitOpError(message: "source type "
4063 "mismatch: "
4064 "expected ")
4065 << expectedSourceType << " or its rank-reduced variant "
4066 << expectedRankReducedSourceType << " (got: " << getSourceType()
4067 << ")";
4068 }
4069
4070 return success();
4071}
4072
4073//===----------------------------------------------------------------------===//
4074// SplatOp
4075//===----------------------------------------------------------------------===//
4076
4077void SplatOp::build(OpBuilder &builder, OperationState &result, Value element,
4078 Type aggregateType, ValueRange dynamicSizes) {
4079 build(odsBuilder&: builder, odsState&: result, aggregate: aggregateType, input: element, dynamicSizes);
4080}
4081
4082void SplatOp::build(OpBuilder &builder, OperationState &result, Value element,
4083 ArrayRef<int64_t> staticShape, ValueRange dynamicSizes) {
4084 auto aggregateType = RankedTensorType::get(shape: staticShape, elementType: element.getType());
4085 build(odsBuilder&: builder, odsState&: result, aggregate: aggregateType, input: element, dynamicSizes);
4086}
4087
4088void SplatOp::build(OpBuilder &builder, OperationState &result, Value element,
4089 ArrayRef<OpFoldResult> sizes) {
4090 SmallVector<int64_t> staticShape;
4091 SmallVector<Value> dynamicSizes;
4092 dispatchIndexOpFoldResults(ofrs: sizes, dynamicVec&: dynamicSizes, staticVec&: staticShape);
4093 build(builder, result, element, staticShape, dynamicSizes);
4094}
4095
4096void SplatOp::getAsmResultNames(
4097 function_ref<void(Value, StringRef)> setNameFn) {
4098 setNameFn(getResult(), "splat");
4099}
4100
4101LogicalResult SplatOp::verify() {
4102 return verifyDynamicDimensionCount(op: getOperation(), type: getType(),
4103 dynamicSizes: getDynamicSizes());
4104}
4105
4106LogicalResult
4107SplatOp::reifyResultShapes(OpBuilder &builder,
4108 ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
4109 reifiedReturnShapes.resize(N: 1, NV: SmallVector<OpFoldResult>(getType().getRank()));
4110 unsigned ctr = 0;
4111 for (int64_t i = 0; i < getType().getRank(); ++i) {
4112 if (getType().isDynamicDim(idx: i)) {
4113 reifiedReturnShapes[0][i] = getDynamicSizes()[ctr++];
4114 } else {
4115 reifiedReturnShapes[0][i] = builder.getIndexAttr(value: getType().getDimSize(idx: i));
4116 }
4117 }
4118 return success();
4119}
4120
4121OpFoldResult SplatOp::fold(FoldAdaptor adaptor) {
4122 auto constOperand = adaptor.getInput();
4123 if (!isa_and_nonnull<IntegerAttr, FloatAttr>(Val: constOperand))
4124 return {};
4125
4126 // Do not fold if the splat is not statically shaped
4127 if (!getType().hasStaticShape())
4128 return {};
4129
4130 // SplatElementsAttr::get treats single value for second arg as being a
4131 // splat.
4132 return SplatElementsAttr::get(type: getType(), list: {constOperand});
4133}
4134
4135namespace {
4136/// Change the type of the result of a `tensor.splat` by making the result
4137/// type statically sized along dimensions that in the original operation were
4138/// defined as dynamic, but the size was defined using a `constant` op. For
4139/// example
4140///
4141/// %c4 = arith.constant 4 : index
4142/// %0 = tensor.splat %arg0[%c4] : tensor<?xf32>
4143///
4144/// to
4145///
4146/// %0 = tensor.splat %arg0 : tensor<4xf32>
4147struct ReplaceSplatStaticShapeDims : OpRewritePattern<SplatOp> {
4148 using OpRewritePattern<SplatOp>::OpRewritePattern;
4149
4150 LogicalResult matchAndRewrite(SplatOp op,
4151 PatternRewriter &rewriter) const override {
4152 SmallVector<Value> foldedDynamicSizes;
4153 RankedTensorType foldedTensorType = foldDynamicToStaticDimSizes(
4154 type: op.getType(), dynamicSizes: op.getDynamicSizes(), foldedDynamicSizes);
4155
4156 // Stop here if no dynamic size was promoted to static.
4157 if (foldedTensorType == op.getType())
4158 return failure();
4159
4160 auto newOp = SplatOp::create(builder&: rewriter, location: op.getLoc(), element: op.getInput(),
4161 aggregateType: foldedTensorType, dynamicSizes: foldedDynamicSizes);
4162 rewriter.replaceOpWithNewOp<tensor::CastOp>(op, args: op.getType(), args&: newOp);
4163 return success();
4164 }
4165};
4166} // namespace
4167
4168void SplatOp::getCanonicalizationPatterns(RewritePatternSet &results,
4169 MLIRContext *context) {
4170 results.add<ReplaceSplatStaticShapeDims>(arg&: context);
4171}
4172
4173//===----------------------------------------------------------------------===//
4174// Common Canonicalizers and Folders.
4175//===----------------------------------------------------------------------===//
4176static bool foldTensorCastPrecondition(DestinationStyleOpInterface op) {
4177 // 1. InsertSliceOp has its own logic about folding tensor.cast ops.
4178 // 2. Exclude DPS ops that are also LoopLike from this interface as they
4179 // might need special handling of attached regions.
4180 if (isa<InsertSliceOp>(Val: op.getOperation()) ||
4181 isa<LoopLikeOpInterface>(Val: op.getOperation()))
4182 return false;
4183
4184 return hasFoldableTensorCastOperand(op);
4185}
4186
4187/// Folds a tensor.cast op into a consuming DestinationStyleOpInterface op if
4188/// the `tensor.cast` has source that is more static than the consuming op.
4189///
4190/// Example:
4191/// ```mlir
4192/// %1 = tensor.cast %0 : tensor<8x16xf32> to tensor<?x?xf32>
4193/// %2 = consumer %1 ... : tensor<?x?xf32> ...
4194/// ```
4195///
4196/// folds into:
4197///
4198/// ```mlir
4199/// %2 = consumer %0 ... : tensor<8x16xf32> ...
4200/// ```
4201/// TODO: Move the pattern to a proper place, so all other DestinationStyleOp
4202/// can add the pattern to their canonicalizers.
4203struct FoldTensorCastProducerOp
4204 : public OpInterfaceRewritePattern<DestinationStyleOpInterface> {
4205 using OpInterfaceRewritePattern<
4206 DestinationStyleOpInterface>::OpInterfaceRewritePattern;
4207
4208 LogicalResult matchAndRewrite(DestinationStyleOpInterface op,
4209 PatternRewriter &rewriter) const override {
4210
4211 // Reject PackOp/UnpackOp (i.e. RelayoutOps) - there are dedicated patterns
4212 // for that instead.
4213 if (!foldTensorCastPrecondition(op) ||
4214 isa<linalg::RelayoutOpInterface>(Val: *op))
4215 return failure();
4216
4217 SmallVector<Type> newResultTypes(op->getResultTypes());
4218 SmallVector<Value> newOperands =
4219 getUpdatedOperandsAfterCastOpFolding(op, newResTy&: newResultTypes);
4220
4221 // Clone op
4222 auto newOp = clone(b&: rewriter, op, newResultTypes, newOperands);
4223
4224 SmallVector<Value, 4> replacements;
4225 replacements.reserve(N: newOp->getNumResults());
4226 for (auto [oldResult, newResult] :
4227 llvm::zip(t: op->getResults(), u: newOp->getResults())) {
4228 if (newResult.getType() != oldResult.getType()) {
4229 replacements.push_back(Elt: tensor::CastOp::create(
4230 builder&: rewriter, location: op->getLoc(), dest: oldResult.getType(), source: newResult));
4231 } else {
4232 replacements.push_back(Elt: newResult);
4233 }
4234 }
4235 rewriter.replaceOp(op, newValues: replacements);
4236
4237 return success();
4238 }
4239};
4240
4241namespace {
4242/// A splat of a scalar condition selects the same arm in every element.
4243struct SelectSplatCondition : OpRewritePattern<arith::SelectOp> {
4244 using Base::Base;
4245
4246 LogicalResult matchAndRewrite(arith::SelectOp select,
4247 PatternRewriter &rewriter) const override {
4248 auto splat = select.getCondition().getDefiningOp<SplatOp>();
4249 if (!splat)
4250 return failure();
4251 rewriter.modifyOpInPlace(
4252 root: select, callable: [&] { select.getConditionMutable().assign(value: splat.getInput()); });
4253 return success();
4254 }
4255};
4256} // namespace
4257
4258//===----------------------------------------------------------------------===//
4259// TensorDialect
4260//===----------------------------------------------------------------------===//
4261
4262void TensorDialect::getCanonicalizationPatterns(
4263 RewritePatternSet &results) const {
4264 results.add<FoldTensorCastProducerOp, SelectSplatCondition>(arg: getContext());
4265}
4266
4267//===----------------------------------------------------------------------===//
4268// TableGen'd op method definitions
4269//===----------------------------------------------------------------------===//
4270
4271#define GET_OP_CLASSES
4272#include "mlir/Dialect/Tensor/IR/TensorOps.cpp.inc"
4273