1//===- VectorOps.cpp - MLIR Vector Dialect Operations ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements convenience types for working with super-vectorization
10// operations, in particular super-vector loads and stores.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/Vector/IR/VectorOps.h"
15
16#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
17#include "mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h"
18#include "mlir/Dialect/Arith/IR/Arith.h"
19#include "mlir/Dialect/Arith/Utils/Utils.h"
20#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
21#include "mlir/Dialect/MemRef/IR/MemRef.h"
22#include "mlir/Dialect/MemRef/IR/MemoryAccessOpInterfaces.h"
23#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
24#include "mlir/Dialect/Tensor/IR/Tensor.h"
25#include "mlir/Dialect/UB/IR/UBMatchers.h"
26#include "mlir/Dialect/Utils/IndexingUtils.h"
27#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
28#include "mlir/Dialect/Utils/VerificationUtils.h"
29#include "mlir/IR/AffineExpr.h"
30#include "mlir/IR/AffineMap.h"
31#include "mlir/IR/Builders.h"
32#include "mlir/IR/BuiltinAttributes.h"
33#include "mlir/IR/BuiltinTypes.h"
34#include "mlir/IR/IRMapping.h"
35#include "mlir/IR/OpImplementation.h"
36#include "mlir/IR/PatternMatch.h"
37#include "mlir/IR/TypeUtilities.h"
38#include "mlir/IR/ValueRange.h"
39#include "mlir/Interfaces/SubsetOpInterface.h"
40#include "mlir/Interfaces/ValueBoundsOpInterface.h"
41#include "mlir/Interfaces/VectorInterfaces.h"
42#include "mlir/Support/LLVM.h"
43#include "mlir/Transforms/InliningUtils.h"
44#include "llvm/ADT/ArrayRef.h"
45#include "llvm/ADT/Repeated.h"
46#include "llvm/ADT/STLExtras.h"
47#include "llvm/ADT/SmallVector.h"
48#include "llvm/ADT/SmallVectorExtras.h"
49#include "llvm/ADT/StringSet.h"
50#include "llvm/Support/Casting.h"
51
52#include <cassert>
53#include <cstdint>
54#include <numeric>
55
56#include "mlir/Dialect/Vector/IR/VectorDialect.cpp.inc"
57// Pull in all enum type and utility function definitions.
58#include "mlir/Dialect/Vector/IR/VectorEnums.cpp.inc"
59
60using namespace mlir;
61using namespace mlir::vector;
62
63/// Helper enum to classify mask value.
64enum class MaskFormat {
65 AllTrue = 0,
66 AllFalse = 1,
67 Unknown = 2,
68};
69
70// Default target shape for unrolling CreateMaskOp and ConstantMaskOp
71static std::optional<SmallVector<int64_t>>
72getDefaultTargetMaskShapeForUnroll() {
73 return SmallVector<int64_t>{8, 8};
74}
75
76// Default target shape for unrolling InterleaveOp and DeinterleaveOp
77static std::optional<SmallVector<int64_t>>
78getDefaultTargetInterleaveShapeForUnroll() {
79 return SmallVector<int64_t>{2, 4};
80}
81
82/// Helper method to classify a mask value. Currently, the method
83/// looks "under the hood" of a constant value with dense attributes
84/// and a constant mask operation (since the client may be called at
85/// various stages during progressive lowering).
86static MaskFormat getMaskFormat(Value mask) {
87 if (auto c = mask.getDefiningOp<arith::ConstantOp>()) {
88 // Inspect constant dense values. We count up for bits that
89 // are set, count down for bits that are cleared, and bail
90 // when a mix is detected.
91 if (auto denseElts = llvm::dyn_cast<DenseIntElementsAttr>(Val: c.getValue())) {
92 int64_t val = 0;
93 for (bool b : denseElts.getValues<bool>())
94 if (b && val >= 0)
95 val++;
96 else if (!b && val <= 0)
97 val--;
98 else
99 return MaskFormat::Unknown;
100 if (val > 0)
101 return MaskFormat::AllTrue;
102 if (val < 0)
103 return MaskFormat::AllFalse;
104 }
105 } else if (auto m = mask.getDefiningOp<ConstantMaskOp>()) {
106 // Inspect constant mask index. If the index exceeds the
107 // dimension size, all bits are set. If the index is zero
108 // or less, no bits are set.
109 ArrayRef<int64_t> masks = m.getMaskDimSizes();
110 auto shape = m.getType().getShape();
111 bool allTrue = true;
112 bool allFalse = true;
113 for (auto [maskIdx, dimSize] : llvm::zip_equal(t&: masks, u&: shape)) {
114 if (maskIdx < dimSize)
115 allTrue = false;
116 if (maskIdx > 0)
117 allFalse = false;
118 }
119 if (allTrue)
120 return MaskFormat::AllTrue;
121 if (allFalse)
122 return MaskFormat::AllFalse;
123 } else if (auto m = mask.getDefiningOp<CreateMaskOp>()) {
124 // Finds all-false create_masks. An all-true create_mask requires all
125 // dims to be constants, so that'll be folded to a constant_mask, then
126 // detected in the constant_mask case.
127 auto maskOperands = m.getOperands();
128 for (Value operand : maskOperands) {
129 if (auto constantOp = operand.getDefiningOp<arith::ConstantOp>()) {
130 int64_t dimSize =
131 llvm::cast<IntegerAttr>(Val: constantOp.getValue()).getInt();
132 if (dimSize <= 0)
133 return MaskFormat::AllFalse;
134 }
135 }
136 return MaskFormat::Unknown;
137 }
138 return MaskFormat::Unknown;
139}
140
141/// Default callback to build a region with a 'vector.yield' terminator with no
142/// arguments.
143void mlir::vector::buildTerminatedBody(OpBuilder &builder, Location loc) {
144 vector::YieldOp::create(builder, location: loc);
145}
146
147// Helper for verifying combining kinds in contractions and reductions.
148static bool isSupportedCombiningKind(CombiningKind combiningKind,
149 Type elementType) {
150 switch (combiningKind) {
151 case CombiningKind::ADD:
152 case CombiningKind::MUL:
153 return elementType.isIntOrIndexOrFloat();
154 case CombiningKind::MINUI:
155 case CombiningKind::MINSI:
156 case CombiningKind::MAXUI:
157 case CombiningKind::MAXSI:
158 case CombiningKind::AND:
159 case CombiningKind::OR:
160 case CombiningKind::XOR:
161 return elementType.isIntOrIndex();
162 case CombiningKind::MINNUMF:
163 case CombiningKind::MAXNUMF:
164 case CombiningKind::MINIMUMF:
165 case CombiningKind::MAXIMUMF:
166 case CombiningKind::MINIMUMNUMF:
167 case CombiningKind::MAXIMUMNUMF:
168 return llvm::isa<FloatType>(Val: elementType);
169 }
170 return false;
171}
172
173/// Returns the effective rank of the vector to read/write for Xfer Ops
174///
175/// When the element type of the shaped type is _a scalar_, this will simply
176/// return the rank of the vector ( the result for xfer_read or the value to
177/// store for xfer_write).
178///
179/// When the element type of the base shaped type is _a vector_, returns the
180/// difference between the original vector type and the element type of the
181/// shaped type.
182///
183/// EXAMPLE 1 (element type is _a scalar_):
184/// - shapedType = tensor<10x20xf32>, vectorType = vector<2x4xf32>
185/// - shapedType.getElementType() = f32 (rank 0)
186/// - vectorType.getRank() = 2
187/// - Result = 2 - 0 = 2
188///
189/// EXAMPLE 2 (element type is _a vector_):
190/// - shapedType = tensor<10xvector<20xf32>>, vectorType = vector<20xf32>
191/// - shapedType.getElementType() = vector<20xf32> (rank 1)
192/// - vectorType.getRank() = 1
193/// - Result = 1 - 1 = 0
194///
195/// This is used to determine the number of minor dimensions for identity maps
196/// in vector transfer Ops.
197static unsigned getEffectiveVectorRankForXferOp(ShapedType shapedType,
198 VectorType vectorType) {
199 unsigned elementVectorRank = 0;
200 VectorType elementVectorType =
201 llvm::dyn_cast<VectorType>(Val: shapedType.getElementType());
202 if (elementVectorType)
203 elementVectorRank += elementVectorType.getRank();
204 return vectorType.getRank() - elementVectorRank;
205}
206
207AffineMap mlir::vector::getTransferMinorIdentityMap(ShapedType shapedType,
208 VectorType vectorType) {
209 // 0-d transfers are to/from tensor<t>/memref<t> and vector<1xt>.
210 // TODO: replace once we have 0-d vectors.
211 if (shapedType.getRank() == 0 &&
212 vectorType.getShape() == ArrayRef<int64_t>{1})
213 return AffineMap::get(
214 /*numDims=*/dimCount: 0, /*numSymbols=*/symbolCount: 0,
215 result: getAffineConstantExpr(constant: 0, context: shapedType.getContext()));
216 return AffineMap::getMinorIdentityMap(
217 dims: shapedType.getRank(),
218 results: getEffectiveVectorRankForXferOp(shapedType, vectorType),
219 context: shapedType.getContext());
220}
221
222/// Check if `write` is of a constant splat and the masked `read` is padded with
223/// the same splat value -- meaning it could be the same value as the initial
224/// constant splat.
225static bool isSplatWriteConsistentWithMaskedRead(vector::TransferWriteOp write,
226 vector::TransferReadOp read) {
227 auto readMask = read.getMask();
228 auto writeMask = write.getMask();
229 // Check if the masks are consistent. The splat value could be the same if the
230 // read is masked (and padded with the splat value), and the write is unmasked
231 // or has the same mask. Note this does not allow the case where the write is
232 // masked and the read is unmasked, as then the read could be of more elements
233 // than the write (which may not be the same value).
234 bool couldBeSameSplat = readMask && (!writeMask || writeMask == readMask);
235 if (!couldBeSameSplat)
236 return false;
237 // Check for constant splat (as the source of the write).
238 DenseElementsAttr splatAttr;
239 if (!matchPattern(value: write.getVector(),
240 pattern: m_Constant<DenseElementsAttr>(bind_value: &splatAttr)) ||
241 !splatAttr.isSplat()) {
242 return false;
243 }
244 // The padding of the read and the constant splat value must be the same.
245 Attribute padAttr;
246 if (!matchPattern(value: read.getPadding(), pattern: m_Constant(bind_value: &padAttr)))
247 return false;
248 return padAttr == splatAttr.getSplatValue<Attribute>();
249}
250
251bool mlir::vector::checkSameValueRAW(vector::TransferWriteOp defWrite,
252 vector::TransferReadOp read) {
253 // An enclosing vector.mask may leave some lanes unwritten or padded, so the
254 // read may not see the written vector as a whole. This conservatively
255 // includes all-true masks. MaskOp::fold removes them during canonicalization,
256 // so this only matters when a caller such as transferOpflowOpt runs first.
257 // TODO: Account for compatible region masks instead of conservatively
258 // rejecting all region-masked operations.
259 if (defWrite.isMasked() || read.isMasked())
260 return false;
261 return !defWrite.hasOutOfBoundsDim() &&
262 defWrite.getIndices() == read.getIndices() &&
263 defWrite.getVectorType() == read.getVectorType() &&
264 defWrite.getPermutationMap() == read.getPermutationMap() &&
265 ((!defWrite.getMask() && !read.getMask()) ||
266 isSplatWriteConsistentWithMaskedRead(write: defWrite, read));
267}
268
269bool mlir::vector::checkSameValueWAW(vector::TransferWriteOp write,
270 vector::TransferWriteOp priorWrite) {
271 // A write under an enclosing vector.mask may not overwrite all lanes of the
272 // prior write. This conservatively includes all-true masks, which
273 // MaskOp::fold removes during canonicalization. Only the later write needs
274 // checking: a region-masked prior write is still dead if the later write
275 // fully overwrites it.
276 // TODO: Handle a region-masked later write when it is known to fully
277 // overwrite the prior write.
278 if (write.isMasked())
279 return false;
280 return priorWrite.getIndices() == write.getIndices() &&
281 priorWrite.getMask() == write.getMask() &&
282 priorWrite.getVectorType() == write.getVectorType() &&
283 priorWrite.getPermutationMap() == write.getPermutationMap();
284}
285
286bool mlir::vector::isDisjointTransferIndices(
287 VectorTransferOpInterface transferA, VectorTransferOpInterface transferB,
288 bool testDynamicValueUsingBounds) {
289 // For simplicity only look at transfer of same type.
290 if (transferA.getVectorType() != transferB.getVectorType())
291 return false;
292 unsigned rankOffset = transferA.getLeadingShapedRank();
293 for (unsigned i = 0, e = transferA.getIndices().size(); i < e; i++) {
294 Value indexA = transferA.getIndices()[i];
295 Value indexB = transferB.getIndices()[i];
296 std::optional<int64_t> cstIndexA = getConstantIntValue(ofr: indexA);
297 std::optional<int64_t> cstIndexB = getConstantIntValue(ofr: indexB);
298
299 if (i < rankOffset) {
300 // For leading dimensions, if we can prove that index are different we
301 // know we are accessing disjoint slices.
302 if (cstIndexA.has_value() && cstIndexB.has_value()) {
303 if (*cstIndexA != *cstIndexB)
304 return true;
305 continue;
306 }
307 if (testDynamicValueUsingBounds) {
308 // First try to see if we can fully compose and simplify the affine
309 // expression as a fast track.
310 FailureOr<uint64_t> delta =
311 affine::fullyComposeAndComputeConstantDelta(value1: indexA, value2: indexB);
312 if (succeeded(Result: delta) && *delta != 0)
313 return true;
314
315 FailureOr<bool> testEqual =
316 ValueBoundsConstraintSet::areEqual(var1: indexA, var2: indexB);
317 if (succeeded(Result: testEqual) && !testEqual.value())
318 return true;
319 }
320 } else {
321 // For this dimension, we slice a part of the memref we need to make sure
322 // the intervals accessed don't overlap.
323 int64_t vectorDim = transferA.getVectorType().getDimSize(idx: i - rankOffset);
324 if (cstIndexA.has_value() && cstIndexB.has_value()) {
325 int64_t distance = std::abs(i: *cstIndexA - *cstIndexB);
326 if (distance >= vectorDim)
327 return true;
328 continue;
329 }
330 if (testDynamicValueUsingBounds) {
331 // First try to see if we can fully compose and simplify the affine
332 // expression as a fast track.
333 FailureOr<int64_t> delta =
334 affine::fullyComposeAndComputeConstantDelta(value1: indexA, value2: indexB);
335 if (succeeded(Result: delta) && std::abs(i: *delta) >= vectorDim)
336 return true;
337
338 FailureOr<int64_t> computeDelta =
339 ValueBoundsConstraintSet::computeConstantDelta(value1: indexA, value2: indexB);
340 if (succeeded(Result: computeDelta)) {
341 if (std::abs(i: computeDelta.value()) >= vectorDim)
342 return true;
343 }
344 }
345 }
346 }
347 return false;
348}
349
350bool mlir::vector::isDisjointTransferSet(VectorTransferOpInterface transferA,
351 VectorTransferOpInterface transferB,
352 bool testDynamicValueUsingBounds) {
353 if (transferA.getBase() != transferB.getBase())
354 return false;
355 return isDisjointTransferIndices(transferA, transferB,
356 testDynamicValueUsingBounds);
357}
358
359// Helper to iterate over n-D vector slice elements. Calculate the next
360// `position` in the n-D vector of size `shape`, applying an offset `offsets`.
361// Modifies the `position` in place. Returns a failure when `position` becomes
362// the end position.
363static LogicalResult incSlicePosition(MutableArrayRef<int64_t> position,
364 ArrayRef<int64_t> shape,
365 ArrayRef<int64_t> offsets) {
366 for (auto [posInDim, dimSize, offsetInDim] :
367 llvm::reverse(C: llvm::zip_equal(t&: position, u&: shape, args&: offsets))) {
368 ++posInDim;
369 if (posInDim < dimSize + offsetInDim)
370 return success();
371
372 // Carry the overflow to the next loop iteration.
373 posInDim = offsetInDim;
374 }
375
376 return failure();
377}
378
379/// Returns the integer numbers in `values`. `values` are expected to be
380/// constant operations.
381SmallVector<int64_t> vector::getAsIntegers(ArrayRef<Value> values) {
382 SmallVector<int64_t> ints;
383 llvm::transform(Range&: values, d_first: std::back_inserter(x&: ints), F: [](Value value) {
384 auto constOp = value.getDefiningOp<arith::ConstantIndexOp>();
385 assert(constOp && "Unexpected non-constant index");
386 return constOp.value();
387 });
388 return ints;
389}
390
391/// Returns the integer numbers in `foldResults`. `foldResults` are expected to
392/// be constant operations.
393SmallVector<int64_t> vector::getAsIntegers(ArrayRef<OpFoldResult> foldResults) {
394 SmallVector<int64_t> ints;
395 llvm::transform(
396 Range&: foldResults, d_first: std::back_inserter(x&: ints), F: [](OpFoldResult foldResult) {
397 assert(isa<Attribute>(foldResult) && "Unexpected non-constant index");
398 return cast<IntegerAttr>(Val: cast<Attribute>(Val&: foldResult)).getInt();
399 });
400 return ints;
401}
402
403/// Convert `foldResults` into Values. Integer attributes are converted to
404/// constant op.
405SmallVector<Value> vector::getAsValues(OpBuilder &builder, Location loc,
406 ArrayRef<OpFoldResult> foldResults) {
407 SmallVector<Value> values;
408 llvm::transform(Range&: foldResults, d_first: std::back_inserter(x&: values),
409 F: [&](OpFoldResult foldResult) {
410 if (auto attr = dyn_cast<Attribute>(Val&: foldResult))
411 return arith::ConstantIndexOp::create(
412 builder, location: loc, value: cast<IntegerAttr>(Val&: attr).getInt())
413 .getResult();
414
415 return cast<Value>(Val&: foldResult);
416 });
417 return values;
418}
419
420std::optional<int64_t> vector::getConstantVscaleMultiplier(Value value) {
421 if (value.getDefiningOp<vector::VectorScaleOp>())
422 return 1;
423 auto mul = value.getDefiningOp<arith::MulIOp>();
424 if (!mul)
425 return {};
426 auto lhs = mul.getLhs();
427 auto rhs = mul.getRhs();
428 if (lhs.getDefiningOp<vector::VectorScaleOp>())
429 return getConstantIntValue(ofr: rhs);
430 if (rhs.getDefiningOp<vector::VectorScaleOp>())
431 return getConstantIntValue(ofr: lhs);
432 return {};
433}
434
435/// Converts numeric attributes to the expected type. Supports
436/// integer-to-integer and float-to-integer conversions. Returns the original
437/// attribute if no conversion is needed or supported.
438static Attribute convertNumericAttr(Attribute attr, Type expectedType) {
439 // Integer-to-integer conversion
440 if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) {
441 if (auto intType = dyn_cast<IntegerType>(Val&: expectedType)) {
442 if (intAttr.getType() != expectedType)
443 return IntegerAttr::get(type: expectedType, value: intAttr.getInt());
444 }
445 return attr;
446 }
447
448 // Float-to-integer bitcast (preserves bit representation)
449 if (auto floatAttr = dyn_cast<FloatAttr>(Val&: attr)) {
450 auto intType = dyn_cast<IntegerType>(Val&: expectedType);
451 if (!intType)
452 return attr;
453
454 APFloat floatVal = floatAttr.getValue();
455 APInt intVal = floatVal.bitcastToAPInt();
456 return IntegerAttr::get(type: expectedType, value: intVal);
457 }
458
459 return attr;
460}
461
462/// Return whether `srcType` can be broadcast to `dstVectorType` under the
463/// semantics of the `vector.broadcast` op.
464BroadcastableToResult mlir::vector::isBroadcastableTo(
465 Type srcType, VectorType dstVectorType,
466 std::pair<VectorDim, VectorDim> *mismatchingDims) {
467 // Broadcast scalar to vector of the same element type.
468 if (isa<VectorElementTypeInterface>(Val: srcType) && dstVectorType &&
469 srcType == getElementTypeOrSelf(type: dstVectorType))
470 return BroadcastableToResult::Success;
471 // From now on, only vectors broadcast.
472 VectorType srcVectorType = llvm::dyn_cast<VectorType>(Val&: srcType);
473 if (!srcVectorType)
474 return BroadcastableToResult::SourceTypeNotAVector;
475
476 int64_t srcRank = srcVectorType.getRank();
477 int64_t dstRank = dstVectorType.getRank();
478 if (srcRank > dstRank)
479 return BroadcastableToResult::SourceRankHigher;
480 // Source has an exact match or singleton value for all trailing dimensions
481 // (all leading dimensions are simply duplicated).
482 int64_t lead = dstRank - srcRank;
483 for (int64_t dimIdx = 0; dimIdx < srcRank; ++dimIdx) {
484 // Have mismatching dims (in the sense of vector.broadcast semantics) been
485 // encountered?
486 bool foundMismatchingDims = false;
487
488 // Check fixed-width dims.
489 int64_t srcDim = srcVectorType.getDimSize(idx: dimIdx);
490 int64_t dstDim = dstVectorType.getDimSize(idx: lead + dimIdx);
491 if (srcDim != 1 && srcDim != dstDim)
492 foundMismatchingDims = true;
493
494 // Check scalable flags.
495 bool srcDimScalableFlag = srcVectorType.getScalableDims()[dimIdx];
496 bool dstDimScalableFlag = dstVectorType.getScalableDims()[lead + dimIdx];
497 if ((srcDim == 1 && srcDimScalableFlag && dstDim != 1) ||
498 // 1 -> [N] is fine, everything else should be rejected when mixing
499 // fixed-width and scalable dims
500 (srcDimScalableFlag != dstDimScalableFlag &&
501 (srcDim != 1 || srcDimScalableFlag)))
502 foundMismatchingDims = true;
503
504 if (foundMismatchingDims) {
505 if (mismatchingDims != nullptr) {
506 mismatchingDims->first.dim = srcDim;
507 mismatchingDims->first.isScalable = srcDimScalableFlag;
508
509 mismatchingDims->second.dim = dstDim;
510 mismatchingDims->second.isScalable = dstDimScalableFlag;
511 }
512 return BroadcastableToResult::DimensionMismatch;
513 }
514 }
515
516 return BroadcastableToResult::Success;
517}
518
519//===----------------------------------------------------------------------===//
520// CombiningKindAttr
521//===----------------------------------------------------------------------===//
522
523namespace mlir {
524namespace vector {
525namespace detail {
526struct BitmaskEnumStorage : public AttributeStorage {
527 using KeyTy = uint64_t;
528
529 BitmaskEnumStorage(KeyTy val) : value(val) {}
530
531 bool operator==(const KeyTy &key) const { return value == key; }
532
533 static BitmaskEnumStorage *construct(AttributeStorageAllocator &allocator,
534 const KeyTy &key) {
535 return new (allocator.allocate<BitmaskEnumStorage>())
536 BitmaskEnumStorage(key);
537 }
538
539 KeyTy value = 0;
540};
541} // namespace detail
542} // namespace vector
543} // namespace mlir
544
545//===----------------------------------------------------------------------===//
546// VectorDialect
547//===----------------------------------------------------------------------===//
548
549namespace {
550/// This class defines the interface for handling inlining with vector dialect
551/// operations.
552struct VectorInlinerInterface : public DialectInlinerInterface {
553 using DialectInlinerInterface::DialectInlinerInterface;
554
555 /// All vector dialect ops can be inlined.
556 bool isLegalToInline(Operation *, Region *, bool, IRMapping &) const final {
557 return true;
558 }
559};
560} // namespace
561
562void VectorDialect::initialize() {
563 addAttributes<
564#define GET_ATTRDEF_LIST
565#include "mlir/Dialect/Vector/IR/VectorAttributes.cpp.inc"
566 >();
567
568 addOperations<
569#define GET_OP_LIST
570#include "mlir/Dialect/Vector/IR/VectorOps.cpp.inc"
571 >();
572
573 addInterfaces<VectorInlinerInterface>();
574
575 declarePromisedInterfaces<memref::IndexedAccessOpInterface, LoadOp, StoreOp,
576 MaskedLoadOp, MaskedStoreOp, ExpandLoadOp,
577 CompressStoreOp>();
578 declarePromisedInterfaces<bufferization::BufferizableOpInterface,
579 TransferReadOp, TransferWriteOp, GatherOp, MaskOp,
580 YieldOp>();
581 declarePromisedInterfaces<SubsetOpInterface, TransferReadOp,
582 TransferWriteOp>();
583 declarePromisedInterface<SubsetExtractionOpInterface, TransferReadOp>();
584 declarePromisedInterface<SubsetInsertionOpInterface, TransferWriteOp>();
585 declarePromisedInterface<ConvertToLLVMPatternInterface, VectorDialect>();
586}
587
588/// Materialize a single constant operation from a given attribute value with
589/// the desired resultant type.
590Operation *VectorDialect::materializeConstant(OpBuilder &builder,
591 Attribute value, Type type,
592 Location loc) {
593 if (matchPattern(attr: value, pattern: ub::m_Poison()))
594 return value.getDialect().materializeConstant(builder, value, type, loc);
595
596 return arith::ConstantOp::materialize(builder, value, type, loc);
597}
598
599IntegerType vector::getVectorSubscriptType(Builder &builder) {
600 return builder.getIntegerType(width: 64);
601}
602
603ArrayAttr vector::getVectorSubscriptAttr(Builder &builder,
604 ArrayRef<int64_t> values) {
605 return builder.getI64ArrayAttr(values);
606}
607
608//===----------------------------------------------------------------------===//
609// MultiDimReductionOp
610//===----------------------------------------------------------------------===//
611
612void vector::MultiDimReductionOp::build(OpBuilder &builder,
613 OperationState &result, Value source,
614 Value acc, ArrayRef<bool> reductionMask,
615 CombiningKind kind,
616 arith::FastMathFlags fastMathFlags) {
617 SmallVector<int64_t> reductionDims;
618 for (const auto &en : llvm::enumerate(First&: reductionMask))
619 if (en.value())
620 reductionDims.push_back(Elt: en.index());
621 build(odsBuilder&: builder, odsState&: result, kind, source, acc, reduction_dims: reductionDims, fastmath: fastMathFlags);
622}
623
624OpFoldResult MultiDimReductionOp::fold(FoldAdaptor adaptor) {
625 // No reduction dims: this is a noop regardless of rank.
626 if (getReductionDims().empty())
627 return getSource();
628 return {};
629}
630
631LogicalResult MultiDimReductionOp::verify() {
632 // Verify the reduction dimensions.
633 int64_t sourceRank = getSourceVectorType().getRank();
634 SmallVector<bool> isReduced(sourceRank, false);
635 for (int64_t dim : getReductionDims()) {
636 if (dim < 0 || dim >= sourceRank)
637 return emitOpError(message: "reduction dimension out of range: ") << dim;
638 if (isReduced[dim])
639 return emitOpError(message: "duplicate reduction dimension: ") << dim;
640 isReduced[dim] = true;
641 }
642
643 SmallVector<int64_t> targetShape;
644 SmallVector<bool> scalableDims;
645 Type inferredReturnType;
646 auto sourceScalableDims = getSourceVectorType().getScalableDims();
647 for (auto [dimIdx, dimSize] :
648 llvm::enumerate(First: getSourceVectorType().getShape()))
649 if (!isReduced[dimIdx]) {
650 targetShape.push_back(Elt: dimSize);
651 scalableDims.push_back(Elt: sourceScalableDims[dimIdx]);
652 }
653 // TODO: update to also allow 0-d vectors when available.
654 if (targetShape.empty())
655 inferredReturnType = getSourceVectorType().getElementType();
656 else
657 inferredReturnType = VectorType::get(
658 shape: targetShape, elementType: getSourceVectorType().getElementType(), scalableDims);
659 if (getType() != inferredReturnType)
660 return emitOpError() << "destination type " << getType()
661 << " is incompatible with source type "
662 << getSourceVectorType();
663
664 return success();
665}
666
667/// Returns the mask type expected by this operation.
668Type MultiDimReductionOp::getExpectedMaskType() {
669 auto vecType = getSourceVectorType();
670 return VectorType::get(shape: vecType.getShape(),
671 elementType: IntegerType::get(context: vecType.getContext(), /*width=*/1),
672 scalableDims: vecType.getScalableDims());
673}
674
675//-----------------------------------------------------------------------------
676// VectorUnrollOpInterface methods
677//-----------------------------------------------------------------------------
678std::optional<SmallVector<int64_t, 4>>
679MultiDimReductionOp::getIterationShapeToUnroll() {
680 return llvm::to_vector<4>(Range: getSourceVectorType().getShape());
681}
682
683namespace {
684// Only unit dimensions that are being reduced are folded. If the dimension is
685// unit, but not reduced, it is not folded, thereby keeping the output type the
686// same. If not all dimensions which are reduced are of unit dimension, this
687// transformation does nothing. This is just a generalization of
688// ElideSingleElementReduction for ReduceOp.
689struct ElideUnitDimsInMultiDimReduction
690 : public OpRewritePattern<MultiDimReductionOp> {
691 using Base::Base;
692
693 LogicalResult matchAndRewrite(MultiDimReductionOp reductionOp,
694 PatternRewriter &rewriter) const override {
695 ArrayRef<int64_t> shape = reductionOp.getSourceVectorType().getShape();
696 for (const auto &dim : enumerate(First&: shape)) {
697 if (reductionOp.isReducedDim(d: dim.index()) && dim.value() != 1)
698 return failure();
699 }
700
701 // Vector mask setup.
702 OpBuilder::InsertionGuard guard(rewriter);
703 Operation *rootOp;
704 Value mask;
705 if (reductionOp.isMasked()) {
706 rewriter.setInsertionPoint(reductionOp.getMaskingOp());
707 rootOp = reductionOp.getMaskingOp();
708 mask = reductionOp.getMaskingOp().getMask();
709 } else {
710 rootOp = reductionOp;
711 }
712
713 Location loc = reductionOp.getLoc();
714 Value acc = reductionOp.getAcc();
715 Value cast;
716 if (auto dstVecType = dyn_cast<VectorType>(Val: reductionOp.getDestType())) {
717 if (mask) {
718 VectorType newMaskType =
719 VectorType::get(shape: dstVecType.getShape(), elementType: rewriter.getI1Type(),
720 scalableDims: dstVecType.getScalableDims());
721 mask = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: newMaskType, source: mask);
722 }
723 cast = vector::ShapeCastOp::create(
724 builder&: rewriter, location: loc, result: reductionOp.getDestType(), source: reductionOp.getSource());
725 } else {
726 // This means we are reducing all the dimensions, and all reduction
727 // dimensions are of size 1. So a simple extraction would do.
728 if (mask)
729 mask = vector::ExtractOp::create(builder&: rewriter, location: loc, source: mask);
730 cast = vector::ExtractOp::create(builder&: rewriter, location: loc, source: reductionOp.getSource());
731 }
732
733 Value result =
734 vector::makeArithReduction(b&: rewriter, loc, kind: reductionOp.getKind(), v1: cast,
735 acc, fastmath: reductionOp.getFastmathAttr(), mask);
736 rewriter.replaceOp(op: rootOp, newValues: result);
737 return success();
738 }
739};
740} // namespace
741
742void MultiDimReductionOp::getCanonicalizationPatterns(
743 RewritePatternSet &results, MLIRContext *context) {
744 results.add<ElideUnitDimsInMultiDimReduction>(arg&: context);
745}
746
747//===----------------------------------------------------------------------===//
748// ReductionOp
749//===----------------------------------------------------------------------===//
750
751void vector::ReductionOp::build(OpBuilder &builder, OperationState &result,
752 CombiningKind kind, Value vector,
753 arith::FastMathFlags fastMathFlags) {
754 build(odsBuilder&: builder, odsState&: result, kind, vector, /*acc=*/Value(), fastMathFlags);
755}
756
757void vector::ReductionOp::build(OpBuilder &builder, OperationState &result,
758 CombiningKind kind, Value vector, Value acc,
759 arith::FastMathFlags fastMathFlags) {
760 build(odsBuilder&: builder, odsState&: result,
761 result: llvm::cast<VectorType>(Val: vector.getType()).getElementType(), kind, vector,
762 acc, fastmath: fastMathFlags);
763}
764
765LogicalResult ReductionOp::verify() {
766 // Verify for 0-D and 1-D vector.
767 int64_t rank = getSourceVectorType().getRank();
768 if (rank > 1)
769 return emitOpError(message: "unsupported reduction rank: ") << rank;
770
771 // Verify supported reduction kind.
772 Type eltType = getResult().getType();
773 if (!isSupportedCombiningKind(combiningKind: getKind(), elementType: eltType))
774 return emitOpError(message: "unsupported reduction type '")
775 << eltType << "' for kind '" << stringifyCombiningKind(val: getKind())
776 << "'";
777
778 return success();
779}
780
781// MaskableOpInterface methods.
782
783/// Returns the mask type expected by this operation.
784Type ReductionOp::getExpectedMaskType() {
785 auto vecType = getSourceVectorType();
786 return VectorType::get(shape: vecType.getShape(),
787 elementType: IntegerType::get(context: vecType.getContext(), /*width=*/1),
788 scalableDims: vecType.getScalableDims());
789}
790
791Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op,
792 OpBuilder &builder, Location loc,
793 Value vector) {
794 switch (op) {
795 case arith::AtomicRMWKind::addf:
796 case arith::AtomicRMWKind::addi:
797 return vector::ReductionOp::create(builder, location: vector.getLoc(),
798 kind: CombiningKind::ADD, vector);
799 case arith::AtomicRMWKind::mulf:
800 case arith::AtomicRMWKind::muli:
801 return vector::ReductionOp::create(builder, location: vector.getLoc(),
802 kind: CombiningKind::MUL, vector);
803 case arith::AtomicRMWKind::minimumf:
804 return vector::ReductionOp::create(builder, location: vector.getLoc(),
805 kind: CombiningKind::MINIMUMF, vector);
806 case arith::AtomicRMWKind::mins:
807 return vector::ReductionOp::create(builder, location: vector.getLoc(),
808 kind: CombiningKind::MINSI, vector);
809 case arith::AtomicRMWKind::minu:
810 return vector::ReductionOp::create(builder, location: vector.getLoc(),
811 kind: CombiningKind::MINUI, vector);
812 case arith::AtomicRMWKind::maximumf:
813 return vector::ReductionOp::create(builder, location: vector.getLoc(),
814 kind: CombiningKind::MAXIMUMF, vector);
815 case arith::AtomicRMWKind::maxs:
816 return vector::ReductionOp::create(builder, location: vector.getLoc(),
817 kind: CombiningKind::MAXSI, vector);
818 case arith::AtomicRMWKind::maxu:
819 return vector::ReductionOp::create(builder, location: vector.getLoc(),
820 kind: CombiningKind::MAXUI, vector);
821 case arith::AtomicRMWKind::andi:
822 return vector::ReductionOp::create(builder, location: vector.getLoc(),
823 kind: CombiningKind::AND, vector);
824 case arith::AtomicRMWKind::ori:
825 return vector::ReductionOp::create(builder, location: vector.getLoc(),
826 kind: CombiningKind::OR, vector);
827 case arith::AtomicRMWKind::minnumf:
828 return vector::ReductionOp::create(builder, location: vector.getLoc(),
829 kind: CombiningKind::MINNUMF, vector);
830 case arith::AtomicRMWKind::maxnumf:
831 return vector::ReductionOp::create(builder, location: vector.getLoc(),
832 kind: CombiningKind::MAXNUMF, vector);
833 case arith::AtomicRMWKind::xori:
834 return vector::ReductionOp::create(builder, location: vector.getLoc(),
835 kind: CombiningKind::XOR, vector);
836 default:
837 (void)emitOptionalError(loc, args: "Reduction operation type not supported");
838 break;
839 }
840 return nullptr;
841}
842
843//-----------------------------------------------------------------------------
844// VectorUnrollOpInterface methods
845//-----------------------------------------------------------------------------
846std::optional<SmallVector<int64_t, 4>>
847ReductionOp::getIterationShapeToUnroll() {
848 return llvm::to_vector<4>(Range: getSourceVectorType().getShape());
849}
850std::optional<SmallVector<int64_t>> ReductionOp::getDefaultUnrollTargetShape() {
851 return SmallVector<int64_t>{2};
852}
853
854namespace {
855struct ElideSingleElementReduction : public OpRewritePattern<ReductionOp> {
856 using Base::Base;
857
858 LogicalResult matchAndRewrite(ReductionOp reductionOp,
859 PatternRewriter &rewriter) const override {
860 // Vector mask setup.
861 OpBuilder::InsertionGuard guard(rewriter);
862 auto maskableOp =
863 cast<vector::MaskableOpInterface>(Val: reductionOp.getOperation());
864 Operation *rootOp;
865 Value mask;
866 if (maskableOp.isMasked()) {
867 rewriter.setInsertionPoint(maskableOp.getMaskingOp());
868 rootOp = maskableOp.getMaskingOp();
869 mask = maskableOp.getMaskingOp().getMask();
870 } else {
871 rootOp = reductionOp;
872 }
873
874 auto vectorType = reductionOp.getSourceVectorType();
875 if (vectorType.getRank() != 0 && vectorType.getDimSize(idx: 0) != 1)
876 return failure();
877
878 Location loc = reductionOp.getLoc();
879 if (mask)
880 mask = ExtractOp::create(builder&: rewriter, location: loc, source: mask);
881 Value result = ExtractOp::create(builder&: rewriter, location: loc, source: reductionOp.getVector());
882
883 if (Value acc = reductionOp.getAcc())
884 result = vector::makeArithReduction(b&: rewriter, loc, kind: reductionOp.getKind(),
885 v1: result, acc,
886 fastmath: reductionOp.getFastmathAttr(), mask);
887
888 rewriter.replaceOp(op: rootOp, newValues: result);
889 return success();
890 }
891};
892} // namespace
893
894void ReductionOp::getCanonicalizationPatterns(RewritePatternSet &results,
895 MLIRContext *context) {
896 results.add<ElideSingleElementReduction>(arg&: context);
897}
898
899//===----------------------------------------------------------------------===//
900// ContractionOp
901//===----------------------------------------------------------------------===//
902
903void vector::ContractionOp::build(OpBuilder &builder, OperationState &result,
904 Value lhs, Value rhs, Value acc,
905 ArrayRef<ArrayRef<AffineExpr>> indexingExprs,
906 ArrayRef<IteratorType> iteratorTypes) {
907 result.addOperands(newOperands: {lhs, rhs, acc});
908 result.addTypes(newTypes: acc.getType());
909 result.addAttribute(
910 name: getIndexingMapsAttrName(name: result.name),
911 attr: builder.getAffineMapArrayAttr(
912 values: AffineMap::inferFromExprList(exprsList: indexingExprs, context: builder.getContext())));
913 result.addAttribute(
914 name: getIteratorTypesAttrName(name: result.name),
915 attr: builder.getArrayAttr(value: llvm::map_to_vector(
916 C&: iteratorTypes, F: [&](IteratorType t) -> mlir::Attribute {
917 return IteratorTypeAttr::get(context: builder.getContext(), value: t);
918 })));
919}
920
921void vector::ContractionOp::build(OpBuilder &builder, OperationState &result,
922 Value lhs, Value rhs, Value acc,
923 ArrayAttr indexingMaps,
924 ArrayAttr iteratorTypes) {
925 build(odsBuilder&: builder, odsState&: result, lhs, rhs, acc, indexingMaps, iteratorTypes,
926 kind: ContractionOp::getDefaultKind());
927}
928
929void vector::ContractionOp::build(OpBuilder &builder, OperationState &result,
930 Value lhs, Value rhs, Value acc,
931 ArrayAttr indexingMaps,
932 ArrayAttr iteratorTypes, CombiningKind kind,
933 arith::FastMathFlags fastMathFlags) {
934 result.addOperands(newOperands: {lhs, rhs, acc});
935 result.addTypes(newTypes: acc.getType());
936 result.addAttribute(name: getIndexingMapsAttrName(name: result.name), attr: indexingMaps);
937 result.addAttribute(name: getIteratorTypesAttrName(name: result.name), attr: iteratorTypes);
938 result.addAttribute(name: getKindAttrName(name: result.name),
939 attr: CombiningKindAttr::get(context: builder.getContext(), value: kind));
940 if (fastMathFlags != arith::FastMathFlags::none)
941 result.addAttribute(
942 name: getFastmathAttrName(name: result.name),
943 attr: arith::FastMathFlagsAttr::get(context: builder.getContext(), value: fastMathFlags));
944}
945
946ParseResult ContractionOp::parse(OpAsmParser &parser, OperationState &result) {
947 OpAsmParser::UnresolvedOperand lhsInfo;
948 OpAsmParser::UnresolvedOperand rhsInfo;
949 OpAsmParser::UnresolvedOperand accInfo;
950 SmallVector<OpAsmParser::UnresolvedOperand, 2> masksInfo;
951 SmallVector<Type, 2> types;
952 Type resultType;
953 auto loc = parser.getCurrentLocation();
954 DictionaryAttr dictAttr;
955 // TODO: Unify linalg op attribute parsing.
956 if (parser.parseAttribute(result&: dictAttr) || parser.parseOperand(result&: lhsInfo) ||
957 parser.parseComma() || parser.parseOperand(result&: rhsInfo) ||
958 parser.parseComma() || parser.parseOperand(result&: accInfo) ||
959 parser.parseTrailingOperandList(result&: masksInfo) ||
960 parser.parseOptionalAttrDict(result&: result.attributes) ||
961 parser.parseColonTypeList(result&: types) ||
962 parser.parseKeywordType(keyword: "into", result&: resultType) ||
963 parser.resolveOperand(operand: lhsInfo, type: types[0], result&: result.operands) ||
964 parser.resolveOperand(operand: rhsInfo, type: types[1], result&: result.operands) ||
965 parser.resolveOperand(operand: accInfo, type: resultType, result&: result.operands) ||
966 parser.addTypeToList(type: resultType, result&: result.types))
967 return failure();
968 result.attributes.append(inStart: dictAttr.getValue().begin(),
969 inEnd: dictAttr.getValue().end());
970
971 // Convert array of string into an array of IteratyType enums. This is needed,
972 // because tests still use the old format when 'iterator_types' attribute is
973 // represented as an array of strings.
974 // TODO: Remove this conversion once tests are fixed.
975 auto iteratorTypes = dyn_cast_or_null<ArrayAttr>(
976 Val: result.attributes.get(name: getIteratorTypesAttrName(name: result.name)));
977 if (!iteratorTypes) {
978 return parser.emitError(loc)
979 << "expected " << getIteratorTypesAttrName(name: result.name)
980 << " array attribute";
981 }
982
983 SmallVector<Attribute> iteratorTypeAttrs;
984
985 for (StringRef s : iteratorTypes.getAsValueRange<StringAttr>()) {
986 auto maybeIteratorType = symbolizeIteratorType(str: s);
987 if (!maybeIteratorType.has_value())
988 return parser.emitError(loc) << "unexpected iterator_type (" << s << ")";
989
990 iteratorTypeAttrs.push_back(
991 Elt: IteratorTypeAttr::get(context: parser.getContext(), value: maybeIteratorType.value()));
992 }
993 result.attributes.set(name: getIteratorTypesAttrName(name: result.name),
994 value: parser.getBuilder().getArrayAttr(value: iteratorTypeAttrs));
995
996 if (!result.attributes.get(name: getKindAttrName(name: result.name))) {
997 result.addAttribute(
998 name: getKindAttrName(name: result.name),
999 attr: CombiningKindAttr::get(context: result.getContext(),
1000 value: ContractionOp::getDefaultKind()));
1001 }
1002 if (masksInfo.empty())
1003 return success();
1004 if (masksInfo.size() != 2)
1005 return parser.emitError(loc: parser.getNameLoc(),
1006 message: "expected zero or exactly 2 vector mask operands");
1007 auto lhsType = llvm::cast<VectorType>(Val&: types[0]);
1008 auto rhsType = llvm::cast<VectorType>(Val&: types[1]);
1009 auto maskElementType = parser.getBuilder().getI1Type();
1010 std::array<VectorType, 2> maskTypes = {
1011 VectorType::Builder(lhsType).setElementType(maskElementType),
1012 VectorType::Builder(rhsType).setElementType(maskElementType)};
1013 if (parser.resolveOperands(operands&: masksInfo, types&: maskTypes, loc, result&: result.operands))
1014 return failure();
1015 return success();
1016}
1017
1018void ContractionOp::print(OpAsmPrinter &p) {
1019 // TODO: Unify printing code with linalg ops.
1020 auto attrNames = getTraitAttrNames();
1021 llvm::StringSet<> traitAttrsSet;
1022 traitAttrsSet.insert_range(R&: attrNames);
1023 NamedAttrList allAttrs(getOperation()->getRawDictionaryAttrs());
1024 getOperation()->getName().walkInherentAttrs(
1025 op: getOperation(),
1026 visitor: [&](StringRef name, Attribute &attr) { allAttrs.append(name, attr); });
1027 SmallVector<NamedAttribute, 8> attrs;
1028 for (auto attr : allAttrs) {
1029 if (attr.getName() == getIteratorTypesAttrName()) {
1030 auto iteratorTypes =
1031 llvm::cast<ArrayAttr>(Val: attr.getValue())
1032 .getAsValueRange<IteratorTypeAttr, IteratorType>();
1033 // Convert IteratorType enums into the string representation. This is
1034 // needed, because tests still use the old format when 'iterator_types'
1035 // attribute is represented as an array of strings.
1036 // TODO: Remove this conversion once tests are fixed.
1037 SmallVector<Attribute> iteratorTypeNames =
1038 llvm::map_to_vector(C&: iteratorTypes, F: [&](IteratorType t) -> Attribute {
1039 return StringAttr::get(context: getContext(), bytes: stringifyIteratorType(val: t));
1040 });
1041
1042 attrs.emplace_back(Args: getIteratorTypesAttrName(),
1043 Args: ArrayAttr::get(context: getContext(), value: iteratorTypeNames));
1044 } else if (traitAttrsSet.count(Key: attr.getName().strref()) > 0) {
1045 // Omit fastmath when it equals the default (none) to keep output clean.
1046 if (attr.getName() == getFastmathAttrName() &&
1047 llvm::cast<arith::FastMathFlagsAttr>(Val: attr.getValue()).getValue() ==
1048 arith::FastMathFlags::none)
1049 continue;
1050 attrs.push_back(Elt: attr);
1051 }
1052 }
1053
1054 auto dictAttr = DictionaryAttr::get(context: getContext(), value: attrs);
1055 p << " " << dictAttr << " " << getLhs() << ", ";
1056 p << getRhs() << ", " << getAcc();
1057
1058 p.printOptionalAttrDict(attrs: allAttrs.getAttrs(), elidedAttrs: attrNames);
1059 p << " : " << getLhs().getType() << ", " << getRhs().getType() << " into "
1060 << getResultType();
1061}
1062
1063static bool verifyDimMap(VectorType lhsType, VectorType rhsType,
1064 const std::vector<std::pair<int64_t, int64_t>> &map) {
1065 for (auto &dimPair : map) {
1066 if (dimPair.first < 0 || dimPair.first >= lhsType.getRank() ||
1067 dimPair.second < 0 || dimPair.second >= rhsType.getRank() ||
1068 lhsType.getDimSize(idx: dimPair.first) != rhsType.getDimSize(idx: dimPair.second))
1069 return false;
1070 }
1071 return true;
1072}
1073
1074static LogicalResult verifyOutputShape(
1075 ContractionOp op, VectorType lhsType, VectorType rhsType, Type accType,
1076 Type resType,
1077 const std::vector<std::pair<int64_t, int64_t>> &contractingDimMap,
1078 const std::vector<std::pair<int64_t, int64_t>> &batchDimMap) {
1079 DenseSet<int64_t> lhsContractingDimSet;
1080 DenseSet<int64_t> rhsContractingDimSet;
1081 for (auto &dimPair : contractingDimMap) {
1082 lhsContractingDimSet.insert(V: dimPair.first);
1083 rhsContractingDimSet.insert(V: dimPair.second);
1084 }
1085 DenseSet<int64_t> rhsBatchDimSet(llvm::from_range,
1086 llvm::make_second_range(c: batchDimMap));
1087
1088 // Add free and batch dimensions from 'lhsType' to 'expectedResultDims'.
1089 SmallVector<int64_t, 4> expectedResultDims;
1090 for (int64_t i = 0, e = lhsType.getRank(); i < e; ++i) {
1091 if (lhsContractingDimSet.count(V: i) > 0)
1092 continue;
1093 expectedResultDims.push_back(Elt: lhsType.getDimSize(idx: i));
1094 }
1095
1096 // Add free dimensions from 'rhsType' to 'expectedResultDims'.
1097 for (int64_t i = 0, e = rhsType.getRank(); i < e; ++i) {
1098 if (rhsContractingDimSet.count(V: i) > 0 || rhsBatchDimSet.count(V: i) > 0)
1099 continue;
1100 expectedResultDims.push_back(Elt: rhsType.getDimSize(idx: i));
1101 }
1102
1103 // The accumulator and the result must agree, regardless of the rank.
1104 if (resType != accType)
1105 return op.emitOpError(message: "expected same accumulator and result type");
1106
1107 // Verify 'expectedResultDims'.
1108 if (expectedResultDims.empty()) {
1109 // No batch or free dimension implies a scalar or 0-D vector result.
1110 if (auto resVectorType = llvm::dyn_cast<VectorType>(Val&: resType))
1111 if (resVectorType.getRank() != 0)
1112 return op.emitOpError(message: "expected scalar or 0-D vector result when "
1113 "there are no batch or free dimensions");
1114 } else {
1115 // At least one batch or free dimension implies a vector result.
1116 auto resVectorType = llvm::dyn_cast<VectorType>(Val&: resType);
1117 auto accVectorType = llvm::dyn_cast<VectorType>(Val&: accType);
1118 if (!resVectorType || !accVectorType)
1119 return op.emitOpError(message: "invalid accumulator/result vector shape");
1120
1121 // Infer expected result vector type. Lhs + rhs map and lhs + rhs vector
1122 // types fully define the result vector type. This assumes the affine maps
1123 // are well-formed, which must have been verified already.
1124 MLIRContext *ctx = op.getContext();
1125 AffineMap lhsMap = op.getIndexingMapsArray()[0];
1126 AffineMap rhsMap = op.getIndexingMapsArray()[1];
1127 if (getUnusedDimsBitVector(maps: {lhsMap, rhsMap}).any())
1128 return op.emitOpError(
1129 message: "expected all dimensions to be either a LHS or a RHS dimension");
1130 SmallVector<AffineExpr, 4> extents(lhsMap.getNumInputs());
1131 for (auto pair :
1132 {std::make_pair(x&: lhsType, y&: lhsMap), std::make_pair(x&: rhsType, y&: rhsMap)}) {
1133 VectorType v = pair.first;
1134 auto map = pair.second;
1135 for (unsigned idx = 0, e = v.getRank(); idx < e; ++idx) {
1136 unsigned pos = map.getDimPosition(idx);
1137 if (!extents[pos])
1138 extents[pos] = getAffineConstantExpr(constant: v.getShape()[idx], context: ctx);
1139 }
1140 }
1141 if (!llvm::all_of(Range&: extents, P: [](AffineExpr e) { return e; }))
1142 return op.emitOpError(message: "expected all dimensions to get an extent as "
1143 "either a LHS or a RHS dimension");
1144
1145 AffineMap resMap = op.getIndexingMapsArray()[2];
1146 auto extentsMap = AffineMap::get(/*dimCount=*/extents.size(),
1147 /*symbolCount=*/0, results: extents, context: ctx);
1148 // Compose the resMap with the extentsMap, which is a constant map.
1149 AffineMap expectedMap = simplifyAffineMap(map: resMap.compose(map: extentsMap));
1150 assert(llvm::all_of(expectedMap.getResults(),
1151 llvm::IsaPred<AffineConstantExpr>) &&
1152 "expected constant extent along all dimensions.");
1153 // Extract the expected shape and build the type.
1154 auto expectedShape =
1155 llvm::map_to_vector<4>(C: expectedMap.getResults(), F: [](AffineExpr e) {
1156 return cast<AffineConstantExpr>(Val&: e).getValue();
1157 });
1158 auto expected =
1159 VectorType::get(shape: expectedShape, elementType: resVectorType.getElementType(),
1160 scalableDims: resVectorType.getScalableDims());
1161 if (resVectorType != expected || accVectorType != expected)
1162 return op.emitOpError(
1163 message: "invalid accumulator/result vector shape, expected: ")
1164 << expected;
1165 }
1166 return success();
1167}
1168
1169// Find a one-sided reduction dim, i.e. a reduction dim that is present in only
1170// one of the inputs.
1171static std::optional<int64_t>
1172getOneSidedReductionDim(vector::ContractionOp *contract) {
1173 auto maps = contract->getIndexingMapsArray();
1174 AffineMap lhsMap = maps[0];
1175 AffineMap rhsMap = maps[1];
1176
1177 for (auto [dim, attr] : llvm::enumerate(First: contract->getIteratorTypes())) {
1178 auto iteratorType = cast<vector::IteratorTypeAttr>(Val: attr).getValue();
1179 if (iteratorType == vector::IteratorType::reduction &&
1180 (!lhsMap.isFunctionOfDim(position: dim) || !rhsMap.isFunctionOfDim(position: dim)))
1181 return dim;
1182 }
1183
1184 return std::nullopt;
1185}
1186
1187LogicalResult ContractionOp::verify() {
1188 VectorType lhsType = getLhsType();
1189 VectorType rhsType = getRhsType();
1190 Type accType = getAccType();
1191 Type resType = getResultType();
1192
1193 if (llvm::isa<IntegerType>(Val: lhsType.getElementType())) {
1194 if (!lhsType.getElementType().isSignlessInteger())
1195 return emitOpError(message: "only supports signless integer types");
1196 }
1197
1198 // Verify that an indexing map was specified for each vector operand.
1199 if (getIndexingMapsArray().size() != 3)
1200 return emitOpError(message: "expected an indexing map for each vector operand");
1201
1202 // Verify that each index map has 'numIterators' inputs, no symbols, and
1203 // that the number of map outputs equals the rank of its associated
1204 // vector operand.
1205 unsigned numIterators = getIteratorTypes().getValue().size();
1206 for (const auto &it : llvm::enumerate(First: getIndexingMapsArray())) {
1207 auto index = it.index();
1208 auto map = it.value();
1209 if (map.getNumSymbols() != 0)
1210 return emitOpError(message: "expected indexing map ")
1211 << index << " to have no symbols";
1212 auto vectorType = llvm::dyn_cast<VectorType>(Val: getOperand(i: index).getType());
1213 unsigned rank = vectorType ? vectorType.getShape().size() : 0;
1214 // Verify that the map has the right number of inputs, outputs, and indices.
1215 // This also correctly accounts for (..) -> () for rank-0 results.
1216 if (map.getNumDims() != numIterators)
1217 return emitOpError(message: "expected indexing map ")
1218 << index << " to have " << numIterators << " number of inputs";
1219 if (map.getNumResults() != rank)
1220 return emitOpError(message: "expected indexing map ")
1221 << index << " to have " << rank << " number of outputs";
1222 if (!map.isProjectedPermutation())
1223 return emitOpError(message: "expected indexing map ")
1224 << index << " to be a projected permutation of its inputs";
1225 }
1226
1227 // TODO: For better code re-use, unify this with linalg.contract.
1228 auto dimIdx = getOneSidedReductionDim(contract: this);
1229 if (dimIdx.has_value())
1230 return emitOpError(message: "reduction dim at index ")
1231 << dimIdx.value() << " must appear in both LHS and RHS";
1232
1233 auto contractingDimMap = getContractingDimMap();
1234 auto batchDimMap = getBatchDimMap();
1235
1236 // Verify at least one contracting dimension pair was specified.
1237 if (contractingDimMap.empty())
1238 return emitOpError(message: "expected at least one contracting dimension pair");
1239
1240 // Verify contracting dimension map was properly constructed.
1241 if (!verifyDimMap(lhsType, rhsType, map: contractingDimMap))
1242 return emitOpError(message: "invalid contracting dimension map");
1243
1244 // Verify batch dimension map was properly constructed.
1245 if (!verifyDimMap(lhsType, rhsType, map: batchDimMap))
1246 return emitOpError(message: "invalid batch dimension map");
1247
1248 // Verify 'accType' and 'resType' shape.
1249 if (failed(Result: verifyOutputShape(op: *this, lhsType, rhsType, accType, resType,
1250 contractingDimMap, batchDimMap)))
1251 return failure();
1252
1253 if (!getKindAttr()) {
1254 return emitOpError(message: "expected 'kind' attribute of type CombiningKind (e.g. "
1255 "'vector.kind<add>')");
1256 }
1257
1258 // Verify supported combining kind.
1259 auto vectorType = llvm::dyn_cast<VectorType>(Val&: resType);
1260 auto elementType = vectorType ? vectorType.getElementType() : resType;
1261 if (!isSupportedCombiningKind(combiningKind: getKind(), elementType))
1262 return emitOpError(message: "unsupported contraction type");
1263
1264 // Delayed calling of IndexingMapOpInterface::verifyImpl.
1265 return cast<IndexingMapOpInterface>(Val: this->getOperation()).verifyImpl();
1266}
1267
1268// MaskableOpInterface methods.
1269
1270/// Returns the mask type expected by this operation. Mostly used for
1271/// verification purposes. It requires the operation to be vectorized."
1272Type ContractionOp::getExpectedMaskType() {
1273 auto indexingMaps = this->getIndexingMapsArray();
1274 AffineMap lhsIdxMap = indexingMaps[0];
1275 AffineMap rhsIdxMap = indexingMaps[1];
1276 VectorType lhsType = this->getLhsType();
1277 VectorType rhsType = this->getRhsType();
1278
1279 unsigned numVecDims = lhsIdxMap.getNumDims();
1280 SmallVector<int64_t> maskShape(numVecDims, ShapedType::kDynamic);
1281 SmallVector<bool> maskShapeScalableDims(numVecDims, false);
1282
1283 // Using the information in the indexing maps, extract the size of each
1284 // dimension in the vector.contract operation from the two input operands.
1285 for (auto [dimIdx, dimSize] : llvm::enumerate(First: lhsType.getShape())) {
1286 maskShape[lhsIdxMap.getDimPosition(idx: dimIdx)] = dimSize;
1287 maskShapeScalableDims[lhsIdxMap.getDimPosition(idx: dimIdx)] =
1288 lhsType.getScalableDims()[dimIdx];
1289 }
1290 for (auto [dimIdx, dimSize] : llvm::enumerate(First: rhsType.getShape())) {
1291 maskShape[rhsIdxMap.getDimPosition(idx: dimIdx)] = dimSize;
1292 maskShapeScalableDims[rhsIdxMap.getDimPosition(idx: dimIdx)] =
1293 rhsType.getScalableDims()[dimIdx];
1294 }
1295
1296 assert(ShapedType::isStaticShape(maskShape) &&
1297 "Mask shape couldn't be computed");
1298
1299 return VectorType::get(shape: maskShape,
1300 elementType: IntegerType::get(context: lhsType.getContext(), /*width=*/1),
1301 scalableDims: maskShapeScalableDims);
1302}
1303
1304SmallVector<StringRef> ContractionOp::getTraitAttrNames() {
1305 return SmallVector<StringRef>{getIndexingMapsAttrName(),
1306 getIteratorTypesAttrName(), getKindAttrName(),
1307 getFastmathAttrName()};
1308}
1309
1310static int64_t getResultIndex(AffineMap map, AffineExpr targetExpr) {
1311 for (int64_t i = 0, e = map.getNumResults(); i < e; ++i)
1312 if (targetExpr == map.getResult(idx: i))
1313 return i;
1314 return -1;
1315}
1316
1317// For every contraction `dim` for which:
1318// * the corresponding iteratory type matches `targetIteratorType`,
1319// returns a pair: <lhsIdx, rhsIdx>, where:
1320// * {lhs|rhs}Idx is the index of `dim` in the (result of) indexing map for
1321// {lhs|rhs}.
1322// If `oneSidedDim` is true, only dims indexed on exactly one of the `contract`
1323// operands are returned.
1324static std::vector<std::pair<int64_t, int64_t>>
1325getDimPairForIterTy(ContractionOp *contract, IteratorType targetIteratorType,
1326 bool oneSidedDim = false) {
1327 SmallVector<AffineMap, 4> indexingMaps(contract->getIndexingMapsArray());
1328 ArrayAttr iteratorTypes = contract->getIteratorTypes();
1329 std::vector<std::pair<int64_t, int64_t>> dimMap;
1330 for (const auto &it : llvm::enumerate(First&: iteratorTypes)) {
1331 auto iteratorType = llvm::cast<IteratorTypeAttr>(Val: it.value()).getValue();
1332 if (iteratorType != targetIteratorType)
1333 continue;
1334 // Search lhs/rhs map results for 'targetExpr'.
1335 auto targetExpr = getAffineDimExpr(position: it.index(), context: contract->getContext());
1336 int64_t lhsDim = getResultIndex(map: indexingMaps[0], targetExpr);
1337 int64_t rhsDim = getResultIndex(map: indexingMaps[1], targetExpr);
1338 bool isOneSidedDim = (lhsDim >= 0) != (rhsDim >= 0);
1339 bool isTwoSidedDim = lhsDim >= 0 && rhsDim >= 0;
1340 if (oneSidedDim ? isOneSidedDim : isTwoSidedDim)
1341 dimMap.emplace_back(args&: lhsDim, args&: rhsDim);
1342 }
1343 return dimMap;
1344}
1345
1346std::vector<std::pair<int64_t, int64_t>> ContractionOp::getContractingDimMap() {
1347 return getDimPairForIterTy(contract: this, targetIteratorType: IteratorType::reduction);
1348}
1349
1350std::vector<std::pair<int64_t, int64_t>> ContractionOp::getBatchDimMap() {
1351 return getDimPairForIterTy(contract: this, targetIteratorType: IteratorType::parallel);
1352}
1353
1354std::vector<std::pair<int64_t, int64_t>> ContractionOp::getFreeDimMap() {
1355 return getDimPairForIterTy(contract: this, targetIteratorType: IteratorType::parallel,
1356 /*oneSidedDim=*/true);
1357}
1358
1359void ContractionOp::getIterationBounds(
1360 SmallVectorImpl<int64_t> &iterationBounds) {
1361 auto lhsShape = getLhsType().getShape();
1362 auto resVectorType = llvm::dyn_cast<VectorType>(Val: getResultType());
1363 SmallVector<AffineMap, 4> indexingMaps(getIndexingMapsArray());
1364 for (const auto &it : llvm::enumerate(First: getIteratorTypes())) {
1365 // Search lhs/rhs map results for 'targetExpr'.
1366 auto targetExpr = getAffineDimExpr(position: it.index(), context: getContext());
1367 auto iteratorType = llvm::cast<IteratorTypeAttr>(Val: it.value()).getValue();
1368 if (iteratorType == IteratorType::reduction) {
1369 // Get reduction dim size from lhs shape (same size in rhsShape).
1370 int64_t lhsDimIndex = getResultIndex(map: indexingMaps[0], targetExpr);
1371 assert(lhsDimIndex >= 0);
1372 iterationBounds.push_back(Elt: lhsShape[lhsDimIndex]);
1373 continue;
1374 }
1375 // Get parallel dimension size from result shape.
1376 int64_t resDimIndex = getResultIndex(map: indexingMaps[2], targetExpr);
1377 assert(resDimIndex >= 0);
1378 assert(resVectorType != nullptr);
1379 iterationBounds.push_back(Elt: resVectorType.getShape()[resDimIndex]);
1380 }
1381}
1382
1383void ContractionOp::getIterationIndexMap(
1384 std::vector<DenseMap<int64_t, int64_t>> &iterationIndexMap) {
1385 unsigned numMaps = getIndexingMapsArray().size();
1386 iterationIndexMap.resize(new_size: numMaps);
1387 for (const auto &it : llvm::enumerate(First: getIndexingMapsArray())) {
1388 auto index = it.index();
1389 auto map = it.value();
1390 for (unsigned i = 0, e = map.getNumResults(); i < e; ++i) {
1391 auto dim = cast<AffineDimExpr>(Val: map.getResult(idx: i));
1392 iterationIndexMap[index][dim.getPosition()] = i;
1393 }
1394 }
1395}
1396
1397//-----------------------------------------------------------------------------
1398// VectorUnrollOpInterface methods
1399//-----------------------------------------------------------------------------
1400std::optional<SmallVector<int64_t, 4>>
1401ContractionOp::getIterationShapeToUnroll() {
1402 SmallVector<int64_t, 4> shape;
1403 getIterationBounds(iterationBounds&: shape);
1404 return shape;
1405}
1406
1407std::optional<SmallVector<int64_t>>
1408ContractionOp::getDefaultUnrollTargetShape() {
1409 SmallVector<int64_t> nativeShape(this->getIteratorTypes().size(), 4);
1410 Type lhsType = this->getLhsType().getElementType();
1411 nativeShape[nativeShape.size() - 1] = lhsType.isF16() ? 4 : 2;
1412 return nativeShape;
1413}
1414
1415/// Return a fused vector::ContractionOp which represents a patterns such as:
1416///
1417/// ```mlir
1418/// %c0 = vector.constant 0: ...
1419/// %c = vector.contract %a, %b, %c0: ...
1420/// %e = add %c, %d: ...
1421/// ```
1422///
1423/// by:
1424///
1425/// ```mlir
1426/// %e = vector.contract %a, %b, %d: ...
1427/// ```
1428///
1429/// Return null if the canonicalization does not apply.
1430// TODO: This should be a folding of Add into Contract in core but while they
1431// live in different dialects, it is not possible without unnatural
1432// dependencies.
1433template <typename AddOpType>
1434struct CanonicalizeContractAdd : public OpRewritePattern<AddOpType> {
1435 using OpRewritePattern<AddOpType>::OpRewritePattern;
1436
1437 LogicalResult matchAndRewrite(AddOpType addOp,
1438 PatternRewriter &rewriter) const override {
1439 auto canonicalize = [&](Value maybeContraction,
1440 Value otherOperand) -> vector::ContractionOp {
1441 vector::ContractionOp contractionOp =
1442 dyn_cast_or_null<vector::ContractionOp>(
1443 Val: maybeContraction.getDefiningOp());
1444 if (!contractionOp)
1445 return vector::ContractionOp();
1446 if (auto maybeZero = dyn_cast_or_null<arith::ConstantOp>(
1447 Val: contractionOp.getAcc().getDefiningOp())) {
1448 if (maybeZero.getValue() ==
1449 rewriter.getZeroAttr(type: contractionOp.getAcc().getType())) {
1450 IRMapping bvm;
1451 bvm.map(from: contractionOp.getAcc(), to: otherOperand);
1452 auto newContraction =
1453 cast<vector::ContractionOp>(Val: rewriter.clone(op&: *contractionOp, mapper&: bvm));
1454 rewriter.replaceOp(addOp, newContraction.getResult());
1455 return newContraction;
1456 }
1457 }
1458 return vector::ContractionOp();
1459 };
1460
1461 Value a = addOp->getOperand(0), b = addOp->getOperand(1);
1462 vector::ContractionOp contract = canonicalize(a, b);
1463 contract = contract ? contract : canonicalize(b, a);
1464 return contract ? success() : failure();
1465 }
1466};
1467
1468void ContractionOp::getCanonicalizationPatterns(RewritePatternSet &results,
1469 MLIRContext *context) {
1470 results.add<CanonicalizeContractAdd<arith::AddIOp>,
1471 CanonicalizeContractAdd<arith::AddFOp>>(arg&: context);
1472}
1473
1474// Returns `true` if `index` is either within [0, maxIndex) or equal to
1475// `poisonValue`.
1476static bool isValidPositiveIndexOrPoison(int64_t index, int64_t poisonValue,
1477 int64_t maxIndex) {
1478 return index == poisonValue || (index >= 0 && index < maxIndex);
1479}
1480
1481//===----------------------------------------------------------------------===//
1482// ExtractOp
1483//===----------------------------------------------------------------------===//
1484
1485void ExtractOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges,
1486 SetIntRangeFn setResultRanges) {
1487 setResultRanges(getResult(), argRanges.front());
1488}
1489
1490void vector::ExtractOp::build(OpBuilder &builder, OperationState &result,
1491 Value source) {
1492 auto vectorTy = cast<VectorType>(Val: source.getType());
1493 build(odsBuilder&: builder, odsState&: result, source, position: SmallVector<int64_t>(vectorTy.getRank(), 0));
1494}
1495
1496void vector::ExtractOp::build(OpBuilder &builder, OperationState &result,
1497 Value source, int64_t position) {
1498 build(odsBuilder&: builder, odsState&: result, source, position: ArrayRef<int64_t>{position});
1499}
1500
1501void vector::ExtractOp::build(OpBuilder &builder, OperationState &result,
1502 Value source, OpFoldResult position) {
1503 build(odsBuilder&: builder, odsState&: result, source, position: ArrayRef<OpFoldResult>{position});
1504}
1505
1506void vector::ExtractOp::build(OpBuilder &builder, OperationState &result,
1507 Value source, ArrayRef<int64_t> position) {
1508 build(odsBuilder&: builder, odsState&: result, source, /*dynamic_position=*/ArrayRef<Value>(),
1509 static_position: builder.getDenseI64ArrayAttr(values: position));
1510}
1511
1512void vector::ExtractOp::build(OpBuilder &builder, OperationState &result,
1513 Value source, ArrayRef<OpFoldResult> position) {
1514 SmallVector<int64_t> staticPos;
1515 SmallVector<Value> dynamicPos;
1516 dispatchIndexOpFoldResults(ofrs: position, dynamicVec&: dynamicPos, staticVec&: staticPos);
1517 build(odsBuilder&: builder, odsState&: result, source, dynamic_position: dynamicPos,
1518 static_position: builder.getDenseI64ArrayAttr(values: staticPos));
1519}
1520
1521LogicalResult
1522ExtractOp::inferReturnTypes(MLIRContext *, std::optional<Location>,
1523 ExtractOp::Adaptor adaptor,
1524 SmallVectorImpl<Type> &inferredReturnTypes) {
1525 auto vectorType = llvm::cast<VectorType>(Val: adaptor.getSource().getType());
1526 if (static_cast<int64_t>(adaptor.getStaticPosition().size()) ==
1527 vectorType.getRank()) {
1528 inferredReturnTypes.push_back(Elt: vectorType.getElementType());
1529 } else {
1530 auto n = std::min<size_t>(a: adaptor.getStaticPosition().size(),
1531 b: vectorType.getRank());
1532 inferredReturnTypes.push_back(Elt: VectorType::get(
1533 shape: vectorType.getShape().drop_front(N: n), elementType: vectorType.getElementType(),
1534 scalableDims: vectorType.getScalableDims().drop_front(N: n)));
1535 }
1536 return success();
1537}
1538
1539LogicalResult vector::ExtractOp::verify() {
1540 if (auto resTy = dyn_cast<VectorType>(Val: getResult().getType()))
1541 if (resTy.getRank() == 0)
1542 return emitError(
1543 message: "expected a scalar instead of a 0-d vector as the result type");
1544
1545 // Note: This check must come before getMixedPosition() to prevent a crash.
1546 auto dynamicMarkersCount =
1547 llvm::count_if(Range: getStaticPosition(), P: ShapedType::isDynamic);
1548 if (static_cast<size_t>(dynamicMarkersCount) != getDynamicPosition().size())
1549 return emitOpError(
1550 message: "mismatch between dynamic and static positions (kDynamic marker but no "
1551 "corresponding dynamic position) -- this can only happen due to an "
1552 "incorrect fold/rewrite");
1553 auto position = getMixedPosition();
1554 if (position.size() > static_cast<unsigned>(getSourceVectorType().getRank()))
1555 return emitOpError(
1556 message: "expected position attribute of rank no greater than vector rank");
1557 for (auto [idx, pos] : llvm::enumerate(First&: position)) {
1558 if (auto attr = dyn_cast<Attribute>(Val&: pos)) {
1559 int64_t constIdx = cast<IntegerAttr>(Val&: attr).getInt();
1560 if (!isValidPositiveIndexOrPoison(
1561 index: constIdx, poisonValue: kPoisonIndex, maxIndex: getSourceVectorType().getDimSize(idx))) {
1562 return emitOpError(message: "expected position attribute #")
1563 << (idx + 1)
1564 << " to be a non-negative integer smaller than the "
1565 "corresponding vector dimension or poison (-1)";
1566 }
1567 }
1568 }
1569 return success();
1570}
1571
1572template <typename IntType>
1573static SmallVector<IntType> extractVector(ArrayAttr arrayAttr) {
1574 return llvm::map_to_vector<4>(
1575 arrayAttr.getAsRange<IntegerAttr>(),
1576 [](IntegerAttr attr) { return static_cast<IntType>(attr.getInt()); });
1577}
1578
1579/// Fold the result of chains of ExtractOp in place by simply concatenating the
1580/// positions. Both static and dynamic positions are supported.
1581///
1582/// BEFORE:
1583/// ```mlir
1584/// %0 = vector.extract %v[1, 2] : vector<3x4x5xf32>
1585/// %1 = vector.extract %0[%idx, 3] : vector<5xf32>
1586/// ```
1587///
1588/// AFTER:
1589/// ```mlir
1590/// %1 = vector.extract %v[1, 2, %idx, 3] : vector<3x4x5xf32>
1591/// ```
1592static LogicalResult foldExtractOpFromExtractChain(ExtractOp extractOp) {
1593 if (!extractOp.getSource().getDefiningOp<ExtractOp>())
1594 return failure();
1595
1596 SmallVector<OpFoldResult> globalPosition = extractOp.getMixedPosition();
1597 ExtractOp currentOp = extractOp;
1598 while (ExtractOp nextOp = currentOp.getSource().getDefiningOp<ExtractOp>()) {
1599 currentOp = nextOp;
1600 SmallVector<OpFoldResult> pos = currentOp.getMixedPosition();
1601 globalPosition.insert(I: globalPosition.begin(), From: pos.begin(), To: pos.end());
1602 }
1603 auto [staticPos, dynPos] = decomposeMixedValues(mixedValues: globalPosition);
1604 Value source = currentOp.getSource();
1605 extractOp->setOperands(
1606 llvm::to_vector(Range: llvm::concat<Value>(Ranges: ValueRange(source), Ranges&: dynPos)));
1607 extractOp.setStaticPosition(staticPos);
1608 return success();
1609}
1610
1611namespace {
1612/// Fold an ExtractOp that is fed by a chain of InsertOps and TransposeOps.
1613/// Walk back a chain of InsertOp/TransposeOp until we hit a match.
1614/// Compose TransposeOp permutations as we walk back.
1615/// This helper class keeps an updated extraction position `extractPosition`
1616/// with extra trailing sentinels.
1617/// The sentinels encode the internal transposition status of the result vector.
1618/// As we iterate, extractPosition is permuted and updated.
1619class ExtractFromInsertTransposeChainState {
1620public:
1621 ExtractFromInsertTransposeChainState(ExtractOp e);
1622
1623 /// Iterate over producing insert and transpose ops until we find a fold.
1624 Value fold();
1625
1626private:
1627 /// Return true if the vector at position `a` is contained within the vector
1628 /// at position `b`. Under insert/extract semantics, this is the same as `a`
1629 /// is a prefix of `b`.
1630 template <typename ContainerA, typename ContainerB>
1631 bool isContainedWithin(const ContainerA &a, const ContainerB &b) {
1632 return a.size() <= b.size() &&
1633 std::equal(a.begin(), a.begin() + a.size(), b.begin());
1634 }
1635
1636 /// Return true if the vector at position `a` intersects the vector at
1637 /// position `b`. Under insert/extract semantics, this is the same as equality
1638 /// of all entries of `a` that are >=0 with the corresponding entries of b.
1639 /// Comparison is on the common prefix (i.e. zip).
1640 template <typename ContainerA, typename ContainerB>
1641 bool intersectsWhereNonNegative(const ContainerA &a, const ContainerB &b) {
1642 for (auto [elemA, elemB] : llvm::zip(a, b)) {
1643 if (elemA < 0 || elemB < 0)
1644 continue;
1645 if (elemA != elemB)
1646 return false;
1647 }
1648 return true;
1649 }
1650
1651 /// Folding is only possible in the absence of an internal permutation in the
1652 /// result vector.
1653 bool canFold() {
1654 return (sentinels == ArrayRef(extractPosition).drop_front(N: extractedRank));
1655 }
1656
1657 // Helper to get the next defining op of interest.
1658 void updateStateForNextIteration(Value v) {
1659 nextInsertOp = v.getDefiningOp<vector::InsertOp>();
1660 nextTransposeOp = v.getDefiningOp<vector::TransposeOp>();
1661 };
1662
1663 // Case 1. If we hit a transpose, just compose the map and iterate.
1664 // Invariant: insert + transpose do not change rank, we can always compose.
1665 LogicalResult handleTransposeOp();
1666
1667 // Case 2: the insert position matches extractPosition exactly, early return.
1668 LogicalResult handleInsertOpWithMatchingPos(Value &res);
1669
1670 /// Case 3: if the insert position is a prefix of extractPosition, extract a
1671 /// portion of the source of the insert.
1672 /// Example:
1673 /// ```
1674 /// %ins = vector.insert %source, %vest[1]: vector<3x4> into vector<2x3x4x5>
1675 /// // extractPosition == [1, 2, 3]
1676 /// %ext = vector.extract %ins[1, 0]: vector<5> from vector<3x4x5>
1677 /// // can fold to vector.extract %source[0, 3]
1678 /// %ext = vector.extract %source[3]: vector<6> from vector<5x6>
1679 /// ```
1680 /// To traverse through %source, we need to set the leading dims to 0 and
1681 /// drop the extra leading dims.
1682 /// This method updates the internal state.
1683 LogicalResult handleInsertOpWithPrefixPos(Value &res);
1684
1685 /// Try to fold in place to extract(source, extractPosition) and return the
1686 /// folded result. Return null if folding is not possible (e.g. due to an
1687 /// internal transposition in the result).
1688 Value tryToFoldExtractOpInPlace(Value source);
1689
1690 ExtractOp extractOp;
1691 int64_t vectorRank;
1692 int64_t extractedRank;
1693
1694 InsertOp nextInsertOp;
1695 TransposeOp nextTransposeOp;
1696
1697 /// Sentinel values that encode the internal permutation status of the result.
1698 /// They are set to (-1, ... , -k) at the beginning and appended to
1699 /// `extractPosition`.
1700 /// In the end, the tail of `extractPosition` must be exactly `sentinels` to
1701 /// ensure that there is no internal transposition.
1702 /// Internal transposition cannot be accounted for with a folding pattern.
1703 // TODO: We could relax the internal transposition with an extra transposition
1704 // operation in a future canonicalizer.
1705 SmallVector<int64_t> sentinels;
1706 SmallVector<int64_t> extractPosition;
1707};
1708} // namespace
1709
1710ExtractFromInsertTransposeChainState::ExtractFromInsertTransposeChainState(
1711 ExtractOp e)
1712 : extractOp(e), vectorRank(extractOp.getSourceVectorType().getRank()),
1713 extractedRank(extractOp.getNumIndices()) {
1714 assert(vectorRank >= extractedRank && "Extracted position overflow");
1715 sentinels.reserve(N: vectorRank - extractedRank);
1716 for (int64_t i = 0, e = vectorRank - extractedRank; i < e; ++i)
1717 sentinels.push_back(Elt: -(i + 1));
1718 extractPosition.assign(in_start: extractOp.getStaticPosition().begin(),
1719 in_end: extractOp.getStaticPosition().end());
1720 llvm::append_range(C&: extractPosition, R&: sentinels);
1721}
1722
1723// Case 1. If we hit a transpose, just compose the map and iterate.
1724// Invariant: insert + transpose do not change rank, we can always compose.
1725LogicalResult ExtractFromInsertTransposeChainState::handleTransposeOp() {
1726 // TODO: Canonicalization for dynamic position not implemented yet.
1727 if (extractOp.hasDynamicPosition())
1728 return failure();
1729
1730 if (!nextTransposeOp)
1731 return failure();
1732
1733 if (nextTransposeOp.getResultVectorType().getRank() == 0)
1734 return success();
1735
1736 AffineMap m = inversePermutation(map: AffineMap::getPermutationMap(
1737 permutation: nextTransposeOp.getPermutation(), context: extractOp.getContext()));
1738 extractPosition = applyPermutationMap(map: m, source: ArrayRef(extractPosition));
1739 return success();
1740}
1741
1742// Case 2: the insert position matches extractPosition exactly, early return.
1743LogicalResult
1744ExtractFromInsertTransposeChainState::handleInsertOpWithMatchingPos(
1745 Value &res) {
1746 // TODO: Canonicalization for dynamic position not implemented yet.
1747 if (extractOp.hasDynamicPosition() || nextInsertOp.hasDynamicPosition())
1748 return failure();
1749
1750 ArrayRef<int64_t> insertedPos = nextInsertOp.getStaticPosition();
1751 if (insertedPos != llvm::ArrayRef(extractPosition).take_front(N: extractedRank))
1752 return failure();
1753 // Case 2.a. early-exit fold.
1754 res = nextInsertOp.getValueToStore();
1755 // Case 2.b. if internal transposition is present, canFold will be false.
1756 return success(IsSuccess: canFold());
1757}
1758
1759/// Case 3: if inserted position is a prefix of extractPosition,
1760/// extract a portion of the source of the insertion.
1761/// This method updates the internal state.
1762LogicalResult
1763ExtractFromInsertTransposeChainState::handleInsertOpWithPrefixPos(Value &res) {
1764 // TODO: Canonicalization for dynamic position not implemented yet.
1765 if (extractOp.hasDynamicPosition() || nextInsertOp.hasDynamicPosition())
1766 return failure();
1767
1768 ArrayRef<int64_t> insertedPos = nextInsertOp.getStaticPosition();
1769 if (!isContainedWithin(a: insertedPos, b: extractPosition))
1770 return failure();
1771 // Set leading dims to zero.
1772 std::fill_n(first: extractPosition.begin(), n: insertedPos.size(), value: 0);
1773 // Drop extra leading dims.
1774 extractPosition.erase(CS: extractPosition.begin(),
1775 CE: extractPosition.begin() + insertedPos.size());
1776 extractedRank = extractPosition.size() - sentinels.size();
1777 // Case 3.a. early-exit fold (break and delegate to post-while path).
1778 res = nextInsertOp.getValueToStore();
1779 // Case 3.b. if internal transposition is present, canFold will be false.
1780 return success();
1781}
1782
1783/// Try to fold in place to extract(source, extractPosition) and return the
1784/// folded result. Return null if folding is not possible (e.g. due to an
1785/// internal transposition in the result).
1786Value ExtractFromInsertTransposeChainState::tryToFoldExtractOpInPlace(
1787 Value source) {
1788 // TODO: Canonicalization for dynamic position not implemented yet.
1789 if (extractOp.hasDynamicPosition())
1790 return Value();
1791
1792 // If we can't fold (either internal transposition, or nothing to fold), bail.
1793 bool nothingToFold = (source == extractOp.getSource());
1794 if (nothingToFold || !canFold())
1795 return Value();
1796
1797 // Otherwise, fold by updating the op inplace and return its result.
1798 OpBuilder b(extractOp.getContext());
1799 extractOp.setStaticPosition(
1800 ArrayRef(extractPosition).take_front(N: extractedRank));
1801 extractOp.getSourceMutable().assign(value: source);
1802 return extractOp.getResult();
1803}
1804
1805/// Iterate over producing insert and transpose ops until we find a fold.
1806Value ExtractFromInsertTransposeChainState::fold() {
1807 // TODO: Canonicalization for dynamic position not implemented yet.
1808 if (extractOp.hasDynamicPosition())
1809 return Value();
1810
1811 Value valueToExtractFrom = extractOp.getSource();
1812 updateStateForNextIteration(v: valueToExtractFrom);
1813 while (nextInsertOp || nextTransposeOp) {
1814 // Case 1. If we hit a transpose, just compose the map and iterate.
1815 // Invariant: insert + transpose do not change rank, we can always compose.
1816 if (succeeded(Result: handleTransposeOp())) {
1817 valueToExtractFrom = nextTransposeOp.getVector();
1818 updateStateForNextIteration(v: valueToExtractFrom);
1819 continue;
1820 }
1821
1822 Value result;
1823 // Case 2: the position match exactly.
1824 if (succeeded(Result: handleInsertOpWithMatchingPos(res&: result)))
1825 return result;
1826
1827 // Case 3: if the inserted position is a prefix of extractPosition, we can
1828 // just extract a portion of the source of the insert.
1829 if (succeeded(Result: handleInsertOpWithPrefixPos(res&: result)))
1830 return tryToFoldExtractOpInPlace(source: result);
1831
1832 // Case 4: extractPositionRef intersects insertedPosRef on non-sentinel
1833 // values. This is a more difficult case and we bail.
1834 ArrayRef<int64_t> insertedPos = nextInsertOp.getStaticPosition();
1835 if (isContainedWithin(a: extractPosition, b: insertedPos) ||
1836 intersectsWhereNonNegative(a: extractPosition, b: insertedPos))
1837 return Value();
1838
1839 // Case 5: No intersection, we forward the extract to insertOp.dest().
1840 valueToExtractFrom = nextInsertOp.getDest();
1841 updateStateForNextIteration(v: valueToExtractFrom);
1842 }
1843 // If after all this we can fold, go for it.
1844 return tryToFoldExtractOpInPlace(source: valueToExtractFrom);
1845}
1846
1847/// Returns true if the operation has a 0-D vector type operand or result.
1848static bool hasZeroDimVectors(Operation *op) {
1849 auto hasZeroDimVectorType = [](Type type) -> bool {
1850 auto vecType = dyn_cast<VectorType>(Val&: type);
1851 return vecType && vecType.getRank() == 0;
1852 };
1853
1854 return llvm::any_of(Range: op->getOperandTypes(), P: hasZeroDimVectorType) ||
1855 llvm::any_of(Range: op->getResultTypes(), P: hasZeroDimVectorType);
1856}
1857
1858/// All BroadcastOps, as well as ShapeCastOps that only prepend 1s, are
1859/// considered to be 'broadcastlike'.
1860static bool isBroadcastLike(Operation *op) {
1861 if (isa<BroadcastOp>(Val: op))
1862 return true;
1863
1864 auto shapeCast = dyn_cast<ShapeCastOp>(Val: op);
1865 if (!shapeCast)
1866 return false;
1867
1868 // Check that shape_cast **only** prepends 1s, like (2,3) -> (1,1,2,3).
1869 // Checking that the destination shape has a prefix of 1s is not sufficient,
1870 // for example (2,3) -> (1,3,2) is not broadcastlike. A sufficient condition
1871 // is that the source shape is a suffix of the destination shape.
1872 VectorType srcType = shapeCast.getSourceVectorType();
1873 ArrayRef<int64_t> srcShape = srcType.getShape();
1874 uint64_t srcRank = srcType.getRank();
1875 ArrayRef<int64_t> dstShape = shapeCast.getType().getShape();
1876 return dstShape.size() >= srcRank && dstShape.take_back(N: srcRank) == srcShape;
1877}
1878
1879/// Fold extract(broadcast(X)) to either extract(X) or just X.
1880///
1881/// Example:
1882///
1883/// broadcast extract [1][2]
1884/// (3, 4) --------> (2, 3, 4) ----------------> (4)
1885///
1886/// becomes
1887/// extract [1]
1888/// (3,4) -------------------------------------> (4)
1889///
1890///
1891/// The variable names used in this implementation correspond to the above
1892/// shapes as,
1893///
1894/// - (3, 4) is `input` shape.
1895/// - (2, 3, 4) is `broadcast` shape.
1896/// - (4) is `extract` shape.
1897///
1898/// This folding is possible when the suffix of `input` shape is the same as
1899/// `extract` shape.
1900static Value foldExtractFromBroadcast(ExtractOp extractOp) {
1901
1902 Operation *defOp = extractOp.getSource().getDefiningOp();
1903 if (!defOp || !isBroadcastLike(op: defOp))
1904 return Value();
1905
1906 Value input = defOp->getOperand(idx: 0);
1907
1908 // Replace extract(broadcast(X)) with X
1909 if (extractOp.getType() == input.getType())
1910 return input;
1911
1912 // Get required types and ranks in the chain
1913 // input -> broadcast -> extract
1914 // (scalars are treated as rank-0).
1915 auto inputType = llvm::dyn_cast<VectorType>(Val: input.getType());
1916 auto extractType = llvm::dyn_cast<VectorType>(Val: extractOp.getType());
1917 unsigned inputRank = inputType ? inputType.getRank() : 0;
1918 unsigned broadcastRank = extractOp.getSourceVectorType().getRank();
1919 unsigned extractRank = extractType ? extractType.getRank() : 0;
1920
1921 // Cannot do without the broadcast if overall the rank increases.
1922 if (extractRank > inputRank)
1923 return Value();
1924
1925 // The above condition guarantees that input is a vector.
1926 assert(inputType && "input must be a vector type because of previous checks");
1927 ArrayRef<int64_t> inputShape = inputType.getShape();
1928
1929 // In the case where there is a broadcast dimension in the suffix, it is not
1930 // possible to replace extract(broadcast(X)) with extract(X). Example:
1931 //
1932 // broadcast extract
1933 // (1) --------> (3,4) ------> (4)
1934 if (extractType &&
1935 extractType.getShape() != inputShape.take_back(N: extractRank))
1936 return Value();
1937
1938 // Replace extract(broadcast(X)) with extract(X).
1939 // First, determine the new extraction position.
1940 unsigned deltaOverall = inputRank - extractRank;
1941 unsigned deltaBroadcast = broadcastRank - inputRank;
1942 SmallVector<OpFoldResult> oldPositions = extractOp.getMixedPosition();
1943 SmallVector<OpFoldResult> newPositions(deltaOverall);
1944 IntegerAttr zero = OpBuilder(extractOp.getContext()).getIndexAttr(value: 0);
1945 for (auto [i, size] : llvm::enumerate(First: inputShape.take_front(N: deltaOverall))) {
1946 newPositions[i] = size == 1 ? zero : oldPositions[i + deltaBroadcast];
1947 }
1948 auto [staticPos, dynPos] = decomposeMixedValues(mixedValues: newPositions);
1949 extractOp->setOperands(
1950 llvm::to_vector(Range: llvm::concat<Value>(Ranges: ValueRange(input), Ranges&: dynPos)));
1951 extractOp.setStaticPosition(staticPos);
1952 return extractOp.getResult();
1953}
1954
1955/// Fold extractOp coming from ShuffleOp.
1956///
1957/// Example:
1958///
1959/// %shuffle = vector.shuffle %a, %b [0, 8, 7, 15]
1960/// : vector<8xf32>, vector<8xf32>
1961/// %extract = vector.extract %shuffle[3] : f32 from vector<4xf32>
1962/// ->
1963/// %extract = vector.extract %b[7] : f32 from vector<8xf32>
1964///
1965static Value foldExtractFromShuffle(ExtractOp extractOp) {
1966 // Dynamic positions are not folded as the resulting code would be more
1967 // complex than the input code.
1968 if (extractOp.hasDynamicPosition())
1969 return Value();
1970
1971 auto shuffleOp = extractOp.getSource().getDefiningOp<ShuffleOp>();
1972 if (!shuffleOp)
1973 return Value();
1974
1975 // TODO: 0-D or multi-dimensional vectors not supported yet.
1976 if (shuffleOp.getResultVectorType().getRank() != 1)
1977 return Value();
1978
1979 int64_t inputVecSize = shuffleOp.getV1().getType().getShape()[0];
1980 auto shuffleMask = shuffleOp.getMask();
1981 int64_t extractIdx = extractOp.getStaticPosition()[0];
1982 int64_t shuffleIdx = shuffleMask[extractIdx];
1983
1984 // Find the shuffled vector to extract from based on the shuffle index.
1985 if (shuffleIdx < inputVecSize) {
1986 extractOp.setOperand(i: 0, value: shuffleOp.getV1());
1987 extractOp.setStaticPosition({shuffleIdx});
1988 } else {
1989 extractOp.setOperand(i: 0, value: shuffleOp.getV2());
1990 extractOp.setStaticPosition({shuffleIdx - inputVecSize});
1991 }
1992
1993 return extractOp.getResult();
1994}
1995
1996// Fold extractOp with source coming from ShapeCast op.
1997static Value foldExtractFromShapeCast(ExtractOp extractOp) {
1998 // TODO: Canonicalization for dynamic position not implemented yet.
1999 if (extractOp.hasDynamicPosition())
2000 return Value();
2001
2002 auto shapeCastOp = extractOp.getSource().getDefiningOp<vector::ShapeCastOp>();
2003 if (!shapeCastOp)
2004 return Value();
2005
2006 // Get the nth dimension size starting from lowest dimension.
2007 auto getDimReverse = [](VectorType type, int64_t n) {
2008 return type.getShape().take_back(N: n + 1).front();
2009 };
2010 int64_t destinationRank =
2011 llvm::isa<VectorType>(Val: extractOp.getType())
2012 ? llvm::cast<VectorType>(Val: extractOp.getType()).getRank()
2013 : 0;
2014 if (destinationRank > shapeCastOp.getSourceVectorType().getRank())
2015 return Value();
2016 if (destinationRank > 0) {
2017 auto destinationType =
2018 llvm::cast<VectorType>(Val: extractOp.getResult().getType());
2019 for (int64_t i = 0; i < destinationRank; i++) {
2020 // The lowest dimension of the destination must match the lowest
2021 // dimension of the shapecast op source.
2022 // TODO: This case could be support in a canonicalization pattern.
2023 if (getDimReverse(shapeCastOp.getSourceVectorType(), i) !=
2024 getDimReverse(destinationType, i))
2025 return Value();
2026 }
2027 }
2028 // Extract the strides associated with the extract op vector source. Then use
2029 // this to calculate a linearized position for the extract.
2030 SmallVector<int64_t> extractedPos(extractOp.getStaticPosition());
2031 std::reverse(first: extractedPos.begin(), last: extractedPos.end());
2032 SmallVector<int64_t, 4> strides;
2033 int64_t stride = 1;
2034 for (int64_t i = 0, e = extractedPos.size(); i < e; i++) {
2035 strides.push_back(Elt: stride);
2036 stride *=
2037 getDimReverse(extractOp.getSourceVectorType(), i + destinationRank);
2038 }
2039
2040 int64_t position = linearize(offsets: extractedPos, basis: strides);
2041 // Then extract the strides associated to the shapeCast op vector source and
2042 // delinearize the position using those strides.
2043 SmallVector<int64_t, 4> newStrides;
2044 int64_t numDimension =
2045 shapeCastOp.getSourceVectorType().getRank() - destinationRank;
2046 stride = 1;
2047 for (int64_t i = 0; i < numDimension; i++) {
2048 newStrides.push_back(Elt: stride);
2049 stride *=
2050 getDimReverse(shapeCastOp.getSourceVectorType(), i + destinationRank);
2051 }
2052 std::reverse(first: newStrides.begin(), last: newStrides.end());
2053 SmallVector<int64_t, 4> newPosition = delinearize(linearIndex: position, strides: newStrides);
2054 // OpBuilder is only used as a helper to build an I64ArrayAttr.
2055 OpBuilder b(extractOp.getContext());
2056 extractOp.setStaticPosition(newPosition);
2057 extractOp.setOperand(i: 0, value: shapeCastOp.getSource());
2058 return extractOp.getResult();
2059}
2060
2061/// Fold an ExtractOp from ExtractStridedSliceOp.
2062static Value foldExtractFromExtractStrided(ExtractOp extractOp) {
2063 // TODO: Canonicalization for dynamic position not implemented yet.
2064 if (extractOp.hasDynamicPosition())
2065 return Value();
2066
2067 auto extractStridedSliceOp =
2068 extractOp.getSource().getDefiningOp<vector::ExtractStridedSliceOp>();
2069 if (!extractStridedSliceOp)
2070 return Value();
2071
2072 // 0-D vectors not supported.
2073 assert(!hasZeroDimVectors(extractOp) && "0-D vectors not supported");
2074 if (hasZeroDimVectors(op: extractStridedSliceOp))
2075 return Value();
2076
2077 // Return if 'extractStridedSliceOp' has non-unit strides.
2078 if (extractStridedSliceOp.hasNonUnitStrides())
2079 return Value();
2080
2081 // Trim offsets for dimensions fully extracted.
2082 auto sliceOffsets =
2083 extractVector<int64_t>(arrayAttr: extractStridedSliceOp.getOffsets());
2084 while (!sliceOffsets.empty()) {
2085 size_t lastOffset = sliceOffsets.size() - 1;
2086 if (sliceOffsets.back() != 0 ||
2087 extractStridedSliceOp.getType().getDimSize(idx: lastOffset) !=
2088 extractStridedSliceOp.getSourceVectorType().getDimSize(idx: lastOffset))
2089 break;
2090 sliceOffsets.pop_back();
2091 }
2092 unsigned destinationRank = 0;
2093 if (auto vecType = llvm::dyn_cast<VectorType>(Val: extractOp.getType()))
2094 destinationRank = vecType.getRank();
2095 // The dimensions of the result need to be untouched by the
2096 // extractStridedSlice op.
2097 if (destinationRank > extractStridedSliceOp.getSourceVectorType().getRank() -
2098 sliceOffsets.size())
2099 return Value();
2100
2101 SmallVector<int64_t> extractedPos(extractOp.getStaticPosition());
2102 assert(extractedPos.size() >= sliceOffsets.size());
2103 for (size_t i = 0, e = sliceOffsets.size(); i < e; i++)
2104 extractedPos[i] = extractedPos[i] + sliceOffsets[i];
2105 extractOp.getSourceMutable().assign(value: extractStridedSliceOp.getSource());
2106
2107 // OpBuilder is only used as a helper to build an I64ArrayAttr.
2108 OpBuilder b(extractOp.getContext());
2109 extractOp.setStaticPosition(extractedPos);
2110 return extractOp.getResult();
2111}
2112
2113/// Fold extract_op fed from a chain of insertStridedSlice ops.
2114static Value foldExtractStridedOpFromInsertChain(ExtractOp extractOp) {
2115 // TODO: Canonicalization for dynamic position not implemented yet.
2116 if (extractOp.hasDynamicPosition())
2117 return Value();
2118
2119 int64_t destinationRank =
2120 llvm::isa<VectorType>(Val: extractOp.getType())
2121 ? llvm::cast<VectorType>(Val: extractOp.getType()).getRank()
2122 : 0;
2123 auto insertOp = extractOp.getSource().getDefiningOp<InsertStridedSliceOp>();
2124 if (!insertOp)
2125 return Value();
2126
2127 // 0-D vectors not supported.
2128 assert(!hasZeroDimVectors(extractOp) && "0-D vectors not supported");
2129 if (hasZeroDimVectors(op: insertOp))
2130 return Value();
2131
2132 while (insertOp) {
2133 int64_t insertRankDiff = insertOp.getDestVectorType().getRank() -
2134 insertOp.getSourceVectorType().getRank();
2135 if (destinationRank > insertOp.getSourceVectorType().getRank())
2136 return Value();
2137 auto insertOffsets = extractVector<int64_t>(arrayAttr: insertOp.getOffsets());
2138 ArrayRef<int64_t> extractOffsets = extractOp.getStaticPosition();
2139
2140 if (llvm::any_of(Range: insertOp.getStrides(), P: [](Attribute attr) {
2141 return llvm::cast<IntegerAttr>(Val&: attr).getInt() != 1;
2142 }))
2143 return Value();
2144 bool disjoint = false;
2145 SmallVector<int64_t, 4> offsetDiffs;
2146 for (unsigned dim = 0, e = extractOffsets.size(); dim < e; ++dim) {
2147 int64_t start = insertOffsets[dim];
2148 int64_t size =
2149 (dim < insertRankDiff)
2150 ? 1
2151 : insertOp.getSourceVectorType().getDimSize(idx: dim - insertRankDiff);
2152 int64_t end = start + size;
2153 int64_t offset = extractOffsets[dim];
2154 // Check if the start of the extract offset is in the interval inserted.
2155 if (start <= offset && offset < end) {
2156 if (dim >= insertRankDiff)
2157 offsetDiffs.push_back(Elt: offset - start);
2158 continue;
2159 }
2160 disjoint = true;
2161 break;
2162 }
2163 // The extract element chunk overlap with the vector inserted.
2164 if (!disjoint) {
2165 // If any of the inner dimensions are only partially inserted we have a
2166 // partial overlap.
2167 int64_t srcRankDiff =
2168 insertOp.getSourceVectorType().getRank() - destinationRank;
2169 for (int64_t i = 0; i < destinationRank; i++) {
2170 if (insertOp.getSourceVectorType().getDimSize(idx: i + srcRankDiff) !=
2171 insertOp.getDestVectorType().getDimSize(idx: i + srcRankDiff +
2172 insertRankDiff))
2173 return Value();
2174 }
2175 extractOp.getSourceMutable().assign(value: insertOp.getValueToStore());
2176 // OpBuilder is only used as a helper to build an I64ArrayAttr.
2177 OpBuilder b(extractOp.getContext());
2178 extractOp.setStaticPosition(offsetDiffs);
2179 return extractOp.getResult();
2180 }
2181 // If the chunk extracted is disjoint from the chunk inserted, keep
2182 // looking in the insert chain.
2183 insertOp = insertOp.getDest().getDefiningOp<InsertStridedSliceOp>();
2184 }
2185 return Value();
2186}
2187
2188/// Try to fold the extraction of a scalar from a vector defined by
2189/// vector.from_elements. E.g.:
2190///
2191/// %0 = vector.from_elements %a, %b : vector<2xf32>
2192/// %1 = vector.extract %0[0] : f32 from vector<2xf32>
2193/// ==> fold to %a
2194static Value foldScalarExtractFromFromElements(ExtractOp extractOp) {
2195 // Dynamic extractions cannot be folded.
2196 if (extractOp.hasDynamicPosition())
2197 return {};
2198
2199 // Look for extract(from_elements).
2200 auto fromElementsOp = extractOp.getSource().getDefiningOp<FromElementsOp>();
2201 if (!fromElementsOp)
2202 return {};
2203
2204 // Scalable vectors are not supported.
2205 auto vecType = llvm::cast<VectorType>(Val: fromElementsOp.getType());
2206 if (vecType.isScalable())
2207 return {};
2208
2209 // Only extractions of scalars are supported.
2210 int64_t rank = vecType.getRank();
2211 ArrayRef<int64_t> indices = extractOp.getStaticPosition();
2212 if (extractOp.getType() != vecType.getElementType())
2213 return {};
2214 assert(static_cast<int64_t>(indices.size()) == rank &&
2215 "unexpected number of indices");
2216
2217 // Compute flattened/linearized index and fold to operand.
2218 int flatIndex = 0;
2219 int stride = 1;
2220 for (int i = rank - 1; i >= 0; --i) {
2221 flatIndex += indices[i] * stride;
2222 stride *= vecType.getDimSize(idx: i);
2223 }
2224 return fromElementsOp.getElements()[flatIndex];
2225}
2226
2227/// If the dynamic indices of `extractOp` or `insertOp` are in fact constants,
2228/// then fold it.
2229template <typename OpType, typename AdaptorType>
2230static Value extractInsertFoldConstantOp(OpType op, AdaptorType adaptor,
2231 SmallVectorImpl<Value> &operands) {
2232 std::vector<int64_t> staticPosition = op.getStaticPosition().vec();
2233 OperandRange dynamicPosition = op.getDynamicPosition();
2234 ArrayRef<Attribute> dynamicPositionAttr = adaptor.getDynamicPosition();
2235 ArrayRef<int64_t> vectorShape;
2236 if constexpr (std::is_same_v<OpType, ExtractOp>)
2237 vectorShape = op.getSourceVectorType().getShape();
2238 else
2239 vectorShape = op.getDestVectorType().getShape();
2240
2241 // If the dynamic operands is empty, it is returned directly.
2242 if (!dynamicPosition.size())
2243 return {};
2244
2245 // `index` is used to iterate over the `dynamicPosition`.
2246 unsigned index = 0;
2247
2248 // `opChange` is a flag. If it is true, it means to update `op` in place.
2249 bool opChange = false;
2250 for (unsigned i = 0, e = staticPosition.size(); i < e; ++i) {
2251 if (ShapedType::isStatic(dValue: staticPosition[i]))
2252 continue;
2253 Attribute positionAttr = dynamicPositionAttr[index];
2254 Value position = dynamicPosition[index++];
2255 if (auto attr = mlir::dyn_cast_if_present<IntegerAttr>(Val&: positionAttr)) {
2256 int64_t value = attr.getInt();
2257 // Do not fold if the value is out of bounds (-1 signifies a poison
2258 // value rather than OOB index).
2259 if (value >= -1 && value < vectorShape[i]) {
2260 staticPosition[i] = attr.getInt();
2261 opChange = true;
2262 continue;
2263 }
2264 }
2265 operands.push_back(Elt: position);
2266 }
2267
2268 if (opChange) {
2269 op.setStaticPosition(staticPosition);
2270 op.getOperation()->setOperands(operands);
2271 // Return the original result to indicate an in-place folding happened.
2272 return op.getResult();
2273 }
2274 return {};
2275}
2276
2277/// Fold an insert or extract operation into an poison value when a poison index
2278/// is found at any dimension of the static position.
2279static Attribute foldPoisonIndexInsertExtractOp(MLIRContext *context,
2280 ArrayRef<int64_t> staticPos,
2281 int64_t poisonVal) {
2282 if (!is_contained(Range&: staticPos, Element: poisonVal))
2283 return {};
2284
2285 return ub::PoisonAttr::get(ctx: context);
2286}
2287
2288/// Fold a vector extract from is a poison source.
2289static Attribute foldPoisonSrcExtractOp(Attribute srcAttr) {
2290 if (matchPattern(attr: srcAttr, pattern: ub::m_Poison()))
2291 return srcAttr;
2292
2293 return {};
2294}
2295
2296/// Fold a vector extract extracting from a DenseElementsAttr.
2297static Attribute foldDenseElementsAttrSrcExtractOp(ExtractOp extractOp,
2298 Attribute srcAttr) {
2299 auto denseAttr = dyn_cast_if_present<DenseElementsAttr>(Val&: srcAttr);
2300 if (!denseAttr) {
2301 return {};
2302 }
2303
2304 if (denseAttr.isSplat()) {
2305 Attribute newAttr = denseAttr.getSplatValue<Attribute>();
2306 if (auto vecDstType = dyn_cast<VectorType>(Val: extractOp.getType()))
2307 newAttr = DenseElementsAttr::get(type: vecDstType, values: newAttr);
2308 return newAttr;
2309 }
2310
2311 auto vecTy = cast<VectorType>(Val: extractOp.getSourceVectorType());
2312 if (vecTy.isScalable())
2313 return {};
2314
2315 if (extractOp.hasDynamicPosition()) {
2316 return {};
2317 }
2318
2319 // Materializing subsets of a large constant array can generally lead to
2320 // explosion in IR size because of different combination of subsets that
2321 // can exist. However, vector.extract is a restricted form of subset
2322 // extract where you can only extract non-overlapping (or the same) subset for
2323 // a given rank of the subset. Because of this property, the IR size can only
2324 // increase at most by `rank * size(array)` from a single constant array being
2325 // extracted by multiple extracts.
2326
2327 // Calculate the linearized position of the continuous chunk of elements to
2328 // extract.
2329 SmallVector<int64_t> completePositions(vecTy.getRank(), 0);
2330 copy(Range: extractOp.getStaticPosition(), Out: completePositions.begin());
2331 int64_t startPos =
2332 linearize(offsets: completePositions, basis: computeStrides(sizes: vecTy.getShape()));
2333 auto denseValuesBegin = denseAttr.value_begin<TypedAttr>() + startPos;
2334
2335 TypedAttr newAttr;
2336 if (auto resVecTy = dyn_cast<VectorType>(Val: extractOp.getType())) {
2337 SmallVector<Attribute> elementValues(
2338 denseValuesBegin, denseValuesBegin + resVecTy.getNumElements());
2339 newAttr = DenseElementsAttr::get(type: resVecTy, values: elementValues);
2340 } else {
2341 newAttr = *denseValuesBegin;
2342 }
2343
2344 return newAttr;
2345}
2346
2347OpFoldResult ExtractOp::fold(FoldAdaptor adaptor) {
2348 // Fold "vector.extract %v[] : vector<2x2xf32> from vector<2x2xf32>" to %v.
2349 // Note: Do not fold "vector.extract %v[] : f32 from vector<f32>" (type
2350 // mismatch).
2351 if (getNumIndices() == 0 && getSource().getType() == getResult().getType())
2352 return getSource();
2353 if (auto res = foldPoisonSrcExtractOp(srcAttr: adaptor.getSource()))
2354 return res;
2355 // Fold `arith.constant` indices into the `vector.extract` operation.
2356 // Do not stop here as this fold may enable subsequent folds that require
2357 // constant indices.
2358 SmallVector<Value> operands = {getSource()};
2359 auto inplaceFolded = extractInsertFoldConstantOp(op: *this, adaptor, operands);
2360
2361 if (auto res = foldPoisonIndexInsertExtractOp(
2362 context: getContext(), staticPos: adaptor.getStaticPosition(), poisonVal: kPoisonIndex))
2363 return res;
2364 if (auto res = foldDenseElementsAttrSrcExtractOp(extractOp: *this, srcAttr: adaptor.getSource()))
2365 return res;
2366 if (succeeded(Result: foldExtractOpFromExtractChain(extractOp: *this)))
2367 return getResult();
2368 if (auto res = ExtractFromInsertTransposeChainState(*this).fold())
2369 return res;
2370 if (auto res = foldExtractFromBroadcast(extractOp: *this))
2371 return res;
2372 if (auto res = foldExtractFromShuffle(extractOp: *this))
2373 return res;
2374 if (auto res = foldExtractFromShapeCast(extractOp: *this))
2375 return res;
2376 if (auto val = foldExtractFromExtractStrided(extractOp: *this))
2377 return val;
2378 if (auto val = foldExtractStridedOpFromInsertChain(extractOp: *this))
2379 return val;
2380 if (auto val = foldScalarExtractFromFromElements(extractOp: *this))
2381 return val;
2382
2383 return inplaceFolded;
2384}
2385
2386namespace {
2387
2388// Pattern to rewrite a ExtractOp(Broadcast) -> Broadcast.
2389class ExtractOpFromBroadcast final : public OpRewritePattern<ExtractOp> {
2390public:
2391 using Base::Base;
2392
2393 LogicalResult matchAndRewrite(ExtractOp extractOp,
2394 PatternRewriter &rewriter) const override {
2395
2396 Operation *defOp = extractOp.getSource().getDefiningOp();
2397 VectorType outType = dyn_cast<VectorType>(Val: extractOp.getType());
2398 if (!defOp || !isBroadcastLike(op: defOp) || !outType)
2399 return failure();
2400
2401 Value source = defOp->getOperand(idx: 0);
2402 if (isBroadcastableTo(srcType: source.getType(), dstVectorType: outType) !=
2403 BroadcastableToResult::Success)
2404 return failure();
2405
2406 rewriter.replaceOpWithNewOp<BroadcastOp>(op: extractOp, args&: outType, args&: source);
2407 return success();
2408 }
2409};
2410
2411// Pattern to rewrite a ExtractOp(CreateMask) -> CreateMask.
2412class ExtractOpFromCreateMask final : public OpRewritePattern<ExtractOp> {
2413public:
2414 using Base::Base;
2415
2416 LogicalResult matchAndRewrite(ExtractOp extractOp,
2417 PatternRewriter &rewriter) const override {
2418 auto createMaskOp =
2419 extractOp.getSource().getDefiningOp<vector::CreateMaskOp>();
2420 if (!createMaskOp)
2421 return failure();
2422
2423 VectorType extractedMaskType =
2424 llvm::dyn_cast<VectorType>(Val: extractOp.getResult().getType());
2425
2426 if (!extractedMaskType)
2427 return failure();
2428
2429 auto maskOperands = createMaskOp.getOperands();
2430 ArrayRef<int64_t> extractOpPos = extractOp.getStaticPosition();
2431 VectorType maskType = createMaskOp.getVectorType();
2432
2433 bool containsUnknownDims = false;
2434 bool allFalse = getMaskFormat(mask: createMaskOp) == MaskFormat::AllFalse;
2435
2436 for (size_t dimIdx = 0; !allFalse && dimIdx < extractOpPos.size();
2437 dimIdx++) {
2438 int64_t pos = extractOpPos[dimIdx];
2439 Value operand = maskOperands[dimIdx];
2440 auto constantOp = operand.getDefiningOp<arith::ConstantOp>();
2441 if (!constantOp) {
2442 // Bounds of this dim unknown.
2443 containsUnknownDims = true;
2444 continue;
2445 }
2446
2447 int64_t createMaskBound =
2448 llvm::cast<IntegerAttr>(Val: constantOp.getValue()).getInt();
2449
2450 if (pos != ShapedType::kDynamic) {
2451 // If any position is outside the range from the `create_mask`, then the
2452 // extracted mask will be all-false.
2453 allFalse |= pos >= createMaskBound;
2454 } else if (createMaskBound < maskType.getDimSize(idx: dimIdx)) {
2455 // This dim is not all-true and since this is a dynamic index we don't
2456 // know if the extraction is within the true or false region.
2457 // Note: Zero dims have already handled via getMaskFormat().
2458 containsUnknownDims = true;
2459 }
2460 }
2461
2462 if (allFalse) {
2463 rewriter.replaceOpWithNewOp<arith::ConstantOp>(
2464 op: extractOp, args: DenseElementsAttr::get(type: extractedMaskType, value: false));
2465 } else if (!containsUnknownDims) {
2466 rewriter.replaceOpWithNewOp<vector::CreateMaskOp>(
2467 op: extractOp, args&: extractedMaskType,
2468 args: maskOperands.drop_front(n: extractOpPos.size()));
2469 } else {
2470 return failure();
2471 }
2472 return success();
2473 }
2474};
2475
2476// Pattern to rewrite a ExtractOp(ConstantMask) -> ConstantMask.
2477class ExtractOpFromConstantMask final : public OpRewritePattern<ExtractOp> {
2478public:
2479 using Base::Base;
2480
2481 LogicalResult matchAndRewrite(ExtractOp extractOp,
2482 PatternRewriter &rewriter) const override {
2483 auto constantMaskOp =
2484 extractOp.getSource().getDefiningOp<vector::ConstantMaskOp>();
2485 if (!constantMaskOp)
2486 return failure();
2487
2488 Type resultType = extractOp.getResult().getType();
2489 auto extractedMaskType = dyn_cast<VectorType>(Val&: resultType);
2490
2491 ArrayRef<int64_t> extractOpPos = extractOp.getStaticPosition();
2492 ArrayRef<int64_t> maskDimSizes = constantMaskOp.getMaskDimSizes();
2493
2494 VectorType maskType = constantMaskOp.getVectorType();
2495
2496 // Check if any extracted position is outside the mask bounds.
2497 for (size_t dimIdx = 0; dimIdx < extractOpPos.size(); dimIdx++) {
2498 int64_t pos = extractOpPos[dimIdx];
2499 if (pos == ShapedType::kDynamic) {
2500 // If the dim is all-true, a dynamic index is fine — any position
2501 // is within the masked region.
2502 if (maskDimSizes[dimIdx] == maskType.getDimSize(idx: dimIdx))
2503 continue;
2504 // Otherwise we don't know if the position is inside or outside of
2505 // the masked area, so bail out.
2506 return failure();
2507 }
2508
2509 // If the position is statically outside of the masked area, the result
2510 // will be all-false.
2511 if (pos >= maskDimSizes[dimIdx]) {
2512 if (extractedMaskType) {
2513 rewriter.replaceOpWithNewOp<arith::ConstantOp>(
2514 op: extractOp, args: DenseElementsAttr::get(type: extractedMaskType, value: false));
2515 } else {
2516 rewriter.replaceOpWithNewOp<arith::ConstantOp>(
2517 op: extractOp, args: rewriter.getIntegerAttr(type: resultType, value: false));
2518 }
2519 return success();
2520 }
2521 }
2522
2523 // All positions are within the mask bounds.
2524 if (extractedMaskType) {
2525 // Vector result: the result is a constant_mask with the remaining
2526 // dimensions.
2527 rewriter.replaceOpWithNewOp<vector::ConstantMaskOp>(
2528 op: extractOp, args&: extractedMaskType,
2529 args: maskDimSizes.drop_front(N: extractOpPos.size()));
2530 } else {
2531 // Scalar result: all positions are within the masked region, so the
2532 // result is true.
2533 rewriter.replaceOpWithNewOp<arith::ConstantOp>(
2534 op: extractOp, args: rewriter.getIntegerAttr(type: resultType, value: true));
2535 }
2536 return success();
2537 }
2538};
2539
2540// Folds extract(shape_cast(..)) into shape_cast when the total element count
2541// does not change.
2542LogicalResult foldExtractFromShapeCastToShapeCast(ExtractOp extractOp,
2543 PatternRewriter &rewriter) {
2544 auto castOp = extractOp.getSource().getDefiningOp<ShapeCastOp>();
2545 if (!castOp)
2546 return failure();
2547
2548 VectorType sourceType = castOp.getSourceVectorType();
2549 auto targetType = dyn_cast<VectorType>(Val: extractOp.getResult().getType());
2550 if (!targetType)
2551 return failure();
2552
2553 if (sourceType.getNumElements() != targetType.getNumElements())
2554 return failure();
2555
2556 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: extractOp, args&: targetType,
2557 args: castOp.getSource());
2558 return success();
2559}
2560
2561/// Try to canonicalize the extraction of a subvector from a vector defined by
2562/// vector.from_elements. E.g.:
2563///
2564/// %0 = vector.from_elements %a, %b, %a, %a : vector<2x2xf32>
2565/// %1 = vector.extract %0[0] : vector<2xf32> from vector<2x2xf32>
2566/// ==> canonicalize to vector.from_elements %a, %b : vector<2xf32>
2567LogicalResult foldExtractFromFromElements(ExtractOp extractOp,
2568 PatternRewriter &rewriter) {
2569 // Dynamic positions are not supported.
2570 if (extractOp.hasDynamicPosition())
2571 return failure();
2572
2573 // Scalar extracts are handled by the folder.
2574 auto resultType = dyn_cast<VectorType>(Val: extractOp.getType());
2575 if (!resultType)
2576 return failure();
2577
2578 // Look for extracts from a from_elements op.
2579 auto fromElementsOp = extractOp.getSource().getDefiningOp<FromElementsOp>();
2580 if (!fromElementsOp)
2581 return failure();
2582 VectorType inputType = fromElementsOp.getType();
2583
2584 // Scalable vectors are not supported.
2585 if (resultType.isScalable() || inputType.isScalable())
2586 return failure();
2587
2588 // Compute the position of first extracted element and flatten/linearize the
2589 // position.
2590 SmallVector<int64_t> firstElementPos =
2591 llvm::to_vector(Range: extractOp.getStaticPosition());
2592 firstElementPos.append(/*NumInputs=*/resultType.getRank(), /*Elt=*/0);
2593 int flatIndex = 0;
2594 int stride = 1;
2595 for (int64_t i = inputType.getRank() - 1; i >= 0; --i) {
2596 flatIndex += firstElementPos[i] * stride;
2597 stride *= inputType.getDimSize(idx: i);
2598 }
2599
2600 // Replace the op with a smaller from_elements op.
2601 rewriter.replaceOpWithNewOp<FromElementsOp>(
2602 op: extractOp, args&: resultType,
2603 args: fromElementsOp.getElements().slice(n: flatIndex,
2604 m: resultType.getNumElements()));
2605 return success();
2606}
2607
2608/// Replace `vector.extract` with `vector.shape_cast`.
2609///
2610/// BEFORE:
2611/// %0 = vector.extract %arg0[0] : vector<4xf32> from vector<1x4xf32>
2612/// AFTER:
2613/// %0 = vector.shape_cast %arg0 : vector<1x4xf32> to vector<4xf32>
2614///
2615/// The canonical form of vector operations that reshape vectors is shape_cast.
2616struct ExtractToShapeCast final : OpRewritePattern<vector::ExtractOp> {
2617 using Base::Base;
2618 LogicalResult matchAndRewrite(vector::ExtractOp extractOp,
2619 PatternRewriter &rewriter) const override {
2620 VectorType sourceType = extractOp.getSourceVectorType();
2621 VectorType outType = dyn_cast<VectorType>(Val: extractOp.getType());
2622 if (!outType)
2623 return failure();
2624
2625 if (sourceType.getNumElements() != outType.getNumElements())
2626 return rewriter.notifyMatchFailure(
2627 arg&: extractOp, msg: "extract to vector with fewer elements");
2628
2629 // Negative values in `position` means that the extacted value is poison.
2630 // There is a vector.extract folder for this.
2631 if (llvm::any_of(Range: extractOp.getMixedPosition(),
2632 P: [](OpFoldResult v) { return !isConstantIntValue(ofr: v, value: 0); }))
2633 return rewriter.notifyMatchFailure(arg&: extractOp,
2634 msg: "leaving for extract poison folder");
2635
2636 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: extractOp, args&: outType,
2637 args: extractOp.getSource());
2638
2639 return success();
2640 }
2641};
2642
2643/// Folds vector.extract from vector.insert when the extract position is a
2644/// prefix of the insert position and the remaining (un-indexed) dimensions
2645/// of the extracted sub-vector are all size 1. In that case the extracted
2646/// value is fully determined by the inserted value.
2647///
2648/// Examples:
2649/// %ins = vector.insert %s, %v [3, 0] : f32 into vector<16x1xf32>
2650/// %ext = vector.extract %ins [3] : vector<1xf32> from vector<16x1xf32>
2651/// folds to:
2652/// %ext = vector.broadcast %s : f32 to vector<1xf32>
2653///
2654/// %ins = vector.insert %s, %v [0, 0] : vector<1xf32> into vector<16x1x1xf32>
2655// %ext = vector.extract %ins [0] : vector<1x1xf32> from vector<16x1x1xf32>
2656/// folds to:
2657/// %ext = vector.shape_cast %arg0 : vector<1xf32> to vector<1x1xf32>
2658struct FoldExtractFromInsertUnitDim final
2659 : OpRewritePattern<vector::ExtractOp> {
2660 using Base::Base;
2661
2662 LogicalResult matchAndRewrite(vector::ExtractOp extractOp,
2663 PatternRewriter &rewriter) const override {
2664 if (extractOp.hasDynamicPosition())
2665 return failure();
2666
2667 auto insertOp = extractOp.getSource().getDefiningOp<vector::InsertOp>();
2668 if (!insertOp || insertOp.hasDynamicPosition())
2669 return failure();
2670
2671 ArrayRef<int64_t> extractPos = extractOp.getStaticPosition();
2672 ArrayRef<int64_t> insertPos = insertOp.getStaticPosition();
2673
2674 // The extract position must be a strict prefix of the insert position.
2675 if (extractPos.size() >= insertPos.size() ||
2676 extractPos != insertPos.take_front(N: extractPos.size()))
2677 return failure();
2678
2679 // The remaining dimensions (those not indexed by the extract) must all
2680 // be size 1 in the source vector type. This guarantees that the inserted
2681 // value fully determines the extracted sub-vector.
2682 auto srcVecType = extractOp.getSourceVectorType();
2683 for (int64_t i = extractPos.size(), e = srcVecType.getRank(); i < e; ++i)
2684 if (srcVecType.getDimSize(idx: i) != 1)
2685 return failure();
2686
2687 Value inserted = insertOp.getValueToStore();
2688 Type extractedType = extractOp.getResult().getType();
2689 if (isa<VectorType>(Val: inserted.getType())) {
2690 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: extractOp, args&: extractedType,
2691 args&: inserted);
2692 } else {
2693 // The inserted value fully determines the extracted sub-vector; broadcast
2694 // it to the extracted type.
2695 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
2696 op: extractOp, args: extractOp.getResult().getType(),
2697 args: insertOp.getValueToStore());
2698 }
2699 return success();
2700 }
2701};
2702
2703} // namespace
2704
2705void ExtractOp::getCanonicalizationPatterns(RewritePatternSet &results,
2706 MLIRContext *context) {
2707 results.add<ExtractOpFromBroadcast, ExtractOpFromCreateMask,
2708 ExtractOpFromConstantMask, ExtractToShapeCast,
2709 FoldExtractFromInsertUnitDim>(arg&: context);
2710 results.add(implFn: foldExtractFromShapeCastToShapeCast);
2711 results.add(implFn: foldExtractFromFromElements);
2712}
2713
2714static void populateFromInt64AttrArray(ArrayAttr arrayAttr,
2715 SmallVectorImpl<int64_t> &results) {
2716 for (auto attr : arrayAttr)
2717 results.push_back(Elt: llvm::cast<IntegerAttr>(Val&: attr).getInt());
2718}
2719
2720//===----------------------------------------------------------------------===//
2721// FmaOp
2722//===----------------------------------------------------------------------===//
2723
2724//-----------------------------------------------------------------------------
2725// VectorUnrollOpInterface methods
2726//-----------------------------------------------------------------------------
2727std::optional<SmallVector<int64_t, 4>> FMAOp::getIterationShapeToUnroll() {
2728 return llvm::to_vector<4>(Range: getVectorType().getShape());
2729}
2730
2731//===----------------------------------------------------------------------===//
2732// ToElementsOp
2733//===----------------------------------------------------------------------===//
2734
2735/// Returns true if all the `operands` are defined by `defOp`.
2736/// Otherwise, returns false.
2737static bool haveSameDefiningOp(OperandRange operands, Operation *defOp) {
2738 if (operands.empty())
2739 return false;
2740
2741 return llvm::all_of(Range&: operands, P: [&](Value operand) {
2742 Operation *currentDef = operand.getDefiningOp();
2743 return currentDef == defOp;
2744 });
2745}
2746
2747/// Folds vector.to_elements(vector.from_elements(%e0, %e1, ...)) into
2748/// (%e0, %e1, ...). For example:
2749///
2750/// %0 = vector.from_elements %a, %b, %c : vector<3xf32>
2751/// %1:3 = vector.to_elements %0 : vector<3xf32>
2752/// user_op %1#0, %1#1, %1#2
2753///
2754/// becomes:
2755///
2756/// user_op %a, %b, %c
2757///
2758static LogicalResult
2759foldToElementsFromElements(ToElementsOp toElementsOp,
2760 SmallVectorImpl<OpFoldResult> &results) {
2761 auto fromElementsOp =
2762 toElementsOp.getSource().getDefiningOp<FromElementsOp>();
2763 if (!fromElementsOp)
2764 return failure();
2765
2766 llvm::append_range(C&: results, R: fromElementsOp.getElements());
2767 return success();
2768}
2769
2770/// Folds vector.to_elements(vector.broadcast(%x)) for the scalar case only.
2771///
2772/// Example:
2773/// %b = vector.broadcast %x : i32 to vector<3xf32>
2774/// %e:3 = vector.to_elements %b : vector<3xf32>
2775/// user_op %e#0, %e#1, %e#2
2776/// becomes:
2777/// user_op %x, %x, %x
2778///
2779/// The vector source case is handled by a canonicalization pattern.
2780static LogicalResult
2781foldToElementsOfBroadcast(ToElementsOp toElementsOp,
2782 SmallVectorImpl<OpFoldResult> &results) {
2783 auto bcastOp = toElementsOp.getSource().getDefiningOp<BroadcastOp>();
2784 if (!bcastOp)
2785 return failure();
2786 // Vectors are handled in the ToElementsOfBroadcast RewritePattern.
2787 if (isa<VectorType>(Val: bcastOp.getSource().getType()))
2788 return failure();
2789
2790 auto resultVecType = cast<VectorType>(Val: toElementsOp.getSource().getType());
2791
2792 Value scalar = bcastOp.getSource();
2793 results.assign(NumElts: resultVecType.getNumElements(), Elt: scalar);
2794 return success();
2795}
2796
2797LogicalResult ToElementsOp::fold(FoldAdaptor adaptor,
2798 SmallVectorImpl<OpFoldResult> &results) {
2799 if (succeeded(Result: foldToElementsFromElements(toElementsOp: *this, results)))
2800 return success();
2801
2802 // Y = ToElements(ShapeCast(X)) -> Y = ToElements(X)
2803 if (auto shapeCast = getSource().getDefiningOp<ShapeCastOp>()) {
2804 setOperand(shapeCast.getSource());
2805 return success();
2806 }
2807
2808 return foldToElementsOfBroadcast(toElementsOp: *this, results);
2809}
2810
2811LogicalResult
2812ToElementsOp::inferReturnTypes(MLIRContext *ctx, std::optional<Location> loc,
2813 ToElementsOp::Adaptor adaptor,
2814 SmallVectorImpl<Type> &inferredReturnTypes) {
2815 auto vecType = cast<VectorType>(Val: adaptor.getSource().getType());
2816 Type elType = vecType.getElementType();
2817 inferredReturnTypes.append(NumInputs: vecType.getNumElements(), Elt: elType);
2818 return success();
2819}
2820
2821/// Canonicalize `vector.to_elements(vector.broadcast(%v))` where `%v` is a
2822/// vector.
2823/// - Build `vector.to_elements %v` and remap each destination element to the
2824/// corresponding source element using broadcast rules (match or 1 →
2825/// replicate).
2826///
2827/// Example:
2828/// %v = vector.broadcast %src : vector<2xf32> to vector<3x2xf32>
2829/// %e:6 = vector.to_elements %v : vector<3x2xf32>
2830/// becomes:
2831/// %src_elems:2 = vector.to_elements %src : vector<2xf32>
2832/// // uses: %src_elems#0, %src_elems#1, %src_elems#0,
2833/// // %src_elems#1, %src_elems#0, %src_elems#1
2834struct ToElementsOfBroadcast final : OpRewritePattern<ToElementsOp> {
2835 using Base::Base;
2836
2837 LogicalResult matchAndRewrite(ToElementsOp toElementsOp,
2838 PatternRewriter &rewriter) const override {
2839 auto bcastOp = toElementsOp.getSource().getDefiningOp<BroadcastOp>();
2840 if (!bcastOp)
2841 return failure();
2842
2843 // Only handle broadcasts from a vector source here.
2844 auto srcType = dyn_cast<VectorType>(Val: bcastOp.getSource().getType());
2845 if (!srcType)
2846 return failure();
2847
2848 auto dstType = cast<VectorType>(Val: toElementsOp.getSource().getType());
2849
2850 ArrayRef<int64_t> dstShape = dstType.getShape();
2851 ArrayRef<int64_t> srcShape = srcType.getShape();
2852
2853 int64_t dstRank = dstShape.size();
2854 int64_t srcRank = srcShape.size();
2855
2856 // Create elements for the broadcast source vector.
2857 auto srcElems = vector::ToElementsOp::create(
2858 builder&: rewriter, location: toElementsOp.getLoc(), source: bcastOp.getSource());
2859
2860 int64_t dstCount = llvm::product_of(Range&: dstShape);
2861
2862 SmallVector<Value> replacements;
2863 replacements.reserve(N: dstCount);
2864
2865 // For each element of the destination, determine which element of the
2866 // source should be used. We walk all destination positions using a single
2867 // counter, decode it into per-dimension indices, then build the matching
2868 // source position: use the same index where sizes match, and use 0 where
2869 // the source size is 1 (replication). This mapping is needed so we can
2870 // replace each result of to_elements with the corresponding element from
2871 // the broadcast source.
2872 // Inner-dimension stretch example:
2873 // %v = vector.broadcast %src : vector<2x1x2xf32> to vector<2x3x2xf32>
2874 // %e:12 = vector.to_elements %v : vector<2x3x2xf32>
2875 // becomes:
2876 // %src_elems:4 = vector.to_elements %src : vector<2x1x2xf32>
2877 // // uses: %src_elems#0, %src_elems#1, %src_elems#0,
2878 // // %src_elems#1, %src_elems#0, %src_elems#1,
2879 // // %src_elems#2, %src_elems#3, %src_elems#2,
2880 // // %src_elems#3, %src_elems#2, %src_elems#3
2881
2882 // Row-major strides for the destination shape.
2883 SmallVector<int64_t> dstStrides = computeStrides(sizes: dstShape);
2884 // Row-major strides for the source shape.
2885 SmallVector<int64_t> srcStrides = computeStrides(sizes: srcShape);
2886 SmallVector<int64_t> dstIdx(dstRank);
2887 SmallVector<int64_t> srcIdx(srcRank);
2888 for (int64_t lin = 0; lin < dstCount; ++lin) {
2889 // Convert linear destination index to per-dimension indices.
2890 dstIdx = delinearize(linearIndex: lin, strides: dstStrides);
2891 for (int64_t k = 0; k < srcRank; ++k)
2892 srcIdx[k] = (srcShape[k] == 1) ? 0 : dstIdx[dstRank - srcRank + k];
2893 // Convert per-dimension source indices back to a linear index.
2894 int64_t srcLin = linearize(offsets: srcIdx, basis: srcStrides);
2895 replacements.push_back(Elt: srcElems.getResult(i: srcLin));
2896 }
2897
2898 rewriter.replaceOp(op: toElementsOp, newValues: replacements);
2899 return success();
2900 }
2901};
2902
2903void ToElementsOp::getCanonicalizationPatterns(RewritePatternSet &results,
2904 MLIRContext *context) {
2905 results.add<ToElementsOfBroadcast>(arg&: context);
2906}
2907
2908//===----------------------------------------------------------------------===//
2909// FromElementsOp
2910//===----------------------------------------------------------------------===//
2911
2912/// Folds vector.from_elements(vector.to_elements(%vector)) into %vector.
2913///
2914/// Case #1: Input and output vectors are the same.
2915///
2916/// %0:3 = vector.to_elements %a : vector<3xf32>
2917/// %1 = vector.from_elements %0#0, %0#1, %0#2 : vector<3xf32>
2918/// user_op %1
2919///
2920/// becomes:
2921///
2922/// user_op %a
2923///
2924static OpFoldResult foldFromElementsToElements(FromElementsOp fromElementsOp) {
2925 OperandRange fromElemsOperands = fromElementsOp.getElements();
2926 if (fromElemsOperands.empty())
2927 return {};
2928
2929 auto toElementsOp = fromElemsOperands[0].getDefiningOp<ToElementsOp>();
2930 if (!toElementsOp)
2931 return {};
2932
2933 if (!haveSameDefiningOp(operands: fromElemsOperands, defOp: toElementsOp))
2934 return {};
2935
2936 // Case #1: Input and output vectors are the same. Forward the input vector.
2937 Value toElementsInput = toElementsOp.getSource();
2938 if (fromElementsOp.getType() == toElementsInput.getType() &&
2939 llvm::equal(LRange&: fromElemsOperands, RRange: toElementsOp.getResults())) {
2940 return toElementsInput;
2941 }
2942
2943 // TODO: Support cases with different input and output shapes and different
2944 // number of elements.
2945
2946 return {};
2947}
2948
2949/// Fold vector.from_elements to a constant when all operands are constants.
2950/// Example:
2951/// %c1 = arith.constant 1 : i32
2952/// %c2 = arith.constant 2 : i32
2953/// %v = vector.from_elements %c1, %c2 : vector<2xi32>
2954/// =>
2955/// %v = arith.constant dense<[1, 2]> : vector<2xi32>
2956///
2957static OpFoldResult foldFromElementsToConstant(FromElementsOp fromElementsOp,
2958 ArrayRef<Attribute> elements) {
2959 // Check for null or poison attributes before any processing.
2960 if (llvm::any_of(Range&: elements, P: [](Attribute attr) {
2961 return !attr || matchPattern(attr, pattern: ub::m_Poison());
2962 }))
2963 return {};
2964
2965 // DenseElementsAttr only supports int/index/float/complex types.
2966 auto resVecType = fromElementsOp.getResult().getType();
2967 auto resEltType = resVecType.getElementType();
2968 if (!resEltType.isIntOrIndexOrFloat() && !isa<ComplexType>(Val: resEltType))
2969 return {};
2970
2971 // Constant attributes might have a different type than the return type.
2972 // Convert them before creating the dense elements attribute.
2973 auto convertedElements = llvm::map_to_vector(C&: elements, F: [&](Attribute attr) {
2974 return convertNumericAttr(attr, expectedType: resEltType);
2975 });
2976
2977 return DenseElementsAttr::get(type: resVecType, values: convertedElements);
2978}
2979
2980OpFoldResult FromElementsOp::fold(FoldAdaptor adaptor) {
2981 if (auto res = foldFromElementsToElements(fromElementsOp: *this))
2982 return res;
2983 if (auto res = foldFromElementsToConstant(fromElementsOp: *this, elements: adaptor.getElements()))
2984 return res;
2985
2986 return {};
2987}
2988
2989/// Rewrite vector.from_elements as vector.broadcast if the elements are the
2990/// same. Example:
2991/// %0 = vector.from_elements %a, %a, %a : vector<3xf32>
2992/// =>
2993/// %0 = vector.broadcast %a : f32 to vector<3xf32>
2994static LogicalResult
2995rewriteFromElementsAsBroadcast(FromElementsOp fromElementsOp,
2996 PatternRewriter &rewriter) {
2997 if (!llvm::all_equal(Range: fromElementsOp.getElements()))
2998 return failure();
2999 rewriter.replaceOpWithNewOp<BroadcastOp>(
3000 op: fromElementsOp, args: fromElementsOp.getType(),
3001 args: fromElementsOp.getElements().front());
3002 return success();
3003}
3004
3005/// Rewrite from_elements on multiple scalar extracts as a shape_cast
3006/// on a single extract. Example:
3007/// %0 = vector.extract %source[0, 0] : i8 from vector<2x2xi8>
3008/// %1 = vector.extract %source[0, 1] : i8 from vector<2x2xi8>
3009/// %2 = vector.from_elements %0, %1 : vector<2xi8>
3010///
3011/// becomes
3012/// %1 = vector.extract %source[0] : vector<1x2xi8> from vector<2x2xi8>
3013/// %2 = vector.shape_cast %1 : vector<1x2xi8> to vector<2xi8>
3014///
3015/// The requirements for this to be valid are
3016///
3017/// i) The elements are extracted from the same vector (%source).
3018///
3019/// ii) The elements form a suffix of %source. Specifically, the number
3020/// of elements is the same as the product of the last N dimension sizes
3021/// of %source, for some N.
3022///
3023/// iii) The elements are extracted contiguously in ascending order.
3024
3025class FromElementsToShapeCast : public OpRewritePattern<FromElementsOp> {
3026
3027 using Base::Base;
3028
3029 LogicalResult matchAndRewrite(FromElementsOp fromElements,
3030 PatternRewriter &rewriter) const override {
3031
3032 // Handled by `rewriteFromElementsAsBroadcast`.
3033 if (fromElements.getType().getNumElements() == 1)
3034 return failure();
3035
3036 // The common source that all elements are extracted from, if one exists.
3037 TypedValue<VectorType> source;
3038 // The position of the combined extract operation, if one is created.
3039 ArrayRef<int64_t> combinedPosition;
3040 // The expected index of extraction of the current element in the loop, if
3041 // elements are extracted contiguously in ascending order.
3042 SmallVector<int64_t> expectedPosition;
3043
3044 for (auto [insertIndex, element] :
3045 llvm::enumerate(First: fromElements.getElements())) {
3046
3047 // Check that the element is from a vector.extract operation.
3048 auto extractOp = element.getDefiningOp<vector::ExtractOp>();
3049 if (!extractOp) {
3050 return rewriter.notifyMatchFailure(arg&: fromElements,
3051 msg: "element not from vector.extract");
3052 }
3053
3054 // Check condition (i) by checking that all elements have the same source
3055 // as the first element.
3056 if (insertIndex == 0) {
3057 source = extractOp.getSource();
3058 } else if (extractOp.getSource() != source) {
3059 return rewriter.notifyMatchFailure(arg&: fromElements,
3060 msg: "element from different vector");
3061 }
3062
3063 ArrayRef<int64_t> position = extractOp.getStaticPosition();
3064 int64_t rank = position.size();
3065 assert(rank == source.getType().getRank() &&
3066 "scalar extract must have full rank position");
3067
3068 // Check condition (ii) by checking that the position that the first
3069 // element is extracted from has sufficient trailing 0s. For example, in
3070 //
3071 // %elm0 = vector.extract %source[1, 0, 0] : i8 from vector<2x3x4xi8>
3072 // [...]
3073 // %elms = vector.from_elements %elm0, [...] : vector<12xi8>
3074 //
3075 // The 2 trailing 0s in the position of extraction of %elm0 cover 3*4 = 12
3076 // elements, which is the number of elements of %n, so this is valid.
3077 if (insertIndex == 0) {
3078 const int64_t numElms = fromElements.getType().getNumElements();
3079 int64_t numSuffixElms = 1;
3080 int64_t index = rank;
3081 while (index > 0 && position[index - 1] == 0 &&
3082 numSuffixElms < numElms) {
3083 numSuffixElms *= source.getType().getDimSize(idx: index - 1);
3084 --index;
3085 }
3086 if (numSuffixElms != numElms) {
3087 return rewriter.notifyMatchFailure(
3088 arg&: fromElements, msg: "elements do not form a suffix of source");
3089 }
3090 expectedPosition = llvm::to_vector(Range&: position);
3091 combinedPosition = position.drop_back(N: rank - index);
3092 }
3093
3094 // Check condition (iii).
3095 else if (expectedPosition != position) {
3096 return rewriter.notifyMatchFailure(
3097 arg&: fromElements, msg: "elements not in ascending order (static order)");
3098 }
3099 increment(indices: expectedPosition, shape: source.getType().getShape());
3100 }
3101
3102 auto extracted = rewriter.createOrFold<vector::ExtractOp>(
3103 location: fromElements.getLoc(), args&: source, args&: combinedPosition);
3104
3105 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
3106 op: fromElements, args: fromElements.getType(), args&: extracted);
3107
3108 return success();
3109 }
3110
3111 /// Increments n-D `indices` by 1 starting from the innermost dimension.
3112 static void increment(MutableArrayRef<int64_t> indices,
3113 ArrayRef<int64_t> shape) {
3114 for (int dim : llvm::reverse(C: llvm::seq<int>(Begin: 0, End: indices.size()))) {
3115 indices[dim] += 1;
3116 if (indices[dim] < shape[dim])
3117 break;
3118 indices[dim] = 0;
3119 }
3120 }
3121};
3122
3123void FromElementsOp::getCanonicalizationPatterns(RewritePatternSet &results,
3124 MLIRContext *context) {
3125 results.add(implFn: rewriteFromElementsAsBroadcast);
3126 results.add<FromElementsToShapeCast>(arg&: context);
3127}
3128
3129//===----------------------------------------------------------------------===//
3130// BroadcastOp
3131//===----------------------------------------------------------------------===//
3132
3133void BroadcastOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges,
3134 SetIntRangeFn setResultRanges) {
3135 setResultRanges(getResult(), argRanges.front());
3136}
3137
3138//-----------------------------------------------------------------------------
3139// VectorUnrollOpInterface methods
3140//-----------------------------------------------------------------------------
3141std::optional<SmallVector<int64_t, 4>>
3142BroadcastOp::getIterationShapeToUnroll() {
3143 return llvm::to_vector<4>(Range: getResultVectorType().getShape());
3144}
3145
3146/// Return the dimensions of the result vector that were formerly ones in the
3147/// source tensor and thus correspond to "dim-1" broadcasting.
3148static llvm::SetVector<int64_t>
3149computeBroadcastedUnitDims(ArrayRef<int64_t> srcShape,
3150 ArrayRef<int64_t> dstShape) {
3151 int64_t rankDiff = dstShape.size() - srcShape.size();
3152 int64_t dstDim = rankDiff;
3153 llvm::SetVector<int64_t> res;
3154 for (auto [s1, s2] :
3155 llvm::zip_equal(t&: srcShape, u: dstShape.drop_front(N: rankDiff))) {
3156 if (s1 != s2) {
3157 assert(s1 == 1 && "expected \"dim-1\" broadcasting");
3158 res.insert(X: dstDim);
3159 }
3160 ++dstDim;
3161 }
3162 return res;
3163}
3164
3165llvm::SetVector<int64_t> BroadcastOp::computeBroadcastedUnitDims() {
3166 // Scalar broadcast is without any unit dim broadcast.
3167 auto srcVectorType = llvm::dyn_cast<VectorType>(Val: getSourceType());
3168 if (!srcVectorType)
3169 return {};
3170 return ::computeBroadcastedUnitDims(srcShape: srcVectorType.getShape(),
3171 dstShape: getResultVectorType().getShape());
3172}
3173
3174/// Broadcast `value` to a vector of `dstShape`, knowing that exactly the
3175/// `broadcastedDims` dimensions in the dstShape are broadcasted.
3176/// This requires (and asserts) that the broadcast is free of "dim-1"
3177/// broadcasting.
3178/// Since vector.broadcast only allows expanding leading dimensions, an extra
3179/// vector.transpose may be inserted to make the broadcast possible.
3180/// `value`, `dstShape` and `broadcastedDims` must be properly specified or
3181/// the helper will assert. This means:
3182/// 1. `dstShape` must not be empty.
3183/// 2. `broadcastedDims` must be confined to [0 .. rank(value.getVectorType)]
3184/// 2. `dstShape` trimmed of the dimensions specified in `broadcastedDims`
3185// must match the `value` shape.
3186Value BroadcastOp::createOrFoldBroadcastOp(
3187 OpBuilder &b, Value value, ArrayRef<int64_t> dstShape,
3188 const llvm::SetVector<int64_t> &broadcastedDims) {
3189 assert(!dstShape.empty() && "unexpected empty dst shape");
3190
3191 // Well-formedness check.
3192 SmallVector<int64_t> checkShape;
3193 for (int i = 0, e = dstShape.size(); i < e; ++i) {
3194 if (broadcastedDims.contains(key: i))
3195 continue;
3196 checkShape.push_back(Elt: dstShape[i]);
3197 }
3198 assert(broadcastedDims.size() == dstShape.size() - checkShape.size() &&
3199 "ill-formed broadcastedDims contains values not confined to "
3200 "destVectorShape");
3201
3202 Location loc = value.getLoc();
3203 Type elementType = getElementTypeOrSelf(type: value.getType());
3204 VectorType srcVectorType = llvm::dyn_cast<VectorType>(Val: value.getType());
3205 VectorType dstVectorType = VectorType::get(shape: dstShape, elementType);
3206
3207 // Step 2. If scalar -> dstShape broadcast, just do it.
3208 if (!srcVectorType) {
3209 assert(checkShape.empty() &&
3210 "ill-formed createOrFoldBroadcastOp arguments");
3211 return b.createOrFold<vector::BroadcastOp>(location: loc, args&: dstVectorType, args&: value);
3212 }
3213
3214 assert(srcVectorType.getShape().equals(checkShape) &&
3215 "ill-formed createOrFoldBroadcastOp arguments");
3216
3217 // Step 3. Since vector.broadcast only allows creating leading dims,
3218 // vector -> dstShape broadcast may require a transpose.
3219 // Traverse the dims in order and construct:
3220 // 1. The leading entries of the broadcastShape that is guaranteed to be
3221 // achievable by a simple broadcast.
3222 // 2. The induced permutation for the subsequent vector.transpose that will
3223 // bring us from `broadcastShape` back to he desired `dstShape`.
3224 // If the induced permutation is not the identity, create a vector.transpose.
3225 SmallVector<int64_t> broadcastShape, permutation(dstShape.size(), -1);
3226 broadcastShape.reserve(N: dstShape.size());
3227 // Consider the example:
3228 // srcShape = 2x4
3229 // dstShape = 1x2x3x4x5
3230 // broadcastedDims = [0, 2, 4]
3231 //
3232 // We want to build:
3233 // broadcastShape = 1x3x5x2x4
3234 // permutation = [0, 2, 4, 1, 3]
3235 // ---V--- -----V-----
3236 // leading broadcast part src shape part
3237 //
3238 // Note that the trailing dims of broadcastShape are exactly the srcShape
3239 // by construction.
3240 // nextSrcShapeDim is used to keep track of where in the permutation the
3241 // "src shape part" occurs.
3242 int64_t nextSrcShapeDim = broadcastedDims.size();
3243 for (int64_t i = 0, e = dstShape.size(); i < e; ++i) {
3244 if (broadcastedDims.contains(key: i)) {
3245 // 3.a. For each dim in the dst shape, if it is a broadcasted dim,
3246 // bring it to the head of the broadcastShape.
3247 // It will need to be permuted back from `broadcastShape.size() - 1` into
3248 // position `i`.
3249 broadcastShape.push_back(Elt: dstShape[i]);
3250 permutation[i] = broadcastShape.size() - 1;
3251 } else {
3252 // 3.b. Otherwise, the dim is not broadcasted, it comes from the src
3253 // shape and needs to be permuted into position `i`.
3254 // Don't touch `broadcastShape` here, the whole srcShape will be
3255 // appended after.
3256 permutation[i] = nextSrcShapeDim++;
3257 }
3258 }
3259 // 3.c. Append the srcShape.
3260 llvm::append_range(C&: broadcastShape, R: srcVectorType.getShape());
3261
3262 // Ensure there are no "dim-1" broadcasts.
3263 assert(::computeBroadcastedUnitDims(srcVectorType.getShape(), broadcastShape)
3264 .empty() &&
3265 "unexpected \"dim-1\" broadcast");
3266
3267 VectorType broadcastType = VectorType::get(shape: broadcastShape, elementType);
3268 assert(vector::isBroadcastableTo(value.getType(), broadcastType) ==
3269 vector::BroadcastableToResult::Success &&
3270 "must be broadcastable");
3271 Value res = b.createOrFold<vector::BroadcastOp>(location: loc, args&: broadcastType, args&: value);
3272 // Step 4. If we find any dimension that indeed needs to be permuted,
3273 // immediately return a new vector.transpose.
3274 for (int64_t i = 0, e = permutation.size(); i < e; ++i)
3275 if (permutation[i] != i)
3276 return b.createOrFold<vector::TransposeOp>(location: loc, args&: res, args&: permutation);
3277 // Otherwise return res.
3278 return res;
3279}
3280
3281LogicalResult BroadcastOp::verify() {
3282 std::pair<VectorDim, VectorDim> mismatchingDims;
3283 BroadcastableToResult res = isBroadcastableTo(
3284 srcType: getSourceType(), dstVectorType: getResultVectorType(), mismatchingDims: &mismatchingDims);
3285 if (res == BroadcastableToResult::Success)
3286 return success();
3287 if (res == BroadcastableToResult::SourceRankHigher)
3288 return emitOpError(message: "source rank higher than destination rank");
3289 if (res == BroadcastableToResult::DimensionMismatch) {
3290 return emitOpError(message: "dimension mismatch (")
3291 << (mismatchingDims.first.isScalable ? "[" : "")
3292 << mismatchingDims.first.dim
3293 << (mismatchingDims.first.isScalable ? "]" : "") << " vs. "
3294 << (mismatchingDims.second.isScalable ? "[" : "")
3295 << mismatchingDims.second.dim
3296 << (mismatchingDims.second.isScalable ? "]" : "") << ")";
3297 }
3298 if (res == BroadcastableToResult::SourceTypeNotAVector)
3299 return emitOpError(message: "source type is not a vector");
3300 llvm_unreachable("unexpected vector.broadcast op error");
3301}
3302
3303// Fold broadcast(shape_cast(x)) into broadcast(x) if x's type is compatible
3304// with broadcast's result type and shape_cast only adds or removes ones in the
3305// leading dimensions.
3306static LogicalResult foldBroadcastOfShapeCast(BroadcastOp broadcastOp) {
3307 auto srcShapeCast = broadcastOp.getSource().getDefiningOp<ShapeCastOp>();
3308 if (!srcShapeCast)
3309 return failure();
3310
3311 VectorType srcType = srcShapeCast.getSourceVectorType();
3312 VectorType destType = broadcastOp.getResultVectorType();
3313 // Check type compatibility.
3314 if (vector::isBroadcastableTo(srcType, dstVectorType: destType) !=
3315 BroadcastableToResult::Success)
3316 return failure();
3317
3318 ArrayRef<int64_t> srcShape = srcType.getShape();
3319 ArrayRef<int64_t> shapecastShape =
3320 srcShapeCast.getResultVectorType().getShape();
3321 // Trailing dimensions should be the same if shape_cast only alters the
3322 // leading dimensions.
3323 unsigned numTrailingDims = std::min(a: srcShape.size(), b: shapecastShape.size());
3324 if (!llvm::equal(LRange: srcShape.take_back(N: numTrailingDims),
3325 RRange: shapecastShape.take_back(N: numTrailingDims)))
3326 return failure();
3327
3328 assert(all_of(srcShape.drop_back(numTrailingDims),
3329 [](int64_t E) { return E == 1; }) &&
3330 all_of(shapecastShape.drop_back(numTrailingDims),
3331 [](int64_t E) { return E == 1; }) &&
3332 "ill-formed shape_cast");
3333
3334 broadcastOp.getSourceMutable().assign(value: srcShapeCast.getSource());
3335 return success();
3336}
3337
3338OpFoldResult BroadcastOp::fold(FoldAdaptor adaptor) {
3339 if (getSourceType() == getResultVectorType())
3340 return getSource();
3341 if (succeeded(Result: foldBroadcastOfShapeCast(broadcastOp: *this)))
3342 return getResult();
3343
3344 if (!adaptor.getSource())
3345 return {};
3346 auto vectorType = getResultVectorType();
3347 if (auto attr = llvm::dyn_cast<IntegerAttr>(Val: adaptor.getSource())) {
3348 if (vectorType.getElementType() != attr.getType())
3349 return {};
3350 return DenseElementsAttr::get(type: vectorType, values: attr);
3351 }
3352 if (auto attr = llvm::dyn_cast<FloatAttr>(Val: adaptor.getSource())) {
3353 if (vectorType.getElementType() != attr.getType())
3354 return {};
3355 return DenseElementsAttr::get(type: vectorType, values: attr);
3356 }
3357 if (auto attr = llvm::dyn_cast<SplatElementsAttr>(Val: adaptor.getSource()))
3358 return DenseElementsAttr::get(type: vectorType, values: attr.getSplatValue<Attribute>());
3359 if (matchPattern(attr: adaptor.getSource(), pattern: ub::m_Poison()))
3360 return ub::PoisonAttr::get(ctx: getContext());
3361 return {};
3362}
3363
3364namespace {
3365
3366// Fold broadcast1(broadcast2(x)) into broadcast1(x).
3367struct BroadcastFolder : public OpRewritePattern<BroadcastOp> {
3368 using Base::Base;
3369
3370 LogicalResult matchAndRewrite(BroadcastOp broadcastOp,
3371 PatternRewriter &rewriter) const override {
3372 auto srcBroadcast = broadcastOp.getSource().getDefiningOp<BroadcastOp>();
3373 if (!srcBroadcast)
3374 return failure();
3375 rewriter.replaceOpWithNewOp<BroadcastOp>(op: broadcastOp,
3376 args: broadcastOp.getResultVectorType(),
3377 args: srcBroadcast.getSource());
3378 return success();
3379 }
3380};
3381
3382/// Replace `vector.broadcast` with `vector.shape_cast`.
3383///
3384/// BEFORE:
3385/// %0 = vector.broadcast %arg0 : vector<4xi8> to vector<1x1x4xi8>
3386/// AFTER:
3387/// %0 = vector.shape_cast %arg0 : vector<4xi8> to vector<1x1x4xi8>
3388///
3389/// The canonical form of vector operations that reshape vectors is shape_cast.
3390struct BroadcastToShapeCast final
3391 : public OpRewritePattern<vector::BroadcastOp> {
3392 using Base::Base;
3393 LogicalResult matchAndRewrite(vector::BroadcastOp broadcast,
3394 PatternRewriter &rewriter) const override {
3395
3396 auto sourceType = dyn_cast<VectorType>(Val: broadcast.getSourceType());
3397 if (!sourceType) {
3398 return rewriter.notifyMatchFailure(
3399 arg&: broadcast, msg: "source is a scalar, shape_cast doesn't support scalar");
3400 }
3401
3402 VectorType outType = broadcast.getType();
3403 if (sourceType.getNumElements() != outType.getNumElements()) {
3404 return rewriter.notifyMatchFailure(
3405 arg&: broadcast, msg: "broadcast to a greater number of elements");
3406 }
3407
3408 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: broadcast, args&: outType,
3409 args: broadcast.getSource());
3410 return success();
3411 }
3412};
3413} // namespace
3414
3415void BroadcastOp::getCanonicalizationPatterns(RewritePatternSet &results,
3416 MLIRContext *context) {
3417 results.add<BroadcastFolder, BroadcastToShapeCast>(arg&: context);
3418}
3419
3420//===----------------------------------------------------------------------===//
3421// ShuffleOp
3422//===----------------------------------------------------------------------===//
3423
3424LogicalResult ShuffleOp::verify() {
3425 VectorType resultType = getResultVectorType();
3426 VectorType v1Type = getV1VectorType();
3427 VectorType v2Type = getV2VectorType();
3428 // Verify ranks.
3429 int64_t resRank = resultType.getRank();
3430 int64_t v1Rank = v1Type.getRank();
3431 int64_t v2Rank = v2Type.getRank();
3432 bool wellFormed0DCase = v1Rank == 0 && v2Rank == 0 && resRank == 1;
3433 bool wellFormedNDCase = v1Rank == resRank && v2Rank == resRank;
3434 if (!wellFormed0DCase && !wellFormedNDCase)
3435 return emitOpError(message: "rank mismatch");
3436
3437 // Verify all but leading dimension sizes.
3438 for (int64_t r = 1; r < v1Rank; ++r) {
3439 int64_t resDim = resultType.getDimSize(idx: r);
3440 int64_t v1Dim = v1Type.getDimSize(idx: r);
3441 int64_t v2Dim = v2Type.getDimSize(idx: r);
3442 if (resDim != v1Dim || v1Dim != v2Dim)
3443 return emitOpError(message: "dimension mismatch");
3444 }
3445 // Verify mask length.
3446 ArrayRef<int64_t> mask = getMask();
3447 int64_t maskLength = mask.size();
3448 if (maskLength <= 0)
3449 return emitOpError(message: "invalid mask length");
3450 if (maskLength != resultType.getDimSize(idx: 0))
3451 return emitOpError(message: "mask length mismatch");
3452 // Verify all indices.
3453 int64_t indexSize = (v1Type.getRank() == 0 ? 1 : v1Type.getDimSize(idx: 0)) +
3454 (v2Type.getRank() == 0 ? 1 : v2Type.getDimSize(idx: 0));
3455 for (auto [idx, maskPos] : llvm::enumerate(First&: mask)) {
3456 if (!isValidPositiveIndexOrPoison(index: maskPos, poisonValue: kPoisonIndex, maxIndex: indexSize))
3457 return emitOpError(message: "mask index #") << (idx + 1) << " out of range";
3458 }
3459 return success();
3460}
3461
3462LogicalResult
3463ShuffleOp::inferReturnTypes(MLIRContext *, std::optional<Location> loc,
3464 ShuffleOp::Adaptor adaptor,
3465 SmallVectorImpl<Type> &inferredReturnTypes) {
3466 auto v1Type = llvm::dyn_cast<VectorType>(Val: adaptor.getV1().getType());
3467 if (!v1Type) {
3468 return emitOptionalError(loc, args: "expected vector type");
3469 }
3470 auto v1Rank = v1Type.getRank();
3471 // Construct resulting type: leading dimension matches mask
3472 // length, all trailing dimensions match the operands.
3473 SmallVector<int64_t, 4> shape;
3474 shape.reserve(N: v1Rank);
3475 shape.push_back(Elt: std::max<size_t>(a: 1, b: adaptor.getMask().size()));
3476 // In the 0-D case there is no trailing shape to append.
3477 if (v1Rank > 0)
3478 llvm::append_range(C&: shape, R: v1Type.getShape().drop_front());
3479 inferredReturnTypes.push_back(
3480 Elt: VectorType::get(shape, elementType: v1Type.getElementType()));
3481 return success();
3482}
3483
3484template <typename T>
3485static bool isStepIndexArray(ArrayRef<T> idxArr, uint64_t begin, size_t width) {
3486 T expected = begin;
3487 return idxArr.size() == width && llvm::all_of(idxArr, [&expected](T value) {
3488 return value == expected++;
3489 });
3490}
3491
3492/// Fold shuffle V1, V2, [0, 1, 2, 3] : <4xi32>, <2xi32> -> V1.
3493/// Fold shuffle V1, V2, [4, 5] : <4xi32>, <2xi32> -> V2.
3494static OpFoldResult foldShuffleIdentityMask(ShuffleOp op) {
3495 auto v1Type = op.getV1VectorType();
3496 auto v2Type = op.getV2VectorType();
3497 auto mask = op.getMask();
3498 if (isStepIndexArray(idxArr: mask, begin: 0, width: v1Type.getDimSize(idx: 0)))
3499 return op.getV1();
3500 if (isStepIndexArray(idxArr: mask, begin: v1Type.getDimSize(idx: 0), width: v2Type.getDimSize(idx: 0)))
3501 return op.getV2();
3502 return {};
3503}
3504
3505/// If a shuffle operand is poison, replace all mask indices that reference it
3506/// with kPoisonIndex. This is an in-place fold.
3507static OpFoldResult foldShufflePoisonOperandToMask(ShuffleOp op) {
3508 bool isV1Poison = matchPattern(value: op.getV1(), pattern: ub::m_Poison());
3509 bool isV2Poison = matchPattern(value: op.getV2(), pattern: ub::m_Poison());
3510 if (!isV1Poison && !isV2Poison)
3511 return {};
3512
3513 int64_t v1Size = op.getV1VectorType().getDimSize(idx: 0);
3514 bool changed = false;
3515 SmallVector<int64_t> newMask = llvm::to_vector(Range: op.getMask());
3516 for (int64_t &idx : newMask) {
3517 if (idx == ShuffleOp::kPoisonIndex)
3518 continue;
3519 if ((isV1Poison && idx < v1Size) || (isV2Poison && idx >= v1Size)) {
3520 idx = ShuffleOp::kPoisonIndex;
3521 changed = true;
3522 }
3523 }
3524
3525 if (!changed)
3526 return {};
3527
3528 op.setMask(newMask);
3529 return op.getResult();
3530}
3531
3532/// Fold shuffle poison, poison -> poison.
3533static OpFoldResult foldShufflePoisonInputs(MLIRContext *context,
3534 Attribute v1Attr,
3535 Attribute v2Attr) {
3536 if (matchPattern(attr: v1Attr, pattern: ub::m_Poison()) &&
3537 matchPattern(attr: v2Attr, pattern: ub::m_Poison()))
3538 return ub::PoisonAttr::get(ctx: context);
3539 return {};
3540}
3541
3542/// Fold a shuffle of constant 1-D inputs by evaluating the mask.
3543static OpFoldResult foldShuffleConstantInputs(ShuffleOp op, Attribute v1Attr,
3544 Attribute v2Attr) {
3545 auto v1Type = op.getV1VectorType();
3546 if (v1Type.getRank() != 1)
3547 return {};
3548
3549 bool isV1Poison = matchPattern(attr: v1Attr, pattern: ub::m_Poison());
3550 bool isV2Poison = matchPattern(attr: v2Attr, pattern: ub::m_Poison());
3551
3552 // Poison input attributes need special handling as they are not
3553 // DenseElementsAttr. If an index is poison, we select the first element of
3554 // the first non-poison input.
3555 SmallVector<Attribute> v1Elements, v2Elements;
3556 Attribute poisonElement;
3557 if (!isV2Poison) {
3558 auto v2DenseAttr = dyn_cast<DenseElementsAttr>(Val&: v2Attr);
3559 if (!v2DenseAttr)
3560 return {};
3561 v2Elements = to_vector(Range: v2DenseAttr.getValues<Attribute>());
3562 poisonElement = v2Elements[0];
3563 }
3564 if (!isV1Poison) {
3565 auto v1DenseAttr = dyn_cast<DenseElementsAttr>(Val&: v1Attr);
3566 if (!v1DenseAttr)
3567 return {};
3568 v1Elements = to_vector(Range: v1DenseAttr.getValues<Attribute>());
3569 poisonElement = v1Elements[0];
3570 }
3571
3572 ArrayRef<int64_t> mask = op.getMask();
3573 SmallVector<Attribute> results;
3574 int64_t v1Size = v1Type.getDimSize(idx: 0);
3575 for (int64_t maskIdx : mask) {
3576 Attribute indexedElm;
3577 // TODO: Return a partial poison vector when supported by the UB dialect.
3578 if (maskIdx == ShuffleOp::kPoisonIndex) {
3579 indexedElm = poisonElement;
3580 } else {
3581 if (maskIdx < v1Size)
3582 indexedElm = isV1Poison ? poisonElement : v1Elements[maskIdx];
3583 else
3584 indexedElm = isV2Poison ? poisonElement : v2Elements[maskIdx - v1Size];
3585 }
3586
3587 results.push_back(Elt: indexedElm);
3588 }
3589
3590 return DenseElementsAttr::get(type: op.getResultVectorType(), values: results);
3591}
3592
3593OpFoldResult vector::ShuffleOp::fold(FoldAdaptor adaptor) {
3594 auto v1Type = getV1VectorType();
3595
3596 assert(!v1Type.isScalable() && !getV2VectorType().isScalable() &&
3597 "Vector shuffle does not support scalable vectors");
3598
3599 // For consistency: 0-D shuffle return type is 1-D, this cannot be a folding
3600 // but must be a canonicalization into a vector.broadcast.
3601 if (v1Type.getRank() == 0)
3602 return {};
3603
3604 if (auto res = foldShuffleIdentityMask(op: *this))
3605 return res;
3606 if (auto res = foldShufflePoisonOperandToMask(op: *this))
3607 return res;
3608
3609 Attribute v1Attr = adaptor.getV1(), v2Attr = adaptor.getV2();
3610 if (!v1Attr || !v2Attr)
3611 return {};
3612
3613 if (auto res = foldShufflePoisonInputs(context: getContext(), v1Attr, v2Attr))
3614 return res;
3615 if (auto res = foldShuffleConstantInputs(op: *this, v1Attr, v2Attr))
3616 return res;
3617
3618 return {};
3619}
3620
3621namespace {
3622
3623// Pattern to rewrite a 0-D shuffle with [0] or [1] mask returning a 1-D vector
3624// to a broadcast.
3625struct Canonicalize0DShuffleOp : public OpRewritePattern<ShuffleOp> {
3626 using Base::Base;
3627
3628 LogicalResult matchAndRewrite(ShuffleOp shuffleOp,
3629 PatternRewriter &rewriter) const override {
3630 VectorType v1VectorType = shuffleOp.getV1VectorType();
3631 ArrayRef<int64_t> mask = shuffleOp.getMask();
3632 if (v1VectorType.getRank() > 0)
3633 return failure();
3634 if (mask.size() != 1)
3635 return failure();
3636 VectorType resType = VectorType::Builder(v1VectorType).setShape(newShape: {1});
3637 if (mask[0] == 0)
3638 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: shuffleOp, args&: resType,
3639 args: shuffleOp.getV1());
3640 else
3641 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: shuffleOp, args&: resType,
3642 args: shuffleOp.getV2());
3643 return success();
3644 }
3645};
3646
3647/// Consider the defining operation `defOp` of `value`. If `defOp` is a
3648/// vector.broadcast with a scalar operand, return the scalar value that is
3649/// splatted. Otherwise return null.
3650///
3651/// Example:
3652///
3653/// scalar_source --> vector.broadcast --> value - return scalar_source
3654static Value getScalarSplatSource(Value value) {
3655 // Block argument:
3656 Operation *defOp = value.getDefiningOp();
3657 if (!defOp)
3658 return {};
3659
3660 auto broadcast = dyn_cast<vector::BroadcastOp>(Val: defOp);
3661
3662 // Not broadcast (and not splat):
3663 if (!broadcast)
3664 return {};
3665
3666 // Broadcast of a vector:
3667 if (isa<VectorType>(Val: broadcast.getSourceType()))
3668 return {};
3669
3670 // Broadcast of a scalar:
3671 return broadcast.getSource();
3672}
3673
3674/// Pattern to rewrite shuffle(splat-like(v), splat-like(v)) as broadcast(v).
3675class ShuffleSplat final : public OpRewritePattern<ShuffleOp> {
3676public:
3677 using Base::Base;
3678
3679 LogicalResult matchAndRewrite(ShuffleOp op,
3680 PatternRewriter &rewriter) const override {
3681 Value splat = getScalarSplatSource(value: op.getV1());
3682 if (!splat || getScalarSplatSource(value: op.getV2()) != splat)
3683 return failure();
3684
3685 rewriter.replaceOpWithNewOp<BroadcastOp>(op, args: op.getType(), args&: splat);
3686 return success();
3687 }
3688};
3689
3690/// Pattern to rewrite a fixed-size interleave via vector.shuffle to
3691/// vector.interleave.
3692class ShuffleInterleave : public OpRewritePattern<ShuffleOp> {
3693public:
3694 using Base::Base;
3695
3696 LogicalResult matchAndRewrite(ShuffleOp op,
3697 PatternRewriter &rewriter) const override {
3698 VectorType resultType = op.getResultVectorType();
3699 if (resultType.isScalable())
3700 return rewriter.notifyMatchFailure(
3701 arg&: op, msg: "ShuffleOp can't represent a scalable interleave");
3702
3703 if (resultType.getRank() != 1)
3704 return rewriter.notifyMatchFailure(
3705 arg&: op, msg: "ShuffleOp can't represent an n-D interleave");
3706
3707 VectorType sourceType = op.getV1VectorType();
3708 if (sourceType != op.getV2VectorType() ||
3709 sourceType.getNumElements() * 2 != resultType.getNumElements()) {
3710 return rewriter.notifyMatchFailure(
3711 arg&: op, msg: "ShuffleOp types don't match an interleave");
3712 }
3713
3714 ArrayRef<int64_t> shuffleMask = op.getMask();
3715 int64_t resultVectorSize = resultType.getNumElements();
3716 for (int i = 0, e = resultVectorSize / 2; i < e; ++i) {
3717 int64_t maskValueA = shuffleMask[i * 2];
3718 int64_t maskValueB = shuffleMask[(i * 2) + 1];
3719 if (maskValueA != i || maskValueB != (resultVectorSize / 2) + i)
3720 return rewriter.notifyMatchFailure(arg&: op,
3721 msg: "ShuffleOp mask not interleaving");
3722 }
3723
3724 rewriter.replaceOpWithNewOp<InterleaveOp>(op, args: op.getV1(), args: op.getV2());
3725 return success();
3726 }
3727};
3728
3729/// Pattern to replace usused shuffle operands / results with poison.
3730///
3731/// Example Input:
3732/// %r = vector.shuffle %v1, %v2 [2, 3, 3, 3] : vector<2xi32>, vector<2xi32>
3733///
3734/// Example Output:
3735/// %0 = ub.poison : vector<2xi32>
3736/// %r = vector.shuffle %0, %v2 [2, 3, 3, 3] : vector<2xi32>, vector<2xi32>
3737class FoldUnusedShuffleOperand final : public OpRewritePattern<ShuffleOp> {
3738public:
3739 using Base::Base;
3740
3741 LogicalResult matchAndRewrite(ShuffleOp op,
3742 PatternRewriter &rewriter) const override {
3743 // Replace with poison if all mask elements are poison.
3744 if (llvm::all_of(Range: op.getMask(), P: [](int64_t mask) {
3745 return mask == ShuffleOp::kPoisonIndex;
3746 })) {
3747 rewriter.replaceOpWithNewOp<ub::PoisonOp>(op, args: op.getType());
3748 return success();
3749 }
3750
3751 // Helper function to replace an operand with poison.
3752 auto replaceOperandWithPoison = [&](OpOperand &operand) {
3753 // Do not replace if the operand is already poison.
3754 if (!matchPattern(value: operand.get(), pattern: ub::m_Poison())) {
3755 Value poison = ub::PoisonOp::create(builder&: rewriter, location: op.getLoc(),
3756 result: operand.get().getType());
3757 rewriter.modifyOpInPlace(root: op, callable: [&]() { operand.set(poison); });
3758 return success();
3759 }
3760 return failure();
3761 };
3762
3763 // Replace V1 with poison if it is not used.
3764 int64_t leadingV1Size = op.getV1VectorType().getRank() > 0
3765 ? op.getV1VectorType().getDimSize(idx: 0)
3766 : 1;
3767 bool isV1Used = llvm::any_of(Range: op.getMask(), P: [&](int64_t mask) {
3768 return mask != ShuffleOp::kPoisonIndex && mask < leadingV1Size;
3769 });
3770 if (!isV1Used && succeeded(Result: replaceOperandWithPoison(op.getV1Mutable())))
3771 return success();
3772
3773 // Replace V2 with poison if it is not used.
3774 bool isV2Used = llvm::any_of(Range: op.getMask(), P: [&](int64_t mask) {
3775 return mask != ShuffleOp::kPoisonIndex && mask >= leadingV1Size;
3776 });
3777 if (!isV2Used && succeeded(Result: replaceOperandWithPoison(op.getV2Mutable())))
3778 return success();
3779
3780 return failure();
3781 }
3782};
3783} // namespace
3784
3785void ShuffleOp::getCanonicalizationPatterns(RewritePatternSet &results,
3786 MLIRContext *context) {
3787 results.add<ShuffleSplat, ShuffleInterleave, Canonicalize0DShuffleOp,
3788 FoldUnusedShuffleOperand>(arg&: context);
3789}
3790
3791//===----------------------------------------------------------------------===//
3792// InsertOp
3793//===----------------------------------------------------------------------===//
3794
3795void vector::InsertOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges,
3796 SetIntRangeFn setResultRanges) {
3797 setResultRanges(getResult(), argRanges[0].rangeUnion(other: argRanges[1]));
3798}
3799
3800void vector::InsertOp::build(OpBuilder &builder, OperationState &result,
3801 Value source, Value dest) {
3802 auto vectorTy = cast<VectorType>(Val: dest.getType());
3803 build(odsBuilder&: builder, odsState&: result, valueToStore: source, dest,
3804 position: SmallVector<int64_t>(vectorTy.getRank(), 0));
3805}
3806
3807void vector::InsertOp::build(OpBuilder &builder, OperationState &result,
3808 Value source, Value dest, int64_t position) {
3809 build(odsBuilder&: builder, odsState&: result, valueToStore: source, dest, position: ArrayRef<int64_t>{position});
3810}
3811
3812void vector::InsertOp::build(OpBuilder &builder, OperationState &result,
3813 Value source, Value dest, OpFoldResult position) {
3814 build(odsBuilder&: builder, odsState&: result, valueToStore: source, dest, position: ArrayRef<OpFoldResult>{position});
3815}
3816
3817void vector::InsertOp::build(OpBuilder &builder, OperationState &result,
3818 Value source, Value dest,
3819 ArrayRef<int64_t> position) {
3820 SmallVector<OpFoldResult> posVals;
3821 posVals.reserve(N: position.size());
3822 llvm::transform(Range&: position, d_first: std::back_inserter(x&: posVals),
3823 F: [&](int64_t pos) { return builder.getI64IntegerAttr(value: pos); });
3824 build(odsBuilder&: builder, odsState&: result, valueToStore: source, dest, position: posVals);
3825}
3826
3827void vector::InsertOp::build(OpBuilder &builder, OperationState &result,
3828 Value source, Value dest,
3829 ArrayRef<OpFoldResult> position) {
3830 SmallVector<int64_t> staticPos;
3831 SmallVector<Value> dynamicPos;
3832 dispatchIndexOpFoldResults(ofrs: position, dynamicVec&: dynamicPos, staticVec&: staticPos);
3833 build(odsBuilder&: builder, odsState&: result, valueToStore: source, dest, dynamic_position: dynamicPos,
3834 static_position: builder.getDenseI64ArrayAttr(values: staticPos));
3835}
3836
3837LogicalResult InsertOp::verify() {
3838 if (auto srcTy = dyn_cast<VectorType>(Val: getValueToStoreType()))
3839 if (srcTy.getRank() == 0)
3840 return emitError(
3841 message: "expected a scalar instead of a 0-d vector as the source operand");
3842
3843 SmallVector<OpFoldResult> position = getMixedPosition();
3844 auto destVectorType = getDestVectorType();
3845 if (position.size() > static_cast<unsigned>(destVectorType.getRank()))
3846 return emitOpError(
3847 message: "expected position attribute of rank no greater than dest vector rank");
3848 auto srcVectorType = llvm::dyn_cast<VectorType>(Val: getValueToStoreType());
3849 if (srcVectorType &&
3850 (static_cast<unsigned>(srcVectorType.getRank()) + position.size() !=
3851 static_cast<unsigned>(destVectorType.getRank())))
3852 return emitOpError(message: "expected position attribute rank + source rank to "
3853 "match dest vector rank");
3854 if (!srcVectorType &&
3855 (position.size() != static_cast<unsigned>(destVectorType.getRank())))
3856 return emitOpError(
3857 message: "expected position attribute rank to match the dest vector rank");
3858 for (auto [idx, pos] : llvm::enumerate(First&: position)) {
3859 if (auto attr = dyn_cast<Attribute>(Val&: pos)) {
3860 int64_t constIdx = cast<IntegerAttr>(Val&: attr).getInt();
3861 if (!isValidPositiveIndexOrPoison(index: constIdx, poisonValue: kPoisonIndex,
3862 maxIndex: destVectorType.getDimSize(idx))) {
3863 return emitOpError(message: "expected position attribute #")
3864 << (idx + 1)
3865 << " to be a non-negative integer smaller than the "
3866 "corresponding "
3867 "dest vector dimension";
3868 }
3869 }
3870 }
3871 return success();
3872}
3873
3874// Calculate the linearized position of the continuous chunk of elements to
3875// insert, based on the shape of the value to insert and the positions to insert
3876// at.
3877static int64_t calculateInsertPosition(VectorType destTy,
3878 ArrayRef<int64_t> positions) {
3879 llvm::SmallVector<int64_t> completePositions(destTy.getRank(), 0);
3880 assert(positions.size() <= completePositions.size() &&
3881 "positions size must be less than or equal to destTy rank");
3882 copy(Range&: positions, Out: completePositions.begin());
3883 return linearize(offsets: completePositions, basis: computeStrides(sizes: destTy.getShape()));
3884}
3885
3886namespace {
3887
3888// If insertOp is only inserting unit dimensions it can be transformed to a
3889// broadcast.
3890class InsertToBroadcast final : public OpRewritePattern<InsertOp> {
3891public:
3892 using Base::Base;
3893
3894 LogicalResult matchAndRewrite(InsertOp insertOp,
3895 PatternRewriter &rewriter) const override {
3896 auto srcVecType =
3897 llvm::dyn_cast<VectorType>(Val: insertOp.getValueToStoreType());
3898 if (!srcVecType || insertOp.getDestVectorType().getNumElements() !=
3899 srcVecType.getNumElements())
3900 return failure();
3901 rewriter.replaceOpWithNewOp<BroadcastOp>(
3902 op: insertOp, args: insertOp.getDestVectorType(), args: insertOp.getValueToStore());
3903 return success();
3904 }
3905};
3906
3907/// Pattern to rewrite a insert(splat-like(v), splat-like(v)) as broadcast(v).
3908class InsertSplatToSplat final : public OpRewritePattern<InsertOp> {
3909public:
3910 using Base::Base;
3911
3912 LogicalResult matchAndRewrite(InsertOp op,
3913 PatternRewriter &rewriter) const override {
3914
3915 Value splat = getScalarSplatSource(value: op.getValueToStore());
3916 if (!splat || getScalarSplatSource(value: op.getDest()) != splat)
3917 return failure();
3918
3919 rewriter.replaceOpWithNewOp<BroadcastOp>(op, args: op.getType(), args&: splat);
3920 return success();
3921 }
3922};
3923
3924/// Pattern to optimize a chain of insertions.
3925///
3926/// This pattern identifies chains of vector.insert operations that:
3927/// 1. Only insert values at static positions.
3928/// 2. Completely initialize all elements in the resulting vector.
3929/// 3. All intermediate insert operations have only one use.
3930///
3931/// When these conditions are met, the entire chain can be replaced with a
3932/// single vector.from_elements operation.
3933///
3934/// To keep this pattern simple, and avoid spending too much time on matching
3935/// fragmented insert chains, this pattern only considers the last insert op in
3936/// the chain.
3937///
3938/// Example transformation:
3939/// %poison = ub.poison : vector<2xi32>
3940/// %0 = vector.insert %c1, %poison[0] : i32 into vector<2xi32>
3941/// %1 = vector.insert %c2, %0[1] : i32 into vector<2xi32>
3942/// ->
3943/// %result = vector.from_elements %c1, %c2 : vector<2xi32>
3944class InsertChainFullyInitialized final : public OpRewritePattern<InsertOp> {
3945public:
3946 using Base::Base;
3947 LogicalResult matchAndRewrite(InsertOp op,
3948 PatternRewriter &rewriter) const override {
3949
3950 VectorType destTy = op.getDestVectorType();
3951 if (destTy.isScalable())
3952 return failure();
3953 // Ensure this is the trailing vector.insert op in a chain of inserts.
3954 for (Operation *user : op.getResult().getUsers())
3955 if (auto insertOp = dyn_cast<InsertOp>(Val: user))
3956 if (insertOp.getDest() == op.getResult())
3957 return failure();
3958
3959 InsertOp currentOp = op;
3960 SmallVector<InsertOp> chainInsertOps;
3961 while (currentOp) {
3962 // Check cond 1: Dynamic position is not supported.
3963 if (currentOp.hasDynamicPosition())
3964 return failure();
3965
3966 chainInsertOps.push_back(Elt: currentOp);
3967 currentOp = currentOp.getDest().getDefiningOp<InsertOp>();
3968 // Check cond 3: Intermediate inserts have only one use to avoid an
3969 // explosion of vectors.
3970 if (currentOp && !currentOp->hasOneUse())
3971 return failure();
3972 }
3973
3974 int64_t vectorSize = destTy.getNumElements();
3975 int64_t initializedCount = 0;
3976 SmallVector<bool> initializedDestIdxs(vectorSize, false);
3977 SmallVector<int64_t> pendingInsertPos;
3978 SmallVector<int64_t> pendingInsertSize;
3979 SmallVector<Value> pendingInsertValues;
3980
3981 for (auto insertOp : chainInsertOps) {
3982 // This pattern can do nothing with poison index.
3983 if (is_contained(Range: insertOp.getStaticPosition(), Element: InsertOp::kPoisonIndex))
3984 return failure();
3985
3986 // Calculate the linearized position for inserting elements.
3987 int64_t insertBeginPosition =
3988 calculateInsertPosition(destTy, positions: insertOp.getStaticPosition());
3989
3990 // The valueToStore operand may be a vector or a scalar. Need to handle
3991 // both cases.
3992 int64_t insertSize = 1;
3993 if (auto srcVectorType =
3994 llvm::dyn_cast<VectorType>(Val: insertOp.getValueToStoreType()))
3995 insertSize = srcVectorType.getNumElements();
3996
3997 assert(insertBeginPosition + insertSize <= vectorSize &&
3998 "insert would overflow the vector");
3999
4000 for (auto index : llvm::seq<int64_t>(Begin: insertBeginPosition,
4001 End: insertBeginPosition + insertSize)) {
4002 if (initializedDestIdxs[index])
4003 continue;
4004 initializedDestIdxs[index] = true;
4005 ++initializedCount;
4006 }
4007
4008 // Defer the creation of ops before we can make sure the pattern can
4009 // succeed.
4010 pendingInsertPos.push_back(Elt: insertBeginPosition);
4011 pendingInsertSize.push_back(Elt: insertSize);
4012 pendingInsertValues.push_back(Elt: insertOp.getValueToStore());
4013
4014 if (initializedCount == vectorSize)
4015 break;
4016 }
4017
4018 // Check cond 2: all positions must be initialized.
4019 if (initializedCount != vectorSize)
4020 return failure();
4021
4022 SmallVector<Value> elements(vectorSize);
4023 for (auto [insertBeginPosition, insertSize, valueToStore] :
4024 llvm::reverse(C: llvm::zip(t&: pendingInsertPos, u&: pendingInsertSize,
4025 args&: pendingInsertValues))) {
4026 auto srcVectorType = llvm::dyn_cast<VectorType>(Val: valueToStore.getType());
4027
4028 if (!srcVectorType) {
4029 elements[insertBeginPosition] = valueToStore;
4030 continue;
4031 }
4032
4033 Repeated<Type> elementToInsertTypes(insertSize,
4034 srcVectorType.getElementType());
4035 // Get all elements from the vector in row-major order.
4036 auto elementsToInsert = vector::ToElementsOp::create(
4037 builder&: rewriter, location: op.getLoc(), elements: elementToInsertTypes, source: valueToStore);
4038 for (int64_t linearIdx = 0; linearIdx < insertSize; linearIdx++) {
4039 elements[insertBeginPosition + linearIdx] =
4040 elementsToInsert.getResult(i: linearIdx);
4041 }
4042 }
4043
4044 rewriter.replaceOpWithNewOp<vector::FromElementsOp>(op, args&: destTy, args&: elements);
4045 return success();
4046 }
4047};
4048
4049} // namespace
4050
4051static Attribute
4052foldDenseElementsAttrDestInsertOp(InsertOp insertOp, Attribute srcAttr,
4053 Attribute dstAttr,
4054 int64_t maxVectorSizeFoldThreshold) {
4055 if (insertOp.hasDynamicPosition())
4056 return {};
4057
4058 auto denseDst = llvm::dyn_cast_if_present<DenseElementsAttr>(Val&: dstAttr);
4059 if (!denseDst)
4060 return {};
4061
4062 if (!srcAttr) {
4063 return {};
4064 }
4065
4066 VectorType destTy = insertOp.getDestVectorType();
4067 if (destTy.isScalable())
4068 return {};
4069
4070 // Make sure we do not create too many large constants.
4071 if (destTy.getNumElements() > maxVectorSizeFoldThreshold &&
4072 !insertOp->hasOneUse())
4073 return {};
4074
4075 // Bail out on poison indices (kPoisonIndex = -1) to avoid computing an
4076 // invalid (negative) linearized position which would cause UB below.
4077 if (is_contained(Range: insertOp.getStaticPosition(), Element: InsertOp::kPoisonIndex))
4078 return {};
4079
4080 // Calculate the linearized position for inserting elements.
4081 int64_t insertBeginPosition =
4082 calculateInsertPosition(destTy, positions: insertOp.getStaticPosition());
4083 SmallVector<Attribute> insertedValues;
4084 Type destEltType = destTy.getElementType();
4085
4086 /// Converts attribute to the expected type if there's
4087 /// a mismatch.
4088 if (auto denseSource = llvm::dyn_cast<DenseElementsAttr>(Val&: srcAttr)) {
4089 for (auto value : denseSource.getValues<Attribute>())
4090 insertedValues.push_back(Elt: convertNumericAttr(attr: value, expectedType: destEltType));
4091 } else {
4092 insertedValues.push_back(Elt: convertNumericAttr(attr: srcAttr, expectedType: destEltType));
4093 }
4094
4095 auto allValues = llvm::to_vector(Range: denseDst.getValues<Attribute>());
4096 copy(Range&: insertedValues, Out: allValues.begin() + insertBeginPosition);
4097 auto newAttr = DenseElementsAttr::get(type: destTy, values: allValues);
4098
4099 return newAttr;
4100}
4101
4102/// Folder to replace the `dest` operand of the insert op with the root dest of
4103/// the insert op use chain.
4104static Value foldInsertUseChain(InsertOp insertOp) {
4105 auto destInsert = insertOp.getDest().getDefiningOp<InsertOp>();
4106 if (!destInsert)
4107 return {};
4108
4109 if (insertOp.getMixedPosition() != destInsert.getMixedPosition())
4110 return {};
4111
4112 insertOp.setOperand(i: 1, value: destInsert.getDest());
4113 return insertOp.getResult();
4114}
4115
4116void InsertOp::getCanonicalizationPatterns(RewritePatternSet &results,
4117 MLIRContext *context) {
4118 results.add<InsertToBroadcast, BroadcastFolder, InsertSplatToSplat,
4119 InsertChainFullyInitialized>(arg&: context);
4120}
4121
4122OpFoldResult InsertOp::fold(FoldAdaptor adaptor) {
4123 // Do not create constants with more than `vectorSizeFoldThreashold` elements,
4124 // unless the source vector constant has a single use.
4125 constexpr int64_t vectorSizeFoldThreshold = 256;
4126 // Fold "vector.insert %v, %dest [] : vector<2x2xf32> from vector<2x2xf32>" to
4127 // %v. Note: Do not fold "vector.insert %v, %dest [] : f32 into vector<f32>"
4128 // (type mismatch).
4129 if (getNumIndices() == 0 && getValueToStoreType() == getType())
4130 return getValueToStore();
4131 // Fold `arith.constant` indices into the `vector.insert` operation.
4132 // Do not stop here as this fold may enable subsequent folds that require
4133 // constant indices.
4134 SmallVector<Value> operands = {getValueToStore(), getDest()};
4135 auto inplaceFolded = extractInsertFoldConstantOp(op: *this, adaptor, operands);
4136
4137 if (auto res = foldInsertUseChain(insertOp: *this))
4138 return res;
4139 if (auto res = foldPoisonIndexInsertExtractOp(
4140 context: getContext(), staticPos: adaptor.getStaticPosition(), poisonVal: kPoisonIndex))
4141 return res;
4142 if (auto res = foldDenseElementsAttrDestInsertOp(
4143 insertOp: *this, srcAttr: adaptor.getValueToStore(), dstAttr: adaptor.getDest(),
4144 maxVectorSizeFoldThreshold: vectorSizeFoldThreshold)) {
4145 return res;
4146 }
4147
4148 return inplaceFolded;
4149}
4150
4151//===----------------------------------------------------------------------===//
4152// InsertStridedSliceOp
4153//===----------------------------------------------------------------------===//
4154
4155void InsertStridedSliceOp::build(OpBuilder &builder, OperationState &result,
4156 Value source, Value dest,
4157 ArrayRef<int64_t> offsets,
4158 ArrayRef<int64_t> strides) {
4159 result.addOperands(newOperands: {source, dest});
4160 auto offsetsAttr = getVectorSubscriptAttr(builder, values: offsets);
4161 auto stridesAttr = getVectorSubscriptAttr(builder, values: strides);
4162 result.addTypes(newTypes: dest.getType());
4163 result.addAttribute(name: InsertStridedSliceOp::getOffsetsAttrName(name: result.name),
4164 attr: offsetsAttr);
4165 result.addAttribute(name: InsertStridedSliceOp::getStridesAttrName(name: result.name),
4166 attr: stridesAttr);
4167}
4168
4169// TODO: Should be moved to Tablegen ConfinedAttr attributes.
4170template <typename OpType>
4171static LogicalResult isIntegerArrayAttrSmallerThanShape(OpType op,
4172 ArrayAttr arrayAttr,
4173 ArrayRef<int64_t> shape,
4174 StringRef attrName) {
4175 if (arrayAttr.size() > shape.size())
4176 return op.emitOpError("expected ")
4177 << attrName << " attribute of rank no greater than vector rank";
4178 return success();
4179}
4180
4181// Returns true if all integers in `arrayAttr` are in the half-open [min, max}
4182// interval. If `halfOpen` is true then the admissible interval is [min, max).
4183// Otherwise, the admissible interval is [min, max].
4184template <typename OpType>
4185static LogicalResult
4186isIntegerArrayAttrConfinedToRange(OpType op, ArrayAttr arrayAttr, int64_t min,
4187 int64_t max, StringRef attrName,
4188 bool halfOpen = true) {
4189 for (auto attr : arrayAttr) {
4190 auto val = llvm::cast<IntegerAttr>(Val&: attr).getInt();
4191 auto upper = max;
4192 if (!halfOpen)
4193 upper += 1;
4194 if (val < min || val >= upper)
4195 return op.emitOpError("expected ") << attrName << " to be confined to ["
4196 << min << ", " << upper << ")";
4197 }
4198 return success();
4199}
4200
4201// Returns true if all integers in `arrayAttr` are in the half-open [min, max}
4202// interval. If `halfOpen` is true then the admissible interval is [min, max).
4203// Otherwise, the admissible interval is [min, max].
4204template <typename OpType>
4205static LogicalResult
4206isIntegerArrayAttrConfinedToShape(OpType op, ArrayAttr arrayAttr,
4207 ArrayRef<int64_t> shape, StringRef attrName,
4208 bool halfOpen = true, int64_t min = 0) {
4209 for (auto [index, attrDimPair] :
4210 llvm::enumerate(First: llvm::zip_first(t&: arrayAttr, u&: shape))) {
4211 int64_t val = llvm::cast<IntegerAttr>(Val: std::get<0>(t&: attrDimPair)).getInt();
4212 int64_t max = std::get<1>(t&: attrDimPair);
4213 if (!halfOpen)
4214 max += 1;
4215 if (val < min || val >= max)
4216 return op.emitOpError("expected ")
4217 << attrName << " dimension " << index << " to be confined to ["
4218 << min << ", " << max << ")";
4219 }
4220 return success();
4221}
4222
4223// Returns true if, for all indices i = 0..shape.size()-1, val is in the
4224// [min, max} interval:
4225// val = `arrayAttr1[i]` + `arrayAttr2[i]`,
4226// If `halfOpen` is true then the admissible interval is [min, max). Otherwise,
4227// the admissible interval is [min, max].
4228template <typename OpType>
4229static LogicalResult isSumOfIntegerArrayAttrConfinedToShape(
4230 OpType op, ArrayAttr arrayAttr1, ArrayAttr arrayAttr2,
4231 ArrayRef<int64_t> shape, StringRef attrName1, StringRef attrName2,
4232 bool halfOpen = true, int64_t min = 1) {
4233 assert(arrayAttr1.size() <= shape.size());
4234 assert(arrayAttr2.size() <= shape.size());
4235 for (auto [index, it] :
4236 llvm::enumerate(First: llvm::zip(t&: arrayAttr1, u&: arrayAttr2, args&: shape))) {
4237 auto val1 = llvm::cast<IntegerAttr>(Val: std::get<0>(t&: it)).getInt();
4238 auto val2 = llvm::cast<IntegerAttr>(Val: std::get<1>(t&: it)).getInt();
4239 int64_t max = std::get<2>(t&: it);
4240 if (!halfOpen)
4241 max += 1;
4242 if (val1 + val2 < 0 || val1 + val2 >= max)
4243 return op.emitOpError("expected sum(")
4244 << attrName1 << ", " << attrName2 << ") dimension " << index
4245 << " to be confined to [" << min << ", " << max << ")";
4246 }
4247 return success();
4248}
4249
4250static ArrayAttr makeI64ArrayAttr(ArrayRef<int64_t> values,
4251 MLIRContext *context) {
4252 auto attrs = llvm::map_range(C&: values, F: [context](int64_t v) -> Attribute {
4253 return IntegerAttr::get(type: IntegerType::get(context, width: 64), value: APInt(64, v));
4254 });
4255 return ArrayAttr::get(context, value: llvm::to_vector<8>(Range&: attrs));
4256}
4257
4258LogicalResult InsertStridedSliceOp::verify() {
4259 auto sourceVectorType = getSourceVectorType();
4260 auto destVectorType = getDestVectorType();
4261 auto offsets = getOffsetsAttr();
4262 auto strides = getStridesAttr();
4263 if (offsets.size() != static_cast<unsigned>(destVectorType.getRank()))
4264 return emitOpError(
4265 message: "expected offsets of same size as destination vector rank");
4266 if (strides.size() != static_cast<unsigned>(sourceVectorType.getRank()))
4267 return emitOpError(message: "expected strides of same size as source vector rank");
4268 if (sourceVectorType.getRank() > destVectorType.getRank())
4269 return emitOpError(
4270 message: "expected source rank to be no greater than destination rank");
4271
4272 auto sourceShape = sourceVectorType.getShape();
4273 auto destShape = destVectorType.getShape();
4274 SmallVector<int64_t, 4> sourceShapeAsDestShape(
4275 destShape.size() - sourceShape.size(), 0);
4276 sourceShapeAsDestShape.append(in_start: sourceShape.begin(), in_end: sourceShape.end());
4277 auto offName = InsertStridedSliceOp::getOffsetsAttrName();
4278 auto stridesName = InsertStridedSliceOp::getStridesAttrName();
4279 if (failed(Result: isIntegerArrayAttrConfinedToShape(op: *this, arrayAttr: offsets, shape: destShape,
4280 attrName: offName)) ||
4281 failed(Result: isIntegerArrayAttrConfinedToRange(op: *this, arrayAttr: strides, /*min=*/1,
4282 /*max=*/1, attrName: stridesName,
4283 /*halfOpen=*/false)) ||
4284 failed(Result: isSumOfIntegerArrayAttrConfinedToShape(
4285 op: *this, arrayAttr1: offsets,
4286 arrayAttr2: makeI64ArrayAttr(values: sourceShapeAsDestShape, context: getContext()), shape: destShape,
4287 attrName1: offName, attrName2: "source vector shape",
4288 /*halfOpen=*/false, /*min=*/1)))
4289 return failure();
4290
4291 unsigned rankDiff = destShape.size() - sourceShape.size();
4292 for (unsigned idx = 0; idx < sourceShape.size(); ++idx) {
4293 if (sourceVectorType.getScalableDims()[idx] !=
4294 destVectorType.getScalableDims()[idx + rankDiff]) {
4295 return emitOpError(message: "mismatching scalable flags (at source vector idx=")
4296 << idx << ")";
4297 }
4298 if (sourceVectorType.getScalableDims()[idx]) {
4299 auto sourceSize = sourceShape[idx];
4300 auto destSize = destShape[idx + rankDiff];
4301 if (sourceSize != destSize) {
4302 return emitOpError(message: "expected size at idx=")
4303 << idx
4304 << (" to match the corresponding base size from the input "
4305 "vector (")
4306 << sourceSize << (" vs ") << destSize << (")");
4307 }
4308 }
4309 }
4310
4311 return success();
4312}
4313
4314namespace {
4315/// Rewrite insert_strided_slice(splat-like(v), splat-like(v)) as v.
4316class FoldInsertStridedSliceSplat final
4317 : public OpRewritePattern<InsertStridedSliceOp> {
4318public:
4319 using Base::Base;
4320
4321 LogicalResult matchAndRewrite(InsertStridedSliceOp insertStridedSliceOp,
4322 PatternRewriter &rewriter) const override {
4323
4324 auto dst = insertStridedSliceOp.getDest();
4325 auto splat = getScalarSplatSource(value: insertStridedSliceOp.getValueToStore());
4326 if (!splat || getScalarSplatSource(value: dst) != splat)
4327 return failure();
4328
4329 rewriter.replaceOp(op: insertStridedSliceOp, newValues: dst);
4330 return success();
4331 }
4332};
4333
4334/// Pattern to rewrite an InsertStridedSliceOp(ExtractStridedSliceOp(dst), dst)
4335/// to dst.
4336class FoldInsertStridedSliceOfExtract final
4337 : public OpRewritePattern<InsertStridedSliceOp> {
4338public:
4339 using Base::Base;
4340
4341 LogicalResult matchAndRewrite(InsertStridedSliceOp insertStridedSliceOp,
4342 PatternRewriter &rewriter) const override {
4343 auto extractStridedSliceOp =
4344 insertStridedSliceOp.getValueToStore()
4345 .getDefiningOp<vector::ExtractStridedSliceOp>();
4346
4347 if (!extractStridedSliceOp)
4348 return failure();
4349
4350 if (extractStridedSliceOp.getOperand() != insertStridedSliceOp.getDest())
4351 return failure();
4352
4353 // Check if have the same strides and offsets.
4354 if (extractStridedSliceOp.getStrides() !=
4355 insertStridedSliceOp.getStrides() ||
4356 extractStridedSliceOp.getOffsets() != insertStridedSliceOp.getOffsets())
4357 return failure();
4358
4359 rewriter.replaceOp(op: insertStridedSliceOp, newValues: insertStridedSliceOp.getDest());
4360 return success();
4361 }
4362};
4363
4364// Pattern to rewrite an InsertStridedSliceOp(ConstantOp into ConstantOp) ->
4365// ConstantOp.
4366class InsertStridedSliceConstantFolder final
4367 : public OpRewritePattern<InsertStridedSliceOp> {
4368public:
4369 using Base::Base;
4370
4371 // Do not create constants with more than `vectorSizeFoldThreashold` elements,
4372 // unless the source vector constant has a single use.
4373 static constexpr int64_t vectorSizeFoldThreshold = 256;
4374
4375 LogicalResult matchAndRewrite(InsertStridedSliceOp op,
4376 PatternRewriter &rewriter) const override {
4377 // Return if 'InsertOp' operand is not defined by a compatible vector
4378 // ConstantOp.
4379 TypedValue<VectorType> destVector = op.getDest();
4380 Attribute vectorDestCst;
4381 if (!matchPattern(value: destVector, pattern: m_Constant(bind_value: &vectorDestCst)))
4382 return failure();
4383
4384 VectorType destTy = destVector.getType();
4385 if (destTy.isScalable())
4386 return failure();
4387
4388 // Make sure we do not create too many large constants.
4389 if (destTy.getNumElements() > vectorSizeFoldThreshold &&
4390 !destVector.hasOneUse())
4391 return failure();
4392
4393 TypedValue<VectorType> sourceValue = op.getValueToStore();
4394 Attribute sourceCst;
4395 if (!matchPattern(value: sourceValue, pattern: m_Constant(bind_value: &sourceCst)))
4396 return failure();
4397
4398 // TODO: Support poison.
4399 if (matchPattern(attr: vectorDestCst, pattern: ub::m_Poison()) ||
4400 matchPattern(attr: sourceCst, pattern: ub::m_Poison()))
4401 return failure();
4402
4403 // TODO: Handle non-unit strides when they become available.
4404 if (op.hasNonUnitStrides())
4405 return failure();
4406
4407 VectorType sliceVecTy = sourceValue.getType();
4408 ArrayRef<int64_t> sliceShape = sliceVecTy.getShape();
4409 int64_t rankDifference = destTy.getRank() - sliceVecTy.getRank();
4410 SmallVector<int64_t, 4> offsets = getI64SubArray(arrayAttr: op.getOffsets());
4411 SmallVector<int64_t, 4> destStrides = computeStrides(sizes: destTy.getShape());
4412
4413 // Calcualte the destination element indices by enumerating all slice
4414 // positions within the destination and linearizing them. The enumeration
4415 // order is lexicographic which yields a sequence of monotonically
4416 // increasing linearized position indices.
4417 // Because the destination may have higher dimensionality then the slice,
4418 // we keep track of two overlapping sets of positions and offsets.
4419 auto denseDest = llvm::cast<DenseElementsAttr>(Val&: vectorDestCst);
4420 auto denseSlice = llvm::cast<DenseElementsAttr>(Val&: sourceCst);
4421 auto sliceValuesIt = denseSlice.value_begin<Attribute>();
4422 auto newValues = llvm::to_vector(Range: denseDest.getValues<Attribute>());
4423 SmallVector<int64_t> currDestPosition(offsets.begin(), offsets.end());
4424 MutableArrayRef<int64_t> currSlicePosition(
4425 currDestPosition.begin() + rankDifference, currDestPosition.end());
4426 ArrayRef<int64_t> sliceOffsets(offsets.begin() + rankDifference,
4427 offsets.end());
4428 do {
4429 int64_t linearizedPosition = linearize(offsets: currDestPosition, basis: destStrides);
4430 assert(linearizedPosition < destTy.getNumElements() && "Invalid index");
4431 assert(sliceValuesIt != denseSlice.value_end<Attribute>() &&
4432 "Invalid slice element");
4433 newValues[linearizedPosition] = *sliceValuesIt;
4434 ++sliceValuesIt;
4435 } while (succeeded(
4436 Result: incSlicePosition(position: currSlicePosition, shape: sliceShape, offsets: sliceOffsets)));
4437
4438 auto newAttr = DenseElementsAttr::get(type: destTy, values: newValues);
4439 rewriter.replaceOpWithNewOp<arith::ConstantOp>(op, args&: newAttr);
4440 return success();
4441 }
4442};
4443
4444} // namespace
4445
4446void vector::InsertStridedSliceOp::getCanonicalizationPatterns(
4447 RewritePatternSet &results, MLIRContext *context) {
4448 results.add<FoldInsertStridedSliceSplat, FoldInsertStridedSliceOfExtract,
4449 InsertStridedSliceConstantFolder>(arg&: context);
4450}
4451
4452OpFoldResult InsertStridedSliceOp::fold(FoldAdaptor adaptor) {
4453 if (getSourceVectorType() == getDestVectorType())
4454 return getValueToStore();
4455 return {};
4456}
4457
4458//===----------------------------------------------------------------------===//
4459// OuterProductOp
4460//===----------------------------------------------------------------------===//
4461
4462/// Build an op without mask, use the type of `acc` as the return type.
4463void OuterProductOp::build(OpBuilder &builder, OperationState &result,
4464 Value lhs, Value rhs, Value acc) {
4465 result.addOperands(newOperands: {lhs, rhs, acc});
4466 result.addTypes(newTypes: acc.getType());
4467}
4468
4469void OuterProductOp::print(OpAsmPrinter &p) {
4470 p << " " << getLhs() << ", " << getRhs();
4471 if (getAcc()) {
4472 p << ", " << getAcc();
4473 SmallVector<NamedAttribute> attrs((*this)->getDiscardableAttrs());
4474 attrs.emplace_back(Args: getKindAttrName(), Args: getKindAttr());
4475 llvm::sort(C&: attrs);
4476 p.printOptionalAttrDict(attrs);
4477 }
4478 p << " : " << getLhs().getType() << ", " << getRhs().getType();
4479}
4480
4481ParseResult OuterProductOp::parse(OpAsmParser &parser, OperationState &result) {
4482 SmallVector<OpAsmParser::UnresolvedOperand, 3> operandsInfo;
4483 Type tLHS, tRHS;
4484 if (parser.parseOperandList(result&: operandsInfo) ||
4485 parser.parseOptionalAttrDict(result&: result.attributes) ||
4486 parser.parseColonType(result&: tLHS) || parser.parseComma() ||
4487 parser.parseType(result&: tRHS))
4488 return failure();
4489 if (operandsInfo.size() < 2)
4490 return parser.emitError(loc: parser.getNameLoc(),
4491 message: "expected at least 2 operands");
4492 VectorType vLHS = llvm::dyn_cast<VectorType>(Val&: tLHS);
4493 VectorType vRHS = llvm::dyn_cast<VectorType>(Val&: tRHS);
4494 if (!vLHS)
4495 return parser.emitError(loc: parser.getNameLoc(),
4496 message: "expected vector type for operand #1");
4497 // The result type is built below from dimension 0 of the operands, which a
4498 // 0-d vector does not have. Only that case has to be caught here; a higher
4499 // rank still reaches the verifier, which rejects it with the same wording.
4500 if (vLHS.getRank() == 0)
4501 return parser.emitError(loc: parser.getNameLoc(),
4502 message: "expected 1-d vector for operand #1");
4503 if (vRHS && vRHS.getRank() == 0)
4504 return parser.emitError(loc: parser.getNameLoc(),
4505 message: "expected 1-d vector for operand #2");
4506
4507 VectorType resType;
4508 if (vRHS) {
4509 SmallVector<bool> scalableDimsRes{vLHS.getScalableDims()[0],
4510 vRHS.getScalableDims()[0]};
4511 resType = VectorType::get(shape: {vLHS.getDimSize(idx: 0), vRHS.getDimSize(idx: 0)},
4512 elementType: vLHS.getElementType(), scalableDims: scalableDimsRes);
4513 } else {
4514 // Scalar RHS operand
4515 SmallVector<bool> scalableDimsRes{vLHS.getScalableDims()[0]};
4516 resType = VectorType::get(shape: {vLHS.getDimSize(idx: 0)}, elementType: vLHS.getElementType(),
4517 scalableDims: scalableDimsRes);
4518 }
4519
4520 if (!result.attributes.get(name: OuterProductOp::getKindAttrName(name: result.name))) {
4521 result.attributes.append(
4522 name: OuterProductOp::getKindAttrName(name: result.name),
4523 attr: CombiningKindAttr::get(context: result.getContext(),
4524 value: OuterProductOp::getDefaultKind()));
4525 }
4526
4527 return failure(
4528 IsFailure: parser.resolveOperand(operand: operandsInfo[0], type: tLHS, result&: result.operands) ||
4529 parser.resolveOperand(operand: operandsInfo[1], type: tRHS, result&: result.operands) ||
4530 (operandsInfo.size() > 2 &&
4531 parser.resolveOperand(operand: operandsInfo[2], type: resType, result&: result.operands)) ||
4532 parser.addTypeToList(type: resType, result&: result.types));
4533}
4534
4535LogicalResult OuterProductOp::verify() {
4536 Type tRHS = getOperandTypeRHS();
4537 VectorType vLHS = getOperandVectorTypeLHS(),
4538 vRHS = llvm::dyn_cast<VectorType>(Val&: tRHS),
4539 vACC = getOperandVectorTypeACC(), vRES = getResultVectorType();
4540
4541 if (vLHS.getRank() != 1)
4542 return emitOpError(message: "expected 1-d vector for operand #1");
4543
4544 if (vRHS) {
4545 // Proper OUTER operation.
4546 if (vRHS.getRank() != 1)
4547 return emitOpError(message: "expected 1-d vector for operand #2");
4548 if (vRES.getRank() != 2)
4549 return emitOpError(message: "expected 2-d vector result");
4550 if (vLHS.getDimSize(idx: 0) != vRES.getDimSize(idx: 0))
4551 return emitOpError(message: "expected #1 operand dim to match result dim #1");
4552 if (vRHS.getDimSize(idx: 0) != vRES.getDimSize(idx: 1))
4553 return emitOpError(message: "expected #2 operand dim to match result dim #2");
4554 if (vLHS.isScalable() && !vRHS.isScalable()) {
4555 // This restriction reflects what's currently supported in terms of
4556 // scalable vectors. However, we could relax this if there's a use case.
4557 return emitOpError(
4558 message: "expected either both or only #2 operand dim to be scalable");
4559 }
4560 } else {
4561 // An AXPY operation.
4562 if (vRES.getRank() != 1)
4563 return emitOpError(message: "expected 1-d vector result");
4564 if (vLHS.getDimSize(idx: 0) != vRES.getDimSize(idx: 0))
4565 return emitOpError(message: "expected #1 operand dim to match result dim #1");
4566 }
4567
4568 if (vACC && vACC != vRES)
4569 return emitOpError(message: "expected operand #3 of same type as result type");
4570
4571 if (!getKindAttr()) {
4572 return emitOpError(message: "expected 'kind' attribute of type CombiningKind (e.g. "
4573 "'vector.kind<add>')");
4574 }
4575
4576 // Verify supported combining kind.
4577 if (!isSupportedCombiningKind(combiningKind: getKind(), elementType: vRES.getElementType()))
4578 return emitOpError(message: "unsupported outerproduct type");
4579
4580 return success();
4581}
4582
4583// MaskableOpInterface methods.
4584
4585/// Returns the mask type expected by this operation. Mostly used for
4586/// verification purposes. It requires the operation to be vectorized."
4587Type OuterProductOp::getExpectedMaskType() {
4588 auto vecType = this->getResultVectorType();
4589 return VectorType::get(shape: vecType.getShape(),
4590 elementType: IntegerType::get(context: vecType.getContext(), /*width=*/1),
4591 scalableDims: vecType.getScalableDims());
4592}
4593
4594//===----------------------------------------------------------------------===//
4595// ExtractStridedSliceOp
4596//===----------------------------------------------------------------------===//
4597
4598// Inference works as follows:
4599// 1. Add 'sizes' from prefix of dims in 'offsets'.
4600// 2. Add sizes from 'vectorType' for remaining dims.
4601// Scalable flags are inherited from 'vectorType'.
4602static Type inferStridedSliceOpResultType(VectorType vectorType,
4603 ArrayAttr offsets, ArrayAttr sizes,
4604 ArrayAttr strides) {
4605 assert(offsets.size() == sizes.size() && offsets.size() == strides.size());
4606 SmallVector<int64_t, 4> shape;
4607 shape.reserve(N: vectorType.getRank());
4608 unsigned idx = 0;
4609 for (unsigned e = offsets.size(); idx < e; ++idx)
4610 shape.push_back(Elt: llvm::cast<IntegerAttr>(Val: sizes[idx]).getInt());
4611 for (unsigned e = vectorType.getShape().size(); idx < e; ++idx)
4612 shape.push_back(Elt: vectorType.getShape()[idx]);
4613
4614 return VectorType::get(shape, elementType: vectorType.getElementType(),
4615 scalableDims: vectorType.getScalableDims());
4616}
4617
4618void ExtractStridedSliceOp::build(OpBuilder &builder, OperationState &result,
4619 Value source, ArrayRef<int64_t> offsets,
4620 ArrayRef<int64_t> sizes,
4621 ArrayRef<int64_t> strides) {
4622 result.addOperands(newOperands: source);
4623 auto offsetsAttr = getVectorSubscriptAttr(builder, values: offsets);
4624 auto sizesAttr = getVectorSubscriptAttr(builder, values: sizes);
4625 auto stridesAttr = getVectorSubscriptAttr(builder, values: strides);
4626 result.addTypes(
4627 newTypes: inferStridedSliceOpResultType(vectorType: llvm::cast<VectorType>(Val: source.getType()),
4628 offsets: offsetsAttr, sizes: sizesAttr, strides: stridesAttr));
4629 result.addAttribute(name: ExtractStridedSliceOp::getOffsetsAttrName(name: result.name),
4630 attr: offsetsAttr);
4631 result.addAttribute(name: ExtractStridedSliceOp::getSizesAttrName(name: result.name),
4632 attr: sizesAttr);
4633 result.addAttribute(name: ExtractStridedSliceOp::getStridesAttrName(name: result.name),
4634 attr: stridesAttr);
4635}
4636
4637LogicalResult ExtractStridedSliceOp::verify() {
4638 auto type = getSourceVectorType();
4639 auto offsets = getOffsetsAttr();
4640 auto sizes = getSizesAttr();
4641 auto strides = getStridesAttr();
4642 if (offsets.size() != sizes.size() || offsets.size() != strides.size())
4643 return emitOpError(
4644 message: "expected offsets, sizes and strides attributes of same size");
4645
4646 auto shape = type.getShape();
4647 auto offName = getOffsetsAttrName();
4648 auto sizesName = getSizesAttrName();
4649 auto stridesName = getStridesAttrName();
4650 if (failed(
4651 Result: isIntegerArrayAttrSmallerThanShape(op: *this, arrayAttr: offsets, shape, attrName: offName)) ||
4652 failed(
4653 Result: isIntegerArrayAttrSmallerThanShape(op: *this, arrayAttr: sizes, shape, attrName: sizesName)) ||
4654 failed(Result: isIntegerArrayAttrSmallerThanShape(op: *this, arrayAttr: strides, shape,
4655 attrName: stridesName)) ||
4656 failed(
4657 Result: isIntegerArrayAttrConfinedToShape(op: *this, arrayAttr: offsets, shape, attrName: offName)) ||
4658 failed(Result: isIntegerArrayAttrConfinedToShape(op: *this, arrayAttr: sizes, shape, attrName: sizesName,
4659 /*halfOpen=*/false,
4660 /*min=*/1)) ||
4661 failed(Result: isIntegerArrayAttrConfinedToRange(op: *this, arrayAttr: strides, /*min=*/1,
4662 /*max=*/1, attrName: stridesName,
4663 /*halfOpen=*/false)) ||
4664 failed(Result: isSumOfIntegerArrayAttrConfinedToShape(op: *this, arrayAttr1: offsets, arrayAttr2: sizes,
4665 shape, attrName1: offName, attrName2: sizesName,
4666 /*halfOpen=*/false)))
4667 return failure();
4668
4669 auto resultType = inferStridedSliceOpResultType(vectorType: getSourceVectorType(),
4670 offsets, sizes, strides);
4671 if (getResult().getType() != resultType)
4672 return emitOpError(message: "expected result type to be ") << resultType;
4673
4674 for (unsigned idx = 0; idx < sizes.size(); ++idx) {
4675 if (type.getScalableDims()[idx]) {
4676 auto inputDim = type.getShape()[idx];
4677 auto inputSize = llvm::cast<IntegerAttr>(Val: sizes[idx]).getInt();
4678 if (inputDim != inputSize)
4679 return emitOpError(message: "expected size at idx=")
4680 << idx
4681 << (" to match the corresponding base size from the input "
4682 "vector (")
4683 << inputSize << (" vs ") << inputDim << (")");
4684 }
4685 }
4686
4687 return success();
4688}
4689
4690// When the source of ExtractStrided comes from a chain of InsertStrided ops try
4691// to use the source of the InsertStrided ops if we can detect that the
4692// extracted vector is a subset of one of the vector inserted.
4693static LogicalResult
4694foldExtractStridedOpFromInsertChain(ExtractStridedSliceOp op) {
4695 // Helper to extract integer out of ArrayAttr.
4696 auto getElement = [](ArrayAttr array, int idx) {
4697 return llvm::cast<IntegerAttr>(Val: array[idx]).getInt();
4698 };
4699 ArrayAttr extractOffsets = op.getOffsets();
4700 ArrayAttr extractStrides = op.getStrides();
4701 ArrayAttr extractSizes = op.getSizes();
4702 auto insertOp = op.getSource().getDefiningOp<InsertStridedSliceOp>();
4703 while (insertOp) {
4704 if (op.getSourceVectorType().getRank() !=
4705 insertOp.getSourceVectorType().getRank())
4706 return failure();
4707 ArrayAttr insertOffsets = insertOp.getOffsets();
4708 ArrayAttr insertStrides = insertOp.getStrides();
4709 // If the rank of extract is greater than the rank of insert, we are likely
4710 // extracting a partial chunk of the vector inserted.
4711 if (extractOffsets.size() > insertOffsets.size())
4712 return failure();
4713 bool patialoverlap = false;
4714 bool disjoint = false;
4715 SmallVector<int64_t, 4> offsetDiffs;
4716 for (unsigned dim = 0, e = extractOffsets.size(); dim < e; ++dim) {
4717 if (getElement(extractStrides, dim) != getElement(insertStrides, dim))
4718 return failure();
4719 int64_t start = getElement(insertOffsets, dim);
4720 int64_t end = start + insertOp.getSourceVectorType().getDimSize(idx: dim);
4721 int64_t offset = getElement(extractOffsets, dim);
4722 int64_t size = getElement(extractSizes, dim);
4723 // Check if the start of the extract offset is in the interval inserted.
4724 if (start <= offset && offset < end) {
4725 // If the extract interval overlaps but is not fully included we may
4726 // have a partial overlap that will prevent any folding.
4727 if (offset + size > end)
4728 patialoverlap = true;
4729 offsetDiffs.push_back(Elt: offset - start);
4730 continue;
4731 }
4732 disjoint = true;
4733 break;
4734 }
4735 // The extract element chunk is a subset of the insert element.
4736 if (!disjoint && !patialoverlap) {
4737 op.setOperand(insertOp.getValueToStore());
4738 // OpBuilder is only used as a helper to build an I64ArrayAttr.
4739 OpBuilder b(op.getContext());
4740 op.setOffsetsAttr(b.getI64ArrayAttr(values: offsetDiffs));
4741 return success();
4742 }
4743 // If the chunk extracted is disjoint from the chunk inserted, keep looking
4744 // in the insert chain.
4745 if (disjoint)
4746 insertOp = insertOp.getDest().getDefiningOp<InsertStridedSliceOp>();
4747 else {
4748 // The extracted vector partially overlap the inserted vector, we cannot
4749 // fold.
4750 return failure();
4751 }
4752 }
4753 return failure();
4754}
4755
4756// ExtractStridedSliceOp(non-splat ConstantOp) -> ConstantOp.
4757static OpFoldResult
4758foldExtractStridedSliceNonSplatConstant(ExtractStridedSliceOp op,
4759 Attribute foldInput) {
4760
4761 auto dense = llvm::dyn_cast_if_present<DenseElementsAttr>(Val&: foldInput);
4762 if (!dense)
4763 return {};
4764
4765 // TODO: Handle non-unit strides when they become available.
4766 if (op.hasNonUnitStrides())
4767 return {};
4768
4769 VectorType sourceVecTy = op.getSourceVectorType();
4770 ArrayRef<int64_t> sourceShape = sourceVecTy.getShape();
4771 SmallVector<int64_t, 4> sourceStrides = computeStrides(sizes: sourceShape);
4772
4773 VectorType sliceVecTy = op.getType();
4774 ArrayRef<int64_t> sliceShape = sliceVecTy.getShape();
4775 int64_t rank = sliceVecTy.getRank();
4776
4777 // Expand offsets and sizes to match the vector rank.
4778 SmallVector<int64_t, 4> offsets(rank, 0);
4779 copy(Range: getI64SubArray(arrayAttr: op.getOffsets()), Out: offsets.begin());
4780
4781 SmallVector<int64_t, 4> sizes(sourceShape);
4782 copy(Range: getI64SubArray(arrayAttr: op.getSizes()), Out: sizes.begin());
4783
4784 // Calculate the slice elements by enumerating all slice positions and
4785 // linearizing them. The enumeration order is lexicographic which yields a
4786 // sequence of monotonically increasing linearized position indices.
4787 const auto denseValuesBegin = dense.value_begin<Attribute>();
4788 SmallVector<Attribute> sliceValues;
4789 sliceValues.reserve(N: sliceVecTy.getNumElements());
4790 SmallVector<int64_t> currSlicePosition(offsets.begin(), offsets.end());
4791 do {
4792 int64_t linearizedPosition = linearize(offsets: currSlicePosition, basis: sourceStrides);
4793 assert(linearizedPosition < sourceVecTy.getNumElements() &&
4794 "Invalid index");
4795 sliceValues.push_back(Elt: *(denseValuesBegin + linearizedPosition));
4796 } while (succeeded(Result: incSlicePosition(position: currSlicePosition, shape: sliceShape, offsets)));
4797
4798 assert(static_cast<int64_t>(sliceValues.size()) ==
4799 sliceVecTy.getNumElements() &&
4800 "Invalid number of slice elements");
4801 return DenseElementsAttr::get(type: sliceVecTy, values: sliceValues);
4802}
4803
4804OpFoldResult ExtractStridedSliceOp::fold(FoldAdaptor adaptor) {
4805 if (getSourceVectorType() == getResult().getType())
4806 return getSource();
4807 if (succeeded(Result: foldExtractStridedOpFromInsertChain(op: *this)))
4808 return getResult();
4809
4810 // ExtractStridedSliceOp(splat ConstantOp) -> ConstantOp.
4811 if (auto splat =
4812 llvm::dyn_cast_if_present<SplatElementsAttr>(Val: adaptor.getSource()))
4813 return DenseElementsAttr::get(type: getType(), values: splat.getSplatValue<Attribute>());
4814
4815 // ExtractStridedSliceOp(non-splat ConstantOp) -> ConstantOp.
4816 return foldExtractStridedSliceNonSplatConstant(op: *this, foldInput: adaptor.getSource());
4817}
4818
4819void ExtractStridedSliceOp::getOffsets(SmallVectorImpl<int64_t> &results) {
4820 populateFromInt64AttrArray(arrayAttr: getOffsets(), results);
4821}
4822
4823namespace {
4824
4825// Pattern to rewrite nested ExtractStridedSliceOp into a single one.
4826//
4827// Example:
4828//
4829// %0 = vector.extract_strided_slice %arg0
4830// {offsets = [1, 2], sizes = [3, 4], strides = [1, 1]}
4831// : vector<4x8x16xf32> to vector<3x4x16xf32>
4832// %1 = vector.extract_strided_slice %0
4833// {offsets = [0, 1], sizes = [2, 2], strides = [1, 1]}
4834// : vector<3x4x16xf32> to vector<2x2x16xf32>
4835//
4836// to
4837//
4838// %1 = vector.extract_strided_slice %arg0
4839// {offsets = [1, 3], sizes = [2, 2], strides = [1, 1]}
4840// : vector<4x8x16xf32> to vector<2x2x16xf32>
4841class StridedSliceFolder final
4842 : public OpRewritePattern<ExtractStridedSliceOp> {
4843public:
4844 using OpRewritePattern<ExtractStridedSliceOp>::OpRewritePattern;
4845
4846 LogicalResult matchAndRewrite(ExtractStridedSliceOp secondOp,
4847 PatternRewriter &rewriter) const override {
4848 auto firstOp = secondOp.getSource().getDefiningOp<ExtractStridedSliceOp>();
4849 if (!firstOp)
4850 return failure();
4851
4852 if (secondOp.hasNonUnitStrides() || firstOp.hasNonUnitStrides())
4853 return failure();
4854
4855 SmallVector<int64_t> firstOffsets = getI64SubArray(arrayAttr: firstOp.getOffsets());
4856 SmallVector<int64_t> firstSizes = getI64SubArray(arrayAttr: firstOp.getSizes());
4857 SmallVector<int64_t> secondOffsets = getI64SubArray(arrayAttr: secondOp.getOffsets());
4858 SmallVector<int64_t> secondSizes = getI64SubArray(arrayAttr: secondOp.getSizes());
4859
4860 unsigned newRank = std::max(a: firstOffsets.size(), b: secondOffsets.size());
4861 SmallVector<int64_t> combinedOffsets(newRank, 0);
4862 SmallVector<int64_t> combinedSizes(newRank);
4863 ArrayRef<int64_t> firstSourceShape =
4864 firstOp.getSourceVectorType().getShape();
4865 for (unsigned i = 0; i < newRank; ++i) {
4866 int64_t off1 = (i < firstOffsets.size()) ? firstOffsets[i] : 0;
4867 int64_t off2 = (i < secondOffsets.size()) ? secondOffsets[i] : 0;
4868 combinedOffsets[i] = off1 + off2;
4869
4870 if (i < secondSizes.size()) {
4871 combinedSizes[i] = secondSizes[i];
4872 } else if (i < firstSizes.size()) {
4873 combinedSizes[i] = firstSizes[i];
4874 } else {
4875 combinedSizes[i] = firstSourceShape[i];
4876 }
4877 }
4878
4879 SmallVector<int64_t> combinedStrides(newRank, 1);
4880 rewriter.replaceOpWithNewOp<ExtractStridedSliceOp>(
4881 op: secondOp, args: firstOp.getSource(), args&: combinedOffsets, args&: combinedSizes,
4882 args&: combinedStrides);
4883 return success();
4884 }
4885};
4886
4887// Pattern to rewrite an ExtractStridedSliceOp(CreateMaskOp) to
4888// CreateMaskOp.
4889//
4890// Example:
4891//
4892// %mask = vector.create_mask %ub : vector<16xi1>
4893// %slice = vector.extract_strided_slice [%offset] [8] [1]
4894//
4895// to
4896//
4897// %new_ub = arith.subi %ub, %offset
4898// %mask = vector.create_mask %new_ub : vector<8xi1>
4899class StridedSliceCreateMaskFolder final
4900 : public OpRewritePattern<ExtractStridedSliceOp> {
4901 using Base::Base;
4902
4903public:
4904 LogicalResult matchAndRewrite(ExtractStridedSliceOp extractStridedSliceOp,
4905 PatternRewriter &rewriter) const override {
4906 Location loc = extractStridedSliceOp.getLoc();
4907 // Return if 'extractStridedSliceOp' operand is not defined by a
4908 // CreateMaskOp.
4909 auto createMaskOp =
4910 extractStridedSliceOp.getSource().getDefiningOp<CreateMaskOp>();
4911 if (!createMaskOp)
4912 return failure();
4913 // Return if 'extractStridedSliceOp' has non-unit strides.
4914 if (extractStridedSliceOp.hasNonUnitStrides())
4915 return failure();
4916 // Gather constant mask dimension sizes.
4917 SmallVector<Value> maskDimSizes(createMaskOp.getOperands());
4918 // Gather strided slice offsets and sizes.
4919 SmallVector<int64_t> sliceOffsets;
4920 populateFromInt64AttrArray(arrayAttr: extractStridedSliceOp.getOffsets(),
4921 results&: sliceOffsets);
4922 SmallVector<int64_t> sliceSizes;
4923 populateFromInt64AttrArray(arrayAttr: extractStridedSliceOp.getSizes(), results&: sliceSizes);
4924
4925 // Compute slice of vector mask region.
4926 SmallVector<Value> sliceMaskDimSizes;
4927 sliceMaskDimSizes.reserve(N: maskDimSizes.size());
4928 // sliceOffsets.size() <= maskDimSizes.size(), so we use llvm::zip and
4929 // only iterate on the leading dim sizes. The tail accounts for the
4930 // remaining dim sizes.
4931 for (auto [maskDimSize, sliceOffset, sliceSize] :
4932 llvm::zip(t&: maskDimSizes, u&: sliceOffsets, args&: sliceSizes)) {
4933 // No need to clamp on min/max values, because create_mask has clamping
4934 // semantics, i.e. the sliceMaskDimSize is allowed to be negative or
4935 // greater than the vector dim size.
4936 IntegerAttr offsetAttr =
4937 rewriter.getIntegerAttr(type: maskDimSize.getType(), value: sliceOffset);
4938 Value offset = arith::ConstantOp::create(builder&: rewriter, location: loc, value: offsetAttr);
4939 Value sliceMaskDimSize =
4940 arith::SubIOp::create(builder&: rewriter, location: loc, lhs: maskDimSize, rhs: offset);
4941 sliceMaskDimSizes.push_back(Elt: sliceMaskDimSize);
4942 }
4943 // Add unchanged dimensions.
4944 llvm::append_range(
4945 C&: sliceMaskDimSizes,
4946 R: llvm::drop_begin(RangeOrContainer&: maskDimSizes, N: sliceMaskDimSizes.size()));
4947 // Replace 'extractStridedSliceOp' with CreateMaskOp with sliced mask
4948 // region.
4949 rewriter.replaceOpWithNewOp<CreateMaskOp>(
4950 op: extractStridedSliceOp, args: extractStridedSliceOp.getResult().getType(),
4951 args&: sliceMaskDimSizes);
4952 return success();
4953 }
4954};
4955
4956// Pattern to rewrite an ExtractStridedSliceOp(ConstantMaskOp) to
4957// ConstantMaskOp.
4958class StridedSliceConstantMaskFolder final
4959 : public OpRewritePattern<ExtractStridedSliceOp> {
4960public:
4961 using Base::Base;
4962
4963 LogicalResult matchAndRewrite(ExtractStridedSliceOp extractStridedSliceOp,
4964 PatternRewriter &rewriter) const override {
4965 // Return if 'extractStridedSliceOp' operand is not defined by a
4966 // ConstantMaskOp.
4967 auto *defOp = extractStridedSliceOp.getSource().getDefiningOp();
4968 auto constantMaskOp = dyn_cast_or_null<ConstantMaskOp>(Val: defOp);
4969 if (!constantMaskOp)
4970 return failure();
4971 // Return if 'extractStridedSliceOp' has non-unit strides.
4972 if (extractStridedSliceOp.hasNonUnitStrides())
4973 return failure();
4974 // Gather constant mask dimension sizes.
4975 ArrayRef<int64_t> maskDimSizes = constantMaskOp.getMaskDimSizes();
4976 // Gather strided slice offsets and sizes.
4977 SmallVector<int64_t> sliceOffsets;
4978 populateFromInt64AttrArray(arrayAttr: extractStridedSliceOp.getOffsets(),
4979 results&: sliceOffsets);
4980 SmallVector<int64_t> sliceSizes;
4981 populateFromInt64AttrArray(arrayAttr: extractStridedSliceOp.getSizes(), results&: sliceSizes);
4982
4983 // Compute slice of vector mask region.
4984 SmallVector<int64_t> sliceMaskDimSizes;
4985 sliceMaskDimSizes.reserve(N: maskDimSizes.size());
4986 for (auto [maskDimSize, sliceOffset, sliceSize] :
4987 llvm::zip(t&: maskDimSizes, u&: sliceOffsets, args&: sliceSizes)) {
4988 int64_t sliceMaskDimSize = std::max(
4989 a: static_cast<int64_t>(0),
4990 b: std::min(a: sliceOffset + sliceSize, b: maskDimSize) - sliceOffset);
4991 sliceMaskDimSizes.push_back(Elt: sliceMaskDimSize);
4992 }
4993 // Add unchanged dimensions.
4994 if (sliceMaskDimSizes.size() < maskDimSizes.size())
4995 for (size_t i = sliceMaskDimSizes.size(); i < maskDimSizes.size(); ++i)
4996 sliceMaskDimSizes.push_back(Elt: maskDimSizes[i]);
4997 // If any of 'sliceMaskDimSizes' are zero, then set all to zero (masked
4998 // region is a conjunction of mask dim intervals).
4999 if (llvm::is_contained(Range&: sliceMaskDimSizes, Element: 0))
5000 sliceMaskDimSizes.assign(NumElts: maskDimSizes.size(), Elt: 0);
5001
5002 // Replace 'extractStridedSliceOp' with ConstantMaskOp with sliced mask
5003 // region.
5004 rewriter.replaceOpWithNewOp<ConstantMaskOp>(
5005 op: extractStridedSliceOp, args: extractStridedSliceOp.getResult().getType(),
5006 args&: sliceMaskDimSizes);
5007 return success();
5008 }
5009};
5010
5011// Pattern to rewrite an ExtractStridedSliceOp(BroadcastOp) to
5012// BroadcastOp(ExtractStrideSliceOp).
5013class StridedSliceBroadcast final
5014 : public OpRewritePattern<ExtractStridedSliceOp> {
5015public:
5016 using Base::Base;
5017
5018 LogicalResult matchAndRewrite(ExtractStridedSliceOp op,
5019 PatternRewriter &rewriter) const override {
5020 auto broadcast = op.getSource().getDefiningOp<BroadcastOp>();
5021 if (!broadcast)
5022 return failure();
5023 auto srcVecType =
5024 llvm::dyn_cast<VectorType>(Val: broadcast.getSource().getType());
5025 unsigned srcRank = srcVecType ? srcVecType.getRank() : 0;
5026 auto dstVecType = llvm::cast<VectorType>(Val: op.getType());
5027 unsigned dstRank = dstVecType.getRank();
5028 unsigned rankDiff = dstRank - srcRank;
5029 // Source dimensions can be broadcasted (1 -> n with n > 1) or sliced
5030 // (n -> m with n > m). If they are originally both broadcasted *and*
5031 // sliced, this can be simplified to just broadcasting.
5032 bool needsSlice = false;
5033 for (unsigned i = 0; i < srcRank; i++) {
5034 if (srcVecType.getDimSize(idx: i) != 1 &&
5035 srcVecType.getDimSize(idx: i) != dstVecType.getDimSize(idx: i + rankDiff)) {
5036 needsSlice = true;
5037 break;
5038 }
5039 }
5040 Value source = broadcast.getSource();
5041 if (needsSlice) {
5042 SmallVector<int64_t> offsets =
5043 getI64SubArray(arrayAttr: op.getOffsets(), /*dropFront=*/rankDiff);
5044 SmallVector<int64_t> sizes =
5045 getI64SubArray(arrayAttr: op.getSizes(), /*dropFront=*/rankDiff);
5046 // Slice attributes (i.e. offsets, sizes, etc) are only required for the
5047 // leading/prefix source dimensions. This is why we use
5048 // `offset.size()` instead of `srcRank` below.
5049 for (unsigned i = 0; i < offsets.size(); i++) {
5050 if (srcVecType.getDimSize(idx: i) == 1) {
5051 // In case this dimension was broadcasted *and* sliced, the offset
5052 // and size need to be updated now that there is no broadcast before
5053 // the slice.
5054 offsets[i] = 0;
5055 sizes[i] = 1;
5056 }
5057 }
5058 source = ExtractStridedSliceOp::create(
5059 builder&: rewriter, location: op->getLoc(), source, offsets, sizes,
5060 strides: getI64SubArray(arrayAttr: op.getStrides(), /*dropFront=*/rankDiff));
5061 }
5062 rewriter.replaceOpWithNewOp<BroadcastOp>(op, args: op.getType(), args&: source);
5063 return success();
5064 }
5065};
5066
5067/// Rewrite extract_strided_slice(splat-like(v)) with broadcast(v).
5068class StridedSliceSplat final : public OpRewritePattern<ExtractStridedSliceOp> {
5069public:
5070 using Base::Base;
5071
5072 LogicalResult matchAndRewrite(ExtractStridedSliceOp op,
5073 PatternRewriter &rewriter) const override {
5074
5075 Value splat = getScalarSplatSource(value: op.getSource());
5076 if (!splat)
5077 return failure();
5078 rewriter.replaceOpWithNewOp<BroadcastOp>(op, args: op.getType(), args&: splat);
5079 return success();
5080 }
5081};
5082
5083/// Pattern to rewrite simple cases of N-D extract_strided_slice, where the
5084/// slice is contiguous, into extract and shape_cast.
5085///
5086/// Example:
5087/// Before:
5088/// %1 = vector.extract_strided_slice %arg0 {
5089/// offsets = [0, 0, 0, 0, 0],
5090/// sizes = [1, 1, 1, 1, 8],
5091/// strides = [1, 1, 1, 1, 1]
5092/// } : vector<8x1x1x2x8xi8> to vector<1x1x1x1x8xi8>
5093/// After:
5094/// %0 = vector.extract %arg0[0, 0, 0, 0]
5095/// : vector<8xi8> from vector<8x1x1x2x8xi8>
5096/// %1 = vector.shape_cast %0
5097/// : vector<8xi8> to vector<1x1x1x1x8xi8>
5098///
5099class ContiguousExtractStridedSliceToExtract final
5100 : public OpRewritePattern<ExtractStridedSliceOp> {
5101public:
5102 using Base::Base;
5103
5104 LogicalResult matchAndRewrite(ExtractStridedSliceOp op,
5105 PatternRewriter &rewriter) const override {
5106 if (op.hasNonUnitStrides())
5107 return failure();
5108 Value source = op.getOperand();
5109 auto sourceType = cast<VectorType>(Val: source.getType());
5110 if (sourceType.isScalable() || sourceType.getRank() == 0)
5111 return failure();
5112
5113 // Compute the number of offsets to pass to ExtractOp::build. That is the
5114 // difference between the source rank and the desired slice rank. We walk
5115 // the dimensions from innermost out, and stop when the next slice dimension
5116 // is not full-size.
5117 SmallVector<int64_t> sizes = getI64SubArray(arrayAttr: op.getSizes());
5118 int numOffsets;
5119 for (numOffsets = sizes.size(); numOffsets > 0; --numOffsets) {
5120 if (sizes[numOffsets - 1] != sourceType.getDimSize(idx: numOffsets - 1))
5121 break;
5122 }
5123
5124 // If the created extract op would have no offsets, then this whole
5125 // extract_strided_slice is the identity and should have been handled by
5126 // other canonicalizations.
5127 if (numOffsets == 0)
5128 return failure();
5129
5130 // If not even the inner-most dimension is full-size, this op can't be
5131 // rewritten as an ExtractOp.
5132 if (numOffsets == sourceType.getRank() &&
5133 static_cast<int>(sizes.size()) == sourceType.getRank())
5134 return failure();
5135
5136 // The outer dimensions must have unit size.
5137 for (int i = 0; i < numOffsets; ++i) {
5138 if (sizes[i] != 1)
5139 return failure();
5140 }
5141
5142 // Avoid generating slices that have leading unit dimensions. The shape_cast
5143 // op that we create below would take bad generic fallback patterns
5144 // (ShapeCastOpRewritePattern).
5145 while (numOffsets < static_cast<int>(sizes.size()) - 1 &&
5146 sizes[numOffsets] == 1) {
5147 ++numOffsets;
5148 }
5149
5150 SmallVector<int64_t> offsets = getI64SubArray(arrayAttr: op.getOffsets());
5151 auto extractOffsets = ArrayRef(offsets).take_front(N: numOffsets);
5152 Value extract = vector::ExtractOp::create(builder&: rewriter, location: op->getLoc(), source,
5153 position: extractOffsets);
5154 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, args: op.getType(), args&: extract);
5155 return success();
5156 }
5157};
5158
5159} // namespace
5160
5161void ExtractStridedSliceOp::getCanonicalizationPatterns(
5162 RewritePatternSet &results, MLIRContext *context) {
5163 // Pattern to rewrite a ExtractStridedSliceOp(ConstantMaskOp) ->
5164 // ConstantMaskOp and ExtractStridedSliceOp(ConstantOp) -> ConstantOp.
5165 results.add<StridedSliceFolder, StridedSliceCreateMaskFolder,
5166 StridedSliceConstantMaskFolder, StridedSliceBroadcast,
5167 StridedSliceSplat, ContiguousExtractStridedSliceToExtract>(
5168 arg&: context);
5169}
5170
5171//===----------------------------------------------------------------------===//
5172// TransferReadOp
5173//===----------------------------------------------------------------------===//
5174
5175/// 1. Builder that sets padding to zero and an empty mask (variant with attrs).
5176/// If `padding` is null, a poison value is used.
5177void TransferReadOp::build(OpBuilder &builder, OperationState &result,
5178 VectorType vectorType, Value source,
5179 ValueRange indices, std::optional<Value> padding,
5180 AffineMapAttr permutationMapAttr,
5181 /*optional*/ ArrayAttr inBoundsAttr) {
5182
5183 Type elemType = llvm::cast<ShapedType>(Val: source.getType()).getElementType();
5184 if (!padding)
5185 padding = ub::PoisonOp::create(builder, location: result.location, result: elemType);
5186 // Delegate to the most general builder (see
5187 // `mlir/Dialect/Vector/IR/VectorOps.cpp.inc`)
5188 build(odsBuilder&: builder, odsState&: result, result: vectorType, base: source, indices, permutation_map: permutationMapAttr,
5189 padding: *padding, /*mask=*/Value(), in_bounds: inBoundsAttr);
5190}
5191
5192/// 2. Builder that sets padding to zero and an empty mask (variant without
5193/// attrs).
5194/// If `padding` is null, a poison value is used.
5195/// If `permutationMap` is null, a minor identity map is used.
5196/// If `inBounds` is null, an empty mask is used.
5197void TransferReadOp::build(OpBuilder &builder, OperationState &result,
5198 VectorType vectorType, Value source,
5199 ValueRange indices, std::optional<Value> padding,
5200 AffineMap permutationMap,
5201 std::optional<ArrayRef<bool>> inBounds) {
5202 if (!permutationMap)
5203 permutationMap = getTransferMinorIdentityMap(
5204 shapedType: llvm::cast<ShapedType>(Val: source.getType()), vectorType);
5205 auto permutationMapAttr = AffineMapAttr::get(value: permutationMap);
5206 auto inBoundsAttr = (inBounds && !inBounds.value().empty())
5207 ? builder.getBoolArrayAttr(values: inBounds.value())
5208 : builder.getBoolArrayAttr(
5209 values: SmallVector<bool>(vectorType.getRank(), false));
5210 // Delegate to Builder 1
5211 build(builder, result, vectorType, source, indices, padding,
5212 permutationMapAttr, inBoundsAttr);
5213}
5214
5215/// 3. Builder that sets permutation map to 'getMinorIdentityMap'.
5216/// If `padding` is null, a poison value is used.
5217/// If `inBounds` is null, an empty mask is used.
5218void TransferReadOp::build(OpBuilder &builder, OperationState &result,
5219 VectorType vectorType, Value source,
5220 ValueRange indices, std::optional<Value> padding,
5221 std::optional<ArrayRef<bool>> inBounds) {
5222 // Delegate to Builder 2
5223 build(builder, result, vectorType, source, indices, padding,
5224 /*permutationMap=*/AffineMap(), inBounds);
5225}
5226
5227template <typename EmitFun>
5228static LogicalResult verifyPermutationMap(AffineMap permutationMap,
5229 EmitFun emitOpError) {
5230 SmallVector<bool, 8> seen(permutationMap.getNumInputs(), false);
5231 for (auto expr : permutationMap.getResults()) {
5232 auto dim = dyn_cast<AffineDimExpr>(Val&: expr);
5233 auto zero = dyn_cast<AffineConstantExpr>(Val&: expr);
5234 if (zero) {
5235 if (zero.getValue() != 0) {
5236 return emitOpError(
5237 "requires a projected permutation_map (at most one dim or the zero "
5238 "constant can appear in each result)");
5239 }
5240 continue;
5241 }
5242 if (!dim) {
5243 return emitOpError("requires a projected permutation_map (at most one "
5244 "dim or the zero constant can appear in each result)");
5245 }
5246 if (seen[dim.getPosition()]) {
5247 return emitOpError(
5248 "requires a permutation_map that is a permutation (found one dim "
5249 "used more than once)");
5250 }
5251 seen[dim.getPosition()] = true;
5252 }
5253 return success();
5254}
5255
5256static LogicalResult
5257verifyTransferOp(VectorTransferOpInterface op, ShapedType shapedType,
5258 VectorType vectorType, VectorType maskType,
5259 VectorType inferredMaskType, AffineMap permutationMap,
5260 ArrayAttr inBounds) {
5261 if (op->hasDiscardableAttr(name: "masked")) {
5262 return op->emitOpError(message: "masked attribute has been removed. "
5263 "Use in_bounds instead.");
5264 }
5265
5266 if (!llvm::isa<MemRefType, RankedTensorType>(Val: shapedType))
5267 return op->emitOpError(
5268 message: "requires source to be a memref or ranked tensor type");
5269
5270 auto elementType = shapedType.getElementType();
5271 DataLayout dataLayout = DataLayout::closest(op);
5272 if (auto vectorElementType = llvm::dyn_cast<VectorType>(Val&: elementType)) {
5273 // Memref or tensor has vector element type.
5274 unsigned sourceVecSize =
5275 dataLayout.getTypeSizeInBits(t: vectorElementType.getElementType()) *
5276 vectorElementType.getShape().back();
5277 unsigned resultVecSize =
5278 dataLayout.getTypeSizeInBits(t: vectorType.getElementType()) *
5279 vectorType.getShape().back();
5280 if (resultVecSize % sourceVecSize != 0)
5281 return op->emitOpError(
5282 message: "requires the bitwidth of the minor 1-D vector to be an integral "
5283 "multiple of the bitwidth of the minor 1-D vector of the source");
5284
5285 unsigned sourceVecEltRank = vectorElementType.getRank();
5286 unsigned resultVecRank = vectorType.getRank();
5287 if (sourceVecEltRank > resultVecRank)
5288 return op->emitOpError(
5289 message: "requires source vector element and vector result ranks to match.");
5290 unsigned rankOffset = resultVecRank - sourceVecEltRank;
5291 // Check that permutation map results match 'rankOffset' of vector type.
5292 if (permutationMap.getNumResults() != rankOffset)
5293 return op->emitOpError(message: "requires a permutation_map with result dims of "
5294 "the same rank as the vector type");
5295
5296 if (maskType)
5297 return op->emitOpError(message: "does not support masks with vector element type");
5298 } else {
5299 // Memref or tensor has scalar element type.
5300 unsigned minorSize =
5301 vectorType.getRank() == 0 ? 1 : vectorType.getShape().back();
5302 unsigned resultVecSize =
5303 dataLayout.getTypeSizeInBits(t: vectorType.getElementType()) * minorSize;
5304 if (resultVecSize % dataLayout.getTypeSizeInBits(t: elementType) != 0)
5305 return op->emitOpError(
5306 message: "requires the bitwidth of the minor 1-D vector to be an integral "
5307 "multiple of the bitwidth of the source element type");
5308
5309 // Check that permutation map results match rank of vector type.
5310 if (permutationMap.getNumResults() != vectorType.getRank())
5311 return op->emitOpError(message: "requires a permutation_map with result dims of "
5312 "the same rank as the vector type");
5313 }
5314
5315 if (permutationMap.getNumSymbols() != 0)
5316 return op->emitOpError(message: "requires permutation_map without symbols");
5317
5318 if (permutationMap.getNumInputs() != shapedType.getRank())
5319 return op->emitOpError(message: "requires a permutation_map with input dims of the "
5320 "same rank as the source type");
5321
5322 if (maskType && maskType != inferredMaskType)
5323 return op->emitOpError(message: "inferred mask type (")
5324 << inferredMaskType << ") and mask operand type (" << maskType
5325 << ") don't match";
5326
5327 if (permutationMap.getNumResults() != static_cast<int64_t>(inBounds.size()))
5328 return op->emitOpError(message: "expects the in_bounds attr of same rank "
5329 "as permutation_map results: ")
5330 << AffineMapAttr::get(value: permutationMap)
5331 << " vs inBounds of size: " << inBounds.size();
5332
5333 return success();
5334}
5335
5336static void printTransferAttrs(OpAsmPrinter &p, VectorTransferOpInterface op) {
5337 SmallVector<NamedAttribute> attrs(op->getDiscardableAttrs());
5338 // Elide in_bounds attribute if all dims are out-of-bounds.
5339 if (llvm::any_of(Range: op.getInBoundsValues(), P: [](bool b) { return b; }))
5340 attrs.emplace_back(Args: op.getInBoundsAttrName(), Args: op.getInBounds());
5341 if (!op.getPermutationMap().isMinorIdentity())
5342 attrs.emplace_back(Args: op.getPermutationMapAttrName(),
5343 Args: AffineMapAttr::get(value: op.getPermutationMap()));
5344 llvm::sort(C&: attrs);
5345 p.printOptionalAttrDict(attrs);
5346}
5347
5348void TransferReadOp::print(OpAsmPrinter &p) {
5349 p << " " << getBase() << "[" << getIndices() << "], " << getPadding();
5350 if (getMask())
5351 p << ", " << getMask();
5352 printTransferAttrs(p, op: *this);
5353 p << " : " << getShapedType() << ", " << getVectorType();
5354}
5355
5356VectorType mlir::vector::inferTransferOpMaskType(VectorType vecType,
5357 AffineMap permMap) {
5358 auto i1Type = IntegerType::get(context: permMap.getContext(), width: 1);
5359 AffineMap invPermMap = inversePermutation(map: compressUnusedDims(map: permMap));
5360 assert(invPermMap && "Inversed permutation map couldn't be computed");
5361 SmallVector<int64_t, 8> maskShape = invPermMap.compose(values: vecType.getShape());
5362
5363 // The MaskOp specification doesn't support 0-D vectors at the moment. Turn a
5364 // 0-D mask into a single-element 1-D mask.
5365 if (maskShape.empty())
5366 maskShape.push_back(Elt: 1);
5367
5368 SmallVector<bool> scalableDims =
5369 applyPermutationMap(map: invPermMap, source: vecType.getScalableDims());
5370
5371 return VectorType::get(shape: maskShape, elementType: i1Type, scalableDims);
5372}
5373
5374ParseResult TransferReadOp::parse(OpAsmParser &parser, OperationState &result) {
5375 auto &builder = parser.getBuilder();
5376 SMLoc typesLoc;
5377 OpAsmParser::UnresolvedOperand sourceInfo;
5378 SmallVector<OpAsmParser::UnresolvedOperand, 8> indexInfo;
5379 OpAsmParser::UnresolvedOperand paddingInfo;
5380 SmallVector<Type, 2> types;
5381 OpAsmParser::UnresolvedOperand maskInfo;
5382 // Parsing with support for paddingValue.
5383 if (parser.parseOperand(result&: sourceInfo) ||
5384 parser.parseOperandList(result&: indexInfo, delimiter: OpAsmParser::Delimiter::Square) ||
5385 parser.parseComma() || parser.parseOperand(result&: paddingInfo))
5386 return failure();
5387 ParseResult hasMask = parser.parseOptionalComma();
5388 if (hasMask.succeeded()) {
5389 if (parser.parseOperand(result&: maskInfo))
5390 return failure();
5391 }
5392 if (parser.parseOptionalAttrDict(result&: result.attributes) ||
5393 parser.getCurrentLocation(loc: &typesLoc) || parser.parseColonTypeList(result&: types))
5394 return failure();
5395 if (types.size() != 2)
5396 return parser.emitError(loc: typesLoc, message: "requires two types");
5397 auto indexType = builder.getIndexType();
5398 auto shapedType = llvm::dyn_cast<ShapedType>(Val&: types[0]);
5399 if (!shapedType || !llvm::isa<MemRefType, RankedTensorType>(Val: shapedType))
5400 return parser.emitError(loc: typesLoc, message: "requires memref or ranked tensor type");
5401 VectorType vectorType = llvm::dyn_cast<VectorType>(Val&: types[1]);
5402 if (!vectorType)
5403 return parser.emitError(loc: typesLoc, message: "requires vector type");
5404 auto permMapAttrName = TransferReadOp::getPermutationMapAttrName(name: result.name);
5405 Attribute permMapAttr = result.attributes.get(name: permMapAttrName);
5406 AffineMap permMap;
5407 if (!permMapAttr) {
5408 if (shapedType.getRank() <
5409 getEffectiveVectorRankForXferOp(shapedType, vectorType))
5410 return parser.emitError(loc: typesLoc,
5411 message: "expected a custom permutation_map when "
5412 "rank(source) != rank(destination)");
5413 permMap = getTransferMinorIdentityMap(shapedType, vectorType);
5414 result.attributes.set(name: permMapAttrName, value: AffineMapAttr::get(value: permMap));
5415 } else {
5416 permMap = llvm::cast<AffineMapAttr>(Val&: permMapAttr).getValue();
5417 }
5418 auto inBoundsAttrName = TransferReadOp::getInBoundsAttrName(name: result.name);
5419 Attribute inBoundsAttr = result.attributes.get(name: inBoundsAttrName);
5420 if (!inBoundsAttr) {
5421 result.addAttribute(name: inBoundsAttrName,
5422 attr: builder.getBoolArrayAttr(
5423 values: SmallVector<bool>(permMap.getNumResults(), false)));
5424 }
5425 if (parser.resolveOperand(operand: sourceInfo, type: shapedType, result&: result.operands) ||
5426 parser.resolveOperands(operands&: indexInfo, type: indexType, result&: result.operands) ||
5427 parser.resolveOperand(operand: paddingInfo, type: shapedType.getElementType(),
5428 result&: result.operands))
5429 return failure();
5430 if (hasMask.succeeded()) {
5431 if (llvm::dyn_cast<VectorType>(Val: shapedType.getElementType()))
5432 return parser.emitError(
5433 loc: maskInfo.location, message: "does not support masks with vector element type");
5434 if (vectorType.getRank() != permMap.getNumResults()) {
5435 return parser.emitError(loc: typesLoc,
5436 message: "expected the same rank for the vector and the "
5437 "results of the permutation map");
5438 }
5439 // Instead of adding the mask type as an op type, compute it based on the
5440 // vector type and the permutation map (to keep the type signature small).
5441 auto maskType = inferTransferOpMaskType(vecType: vectorType, permMap);
5442 if (parser.resolveOperand(operand: maskInfo, type: maskType, result&: result.operands))
5443 return failure();
5444 }
5445 result.addAttribute(name: TransferReadOp::getOperandSegmentSizeAttr(),
5446 attr: builder.getDenseI32ArrayAttr(
5447 values: {1, static_cast<int32_t>(indexInfo.size()), 1,
5448 static_cast<int32_t>(hasMask.succeeded())}));
5449 return parser.addTypeToList(type: vectorType, result&: result.types);
5450}
5451
5452LogicalResult TransferReadOp::verify() {
5453 // Consistency of elemental types in source and vector.
5454 ShapedType shapedType = getShapedType();
5455 VectorType vectorType = getVectorType();
5456 VectorType maskType = getMaskType();
5457 auto paddingType = getPadding().getType();
5458 auto permutationMap = getPermutationMap();
5459 VectorType inferredMaskType =
5460 maskType ? inferTransferOpMaskType(vecType: vectorType, permMap: permutationMap)
5461 : VectorType();
5462 auto sourceElementType = shapedType.getElementType();
5463
5464 if (static_cast<int64_t>(getIndices().size()) != shapedType.getRank())
5465 return emitOpError(message: "requires ") << shapedType.getRank() << " indices";
5466
5467 if (failed(Result: verifyTransferOp(op: cast<VectorTransferOpInterface>(Val: getOperation()),
5468 shapedType, vectorType, maskType,
5469 inferredMaskType, permutationMap, inBounds: getInBounds())))
5470 return failure();
5471
5472 if (auto sourceVectorElementType =
5473 llvm::dyn_cast<VectorType>(Val&: sourceElementType)) {
5474 // Source has vector element type.
5475 // Check that 'sourceVectorElementType' and 'paddingType' types match.
5476 if (sourceVectorElementType != paddingType)
5477 return emitOpError(
5478 message: "requires source element type and padding type to match.");
5479
5480 } else {
5481 // Check that 'paddingType' is valid to store in a vector type.
5482 if (!VectorType::isValidElementType(t: paddingType))
5483 return emitOpError(message: "requires valid padding vector elemental type");
5484
5485 // Check that padding type and vector element types match.
5486 if (paddingType != sourceElementType)
5487 return emitOpError(
5488 message: "requires formal padding and source of the same elemental type");
5489 }
5490
5491 return verifyPermutationMap(permutationMap,
5492 emitOpError: [&](Twine t) { return emitOpError(message: t); });
5493}
5494
5495// MaskableOpInterface methods.
5496
5497/// Returns the mask type expected by this operation. Mostly used for
5498/// verification purposes. It requires the operation to be vectorized."
5499Type TransferReadOp::getExpectedMaskType() {
5500 return inferTransferOpMaskType(vecType: getVectorType(), permMap: getPermutationMap());
5501}
5502
5503//===----------------------------------------------------------------------===//
5504// TransferReadOp: VectorTransferOpInterface methods.
5505//===----------------------------------------------------------------------===//
5506VectorType TransferReadOp::getVectorType() { return getResult().getType(); }
5507
5508Value TransferReadOp::getVector() { return getResult(); }
5509
5510template <typename TransferOp>
5511static bool isInBounds(TransferOp op, int64_t resultIdx, int64_t indicesIdx) {
5512 // TODO: support more aggressive createOrFold on:
5513 // op.getIndices()[indicesIdx] + vectorType < dim(op.getSource(), indicesIdx)
5514 if (op.getShapedType().isDynamicDim(indicesIdx))
5515 return false;
5516 // Scalable dimensions are `vscale` times larger at runtime, so the static
5517 // size is only a lower bound and cannot prove that the transfer fits.
5518 if (op.getVectorType().getScalableDims()[resultIdx])
5519 return false;
5520 Value index = op.getIndices()[indicesIdx];
5521 std::optional<int64_t> cstOp = getConstantIntValue(ofr: index);
5522 if (!cstOp.has_value())
5523 return false;
5524
5525 int64_t sourceSize = op.getShapedType().getDimSize(indicesIdx);
5526 int64_t vectorSize = op.getVectorType().getDimSize(resultIdx);
5527 // Largest index at which a full vector still fits. Computed as a subtraction
5528 // rather than adding to the index, which could overflow. The subtraction is
5529 // safe only because of the `isDynamicDim` early return above: `sourceSize`
5530 // is a real static extent here, never `ShapedType::kDynamic`, which is
5531 // `INT64_MIN` and would make this signed overflow.
5532 int64_t maxStart = sourceSize - vectorSize;
5533
5534 // `in_bounds` guarantees that the transfer stays within the source *including
5535 // its starting point*, so a negative index is not in bounds.
5536 return *cstOp >= 0 && *cstOp <= maxStart;
5537}
5538
5539template <typename TransferOp>
5540static LogicalResult foldTransferInBoundsAttribute(TransferOp op) {
5541 // TODO: support 0-d corner case.
5542 // TODO: Be less conservative.
5543 if (op.getTransferRank() == 0)
5544 return failure();
5545 AffineMap permutationMap = op.getPermutationMap();
5546 bool changed = false;
5547 SmallVector<bool, 4> newInBounds;
5548 newInBounds.reserve(N: op.getTransferRank());
5549 // Idxs of non-bcast dims - used when analysing bcast dims.
5550 SmallVector<unsigned> nonBcastDims;
5551
5552 // 1. Process non-broadcast dims
5553 for (unsigned i = 0; i < op.getTransferRank(); ++i) {
5554 // 1.1. Already marked as in-bounds, nothing to see here.
5555 if (op.isDimInBounds(i)) {
5556 newInBounds.push_back(Elt: true);
5557 continue;
5558 }
5559 // 1.2. Currently out-of-bounds, check whether we can statically determine
5560 // it is inBounds.
5561 bool inBounds = false;
5562 auto dimExpr = dyn_cast<AffineDimExpr>(Val: permutationMap.getResult(idx: i));
5563 if (dimExpr) {
5564 inBounds = isInBounds(op, /*resultIdx=*/i,
5565 /*indicesIdx=*/dimExpr.getPosition());
5566 nonBcastDims.push_back(Elt: i);
5567 }
5568
5569 newInBounds.push_back(Elt: inBounds);
5570 // We commit the pattern if it is "more inbounds".
5571 changed |= inBounds;
5572 }
5573
5574 // 2. Handle broadcast dims
5575 // If all non-broadcast dims are "in bounds", then all bcast dims should be
5576 // "in bounds" as well.
5577 bool allNonBcastDimsInBounds = llvm::all_of(
5578 nonBcastDims, [&newInBounds](unsigned idx) { return newInBounds[idx]; });
5579 if (allNonBcastDimsInBounds) {
5580 for (size_t idx : permutationMap.getBroadcastDims()) {
5581 changed |= !newInBounds[idx];
5582 newInBounds[idx] = true;
5583 }
5584 }
5585
5586 if (!changed)
5587 return failure();
5588 // OpBuilder is only used as a helper to build an I64ArrayAttr.
5589 OpBuilder b(op.getContext());
5590 op.setInBoundsAttr(b.getBoolArrayAttr(values: newInBounds));
5591 return success();
5592}
5593
5594template <typename TransferOp>
5595static LogicalResult foldTransferFullMask(TransferOp op) {
5596 auto mask = op.getMask();
5597 if (!mask)
5598 return failure();
5599
5600 if (getMaskFormat(mask) != MaskFormat::AllTrue)
5601 return failure();
5602
5603 op.getMaskMutable().clear();
5604 return success();
5605}
5606
5607/// When the vector type is `vector<1xT>`, the permutation map is irrelevant:
5608/// the single vector lane always has iteration offset 0, so the element is at
5609/// `indices` regardless of which source dimension the map points at. Replace
5610/// with the minor identity to unblock lowering to vector.load / vector.store.
5611template <typename TransferOp>
5612static LogicalResult foldSize1TransferPermutationMap(TransferOp op) {
5613 VectorType vecType = op.getVectorType();
5614 if (vecType.getRank() != 1 || vecType.getShape()[0] != 1 ||
5615 vecType.isScalable())
5616 return failure();
5617
5618 AffineMap map = op.getPermutationMap();
5619 if (map.isMinorIdentity())
5620 return failure();
5621
5622 int64_t srcRank = op.getShapedType().getRank();
5623 if (srcRank < 1)
5624 return failure();
5625
5626 AffineMap minorIdentity =
5627 AffineMap::getMinorIdentityMap(dims: srcRank, results: 1, context: op.getContext());
5628 op.setPermutationMapAttr(AffineMapAttr::get(value: minorIdentity));
5629 return success();
5630}
5631
5632/// ```
5633/// %w0 = vector.transfer_write %v0, %arg0[%c1, %c0] {in_bounds = [true, true]}
5634/// : vector<1x4xf32>, tensor<4x4xf32>
5635/// %0 = vector.transfer_read %w0[%c1, %c0], %cf0 {in_bounds = [true, true]}
5636/// : tensor<4x4xf32>, vector<1x4xf32>
5637/// ```
5638/// -> Folds into
5639/// ```
5640/// %v0
5641/// ```
5642static Value foldRAW(TransferReadOp readOp) {
5643 if (!llvm::isa<RankedTensorType>(Val: readOp.getShapedType()))
5644 return {};
5645 auto defWrite = readOp.getBase().getDefiningOp<vector::TransferWriteOp>();
5646 while (defWrite) {
5647 if (checkSameValueRAW(defWrite, read: readOp))
5648 return defWrite.getVector();
5649 if (!isDisjointTransferIndices(
5650 transferA: cast<VectorTransferOpInterface>(Val: defWrite.getOperation()),
5651 transferB: cast<VectorTransferOpInterface>(Val: readOp.getOperation())))
5652 break;
5653 defWrite = defWrite.getBase().getDefiningOp<vector::TransferWriteOp>();
5654 }
5655 return {};
5656}
5657
5658OpFoldResult TransferReadOp::fold(FoldAdaptor) {
5659 if (Value vec = foldRAW(readOp: *this))
5660 return vec;
5661 /// transfer_read(memrefcast) -> transfer_read
5662 if (succeeded(Result: foldTransferInBoundsAttribute(op: *this)))
5663 return getResult();
5664 if (succeeded(Result: foldTransferFullMask(op: *this)))
5665 return getResult();
5666 if (succeeded(Result: foldSize1TransferPermutationMap(op: *this)))
5667 return getResult();
5668 if (succeeded(Result: memref::foldMemRefCast(op: *this)))
5669 return getResult();
5670 if (succeeded(Result: tensor::foldTensorCast(op: *this)))
5671 return getResult();
5672 return OpFoldResult();
5673}
5674
5675//-----------------------------------------------------------------------------
5676// VectorUnrollOpInterface methods
5677//-----------------------------------------------------------------------------
5678std::optional<SmallVector<int64_t, 4>>
5679TransferReadOp::getIterationShapeToUnroll() {
5680 return llvm::to_vector<4>(Range: getVectorType().getShape());
5681}
5682
5683void TransferReadOp::getEffects(
5684 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
5685 &effects) {
5686 if (llvm::isa<MemRefType>(Val: getShapedType()))
5687 effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &getBaseMutable(),
5688 Args: SideEffects::DefaultResource::get());
5689}
5690
5691Speculation::Speculatability TransferReadOp::getSpeculatability() {
5692 if (hasPureTensorSemantics())
5693 return Speculation::Speculatable;
5694 return Speculation::NotSpeculatable;
5695}
5696
5697/// Given a projected permutation, inverse an affine map, making the unused dims
5698/// 0 in the result.
5699static AffineMap inverseWithUnusedDims(AffineMap map) {
5700 assert(map.isProjectedPermutation() &&
5701 "expected a projected permutation map");
5702 SmallVector<AffineExpr> results(map.getNumInputs(),
5703 getAffineConstantExpr(constant: 0, context: map.getContext()));
5704 for (auto [idx, result] : llvm::enumerate(First: map.getResults())) {
5705 // We should only have dim exprs because this is a projected permutation.
5706 int64_t pos = cast<AffineDimExpr>(Val: result).getPosition();
5707 results[pos] = getAffineDimExpr(position: idx, context: map.getContext());
5708 }
5709 return AffineMap::get(/*dimCount=*/map.getNumResults(), /*symbolCount=*/0,
5710 results, context: map.getContext());
5711}
5712
5713namespace {
5714/// Store to load forwarding for transfer operations with permuation maps.
5715/// Even if the permutation maps are different we can still propagate the store
5716/// into the load if the size of the dimensions read and written match. Then we
5717/// can replace the transfer_read + transfer_write by vector.broadcast and
5718/// vector.transpose.
5719/// Example:
5720/// ```
5721/// %w0 = vector.transfer_write %v0, %arg0[%c0, %c0, %c0]
5722/// {in_bounds = [true, true],
5723/// permutation_map = affine_map<(d0, d1, d2) -> (d2, d1)>} :
5724/// vector<4x1xf32>, tensor<4x4x4xf32>
5725/// %r = vector.transfer_read %w0[%c0, %c0, %c0], %cf0
5726/// {in_bounds = [true, true, true, true],
5727/// permutation_map = affine_map<(d0, d1, d2) -> (d1, 0, d2, 0)>} :
5728/// tensor<4x4x4xf32>, vector<1x100x4x5xf32>
5729/// ```
5730/// To:
5731/// ```
5732/// %0 = vector.broadcast %arg1 : vector<4x1xf32> to vector<100x5x4x1xf32>
5733/// %r = vector.transpose %0, [3, 0, 2, 1] :
5734/// vector<100x5x4x1xf32> to vector<1x100x4x5xf32>
5735/// ```
5736struct TransferReadAfterWriteToBroadcast
5737 : public OpRewritePattern<TransferReadOp> {
5738 using Base::Base;
5739
5740 LogicalResult matchAndRewrite(TransferReadOp readOp,
5741 PatternRewriter &rewriter) const override {
5742 auto defWrite = readOp.getBase().getDefiningOp<vector::TransferWriteOp>();
5743 if (!defWrite)
5744 return failure();
5745 // Bail if we need an alias analysis.
5746 if (!readOp.hasPureTensorSemantics() || !defWrite.hasPureTensorSemantics())
5747 return failure();
5748 // Bail in the masked case (too complex atm and needed to properly account
5749 // for padding). This includes an enclosing vector.mask.
5750 // TODO: Support mask operands and reads nested in vector.mask while
5751 // properly accounting for padding.
5752 if (readOp.getMask() || defWrite.getMask() || readOp.isMasked())
5753 return failure();
5754 // If indices are not the same a shift may be required, bail.
5755 if (readOp.getIndices() != defWrite.getIndices())
5756 return failure();
5757 // Bail if we need a bounds analysis.
5758 if (readOp.hasOutOfBoundsDim() || defWrite.hasOutOfBoundsDim())
5759 return failure();
5760 // TODO: If the written transfer chunk is a superset of the read transfer
5761 // chunk we could do an extract_strided_slice.
5762 if (readOp.getTransferChunkAccessed() !=
5763 defWrite.getTransferChunkAccessed())
5764 return failure();
5765 // WriteMap: tensor -> w_vec
5766 // ReadMap: tensor -> r_vec
5767 //
5768 // inv(WriteMap): w_vec -> tensor
5769 // inv(WriteMap) o ReadMap: w_vec -> r_vec
5770 AffineMap readMap = readOp.getPermutationMap();
5771 AffineMap writeMap = defWrite.getPermutationMap();
5772 AffineMap invWriteMap = inverseWithUnusedDims(map: writeMap);
5773 AffineMap composedMap = readMap.compose(map: invWriteMap);
5774 // If there are any unused dims in the composedMap, we have to drop some
5775 // unit dims from the written vector before we can do transpose(broadcast).
5776 // TODO: Support this case.
5777 if (getUnusedDimsBitVector(maps: composedMap).any())
5778 return failure();
5779 // readVec = transpose(broadcast(writeVec))
5780 //
5781 // Build a transpose permutation for the above transpose operation.
5782 //
5783 // Treat the composed map as having extra leading dimensions which are
5784 // the broadcasted dimensions, and treat the zeros as these new broadcasted
5785 // dimensions.
5786 SmallVector<unsigned> broadcastedDims = composedMap.getBroadcastDims();
5787 int64_t numBroadcastedDims = broadcastedDims.size();
5788 auto invPerm = llvm::to_vector_of<int64_t>(Range&: broadcastedDims);
5789 invPerm.resize(N: composedMap.getNumResults());
5790 for (auto [idx, expr] : llvm::enumerate(First: composedMap.getResults())) {
5791 if (auto dim = dyn_cast<AffineDimExpr>(Val: expr)) {
5792 int64_t effectiveDim = dim.getPosition() + numBroadcastedDims;
5793 invPerm[effectiveDim] = idx;
5794 }
5795 }
5796 // Applying the inverse permutation on the readVecTy will give us the
5797 // broadcast result type.
5798 VectorType readVecTy = readOp.getVectorType();
5799 SmallVector<int64_t> permutation = invertPermutationVector(permutation: invPerm);
5800 auto broadcastedVecTy =
5801 VectorType::get(shape: applyPermutation(input: readVecTy.getShape(), permutation: invPerm),
5802 elementType: readVecTy.getElementType(),
5803 scalableDims: applyPermutation(input: readVecTy.getScalableDims(), permutation: invPerm));
5804 // Build the transpose(broadcast) transformation.
5805 Value vec = defWrite.getVector();
5806 Location loc = readOp.getLoc();
5807 vec = vector::BroadcastOp::create(builder&: rewriter, location: loc, result: broadcastedVecTy, source: vec);
5808 rewriter.replaceOpWithNewOp<vector::TransposeOp>(op: readOp, args&: vec, args&: permutation);
5809 return success();
5810 }
5811};
5812} // namespace
5813
5814void TransferReadOp::getCanonicalizationPatterns(RewritePatternSet &results,
5815 MLIRContext *context) {
5816 results.add<TransferReadAfterWriteToBroadcast>(arg&: context);
5817}
5818
5819FailureOr<std::optional<SmallVector<Value>>>
5820TransferReadOp::bubbleDownCasts(OpBuilder &builder) {
5821 if (!hasPureBufferSemantics())
5822 return failure();
5823 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(),
5824 results: getResult());
5825}
5826
5827//===----------------------------------------------------------------------===//
5828// TransferWriteOp
5829//===----------------------------------------------------------------------===//
5830
5831/// 1. Builder with type inference.
5832void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
5833 Value vector, Value dest, ValueRange indices,
5834 AffineMapAttr permutationMapAttr,
5835 /*optional*/ Value mask,
5836 /*optional*/ ArrayAttr inBoundsAttr) {
5837 Type resultType = llvm::dyn_cast<RankedTensorType>(Val: dest.getType());
5838 build(odsBuilder&: builder, odsState&: result, result: resultType, valueToStore: vector, base: dest, indices, permutation_map: permutationMapAttr,
5839 mask, in_bounds: inBoundsAttr);
5840}
5841
5842/// 2. Builder with type inference that sets an empty mask (variant with attrs).
5843void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
5844 Value vector, Value dest, ValueRange indices,
5845 AffineMapAttr permutationMapAttr,
5846 /*optional*/ ArrayAttr inBoundsAttr) {
5847 build(builder, result, vector, dest, indices, permutationMapAttr,
5848 /*mask=*/Value(), inBoundsAttr);
5849}
5850
5851/// 3. Builder with type inference that sets an empty mask (variant without
5852/// attrs). If `permutationMap` is null, a minor identity map is used.
5853void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
5854 Value vector, Value dest, ValueRange indices,
5855 AffineMap permutationMap,
5856 std::optional<ArrayRef<bool>> inBounds) {
5857 if (!permutationMap)
5858 permutationMap =
5859 getTransferMinorIdentityMap(shapedType: llvm::cast<ShapedType>(Val: dest.getType()),
5860 vectorType: llvm::cast<VectorType>(Val: vector.getType()));
5861 auto permutationMapAttr = AffineMapAttr::get(value: permutationMap);
5862 auto inBoundsAttr =
5863 (inBounds && !inBounds.value().empty())
5864 ? builder.getBoolArrayAttr(values: inBounds.value())
5865 : builder.getBoolArrayAttr(values: SmallVector<bool>(
5866 llvm::cast<VectorType>(Val: vector.getType()).getRank(), false));
5867 build(builder, result, vector, dest, indices, permutationMapAttr,
5868 /*mask=*/Value(), inBoundsAttr);
5869}
5870
5871/// 4. Builder with type inference that sets an empty mask and sets permutation
5872/// map to 'getMinorIdentityMap'.
5873void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
5874 Value vector, Value dest, ValueRange indices,
5875 std::optional<ArrayRef<bool>> inBounds) {
5876 build(builder, result, vector, dest, indices, /*permutationMap=*/AffineMap(),
5877 inBounds);
5878}
5879
5880ParseResult TransferWriteOp::parse(OpAsmParser &parser,
5881 OperationState &result) {
5882 auto &builder = parser.getBuilder();
5883 SMLoc typesLoc;
5884 OpAsmParser::UnresolvedOperand vectorInfo, sourceInfo;
5885 SmallVector<OpAsmParser::UnresolvedOperand, 8> indexInfo;
5886 SmallVector<Type, 2> types;
5887 OpAsmParser::UnresolvedOperand maskInfo;
5888 if (parser.parseOperand(result&: vectorInfo) || parser.parseComma() ||
5889 parser.parseOperand(result&: sourceInfo) ||
5890 parser.parseOperandList(result&: indexInfo, delimiter: OpAsmParser::Delimiter::Square))
5891 return failure();
5892 ParseResult hasMask = parser.parseOptionalComma();
5893 if (hasMask.succeeded() && parser.parseOperand(result&: maskInfo))
5894 return failure();
5895 if (parser.parseOptionalAttrDict(result&: result.attributes) ||
5896 parser.getCurrentLocation(loc: &typesLoc) || parser.parseColonTypeList(result&: types))
5897 return failure();
5898 if (types.size() != 2)
5899 return parser.emitError(loc: typesLoc, message: "requires two types");
5900 auto indexType = builder.getIndexType();
5901 VectorType vectorType = llvm::dyn_cast<VectorType>(Val&: types[0]);
5902 if (!vectorType)
5903 return parser.emitError(loc: typesLoc, message: "requires vector type");
5904 ShapedType shapedType = llvm::dyn_cast<ShapedType>(Val&: types[1]);
5905 if (!shapedType || !llvm::isa<MemRefType, RankedTensorType>(Val: shapedType))
5906 return parser.emitError(loc: typesLoc, message: "requires memref or ranked tensor type");
5907 auto permMapAttrName =
5908 TransferWriteOp::getPermutationMapAttrName(name: result.name);
5909 auto permMapAttr = result.attributes.get(name: permMapAttrName);
5910 AffineMap permMap;
5911 if (!permMapAttr) {
5912 if (shapedType.getRank() <
5913 getEffectiveVectorRankForXferOp(shapedType, vectorType))
5914 return parser.emitError(loc: typesLoc,
5915 message: "expected a custom permutation_map when "
5916 "rank(source) != rank(destination)");
5917 permMap = getTransferMinorIdentityMap(shapedType, vectorType);
5918 result.attributes.set(name: permMapAttrName, value: AffineMapAttr::get(value: permMap));
5919 } else {
5920 permMap = llvm::cast<AffineMapAttr>(Val&: permMapAttr).getValue();
5921 }
5922 auto inBoundsAttrName = TransferWriteOp::getInBoundsAttrName(name: result.name);
5923 Attribute inBoundsAttr = result.attributes.get(name: inBoundsAttrName);
5924 if (!inBoundsAttr) {
5925 result.addAttribute(name: inBoundsAttrName,
5926 attr: builder.getBoolArrayAttr(
5927 values: SmallVector<bool>(permMap.getNumResults(), false)));
5928 }
5929 if (parser.resolveOperand(operand: vectorInfo, type: vectorType, result&: result.operands) ||
5930 parser.resolveOperand(operand: sourceInfo, type: shapedType, result&: result.operands) ||
5931 parser.resolveOperands(operands&: indexInfo, type: indexType, result&: result.operands))
5932 return failure();
5933 if (hasMask.succeeded()) {
5934 if (llvm::dyn_cast<VectorType>(Val: shapedType.getElementType()))
5935 return parser.emitError(
5936 loc: maskInfo.location, message: "does not support masks with vector element type");
5937 if (vectorType.getRank() != permMap.getNumResults()) {
5938 return parser.emitError(loc: typesLoc,
5939 message: "expected the same rank for the vector and the "
5940 "results of the permutation map");
5941 }
5942 auto maskType = inferTransferOpMaskType(vecType: vectorType, permMap);
5943 if (parser.resolveOperand(operand: maskInfo, type: maskType, result&: result.operands))
5944 return failure();
5945 }
5946 result.addAttribute(name: TransferWriteOp::getOperandSegmentSizeAttr(),
5947 attr: builder.getDenseI32ArrayAttr(
5948 values: {1, 1, static_cast<int32_t>(indexInfo.size()),
5949 static_cast<int32_t>(hasMask.succeeded())}));
5950 return failure(IsFailure: llvm::isa<RankedTensorType>(Val: shapedType) &&
5951 parser.addTypeToList(type: shapedType, result&: result.types));
5952}
5953
5954void TransferWriteOp::print(OpAsmPrinter &p) {
5955 p << " " << getVector() << ", " << getBase() << "[" << getIndices() << "]";
5956 if (getMask())
5957 p << ", " << getMask();
5958 printTransferAttrs(p, op: *this);
5959 p << " : " << getVectorType() << ", " << getShapedType();
5960}
5961
5962LogicalResult TransferWriteOp::verify() {
5963 // Consistency of elemental types in shape and vector.
5964 ShapedType shapedType = getShapedType();
5965 VectorType vectorType = getVectorType();
5966 VectorType maskType = getMaskType();
5967 auto permutationMap = getPermutationMap();
5968 VectorType inferredMaskType =
5969 maskType ? inferTransferOpMaskType(vecType: vectorType, permMap: permutationMap)
5970 : VectorType();
5971
5972 if (llvm::size(Range: getIndices()) != shapedType.getRank())
5973 return emitOpError(message: "requires ") << shapedType.getRank() << " indices";
5974
5975 // We do not allow broadcast dimensions on TransferWriteOps for the moment,
5976 // as the semantics is unclear. This can be revisited later if necessary.
5977 if (hasBroadcastDim())
5978 return emitOpError(message: "should not have broadcast dimensions");
5979
5980 if (failed(Result: verifyTransferOp(op: cast<VectorTransferOpInterface>(Val: getOperation()),
5981 shapedType, vectorType, maskType,
5982 inferredMaskType, permutationMap, inBounds: getInBounds())))
5983 return failure();
5984
5985 return verifyPermutationMap(permutationMap,
5986 emitOpError: [&](Twine t) { return emitOpError(message: t); });
5987}
5988
5989//===----------------------------------------------------------------------===//
5990// TransferWriteOp: MaskableOpInterface methods.
5991//===----------------------------------------------------------------------===//
5992
5993/// Returns the mask type expected by this operation. Mostly used for
5994/// verification purposes.
5995Type TransferWriteOp::getExpectedMaskType() {
5996 return inferTransferOpMaskType(vecType: getVectorType(), permMap: getPermutationMap());
5997}
5998
5999//===----------------------------------------------------------------------===//
6000// TransferWriteOp: VectorTransferOpInterface methods.
6001//===----------------------------------------------------------------------===//
6002Value TransferWriteOp::getVector() { return getOperand(i: 0); }
6003VectorType TransferWriteOp::getVectorType() {
6004 return cast<VectorType>(Val: getValueToStore().getType());
6005}
6006
6007//===----------------------------------------------------------------------===//
6008// TransferWriteOp: fold methods.
6009//===----------------------------------------------------------------------===//
6010/// Fold:
6011/// ```
6012/// %t1 = ...
6013/// %v = vector.transfer_read %t0[%c0...], {in_bounds = [true...]} :
6014/// tensor<static_sizesxf32>, vector<static_sizesxf32>
6015/// %t2 = vector.transfer_write %v, %t1[%c0...] {in_bounds = [true...]} :
6016/// vector<static_sizesxf32>, tensor<static_sizesxf32>
6017/// ```
6018///
6019/// into:
6020///
6021/// ```
6022/// %t0
6023/// ```
6024///
6025/// The producer of t1 may or may not be DCE'd depending on whether it is a
6026/// block argument or has side effects.
6027static LogicalResult foldReadInitWrite(TransferWriteOp write,
6028 ArrayRef<Attribute>,
6029 SmallVectorImpl<OpFoldResult> &results) {
6030 // TODO: support 0-d corner case.
6031 if (write.getTransferRank() == 0)
6032 return failure();
6033 auto rankedTensorType =
6034 llvm::dyn_cast<RankedTensorType>(Val: write.getBase().getType());
6035 // If not operating on tensors, bail.
6036 if (!rankedTensorType)
6037 return failure();
6038 // If no read, bail.
6039 auto read = write.getVector().getDefiningOp<vector::TransferReadOp>();
6040 if (!read)
6041 return failure();
6042 // TODO: support 0-d corner case.
6043 if (read.getTransferRank() == 0)
6044 return failure();
6045 // For now, only accept minor identity. Future: composition is minor identity.
6046 if (!read.getPermutationMap().isMinorIdentity() ||
6047 !write.getPermutationMap().isMinorIdentity())
6048 return failure();
6049 // Bail on mismatching ranks.
6050 if (read.getTransferRank() != write.getTransferRank())
6051 return failure();
6052 // Bail on potential out-of-bounds accesses.
6053 if (read.hasOutOfBoundsDim() || write.hasOutOfBoundsDim())
6054 return failure();
6055 // Masked transfers have padding/select semantics and are not identity folds.
6056 if (read.getMask() || write.getMask())
6057 return failure();
6058 // Tensor types must be the same.
6059 if (read.getBase().getType() != rankedTensorType)
6060 return failure();
6061 // Vector types must be the same.
6062 if (read.getVectorType() != write.getVectorType())
6063 return failure();
6064 // Vector and Tensor shapes must match.
6065 if (read.getVectorType().getShape() != rankedTensorType.getShape())
6066 return failure();
6067 // If any index is nonzero.
6068 auto isNotConstantZero = [](Value v) {
6069 auto cstOp = getConstantIntValue(ofr: v);
6070 return !cstOp.has_value() || cstOp.value() != 0;
6071 };
6072 if (llvm::any_of(Range: read.getIndices(), P: isNotConstantZero) ||
6073 llvm::any_of(Range: write.getIndices(), P: isNotConstantZero))
6074 return failure();
6075 // Success.
6076 results.push_back(Elt: read.getBase());
6077 return success();
6078}
6079
6080static bool checkSameValueWAR(vector::TransferReadOp read,
6081 vector::TransferWriteOp write) {
6082 return read.getBase() == write.getBase() &&
6083 read.getIndices() == write.getIndices() &&
6084 read.getPermutationMap() == write.getPermutationMap() &&
6085 read.getVectorType() == write.getVectorType() && !read.getMask() &&
6086 !write.getMask();
6087}
6088/// Fold transfer_write write after read:
6089/// ```
6090/// %t0 = ...
6091/// %v = vector.transfer_read %t0[%c0...] :
6092/// tensor<static_sizesxf32>, vector<static_sizesxf32>
6093/// %t1 = vector.transfer_write %v, %t0[%c0...] :
6094/// vector<static_sizesxf32>, tensor<static_sizesxf32>
6095/// ```
6096///
6097/// into:
6098///
6099/// ```
6100/// %t0
6101/// ```
6102static LogicalResult foldWAR(TransferWriteOp write,
6103 SmallVectorImpl<OpFoldResult> &results) {
6104 if (!llvm::isa<RankedTensorType>(Val: write.getBase().getType()))
6105 return failure();
6106 auto read = write.getVector().getDefiningOp<vector::TransferReadOp>();
6107 if (!read)
6108 return failure();
6109
6110 if (!checkSameValueWAR(read, write))
6111 return failure();
6112 results.push_back(Elt: read.getBase());
6113 return success();
6114}
6115
6116LogicalResult TransferWriteOp::fold(FoldAdaptor adaptor,
6117 SmallVectorImpl<OpFoldResult> &results) {
6118 if (succeeded(Result: foldReadInitWrite(write: *this, adaptor.getOperands(), results)))
6119 return success();
6120 if (succeeded(Result: foldWAR(write: *this, results)))
6121 return success();
6122 if (succeeded(Result: foldTransferInBoundsAttribute(op: *this)))
6123 return success();
6124 if (succeeded(Result: foldTransferFullMask(op: *this)))
6125 return success();
6126 if (succeeded(Result: foldSize1TransferPermutationMap(op: *this)))
6127 return success();
6128 return memref::foldMemRefCast(op: *this);
6129}
6130
6131//===----------------------------------------------------------------------===//
6132// TransferWriteOp: other methods.
6133//===----------------------------------------------------------------------===//
6134std::optional<SmallVector<int64_t, 4>>
6135TransferWriteOp::getIterationShapeToUnroll() {
6136 return llvm::to_vector<4>(Range: getVectorType().getShape());
6137}
6138
6139void TransferWriteOp::getEffects(
6140 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
6141 &effects) {
6142 if (llvm::isa<MemRefType>(Val: getShapedType()))
6143 effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &getBaseMutable(),
6144 Args: SideEffects::DefaultResource::get());
6145}
6146
6147Speculation::Speculatability TransferWriteOp::getSpeculatability() {
6148 if (hasPureTensorSemantics())
6149 return Speculation::Speculatable;
6150 return Speculation::NotSpeculatable;
6151}
6152
6153namespace {
6154/// Remove dead transfer write from the SSA chain so that it an be eliminated by
6155/// DCE
6156/// ```
6157/// %w0 = vector.transfer_write %v0, %arg0[%c1, %c0] {in_bounds = [true, true]}
6158/// : vector<1x4xf32>, tensor<4x4xf32>
6159/// %w1 = vector.transfer_write %v0, %w0[%c2, %c0] {in_bounds = [true, true]}
6160/// : vector<1x4xf32>, tensor<4x4xf32>
6161/// %w2 = vector.transfer_write %v1, %w1[%c1, %c0] {in_bounds = [true, true]}
6162/// : vector<1x4xf32>, tensor<4x4xf32>
6163/// ```
6164///
6165/// into:
6166///
6167/// ```
6168/// %w0 = vector.transfer_write %v0, %arg0[%c1, %c0] {in_bounds = [true, true]}
6169/// : vector<1x4xf32>, tensor<4x4xf32>
6170/// %w1 = vector.transfer_write %v0, %arg0[%c2, %c0] {in_bounds = [true, true]}
6171/// : vector<1x4xf32>, tensor<4x4xf32>
6172/// %w2 = vector.transfer_write %v1, %w1[%c1, %c0] {in_bounds = [true, true]}
6173/// : vector<1x4xf32>, tensor<4x4xf32>
6174/// ```
6175///
6176/// `%w0 = vector.transfer_write` op will be removed by DCE if it doesn't have
6177/// any other uses.
6178class FoldWaw final : public OpRewritePattern<TransferWriteOp> {
6179public:
6180 using Base::Base;
6181 LogicalResult matchAndRewrite(TransferWriteOp writeOp,
6182 PatternRewriter &rewriter) const override {
6183 if (!llvm::isa<RankedTensorType>(Val: writeOp.getShapedType()))
6184 return failure();
6185 vector::TransferWriteOp writeToModify = writeOp;
6186
6187 auto defWrite = writeOp.getBase().getDefiningOp<vector::TransferWriteOp>();
6188 while (defWrite) {
6189 if (checkSameValueWAW(write: writeOp, priorWrite: defWrite)) {
6190 rewriter.modifyOpInPlace(root: writeToModify, callable: [&]() {
6191 writeToModify.getBaseMutable().assign(value: defWrite.getBase());
6192 });
6193 return success();
6194 }
6195 if (!isDisjointTransferIndices(
6196 transferA: cast<VectorTransferOpInterface>(Val: defWrite.getOperation()),
6197 transferB: cast<VectorTransferOpInterface>(Val: writeOp.getOperation())))
6198 break;
6199 // If the previous write op doesn't have any other use we an safely look
6200 // at the previous store to see if it can be removed.
6201 if (!defWrite->hasOneUse())
6202 break;
6203 writeToModify = defWrite;
6204 defWrite = defWrite.getBase().getDefiningOp<vector::TransferWriteOp>();
6205 }
6206 return failure();
6207 }
6208};
6209
6210/// Rewrite tensor::ExtractSliceOp(vector::TransferWriteOp) to
6211/// vector::TransferWriteOp(tensor::ExtractSliceOp) if the full slice is
6212/// overwritten and inserted into another tensor. After this rewrite, the
6213/// operations bufferize in-place since all of them work on the same slice.
6214///
6215/// For example:
6216/// ```mlir
6217/// %0 = vector.transfer_write %vec, %init_tensor[%c0, %c0]
6218/// : vector<8x16xf32>, tensor<8x16xf32>
6219/// %1 = tensor.extract_slice %0[0, 0] [%sz0, %sz1] [1, 1]
6220/// : tensor<8x16xf32> to tensor<?x?xf32>
6221/// %r = tensor.insert_slice %1 into %iter_arg[%iv0, %iv1] [%sz0, %sz1] [1, 1]
6222/// : tensor<?x?xf32> into tensor<27x37xf32>
6223/// ```
6224/// folds to
6225/// ```mlir
6226/// %0 = tensor.extract_slice %iter_arg[%iv0, %iv1] [%sz0, %sz1] [1, 1]
6227/// : tensor<27x37xf32> to tensor<?x?xf32>
6228/// %1 = vector.transfer_write %vec, %0[%c0, %c0]
6229/// : vector<8x16xf32>, tensor<?x?xf32>
6230/// %r = tensor.insert_slice %1 into %iter_arg[%iv0, %iv1] [%sz0, %sz1] [1, 1]
6231/// : tensor<?x?xf32> into tensor<27x37xf32>
6232/// ```
6233struct SwapExtractSliceOfTransferWrite
6234 : public OpRewritePattern<tensor::InsertSliceOp> {
6235public:
6236 using Base::Base;
6237
6238 LogicalResult matchAndRewrite(tensor::InsertSliceOp insertOp,
6239 PatternRewriter &rewriter) const override {
6240 if (!insertOp.hasUnitStride())
6241 return failure();
6242 auto extractOp =
6243 insertOp.getSource().getDefiningOp<tensor::ExtractSliceOp>();
6244 if (!extractOp || !extractOp.hasUnitStride() || !extractOp->hasOneUse())
6245 return failure();
6246 auto transferOp = extractOp.getSource().getDefiningOp<TransferWriteOp>();
6247 if (!transferOp || !transferOp->hasOneUse())
6248 return failure();
6249
6250 // Fail if vector::TransferWriteOp or tensor::ExtractSliceOp is
6251 // rank-reducing.
6252 if (insertOp.getSourceType().getRank() != transferOp.getTransferRank()) {
6253 return rewriter.notifyMatchFailure(arg&: insertOp,
6254 msg: "use-def chain is rank-reducing");
6255 }
6256
6257 // Fail if tensor::ExtractSliceOp has non-zero offset.
6258 if (!extractOp.hasZeroOffset()) {
6259 return rewriter.notifyMatchFailure(arg&: insertOp,
6260 msg: "ExtractSliceOp has non-zero offset");
6261 }
6262
6263 // Fail if tensor::TransferWriteOp has non-zero offset.
6264 if (!llvm::all_of(Range: transferOp.getIndices(), P: [](Value value) {
6265 return getConstantIntValue(ofr: value) == static_cast<int64_t>(0);
6266 })) {
6267 return rewriter.notifyMatchFailure(arg&: insertOp,
6268 msg: "TranferWriteOp has non-zero offset");
6269 }
6270
6271 // Fail if tensor::ExtractSliceOp and tensor::InsertSliceOp sizes differ.
6272 if (insertOp.getMixedSizes().size() != extractOp.getMixedSizes().size()) {
6273 return rewriter.notifyMatchFailure(
6274 arg&: insertOp, msg: "InsertSliceOp and ExtractSliceOp ranks differ");
6275 }
6276
6277 for (auto [insertSize, extractSize] :
6278 llvm::zip_equal(t: insertOp.getMixedSizes(), u: extractOp.getMixedSizes())) {
6279 if (!isEqualConstantIntOrValue(ofr1: insertSize, ofr2: extractSize)) {
6280 return rewriter.notifyMatchFailure(
6281 arg&: insertOp, msg: "InsertSliceOp and ExtractSliceOp sizes differ");
6282 }
6283 }
6284
6285 // Fail if the vector::TransferWriteOp may not overwrite the full tensor.
6286 assert(transferOp.getVectorType().hasStaticShape() &&
6287 "expected vector to have a static shape");
6288 ArrayRef<int64_t> vectorShape = transferOp.getVectorType().getShape();
6289 SmallVector<int64_t> resultShape = applyPermutationMap(
6290 map: transferOp.getPermutationMap(), source: transferOp.getShapedType().getShape());
6291 if (transferOp.getMask() || !vectorShape.equals(RHS: resultShape)) {
6292 return rewriter.notifyMatchFailure(
6293 arg&: insertOp, msg: "TransferWriteOp may not write the full tensor.");
6294 }
6295
6296 // Swap the tensor::ExtractSliceOp in front of the vector::TransferWriteOp.
6297 // Set all in_bounds to false and let the folder infer them.
6298 SmallVector<bool> newInBounds(vectorShape.size(), false);
6299 auto newExtractOp = tensor::ExtractSliceOp::create(
6300 builder&: rewriter, location: extractOp.getLoc(), resultType: insertOp.getSourceType(),
6301 source: insertOp.getDest(), offsets: insertOp.getMixedOffsets(),
6302 sizes: insertOp.getMixedSizes(), strides: insertOp.getMixedStrides());
6303 auto newTransferWriteOp = TransferWriteOp::create(
6304 builder&: rewriter, location: transferOp.getLoc(), vector: transferOp.getVector(),
6305 dest: newExtractOp.getResult(), indices: transferOp.getIndices(),
6306 permutationMapAttr: transferOp.getPermutationMapAttr(),
6307 inBoundsAttr: rewriter.getBoolArrayAttr(values: newInBounds));
6308 rewriter.modifyOpInPlace(root: insertOp, callable: [&]() {
6309 insertOp.getSourceMutable().assign(value: newTransferWriteOp.getResult());
6310 });
6311 return success();
6312 }
6313};
6314
6315} // namespace
6316
6317void TransferWriteOp::getCanonicalizationPatterns(RewritePatternSet &results,
6318 MLIRContext *context) {
6319 results.add<FoldWaw, SwapExtractSliceOfTransferWrite>(arg&: context);
6320}
6321
6322FailureOr<std::optional<SmallVector<Value>>>
6323TransferWriteOp::bubbleDownCasts(OpBuilder &builder) {
6324 if (!hasPureBufferSemantics())
6325 return failure();
6326 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(),
6327 results: ValueRange());
6328}
6329
6330//===----------------------------------------------------------------------===//
6331// LoadOp
6332//===----------------------------------------------------------------------===//
6333
6334static ParseResult parseBoolAttr(OpAsmParser &parser, BoolAttr &result) {
6335 Attribute attr;
6336 if (parser.parseAttribute(result&: attr))
6337 return failure();
6338 result = dyn_cast<BoolAttr>(Val&: attr);
6339 if (!result)
6340 return parser.emitError(loc: parser.getCurrentLocation(),
6341 message: "expected boolean attribute");
6342 return success();
6343}
6344
6345static void printBoolAttr(OpAsmPrinter &printer, Operation *, BoolAttr attr) {
6346 printer.printAttribute(attr);
6347}
6348
6349static LogicalResult verifyLoadStoreMemRefLayout(Operation *op,
6350 VectorType vecTy,
6351 MemRefType memRefTy) {
6352 // If rank==0 or size==1 it's equivalent to scalar load/store, so we don't
6353 // need any strides limitations.
6354 if (!vecTy.isScalable() &&
6355 (vecTy.getRank() == 0 || vecTy.getNumElements() == 1))
6356 return success();
6357
6358 if (!memRefTy.isLastDimUnitStride())
6359 return op->emitOpError(message: "most minor memref dim must have unit stride");
6360 return success();
6361}
6362
6363LogicalResult vector::LoadOp::verify() {
6364 VectorType resVecTy = getVectorType();
6365 MemRefType memRefTy = getMemRefType();
6366
6367 if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: resVecTy, memRefTy)))
6368 return failure();
6369
6370 // Negative strides are not supported on vector.load. The lowering to LLVM
6371 // emits arithmetic operations (e.g., GEP, mul) with nuw flags that assume
6372 // non-negative strides to avoid undefined behavior.
6373 if (memref::hasNegativeStaticStride(memRefTy))
6374 return emitOpError(message: "memref strides must be non-negative");
6375
6376 if (memRefTy.getRank() < resVecTy.getRank())
6377 return emitOpError(
6378 message: "destination memref has lower rank than the result vector");
6379
6380 // Checks for vector memrefs.
6381 Type memElemTy = memRefTy.getElementType();
6382 if (auto memVecTy = llvm::dyn_cast<VectorType>(Val&: memElemTy)) {
6383 if (memVecTy != resVecTy)
6384 return emitOpError(message: "base memref and result vector types should match");
6385 memElemTy = memVecTy.getElementType();
6386 }
6387
6388 if (resVecTy.getElementType() != memElemTy)
6389 return emitOpError(message: "base and result element types should match");
6390 if (llvm::size(Range: getIndices()) != memRefTy.getRank())
6391 return emitOpError(message: "requires ") << memRefTy.getRank() << " indices";
6392 return success();
6393}
6394
6395OpFoldResult LoadOp::fold(FoldAdaptor) {
6396 if (succeeded(Result: memref::foldMemRefCast(op: *this)))
6397 return getResult();
6398 return OpFoldResult();
6399}
6400
6401FailureOr<std::optional<SmallVector<Value>>>
6402LoadOp::bubbleDownCasts(OpBuilder &builder) {
6403 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(),
6404 results: getResult());
6405}
6406
6407//-----------------------------------------------------------------------------
6408// VectorUnrollOpInterface methods
6409//-----------------------------------------------------------------------------
6410std::optional<SmallVector<int64_t, 4>> LoadOp::getIterationShapeToUnroll() {
6411 return llvm::to_vector<4>(Range: getVectorType().getShape());
6412}
6413
6414//===----------------------------------------------------------------------===//
6415// StoreOp
6416//===----------------------------------------------------------------------===//
6417
6418LogicalResult vector::StoreOp::verify() {
6419 VectorType valueVecTy = getVectorType();
6420 MemRefType memRefTy = getMemRefType();
6421
6422 if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: valueVecTy, memRefTy)))
6423 return failure();
6424
6425 // Negative strides are not supported on vector.store. The lowering to LLVM
6426 // emits arithmetic operations (e.g., GEP, mul) with nuw flags that assume
6427 // non-negative strides to avoid undefined behavior.
6428 if (memref::hasNegativeStaticStride(memRefTy))
6429 return emitOpError(message: "memref strides must be non-negative");
6430
6431 if (memRefTy.getRank() < valueVecTy.getRank())
6432 return emitOpError(message: "source memref has lower rank than the vector to store");
6433
6434 // Checks for vector memrefs.
6435 Type memElemTy = memRefTy.getElementType();
6436 if (auto memVecTy = llvm::dyn_cast<VectorType>(Val&: memElemTy)) {
6437 if (memVecTy != valueVecTy)
6438 return emitOpError(
6439 message: "base memref and valueToStore vector types should match");
6440 memElemTy = memVecTy.getElementType();
6441 }
6442
6443 if (valueVecTy.getElementType() != memElemTy)
6444 return emitOpError(message: "base and valueToStore element type should match");
6445 if (llvm::size(Range: getIndices()) != memRefTy.getRank())
6446 return emitOpError(message: "requires ") << memRefTy.getRank() << " indices";
6447 return success();
6448}
6449
6450LogicalResult StoreOp::fold(FoldAdaptor adaptor,
6451 SmallVectorImpl<OpFoldResult> &results) {
6452 return memref::foldMemRefCast(op: *this);
6453}
6454
6455FailureOr<std::optional<SmallVector<Value>>>
6456StoreOp::bubbleDownCasts(OpBuilder &builder) {
6457 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(),
6458 results: ValueRange());
6459}
6460
6461//-----------------------------------------------------------------------------
6462// VectorUnrollOpInterface methods
6463//-----------------------------------------------------------------------------
6464std::optional<SmallVector<int64_t, 4>> StoreOp::getIterationShapeToUnroll() {
6465 return llvm::to_vector<4>(Range: getVectorType().getShape());
6466}
6467
6468//===----------------------------------------------------------------------===//
6469// MaskedLoadOp
6470//===----------------------------------------------------------------------===//
6471
6472LogicalResult MaskedLoadOp::verify() {
6473 VectorType maskVType = getMaskVectorType();
6474 VectorType passVType = getPassThruVectorType();
6475 VectorType resVType = getVectorType();
6476 MemRefType memType = getMemRefType();
6477
6478 if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: resVType, memRefTy: memType)))
6479 return failure();
6480
6481 // Negative strides are not supported on vector.maskedload. The lowering to
6482 // LLVM emits arithmetic operations (e.g., GEP, mul) with nuw flags that
6483 // assume non-negative strides to avoid undefined behavior.
6484 if (memref::hasNegativeStaticStride(memRefTy: memType))
6485 return emitOpError(message: "memref strides must be non-negative");
6486
6487 if (failed(
6488 Result: verifyElementTypesMatch(op: *this, lhs: memType, rhs: resVType, lhsName: "base", rhsName: "result")))
6489 return failure();
6490 if (llvm::size(Range: getIndices()) != memType.getRank())
6491 return emitOpError(message: "requires ") << memType.getRank() << " indices";
6492 if (resVType.getShape() != maskVType.getShape())
6493 return emitOpError(message: "expected result shape to match mask shape");
6494 if (resVType != passVType)
6495 return emitOpError(message: "expected pass_thru of same type as result type");
6496 return success();
6497}
6498
6499namespace {
6500class MaskedLoadFolder final : public OpRewritePattern<MaskedLoadOp> {
6501public:
6502 using Base::Base;
6503 LogicalResult matchAndRewrite(MaskedLoadOp load,
6504 PatternRewriter &rewriter) const override {
6505 switch (getMaskFormat(mask: load.getMask())) {
6506 case MaskFormat::AllTrue:
6507 rewriter.replaceOpWithNewOp<vector::LoadOp>(
6508 op: load, args: load.getType(), args: load.getBase(), args: load.getIndices());
6509 return success();
6510 case MaskFormat::AllFalse:
6511 rewriter.replaceOp(op: load, newValues: load.getPassThru());
6512 return success();
6513 case MaskFormat::Unknown:
6514 return failure();
6515 }
6516 llvm_unreachable("Unexpected 1DMaskFormat on MaskedLoad");
6517 }
6518};
6519} // namespace
6520
6521void MaskedLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
6522 MLIRContext *context) {
6523 results.add<MaskedLoadFolder>(arg&: context);
6524}
6525
6526OpFoldResult MaskedLoadOp::fold(FoldAdaptor) {
6527 if (succeeded(Result: memref::foldMemRefCast(op: *this)))
6528 return getResult();
6529 return OpFoldResult();
6530}
6531
6532FailureOr<std::optional<SmallVector<Value>>>
6533MaskedLoadOp::bubbleDownCasts(OpBuilder &builder) {
6534 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(),
6535 results: getResult());
6536}
6537
6538//===----------------------------------------------------------------------===//
6539// MaskedStoreOp
6540//===----------------------------------------------------------------------===//
6541
6542LogicalResult MaskedStoreOp::verify() {
6543 VectorType maskVType = getMaskVectorType();
6544 VectorType valueVType = getVectorType();
6545 MemRefType memType = getMemRefType();
6546
6547 if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: valueVType, memRefTy: memType)))
6548 return failure();
6549
6550 // Negative strides are not supported on vector.maskedstore. The lowering to
6551 // LLVM emits arithmetic operations (e.g., GEP, mul) with nuw flags that
6552 // assume non-negative strides to avoid undefined behavior.
6553 if (memref::hasNegativeStaticStride(memRefTy: memType))
6554 return emitOpError(message: "memref strides must be non-negative");
6555
6556 if (failed(Result: verifyElementTypesMatch(op: *this, lhs: memType, rhs: valueVType, lhsName: "base",
6557 rhsName: "valueToStore")))
6558 return failure();
6559 if (llvm::size(Range: getIndices()) != memType.getRank())
6560 return emitOpError(message: "requires ") << memType.getRank() << " indices";
6561 if (valueVType.getShape() != maskVType.getShape())
6562 return emitOpError(message: "expected valueToStore shape to match mask shape");
6563 return success();
6564}
6565
6566namespace {
6567class MaskedStoreFolder final : public OpRewritePattern<MaskedStoreOp> {
6568public:
6569 using Base::Base;
6570 LogicalResult matchAndRewrite(MaskedStoreOp store,
6571 PatternRewriter &rewriter) const override {
6572 switch (getMaskFormat(mask: store.getMask())) {
6573 case MaskFormat::AllTrue:
6574 rewriter.replaceOpWithNewOp<vector::StoreOp>(
6575 op: store, args: store.getValueToStore(), args: store.getBase(), args: store.getIndices());
6576 return success();
6577 case MaskFormat::AllFalse:
6578 rewriter.eraseOp(op: store);
6579 return success();
6580 case MaskFormat::Unknown:
6581 return failure();
6582 }
6583 llvm_unreachable("Unexpected 1DMaskFormat on MaskedStore");
6584 }
6585};
6586} // namespace
6587
6588void MaskedStoreOp::getCanonicalizationPatterns(RewritePatternSet &results,
6589 MLIRContext *context) {
6590 results.add<MaskedStoreFolder>(arg&: context);
6591}
6592
6593LogicalResult MaskedStoreOp::fold(FoldAdaptor adaptor,
6594 SmallVectorImpl<OpFoldResult> &results) {
6595 return memref::foldMemRefCast(op: *this);
6596}
6597
6598FailureOr<std::optional<SmallVector<Value>>>
6599MaskedStoreOp::bubbleDownCasts(OpBuilder &builder) {
6600 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(),
6601 results: ValueRange());
6602}
6603
6604//===----------------------------------------------------------------------===//
6605// GatherOp
6606//===----------------------------------------------------------------------===//
6607
6608LogicalResult GatherOp::verify() {
6609 VectorType indVType = getIndexVectorType();
6610 VectorType maskVType = getMaskVectorType();
6611 VectorType resVType = getVectorType();
6612 ShapedType baseType = getBaseType();
6613
6614 if (!llvm::isa<MemRefType, RankedTensorType>(Val: baseType))
6615 return emitOpError(message: "requires base to be a memref or ranked tensor type");
6616
6617 // Negative strides are not supported on vector.gather.
6618 // The lowering to LLVM emits arithmetic operations (e.g., GEP, mul) with nuw
6619 // flags that assume non-negative strides to avoid undefined behavior.
6620 if (auto memRefType = dyn_cast<MemRefType>(Val&: baseType))
6621 if (memref::hasNegativeStaticStride(memRefTy: memRefType))
6622 return emitOpError(message: "memref strides must be non-negative");
6623
6624 if (failed(
6625 Result: verifyElementTypesMatch(op: *this, lhs: baseType, rhs: resVType, lhsName: "base", rhsName: "result")))
6626 return failure();
6627 if (llvm::size(Range: getOffsets()) != baseType.getRank())
6628 return emitOpError(message: "requires ") << baseType.getRank() << " indices";
6629 if (resVType.getShape() != indVType.getShape())
6630 return emitOpError(message: "expected result dim to match indices dim");
6631 if (resVType.getShape() != maskVType.getShape())
6632 return emitOpError(message: "expected result dim to match mask dim");
6633 if (resVType != getPassThruVectorType())
6634 return emitOpError(message: "expected pass_thru of same type as result type");
6635 if (getAlignmentAttr() && !isa<MemRefType>(Val: baseType)) {
6636 return emitOpError(
6637 message: "alignment is only supported for memref bases, not tensor bases");
6638 }
6639 return success();
6640}
6641
6642//-----------------------------------------------------------------------------
6643// MaskableOpInterface methods.
6644//-----------------------------------------------------------------------------
6645/// Cheeck if `indexVec` is constant 1D vec of consecutive values [0, 1, 2, ...]
6646static LogicalResult isZeroBasedContiguousSeq(Value indexVec) {
6647 auto vecType = dyn_cast<VectorType>(Val: indexVec.getType());
6648 if (!vecType || vecType.getRank() != 1 || vecType.isScalable())
6649 return failure();
6650
6651 if (indexVec.getDefiningOp<StepOp>())
6652 return success();
6653
6654 DenseIntElementsAttr elements;
6655 if (!matchPattern(value: indexVec, pattern: m_Constant(bind_value: &elements)))
6656 return failure();
6657
6658 return success(
6659 IsSuccess: llvm::equal(LRange&: elements, RRange: llvm::seq<int64_t>(Begin: 0, End: vecType.getNumElements())));
6660}
6661
6662//-----------------------------------------------------------------------------
6663// VectorUnrollOpInterface methods
6664//-----------------------------------------------------------------------------
6665std::optional<SmallVector<int64_t, 4>> GatherOp::getIterationShapeToUnroll() {
6666 return llvm::to_vector<4>(Range: getVectorType().getShape());
6667}
6668
6669namespace {
6670class GatherFolder final : public OpRewritePattern<GatherOp> {
6671public:
6672 using Base::Base;
6673 LogicalResult matchAndRewrite(GatherOp gather,
6674 PatternRewriter &rewriter) const override {
6675 switch (getMaskFormat(mask: gather.getMask())) {
6676 case MaskFormat::AllTrue:
6677 return failure(); // no unmasked equivalent
6678 case MaskFormat::AllFalse:
6679 rewriter.replaceOp(op: gather, newValues: gather.getPassThru());
6680 return success();
6681 case MaskFormat::Unknown:
6682 return failure();
6683 }
6684 llvm_unreachable("Unexpected 1DMaskFormat on GatherFolder");
6685 }
6686};
6687
6688/// Fold gathers with consecutive offsets [0, 1, 2, ...] into contiguous
6689/// maskedload. Only 1D fixed vectors are supported for now.
6690class FoldContiguousGather final : public OpRewritePattern<GatherOp> {
6691public:
6692 using Base::Base;
6693 LogicalResult matchAndRewrite(GatherOp op,
6694 PatternRewriter &rewriter) const override {
6695 if (!isa<MemRefType>(Val: op.getBase().getType()))
6696 return rewriter.notifyMatchFailure(arg&: op, msg: "base must be of memref type");
6697
6698 if (failed(Result: isZeroBasedContiguousSeq(indexVec: op.getIndices())))
6699 return failure();
6700
6701 rewriter.replaceOpWithNewOp<MaskedLoadOp>(op, args: op.getType(), args: op.getBase(),
6702 args: op.getOffsets(), args: op.getMask(),
6703 args: op.getPassThru());
6704 return success();
6705 }
6706};
6707} // namespace
6708
6709void GatherOp::getCanonicalizationPatterns(RewritePatternSet &results,
6710 MLIRContext *context) {
6711 results.add<GatherFolder, FoldContiguousGather>(arg&: context);
6712}
6713
6714FailureOr<std::optional<SmallVector<Value>>>
6715GatherOp::bubbleDownCasts(OpBuilder &builder) {
6716 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(),
6717 results: getResult());
6718}
6719
6720//===----------------------------------------------------------------------===//
6721// ScatterOp
6722//===----------------------------------------------------------------------===//
6723
6724LogicalResult ScatterOp::verify() {
6725 VectorType indVType = getIndexVectorType();
6726 VectorType maskVType = getMaskVectorType();
6727 VectorType valueVType = getVectorType();
6728 ShapedType baseType = getBaseType();
6729
6730 if (!llvm::isa<MemRefType, RankedTensorType>(Val: baseType))
6731 return emitOpError(message: "requires base to be a memref or ranked tensor type");
6732
6733 // Negative strides are not supported on vector.scatter.
6734 // The lowering to LLVM emits arithmetic operations (e.g., GEP, mul) with nuw
6735 // flags that assume non-negative strides to avoid undefined behavior.
6736 if (auto memRefType = dyn_cast<MemRefType>(Val&: baseType))
6737 if (memref::hasNegativeStaticStride(memRefTy: memRefType))
6738 return emitOpError(message: "memref strides must be non-negative");
6739
6740 if (failed(Result: verifyElementTypesMatch(op: *this, lhs: baseType, rhs: valueVType, lhsName: "base",
6741 rhsName: "valueToStore")))
6742 return failure();
6743 if (llvm::size(Range: getOffsets()) != baseType.getRank())
6744 return emitOpError(message: "requires ") << baseType.getRank() << " indices";
6745 if (valueVType.getShape() != indVType.getShape())
6746 return emitOpError(message: "expected valueToStore dim to match indices dim");
6747 if (valueVType.getShape() != maskVType.getShape())
6748 return emitOpError(message: "expected valueToStore dim to match mask dim");
6749 if (getAlignmentAttr() && !isa<MemRefType>(Val: baseType)) {
6750 return emitOpError(
6751 message: "alignment is only supported for memref bases, not tensor bases");
6752 }
6753 return success();
6754}
6755namespace {
6756class ScatterFolder final : public OpRewritePattern<ScatterOp> {
6757public:
6758 using Base::Base;
6759 LogicalResult matchAndRewrite(ScatterOp scatter,
6760 PatternRewriter &rewriter) const override {
6761 ShapedType baseType = scatter.getBaseType();
6762 bool isMemRef = isa<MemRefType>(Val: baseType);
6763 if (!isMemRef && !isa<RankedTensorType>(Val: baseType))
6764 return failure();
6765
6766 // Memrefs have no result, so an all-false mask can simply erase the op.
6767 // Tensors carry the updated value, so we must replace uses with the
6768 // original base tensor instead of erasing.
6769 switch (getMaskFormat(mask: scatter.getMask())) {
6770 case MaskFormat::AllTrue:
6771 return failure(); // no unmasked equivalent
6772 case MaskFormat::AllFalse:
6773 if (isMemRef)
6774 rewriter.eraseOp(op: scatter);
6775 else
6776 rewriter.replaceOp(op: scatter, newValues: scatter.getBase());
6777 return success();
6778 case MaskFormat::Unknown:
6779 return failure();
6780 }
6781 llvm_unreachable("Unexpected 1DMaskFormat on ScatterFolder");
6782 }
6783};
6784
6785/// Fold scatters with consecutive offsets [0, 1, 2, ...] into contiguous
6786/// maskedstore. Only 1D fixed vectors are supported for now.
6787class FoldContiguousScatter final : public OpRewritePattern<ScatterOp> {
6788public:
6789 using Base::Base;
6790 LogicalResult matchAndRewrite(ScatterOp op,
6791 PatternRewriter &rewriter) const override {
6792 // Fold only for memrefs: the replacement uses maskedstore, which does not
6793 // support tensor bases. Tensor cases intentionally bail out.
6794 if (!isa<MemRefType>(Val: op.getBase().getType()))
6795 return failure();
6796
6797 if (failed(Result: isZeroBasedContiguousSeq(indexVec: op.getIndices())))
6798 return failure();
6799
6800 rewriter.replaceOpWithNewOp<MaskedStoreOp>(
6801 op, args: op.getBase(), args: op.getOffsets(), args: op.getMask(), args: op.getValueToStore());
6802 return success();
6803 }
6804};
6805} // namespace
6806
6807void ScatterOp::getCanonicalizationPatterns(RewritePatternSet &results,
6808 MLIRContext *context) {
6809 results.add<ScatterFolder, FoldContiguousScatter>(arg&: context);
6810}
6811
6812FailureOr<std::optional<SmallVector<Value>>>
6813ScatterOp::bubbleDownCasts(OpBuilder &builder) {
6814 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(),
6815 results: ValueRange());
6816}
6817
6818//===----------------------------------------------------------------------===//
6819// ExpandLoadOp
6820//===----------------------------------------------------------------------===//
6821
6822LogicalResult ExpandLoadOp::verify() {
6823 VectorType maskVType = getMaskVectorType();
6824 VectorType passVType = getPassThruVectorType();
6825 VectorType resVType = getVectorType();
6826 MemRefType memType = getMemRefType();
6827
6828 if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: resVType, memRefTy: memType)))
6829 return failure();
6830
6831 // Negative strides are not supported on vector.expandload. The lowering to
6832 // LLVM emits arithmetic operations (e.g., GEP, mul) with nuw flags that
6833 // assume non-negative strides to avoid undefined behavior.
6834 if (memref::hasNegativeStaticStride(memRefTy: memType))
6835 return emitOpError(message: "memref strides must be non-negative");
6836
6837 if (failed(
6838 Result: verifyElementTypesMatch(op: *this, lhs: memType, rhs: resVType, lhsName: "base", rhsName: "result")))
6839 return failure();
6840 if (llvm::size(Range: getIndices()) != memType.getRank())
6841 return emitOpError(message: "requires ") << memType.getRank() << " indices";
6842 if (resVType.getShape() != maskVType.getShape())
6843 return emitOpError(message: "expected result shape to match mask shape");
6844 if (resVType.getScalableDims() != maskVType.getScalableDims())
6845 return emitOpError(
6846 message: "expected result scalable dims to match mask scalable dims");
6847 if (resVType != passVType)
6848 return emitOpError(message: "expected pass_thru of same type as result type");
6849 return success();
6850}
6851
6852namespace {
6853class ExpandLoadFolder final : public OpRewritePattern<ExpandLoadOp> {
6854public:
6855 using Base::Base;
6856 LogicalResult matchAndRewrite(ExpandLoadOp expand,
6857 PatternRewriter &rewriter) const override {
6858 switch (getMaskFormat(mask: expand.getMask())) {
6859 case MaskFormat::AllTrue:
6860 rewriter.replaceOpWithNewOp<vector::LoadOp>(
6861 op: expand, args: expand.getType(), args: expand.getBase(), args: expand.getIndices());
6862 return success();
6863 case MaskFormat::AllFalse:
6864 rewriter.replaceOp(op: expand, newValues: expand.getPassThru());
6865 return success();
6866 case MaskFormat::Unknown:
6867 return failure();
6868 }
6869 llvm_unreachable("Unexpected 1DMaskFormat on ExpandLoadFolder");
6870 }
6871};
6872} // namespace
6873
6874void ExpandLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
6875 MLIRContext *context) {
6876 results.add<ExpandLoadFolder>(arg&: context);
6877}
6878
6879FailureOr<std::optional<SmallVector<Value>>>
6880ExpandLoadOp::bubbleDownCasts(OpBuilder &builder) {
6881 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(),
6882 results: getResult());
6883}
6884
6885//===----------------------------------------------------------------------===//
6886// CompressStoreOp
6887//===----------------------------------------------------------------------===//
6888
6889LogicalResult CompressStoreOp::verify() {
6890 VectorType maskVType = getMaskVectorType();
6891 VectorType valueVType = getVectorType();
6892 MemRefType memType = getMemRefType();
6893
6894 if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: valueVType, memRefTy: memType)))
6895 return failure();
6896
6897 // Negative strides are not supported on vector.compressstore. The lowering
6898 // to LLVM emits arithmetic operations (e.g., GEP, mul) with nuw flags that
6899 // assume non-negative strides to avoid undefined behavior.
6900 if (memref::hasNegativeStaticStride(memRefTy: memType))
6901 return emitOpError(message: "memref strides must be non-negative");
6902
6903 if (failed(Result: verifyElementTypesMatch(op: *this, lhs: memType, rhs: valueVType, lhsName: "base",
6904 rhsName: "valueToStore")))
6905 return failure();
6906 if (llvm::size(Range: getIndices()) != memType.getRank())
6907 return emitOpError(message: "requires ") << memType.getRank() << " indices";
6908 if (valueVType.getShape() != maskVType.getShape())
6909 return emitOpError(message: "expected valueToStore shape to match mask shape");
6910 if (valueVType.getScalableDims() != maskVType.getScalableDims())
6911 return emitOpError(
6912 message: "expected valueToStore scalable dims to match mask scalable dims");
6913 return success();
6914}
6915
6916namespace {
6917class CompressStoreFolder final : public OpRewritePattern<CompressStoreOp> {
6918public:
6919 using Base::Base;
6920 LogicalResult matchAndRewrite(CompressStoreOp compress,
6921 PatternRewriter &rewriter) const override {
6922 switch (getMaskFormat(mask: compress.getMask())) {
6923 case MaskFormat::AllTrue:
6924 rewriter.replaceOpWithNewOp<vector::StoreOp>(
6925 op: compress, args: compress.getValueToStore(), args: compress.getBase(),
6926 args: compress.getIndices());
6927 return success();
6928 case MaskFormat::AllFalse:
6929 rewriter.eraseOp(op: compress);
6930 return success();
6931 case MaskFormat::Unknown:
6932 return failure();
6933 }
6934 llvm_unreachable("Unexpected 1DMaskFormat on CompressStoreFolder");
6935 }
6936};
6937} // namespace
6938
6939void CompressStoreOp::getCanonicalizationPatterns(RewritePatternSet &results,
6940 MLIRContext *context) {
6941 results.add<CompressStoreFolder>(arg&: context);
6942}
6943
6944FailureOr<std::optional<SmallVector<Value>>>
6945CompressStoreOp::bubbleDownCasts(OpBuilder &builder) {
6946 return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(),
6947 results: ValueRange());
6948}
6949
6950//===----------------------------------------------------------------------===//
6951// ShapeCastOp
6952//===----------------------------------------------------------------------===//
6953
6954void ShapeCastOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges,
6955 SetIntRangeFn setResultRanges) {
6956 setResultRanges(getResult(), argRanges.front());
6957}
6958
6959LogicalResult ShapeCastOp::verify() {
6960
6961 VectorType sourceType = getSourceVectorType();
6962 VectorType resultType = getResultVectorType();
6963
6964 // Check that element type is preserved
6965 if (failed(Result: verifyElementTypesMatch(op: *this, lhs: sourceType, rhs: resultType, lhsName: "source",
6966 rhsName: "result")))
6967 return failure();
6968
6969 // Check that number of elements is preserved
6970 int64_t sourceNElms = sourceType.getNumElements();
6971 int64_t resultNElms = resultType.getNumElements();
6972 if (sourceNElms != resultNElms) {
6973 return emitOpError() << "has different number of elements at source ("
6974 << sourceNElms << ") and result (" << resultNElms
6975 << ")";
6976 }
6977
6978 // Check that (non-)scalability is preserved
6979 int64_t sourceNScalableDims = sourceType.getNumScalableDims();
6980 int64_t resultNScalableDims = resultType.getNumScalableDims();
6981 if (sourceNScalableDims != resultNScalableDims)
6982 return emitOpError() << "has different number of scalable dims at source ("
6983 << sourceNScalableDims << ") and result ("
6984 << resultNScalableDims << ")";
6985
6986 return success();
6987}
6988
6989/// Check whether this ShapeCastOp is effectively a BroadcastOp.
6990///
6991/// The only case in which this method can return `true` is when the underlying
6992/// op merely adds leading unit dimensions, e.g.:
6993/// %res = vector.shape_cast %src : vector<8x4xi32> to vector<1x8x4xi32>
6994///
6995bool ShapeCastOp::isBroadcastLike() {
6996 auto srcType = getSourceVectorType();
6997 auto resType = getResultVectorType();
6998
6999 // Is srcType broadcastable to resType?
7000 std::pair<VectorDim, VectorDim> mismatchingDims;
7001 if (isBroadcastableTo(srcType, dstVectorType: resType, mismatchingDims: &mismatchingDims) !=
7002 BroadcastableToResult::Success)
7003 return false;
7004
7005 // Do ranks mismatch?
7006 //
7007 // The only case where ranks match and this ShapeCastOp is also a broadcast,
7008 // is when it's effectively a NOp, but that's an uninteresting edge case.
7009 size_t rankDiff = resType.getRank() - srcType.getRank();
7010 if (rankDiff == 0)
7011 return false;
7012
7013 // Are all newly added leading dims unit?
7014 if (!llvm::all_of(Range: resType.getShape().take_front(N: rankDiff),
7015 P: [](int64_t dim) { return dim == 1; }))
7016 return false;
7017
7018 // Do all trailing dims match?
7019 return resType.getShape().take_back(N: srcType.getRank()) == srcType.getShape();
7020}
7021
7022/// Return true if `transpose` does not permute a pair of non-unit dims.
7023/// By `order preserving` we mean that the flattened versions of the input and
7024/// output vectors are (numerically) identical. In other words `transpose` is
7025/// effectively a shape cast.
7026static bool isOrderPreserving(TransposeOp transpose) {
7027 ArrayRef<int64_t> permutation = transpose.getPermutation();
7028 VectorType sourceType = transpose.getSourceVectorType();
7029 ArrayRef<int64_t> inShape = sourceType.getShape();
7030 ArrayRef<bool> inDimIsScalable = sourceType.getScalableDims();
7031 auto isNonScalableUnitDim = [&](int64_t dim) {
7032 return inShape[dim] == 1 && !inDimIsScalable[dim];
7033 };
7034 int64_t current = 0;
7035 for (auto p : permutation) {
7036 if (!isNonScalableUnitDim(p)) {
7037 if (p < current) {
7038 return false;
7039 }
7040 current = p;
7041 }
7042 }
7043 return true;
7044}
7045
7046OpFoldResult ShapeCastOp::fold(FoldAdaptor adaptor) {
7047
7048 VectorType resultType = getType();
7049
7050 // No-op shape cast.
7051 if (getSource().getType() == resultType)
7052 return getSource();
7053
7054 // shape_cast(shape_cast(x)) -> shape_cast(x)
7055 if (auto precedingShapeCast = getSource().getDefiningOp<ShapeCastOp>()) {
7056 setOperand(precedingShapeCast.getSource());
7057 return getResult();
7058 }
7059
7060 // shape_cast(transpose(x)) -> shape_cast(x)
7061 if (auto transpose = getSource().getDefiningOp<TransposeOp>()) {
7062 if (isOrderPreserving(transpose)) {
7063 setOperand(transpose.getVector());
7064 return getResult();
7065 }
7066 return {};
7067 }
7068
7069 // Y = shape_cast(broadcast(X))
7070 // -> X, if X and Y have same type
7071 if (auto bcastOp = getSource().getDefiningOp<BroadcastOp>()) {
7072 if (bcastOp.getSourceType() == resultType)
7073 return bcastOp.getSource();
7074 }
7075
7076 // shape_cast(constant) -> constant
7077 if (auto denseAttr =
7078 dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getSource()))
7079 return denseAttr.reshape(newType: getType());
7080
7081 // shape_cast(poison) -> poison
7082 if (matchPattern(attr: adaptor.getSource(), pattern: ub::m_Poison()))
7083 return ub::PoisonAttr::get(ctx: getContext());
7084
7085 return {};
7086}
7087
7088//-----------------------------------------------------------------------------
7089// VectorUnrollOpInterface methods
7090//-----------------------------------------------------------------------------
7091std::optional<SmallVector<int64_t, 4>>
7092ShapeCastOp::getIterationShapeToUnroll() {
7093 return llvm::to_vector<4>(Range: getResultVectorType().getShape());
7094}
7095
7096std::optional<SmallVector<int64_t>> ShapeCastOp::getDefaultUnrollTargetShape() {
7097 auto resultShape = this->getResultVectorType().getShape();
7098 // Special case with leading unit dims and different inner dim
7099 // for result and target shape.
7100 if (resultShape.size() == 2 && resultShape[0] == 1 && resultShape[1] == 32) {
7101 return SmallVector<int64_t>{1, 16};
7102 }
7103 if (resultShape.size() == 2 && resultShape[0] == 2 && resultShape[1] == 1) {
7104 return SmallVector<int64_t>{1, 1};
7105 }
7106 // Multi-group cases: tile contiguous per reassociation group
7107 // but strided in the whole result.
7108 auto sourceShape = this->getSourceVectorType().getShape();
7109 if (resultShape.size() == 3 && resultShape[0] == 8 && resultShape[1] == 1 &&
7110 resultShape[2] == 32) {
7111 return SmallVector<int64_t>{8, 1, 4};
7112 }
7113 if (sourceShape.size() == 3 && resultShape.size() == 2 &&
7114 resultShape[0] == 4 && resultShape[1] == 4) {
7115 return SmallVector<int64_t>{2, 2};
7116 }
7117 if (resultShape.size() == 3 && resultShape[0] == 2 && resultShape[1] == 8 &&
7118 resultShape[2] == 4) {
7119 return SmallVector<int64_t>{2, 2, 2};
7120 }
7121 // Default case: [2,4] for all tests.
7122 return SmallVector<int64_t>{2, 4};
7123}
7124
7125namespace {
7126
7127/// Helper function that computes a new vector type based on the input vector
7128/// type by removing the _trailing_ unit dims:
7129///
7130/// vector<4x1x1xi1> --> vector<4x1xi1>
7131///
7132static VectorType trimTrailingUnitDims(VectorType oldType) {
7133 ArrayRef<int64_t> oldShape = oldType.getShape();
7134 ArrayRef<int64_t> newShape = oldShape;
7135
7136 ArrayRef<bool> oldScalableDims = oldType.getScalableDims();
7137 ArrayRef<bool> newScalableDims = oldScalableDims;
7138
7139 while (!newShape.empty() && newShape.back() == 1 && !newScalableDims.back()) {
7140 newShape = newShape.drop_back(N: 1);
7141 newScalableDims = newScalableDims.drop_back(N: 1);
7142 }
7143
7144 // Make sure we have at least 1 dimension.
7145 // TODO: Add support for 0-D vectors.
7146 if (newShape.empty()) {
7147 newShape = oldShape.take_back();
7148 newScalableDims = oldScalableDims.take_back();
7149 }
7150
7151 return VectorType::get(shape: newShape, elementType: oldType.getElementType(), scalableDims: newScalableDims);
7152}
7153
7154/// Helper function that computes a new vector type based on the input vector
7155/// type by removing the _leading_ unit dims:
7156///
7157/// vector<1x1x4xi1> --> vector<4x1xi1>
7158///
7159static VectorType trimLeadingUnitDims(VectorType oldType) {
7160 ArrayRef<int64_t> oldShape = oldType.getShape();
7161 ArrayRef<int64_t> newShape = oldShape;
7162
7163 ArrayRef<bool> oldScalableDims = oldType.getScalableDims();
7164 ArrayRef<bool> newScalableDims = oldScalableDims;
7165
7166 while (!newShape.empty() && newShape.front() == 1 &&
7167 !newScalableDims.front()) {
7168 newShape = newShape.drop_front(N: 1);
7169 newScalableDims = newScalableDims.drop_front(N: 1);
7170 }
7171
7172 // Make sure we have at least 1 dimension.
7173 // TODO: Add support for 0-D vectors.
7174 if (newShape.empty()) {
7175 newShape = oldShape.take_back();
7176 newScalableDims = oldScalableDims.take_back();
7177 }
7178
7179 return VectorType::get(shape: newShape, elementType: oldType.getElementType(), scalableDims: newScalableDims);
7180}
7181
7182enum class UnitDimSide { Leading, Trailing };
7183
7184/// Folds qualifying shape_cast(create_mask) into a new create_mask
7185///
7186/// Looks at `vector.shape_cast` Ops that simply "drop" either the _trailing_ or
7187/// the _leading_unit dimension (configured with the template parameter). If the
7188/// input vector comes from `vector.create_mask` for which the corresponding
7189/// mask input value is 1 (e.g. `%c1` below), then it is safe to fold shape_cast
7190/// into create_mask.
7191///
7192/// EX 1 (trailing unit dims)
7193/// BEFORE:
7194/// %1 = vector.create_mask %c1, %dim, %c1, %c1 : vector<1x[4]x1x1xi1>
7195/// %2 = vector.shape_cast %1 : vector<1x[4]x1x1xi1> to vector<1x[4]xi1>
7196/// AFTER:
7197/// %0 = vector.create_mask %c1, %dim : vector<1x[4]xi1>
7198///
7199/// EX 2 (leading unit dims)
7200/// BEFORE:
7201/// %1 = vector.create_mask %c1, %c1, %dim, %c1 : vector<1x1x[4]x1xi1>
7202/// %2 = vector.shape_cast %1 : vector<1x1x[4]x1xi1> to vector<[4]x1xi1>
7203/// AFTER:
7204/// %0 = vector.create_mask %c1, %dim : vector<[4]xi1>
7205template <UnitDimSide Side>
7206class ShapeCastCreateMaskFolderBoundaryUnitDim final
7207 : public OpRewritePattern<ShapeCastOp> {
7208public:
7209 using Base::Base;
7210
7211 LogicalResult matchAndRewrite(ShapeCastOp shapeOp,
7212 PatternRewriter &rewriter) const override {
7213 Value shapeOpSrc = shapeOp->getOperand(idx: 0);
7214 auto createMaskOp = shapeOpSrc.getDefiningOp<vector::CreateMaskOp>();
7215 auto constantMaskOp = shapeOpSrc.getDefiningOp<vector::ConstantMaskOp>();
7216 if (!createMaskOp && !constantMaskOp)
7217 return failure();
7218
7219 VectorType shapeOpResTy = shapeOp.getResultVectorType();
7220 VectorType shapeOpSrcTy = shapeOp.getSourceVectorType();
7221
7222 VectorType newVecType = (Side == UnitDimSide::Trailing)
7223 ? trimTrailingUnitDims(oldType: shapeOpSrcTy)
7224 : trimLeadingUnitDims(oldType: shapeOpSrcTy);
7225
7226 if (newVecType != shapeOpResTy) {
7227 return (Side == UnitDimSide::Trailing)
7228 ? rewriter.notifyMatchFailure(
7229 arg&: shapeOp, msg: "Non-trailing-unit-dim dropping shape_cast Op")
7230 : rewriter.notifyMatchFailure(
7231 arg&: shapeOp, msg: "Non-leading-unit-dim dropping shape_cast Op");
7232 }
7233
7234 auto numDimsToDrop = shapeOpSrcTy.getRank() - shapeOpResTy.getRank();
7235
7236 // No unit dims to drop
7237 if (!numDimsToDrop) {
7238 return (Side == UnitDimSide::Trailing)
7239 ? rewriter.notifyMatchFailure(
7240 arg&: shapeOp, msg: "Non-trailing-unit-dim dropping shape_cast Op")
7241 : rewriter.notifyMatchFailure(
7242 arg&: shapeOp, msg: "Non-leading-unit-dim dropping shape_cast Op");
7243 }
7244
7245 if (createMaskOp) {
7246 auto maskOperands = createMaskOp.getOperands();
7247
7248 auto maskOperandsToDrop = (Side == UnitDimSide::Trailing)
7249 ? maskOperands.take_back(n: numDimsToDrop)
7250 : maskOperands.take_front(n: numDimsToDrop);
7251
7252 // Check that every mask-dim-size to-be-dropped is constant and == 1. We
7253 // could also check for == 0, but that's left as a TODO.
7254 if (llvm::any_of(maskOperandsToDrop, [](Value maskDim) {
7255 auto cst = maskDim.getDefiningOp<arith::ConstantIndexOp>();
7256 return !cst || (cst.value() != 1);
7257 }))
7258 return failure();
7259
7260 auto newMaskOperands = (Side == UnitDimSide::Trailing)
7261 ? maskOperands.drop_back(n: numDimsToDrop)
7262 : maskOperands.drop_front(n: numDimsToDrop);
7263 rewriter.replaceOpWithNewOp<vector::CreateMaskOp>(op: shapeOp, args&: shapeOpResTy,
7264 args&: newMaskOperands);
7265 return success();
7266 }
7267
7268 if (constantMaskOp) {
7269 auto maskDimSizes = constantMaskOp.getMaskDimSizes();
7270
7271 ArrayRef<int64_t> maskDimSizesToDrop =
7272 (Side == UnitDimSide::Trailing)
7273 ? maskDimSizes.take_back(N: numDimsToDrop)
7274 : maskDimSizes.take_front(N: numDimsToDrop);
7275
7276 // Check that every mask-dim-size to-be-dropped is constant and == 1. We
7277 // could also check for == 0, but that's left as a TODO.
7278 if (llvm::any_of(maskDimSizesToDrop,
7279 [](int64_t dim) { return dim != 1; }))
7280 return failure();
7281
7282 ArrayRef<int64_t> newMaskDimSizes =
7283 (Side == UnitDimSide::Trailing)
7284 ? maskDimSizes.drop_back(N: numDimsToDrop)
7285 : maskDimSizes.drop_front(N: numDimsToDrop);
7286 rewriter.replaceOpWithNewOp<vector::ConstantMaskOp>(op: shapeOp, args&: shapeOpResTy,
7287 args&: newMaskDimSizes);
7288 return success();
7289 }
7290
7291 return failure();
7292 }
7293};
7294
7295/// Pattern to rewrite Y = ShapeCast(Broadcast(X)) as Y = Broadcast(X)
7296class ShapeCastBroadcastFolder final : public OpRewritePattern<ShapeCastOp> {
7297public:
7298 using Base::Base;
7299
7300 LogicalResult matchAndRewrite(ShapeCastOp shapeCastOp,
7301 PatternRewriter &rewriter) const override {
7302 auto oldBcOp = shapeCastOp.getSource().getDefiningOp<vector::BroadcastOp>();
7303 if (!oldBcOp)
7304 return failure();
7305
7306 // Replace Y = ShapeCast(Broadcast(X)) with Y = Broadcast(X)
7307 // Example
7308 // %oldBc = vector.broadcast %src : vector<3xf32> to vector<2x4x3xf32>
7309 // %res = vector.shape_cast %oldBc : vector<2x4x3xf32> to vector<8x3xf32>
7310 // to
7311 // %newBc = vector.broadcast %src : vector<3xf32> to vector<8x3xf32>
7312 VectorType resVectorType = shapeCastOp.getResultVectorType();
7313
7314 // Limit this pattern to vectors with at most one scalable dimension for
7315 // now.
7316 if (resVectorType.getNumScalableDims() > 1)
7317 return failure();
7318
7319 auto srcVectorType = dyn_cast<VectorType>(Val: oldBcOp.getSourceType());
7320 bool srcIsScalar = !srcVectorType;
7321
7322 if (!srcIsScalar) {
7323 if (isBroadcastableTo(srcType: srcVectorType, dstVectorType: resVectorType) !=
7324 BroadcastableToResult::Success) {
7325 return failure();
7326 }
7327 // For a fixed source shape, the result strides determine which source
7328 // element supplies each linear position in the broadcast result (see the
7329 // `BroadcastOp` definition for the closed-form mapping). Since
7330 // `shape_cast` preserves the flattened 1-D (row-major) element order,
7331 // folding into `%newBc` is valid when every non-unit `%src` dimension has
7332 // the same stride (both static product and scalability) in `%oldBc` and
7333 // `%res`. Note that a scalable `[1]` has runtime size `vscale` and is not
7334 // a unit dimension.
7335 VectorType oldBcVectorType = oldBcOp.getResultVectorType();
7336 int64_t srcRank = srcVectorType.getRank();
7337 int64_t oldBcRank = oldBcVectorType.getRank();
7338 int64_t resRank = resVectorType.getRank();
7339 ArrayRef<int64_t> srcShape = srcVectorType.getShape();
7340 ArrayRef<int64_t> oldBcShape = oldBcVectorType.getShape();
7341 ArrayRef<int64_t> resShape = resVectorType.getShape();
7342 ArrayRef<bool> srcScalable = srcVectorType.getScalableDims();
7343 ArrayRef<bool> oldBcScalable = oldBcVectorType.getScalableDims();
7344 ArrayRef<bool> resScalable = resVectorType.getScalableDims();
7345
7346 // Iterate from the innermost (trailing) dimension outward, aligning each
7347 // `%src` dimension with the corresponding trailing dimension of `%oldBc`
7348 // and `%res`.
7349 int64_t oldBcStride = 1, resStride = 1;
7350 bool isOldBcStrideScalable = false, isResStrideScalable = false;
7351 for (int64_t i = 0; i < srcRank; ++i) {
7352 int64_t srcDim = srcRank - 1 - i;
7353 int64_t oldBcDim = oldBcRank - 1 - i;
7354 int64_t resDim = resRank - 1 - i;
7355
7356 bool isUnitDim = srcShape[srcDim] == 1 && !srcScalable[srcDim];
7357 if (!isUnitDim && (oldBcStride != resStride ||
7358 isOldBcStrideScalable != isResStrideScalable)) {
7359 return failure();
7360 }
7361
7362 oldBcStride *= oldBcShape[oldBcDim];
7363 resStride *= resShape[resDim];
7364 isOldBcStrideScalable |= oldBcScalable[oldBcDim];
7365 isResStrideScalable |= resScalable[resDim];
7366 }
7367 }
7368
7369 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: shapeCastOp, args&: resVectorType,
7370 args: oldBcOp.getSource());
7371 return success();
7372 }
7373};
7374
7375/// Pattern to rewrite Y = ShapeCast(FromElements(X)) as Y = FromElements(X)
7376///
7377/// BEFORE:
7378/// %1 = vector.from_elements %c1, %c2, %c3 : vector<3xf32>
7379/// %2 = vector.shape_cast %1 : vector<3xf32> to vector<1x3xf32>
7380/// AFTER:
7381/// %2 = vector.from_elements %c1, %c2, %c3 : vector<1x3xf32>
7382///
7383/// Note: this transformation is implemented as an OpRewritePattern, not as a
7384/// fold, because we have to create new op FromElementsOp with updated result
7385/// type. This cannot be done with a fold, because fold cannot create new ops
7386/// and the existing FromElementsOp result type differs from the ShapeCastOp
7387/// result type. Mutating the FromElementsOp (not root op) would violate the
7388/// fold contract and break other users.
7389class FoldShapeCastOfFromElements final : public OpRewritePattern<ShapeCastOp> {
7390public:
7391 using Base::Base;
7392
7393 LogicalResult matchAndRewrite(ShapeCastOp shapeCastOp,
7394 PatternRewriter &rewriter) const override {
7395 auto fromElements = shapeCastOp.getSource().getDefiningOp<FromElementsOp>();
7396 if (!fromElements)
7397 return failure();
7398
7399 rewriter.replaceOpWithNewOp<FromElementsOp>(
7400 op: shapeCastOp, args: shapeCastOp.getResultVectorType(),
7401 args: fromElements.getElements());
7402 return success();
7403 }
7404};
7405
7406} // namespace
7407
7408void ShapeCastOp::getCanonicalizationPatterns(RewritePatternSet &results,
7409 MLIRContext *context) {
7410 results.add<ShapeCastCreateMaskFolderBoundaryUnitDim<UnitDimSide::Leading>,
7411 ShapeCastCreateMaskFolderBoundaryUnitDim<UnitDimSide::Trailing>,
7412 ShapeCastBroadcastFolder, FoldShapeCastOfFromElements>(arg&: context);
7413}
7414
7415//===----------------------------------------------------------------------===//
7416// VectorBitCastOp
7417//===----------------------------------------------------------------------===//
7418
7419LogicalResult BitCastOp::verify() {
7420 auto sourceVectorType = getSourceVectorType();
7421 auto resultVectorType = getResultVectorType();
7422
7423 for (int64_t i = 0, e = sourceVectorType.getRank() - 1; i < e; i++) {
7424 if (sourceVectorType.getDimSize(idx: i) != resultVectorType.getDimSize(idx: i))
7425 return emitOpError(message: "dimension size mismatch at: ") << i;
7426 }
7427
7428 DataLayout dataLayout = DataLayout::closest(op: *this);
7429 auto sourceElementBits =
7430 dataLayout.getTypeSizeInBits(t: sourceVectorType.getElementType());
7431 auto resultElementBits =
7432 dataLayout.getTypeSizeInBits(t: resultVectorType.getElementType());
7433
7434 if (sourceVectorType.getRank() == 0) {
7435 if (sourceElementBits != resultElementBits)
7436 return emitOpError(message: "source/result bitwidth of the 0-D vector element "
7437 "types must be equal");
7438 } else if (sourceElementBits * sourceVectorType.getShape().back() !=
7439 resultElementBits * resultVectorType.getShape().back()) {
7440 return emitOpError(
7441 message: "source/result bitwidth of the minor 1-D vectors must be equal");
7442 }
7443
7444 return success();
7445}
7446
7447OpFoldResult BitCastOp::fold(FoldAdaptor adaptor) {
7448 // Nop cast.
7449 if (getSource().getType() == getResult().getType())
7450 return getSource();
7451
7452 // Canceling bitcasts.
7453 if (auto otherOp = getSource().getDefiningOp<BitCastOp>()) {
7454 if (getResult().getType() == otherOp.getSource().getType())
7455 return otherOp.getSource();
7456
7457 setOperand(otherOp.getSource());
7458 return getResult();
7459 }
7460
7461 Attribute sourceConstant = adaptor.getSource();
7462 if (!sourceConstant)
7463 return {};
7464
7465 Type srcElemType = getSourceVectorType().getElementType();
7466 Type dstElemType = getResultVectorType().getElementType();
7467
7468 if (auto floatPack = llvm::dyn_cast<DenseFPElementsAttr>(Val&: sourceConstant)) {
7469 if (floatPack.isSplat()) {
7470 auto splat = floatPack.getSplatValue<FloatAttr>();
7471
7472 // Casting fp16 into fp32.
7473 if (srcElemType.isF16() && dstElemType.isF32()) {
7474 uint32_t bits = static_cast<uint32_t>(
7475 splat.getValue().bitcastToAPInt().getZExtValue());
7476 // Duplicate the 16-bit pattern.
7477 bits = (bits << 16) | (bits & 0xffff);
7478 APInt intBits(32, bits);
7479 APFloat floatBits(llvm::APFloat::IEEEsingle(), intBits);
7480 return DenseElementsAttr::get(type: getResultVectorType(), values: floatBits);
7481 }
7482 }
7483 }
7484
7485 if (auto intPack = llvm::dyn_cast<DenseIntElementsAttr>(Val&: sourceConstant)) {
7486 if (intPack.isSplat()) {
7487 auto splat = intPack.getSplatValue<IntegerAttr>();
7488
7489 if (llvm::isa<IntegerType>(Val: dstElemType) && srcElemType.isIntOrFloat()) {
7490 uint64_t srcBitWidth = srcElemType.getIntOrFloatBitWidth();
7491 uint64_t dstBitWidth = dstElemType.getIntOrFloatBitWidth();
7492
7493 // Casting to a larger integer bit width.
7494 if (dstBitWidth > srcBitWidth && dstBitWidth % srcBitWidth == 0) {
7495 APInt intBits = splat.getValue().zext(width: dstBitWidth);
7496
7497 // Duplicate the lower width element.
7498 for (uint64_t i = 0; i < dstBitWidth / srcBitWidth - 1; i++)
7499 intBits = (intBits << srcBitWidth) | intBits;
7500 return DenseElementsAttr::get(type: getResultVectorType(), values: intBits);
7501 }
7502 }
7503 }
7504 }
7505
7506 return {};
7507}
7508
7509//-----------------------------------------------------------------------------
7510// VectorUnrollOpInterface methods
7511//-----------------------------------------------------------------------------
7512std::optional<SmallVector<int64_t, 4>> BitCastOp::getIterationShapeToUnroll() {
7513 return llvm::to_vector<4>(Range: getResultVectorType().getShape());
7514}
7515
7516std::optional<SmallVector<int64_t>> BitCastOp::getDefaultUnrollTargetShape() {
7517 return SmallVector<int64_t>{4, 4};
7518}
7519
7520//===----------------------------------------------------------------------===//
7521// TypeCastOp
7522//===----------------------------------------------------------------------===//
7523
7524static SmallVector<int64_t, 8> extractShape(MemRefType memRefType) {
7525 auto vectorType = llvm::dyn_cast<VectorType>(Val: memRefType.getElementType());
7526 SmallVector<int64_t, 8> res(memRefType.getShape());
7527 if (vectorType)
7528 res.append(in_start: vectorType.getShape().begin(), in_end: vectorType.getShape().end());
7529 return res;
7530}
7531
7532/// Build the canonical memRefType with a single vector.
7533/// E.g. memref<4 x 5 x vector<6 x f32>> -> memref<vector<4 x 5 x 6 x f32>>.
7534void TypeCastOp::build(OpBuilder &builder, OperationState &result,
7535 Value source) {
7536 result.addOperands(newOperands: source);
7537 MemRefType memRefType = llvm::cast<MemRefType>(Val: source.getType());
7538 VectorType vectorType =
7539 VectorType::get(shape: extractShape(memRefType),
7540 elementType: getElementTypeOrSelf(type: getElementTypeOrSelf(type: memRefType)));
7541 result.addTypes(newTypes: MemRefType::get(shape: {}, elementType: vectorType, layout: MemRefLayoutAttrInterface(),
7542 memorySpace: memRefType.getMemorySpace()));
7543}
7544
7545LogicalResult TypeCastOp::verify() {
7546 MemRefType canonicalType = getMemRefType().canonicalizeStridedLayout();
7547 if (!canonicalType.getLayout().isIdentity())
7548 return emitOpError(message: "expects operand to be a memref with identity layout");
7549 if (!getResultMemRefType().getLayout().isIdentity())
7550 return emitOpError(message: "expects result to be a memref with identity layout");
7551 if (getResultMemRefType().getMemorySpace() !=
7552 getMemRefType().getMemorySpace())
7553 return emitOpError(message: "expects result in same memory space");
7554
7555 auto sourceType = getMemRefType();
7556 auto resultType = getResultMemRefType();
7557 if (getElementTypeOrSelf(type: getElementTypeOrSelf(type: sourceType)) !=
7558 getElementTypeOrSelf(type: getElementTypeOrSelf(type: resultType)))
7559 return emitOpError(
7560 message: "expects result and operand with same underlying scalar type: ")
7561 << resultType;
7562 if (extractShape(memRefType: sourceType) != extractShape(memRefType: resultType))
7563 return emitOpError(
7564 message: "expects concatenated result and operand shapes to be equal: ")
7565 << resultType;
7566 return success();
7567}
7568
7569//===----------------------------------------------------------------------===//
7570// TransposeOp
7571//===----------------------------------------------------------------------===//
7572
7573void vector::TransposeOp::build(OpBuilder &builder, OperationState &result,
7574 Value vector, ArrayRef<int64_t> permutation) {
7575 VectorType vt = llvm::cast<VectorType>(Val: vector.getType());
7576 SmallVector<int64_t, 4> transposedShape(vt.getRank());
7577 SmallVector<bool, 4> transposedScalableDims(vt.getRank());
7578 for (unsigned i = 0; i < permutation.size(); ++i) {
7579 transposedShape[i] = vt.getShape()[permutation[i]];
7580 transposedScalableDims[i] = vt.getScalableDims()[permutation[i]];
7581 }
7582
7583 result.addOperands(newOperands: vector);
7584 result.addTypes(newTypes: VectorType::get(shape: transposedShape, elementType: vt.getElementType(),
7585 scalableDims: transposedScalableDims));
7586 result.addAttribute(name: TransposeOp::getPermutationAttrName(name: result.name),
7587 attr: builder.getDenseI64ArrayAttr(values: permutation));
7588}
7589
7590OpFoldResult vector::TransposeOp::fold(FoldAdaptor adaptor) {
7591 // Eliminate splat constant transpose ops.
7592 if (auto splat =
7593 llvm::dyn_cast_if_present<SplatElementsAttr>(Val: adaptor.getVector()))
7594 return splat.reshape(newType: getResultVectorType());
7595
7596 // Eliminate poison transpose ops.
7597 if (matchPattern(attr: adaptor.getVector(), pattern: ub::m_Poison()))
7598 return ub::PoisonAttr::get(ctx: getContext());
7599
7600 // Eliminate identity transposes, and more generally any transposes that
7601 // preserves the shape without permuting elements.
7602 //
7603 // Examples of what to fold:
7604 // %0 = vector.transpose %arg, [0, 1] : vector<1x1xi8> to vector<1x1xi8>
7605 // %0 = vector.transpose %arg, [0, 1] : vector<2x2xi8> to vector<2x2xi8>
7606 // %0 = vector.transpose %arg, [1, 0] : vector<1x1xi8> to vector<1x1xi8>
7607 //
7608 // Example of what NOT to fold:
7609 // %0 = vector.transpose %arg, [1, 0] : vector<2x2xi8> to vector<2x2xi8>
7610 //
7611 if (getSourceVectorType() == getResultVectorType() &&
7612 isOrderPreserving(transpose: *this))
7613 return getVector();
7614
7615 return {};
7616}
7617
7618LogicalResult vector::TransposeOp::verify() {
7619 VectorType vectorType = getSourceVectorType();
7620 VectorType resultType = getResultVectorType();
7621 int64_t rank = resultType.getRank();
7622 if (vectorType.getRank() != rank)
7623 return emitOpError(message: "vector result rank mismatch: ") << rank;
7624 // Verify transposition array.
7625 ArrayRef<int64_t> perm = getPermutation();
7626 int64_t size = perm.size();
7627 if (rank != size)
7628 return emitOpError(message: "transposition length mismatch: ") << size;
7629 SmallVector<bool, 8> seen(rank, false);
7630 for (const auto &ta : llvm::enumerate(First&: perm)) {
7631 if (ta.value() < 0 || ta.value() >= rank)
7632 return emitOpError(message: "transposition index out of range: ") << ta.value();
7633 if (seen[ta.value()])
7634 return emitOpError(message: "duplicate position index: ") << ta.value();
7635 seen[ta.value()] = true;
7636 if (resultType.getDimSize(idx: ta.index()) != vectorType.getDimSize(idx: ta.value()))
7637 return emitOpError(message: "dimension size mismatch at: ") << ta.value();
7638 }
7639 return success();
7640}
7641
7642void TransposeOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges,
7643 SetIntRangeFn setResultRanges) {
7644 setResultRanges(getResult(), argRanges.front());
7645}
7646
7647//-----------------------------------------------------------------------------
7648// VectorUnrollOpInterface methods
7649//-----------------------------------------------------------------------------
7650std::optional<SmallVector<int64_t, 4>>
7651TransposeOp::getIterationShapeToUnroll() {
7652 return llvm::to_vector<4>(Range: getResultVectorType().getShape());
7653}
7654
7655std::optional<SmallVector<int64_t>> TransposeOp::getDefaultUnrollTargetShape() {
7656 return SmallVector<int64_t>{1, 3, 4, 2};
7657}
7658
7659namespace {
7660
7661// Rewrites two back-to-back TransposeOp operations into a single TransposeOp.
7662class TransposeFolder final : public OpRewritePattern<vector::TransposeOp> {
7663public:
7664 using Base::Base;
7665
7666 LogicalResult matchAndRewrite(vector::TransposeOp transposeOp,
7667 PatternRewriter &rewriter) const override {
7668 // Composes two permutations: result[i] = permutation1[permutation2[i]].
7669 auto composePermutations = [](ArrayRef<int64_t> permutation1,
7670 ArrayRef<int64_t> permutation2) {
7671 SmallVector<int64_t, 4> result;
7672 for (auto index : permutation2)
7673 result.push_back(Elt: permutation1[index]);
7674 return result;
7675 };
7676
7677 // Return if the input of 'transposeOp' is not defined by another transpose.
7678 vector::TransposeOp parentTransposeOp =
7679 transposeOp.getVector().getDefiningOp<vector::TransposeOp>();
7680 if (!parentTransposeOp)
7681 return failure();
7682
7683 SmallVector<int64_t, 4> permutation = composePermutations(
7684 parentTransposeOp.getPermutation(), transposeOp.getPermutation());
7685 // Replace 'transposeOp' with a new transpose operation.
7686 rewriter.replaceOpWithNewOp<vector::TransposeOp>(
7687 op: transposeOp, args: transposeOp.getResult().getType(),
7688 args: parentTransposeOp.getVector(), args&: permutation);
7689 return success();
7690 }
7691};
7692
7693/// Replace transpose(splat-like(v)) with broadcast(v)
7694class FoldTransposeSplat final : public OpRewritePattern<TransposeOp> {
7695public:
7696 using Base::Base;
7697
7698 LogicalResult matchAndRewrite(TransposeOp transposeOp,
7699 PatternRewriter &rewriter) const override {
7700 Value splat = getScalarSplatSource(value: transposeOp.getVector());
7701 if (!splat)
7702 return failure();
7703
7704 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
7705 op: transposeOp, args: transposeOp.getResultVectorType(), args&: splat);
7706 return success();
7707 }
7708};
7709
7710/// Folds transpose(create_mask) into a new transposed create_mask.
7711class FoldTransposeCreateMask final : public OpRewritePattern<TransposeOp> {
7712public:
7713 using Base::Base;
7714
7715 LogicalResult matchAndRewrite(TransposeOp transpOp,
7716 PatternRewriter &rewriter) const override {
7717 Value transposeSrc = transpOp.getVector();
7718 auto createMaskOp = transposeSrc.getDefiningOp<vector::CreateMaskOp>();
7719 auto constantMaskOp = transposeSrc.getDefiningOp<vector::ConstantMaskOp>();
7720 if (!createMaskOp && !constantMaskOp)
7721 return failure();
7722
7723 // Get the transpose permutation and apply it to the vector.create_mask or
7724 // vector.constant_mask operands.
7725 ArrayRef<int64_t> permutation = transpOp.getPermutation();
7726
7727 if (createMaskOp) {
7728 auto maskOperands = createMaskOp.getOperands();
7729 SmallVector<Value> newOperands(maskOperands.begin(), maskOperands.end());
7730 applyPermutationToVector(inVec&: newOperands, permutation);
7731
7732 rewriter.replaceOpWithNewOp<vector::CreateMaskOp>(
7733 op: transpOp, args: transpOp.getResultVectorType(), args&: newOperands);
7734 return success();
7735 }
7736
7737 // ConstantMaskOp case.
7738 auto maskDimSizes = constantMaskOp.getMaskDimSizes();
7739 auto newMaskDimSizes = applyPermutation(input: maskDimSizes, permutation);
7740
7741 rewriter.replaceOpWithNewOp<vector::ConstantMaskOp>(
7742 op: transpOp, args: transpOp.getResultVectorType(), args&: newMaskDimSizes);
7743 return success();
7744 }
7745};
7746
7747/// Folds transpose(shape_cast) into a new shape_cast.
7748class FoldTransposeShapeCast final : public OpRewritePattern<TransposeOp> {
7749public:
7750 using Base::Base;
7751
7752 LogicalResult matchAndRewrite(TransposeOp transposeOp,
7753 PatternRewriter &rewriter) const override {
7754 auto shapeCastOp =
7755 transposeOp.getVector().getDefiningOp<vector::ShapeCastOp>();
7756 if (!shapeCastOp)
7757 return failure();
7758 if (!isOrderPreserving(transpose: transposeOp))
7759 return failure();
7760
7761 VectorType resultType = transposeOp.getType();
7762
7763 // We don't need to check isValidShapeCast at this point, because it is
7764 // guaranteed that merging the transpose into the the shape_cast is a valid
7765 // shape_cast, because the transpose just inserts/removes ones.
7766
7767 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: transposeOp, args&: resultType,
7768 args: shapeCastOp.getSource());
7769 return success();
7770 }
7771};
7772
7773/// Folds transpose(from_elements(...)) into a new from_elements with permuted
7774/// operands matching the transposed shape.
7775///
7776/// Example:
7777///
7778/// %v = vector.from_elements %a00, %a01, %a02, %a10, %a11, %a12 :
7779/// vector<2x3xi32> %t = vector.transpose %v, [1, 0] : vector<2x3xi32> to
7780/// vector<3x2xi32>
7781///
7782/// becomes ->
7783///
7784/// %r = vector.from_elements %a00, %a10, %a01, %a11, %a02, %a12 :
7785/// vector<3x2xi32>
7786///
7787class FoldTransposeFromElements final : public OpRewritePattern<TransposeOp> {
7788public:
7789 using Base::Base;
7790 LogicalResult matchAndRewrite(vector::TransposeOp transposeOp,
7791 PatternRewriter &rewriter) const override {
7792 auto fromElementsOp =
7793 transposeOp.getVector().getDefiningOp<vector::FromElementsOp>();
7794 if (!fromElementsOp)
7795 return failure();
7796
7797 VectorType srcTy = fromElementsOp.getResult().getType();
7798 VectorType dstTy = transposeOp.getType();
7799
7800 ArrayRef<int64_t> permutation = transposeOp.getPermutation();
7801 int64_t rank = srcTy.getRank();
7802
7803 // Build inverse permutation to map destination indices back to source.
7804 SmallVector<int64_t> inversePerm(rank, 0);
7805 for (int64_t i = 0; i < rank; ++i)
7806 inversePerm[permutation[i]] = i;
7807
7808 ArrayRef<int64_t> srcShape = srcTy.getShape();
7809 ArrayRef<int64_t> dstShape = dstTy.getShape();
7810 SmallVector<int64_t> srcIdx(rank, 0);
7811 SmallVector<int64_t> dstIdx(rank, 0);
7812 SmallVector<int64_t> srcStrides = computeStrides(sizes: srcShape);
7813 SmallVector<int64_t> dstStrides = computeStrides(sizes: dstShape);
7814
7815 auto elementsOld = fromElementsOp.getElements();
7816 SmallVector<Value> elementsNew;
7817 int64_t dstNumElements = dstTy.getNumElements();
7818 elementsNew.reserve(N: dstNumElements);
7819
7820 // For each element in destination row-major order, pick the corresponding
7821 // source element.
7822 for (int64_t linearIdx = 0; linearIdx < dstNumElements; ++linearIdx) {
7823 // Pick the destination element index.
7824 dstIdx = delinearize(linearIndex: linearIdx, strides: dstStrides);
7825 // Map the destination element index to the source element index.
7826 for (int64_t j = 0; j < rank; ++j)
7827 srcIdx[j] = dstIdx[inversePerm[j]];
7828 // Linearize the source element index.
7829 int64_t srcLin = linearize(offsets: srcIdx, basis: srcStrides);
7830 // Add the source element to the new elements.
7831 elementsNew.push_back(Elt: elementsOld[srcLin]);
7832 }
7833
7834 rewriter.replaceOpWithNewOp<FromElementsOp>(op: transposeOp, args&: dstTy,
7835 args&: elementsNew);
7836 return success();
7837 }
7838};
7839
7840/// Folds transpose(broadcast(x)) to broadcast(x) if the transpose is
7841/// 'order preserving', where 'order preserving' means the flattened
7842/// inputs and outputs of the transpose have identical (numerical) values.
7843///
7844/// Example:
7845/// ```
7846/// %0 = vector.broadcast %input : vector<1x1xi32> to vector<1x8xi32>
7847/// %1 = vector.transpose %0, [1, 0] : vector<1x8xi32>
7848/// to vector<8x1xi32>
7849/// ```
7850/// can be rewritten as the equivalent
7851/// ```
7852/// %0 = vector.broadcast %input : vector<1x1xi32> to vector<8x1xi32>.
7853/// ```
7854/// The algorithm works by partitioning dimensions into groups that can be
7855/// locally permuted while preserving order, and checks that the transpose
7856/// only permutes within these groups.
7857///
7858/// Groups are either contiguous sequences of 1s, or non-1s (1-element groups).
7859/// Consider broadcasting 4x1x1x7 to 2x3x4x5x6x7. This is equivalent to
7860/// broadcasting from 1x1x4x1x1x7.
7861/// ^^^ ^ ^^^ ^
7862/// groups: 0 1 2 3
7863/// Order preserving permutations for this example are ones that only permute
7864/// within the groups [0,1] and [3,4], like (1 0 2 4 3 5 6).
7865class FoldTransposeBroadcast : public OpRewritePattern<vector::TransposeOp> {
7866public:
7867 using Base::Base;
7868 FoldTransposeBroadcast(MLIRContext *context, PatternBenefit benefit = 1)
7869 : OpRewritePattern<vector::TransposeOp>(context, benefit) {}
7870
7871 LogicalResult matchAndRewrite(vector::TransposeOp transpose,
7872 PatternRewriter &rewriter) const override {
7873
7874 vector::BroadcastOp broadcast =
7875 transpose.getVector().getDefiningOp<vector::BroadcastOp>();
7876 if (!broadcast) {
7877 return rewriter.notifyMatchFailure(arg&: transpose,
7878 msg: "not preceded by a broadcast");
7879 }
7880
7881 auto inputType = dyn_cast<VectorType>(Val: broadcast.getSourceType());
7882 VectorType outputType = transpose.getResultVectorType();
7883
7884 // transpose(broadcast(scalar)) -> broadcast(scalar) is always valid
7885 bool inputIsScalar = !inputType;
7886 if (inputIsScalar) {
7887 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: transpose, args&: outputType,
7888 args: broadcast.getSource());
7889 return success();
7890 }
7891
7892 ArrayRef<int64_t> permutation = transpose.getPermutation();
7893 ArrayRef<int64_t> inputShape = inputType.getShape();
7894 int64_t inputRank = inputType.getRank();
7895 int64_t outputRank = transpose.getType().getRank();
7896 int64_t deltaRank = outputRank - inputRank;
7897
7898 int low = 0;
7899 for (int inputIndex = 0; inputIndex < inputRank; ++inputIndex) {
7900 bool notOne = inputShape[inputIndex] != 1;
7901 bool prevNotOne = (inputIndex != 0 && inputShape[inputIndex - 1] != 1);
7902 bool groupEndFound = notOne || prevNotOne;
7903 if (groupEndFound) {
7904 int high = inputIndex + deltaRank;
7905 // Return failure if not all permutation destinations for indices in
7906 // [low, high) are in [low, high), i.e. the permutation is not local to
7907 // the group.
7908 for (int i = low; i < high; ++i) {
7909 if (permutation[i] < low || permutation[i] >= high) {
7910 return rewriter.notifyMatchFailure(
7911 arg&: transpose, msg: "permutation not local to group");
7912 }
7913 }
7914 low = high;
7915 }
7916 }
7917
7918 // We don't need to check the final group [low, outputRank) because if it is
7919 // not locally bound, there must be a preceding group that already failed
7920 // the check (impossible to have just 1 non-locally bound group).
7921
7922 // The preceding logic also ensures that at this point, the output of the
7923 // transpose is definitely broadcastable from the input shape, assert so:
7924 assert(vector::isBroadcastableTo(inputType, outputType) ==
7925 vector::BroadcastableToResult::Success &&
7926 "not broadcastable directly to transpose output");
7927
7928 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: transpose, args&: outputType,
7929 args: broadcast.getSource());
7930
7931 return success();
7932 }
7933};
7934
7935} // namespace
7936
7937void vector::TransposeOp::getCanonicalizationPatterns(
7938 RewritePatternSet &results, MLIRContext *context) {
7939 results.add<FoldTransposeCreateMask, FoldTransposeShapeCast, TransposeFolder,
7940 FoldTransposeSplat, FoldTransposeFromElements,
7941 FoldTransposeBroadcast>(arg&: context);
7942}
7943
7944//===----------------------------------------------------------------------===//
7945// ConstantMaskOp
7946//===----------------------------------------------------------------------===//
7947
7948void ConstantMaskOp::build(OpBuilder &builder, OperationState &result,
7949 VectorType type, ConstantMaskKind kind) {
7950 assert(kind == ConstantMaskKind::AllTrue ||
7951 kind == ConstantMaskKind::AllFalse);
7952 auto maskDimSizes = llvm::to_vector(Range: type.getShape());
7953 if (maskDimSizes.empty())
7954 maskDimSizes.push_back(Elt: 1);
7955 if (kind == ConstantMaskKind::AllFalse)
7956 maskDimSizes.assign(NumElts: maskDimSizes.size(), Elt: 0);
7957 build(odsBuilder&: builder, odsState&: result, resultType0: type, mask_dim_sizes: maskDimSizes);
7958}
7959
7960LogicalResult ConstantMaskOp::verify() {
7961 auto resultType = llvm::cast<VectorType>(Val: getResult().getType());
7962 // Check the corner case of 0-D vectors first.
7963 if (resultType.getRank() == 0) {
7964 if (getMaskDimSizes().size() != 1)
7965 return emitError(message: "array attr must have length 1 for 0-D vectors");
7966 auto dim = getMaskDimSizes()[0];
7967 if (dim != 0 && dim != 1)
7968 return emitError(message: "mask dim size must be either 0 or 1 for 0-D vectors");
7969 return success();
7970 }
7971
7972 // Verify that array attr size matches the rank of the vector result.
7973 if (static_cast<int64_t>(getMaskDimSizes().size()) != resultType.getRank())
7974 return emitOpError(
7975 message: "must specify array attr of size equal vector result rank");
7976 // Verify that each array attr element is in bounds of corresponding vector
7977 // result dimension size.
7978 auto resultShape = resultType.getShape();
7979 auto resultScalableDims = resultType.getScalableDims();
7980 ArrayRef<int64_t> maskDimSizes = getMaskDimSizes();
7981 for (const auto [index, maskDimSize] : llvm::enumerate(First&: maskDimSizes)) {
7982 if (maskDimSize < 0 || maskDimSize > resultShape[index])
7983 return emitOpError(
7984 message: "array attr of size out of bounds of vector result dimension size");
7985 if (resultScalableDims[index] && maskDimSize != 0 &&
7986 maskDimSize != resultShape[index])
7987 return emitOpError(
7988 message: "only supports 'none set' or 'all set' scalable dimensions");
7989 }
7990 // Verify that if one mask dim size is zero, they all should be zero (because
7991 // the mask region is a conjunction of each mask dimension interval).
7992 bool anyZeros = llvm::is_contained(Range&: maskDimSizes, Element: 0);
7993 bool allZeros = llvm::all_of(Range&: maskDimSizes, P: [](int64_t s) { return s == 0; });
7994 if (anyZeros && !allZeros)
7995 return emitOpError(message: "expected all mask dim sizes to be zeros, "
7996 "as a result of conjunction with zero mask dim");
7997 return success();
7998}
7999
8000bool ConstantMaskOp::isAllOnesMask() {
8001 auto resultType = getVectorType();
8002 // Check the corner case of 0-D vectors first.
8003 if (resultType.getRank() == 0) {
8004 assert(getMaskDimSizes().size() == 1 && "invalid sizes for zero rank mask");
8005 return getMaskDimSizes()[0] == 1;
8006 }
8007 for (const auto [resultSize, maskDimSize] :
8008 llvm::zip_equal(t: resultType.getShape(), u: getMaskDimSizes())) {
8009 if (maskDimSize < resultSize)
8010 return false;
8011 }
8012 return true;
8013}
8014
8015OpFoldResult ConstantMaskOp::fold(FoldAdaptor adaptor) {
8016 ArrayRef<int64_t> bounds = getMaskDimSizes();
8017 ArrayRef<int64_t> vectorSizes = getVectorType().getShape();
8018
8019 auto createBoolSplat = [&](bool x) {
8020 return SplatElementsAttr::get(type: getVectorType(),
8021 values: BoolAttr::get(context: getContext(), value: x));
8022 };
8023
8024 // Check the corner case of 0-D vectors first.
8025 if (vectorSizes.empty()) {
8026 assert(bounds.size() == 1 && "invalid sizes for zero rank mask");
8027 return createBoolSplat(bounds[0] == 1);
8028 }
8029 // Fold vector.constant_mask to splat if possible.
8030 if (bounds == vectorSizes)
8031 return createBoolSplat(true);
8032 if (llvm::all_of(Range&: bounds, P: [](int64_t x) { return x == 0; }))
8033 return createBoolSplat(false);
8034 return OpFoldResult();
8035}
8036
8037std::optional<SmallVector<int64_t>>
8038ConstantMaskOp::getDefaultUnrollTargetShape() {
8039 return getDefaultTargetMaskShapeForUnroll();
8040}
8041
8042//===----------------------------------------------------------------------===//
8043// CreateMaskOp
8044//===----------------------------------------------------------------------===//
8045
8046void CreateMaskOp::build(OpBuilder &builder, OperationState &result,
8047 VectorType type,
8048 ArrayRef<OpFoldResult> mixedOperands) {
8049 SmallVector<Value> operands =
8050 getValueOrCreateConstantIndexOp(b&: builder, loc: result.location, valueOrAttrVec: mixedOperands);
8051 build(odsBuilder&: builder, odsState&: result, resultType0: type, mask_dim_sizes: operands);
8052}
8053
8054LogicalResult CreateMaskOp::verify() {
8055 auto vectorType = llvm::cast<VectorType>(Val: getResult().getType());
8056 // Verify that an operand was specified for each result vector each dimension.
8057 if (vectorType.getRank() == 0) {
8058 if (getNumOperands() != 1)
8059 return emitOpError(
8060 message: "must specify exactly one operand for 0-D create_mask");
8061 } else if (getNumOperands() !=
8062 llvm::cast<VectorType>(Val: getResult().getType()).getRank()) {
8063 return emitOpError(
8064 message: "must specify an operand for each result vector dimension");
8065 }
8066 return success();
8067}
8068
8069std::optional<SmallVector<int64_t>>
8070CreateMaskOp::getDefaultUnrollTargetShape() {
8071 return getDefaultTargetMaskShapeForUnroll();
8072}
8073
8074namespace {
8075
8076/// Pattern to rewrite a CreateMaskOp with a ConstantMaskOp.
8077///
8078/// Ex 1:
8079/// %c2 = arith.constant 2 : index
8080/// %c3 = arith.constant 3 : index
8081/// %0 = vector.create_mask %c3, %c2 : vector<4x3xi1>
8082/// Becomes:
8083/// vector.constant_mask [3, 2] : vector<4x3xi1>
8084///
8085/// Ex 2:
8086/// %c_neg_1 = arith.constant -1 : index
8087/// %0 = vector.create_mask %c_neg_1 : vector<[8]xi1>
8088/// becomes:
8089/// vector.constant_mask [0] : vector<[8]xi1>
8090///
8091/// Ex 3:
8092/// %c8 = arith.constant 8 : index
8093/// %c16 = arith.constant 16 : index
8094/// %0 = vector.vscale
8095/// %1 = arith.muli %0, %c16 : index
8096/// %10 = vector.create_mask %c8, %1 : vector<8x[16]xi1>
8097/// becomes:
8098/// %0 = vector.constant_mask [8, 16] : vector<8x[16]xi1>
8099class CreateMaskFolder final : public OpRewritePattern<CreateMaskOp> {
8100public:
8101 using Base::Base;
8102
8103 LogicalResult matchAndRewrite(CreateMaskOp createMaskOp,
8104 PatternRewriter &rewriter) const override {
8105 VectorType maskType = createMaskOp.getVectorType();
8106 ArrayRef<int64_t> maskTypeDimSizes = maskType.getShape();
8107 ArrayRef<bool> maskTypeDimScalableFlags = maskType.getScalableDims();
8108
8109 // Special case: Rank zero shape.
8110 constexpr std::array<int64_t, 1> rankZeroShape{1};
8111 constexpr std::array<bool, 1> rankZeroScalableDims{false};
8112 if (maskType.getRank() == 0) {
8113 maskTypeDimSizes = rankZeroShape;
8114 maskTypeDimScalableFlags = rankZeroScalableDims;
8115 }
8116
8117 // Determine if this CreateMaskOp can be folded to a ConstantMaskOp and
8118 // collect the `constantDims` (for the ConstantMaskOp).
8119 SmallVector<int64_t, 4> constantDims;
8120 for (auto [i, dimSize] : llvm::enumerate(First: createMaskOp.getOperands())) {
8121 if (auto intSize = getConstantIntValue(ofr: dimSize)) {
8122 // Constant value.
8123 // If the mask dim is non-scalable this can be any value.
8124 // If the mask dim is scalable only zero (all-false) is supported.
8125 if (maskTypeDimScalableFlags[i] && intSize >= 0)
8126 return failure();
8127 constantDims.push_back(Elt: *intSize);
8128 } else if (auto vscaleMultiplier = getConstantVscaleMultiplier(value: dimSize)) {
8129 // Constant vscale multiple (e.g. 4 x vscale).
8130 // Must be all-true to fold to a ConstantMask.
8131 if (vscaleMultiplier < maskTypeDimSizes[i])
8132 return failure();
8133 constantDims.push_back(Elt: *vscaleMultiplier);
8134 } else {
8135 return failure();
8136 }
8137 }
8138
8139 // Clamp values to constant_mask bounds.
8140 for (auto [value, maskDimSize] : llvm::zip(t&: constantDims, u&: maskTypeDimSizes))
8141 value = std::clamp<int64_t>(val: value, lo: 0, hi: maskDimSize);
8142
8143 // If one of dim sizes is zero, set all dims to zero.
8144 if (llvm::is_contained(Range&: constantDims, Element: 0))
8145 constantDims.assign(NumElts: constantDims.size(), Elt: 0);
8146
8147 // Replace 'createMaskOp' with ConstantMaskOp.
8148 rewriter.replaceOpWithNewOp<ConstantMaskOp>(op: createMaskOp, args&: maskType,
8149 args&: constantDims);
8150 return success();
8151 }
8152};
8153
8154} // namespace
8155
8156void CreateMaskOp::getCanonicalizationPatterns(RewritePatternSet &results,
8157 MLIRContext *context) {
8158 results.add<CreateMaskFolder>(arg&: context);
8159}
8160
8161//===----------------------------------------------------------------------===//
8162// MaskOp
8163//===----------------------------------------------------------------------===//
8164
8165void MaskOp::build(
8166 OpBuilder &builder, OperationState &result, Value mask,
8167 Operation *maskableOp,
8168 function_ref<void(OpBuilder &, Operation *)> maskRegionBuilder) {
8169 assert(maskRegionBuilder &&
8170 "builder callback for 'maskRegion' must be present");
8171
8172 result.addOperands(newOperands: mask);
8173 OpBuilder::InsertionGuard guard(builder);
8174 Region *maskRegion = result.addRegion();
8175 builder.createBlock(parent: maskRegion);
8176 maskRegionBuilder(builder, maskableOp);
8177}
8178
8179void MaskOp::build(
8180 OpBuilder &builder, OperationState &result, TypeRange resultTypes,
8181 Value mask, Operation *maskableOp,
8182 function_ref<void(OpBuilder &, Operation *)> maskRegionBuilder) {
8183 build(odsBuilder&: builder, odsState&: result, resultTypes, mask, /*passthru=*/Value(), maskableOp,
8184 maskRegion: maskRegionBuilder);
8185}
8186
8187void MaskOp::build(
8188 OpBuilder &builder, OperationState &result, TypeRange resultTypes,
8189 Value mask, Value passthru, Operation *maskableOp,
8190 function_ref<void(OpBuilder &, Operation *)> maskRegionBuilder) {
8191 build(builder, result, mask, maskableOp, maskRegionBuilder);
8192 if (passthru)
8193 result.addOperands(newOperands: passthru);
8194 result.addTypes(newTypes&: resultTypes);
8195}
8196
8197ParseResult MaskOp::parse(OpAsmParser &parser, OperationState &result) {
8198 // Create the op region.
8199 result.regions.reserve(N: 1);
8200 Region &maskRegion = *result.addRegion();
8201
8202 auto &builder = parser.getBuilder();
8203
8204 // Parse all the operands.
8205 OpAsmParser::UnresolvedOperand mask;
8206 if (parser.parseOperand(result&: mask))
8207 return failure();
8208
8209 // Optional passthru operand.
8210 OpAsmParser::UnresolvedOperand passthru;
8211 ParseResult parsePassthru = parser.parseOptionalComma();
8212 if (parsePassthru.succeeded() && parser.parseOperand(result&: passthru))
8213 return failure();
8214
8215 // Parse op region.
8216 if (parser.parseRegion(region&: maskRegion, /*arguments=*/{}, /*argTypes=*/enableNameShadowing: {}))
8217 return failure();
8218
8219 MaskOp::ensureTerminator(region&: maskRegion, builder, loc: result.location);
8220
8221 // Parse the optional attribute list.
8222 if (parser.parseOptionalAttrDict(result&: result.attributes))
8223 return failure();
8224
8225 // Parse all the types.
8226 Type maskType;
8227 if (parser.parseColonType(result&: maskType))
8228 return failure();
8229
8230 SmallVector<Type> resultTypes;
8231 if (parser.parseOptionalArrowTypeList(result&: resultTypes))
8232 return failure();
8233 result.types.append(RHS: resultTypes);
8234
8235 // Resolve operands.
8236 if (parser.resolveOperand(operand: mask, type: maskType, result&: result.operands))
8237 return failure();
8238
8239 if (parsePassthru.succeeded()) {
8240 if (resultTypes.empty())
8241 return parser.emitError(
8242 loc: parser.getNameLoc(),
8243 message: "expects a result if passthru operand is provided");
8244
8245 if (parser.resolveOperand(operand: passthru, type: resultTypes[0], result&: result.operands))
8246 return failure();
8247 }
8248
8249 return success();
8250}
8251
8252void mlir::vector::MaskOp::print(OpAsmPrinter &p) {
8253 p << " " << getMask();
8254 if (getPassthru())
8255 p << ", " << getPassthru();
8256
8257 // Print single masked operation and skip terminator.
8258 p << " { ";
8259 Block *singleBlock = &getMaskRegion().getBlocks().front();
8260 if (singleBlock && !singleBlock->getOperations().empty())
8261 p.printCustomOrGenericOp(op: &singleBlock->front());
8262 p << " }";
8263
8264 p.printOptionalAttrDict(
8265 attrs: getOperation()->getDiscardableAttrDictionary().getValue());
8266
8267 p << " : " << getMask().getType();
8268 if (getNumResults() > 0)
8269 p << " -> " << getResultTypes();
8270}
8271
8272void MaskOp::ensureTerminator(Region &region, Builder &builder, Location loc) {
8273 // 1. For an empty `vector.mask`, create a default terminator.
8274 if (region.empty() || region.front().empty()) {
8275 OpTrait::SingleBlockImplicitTerminator<vector::YieldOp>::Impl<
8276 MaskOp>::ensureTerminator(region, builder, loc);
8277 return;
8278 }
8279
8280 // 2. For a non-empty `vector.mask` with an explicit terminator, do nothing.
8281 Block &block = region.front();
8282 if (isa<vector::YieldOp>(Val: block.back()))
8283 return;
8284
8285 // 3. For a non-empty `vector.mask` without an explicit terminator:
8286
8287 // Create default terminator if the number of masked operations is not
8288 // one. This case will trigger a verification failure.
8289 if (block.getOperations().size() != 1) {
8290 OpTrait::SingleBlockImplicitTerminator<vector::YieldOp>::Impl<
8291 MaskOp>::ensureTerminator(region, builder, loc);
8292 return;
8293 }
8294
8295 // Create a terminator that yields the results from the masked operation.
8296 OpBuilder opBuilder(builder.getContext());
8297 Operation *maskedOp = &block.front();
8298 opBuilder.setInsertionPointToEnd(&block);
8299 vector::YieldOp::create(builder&: opBuilder, location: loc, operands: maskedOp->getResults());
8300}
8301
8302LogicalResult MaskOp::verify() {
8303 // Structural checks.
8304 Block &block = getMaskRegion().getBlocks().front();
8305 if (block.getOperations().empty())
8306 return emitOpError(message: "expects a terminator within the mask region");
8307
8308 unsigned numMaskRegionOps = block.getOperations().size();
8309 if (numMaskRegionOps > 2)
8310 return emitOpError(message: "expects only one operation to mask");
8311
8312 // Terminator checks.
8313 auto terminator = dyn_cast<vector::YieldOp>(Val&: block.back());
8314 if (!terminator)
8315 return emitOpError(message: "expects a terminator within the mask region");
8316
8317 if (terminator->getNumOperands() != getNumResults())
8318 return emitOpError(
8319 message: "expects number of results to match mask region yielded values");
8320
8321 // Empty vector.mask. Nothing else to check.
8322 if (numMaskRegionOps == 1)
8323 return success();
8324
8325 auto maskableOp = dyn_cast<MaskableOpInterface>(Val&: block.front());
8326 if (!maskableOp)
8327 return emitOpError(message: "expects a MaskableOpInterface within the mask region");
8328
8329 // Result checks.
8330 if (maskableOp->getNumResults() != getNumResults())
8331 return emitOpError(message: "expects number of results to match maskable operation "
8332 "number of results");
8333
8334 if (!llvm::equal(LRange: maskableOp->getResults(), RRange: terminator.getOperands()))
8335 return emitOpError(message: "expects all the results from the MaskableOpInterface "
8336 "to match all the values returned by the terminator");
8337
8338 if (!llvm::equal(LRange: maskableOp->getResultTypes(), RRange: getResultTypes()))
8339 return emitOpError(
8340 message: "expects result type to match maskable operation result type");
8341
8342 if (llvm::count_if(Range: maskableOp->getResultTypes(),
8343 P: [](Type t) { return llvm::isa<VectorType>(Val: t); }) > 1)
8344 return emitOpError(message: "multiple vector results not supported");
8345
8346 // Mask checks.
8347 Type expectedMaskType = maskableOp.getExpectedMaskType();
8348 if (getMask().getType() != expectedMaskType)
8349 return emitOpError(message: "expects a ")
8350 << expectedMaskType << " mask for the maskable operation";
8351
8352 auto transferOp =
8353 dyn_cast<VectorTransferOpInterface>(Val: maskableOp.getOperation());
8354 if (transferOp && transferOp.getMask())
8355 return emitOpError(
8356 message: "doesn't expect a mask operand for the masked transfer operation");
8357
8358 // Passthru checks.
8359 Value passthru = getPassthru();
8360 if (passthru) {
8361 if (!maskableOp.supportsPassthru())
8362 return emitOpError(
8363 message: "doesn't expect a passthru argument for this maskable operation");
8364
8365 if (maskableOp->getNumResults() != 1)
8366 return emitOpError(message: "expects result when passthru argument is provided");
8367
8368 if (passthru.getType() != maskableOp->getResultTypes()[0])
8369 return emitOpError(message: "expects passthru type to match result type");
8370 }
8371
8372 return success();
8373}
8374
8375/// Folds empty `vector.mask` with no passthru operand and with or without
8376/// return values. For example:
8377///
8378/// %0 = vector.mask %mask { vector.yield %a : vector<8xf32> } :
8379/// vector<8xi1> -> vector<8xf32>
8380/// %1 = user_op %0 : vector<8xf32>
8381///
8382/// becomes:
8383///
8384/// %0 = user_op %a : vector<8xf32>
8385///
8386/// Empty `vector.mask` with passthru operand are handled by the canonicalizer
8387/// as it requires creating new operations.
8388
8389static LogicalResult foldEmptyMaskOp(MaskOp maskOp, MaskOp::FoldAdaptor adaptor,
8390 SmallVectorImpl<OpFoldResult> &results) {
8391 if (!maskOp.isEmpty() || maskOp.hasPassthru())
8392 return failure();
8393
8394 Block *block = maskOp.getMaskBlock();
8395 auto terminator = cast<vector::YieldOp>(Val&: block->front());
8396 if (terminator.getNumOperands() == 0)
8397 return failure();
8398
8399 // `vector.mask` has results, propagate the results.
8400 llvm::append_range(C&: results, R: terminator.getOperands());
8401 return success();
8402}
8403
8404LogicalResult MaskOp::fold(FoldAdaptor adaptor,
8405 SmallVectorImpl<OpFoldResult> &results) {
8406 if (succeeded(Result: foldEmptyMaskOp(maskOp: *this, adaptor, results)))
8407 return success();
8408
8409 MaskFormat maskFormat = getMaskFormat(mask: getMask());
8410 if (maskFormat != MaskFormat::AllTrue)
8411 return failure();
8412
8413 // Move maskable operation outside of the `vector.mask` region.
8414 // If there is no maskable op (empty body), the fold cannot proceed; the
8415 // canonicalizer handles this case instead.
8416 Operation *maskableOp = getMaskableOp();
8417 if (!maskableOp)
8418 return failure();
8419 maskableOp->dropAllUses();
8420 maskableOp->moveBefore(existingOp: getOperation());
8421
8422 llvm::append_range(C&: results, R: maskableOp->getResults());
8423 return success();
8424}
8425
8426/// Canonialize empty `vector.mask` operations that can't be handled in
8427/// `VectorMask::fold` as they require creating new operations.
8428///
8429/// Example 1: Empty `vector.mask` with passthru operand.
8430///
8431/// %0 = vector.mask %mask, %passthru { vector.yield %a : vector<8xf32> } :
8432/// vector<8xi1> -> vector<8xf32>
8433///
8434/// becomes:
8435///
8436/// %0 = arith.select %mask, %a, %passthru : vector<8xf32>
8437///
8438class CanonializeEmptyMaskOp : public OpRewritePattern<MaskOp> {
8439 using Base::Base;
8440
8441 LogicalResult matchAndRewrite(MaskOp maskOp,
8442 PatternRewriter &rewriter) const override {
8443 if (!maskOp.isEmpty())
8444 return failure();
8445
8446 if (!maskOp.hasPassthru())
8447 return failure();
8448
8449 // arith.select with a vector condition requires the value types to be
8450 // vectors of the same shape. Since vector.mask always has a vector mask
8451 // type, bail out when any result type doesn't match the mask shape to
8452 // avoid creating invalid IR.
8453 VectorType maskType = maskOp.getMask().getType();
8454 for (Type resultType : maskOp.getResultTypes()) {
8455 auto vecResultType = dyn_cast<VectorType>(Val&: resultType);
8456 if (!vecResultType || vecResultType.getShape() != maskType.getShape())
8457 return failure();
8458 }
8459
8460 Block *block = maskOp.getMaskBlock();
8461 auto terminator = cast<vector::YieldOp>(Val&: block->front());
8462 assert(terminator.getNumOperands() == 1 &&
8463 "expected one result when passthru is provided");
8464
8465 rewriter.replaceOpWithNewOp<arith::SelectOp>(
8466 op: maskOp, args: maskOp.getResultTypes(), args: maskOp.getMask(),
8467 args: terminator.getOperand(i: 0), args: maskOp.getPassthru());
8468
8469 return success();
8470 }
8471};
8472
8473void MaskOp::getCanonicalizationPatterns(RewritePatternSet &results,
8474 MLIRContext *context) {
8475 results.add<CanonializeEmptyMaskOp>(arg&: context);
8476}
8477
8478// MaskingOpInterface definitions.
8479
8480/// Returns the operation masked by this 'vector.mask'.
8481Operation *MaskOp::getMaskableOp() {
8482 Block *block = getMaskBlock();
8483 if (block->getOperations().size() < 2)
8484 return nullptr;
8485
8486 return &block->front();
8487}
8488
8489/// Returns true if 'vector.mask' has a passthru value.
8490bool MaskOp::hasPassthru() { return getPassthru() != Value(); }
8491
8492//===----------------------------------------------------------------------===//
8493// ScanOp
8494//===----------------------------------------------------------------------===//
8495
8496LogicalResult ScanOp::verify() {
8497 VectorType srcType = getSourceType();
8498 VectorType initialType = getInitialValueType();
8499 // Check reduction dimension < rank.
8500 int64_t srcRank = srcType.getRank();
8501 int64_t reductionDim = getReductionDim();
8502 if (reductionDim >= srcRank)
8503 return emitOpError(message: "reduction dimension ")
8504 << reductionDim << " has to be less than " << srcRank;
8505
8506 // Check that rank(initial_value) = rank(src) - 1.
8507 int64_t initialValueRank = initialType.getRank();
8508 if (initialValueRank != srcRank - 1)
8509 return emitOpError(message: "initial value rank ")
8510 << initialValueRank << " has to be equal to " << srcRank - 1;
8511
8512 // Check shapes of initial value and src.
8513 ArrayRef<int64_t> srcShape = srcType.getShape();
8514 ArrayRef<int64_t> initialValueShapes = initialType.getShape();
8515 SmallVector<int64_t> expectedShape;
8516 for (int i = 0; i < srcRank; i++) {
8517 if (i != reductionDim)
8518 expectedShape.push_back(Elt: srcShape[i]);
8519 }
8520 if (!llvm::equal(LRange&: initialValueShapes, RRange&: expectedShape)) {
8521 return emitOpError(message: "incompatible input/initial value shapes");
8522 }
8523
8524 // Verify supported reduction kind.
8525 Type eltType = getDestType().getElementType();
8526 if (!isSupportedCombiningKind(combiningKind: getKind(), elementType: eltType))
8527 return emitOpError(message: "unsupported reduction type ")
8528 << eltType << " for kind '" << stringifyCombiningKind(val: getKind())
8529 << "'";
8530
8531 return success();
8532}
8533
8534void mlir::vector::populateVectorToVectorCanonicalizationPatterns(
8535 RewritePatternSet &patterns, PatternBenefit benefit) {
8536 patterns
8537 .add<CreateMaskFolder, MaskedLoadFolder, MaskedStoreFolder, GatherFolder,
8538 ScatterFolder, ExpandLoadFolder, CompressStoreFolder,
8539 StridedSliceConstantMaskFolder, TransposeFolder>(
8540 arg: patterns.getContext(), args&: benefit);
8541}
8542
8543Value mlir::vector::makeArithReduction(OpBuilder &b, Location loc,
8544 CombiningKind kind, Value v1, Value acc,
8545 arith::FastMathFlagsAttr fastmath,
8546 Value mask) {
8547 Type t1 = getElementTypeOrSelf(type: v1.getType());
8548 Type tAcc = getElementTypeOrSelf(type: acc.getType());
8549 Value result;
8550
8551 switch (kind) {
8552 case CombiningKind::ADD:
8553 if (t1.isIntOrIndex() && tAcc.isIntOrIndex())
8554 result = b.createOrFold<arith::AddIOp>(location: loc, args&: v1, args&: acc);
8555 else if (llvm::isa<FloatType>(Val: t1) && llvm::isa<FloatType>(Val: tAcc))
8556 result = b.createOrFold<arith::AddFOp>(location: loc, args&: v1, args&: acc, args&: fastmath);
8557 else
8558 llvm_unreachable("invalid value types for ADD reduction");
8559 break;
8560 case CombiningKind::AND:
8561 assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values");
8562 result = b.createOrFold<arith::AndIOp>(location: loc, args&: v1, args&: acc);
8563 break;
8564 case CombiningKind::MAXNUMF:
8565 assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) &&
8566 "expected float values");
8567 result = b.createOrFold<arith::MaxNumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath);
8568 break;
8569 case CombiningKind::MAXIMUMNUMF:
8570 assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) &&
8571 "expected float values");
8572 result = b.createOrFold<arith::MaximumNumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath);
8573 break;
8574 case CombiningKind::MAXIMUMF:
8575 assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) &&
8576 "expected float values");
8577 result = b.createOrFold<arith::MaximumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath);
8578 break;
8579 case CombiningKind::MINNUMF:
8580 assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) &&
8581 "expected float values");
8582 result = b.createOrFold<arith::MinNumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath);
8583 break;
8584 case CombiningKind::MINIMUMNUMF:
8585 assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) &&
8586 "expected float values");
8587 result = b.createOrFold<arith::MinimumNumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath);
8588 break;
8589 case CombiningKind::MINIMUMF:
8590 assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) &&
8591 "expected float values");
8592 result = b.createOrFold<arith::MinimumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath);
8593 break;
8594 case CombiningKind::MAXSI:
8595 assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values");
8596 result = b.createOrFold<arith::MaxSIOp>(location: loc, args&: v1, args&: acc);
8597 break;
8598 case CombiningKind::MINSI:
8599 assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values");
8600 result = b.createOrFold<arith::MinSIOp>(location: loc, args&: v1, args&: acc);
8601 break;
8602 case CombiningKind::MAXUI:
8603 assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values");
8604 result = b.createOrFold<arith::MaxUIOp>(location: loc, args&: v1, args&: acc);
8605 break;
8606 case CombiningKind::MINUI:
8607 assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values");
8608 result = b.createOrFold<arith::MinUIOp>(location: loc, args&: v1, args&: acc);
8609 break;
8610 case CombiningKind::MUL:
8611 if (t1.isIntOrIndex() && tAcc.isIntOrIndex())
8612 result = b.createOrFold<arith::MulIOp>(location: loc, args&: v1, args&: acc);
8613 else if (llvm::isa<FloatType>(Val: t1) && llvm::isa<FloatType>(Val: tAcc))
8614 result = b.createOrFold<arith::MulFOp>(location: loc, args&: v1, args&: acc, args&: fastmath);
8615 else
8616 llvm_unreachable("invalid value types for MUL reduction");
8617 break;
8618 case CombiningKind::OR:
8619 assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values");
8620 result = b.createOrFold<arith::OrIOp>(location: loc, args&: v1, args&: acc);
8621 break;
8622 case CombiningKind::XOR:
8623 assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values");
8624 result = b.createOrFold<arith::XOrIOp>(location: loc, args&: v1, args&: acc);
8625 break;
8626 };
8627
8628 assert(result && "unknown CombiningKind");
8629 return selectPassthru(builder&: b, mask, newValue: result, passthru: acc);
8630}
8631
8632//===----------------------------------------------------------------------===//
8633// StepOp
8634//===----------------------------------------------------------------------===//
8635
8636void StepOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges,
8637 SetIntRangeFn setResultRanges) {
8638 auto resultType = cast<VectorType>(Val: getType());
8639 if (resultType.isScalable()) {
8640 return;
8641 }
8642 unsigned bitwidth = ConstantIntRanges::getStorageBitwidth(type: resultType);
8643 // The result holds the sequence [0, 1, ..., N-1], with each value truncated
8644 // to the result element type.
8645 uint64_t maxIndex = resultType.getDimSize(idx: 0) - 1;
8646 APInt umin = APInt::getZero(numBits: bitwidth);
8647 APInt umax = APInt::getMaxValue(numBits: bitwidth).ugt(RHS: maxIndex)
8648 ? APInt(bitwidth, maxIndex)
8649 : APInt::getMaxValue(numBits: bitwidth);
8650 setResultRanges(getResult(), ConstantIntRanges::fromUnsigned(umin, umax));
8651}
8652
8653std::optional<SmallVector<int64_t>> StepOp::getDefaultUnrollTargetShape() {
8654 return SmallVector<int64_t>{8};
8655}
8656
8657namespace {
8658
8659/// Fold `vector.step -> arith.cmpi` when the step value is compared to a
8660/// constant large enough such that the result is the same at all indices.
8661///
8662/// For example, rewrite the 'greater than' comparison below,
8663///
8664/// ```mlir
8665/// %cst = arith.constant dense<7> : vector<3xindex>
8666/// %stp = vector.step : vector<3xindex>
8667/// %out = arith.cmpi ugt, %stp, %cst : vector<3xindex>
8668/// ```
8669///
8670/// as,
8671///
8672/// ```mlir
8673/// %out = arith.constant dense<false> : vector<3xi1>.
8674/// ```
8675///
8676/// Above `[0, 1, 2] > [7, 7, 7]` => `[false, false, false]`. Because the result
8677/// is false at ALL indices we fold. If the constant was 1, then
8678/// `[0, 1, 2] > [1, 1, 1]` => `[false, false, true]` and we do fold,
8679/// conservatively preferring the 'compact' vector.step representation.
8680///
8681/// Note: this folder only works for the case where the constant (`%cst` above)
8682/// is the second operand of the comparison. The arith.cmpi canonicalizer will
8683/// ensure that constants are always second (on the right).
8684struct StepCompareFolder : public OpRewritePattern<StepOp> {
8685 using Base::Base;
8686
8687 LogicalResult matchAndRewrite(StepOp stepOp,
8688 PatternRewriter &rewriter) const override {
8689 const int64_t stepSize = stepOp.getResult().getType().getNumElements();
8690
8691 for (OpOperand &use : stepOp.getResult().getUses()) {
8692 auto cmpiOp = dyn_cast<arith::CmpIOp>(Val: use.getOwner());
8693 if (!cmpiOp)
8694 continue;
8695
8696 // arith.cmpi canonicalizer makes constants final operands.
8697 const unsigned stepOperandNumber = use.getOperandNumber();
8698 if (stepOperandNumber != 0)
8699 continue;
8700
8701 // Check that operand 1 is a constant.
8702 unsigned constOperandNumber = 1;
8703 Value otherOperand = cmpiOp.getOperand(i: constOperandNumber);
8704 std::optional<int64_t> maybeConstValue =
8705 getConstantIntValue(ofr: otherOperand);
8706 if (!maybeConstValue.has_value())
8707 continue;
8708
8709 int64_t constValue = maybeConstValue.value();
8710 arith::CmpIPredicate pred = cmpiOp.getPredicate();
8711
8712 auto maybeSplat = [&]() -> std::optional<bool> {
8713 // Handle ult (unsigned less than) and uge (unsigned greater equal).
8714 if ((pred == arith::CmpIPredicate::ult ||
8715 pred == arith::CmpIPredicate::uge) &&
8716 stepSize <= constValue)
8717 return pred == arith::CmpIPredicate::ult;
8718
8719 // Handle ule and ugt.
8720 if ((pred == arith::CmpIPredicate::ule ||
8721 pred == arith::CmpIPredicate::ugt) &&
8722 stepSize - 1 <= constValue) {
8723 return pred == arith::CmpIPredicate::ule;
8724 }
8725
8726 // Handle eq and ne.
8727 if ((pred == arith::CmpIPredicate::eq ||
8728 pred == arith::CmpIPredicate::ne) &&
8729 stepSize <= constValue)
8730 return pred == arith::CmpIPredicate::ne;
8731
8732 return std::nullopt;
8733 }();
8734
8735 if (!maybeSplat.has_value())
8736 continue;
8737
8738 rewriter.setInsertionPointAfter(cmpiOp);
8739
8740 auto type = dyn_cast<VectorType>(Val: cmpiOp.getResult().getType());
8741 if (!type)
8742 continue;
8743
8744 auto boolAttr = DenseElementsAttr::get(type, value: maybeSplat.value());
8745 Value splat = mlir::arith::ConstantOp::create(builder&: rewriter, location: cmpiOp.getLoc(),
8746 result: type, value: boolAttr);
8747
8748 rewriter.replaceOp(op: cmpiOp, newValues: splat);
8749 return success();
8750 }
8751
8752 return failure();
8753 }
8754};
8755} // namespace
8756
8757void StepOp::getCanonicalizationPatterns(RewritePatternSet &results,
8758 MLIRContext *context) {
8759 results.add<StepCompareFolder>(arg&: context);
8760}
8761
8762//===----------------------------------------------------------------------===//
8763// Vector Masking Utilities
8764//===----------------------------------------------------------------------===//
8765
8766/// Create the vector.yield-ended region of a vector.mask op with `maskableOp`
8767/// as masked operation.
8768void mlir::vector::createMaskOpRegion(OpBuilder &builder,
8769 Operation *maskableOp) {
8770 assert(maskableOp->getBlock() && "MaskableOp must be inserted into a block");
8771 Block *insBlock = builder.getInsertionBlock();
8772 // Create a block and move the op to that block.
8773 insBlock->getOperations().splice(
8774 where: insBlock->begin(), L2&: maskableOp->getBlock()->getOperations(), N: maskableOp);
8775 YieldOp::create(builder, location: maskableOp->getLoc(), operands: maskableOp->getResults());
8776}
8777
8778/// Creates a vector.mask operation around a maskable operation. Returns the
8779/// vector.mask operation if the mask provided is valid. Otherwise, returns
8780/// the maskable operation itself.
8781Operation *mlir::vector::maskOperation(OpBuilder &builder,
8782 Operation *maskableOp, Value mask,
8783 Value passthru) {
8784 if (!mask)
8785 return maskableOp;
8786 if (passthru)
8787 return MaskOp::create(builder, location: maskableOp->getLoc(),
8788 resultTypes: maskableOp->getResultTypes(), mask, passthru,
8789 maskableOp, maskRegion: createMaskOpRegion);
8790 return MaskOp::create(builder, location: maskableOp->getLoc(),
8791 resultTypes: maskableOp->getResultTypes(), mask, maskableOp,
8792 maskRegion: createMaskOpRegion);
8793}
8794
8795/// Creates a vector select operation that picks values from `newValue` or
8796/// `passthru` for each result vector lane based on `mask`. This utility is used
8797/// to propagate the pass-thru value of vector.mask or for cases where only the
8798/// pass-thru value propagation is needed. VP intrinsics do not support
8799/// pass-thru values and every mask-out lane is set to poison. LLVM backends are
8800/// usually able to match op + select patterns and fold them into a native
8801/// target instructions.
8802Value mlir::vector::selectPassthru(OpBuilder &builder, Value mask,
8803 Value newValue, Value passthru) {
8804 if (!mask)
8805 return newValue;
8806
8807 return arith::SelectOp::create(builder, location: newValue.getLoc(), result: newValue.getType(),
8808 condition: mask, true_value: newValue, false_value: passthru);
8809}
8810
8811//===----------------------------------------------------------------------===//
8812// InterleaveOp
8813//===----------------------------------------------------------------------===//
8814
8815namespace {
8816
8817/// This folder works on the following round-trip identity:
8818/// interleave(deinterleave(x).even, deinterleave(x).odd) -> x
8819struct InterleaveDeinterleaveFolder : public OpRewritePattern<InterleaveOp> {
8820 using Base::Base;
8821
8822 LogicalResult matchAndRewrite(InterleaveOp interleaveOp,
8823 PatternRewriter &rewriter) const override {
8824 auto lhsDefOp = interleaveOp.getLhs().getDefiningOp<DeinterleaveOp>();
8825 auto rhsDefOp = interleaveOp.getRhs().getDefiningOp<DeinterleaveOp>();
8826 if (!lhsDefOp || !rhsDefOp || lhsDefOp != rhsDefOp)
8827 return failure();
8828 for (auto [idx, operand] : llvm::enumerate(First: interleaveOp.getOperands())) {
8829 if (cast<OpResult>(Val&: operand).getResultNumber() != idx)
8830 return failure();
8831 }
8832 rewriter.replaceOp(op: interleaveOp, newValues: lhsDefOp.getSource());
8833 return success();
8834 }
8835};
8836} // namespace
8837
8838void InterleaveOp::getCanonicalizationPatterns(RewritePatternSet &results,
8839 MLIRContext *context) {
8840 results.add<InterleaveDeinterleaveFolder>(arg&: context);
8841}
8842
8843OpFoldResult InterleaveOp::fold(FoldAdaptor adaptor) {
8844 // interleave(splat(x), splat(x)) -> widened splat(x)
8845 auto splat = dyn_cast_if_present<SplatElementsAttr>(Val: adaptor.getLhs());
8846 if (!splat || adaptor.getLhs() != adaptor.getRhs())
8847 return {};
8848 return SplatElementsAttr::get(type: getResultVectorType(),
8849 values: splat.getSplatValue<Attribute>());
8850}
8851
8852//-----------------------------------------------------------------------------
8853// VectorUnrollOpInterface methods
8854//-----------------------------------------------------------------------------
8855std::optional<SmallVector<int64_t, 4>>
8856InterleaveOp::getIterationShapeToUnroll() {
8857 return llvm::to_vector<4>(Range: getResultVectorType().getShape());
8858}
8859
8860std::optional<SmallVector<int64_t>>
8861InterleaveOp::getDefaultUnrollTargetShape() {
8862 return getDefaultTargetInterleaveShapeForUnroll();
8863}
8864
8865//===----------------------------------------------------------------------===//
8866// DeinterleaveOp
8867//===----------------------------------------------------------------------===//
8868//-----------------------------------------------------------------------------
8869// VectorUnrollOpInterface methods
8870//-----------------------------------------------------------------------------
8871
8872std::optional<SmallVector<int64_t, 4>>
8873DeinterleaveOp::getIterationShapeToUnroll() {
8874 return llvm::to_vector<4>(Range: getResultVectorType().getShape());
8875}
8876std::optional<SmallVector<int64_t>>
8877DeinterleaveOp::getDefaultUnrollTargetShape() {
8878 return getDefaultTargetInterleaveShapeForUnroll();
8879}
8880
8881//===----------------------------------------------------------------------===//
8882// VectorDialect canonicalization
8883//===----------------------------------------------------------------------===//
8884
8885namespace {
8886/// A broadcast of a scalar condition selects the same arm in every lane.
8887struct SelectBroadcastCondition : OpRewritePattern<arith::SelectOp> {
8888 using Base::Base;
8889
8890 LogicalResult matchAndRewrite(arith::SelectOp select,
8891 PatternRewriter &rewriter) const override {
8892 auto broadcast = select.getCondition().getDefiningOp<BroadcastOp>();
8893 if (!broadcast || !broadcast.getSourceType().isSignlessInteger(width: 1))
8894 return failure();
8895 rewriter.modifyOpInPlace(root: select, callable: [&] {
8896 select.getConditionMutable().assign(value: broadcast.getSource());
8897 });
8898 return success();
8899 }
8900};
8901} // namespace
8902
8903void VectorDialect::getCanonicalizationPatterns(
8904 RewritePatternSet &results) const {
8905 results.add<SelectBroadcastCondition>(arg: getContext());
8906}
8907
8908//===----------------------------------------------------------------------===//
8909// TableGen'd op method definitions
8910//===----------------------------------------------------------------------===//
8911
8912#define GET_ATTRDEF_CLASSES
8913#include "mlir/Dialect/Vector/IR/VectorAttributes.cpp.inc"
8914
8915#define GET_OP_CLASSES
8916#include "mlir/Dialect/Vector/IR/VectorOps.cpp.inc"
8917