1//===- VectorDropLeadUnitDim.cpp - Conversion within the Vector dialect ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include <numeric>
10
11#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
12#include "mlir/Dialect/Vector/IR/VectorOps.h"
13#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
14#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
15#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
16#include "mlir/IR/Builders.h"
17#include "mlir/IR/TypeUtilities.h"
18#include "llvm/ADT/STLExtras.h"
19
20#define DEBUG_TYPE "vector-drop-unit-dim"
21
22using namespace mlir;
23using namespace mlir::vector;
24
25// Trims leading one dimensions (fixed-width) from `oldType` and returns the
26// result type. Returns `vector<1xT>` if `oldType` only has one element.
27static VectorType trimLeadingUnitDims(VectorType oldType,
28 bool trimOnlyOneDim = false,
29 bool allowRank0 = false) {
30 ArrayRef<int64_t> oldShape = oldType.getShape();
31 ArrayRef<int64_t> newShape = oldShape;
32
33 ArrayRef<bool> oldScalableDims = oldType.getScalableDims();
34 ArrayRef<bool> newScalableDims = oldScalableDims;
35
36 while (!newShape.empty() && newShape.front() == 1 &&
37 !newScalableDims.front()) {
38 newShape = newShape.drop_front(N: 1);
39 newScalableDims = newScalableDims.drop_front(N: 1);
40
41 if (trimOnlyOneDim)
42 break;
43 }
44
45 // Make sure we have at least 1 dimension per vector type requirements.
46 if (newShape.empty() && !allowRank0) {
47 newShape = oldShape.take_back();
48 newScalableDims = oldType.getScalableDims().take_back();
49 }
50 return VectorType::get(shape: newShape, elementType: oldType.getElementType(), scalableDims: newScalableDims);
51}
52
53// Returns true if one of the leading `numDims` dims of `xferOp` may be out of
54// bounds. Dropping such a dim would make it a non-vector dim, and those must be
55// in bounds.
56static bool isLeadingDimOutOfBounds(VectorTransferOpInterface xferOp,
57 int64_t numDims) {
58 return llvm::any_of(Range: llvm::seq<int64_t>(Size: numDims),
59 P: [&](int64_t dim) { return !xferOp.isDimInBounds(dim); });
60}
61
62/// Return a smallVector of size `rank` containing all zeros.
63static SmallVector<int64_t> splatZero(int64_t rank) {
64 return SmallVector<int64_t>(rank, 0);
65}
66
67static Operation *createWithProperties(OpBuilder &builder, Operation *op,
68 ValueRange operands,
69 TypeRange resultTypes) {
70 OperationState state(op->getLoc(), op->getName(), operands, resultTypes,
71 op->getDiscardableAttrDictionary().getValue());
72 state.propertiesAttr = op->getPropertiesAsAttribute();
73 return builder.create(state);
74}
75namespace {
76
77// Casts away leading one dimensions in vector.extract_strided_slice's vector
78// input by inserting vector.broadcast.
79struct CastAwayExtractStridedSliceLeadingOneDim
80 : public OpRewritePattern<vector::ExtractStridedSliceOp> {
81 using Base::Base;
82
83 LogicalResult matchAndRewrite(vector::ExtractStridedSliceOp extractOp,
84 PatternRewriter &rewriter) const override {
85 // vector.extract_strided_slice requires the input and output vector to have
86 // the same rank. Here we drop leading one dimensions from the input vector
87 // type to make sure we don't cause mismatch.
88 VectorType oldSrcType = extractOp.getSourceVectorType();
89 VectorType newSrcType = trimLeadingUnitDims(oldType: oldSrcType);
90
91 if (newSrcType.getRank() == oldSrcType.getRank())
92 return failure();
93
94 int64_t dropCount = oldSrcType.getRank() - newSrcType.getRank();
95
96 VectorType oldDstType = extractOp.getType();
97 VectorType newDstType =
98 VectorType::get(shape: oldDstType.getShape().drop_front(N: dropCount),
99 elementType: oldDstType.getElementType(),
100 scalableDims: oldDstType.getScalableDims().drop_front(N: dropCount));
101
102 Location loc = extractOp.getLoc();
103
104 Value newSrcVector = rewriter.createOrFold<ShapeCastOp>(
105 location: loc, args&: newSrcType, args: extractOp.getSource());
106
107 // The offsets/sizes/strides attribute can have a less number of elements
108 // than the input vector's rank: it is meant for the leading dimensions.
109 auto newOffsets = rewriter.getArrayAttr(
110 value: extractOp.getOffsets().getValue().drop_front(N: dropCount));
111 auto newSizes = rewriter.getArrayAttr(
112 value: extractOp.getSizes().getValue().drop_front(N: dropCount));
113 auto newStrides = rewriter.getArrayAttr(
114 value: extractOp.getStrides().getValue().drop_front(N: dropCount));
115
116 auto newExtractOp = vector::ExtractStridedSliceOp::create(
117 builder&: rewriter, location: loc, resultType0: newDstType, source: newSrcVector, offsets: newOffsets, sizes: newSizes,
118 strides: newStrides);
119
120 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: extractOp, args&: oldDstType,
121 args&: newExtractOp);
122
123 return success();
124 }
125};
126
127// Casts away leading one dimensions in vector.insert_strided_slice's vector
128// inputs by inserting vector.broadcast.
129struct CastAwayInsertStridedSliceLeadingOneDim
130 : public OpRewritePattern<vector::InsertStridedSliceOp> {
131 using Base::Base;
132
133 LogicalResult matchAndRewrite(vector::InsertStridedSliceOp insertOp,
134 PatternRewriter &rewriter) const override {
135 VectorType oldSrcType = insertOp.getSourceVectorType();
136 VectorType newSrcType = trimLeadingUnitDims(oldType: oldSrcType);
137 VectorType oldDstType = insertOp.getDestVectorType();
138 VectorType newDstType = trimLeadingUnitDims(oldType: oldDstType);
139
140 int64_t srcDropCount = oldSrcType.getRank() - newSrcType.getRank();
141 int64_t dstDropCount = oldDstType.getRank() - newDstType.getRank();
142 if (srcDropCount == 0 && dstDropCount == 0)
143 return failure();
144
145 // Trim leading one dimensions from both operands.
146 Location loc = insertOp.getLoc();
147
148 Value newSrcVector = rewriter.createOrFold<vector::ShapeCastOp>(
149 location: loc, args&: newSrcType, args: insertOp.getValueToStore());
150 Value newDstVector = rewriter.createOrFold<vector::ShapeCastOp>(
151 location: loc, args&: newDstType, args: insertOp.getDest());
152
153 auto newOffsets = rewriter.getArrayAttr(
154 value: insertOp.getOffsets().getValue().take_back(N: newDstType.getRank()));
155 auto newStrides = rewriter.getArrayAttr(
156 value: insertOp.getStrides().getValue().take_back(N: newSrcType.getRank()));
157
158 auto newInsertOp = vector::InsertStridedSliceOp::create(
159 builder&: rewriter, location: loc, result: newDstType, valueToStore: newSrcVector, dest: newDstVector, offsets: newOffsets,
160 strides: newStrides);
161
162 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: insertOp, args&: oldDstType,
163 args&: newInsertOp);
164
165 return success();
166 }
167};
168
169// Casts away leading one dimensions in vector.insert's vector inputs by
170// inserting vector.shape_cast.
171struct CastAwayInsertLeadingOneDim : public OpRewritePattern<vector::InsertOp> {
172 using Base::Base;
173
174 LogicalResult matchAndRewrite(vector::InsertOp insertOp,
175 PatternRewriter &rewriter) const override {
176 Type oldSrcType = insertOp.getValueToStoreType();
177 Type newSrcType = oldSrcType;
178 int64_t oldSrcRank = 0, newSrcRank = 0;
179 if (auto type = dyn_cast<VectorType>(Val&: oldSrcType)) {
180 newSrcType = trimLeadingUnitDims(oldType: type);
181 oldSrcRank = type.getRank();
182 newSrcRank = cast<VectorType>(Val&: newSrcType).getRank();
183 }
184
185 VectorType oldDstType = insertOp.getDestVectorType();
186 VectorType newDstType = trimLeadingUnitDims(oldType: oldDstType);
187
188 int64_t srcDropCount = oldSrcRank - newSrcRank;
189 int64_t dstDropCount = oldDstType.getRank() - newDstType.getRank();
190 if (srcDropCount == 0 && dstDropCount == 0)
191 return failure();
192
193 // Trim leading one dimensions from both operands.
194 Location loc = insertOp.getLoc();
195
196 Value newSrcVector = insertOp.getValueToStore();
197 if (oldSrcRank != 0) {
198 newSrcVector = rewriter.createOrFold<vector::ShapeCastOp>(
199 location: loc, args: cast<VectorType>(Val&: newSrcType), args: insertOp.getValueToStore());
200 }
201 Value newDstVector = rewriter.createOrFold<vector::ShapeCastOp>(
202 location: loc, args&: newDstType, args: insertOp.getDest());
203
204 // New position rank needs to be computed in two steps: (1) if destination
205 // type has leading unit dims, we also trim the position array accordingly,
206 // then (2) if source type also has leading unit dims, we need to append
207 // zeroes to the position array accordingly.
208 unsigned oldPosRank = insertOp.getNumIndices();
209 unsigned newPosRank = std::max<int64_t>(a: 0, b: oldPosRank - dstDropCount);
210 SmallVector<OpFoldResult> oldPosition = insertOp.getMixedPosition();
211 SmallVector<OpFoldResult> newPosition =
212 llvm::to_vector(Range: ArrayRef(oldPosition).take_back(N: newPosRank));
213 newPosition.resize(N: newDstType.getRank() - newSrcRank,
214 NV: rewriter.getI64IntegerAttr(value: 0));
215
216 auto newInsertOp = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: newSrcVector,
217 dest: newDstVector, position: newPosition);
218
219 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: insertOp, args&: oldDstType,
220 args&: newInsertOp);
221
222 return success();
223 }
224};
225
226static Value dropUnitDimsFromMask(OpBuilder &b, Location loc, Value mask,
227 VectorType newType, AffineMap newMap) {
228 VectorType newMaskType = inferTransferOpMaskType(vecType: newType, permMap: newMap);
229
230 return vector::ShapeCastOp::create(builder&: b, location: loc, result: newMaskType, source: mask);
231}
232
233// Turns vector.transfer_read on vector with leading 1 dimensions into
234// vector.shape_cast followed by vector.transfer_read on vector without leading
235// 1 dimensions.
236struct CastAwayTransferReadLeadingOneDim
237 : public OpRewritePattern<vector::TransferReadOp> {
238 using Base::Base;
239
240 LogicalResult matchAndRewrite(vector::TransferReadOp read,
241 PatternRewriter &rewriter) const override {
242 // TODO(#78787): Not supported masked op yet.
243 if (cast<MaskableOpInterface>(Val: read.getOperation()).isMasked())
244 return failure();
245
246 if (read.getTransferRank() == 0)
247 return rewriter.notifyMatchFailure(
248 arg&: read, msg: "Nothing to trim - the transfer itself has rank zero");
249
250 auto shapedType = cast<ShapedType>(Val: read.getBase().getType());
251 if (shapedType.getElementType() != read.getVectorType().getElementType())
252 return failure();
253
254 VectorType oldType = read.getVectorType();
255 VectorType newType = trimLeadingUnitDims(oldType);
256
257 if (newType == oldType)
258 return failure();
259
260 if (isLeadingDimOutOfBounds(xferOp: read, numDims: oldType.getRank() - newType.getRank()))
261 return rewriter.notifyMatchFailure(arg&: read, msg: "dropped dim is out of bounds");
262
263 AffineMap oldMap = read.getPermutationMap();
264 ArrayRef<AffineExpr> newResults =
265 oldMap.getResults().take_back(N: newType.getRank());
266 AffineMap newMap =
267 AffineMap::get(dimCount: oldMap.getNumDims(), symbolCount: oldMap.getNumSymbols(), results: newResults,
268 context: rewriter.getContext());
269
270 ArrayAttr inBoundsAttr;
271 if (read.getInBounds())
272 inBoundsAttr = rewriter.getArrayAttr(
273 value: read.getInBoundsAttr().getValue().take_back(N: newType.getRank()));
274
275 Value mask = Value();
276 if (read.getMask())
277 mask = dropUnitDimsFromMask(b&: rewriter, loc: read.getLoc(), mask: read.getMask(),
278 newType, newMap);
279
280 auto newRead = vector::TransferReadOp::create(
281 builder&: rewriter, location: read.getLoc(), result: newType, base: read.getBase(), indices: read.getIndices(),
282 permutation_map: AffineMapAttr::get(value: newMap), padding: read.getPadding(), mask, in_bounds: inBoundsAttr);
283 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: read, args&: oldType, args&: newRead);
284
285 return success();
286 }
287};
288
289// Turns vector.transfer_write on vector with leading 1 dimensions into
290// vector.shape_cast followed by vector.transfer_write on vector without leading
291// 1 dimensions.
292struct CastAwayTransferWriteLeadingOneDim
293 : public OpRewritePattern<vector::TransferWriteOp> {
294 using Base::Base;
295
296 LogicalResult matchAndRewrite(vector::TransferWriteOp write,
297 PatternRewriter &rewriter) const override {
298 // TODO(#78787): Not supported masked op yet.
299 if (cast<MaskableOpInterface>(Val: write.getOperation()).isMasked())
300 return failure();
301
302 if (write.getTransferRank() == 0)
303 return rewriter.notifyMatchFailure(
304 arg&: write, msg: "Nothing to trim - the transfer itself has rank zero");
305
306 auto shapedType = dyn_cast<ShapedType>(Val: write.getBase().getType());
307 if (shapedType.getElementType() != write.getVectorType().getElementType())
308 return failure();
309
310 VectorType oldType = write.getVectorType();
311 VectorType newType = trimLeadingUnitDims(oldType);
312 if (newType == oldType)
313 return failure();
314
315 if (isLeadingDimOutOfBounds(xferOp: write, numDims: oldType.getRank() - newType.getRank()))
316 return rewriter.notifyMatchFailure(arg&: write, msg: "dropped dim is out of bounds");
317
318 AffineMap oldMap = write.getPermutationMap();
319 ArrayRef<AffineExpr> newResults =
320 oldMap.getResults().take_back(N: newType.getRank());
321 AffineMap newMap =
322 AffineMap::get(dimCount: oldMap.getNumDims(), symbolCount: oldMap.getNumSymbols(), results: newResults,
323 context: rewriter.getContext());
324
325 ArrayAttr inBoundsAttr;
326 if (write.getInBounds())
327 inBoundsAttr = rewriter.getArrayAttr(
328 value: write.getInBoundsAttr().getValue().take_back(N: newType.getRank()));
329
330 auto newVector = rewriter.createOrFold<vector::ShapeCastOp>(
331 location: write.getLoc(), args&: newType, args: write.getVector());
332
333 if (write.getMask()) {
334 Value newMask = dropUnitDimsFromMask(b&: rewriter, loc: write.getLoc(),
335 mask: write.getMask(), newType, newMap);
336 rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
337 op: write, args&: newVector, args: write.getBase(), args: write.getIndices(),
338 args: AffineMapAttr::get(value: newMap), args&: newMask, args&: inBoundsAttr);
339 return success();
340 }
341
342 rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
343 op: write, args&: newVector, args: write.getBase(), args: write.getIndices(),
344 args: AffineMapAttr::get(value: newMap), args&: inBoundsAttr);
345 return success();
346 }
347};
348
349} // namespace
350
351// Takes `oldVal` and "drops" the leading unit dim with either ShapeCastOp or
352// ExtractOp. The latter is used for rank-1 vectors to make sure that a scalar
353// (as opposed to rank-0 vector) is generated. This is a requirement of e.g.
354// ContractOp.
355static Value dropLeadingUnitDimViaShapeCastOrExtract(RewriterBase &rewriter,
356 Location loc,
357 mlir::Value oldVal) {
358 auto oldValTy = cast<VectorType>(Val: oldVal.getType());
359 if (oldValTy.getRank() == 1) {
360 return rewriter.createOrFold<ExtractOp>(location: loc, args&: oldVal, args: 0);
361 }
362
363 return rewriter.createOrFold<ShapeCastOp>(
364 location: loc,
365 args: trimLeadingUnitDims(oldType: oldValTy,
366 /*trimOnlyOneDim=*/true,
367 /*allowRank0=*/true),
368 args&: oldVal);
369}
370
371// Takes `oldVal` and "adds" leading unit dim with either ShapeCastOp or
372// BroadcastOp. The latter is used for scalar inputs as ShapeCastOp cannot
373// "broadcast" from a scalar. Scalars are used (instead of rank-0 vectors) as
374// ContractOp operands.
375static Value restoreLeadingUnitDimViaShapeCastOrBcast(RewriterBase &rewriter,
376 Location loc,
377 mlir::Value oldVal,
378 mlir::Type newTy) {
379 if (!isa<VectorType>(Val: oldVal.getType())) {
380 return rewriter.createOrFold<BroadcastOp>(location: loc, args&: newTy, args&: oldVal);
381 }
382
383 return rewriter.createOrFold<ShapeCastOp>(location: loc, args&: newTy, args&: oldVal);
384}
385
386FailureOr<Value>
387mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp,
388 MaskingOpInterface maskingOp,
389 RewriterBase &rewriter) {
390 VectorType oldAccType = dyn_cast<VectorType>(Val: contractOp.getAccType());
391 if (oldAccType == nullptr)
392 return failure();
393 if (oldAccType.getRank() < 1)
394 return failure();
395 if (oldAccType.getShape()[0] != 1 || oldAccType.getScalableDims()[0])
396 return failure();
397
398 auto oldIndexingMaps = contractOp.getIndexingMapsArray();
399 SmallVector<AffineMap> newIndexingMaps;
400
401 auto oldIteratorTypes = contractOp.getIteratorTypes();
402 SmallVector<Attribute> newIteratorTypes;
403
404 // 0-th dim from the accumulator
405 int64_t dimToDrop = oldIndexingMaps[2].getDimPosition(idx: 0);
406
407 if (!isParallelIterator(attr: oldIteratorTypes[dimToDrop]))
408 // only parallel type iterators can be dropped.
409 return failure();
410
411 for (const auto &it : llvm::enumerate(First&: oldIteratorTypes)) {
412 int64_t currDim = it.index();
413 if (currDim == dimToDrop)
414 continue;
415 newIteratorTypes.push_back(Elt: it.value());
416 }
417
418 SmallVector<Value> operands = {contractOp.getLhs(), contractOp.getRhs(),
419 contractOp.getAcc()};
420 SmallVector<Value> newOperands;
421 auto loc = contractOp.getLoc();
422
423 for (const auto &it : llvm::enumerate(First&: oldIndexingMaps)) {
424 // Check if the dim to be dropped exists as a leading dim in the operand
425 // if it does then we use vector.extract to drop it.
426 bool validExtract = false;
427 SmallVector<AffineExpr> results;
428 auto map = it.value();
429 int64_t orginalZeroDim = it.value().getDimPosition(idx: 0);
430 if (orginalZeroDim != dimToDrop) {
431 // There are two reasons to be in this path, 1. We need to
432 // transpose the operand to make the dim to be dropped
433 // leading. 2. The dim to be dropped does not exist and in
434 // that case we dont want to add a unit transpose but we must
435 // check all the indices to make sure this is the case.
436 bool transposeNeeded = false;
437 SmallVector<int64_t> perm;
438 SmallVector<AffineExpr> transposeResults;
439
440 for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) {
441 int64_t currDim = map.getDimPosition(idx: i);
442 if (currDim == dimToDrop) {
443 transposeNeeded = true;
444 perm.insert(I: perm.begin(), Elt: i);
445 auto targetExpr = rewriter.getAffineDimExpr(position: currDim);
446 transposeResults.insert(I: transposeResults.begin(), Elt: targetExpr);
447 } else {
448 perm.push_back(Elt: i);
449 auto targetExpr = rewriter.getAffineDimExpr(position: currDim);
450 transposeResults.push_back(Elt: targetExpr);
451 }
452 }
453
454 // Checks if only the outer, unit dimensions (of size 1) are permuted.
455 // Such transposes do not materially effect the underlying vector and can
456 // be omitted. EG: perm [1, 0, 2] applied to vector<1x1x8xi32>
457 bool transposeNonOuterUnitDims = false;
458 auto operandShape = cast<ShapedType>(Val: operands[it.index()].getType());
459 for (auto [index, dim] :
460 llvm::enumerate(First: ArrayRef<int64_t>(perm).drop_back(N: 1))) {
461 if (dim != static_cast<int64_t>(index) &&
462 operandShape.getDimSize(idx: index) != 1) {
463 transposeNonOuterUnitDims = true;
464 break;
465 }
466 }
467
468 // Do the transpose now if needed so that we can drop the
469 // correct dim using extract later.
470 if (transposeNeeded) {
471 map = AffineMap::get(dimCount: map.getNumDims(), symbolCount: 0, results: transposeResults,
472 context: contractOp.getContext());
473 if (transposeNonOuterUnitDims) {
474 // TODO: While the discussion on the validity of folding
475 // TransposeOp into ShapeCastOp continues, see e.g.
476 // * https://github.com/llvm/llvm-project/pull/219611,
477 // keep the explicit TransposeOp here. Note that existing TransposeOp
478 // folders already turn it into a ShapeCastOp, as demonstrated by the
479 // tests. Revisit this and consider inserting ShapeCastOp directly
480 // once the discussion progresses.
481 operands[it.index()] = rewriter.createOrFold<vector::TransposeOp>(
482 location: loc, args&: operands[it.index()], args&: perm);
483 }
484 }
485 }
486 // We have taken care to have the dim to be dropped be
487 // the leading dim. If its still not leading that means it
488 // does not exist in this operand and hence we do not need
489 // an extract.
490 if (map.getDimPosition(idx: 0) == dimToDrop)
491 validExtract = true;
492
493 for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) {
494 int64_t currDim = map.getDimPosition(idx: i);
495 if (currDim == dimToDrop)
496 // This is the dim we are dropping.
497 continue;
498 auto targetExpr = rewriter.getAffineDimExpr(
499 position: currDim < dimToDrop ? currDim : currDim - 1);
500 results.push_back(Elt: targetExpr);
501 }
502 newIndexingMaps.push_back(Elt: AffineMap::get(dimCount: map.getNumDims() - 1, symbolCount: 0, results,
503 context: contractOp.getContext()));
504 // Extract if its a valid extraction, otherwise use the operand
505 // without extraction.
506 auto oldVal = operands[it.index()];
507 newOperands.push_back(
508 Elt: validExtract
509 ? dropLeadingUnitDimViaShapeCastOrExtract(rewriter, loc, oldVal)
510 : oldVal);
511 }
512
513 // Depending on whether this vector.contract is masked, the replacing Op
514 // should either be a new vector.contract Op or vector.mask Op.
515 Operation *newOp = vector::ContractionOp::create(
516 builder&: rewriter, location: loc, lhs: newOperands[0], rhs: newOperands[1], acc: newOperands[2],
517 indexingMaps: rewriter.getAffineMapArrayAttr(values: newIndexingMaps),
518 iteratorTypes: rewriter.getArrayAttr(value: newIteratorTypes), kind: contractOp.getKind(),
519 fastMathFlags: contractOp.getFastmath());
520
521 if (maskingOp) {
522 auto newMask = rewriter.createOrFold<ShapeCastOp>(
523 location: loc,
524 args: trimLeadingUnitDims(oldType: cast<VectorType>(Val: maskingOp.getMask().getType()),
525 /*trimOnlyOneDim=*/true, /*allowRank0=*/true),
526 args: maskingOp.getMask());
527
528 newOp = mlir::vector::maskOperation(builder&: rewriter, maskableOp: newOp, mask: newMask);
529 }
530
531 return restoreLeadingUnitDimViaShapeCastOrBcast(
532 rewriter, loc, oldVal: newOp->getResult(idx: 0), newTy: contractOp->getResultTypes()[0]);
533}
534
535namespace {
536
537/// Turns vector.contract on vector with leading 1 dimensions into
538/// vector.shape_cast followed by vector.contract on vector without leading
539/// 1 dimensions. Also performs transpose of lhs and rhs operands if required.
540///
541/// TODO: While the discussion on the validity of folding TransposeOp into
542/// ShapeCastOp continues, see e.g.
543/// * https://github.com/llvm/llvm-project/pull/219611,
544/// keep the explicit TransposeOp here. Once the discussion settles, revisit and
545/// consider replacing TransposeOp with ShapeCastOp.
546struct CastAwayContractionLeadingOneDim
547 : public MaskableOpRewritePattern<vector::ContractionOp> {
548 using MaskableOpRewritePattern::MaskableOpRewritePattern;
549
550 FailureOr<Value>
551 matchAndRewriteMaskableOp(vector::ContractionOp contractOp,
552 MaskingOpInterface maskingOp,
553 PatternRewriter &rewriter) const override {
554 return castAwayContractionLeadingOneDim(contractOp, maskingOp, rewriter);
555 }
556};
557
558/// Looks at elementwise operations on vectors with at least one leading
559/// dimension equal 1, e.g. vector<1x[4]x1xf32> (but not vector<2x[4]x1xf32>),
560/// and cast aways the leading one dimensions (_plural_) and then broadcasts
561/// the results.
562///
563/// Example before:
564/// %1 = arith.mulf %arg0, %arg1 : vector<1x4x1xf32>
565/// Example after:
566/// %2 = arith.mulf %0, %1 : vector<4x1xf32>
567/// %3 = vector.broadcast %2 : vector<4x1xf32> to vector<1x4x1xf32>
568///
569/// Does support scalable vectors.
570class CastAwayElementwiseLeadingOneDim : public RewritePattern {
571public:
572 CastAwayElementwiseLeadingOneDim(MLIRContext *context,
573 PatternBenefit benefit = 1)
574 : RewritePattern(MatchAnyOpTypeTag(), benefit, context) {}
575
576 LogicalResult matchAndRewrite(Operation *op,
577 PatternRewriter &rewriter) const override {
578 if (!OpTrait::hasElementwiseMappableTraits(op) || op->getNumResults() != 1)
579 return failure();
580 auto vecType = dyn_cast<VectorType>(Val: op->getResultTypes()[0]);
581 if (!vecType)
582 return failure();
583 VectorType newVecType = trimLeadingUnitDims(oldType: vecType);
584 if (newVecType == vecType)
585 return failure();
586 int64_t dropDim = vecType.getRank() - newVecType.getRank();
587 SmallVector<Value, 4> newOperands;
588 for (Value operand : op->getOperands()) {
589 if (auto opVecType = dyn_cast<VectorType>(Val: operand.getType())) {
590 newOperands.push_back(Elt: vector::ExtractOp::create(
591 builder&: rewriter, location: op->getLoc(), source: operand, position: splatZero(rank: dropDim)));
592 } else {
593 newOperands.push_back(Elt: operand);
594 }
595 }
596 Operation *newOp =
597 createWithProperties(builder&: rewriter, op, operands: newOperands, resultTypes: TypeRange{newVecType});
598 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, args&: vecType,
599 args: newOp->getResult(idx: 0));
600 return success();
601 }
602};
603} // namespace
604
605// Drops `dropDim` leading dimensions from `operand` using vector.extract when
606// those dims are all non-scalable units (the cheap, structural rewrite); falls
607// back to vector.shape_cast otherwise.
608static Value dropLeadingOneDimsFromOperand(OpBuilder &b, Location loc,
609 Value operand, int64_t nDropped) {
610 auto oldType = cast<VectorType>(Val: operand.getType());
611 ArrayRef<int64_t> leadingShape = oldType.getShape().take_front(N: nDropped);
612 ArrayRef<bool> leadingScalable =
613 oldType.getScalableDims().take_front(N: nDropped);
614 bool extractable =
615 llvm::all_of(Range&: leadingShape, P: [](int64_t d) { return d == 1; }) &&
616 llvm::none_of(Range&: leadingScalable, P: [](bool s) { return s; });
617 if (extractable)
618 return vector::ExtractOp::create(builder&: b, location: loc, source: operand, position: splatZero(rank: nDropped));
619 VectorType newType = VectorType::get(
620 shape: oldType.getShape().drop_front(N: nDropped), elementType: oldType.getElementType(),
621 scalableDims: oldType.getScalableDims().drop_front(N: nDropped));
622 return vector::ShapeCastOp::create(builder&: b, location: loc, result: newType, source: operand);
623}
624
625namespace {
626
627// Drops leading 1 dimensions from load-like memory operaitons. REmoves leading
628// unit dimensions from the result types and then broadcasts back in those 1s,
629// while also extracting (or shape_cast-ing) any leading unit dimensions on
630// the input operands.
631template <typename OpTy>
632struct CastAwayLoadLikeLeadingOneDim : public OpRewritePattern<OpTy> {
633 using OpRewritePattern<OpTy>::OpRewritePattern;
634
635 LogicalResult matchAndRewrite(OpTy op,
636 PatternRewriter &rewriter) const override {
637 VectorType oldResultType = op.getVectorType();
638 VectorType newResultType = trimLeadingUnitDims(oldType: oldResultType);
639 if (newResultType == oldResultType)
640 return failure();
641 int64_t nDropped = oldResultType.getRank() - newResultType.getRank();
642
643 Location loc = op.getLoc();
644 SmallVector<Value> newOperands;
645 newOperands.reserve(N: op->getNumOperands());
646 for (Value operand : op->getOperands()) {
647 if (isa<VectorType>(Val: operand.getType())) {
648 newOperands.push_back(
649 Elt: dropLeadingOneDimsFromOperand(b&: rewriter, loc, operand, nDropped));
650 } else {
651 newOperands.push_back(Elt: operand);
652 }
653 }
654
655 Operation *newOp = createWithProperties(rewriter, op, newOperands,
656 TypeRange{newResultType});
657 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, oldResultType,
658 newOp->getResult(idx: 0));
659 return success();
660 }
661};
662
663// Drops leading 1 dimensions from store-like memory ops. Extracts or
664// `shape_cast`s away those leading unit dimensions and leaves any scalar
665// operands alone.
666template <typename OpTy>
667struct CastAwayStoreLikeLeadingOneDim : public OpRewritePattern<OpTy> {
668 using OpRewritePattern<OpTy>::OpRewritePattern;
669
670 LogicalResult matchAndRewrite(OpTy op,
671 PatternRewriter &rewriter) const override {
672 VectorType oldVecType = op.getVectorType();
673 VectorType newVecType = trimLeadingUnitDims(oldType: oldVecType);
674 if (newVecType == oldVecType)
675 return failure();
676 int64_t nDropped = oldVecType.getRank() - newVecType.getRank();
677
678 Location loc = op.getLoc();
679 SmallVector<Value> newOperands;
680 newOperands.reserve(N: op->getNumOperands());
681 for (Value operand : op->getOperands()) {
682 if (isa<VectorType>(Val: operand.getType())) {
683 newOperands.push_back(
684 Elt: dropLeadingOneDimsFromOperand(b&: rewriter, loc, operand, nDropped));
685 } else {
686 newOperands.push_back(Elt: operand);
687 }
688 }
689
690 Operation *newOp =
691 createWithProperties(rewriter, op, newOperands, op->getResultTypes());
692 rewriter.replaceOp(op, newOp->getResults());
693 return success();
694 }
695};
696
697// Drops leading 1 dimensions from vector.constant_mask and inserts a
698// vector.broadcast back to the original shape.
699struct CastAwayConstantMaskLeadingOneDim
700 : public OpRewritePattern<vector::ConstantMaskOp> {
701 using Base::Base;
702
703 LogicalResult matchAndRewrite(vector::ConstantMaskOp mask,
704 PatternRewriter &rewriter) const override {
705 VectorType oldType = mask.getType();
706 VectorType newType = trimLeadingUnitDims(oldType);
707
708 if (newType == oldType)
709 return failure();
710
711 int64_t dropDim = oldType.getRank() - newType.getRank();
712 ArrayRef<int64_t> dimSizes = mask.getMaskDimSizes();
713
714 // If any of the dropped unit dims has a size of `0`, the entire mask is a
715 // zero mask, else the unit dim has no effect on the mask.
716 int64_t flatLeadingSize =
717 llvm::product_of(Range: dimSizes.take_front(N: dropDim + 1));
718 SmallVector<int64_t> newDimSizes = {flatLeadingSize};
719 newDimSizes.append(in_start: dimSizes.begin() + dropDim + 1, in_end: dimSizes.end());
720
721 auto newMask = vector::ConstantMaskOp::create(builder&: rewriter, location: mask.getLoc(),
722 resultType0: newType, mask_dim_sizes: newDimSizes);
723 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: mask, args&: oldType, args&: newMask);
724 return success();
725 }
726};
727
728} // namespace
729
730void mlir::vector::populateCastAwayVectorLeadingOneDimPatterns(
731 RewritePatternSet &patterns, PatternBenefit benefit) {
732 patterns
733 .add<CastAwayExtractStridedSliceLeadingOneDim,
734 CastAwayInsertStridedSliceLeadingOneDim, CastAwayInsertLeadingOneDim,
735 CastAwayConstantMaskLeadingOneDim, CastAwayTransferReadLeadingOneDim,
736 CastAwayTransferWriteLeadingOneDim, CastAwayElementwiseLeadingOneDim,
737 CastAwayContractionLeadingOneDim,
738 CastAwayLoadLikeLeadingOneDim<vector::LoadOp>,
739 CastAwayLoadLikeLeadingOneDim<vector::MaskedLoadOp>,
740 CastAwayLoadLikeLeadingOneDim<vector::ExpandLoadOp>,
741 CastAwayLoadLikeLeadingOneDim<vector::GatherOp>,
742 CastAwayStoreLikeLeadingOneDim<vector::StoreOp>,
743 CastAwayStoreLikeLeadingOneDim<vector::MaskedStoreOp>,
744 CastAwayStoreLikeLeadingOneDim<vector::CompressStoreOp>,
745 CastAwayStoreLikeLeadingOneDim<vector::ScatterOp>>(
746 arg: patterns.getContext(), args&: benefit);
747}
748