1//===- VectorTransforms.cpp - Conversion within the Vector dialect --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements target-independent rewrites as 1->N patterns.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
14
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/Arith/Utils/Utils.h"
17#include "mlir/Dialect/MemRef/IR/MemRef.h"
18#include "mlir/Dialect/SCF/IR/SCF.h"
19#include "mlir/Dialect/Utils/IndexingUtils.h"
20#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
21#include "mlir/Dialect/Vector/IR/VectorOps.h"
22#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
23#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
24#include "mlir/IR/BuiltinTypes.h"
25#include "mlir/IR/Location.h"
26#include "mlir/IR/Matchers.h"
27#include "mlir/IR/PatternMatch.h"
28#include "mlir/IR/TypeUtilities.h"
29
30#include "llvm/ADT/STLExtras.h"
31#include "llvm/ADT/SmallVectorExtras.h"
32#include "llvm/Support/FormatVariadic.h"
33
34#include <cassert>
35#include <cstdint>
36#include <functional>
37#include <optional>
38
39#define DEBUG_TYPE "vector-to-vector"
40
41using namespace mlir;
42using namespace mlir::vector;
43
44static Operation *createWithProperties(OpBuilder &builder, Operation *op,
45 ValueRange operands, TypeRange types) {
46 OperationState state(op->getLoc(), op->getName(), operands, types,
47 op->getDiscardableAttrDictionary().getValue());
48 state.propertiesAttr = op->getPropertiesAsAttribute();
49 return builder.create(state);
50}
51
52// Helper to find an index in an affine map.
53static std::optional<int64_t> getResultIndex(AffineMap map, int64_t index) {
54 for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) {
55 int64_t idx = map.getDimPosition(idx: i);
56 if (idx == index)
57 return i;
58 }
59 return std::nullopt;
60}
61
62namespace {
63
64/// Convert MulIOp/MulFOp + MultiDimReductionOp<add> into ContractionOp.
65/// Ex:
66/// ```
67/// %0 = arith.mulf %arg0, %arg1 : vector<8x32x16xf32>
68/// %1 = vector.multi_reduction add, %0 [1]
69/// : vector<8x32x16xf32> to vector<8x16xf32>
70/// ```
71/// Gets converted to:
72/// ```
73/// %1 = vector.contract {indexing_maps = [
74/// affine_map<(d0, d1, d2) -> (d0, d1, d2)>,
75/// affine_map<(d0, d1, d2) -> (d0, d1, d2)>,
76/// affine_map<(d0, d1, d2) -> (d0, d1)>],
77/// iterator_types = ["parallel", "parallel", "reduction"],
78/// kind = add} %0, %arg1, %cst_f0
79/// : vector<8x32x16xf32>, vector<8x32x16xf32> into vector<8x32xf32>
80/// ```
81struct MultiReduceToContract
82 : public OpRewritePattern<vector::MultiDimReductionOp> {
83 MultiReduceToContract(MLIRContext *context, bool allowFastMathFlagLoss,
84 PatternBenefit benefit = 1)
85 : OpRewritePattern(context, benefit),
86 allowFastMathFlagLoss(allowFastMathFlagLoss) {}
87
88 LogicalResult matchAndRewrite(vector::MultiDimReductionOp reduceOp,
89 PatternRewriter &rewriter) const override {
90 // TODO: Support the other combining kinds that vector.contract supports.
91 if (reduceOp.getKind() != vector::CombiningKind::ADD)
92 return failure();
93 Operation *mulOp = reduceOp.getSource().getDefiningOp();
94 if (!mulOp || !isa<arith::MulIOp, arith::MulFOp>(Val: mulOp))
95 return failure();
96 // The fast-math flags of a vector.contract apply to both its elementwise
97 // multiply and its combining op. So the contract can only take the flags
98 // that both the multiply and the reduction carry; then neither of the two
99 // gains a flag it did not have. In particular, `contract` is kept only if
100 // both carry it. An `arith.muli` has no flags, so an integer contraction
101 // gets none. If any flag would be lost, convert only when allowed.
102 arith::FastMathFlags mulFlags = arith::FastMathFlags::none;
103 if (auto mulFOp = dyn_cast<arith::MulFOp>(Val: mulOp))
104 mulFlags = mulFOp.getFastmath();
105 arith::FastMathFlags reduceFlags = reduceOp.getFastmath();
106 if (!allowFastMathFlagLoss && mulFlags != reduceFlags)
107 return rewriter.notifyMatchFailure(
108 arg&: reduceOp, msg: "converting would drop fast-math flags");
109 arith::FastMathFlags fastMathFlags = mulFlags & reduceFlags;
110 SmallVector<bool> reductionMask = reduceOp.getReductionMask();
111 auto srcMap = rewriter.getMultiDimIdentityMap(rank: reductionMask.size());
112 SmallVector<AffineExpr> exprs;
113 SmallVector<vector::IteratorType> iteratorTypes;
114 for (const auto &isReduceDim : llvm::enumerate(First&: reductionMask)) {
115 if (!isReduceDim.value()) {
116 iteratorTypes.push_back(Elt: vector::IteratorType::parallel);
117 exprs.push_back(Elt: rewriter.getAffineDimExpr(position: isReduceDim.index()));
118 } else {
119 iteratorTypes.push_back(Elt: vector::IteratorType::reduction);
120 }
121 }
122 auto dstMap =
123 AffineMap::get(/*dimCount=*/reductionMask.size(),
124 /*symbolCount=*/0, results: exprs, context: reduceOp.getContext());
125 rewriter.replaceOpWithNewOp<mlir::vector::ContractionOp>(
126 op: reduceOp, args: mulOp->getOperand(idx: 0), args: mulOp->getOperand(idx: 1), args: reduceOp.getAcc(),
127 args: rewriter.getAffineMapArrayAttr(values: {srcMap, srcMap, dstMap}),
128 args: rewriter.getArrayAttr(value: llvm::map_to_vector(
129 C&: iteratorTypes,
130 F: [&](IteratorType t) -> mlir::Attribute {
131 return IteratorTypeAttr::get(context: rewriter.getContext(), value: t);
132 })),
133 args: CombiningKind::ADD, args&: fastMathFlags);
134 return success();
135 }
136
137private:
138 bool allowFastMathFlagLoss;
139};
140
141/// Merge LHS/RHS (A/B) TransposeOp into ContractionOp user.
142/// Ex:
143/// ```
144/// %0 = vector.transpose %arg0, [2, 0, 1]
145/// : vector<32x16x8xf32> to vector<8x32x16xf32>
146/// %1 = vector.contract {indexing_maps = [
147/// affine_map<(d0, d1, d2) -> (d0, d1, d2)>,
148/// affine_map<(d0, d1, d2) -> (d0, d1, d2)>,
149/// affine_map<(d0, d1, d2) -> (d0, d1)>],
150/// iterator_types = ["parallel", "parallel", "reduction"],
151/// kind = add} %0, %arg1, %cst_f0
152/// : vector<8x32x16xf32>, vector<8x32x16xf32> into vector<8x32xf32>
153/// ```
154/// Gets converted to:
155/// ```
156/// %1 = vector.contract {indexing_maps = [
157/// affine_map<(d0, d1, d2) -> (d1, d2, d0)>,
158/// affine_map<(d0, d1, d2) -> (d0, d1, d2)>,
159/// affine_map<(d0, d1, d2) -> (d0, d1)>],
160/// iterator_types = ["parallel", "parallel", "reduction"],
161/// kind = add} %arg0, %arg1, %cst_f0
162/// : vector<8x32x16xf32>, vector<8x32x16xf32> into vector<8x32xf32>
163/// ```
164struct CombineContractABTranspose final
165 : public OpRewritePattern<vector::ContractionOp> {
166 using Base::Base;
167
168 LogicalResult matchAndRewrite(vector::ContractionOp contractOp,
169 PatternRewriter &rewriter) const override {
170 SmallVector<AffineMap> maps =
171 llvm::to_vector<4>(Range: contractOp.getIndexingMapsArray());
172 Value lhs = contractOp.getLhs();
173 Value rhs = contractOp.getRhs();
174 size_t index = 0;
175 bool changed = false;
176 for (Value *operand : {&lhs, &rhs}) {
177 AffineMap &map = maps[index++];
178 auto transposeOp = operand->getDefiningOp<vector::TransposeOp>();
179 if (!transposeOp)
180 continue;
181 AffineMap permutationMap = AffineMap::getPermutationMap(
182 permutation: transposeOp.getPermutation(), context: contractOp.getContext());
183 map = inversePermutation(map: permutationMap).compose(map);
184 *operand = transposeOp.getVector();
185 changed = true;
186 }
187 if (!changed)
188 return failure();
189 rewriter.replaceOpWithNewOp<vector::ContractionOp>(
190 op: contractOp, args&: lhs, args&: rhs, args: contractOp.getAcc(),
191 args: rewriter.getAffineMapArrayAttr(values: maps), args: contractOp.getIteratorTypes(),
192 args: contractOp.getKind(), args: contractOp.getFastmath());
193 return success();
194 }
195};
196
197/// Merges accumulator and result transposes into contract.
198///
199/// For example:
200/// ```mlir
201/// %accT = vector.transpose %acc, [0, 2, 1]
202/// : vector<2x8x4xf32> to vector<2x4x8xf32>
203/// %contract = vector.contract {
204/// indexing_maps = [
205/// affine_map<(d0, d1, d2, d3) -> (d0, d3, d1)>,
206/// affine_map<(d0, d1, d2, d3) -> (d3, d2)>,
207/// affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
208/// ],
209/// iterator_types = ["parallel", "parallel", "parallel", "reduction"],
210/// kind = #vector.kind<add>
211/// } %lhs, %rhs, %accT
212/// : vector<2x4x4xf32>, vector<4x8xf32> into vector<2x4x8xf32>
213/// %0 = vector.transpose %contract, [0, 2, 1]
214/// : vector<2x4x8xf32> to vector<2x8x4>
215/// ```
216/// Becomes:
217/// ```mlir
218/// %0 = vector.contract {
219/// indexing_maps = [
220/// affine_map<(d0, d1, d2, d3) -> (d0, d3, d1)>,
221/// affine_map<(d0, d1, d2, d3) -> (d3, d2)>,
222/// affine_map<(d0, d1, d2, d3) -> (d0, d2, d1)>
223/// ],
224/// iterator_types = ["parallel", "parallel", "parallel", "reduction"],
225/// kind = #vector.kind<add>
226/// } %lhs, %rhs, %acc
227/// : vector<2x4x4xf32>, vector<4x8xf32> into vector<2x8x4xf32>
228/// ```
229struct CombineContractResultTranspose final
230 : public OpRewritePattern<vector::TransposeOp> {
231 using Base::Base;
232
233 LogicalResult matchAndRewrite(vector::TransposeOp resTOp,
234 PatternRewriter &rewriter) const override {
235 auto contractOp = resTOp.getVector().getDefiningOp<vector::ContractionOp>();
236 if (!contractOp || !contractOp->hasOneUse())
237 return failure();
238
239 auto accTOp = contractOp.getAcc().getDefiningOp<vector::TransposeOp>();
240 if (!accTOp)
241 return failure();
242
243 MLIRContext *context = contractOp.getContext();
244 auto maps = llvm::to_vector<3>(Range: contractOp.getIndexingMapsArray());
245 AffineMap contractMap = maps.back();
246
247 // Accumulator transpose performs f(A) -> B. Contract performs g(C) -> B.
248 // To index into A in contract, we need revert(f)(g(C)) -> A.
249 auto accTMap =
250 AffineMap::getPermutationMap(permutation: accTOp.getPermutation(), context);
251
252 // Contract performs g(C) -> D. Result transpose performs h(D) -> E.
253 // To index into E in contract, we need h(g(C)) -> E.
254 auto resTMap =
255 AffineMap::getPermutationMap(permutation: resTOp.getPermutation(), context);
256 auto combinedResMap = resTMap.compose(map: contractMap);
257
258 // The accumulator and result share the same indexing map. So they should be
259 // the same to be able to merge. This means combinedResMap is the same as
260 // inversePermutation(accTMap).compose(contractMap), which means
261 if (inversePermutation(map: accTMap) != resTMap)
262 return failure();
263 maps.back() = combinedResMap;
264
265 rewriter.replaceOpWithNewOp<vector::ContractionOp>(
266 op: resTOp, args: contractOp.getLhs(), args: contractOp.getRhs(), args: accTOp.getVector(),
267 args: rewriter.getAffineMapArrayAttr(values: maps), args: contractOp.getIteratorTypes(),
268 args: contractOp.getKind(), args: contractOp.getFastmath());
269 return success();
270 }
271};
272
273/// Merge BroadcastOp (and broadcast-like ShapeCastOp) into ContractionOp user.
274/// Ex:
275/// ```
276/// %0 = vector.broadcast %arg0 : vector<32x16xf32> to vector<8x32x16xf32>
277/// %1 = vector.contract {indexing_maps = [
278/// affine_map<(d0, d1, d2) -> (d0, d1, d2)>,
279/// affine_map<(d0, d1, d2) -> (d0, d1, d2)>,
280/// affine_map<(d0, d1, d2) -> (d0, d1)>],
281/// iterator_types = ["parallel", "parallel", "reduction"],
282/// kind = add} %0, %arg1, %cst_f0
283/// : vector<8x32x16xf32>, vector<8x32x16xf32> into vector<8x32xf32>
284/// ```
285/// Gets converted to:
286/// ```
287/// %1 = vector.contract {indexing_maps = [
288/// affine_map<(d0, d1, d2) -> (d1, d2)>,
289/// affine_map<(d0, d1, d2) -> (d0, d1, d2)>,
290/// affine_map<(d0, d1, d2) -> (d0, d1)>],
291/// iterator_types = ["parallel", "parallel", "reduction"],
292/// kind = add} %arg0, %arg1, %cst_f0
293/// : vector<32x16xf32>, vector<8x32x16xf32> into vector<8x32xf32>
294/// ```
295///
296/// For masked vector.contract, the mask requires updating when a dimension is
297/// dropped. In such cases, the dropped dimensions must correspond to the mask's
298/// leading unit dimensions. Supporting more generic cases (e.g. non-unit dims)
299/// is not supported.
300FailureOr<Value> combineContractAndBroadcast(vector::ContractionOp contractOp,
301 MaskingOpInterface maskingOp,
302 PatternRewriter &rewriter) {
303 SmallVector<AffineMap> maps =
304 llvm::to_vector<4>(Range: contractOp.getIndexingMapsArray());
305 Value lhs = contractOp.getLhs();
306 Value rhs = contractOp.getRhs();
307 size_t index = 0;
308 bool changed = false;
309 for (Value *operand : {&lhs, &rhs}) {
310 AffineMap &map = maps[index++];
311
312 // Accept operands defined by vector.broadcast and broadcast-like
313 // vector.shape_cast.
314 auto sc = operand->getDefiningOp<vector::ShapeCastOp>();
315 auto broadcast = operand->getDefiningOp<vector::BroadcastOp>();
316 if (!broadcast && !sc)
317 continue;
318
319 if (sc && !sc.isBroadcastLike())
320 return rewriter.notifyMatchFailure(
321 arg&: contractOp, msg: "Operand defined via vector.shape_cast that has "
322 "non-broadcast semantics");
323
324 // Get the source and the result types.
325 VectorType srcType = sc ? sc.getSourceVectorType()
326 : dyn_cast<VectorType>(Val: broadcast.getSourceType());
327 VectorType resType =
328 sc ? sc.getResultVectorType() : broadcast.getResultVectorType();
329
330 // contractionOp can only take vector as operands.
331 // auto srcType = dyn_cast<VectorType>(broadcast.getSourceVectorType());
332 if (!srcType || srcType.getRank() >= resType.getRank())
333 continue;
334 int64_t rankDiff = resType.getRank() - srcType.getRank();
335 bool innerDimBroadcast = false;
336 SmallVector<AffineExpr> originalDims;
337 for (const auto &dim : llvm::enumerate(First: srcType.getShape())) {
338 if (dim.value() != resType.getDimSize(idx: rankDiff + dim.index())) {
339 innerDimBroadcast = true;
340 break;
341 }
342 originalDims.push_back(Elt: rewriter.getAffineDimExpr(position: dim.index() + rankDiff));
343 }
344 // Contract doesn't support inner dimension broadcast. Once this is
345 // relaxed we can remove this case.
346 if (innerDimBroadcast)
347 continue;
348
349 // It would be incorrect to fold a broadcast onto a reduction dimension
350 // of non-unit size.
351 bool nonUnitDimReductionBroadcast = false;
352 for (int64_t i = 0; i < rankDiff; ++i) {
353 if (resType.getDimSize(idx: i) != 1 &&
354 isReductionIterator(attr: contractOp.getIteratorTypes()
355 .getValue()[map.getDimPosition(idx: i)])) {
356 nonUnitDimReductionBroadcast = true;
357 break;
358 }
359 }
360 if (nonUnitDimReductionBroadcast)
361 continue;
362
363 AffineMap broadcastMap = AffineMap::get(dimCount: resType.getRank(), symbolCount: 0, results: originalDims,
364 context: contractOp.getContext());
365 map = broadcastMap.compose(map);
366 *operand = broadcast ? broadcast.getSource() : sc.getSource();
367 changed = true;
368 }
369
370 if (!changed)
371 return failure();
372
373 // Determine which dims are usused, now that the maps have been composed
374 // with the broadcast maps.
375 llvm::SmallBitVector unusedDimsBitVector = getUnusedDimsBitVector(maps);
376 // Compress unused dims.
377 for (auto &m : maps)
378 m = compressDims(map: m, unusedDims: unusedDimsBitVector);
379 // Compute the combined iterators.
380 SmallVector<Attribute> iterators;
381 for (unsigned i = 0, e = unusedDimsBitVector.size(); i < e; ++i) {
382 if (!unusedDimsBitVector.test(Idx: i))
383 iterators.push_back(Elt: contractOp.getIteratorTypes().getValue()[i]);
384 }
385
386 // Check whether any of the unused dims is non-unit, e.g.:
387 // * vector.broadcast %arg0 : vector<8x4xi32> to vector<2x8x4xi32>
388 // This is only required when collapsing a mask. If there is no mask, skip.
389 VectorType oldMaskType;
390 bool isAnyUnusedDimNonUnit = false;
391 if (maskingOp) {
392 oldMaskType = cast<VectorType>(Val: maskingOp.getMask().getType());
393 for (unsigned i = 0, e = unusedDimsBitVector.size(); i < e; ++i) {
394 if (unusedDimsBitVector.test(Idx: i) && oldMaskType.getShape()[i] != 1) {
395 isAnyUnusedDimNonUnit = true;
396 break;
397 }
398 }
399 }
400
401 // Bail if no reduction iterator survives on both LHS and RHS: folding a
402 // one-sided unit-dim broadcast would leave an unpaired reduction iterator
403 // and produce an invalid contract.
404 bool hasReductionIteratorApplyingOnBothSides = false;
405 for (unsigned i = 0; i < iterators.size(); ++i) {
406 if (!isReductionIterator(attr: iterators[i]))
407 continue;
408 if (!getResultIndex(map: maps[0], index: i) || !getResultIndex(map: maps[1], index: i))
409 return failure();
410 hasReductionIteratorApplyingOnBothSides = true;
411 }
412 if (!hasReductionIteratorApplyingOnBothSides)
413 return failure();
414
415 // If the compressed maps have a dimension that is not used by either LHS or
416 // RHS then the ContractionOp verifier would fail.
417 if (getUnusedDimsBitVector(maps: {maps[0], maps[1]}).any())
418 return failure();
419
420 Operation *newOp = vector::ContractionOp::create(
421 builder&: rewriter, location: contractOp.getLoc(), lhs, rhs, acc: contractOp.getAcc(),
422 indexingMaps: rewriter.getAffineMapArrayAttr(values: maps), iteratorTypes: rewriter.getArrayAttr(value: iterators),
423 kind: contractOp.getKind(), fastMathFlags: contractOp.getFastmath());
424
425 // Handle the mask.
426 if (maskingOp) {
427 if (isAnyUnusedDimNonUnit)
428 return rewriter.notifyMatchFailure(arg&: contractOp,
429 msg: "Cannont drop non-unit mask dim.");
430 assert(unusedDimsBitVector.size() ==
431 static_cast<size_t>(oldMaskType.getRank()) &&
432 "The mask rank is incorrect!");
433
434 // If a dimension has been dropped, update the mask accordingly. Otherwise,
435 // keep it as is.
436 Value mask = maskingOp.getMask();
437 if (unusedDimsBitVector.count() != 0) {
438 // At this point, two assumptions are made:
439 // * The unused dimensions are the leading mask dimensions
440 // (vector.contract does not support inner dim broadcasting).
441 // * The unused dimensions are all unit.
442 // These conditions are effectively verified in the blocks preceeding this
443 // one.
444 auto newShape =
445 oldMaskType.getShape().drop_front(N: unusedDimsBitVector.count());
446 auto newShapeScalableDims =
447 oldMaskType.getScalableDims().drop_front(N: unusedDimsBitVector.count());
448 VectorType maskOpType =
449 VectorType::get(shape: newShape, elementType: rewriter.getI1Type(), scalableDims: newShapeScalableDims);
450 mask = vector::ShapeCastOp::create(builder&: rewriter, location: contractOp.getLoc(),
451 result: maskOpType, source: maskingOp.getMask())
452 .getResult();
453 }
454
455 newOp = mlir::vector::maskOperation(builder&: rewriter, maskableOp: newOp, mask);
456 }
457 return newOp->getResult(idx: 0);
458}
459
460struct CombineContractBroadcastMask
461 : public MaskableOpRewritePattern<vector::ContractionOp> {
462 using MaskableOpRewritePattern::MaskableOpRewritePattern;
463 FailureOr<Value>
464
465 matchAndRewriteMaskableOp(vector::ContractionOp contractOp,
466 MaskingOpInterface maskingOp,
467 PatternRewriter &rewriter) const override {
468 return combineContractAndBroadcast(contractOp, maskingOp, rewriter);
469 }
470};
471
472/// Reorders cast(broadcast) to broadcast(cast). This makes broadcast ops and
473/// contraction ops closer, which kicks in CombineContractBroadcast pattern when
474/// casting ops are around these operations.
475/// Ex:
476/// ```
477/// %0 = vector.broadcast %arg0 : vector<32x16xi8> to vector<8x32x16xi8>
478/// %1 = arith.extsi %0 : vector<8x32x16xi8> to vector<8x32x16xi32>
479/// ```
480/// Gets converted to:
481/// ```
482/// %0 = arith.extsi %0 : vector<32x16xi8> to vector<32x16xi32>
483/// %1 = vector.broadcast %arg0 : vector<32x16xi32> to vector<8x32x16xi32>
484/// ```
485struct ReorderCastOpsOnBroadcast
486 : public OpInterfaceRewritePattern<CastOpInterface> {
487 using OpInterfaceRewritePattern<CastOpInterface>::OpInterfaceRewritePattern;
488
489 LogicalResult matchAndRewrite(CastOpInterface op,
490 PatternRewriter &rewriter) const override {
491 if (op->getNumOperands() != 1)
492 return failure();
493 if (!isa<VectorType>(Val: op->getResult(idx: 0).getType()))
494 return failure();
495 auto bcastOp = op->getOperand(idx: 0).getDefiningOp<vector::BroadcastOp>();
496 if (!bcastOp)
497 return failure();
498
499 Type castResTy = getElementTypeOrSelf(val: op->getResult(idx: 0));
500 if (auto vecTy = dyn_cast<VectorType>(Val: bcastOp.getSourceType()))
501 castResTy = vecTy.clone(elementType: castResTy);
502 auto *castOp =
503 createWithProperties(builder&: rewriter, op, operands: bcastOp.getSource(), types: castResTy);
504 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
505 op, args: op->getResult(idx: 0).getType(), args: castOp->getResult(idx: 0));
506 return success();
507 }
508};
509
510/// Reorders elementwise(transpose) to transpose(elementwise). This makes
511/// transpose ops and contraction ops closer, which kicks in
512/// CombineContractABTranspose pattern when elementwise ops are between these
513/// operations. Ex:
514/// ```
515/// %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
516/// %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
517/// %r = arith.addf %at, %bt : vector<2x4xf32>
518/// ```
519/// Gets converted to:
520/// ```
521/// %0 = arith.addf %a, %b : vector<4x2xf32>
522/// %r = vector.transpose %0, [1, 0] : vector<2x4xf32>
523/// ```
524struct ReorderElementwiseOpsOnTranspose final
525 : public OpTraitRewritePattern<OpTrait::Elementwise> {
526 using Base::Base;
527 LogicalResult matchAndRewrite(Operation *op,
528 PatternRewriter &rewriter) const override {
529 if (op->getNumResults() != 1 || op->getNumRegions() != 0)
530 return failure();
531
532 // Make sure all operands are transpose/constant ops and collect their
533 // transposition maps.
534 SmallVector<ArrayRef<int64_t>> transposeMaps;
535 transposeMaps.reserve(N: op->getNumOperands());
536 // Record the initial type before transposition. We'll use its shape later.
537 // Any type will do here as we will check all transpose maps are the same.
538 VectorType srcType;
539 for (Value operand : op->getOperands()) {
540 auto transposeOp = operand.getDefiningOp<vector::TransposeOp>();
541 if (transposeOp) {
542 transposeMaps.push_back(Elt: transposeOp.getPermutation());
543 srcType = transposeOp.getSourceVectorType();
544 } else if (!matchPattern(value: operand, pattern: m_Constant())) {
545 return failure();
546 }
547 }
548 if (transposeMaps.empty())
549 return failure();
550 // This is an elementwise op, so all transposed operands should have the
551 // same type. We need to additionally check that all transposes uses the
552 // same map.
553 if (!llvm::all_equal(Range&: transposeMaps))
554 return rewriter.notifyMatchFailure(arg&: op, msg: "different transpose map");
555
556 SmallVector<Value> srcValues;
557 srcValues.reserve(N: op->getNumOperands());
558
559 // If there are constant operands, we need to insert inverse transposes for
560 // them. Calculate the inverse order first.
561 auto order = transposeMaps.front();
562 SmallVector<int64_t> invOrder(order.size());
563 for (int i = 0, e = order.size(); i < e; ++i)
564 invOrder[order[i]] = i;
565
566 for (Value operand : op->getOperands()) {
567 auto transposeOp = operand.getDefiningOp<vector::TransposeOp>();
568 if (transposeOp) {
569 srcValues.push_back(Elt: transposeOp.getVector());
570 } else {
571 // This is a constant. Create a reverse transpose op for it.
572 auto vectorType =
573 srcType.clone(elementType: cast<VectorType>(Val: operand.getType()).getElementType());
574 srcValues.push_back(Elt: vector::TransposeOp::create(
575 builder&: rewriter, location: operand.getLoc(), result: vectorType, vector: operand, permutation: invOrder));
576 }
577 }
578
579 auto vectorType = srcType.clone(
580 elementType: cast<VectorType>(Val: op->getResultTypes()[0]).getElementType());
581 Operation *elementwiseOp =
582 createWithProperties(builder&: rewriter, op, operands: srcValues, types: vectorType);
583 rewriter.replaceOpWithNewOp<vector::TransposeOp>(
584 op, args: op->getResultTypes()[0], args: elementwiseOp->getResult(idx: 0),
585 args&: transposeMaps.front());
586 return success();
587 }
588};
589
590// Returns the values in `arrayAttr` as an integer vector.
591static SmallVector<int64_t> getIntValueVector(ArrayAttr arrayAttr) {
592 return llvm::map_to_vector<4>(C: arrayAttr.getAsRange<IntegerAttr>(),
593 F: [](IntegerAttr attr) { return attr.getInt(); });
594}
595
596// Shuffles vector.bitcast op after vector.extract op.
597//
598// This transforms IR like:
599// %0 = vector.bitcast %src : vector<4xf32> to vector<8xf16>
600// %1 = vector.extract %0[3] : f16 from vector<8xf16>
601// Into:
602// %0 = vector.extract %src[1] : f32 from vector<4xf32>
603// %1 = vector.bitcast %0: vector<1xf32> to vector<2xf16>
604// %2 = vector.extract %1[1] : f16 from vector<2xf16>
605struct BubbleDownVectorBitCastForExtract
606 : public OpRewritePattern<vector::ExtractOp> {
607 using Base::Base;
608
609 LogicalResult matchAndRewrite(vector::ExtractOp extractOp,
610 PatternRewriter &rewriter) const override {
611 // Only support extracting scalars for now.
612 if (extractOp.getSourceVectorType().getRank() != 1)
613 return failure();
614
615 auto castOp = extractOp.getSource().getDefiningOp<vector::BitCastOp>();
616 if (!castOp)
617 return failure();
618
619 VectorType castSrcType = castOp.getSourceVectorType();
620 VectorType castDstType = castOp.getResultVectorType();
621 assert(castSrcType.getRank() == castDstType.getRank());
622
623 // Fail to match if we only have one element in the cast op source.
624 // This is to avoid infinite loop given that this pattern can generate
625 // such cases.
626 if (castSrcType.getNumElements() == 1)
627 return failure();
628
629 // Only support casting to a larger number of elements or now.
630 // E.g., vector<4xf32> -> vector<8xf16>.
631 if (castSrcType.getNumElements() > castDstType.getNumElements())
632 return failure();
633
634 unsigned expandRatio =
635 castDstType.getNumElements() / castSrcType.getNumElements();
636
637 // Get the first element of the mixed position as integer.
638 auto mixedPos = extractOp.getMixedPosition();
639 if (!mixedPos.empty() && !isa<Attribute>(Val: mixedPos[0]))
640 return failure();
641 uint64_t index = cast<IntegerAttr>(Val: cast<Attribute>(Val&: mixedPos[0])).getInt();
642
643 // Get the single scalar (as a vector) in the source value that packs the
644 // desired scalar. E.g. extract vector<1xf32> from vector<4xf32>
645 Location loc = extractOp.getLoc();
646 Value packedValue = vector::ExtractOp::create(
647 builder&: rewriter, location: loc, source: castOp.getSource(), position: index / expandRatio);
648 Type packedVecType = VectorType::get(/*shape=*/{1}, elementType: packedValue.getType());
649 Value zero = arith::ConstantOp::create(builder&: rewriter, location: loc, result: packedVecType,
650 value: rewriter.getZeroAttr(type: packedVecType));
651 packedValue = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: packedValue, dest: zero,
652 /*position=*/0);
653
654 // Cast it to a vector with the desired scalar's type.
655 // E.g. f32 -> vector<2xf16>
656 VectorType packedType =
657 VectorType::get(shape: {expandRatio}, elementType: castDstType.getElementType());
658 Value castedValue =
659 vector::BitCastOp::create(builder&: rewriter, location: loc, result: packedType, source: packedValue);
660
661 // Finally extract the desired scalar.
662 rewriter.replaceOpWithNewOp<vector::ExtractOp>(op: extractOp, args&: castedValue,
663 args: index % expandRatio);
664 return success();
665 }
666};
667
668// Shuffles vector.bitcast op after vector.extract_strided_slice op.
669//
670// This transforms IR like:
671// %cast = vector.bitcast %arg0: vector<4xf32> to vector<8xf16>
672// %0 = vector.extract_strided_slice %cast {
673// offsets = [4], sizes = [4], strides = [1]
674// } : vector<8xf16> to vector<4xf16>
675// Into:
676// %0 = vector.extract_strided_slice %src {
677// offsets = [2], sizes = [2], strides = [1]
678// } : vector<4xf32> to vector<2xf32>
679// %1 = vector.bitcast %0 : vector<2xf32> to vector<4xf16>
680struct BubbleDownBitCastForStridedSliceExtract
681 : public OpRewritePattern<vector::ExtractStridedSliceOp> {
682 using Base::Base;
683
684 LogicalResult matchAndRewrite(vector::ExtractStridedSliceOp extractOp,
685 PatternRewriter &rewriter) const override {
686 auto castOp = extractOp.getSource().getDefiningOp<vector::BitCastOp>();
687 if (!castOp)
688 return failure();
689
690 VectorType castSrcType = castOp.getSourceVectorType();
691 VectorType castDstType = castOp.getResultVectorType();
692 assert(castSrcType.getRank() == castDstType.getRank());
693
694 int64_t castSrcLastDim = castSrcType.getShape().back();
695 int64_t castDstLastDim = castDstType.getShape().back();
696 // Require casting to more elements for now; other cases to be implemented.
697 if (castSrcLastDim > castDstLastDim)
698 return failure();
699
700 // Only accept all one strides for now.
701 if (llvm::any_of(Range: extractOp.getStrides().getAsValueRange<IntegerAttr>(),
702 P: [](const APInt &val) { return !val.isOne(); }))
703 return failure();
704
705 unsigned rank = extractOp.getSourceVectorType().getRank();
706 assert(castDstLastDim % castSrcLastDim == 0);
707 int64_t expandRatio = castDstLastDim / castSrcLastDim;
708
709 // If we have a less number of offsets than the rank, then implicitly we
710 // are selecting the full range for the last bitcasted dimension; other
711 // dimensions aren't affected. Otherwise, we need to scale down the last
712 // dimension's offset given we are extracting from less elements now.
713 ArrayAttr newOffsets = extractOp.getOffsets();
714 if (newOffsets.size() == rank) {
715 SmallVector<int64_t> offsets = getIntValueVector(arrayAttr: newOffsets);
716 if (offsets.back() % expandRatio != 0)
717 return failure();
718 offsets.back() = offsets.back() / expandRatio;
719 newOffsets = rewriter.getI64ArrayAttr(values: offsets);
720 }
721
722 // Similarly for sizes.
723 ArrayAttr newSizes = extractOp.getSizes();
724 if (newSizes.size() == rank) {
725 SmallVector<int64_t> sizes = getIntValueVector(arrayAttr: newSizes);
726 if (sizes.back() % expandRatio != 0)
727 return failure();
728 sizes.back() = sizes.back() / expandRatio;
729 newSizes = rewriter.getI64ArrayAttr(values: sizes);
730 }
731
732 SmallVector<int64_t> dims =
733 llvm::to_vector<4>(Range: cast<VectorType>(Val: extractOp.getType()).getShape());
734 dims.back() = dims.back() / expandRatio;
735 VectorType newExtractType =
736 VectorType::get(shape: dims, elementType: castSrcType.getElementType());
737
738 auto newExtractOp = vector::ExtractStridedSliceOp::create(
739 builder&: rewriter, location: extractOp.getLoc(), resultType0: newExtractType, source: castOp.getSource(),
740 offsets: newOffsets, sizes: newSizes, strides: extractOp.getStrides());
741
742 rewriter.replaceOpWithNewOp<vector::BitCastOp>(
743 op: extractOp, args: extractOp.getType(), args&: newExtractOp);
744
745 return success();
746 }
747};
748
749// Shuffles vector.bitcast op before vector.insert_strided_slice op.
750//
751// This transforms IR like:
752// %0 = vector.insert %val, %dst[4] : vector<32xi4> into vector<8x32xi4>
753// %1 = vector.bitcast %0 : vector<8x32xi4> to vector<8x16xi8>
754// Into:
755// %0 = vector.bitcast %val : vector<32xi4> to vector<16xi8>
756// %1 = vector.bitcast %dst : vector<8x32xi4> to vector<8x16xi8>
757// %2 = vector.insert %0, %1 [4] : vector<16xi8> into vector<8x16xi8>
758//
759struct BubbleUpBitCastForInsert : public OpRewritePattern<vector::BitCastOp> {
760 using Base::Base;
761
762 LogicalResult matchAndRewrite(vector::BitCastOp bitcastOp,
763 PatternRewriter &rewriter) const override {
764 VectorType castSrcType = bitcastOp.getSourceVectorType();
765 VectorType castDstType = bitcastOp.getResultVectorType();
766
767 // 0-D and scalable vectors are not supported yet.
768 if (castSrcType.getRank() == 0 || castSrcType.isScalable() ||
769 castDstType.isScalable())
770 return failure();
771
772 int64_t castSrcLastDim = castSrcType.getShape().back();
773 int64_t castDstLastDim = castDstType.getShape().back();
774 bool isNumElemsShrink = castSrcLastDim >= castDstLastDim;
775 int64_t ratio;
776 if (isNumElemsShrink) {
777 assert(castSrcLastDim % castDstLastDim == 0);
778 ratio = castSrcLastDim / castDstLastDim;
779 } else {
780 assert(castDstLastDim % castSrcLastDim == 0);
781 ratio = castDstLastDim / castSrcLastDim;
782 }
783
784 auto insertOp = bitcastOp.getSource().getDefiningOp<vector::InsertOp>();
785 if (!insertOp)
786 return failure();
787
788 // Only vector sources are supported for now.
789 auto insertSrcType = dyn_cast<VectorType>(Val: insertOp.getValueToStoreType());
790 if (!insertSrcType)
791 return failure();
792
793 // Bitcast the source.
794 SmallVector<int64_t> srcDims(insertSrcType.getShape());
795 srcDims.back() =
796 isNumElemsShrink ? srcDims.back() / ratio : srcDims.back() * ratio;
797 VectorType newCastSrcType =
798 VectorType::get(shape: srcDims, elementType: castDstType.getElementType());
799 auto newCastSrcOp =
800 vector::BitCastOp::create(builder&: rewriter, location: bitcastOp.getLoc(), result: newCastSrcType,
801 source: insertOp.getValueToStore());
802
803 SmallVector<int64_t> dstDims(insertOp.getDestVectorType().getShape());
804 dstDims.back() =
805 isNumElemsShrink ? dstDims.back() / ratio : dstDims.back() * ratio;
806 VectorType newCastDstType =
807 VectorType::get(shape: dstDims, elementType: castDstType.getElementType());
808
809 // Bitcast the destination.
810 auto newCastDstOp = vector::BitCastOp::create(
811 builder&: rewriter, location: bitcastOp.getLoc(), result: newCastDstType, source: insertOp.getDest());
812
813 // Generate new insert.
814 rewriter.replaceOpWithNewOp<vector::InsertOp>(
815 op: bitcastOp, args&: newCastSrcOp, args&: newCastDstOp, args: insertOp.getMixedPosition());
816 return success();
817 }
818};
819
820// Shuffles vector.bitcast op before vector.insert_strided_slice op.
821//
822// This transforms IR like:
823// %0 = vector.insert_strided_slice %src, %dst {
824// offsets = [0], strides = [1]} : vector<4xf16> into vector<8xf16>
825// %1 = vector.bitcast %0: vector<8xf16> to vector<4xf32>
826// Into:
827// %0 = vector.bitcast %src : vector<4xf16> to vector<2xf32>
828// %1 = vector.bitcast %dst : vector<8xf16> to vector<4xf32>
829// %2 = vector.insert_strided_slice %src, %dst {
830// offsets = [0], strides = [1]} : vector<2xf32> into vector<4xf32>
831struct BubbleUpBitCastForStridedSliceInsert
832 : public OpRewritePattern<vector::BitCastOp> {
833 using Base::Base;
834
835 LogicalResult matchAndRewrite(vector::BitCastOp bitcastOp,
836 PatternRewriter &rewriter) const override {
837 VectorType castSrcType = bitcastOp.getSourceVectorType();
838 VectorType castDstType = bitcastOp.getResultVectorType();
839 assert(castSrcType.getRank() == castDstType.getRank());
840 // Skip 0-D vector which will not from InsertStridedSliceOp.
841 if (castSrcType.getRank() == 0)
842 return failure();
843
844 int64_t castSrcLastDim = castSrcType.getShape().back();
845 int64_t castDstLastDim = castDstType.getShape().back();
846 // Require casting to less elements for now; other cases to be implemented.
847 if (castSrcLastDim < castDstLastDim)
848 return failure();
849
850 assert(castSrcLastDim % castDstLastDim == 0);
851 int64_t shrinkRatio = castSrcLastDim / castDstLastDim;
852
853 auto insertOp =
854 bitcastOp.getSource().getDefiningOp<vector::InsertStridedSliceOp>();
855 if (!insertOp)
856 return failure();
857
858 // Only accept all one strides for now.
859 if (llvm::any_of(Range: insertOp.getStrides().getAsValueRange<IntegerAttr>(),
860 P: [](const APInt &val) { return !val.isOne(); }))
861 return failure();
862
863 unsigned rank = insertOp.getSourceVectorType().getRank();
864 // Require insert op to have the same rank for the source and destination
865 // vector; other cases to be implemented.
866 if (rank != insertOp.getDestVectorType().getRank())
867 return failure();
868
869 // Requires that shape of insert op src is castable to dstType.
870 unsigned sourceWidth = castSrcType.getElementType().getIntOrFloatBitWidth();
871 unsigned destinationWidth =
872 castDstType.getElementType().getIntOrFloatBitWidth();
873 unsigned numElements = destinationWidth / sourceWidth;
874 if (insertOp.getSourceVectorType().getNumElements() % numElements != 0)
875 return failure();
876
877 ArrayAttr newOffsets = insertOp.getOffsets();
878 assert(newOffsets.size() == rank);
879 SmallVector<int64_t> offsets = getIntValueVector(arrayAttr: newOffsets);
880 if (offsets.back() % shrinkRatio != 0)
881 return failure();
882 offsets.back() = offsets.back() / shrinkRatio;
883 newOffsets = rewriter.getI64ArrayAttr(values: offsets);
884
885 SmallVector<int64_t> srcDims =
886 llvm::to_vector<4>(Range: insertOp.getSourceVectorType().getShape());
887 srcDims.back() = srcDims.back() / shrinkRatio;
888 VectorType newCastSrcType =
889 VectorType::get(shape: srcDims, elementType: castDstType.getElementType());
890
891 auto newCastSrcOp =
892 vector::BitCastOp::create(builder&: rewriter, location: bitcastOp.getLoc(), result: newCastSrcType,
893 source: insertOp.getValueToStore());
894
895 SmallVector<int64_t> dstDims =
896 llvm::to_vector<4>(Range: insertOp.getDestVectorType().getShape());
897 dstDims.back() = dstDims.back() / shrinkRatio;
898 VectorType newCastDstType =
899 VectorType::get(shape: dstDims, elementType: castDstType.getElementType());
900
901 auto newCastDstOp = vector::BitCastOp::create(
902 builder&: rewriter, location: bitcastOp.getLoc(), result: newCastDstType, source: insertOp.getDest());
903
904 rewriter.replaceOpWithNewOp<vector::InsertStridedSliceOp>(
905 op: bitcastOp, args: bitcastOp.getType(), args&: newCastSrcOp, args&: newCastDstOp, args&: newOffsets,
906 args: insertOp.getStrides());
907
908 return success();
909 }
910};
911
912// Breaks down vector.bitcast op
913//
914// This transforms IR like:
915// %1 = vector.bitcast %0: vector<8xf16> to vector<4xf32>
916// Into:
917// %cst = vector.broadcast %c0_f32 : f32 to vector<4xf32>
918// %1 = vector.extract_strided_slice %0 {
919// offsets = [0], sizes = [4], strides = [1]
920// } : vector<8xf16> to vector<4xf16>
921// %2 = vector.bitcast %1 : vector<4xf16> to vector<2xf32>
922// %4 = vector.insert_strided_slice %2, %cst {
923// offsets = [0], strides = [1]} : vector<2xf32> into vector<4xf32>
924// %5 = vector.extract_strided_slice %0 {
925// offsets = [4], sizes = [4], strides = [1]
926// } : vector<8xf16> to vector<4xf16>
927// %6 = vector.bitcast %5 : vector<4xf16> to vector<2xf32>
928// %7 = vector.insert_strided_slice %6, %cst {
929// offsets = [2], strides = [1]} : vector<2xf32> into vector<4xf32>
930struct BreakDownVectorBitCast : public OpRewritePattern<vector::BitCastOp> {
931 using Base::Base;
932
933public:
934 BreakDownVectorBitCast(MLIRContext *context,
935 std::function<bool(vector::BitCastOp)> controlFn,
936 PatternBenefit benefit)
937 : OpRewritePattern(context, benefit), controlFn(std::move(controlFn)) {}
938
939 LogicalResult matchAndRewrite(vector::BitCastOp bitcastOp,
940 PatternRewriter &rewriter) const override {
941
942 if (controlFn && !controlFn(bitcastOp))
943 return failure();
944
945 VectorType castSrcType = bitcastOp.getSourceVectorType();
946 VectorType castDstType = bitcastOp.getResultVectorType();
947 assert(castSrcType.getRank() == castDstType.getRank());
948
949 // This transformation builds on top of
950 // vector.{extract|insert}_strided_slice, which do not support
951 // extracting/inserting "scallable sub-vectors". Bail out.
952 if (castSrcType.isScalable())
953 return rewriter.notifyMatchFailure(arg&: bitcastOp,
954 msg: "Scalable vectors are not supported");
955
956 // Only support rank 1 case for now.
957 if (castSrcType.getRank() != 1)
958 return failure();
959
960 int64_t castSrcLastDim = castSrcType.getShape().back();
961 int64_t castDstLastDim = castDstType.getShape().back();
962 // Require casting to less elements for now; other cases to be implemented.
963 if (castSrcLastDim < castDstLastDim)
964 return failure();
965
966 assert(castSrcLastDim % castDstLastDim == 0);
967 int64_t shrinkRatio = castSrcLastDim / castDstLastDim;
968 // Nothing to do if it is already bitcasting to a single element.
969 if (castSrcLastDim == shrinkRatio)
970 return failure();
971
972 Location loc = bitcastOp.getLoc();
973 Type elemType = castDstType.getElementType();
974 assert(elemType.isSignlessIntOrIndexOrFloat());
975
976 Value zero = arith::ConstantOp::create(builder&: rewriter, location: loc, result: elemType,
977 value: rewriter.getZeroAttr(type: elemType));
978 Value res = BroadcastOp::create(builder&: rewriter, location: loc, result: castDstType, source: zero);
979
980 SmallVector<int64_t> sliceShape = {castDstLastDim};
981 SmallVector<int64_t> strides = {1};
982 VectorType newCastDstType =
983 VectorType::get(shape: SmallVector<int64_t>{castDstLastDim / shrinkRatio},
984 elementType: castDstType.getElementType());
985
986 for (int i = 0, e = shrinkRatio; i < e; ++i) {
987 Value extracted = ExtractStridedSliceOp::create(
988 builder&: rewriter, location: loc, source: bitcastOp.getSource(),
989 offsets: ArrayRef<int64_t>{i * castDstLastDim}, sizes: sliceShape, strides);
990 Value bitcast =
991 BitCastOp::create(builder&: rewriter, location: loc, result: newCastDstType, source: extracted);
992 res = InsertStridedSliceOp::create(
993 builder&: rewriter, location: loc, valueToStore: bitcast, dest: res,
994 offsets: ArrayRef<int64_t>{i * castDstLastDim / shrinkRatio}, strides);
995 }
996 rewriter.replaceOp(op: bitcastOp, newValues: res);
997 return success();
998 }
999
1000private:
1001 std::function<bool(BitCastOp)> controlFn;
1002};
1003
1004static bool haveSameShapeAndScaling(Type t, Type u) {
1005 auto tVec = dyn_cast<VectorType>(Val&: t);
1006 auto uVec = dyn_cast<VectorType>(Val&: u);
1007 if (!tVec) {
1008 return !uVec;
1009 }
1010 if (!uVec) {
1011 return false;
1012 }
1013 return tVec.getShape() == uVec.getShape() &&
1014 tVec.getScalableDims() == uVec.getScalableDims();
1015}
1016
1017/// If `type` is shaped, clone it with `newElementType`. Otherwise,
1018/// return `newElementType`.
1019static Type cloneOrReplace(Type type, Type newElementType) {
1020 if (auto shapedType = dyn_cast<ShapedType>(Val&: type)) {
1021 return shapedType.clone(elementType: newElementType);
1022 }
1023 return newElementType;
1024}
1025
1026/// If `value` is the result of a broadcast operation, return the input
1027/// of the broadcast operation.
1028static Value getBroadcastLikeSource(Value value) {
1029
1030 Operation *op = value.getDefiningOp();
1031 if (!op)
1032 return {};
1033
1034 if (auto broadcast = dyn_cast<vector::BroadcastOp>(Val: op))
1035 return broadcast.getSource();
1036
1037 return {};
1038}
1039
1040/// Reorders elementwise(broadcast) to broadcast(elementwise). Ex:
1041///
1042/// Example:
1043/// ```
1044/// %a = vector.broadcast %arg1 : index to vector<1x4xindex>
1045/// %b = vector.broadcast %arg2 : index to vector<1x4xindex>
1046/// %r = arith.addi %a, %b : vector<1x4xindex>
1047/// ```
1048/// Gets converted to:
1049/// ```
1050/// %r = arith.addi %arg0, %arg1 : index
1051/// %b = vector.broadcast %r : index to vector<1x4xindex>
1052/// ```
1053struct ReorderElementwiseOpsOnBroadcast final
1054 : public OpTraitRewritePattern<OpTrait::Elementwise> {
1055 using Base::Base;
1056 LogicalResult matchAndRewrite(Operation *op,
1057 PatternRewriter &rewriter) const override {
1058 if (op->getNumResults() != 1)
1059 return failure();
1060 auto resultType = dyn_cast<VectorType>(Val: op->getResult(idx: 0).getType());
1061 if (!resultType)
1062 return failure();
1063 if (!OpTrait::hasElementwiseMappableTraits(op))
1064 return rewriter.notifyMatchFailure(
1065 arg&: op, msg: "Op doesn't have ElementwiseMappableTraits");
1066 if (op->getNumOperands() == 0)
1067 return failure();
1068
1069 Type resultElemType = resultType.getElementType();
1070
1071 // Select the source shape for the reordered computation. Prefer the first
1072 // non-constant vector source so that scalar sources can be broadcast to its
1073 // shape. The compatibility check below ensures that all vector sources have
1074 // the same shape and scalable dimensions.
1075 Value broadcastSource;
1076 Value firstBroadcastSource;
1077 for (Value operand : op->getOperands()) {
1078 Operation *definingOp = operand.getDefiningOp();
1079 if (!definingOp)
1080 return failure();
1081 if (definingOp->hasTrait<OpTrait::ConstantLike>())
1082 continue;
1083 Value source = getBroadcastLikeSource(value: operand);
1084 if (!source)
1085 return failure();
1086 if (!firstBroadcastSource)
1087 firstBroadcastSource = source;
1088 if (isa<VectorType>(Val: source.getType())) {
1089 broadcastSource = source;
1090 break;
1091 }
1092 }
1093 // If all non-constant operands are scalar, choose the first source.
1094 if (!broadcastSource)
1095 broadcastSource = firstBroadcastSource;
1096 if (!broadcastSource)
1097 return failure();
1098 Type unbroadcastResultType =
1099 cloneOrReplace(type: broadcastSource.getType(), newElementType: resultElemType);
1100
1101 // Some ops, e.g. `vector.fma`, only accept vector types. For such ops, a
1102 // vector broadcast source is needed to determine the type of the reordered
1103 // op. Scalar sources can then be promoted to that vector type.
1104 // TODO: Support the case where all broadcast sources are scalars by
1105 // promoting them to single element vectors.
1106 if (isa<vector::FMAOp>(Val: op) && !isa<VectorType>(Val: unbroadcastResultType)) {
1107 return rewriter.notifyMatchFailure(
1108 arg&: op, msg: "Op only accepts vector types, but the broadcast source is a "
1109 "scalar");
1110 }
1111
1112 // Make sure that all operands are broadcasts from compatible source types.
1113 // Scalar sources are allowed when a vector source is available and are
1114 // promoted to the vector source type selected above.
1115 if (!llvm::all_of(Range: op->getOperands(), P: [broadcastSource](Value val) {
1116 if (auto source = getBroadcastLikeSource(value: val))
1117 return haveSameShapeAndScaling(t: source.getType(),
1118 u: broadcastSource.getType()) ||
1119 (isa<VectorType>(Val: broadcastSource.getType()) &&
1120 !isa<VectorType>(Val: source.getType()));
1121 SplatElementsAttr splatConst;
1122 return matchPattern(value: val, pattern: m_Constant(bind_value: &splatConst));
1123 })) {
1124 return rewriter.notifyMatchFailure(
1125 arg&: op,
1126 msg: "not all operands are constants or broadcasts from the same type");
1127 }
1128
1129 // Collect the source values before broadcasting
1130 SmallVector<Value> srcValues;
1131 srcValues.reserve(N: op->getNumOperands());
1132 for (Value operand : op->getOperands()) {
1133 SplatElementsAttr splatConst;
1134 if (matchPattern(value: operand, pattern: m_Constant(bind_value: &splatConst))) {
1135 Attribute newConst;
1136 Type elementType = getElementTypeOrSelf(type: operand.getType());
1137 Type newType = cloneOrReplace(type: unbroadcastResultType, newElementType: elementType);
1138 if (auto newTypeShaped = dyn_cast<ShapedType>(Val&: newType)) {
1139 newConst = splatConst.resizeSplat(newType: newTypeShaped);
1140 } else {
1141 newConst = splatConst.getSplatValue<Attribute>();
1142 }
1143 Operation *newConstOp =
1144 operand.getDefiningOp()->getDialect()->materializeConstant(
1145 builder&: rewriter, value: newConst, type: newType, loc: operand.getLoc());
1146 srcValues.push_back(Elt: newConstOp->getResult(idx: 0));
1147 } else {
1148 Value source = operand.getDefiningOp()->getOperand(idx: 0);
1149 if (isa<VectorType>(Val: broadcastSource.getType()) &&
1150 !isa<VectorType>(Val: source.getType()))
1151 source = vector::BroadcastOp::create(
1152 builder&: rewriter, location: operand.getLoc(),
1153 result: cloneOrReplace(type: broadcastSource.getType(), newElementType: source.getType()),
1154 source);
1155 srcValues.push_back(Elt: source);
1156 }
1157 }
1158
1159 // Create the "elementwise" Op
1160 Operation *elementwiseOp =
1161 createWithProperties(builder&: rewriter, op, operands: srcValues, types: unbroadcastResultType);
1162
1163 // Replace the original Op with the elementwise Op
1164 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
1165 op, args&: resultType, args: elementwiseOp->getResults());
1166
1167 return success();
1168 }
1169};
1170
1171/// Pattern to rewrite a ExtractOp(Elementwise) -> Elementwise(ExtractOp).
1172/// This may result in cleaner code when extracting a single value
1173/// from multi-element vector and also to help canonicalize 1-element vectors to
1174/// scalars.
1175///
1176/// Example:
1177/// ```
1178/// %0 = arith.addf %arg0, %arg1 : vector<4xf32>
1179/// %1 = vector.extract %0[1] : f32 from vector<4xf32>
1180/// ```
1181/// Gets converted to:
1182/// ```
1183/// %0 = vector.extract %arg0[1] : f32 from vector<4xf32>
1184/// %1 = vector.extract %arg1[1] : f32 from vector<4xf32>
1185/// %2 = arith.addf %0, %1 : f32
1186/// ```
1187class ExtractOpFromElementwise final
1188 : public OpRewritePattern<vector::ExtractOp> {
1189public:
1190 using Base::Base;
1191
1192 LogicalResult matchAndRewrite(vector::ExtractOp op,
1193 PatternRewriter &rewriter) const override {
1194 Operation *eltwise = op.getSource().getDefiningOp();
1195
1196 // TODO: vector::FMAOp is not an ElemetwiseMappable even if it claims to be,
1197 // as it doesn't support scalars.
1198 if (!eltwise || !OpTrait::hasElementwiseMappableTraits(op: eltwise) ||
1199 isa<vector::FMAOp>(Val: eltwise))
1200 return rewriter.notifyMatchFailure(arg&: op, msg: "not an elementwise op");
1201
1202 if (eltwise->getNumResults() != 1)
1203 return rewriter.notifyMatchFailure(arg&: op, msg: "expected single result");
1204
1205 if (!eltwise->hasOneUse())
1206 return rewriter.notifyMatchFailure(arg&: op, msg: "expected single op use");
1207
1208 if (!llvm::all_equal(Range: eltwise->getOperandTypes()))
1209 return rewriter.notifyMatchFailure(arg&: op, msg: "operand types are different");
1210
1211 // Dynamic position can cause dominance issues, so conservatively fail for
1212 // now.
1213 if (!op.getDynamicPosition().empty())
1214 return rewriter.notifyMatchFailure(
1215 arg&: op, msg: "dynamic position not yet implemented");
1216
1217 Type dstType = op.getType();
1218
1219 OpBuilder::InsertionGuard g(rewriter);
1220 rewriter.setInsertionPoint(eltwise);
1221
1222 IRMapping mapping;
1223 Location loc = eltwise->getLoc();
1224 SmallVector<OpFoldResult> pos = op.getMixedPosition();
1225 for (Value arg : eltwise->getOperands()) {
1226 Value newArg = vector::ExtractOp::create(builder&: rewriter, location: loc, source: arg, position: pos);
1227 mapping.map(from: arg, to: newArg);
1228 }
1229
1230 Operation *newEltwise = rewriter.clone(op&: *eltwise, mapper&: mapping);
1231 newEltwise->getResult(idx: 0).setType(dstType);
1232
1233 rewriter.replaceOp(op, newOp: newEltwise);
1234 rewriter.eraseOp(op: eltwise);
1235 return success();
1236 }
1237};
1238
1239/// Check if the element type is suitable for vector.load/store sinking.
1240/// Element type must be index or byte-aligned integer or floating-point type.
1241static bool isSupportedMemSinkElementType(Type type) {
1242 if (isa<IndexType>(Val: type))
1243 return true;
1244
1245 return type.isIntOrFloat() && type.getIntOrFloatBitWidth() % 8 == 0;
1246}
1247
1248/// Pattern to rewrite `vector.extract(vector.load) -> vector/memref.load.
1249/// Only index and byte-aligned integer and floating-point element types are
1250/// supported for now.
1251///
1252/// Example:
1253/// ```
1254/// vector.load %arg0[%arg1] : memref<?xf32>, vector<4xf32>
1255/// vector.extract %0[1] : f32 from vector<4xf32>
1256/// ```
1257/// Gets converted to:
1258/// ```
1259/// %c1 = arith.constant 1 : index
1260/// %0 = arith.addi %arg1, %c1 overflow<nsw> : index
1261/// %1 = memref.load %arg0[%0] : memref<?xf32>
1262/// ```
1263class ExtractOpFromLoad final : public OpRewritePattern<vector::ExtractOp> {
1264public:
1265 using Base::Base;
1266
1267 LogicalResult matchAndRewrite(vector::ExtractOp op,
1268 PatternRewriter &rewriter) const override {
1269 auto loadOp = op.getSource().getDefiningOp<vector::LoadOp>();
1270 if (!loadOp)
1271 return rewriter.notifyMatchFailure(arg&: op, msg: "expected a load op");
1272
1273 // Checking for single use so we won't duplicate load ops.
1274 if (!loadOp->hasOneUse())
1275 return rewriter.notifyMatchFailure(arg&: op, msg: "expected single op use");
1276
1277 VectorType loadVecType = loadOp.getVectorType();
1278 if (loadVecType.isScalable())
1279 return rewriter.notifyMatchFailure(arg&: op,
1280 msg: "scalable vectors are not supported");
1281
1282 MemRefType memType = loadOp.getMemRefType();
1283
1284 // Non-byte-aligned types are tricky and may require special handling,
1285 // ignore them for now.
1286 if (!isSupportedMemSinkElementType(type: memType.getElementType()))
1287 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported element type");
1288
1289 int64_t rankOffset = memType.getRank() - loadVecType.getRank();
1290 if (rankOffset < 0)
1291 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported ranks combination");
1292
1293 auto extractVecType = dyn_cast<VectorType>(Val: op.getResult().getType());
1294 int64_t finalRank = 0;
1295 if (extractVecType)
1296 finalRank = extractVecType.getRank();
1297
1298 SmallVector<Value> indices = loadOp.getIndices();
1299 SmallVector<OpFoldResult> extractPos = op.getMixedPosition();
1300
1301 // There may be memory stores between the load and the extract op, so we
1302 // need to make sure that the new load op is inserted at the same place as
1303 // the original load op.
1304 OpBuilder::InsertionGuard g(rewriter);
1305 rewriter.setInsertionPoint(loadOp);
1306 Location loc = loadOp.getLoc();
1307 ArithIndexingBuilder idxBuilderf(rewriter, loc);
1308 for (auto i : llvm::seq<int64_t>(Begin: rankOffset, End: indices.size() - finalRank)) {
1309 OpFoldResult pos = extractPos[i - rankOffset];
1310 if (isZeroInteger(v: pos))
1311 continue;
1312
1313 Value offset = getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr: pos);
1314 indices[i] = idxBuilderf.add(lhs: indices[i], rhs: offset);
1315 }
1316
1317 Value base = loadOp.getBase();
1318 if (extractVecType) {
1319 rewriter.replaceOpWithNewOp<vector::LoadOp>(op, args&: extractVecType, args&: base,
1320 args&: indices);
1321 } else {
1322 rewriter.replaceOpWithNewOp<memref::LoadOp>(op, args&: base, args&: indices);
1323 }
1324 // We checked for single use so we can safely erase the load op.
1325 rewriter.eraseOp(op: loadOp);
1326 return success();
1327 }
1328};
1329
1330/// Pattern to rewrite vector.store(vector.broadcast) -> vector/memref.store.
1331///
1332/// Example:
1333/// ```
1334/// %0 = vector.broadcast %arg2 : f32 to vector<1xf32>
1335/// vector.store %0, %arg0[%arg1] : memref<?xf32>, vector<1xf32>
1336/// ```
1337/// Gets converted to:
1338/// ```
1339/// memref.store %arg2, %arg0[%arg1] : memref<?xf32>
1340/// ```
1341class StoreOpFromBroadcast final : public OpRewritePattern<vector::StoreOp> {
1342public:
1343 using Base::Base;
1344
1345 LogicalResult matchAndRewrite(vector::StoreOp op,
1346 PatternRewriter &rewriter) const override {
1347 VectorType vecType = op.getVectorType();
1348 if (vecType.isScalable())
1349 return rewriter.notifyMatchFailure(arg&: op,
1350 msg: "scalable vectors are not supported");
1351
1352 if (isa<VectorType>(Val: op.getMemRefType().getElementType()))
1353 return rewriter.notifyMatchFailure(
1354 arg&: op, msg: "memrefs of vectors are not supported");
1355
1356 if (vecType.getNumElements() != 1)
1357 return rewriter.notifyMatchFailure(
1358 arg&: op, msg: "only 1-element vectors are supported");
1359
1360 Value toStore = op.getValueToStore();
1361 Value source = getBroadcastLikeSource(value: toStore);
1362 if (!source)
1363 return rewriter.notifyMatchFailure(
1364 arg&: op, msg: "value to store is not from a broadcast");
1365
1366 // Checking for single use so we can remove broadcast.
1367 Operation *broadcast = toStore.getDefiningOp();
1368 if (!broadcast->hasOneUse())
1369 return rewriter.notifyMatchFailure(arg&: op, msg: "expected single op use");
1370
1371 Value base = op.getBase();
1372 ValueRange indices = op.getIndices();
1373
1374 if (isa<VectorType>(Val: source.getType())) {
1375 rewriter.replaceOpWithNewOp<vector::StoreOp>(op, args&: source, args&: base, args&: indices);
1376 } else {
1377 rewriter.replaceOpWithNewOp<memref::StoreOp>(op, args&: source, args&: base, args&: indices);
1378 }
1379 rewriter.eraseOp(op: broadcast);
1380 return success();
1381 }
1382};
1383
1384// Helper that returns a vector comparison that constructs a mask:
1385// mask = [0,1,..,n-1] + [o,o,..,o] < [b,b,..,b]
1386//
1387// If `dim == 0` then the result will be a 0-D vector.
1388//
1389// NOTE: The LLVM::GetActiveLaneMaskOp intrinsic would provide an alternative,
1390// much more compact, IR for this operation, but LLVM eventually
1391// generates more elaborate instructions for this intrinsic since it
1392// is very conservative on the boundary conditions.
1393static Value buildVectorComparison(PatternRewriter &rewriter, Operation *op,
1394 bool force32BitVectorIndices, int64_t dim,
1395 Value b, Value *off = nullptr) {
1396 auto loc = op->getLoc();
1397 // If we can assume all indices fit in 32-bit, we perform the vector
1398 // comparison in 32-bit to get a higher degree of SIMD parallelism.
1399 // Otherwise we perform the vector comparison using 64-bit indices.
1400 Type idxType =
1401 force32BitVectorIndices ? rewriter.getI32Type() : rewriter.getI64Type();
1402 DenseIntElementsAttr indicesAttr;
1403 if (dim == 0 && force32BitVectorIndices) {
1404 indicesAttr = DenseIntElementsAttr::get(
1405 type: VectorType::get(shape: ArrayRef<int64_t>{}, elementType: idxType), arg: ArrayRef<int32_t>{0});
1406 } else if (dim == 0) {
1407 indicesAttr = DenseIntElementsAttr::get(
1408 type: VectorType::get(shape: ArrayRef<int64_t>{}, elementType: idxType), arg: ArrayRef<int64_t>{0});
1409 } else if (force32BitVectorIndices) {
1410 indicesAttr = rewriter.getI32VectorAttr(
1411 values: llvm::to_vector<4>(Range: llvm::seq<int32_t>(Begin: 0, End: dim)));
1412 } else {
1413 indicesAttr = rewriter.getI64VectorAttr(
1414 values: llvm::to_vector<4>(Range: llvm::seq<int64_t>(Begin: 0, End: dim)));
1415 }
1416 Value indices = arith::ConstantOp::create(builder&: rewriter, location: loc, value: indicesAttr);
1417 // Add in an offset if requested.
1418 if (off) {
1419 Value o = getValueOrCreateCastToIndexLike(b&: rewriter, loc, targetType: idxType, value: *off);
1420 Value ov = vector::BroadcastOp::create(builder&: rewriter, location: loc, result: indices.getType(), source: o);
1421 indices = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: ov, rhs: indices);
1422 }
1423 // Construct the vector comparison.
1424 // When using 32-bit indices, cap `b` at INT32_MAX before casting to prevent
1425 // signed overflow for large index values (e.g., 2^51 wrapping to 0 in i32).
1426 // Note: for fixed-size vectors, `dim` is a tighter bound (since any b >= dim
1427 // already implies all-true), but we use INT32_MAX for uniformity with the
1428 // scalable-vector path.
1429 if (force32BitVectorIndices) {
1430 Value maxBound =
1431 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: (1LL << 31) - 1);
1432 b = arith::MinSIOp::create(builder&: rewriter, location: loc, lhs: b, rhs: maxBound);
1433 }
1434 Value bound = getValueOrCreateCastToIndexLike(b&: rewriter, loc, targetType: idxType, value: b);
1435 Value bounds =
1436 vector::BroadcastOp::create(builder&: rewriter, location: loc, result: indices.getType(), source: bound);
1437 return arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::slt,
1438 lhs: indices, rhs: bounds);
1439}
1440
1441template <typename ConcreteOp>
1442struct MaterializeTransferMask : public OpRewritePattern<ConcreteOp> {
1443public:
1444 explicit MaterializeTransferMask(MLIRContext *context, bool enableIndexOpt,
1445 PatternBenefit benefit = 1)
1446 : mlir::OpRewritePattern<ConcreteOp>(context, benefit),
1447 force32BitVectorIndices(enableIndexOpt) {}
1448
1449 LogicalResult matchAndRewrite(ConcreteOp xferOp,
1450 PatternRewriter &rewriter) const override {
1451 if (!xferOp.hasOutOfBoundsDim())
1452 return failure();
1453
1454 if (xferOp.getVectorType().getRank() > 1 || xferOp.getIndices().empty())
1455 return failure();
1456
1457 Location loc = xferOp->getLoc();
1458 VectorType vtp = xferOp.getVectorType();
1459
1460 // Create the in-bounds mask with all elements between [0 .. dim - offset)
1461 // set and [dim - offset .. vector_length) unset.
1462 //
1463 // TODO: when the leaf transfer rank is k > 1, we need the last `k`
1464 // dimensions here.
1465 unsigned lastIndex = llvm::size(xferOp.getIndices()) - 1;
1466 Value off = xferOp.getIndices()[lastIndex];
1467 Value dim =
1468 vector::createOrFoldDimOp(b&: rewriter, loc, source: xferOp.getBase(), dim: lastIndex);
1469 Value b = arith::SubIOp::create(builder&: rewriter, location: loc, result: dim.getType(), lhs: dim, rhs: off);
1470 Value mask = vector::CreateMaskOp::create(
1471 builder&: rewriter, location: loc,
1472 resultType0: VectorType::get(shape: vtp.getShape(), elementType: rewriter.getI1Type(),
1473 scalableDims: vtp.getScalableDims()),
1474 mask_dim_sizes: b);
1475 if (xferOp.getMask()) {
1476 // Intersect the in-bounds with the mask specified as an op parameter.
1477 mask = arith::AndIOp::create(rewriter, loc, mask, xferOp.getMask());
1478 }
1479
1480 rewriter.modifyOpInPlace(xferOp, [&]() {
1481 xferOp.getMaskMutable().assign(mask);
1482 xferOp.setInBoundsAttr(rewriter.getBoolArrayAttr(values: {true}));
1483 });
1484
1485 return success();
1486 }
1487
1488private:
1489 const bool force32BitVectorIndices;
1490};
1491
1492/// Conversion pattern for a `vector.create_mask` (0-D and 1-D only).
1493class VectorCreateMaskOpConversion
1494 : public OpRewritePattern<vector::CreateMaskOp> {
1495public:
1496 explicit VectorCreateMaskOpConversion(MLIRContext *context,
1497 bool enableIndexOpt,
1498 PatternBenefit benefit = 1)
1499 : mlir::OpRewritePattern<vector::CreateMaskOp>(context, benefit),
1500 force32BitVectorIndices(enableIndexOpt) {}
1501
1502 LogicalResult matchAndRewrite(vector::CreateMaskOp op,
1503 PatternRewriter &rewriter) const override {
1504 auto dstType = op.getType();
1505 if (cast<VectorType>(Val&: dstType).isScalable())
1506 return failure();
1507 int64_t rank = dstType.getRank();
1508 if (rank > 1)
1509 return failure();
1510 rewriter.replaceOp(
1511 op, newValues: buildVectorComparison(rewriter, op, force32BitVectorIndices,
1512 dim: rank == 0 ? 0 : dstType.getDimSize(idx: 0),
1513 b: op.getOperand(i: 0)));
1514 return success();
1515 }
1516
1517private:
1518 const bool force32BitVectorIndices;
1519};
1520
1521/// Returns true if all the `i1` elements of `constantOp` are set to `value`.
1522static bool allI1ConstantValuesSetTo(arith::ConstantOp constantOp, bool value) {
1523 auto denseAttr = dyn_cast<DenseIntElementsAttr>(Val: constantOp.getValue());
1524 // TODO: Support non-dense constant.
1525 if (!denseAttr)
1526 return false;
1527
1528 assert(denseAttr.getElementType().isInteger(1) && "Unexpected type");
1529 return denseAttr.isSplat() && denseAttr.getSplatValue<bool>() == value;
1530}
1531
1532/// Folds a select operation between an all-true and all-false vector. For now,
1533/// only single element vectors (i.e., vector<1xi1>) are supported. That is:
1534///
1535/// %true = arith.constant dense<true> : vector<1xi1>
1536/// %false = arith.constant dense<false> : vector<1xi1>
1537/// %result = arith.select %cond, %true, %false : i1, vector<1xi1>
1538/// =>
1539/// %result = vector.broadcast %cond : i1 to vector<1xi1>
1540///
1541/// InstCombine seems to handle vectors with multiple elements but not the
1542/// single element ones.
1543struct FoldI1Select : public OpRewritePattern<arith::SelectOp> {
1544 using Base::Base;
1545
1546 LogicalResult matchAndRewrite(arith::SelectOp selectOp,
1547 PatternRewriter &rewriter) const override {
1548 auto vecType = dyn_cast<VectorType>(Val: selectOp.getType());
1549 if (!vecType || !vecType.getElementType().isInteger(width: 1))
1550 return failure();
1551
1552 // Only scalar conditions can be folded.
1553 Value cond = selectOp.getCondition();
1554 if (isa<VectorType>(Val: cond.getType()))
1555 return failure();
1556
1557 // TODO: Support n-D and scalable vectors.
1558 if (vecType.getRank() != 1 || vecType.isScalable())
1559 return failure();
1560
1561 // TODO: Support vectors with multiple elements.
1562 if (vecType.getShape()[0] != 1)
1563 return failure();
1564
1565 auto trueConst = selectOp.getTrueValue().getDefiningOp<arith::ConstantOp>();
1566 if (!trueConst || !allI1ConstantValuesSetTo(constantOp: trueConst, value: true))
1567 return failure();
1568
1569 auto falseConst =
1570 selectOp.getFalseValue().getDefiningOp<arith::ConstantOp>();
1571 if (!falseConst || !allI1ConstantValuesSetTo(constantOp: falseConst, value: false))
1572 return failure();
1573
1574 // Replace select with its condition broadcasted to single element vector.
1575 auto elemType = rewriter.getIntegerType(width: vecType.getNumElements());
1576 auto bcastType = VectorType::get(/*shape=*/{1}, elementType: elemType);
1577 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: selectOp, args&: bcastType, args&: cond);
1578 return success();
1579 }
1580};
1581
1582/// Returns the number of dims can be folded away from transfer ops. It returns
1583/// a failure if it can not determine the number of dims to be folded.
1584///
1585/// Ex 1: returns "2" if `srcType` is memref<512x16x1x1xf32> and
1586/// `vectorType` is vector<16x16x1x1xf32>
1587/// (there two inner most dims can be dropped by memref.subview ops)
1588///
1589/// Ex 2: returns "1" if `srcType` is memref<512x16x1x1xf32> with
1590/// [8192, 16, 8, 1] strides and `vectorType` is vector<16x16x1x1xf32>
1591/// (only the inner most unit dim of `srcType` can be dropped)
1592///
1593/// Ex 3: return "0" if `srcType` is memref<512x16x1x1xf32> and
1594/// `vectorType` is vector<16x16x1x[1]xf32>
1595/// (the most inner dim in `vectorType` is not a unit dim (it's a "scalable
1596/// unit")
1597static FailureOr<size_t>
1598getTransferFoldableInnerUnitDims(MemRefType srcType, VectorType vectorType) {
1599 SmallVector<int64_t> srcStrides;
1600 int64_t srcOffset;
1601 if (failed(Result: srcType.getStridesAndOffset(strides&: srcStrides, offset&: srcOffset)))
1602 return failure();
1603
1604 auto isUnitDim = [](VectorType type, int dim) {
1605 return type.getDimSize(idx: dim) == 1 && !type.getScalableDims()[dim];
1606 };
1607
1608 // According to vector.transfer_read/write semantics, the vector can be a
1609 // slice. Thus, we have to offset the check index with `rankDiff` in
1610 // `srcStrides` and source dim sizes.
1611 size_t result = 0;
1612 int rankDiff = srcType.getRank() - vectorType.getRank();
1613 for (int64_t i = 0, e = vectorType.getRank(); i < e; ++i) {
1614 // Check that the inner dim size is 1 for both memref type and vector slice.
1615 // It can be folded only if they are 1 and the stride is 1.
1616 int dim = vectorType.getRank() - i - 1;
1617 if (srcStrides[dim + rankDiff] != 1 ||
1618 srcType.getDimSize(idx: dim + rankDiff) != 1 || !isUnitDim(vectorType, dim))
1619 break;
1620 result++;
1621 }
1622 return result;
1623}
1624
1625/// Drop inner most contiguous unit dimensions from transfer_read operand.
1626class DropInnerMostUnitDimsTransferRead
1627 : public OpRewritePattern<vector::TransferReadOp> {
1628 using Base::Base;
1629
1630 LogicalResult matchAndRewrite(vector::TransferReadOp readOp,
1631 PatternRewriter &rewriter) const override {
1632 // TODO: support 0-d corner case.
1633 if (readOp.getTransferRank() == 0)
1634 return failure();
1635
1636 auto srcType = dyn_cast<MemRefType>(Val: readOp.getBase().getType());
1637 if (!srcType)
1638 return failure();
1639
1640 if (!readOp.getPermutationMap().isMinorIdentity())
1641 return failure();
1642
1643 auto targetType = readOp.getVectorType();
1644 if (targetType.getRank() <= 1)
1645 return failure();
1646
1647 FailureOr<size_t> maybeDimsToDrop =
1648 getTransferFoldableInnerUnitDims(srcType, vectorType: targetType);
1649 if (failed(Result: maybeDimsToDrop))
1650 return failure();
1651
1652 size_t dimsToDrop = maybeDimsToDrop.value();
1653 if (dimsToDrop == 0)
1654 return failure();
1655
1656 auto inBounds = readOp.getInBoundsValues();
1657 auto droppedInBounds = ArrayRef<bool>(inBounds).take_back(N: dimsToDrop);
1658 if (llvm::is_contained(Range&: droppedInBounds, Element: false))
1659 return failure();
1660
1661 auto resultTargetVecType =
1662 VectorType::get(shape: targetType.getShape().drop_back(N: dimsToDrop),
1663 elementType: targetType.getElementType(),
1664 scalableDims: targetType.getScalableDims().drop_back(N: dimsToDrop));
1665
1666 auto loc = readOp.getLoc();
1667 SmallVector<OpFoldResult> sizes =
1668 memref::getMixedSizes(builder&: rewriter, loc, value: readOp.getBase());
1669 SmallVector<OpFoldResult> offsets(srcType.getRank(),
1670 rewriter.getIndexAttr(value: 0));
1671 SmallVector<OpFoldResult> strides(srcType.getRank(),
1672 rewriter.getIndexAttr(value: 1));
1673 MemRefType resultMemrefType = memref::SubViewOp::inferRankReducedResultType(
1674 resultShape: srcType.getShape().drop_back(N: dimsToDrop), sourceMemRefType: srcType, staticOffsets: offsets, staticSizes: sizes,
1675 staticStrides: strides);
1676 ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
1677 value: readOp.getInBoundsAttr().getValue().drop_back(N: dimsToDrop));
1678 Value rankedReducedView =
1679 memref::SubViewOp::create(builder&: rewriter, location: loc, resultType: resultMemrefType,
1680 source: readOp.getBase(), offsets, sizes, strides);
1681 auto permMap = getTransferMinorIdentityMap(
1682 shapedType: cast<ShapedType>(Val: rankedReducedView.getType()), vectorType: resultTargetVecType);
1683
1684 // If there is a mask, shape_cast it to drop the same inner unit dims.
1685 Value mask = readOp.getMask();
1686 if (mask) {
1687 auto maskType = cast<VectorType>(Val: mask.getType());
1688 auto reducedMaskType = VectorType::get(
1689 shape: maskType.getShape().drop_back(N: dimsToDrop), elementType: maskType.getElementType(),
1690 scalableDims: maskType.getScalableDims().drop_back(N: dimsToDrop));
1691 mask = rewriter.createOrFold<vector::ShapeCastOp>(location: loc, args&: reducedMaskType,
1692 args&: mask);
1693 }
1694
1695 Value result = vector::TransferReadOp::create(
1696 builder&: rewriter, location: loc, result: resultTargetVecType, base: rankedReducedView,
1697 indices: readOp.getIndices().drop_back(n: dimsToDrop), permutation_map: AffineMapAttr::get(value: permMap),
1698 padding: readOp.getPadding(), mask, in_bounds: inBoundsAttr);
1699 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: readOp, args&: targetType,
1700 args&: result);
1701 return success();
1702 }
1703};
1704
1705/// Drop inner most contiguous unit dimensions from transfer_write operand.
1706/// E.g.,
1707/// vector.transfer_write %arg1, %arg0[%c0, %arg2, %c0, %c0, %c0]
1708/// {in_bounds = [true, true, true, true, true]}
1709/// : vector<1x16x16x1x1xf32>, memref<1x512x16x1x1xf32>
1710///
1711/// will be replaced with
1712///
1713/// %subview = memref.subview %arg0
1714/// [0, 0, 0, 0, 0] [1, 512, 16, 1, 1] [1, 1, 1, 1, 1]
1715/// : memref<1x512x16x1x1xf32> to memref<1x512x16xf32>
1716/// %0 = vector.shape_cast %arg1 : vector<1x16x16x1x1xf32>
1717/// to vector<1x16x16xf32>
1718/// vector.transfer_write %0, %subview[%c0, %arg2, %c0]
1719/// {in_bounds = [true, true, true]}
1720/// : vector<1x16x16xf32>, memref<1x512x16xf32>
1721///
1722/// Note, this pattern will not collapse "scalable unit" dims (i.e. `[1]`).
1723class DropInnerMostUnitDimsTransferWrite
1724 : public OpRewritePattern<vector::TransferWriteOp> {
1725 using Base::Base;
1726
1727 LogicalResult matchAndRewrite(vector::TransferWriteOp writeOp,
1728 PatternRewriter &rewriter) const override {
1729 // TODO: support 0-d corner case.
1730 if (writeOp.getTransferRank() == 0)
1731 return failure();
1732
1733 auto srcType = dyn_cast<MemRefType>(Val: writeOp.getBase().getType());
1734 if (!srcType)
1735 return failure();
1736
1737 if (!writeOp.getPermutationMap().isMinorIdentity())
1738 return failure();
1739
1740 auto targetType = writeOp.getVectorType();
1741 if (targetType.getRank() <= 1)
1742 return failure();
1743
1744 FailureOr<size_t> maybeDimsToDrop =
1745 getTransferFoldableInnerUnitDims(srcType, vectorType: targetType);
1746 if (failed(Result: maybeDimsToDrop))
1747 return failure();
1748
1749 size_t dimsToDrop = maybeDimsToDrop.value();
1750 if (dimsToDrop == 0)
1751 return failure();
1752
1753 auto inBounds = writeOp.getInBoundsValues();
1754 auto droppedInBounds = ArrayRef<bool>(inBounds).take_back(N: dimsToDrop);
1755 if (llvm::is_contained(Range&: droppedInBounds, Element: false))
1756 return failure();
1757
1758 auto resultTargetVecType =
1759 VectorType::get(shape: targetType.getShape().drop_back(N: dimsToDrop),
1760 elementType: targetType.getElementType(),
1761 scalableDims: targetType.getScalableDims().drop_back(N: dimsToDrop));
1762
1763 Location loc = writeOp.getLoc();
1764 SmallVector<OpFoldResult> sizes =
1765 memref::getMixedSizes(builder&: rewriter, loc, value: writeOp.getBase());
1766 SmallVector<OpFoldResult> offsets(srcType.getRank(),
1767 rewriter.getIndexAttr(value: 0));
1768 SmallVector<OpFoldResult> strides(srcType.getRank(),
1769 rewriter.getIndexAttr(value: 1));
1770 MemRefType resultMemrefType = memref::SubViewOp::inferRankReducedResultType(
1771 resultShape: srcType.getShape().drop_back(N: dimsToDrop), sourceMemRefType: srcType, staticOffsets: offsets, staticSizes: sizes,
1772 staticStrides: strides);
1773 ArrayAttr inBoundsAttr = rewriter.getArrayAttr(
1774 value: writeOp.getInBoundsAttr().getValue().drop_back(N: dimsToDrop));
1775
1776 Value rankedReducedView =
1777 memref::SubViewOp::create(builder&: rewriter, location: loc, resultType: resultMemrefType,
1778 source: writeOp.getBase(), offsets, sizes, strides);
1779 auto permMap = getTransferMinorIdentityMap(
1780 shapedType: cast<ShapedType>(Val: rankedReducedView.getType()), vectorType: resultTargetVecType);
1781
1782 auto shapeCast = rewriter.createOrFold<vector::ShapeCastOp>(
1783 location: loc, args&: resultTargetVecType, args: writeOp.getVector());
1784
1785 // If there is a mask, shape_cast it to drop the same inner unit dims.
1786 Value mask = writeOp.getMask();
1787 if (mask) {
1788 auto maskType = cast<VectorType>(Val: mask.getType());
1789 auto reducedMaskType = VectorType::get(
1790 shape: maskType.getShape().drop_back(N: dimsToDrop), elementType: maskType.getElementType(),
1791 scalableDims: maskType.getScalableDims().drop_back(N: dimsToDrop));
1792 mask = rewriter.createOrFold<vector::ShapeCastOp>(location: loc, args&: reducedMaskType,
1793 args&: mask);
1794 }
1795
1796 rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
1797 op: writeOp, args&: shapeCast, args&: rankedReducedView,
1798 args: writeOp.getIndices().drop_back(n: dimsToDrop), args: AffineMapAttr::get(value: permMap),
1799 args&: mask, args&: inBoundsAttr);
1800 return success();
1801 }
1802};
1803
1804/// Canonicalization of a `vector.contract %a, %b, %c` with row-major matmul
1805/// semantics to a contraction suitable for MMT (matrix matrix multiplication
1806/// with the RHS transposed) lowering.
1807struct CanonicalizeContractMatmulToMMT final
1808 : OpRewritePattern<vector::ContractionOp> {
1809 using Base::Base;
1810
1811 using FilterConstraintType =
1812 std::function<LogicalResult(vector::ContractionOp op)>;
1813
1814 CanonicalizeContractMatmulToMMT(MLIRContext *context, PatternBenefit benefit,
1815 FilterConstraintType constraint)
1816 : OpRewritePattern<vector::ContractionOp>(context, benefit),
1817 filter(std::move(constraint)) {}
1818
1819 LogicalResult matchAndRewrite(vector::ContractionOp op,
1820 PatternRewriter &rewriter) const override {
1821 if (failed(Result: filter(op)))
1822 return failure();
1823 // The mask is not updated: the rewrite may create ops inside the
1824 // vector.mask region or swap the m and n iteration dimensions.
1825 if (op.isMasked())
1826 return rewriter.notifyMatchFailure(arg&: op, msg: "masked contraction");
1827
1828 Location loc = op.getLoc();
1829 Value lhs = op.getLhs();
1830 Value rhs = op.getRhs();
1831 Value res = op.getAcc();
1832
1833 // Set up the parallel/reduction structure in right form.
1834 using MapList = ArrayRef<ArrayRef<AffineExpr>>;
1835 auto infer = [&](MapList m) {
1836 return AffineMap::inferFromExprList(exprsList: m, context: op.getContext());
1837 };
1838 AffineExpr m;
1839 AffineExpr n;
1840 AffineExpr k;
1841 bindDims(ctx: rewriter.getContext(), exprs&: m, exprs&: n, exprs&: k);
1842 static constexpr std::array<int64_t, 2> perm = {1, 0};
1843 auto iteratorTypes = op.getIteratorTypes().getValue();
1844 SmallVector<AffineMap, 4> maps = op.getIndexingMapsArray();
1845 if (iteratorTypes.size() != 3 ||
1846 !vector::isParallelIterator(attr: iteratorTypes[0]) ||
1847 !vector::isParallelIterator(attr: iteratorTypes[1]) ||
1848 !vector::isReductionIterator(attr: iteratorTypes[2]))
1849 return rewriter.notifyMatchFailure(arg&: op, msg: "contraction is not a gemm");
1850
1851 // The canonical form is "TNT" = A row-major, B col-major, C row-major.
1852 const auto canonicalForm = infer({{m, k}, {n, k}, {m, n}});
1853 if (maps == canonicalForm)
1854 return rewriter.notifyMatchFailure(arg&: op, msg: "already in the canonical form");
1855
1856 // Create a vector transpose making sure to emit zero/sign-extend at the
1857 // end.
1858 auto createTranspose = [&rewriter, loc](Value mat) -> Value {
1859 if (auto sext = mat.getDefiningOp<arith::ExtSIOp>()) {
1860 Value trans =
1861 vector::TransposeOp::create(builder&: rewriter, location: loc, vector: sext.getIn(), permutation: perm);
1862 VectorType newType =
1863 cast<VectorType>(Val: trans.getType())
1864 .clone(elementType: cast<VectorType>(Val: mat.getType()).getElementType());
1865 return arith::ExtSIOp::create(builder&: rewriter, location: loc, out: newType, in: trans);
1866 }
1867 if (auto zext = mat.getDefiningOp<arith::ExtUIOp>()) {
1868 Value trans =
1869 vector::TransposeOp::create(builder&: rewriter, location: loc, vector: zext.getIn(), permutation: perm);
1870 VectorType newType =
1871 VectorType::get(shape: cast<VectorType>(Val: trans.getType()).getShape(),
1872 elementType: cast<VectorType>(Val: mat.getType()).getElementType());
1873 return arith::ExtUIOp::create(builder&: rewriter, location: loc, out: newType, in: trans);
1874 }
1875 return vector::TransposeOp::create(builder&: rewriter, location: loc, vector: mat, permutation: perm);
1876 };
1877
1878 if (maps == infer({{m, k}, {k, n}, {m, n}})) {
1879 rhs = createTranspose(rhs);
1880 } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
1881 lhs = createTranspose(lhs);
1882 } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
1883 rhs = createTranspose(rhs);
1884 lhs = createTranspose(lhs);
1885 } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
1886 std::swap(a&: rhs, b&: lhs);
1887 rhs = createTranspose(rhs);
1888 lhs = createTranspose(lhs);
1889 } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
1890 std::swap(a&: rhs, b&: lhs);
1891 rhs = createTranspose(rhs);
1892 } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
1893 std::swap(a&: lhs, b&: rhs);
1894 lhs = createTranspose(lhs);
1895 } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
1896 std::swap(a&: lhs, b&: rhs);
1897 } else {
1898 return rewriter.notifyMatchFailure(arg&: op, msg: "unhandled contraction form");
1899 }
1900 rewriter.replaceOpWithNewOp<vector::ContractionOp>(
1901 op, args&: lhs, args&: rhs, args&: res, args: rewriter.getAffineMapArrayAttr(values: canonicalForm),
1902 args: op.getIteratorTypes(), args: op.getKind(), args: op.getFastmath());
1903 return success();
1904 };
1905
1906private:
1907 FilterConstraintType filter;
1908};
1909
1910/// Pattern to fold arithmetic extensions on floating point data types into
1911/// vector contraction operations. linalg.matmul introduces arithmetic
1912/// extensions on its operands. Please mlir snippets below for more details.
1913/// ```mlir
1914/// "linalg.matmul"(%lhs, %rhs, %acc) ({
1915/// ^bb0(%arg1: f16, %arg2: f16, %arg3: f32):
1916/// %lhs_f32 = "arith.extf"(%arg1) : (f16) -> f32
1917/// %rhs_f32 = "arith.extf"(%arg2) : (f16) -> f32
1918/// %mul = "arith.mulf"(%lhs_f32, %rhs_f32) : (f32, f32) -> f32
1919/// %acc = "arith.addf"(%arg3, %mul) : (f32, f32) -> f32
1920/// "linalg.yield"(%acc) : (f32) -> ()
1921/// })
1922/// ```
1923/// This restricts the native usage of mixed precision NVIDIA Ampere Tensor
1924/// Cores, i.e, `mma.sync.*.f32.f16.f16.f32` and `mma.sync.*.f32.bf16.bf16.f32`.
1925/// This pattern folds the arithmetic extensions into the vector contraction and
1926/// enables the usage of native mixed precision Tensor Core instructions.
1927template <typename ExtOp>
1928struct FoldArithExtIntoContractionOp
1929 : public OpRewritePattern<vector::ContractionOp> {
1930 using Base::Base;
1931
1932 LogicalResult matchAndRewrite(vector::ContractionOp contractOp,
1933 PatternRewriter &rewriter) const override {
1934
1935 auto lhsDefOp = contractOp.getLhs().getDefiningOp<ExtOp>();
1936 auto rhsDefOp = contractOp.getRhs().getDefiningOp<ExtOp>();
1937
1938 if (!lhsDefOp || !rhsDefOp) {
1939 return rewriter.notifyMatchFailure(arg&: contractOp,
1940 msg: "no defining op on contract operands");
1941 }
1942
1943 rewriter.replaceOpWithNewOp<vector::ContractionOp>(
1944 contractOp, lhsDefOp->getOperand(0), rhsDefOp->getOperand(0),
1945 contractOp.getAcc(), contractOp.getIndexingMapsAttr(),
1946 contractOp.getIteratorTypesAttr(), contractOp.getKind(),
1947 contractOp.getFastmath());
1948
1949 return success();
1950 }
1951};
1952
1953/// Pattern to fold chained reduction to a series of vector additions and a
1954/// final reduction. This form should require fewer subgroup operations.
1955///
1956/// ```mlir
1957/// %a = vector.reduction <add> %x, %acc
1958/// %b = vector.reduction <add> %y, %a
1959/// ==>
1960/// %a = arith.addf %x, %y
1961/// %b = vector.reduction <add> %a, %acc
1962/// ```
1963struct ChainedReduction final : OpRewritePattern<vector::ReductionOp> {
1964 using Base::Base;
1965
1966 LogicalResult matchAndRewrite(vector::ReductionOp op,
1967 PatternRewriter &rewriter) const override {
1968 // TODO: Handle other combining kinds.
1969 if (op.getKind() != vector::CombiningKind::ADD)
1970 return failure();
1971
1972 // Accumulator is optional.
1973 Value acc = op.getAcc();
1974 if (!acc)
1975 return failure();
1976
1977 if (!acc.getType().isIntOrFloat())
1978 return failure();
1979
1980 auto parentReduction = acc.getDefiningOp<vector::ReductionOp>();
1981 if (!parentReduction)
1982 return failure();
1983
1984 Location loc = op.getLoc();
1985 Value vAdd;
1986 if (isa<IntegerType>(Val: acc.getType())) {
1987 vAdd = rewriter.createOrFold<arith::AddIOp>(
1988 location: loc, args: parentReduction.getVector(), args: op.getVector());
1989 } else {
1990 vAdd = arith::AddFOp::create(builder&: rewriter, location: loc, lhs: parentReduction.getVector(),
1991 rhs: op.getVector());
1992 }
1993 rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, args: op.getKind(), args&: vAdd,
1994 args: parentReduction.getAcc());
1995 return success();
1996 }
1997};
1998
1999// Helper function dropping unit non-scalable dimension from a VectorType
2000// keeping at least 1 dimension to avoid generating 0-D vectors. Scalable unit
2001// dimensions are not dropped. Folding such dimensions would require "shifting"
2002// the scalable flag onto some other fixed-width dim (e.g. vector<[1]x4xf32> ->
2003// vector<[4]xf32>). This could be implemented in the future.
2004static VectorType dropNonScalableUnitDimFromType(VectorType inVecTy) {
2005 auto inVecShape = inVecTy.getShape();
2006 SmallVector<int64_t> newShape;
2007 SmallVector<bool> newScalableDims;
2008 for (auto [dim, isScalable] :
2009 llvm::zip_equal(t&: inVecShape, u: inVecTy.getScalableDims())) {
2010 if (dim == 1 && !isScalable)
2011 continue;
2012
2013 newShape.push_back(Elt: dim);
2014 newScalableDims.push_back(Elt: isScalable);
2015 }
2016 // All dims have been dropped, return vector<1xeType>.
2017 if (newShape.empty()) {
2018 newShape.push_back(Elt: 1);
2019 newScalableDims.push_back(Elt: false);
2020 }
2021
2022 return VectorType::get(shape: newShape, elementType: inVecTy.getElementType(), scalableDims: newScalableDims);
2023}
2024
2025/// For vectors with at least one unit dim, replaces:
2026/// elementwise(a, b)
2027/// with:
2028/// sc_a = shape_cast(a)
2029/// sc_b = shape_cast(b)
2030/// res = elementwise(sc_a, sc_b)
2031/// return shape_cast(res)
2032/// The newly inserted shape_cast Ops fold (before elementwise Op) and then
2033/// restore (after elementwise Op) the unit dim. Vectors `a` and `b` are
2034/// required to be rank > 1.
2035///
2036/// Ex:
2037/// %mul = arith.mulf %B_row, %A_row : vector<1x[4]xf32>
2038/// %cast = vector.shape_cast %mul : vector<1x[4]xf32> to vector<[4]xf32>
2039///
2040/// gets converted to:
2041///
2042/// %B_row_sc = vector.shape_cast %B_row : vector<1x[4]xf32> to vector<[4]xf32>
2043/// %A_row_sc = vector.shape_cast %A_row : vector<1x[4]xf32> to vector<[4]xf32>
2044/// %mul = arith.mulf %B_row_sc, %A_row_sc : vector<[4]xf32>
2045/// %cast_new = vector.shape_cast %mul : vector<[4]xf32> to vector<1x[4]xf32>
2046/// %cast = vector.shape_cast %cast_new : vector<1x[4]xf32> to vector<[4]xf32>
2047///
2048/// Patterns for folding shape_casts should instantly eliminate `%cast_new` and
2049/// `%cast`.
2050struct DropUnitDimFromElementwiseOps final
2051 : public OpTraitRewritePattern<OpTrait::Elementwise> {
2052 using Base::Base;
2053 LogicalResult matchAndRewrite(Operation *op,
2054 PatternRewriter &rewriter) const override {
2055 if (op->getNumResults() != 1 || op->getNumRegions() != 0)
2056 return failure();
2057
2058 auto resultVectorType = dyn_cast<VectorType>(Val: op->getResult(idx: 0).getType());
2059 if (!resultVectorType)
2060 return failure();
2061
2062 // Check the operand pre-conditions. For `Elementwise` ops all operands are
2063 // guaranteed to have identical shapes (with some exceptions such as
2064 // `arith.select`) and it suffices to only check one of them.
2065 auto sourceVectorType = dyn_cast<VectorType>(Val: op->getOperand(idx: 0).getType());
2066 if (!sourceVectorType)
2067 return failure();
2068 if (sourceVectorType.getRank() < 2)
2069 return failure();
2070
2071 SmallVector<Value> newOperands;
2072 auto loc = op->getLoc();
2073 for (auto operand : op->getOperands()) {
2074 auto opVectorType = cast<VectorType>(Val: operand.getType());
2075 auto newVType = dropNonScalableUnitDimFromType(inVecTy: opVectorType);
2076 if (newVType == opVectorType)
2077 return rewriter.notifyMatchFailure(arg&: op, msg: "No unit dimension to remove.");
2078
2079 auto opSC = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: newVType, source: operand);
2080 newOperands.push_back(Elt: opSC);
2081 }
2082
2083 VectorType newResultVectorType =
2084 dropNonScalableUnitDimFromType(inVecTy: resultVectorType);
2085 // Create an updated elementwise Op without unit dim.
2086 Operation *elementwiseOp =
2087 createWithProperties(builder&: rewriter, op, operands: newOperands, types: newResultVectorType);
2088
2089 // Restore the unit dim by applying vector.shape_cast to the result.
2090 rewriter.replaceOpWithNewOp<ShapeCastOp>(op, args&: resultVectorType,
2091 args: elementwiseOp->getResult(idx: 0));
2092
2093 return success();
2094 }
2095};
2096
2097/// A pattern to drop unit dims from vector.transpose.
2098///
2099/// Example:
2100///
2101/// BEFORE:
2102/// ```mlir
2103/// %transpose = vector.transpose %vector, [3, 0, 1, 2]
2104/// : vector<1x1x4x[4]xf32> to vector<[4]x1x1x4xf32>
2105/// ```
2106///
2107/// AFTER:
2108/// ```mlir
2109/// %dropDims = vector.shape_cast %vector
2110/// : vector<1x1x4x[4]xf32> to vector<4x[4]xf32>
2111/// %transpose = vector.transpose %0, [1, 0]
2112/// : vector<4x[4]xf32> to vector<[4]x4xf32>
2113/// %restoreDims = vector.shape_cast %transpose
2114/// : vector<[4]x4xf32> to vector<[4]x1x1x4xf32>
2115/// ```
2116struct DropUnitDimsFromTransposeOp final
2117 : OpRewritePattern<vector::TransposeOp> {
2118 using Base::Base;
2119
2120 LogicalResult matchAndRewrite(vector::TransposeOp op,
2121 PatternRewriter &rewriter) const override {
2122 VectorType sourceType = op.getSourceVectorType();
2123 VectorType sourceTypeWithoutUnitDims =
2124 dropNonScalableUnitDimFromType(inVecTy: sourceType);
2125
2126 if (sourceType == sourceTypeWithoutUnitDims)
2127 return failure();
2128
2129 // Construct a map from dimIdx -> number of dims dropped before dimIdx.
2130 auto sourceDims = llvm::to_vector(Range: vector::getDims(vType: sourceType));
2131 SmallVector<int64_t> droppedDimsBefore(sourceType.getRank());
2132 int64_t droppedDims = 0;
2133 for (auto [i, dim] : llvm::enumerate(First&: sourceDims)) {
2134 droppedDimsBefore[i] = droppedDims;
2135 if (dim == std::make_tuple(args: 1, args: false))
2136 ++droppedDims;
2137 }
2138
2139 // Drop unit dims from transpose permutation.
2140 ArrayRef<int64_t> perm = op.getPermutation();
2141 SmallVector<int64_t> newPerm;
2142 for (int64_t idx : perm) {
2143 if (sourceDims[idx] == std::make_tuple(args: 1, args: false))
2144 continue;
2145 newPerm.push_back(Elt: idx - droppedDimsBefore[idx]);
2146 }
2147
2148 // Fixup for `newPerm`. The `sourceTypeWithoutUnitDims` could be vector<1xT>
2149 // type when the dimensions are unit dimensions. In this case, the newPerm
2150 // should be [0].
2151 if (newPerm.empty()) {
2152 newPerm.push_back(Elt: 0);
2153 }
2154
2155 Location loc = op.getLoc();
2156 // Drop the unit dims via shape_cast.
2157 auto dropDimsShapeCast = vector::ShapeCastOp::create(
2158 builder&: rewriter, location: loc, result: sourceTypeWithoutUnitDims, source: op.getVector());
2159 // Create the new transpose.
2160 auto transposeWithoutUnitDims =
2161 vector::TransposeOp::create(builder&: rewriter, location: loc, vector: dropDimsShapeCast, permutation: newPerm);
2162 // Restore the unit dims via shape cast.
2163 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
2164 op, args: op.getResultVectorType(), args&: transposeWithoutUnitDims);
2165
2166 return success();
2167 }
2168};
2169
2170/// A pattern to drop unit dims from the iter_args of an scf.for.
2171///
2172/// Example:
2173///
2174/// BEFORE:
2175/// ```mlir
2176/// %res = scf.for ... iter_args(%iter = %init) -> vector<[4]x1x1x4xf32> {
2177/// ...
2178/// scf.yield %
2179/// }
2180/// ```
2181///
2182/// AFTER:
2183/// ```mlir
2184/// %drop = vector.shape_cast %init
2185/// : vector<4x1x1x[4]xf32> to vector<4x[4]xf32>
2186/// %new_loop = scf.for ... iter_args(%iter = %drop) -> vector<[4]x4xf32> {
2187/// %new_iter = vector.shape_cast %iter
2188/// : vector<[4]x4xf32> to vector<[4]x1x1x4xf32>
2189/// ...
2190/// }
2191/// %res = vector.shape_cast %new_loop
2192/// : vector<[4]x4xf32> to vector<[4]x1x1x4xf32>
2193/// ```
2194struct DropUnitDimsFromScfForOp final : OpRewritePattern<scf::ForOp> {
2195 using Base::Base;
2196
2197 LogicalResult matchAndRewrite(scf::ForOp forOp,
2198 PatternRewriter &rewriter) const override {
2199 /// Find the first iter_arg with droppable unit dims. Further applications
2200 /// of this pattern will apply to later arguments.
2201 for (OpOperand &operand : forOp.getInitArgsMutable()) {
2202 auto vectorType = dyn_cast<VectorType>(Val: operand.get().getType());
2203 if (!vectorType)
2204 continue;
2205
2206 VectorType newVectorType = dropNonScalableUnitDimFromType(inVecTy: vectorType);
2207 if (vectorType == newVectorType)
2208 continue;
2209
2210 // Create a new ForOp with that iter operand replaced.
2211 auto castFn = [](OpBuilder &b, Location loc, Type type, Value source) {
2212 return vector::ShapeCastOp::create(builder&: b, location: loc, result: type, source);
2213 };
2214
2215 Value replacement =
2216 castFn(rewriter, forOp.getLoc(), newVectorType, operand.get());
2217 rewriter.replaceOp(op: forOp,
2218 newValues: replaceAndCastForOpIterArg(rewriter, forOp, operand,
2219 replacement, castFn));
2220 return success();
2221 }
2222 return failure();
2223 }
2224};
2225
2226/// Pattern to eliminate redundant zero-constants added to reduction operands.
2227/// It's enough for there to be one initial zero value, so we can eliminate the
2228/// extra ones that feed into `vector.reduction <add>`. These get created by the
2229/// `ChainedReduction` pattern.
2230///
2231/// ```mlir
2232/// %a = arith.addf %x, %zero
2233/// %b = arith.addf %a, %y
2234/// %c = vector.reduction <add> %b, %acc
2235/// ==>
2236/// %b = arith.addf %a, %y
2237/// %c = vector.reduction <add> %b, %acc
2238/// ```
2239struct ReduceRedundantZero final : OpRewritePattern<vector::ReductionOp> {
2240 using Base::Base;
2241
2242 LogicalResult matchAndRewrite(vector::ReductionOp op,
2243 PatternRewriter &rewriter) const override {
2244 // TODO: Handle other reduction kinds and their identity values.
2245 if (op.getKind() != vector::CombiningKind::ADD)
2246 return failure();
2247
2248 Type elemType = op.getSourceVectorType().getElementType();
2249 // The integer case should be handled by `arith.addi` folders, only check
2250 // for floats here.
2251 if (!isa<FloatType>(Val: elemType))
2252 return failure();
2253
2254 auto vAdd = op.getVector().getDefiningOp<arith::AddFOp>();
2255 if (!vAdd)
2256 return failure();
2257 auto addLhs = vAdd.getLhs().getDefiningOp<arith::AddFOp>();
2258 if (!addLhs)
2259 return failure();
2260
2261 if (!matchPattern(value: addLhs.getRhs(), pattern: m_AnyZeroFloat()))
2262 return failure();
2263
2264 auto newAdd = arith::AddFOp::create(builder&: rewriter, location: vAdd.getLoc(),
2265 lhs: addLhs.getLhs(), rhs: vAdd.getRhs());
2266 rewriter.replaceOpWithNewOp<vector::ReductionOp>(op, args: op.getKind(), args&: newAdd,
2267 args: op.getAcc());
2268 return success();
2269 }
2270};
2271
2272/// Example:
2273/// ```
2274/// %a = vector.reduction <add> %x : vector<2xf32> into f32
2275/// ```
2276/// is transformed into:
2277/// ```
2278/// %y = vector.extract %x[0] : f32 from vector<2xf32>
2279/// %z = vector.extract %x[1] : f32 from vector<2xf32>
2280/// %a = arith.addf %y, %z : f32
2281/// ```
2282struct BreakDownVectorReduction final : OpRewritePattern<vector::ReductionOp> {
2283 BreakDownVectorReduction(MLIRContext *context,
2284 unsigned maxNumElementsToExtract,
2285 PatternBenefit benefit)
2286 : OpRewritePattern(context, benefit),
2287 maxNumElementsToExtract(maxNumElementsToExtract) {}
2288
2289 LogicalResult matchAndRewrite(vector::ReductionOp op,
2290 PatternRewriter &rewriter) const override {
2291 VectorType type = op.getSourceVectorType();
2292 if (type.isScalable() || op.isMasked())
2293 return failure();
2294 assert(type.getRank() == 1 && "Expected a 1-d vector");
2295
2296 int64_t numElems = type.getNumElements();
2297 if (numElems > maxNumElementsToExtract) {
2298 return rewriter.notifyMatchFailure(
2299 arg&: op, msg: llvm::formatv(Fmt: "has too many vector elements ({0}) to break down "
2300 "(max allowed: {1})",
2301 Vals&: numElems, Vals: maxNumElementsToExtract));
2302 }
2303
2304 Location loc = op.getLoc();
2305 SmallVector<Value> extracted(numElems, nullptr);
2306 for (auto [idx, extractedElem] : llvm::enumerate(First&: extracted))
2307 extractedElem = vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getVector(),
2308 position: static_cast<int64_t>(idx));
2309
2310 Value res = extracted.front();
2311 for (auto extractedElem : llvm::drop_begin(RangeOrContainer&: extracted))
2312 res = vector::makeArithReduction(b&: rewriter, loc, kind: op.getKind(), v1: res,
2313 acc: extractedElem, fastmath: op.getFastmathAttr());
2314 if (Value acc = op.getAcc())
2315 res = vector::makeArithReduction(b&: rewriter, loc, kind: op.getKind(), v1: res, acc,
2316 fastmath: op.getFastmathAttr());
2317
2318 rewriter.replaceOp(op, newValues: res);
2319 return success();
2320 }
2321
2322private:
2323 unsigned maxNumElementsToExtract = 0;
2324};
2325
2326/// Fold `mulf(tr(broadcast(A)), broadcast(B))` into `vector.outerproduct(A,
2327/// B)`.
2328/// Example:
2329/// %lhsBcast = vector.broadcast %lhs : vector<4xi32> to vector<4x4xi32>
2330/// %lhsT = vector.transpose %lhsBcast, [1, 0] : vector<4x4xi32> to
2331/// vector<4x4xi32> %rhsBcast = vector.broadcast %rhs : vector<4xi32> to
2332/// vector<4x4xi32> %mul = arith.muli %lhsT, %rhsBcast : vector<4x4xi32>
2333///
2334/// Becomes :
2335///
2336/// %res = vector.outerproduct %lhs, %rhs : vector<4xi32>, vector<4xi32>
2337///
2338/// Supports only 1D-to-2D broadcasts. The following cases are not supported.
2339/// %ex1 = vector.broadcast %lhsCast : vector<1x4xf32> to vector<4x4xf32>
2340/// %ex2 = vector.broadcast %lhsCast : f32 to vector<4x4xf32>
2341/// %ex3 = vector.broadcast %lhsCast : vector<1x1xf32> to vector<4x4xf32>
2342template <typename MulOpType>
2343struct FoldArithToVectorOuterProduct : public OpRewritePattern<MulOpType> {
2344 using OpRewritePattern<MulOpType>::OpRewritePattern;
2345 // Returns whether a vector.broadcast matches requirements for an outerproduct
2346 // pattern. aka a 1D-to-2D broadcastOp without broadcasted unit dimension.
2347 bool isValidBroadcastSource(vector::BroadcastOp broadcastOp) const {
2348 // Fail if it is not a 1-to-2 dimension to broadcast to avoid generating
2349 // shape_casts/broadcasts which does not belong in this pattern.
2350 if (!broadcastOp.computeBroadcastedUnitDims().empty())
2351 return false;
2352 // Avoid broadcast like f32 or vector<f32> -> ResType
2353 auto srcType = dyn_cast<VectorType>(Val: broadcastOp.getSourceType());
2354 return srcType && srcType.getRank() != 2;
2355 }
2356
2357 LogicalResult matchAndRewrite(MulOpType mulOp,
2358 PatternRewriter &rewriter) const override {
2359 auto resType = llvm::dyn_cast<VectorType>(mulOp.getResult().getType());
2360 if (!resType)
2361 return failure();
2362 if (resType.getRank() != 2)
2363 return failure();
2364 /// If operandA can be written as tr(broadcast(A)) and operandB as
2365 /// broadcast(B) where broadcasts are 1D-to-2D, create and return
2366 /// vector.outerproduct(A, B). Returns failure() otherwise.
2367 auto matchOuterProduct =
2368 [&](Value operandA,
2369 Value operandB) -> FailureOr<vector::OuterProductOp> {
2370 auto transposedLhs = operandA.getDefiningOp<vector::TransposeOp>();
2371 if (!transposedLhs)
2372 return failure();
2373 // Fail unless this is a true 2-D matrix transpose.
2374 ArrayRef<int64_t> permutation = transposedLhs.getPermutation();
2375 if (permutation.size() != 2 || permutation[0] != 1 || permutation[1] != 0)
2376 return failure();
2377
2378 auto broadcastedLhs =
2379 transposedLhs.getVector().getDefiningOp<vector::BroadcastOp>();
2380 if (!broadcastedLhs || !isValidBroadcastSource(broadcastOp: broadcastedLhs))
2381 return failure();
2382
2383 auto broadcastedRhs = operandB.getDefiningOp<vector::BroadcastOp>();
2384 if (!broadcastedRhs || !isValidBroadcastSource(broadcastOp: broadcastedRhs))
2385 return failure();
2386
2387 return vector::OuterProductOp::create(
2388 rewriter, mulOp->getLoc(), resType, broadcastedLhs.getSource(),
2389 broadcastedRhs.getSource(), Value(), vector::CombiningKind::ADD);
2390 };
2391
2392 Value lhs = mulOp->getOperand(0), rhs = mulOp->getOperand(1);
2393 auto maybeOuterP = matchOuterProduct(lhs, rhs);
2394 // Handle commutativity, the transposed op is the outerproduct LHS.
2395 if (failed(maybeOuterP))
2396 maybeOuterP = matchOuterProduct(rhs, lhs);
2397 if (failed(maybeOuterP))
2398 return failure();
2399 rewriter.replaceOp(mulOp, maybeOuterP->getResult());
2400 return success();
2401 }
2402};
2403
2404} // namespace
2405
2406void mlir::vector::populateFoldArithExtensionPatterns(
2407 RewritePatternSet &patterns) {
2408 patterns.add<FoldArithExtIntoContractionOp<arith::ExtFOp>,
2409 FoldArithExtIntoContractionOp<arith::ExtSIOp>>(
2410 arg: patterns.getContext());
2411}
2412
2413void mlir::vector::populateVectorMaskMaterializationPatterns(
2414 RewritePatternSet &patterns, bool force32BitVectorIndices,
2415 PatternBenefit benefit) {
2416 patterns.add<VectorCreateMaskOpConversion,
2417 MaterializeTransferMask<vector::TransferReadOp>,
2418 MaterializeTransferMask<vector::TransferWriteOp>>(
2419 arg: patterns.getContext(), args&: force32BitVectorIndices, args&: benefit);
2420 patterns.add<FoldI1Select>(arg: patterns.getContext(), args&: benefit);
2421}
2422
2423void mlir::vector::populateDropUnitDimWithShapeCastPatterns(
2424 RewritePatternSet &patterns, PatternBenefit benefit) {
2425 patterns.add<DropUnitDimFromElementwiseOps, DropUnitDimsFromScfForOp,
2426 DropUnitDimsFromTransposeOp>(arg: patterns.getContext(), args&: benefit);
2427}
2428
2429void mlir::vector::populateBubbleVectorBitCastOpPatterns(
2430 RewritePatternSet &patterns, PatternBenefit benefit) {
2431 patterns.add<BubbleDownVectorBitCastForExtract,
2432 BubbleDownBitCastForStridedSliceExtract,
2433 BubbleUpBitCastForInsert, BubbleUpBitCastForStridedSliceInsert>(
2434 arg: patterns.getContext(), args&: benefit);
2435}
2436
2437void mlir::vector::populateBreakDownVectorBitCastOpPatterns(
2438 RewritePatternSet &patterns,
2439 std::function<bool(vector::BitCastOp)> controlFn, PatternBenefit benefit) {
2440 patterns.add<BreakDownVectorBitCast>(arg: patterns.getContext(),
2441 args: std::move(controlFn), args&: benefit);
2442}
2443
2444void mlir::vector::populateVectorContractCanonicalizeMatmulToMMT(
2445 RewritePatternSet &patterns,
2446 std::function<LogicalResult(vector::ContractionOp)> constraint,
2447 PatternBenefit benefit) {
2448 patterns.add<CanonicalizeContractMatmulToMMT>(arg: patterns.getContext(), args&: benefit,
2449 args: std::move(constraint));
2450}
2451
2452void mlir::vector::populateVectorReductionToContractPatterns(
2453 RewritePatternSet &patterns, PatternBenefit benefit,
2454 bool allowFastMathFlagLoss) {
2455 patterns.add<MultiReduceToContract>(arg: patterns.getContext(),
2456 args&: allowFastMathFlagLoss, args&: benefit);
2457 patterns.add<CombineContractBroadcastMask, CombineContractABTranspose,
2458 CombineContractResultTranspose>(arg: patterns.getContext(), args&: benefit);
2459}
2460
2461void mlir::vector::populateDropInnerMostUnitDimsXferOpPatterns(
2462 RewritePatternSet &patterns, PatternBenefit benefit) {
2463 patterns.add<DropInnerMostUnitDimsTransferRead,
2464 DropInnerMostUnitDimsTransferWrite>(arg: patterns.getContext(),
2465 args&: benefit);
2466}
2467
2468void mlir::vector::populateSinkVectorOpsPatterns(RewritePatternSet &patterns,
2469 PatternBenefit benefit) {
2470 patterns.add<ReorderElementwiseOpsOnTranspose, ReorderCastOpsOnBroadcast,
2471 ReorderElementwiseOpsOnBroadcast, ExtractOpFromElementwise>(
2472 arg: patterns.getContext(), args&: benefit);
2473}
2474
2475void mlir::vector::populateSinkVectorMemOpsPatterns(RewritePatternSet &patterns,
2476 PatternBenefit benefit) {
2477 // TODO: Consider converting these patterns to canonicalizations.
2478 patterns.add<ExtractOpFromLoad, StoreOpFromBroadcast>(arg: patterns.getContext(),
2479 args&: benefit);
2480}
2481
2482void mlir::vector::populateChainedVectorReductionFoldingPatterns(
2483 RewritePatternSet &patterns, PatternBenefit benefit) {
2484 patterns.add<ChainedReduction>(arg: patterns.getContext(), args&: benefit);
2485 patterns.add<ReduceRedundantZero>(arg: patterns.getContext(),
2486 args: PatternBenefit(benefit.getBenefit() + 1));
2487}
2488
2489void mlir::vector::populateBreakDownVectorReductionPatterns(
2490 RewritePatternSet &patterns, unsigned maxNumElementsToExtract,
2491 PatternBenefit benefit) {
2492 patterns.add<BreakDownVectorReduction>(arg: patterns.getContext(),
2493 args&: maxNumElementsToExtract, args&: benefit);
2494}
2495
2496void mlir::vector::populateElementwiseToVectorOpsPatterns(
2497 RewritePatternSet &patterns) {
2498 patterns.add<FoldArithToVectorOuterProduct<arith::MulFOp>,
2499 FoldArithToVectorOuterProduct<arith::MulIOp>>(
2500 arg: patterns.getContext());
2501}
2502
2503//===----------------------------------------------------------------------===//
2504// TableGen'd enum attribute definitions
2505//===----------------------------------------------------------------------===//
2506
2507#include "mlir/Dialect/Vector/Transforms/VectorTransformsEnums.cpp.inc"
2508