1//===- VectorLinearize.cpp - vector linearization transforms --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements patterns and pass for linearizing ND vectors into 1D.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/UB/IR/UBOps.h"
14#include "mlir/Dialect/Vector/IR/VectorOps.h"
15#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
16#include "mlir/IR/Attributes.h"
17#include "mlir/IR/BuiltinAttributes.h"
18#include "mlir/IR/Matchers.h"
19#include "mlir/IR/Operation.h"
20#include "mlir/IR/PatternMatch.h"
21#include "mlir/IR/TypeUtilities.h"
22#include "mlir/Transforms/DialectConversion.h"
23#include "llvm/ADT/ArrayRef.h"
24#include <cstdint>
25#include <limits>
26#include <numeric>
27#include <optional>
28
29using namespace mlir;
30
31static FailureOr<Attribute>
32linearizeConstAttr(Location loc, ConversionPatternRewriter &rewriter,
33 VectorType resType, Attribute value) {
34
35 if (auto dstElementsAttr = dyn_cast<DenseElementsAttr>(Val&: value)) {
36 if (resType.isScalable() && !isa<SplatElementsAttr>(Val: value))
37 return rewriter.notifyMatchFailure(
38 arg&: loc,
39 msg: "Cannot linearize a constant scalable vector that's not a splat");
40
41 return dstElementsAttr.reshape(newType: resType);
42 }
43
44 if (auto poisonAttr = dyn_cast<ub::PoisonAttr>(Val&: value))
45 return poisonAttr;
46
47 return rewriter.notifyMatchFailure(arg&: loc, msg: "unsupported attr type");
48}
49
50namespace {
51
52struct LinearizeConstantLike final
53 : OpTraitConversionPattern<OpTrait::ConstantLike> {
54 using Base::Base;
55
56 LinearizeConstantLike(const TypeConverter &typeConverter,
57 MLIRContext *context, PatternBenefit benefit = 1)
58 : OpTraitConversionPattern(typeConverter, context, benefit) {}
59 LogicalResult
60 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
61 ConversionPatternRewriter &rewriter) const override {
62 Location loc = op->getLoc();
63 if (op->getNumResults() != 1)
64 return rewriter.notifyMatchFailure(arg&: loc, msg: "expected 1 result");
65
66 const TypeConverter &typeConverter = *getTypeConverter();
67 auto resType =
68 typeConverter.convertType<VectorType>(t: op->getResult(idx: 0).getType());
69 assert(resType && "expected 1-D vector type");
70
71 Attribute value;
72 if (!matchPattern(op, pattern: m_Constant(bind_value: &value)))
73 return rewriter.notifyMatchFailure(arg&: loc,
74 msg: "could not fold constant-like op");
75
76 FailureOr<Attribute> newValue =
77 linearizeConstAttr(loc, rewriter, resType, value);
78 if (failed(Result: newValue))
79 return failure();
80
81 Operation *newOp = op->getDialect()->materializeConstant(
82 builder&: rewriter, value: *newValue, type: resType, loc);
83 if (!newOp)
84 return rewriter.notifyMatchFailure(
85 arg&: loc, msg: "could not materialize linearized constant");
86 newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary());
87 rewriter.replaceOp(op, newOp);
88 return success();
89 }
90};
91
92struct LinearizeVectorizable final
93 : OpTraitConversionPattern<OpTrait::Vectorizable> {
94 using Base::Base;
95
96public:
97 LinearizeVectorizable(const TypeConverter &typeConverter,
98 MLIRContext *context, PatternBenefit benefit = 1)
99 : OpTraitConversionPattern(typeConverter, context, benefit) {}
100 LogicalResult
101 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
102 ConversionPatternRewriter &rewriter) const override {
103 FailureOr<Operation *> newOp =
104 convertOpResultTypes(op, operands, converter: *getTypeConverter(), rewriter);
105 if (failed(Result: newOp))
106 return failure();
107
108 rewriter.replaceOp(op, newValues: (*newOp)->getResults());
109 return success();
110 }
111};
112
113template <typename TOp>
114static bool stridesAllOne(TOp op) {
115 static_assert(
116 std::is_same_v<TOp, vector::ExtractStridedSliceOp> ||
117 std::is_same_v<TOp, vector::InsertStridedSliceOp>,
118 "expected vector.extract_strided_slice or vector.insert_strided_slice");
119 ArrayAttr strides = op.getStrides();
120 return llvm::all_of(Range&: strides, P: isOneInteger);
121}
122
123/// Convert an array of attributes into a vector of integers, if possible.
124static FailureOr<SmallVector<int64_t>> intsFromArrayAttr(ArrayAttr attrs) {
125 if (!attrs)
126 return failure();
127 SmallVector<int64_t> ints;
128 ints.reserve(N: attrs.size());
129 for (auto attr : attrs) {
130 if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) {
131 ints.push_back(Elt: intAttr.getInt());
132 } else {
133 return failure();
134 }
135 }
136 return ints;
137}
138
139/// Consider inserting a vector of shape `small` into a vector of shape `large`,
140/// at position `offsets`: this function enumeratates all the indices in `large`
141/// that are written to. The enumeration is with row-major ordering.
142///
143/// Example: insert a 1x2 vector into a 4x5 vector at position (1,3). The 2
144/// positions written to are (1,3) and (1,4), which have linearized indices 8
145/// and 9. So [8,9] is returned.
146///
147/// The length of the returned vector is equal to the number of elements in
148/// the shape `small` (i.e. the product of dimensions of `small`).
149SmallVector<int64_t> static getStridedSliceInsertionIndices(
150 ArrayRef<int64_t> small, ArrayRef<int64_t> large,
151 ArrayRef<int64_t> offsets) {
152
153 // Example of alignment between, `large`, `small` and `offsets`:
154 // large = 4, 5, 6, 7, 8
155 // small = 1, 6, 7, 8
156 // offsets = 2, 3, 0
157 //
158 // `offsets` has implicit trailing 0s, `small` has implicit leading 1s.
159 assert((large.size() >= small.size()) &&
160 "rank of 'large' cannot be lower than rank of 'small'");
161 assert((large.size() >= offsets.size()) &&
162 "rank of 'large' cannot be lower than the number of offsets");
163 unsigned delta = large.size() - small.size();
164 unsigned nOffsets = offsets.size();
165 auto getSmall = [&](int64_t i) -> int64_t {
166 return i >= delta ? small[i - delta] : 1;
167 };
168 auto getOffset = [&](int64_t i) -> int64_t {
169 return i < nOffsets ? offsets[i] : 0;
170 };
171
172 // Using 2 vectors of indices, at each iteration populate the updated set of
173 // indices based on the old set of indices, and the size of the small vector
174 // in the current iteration.
175 SmallVector<int64_t> indices{0};
176 int64_t stride = 1;
177 for (int i = large.size() - 1; i >= 0; --i) {
178 int64_t currentSize = indices.size();
179 int64_t smallSize = getSmall(i);
180 int64_t nextSize = currentSize * smallSize;
181 SmallVector<int64_t> nextIndices(nextSize);
182 int64_t *base = nextIndices.begin();
183 int64_t offset = getOffset(i) * stride;
184 for (int j = 0; j < smallSize; ++j) {
185 for (int k = 0; k < currentSize; ++k) {
186 base[k] = indices[k] + offset;
187 }
188 offset += stride;
189 base += currentSize;
190 }
191 stride *= large[i];
192 indices = std::move(nextIndices);
193 }
194 return indices;
195}
196
197/// This pattern converts a vector.extract_strided_slice operation into a
198/// vector.shuffle operation that has a rank-1 (linearized) operand and result.
199///
200/// For example, the following:
201///
202/// ```
203/// vector.extract_strided_slice %source
204/// { offsets = [..], strides = [..], sizes = [..] }
205/// ```
206///
207/// is converted to :
208/// ```
209/// %source_1d = vector.shape_cast %source
210/// %out_1d = vector.shuffle %source_1d, %source_1d [ shuffle_indices_1d ]
211/// %out_nd = vector.shape_cast %out_1d
212/// ```
213///
214/// `shuffle_indices_1d` is computed using the offsets and sizes of the original
215/// vector.extract_strided_slice operation.
216struct LinearizeVectorExtractStridedSlice final
217 : public mlir::OpConversionPattern<mlir::vector::ExtractStridedSliceOp> {
218 using Base::Base;
219 LinearizeVectorExtractStridedSlice(const TypeConverter &typeConverter,
220 MLIRContext *context,
221 PatternBenefit benefit = 1)
222 : OpConversionPattern(typeConverter, context, benefit) {}
223
224 LogicalResult
225 matchAndRewrite(vector::ExtractStridedSliceOp extractStridedSliceOp,
226 OpAdaptor adaptor,
227 ConversionPatternRewriter &rewriter) const override {
228
229 VectorType flatOutputType = getTypeConverter()->convertType<VectorType>(
230 t: extractStridedSliceOp.getType());
231 assert(flatOutputType && "vector type expected");
232
233 // Expect a legalization failure if the strides are not all 1 (if ever the
234 // verifier for extract_strided_slice allows non-1 strides).
235 if (!stridesAllOne(op: extractStridedSliceOp)) {
236 return rewriter.notifyMatchFailure(
237 arg&: extractStridedSliceOp,
238 msg: "extract_strided_slice with strides != 1 not supported");
239 }
240
241 FailureOr<SmallVector<int64_t>> offsets =
242 intsFromArrayAttr(attrs: extractStridedSliceOp.getOffsets());
243 if (failed(Result: offsets)) {
244 return rewriter.notifyMatchFailure(arg&: extractStridedSliceOp,
245 msg: "failed to get integer offsets");
246 }
247
248 ArrayRef<int64_t> inputShape =
249 extractStridedSliceOp.getSourceVectorType().getShape();
250
251 ArrayRef<int64_t> outputShape = extractStridedSliceOp.getType().getShape();
252
253 SmallVector<int64_t> indices = getStridedSliceInsertionIndices(
254 small: outputShape, large: inputShape, offsets: offsets.value());
255
256 Value srcVector = adaptor.getSource();
257 rewriter.replaceOpWithNewOp<vector::ShuffleOp>(
258 op: extractStridedSliceOp, args&: flatOutputType, args&: srcVector, args&: srcVector, args&: indices);
259 return success();
260 }
261};
262
263/// This pattern converts a vector.insert_strided_slice operation into a
264/// vector.shuffle operation that has rank-1 (linearized) operands and result.
265///
266/// For example, the following:
267/// ```
268/// %0 = vector.insert_strided_slice %to_store, %into
269/// {offsets = [1, 0, 0, 0], strides = [1, 1]}
270/// : vector<2x2xi8> into vector<2x1x3x2xi8>
271/// ```
272///
273/// is converted to
274/// ```
275/// %to_store_1d
276/// = vector.shape_cast %to_store : vector<2x2xi8> to vector<4xi8>
277/// %into_1d = vector.shape_cast %into : vector<2x1x3x2xi8> to vector<12xi8>
278/// %out_1d = vector.shuffle %into_1d, %to_store_1d [ shuffle_indices_1d ]
279/// %out_nd = vector.shape_cast %out_1d : vector<12xi8> to vector<2x1x3x2xi8>
280/// ```
281///
282/// where shuffle_indices_1d in this case is
283/// [0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 10, 11].
284/// ^^^^^^^^^^^^^^
285/// to_store_1d
286///
287struct LinearizeVectorInsertStridedSlice final
288 : public mlir::OpConversionPattern<mlir::vector::InsertStridedSliceOp> {
289 using Base::Base;
290 LinearizeVectorInsertStridedSlice(const TypeConverter &typeConverter,
291 MLIRContext *context,
292 PatternBenefit benefit = 1)
293 : OpConversionPattern(typeConverter, context, benefit) {}
294
295 LogicalResult
296 matchAndRewrite(vector::InsertStridedSliceOp insertStridedSliceOp,
297 OpAdaptor adaptor,
298 ConversionPatternRewriter &rewriter) const override {
299
300 // Expect a legalization failure if the strides are not all 1 (if ever the
301 // verifier for insert_strided_slice allows non-1 strides).
302 if (!stridesAllOne(op: insertStridedSliceOp)) {
303 return rewriter.notifyMatchFailure(
304 arg&: insertStridedSliceOp,
305 msg: "insert_strided_slice with strides != 1 not supported");
306 }
307
308 VectorType inputType = insertStridedSliceOp.getValueToStore().getType();
309 ArrayRef<int64_t> inputShape = inputType.getShape();
310
311 VectorType outputType = insertStridedSliceOp.getType();
312 ArrayRef<int64_t> outputShape = outputType.getShape();
313 int64_t nOutputElements = outputType.getNumElements();
314
315 FailureOr<SmallVector<int64_t>> offsets =
316 intsFromArrayAttr(attrs: insertStridedSliceOp.getOffsets());
317 if (failed(Result: offsets)) {
318 return rewriter.notifyMatchFailure(arg&: insertStridedSliceOp,
319 msg: "failed to get integer offsets");
320 }
321 SmallVector<int64_t> sliceIndices = getStridedSliceInsertionIndices(
322 small: inputShape, large: outputShape, offsets: offsets.value());
323
324 SmallVector<int64_t> indices(nOutputElements);
325 std::iota(first: indices.begin(), last: indices.end(), value: 0);
326 for (auto [index, sliceIndex] : llvm::enumerate(First&: sliceIndices)) {
327 indices[sliceIndex] = index + nOutputElements;
328 }
329
330 Value flatToStore = adaptor.getValueToStore();
331 Value flatDest = adaptor.getDest();
332 rewriter.replaceOpWithNewOp<vector::ShuffleOp>(op: insertStridedSliceOp,
333 args: flatDest.getType(), args&: flatDest,
334 args&: flatToStore, args&: indices);
335 return success();
336 }
337};
338
339/// This pattern converts the ShuffleOp that works on nD (n > 1)
340/// vectors to a ShuffleOp that works on linearized vectors.
341/// Following,
342/// vector.shuffle %v1, %v2 [ shuffle_indices ]
343/// is converted to :
344/// %v1_1d = vector.shape_cast %v1
345/// %v2_1d = vector.shape_cast %v2
346/// %out_1d = vector.shuffle %v1_1d, %v2_1d [ shuffle_indices_1d ]
347/// %out_nd = vector.shape_cast %out_1d
348// `shuffle_indices_1d` is computed using the sizes and `shuffle_indices`
349/// of the original shuffle operation.
350struct LinearizeVectorShuffle final
351 : public OpConversionPattern<vector::ShuffleOp> {
352 using Base::Base;
353 LinearizeVectorShuffle(const TypeConverter &typeConverter,
354 MLIRContext *context, PatternBenefit benefit = 1)
355 : OpConversionPattern(typeConverter, context, benefit) {}
356
357 LogicalResult
358 matchAndRewrite(vector::ShuffleOp shuffleOp, OpAdaptor adaptor,
359 ConversionPatternRewriter &rewriter) const override {
360 VectorType dstType =
361 getTypeConverter()->convertType<VectorType>(t: shuffleOp.getType());
362 assert(dstType && "vector type destination expected.");
363
364 Value vec1 = adaptor.getV1();
365 Value vec2 = adaptor.getV2();
366 int shuffleSliceLen = 1;
367 int rank = shuffleOp.getV1().getType().getRank();
368
369 // If rank > 1, we need to do the shuffle in the granularity of slices
370 // instead of scalars. Size of the slice is equal to the rank-1 innermost
371 // dims. Mask of the shuffle op specifies which slice to take from the
372 // outermost dim.
373 if (rank > 1) {
374 llvm::ArrayRef<int64_t> shape = shuffleOp.getV1().getType().getShape();
375 for (unsigned i = 1; i < shape.size(); ++i) {
376 shuffleSliceLen *= shape[i];
377 }
378 }
379
380 // For each value in the mask, we generate the indices of the source vectors
381 // that need to be shuffled to the destination vector. If shuffleSliceLen >
382 // 1 we need to shuffle the slices (consecutive shuffleSliceLen number of
383 // elements) instead of scalars.
384 ArrayRef<int64_t> mask = shuffleOp.getMask();
385 int64_t totalSizeOfShuffledElmnts = mask.size() * shuffleSliceLen;
386 llvm::SmallVector<int64_t, 2> indices(totalSizeOfShuffledElmnts);
387 for (auto [i, value] : llvm::enumerate(First&: mask)) {
388 std::iota(first: indices.begin() + shuffleSliceLen * i,
389 last: indices.begin() + shuffleSliceLen * (i + 1),
390 value: shuffleSliceLen * value);
391 }
392
393 rewriter.replaceOpWithNewOp<vector::ShuffleOp>(op: shuffleOp, args&: dstType, args&: vec1,
394 args&: vec2, args&: indices);
395 return success();
396 }
397};
398
399/// This pattern linearizes `vector.extract` operations. It generates a 1-D
400/// version of the `vector.extract` operation when extracting a scalar from a
401/// vector. It generates a 1-D `vector.shuffle` operation when extracting a
402/// subvector from a larger vector.
403///
404/// Example #1:
405///
406/// %0 = vector.extract %arg0[1]: vector<8x2xf32> from vector<2x8x2xf32>
407///
408/// is converted to:
409///
410/// %0 = vector.shape_cast %arg0 : vector<2x8x2xf32> to vector<32xf32>
411/// %1 = vector.shuffle %0, %0 [16, 17, 18, 19, 20, 21, 22, 23,
412/// 24, 25, 26, 27, 28, 29, 30, 31] :
413/// vector<32xf32>, vector<32xf32>
414/// %2 = vector.shape_cast %1 : vector<16xf32> to vector<8x2xf32>
415///
416/// Example #2:
417///
418/// %0 = vector.extract %arg0[1, 2] : i32 from vector<2x4xi32>
419///
420/// is converted to:
421///
422/// %0 = vector.shape_cast %arg0 : vector<2x4xi32> to vector<8xi32>
423/// %1 = vector.extract %0[6] : i32 from vector<8xi32>
424///
425struct LinearizeVectorExtract final
426 : public OpConversionPattern<vector::ExtractOp> {
427 using Base::Base;
428 LinearizeVectorExtract(const TypeConverter &typeConverter,
429 MLIRContext *context, PatternBenefit benefit = 1)
430 : OpConversionPattern(typeConverter, context, benefit) {}
431 LogicalResult
432 matchAndRewrite(vector::ExtractOp extractOp, OpAdaptor adaptor,
433 ConversionPatternRewriter &rewriter) const override {
434 Type dstTy = getTypeConverter()->convertType(t: extractOp.getType());
435 assert(dstTy && "expected 1-D vector type");
436
437 // Dynamic position is not supported.
438 if (extractOp.hasDynamicPosition())
439 return rewriter.notifyMatchFailure(arg&: extractOp,
440 msg: "dynamic position is not supported.");
441
442 llvm::ArrayRef<int64_t> shape = extractOp.getSource().getType().getShape();
443 int64_t size = extractOp.getSource().getType().getNumElements();
444
445 // Compute linearized offset.
446 int64_t linearizedOffset = 0;
447 llvm::ArrayRef<int64_t> offsets = extractOp.getStaticPosition();
448 for (auto [i, off] : llvm::enumerate(First&: offsets)) {
449 size /= shape[i];
450 linearizedOffset += offsets[i] * size;
451 }
452
453 Value srcVector = adaptor.getSource();
454 if (!isa<VectorType>(Val: extractOp.getType())) {
455 // Scalar case: generate a 1-D extract.
456 Value result = rewriter.createOrFold<vector::ExtractOp>(
457 location: extractOp.getLoc(), args&: srcVector, args&: linearizedOffset);
458 rewriter.replaceOp(op: extractOp, newValues: result);
459 return success();
460 }
461
462 // Vector case: generate a shuffle.
463
464 llvm::SmallVector<int64_t, 2> indices(size);
465 std::iota(first: indices.begin(), last: indices.end(), value: linearizedOffset);
466 rewriter.replaceOpWithNewOp<vector::ShuffleOp>(op: extractOp, args&: dstTy, args&: srcVector,
467 args&: srcVector, args&: indices);
468
469 return success();
470 }
471};
472
473/// This pattern linearizes `vector.insert` operations. It generates a 1-D
474/// version of the `vector.insert` operation when inserting a scalar into a
475/// vector. It generates a 1-D `vector.shuffle` operation when inserting a
476/// vector into another vector.
477///
478/// Example #1:
479///
480/// %0 = vector.insert %source, %destination[0] :
481/// vector<2x4xf32> into vector<2x2x4xf32>
482///
483/// is converted to:
484///
485/// %0 = vector.shape_cast %source : vector<2x4xf32> to vector<8xf32>
486/// %1 = vector.shape_cast %destination :
487/// vector<2x2x4xf32> to vector<16xf32>
488/// %2 = vector.shuffle %1, %0 [16, 17, 18, 19, 20, 21, 22, 23
489/// 8, 9, 10, 11, 12, 13, 14, 15] :
490/// vector<16xf32>, vector<8xf32>
491/// %3 = vector.shape_cast %2 : vector<16xf32> to vector<2x2x4xf32>
492///
493/// Example #2:
494///
495/// %0 = vector.insert %source, %destination[1, 2]: f32 into vector<2x4xf32>
496///
497/// is converted to:
498///
499/// %0 = vector.shape_cast %destination : vector<2x4xf32> to vector<8xf32>
500/// %1 = vector.insert %source, %0[6]: f32 into vector<8xf32>
501/// %2 = vector.shape_cast %1 : vector<8xf32> to vector<2x4xf32>
502///
503struct LinearizeVectorInsert final
504 : public OpConversionPattern<vector::InsertOp> {
505 using Base::Base;
506 LinearizeVectorInsert(const TypeConverter &typeConverter,
507 MLIRContext *context, PatternBenefit benefit = 1)
508 : OpConversionPattern(typeConverter, context, benefit) {}
509 LogicalResult
510 matchAndRewrite(vector::InsertOp insertOp, OpAdaptor adaptor,
511 ConversionPatternRewriter &rewriter) const override {
512 VectorType dstTy = getTypeConverter()->convertType<VectorType>(
513 t: insertOp.getDestVectorType());
514 assert(dstTy && "vector type destination expected.");
515
516 // Dynamic position is not supported.
517 if (insertOp.hasDynamicPosition())
518 return rewriter.notifyMatchFailure(arg&: insertOp,
519 msg: "dynamic position is not supported.");
520 auto srcTy = insertOp.getValueToStoreType();
521 auto srcAsVec = dyn_cast<VectorType>(Val&: srcTy);
522 uint64_t srcSize = srcAsVec ? srcAsVec.getNumElements() : 1;
523
524 auto dstShape = insertOp.getDestVectorType().getShape();
525 const auto dstSize = insertOp.getDestVectorType().getNumElements();
526 auto dstSizeForOffsets = dstSize;
527
528 // Compute linearized offset.
529 int64_t linearizedOffset = 0;
530 auto offsetsNd = insertOp.getStaticPosition();
531 for (auto [dim, offset] : llvm::enumerate(First&: offsetsNd)) {
532 dstSizeForOffsets /= dstShape[dim];
533 linearizedOffset += offset * dstSizeForOffsets;
534 }
535
536 Location loc = insertOp.getLoc();
537 Value valueToStore = adaptor.getValueToStore();
538
539 if (!isa<VectorType>(Val: valueToStore.getType())) {
540 // Scalar case: generate a 1-D insert.
541 Value result = rewriter.createOrFold<vector::InsertOp>(
542 location: loc, args&: valueToStore, args: adaptor.getDest(), args&: linearizedOffset);
543 rewriter.replaceOp(op: insertOp, newValues: result);
544 return success();
545 }
546
547 // Vector case: generate a shuffle.
548 llvm::SmallVector<int64_t, 2> indices(dstSize);
549 auto *origValsUntil = indices.begin();
550 std::advance(i&: origValsUntil, n: linearizedOffset);
551
552 // Original values that remain [0, offset).
553 std::iota(first: indices.begin(), last: origValsUntil, value: 0);
554 auto *newValsUntil = origValsUntil;
555 std::advance(i&: newValsUntil, n: srcSize);
556 // New values [offset, offset+srcNumElements).
557 std::iota(first: origValsUntil, last: newValsUntil, value: dstSize);
558 // The rest of original values [offset+srcNumElements, end);
559 std::iota(first: newValsUntil, last: indices.end(), value: linearizedOffset + srcSize);
560
561 Value result = rewriter.createOrFold<vector::ShuffleOp>(
562 location: loc, args&: dstTy, args: adaptor.getDest(), args&: valueToStore, args&: indices);
563
564 rewriter.replaceOp(op: insertOp, newValues: result);
565 return success();
566 }
567};
568
569/// This pattern converts the BitCastOp that works on nD (n > 1)
570/// vectors to a BitCastOp that works on linearized vectors.
571/// Following,
572/// vector.bitcast %v1: vector<4x2xf32> to vector<4x4xf16>
573/// is converted to :
574/// %v1_1d = vector.shape_cast %v1: vector<4x2xf32> to vector<8xf32>
575/// %out_1d = vector.bitcast %v1_1d: vector<8xf32> to vector<16xf16>
576/// %out_nd = vector.shape_cast %out_1d: vector<16xf16> to vector<4x4xf16>
577struct LinearizeVectorBitCast final
578 : public OpConversionPattern<vector::BitCastOp> {
579 using Base::Base;
580 LinearizeVectorBitCast(const TypeConverter &typeConverter,
581 MLIRContext *context, PatternBenefit benefit = 1)
582 : OpConversionPattern(typeConverter, context, benefit) {}
583 LogicalResult
584 matchAndRewrite(vector::BitCastOp castOp, OpAdaptor adaptor,
585 ConversionPatternRewriter &rewriter) const override {
586 auto resType = getTypeConverter()->convertType(t: castOp.getType());
587 assert(resType && "expected 1-D vector type");
588 rewriter.replaceOpWithNewOp<vector::BitCastOp>(op: castOp, args&: resType,
589 args: adaptor.getSource());
590 return mlir::success();
591 }
592};
593
594/// This pattern converts the CreateMaskOp to work on a linearized vector.
595/// It currently supports only 2D masks with a unit outer dimension.
596/// Following,
597/// vector.create_mask %arg0, %arg1 : vector<1x4xi1>
598/// is converted to:
599/// %zero = arith.constant 0 : index
600/// %cmpi = arith.cmpi sgt, %arg0, %zero : index
601/// %index = arith.index_cast %cmpi : i1 to index
602/// %mul = arith.andi %index, %arg1 : index
603/// %mask = vector.create_mask %mul : vector<4xi1>
604/// %shape_cast = vector.shape_cast %mask : vector<4xi1> to vector<1x4xi1>
605struct LinearizeVectorCreateMask final
606 : OpConversionPattern<vector::CreateMaskOp> {
607 using Base::Base;
608
609 LinearizeVectorCreateMask(const TypeConverter &typeConverter,
610 MLIRContext *context, PatternBenefit benefit = 1)
611 : OpConversionPattern(typeConverter, context, benefit) {}
612
613 LogicalResult
614 matchAndRewrite(vector::CreateMaskOp createMaskOp, OpAdaptor adaptor,
615 ConversionPatternRewriter &rewriter) const override {
616 Location loc = createMaskOp.getLoc();
617 VectorType srcTy = createMaskOp.getType();
618 auto srcShape = srcTy.getShape();
619 if (srcShape.size() != 2)
620 return rewriter.notifyMatchFailure(arg&: createMaskOp,
621 msg: "only 2D mask is supported.");
622
623 if (srcShape[0] != 1)
624 return rewriter.notifyMatchFailure(
625 arg&: createMaskOp, msg: "only unit outer dimension is supported.");
626
627 auto dstTy = getTypeConverter()->convertType(t: srcTy);
628 if (!dstTy)
629 return rewriter.notifyMatchFailure(arg&: createMaskOp, msg: "cannot convert type.");
630
631 // Compare the first operand with 0. If it is greater than 0, the
632 // corresponding mask element is set to true, otherwise false.
633 // The result of the comparison is then multiplied with
634 // the second operand of create_mask to get the 1D mask.
635 auto firstOperand = adaptor.getOperands().front();
636 auto zero = mlir::arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
637 auto isNonZero = rewriter.createOrFold<mlir::arith::CmpIOp>(
638 location: loc, args: mlir::arith::CmpIPredicate::sgt, args&: firstOperand, args&: zero);
639 auto isNonZeroIndex = rewriter.createOrFold<mlir::arith::IndexCastOp>(
640 location: loc, args: rewriter.getIndexType(), args&: isNonZero);
641 auto secondOperand = adaptor.getOperands().back();
642 auto maskSize = rewriter.createOrFold<mlir::arith::AndIOp>(
643 location: loc, args: rewriter.getIndexType(), args&: isNonZeroIndex, args&: secondOperand);
644
645 auto newMask =
646 mlir::vector::CreateMaskOp::create(builder&: rewriter, location: loc, resultType0: dstTy, mask_dim_sizes: maskSize);
647 rewriter.replaceOp(op: createMaskOp, newOp: newMask);
648 return success();
649 }
650};
651
652/// This pattern linearizes vector.load from vector<1x1x...xN> to vector<N>
653/// It currently supports linearization where all but the last dimension are 1
654/// The following,
655/// vector.load %arg0[%c0, %c0] : memref<1x4xf32>, vector<1x4xf32>
656/// is converted to:
657/// vector.load %arg0[%c0, %c0] : memref<1x4xf32>, vector<4xf32>
658/// vector.shape_cast %load_result : vector<4xf32> to vector<1x4xf32>
659/// For generic cases, the vector unroll pass should be used to unroll the load
660/// to vector<1x1x...xN> form and then linearized
661struct LinearizeVectorLoad final : public OpConversionPattern<vector::LoadOp> {
662 using Base::Base;
663 LinearizeVectorLoad(const TypeConverter &typeConverter, MLIRContext *context,
664 PatternBenefit benefit = 1)
665 : OpConversionPattern(typeConverter, context, benefit) {}
666
667 LogicalResult
668 matchAndRewrite(vector::LoadOp loadOp, OpAdaptor adaptor,
669 ConversionPatternRewriter &rewriter) const override {
670 VectorType vecTy = loadOp.getType();
671 if (!vecTy)
672 return rewriter.notifyMatchFailure(arg&: loadOp, msg: "expected vector type");
673
674 auto shape = vecTy.getShape();
675 auto scalableDims = vecTy.getScalableDims();
676 // All but the last dim must be 1, and only the last dim may be scalable (if
677 // any).
678 if (!llvm::all_of(Range: shape.drop_back(N: 1), P: [](auto d) { return d == 1; }))
679 return rewriter.notifyMatchFailure(arg&: loadOp,
680 msg: "only vector<1x1x...xN> supported");
681
682 if (llvm::any_of(Range: scalableDims.drop_back(N: 1), P: [](bool s) { return s; }))
683 return rewriter.notifyMatchFailure(arg&: loadOp,
684 msg: "only innermost dim may be scalable");
685
686 auto linearTy = typeConverter->convertType<VectorType>(t: vecTy);
687
688 auto newLoad =
689 vector::LoadOp::create(builder&: rewriter, location: loadOp.getLoc(), resultType: linearTy,
690 base: adaptor.getBase(), indices: adaptor.getIndices());
691 rewriter.replaceOp(op: loadOp, newValues: newLoad.getResult());
692 return success();
693 }
694};
695
696/// This pattern linearizes vector.store from vector<1x1x...xN> to vector<N>
697/// It currently supports linearization where all but the last dimension are 1
698/// The following,
699/// vector.store %arg0, %arg1[%c0, %c0]s
700/// : vector<1x4xf32>, memref<1x4xf32>
701/// is converted to:
702/// vector.shape_cast %arg0 : vector<1x4xf32> to vector<4xf32>
703/// vector.store %arg0, %arg1[%c0, %c0]
704/// : vector<4xf32>, memref<1x4xf32>
705/// For generic cases, the vector unroll pass should be used to unroll the store
706/// to vector<1x1x...xN> form and then linearized
707struct LinearizeVectorStore final
708 : public OpConversionPattern<vector::StoreOp> {
709 using Base::Base;
710 LinearizeVectorStore(const TypeConverter &typeConverter, MLIRContext *context,
711 PatternBenefit benefit = 1)
712 : OpConversionPattern(typeConverter, context, benefit) {}
713
714 LogicalResult
715 matchAndRewrite(vector::StoreOp storeOp, OpAdaptor adaptor,
716 ConversionPatternRewriter &rewriter) const override {
717 VectorType vecTy = storeOp.getValueToStore().getType();
718 if (!vecTy)
719 return rewriter.notifyMatchFailure(arg&: storeOp, msg: "expected vector type");
720
721 auto shape = vecTy.getShape();
722 auto scalableDims = vecTy.getScalableDims();
723 // All but the last dim must be 1, and only the last dim may be scalable (if
724 // any).
725 if (!llvm::all_of(Range: shape.drop_back(N: 1), P: [](auto d) { return d == 1; }))
726 return rewriter.notifyMatchFailure(arg&: storeOp,
727 msg: "only vector<1x1x...xN> supported");
728
729 if (llvm::any_of(Range: scalableDims.drop_back(N: 1), P: [](bool s) { return s; }))
730 return rewriter.notifyMatchFailure(arg&: storeOp,
731 msg: "only innermost dim may be scalable");
732
733 rewriter.replaceOpWithNewOp<vector::StoreOp>(
734 op: storeOp, args: adaptor.getValueToStore(), args: adaptor.getBase(),
735 args: adaptor.getIndices());
736 return success();
737 }
738};
739
740/// This pattern linearizes `vector.from_elements` operations by converting
741/// the result type to a 1-D vector while preserving all element values.
742/// The transformation creates a linearized `vector.from_elements` followed by
743/// a `vector.shape_cast` to restore the original multidimensional shape.
744///
745/// Example:
746///
747/// %0 = vector.from_elements %a, %b, %c, %d : vector<2x2xf32>
748///
749/// is converted to:
750///
751/// %0 = vector.from_elements %a, %b, %c, %d : vector<4xf32>
752/// %1 = vector.shape_cast %0 : vector<4xf32> to vector<2x2xf32>
753///
754struct LinearizeVectorFromElements final
755 : public OpConversionPattern<vector::FromElementsOp> {
756 using Base::Base;
757 LinearizeVectorFromElements(const TypeConverter &typeConverter,
758 MLIRContext *context, PatternBenefit benefit = 1)
759 : OpConversionPattern(typeConverter, context, benefit) {}
760 LogicalResult
761 matchAndRewrite(vector::FromElementsOp fromElementsOp, OpAdaptor adaptor,
762 ConversionPatternRewriter &rewriter) const override {
763 VectorType dstTy =
764 getTypeConverter()->convertType<VectorType>(t: fromElementsOp.getType());
765 assert(dstTy && "vector type destination expected.");
766
767 OperandRange elements = fromElementsOp.getElements();
768 assert(elements.size() == static_cast<size_t>(dstTy.getNumElements()) &&
769 "expected same number of elements");
770 rewriter.replaceOpWithNewOp<vector::FromElementsOp>(op: fromElementsOp, args&: dstTy,
771 args&: elements);
772 return success();
773 }
774};
775
776/// This pattern linearizes the operand in `vector.to_elements` operations
777/// by converting the source type to a 1-D vector while preserving all element
778/// values. The transformation creates a linearized `vector.shape_cast`
779/// followed by a `vector.to_elements`.
780///
781/// Example:
782///
783/// %0:4 = vector.to_elements %v : vector<2x2xf32>
784///
785/// is converted to:
786///
787/// %vector_cast = vector.shape_cast %v : vector<2x2xf32> to vector<4xf32>
788/// %0:4 = vector.to_elements %vector_cast : vector<4xf32>
789///
790struct LinearizeVectorToElements final
791 : public OpConversionPattern<vector::ToElementsOp> {
792 using Base::Base;
793
794 LinearizeVectorToElements(const TypeConverter &typeConverter,
795 MLIRContext *context, PatternBenefit benefit = 1)
796 : OpConversionPattern(typeConverter, context, benefit) {}
797
798 LogicalResult
799 matchAndRewrite(vector::ToElementsOp toElementsOp, OpAdaptor adaptor,
800 ConversionPatternRewriter &rewriter) const override {
801
802 VectorType vecType = toElementsOp.getSource().getType();
803 if (vecType.getRank() <= 1)
804 return rewriter.notifyMatchFailure(
805 arg&: toElementsOp, msg: "the rank is already less than or equal to 1");
806
807 assert(vecType.getNumScalableDims() == 0 &&
808 "to_elements does not support scalable vectors");
809 auto vec1DType =
810 VectorType::get(shape: {vecType.getNumElements()}, elementType: vecType.getElementType());
811 Value shapeCast = vector::ShapeCastOp::create(
812 builder&: rewriter, location: toElementsOp.getLoc(), result: vec1DType, source: toElementsOp.getSource());
813 auto newToElementsOp =
814 vector::ToElementsOp::create(builder&: rewriter, location: toElementsOp.getLoc(),
815 elements: toElementsOp.getResultTypes(), source: shapeCast);
816 rewriter.replaceOp(op: toElementsOp, newOp: newToElementsOp);
817 return success();
818 }
819};
820
821/// Convert broadcasts from scalars or 1-element vectors, such as
822///
823/// ```mlir
824/// vector.broadcast %value : f32 to vector<4x4xf32>
825/// ```
826///
827/// to broadcasts to rank-1 vectors, with shape_casts before/after as needed.
828/// The above becomes,
829///
830/// ```mlir
831/// %out_1d = vector.broadcast %value : f32 to vector<16xf32>
832/// %out_nd = vector.shape_cast %out_1d : vector<16xf32> to vector<4x4xf32>
833/// ```
834struct LinearizeVectorBroadcast final
835 : public OpConversionPattern<vector::BroadcastOp> {
836 using Base::Base;
837
838 LinearizeVectorBroadcast(const TypeConverter &typeConverter,
839 MLIRContext *context, PatternBenefit benefit = 1)
840 : OpConversionPattern(typeConverter, context, benefit) {}
841
842 LogicalResult
843 matchAndRewrite(vector::BroadcastOp broadcastOp, OpAdaptor adaptor,
844 ConversionPatternRewriter &rewriter) const override {
845
846 int numElements = 1;
847 Type sourceType = broadcastOp.getSourceType();
848 if (auto vecType = dyn_cast<VectorType>(Val&: sourceType)) {
849 numElements = vecType.getNumElements();
850 }
851
852 if (numElements != 1) {
853 return rewriter.notifyMatchFailure(
854 arg&: broadcastOp, msg: "only broadcasts of single elements can be linearized.");
855 }
856
857 auto dstTy = getTypeConverter()->convertType(t: broadcastOp.getType());
858 rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: broadcastOp, args&: dstTy,
859 args: adaptor.getSource());
860
861 return success();
862 }
863};
864
865/// Linearize `vector.interleave` to operate on flattened 1D operands and
866/// result. The flattening is commutative here, the order of flatten and
867/// interleave ops does not matter, so this transform is valid for
868/// any ND shape.
869///
870/// Example:
871/// vector.interleave %a, %b : vector<4x1xT> -> vector<4x2xT>
872/// becomes:
873/// vector.interleave %a_flat, %b_flat : vector<4xT> -> vector<8xT>
874struct LinearizeVectorInterleave final
875 : public OpConversionPattern<vector::InterleaveOp> {
876 using Base::Base;
877 LinearizeVectorInterleave(const TypeConverter &typeConverter,
878 MLIRContext *context, PatternBenefit benefit = 1)
879 : OpConversionPattern(typeConverter, context, benefit) {}
880
881 LogicalResult
882 matchAndRewrite(vector::InterleaveOp op, OpAdaptor adaptor,
883 ConversionPatternRewriter &rewriter) const override {
884 VectorType flatResultTy =
885 getTypeConverter()->convertType<VectorType>(t: op.getResultVectorType());
886 if (!flatResultTy)
887 return rewriter.notifyMatchFailure(arg&: op, msg: "failed to linearize result type");
888 rewriter.replaceOpWithNewOp<vector::InterleaveOp>(
889 op, args&: flatResultTy, args: adaptor.getLhs(), args: adaptor.getRhs());
890 return success();
891 }
892};
893
894/// Linearize `vector.deinterleave` to operate on a flattened 1D source.
895/// The flattening is commutative here, the order of flatten and deinterleave
896/// ops does not matter, so this transform is valid for any ND shape.
897///
898/// Example:
899/// %even, %odd = vector.deinterleave %src : vector<4x2xT> -> vector<4x1xT>
900/// becomes:
901/// %even, %odd = vector.deinterleave %src_flat : vector<8xT> -> vector<4xT>
902/// This linearization only works if the innermost dimension is even.
903/// Consider vector<2x5xT>:
904/// [[0,1,2,3,4],
905/// [5,6,7,8,9]]
906/// Non-linearized deinterleave returns the odd part: [[1, 3], [6, 8]]
907/// Linearized deinterleave would the odd part: [1, 3, 5, 7, 9]
908struct LinearizeVectorDeinterleave final
909 : public OpConversionPattern<vector::DeinterleaveOp> {
910 using Base::Base;
911 LinearizeVectorDeinterleave(const TypeConverter &typeConverter,
912 MLIRContext *context, PatternBenefit benefit = 1)
913 : OpConversionPattern(typeConverter, context, benefit) {}
914
915 LogicalResult
916 matchAndRewrite(vector::DeinterleaveOp op, OpAdaptor adaptor,
917 ConversionPatternRewriter &rewriter) const override {
918 if (op.getSource().getType().getShape().back() % 2)
919 return failure();
920 auto newOp = vector::DeinterleaveOp::create(builder&: rewriter, location: op.getLoc(),
921 source: adaptor.getSource());
922 rewriter.replaceOp(op, newValues: newOp.getResults());
923 return success();
924 }
925};
926
927} // namespace
928
929/// This method defines the set of operations that are linearizable, and hence
930/// that are considered illegal for the conversion target.
931static bool isLinearizable(Operation *op) {
932
933 // Only ops that are in the vector dialect, are ConstantLike, or
934 // are Vectorizable might be linearized currently.
935 StringLiteral vectorDialect = vector::VectorDialect::getDialectNamespace();
936 StringRef opDialect = op->getDialect()->getNamespace();
937 bool supported = (opDialect == vectorDialect) ||
938 op->hasTrait<OpTrait::ConstantLike>() ||
939 op->hasTrait<OpTrait::Vectorizable>();
940 if (!supported)
941 return false;
942
943 return TypeSwitch<Operation *, bool>(op)
944 // As type legalization is done with vector.shape_cast, shape_cast
945 // itself cannot be linearized (will create new shape_casts to linearize
946 // ad infinitum).
947 .Case(caseFn: [&](vector::ShapeCastOp) { return false; })
948 // The operations
949 // - vector.extract_strided_slice
950 // - vector.extract
951 // - vector.insert_strided_slice
952 // - vector.insert
953 // are linearized to a rank-1 vector.shuffle by the current patterns.
954 // vector.shuffle only supports fixed size vectors, so it is impossible to
955 // use this approach to linearize these ops if they operate on scalable
956 // vectors.
957 .Case(caseFn: [&](vector::ExtractStridedSliceOp extractOp) {
958 return !extractOp.getType().isScalable();
959 })
960 .Case(caseFn: [&](vector::InsertStridedSliceOp insertOp) {
961 return !insertOp.getType().isScalable();
962 })
963 .Case(caseFn: [&](vector::InsertOp insertOp) {
964 return !insertOp.getType().isScalable();
965 })
966 .Case(caseFn: [&](vector::ExtractOp extractOp) {
967 return !extractOp.getSourceVectorType().isScalable();
968 })
969 .Default(defaultFn: [&](auto) { return true; });
970}
971
972void mlir::vector::populateForVectorLinearize(TypeConverter &typeConverter,
973 ConversionTarget &target) {
974
975 auto convertType = [](Type type) -> std::optional<Type> {
976 VectorType vectorType = dyn_cast<VectorType>(Val&: type);
977 if (!vectorType || !isLinearizableVector(type: vectorType))
978 return type;
979
980 VectorType linearizedType =
981 VectorType::get(shape: vectorType.getNumElements(),
982 elementType: vectorType.getElementType(), scalableDims: vectorType.isScalable());
983 return linearizedType;
984 };
985 typeConverter.addConversion(callback&: convertType);
986
987 auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs,
988 Location loc) -> Value {
989 if (inputs.size() != 1)
990 return nullptr;
991
992 Value value = inputs.front();
993 if (!isa<VectorType>(Val: type) || !isa<VectorType>(Val: value.getType()))
994 return nullptr;
995
996 return vector::ShapeCastOp::create(builder, location: loc, result: type, source: value);
997 };
998 typeConverter.addSourceMaterialization(callback&: materializeCast);
999 typeConverter.addTargetMaterialization(callback&: materializeCast);
1000
1001 target.markUnknownOpDynamicallyLegal(
1002 fn: [=](Operation *op) -> std::optional<bool> {
1003 if (!isLinearizable(op))
1004 return true;
1005 // This will return true if, for all operand and result types `t`,
1006 // convertType(t) = t. This is true if there are no rank>=2 vectors.
1007 return typeConverter.isLegal(op);
1008 });
1009}
1010
1011/// Get the set of operand/result types to check for sufficiently
1012/// small inner-most dimension size.
1013static SmallVector<std::pair<Type, unsigned>>
1014getTypeBitWidthBoundPairs(Operation *op, unsigned targetBitWidth) {
1015
1016 if (auto insertOp = dyn_cast<vector::InsertOp>(Val: op)) {
1017 unsigned w = targetBitWidth < std::numeric_limits<unsigned>::max()
1018 ? targetBitWidth + 1
1019 : targetBitWidth;
1020 return {{insertOp.getValueToStoreType(), w}};
1021 }
1022
1023 auto resultTypes = op->getResultTypes();
1024 SmallVector<std::pair<Type, unsigned>> resultsWithBitWidth;
1025 resultsWithBitWidth.reserve(N: resultTypes.size());
1026 for (Type type : resultTypes) {
1027 resultsWithBitWidth.push_back(Elt: {type, targetBitWidth});
1028 }
1029 return resultsWithBitWidth;
1030}
1031
1032/// Returns true if any checked vector type of op's inner-most dimension
1033/// size is equal to or exceeds target bit width.
1034static bool hasWideInnerDim(Operation *op, unsigned targetBitWidth) {
1035 // Check on bitwidths.
1036 SmallVector<std::pair<Type, unsigned>> toCheck =
1037 getTypeBitWidthBoundPairs(op, targetBitWidth);
1038
1039 // If `type` is VectorType with trailing dimension of (bit) size greater than
1040 // or equal to `targetBitWidth`, its defining op is considered legal.
1041 return llvm::any_of(Range&: toCheck, P: [&](std::pair<Type, unsigned> typeWidth) {
1042 VectorType vecType = dyn_cast<VectorType>(Val&: typeWidth.first);
1043
1044 // Not linearizable for reasons other than what this function checks.
1045 if (!vecType || vecType.getRank() == 0)
1046 return false;
1047
1048 // The width of the type 'index' is unbounded (and therefore potentially
1049 // above the target width).
1050 if (vecType.getElementType().isIndex())
1051 return true;
1052
1053 unsigned finalDimSize = vecType.getShape().back();
1054 unsigned nbBitsPerElm = vecType.getElementTypeBitWidth();
1055 unsigned trailingVecDimBitWidth = finalDimSize * nbBitsPerElm;
1056 return trailingVecDimBitWidth >= typeWidth.second;
1057 });
1058}
1059
1060void mlir::vector::populateVectorLinearizeBitWidthTargetRules(
1061 ConversionTarget &target, unsigned targetBitWidth) {
1062 target.markUnknownOpDynamicallyLegal(
1063 fn: [=](Operation *op) -> std::optional<bool> {
1064 if (hasWideInnerDim(op, targetBitWidth))
1065 return true;
1066 return std::nullopt;
1067 });
1068}
1069
1070void mlir::vector::populateVectorLinearizeBasePatterns(
1071 const TypeConverter &typeConverter, const ConversionTarget &target,
1072 RewritePatternSet &patterns) {
1073 patterns
1074 .add<LinearizeConstantLike, LinearizeVectorizable, LinearizeVectorBitCast,
1075 LinearizeVectorCreateMask, LinearizeVectorLoad, LinearizeVectorStore,
1076 LinearizeVectorBroadcast, LinearizeVectorFromElements,
1077 LinearizeVectorToElements, LinearizeVectorInterleave,
1078 LinearizeVectorDeinterleave>(arg: typeConverter, args: patterns.getContext());
1079}
1080
1081void mlir::vector::populateVectorLinearizeShuffleLikeOpsPatterns(
1082 const TypeConverter &typeConverter, const ConversionTarget &target,
1083 RewritePatternSet &patterns) {
1084 patterns.add<LinearizeVectorShuffle, LinearizeVectorExtract,
1085 LinearizeVectorInsert, LinearizeVectorExtractStridedSlice,
1086 LinearizeVectorInsertStridedSlice>(arg: typeConverter,
1087 args: patterns.getContext());
1088}
1089