1//===- LowerVectorMultiReduction.cpp - Lower `vector.multi_reduction` op --===//
2//
3/// Part of the LLVM Project, under the Apache License v2.0 with LLVM
4/// Exceptions. See https://llvm.org/LICENSE.txt for license information.
5/// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements target-independent rewrites and utilities to lower the
10// 'vector.multi_reduction' operation.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/Arith/IR/Arith.h"
15#include "mlir/Dialect/Func/IR/FuncOps.h"
16#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
17#include "mlir/Dialect/Vector/Transforms/Passes.h"
18#include "mlir/IR/Builders.h"
19#include "mlir/IR/TypeUtilities.h"
20#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
21
22namespace mlir {
23namespace vector {
24#define GEN_PASS_DEF_LOWERVECTORMULTIREDUCTION
25#include "mlir/Dialect/Vector/Transforms/Passes.h.inc"
26} // namespace vector
27} // namespace mlir
28
29#define DEBUG_TYPE "vector-multi-reduction"
30
31using namespace mlir;
32
33namespace {
34/// This file implements the following transformations as composable atomic
35/// patterns.
36
37/// Converts vector.multi_reduction into inner-most/outer-most reduction form
38/// by using vector.transpose
39class InnerOuterDimReductionConversion
40 : public OpRewritePattern<vector::MultiDimReductionOp> {
41public:
42 using Base::Base;
43
44 explicit InnerOuterDimReductionConversion(
45 MLIRContext *context, vector::VectorMultiReductionLowering options,
46 PatternBenefit benefit = 1)
47 : mlir::OpRewritePattern<vector::MultiDimReductionOp>(context, benefit),
48 useInnerDimsForReduction(
49 options == vector::VectorMultiReductionLowering::InnerReduction) {}
50
51 LogicalResult matchAndRewrite(vector::MultiDimReductionOp multiReductionOp,
52 PatternRewriter &rewriter) const override {
53 // Vector mask setup.
54 OpBuilder::InsertionGuard guard(rewriter);
55 auto maskableOp =
56 cast<vector::MaskableOpInterface>(Val: multiReductionOp.getOperation());
57 Operation *rootOp;
58 if (maskableOp.isMasked()) {
59 rewriter.setInsertionPoint(maskableOp.getMaskingOp());
60 rootOp = maskableOp.getMaskingOp();
61 } else {
62 rootOp = multiReductionOp;
63 }
64
65 auto src = multiReductionOp.getSource();
66 auto loc = multiReductionOp.getLoc();
67 auto srcRank = multiReductionOp.getSourceVectorType().getRank();
68
69 // Separate reduction and parallel dims
70 ArrayRef<int64_t> reductionDims = multiReductionOp.getReductionDims();
71 llvm::SmallDenseSet<int64_t> reductionDimsSet(reductionDims.begin(),
72 reductionDims.end());
73 int64_t reductionSize = reductionDims.size();
74 SmallVector<int64_t, 4> parallelDims;
75 for (int64_t i = 0; i < srcRank; ++i)
76 if (!reductionDimsSet.contains(V: i))
77 parallelDims.push_back(Elt: i);
78
79 // Add transpose only if inner-most/outer-most dimensions are not parallel
80 // and there are parallel dims.
81 if (parallelDims.empty())
82 return failure();
83 if (useInnerDimsForReduction &&
84 (parallelDims ==
85 llvm::to_vector<4>(Range: llvm::seq<int64_t>(Begin: 0, End: parallelDims.size()))))
86 return failure();
87
88 if (!useInnerDimsForReduction &&
89 (parallelDims == llvm::to_vector<4>(Range: llvm::seq<int64_t>(
90 Begin: reductionDims.size(),
91 End: parallelDims.size() + reductionDims.size()))))
92 return failure();
93
94 SmallVector<int64_t, 4> indices;
95 if (useInnerDimsForReduction) {
96 indices.append(in_start: parallelDims.begin(), in_end: parallelDims.end());
97 indices.append(in_start: reductionDims.begin(), in_end: reductionDims.end());
98 } else {
99 indices.append(in_start: reductionDims.begin(), in_end: reductionDims.end());
100 indices.append(in_start: parallelDims.begin(), in_end: parallelDims.end());
101 }
102
103 // If masked, transpose the original mask.
104 Value transposedMask;
105 if (maskableOp.isMasked()) {
106 transposedMask = vector::TransposeOp::create(
107 builder&: rewriter, location: loc, vector: maskableOp.getMaskingOp().getMask(), permutation: indices);
108 }
109
110 // Transpose reduction source.
111 auto transposeOp = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: src, permutation: indices);
112 SmallVector<bool> reductionMask(srcRank, false);
113 for (int i = 0; i < reductionSize; ++i) {
114 if (useInnerDimsForReduction)
115 reductionMask[srcRank - i - 1] = true;
116 else
117 reductionMask[i] = true;
118 }
119
120 Operation *newMultiRedOp = vector::MultiDimReductionOp::create(
121 builder&: rewriter, location: multiReductionOp.getLoc(), source: transposeOp.getResult(),
122 acc: multiReductionOp.getAcc(), reductionMask, kind: multiReductionOp.getKind(),
123 fastMathFlags: multiReductionOp.getFastmath());
124 newMultiRedOp =
125 mlir::vector::maskOperation(builder&: rewriter, maskableOp: newMultiRedOp, mask: transposedMask);
126
127 rewriter.replaceOp(op: rootOp, newValues: newMultiRedOp->getResult(idx: 0));
128 return success();
129 }
130
131private:
132 const bool useInnerDimsForReduction;
133};
134
135/// Flattens vector.multi_reduction to 2D
136///
137/// Given all reduction dimensions are either inner most or outer most,
138/// flattens all reduction and parallel dimensions so that there are only 2Ds.
139///
140/// BEFORE
141/// vector.multi_reduction <add>, %vec, %acc [2, 3] : vector<2x3x4x5xi32> to
142/// vector<2x3xi32>
143/// AFTER
144/// %vec_sc = vector.shape_cast %vec
145/// %acc_sc = vector.shape_cast %acc
146/// %res = vector.multi_reduction <add>, %vec_sc, %acc_cs [1] :
147/// vector<6x20xi32> to vector<6xi32> %res_sc = vector.shape_cast %res
148class FlattenMultiReduction
149 : public OpRewritePattern<vector::MultiDimReductionOp> {
150public:
151 using Base::Base;
152
153 explicit FlattenMultiReduction(MLIRContext *context,
154 vector::VectorMultiReductionLowering options,
155 PatternBenefit benefit = 1)
156 : mlir::OpRewritePattern<vector::MultiDimReductionOp>(context, benefit),
157 useInnerDimsForReduction(
158 options == vector::VectorMultiReductionLowering::InnerReduction) {}
159
160 LogicalResult matchAndRewrite(vector::MultiDimReductionOp multiReductionOp,
161 PatternRewriter &rewriter) const override {
162 // Vector mask setup.
163 OpBuilder::InsertionGuard guard(rewriter);
164 auto maskableOp =
165 cast<vector::MaskableOpInterface>(Val: multiReductionOp.getOperation());
166 Operation *rootOp;
167 if (maskableOp.isMasked()) {
168 rewriter.setInsertionPoint(maskableOp.getMaskingOp());
169 rootOp = maskableOp.getMaskingOp();
170 } else {
171 rootOp = multiReductionOp;
172 }
173
174 auto srcRank = multiReductionOp.getSourceVectorType().getRank();
175 auto srcShape = multiReductionOp.getSourceVectorType().getShape();
176 auto srcScalableDims =
177 multiReductionOp.getSourceVectorType().getScalableDims();
178 auto loc = multiReductionOp.getLoc();
179
180 // If rank less than 2, nothing to do.
181 if (srcRank < 2)
182 return failure();
183
184 // Allow only 1 scalable dimensions. Otherwise we could end-up with e.g.
185 // `vscale * vscale` that's currently not modelled.
186 if (llvm::count(Range&: srcScalableDims, Element: true) > 1)
187 return failure();
188
189 // If already rank-2 ["parallel", "reduce"] or ["reduce", "parallel"] bail.
190 SmallVector<bool> reductionMask = multiReductionOp.getReductionMask();
191 if (srcRank == 2 && reductionMask.front() != reductionMask.back())
192 return failure();
193
194 // 1. Separate reduction and parallel dims.
195 SmallVector<int64_t, 4> parallelDims, parallelShapes;
196 SmallVector<bool, 4> parallelScalableDims;
197 SmallVector<int64_t, 4> reductionDims, reductionShapes;
198 bool isReductionDimScalable = false;
199 for (const auto &it : llvm::enumerate(First&: reductionMask)) {
200 int64_t i = it.index();
201 bool isReduction = it.value();
202 if (isReduction) {
203 reductionDims.push_back(Elt: i);
204 reductionShapes.push_back(Elt: srcShape[i]);
205 isReductionDimScalable |= srcScalableDims[i];
206 } else {
207 parallelDims.push_back(Elt: i);
208 parallelShapes.push_back(Elt: srcShape[i]);
209 parallelScalableDims.push_back(Elt: srcScalableDims[i]);
210 }
211 }
212
213 // 2. Compute flattened parallel and reduction sizes.
214 int flattenedParallelDim = 0;
215 int flattenedReductionDim = 0;
216 if (!parallelShapes.empty()) {
217 flattenedParallelDim = 1;
218 for (auto d : parallelShapes)
219 flattenedParallelDim *= d;
220 }
221 if (!reductionShapes.empty()) {
222 flattenedReductionDim = 1;
223 for (auto d : reductionShapes)
224 flattenedReductionDim *= d;
225 }
226 // We must at least have some parallel or some reduction.
227 assert((flattenedParallelDim || flattenedReductionDim) &&
228 "expected at least one parallel or reduction dim");
229
230 // 3. Fail if reduction/parallel dims are not contiguous.
231 // Check parallelDims are exactly [0 .. size).
232 int64_t counter = 0;
233 if (useInnerDimsForReduction &&
234 llvm::any_of(Range&: parallelDims, P: [&](int64_t i) { return i != counter++; }))
235 return failure();
236 // Check parallelDims are exactly {reductionDims.size()} + [0 .. size).
237 counter = reductionDims.size();
238 if (!useInnerDimsForReduction &&
239 llvm::any_of(Range&: parallelDims, P: [&](int64_t i) { return i != counter++; }))
240 return failure();
241
242 // 4. Shape cast to collapse consecutive parallel (resp. reduction dim) into
243 // a single parallel (resp. reduction) dim.
244 SmallVector<bool, 2> mask;
245 SmallVector<bool, 2> scalableDims;
246 SmallVector<int64_t, 2> vectorShape;
247 bool isParallelDimScalable = llvm::is_contained(Range&: parallelScalableDims, Element: true);
248 if (flattenedParallelDim) {
249 mask.push_back(Elt: false);
250 vectorShape.push_back(Elt: flattenedParallelDim);
251 scalableDims.push_back(Elt: isParallelDimScalable);
252 }
253 if (flattenedReductionDim) {
254 mask.push_back(Elt: true);
255 vectorShape.push_back(Elt: flattenedReductionDim);
256 scalableDims.push_back(Elt: isReductionDimScalable);
257 }
258 if (!useInnerDimsForReduction && vectorShape.size() == 2) {
259 std::swap(a&: mask.front(), b&: mask.back());
260 std::swap(a&: vectorShape.front(), b&: vectorShape.back());
261 std::swap(a&: scalableDims.front(), b&: scalableDims.back());
262 }
263
264 Value newVectorMask;
265 if (maskableOp.isMasked()) {
266 Value vectorMask = maskableOp.getMaskingOp().getMask();
267 auto maskCastedType = VectorType::get(
268 shape: vectorShape,
269 elementType: llvm::cast<VectorType>(Val: vectorMask.getType()).getElementType(),
270 scalableDims);
271 newVectorMask = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: maskCastedType,
272 source: vectorMask);
273 }
274
275 auto castedType = VectorType::get(
276 shape: vectorShape, elementType: multiReductionOp.getSourceVectorType().getElementType(),
277 scalableDims);
278 Value cast = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: castedType,
279 source: multiReductionOp.getSource());
280
281 Value acc = multiReductionOp.getAcc();
282 if (flattenedParallelDim) {
283 auto accType = VectorType::get(
284 shape: {flattenedParallelDim},
285 elementType: multiReductionOp.getSourceVectorType().getElementType(),
286 /*scalableDims=*/{isParallelDimScalable});
287 acc = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: accType, source: acc);
288 }
289 // 6. Creates the flattened form of vector.multi_reduction with inner/outer
290 // most dim as reduction.
291 Operation *newMultiDimRedOp = vector::MultiDimReductionOp::create(
292 builder&: rewriter, location: loc, source: cast, acc, reductionMask: mask, kind: multiReductionOp.getKind(),
293 fastMathFlags: multiReductionOp.getFastmath());
294 newMultiDimRedOp =
295 mlir::vector::maskOperation(builder&: rewriter, maskableOp: newMultiDimRedOp, mask: newVectorMask);
296
297 // 7. If there are no parallel shapes, the result is a scalar.
298 // TODO: support 0-d vectors when available.
299 if (parallelShapes.empty()) {
300 rewriter.replaceOp(op: rootOp, newValues: newMultiDimRedOp->getResult(idx: 0));
301 return success();
302 }
303
304 // 8. Shape cast the flattened result back to the original n-D parallel
305 // shape.
306 VectorType outputCastedType = VectorType::get(
307 shape: parallelShapes, elementType: multiReductionOp.getSourceVectorType().getElementType(),
308 scalableDims: parallelScalableDims);
309 rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(
310 op: rootOp, args&: outputCastedType, args: newMultiDimRedOp->getResult(idx: 0));
311 return success();
312 }
313
314private:
315 const bool useInnerDimsForReduction;
316};
317
318/// Lowers 2D vector.multi_reduction to a squence of Arith Ops
319///
320/// The reduction dimension must be the outer-most dimension.
321///
322/// BEFORE:
323///
324/// %1 = vector.multi_reduction <mul>, %src, %acc [0] : vector<4x2xf32> to
325/// vector<2xf32>
326///
327/// AFTER:
328///
329/// // Prod 1.
330/// %vec_0 = vector.extract %src[0] : vector<2xf32> from vector<4x2xf32>
331/// %mul_0 = arith.mulf %vec_0, %acc : vector<2xf32>
332///
333/// // Prod 2.
334/// %vec_1 = vector.extract %src[1] : vector<2xf32> from vector<4x2xf32>
335/// %mul_2 = arith.mulf %vec_1, %mul_0 : vector<2xf32>
336///
337/// // Prod 3.
338/// %vec_3 = vector.extract %src[2] : vector<2xf32> from vector<4x2xf32>
339/// %mul_3 = arith.mulf %vec_3, %mul_2 : vector<2xf32>
340///
341/// // Prod 4.
342/// %vec_4 = vector.extract %src[3] : vector<2xf32> from vector<4x2xf32>
343/// %res = arith.mulf %vec_4, %mul_3 : vector<2xf32>
344struct TwoDimMultiReductionToElementWise
345 : public vector::MaskableOpRewritePattern<vector::MultiDimReductionOp> {
346 using MaskableOpRewritePattern::MaskableOpRewritePattern;
347
348 FailureOr<Value>
349 matchAndRewriteMaskableOp(vector::MultiDimReductionOp multiReductionOp,
350 vector::MaskingOpInterface maskingOp,
351 PatternRewriter &rewriter) const override {
352 auto srcRank = multiReductionOp.getSourceVectorType().getRank();
353 // Rank-2 ["parallel", "reduce"] or bail.
354 if (srcRank != 2)
355 return failure();
356
357 if (multiReductionOp.isReducedDim(d: 1) || !multiReductionOp.isReducedDim(d: 0))
358 return failure();
359
360 Value mask = maskingOp ? maskingOp.getMask() : Value();
361
362 auto loc = multiReductionOp.getLoc();
363 Value source = multiReductionOp.getSource();
364 ArrayRef<int64_t> srcShape =
365 multiReductionOp.getSourceVectorType().getShape();
366 int outerDim = srcShape[0];
367
368 Value result = multiReductionOp.getAcc();
369 for (int64_t i = 0; i < outerDim; i++) {
370 auto v = vector::ExtractOp::create(builder&: rewriter, location: loc, source, position: i);
371 Value m = mask ? Value(vector::ExtractOp::create(builder&: rewriter, location: loc, source: mask, position: i))
372 : nullptr;
373 result =
374 makeArithReduction(b&: rewriter, loc, kind: multiReductionOp.getKind(), v1: v,
375 acc: result, fastmath: multiReductionOp.getFastmathAttr(), mask: m);
376 }
377
378 return result;
379 }
380};
381
382/// Lowers 2D vector.multi_reduction to a sequence of vector.reduction Ops.
383///
384/// The reduction dimension must be the inner-most dimension.
385///
386/// BEFORE:
387/// vector.multi_reduction <mul>, %src, %acc [1] : vector<2x4xf32> to
388/// vector<2xf32>
389///
390/// AFTER:
391/// // 1st reduction
392/// %v_0 = vector.extract %src[0] : vector<4xf32> from vector<2x4xf32>
393/// %a_0 = vector.extract %acc[0] : f32 from vector<2xf32>
394/// %red_1 = vector.reduction <mul>, %v_0, %a_1 : vector<4xf32> into f32
395/// %res_tmp = vector.insert %red_1, %res [0] : f32 into vector<2xf32>
396///
397/// // 2nd reduction
398/// %v_1 = vector.extract %src[1] : vector<4xf32> from vector<2x4xf32>
399/// %a_1 = vector.extract %acc[1] : f32 from vector<2xf32>
400/// %red_2 = vector.reduction <mul>, %v_1, %a_1 : vector<4xf32> into f32
401/// %res_final = vector.insert %red_2, %res_tmp [1] : f32 into vector<2xf32>
402struct TwoDimMultiReductionToReduction
403 : public vector::MaskableOpRewritePattern<vector::MultiDimReductionOp> {
404 using MaskableOpRewritePattern::MaskableOpRewritePattern;
405
406 FailureOr<Value>
407 matchAndRewriteMaskableOp(vector::MultiDimReductionOp multiReductionOp,
408 vector::MaskingOpInterface maskingOp,
409 PatternRewriter &rewriter) const override {
410 auto srcRank = multiReductionOp.getSourceVectorType().getRank();
411 // Rank-2 ["reduce", "parallel"] or bail.
412 if (srcRank != 2)
413 return failure();
414
415 if (multiReductionOp.isReducedDim(d: 0) || !multiReductionOp.isReducedDim(d: 1))
416 return failure();
417
418 Value mask = maskingOp ? maskingOp.getMask() : nullptr;
419
420 auto loc = multiReductionOp.getLoc();
421 Value source = multiReductionOp.getSource();
422 Value acc = multiReductionOp.getAcc();
423 int outerDim = multiReductionOp.getSourceVectorType().getShape()[0];
424
425 Value result = arith::ConstantOp::create(
426 builder&: rewriter, location: loc, result: multiReductionOp.getDestType(),
427 value: rewriter.getZeroAttr(type: multiReductionOp.getDestType()));
428
429 SmallVector<Value> vectors(outerDim);
430 for (int64_t i = 0; i < outerDim; ++i) {
431 Value v = vector::ExtractOp::create(builder&: rewriter, location: loc, source, position: i);
432 Value a = vector::ExtractOp::create(builder&: rewriter, location: loc, source: acc, position: i);
433
434 Operation *reductionOp =
435 vector::ReductionOp::create(builder&: rewriter, location: loc, kind: multiReductionOp.getKind(),
436 vector: v, acc: a, fastMathFlags: multiReductionOp.getFastmath());
437
438 if (mask) {
439 Value m = vector::ExtractOp::create(builder&: rewriter, location: loc, source: mask, position: i);
440 reductionOp = mlir::vector::maskOperation(builder&: rewriter, maskableOp: reductionOp, mask: m);
441 }
442
443 result = vector::InsertOp::create(builder&: rewriter, location: loc,
444 valueToStore: reductionOp->getResult(idx: 0), dest: result, position: i);
445 }
446
447 return result;
448 }
449};
450
451/// Converts 1D vector.multi_reduction directly to vector.reduction.
452///
453/// Example:
454/// ```mlir
455/// // Before
456/// %r = vector.multi_reduction <add>, %v, %acc [0] : vector<Nxf32> to f32
457///
458/// // After
459/// %r = vector.reduction <add>, %v, %acc : vector<Nxf32> into f32
460/// ```
461struct OneDimMultiReductionToReduction
462 : public vector::MaskableOpRewritePattern<vector::MultiDimReductionOp> {
463 using MaskableOpRewritePattern::MaskableOpRewritePattern;
464
465 FailureOr<Value>
466 matchAndRewriteMaskableOp(vector::MultiDimReductionOp multiReductionOp,
467 vector::MaskingOpInterface maskingOp,
468 PatternRewriter &rewriter) const override {
469 auto srcRank = multiReductionOp.getSourceVectorType().getRank();
470 if (srcRank != 1)
471 return failure();
472
473 if (!multiReductionOp.isReducedDim(d: 0))
474 return failure();
475
476 auto loc = multiReductionOp.getLoc();
477 Value mask = maskingOp ? maskingOp.getMask() : Value();
478
479 Operation *reductionOp = vector::ReductionOp::create(
480 builder&: rewriter, location: loc, kind: multiReductionOp.getKind(), vector: multiReductionOp.getSource(),
481 acc: multiReductionOp.getAcc(), fastMathFlags: multiReductionOp.getFastmath());
482
483 if (mask)
484 reductionOp = mlir::vector::maskOperation(builder&: rewriter, maskableOp: reductionOp, mask);
485
486 return reductionOp->getResult(idx: 0);
487 }
488};
489
490struct LowerVectorMultiReductionPass
491 : public vector::impl::LowerVectorMultiReductionBase<
492 LowerVectorMultiReductionPass> {
493 LowerVectorMultiReductionPass(vector::VectorMultiReductionLowering option) {
494 this->loweringStrategy = option;
495 }
496
497 void runOnOperation() override {
498 Operation *op = getOperation();
499 MLIRContext *context = op->getContext();
500
501 RewritePatternSet patterns(context);
502 mlir::vector::populateVectorMultiReductionReorderPatterns(
503 patterns, options: this->loweringStrategy);
504 if (failed(Result: applyPatternsGreedily(op, patterns: std::move(patterns))))
505 signalPassFailure();
506
507 RewritePatternSet flatteningPatterns(context);
508 mlir::vector::populateVectorMultiReductionFlatteningPatterns(
509 patterns&: flatteningPatterns, options: this->loweringStrategy);
510 if (failed(Result: applyPatternsGreedily(op, patterns: std::move(flatteningPatterns))))
511 signalPassFailure();
512
513 RewritePatternSet unrollingPatterns(context);
514 mlir::vector::populateVectorMultiReductionUnrollingPatterns(
515 patterns&: unrollingPatterns, options: this->loweringStrategy);
516 if (failed(Result: applyPatternsGreedily(op, patterns: std::move(unrollingPatterns))))
517 signalPassFailure();
518 }
519
520 void getDependentDialects(DialectRegistry &registry) const override {
521 registry.insert<vector::VectorDialect>();
522 }
523};
524
525} // namespace
526
527void mlir::vector::populateVectorMultiReductionReorderPatterns(
528 RewritePatternSet &patterns, VectorMultiReductionLowering options,
529 PatternBenefit benefit) {
530 patterns.add<InnerOuterDimReductionConversion>(arg: patterns.getContext(), args&: options,
531 args&: benefit);
532}
533
534void mlir::vector::populateVectorMultiReductionFlatteningPatterns(
535 RewritePatternSet &patterns, VectorMultiReductionLowering options,
536 PatternBenefit benefit) {
537 patterns.add<FlattenMultiReduction>(arg: patterns.getContext(), args&: options, args&: benefit);
538}
539
540void mlir::vector::populateVectorMultiReductionUnrollingPatterns(
541 RewritePatternSet &patterns, VectorMultiReductionLowering options,
542 PatternBenefit benefit) {
543 patterns.add<OneDimMultiReductionToReduction>(arg: patterns.getContext(), args&: benefit);
544 if (options == VectorMultiReductionLowering ::InnerReduction)
545 patterns.add<TwoDimMultiReductionToReduction>(arg: patterns.getContext(),
546 args&: benefit);
547 else
548 patterns.add<TwoDimMultiReductionToElementWise>(arg: patterns.getContext(),
549 args&: benefit);
550}
551
552std::unique_ptr<Pass> vector::createLowerVectorMultiReductionPass(
553 vector::VectorMultiReductionLowering option) {
554 return std::make_unique<LowerVectorMultiReductionPass>(args&: option);
555}
556