| 1 | //===- LowerVectorMultiReduction.cpp - Lower `vector.multi_reduction` op --===// |
| 2 | // |
| 3 | /// Part of the LLVM Project, under the Apache License v2.0 with LLVM |
| 4 | /// Exceptions. See https://llvm.org/LICENSE.txt for license information. |
| 5 | /// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements target-independent rewrites and utilities to lower the |
| 10 | // 'vector.multi_reduction' operation. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 15 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 16 | #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" |
| 17 | #include "mlir/Dialect/Vector/Transforms/Passes.h" |
| 18 | #include "mlir/IR/Builders.h" |
| 19 | #include "mlir/IR/TypeUtilities.h" |
| 20 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 21 | |
| 22 | namespace mlir { |
| 23 | namespace vector { |
| 24 | #define GEN_PASS_DEF_LOWERVECTORMULTIREDUCTION |
| 25 | #include "mlir/Dialect/Vector/Transforms/Passes.h.inc" |
| 26 | } // namespace vector |
| 27 | } // namespace mlir |
| 28 | |
| 29 | #define DEBUG_TYPE "vector-multi-reduction" |
| 30 | |
| 31 | using namespace mlir; |
| 32 | |
| 33 | namespace { |
| 34 | /// This file implements the following transformations as composable atomic |
| 35 | /// patterns. |
| 36 | |
| 37 | /// Converts vector.multi_reduction into inner-most/outer-most reduction form |
| 38 | /// by using vector.transpose |
| 39 | class InnerOuterDimReductionConversion |
| 40 | : public OpRewritePattern<vector::MultiDimReductionOp> { |
| 41 | public: |
| 42 | using Base::Base; |
| 43 | |
| 44 | explicit InnerOuterDimReductionConversion( |
| 45 | MLIRContext *context, vector::VectorMultiReductionLowering options, |
| 46 | PatternBenefit benefit = 1) |
| 47 | : mlir::OpRewritePattern<vector::MultiDimReductionOp>(context, benefit), |
| 48 | useInnerDimsForReduction( |
| 49 | options == vector::VectorMultiReductionLowering::InnerReduction) {} |
| 50 | |
| 51 | LogicalResult matchAndRewrite(vector::MultiDimReductionOp multiReductionOp, |
| 52 | PatternRewriter &rewriter) const override { |
| 53 | // Vector mask setup. |
| 54 | OpBuilder::InsertionGuard guard(rewriter); |
| 55 | auto maskableOp = |
| 56 | cast<vector::MaskableOpInterface>(Val: multiReductionOp.getOperation()); |
| 57 | Operation *rootOp; |
| 58 | if (maskableOp.isMasked()) { |
| 59 | rewriter.setInsertionPoint(maskableOp.getMaskingOp()); |
| 60 | rootOp = maskableOp.getMaskingOp(); |
| 61 | } else { |
| 62 | rootOp = multiReductionOp; |
| 63 | } |
| 64 | |
| 65 | auto src = multiReductionOp.getSource(); |
| 66 | auto loc = multiReductionOp.getLoc(); |
| 67 | auto srcRank = multiReductionOp.getSourceVectorType().getRank(); |
| 68 | |
| 69 | // Separate reduction and parallel dims |
| 70 | ArrayRef<int64_t> reductionDims = multiReductionOp.getReductionDims(); |
| 71 | llvm::SmallDenseSet<int64_t> reductionDimsSet(reductionDims.begin(), |
| 72 | reductionDims.end()); |
| 73 | int64_t reductionSize = reductionDims.size(); |
| 74 | SmallVector<int64_t, 4> parallelDims; |
| 75 | for (int64_t i = 0; i < srcRank; ++i) |
| 76 | if (!reductionDimsSet.contains(V: i)) |
| 77 | parallelDims.push_back(Elt: i); |
| 78 | |
| 79 | // Add transpose only if inner-most/outer-most dimensions are not parallel |
| 80 | // and there are parallel dims. |
| 81 | if (parallelDims.empty()) |
| 82 | return failure(); |
| 83 | if (useInnerDimsForReduction && |
| 84 | (parallelDims == |
| 85 | llvm::to_vector<4>(Range: llvm::seq<int64_t>(Begin: 0, End: parallelDims.size())))) |
| 86 | return failure(); |
| 87 | |
| 88 | if (!useInnerDimsForReduction && |
| 89 | (parallelDims == llvm::to_vector<4>(Range: llvm::seq<int64_t>( |
| 90 | Begin: reductionDims.size(), |
| 91 | End: parallelDims.size() + reductionDims.size())))) |
| 92 | return failure(); |
| 93 | |
| 94 | SmallVector<int64_t, 4> indices; |
| 95 | if (useInnerDimsForReduction) { |
| 96 | indices.append(in_start: parallelDims.begin(), in_end: parallelDims.end()); |
| 97 | indices.append(in_start: reductionDims.begin(), in_end: reductionDims.end()); |
| 98 | } else { |
| 99 | indices.append(in_start: reductionDims.begin(), in_end: reductionDims.end()); |
| 100 | indices.append(in_start: parallelDims.begin(), in_end: parallelDims.end()); |
| 101 | } |
| 102 | |
| 103 | // If masked, transpose the original mask. |
| 104 | Value transposedMask; |
| 105 | if (maskableOp.isMasked()) { |
| 106 | transposedMask = vector::TransposeOp::create( |
| 107 | builder&: rewriter, location: loc, vector: maskableOp.getMaskingOp().getMask(), permutation: indices); |
| 108 | } |
| 109 | |
| 110 | // Transpose reduction source. |
| 111 | auto transposeOp = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: src, permutation: indices); |
| 112 | SmallVector<bool> reductionMask(srcRank, false); |
| 113 | for (int i = 0; i < reductionSize; ++i) { |
| 114 | if (useInnerDimsForReduction) |
| 115 | reductionMask[srcRank - i - 1] = true; |
| 116 | else |
| 117 | reductionMask[i] = true; |
| 118 | } |
| 119 | |
| 120 | Operation *newMultiRedOp = vector::MultiDimReductionOp::create( |
| 121 | builder&: rewriter, location: multiReductionOp.getLoc(), source: transposeOp.getResult(), |
| 122 | acc: multiReductionOp.getAcc(), reductionMask, kind: multiReductionOp.getKind(), |
| 123 | fastMathFlags: multiReductionOp.getFastmath()); |
| 124 | newMultiRedOp = |
| 125 | mlir::vector::maskOperation(builder&: rewriter, maskableOp: newMultiRedOp, mask: transposedMask); |
| 126 | |
| 127 | rewriter.replaceOp(op: rootOp, newValues: newMultiRedOp->getResult(idx: 0)); |
| 128 | return success(); |
| 129 | } |
| 130 | |
| 131 | private: |
| 132 | const bool useInnerDimsForReduction; |
| 133 | }; |
| 134 | |
| 135 | /// Flattens vector.multi_reduction to 2D |
| 136 | /// |
| 137 | /// Given all reduction dimensions are either inner most or outer most, |
| 138 | /// flattens all reduction and parallel dimensions so that there are only 2Ds. |
| 139 | /// |
| 140 | /// BEFORE |
| 141 | /// vector.multi_reduction <add>, %vec, %acc [2, 3] : vector<2x3x4x5xi32> to |
| 142 | /// vector<2x3xi32> |
| 143 | /// AFTER |
| 144 | /// %vec_sc = vector.shape_cast %vec |
| 145 | /// %acc_sc = vector.shape_cast %acc |
| 146 | /// %res = vector.multi_reduction <add>, %vec_sc, %acc_cs [1] : |
| 147 | /// vector<6x20xi32> to vector<6xi32> %res_sc = vector.shape_cast %res |
| 148 | class FlattenMultiReduction |
| 149 | : public OpRewritePattern<vector::MultiDimReductionOp> { |
| 150 | public: |
| 151 | using Base::Base; |
| 152 | |
| 153 | explicit FlattenMultiReduction(MLIRContext *context, |
| 154 | vector::VectorMultiReductionLowering options, |
| 155 | PatternBenefit benefit = 1) |
| 156 | : mlir::OpRewritePattern<vector::MultiDimReductionOp>(context, benefit), |
| 157 | useInnerDimsForReduction( |
| 158 | options == vector::VectorMultiReductionLowering::InnerReduction) {} |
| 159 | |
| 160 | LogicalResult matchAndRewrite(vector::MultiDimReductionOp multiReductionOp, |
| 161 | PatternRewriter &rewriter) const override { |
| 162 | // Vector mask setup. |
| 163 | OpBuilder::InsertionGuard guard(rewriter); |
| 164 | auto maskableOp = |
| 165 | cast<vector::MaskableOpInterface>(Val: multiReductionOp.getOperation()); |
| 166 | Operation *rootOp; |
| 167 | if (maskableOp.isMasked()) { |
| 168 | rewriter.setInsertionPoint(maskableOp.getMaskingOp()); |
| 169 | rootOp = maskableOp.getMaskingOp(); |
| 170 | } else { |
| 171 | rootOp = multiReductionOp; |
| 172 | } |
| 173 | |
| 174 | auto srcRank = multiReductionOp.getSourceVectorType().getRank(); |
| 175 | auto srcShape = multiReductionOp.getSourceVectorType().getShape(); |
| 176 | auto srcScalableDims = |
| 177 | multiReductionOp.getSourceVectorType().getScalableDims(); |
| 178 | auto loc = multiReductionOp.getLoc(); |
| 179 | |
| 180 | // If rank less than 2, nothing to do. |
| 181 | if (srcRank < 2) |
| 182 | return failure(); |
| 183 | |
| 184 | // Allow only 1 scalable dimensions. Otherwise we could end-up with e.g. |
| 185 | // `vscale * vscale` that's currently not modelled. |
| 186 | if (llvm::count(Range&: srcScalableDims, Element: true) > 1) |
| 187 | return failure(); |
| 188 | |
| 189 | // If already rank-2 ["parallel", "reduce"] or ["reduce", "parallel"] bail. |
| 190 | SmallVector<bool> reductionMask = multiReductionOp.getReductionMask(); |
| 191 | if (srcRank == 2 && reductionMask.front() != reductionMask.back()) |
| 192 | return failure(); |
| 193 | |
| 194 | // 1. Separate reduction and parallel dims. |
| 195 | SmallVector<int64_t, 4> parallelDims, parallelShapes; |
| 196 | SmallVector<bool, 4> parallelScalableDims; |
| 197 | SmallVector<int64_t, 4> reductionDims, reductionShapes; |
| 198 | bool isReductionDimScalable = false; |
| 199 | for (const auto &it : llvm::enumerate(First&: reductionMask)) { |
| 200 | int64_t i = it.index(); |
| 201 | bool isReduction = it.value(); |
| 202 | if (isReduction) { |
| 203 | reductionDims.push_back(Elt: i); |
| 204 | reductionShapes.push_back(Elt: srcShape[i]); |
| 205 | isReductionDimScalable |= srcScalableDims[i]; |
| 206 | } else { |
| 207 | parallelDims.push_back(Elt: i); |
| 208 | parallelShapes.push_back(Elt: srcShape[i]); |
| 209 | parallelScalableDims.push_back(Elt: srcScalableDims[i]); |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | // 2. Compute flattened parallel and reduction sizes. |
| 214 | int flattenedParallelDim = 0; |
| 215 | int flattenedReductionDim = 0; |
| 216 | if (!parallelShapes.empty()) { |
| 217 | flattenedParallelDim = 1; |
| 218 | for (auto d : parallelShapes) |
| 219 | flattenedParallelDim *= d; |
| 220 | } |
| 221 | if (!reductionShapes.empty()) { |
| 222 | flattenedReductionDim = 1; |
| 223 | for (auto d : reductionShapes) |
| 224 | flattenedReductionDim *= d; |
| 225 | } |
| 226 | // We must at least have some parallel or some reduction. |
| 227 | assert((flattenedParallelDim || flattenedReductionDim) && |
| 228 | "expected at least one parallel or reduction dim" ); |
| 229 | |
| 230 | // 3. Fail if reduction/parallel dims are not contiguous. |
| 231 | // Check parallelDims are exactly [0 .. size). |
| 232 | int64_t counter = 0; |
| 233 | if (useInnerDimsForReduction && |
| 234 | llvm::any_of(Range&: parallelDims, P: [&](int64_t i) { return i != counter++; })) |
| 235 | return failure(); |
| 236 | // Check parallelDims are exactly {reductionDims.size()} + [0 .. size). |
| 237 | counter = reductionDims.size(); |
| 238 | if (!useInnerDimsForReduction && |
| 239 | llvm::any_of(Range&: parallelDims, P: [&](int64_t i) { return i != counter++; })) |
| 240 | return failure(); |
| 241 | |
| 242 | // 4. Shape cast to collapse consecutive parallel (resp. reduction dim) into |
| 243 | // a single parallel (resp. reduction) dim. |
| 244 | SmallVector<bool, 2> mask; |
| 245 | SmallVector<bool, 2> scalableDims; |
| 246 | SmallVector<int64_t, 2> vectorShape; |
| 247 | bool isParallelDimScalable = llvm::is_contained(Range&: parallelScalableDims, Element: true); |
| 248 | if (flattenedParallelDim) { |
| 249 | mask.push_back(Elt: false); |
| 250 | vectorShape.push_back(Elt: flattenedParallelDim); |
| 251 | scalableDims.push_back(Elt: isParallelDimScalable); |
| 252 | } |
| 253 | if (flattenedReductionDim) { |
| 254 | mask.push_back(Elt: true); |
| 255 | vectorShape.push_back(Elt: flattenedReductionDim); |
| 256 | scalableDims.push_back(Elt: isReductionDimScalable); |
| 257 | } |
| 258 | if (!useInnerDimsForReduction && vectorShape.size() == 2) { |
| 259 | std::swap(a&: mask.front(), b&: mask.back()); |
| 260 | std::swap(a&: vectorShape.front(), b&: vectorShape.back()); |
| 261 | std::swap(a&: scalableDims.front(), b&: scalableDims.back()); |
| 262 | } |
| 263 | |
| 264 | Value newVectorMask; |
| 265 | if (maskableOp.isMasked()) { |
| 266 | Value vectorMask = maskableOp.getMaskingOp().getMask(); |
| 267 | auto maskCastedType = VectorType::get( |
| 268 | shape: vectorShape, |
| 269 | elementType: llvm::cast<VectorType>(Val: vectorMask.getType()).getElementType(), |
| 270 | scalableDims); |
| 271 | newVectorMask = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: maskCastedType, |
| 272 | source: vectorMask); |
| 273 | } |
| 274 | |
| 275 | auto castedType = VectorType::get( |
| 276 | shape: vectorShape, elementType: multiReductionOp.getSourceVectorType().getElementType(), |
| 277 | scalableDims); |
| 278 | Value cast = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: castedType, |
| 279 | source: multiReductionOp.getSource()); |
| 280 | |
| 281 | Value acc = multiReductionOp.getAcc(); |
| 282 | if (flattenedParallelDim) { |
| 283 | auto accType = VectorType::get( |
| 284 | shape: {flattenedParallelDim}, |
| 285 | elementType: multiReductionOp.getSourceVectorType().getElementType(), |
| 286 | /*scalableDims=*/{isParallelDimScalable}); |
| 287 | acc = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: accType, source: acc); |
| 288 | } |
| 289 | // 6. Creates the flattened form of vector.multi_reduction with inner/outer |
| 290 | // most dim as reduction. |
| 291 | Operation *newMultiDimRedOp = vector::MultiDimReductionOp::create( |
| 292 | builder&: rewriter, location: loc, source: cast, acc, reductionMask: mask, kind: multiReductionOp.getKind(), |
| 293 | fastMathFlags: multiReductionOp.getFastmath()); |
| 294 | newMultiDimRedOp = |
| 295 | mlir::vector::maskOperation(builder&: rewriter, maskableOp: newMultiDimRedOp, mask: newVectorMask); |
| 296 | |
| 297 | // 7. If there are no parallel shapes, the result is a scalar. |
| 298 | // TODO: support 0-d vectors when available. |
| 299 | if (parallelShapes.empty()) { |
| 300 | rewriter.replaceOp(op: rootOp, newValues: newMultiDimRedOp->getResult(idx: 0)); |
| 301 | return success(); |
| 302 | } |
| 303 | |
| 304 | // 8. Shape cast the flattened result back to the original n-D parallel |
| 305 | // shape. |
| 306 | VectorType outputCastedType = VectorType::get( |
| 307 | shape: parallelShapes, elementType: multiReductionOp.getSourceVectorType().getElementType(), |
| 308 | scalableDims: parallelScalableDims); |
| 309 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>( |
| 310 | op: rootOp, args&: outputCastedType, args: newMultiDimRedOp->getResult(idx: 0)); |
| 311 | return success(); |
| 312 | } |
| 313 | |
| 314 | private: |
| 315 | const bool useInnerDimsForReduction; |
| 316 | }; |
| 317 | |
| 318 | /// Lowers 2D vector.multi_reduction to a squence of Arith Ops |
| 319 | /// |
| 320 | /// The reduction dimension must be the outer-most dimension. |
| 321 | /// |
| 322 | /// BEFORE: |
| 323 | /// |
| 324 | /// %1 = vector.multi_reduction <mul>, %src, %acc [0] : vector<4x2xf32> to |
| 325 | /// vector<2xf32> |
| 326 | /// |
| 327 | /// AFTER: |
| 328 | /// |
| 329 | /// // Prod 1. |
| 330 | /// %vec_0 = vector.extract %src[0] : vector<2xf32> from vector<4x2xf32> |
| 331 | /// %mul_0 = arith.mulf %vec_0, %acc : vector<2xf32> |
| 332 | /// |
| 333 | /// // Prod 2. |
| 334 | /// %vec_1 = vector.extract %src[1] : vector<2xf32> from vector<4x2xf32> |
| 335 | /// %mul_2 = arith.mulf %vec_1, %mul_0 : vector<2xf32> |
| 336 | /// |
| 337 | /// // Prod 3. |
| 338 | /// %vec_3 = vector.extract %src[2] : vector<2xf32> from vector<4x2xf32> |
| 339 | /// %mul_3 = arith.mulf %vec_3, %mul_2 : vector<2xf32> |
| 340 | /// |
| 341 | /// // Prod 4. |
| 342 | /// %vec_4 = vector.extract %src[3] : vector<2xf32> from vector<4x2xf32> |
| 343 | /// %res = arith.mulf %vec_4, %mul_3 : vector<2xf32> |
| 344 | struct TwoDimMultiReductionToElementWise |
| 345 | : public vector::MaskableOpRewritePattern<vector::MultiDimReductionOp> { |
| 346 | using MaskableOpRewritePattern::MaskableOpRewritePattern; |
| 347 | |
| 348 | FailureOr<Value> |
| 349 | matchAndRewriteMaskableOp(vector::MultiDimReductionOp multiReductionOp, |
| 350 | vector::MaskingOpInterface maskingOp, |
| 351 | PatternRewriter &rewriter) const override { |
| 352 | auto srcRank = multiReductionOp.getSourceVectorType().getRank(); |
| 353 | // Rank-2 ["parallel", "reduce"] or bail. |
| 354 | if (srcRank != 2) |
| 355 | return failure(); |
| 356 | |
| 357 | if (multiReductionOp.isReducedDim(d: 1) || !multiReductionOp.isReducedDim(d: 0)) |
| 358 | return failure(); |
| 359 | |
| 360 | Value mask = maskingOp ? maskingOp.getMask() : Value(); |
| 361 | |
| 362 | auto loc = multiReductionOp.getLoc(); |
| 363 | Value source = multiReductionOp.getSource(); |
| 364 | ArrayRef<int64_t> srcShape = |
| 365 | multiReductionOp.getSourceVectorType().getShape(); |
| 366 | int outerDim = srcShape[0]; |
| 367 | |
| 368 | Value result = multiReductionOp.getAcc(); |
| 369 | for (int64_t i = 0; i < outerDim; i++) { |
| 370 | auto v = vector::ExtractOp::create(builder&: rewriter, location: loc, source, position: i); |
| 371 | Value m = mask ? Value(vector::ExtractOp::create(builder&: rewriter, location: loc, source: mask, position: i)) |
| 372 | : nullptr; |
| 373 | result = |
| 374 | makeArithReduction(b&: rewriter, loc, kind: multiReductionOp.getKind(), v1: v, |
| 375 | acc: result, fastmath: multiReductionOp.getFastmathAttr(), mask: m); |
| 376 | } |
| 377 | |
| 378 | return result; |
| 379 | } |
| 380 | }; |
| 381 | |
| 382 | /// Lowers 2D vector.multi_reduction to a sequence of vector.reduction Ops. |
| 383 | /// |
| 384 | /// The reduction dimension must be the inner-most dimension. |
| 385 | /// |
| 386 | /// BEFORE: |
| 387 | /// vector.multi_reduction <mul>, %src, %acc [1] : vector<2x4xf32> to |
| 388 | /// vector<2xf32> |
| 389 | /// |
| 390 | /// AFTER: |
| 391 | /// // 1st reduction |
| 392 | /// %v_0 = vector.extract %src[0] : vector<4xf32> from vector<2x4xf32> |
| 393 | /// %a_0 = vector.extract %acc[0] : f32 from vector<2xf32> |
| 394 | /// %red_1 = vector.reduction <mul>, %v_0, %a_1 : vector<4xf32> into f32 |
| 395 | /// %res_tmp = vector.insert %red_1, %res [0] : f32 into vector<2xf32> |
| 396 | /// |
| 397 | /// // 2nd reduction |
| 398 | /// %v_1 = vector.extract %src[1] : vector<4xf32> from vector<2x4xf32> |
| 399 | /// %a_1 = vector.extract %acc[1] : f32 from vector<2xf32> |
| 400 | /// %red_2 = vector.reduction <mul>, %v_1, %a_1 : vector<4xf32> into f32 |
| 401 | /// %res_final = vector.insert %red_2, %res_tmp [1] : f32 into vector<2xf32> |
| 402 | struct TwoDimMultiReductionToReduction |
| 403 | : public vector::MaskableOpRewritePattern<vector::MultiDimReductionOp> { |
| 404 | using MaskableOpRewritePattern::MaskableOpRewritePattern; |
| 405 | |
| 406 | FailureOr<Value> |
| 407 | matchAndRewriteMaskableOp(vector::MultiDimReductionOp multiReductionOp, |
| 408 | vector::MaskingOpInterface maskingOp, |
| 409 | PatternRewriter &rewriter) const override { |
| 410 | auto srcRank = multiReductionOp.getSourceVectorType().getRank(); |
| 411 | // Rank-2 ["reduce", "parallel"] or bail. |
| 412 | if (srcRank != 2) |
| 413 | return failure(); |
| 414 | |
| 415 | if (multiReductionOp.isReducedDim(d: 0) || !multiReductionOp.isReducedDim(d: 1)) |
| 416 | return failure(); |
| 417 | |
| 418 | Value mask = maskingOp ? maskingOp.getMask() : nullptr; |
| 419 | |
| 420 | auto loc = multiReductionOp.getLoc(); |
| 421 | Value source = multiReductionOp.getSource(); |
| 422 | Value acc = multiReductionOp.getAcc(); |
| 423 | int outerDim = multiReductionOp.getSourceVectorType().getShape()[0]; |
| 424 | |
| 425 | Value result = arith::ConstantOp::create( |
| 426 | builder&: rewriter, location: loc, result: multiReductionOp.getDestType(), |
| 427 | value: rewriter.getZeroAttr(type: multiReductionOp.getDestType())); |
| 428 | |
| 429 | SmallVector<Value> vectors(outerDim); |
| 430 | for (int64_t i = 0; i < outerDim; ++i) { |
| 431 | Value v = vector::ExtractOp::create(builder&: rewriter, location: loc, source, position: i); |
| 432 | Value a = vector::ExtractOp::create(builder&: rewriter, location: loc, source: acc, position: i); |
| 433 | |
| 434 | Operation *reductionOp = |
| 435 | vector::ReductionOp::create(builder&: rewriter, location: loc, kind: multiReductionOp.getKind(), |
| 436 | vector: v, acc: a, fastMathFlags: multiReductionOp.getFastmath()); |
| 437 | |
| 438 | if (mask) { |
| 439 | Value m = vector::ExtractOp::create(builder&: rewriter, location: loc, source: mask, position: i); |
| 440 | reductionOp = mlir::vector::maskOperation(builder&: rewriter, maskableOp: reductionOp, mask: m); |
| 441 | } |
| 442 | |
| 443 | result = vector::InsertOp::create(builder&: rewriter, location: loc, |
| 444 | valueToStore: reductionOp->getResult(idx: 0), dest: result, position: i); |
| 445 | } |
| 446 | |
| 447 | return result; |
| 448 | } |
| 449 | }; |
| 450 | |
| 451 | /// Converts 1D vector.multi_reduction directly to vector.reduction. |
| 452 | /// |
| 453 | /// Example: |
| 454 | /// ```mlir |
| 455 | /// // Before |
| 456 | /// %r = vector.multi_reduction <add>, %v, %acc [0] : vector<Nxf32> to f32 |
| 457 | /// |
| 458 | /// // After |
| 459 | /// %r = vector.reduction <add>, %v, %acc : vector<Nxf32> into f32 |
| 460 | /// ``` |
| 461 | struct OneDimMultiReductionToReduction |
| 462 | : public vector::MaskableOpRewritePattern<vector::MultiDimReductionOp> { |
| 463 | using MaskableOpRewritePattern::MaskableOpRewritePattern; |
| 464 | |
| 465 | FailureOr<Value> |
| 466 | matchAndRewriteMaskableOp(vector::MultiDimReductionOp multiReductionOp, |
| 467 | vector::MaskingOpInterface maskingOp, |
| 468 | PatternRewriter &rewriter) const override { |
| 469 | auto srcRank = multiReductionOp.getSourceVectorType().getRank(); |
| 470 | if (srcRank != 1) |
| 471 | return failure(); |
| 472 | |
| 473 | if (!multiReductionOp.isReducedDim(d: 0)) |
| 474 | return failure(); |
| 475 | |
| 476 | auto loc = multiReductionOp.getLoc(); |
| 477 | Value mask = maskingOp ? maskingOp.getMask() : Value(); |
| 478 | |
| 479 | Operation *reductionOp = vector::ReductionOp::create( |
| 480 | builder&: rewriter, location: loc, kind: multiReductionOp.getKind(), vector: multiReductionOp.getSource(), |
| 481 | acc: multiReductionOp.getAcc(), fastMathFlags: multiReductionOp.getFastmath()); |
| 482 | |
| 483 | if (mask) |
| 484 | reductionOp = mlir::vector::maskOperation(builder&: rewriter, maskableOp: reductionOp, mask); |
| 485 | |
| 486 | return reductionOp->getResult(idx: 0); |
| 487 | } |
| 488 | }; |
| 489 | |
| 490 | struct LowerVectorMultiReductionPass |
| 491 | : public vector::impl::LowerVectorMultiReductionBase< |
| 492 | LowerVectorMultiReductionPass> { |
| 493 | LowerVectorMultiReductionPass(vector::VectorMultiReductionLowering option) { |
| 494 | this->loweringStrategy = option; |
| 495 | } |
| 496 | |
| 497 | void runOnOperation() override { |
| 498 | Operation *op = getOperation(); |
| 499 | MLIRContext *context = op->getContext(); |
| 500 | |
| 501 | RewritePatternSet patterns(context); |
| 502 | mlir::vector::populateVectorMultiReductionReorderPatterns( |
| 503 | patterns, options: this->loweringStrategy); |
| 504 | if (failed(Result: applyPatternsGreedily(op, patterns: std::move(patterns)))) |
| 505 | signalPassFailure(); |
| 506 | |
| 507 | RewritePatternSet flatteningPatterns(context); |
| 508 | mlir::vector::populateVectorMultiReductionFlatteningPatterns( |
| 509 | patterns&: flatteningPatterns, options: this->loweringStrategy); |
| 510 | if (failed(Result: applyPatternsGreedily(op, patterns: std::move(flatteningPatterns)))) |
| 511 | signalPassFailure(); |
| 512 | |
| 513 | RewritePatternSet unrollingPatterns(context); |
| 514 | mlir::vector::populateVectorMultiReductionUnrollingPatterns( |
| 515 | patterns&: unrollingPatterns, options: this->loweringStrategy); |
| 516 | if (failed(Result: applyPatternsGreedily(op, patterns: std::move(unrollingPatterns)))) |
| 517 | signalPassFailure(); |
| 518 | } |
| 519 | |
| 520 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 521 | registry.insert<vector::VectorDialect>(); |
| 522 | } |
| 523 | }; |
| 524 | |
| 525 | } // namespace |
| 526 | |
| 527 | void mlir::vector::populateVectorMultiReductionReorderPatterns( |
| 528 | RewritePatternSet &patterns, VectorMultiReductionLowering options, |
| 529 | PatternBenefit benefit) { |
| 530 | patterns.add<InnerOuterDimReductionConversion>(arg: patterns.getContext(), args&: options, |
| 531 | args&: benefit); |
| 532 | } |
| 533 | |
| 534 | void mlir::vector::populateVectorMultiReductionFlatteningPatterns( |
| 535 | RewritePatternSet &patterns, VectorMultiReductionLowering options, |
| 536 | PatternBenefit benefit) { |
| 537 | patterns.add<FlattenMultiReduction>(arg: patterns.getContext(), args&: options, args&: benefit); |
| 538 | } |
| 539 | |
| 540 | void mlir::vector::populateVectorMultiReductionUnrollingPatterns( |
| 541 | RewritePatternSet &patterns, VectorMultiReductionLowering options, |
| 542 | PatternBenefit benefit) { |
| 543 | patterns.add<OneDimMultiReductionToReduction>(arg: patterns.getContext(), args&: benefit); |
| 544 | if (options == VectorMultiReductionLowering ::InnerReduction) |
| 545 | patterns.add<TwoDimMultiReductionToReduction>(arg: patterns.getContext(), |
| 546 | args&: benefit); |
| 547 | else |
| 548 | patterns.add<TwoDimMultiReductionToElementWise>(arg: patterns.getContext(), |
| 549 | args&: benefit); |
| 550 | } |
| 551 | |
| 552 | std::unique_ptr<Pass> vector::createLowerVectorMultiReductionPass( |
| 553 | vector::VectorMultiReductionLowering option) { |
| 554 | return std::make_unique<LowerVectorMultiReductionPass>(args&: option); |
| 555 | } |
| 556 | |