| 1 | //===- LowerVectorBroadcast.cpp - Lower 'vector.broadcast' operation ------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements target-independent rewrites and utilities to lower the |
| 10 | // 'vector.broadcast' operation. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 15 | #include "mlir/Dialect/UB/IR/UBOps.h" |
| 16 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 17 | #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" |
| 18 | #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" |
| 19 | #include "mlir/Dialect/Vector/Utils/VectorUtils.h" |
| 20 | #include "mlir/IR/BuiltinTypes.h" |
| 21 | #include "mlir/IR/Location.h" |
| 22 | #include "mlir/IR/PatternMatch.h" |
| 23 | #include "mlir/IR/TypeUtilities.h" |
| 24 | |
| 25 | #define DEBUG_TYPE "vector-broadcast-lowering" |
| 26 | |
| 27 | using namespace mlir; |
| 28 | using namespace mlir::vector; |
| 29 | |
| 30 | namespace { |
| 31 | |
| 32 | /// Convert a vector.broadcast with a vector operand to a lower rank |
| 33 | /// vector.broadcast. vector.broadcast with a scalar operand is expected to be |
| 34 | /// convertible to the lower level target dialect (LLVM, SPIR-V, etc.) directly. |
| 35 | class BroadcastOpLowering : public OpRewritePattern<vector::BroadcastOp> { |
| 36 | public: |
| 37 | using Base::Base; |
| 38 | |
| 39 | LogicalResult matchAndRewrite(vector::BroadcastOp op, |
| 40 | PatternRewriter &rewriter) const override { |
| 41 | auto loc = op.getLoc(); |
| 42 | VectorType dstType = op.getResultVectorType(); |
| 43 | VectorType srcType = dyn_cast<VectorType>(Val: op.getSourceType()); |
| 44 | Type eltType = dstType.getElementType(); |
| 45 | |
| 46 | // A broadcast from a scalar is considered to be in the lowered form. |
| 47 | if (!srcType) |
| 48 | return rewriter.notifyMatchFailure( |
| 49 | arg&: op, msg: "broadcast from scalar already in lowered form" ); |
| 50 | |
| 51 | // Determine rank of source and destination. |
| 52 | int64_t srcRank = srcType.getRank(); |
| 53 | int64_t dstRank = dstType.getRank(); |
| 54 | |
| 55 | // Single-element fixed-size source: extract the scalar and broadcast it. |
| 56 | if (srcType.getNumElements() == 1 && !srcType.isScalable()) { |
| 57 | SmallVector<int64_t> fullRankPosition(srcRank, 0); |
| 58 | Value ext = vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getSource(), |
| 59 | position: fullRankPosition); |
| 60 | assert(!isa<VectorType>(ext.getType()) && "expected scalar" ); |
| 61 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op, args&: dstType, args&: ext); |
| 62 | return success(); |
| 63 | } |
| 64 | |
| 65 | // Duplicate this rank. |
| 66 | // For example: |
| 67 | // %x = broadcast %y : k-D to n-D, k < n |
| 68 | // becomes: |
| 69 | // %b = broadcast %y : k-D to (n-1)-D |
| 70 | // %x = [%b,%b,%b,%b] : n-D |
| 71 | // becomes: |
| 72 | // %b = [%y,%y] : (n-1)-D |
| 73 | // %x = [%b,%b,%b,%b] : n-D |
| 74 | if (srcRank < dstRank) { |
| 75 | // Duplication. |
| 76 | if (dstType.getScalableDims()[0]) |
| 77 | return rewriter.notifyMatchFailure( |
| 78 | arg&: op, msg: "Vector broadcasting over a scalable dimension is not " |
| 79 | "currently supported" ); |
| 80 | VectorType resType = VectorType::Builder(dstType).dropDim(pos: 0); |
| 81 | Value bcst = |
| 82 | vector::BroadcastOp::create(builder&: rewriter, location: loc, result: resType, source: op.getSource()); |
| 83 | Value result = ub::PoisonOp::create(builder&: rewriter, location: loc, result: dstType); |
| 84 | for (int64_t d = 0, dim = dstType.getDimSize(idx: 0); d < dim; ++d) |
| 85 | result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: bcst, dest: result, position: d); |
| 86 | rewriter.replaceOp(op, newValues: result); |
| 87 | return success(); |
| 88 | } |
| 89 | |
| 90 | // Find non-matching dimension, if any. |
| 91 | assert(srcRank == dstRank); |
| 92 | int64_t m = -1; |
| 93 | for (int64_t r = 0; r < dstRank; r++) |
| 94 | if (srcType.getDimSize(idx: r) != dstType.getDimSize(idx: r)) { |
| 95 | m = r; |
| 96 | break; |
| 97 | } |
| 98 | |
| 99 | // All trailing dimensions are the same. Simply pass through. |
| 100 | if (m == -1) { |
| 101 | rewriter.replaceOp(op, newValues: op.getSource()); |
| 102 | return success(); |
| 103 | } |
| 104 | |
| 105 | // Any non-matching dimension forces a stretch along this rank. |
| 106 | // For example: |
| 107 | // %x = broadcast %y : vector<4x1x2xf32> to vector<4x2x2xf32> |
| 108 | // becomes: |
| 109 | // %a = broadcast %y[0] : vector<1x2xf32> to vector<2x2xf32> |
| 110 | // %b = broadcast %y[1] : vector<1x2xf32> to vector<2x2xf32> |
| 111 | // %c = broadcast %y[2] : vector<1x2xf32> to vector<2x2xf32> |
| 112 | // %d = broadcast %y[3] : vector<1x2xf32> to vector<2x2xf32> |
| 113 | // %x = [%a,%b,%c,%d] |
| 114 | // becomes: |
| 115 | // %u = broadcast %y[0][0] : vector<2xf32> to vector <2x2xf32> |
| 116 | // %v = broadcast %y[1][0] : vector<2xf32> to vector <2x2xf32> |
| 117 | // %a = [%u, %v] |
| 118 | // .. |
| 119 | // %x = [%a,%b,%c,%d] |
| 120 | VectorType resType = |
| 121 | VectorType::get(shape: dstType.getShape().drop_front(), elementType: eltType, |
| 122 | scalableDims: dstType.getScalableDims().drop_front()); |
| 123 | |
| 124 | // For a scalable outer dimension we would need to emit an scf.for loop, |
| 125 | // which is not yet supported. Check before creating any IR so that |
| 126 | // returning failure() does not violate the pattern API contract. |
| 127 | if (dstType.getScalableDims()[0]) { |
| 128 | // TODO: For scalable vectors we should emit an scf.for loop. |
| 129 | return failure(); |
| 130 | } |
| 131 | |
| 132 | Value result = ub::PoisonOp::create(builder&: rewriter, location: loc, result: dstType); |
| 133 | if (m == 0) { |
| 134 | // Stetch at start. |
| 135 | Value ext = vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getSource(), position: 0); |
| 136 | Value bcst = vector::BroadcastOp::create(builder&: rewriter, location: loc, result: resType, source: ext); |
| 137 | for (int64_t d = 0, dim = dstType.getDimSize(idx: 0); d < dim; ++d) |
| 138 | result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: bcst, dest: result, position: d); |
| 139 | } else { |
| 140 | // Stetch not at start. |
| 141 | for (int64_t d = 0, dim = dstType.getDimSize(idx: 0); d < dim; ++d) { |
| 142 | Value ext = vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getSource(), position: d); |
| 143 | Value bcst = vector::BroadcastOp::create(builder&: rewriter, location: loc, result: resType, source: ext); |
| 144 | result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: bcst, dest: result, position: d); |
| 145 | } |
| 146 | } |
| 147 | rewriter.replaceOp(op, newValues: result); |
| 148 | return success(); |
| 149 | } |
| 150 | }; |
| 151 | } // namespace |
| 152 | |
| 153 | void mlir::vector::populateVectorBroadcastLoweringPatterns( |
| 154 | RewritePatternSet &patterns, PatternBenefit benefit) { |
| 155 | patterns.add<BroadcastOpLowering>(arg: patterns.getContext(), args&: benefit); |
| 156 | } |
| 157 | |