| 1 | //===- VectorMaskElimination.cpp - Eliminate Vector Masks -----------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 10 | #include "mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h" |
| 11 | #include "mlir/Dialect/Vector/Transforms/Passes.h" |
| 12 | #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" |
| 13 | #include "mlir/Interfaces/FunctionInterfaces.h" |
| 14 | |
| 15 | namespace mlir { |
| 16 | namespace vector { |
| 17 | |
| 18 | #define GEN_PASS_DEF_ELIMINATEVECTORMASKS |
| 19 | #include "mlir/Dialect/Vector/Transforms/Passes.h.inc" |
| 20 | |
| 21 | } // namespace vector |
| 22 | } // namespace mlir |
| 23 | |
| 24 | using namespace mlir; |
| 25 | using namespace mlir::vector; |
| 26 | namespace { |
| 27 | |
| 28 | /// Attempts to resolve a CreateMaskOp to an all-true constant mask. All-true |
| 29 | /// masks can then be eliminated by simple folds. `vscaleRange` is required to |
| 30 | /// reason about scalable dimensions; without it only fixed-size dimensions can |
| 31 | /// be proven all-true. |
| 32 | LogicalResult |
| 33 | resolveAllTrueCreateMaskOp(IRRewriter &rewriter, |
| 34 | vector::CreateMaskOp createMaskOp, |
| 35 | std::optional<VscaleRange> vscaleRange) { |
| 36 | auto maskType = createMaskOp.getVectorType(); |
| 37 | auto maskTypeDimScalableFlags = maskType.getScalableDims(); |
| 38 | auto maskTypeDimSizes = maskType.getShape(); |
| 39 | |
| 40 | // Special case: Rank zero shape. |
| 41 | constexpr std::array<int64_t, 1> rankZeroShape{1}; |
| 42 | constexpr std::array<bool, 1> rankZeroScalableDims{false}; |
| 43 | if (maskType.getRank() == 0) { |
| 44 | maskTypeDimSizes = rankZeroShape; |
| 45 | maskTypeDimScalableFlags = rankZeroScalableDims; |
| 46 | } |
| 47 | |
| 48 | struct UnknownMaskDim { |
| 49 | size_t position; |
| 50 | Value dimSize; |
| 51 | }; |
| 52 | |
| 53 | // Loop over the CreateMaskOp operands and collect unknown dims (i.e. dims |
| 54 | // that are not obviously constant). If any constant dimension is not all-true |
| 55 | // bail out early (as this transform only trying to resolve all-true masks). |
| 56 | // This avoids doing value-bounds anaylis in cases like: |
| 57 | // `%mask = vector.create_mask %dynamicValue, %c2 : vector<8x4xi1>` |
| 58 | // ...where it is known the mask is not all-true by looking at `%c2`. |
| 59 | SmallVector<UnknownMaskDim> unknownDims; |
| 60 | for (auto [i, dimSize] : llvm::enumerate(First: createMaskOp.getOperands())) { |
| 61 | if (auto intSize = getConstantIntValue(ofr: dimSize)) { |
| 62 | // Mask not all-true for this dim. |
| 63 | if (maskTypeDimScalableFlags[i] || intSize < maskTypeDimSizes[i]) |
| 64 | return failure(); |
| 65 | } else if (auto vscaleMultiplier = getConstantVscaleMultiplier(value: dimSize)) { |
| 66 | // Mask not all-true for this dim. |
| 67 | if (vscaleMultiplier < maskTypeDimSizes[i]) |
| 68 | return failure(); |
| 69 | } else { |
| 70 | // Unknown (without further analysis). |
| 71 | unknownDims.push_back(Elt: UnknownMaskDim{.position: i, .dimSize: dimSize}); |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | for (auto [i, dimSize] : unknownDims) { |
| 76 | // Compute the lower bound for the unknown dimension (i.e. the smallest |
| 77 | // value it could be). |
| 78 | |
| 79 | // Fixed-width case: without a `vscale` range the bound is a plain constant, |
| 80 | // which can only prove a fixed-size dimension all-true. |
| 81 | if (!vscaleRange) { |
| 82 | // A constant bound cannot prove a scalable dim, whose runtime size is |
| 83 | // `vscale` times the size in the type. Checked first to skip the query. |
| 84 | if (maskTypeDimScalableFlags[i]) |
| 85 | return failure(); |
| 86 | FailureOr<int64_t> constantLowerBound = |
| 87 | ValueBoundsConstraintSet::computeConstantBound( |
| 88 | type: presburger::BoundType::LB, var: dimSize); |
| 89 | if (failed(Result: constantLowerBound)) |
| 90 | return failure(); |
| 91 | // If LB < the mask dim size then this dim is not all-true. |
| 92 | if (*constantLowerBound < maskTypeDimSizes[i]) |
| 93 | return failure(); |
| 94 | continue; |
| 95 | } |
| 96 | |
| 97 | // Scalable case: with a `vscale` range the bound has the form |
| 98 | // `base + n * vscale`, which can prove either kind of dimension all-true. |
| 99 | FailureOr<ConstantOrScalableBound> dimLowerBound = |
| 100 | vector::ScalableValueBoundsConstraintSet::computeScalableBound( |
| 101 | value: dimSize, dim: {}, vscaleMin: vscaleRange->vscaleMin, vscaleMax: vscaleRange->vscaleMax, |
| 102 | boundType: presburger::BoundType::LB); |
| 103 | if (failed(Result: dimLowerBound)) |
| 104 | return failure(); |
| 105 | auto dimLowerBoundSize = dimLowerBound->getSize(); |
| 106 | if (failed(Result: dimLowerBoundSize)) |
| 107 | return failure(); |
| 108 | if (dimLowerBoundSize->scalable) { |
| 109 | // 1. The lower bound, LB, is scalable. If LB is < the mask dim size then |
| 110 | // this dim is not all-true. |
| 111 | if (dimLowerBoundSize->baseSize < maskTypeDimSizes[i]) |
| 112 | return failure(); |
| 113 | } else { |
| 114 | // 2. The lower bound, LB, is a constant. |
| 115 | // - If the mask dim size is scalable then this dim is not all-true. |
| 116 | if (maskTypeDimScalableFlags[i]) |
| 117 | return failure(); |
| 118 | // - If LB < the _fixed-size_ mask dim size then this dim is not all-true. |
| 119 | if (dimLowerBoundSize->baseSize < maskTypeDimSizes[i]) |
| 120 | return failure(); |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | // Replace createMaskOp with an all-true constant. This should result in the |
| 125 | // mask being removed in most cases (as xfer ops + vector.mask have folds to |
| 126 | // remove all-true masks). |
| 127 | auto allTrue = vector::ConstantMaskOp::create( |
| 128 | builder&: rewriter, location: createMaskOp.getLoc(), type: maskType, kind: ConstantMaskKind::AllTrue); |
| 129 | rewriter.replaceAllUsesWith(from: createMaskOp, to: allTrue); |
| 130 | return success(); |
| 131 | } |
| 132 | |
| 133 | } // namespace |
| 134 | |
| 135 | namespace mlir::vector { |
| 136 | |
| 137 | void eliminateVectorMasks(IRRewriter &rewriter, FunctionOpInterface function, |
| 138 | std::optional<VscaleRange> vscaleRange) { |
| 139 | // Early exit for functions without a body. |
| 140 | if (function.isExternal()) |
| 141 | return; |
| 142 | |
| 143 | OpBuilder::InsertionGuard g(rewriter); |
| 144 | |
| 145 | // Build worklist so we can safely insert new ops in |
| 146 | // `resolveAllTrueCreateMaskOp()`. |
| 147 | SmallVector<vector::CreateMaskOp> worklist; |
| 148 | function.walk(callback: [&](vector::CreateMaskOp createMaskOp) { |
| 149 | worklist.push_back(Elt: createMaskOp); |
| 150 | }); |
| 151 | |
| 152 | rewriter.setInsertionPointToStart(&function.front()); |
| 153 | for (auto mask : worklist) |
| 154 | (void)resolveAllTrueCreateMaskOp(rewriter, createMaskOp: mask, vscaleRange); |
| 155 | } |
| 156 | |
| 157 | namespace { |
| 158 | struct EliminateVectorMasksPass |
| 159 | : public impl::EliminateVectorMasksBase<EliminateVectorMasksPass> { |
| 160 | using Base::Base; |
| 161 | |
| 162 | // Checked here rather than in runOnOperation so that a bad range is reported |
| 163 | // once, not once per function. |
| 164 | LogicalResult initialize(MLIRContext *context) override { |
| 165 | bool unset = !vscaleMin && !vscaleMax; |
| 166 | bool valid = vscaleMin && vscaleMax && vscaleMin <= vscaleMax; |
| 167 | if (unset || valid) |
| 168 | return success(); |
| 169 | return emitError(loc: UnknownLoc::get(context)) |
| 170 | << "invalid vscale range 'vscale-min=" |
| 171 | << static_cast<unsigned>(vscaleMin) |
| 172 | << " vscale-max=" << static_cast<unsigned>(vscaleMax) |
| 173 | << "': expected both to be 0 (unknown), or both non-zero with " |
| 174 | "'vscale-min' <= 'vscale-max'" ; |
| 175 | } |
| 176 | |
| 177 | void runOnOperation() override { |
| 178 | std::optional<VscaleRange> vscaleRange; |
| 179 | if (vscaleMin && vscaleMax) |
| 180 | vscaleRange = VscaleRange{.vscaleMin: vscaleMin, .vscaleMax: vscaleMax}; |
| 181 | |
| 182 | IRRewriter rewriter(&getContext()); |
| 183 | eliminateVectorMasks(rewriter, function: getOperation(), vscaleRange); |
| 184 | } |
| 185 | }; |
| 186 | } // namespace |
| 187 | |
| 188 | } // namespace mlir::vector |
| 189 | |