1//===- VectorMaskElimination.cpp - Eliminate Vector Masks -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Utils/StaticValueUtils.h"
10#include "mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h"
11#include "mlir/Dialect/Vector/Transforms/Passes.h"
12#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
13#include "mlir/Interfaces/FunctionInterfaces.h"
14
15namespace mlir {
16namespace vector {
17
18#define GEN_PASS_DEF_ELIMINATEVECTORMASKS
19#include "mlir/Dialect/Vector/Transforms/Passes.h.inc"
20
21} // namespace vector
22} // namespace mlir
23
24using namespace mlir;
25using namespace mlir::vector;
26namespace {
27
28/// Attempts to resolve a CreateMaskOp to an all-true constant mask. All-true
29/// masks can then be eliminated by simple folds. `vscaleRange` is required to
30/// reason about scalable dimensions; without it only fixed-size dimensions can
31/// be proven all-true.
32LogicalResult
33resolveAllTrueCreateMaskOp(IRRewriter &rewriter,
34 vector::CreateMaskOp createMaskOp,
35 std::optional<VscaleRange> vscaleRange) {
36 auto maskType = createMaskOp.getVectorType();
37 auto maskTypeDimScalableFlags = maskType.getScalableDims();
38 auto maskTypeDimSizes = maskType.getShape();
39
40 // Special case: Rank zero shape.
41 constexpr std::array<int64_t, 1> rankZeroShape{1};
42 constexpr std::array<bool, 1> rankZeroScalableDims{false};
43 if (maskType.getRank() == 0) {
44 maskTypeDimSizes = rankZeroShape;
45 maskTypeDimScalableFlags = rankZeroScalableDims;
46 }
47
48 struct UnknownMaskDim {
49 size_t position;
50 Value dimSize;
51 };
52
53 // Loop over the CreateMaskOp operands and collect unknown dims (i.e. dims
54 // that are not obviously constant). If any constant dimension is not all-true
55 // bail out early (as this transform only trying to resolve all-true masks).
56 // This avoids doing value-bounds anaylis in cases like:
57 // `%mask = vector.create_mask %dynamicValue, %c2 : vector<8x4xi1>`
58 // ...where it is known the mask is not all-true by looking at `%c2`.
59 SmallVector<UnknownMaskDim> unknownDims;
60 for (auto [i, dimSize] : llvm::enumerate(First: createMaskOp.getOperands())) {
61 if (auto intSize = getConstantIntValue(ofr: dimSize)) {
62 // Mask not all-true for this dim.
63 if (maskTypeDimScalableFlags[i] || intSize < maskTypeDimSizes[i])
64 return failure();
65 } else if (auto vscaleMultiplier = getConstantVscaleMultiplier(value: dimSize)) {
66 // Mask not all-true for this dim.
67 if (vscaleMultiplier < maskTypeDimSizes[i])
68 return failure();
69 } else {
70 // Unknown (without further analysis).
71 unknownDims.push_back(Elt: UnknownMaskDim{.position: i, .dimSize: dimSize});
72 }
73 }
74
75 for (auto [i, dimSize] : unknownDims) {
76 // Compute the lower bound for the unknown dimension (i.e. the smallest
77 // value it could be).
78
79 // Fixed-width case: without a `vscale` range the bound is a plain constant,
80 // which can only prove a fixed-size dimension all-true.
81 if (!vscaleRange) {
82 // A constant bound cannot prove a scalable dim, whose runtime size is
83 // `vscale` times the size in the type. Checked first to skip the query.
84 if (maskTypeDimScalableFlags[i])
85 return failure();
86 FailureOr<int64_t> constantLowerBound =
87 ValueBoundsConstraintSet::computeConstantBound(
88 type: presburger::BoundType::LB, var: dimSize);
89 if (failed(Result: constantLowerBound))
90 return failure();
91 // If LB < the mask dim size then this dim is not all-true.
92 if (*constantLowerBound < maskTypeDimSizes[i])
93 return failure();
94 continue;
95 }
96
97 // Scalable case: with a `vscale` range the bound has the form
98 // `base + n * vscale`, which can prove either kind of dimension all-true.
99 FailureOr<ConstantOrScalableBound> dimLowerBound =
100 vector::ScalableValueBoundsConstraintSet::computeScalableBound(
101 value: dimSize, dim: {}, vscaleMin: vscaleRange->vscaleMin, vscaleMax: vscaleRange->vscaleMax,
102 boundType: presburger::BoundType::LB);
103 if (failed(Result: dimLowerBound))
104 return failure();
105 auto dimLowerBoundSize = dimLowerBound->getSize();
106 if (failed(Result: dimLowerBoundSize))
107 return failure();
108 if (dimLowerBoundSize->scalable) {
109 // 1. The lower bound, LB, is scalable. If LB is < the mask dim size then
110 // this dim is not all-true.
111 if (dimLowerBoundSize->baseSize < maskTypeDimSizes[i])
112 return failure();
113 } else {
114 // 2. The lower bound, LB, is a constant.
115 // - If the mask dim size is scalable then this dim is not all-true.
116 if (maskTypeDimScalableFlags[i])
117 return failure();
118 // - If LB < the _fixed-size_ mask dim size then this dim is not all-true.
119 if (dimLowerBoundSize->baseSize < maskTypeDimSizes[i])
120 return failure();
121 }
122 }
123
124 // Replace createMaskOp with an all-true constant. This should result in the
125 // mask being removed in most cases (as xfer ops + vector.mask have folds to
126 // remove all-true masks).
127 auto allTrue = vector::ConstantMaskOp::create(
128 builder&: rewriter, location: createMaskOp.getLoc(), type: maskType, kind: ConstantMaskKind::AllTrue);
129 rewriter.replaceAllUsesWith(from: createMaskOp, to: allTrue);
130 return success();
131}
132
133} // namespace
134
135namespace mlir::vector {
136
137void eliminateVectorMasks(IRRewriter &rewriter, FunctionOpInterface function,
138 std::optional<VscaleRange> vscaleRange) {
139 // Early exit for functions without a body.
140 if (function.isExternal())
141 return;
142
143 OpBuilder::InsertionGuard g(rewriter);
144
145 // Build worklist so we can safely insert new ops in
146 // `resolveAllTrueCreateMaskOp()`.
147 SmallVector<vector::CreateMaskOp> worklist;
148 function.walk(callback: [&](vector::CreateMaskOp createMaskOp) {
149 worklist.push_back(Elt: createMaskOp);
150 });
151
152 rewriter.setInsertionPointToStart(&function.front());
153 for (auto mask : worklist)
154 (void)resolveAllTrueCreateMaskOp(rewriter, createMaskOp: mask, vscaleRange);
155}
156
157namespace {
158struct EliminateVectorMasksPass
159 : public impl::EliminateVectorMasksBase<EliminateVectorMasksPass> {
160 using Base::Base;
161
162 // Checked here rather than in runOnOperation so that a bad range is reported
163 // once, not once per function.
164 LogicalResult initialize(MLIRContext *context) override {
165 bool unset = !vscaleMin && !vscaleMax;
166 bool valid = vscaleMin && vscaleMax && vscaleMin <= vscaleMax;
167 if (unset || valid)
168 return success();
169 return emitError(loc: UnknownLoc::get(context))
170 << "invalid vscale range 'vscale-min="
171 << static_cast<unsigned>(vscaleMin)
172 << " vscale-max=" << static_cast<unsigned>(vscaleMax)
173 << "': expected both to be 0 (unknown), or both non-zero with "
174 "'vscale-min' <= 'vscale-max'";
175 }
176
177 void runOnOperation() override {
178 std::optional<VscaleRange> vscaleRange;
179 if (vscaleMin && vscaleMax)
180 vscaleRange = VscaleRange{.vscaleMin: vscaleMin, .vscaleMax: vscaleMax};
181
182 IRRewriter rewriter(&getContext());
183 eliminateVectorMasks(rewriter, function: getOperation(), vscaleRange);
184 }
185};
186} // namespace
187
188} // namespace mlir::vector
189