| 1 | //===- ConversionUtils.cpp ------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Utility functions for TOSA lowering |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/Tosa/Utils/ConversionUtils.h" |
| 14 | #include "mlir/Dialect/Tosa/IR/TosaOps.h" |
| 15 | #include "llvm/ADT/SmallVectorExtras.h" |
| 16 | |
| 17 | using namespace mlir; |
| 18 | using namespace mlir::tosa; |
| 19 | |
| 20 | SmallVector<utils::IteratorType> |
| 21 | mlir::tosa::getNParallelLoopsAttrs(unsigned nParallelLoops) { |
| 22 | return SmallVector<utils::IteratorType>(nParallelLoops, |
| 23 | utils::IteratorType::parallel); |
| 24 | } |
| 25 | |
| 26 | SmallVector<Value> |
| 27 | mlir::tosa::condenseValues(const SmallVector<Value> &values) { |
| 28 | SmallVector<Value> condensedValues; |
| 29 | for (auto value : values) |
| 30 | if (value) |
| 31 | condensedValues.push_back(Elt: value); |
| 32 | return condensedValues; |
| 33 | } |
| 34 | |
| 35 | Value mlir::tosa::clampFloatHelper(Location loc, Value arg, Value min, |
| 36 | Value max, OpBuilder &rewriter) { |
| 37 | Value minValue = arith::MinimumFOp::create(builder&: rewriter, location: loc, lhs: arg, rhs: max); |
| 38 | return arith::MaximumFOp::create(builder&: rewriter, location: loc, lhs: minValue, rhs: min); |
| 39 | } |
| 40 | |
| 41 | Value mlir::tosa::clampIntHelper(Location loc, Value arg, Value min, Value max, |
| 42 | OpBuilder &rewriter, bool isUnsigned) { |
| 43 | if (isUnsigned) { |
| 44 | auto minOrArg = arith::MaxUIOp::create(builder&: rewriter, location: loc, lhs: min, rhs: arg); |
| 45 | return arith::MinUIOp::create(builder&: rewriter, location: loc, lhs: max, rhs: minOrArg); |
| 46 | } |
| 47 | auto minOrArg = arith::MaxSIOp::create(builder&: rewriter, location: loc, lhs: min, rhs: arg); |
| 48 | return arith::MinSIOp::create(builder&: rewriter, location: loc, lhs: max, rhs: minOrArg); |
| 49 | } |
| 50 | |
| 51 | bool mlir::tosa::validIntegerRange(IntegerType ty, int64_t value) { |
| 52 | uint64_t bitwidth = ty.getIntOrFloatBitWidth(); |
| 53 | if (ty.getSignedness() == IntegerType::Unsigned) { |
| 54 | uint64_t uvalue = value; |
| 55 | APInt intMin = APInt::getMinValue(numBits: bitwidth); |
| 56 | APInt intMax = APInt::getMaxValue(numBits: bitwidth); |
| 57 | return uvalue >= intMin.getZExtValue() && uvalue <= intMax.getZExtValue(); |
| 58 | } |
| 59 | |
| 60 | APInt intMin = APInt::getSignedMinValue(numBits: bitwidth); |
| 61 | APInt intMax = APInt::getSignedMaxValue(numBits: bitwidth); |
| 62 | return value >= intMin.getSExtValue() && value <= intMax.getSExtValue(); |
| 63 | } |
| 64 | |
| 65 | APFloat mlir::tosa::getFloatMinMaxIdentity(const llvm::fltSemantics &semantics, |
| 66 | bool negative, |
| 67 | bool allowNonFinites) { |
| 68 | if (allowNonFinites && APFloat::semanticsHasInf(semantics)) |
| 69 | return APFloat::getInf(Sem: semantics, Negative: negative); |
| 70 | return APFloat::getLargest(Sem: semantics, Negative: negative); |
| 71 | } |
| 72 | |
| 73 | namespace { |
| 74 | // Given two tensors of high and low ranks, derive the output shape |
| 75 | // to reshape the lower rank to. |
| 76 | // Examples: |
| 77 | // If lower=[c], higher=[a, b, c], [c] reshaped into [1, 1, c]. |
| 78 | // If lower=[b, c], higher=[a, b, c], [b, c] reshaped into [1, b, c]. |
| 79 | // If lower=[a], higher=[a, a], [a] reshaped into [1, a]. |
| 80 | // If lower=[a], target=[a, b, a], [a] reshaped into [1, 1, a]. |
| 81 | // If lower=[], target=[a, b, c], [] reshaped into [1, 1, 1]. |
| 82 | // If lower=[c], higher=[?, ?, c], [c] reshaped into [1, 1, c]. |
| 83 | // If lower=[?], higher=[?, ?, ?], [?] reshaped into [1, 1, ?]. |
| 84 | LogicalResult |
| 85 | computeReshapeOutput(ArrayRef<int64_t> higherRankShape, |
| 86 | ArrayRef<int64_t> lowerRankShape, |
| 87 | SmallVectorImpl<int64_t> &reshapeOutputShape) { |
| 88 | // Initialize new shapes with [1] * higherRank. |
| 89 | int64_t higherRank = higherRankShape.size(); |
| 90 | int64_t lowerRank = lowerRankShape.size(); |
| 91 | reshapeOutputShape.assign(NumElts: higherRank, Elt: 1); |
| 92 | |
| 93 | int64_t higherRankDim; |
| 94 | int64_t lowerRankDim; |
| 95 | const int64_t rankDiff = higherRank - lowerRank; |
| 96 | |
| 97 | for (int64_t i = lowerRank - 1; i >= 0; i--) { |
| 98 | higherRankDim = higherRankShape[i + rankDiff]; |
| 99 | lowerRankDim = lowerRankShape[i]; |
| 100 | |
| 101 | auto isStaticDimAndNotEqualToOne = [](int64_t dim) { |
| 102 | return dim != 1 && dim != ShapedType::kDynamic; |
| 103 | }; |
| 104 | |
| 105 | if (isStaticDimAndNotEqualToOne(lowerRankDim) && |
| 106 | isStaticDimAndNotEqualToOne(higherRankDim) && |
| 107 | lowerRankDim != higherRankDim) |
| 108 | return failure(); |
| 109 | |
| 110 | reshapeOutputShape[i + rankDiff] = lowerRankDim == 1 ? 1 : lowerRankDim; |
| 111 | } |
| 112 | return success(); |
| 113 | } |
| 114 | } // namespace |
| 115 | |
| 116 | LogicalResult mlir::tosa::EqualizeRanks(PatternRewriter &rewriter, Location loc, |
| 117 | Value &input1, Value &input2) { |
| 118 | ImplicitLocOpBuilder builder(loc, rewriter); |
| 119 | return EqualizeRanks(builder, input1, input2); |
| 120 | } |
| 121 | |
| 122 | LogicalResult mlir::tosa::EqualizeRanks(ImplicitLocOpBuilder &builder, |
| 123 | Value &input1, Value &input2) { |
| 124 | auto input1Ty = llvm::dyn_cast<RankedTensorType>(Val: input1.getType()); |
| 125 | auto input2Ty = llvm::dyn_cast<RankedTensorType>(Val: input2.getType()); |
| 126 | |
| 127 | if (!input1Ty || !input2Ty) { |
| 128 | return failure(); |
| 129 | } |
| 130 | |
| 131 | int64_t input1Rank = input1Ty.getRank(); |
| 132 | int64_t input2Rank = input2Ty.getRank(); |
| 133 | |
| 134 | if (input1Rank == input2Rank) |
| 135 | return success(); |
| 136 | |
| 137 | Value higherTensorValue, lowerTensorValue; |
| 138 | if (input1Rank > input2Rank) { |
| 139 | higherTensorValue = input1; |
| 140 | lowerTensorValue = input2; |
| 141 | } else { |
| 142 | higherTensorValue = input2; |
| 143 | lowerTensorValue = input1; |
| 144 | } |
| 145 | |
| 146 | ArrayRef<int64_t> higherRankShape = |
| 147 | llvm::cast<RankedTensorType>(Val: higherTensorValue.getType()).getShape(); |
| 148 | ArrayRef<int64_t> lowerRankShape = |
| 149 | llvm::cast<RankedTensorType>(Val: lowerTensorValue.getType()).getShape(); |
| 150 | |
| 151 | SmallVector<int64_t, 4> reshapeOutputShape; |
| 152 | |
| 153 | if (computeReshapeOutput(higherRankShape, lowerRankShape, reshapeOutputShape) |
| 154 | .failed()) |
| 155 | return failure(); |
| 156 | |
| 157 | auto reshapeInputType = |
| 158 | llvm::cast<RankedTensorType>(Val: lowerTensorValue.getType()); |
| 159 | auto reshapeOutputType = RankedTensorType::get( |
| 160 | shape: ArrayRef<int64_t>(reshapeOutputShape), elementType: reshapeInputType.getElementType()); |
| 161 | auto reshapeOutputShapeValue = getTosaConstShape(builder, shape: reshapeOutputShape); |
| 162 | |
| 163 | auto reshapeLower = tosa::ReshapeOp::create( |
| 164 | builder, output: reshapeOutputType, input1: lowerTensorValue, shape: reshapeOutputShapeValue); |
| 165 | |
| 166 | if (input1Rank > input2Rank) { |
| 167 | input1 = higherTensorValue; |
| 168 | input2 = reshapeLower.getResult(); |
| 169 | } else { |
| 170 | input1 = reshapeLower.getResult(); |
| 171 | input2 = higherTensorValue; |
| 172 | } |
| 173 | |
| 174 | return success(); |
| 175 | } |
| 176 | |
| 177 | Value mlir::tosa::getTosaConstShape(ImplicitLocOpBuilder &builder, |
| 178 | llvm::ArrayRef<int64_t> shape) { |
| 179 | auto attr = builder.getIndexTensorAttr(values: convertFromMlirShape(shape)); |
| 180 | auto type = mlir::tosa::shapeType::get(context: builder.getContext(), rank: shape.size()); |
| 181 | mlir::Operation *mlirOp = tosa::ConstShapeOp::create(builder, output: type, values: attr); |
| 182 | return mlirOp->getResult(idx: 0); |
| 183 | } |
| 184 | |
| 185 | Value mlir::tosa::getTosaConstShape(PatternRewriter &rewriter, Location loc, |
| 186 | llvm::ArrayRef<int64_t> shape) { |
| 187 | ImplicitLocOpBuilder builder(loc, rewriter); |
| 188 | return getTosaConstShape(builder, shape); |
| 189 | } |
| 190 | |
| 191 | SmallVector<int64_t> mlir::tosa::convertFromMlirShape(ArrayRef<int64_t> shape) { |
| 192 | return map_to_vector( |
| 193 | C&: shape, F: [](int64_t dim) { return ShapedType::isDynamic(dValue: dim) ? -1 : dim; }); |
| 194 | } |
| 195 | |
| 196 | bool mlir::tosa::getConstShapeValues(Operation *op, |
| 197 | llvm::SmallVector<int64_t> &resultShape) { |
| 198 | if (!op) { |
| 199 | return false; |
| 200 | } |
| 201 | if (auto constOp = mlir::dyn_cast<tosa::ConstShapeOp>(Val: op)) { |
| 202 | DenseElementsAttr elementsAttr = constOp.getValuesAttr(); |
| 203 | for (int i = 0; i < elementsAttr.size(); i++) { |
| 204 | int64_t val = elementsAttr.getValues<int64_t>()[i]; |
| 205 | resultShape.push_back(Elt: val); |
| 206 | } |
| 207 | return true; |
| 208 | } |
| 209 | // for undefined op, return false. |
| 210 | return false; |
| 211 | } |
| 212 | |
| 213 | // returns a small vector of int64_t values that attr contains |
| 214 | SmallVector<int64_t> |
| 215 | mlir::tosa::convertFromIntAttr(const DenseElementsAttr &attr, const int rank) { |
| 216 | if (attr.isSplat()) { |
| 217 | int64_t v = attr.getSplatValue<APInt>().getSExtValue(); |
| 218 | return SmallVector<int64_t>(rank, v); |
| 219 | } |
| 220 | |
| 221 | if (auto intArrayAttr = llvm::dyn_cast<DenseIntElementsAttr>(Val: attr)) { |
| 222 | SmallVector<int64_t> vec; |
| 223 | for (APInt val : intArrayAttr.getValues<APInt>()) { |
| 224 | vec.push_back(Elt: val.getSExtValue()); |
| 225 | } |
| 226 | return vec; |
| 227 | } |
| 228 | return {}; |
| 229 | } |
| 230 | |
| 231 | bool mlir::tosa::hasUniqueConstantScatterIndices( |
| 232 | ShapedType indicesType, DenseIntElementsAttr indicesAttr) { |
| 233 | const llvm::ArrayRef<int64_t> indicesShape = indicesType.getShape(); |
| 234 | const unsigned int indicesRank = indicesShape.size(); |
| 235 | const unsigned int lastDimSize = indicesShape[indicesRank - 1]; |
| 236 | |
| 237 | // check each batch of indices from the flat indicesAttr values |
| 238 | // for duplicates |
| 239 | auto const indicesValues = indicesAttr.getValues<APInt>(); |
| 240 | assert( |
| 241 | (indicesValues.size() % lastDimSize == 0) && |
| 242 | "Constant indices data length should be a multiple of indicesShape[-1]" ); |
| 243 | |
| 244 | std::vector<APInt> indices(lastDimSize); |
| 245 | for (auto beg = indicesValues.begin(); beg < indicesValues.end(); |
| 246 | beg += lastDimSize) { |
| 247 | std::copy(first: beg, last: beg + lastDimSize, result: indices.begin()); |
| 248 | std::sort(first: indices.begin(), last: indices.end(), |
| 249 | comp: [](const APInt &a, const APInt &b) { return a.slt(RHS: b); }); |
| 250 | if (std::adjacent_find(first: indices.begin(), last: indices.end()) != indices.end()) { |
| 251 | // found duplicate values in indices in batch |
| 252 | return false; |
| 253 | } |
| 254 | } |
| 255 | |
| 256 | return true; |
| 257 | } |
| 258 | |
| 259 | template <typename T> |
| 260 | FailureOr<T> mlir::tosa::getConstantScalarIntValue(Value val) { |
| 261 | ElementsAttr attr; |
| 262 | if (!matchPattern(value: val, pattern: m_Constant(bind_value: &attr))) |
| 263 | return failure(); |
| 264 | |
| 265 | if (!llvm::isa<IntegerType>(Val: attr.getElementType()) || |
| 266 | attr.getNumElements() != 1) |
| 267 | return failure(); |
| 268 | |
| 269 | return attr.getValues<T>()[0]; |
| 270 | } |
| 271 | |
| 272 | template FailureOr<int32_t> |
| 273 | mlir::tosa::getConstantScalarIntValue<int32_t>(Value val); |
| 274 | |