| 1 | //===- TosaToLinalgNamed.cpp - Lowering Tosa to Linalg Named Ops ----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // These rewriters lower from the Tosa to the Linalg named ops. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "TosaToLinalgUtils.h" |
| 14 | #include "mlir/Conversion/TosaToLinalg/TosaToLinalg.h" |
| 15 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 16 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 17 | #include "mlir/Dialect/Linalg/IR/Linalg.h" |
| 18 | #include "mlir/Dialect/Math/IR/Math.h" |
| 19 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 20 | #include "mlir/Dialect/Tosa/IR/TosaOps.h" |
| 21 | #include "mlir/Dialect/Tosa/Utils/ConversionUtils.h" |
| 22 | #include "mlir/Dialect/Utils/ReshapeOpsUtils.h" |
| 23 | #include "mlir/IR/Matchers.h" |
| 24 | #include "mlir/IR/PatternMatch.h" |
| 25 | #include "mlir/Transforms/DialectConversion.h" |
| 26 | #include "llvm/ADT/SmallVectorExtras.h" |
| 27 | #include "llvm/ADT/TypeSwitch.h" |
| 28 | |
| 29 | #include <optional> |
| 30 | #include <type_traits> |
| 31 | |
| 32 | using namespace mlir; |
| 33 | using namespace mlir::tosa; |
| 34 | |
| 35 | // Return the linalg.elementwise kind that is semantically equivalent to the |
| 36 | // given TOSA elementwise op for the provided element type, or std::nullopt if |
| 37 | // the op has no direct linalg.elementwise counterpart. Only ops whose TOSA |
| 38 | // lowering maps to a single linalg.elementwise function are handled; the more |
| 39 | // involved lowerings (e.g. quantized negate, mul with shift, casts) are left to |
| 40 | // the generic TosaToLinalg conversion. |
| 41 | static std::optional<linalg::ElementwiseKind> |
| 42 | getElementwiseKind(Operation *op, Type elementTy) { |
| 43 | using linalg::ElementwiseKind; |
| 44 | const bool isFloat = isa<FloatType>(Val: elementTy); |
| 45 | const bool isSignlessInt = elementTy.isSignlessInteger(); |
| 46 | const bool isBoolean = elementTy.isInteger(width: 1); |
| 47 | |
| 48 | // Unary ops. linalg.elementwise only defines these on floating point types. |
| 49 | if (isFloat) { |
| 50 | if (isa<tosa::AbsOp>(Val: op)) |
| 51 | return ElementwiseKind::abs; |
| 52 | if (isa<tosa::CeilOp>(Val: op)) |
| 53 | return ElementwiseKind::ceil; |
| 54 | if (isa<tosa::FloorOp>(Val: op)) |
| 55 | return ElementwiseKind::floor; |
| 56 | if (isa<tosa::ExpOp>(Val: op)) |
| 57 | return ElementwiseKind::exp; |
| 58 | if (isa<tosa::LogOp>(Val: op)) |
| 59 | return ElementwiseKind::log; |
| 60 | if (isa<tosa::RsqrtOp>(Val: op)) |
| 61 | return ElementwiseKind::rsqrt; |
| 62 | if (isa<tosa::SinOp>(Val: op)) |
| 63 | return ElementwiseKind::sin; |
| 64 | if (isa<tosa::CosOp>(Val: op)) |
| 65 | return ElementwiseKind::cos; |
| 66 | if (isa<tosa::TanhOp>(Val: op)) |
| 67 | return ElementwiseKind::tanh; |
| 68 | if (isa<tosa::ErfOp>(Val: op)) |
| 69 | return ElementwiseKind::erf; |
| 70 | if (isa<tosa::ReciprocalOp>(Val: op)) |
| 71 | return ElementwiseKind::reciprocal; |
| 72 | if (isa<tosa::PowOp>(Val: op)) |
| 73 | return ElementwiseKind::powf; |
| 74 | } |
| 75 | |
| 76 | // Binary ops defined on both floating point and (non-boolean) integers. |
| 77 | if (isFloat || (isSignlessInt && !isBoolean)) { |
| 78 | if (isa<tosa::AddOp>(Val: op)) |
| 79 | return ElementwiseKind::add; |
| 80 | if (isa<tosa::SubOp>(Val: op)) |
| 81 | return ElementwiseKind::sub; |
| 82 | } |
| 83 | |
| 84 | // Signed integer division. |
| 85 | if (isSignlessInt && !isBoolean && isa<tosa::IntDivOp>(Val: op)) |
| 86 | return ElementwiseKind::div; |
| 87 | |
| 88 | // Multiply maps to a plain elementwise multiply only when no rescaling shift |
| 89 | // is requested (the shift must be a constant zero) and the inputs already |
| 90 | // share the result element type. The shifted/rescaled integer variant is left |
| 91 | // to the generic conversion. |
| 92 | if (auto mulOp = dyn_cast<tosa::MulOp>(Val: op)) { |
| 93 | DenseElementsAttr shiftAttr; |
| 94 | const bool zeroShift = |
| 95 | matchPattern(value: mulOp.getShift(), pattern: m_Constant(bind_value: &shiftAttr)) && |
| 96 | shiftAttr.getValues<APInt>()[0].isZero(); |
| 97 | const bool matchingTypes = |
| 98 | cast<ShapedType>(Val: mulOp.getInput1().getType()).getElementType() == |
| 99 | elementTy && |
| 100 | cast<ShapedType>(Val: mulOp.getInput2().getType()).getElementType() == |
| 101 | elementTy; |
| 102 | if ((isFloat || (isSignlessInt && !isBoolean)) && zeroShift && |
| 103 | matchingTypes) |
| 104 | return ElementwiseKind::mul; |
| 105 | } |
| 106 | |
| 107 | // Negate maps to floating-point negation only when the zero points are zero. |
| 108 | // The quantized integer variant is left to the generic conversion. |
| 109 | if (auto negateOp = dyn_cast<tosa::NegateOp>(Val: op)) { |
| 110 | if (isFloat) { |
| 111 | FailureOr<int64_t> inZp = negateOp.getInput1ZeroPoint(); |
| 112 | FailureOr<int64_t> outZp = negateOp.getOutputZeroPoint(); |
| 113 | if (succeeded(Result: inZp) && succeeded(Result: outZp) && *inZp == 0 && *outZp == 0) |
| 114 | return ElementwiseKind::negf; |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | // Maximum/minimum. The NaN "IGNORE" mode requires an additional compare and |
| 119 | // select that linalg.elementwise cannot express, so only the default |
| 120 | // "PROPAGATE" mode is handled for floating point types. |
| 121 | auto nanModeIsPropagate = [](auto tosaOp) { |
| 122 | return tosaOp.getNanMode() == NanPropagationMode::PROPAGATE; |
| 123 | }; |
| 124 | if (isFloat || (isSignlessInt && !isBoolean)) { |
| 125 | if (auto maxOp = dyn_cast<tosa::MaximumOp>(Val: op)) { |
| 126 | if (!isFloat || nanModeIsPropagate(maxOp)) |
| 127 | return ElementwiseKind::max_signed; |
| 128 | } |
| 129 | if (auto minOp = dyn_cast<tosa::MinimumOp>(Val: op)) { |
| 130 | if (!isFloat || nanModeIsPropagate(minOp)) |
| 131 | return ElementwiseKind::min_signed; |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | // Select (ternary) for valid element types. |
| 136 | if ((isFloat || isSignlessInt || isBoolean) && isa<tosa::SelectOp>(Val: op)) |
| 137 | return ElementwiseKind::select; |
| 138 | |
| 139 | return std::nullopt; |
| 140 | } |
| 141 | |
| 142 | std::optional<linalg::ElementwiseKind> |
| 143 | mlir::tosa::isConvertibleToLinalgElementwise(Operation *op) { |
| 144 | if (op->getNumResults() != 1) |
| 145 | return std::nullopt; |
| 146 | |
| 147 | // Make sure all operands and result have static sizes |
| 148 | for (Value operand : op->getOperands()) { |
| 149 | auto operandTy = dyn_cast<RankedTensorType>(Val: operand.getType()); |
| 150 | if (!operandTy || !operandTy.hasStaticShape()) |
| 151 | return std::nullopt; |
| 152 | } |
| 153 | auto resultTy = dyn_cast<RankedTensorType>(Val: op->getResult(idx: 0).getType()); |
| 154 | if (!resultTy || !resultTy.hasStaticShape()) |
| 155 | return std::nullopt; |
| 156 | |
| 157 | return getElementwiseKind(op, elementTy: resultTy.getElementType()); |
| 158 | } |
| 159 | |
| 160 | static mlir::Value applyPad(Location loc, Value input, ArrayRef<int64_t> pad, |
| 161 | TypedAttr padAttr, OpBuilder &rewriter) { |
| 162 | // Input should be padded only if necessary. |
| 163 | if (llvm::all_of(Range&: pad, P: [](int64_t p) { return p == 0; })) |
| 164 | return input; |
| 165 | |
| 166 | ShapedType inputTy = cast<ShapedType>(Val: input.getType()); |
| 167 | Type inputETy = inputTy.getElementType(); |
| 168 | auto inputShape = inputTy.getShape(); |
| 169 | |
| 170 | assert((inputShape.size() * 2) == pad.size()); |
| 171 | |
| 172 | SmallVector<int64_t, 4> paddedShape; |
| 173 | SmallVector<OpFoldResult, 8> lowIndices; |
| 174 | SmallVector<OpFoldResult, 8> highIndices; |
| 175 | for (size_t i : llvm::seq(Size: inputShape.size())) { |
| 176 | auto lowPad = pad[i * 2]; |
| 177 | auto highPad = pad[i * 2 + 1]; |
| 178 | if (ShapedType::isDynamic(dValue: inputShape[i])) |
| 179 | paddedShape.push_back(Elt: inputShape[i]); |
| 180 | else |
| 181 | paddedShape.push_back(Elt: inputShape[i] + highPad + lowPad); |
| 182 | lowIndices.push_back(Elt: rewriter.getIndexAttr(value: lowPad)); |
| 183 | highIndices.push_back(Elt: rewriter.getIndexAttr(value: highPad)); |
| 184 | } |
| 185 | |
| 186 | Value padValue = arith::ConstantOp::create(builder&: rewriter, location: loc, value: padAttr); |
| 187 | |
| 188 | return tensor::PadOp::create(builder&: rewriter, location: loc, |
| 189 | resultType: RankedTensorType::get(shape: paddedShape, elementType: inputETy), |
| 190 | source: input, low: lowIndices, high: highIndices, constantPadValue: padValue); |
| 191 | } |
| 192 | |
| 193 | static mlir::Value |
| 194 | linalgIntBroadcastExtSIAdd(PatternRewriter &rewriter, Location loc, Value bias, |
| 195 | Value conv, Value result, |
| 196 | ArrayRef<AffineMap> indexingMaps) { |
| 197 | ShapedType resultTy = cast<ShapedType>(Val: conv.getType()); |
| 198 | return linalg::GenericOp::create( |
| 199 | builder&: rewriter, location: loc, resultTensorTypes: resultTy, inputs: ValueRange({bias, conv}), outputs: result, |
| 200 | indexingMaps, iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultTy.getRank()), |
| 201 | odsArg5: [](OpBuilder &builder, Location loc, ValueRange args) { |
| 202 | Value biasVal = args[0]; |
| 203 | Type resType = args[1].getType(); |
| 204 | if (resType != biasVal.getType()) { |
| 205 | biasVal = |
| 206 | arith::ExtSIOp::create(builder, location: loc, out: resType, in: biasVal); |
| 207 | } |
| 208 | Value added = |
| 209 | arith::AddIOp::create(builder, location: loc, lhs: biasVal, rhs: args[1]); |
| 210 | linalg::YieldOp::create(builder, location: loc, values: added); |
| 211 | }) |
| 212 | .getResult(i: 0); |
| 213 | } |
| 214 | |
| 215 | // Construct the affine map that a linalg generic would use to broadcast the |
| 216 | // source tensor into the shape of the result tensor. |
| 217 | static AffineMap getBroadcastingMap(PatternRewriter &rewriter, Value source, |
| 218 | Value result) { |
| 219 | ShapedType resultTy = cast<ShapedType>(Val: result.getType()); |
| 220 | ShapedType sourceTy = cast<ShapedType>(Val: source.getType()); |
| 221 | const int64_t resultRank = resultTy.getRank(); |
| 222 | const int64_t sourceRank = sourceTy.getRank(); |
| 223 | |
| 224 | // The source tensor is broadcast to all the outer dimensions of the |
| 225 | // result tensor. |
| 226 | SmallVector<AffineExpr> sourceDims; |
| 227 | // In the case of a rank one source tensor with a single element TOSA |
| 228 | // specifies that the value be broadcast meaning we need an edge case for a |
| 229 | // constant map. |
| 230 | assert(sourceTy.hasStaticShape() && |
| 231 | "Dynamic broadcasting shapes not supported!" ); |
| 232 | if (sourceRank == 1 && sourceTy.getDimSize(idx: 0) == 1) { |
| 233 | sourceDims.push_back(Elt: rewriter.getAffineConstantExpr(constant: 0)); |
| 234 | } else { |
| 235 | for (auto dim : llvm::seq<int64_t>(Begin: 0, End: sourceRank)) { |
| 236 | auto expr = rewriter.getAffineDimExpr(position: dim + resultRank - sourceRank); |
| 237 | sourceDims.push_back(Elt: expr); |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | return AffineMap::get(/*dimCount=*/resultRank, |
| 242 | /*symbolCount=*/0, results: sourceDims, context: rewriter.getContext()); |
| 243 | } |
| 244 | |
| 245 | // Broadcast the source value to all the outer dimensions of the result value. |
| 246 | // If required, the element type is expanded using an arith.extsi or arith.extf |
| 247 | // operation as appropriate. |
| 248 | static mlir::Value linalgBroadcastAndMaybeExt(PatternRewriter &rewriter, |
| 249 | Location loc, Value source, |
| 250 | Value result) { |
| 251 | ShapedType resultTy = cast<ShapedType>(Val: result.getType()); |
| 252 | const int64_t resultRank = resultTy.getRank(); |
| 253 | // Creating maps for the input and output of the broacast-like generic op. |
| 254 | SmallVector<AffineMap, 2> indexingMaps; |
| 255 | indexingMaps.push_back(Elt: getBroadcastingMap(rewriter, source, result)); |
| 256 | indexingMaps.push_back(Elt: rewriter.getMultiDimIdentityMap(rank: resultRank)); |
| 257 | |
| 258 | // Build the broadcast-like operation as a linalg.generic. |
| 259 | return linalg::GenericOp::create( |
| 260 | builder&: rewriter, location: loc, resultTensorTypes: resultTy, inputs: ValueRange({source}), outputs: result, |
| 261 | indexingMaps, iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultTy.getRank()), |
| 262 | odsArg5: [&resultTy](OpBuilder &builder, Location loc, ValueRange args) { |
| 263 | Value biasVal = args[0]; |
| 264 | Type resType = args[1].getType(); |
| 265 | if (resType != biasVal.getType()) { |
| 266 | biasVal = |
| 267 | resultTy.getElementType().isFloat() |
| 268 | ? arith::ExtFOp::create( |
| 269 | builder, location: loc, resultTypes: TypeRange{resType}, |
| 270 | operands: ValueRange{biasVal}, properties: arith::ExtFOp::Properties{}) |
| 271 | .getResult() |
| 272 | : arith::ExtSIOp::create(builder, location: loc, out: resType, |
| 273 | in: biasVal) |
| 274 | .getResult(); |
| 275 | } |
| 276 | linalg::YieldOp::create(builder, location: loc, values: biasVal); |
| 277 | }) |
| 278 | .getResult(i: 0); |
| 279 | } |
| 280 | |
| 281 | static mlir::Value reifyConstantDim(int64_t attr, |
| 282 | ImplicitLocOpBuilder &builder) { |
| 283 | return arith::ConstantIndexOp::create(builder, value: attr); |
| 284 | } |
| 285 | |
| 286 | // Calculating the output width/height using the formula: |
| 287 | // H = ((IH+pad_top+pad_bottom-(dilation_y*(KH-1)+1))/stride_y)+1 |
| 288 | // W = ((IW+pad_left+pad_right-(dilation_x*(KW-1)+1))/stride_x)+1 |
| 289 | |
| 290 | static mlir::Value getConvOrPoolOutputDim(Location loc, Value inputDim, |
| 291 | int64_t padBeforeAttr, |
| 292 | int64_t padAfterAttr, Value kernelDim, |
| 293 | int64_t strideAttr, |
| 294 | int64_t dilationAttr, |
| 295 | OpBuilder &rewriter) { |
| 296 | ImplicitLocOpBuilder builder(loc, rewriter); |
| 297 | auto one = arith::ConstantOp::create(builder&: rewriter, location: loc, |
| 298 | value: IntegerAttr::get(type: inputDim.getType(), value: 1)); |
| 299 | Value padBefore = reifyConstantDim(attr: padBeforeAttr, builder); |
| 300 | Value paddedBefore = arith::AddIOp::create(builder, lhs: inputDim, rhs: padBefore); |
| 301 | Value padAfter = reifyConstantDim(attr: padAfterAttr, builder); |
| 302 | Value paddedAfter = arith::AddIOp::create(builder, lhs: paddedBefore, rhs: padAfter); |
| 303 | |
| 304 | Value subOne = arith::SubIOp::create(builder, lhs: kernelDim, rhs: one); |
| 305 | Value dilation = reifyConstantDim(attr: dilationAttr, builder); |
| 306 | Value dilated = arith::MulIOp::create(builder, lhs: dilation, rhs: subOne); |
| 307 | Value addOne = arith::AddIOp::create(builder, lhs: dilated, rhs: one); |
| 308 | |
| 309 | Value subtract = arith::SubIOp::create(builder, lhs: paddedAfter, rhs: addOne); |
| 310 | Value stride = reifyConstantDim(attr: strideAttr, builder); |
| 311 | Value divide = arith::DivUIOp::create(builder, lhs: subtract, rhs: stride); |
| 312 | return arith::AddIOp::create(builder, lhs: divide, rhs: one); |
| 313 | } |
| 314 | |
| 315 | // Creates a vector of the dynamic output dims for Conv2D and Depthwise_Conv2D |
| 316 | static SmallVector<Value> inferDynamicDimsForConv( |
| 317 | Location loc, Value input, Value weight, ShapedType resultTy, |
| 318 | ArrayRef<int64_t> padAttr, ArrayRef<int64_t> strideAttr, |
| 319 | ArrayRef<int64_t> dilationAttr, ArrayRef<int64_t> inputSizeDims, |
| 320 | ArrayRef<int64_t> kernelSizeDims, OpBuilder &rewriter) { |
| 321 | ShapedType inputTy = cast<ShapedType>(Val: input.getType()); |
| 322 | int64_t inputRank = inputTy.getRank(); |
| 323 | |
| 324 | SmallVector<Value> dynDims; |
| 325 | dynDims.resize(N: resultTy.getRank()); |
| 326 | |
| 327 | for (uint32_t i = 0, s = inputSizeDims.size(); i < s; ++i) { |
| 328 | int64_t inputDim = inputSizeDims[i]; |
| 329 | int64_t kernelDim = kernelSizeDims[i]; |
| 330 | if (resultTy.isDynamicDim(idx: inputDim)) { |
| 331 | auto padTop = padAttr[i * 2]; |
| 332 | auto padBottom = padAttr[i * 2 + 1]; |
| 333 | auto stride = strideAttr[i]; |
| 334 | auto dilation = dilationAttr[i]; |
| 335 | Value initDynDim = tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: inputDim); |
| 336 | Value kernelDynDim = |
| 337 | tensor::DimOp::create(builder&: rewriter, location: loc, source: weight, index: kernelDim); |
| 338 | // H = F(IH, pad_top, pad_bottom, dilation_y, KH, stride_y) |
| 339 | dynDims[inputDim] = |
| 340 | getConvOrPoolOutputDim(loc, inputDim: initDynDim, padBeforeAttr: padTop, padAfterAttr: padBottom, |
| 341 | kernelDim: kernelDynDim, strideAttr: stride, dilationAttr: dilation, rewriter); |
| 342 | } |
| 343 | } |
| 344 | |
| 345 | // Get the batch/channels dimensions. |
| 346 | for (int i = 0; i < inputRank; i++) { |
| 347 | if (resultTy.isDynamicDim(idx: i) && !dynDims[i]) |
| 348 | dynDims[i] = tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: i); |
| 349 | } |
| 350 | |
| 351 | SmallVector<Value> filteredDims = condenseValues(values: dynDims); |
| 352 | return filteredDims; |
| 353 | } |
| 354 | |
| 355 | // Creates a map to collapse the last dimension of the Depthwise convolution op |
| 356 | // due to a shape mismatch |
| 357 | static void createDepthwiseConvCollapseMap( |
| 358 | int64_t outputRank, SmallVector<ReassociationExprs, 4> &reassociationMap, |
| 359 | OpBuilder &rewriter) { |
| 360 | reassociationMap.resize(N: outputRank); |
| 361 | for (int i = 0; i < outputRank; i++) { |
| 362 | reassociationMap[i].push_back(Elt: rewriter.getAffineDimExpr(position: i)); |
| 363 | } |
| 364 | reassociationMap[outputRank - 1].push_back( |
| 365 | Elt: rewriter.getAffineDimExpr(position: outputRank)); |
| 366 | } |
| 367 | |
| 368 | namespace { |
| 369 | |
| 370 | template <typename TosaConvOp, typename LinalgConvOp, typename LinalgConvQOp> |
| 371 | class ConvConverter : public OpConversionPattern<TosaConvOp> { |
| 372 | public: |
| 373 | using OpConversionPattern<TosaConvOp>::OpConversionPattern; |
| 374 | LogicalResult |
| 375 | matchAndRewrite(TosaConvOp op, typename TosaConvOp::Adaptor adaptor, |
| 376 | ConversionPatternRewriter &rewriter) const final { |
| 377 | Location loc = op->getLoc(); |
| 378 | Value input = op->getOperand(0); |
| 379 | Value weight = op->getOperand(1); |
| 380 | Value bias = op->getOperand(2); |
| 381 | |
| 382 | ShapedType inputTy = cast<ShapedType>(Val: input.getType()); |
| 383 | ShapedType weightTy = cast<ShapedType>(Val: weight.getType()); |
| 384 | ShapedType biasTy = cast<ShapedType>(Val: bias.getType()); |
| 385 | ShapedType resultTy = cast<ShapedType>(op->getResult(0).getType()); |
| 386 | |
| 387 | Type inputETy = inputTy.getElementType(); |
| 388 | |
| 389 | DenseI64ArrayAttr padAttr = op.getPadAttr(); |
| 390 | DenseI64ArrayAttr strideTosaAttr = op.getStrideAttr(); |
| 391 | DenseI64ArrayAttr dilationTosaAttr = op.getDilationAttr(); |
| 392 | |
| 393 | Type accETy = op.getAccType(); |
| 394 | Type accTy = RankedTensorType::get(shape: resultTy.getShape(), elementType: accETy); |
| 395 | |
| 396 | // Get and verify zero points. |
| 397 | FailureOr<int64_t> maybeIZp = op.getInputZeroPoint(); |
| 398 | if (failed(Result: maybeIZp)) |
| 399 | return rewriter.notifyMatchFailure( |
| 400 | op, "input zero point cannot be statically determined" ); |
| 401 | |
| 402 | FailureOr<int64_t> maybeWZp = op.getWeightZeroPoint(); |
| 403 | if (failed(Result: maybeWZp)) |
| 404 | return rewriter.notifyMatchFailure( |
| 405 | op, "weight zero point cannot be statically determined" ); |
| 406 | |
| 407 | const int64_t inputZpVal = *maybeIZp; |
| 408 | const int64_t weightZpVal = *maybeWZp; |
| 409 | |
| 410 | if (op.verifyInputZeroPoint(inputZpVal).failed()) |
| 411 | return rewriter.notifyMatchFailure( |
| 412 | op, "input zero point must be zero for non-int8 integer types" ); |
| 413 | |
| 414 | if (op.verifyWeightZeroPoint(weightZpVal).failed()) |
| 415 | return rewriter.notifyMatchFailure( |
| 416 | op, "weight zero point must be zero for non-int8 integer types" ); |
| 417 | |
| 418 | bool hasZp = (inputZpVal != 0) || (weightZpVal != 0); |
| 419 | |
| 420 | if (!weightTy.hasStaticShape() || !biasTy.hasStaticShape()) |
| 421 | return rewriter.notifyMatchFailure( |
| 422 | op, "tosa.conv ops require static shapes for weight and bias" ); |
| 423 | |
| 424 | if (inputETy.isUnsignedInteger()) |
| 425 | return rewriter.notifyMatchFailure( |
| 426 | op, "tosa.conv ops does not support unsigned integer input" ); |
| 427 | |
| 428 | llvm::SmallVector<int64_t> inputSizeDims; |
| 429 | llvm::SmallVector<int64_t> kernelSizeDims; |
| 430 | for (int i = 1; i < resultTy.getRank() - 1; i++) { |
| 431 | inputSizeDims.push_back(Elt: i); |
| 432 | kernelSizeDims.push_back(Elt: i); |
| 433 | } |
| 434 | |
| 435 | SmallVector<Value> filteredDims = inferDynamicDimsForConv( |
| 436 | loc, input, weight, resultTy, padAttr: padAttr.asArrayRef(), |
| 437 | strideAttr: strideTosaAttr.asArrayRef(), dilationAttr: dilationTosaAttr.asArrayRef(), |
| 438 | inputSizeDims, kernelSizeDims, rewriter); |
| 439 | |
| 440 | auto weightShape = weightTy.getShape(); |
| 441 | |
| 442 | // Apply padding as necessary. |
| 443 | TypedAttr zeroAttr = rewriter.getZeroAttr(type: inputETy); |
| 444 | if (hasZp) { |
| 445 | int64_t intMin = |
| 446 | APInt::getSignedMinValue(numBits: inputETy.getIntOrFloatBitWidth()) |
| 447 | .getSExtValue(); |
| 448 | int64_t intMax = |
| 449 | APInt::getSignedMaxValue(numBits: inputETy.getIntOrFloatBitWidth()) |
| 450 | .getSExtValue(); |
| 451 | |
| 452 | if (inputZpVal < intMin || inputZpVal > intMax) |
| 453 | return rewriter.notifyMatchFailure( |
| 454 | op, "tosa.conv op quantization has zp outside of input range" ); |
| 455 | |
| 456 | zeroAttr = rewriter.getIntegerAttr(type: inputETy, value: inputZpVal); |
| 457 | } |
| 458 | |
| 459 | llvm::SmallVector<int64_t> pad; |
| 460 | pad.resize(N: 2, NV: 0); |
| 461 | llvm::append_range(C&: pad, R: padAttr.asArrayRef()); |
| 462 | pad.resize(N: pad.size() + 2, NV: 0); |
| 463 | input = applyPad(loc, input, pad, padAttr: zeroAttr, rewriter); |
| 464 | |
| 465 | if (4 == inputTy.getRank()) { |
| 466 | // For 2D convolutions, we need to check if the target convolution op |
| 467 | // wants a HWCF kernel layout. |
| 468 | bool wantHwcf = |
| 469 | hasZp ? std::is_same_v<LinalgConvQOp, linalg::Conv2DNhwcHwcfQOp> |
| 470 | : std::is_same_v<LinalgConvOp, linalg::Conv2DNhwcHwcfOp>; |
| 471 | if (wantHwcf) { |
| 472 | // Transpose the kernel to match dimension ordering of the linalg |
| 473 | // convolution operation. |
| 474 | // TODO(suderman): See if this can be efficiently folded - check whether |
| 475 | // the input is used anywhere else, if not fold the constant. |
| 476 | SmallVector<int32_t> weightPerm; |
| 477 | for (int i = 1; i < resultTy.getRank(); i++) |
| 478 | weightPerm.push_back(Elt: i); |
| 479 | weightPerm.push_back(Elt: 0); |
| 480 | |
| 481 | SmallVector<int64_t> newWeightShape; |
| 482 | for (auto dim : weightPerm) |
| 483 | newWeightShape.push_back(Elt: weightShape[dim]); |
| 484 | auto weightPermAttr = rewriter.getDenseI32ArrayAttr(values: weightPerm); |
| 485 | Type newWeightTy = |
| 486 | RankedTensorType::get(shape: newWeightShape, elementType: weightTy.getElementType()); |
| 487 | weight = tosa::TransposeOp::create(builder&: rewriter, location: loc, output: newWeightTy, input1: weight, |
| 488 | perms: weightPermAttr); |
| 489 | } |
| 490 | } |
| 491 | |
| 492 | // For Conv3D transpose the kernel to match dimension ordering of the linalg |
| 493 | // convolution operation. Conv2D has a 1-1 mapping in linalg so better to |
| 494 | // map directly and then transpose later if desired. |
| 495 | if (5 == inputTy.getRank()) { |
| 496 | // TODO(suderman): See if this can be efficiently folded - check whether |
| 497 | // the input is used anywhere else, if not fold the constant. |
| 498 | SmallVector<int32_t> weightPerm; |
| 499 | for (int i = 1; i < resultTy.getRank(); i++) |
| 500 | weightPerm.push_back(Elt: i); |
| 501 | weightPerm.push_back(Elt: 0); |
| 502 | |
| 503 | SmallVector<int64_t> newWeightShape; |
| 504 | for (auto dim : weightPerm) |
| 505 | newWeightShape.push_back(Elt: weightShape[dim]); |
| 506 | auto weightPermAttr = rewriter.getDenseI32ArrayAttr(values: weightPerm); |
| 507 | Type newWeightTy = |
| 508 | RankedTensorType::get(shape: newWeightShape, elementType: weightTy.getElementType()); |
| 509 | weight = tosa::TransposeOp::create(builder&: rewriter, location: loc, output: newWeightTy, input1: weight, |
| 510 | perms: weightPermAttr); |
| 511 | } |
| 512 | |
| 513 | // Extract the attributes for convolution. |
| 514 | ArrayRef<int64_t> stride = strideTosaAttr; |
| 515 | ArrayRef<int64_t> dilation = dilationTosaAttr; |
| 516 | |
| 517 | // Create the convolution op. |
| 518 | auto strideAttr = rewriter.getI64TensorAttr(values: stride); |
| 519 | auto dilationAttr = rewriter.getI64TensorAttr(values: dilation); |
| 520 | |
| 521 | Value biasEmptyTensor = tensor::EmptyOp::create( |
| 522 | builder&: rewriter, location: loc, staticShape: resultTy.getShape(), elementType: accETy, dynamicSizes: filteredDims); |
| 523 | |
| 524 | Value broadcastBias = |
| 525 | linalgBroadcastAndMaybeExt(rewriter, loc, source: bias, result: biasEmptyTensor); |
| 526 | |
| 527 | if (hasZp) { |
| 528 | auto iZp = rewriter.getI32IntegerAttr(value: inputZpVal); |
| 529 | auto kZp = rewriter.getI32IntegerAttr(value: weightZpVal); |
| 530 | |
| 531 | auto iZpVal = arith::ConstantOp::create(builder&: rewriter, location: loc, value: iZp); |
| 532 | auto kZpVal = arith::ConstantOp::create(builder&: rewriter, location: loc, value: kZp); |
| 533 | |
| 534 | Value conv = LinalgConvQOp::create( |
| 535 | rewriter, loc, resultTy, |
| 536 | ValueRange{input, weight, iZpVal, kZpVal}, |
| 537 | ValueRange{broadcastBias}, strideAttr, dilationAttr) |
| 538 | ->getResult(0); |
| 539 | |
| 540 | rewriter.replaceOp(op, conv); |
| 541 | return success(); |
| 542 | } |
| 543 | |
| 544 | Value conv = LinalgConvOp::create( |
| 545 | rewriter, loc, accTy, ValueRange{input, weight}, |
| 546 | ValueRange{broadcastBias}, strideAttr, dilationAttr) |
| 547 | ->getResult(0); |
| 548 | |
| 549 | // We may need to truncate back to the result type if the accumulator was |
| 550 | // wider than the result. |
| 551 | if (resultTy != accTy) |
| 552 | conv = tosa::CastOp::create(builder&: rewriter, location: loc, output: resultTy, input: conv); |
| 553 | |
| 554 | rewriter.replaceOp(op, conv); |
| 555 | return success(); |
| 556 | } |
| 557 | }; |
| 558 | |
| 559 | class DepthwiseConvConverter |
| 560 | : public OpConversionPattern<tosa::DepthwiseConv2DOp> { |
| 561 | public: |
| 562 | using OpConversionPattern<tosa::DepthwiseConv2DOp>::OpConversionPattern; |
| 563 | LogicalResult |
| 564 | matchAndRewrite(tosa::DepthwiseConv2DOp op, OpAdaptor adaptor, |
| 565 | ConversionPatternRewriter &rewriter) const final { |
| 566 | Location loc = op->getLoc(); |
| 567 | Value input = op->getOperand(idx: 0); |
| 568 | Value weight = op->getOperand(idx: 1); |
| 569 | Value bias = op->getOperand(idx: 2); |
| 570 | |
| 571 | ShapedType inputTy = cast<ShapedType>(Val: input.getType()); |
| 572 | ShapedType weightTy = cast<ShapedType>(Val: weight.getType()); |
| 573 | ShapedType biasTy = cast<ShapedType>(Val: bias.getType()); |
| 574 | ShapedType resultTy = cast<ShapedType>(Val: op->getResult(idx: 0).getType()); |
| 575 | int64_t resultRank = resultTy.getRank(); |
| 576 | |
| 577 | Type inputETy = inputTy.getElementType(); |
| 578 | Type resultETy = resultTy.getElementType(); |
| 579 | |
| 580 | auto padAttr = op.getPadAttr(); |
| 581 | auto strideTosaAttr = op.getStrideAttr(); |
| 582 | auto dilationTosaAttr = op.getDilationAttr(); |
| 583 | |
| 584 | Type accETy = op.getAccType(); |
| 585 | |
| 586 | if (!weightTy.hasStaticShape() || !biasTy.hasStaticShape()) |
| 587 | return rewriter.notifyMatchFailure( |
| 588 | arg&: op, msg: "tosa.depthwise_conv ops require static shapes" ); |
| 589 | |
| 590 | // Compute output dynamic dims |
| 591 | SmallVector<Value> filteredDims = inferDynamicDimsForConv( |
| 592 | loc, input, weight, resultTy, padAttr: padAttr.asArrayRef(), |
| 593 | strideAttr: strideTosaAttr.asArrayRef(), dilationAttr: dilationTosaAttr.asArrayRef(), |
| 594 | /*inputSizeDims=*/{1, 2}, |
| 595 | /*kernelSizeDims=*/{0, 1}, rewriter); |
| 596 | |
| 597 | // Get and verify zero points. |
| 598 | |
| 599 | FailureOr<int64_t> maybeIZp = op.getInputZeroPoint(); |
| 600 | FailureOr<int64_t> maybeWZp = op.getWeightZeroPoint(); |
| 601 | if (failed(Result: maybeIZp)) |
| 602 | return rewriter.notifyMatchFailure( |
| 603 | arg&: op, msg: "input zero point cannot be statically determined" ); |
| 604 | if (failed(Result: maybeWZp)) |
| 605 | return rewriter.notifyMatchFailure( |
| 606 | arg&: op, msg: "weight zero point cannot be statically determined" ); |
| 607 | |
| 608 | const int64_t inputZpVal = *maybeIZp; |
| 609 | const int64_t weightZpVal = *maybeWZp; |
| 610 | |
| 611 | if (op.verifyInputZeroPoint(zp: inputZpVal).failed()) |
| 612 | return rewriter.notifyMatchFailure( |
| 613 | arg&: op, msg: "input zero point must be zero for non-int8 integer types" ); |
| 614 | |
| 615 | if (op.verifyWeightZeroPoint(zp: weightZpVal).failed()) |
| 616 | return rewriter.notifyMatchFailure( |
| 617 | arg&: op, msg: "weight zero point must be zero for non-int8 integer types" ); |
| 618 | |
| 619 | bool hasNullZps = (inputZpVal == 0) && (weightZpVal == 0); |
| 620 | auto weightShape = weightTy.getShape(); |
| 621 | auto resultShape = resultTy.getShape(); |
| 622 | |
| 623 | // Apply padding as necessary. |
| 624 | TypedAttr zeroAttr = rewriter.getZeroAttr(type: inputETy); |
| 625 | if (!hasNullZps) { |
| 626 | int64_t intMin = |
| 627 | APInt::getSignedMinValue(numBits: inputETy.getIntOrFloatBitWidth()) |
| 628 | .getSExtValue(); |
| 629 | int64_t intMax = |
| 630 | APInt::getSignedMaxValue(numBits: inputETy.getIntOrFloatBitWidth()) |
| 631 | .getSExtValue(); |
| 632 | |
| 633 | if (inputZpVal < intMin || inputZpVal > intMax) |
| 634 | return rewriter.notifyMatchFailure( |
| 635 | arg&: op, msg: "tosa.depthwise_conv op quantization has zp outside of input " |
| 636 | "range" ); |
| 637 | |
| 638 | zeroAttr = rewriter.getIntegerAttr(type: inputETy, value: inputZpVal); |
| 639 | } |
| 640 | |
| 641 | llvm::SmallVector<int64_t> pad; |
| 642 | pad.resize(N: 2, NV: 0); |
| 643 | llvm::append_range(C&: pad, R: padAttr.asArrayRef()); |
| 644 | pad.resize(N: pad.size() + 2, NV: 0); |
| 645 | |
| 646 | input = applyPad(loc, input, pad, padAttr: zeroAttr, rewriter); |
| 647 | |
| 648 | // Extract the attributes for convolution. |
| 649 | ArrayRef<int64_t> stride = strideTosaAttr; |
| 650 | ArrayRef<int64_t> dilation = dilationTosaAttr; |
| 651 | |
| 652 | // Create the convolution op. |
| 653 | auto strideAttr = rewriter.getI64TensorAttr(values: stride); |
| 654 | auto dilationAttr = rewriter.getI64TensorAttr(values: dilation); |
| 655 | ShapedType linalgConvTy = |
| 656 | RankedTensorType::get(shape: {resultShape[0], resultShape[1], resultShape[2], |
| 657 | weightShape[2], weightShape[3]}, |
| 658 | elementType: accETy); |
| 659 | |
| 660 | auto resultZeroAttr = rewriter.getZeroAttr(type: accETy); |
| 661 | Value emptyTensor = tensor::EmptyOp::create( |
| 662 | builder&: rewriter, location: loc, staticShape: linalgConvTy.getShape(), elementType: accETy, dynamicSizes: filteredDims); |
| 663 | Value zero = arith::ConstantOp::create(builder&: rewriter, location: loc, value: resultZeroAttr); |
| 664 | Value zeroTensor = linalg::FillOp::create(builder&: rewriter, location: loc, inputs: ValueRange{zero}, |
| 665 | outputs: ValueRange{emptyTensor}) |
| 666 | .result(); |
| 667 | |
| 668 | Value biasEmptyTensor = tensor::EmptyOp::create( |
| 669 | builder&: rewriter, location: loc, staticShape: resultTy.getShape(), elementType: resultETy, dynamicSizes: filteredDims); |
| 670 | |
| 671 | // Broadcast the initial value to the output tensor before convolving. |
| 672 | SmallVector<AffineMap, 4> indexingMaps; |
| 673 | indexingMaps.push_back(Elt: getBroadcastingMap(rewriter, source: bias, result: biasEmptyTensor)); |
| 674 | indexingMaps.push_back(Elt: rewriter.getMultiDimIdentityMap(rank: resultRank)); |
| 675 | indexingMaps.push_back(Elt: rewriter.getMultiDimIdentityMap(rank: resultRank)); |
| 676 | |
| 677 | if (hasNullZps) { |
| 678 | Value conv = linalg::DepthwiseConv2DNhwcHwcmOp::create( |
| 679 | builder&: rewriter, location: loc, resultTensorTypes: linalgConvTy, inputs: ValueRange{input, weight}, |
| 680 | outputs: ValueRange{zeroTensor}, strides: strideAttr, dilations: dilationAttr) |
| 681 | .getResult(i: 0); |
| 682 | |
| 683 | // We may need to truncate back to the result type if the accumulator was |
| 684 | // wider than the result. |
| 685 | if (accETy != resultETy) |
| 686 | conv = tosa::CastOp::create( |
| 687 | builder&: rewriter, location: loc, |
| 688 | output: RankedTensorType::get(shape: cast<ShapedType>(Val: conv.getType()).getShape(), |
| 689 | elementType: resultETy), |
| 690 | input: conv); |
| 691 | |
| 692 | SmallVector<ReassociationExprs, 4> reassociationMap; |
| 693 | createDepthwiseConvCollapseMap(outputRank: resultRank, reassociationMap, rewriter); |
| 694 | Value convReshape = tensor::CollapseShapeOp::create( |
| 695 | builder&: rewriter, location: loc, resultType: resultTy, src: conv, reassociation: reassociationMap); |
| 696 | |
| 697 | Value result = |
| 698 | linalg::GenericOp::create( |
| 699 | builder&: rewriter, location: loc, resultTensorTypes: resultTy, inputs: ValueRange({bias, convReshape}), |
| 700 | outputs: biasEmptyTensor, indexingMaps, iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultRank), |
| 701 | odsArg5: [&](OpBuilder &nestedBuilder, Location nestedLoc, |
| 702 | ValueRange args) { |
| 703 | Value added; |
| 704 | if (llvm::isa<FloatType>(Val: inputETy)) |
| 705 | added = arith::AddFOp::create(builder&: nestedBuilder, location: loc, lhs: args[0], |
| 706 | rhs: args[1]); |
| 707 | else |
| 708 | added = arith::AddIOp::create(builder&: nestedBuilder, location: loc, lhs: args[0], |
| 709 | rhs: args[1]); |
| 710 | linalg::YieldOp::create(builder&: nestedBuilder, location: nestedLoc, values: added); |
| 711 | }) |
| 712 | .getResult(i: 0); |
| 713 | rewriter.replaceOp(op, newValues: result); |
| 714 | } else { |
| 715 | IntegerAttr iZp = rewriter.getI32IntegerAttr(value: inputZpVal); |
| 716 | IntegerAttr wZp = rewriter.getI32IntegerAttr(value: weightZpVal); |
| 717 | auto iZpVal = arith::ConstantOp::create(builder&: rewriter, location: loc, value: iZp); |
| 718 | auto kZpVal = arith::ConstantOp::create(builder&: rewriter, location: loc, value: wZp); |
| 719 | Value conv = linalg::DepthwiseConv2DNhwcHwcmQOp::create( |
| 720 | builder&: rewriter, location: loc, resultTensorTypes: linalgConvTy, |
| 721 | inputs: ValueRange{input, weight, iZpVal, kZpVal}, |
| 722 | outputs: ValueRange{zeroTensor}, strides: strideAttr, dilations: dilationAttr) |
| 723 | .getResult(i: 0); |
| 724 | SmallVector<ReassociationExprs, 4> reassociationMap; |
| 725 | createDepthwiseConvCollapseMap(outputRank: resultRank, reassociationMap, rewriter); |
| 726 | Value convReshape = tensor::CollapseShapeOp::create( |
| 727 | builder&: rewriter, location: loc, resultType: resultTy, src: conv, reassociation: reassociationMap); |
| 728 | Value result = linalgIntBroadcastExtSIAdd( |
| 729 | rewriter, loc, bias, conv: convReshape, result: biasEmptyTensor, indexingMaps); |
| 730 | rewriter.replaceOp(op, newValues: result); |
| 731 | } |
| 732 | return success(); |
| 733 | } |
| 734 | }; |
| 735 | |
| 736 | // Lower TOSA elementwise ops that have a direct linalg.elementwise named op |
| 737 | // counterpart. Unlike linalg.generic, linalg.elementwise dispatches on a single |
| 738 | // function kind and infers the body, so the conversion is a handful of checks |
| 739 | // (kind, ranks, static shapes) followed by construction of the indexing maps |
| 740 | // and the named op. |
| 741 | template <typename TosaOp> |
| 742 | class ElementwiseConverter : public OpConversionPattern<TosaOp> { |
| 743 | public: |
| 744 | using OpConversionPattern<TosaOp>::OpConversionPattern; |
| 745 | using typename OpConversionPattern<TosaOp>::OpAdaptor; |
| 746 | |
| 747 | LogicalResult |
| 748 | matchAndRewrite(TosaOp op, OpAdaptor adaptor, |
| 749 | ConversionPatternRewriter &rewriter) const final { |
| 750 | std::optional<linalg::ElementwiseKind> kind = |
| 751 | isConvertibleToLinalgElementwise(op); |
| 752 | if (!kind) |
| 753 | return rewriter.notifyMatchFailure( |
| 754 | op, "op has no linalg.elementwise counterpart" ); |
| 755 | |
| 756 | Location loc = op.getLoc(); |
| 757 | auto resultTy = cast<RankedTensorType>(op.getType()); |
| 758 | const int64_t rank = resultTy.getRank(); |
| 759 | |
| 760 | const unsigned arity = |
| 761 | llvm::to_underlying(E: linalg::getArityGroupAndKind(kind: *kind).arityGroup); |
| 762 | |
| 763 | // Trailing metadata operands (e.g. MulOp's shift, NegateOp's zero points) |
| 764 | // are not forwarded to linalg.elementwise. |
| 765 | ValueRange inputs = adaptor.getOperands().take_front(arity); |
| 766 | |
| 767 | // Build one indexing map per input, broadcasting static size-1 dimensions, |
| 768 | // followed by the identity map for the output. |
| 769 | SmallVector<AffineMap> indexingMaps; |
| 770 | for (Value operand : inputs) { |
| 771 | ArrayRef<int64_t> shape = cast<ShapedType>(Val: operand.getType()).getShape(); |
| 772 | SmallVector<AffineExpr> exprs; |
| 773 | for (const auto &dim : llvm::enumerate(First&: shape)) { |
| 774 | bool requiresBroadcast = |
| 775 | dim.value() == 1 && resultTy.getDimSize(dim.index()) != 1; |
| 776 | exprs.push_back(Elt: requiresBroadcast |
| 777 | ? rewriter.getAffineConstantExpr(constant: 0) |
| 778 | : rewriter.getAffineDimExpr(position: dim.index())); |
| 779 | } |
| 780 | indexingMaps.push_back(Elt: AffineMap::get(dimCount: rank, /*symbolCount=*/0, results: exprs, |
| 781 | context: rewriter.getContext())); |
| 782 | } |
| 783 | indexingMaps.push_back(Elt: rewriter.getMultiDimIdentityMap(rank)); |
| 784 | |
| 785 | // Shapes are guaranteed static by isConvertibleToLinalgElementwise. |
| 786 | Value emptyTensor = tensor::EmptyOp::create( |
| 787 | rewriter, loc, resultTy.getShape(), resultTy.getElementType()); |
| 788 | |
| 789 | rewriter.replaceOpWithNewOp<linalg::ElementwiseOp>( |
| 790 | op, inputs, ValueRange{emptyTensor}, *kind, |
| 791 | rewriter.getAffineMapArrayAttr(values: indexingMaps)); |
| 792 | return success(); |
| 793 | } |
| 794 | }; |
| 795 | |
| 796 | static SmallVector<ReassociationIndices> |
| 797 | getMatMulBatchCollapseReassociation(int64_t rank) { |
| 798 | SmallVector<ReassociationIndices> reassociation(3); |
| 799 | for (int64_t dim = 0; dim < rank - 2; ++dim) |
| 800 | reassociation[0].push_back(Elt: dim); |
| 801 | reassociation[1].push_back(Elt: rank - 2); |
| 802 | reassociation[2].push_back(Elt: rank - 1); |
| 803 | return reassociation; |
| 804 | } |
| 805 | |
| 806 | class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> { |
| 807 | public: |
| 808 | using OpConversionPattern<tosa::MatMulOp>::OpConversionPattern; |
| 809 | LogicalResult |
| 810 | matchAndRewrite(tosa::MatMulOp op, OpAdaptor adaptor, |
| 811 | ConversionPatternRewriter &rewriter) const final { |
| 812 | Location loc = op.getLoc(); |
| 813 | auto aType = dyn_cast<RankedTensorType>(Val: adaptor.getA().getType()); |
| 814 | auto bType = dyn_cast<RankedTensorType>(Val: adaptor.getB().getType()); |
| 815 | auto outputType = dyn_cast<RankedTensorType>(Val: op.getType()); |
| 816 | if (!aType || !bType || !outputType) |
| 817 | return rewriter.notifyMatchFailure(arg&: op, msg: "only supports ranked tensors" ); |
| 818 | |
| 819 | FailureOr<int64_t> maybeAZp = op.getAZeroPoint(); |
| 820 | FailureOr<int64_t> maybeBZp = op.getBZeroPoint(); |
| 821 | if (failed(Result: maybeAZp)) |
| 822 | return rewriter.notifyMatchFailure( |
| 823 | arg&: op, msg: "input A zero point cannot be statically determined" ); |
| 824 | if (failed(Result: maybeBZp)) |
| 825 | return rewriter.notifyMatchFailure( |
| 826 | arg&: op, msg: "input B zero point cannot be statically determined" ); |
| 827 | |
| 828 | const int64_t aZpVal = *maybeAZp; |
| 829 | const int64_t bZpVal = *maybeBZp; |
| 830 | |
| 831 | if (op.verifyAZeroPoint(zp: aZpVal).failed()) |
| 832 | return rewriter.notifyMatchFailure( |
| 833 | arg&: op, msg: "input A zero point must be zero for non-int8 integer types" ); |
| 834 | |
| 835 | if (op.verifyBZeroPoint(zp: bZpVal).failed()) |
| 836 | return rewriter.notifyMatchFailure( |
| 837 | arg&: op, msg: "input B zero point must be zero for non-int8 integer types" ); |
| 838 | |
| 839 | const int64_t outputRank = outputType.getRank(); |
| 840 | auto [a, b, zeroTensor] = |
| 841 | prepareMatMulOperands(rewriter, loc, a: adaptor.getA(), b: adaptor.getB(), |
| 842 | outputType, /*transposeB=*/false); |
| 843 | |
| 844 | Value aZp, bZp; |
| 845 | const bool hasNonzeroZeroPoint = aZpVal != 0 || bZpVal != 0; |
| 846 | if (hasNonzeroZeroPoint) { |
| 847 | aZp = arith::ConstantOp::create(builder&: rewriter, location: loc, |
| 848 | value: rewriter.getI32IntegerAttr(value: aZpVal)); |
| 849 | bZp = arith::ConstantOp::create(builder&: rewriter, location: loc, |
| 850 | value: rewriter.getI32IntegerAttr(value: bZpVal)); |
| 851 | } |
| 852 | |
| 853 | if (outputRank == 2) { |
| 854 | if (!hasNonzeroZeroPoint) { |
| 855 | rewriter.replaceOpWithNewOp<linalg::MatmulOp>(op, args: TypeRange{outputType}, |
| 856 | args: ValueRange{a, b}, |
| 857 | args: ValueRange{zeroTensor}); |
| 858 | } else { |
| 859 | rewriter.replaceOpWithNewOp<linalg::QuantizedMatmulOp>( |
| 860 | op, args: TypeRange{outputType}, args: ValueRange{a, b, aZp, bZp}, |
| 861 | args: ValueRange{zeroTensor}); |
| 862 | } |
| 863 | return success(); |
| 864 | } |
| 865 | |
| 866 | if (outputRank == 3) { |
| 867 | if (!hasNonzeroZeroPoint) { |
| 868 | rewriter.replaceOpWithNewOp<linalg::BatchMatmulOp>( |
| 869 | op, args: TypeRange{outputType}, args: ValueRange{a, b}, |
| 870 | args: ValueRange{zeroTensor}); |
| 871 | } else { |
| 872 | rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>( |
| 873 | op, args: TypeRange{outputType}, args: ValueRange{a, b, aZp, bZp}, |
| 874 | args: ValueRange{zeroTensor}); |
| 875 | } |
| 876 | return success(); |
| 877 | } |
| 878 | |
| 879 | SmallVector<ReassociationIndices> reassociation = |
| 880 | getMatMulBatchCollapseReassociation(rank: outputRank); |
| 881 | auto collapse = [&](Value value) { |
| 882 | auto type = cast<RankedTensorType>(Val: value.getType()); |
| 883 | auto collapsedType = |
| 884 | tensor::CollapseShapeOp::inferCollapsedType(type, reassociation); |
| 885 | return tensor::CollapseShapeOp::create(builder&: rewriter, location: loc, resultType: collapsedType, |
| 886 | src: value, reassociation); |
| 887 | }; |
| 888 | |
| 889 | Value collapsedA = collapse(a); |
| 890 | Value collapsedB = collapse(b); |
| 891 | Value collapsedInit = collapse(zeroTensor); |
| 892 | auto collapsedOutputType = cast<RankedTensorType>(Val: collapsedInit.getType()); |
| 893 | Value collapsedResult; |
| 894 | |
| 895 | if (!hasNonzeroZeroPoint) { |
| 896 | collapsedResult = |
| 897 | linalg::BatchMatmulOp::create( |
| 898 | builder&: rewriter, location: loc, resultTensorTypes: TypeRange{collapsedOutputType}, |
| 899 | inputs: ValueRange{collapsedA, collapsedB}, outputs: ValueRange{collapsedInit}) |
| 900 | .getResult(i: 0); |
| 901 | } else { |
| 902 | collapsedResult = linalg::QuantizedBatchMatmulOp::create( |
| 903 | builder&: rewriter, location: loc, resultTensorTypes: TypeRange{collapsedOutputType}, |
| 904 | inputs: ValueRange{collapsedA, collapsedB, aZp, bZp}, |
| 905 | outputs: ValueRange{collapsedInit}) |
| 906 | .getResult(i: 0); |
| 907 | } |
| 908 | |
| 909 | Value result = tensor::ExpandShapeOp::create( |
| 910 | builder&: rewriter, location: loc, resultType: outputType, src: collapsedResult, reassociation, |
| 911 | outputShape: tensor::getMixedSizes(builder&: rewriter, loc, value: zeroTensor)); |
| 912 | rewriter.replaceOp(op, newValues: result); |
| 913 | return success(); |
| 914 | } |
| 915 | }; |
| 916 | |
| 917 | class MatMulTConverter : public OpConversionPattern<tosa::MatMulTOp> { |
| 918 | public: |
| 919 | using OpConversionPattern<tosa::MatMulTOp>::OpConversionPattern; |
| 920 | LogicalResult |
| 921 | matchAndRewrite(tosa::MatMulTOp op, OpAdaptor adaptor, |
| 922 | ConversionPatternRewriter &rewriter) const final { |
| 923 | Location loc = op.getLoc(); |
| 924 | auto aType = dyn_cast<RankedTensorType>(Val: adaptor.getA().getType()); |
| 925 | auto bType = dyn_cast<RankedTensorType>(Val: adaptor.getB().getType()); |
| 926 | auto outputType = dyn_cast<RankedTensorType>(Val: op.getType()); |
| 927 | if (!aType || !bType || !outputType) |
| 928 | return rewriter.notifyMatchFailure(arg&: op, msg: "only supports ranked tensors" ); |
| 929 | |
| 930 | FailureOr<int64_t> maybeAZp = op.getAZeroPoint(); |
| 931 | FailureOr<int64_t> maybeBZp = op.getBZeroPoint(); |
| 932 | if (failed(Result: maybeAZp)) |
| 933 | return rewriter.notifyMatchFailure( |
| 934 | arg&: op, msg: "input A zero point cannot be statically determined" ); |
| 935 | if (failed(Result: maybeBZp)) |
| 936 | return rewriter.notifyMatchFailure( |
| 937 | arg&: op, msg: "input B zero point cannot be statically determined" ); |
| 938 | |
| 939 | const int64_t aZpVal = *maybeAZp; |
| 940 | const int64_t bZpVal = *maybeBZp; |
| 941 | |
| 942 | if (aZpVal != 0 || bZpVal != 0) |
| 943 | return rewriter.notifyMatchFailure( |
| 944 | arg&: op, msg: "nonzero zero points require the generic MATMUL_T lowering" ); |
| 945 | |
| 946 | const int64_t outputRank = outputType.getRank(); |
| 947 | auto [a, b, zeroTensor] = |
| 948 | prepareMatMulOperands(rewriter, loc, a: adaptor.getA(), b: adaptor.getB(), |
| 949 | outputType, /*transposeB=*/true); |
| 950 | |
| 951 | if (outputRank == 2) { |
| 952 | rewriter.replaceOpWithNewOp<linalg::MatmulTransposeBOp>( |
| 953 | op, args: TypeRange{outputType}, args: ValueRange{a, b}, args: ValueRange{zeroTensor}); |
| 954 | return success(); |
| 955 | } |
| 956 | |
| 957 | if (outputRank == 3) { |
| 958 | rewriter.replaceOpWithNewOp<linalg::BatchMatmulTransposeBOp>( |
| 959 | op, args: TypeRange{outputType}, args: ValueRange{a, b}, args: ValueRange{zeroTensor}); |
| 960 | return success(); |
| 961 | } |
| 962 | |
| 963 | SmallVector<ReassociationIndices> reassociation = |
| 964 | getMatMulBatchCollapseReassociation(rank: outputRank); |
| 965 | auto collapse = [&](Value value) { |
| 966 | auto type = cast<RankedTensorType>(Val: value.getType()); |
| 967 | auto collapsedType = |
| 968 | tensor::CollapseShapeOp::inferCollapsedType(type, reassociation); |
| 969 | return tensor::CollapseShapeOp::create(builder&: rewriter, location: loc, resultType: collapsedType, |
| 970 | src: value, reassociation); |
| 971 | }; |
| 972 | Value collapsedA = collapse(a); |
| 973 | Value collapsedB = collapse(b); |
| 974 | Value collapsedInit = collapse(zeroTensor); |
| 975 | auto collapsedOutputType = cast<RankedTensorType>(Val: collapsedInit.getType()); |
| 976 | Value collapsedResult = |
| 977 | linalg::BatchMatmulTransposeBOp::create( |
| 978 | builder&: rewriter, location: loc, resultTensorTypes: TypeRange{collapsedOutputType}, |
| 979 | inputs: ValueRange{collapsedA, collapsedB}, outputs: ValueRange{collapsedInit}) |
| 980 | .getResult(i: 0); |
| 981 | Value result = tensor::ExpandShapeOp::create( |
| 982 | builder&: rewriter, location: loc, resultType: outputType, src: collapsedResult, reassociation, |
| 983 | outputShape: tensor::getMixedSizes(builder&: rewriter, loc, value: zeroTensor)); |
| 984 | rewriter.replaceOp(op, newValues: result); |
| 985 | return success(); |
| 986 | } |
| 987 | }; |
| 988 | |
| 989 | class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> { |
| 990 | public: |
| 991 | MaxPool2dConverter(const TypeConverter &typeConverter, MLIRContext *context, |
| 992 | bool allowNonFinites) |
| 993 | : OpConversionPattern(typeConverter, context), |
| 994 | allowNonFinites(allowNonFinites) {} |
| 995 | |
| 996 | // Compute the dynamic output sizes of the maxpool operation. |
| 997 | static SmallVector<Value> |
| 998 | computeDynamicOutputSizes(tosa::MaxPool2dOp op, OpAdaptor adaptor, |
| 999 | ConversionPatternRewriter &rewriter) { |
| 1000 | TensorType resultTy = op.getType(); |
| 1001 | Location loc = op.getLoc(); |
| 1002 | |
| 1003 | Value input = adaptor.getInput(); |
| 1004 | ArrayRef<int64_t> kernel = op.getKernel(); |
| 1005 | ArrayRef<int64_t> pad = op.getPad(); |
| 1006 | ArrayRef<int64_t> stride = op.getStride(); |
| 1007 | |
| 1008 | SmallVector<Value> dynamicDims; |
| 1009 | |
| 1010 | // Batch dimension |
| 1011 | if (resultTy.isDynamicDim(idx: 0)) |
| 1012 | dynamicDims.push_back(Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: 0)); |
| 1013 | |
| 1014 | // Height/width dimensions |
| 1015 | for (int64_t dim : {1, 2}) { |
| 1016 | if (!resultTy.isDynamicDim(idx: dim)) |
| 1017 | continue; |
| 1018 | |
| 1019 | // Index into the attribute arrays |
| 1020 | int64_t index = dim - 1; |
| 1021 | |
| 1022 | // Input height/width |
| 1023 | Value ihw = tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: dim); |
| 1024 | |
| 1025 | // Kernel height/width |
| 1026 | Value khw = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: kernel[index]); |
| 1027 | |
| 1028 | // Output height/width |
| 1029 | Value ohw = getConvOrPoolOutputDim(loc, inputDim: ihw, padBeforeAttr: pad[index * 2], |
| 1030 | padAfterAttr: pad[index * 2 + 1], kernelDim: khw, strideAttr: stride[index], |
| 1031 | /*dilationAttr=*/1, rewriter); |
| 1032 | dynamicDims.push_back(Elt: ohw); |
| 1033 | } |
| 1034 | |
| 1035 | // Channel dimension |
| 1036 | if (resultTy.isDynamicDim(idx: 3)) |
| 1037 | dynamicDims.push_back(Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: 3)); |
| 1038 | |
| 1039 | return dynamicDims; |
| 1040 | } |
| 1041 | |
| 1042 | LogicalResult |
| 1043 | matchAndRewrite(tosa::MaxPool2dOp op, OpAdaptor adaptor, |
| 1044 | ConversionPatternRewriter &rewriter) const final { |
| 1045 | Location loc = op.getLoc(); |
| 1046 | Value input = adaptor.getInput(); |
| 1047 | ShapedType inputTy = cast<ShapedType>(Val: input.getType()); |
| 1048 | |
| 1049 | bool isUnsigned = op.getType().getElementType().isUnsignedInteger(); |
| 1050 | ShapedType resultTy = |
| 1051 | getTypeConverter()->convertType<ShapedType>(t: op.getType()); |
| 1052 | if (!resultTy) |
| 1053 | return rewriter.notifyMatchFailure(arg&: op, msg: "failed to convert type" ); |
| 1054 | Type resultETy = inputTy.getElementType(); |
| 1055 | NanPropagationMode nanMode = op.getNanMode(); |
| 1056 | |
| 1057 | SmallVector<Value> dynamicDims = |
| 1058 | computeDynamicOutputSizes(op, adaptor, rewriter); |
| 1059 | |
| 1060 | // Determine what the initial value needs to be for the max pool op. |
| 1061 | TypedAttr initialAttr; |
| 1062 | if (resultETy.isF32() || resultETy.isBF16() || resultETy.isF16()) { |
| 1063 | const llvm::fltSemantics &semantics = |
| 1064 | cast<FloatType>(Val&: resultETy).getFloatSemantics(); |
| 1065 | if (llvm::APFloat::semanticsHasNaN(semantics)) |
| 1066 | initialAttr = rewriter.getFloatAttr( |
| 1067 | type: resultETy, |
| 1068 | value: nanMode == NanPropagationMode::IGNORE |
| 1069 | ? APFloat::getNaN(Sem: semantics) |
| 1070 | : getFloatMinMaxIdentity(semantics, |
| 1071 | /*negative=*/true, allowNonFinites)); |
| 1072 | } |
| 1073 | |
| 1074 | else if (isUnsigned) |
| 1075 | initialAttr = rewriter.getIntegerAttr( |
| 1076 | type: resultETy, value: APInt::getZero(numBits: resultETy.getIntOrFloatBitWidth())); |
| 1077 | else if (isa<IntegerType>(Val: resultETy)) |
| 1078 | initialAttr = rewriter.getIntegerAttr( |
| 1079 | type: resultETy, |
| 1080 | value: APInt::getSignedMinValue(numBits: resultETy.getIntOrFloatBitWidth())); |
| 1081 | |
| 1082 | if (!initialAttr) |
| 1083 | return rewriter.notifyMatchFailure( |
| 1084 | arg&: op, msg: "Unsupported initial value for tosa.maxpool_2d op" ); |
| 1085 | |
| 1086 | // Apply padding as necessary. |
| 1087 | llvm::SmallVector<int64_t> pad; |
| 1088 | pad.resize(N: 2, NV: 0); |
| 1089 | llvm::append_range(C&: pad, R: op.getPad()); |
| 1090 | pad.resize(N: pad.size() + 2, NV: 0); |
| 1091 | |
| 1092 | // The initial value doubles as the padding value. This is safe for every |
| 1093 | // seed above because the verifier rejects a pad as wide as the kernel on |
| 1094 | // any side, so no window consists purely of padding. |
| 1095 | Value paddedInput = applyPad(loc, input, pad, padAttr: initialAttr, rewriter); |
| 1096 | |
| 1097 | Value initialValue = arith::ConstantOp::create(builder&: rewriter, location: loc, value: initialAttr); |
| 1098 | |
| 1099 | ArrayRef<int64_t> kernel = op.getKernel(); |
| 1100 | ArrayRef<int64_t> stride = op.getStride(); |
| 1101 | |
| 1102 | Attribute strideAttr = rewriter.getI64VectorAttr(values: stride); |
| 1103 | Attribute dilationAttr = rewriter.getI64VectorAttr(values: {1, 1}); |
| 1104 | |
| 1105 | // Create the linalg op that performs pooling. |
| 1106 | Value emptyTensor = |
| 1107 | tensor::EmptyOp::create(builder&: rewriter, location: loc, staticShape: resultTy.getShape(), |
| 1108 | elementType: resultTy.getElementType(), dynamicSizes: dynamicDims); |
| 1109 | |
| 1110 | Value filledEmptyTensor = |
| 1111 | linalg::FillOp::create(builder&: rewriter, location: loc, inputs: initialValue, outputs: emptyTensor) |
| 1112 | .result(); |
| 1113 | |
| 1114 | Value fakeWindowDims = |
| 1115 | tensor::EmptyOp::create(builder&: rewriter, location: loc, staticShape: kernel, elementType: resultETy); |
| 1116 | |
| 1117 | if (isUnsigned) { |
| 1118 | rewriter.replaceOpWithNewOp<linalg::PoolingNhwcMaxUnsignedOp>( |
| 1119 | op, args: ArrayRef<Type>{resultTy}, args: ValueRange{paddedInput, fakeWindowDims}, |
| 1120 | args&: filledEmptyTensor, args&: strideAttr, args&: dilationAttr); |
| 1121 | return llvm::success(); |
| 1122 | } |
| 1123 | |
| 1124 | auto resultOp = linalg::PoolingNhwcMaxOp::create( |
| 1125 | builder&: rewriter, location: op->getLoc(), resultTensorTypes: ArrayRef<Type>{resultTy}, |
| 1126 | inputs: ValueRange{paddedInput, fakeWindowDims}, outputs: filledEmptyTensor, strides: strideAttr, |
| 1127 | dilations: dilationAttr); |
| 1128 | |
| 1129 | rewriter.replaceOp(op, newOp: resultOp); |
| 1130 | |
| 1131 | // NaN propagation has no meaning for non floating point types. |
| 1132 | if (!isa<FloatType>(Val: getElementTypeOrSelf(type: inputTy))) |
| 1133 | return success(); |
| 1134 | |
| 1135 | // "PROPAGATE" mode matches the behaviour of the LinAlg named op, so no |
| 1136 | // compare and select materialization is required. |
| 1137 | // |
| 1138 | // In the case of "IGNORE" we need to insert a compare and select. Since |
| 1139 | // we've already produced a named op we will just take its body and modify |
| 1140 | // it to include the appropriate checks. A NaN accumulator represents a |
| 1141 | // window with no finite values yet. The first finite input replaces it; |
| 1142 | // input NaNs are ignored. An all-NaN window therefore remains NaN. |
| 1143 | if (nanMode == NanPropagationMode::IGNORE) { |
| 1144 | auto genericOp = linalg::GenericOp::create( |
| 1145 | builder&: rewriter, location: loc, resultTensorTypes: resultOp.getType(i: 0), inputs: resultOp.getInputs(), |
| 1146 | outputs: resultOp.getOutputs(), indexingMaps: resultOp.getIndexingMapsArray(), |
| 1147 | iteratorTypes: resultOp.getIteratorTypesArray(), |
| 1148 | odsArg5: [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) { |
| 1149 | IRMapping map; |
| 1150 | auto oldBlock = resultOp.getRegion().begin(); |
| 1151 | auto oldArgs = oldBlock->getArguments(); |
| 1152 | auto &oldMaxOp = *resultOp.getBlock()->begin(); |
| 1153 | map.map(from&: oldArgs, to&: blockArgs); |
| 1154 | auto *newOp = opBuilder.clone(op&: oldMaxOp, mapper&: map); |
| 1155 | Value inputIsNaN = |
| 1156 | arith::CmpFOp::create(builder&: opBuilder, location: loc, predicate: arith::CmpFPredicate::UNO, |
| 1157 | lhs: blockArgs.front(), rhs: blockArgs.front()); |
| 1158 | Value accumulatorIsNaN = |
| 1159 | arith::CmpFOp::create(builder&: opBuilder, location: loc, predicate: arith::CmpFPredicate::UNO, |
| 1160 | lhs: blockArgs.back(), rhs: blockArgs.back()); |
| 1161 | Value finiteMaximum = |
| 1162 | arith::SelectOp::create(builder&: opBuilder, location: loc, condition: accumulatorIsNaN, |
| 1163 | true_value: blockArgs.front(), false_value: newOp->getResult(idx: 0)); |
| 1164 | Value result = arith::SelectOp::create( |
| 1165 | builder&: opBuilder, location: loc, condition: inputIsNaN, true_value: blockArgs.back(), false_value: finiteMaximum); |
| 1166 | linalg::YieldOp::create(builder&: opBuilder, location: loc, values: result); |
| 1167 | }); |
| 1168 | rewriter.replaceOp(op: resultOp, newOp: genericOp); |
| 1169 | } |
| 1170 | |
| 1171 | return success(); |
| 1172 | } |
| 1173 | |
| 1174 | private: |
| 1175 | bool allowNonFinites; |
| 1176 | }; |
| 1177 | |
| 1178 | class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> { |
| 1179 | public: |
| 1180 | using OpRewritePattern<tosa::AvgPool2dOp>::OpRewritePattern; |
| 1181 | |
| 1182 | LogicalResult matchAndRewrite(tosa::AvgPool2dOp op, |
| 1183 | PatternRewriter &rewriter) const final { |
| 1184 | Location loc = op.getLoc(); |
| 1185 | Value input = op.getInput(); |
| 1186 | ShapedType inputTy = cast<ShapedType>(Val: input.getType()); |
| 1187 | Type inElementTy = inputTy.getElementType(); |
| 1188 | |
| 1189 | ShapedType resultTy = cast<ShapedType>(Val: op.getType()); |
| 1190 | Type resultETy = cast<ShapedType>(Val: op.getType()).getElementType(); |
| 1191 | |
| 1192 | Type accETy = op.getAccType(); |
| 1193 | ShapedType accTy = resultTy.clone(elementType: accETy); |
| 1194 | |
| 1195 | auto dynamicDimsOr = |
| 1196 | checkHasDynamicBatchDims(rewriter, op, params: {input, op.getOutput()}); |
| 1197 | if (!dynamicDimsOr.has_value()) |
| 1198 | return failure(); |
| 1199 | SmallVector<Value> dynamicDims = *dynamicDimsOr; |
| 1200 | |
| 1201 | FailureOr<int64_t> maybeIZp = op.getInputZeroPoint(); |
| 1202 | FailureOr<int64_t> maybeOZp = op.getOutputZeroPoint(); |
| 1203 | if (failed(Result: maybeIZp)) |
| 1204 | return rewriter.notifyMatchFailure( |
| 1205 | arg&: op, msg: "input zero point could not be statically determined" ); |
| 1206 | if (failed(Result: maybeOZp)) |
| 1207 | return rewriter.notifyMatchFailure( |
| 1208 | arg&: op, msg: "output zero point could not be statically determined" ); |
| 1209 | |
| 1210 | const int64_t inputZpVal = *maybeIZp; |
| 1211 | const int64_t outputZpVal = *maybeOZp; |
| 1212 | |
| 1213 | // Apply padding as necessary. |
| 1214 | llvm::SmallVector<int64_t> pad; |
| 1215 | pad.resize(N: 2, NV: 0); |
| 1216 | llvm::append_range(C&: pad, R: op.getPad()); |
| 1217 | pad.resize(N: pad.size() + 2, NV: 0); |
| 1218 | TypedAttr padAttr = rewriter.getZeroAttr(type: inElementTy); |
| 1219 | // Unsupported element type |
| 1220 | if (!padAttr) |
| 1221 | return failure(); |
| 1222 | Value paddedInput = applyPad(loc, input, pad, padAttr, rewriter); |
| 1223 | |
| 1224 | auto initialAttr = rewriter.getZeroAttr(type: accETy); |
| 1225 | Value initialValue = arith::ConstantOp::create(builder&: rewriter, location: loc, value: initialAttr); |
| 1226 | |
| 1227 | ArrayRef<int64_t> kernel = op.getKernel(); |
| 1228 | ArrayRef<int64_t> stride = op.getStride(); |
| 1229 | |
| 1230 | Attribute strideAttr = rewriter.getI64VectorAttr(values: stride); |
| 1231 | Attribute dilationAttr = rewriter.getI64VectorAttr(values: {1, 1}); |
| 1232 | |
| 1233 | // Create the linalg op that performs pooling. |
| 1234 | Value poolEmptyTensor = tensor::EmptyOp::create( |
| 1235 | builder&: rewriter, location: loc, staticShape: accTy.getShape(), elementType: accETy, dynamicSizes: dynamicDims); |
| 1236 | |
| 1237 | Value filledEmptyTensor = |
| 1238 | linalg::FillOp::create(builder&: rewriter, location: loc, inputs: ValueRange{initialValue}, |
| 1239 | outputs: ValueRange{poolEmptyTensor}) |
| 1240 | .result(); |
| 1241 | |
| 1242 | Value fakeWindowDims = |
| 1243 | tensor::EmptyOp::create(builder&: rewriter, location: loc, staticShape: kernel, elementType: accETy); |
| 1244 | |
| 1245 | // Sum across the pooled region. |
| 1246 | Value poolingOp = linalg::PoolingNhwcSumOp::create( |
| 1247 | builder&: rewriter, location: loc, resultTensorTypes: ArrayRef<Type>{accTy}, |
| 1248 | inputs: ValueRange{paddedInput, fakeWindowDims}, |
| 1249 | outputs: filledEmptyTensor, strides: strideAttr, dilations: dilationAttr) |
| 1250 | .getResult(i: 0); |
| 1251 | |
| 1252 | // Normalize the summed value by the number of elements grouped in each |
| 1253 | // pool. |
| 1254 | Value iH = tensor::DimOp::create(builder&: rewriter, location: loc, source: poolingOp, index: 1); |
| 1255 | Value iW = tensor::DimOp::create(builder&: rewriter, location: loc, source: poolingOp, index: 2); |
| 1256 | |
| 1257 | auto one = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1); |
| 1258 | iH = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: iH, rhs: one); |
| 1259 | iW = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: iW, rhs: one); |
| 1260 | |
| 1261 | Value genericEmptyTensor = tensor::EmptyOp::create( |
| 1262 | builder&: rewriter, location: loc, staticShape: resultTy.getShape(), elementType: resultETy, dynamicSizes: dynamicDims); |
| 1263 | |
| 1264 | auto affineMap = rewriter.getMultiDimIdentityMap(rank: resultTy.getRank()); |
| 1265 | auto genericOp = linalg::GenericOp::create( |
| 1266 | builder&: rewriter, location: loc, resultTensorTypes: ArrayRef<Type>({resultTy}), inputs: ValueRange{poolingOp}, |
| 1267 | outputs: ValueRange{genericEmptyTensor}, |
| 1268 | indexingMaps: ArrayRef<AffineMap>({affineMap, affineMap}), |
| 1269 | iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultTy.getRank()), |
| 1270 | odsArg5: [&](OpBuilder &b, Location loc, ValueRange args) { |
| 1271 | auto zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 1272 | |
| 1273 | // Determines what the portion of valid input is covered by the |
| 1274 | // kernel. |
| 1275 | auto padFn = [&](Value valid, Value pos, int64_t pad) -> Value { |
| 1276 | if (pad == 0) |
| 1277 | return valid; |
| 1278 | |
| 1279 | auto padVal = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: pad); |
| 1280 | Value dpos = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: pos, rhs: padVal); |
| 1281 | |
| 1282 | Value offset = arith::MinSIOp::create(builder&: rewriter, location: loc, lhs: dpos, rhs: zero); |
| 1283 | return arith::AddIOp::create(builder&: rewriter, location: loc, lhs: valid, rhs: offset) |
| 1284 | ->getResult(idx: 0); |
| 1285 | }; |
| 1286 | |
| 1287 | auto coverageFn = [&](int64_t i, Value isize) -> Value { |
| 1288 | Value strideVal = |
| 1289 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: stride[i - 1]); |
| 1290 | Value val = |
| 1291 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: kernel[i - 1]); |
| 1292 | |
| 1293 | // Find the position relative to the input tensor's ends. |
| 1294 | Value left = linalg::IndexOp::create(builder&: rewriter, location: loc, dim: i); |
| 1295 | Value right = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: isize, rhs: left); |
| 1296 | left = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: left, rhs: strideVal); |
| 1297 | right = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: right, rhs: strideVal); |
| 1298 | |
| 1299 | // Determine how much padding was included. |
| 1300 | val = padFn(val, left, pad[i * 2]); |
| 1301 | val = padFn(val, right, pad[i * 2 + 1]); |
| 1302 | return arith::MaxSIOp::create(builder&: rewriter, location: loc, lhs: one, rhs: val); |
| 1303 | }; |
| 1304 | |
| 1305 | // Compute the indices from either end. |
| 1306 | Value kH3 = coverageFn(1, iH); |
| 1307 | Value kW3 = coverageFn(2, iW); |
| 1308 | |
| 1309 | // Compute the total number of elements and normalize. |
| 1310 | auto count = arith::IndexCastOp::create( |
| 1311 | builder&: rewriter, location: loc, out: rewriter.getI32Type(), |
| 1312 | in: arith::MulIOp::create(builder&: rewriter, location: loc, lhs: kH3, rhs: kW3)); |
| 1313 | |
| 1314 | // Divide by the number of summed values. For floats this is just |
| 1315 | // a div however for quantized values input normalization had |
| 1316 | // to be applied. |
| 1317 | Value poolVal = args[0]; |
| 1318 | if (isa<FloatType>(Val: accETy)) { |
| 1319 | auto countF = arith::SIToFPOp::create(builder&: rewriter, location: loc, out: accETy, in: count); |
| 1320 | poolVal = arith::DivFOp::create(builder&: rewriter, location: loc, lhs: poolVal, rhs: countF) |
| 1321 | ->getResult(idx: 0); |
| 1322 | if (accETy.getIntOrFloatBitWidth() > |
| 1323 | resultETy.getIntOrFloatBitWidth()) |
| 1324 | poolVal = |
| 1325 | arith::TruncFOp::create(builder&: rewriter, location: loc, out: resultETy, in: poolVal); |
| 1326 | } else { |
| 1327 | |
| 1328 | // If we have quantization information we need to apply an offset |
| 1329 | // for the input zp value. |
| 1330 | if (inputZpVal != 0) { |
| 1331 | auto inputZp = arith::ConstantOp::create( |
| 1332 | builder&: rewriter, location: loc, value: b.getIntegerAttr(type: accETy, value: inputZpVal)); |
| 1333 | Value offset = |
| 1334 | arith::MulIOp::create(builder&: rewriter, location: loc, result: accETy, lhs: count, rhs: inputZp); |
| 1335 | poolVal = |
| 1336 | arith::SubIOp::create(builder&: rewriter, location: loc, result: accETy, lhs: poolVal, rhs: offset); |
| 1337 | } |
| 1338 | |
| 1339 | // Compute: k = 32 - count_leading_zeros(value - 1) |
| 1340 | Value one32 = arith::ConstantOp::create( |
| 1341 | builder&: rewriter, location: loc, value: rewriter.getI32IntegerAttr(value: 1)); |
| 1342 | Value thirtyTwo32 = arith::ConstantOp::create( |
| 1343 | builder&: rewriter, location: loc, value: rewriter.getI32IntegerAttr(value: 32)); |
| 1344 | |
| 1345 | Value countSubOne = |
| 1346 | arith::SubIOp::create(builder&: rewriter, location: loc, lhs: count, rhs: one32); |
| 1347 | Value leadingZeros = |
| 1348 | math::CountLeadingZerosOp::create(builder&: rewriter, location: loc, operand: countSubOne); |
| 1349 | Value k = |
| 1350 | arith::SubIOp::create(builder&: rewriter, location: loc, lhs: thirtyTwo32, rhs: leadingZeros); |
| 1351 | |
| 1352 | // Compute: numerator = ((1 << 30) + 1) << k |
| 1353 | Value k64 = |
| 1354 | arith::ExtUIOp::create(builder&: rewriter, location: loc, out: rewriter.getI64Type(), in: k); |
| 1355 | Value thirtyShiftPlusOne = arith::ConstantOp::create( |
| 1356 | builder&: rewriter, location: loc, value: rewriter.getI64IntegerAttr(value: (1 << 30) + 1)); |
| 1357 | Value numerator = |
| 1358 | arith::ShLIOp::create(builder&: rewriter, location: loc, lhs: thirtyShiftPlusOne, rhs: k64); |
| 1359 | |
| 1360 | // Compute: scale.multiplier = numerator / value; |
| 1361 | Value count64 = arith::ExtUIOp::create( |
| 1362 | builder&: rewriter, location: loc, out: rewriter.getI64Type(), in: count); |
| 1363 | Value multiplier = |
| 1364 | arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: numerator, rhs: count64); |
| 1365 | multiplier = arith::TruncIOp::create( |
| 1366 | builder&: rewriter, location: loc, out: rewriter.getI32Type(), in: multiplier); |
| 1367 | |
| 1368 | // Compute: scale.shift = 30 + k |
| 1369 | Value k8 = |
| 1370 | arith::TruncIOp::create(builder&: rewriter, location: loc, out: rewriter.getI8Type(), in: k); |
| 1371 | Value thirty8 = arith::ConstantOp::create( |
| 1372 | builder&: rewriter, location: loc, value: rewriter.getI8IntegerAttr(value: 30)); |
| 1373 | Value shift = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: k8, rhs: thirty8); |
| 1374 | |
| 1375 | auto roundingAttr = RoundingModeAttr::get( |
| 1376 | context: rewriter.getContext(), value: RoundingMode::SINGLE_ROUND); |
| 1377 | |
| 1378 | auto scaled = tosa::ApplyScaleOp::create( |
| 1379 | builder&: rewriter, location: loc, output: rewriter.getI32Type(), value: poolVal, |
| 1380 | multiplier, shift, rounding_mode: roundingAttr) |
| 1381 | .getResult(); |
| 1382 | |
| 1383 | // If we have quantization information we need to apply output |
| 1384 | // zeropoint. |
| 1385 | if (outputZpVal != 0) { |
| 1386 | auto outputZp = arith::ConstantOp::create( |
| 1387 | builder&: rewriter, location: loc, |
| 1388 | value: b.getIntegerAttr(type: scaled.getType(), value: outputZpVal)); |
| 1389 | scaled = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: scaled, rhs: outputZp) |
| 1390 | .getResult(); |
| 1391 | } |
| 1392 | |
| 1393 | // Apply Clip. |
| 1394 | int64_t outBitwidth = resultETy.getIntOrFloatBitWidth(); |
| 1395 | |
| 1396 | auto min = arith::ConstantIntOp::create( |
| 1397 | builder&: rewriter, location: loc, type: accETy, |
| 1398 | value: APInt::getSignedMinValue(numBits: outBitwidth).getSExtValue()); |
| 1399 | auto max = arith::ConstantIntOp::create( |
| 1400 | builder&: rewriter, location: loc, type: accETy, |
| 1401 | value: APInt::getSignedMaxValue(numBits: outBitwidth).getSExtValue()); |
| 1402 | auto clamp = clampIntHelper(loc, arg: scaled, min, max, rewriter, |
| 1403 | /*isUnsigned=*/false); |
| 1404 | |
| 1405 | poolVal = clamp; |
| 1406 | // Convert type. |
| 1407 | if (resultETy != clamp.getType()) { |
| 1408 | poolVal = |
| 1409 | arith::TruncIOp::create(builder&: rewriter, location: loc, out: resultETy, in: poolVal); |
| 1410 | } |
| 1411 | } |
| 1412 | |
| 1413 | linalg::YieldOp::create(builder&: rewriter, location: loc, values: poolVal); |
| 1414 | }); |
| 1415 | |
| 1416 | rewriter.replaceOp(op, newValues: genericOp.getResult(i: 0)); |
| 1417 | return success(); |
| 1418 | } |
| 1419 | }; |
| 1420 | |
| 1421 | class TransposeConverter : public OpRewritePattern<tosa::TransposeOp> { |
| 1422 | public: |
| 1423 | using OpRewritePattern<tosa::TransposeOp>::OpRewritePattern; |
| 1424 | |
| 1425 | LogicalResult matchAndRewrite(tosa::TransposeOp op, |
| 1426 | PatternRewriter &rewriter) const final { |
| 1427 | const llvm::ArrayRef<int32_t> constantPerms = op.getPerms(); |
| 1428 | |
| 1429 | Location loc = op.getLoc(); |
| 1430 | // The verifier should have made sure we have a valid TOSA permutation |
| 1431 | // tensor. isPermutationVector doesn't actually check the TOSA perms we |
| 1432 | // expect. |
| 1433 | SmallVector<OpFoldResult> inputSizes = |
| 1434 | tensor::getMixedSizes(builder&: rewriter, loc, value: op.getInput1()); |
| 1435 | auto permutedSizes = |
| 1436 | applyTOSAPermutation<OpFoldResult>(input: inputSizes, perms: constantPerms); |
| 1437 | |
| 1438 | auto permutedInit = |
| 1439 | tensor::EmptyOp::create(builder&: rewriter, location: loc, sizes: permutedSizes, |
| 1440 | elementType: op.getInput1().getType().getElementType()); |
| 1441 | rewriter.replaceOpWithNewOp<linalg::TransposeOp>( |
| 1442 | op, args: op.getInput1(), args&: permutedInit, |
| 1443 | args: llvm::map_to_vector(C: constantPerms, |
| 1444 | F: [](int32_t v) -> int64_t { return v; })); |
| 1445 | return success(); |
| 1446 | } |
| 1447 | }; |
| 1448 | } // namespace |
| 1449 | |
| 1450 | void mlir::tosa::populateTosaToLinalgNamedConversionPatterns( |
| 1451 | const TypeConverter &converter, RewritePatternSet *patterns, |
| 1452 | const TosaToLinalgNamedOptions &options) { |
| 1453 | if (options.preferConv2DKernelLayoutHWCF) { |
| 1454 | patterns->add<ConvConverter<tosa::Conv2DOp, linalg::Conv2DNhwcHwcfOp, |
| 1455 | linalg::Conv2DNhwcHwcfQOp>>( |
| 1456 | arg: patterns->getContext()); |
| 1457 | } else { |
| 1458 | patterns->add<ConvConverter<tosa::Conv2DOp, linalg::Conv2DNhwcFhwcOp, |
| 1459 | linalg::Conv2DNhwcFhwcQOp>>( |
| 1460 | arg: patterns->getContext()); |
| 1461 | } |
| 1462 | patterns->add< |
| 1463 | // clang-format off |
| 1464 | ConvConverter<tosa::Conv3DOp, linalg::Conv3DNdhwcDhwcfOp, linalg::Conv3DNdhwcDhwcfQOp>, |
| 1465 | DepthwiseConvConverter, |
| 1466 | MatMulConverter, |
| 1467 | MatMulTConverter, |
| 1468 | AvgPool2dConverter, |
| 1469 | TransposeConverter |
| 1470 | >(arg: patterns->getContext()); |
| 1471 | |
| 1472 | patterns->add< |
| 1473 | ElementwiseConverter<tosa::AbsOp>, |
| 1474 | ElementwiseConverter<tosa::CeilOp>, |
| 1475 | ElementwiseConverter<tosa::FloorOp>, |
| 1476 | ElementwiseConverter<tosa::ExpOp>, |
| 1477 | ElementwiseConverter<tosa::LogOp>, |
| 1478 | ElementwiseConverter<tosa::RsqrtOp>, |
| 1479 | ElementwiseConverter<tosa::SinOp>, |
| 1480 | ElementwiseConverter<tosa::CosOp>, |
| 1481 | ElementwiseConverter<tosa::TanhOp>, |
| 1482 | ElementwiseConverter<tosa::ErfOp>, |
| 1483 | ElementwiseConverter<tosa::ReciprocalOp>, |
| 1484 | ElementwiseConverter<tosa::PowOp>, |
| 1485 | ElementwiseConverter<tosa::AddOp>, |
| 1486 | ElementwiseConverter<tosa::SubOp>, |
| 1487 | ElementwiseConverter<tosa::IntDivOp>, |
| 1488 | ElementwiseConverter<tosa::MulOp>, |
| 1489 | ElementwiseConverter<tosa::NegateOp>, |
| 1490 | ElementwiseConverter<tosa::MaximumOp>, |
| 1491 | ElementwiseConverter<tosa::MinimumOp>, |
| 1492 | ElementwiseConverter<tosa::SelectOp> |
| 1493 | >(arg: patterns->getContext()); |
| 1494 | |
| 1495 | patterns->add< |
| 1496 | MaxPool2dConverter |
| 1497 | >(arg: converter, args: patterns->getContext(), args: options.allowNonFinites); |
| 1498 | // clang-format on |
| 1499 | } |
| 1500 | |