| 1 | //===- TosaToTensor.cpp - Lowering Tosa to Tensor Dialect -------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // These rewriters lower from the Tosa to the Tensor dialect. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Conversion/TosaToTensor/TosaToTensor.h" |
| 14 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 15 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 16 | #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" |
| 17 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 18 | #include "mlir/Dialect/Tosa/IR/TosaOps.h" |
| 19 | #include "mlir/Dialect/Tosa/Utils/ConversionUtils.h" |
| 20 | #include "mlir/IR/PatternMatch.h" |
| 21 | #include "mlir/Transforms/DialectConversion.h" |
| 22 | #include "llvm/ADT/STLExtras.h" |
| 23 | |
| 24 | #include <numeric> |
| 25 | |
| 26 | using namespace mlir; |
| 27 | using namespace tosa; |
| 28 | |
| 29 | namespace { |
| 30 | |
| 31 | // Infer the type to which the input of a 'tosa.reshape' op must be cast when |
| 32 | // lowered. |
| 33 | TensorType inferReshapeInputType(TypedValue<TensorType> input, |
| 34 | ArrayRef<int64_t> newShape) { |
| 35 | // No need to cast input for non-empty target shape |
| 36 | if (!newShape.empty()) |
| 37 | return input.getType(); |
| 38 | |
| 39 | // The input type must be cast into a tensor with the same rank and all static |
| 40 | // dimensions set to 1. This prevents the generation of a |
| 41 | // tensor.collapse_shape op that converts a dynamically shaped tensor into a |
| 42 | // 0D tensor. While such construct is not incorrect on its own, bufferization |
| 43 | // cannot properly handle it at the moment, so we avoid it. |
| 44 | SmallVector<int64_t> shape(input.getType().getRank(), 1); |
| 45 | return input.getType().clone(shape); |
| 46 | } |
| 47 | |
| 48 | // Infer the result type of 'tensor.expand_shape' in the collapse-expand |
| 49 | // pair emitted for a 'tosa.reshape' op. |
| 50 | TensorType inferReshapeExpandedType(TensorType inputType, |
| 51 | ArrayRef<int64_t> newShape) { |
| 52 | // Special case for 0D output tensor. Note: Watch out when using Type::clone() |
| 53 | // with just '{}', as it will invoke the incorrect overload. |
| 54 | if (newShape.empty()) |
| 55 | return inputType.clone(shape: ArrayRef<int64_t>{}); |
| 56 | |
| 57 | // Check if the input is static, and if so, get its total size |
| 58 | bool inputIsStatic = inputType.hasStaticShape(); |
| 59 | int64_t totalSize = inputIsStatic ? inputType.getNumElements() : -1; |
| 60 | |
| 61 | // Compute result shape |
| 62 | auto resultShape = |
| 63 | llvm::map_to_vector(C&: newShape, F: [&](int64_t size) -> int64_t { |
| 64 | // If this is not a placeholder, do not change it. |
| 65 | if (size >= 0) |
| 66 | return size; |
| 67 | |
| 68 | // If we do not know the total size of the tensor, keep this dimension |
| 69 | // dynamic in the result shape. |
| 70 | if (!inputIsStatic) |
| 71 | return ShapedType::kDynamic; |
| 72 | |
| 73 | // Calculate the product of all elements in 'newShape' except for the -1 |
| 74 | // placeholder, which we discard by negating the result. |
| 75 | int64_t totalSizeNoPlaceholder = -llvm::product_of(Range&: newShape); |
| 76 | |
| 77 | // If there is a 0 component in 'newShape', resolve the placeholder as |
| 78 | // 0. |
| 79 | if (totalSizeNoPlaceholder == 0) |
| 80 | return 0; |
| 81 | |
| 82 | // Resolve the placeholder as the quotient between the total tensor size |
| 83 | // and the product of all other sizes. |
| 84 | return totalSize / totalSizeNoPlaceholder; |
| 85 | }); |
| 86 | |
| 87 | bool resultIsStatic = ShapedType::isStaticShape(dSizes: resultShape); |
| 88 | |
| 89 | // A syntactic restriction in 'tensor.expand_shape' forbids a dynamically |
| 90 | // shaped input from being reshaped into a statically shaped result. We may |
| 91 | // simply turn the first result dimension dynamic to address this. |
| 92 | if (!inputIsStatic && resultIsStatic) |
| 93 | resultShape[0] = ShapedType::kDynamic; |
| 94 | |
| 95 | // The 'tensor.expand_shape' op also forbids a statically shaped input from |
| 96 | // being reshaped into a dynamically shaped result, but the placeholder |
| 97 | // inference algorithm above guarantees that this will never be the case. |
| 98 | assert(!inputIsStatic || resultIsStatic); |
| 99 | |
| 100 | // Create result type |
| 101 | return inputType.clone(shape: resultShape); |
| 102 | } |
| 103 | |
| 104 | // Infer the result type of 'tensor.collapse_shape' in the collapse-expand |
| 105 | // pair emitted for a 'tosa.reshape' op. |
| 106 | TensorType inferReshapeCollapsedType(TensorType lhsType, TensorType rhsType) { |
| 107 | auto lhsShape = lhsType.getShape(); |
| 108 | auto rhsShape = rhsType.getShape(); |
| 109 | |
| 110 | if (lhsShape.empty() || rhsShape.empty()) |
| 111 | return lhsType.clone(shape: ArrayRef<int64_t>{}); |
| 112 | |
| 113 | if (ShapedType::isDynamicShape(dSizes: lhsShape) || |
| 114 | ShapedType::isDynamicShape(dSizes: rhsShape)) |
| 115 | return lhsType.clone(shape: {ShapedType::kDynamic}); |
| 116 | |
| 117 | SmallVector<int64_t> intermediateShape; |
| 118 | unsigned currLhsDim = 0, currRhsDim = 0; |
| 119 | while (currLhsDim < lhsShape.size() && currRhsDim < rhsShape.size()) { |
| 120 | int64_t rhsSize = rhsShape[currRhsDim]; |
| 121 | int64_t lhsSize = lhsShape[currLhsDim]; |
| 122 | while (lhsSize != rhsSize && currLhsDim < lhsShape.size() && |
| 123 | currRhsDim < rhsShape.size()) { |
| 124 | if (lhsSize < rhsSize) { |
| 125 | currLhsDim++; |
| 126 | if (currLhsDim < lhsShape.size()) { |
| 127 | lhsSize *= lhsShape[currLhsDim]; |
| 128 | } |
| 129 | } else { |
| 130 | currRhsDim++; |
| 131 | if (currRhsDim < rhsShape.size()) { |
| 132 | rhsSize *= rhsShape[currRhsDim]; |
| 133 | } |
| 134 | } |
| 135 | } |
| 136 | if (lhsSize == rhsSize) { |
| 137 | intermediateShape.push_back(Elt: lhsSize); |
| 138 | } |
| 139 | currRhsDim++; |
| 140 | currLhsDim++; |
| 141 | } |
| 142 | |
| 143 | // Static shapes are guaranteed to be compatible by the op verifier, so all |
| 144 | // leftover dimensions should be 1. |
| 145 | for (; currLhsDim < lhsShape.size(); currLhsDim++) { |
| 146 | assert(lhsShape[currLhsDim] == 1); |
| 147 | } |
| 148 | for (; currRhsDim < rhsShape.size(); currRhsDim++) { |
| 149 | assert(rhsShape[currRhsDim] == 1); |
| 150 | } |
| 151 | |
| 152 | return lhsType.clone(shape: intermediateShape); |
| 153 | } |
| 154 | |
| 155 | SmallVector<ReassociationExprs> |
| 156 | createReassociationMapForCollapse(OpBuilder &builder, Type srcType, |
| 157 | Type dstType) { |
| 158 | auto srcShape = cast<TensorType>(Val&: srcType).getShape(); |
| 159 | auto dstShape = cast<TensorType>(Val&: dstType).getShape(); |
| 160 | |
| 161 | if (srcShape.empty() || dstShape.empty()) |
| 162 | return {}; |
| 163 | |
| 164 | if (ShapedType::isDynamicShape(dSizes: srcShape) || |
| 165 | ShapedType::isDynamicShape(dSizes: dstShape)) { |
| 166 | assert(dstShape.size() == 1); |
| 167 | SmallVector<AffineExpr, 2> exprs; |
| 168 | for (auto i : llvm::seq<int64_t>(Size: srcShape.size())) |
| 169 | exprs.push_back(Elt: builder.getAffineDimExpr(position: i)); |
| 170 | return {exprs}; |
| 171 | } |
| 172 | |
| 173 | SmallVector<ReassociationExprs> reassociationMap(dstShape.size()); |
| 174 | unsigned currSrcDim = 0, currDstDim = 0; |
| 175 | while (currSrcDim < srcShape.size() && currDstDim < dstShape.size()) { |
| 176 | int64_t dstSize = dstShape[currDstDim]; |
| 177 | int64_t srcSize = srcShape[currSrcDim]; |
| 178 | while (srcSize < dstSize && currSrcDim < srcShape.size()) { |
| 179 | reassociationMap[currDstDim].push_back( |
| 180 | Elt: builder.getAffineDimExpr(position: currSrcDim++)); |
| 181 | srcSize *= srcShape[currSrcDim]; |
| 182 | } |
| 183 | if (srcSize == dstSize) { |
| 184 | reassociationMap[currDstDim].push_back( |
| 185 | Elt: builder.getAffineDimExpr(position: currSrcDim++)); |
| 186 | // If the next dim in collapsedShape is not 1, treat subsequent dims in |
| 187 | // expandedShape which are 1 to be collapsed. |
| 188 | if (currDstDim == dstShape.size() - 1 || dstShape[currDstDim + 1] != 1) { |
| 189 | while (currSrcDim < srcShape.size() && srcShape[currSrcDim] == 1) { |
| 190 | reassociationMap[currDstDim].push_back( |
| 191 | Elt: builder.getAffineDimExpr(position: currSrcDim++)); |
| 192 | } |
| 193 | } |
| 194 | } |
| 195 | currDstDim++; |
| 196 | } |
| 197 | |
| 198 | // If the source and target shapes are compatible, both iterators must have |
| 199 | // reached the end. This condition is guaranteed by the op verifier for |
| 200 | // static shapes. |
| 201 | assert(currSrcDim == srcShape.size() && currDstDim == dstShape.size()); |
| 202 | return reassociationMap; |
| 203 | } |
| 204 | |
| 205 | // Create a tensor.collapse_shape op that reshapes the input into the given |
| 206 | // result type. |
| 207 | Value createCollapse(OpBuilder &builder, Location loc, TensorType resultType, |
| 208 | Value input) { |
| 209 | auto reassociationMap = |
| 210 | createReassociationMapForCollapse(builder, srcType: input.getType(), dstType: resultType); |
| 211 | return builder.createOrFold<tensor::CollapseShapeOp>(location: loc, args&: resultType, args&: input, |
| 212 | args&: reassociationMap); |
| 213 | } |
| 214 | |
| 215 | // Create a tensor.expand_shape op that reshapes the input into the given result |
| 216 | // type. |
| 217 | Value createExpand(OpBuilder &builder, Location loc, TensorType resultType, |
| 218 | Value input) { |
| 219 | auto reassociationMap = |
| 220 | createReassociationMapForCollapse(builder, srcType: resultType, dstType: input.getType()); |
| 221 | return builder.createOrFold<tensor::ExpandShapeOp>(location: loc, args&: resultType, args&: input, |
| 222 | args&: reassociationMap); |
| 223 | } |
| 224 | |
| 225 | class ReshapeConverter : public OpConversionPattern<tosa::ReshapeOp> { |
| 226 | public: |
| 227 | using OpConversionPattern<tosa::ReshapeOp>::OpConversionPattern; |
| 228 | |
| 229 | LogicalResult |
| 230 | matchAndRewrite(tosa::ReshapeOp reshape, OpAdaptor adaptor, |
| 231 | ConversionPatternRewriter &rewriter) const final { |
| 232 | auto loc = reshape.getLoc(); |
| 233 | auto resultType = |
| 234 | getTypeConverter()->convertType<ShapedType>(t: reshape.getType()); |
| 235 | if (!resultType) { |
| 236 | return rewriter.notifyMatchFailure(arg: reshape.getLoc(), |
| 237 | msg: "could not convert result type" ); |
| 238 | } |
| 239 | auto input = dyn_cast<TypedValue<TensorType>>(Val: adaptor.getInput1()); |
| 240 | if (!input) { |
| 241 | return rewriter.notifyMatchFailure(arg: reshape.getLoc(), |
| 242 | msg: "expected input type to be tensor" ); |
| 243 | } |
| 244 | |
| 245 | llvm::SmallVector<int64_t> newShape; |
| 246 | if (!tosa::getConstShapeValues(op: reshape.getShape().getDefiningOp(), |
| 247 | result_shape&: newShape)) { |
| 248 | return failure(); |
| 249 | } |
| 250 | |
| 251 | // Infer all intermediate types |
| 252 | auto inputType = inferReshapeInputType(input, newShape); |
| 253 | auto expandedType = inferReshapeExpandedType(inputType, newShape); |
| 254 | auto collapsedType = inferReshapeCollapsedType(lhsType: inputType, rhsType: expandedType); |
| 255 | |
| 256 | // Cast input if needed |
| 257 | auto castInput = |
| 258 | rewriter.createOrFold<tensor::CastOp>(location: loc, args&: inputType, args&: input); |
| 259 | |
| 260 | // Emit collaspe-expand pair |
| 261 | auto collapsed = createCollapse(builder&: rewriter, loc, resultType: collapsedType, input: castInput); |
| 262 | auto expanded = createExpand(builder&: rewriter, loc, resultType: expandedType, input: collapsed); |
| 263 | |
| 264 | // Cast to final result type if needed |
| 265 | auto result = |
| 266 | rewriter.createOrFold<tensor::CastOp>(location: loc, args&: resultType, args&: expanded); |
| 267 | rewriter.replaceOp(op: reshape, newValues: result); |
| 268 | return success(); |
| 269 | } |
| 270 | }; |
| 271 | |
| 272 | class SliceConverter : public OpConversionPattern<tosa::SliceOp> { |
| 273 | public: |
| 274 | using OpConversionPattern<tosa::SliceOp>::OpConversionPattern; |
| 275 | |
| 276 | LogicalResult |
| 277 | matchAndRewrite(tosa::SliceOp sliceOp, OpAdaptor adaptor, |
| 278 | ConversionPatternRewriter &rewriter) const final { |
| 279 | Location loc = sliceOp.getLoc(); |
| 280 | Value input = adaptor.getInput1(); |
| 281 | ShapedType resultType = cast<ShapedType>(Val: sliceOp.getType()); |
| 282 | if (llvm::isa<UnrankedTensorType>(Val: resultType)) |
| 283 | return failure(); |
| 284 | |
| 285 | ElementsAttr startElems; |
| 286 | ElementsAttr sizeElems; |
| 287 | |
| 288 | if (!matchPattern(value: sliceOp.getStart(), pattern: m_Constant(bind_value: &startElems))) |
| 289 | return rewriter.notifyMatchFailure( |
| 290 | arg&: sliceOp, msg: "start of slice must be a static ranked shape" ); |
| 291 | |
| 292 | if (!matchPattern(value: sliceOp.getSize(), pattern: m_Constant(bind_value: &sizeElems))) |
| 293 | return rewriter.notifyMatchFailure( |
| 294 | arg&: sliceOp, msg: "size of slice must be a static ranked shape" ); |
| 295 | |
| 296 | llvm::SmallVector<int64_t> sliceStarts = |
| 297 | llvm::to_vector(Range: startElems.getValues<int64_t>()); |
| 298 | llvm::SmallVector<int64_t> sliceSizes = |
| 299 | llvm::to_vector(Range: sizeElems.getValues<int64_t>()); |
| 300 | |
| 301 | SmallVector<int64_t> strides, sizes; |
| 302 | strides.resize(N: cast<ShapedType>(Val: sliceOp.getType()).getRank(), NV: 1); |
| 303 | |
| 304 | SmallVector<Value> dynSizes; |
| 305 | for (const auto &i : llvm::enumerate(First&: sliceSizes)) { |
| 306 | int64_t size = i.value(); |
| 307 | size_t index = i.index(); |
| 308 | sizes.push_back(Elt: size == -1 ? ShapedType::kDynamic : size); |
| 309 | if (ShapedType::isStatic(dValue: sizes.back())) |
| 310 | continue; |
| 311 | |
| 312 | auto dim = tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index); |
| 313 | auto offset = arith::ConstantOp::create( |
| 314 | builder&: rewriter, location: loc, value: rewriter.getIndexAttr(value: sliceStarts[index])); |
| 315 | dynSizes.push_back(Elt: arith::SubIOp::create(builder&: rewriter, location: loc, lhs: dim, rhs: offset)); |
| 316 | } |
| 317 | |
| 318 | auto newSliceOp = tensor::ExtractSliceOp::create( |
| 319 | builder&: rewriter, location: sliceOp.getLoc(), result: sliceOp.getType(), source: input, offsets: ValueRange({}), |
| 320 | sizes: dynSizes, strides: ValueRange({}), static_offsets: rewriter.getDenseI64ArrayAttr(values: sliceStarts), |
| 321 | static_sizes: rewriter.getDenseI64ArrayAttr(values: sizes), |
| 322 | static_strides: rewriter.getDenseI64ArrayAttr(values: strides)); |
| 323 | |
| 324 | // Remove const_shape ops when it no longer has use point. |
| 325 | Operation *startConstShape = sliceOp.getStart().getDefiningOp(); |
| 326 | if (startConstShape->getResult(idx: 0).hasOneUse()) |
| 327 | rewriter.eraseOp(op: startConstShape); |
| 328 | |
| 329 | Operation *sizeConstShape = sliceOp.getSize().getDefiningOp(); |
| 330 | if (sizeConstShape->getResult(idx: 0).hasOneUse()) |
| 331 | rewriter.eraseOp(op: sizeConstShape); |
| 332 | |
| 333 | rewriter.replaceOp(op: sliceOp, newValues: newSliceOp.getResult()); |
| 334 | return success(); |
| 335 | } |
| 336 | }; |
| 337 | |
| 338 | class PadConverter : public OpConversionPattern<tosa::PadOp> { |
| 339 | public: |
| 340 | using Base::Base; |
| 341 | |
| 342 | LogicalResult |
| 343 | matchAndRewrite(tosa::PadOp padOp, OpAdaptor adaptor, |
| 344 | ConversionPatternRewriter &rewriter) const final { |
| 345 | auto loc = padOp.getLoc(); |
| 346 | auto input = padOp.getInput1(); |
| 347 | |
| 348 | ElementsAttr paddingElems; |
| 349 | if (!matchPattern(value: padOp.getPadding(), pattern: m_Constant(bind_value: &paddingElems))) { |
| 350 | return rewriter.notifyMatchFailure( |
| 351 | arg&: padOp, msg: "padding must be a static shape value" ); |
| 352 | } |
| 353 | llvm::SmallVector<int64_t> paddingVals; |
| 354 | for (auto idx : paddingElems.getValues<IntegerAttr>()) { |
| 355 | paddingVals.push_back(Elt: static_cast<int64_t>(idx.getInt())); |
| 356 | } |
| 357 | |
| 358 | // The TOSA specification requires a negative padding to be diagnosed. |
| 359 | // We emit a runtime assertion, as this could be dead or unreachable code. |
| 360 | if (llvm::any_of(Range&: paddingVals, P: [](int64_t val) { return val < 0; })) { |
| 361 | Value fail = |
| 362 | arith::ConstantOp::create(builder&: rewriter, location: loc, value: rewriter.getBoolAttr(value: false)); |
| 363 | cf::AssertOp::create( |
| 364 | builder&: rewriter, location: loc, arg: fail, |
| 365 | msg: rewriter.getStringAttr(bytes: "tosa.pad: padding must be non-negative" )); |
| 366 | } |
| 367 | |
| 368 | ShapedType inputTy = cast<ShapedType>(Val: input.getType()); |
| 369 | int64_t rank = inputTy.getRank(); |
| 370 | |
| 371 | // Setup the default constantAttr. |
| 372 | |
| 373 | Value padConstant = rewriter.createOrFold<tensor::ExtractOp>( |
| 374 | location: loc, args: padOp.getPadConst(), |
| 375 | args: ValueRange({arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0)})); |
| 376 | |
| 377 | if (!padConstant) { |
| 378 | return rewriter.notifyMatchFailure( |
| 379 | arg&: padOp, msg: "tosa.pad was unable to determine the pad constant value." ); |
| 380 | } |
| 381 | |
| 382 | SmallVector<OpFoldResult, 3> lowValues; |
| 383 | SmallVector<OpFoldResult, 3> highValues; |
| 384 | |
| 385 | lowValues.reserve(N: rank); |
| 386 | highValues.reserve(N: rank); |
| 387 | |
| 388 | for (int i = 0; i < rank; i++) { |
| 389 | Value lowVal = arith::ConstantOp::create( |
| 390 | builder&: rewriter, location: loc, value: rewriter.getIndexAttr(value: paddingVals[2 * i])); |
| 391 | Value highVal = arith::ConstantOp::create( |
| 392 | builder&: rewriter, location: loc, value: rewriter.getIndexAttr(value: paddingVals[2 * i + 1])); |
| 393 | lowValues.push_back(Elt: lowVal); |
| 394 | highValues.push_back(Elt: highVal); |
| 395 | } |
| 396 | |
| 397 | auto newPadOp = tensor::PadOp::create(builder&: rewriter, location: loc, resultType: padOp.getType(), source: input, |
| 398 | low: lowValues, high: highValues, constantPadValue: padConstant); |
| 399 | |
| 400 | rewriter.replaceOp(op: padOp, newValues: newPadOp.getResult()); |
| 401 | return success(); |
| 402 | } |
| 403 | }; |
| 404 | |
| 405 | struct ConcatConverter : public OpConversionPattern<tosa::ConcatOp> { |
| 406 | using OpConversionPattern<tosa::ConcatOp>::OpConversionPattern; |
| 407 | |
| 408 | LogicalResult |
| 409 | matchAndRewrite(tosa::ConcatOp op, OpAdaptor adaptor, |
| 410 | ConversionPatternRewriter &rewriter) const override { |
| 411 | auto resultType = |
| 412 | getTypeConverter()->convertType<RankedTensorType>(t: op.getType()); |
| 413 | if (!resultType) |
| 414 | return failure(); |
| 415 | |
| 416 | rewriter.replaceOpWithNewOp<tensor::ConcatOp>( |
| 417 | op, args&: resultType, args: rewriter.getI64IntegerAttr(value: op.getAxis()), |
| 418 | args: adaptor.getInput1()); |
| 419 | return success(); |
| 420 | } |
| 421 | }; |
| 422 | |
| 423 | } // namespace |
| 424 | |
| 425 | void mlir::tosa::populateTosaToTensorConversionPatterns( |
| 426 | const TypeConverter &converter, RewritePatternSet *patterns) { |
| 427 | patterns |
| 428 | ->add<ConcatConverter, PadConverter, ReshapeConverter, SliceConverter>( |
| 429 | arg: converter, args: patterns->getContext()); |
| 430 | } |
| 431 | |