| 1 | //===- TosaToSPIRVTosaOps.cpp - TOSA to SPIR-V Graph/TOSA patterns --------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements patterns to convert TOSA IR to SPIR-V Graph/TOSA. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Conversion/TosaToSPIRVTosa/TosaToSPIRVTosa.h" |
| 14 | #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h" |
| 15 | #include "mlir/Dialect/Tosa/IR/TosaOps.h" |
| 16 | #include "mlir/Transforms/DialectConversion.h" |
| 17 | |
| 18 | #define DEBUG_TYPE "tosa-to-spirv-tosa-ops-pattern" |
| 19 | |
| 20 | namespace mlir::tosa { |
| 21 | namespace { |
| 22 | |
| 23 | constexpr unsigned maxConcatOpInputs = 64; |
| 24 | |
| 25 | template <typename OpAdaptor> |
| 26 | spirv::TosaExtNaNPropagationModeType getNanMode(OpAdaptor adaptor) { |
| 27 | return static_cast<spirv::TosaExtNaNPropagationModeType>( |
| 28 | adaptor.getNanMode()); |
| 29 | } |
| 30 | |
| 31 | template <typename OpAdaptor> |
| 32 | spirv::TosaExtResizeModeType getResizeMode(OpAdaptor adaptor) { |
| 33 | return static_cast<spirv::TosaExtResizeModeType>(adaptor.getMode()); |
| 34 | } |
| 35 | |
| 36 | template <typename OpAdaptor> |
| 37 | spirv::TosaExtRoundingModeType getRoundingMode(OpAdaptor adaptor) { |
| 38 | return static_cast<spirv::TosaExtRoundingModeType>(adaptor.getRoundingMode()); |
| 39 | } |
| 40 | |
| 41 | spirv::TosaExtAccType getAccType(Type accType) { |
| 42 | if (accType.isInteger(width: 32)) |
| 43 | return spirv::TosaExtAccType::INT32; |
| 44 | else if (accType.isF16()) |
| 45 | return spirv::TosaExtAccType::FP16; |
| 46 | else if (accType.isF32()) |
| 47 | return spirv::TosaExtAccType::FP32; |
| 48 | else if (accType.isInteger(width: 48)) |
| 49 | return spirv::TosaExtAccType::INT48; |
| 50 | llvm_unreachable("unknown accumulator type" ); |
| 51 | } |
| 52 | |
| 53 | DenseIntElementsAttr getI32TensorArmAttr(ArrayRef<int32_t> values, |
| 54 | ConversionPatternRewriter &rewriter) { |
| 55 | return DenseIntElementsAttr::get( |
| 56 | type: spirv::TensorArmType::get(shape: static_cast<int64_t>(values.size()), |
| 57 | elementType: IntegerType::get(context: rewriter.getContext(), width: 32)), |
| 58 | arg&: values); |
| 59 | } |
| 60 | |
| 61 | // TOSA stores many integer array attributes as i64 in MLIR, while the |
| 62 | // SPIR-V TOSA extended instruction set models the same attributes as i32. |
| 63 | DenseIntElementsAttr getI32TensorArmAttr(ArrayRef<int64_t> values, |
| 64 | ConversionPatternRewriter &rewriter) { |
| 65 | SmallVector<int32_t> i32Values(values.begin(), values.end()); |
| 66 | return getI32TensorArmAttr(values: i32Values, rewriter); |
| 67 | } |
| 68 | |
| 69 | FailureOr<DenseElementsAttr> |
| 70 | convertDenseElementsAttr(DenseElementsAttr values, ShapedType convertedType) { |
| 71 | Type convertedElementType = convertedType.getElementType(); |
| 72 | if (values.getElementType() == convertedElementType) |
| 73 | return values.reshape(newType: convertedType); |
| 74 | |
| 75 | // Constant attributes still have the source TOSA element type. Rebuild them |
| 76 | // for the converted SPIR-V tensor type, including integer type-converter |
| 77 | // changes such as index to i32, i4 to i8, and i48 to i64. |
| 78 | auto integerType = dyn_cast<IntegerType>(Val&: convertedElementType); |
| 79 | if (!integerType) |
| 80 | return failure(); |
| 81 | |
| 82 | // The SPIR-V ARM tensor type does not represent scalar shape constants |
| 83 | // directly, so model tensor<0xindex> as a one-element i32 tensor. |
| 84 | if (values.empty() && values.getElementType().isIndex()) |
| 85 | return DenseIntElementsAttr::get(type: convertedType, list: {1}); |
| 86 | |
| 87 | DenseElementsAttr convertedValues = |
| 88 | values.mapValues(newElementType: integerType, mapping: [&](const APInt &value) { |
| 89 | return value.sextOrTrunc(width: integerType.getWidth()); |
| 90 | }); |
| 91 | return convertedValues.reshape(newType: convertedType); |
| 92 | } |
| 93 | |
| 94 | // Split a large concat into smaller concat operations so the generated SPIR-V |
| 95 | // instructions stay below the binary operand count limit. |
| 96 | LogicalResult splitConcat(tosa::ConcatOp op, Type resultType, int32_t axis, |
| 97 | ValueRange inputs, |
| 98 | ConversionPatternRewriter &rewriter) { |
| 99 | auto resultTensorType = dyn_cast<spirv::TensorArmType>(Val&: resultType); |
| 100 | if (!resultTensorType) |
| 101 | return rewriter.notifyMatchFailure(arg&: op, msg: "expected tensor result type" ); |
| 102 | if (!resultTensorType.hasRank()) |
| 103 | return rewriter.notifyMatchFailure(arg&: op, |
| 104 | msg: "expected ranked tensor result type" ); |
| 105 | |
| 106 | SmallVector<Value> concatInputs; |
| 107 | SmallVector<int64_t> concatShape(resultTensorType.getShape()); |
| 108 | concatShape[axis] = 0; |
| 109 | |
| 110 | for (auto [index, input] : llvm::enumerate(First&: inputs)) { |
| 111 | auto inputType = dyn_cast<spirv::TensorArmType>(Val: input.getType()); |
| 112 | if (!inputType) |
| 113 | return rewriter.notifyMatchFailure(arg&: op, msg: "expected tensor input type" ); |
| 114 | if (!inputType.hasRank()) |
| 115 | return rewriter.notifyMatchFailure(arg&: op, |
| 116 | msg: "expected ranked tensor input type" ); |
| 117 | |
| 118 | int64_t inputAxisDim = inputType.getShape()[axis]; |
| 119 | if (ShapedType::isDynamic(dValue: inputAxisDim) || |
| 120 | ShapedType::isDynamic(dValue: concatShape[axis])) |
| 121 | concatShape[axis] = ShapedType::kDynamic; |
| 122 | else |
| 123 | concatShape[axis] += inputAxisDim; |
| 124 | |
| 125 | concatInputs.push_back(Elt: input); |
| 126 | if (concatInputs.size() != maxConcatOpInputs || index == inputs.size() - 1) |
| 127 | continue; |
| 128 | |
| 129 | Type concatType = spirv::TensorArmType::get( |
| 130 | shape: concatShape, elementType: resultTensorType.getElementType()); |
| 131 | auto concat = spirv::TosaConcatOp::create(builder&: rewriter, location: op.getLoc(), output: concatType, |
| 132 | axis, input1: concatInputs); |
| 133 | concatInputs.clear(); |
| 134 | concatInputs.push_back(Elt: concat.getOutput()); |
| 135 | } |
| 136 | |
| 137 | rewriter.replaceOpWithNewOp<spirv::TosaConcatOp>(op, args&: resultType, args&: axis, |
| 138 | args&: concatInputs); |
| 139 | return success(); |
| 140 | } |
| 141 | |
| 142 | template <typename SourceOp, auto Replace> |
| 143 | struct TosaOpConvert final : public OpConversionPattern<SourceOp> { |
| 144 | using OpConversionPattern<SourceOp>::OpConversionPattern; |
| 145 | |
| 146 | LogicalResult |
| 147 | matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor, |
| 148 | ConversionPatternRewriter &rewriter) const override { |
| 149 | Type type = this->getTypeConverter()->convertType(op.getType()); |
| 150 | if (!type) |
| 151 | return rewriter.notifyMatchFailure(op, "type conversion failed" ); |
| 152 | return Replace(op, adaptor, type, rewriter); |
| 153 | } |
| 154 | }; |
| 155 | |
| 156 | template <typename SourceOp, auto Replace> |
| 157 | struct TosaMultiResultOpConvert final : public OpConversionPattern<SourceOp> { |
| 158 | using OpConversionPattern<SourceOp>::OpConversionPattern; |
| 159 | |
| 160 | LogicalResult |
| 161 | matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor, |
| 162 | ConversionPatternRewriter &rewriter) const override { |
| 163 | SmallVector<Type> types; |
| 164 | if (failed(this->getTypeConverter()->convertTypes(op->getResultTypes(), |
| 165 | types))) |
| 166 | return rewriter.notifyMatchFailure(op, "type conversion failed" ); |
| 167 | return Replace(op, adaptor, types, rewriter); |
| 168 | } |
| 169 | }; |
| 170 | |
| 171 | template <typename SourceOp, typename TargetOp> |
| 172 | LogicalResult replaceUnaryInput1(SourceOp op, |
| 173 | typename SourceOp::Adaptor adaptor, Type type, |
| 174 | ConversionPatternRewriter &rewriter) { |
| 175 | rewriter.replaceOpWithNewOp<TargetOp>(op, type, adaptor.getInput1()); |
| 176 | return success(); |
| 177 | } |
| 178 | |
| 179 | template <typename SourceOp, typename TargetOp> |
| 180 | LogicalResult replaceUnaryInput(SourceOp op, typename SourceOp::Adaptor adaptor, |
| 181 | Type type, |
| 182 | ConversionPatternRewriter &rewriter) { |
| 183 | rewriter.replaceOpWithNewOp<TargetOp>(op, type, adaptor.getInput()); |
| 184 | return success(); |
| 185 | } |
| 186 | |
| 187 | template <typename SourceOp, typename TargetOp> |
| 188 | LogicalResult |
| 189 | replaceBinaryElementwise(SourceOp op, typename SourceOp::Adaptor adaptor, |
| 190 | Type type, ConversionPatternRewriter &rewriter) { |
| 191 | rewriter.replaceOpWithNewOp<TargetOp>(op, type, adaptor.getInput1(), |
| 192 | adaptor.getInput2()); |
| 193 | return success(); |
| 194 | } |
| 195 | |
| 196 | template <typename SourceOp, typename TargetOp> |
| 197 | LogicalResult |
| 198 | replaceBinaryNanModeElementwise(SourceOp op, typename SourceOp::Adaptor adaptor, |
| 199 | Type type, |
| 200 | ConversionPatternRewriter &rewriter) { |
| 201 | rewriter.replaceOpWithNewOp<TargetOp>( |
| 202 | op, type, getNanMode(adaptor), adaptor.getInput1(), adaptor.getInput2()); |
| 203 | return success(); |
| 204 | } |
| 205 | |
| 206 | template <typename SourceOp, typename TargetOp> |
| 207 | LogicalResult replaceReduction(SourceOp op, typename SourceOp::Adaptor adaptor, |
| 208 | Type type, ConversionPatternRewriter &rewriter) { |
| 209 | rewriter.replaceOpWithNewOp<TargetOp>(op, type, adaptor.getAxis(), |
| 210 | adaptor.getInput()); |
| 211 | return success(); |
| 212 | } |
| 213 | |
| 214 | template <typename SourceOp, typename TargetOp> |
| 215 | LogicalResult |
| 216 | replaceNanModeReduction(SourceOp op, typename SourceOp::Adaptor adaptor, |
| 217 | Type type, ConversionPatternRewriter &rewriter) { |
| 218 | rewriter.replaceOpWithNewOp<TargetOp>( |
| 219 | op, type, adaptor.getAxis(), getNanMode(adaptor), adaptor.getInput()); |
| 220 | return success(); |
| 221 | } |
| 222 | |
| 223 | LogicalResult replaceAvgPool2d(tosa::AvgPool2dOp op, |
| 224 | tosa::AvgPool2dOpAdaptor adaptor, Type type, |
| 225 | ConversionPatternRewriter &rewriter) { |
| 226 | rewriter.replaceOpWithNewOp<spirv::TosaAvgPool2DOp>( |
| 227 | op, args&: type, args: getI32TensorArmAttr(values: adaptor.getKernel(), rewriter), |
| 228 | args: getI32TensorArmAttr(values: adaptor.getStride(), rewriter), |
| 229 | args: getI32TensorArmAttr(values: adaptor.getPad(), rewriter), |
| 230 | args: getAccType(accType: adaptor.getAccType()), args: adaptor.getInput(), |
| 231 | args: adaptor.getInputZp(), args: adaptor.getOutputZp()); |
| 232 | return success(); |
| 233 | } |
| 234 | |
| 235 | template <typename SourceOp, typename TargetOp> |
| 236 | LogicalResult replaceConvolution(SourceOp op, |
| 237 | typename SourceOp::Adaptor adaptor, Type type, |
| 238 | ConversionPatternRewriter &rewriter) { |
| 239 | rewriter.replaceOpWithNewOp<TargetOp>( |
| 240 | op, type, getI32TensorArmAttr(adaptor.getPad(), rewriter), |
| 241 | getI32TensorArmAttr(adaptor.getStride(), rewriter), |
| 242 | getI32TensorArmAttr(adaptor.getDilation(), rewriter), |
| 243 | getAccType(adaptor.getAccType()), adaptor.getLocalBound(), |
| 244 | adaptor.getInput(), adaptor.getWeight(), adaptor.getBias(), |
| 245 | adaptor.getInputZp(), adaptor.getWeightZp()); |
| 246 | return success(); |
| 247 | } |
| 248 | |
| 249 | LogicalResult replaceFFT2d(tosa::FFT2dOp op, tosa::FFT2dOpAdaptor adaptor, |
| 250 | ArrayRef<Type> types, |
| 251 | ConversionPatternRewriter &rewriter) { |
| 252 | auto structType = spirv::StructType::get(memberTypes: types); |
| 253 | auto result = spirv::TosaFFT2DOp::create( |
| 254 | builder&: rewriter, location: op.getLoc(), output: structType, inverse: adaptor.getInverse(), |
| 255 | local_bound: adaptor.getLocalBound(), input_real: adaptor.getInputReal(), input_imag: adaptor.getInputImag()); |
| 256 | auto outputReal = |
| 257 | spirv::CompositeExtractOp::create(builder&: rewriter, location: op.getLoc(), composite: result, indices: {0}); |
| 258 | auto outputImag = |
| 259 | spirv::CompositeExtractOp::create(builder&: rewriter, location: op.getLoc(), composite: result, indices: {1}); |
| 260 | rewriter.replaceOp(op, newValues: {outputReal, outputImag}); |
| 261 | return success(); |
| 262 | } |
| 263 | |
| 264 | LogicalResult replaceMatMul(tosa::MatMulOp op, tosa::MatMulOpAdaptor adaptor, |
| 265 | Type type, ConversionPatternRewriter &rewriter) { |
| 266 | rewriter.replaceOpWithNewOp<spirv::TosaMatMulOp>( |
| 267 | op, args&: type, args: adaptor.getA(), args: adaptor.getB(), args: adaptor.getAZp(), |
| 268 | args: adaptor.getBZp()); |
| 269 | return success(); |
| 270 | } |
| 271 | |
| 272 | LogicalResult replaceMaxPool2d(tosa::MaxPool2dOp op, |
| 273 | tosa::MaxPool2dOpAdaptor adaptor, Type type, |
| 274 | ConversionPatternRewriter &rewriter) { |
| 275 | rewriter.replaceOpWithNewOp<spirv::TosaMaxPool2DOp>( |
| 276 | op, args&: type, args: getI32TensorArmAttr(values: adaptor.getKernel(), rewriter), |
| 277 | args: getI32TensorArmAttr(values: adaptor.getStride(), rewriter), |
| 278 | args: getI32TensorArmAttr(values: adaptor.getPad(), rewriter), args: getNanMode(adaptor), |
| 279 | args: adaptor.getInput()); |
| 280 | return success(); |
| 281 | } |
| 282 | |
| 283 | LogicalResult replaceRFFT2d(tosa::RFFT2dOp op, tosa::RFFT2dOpAdaptor adaptor, |
| 284 | ArrayRef<Type> types, |
| 285 | ConversionPatternRewriter &rewriter) { |
| 286 | auto structType = spirv::StructType::get(memberTypes: types); |
| 287 | auto result = spirv::TosaRFFT2DOp::create(builder&: rewriter, location: op.getLoc(), output: structType, |
| 288 | local_bound: adaptor.getLocalBound(), |
| 289 | input_real: adaptor.getInputReal()); |
| 290 | auto outputReal = |
| 291 | spirv::CompositeExtractOp::create(builder&: rewriter, location: op.getLoc(), composite: result, indices: {0}); |
| 292 | auto outputImag = |
| 293 | spirv::CompositeExtractOp::create(builder&: rewriter, location: op.getLoc(), composite: result, indices: {1}); |
| 294 | rewriter.replaceOp(op, newValues: {outputReal, outputImag}); |
| 295 | return success(); |
| 296 | } |
| 297 | |
| 298 | LogicalResult replaceTransposeConv2d(tosa::TransposeConv2DOp op, |
| 299 | tosa::TransposeConv2DOpAdaptor adaptor, |
| 300 | Type type, |
| 301 | ConversionPatternRewriter &rewriter) { |
| 302 | rewriter.replaceOpWithNewOp<spirv::TosaTransposeConv2DOp>( |
| 303 | op, args&: type, args: getI32TensorArmAttr(values: adaptor.getOutPad(), rewriter), |
| 304 | args: getI32TensorArmAttr(values: adaptor.getStride(), rewriter), |
| 305 | args: getAccType(accType: adaptor.getAccType()), args: adaptor.getLocalBound(), |
| 306 | args: adaptor.getInput(), args: adaptor.getWeight(), args: adaptor.getBias(), |
| 307 | args: adaptor.getInputZp(), args: adaptor.getWeightZp()); |
| 308 | return success(); |
| 309 | } |
| 310 | |
| 311 | LogicalResult replaceClamp(tosa::ClampOp op, tosa::ClampOpAdaptor adaptor, |
| 312 | Type type, ConversionPatternRewriter &rewriter) { |
| 313 | rewriter.replaceOpWithNewOp<spirv::TosaClampOp>( |
| 314 | op, args&: type, args: adaptor.getMinVal(), args: adaptor.getMaxVal(), args: getNanMode(adaptor), |
| 315 | args: adaptor.getInput()); |
| 316 | return success(); |
| 317 | } |
| 318 | |
| 319 | LogicalResult |
| 320 | replaceArithmeticRightShift(tosa::ArithmeticRightShiftOp op, |
| 321 | tosa::ArithmeticRightShiftOpAdaptor adaptor, |
| 322 | Type type, ConversionPatternRewriter &rewriter) { |
| 323 | rewriter.replaceOpWithNewOp<spirv::TosaArithmeticRightShiftOp>( |
| 324 | op, args&: type, args: adaptor.getRound(), args: adaptor.getInput1(), args: adaptor.getInput2()); |
| 325 | return success(); |
| 326 | } |
| 327 | |
| 328 | LogicalResult replaceMul(tosa::MulOp op, tosa::MulOpAdaptor adaptor, Type type, |
| 329 | ConversionPatternRewriter &rewriter) { |
| 330 | rewriter.replaceOpWithNewOp<spirv::TosaMulOp>( |
| 331 | op, args&: type, args: adaptor.getInput1(), args: adaptor.getInput2(), args: adaptor.getShift()); |
| 332 | return success(); |
| 333 | } |
| 334 | |
| 335 | LogicalResult replaceTable(tosa::TableOp op, tosa::TableOpAdaptor adaptor, |
| 336 | Type type, ConversionPatternRewriter &rewriter) { |
| 337 | rewriter.replaceOpWithNewOp<spirv::TosaTableOp>(op, args&: type, args: adaptor.getInput1(), |
| 338 | args: adaptor.getTable()); |
| 339 | return success(); |
| 340 | } |
| 341 | |
| 342 | LogicalResult replaceNegate(tosa::NegateOp op, tosa::NegateOpAdaptor adaptor, |
| 343 | Type type, ConversionPatternRewriter &rewriter) { |
| 344 | rewriter.replaceOpWithNewOp<spirv::TosaNegateOp>( |
| 345 | op, args&: type, args: adaptor.getInput1(), args: adaptor.getInput1Zp(), |
| 346 | args: adaptor.getOutputZp()); |
| 347 | return success(); |
| 348 | } |
| 349 | |
| 350 | LogicalResult replaceSelect(tosa::SelectOp op, tosa::SelectOpAdaptor adaptor, |
| 351 | Type type, ConversionPatternRewriter &rewriter) { |
| 352 | rewriter.replaceOpWithNewOp<spirv::TosaSelectOp>( |
| 353 | op, args&: type, args: adaptor.getInput1(), args: adaptor.getInput2(), args: adaptor.getInput3()); |
| 354 | return success(); |
| 355 | } |
| 356 | |
| 357 | LogicalResult replaceConcat(tosa::ConcatOp op, tosa::ConcatOpAdaptor adaptor, |
| 358 | Type type, ConversionPatternRewriter &rewriter) { |
| 359 | // Large TOSA concats can produce SPIR-V instructions with too many |
| 360 | // operands and fail validation. Split them into conservative 64-input |
| 361 | // chunks to keep the generated SPIR-V valid. |
| 362 | if (adaptor.getInput1().size() > maxConcatOpInputs) |
| 363 | return splitConcat(op, resultType: type, axis: adaptor.getAxis(), inputs: adaptor.getInput1(), |
| 364 | rewriter); |
| 365 | |
| 366 | rewriter.replaceOpWithNewOp<spirv::TosaConcatOp>(op, args&: type, args: adaptor.getAxis(), |
| 367 | args: adaptor.getInput1()); |
| 368 | return success(); |
| 369 | } |
| 370 | |
| 371 | LogicalResult replacePad(tosa::PadOp op, tosa::PadOpAdaptor adaptor, Type type, |
| 372 | ConversionPatternRewriter &rewriter) { |
| 373 | rewriter.replaceOpWithNewOp<spirv::TosaPadOp>(op, args&: type, args: adaptor.getInput1(), |
| 374 | args: adaptor.getPadding(), |
| 375 | args: adaptor.getPadConst()); |
| 376 | return success(); |
| 377 | } |
| 378 | |
| 379 | LogicalResult replaceReshape(tosa::ReshapeOp op, tosa::ReshapeOpAdaptor adaptor, |
| 380 | Type type, ConversionPatternRewriter &rewriter) { |
| 381 | rewriter.replaceOpWithNewOp<spirv::TosaReshapeOp>( |
| 382 | op, args&: type, args: adaptor.getInput1(), args: adaptor.getShape()); |
| 383 | return success(); |
| 384 | } |
| 385 | |
| 386 | LogicalResult replaceReverse(tosa::ReverseOp op, tosa::ReverseOpAdaptor adaptor, |
| 387 | Type type, ConversionPatternRewriter &rewriter) { |
| 388 | rewriter.replaceOpWithNewOp<spirv::TosaReverseOp>(op, args&: type, args: adaptor.getAxis(), |
| 389 | args: adaptor.getInput1()); |
| 390 | return success(); |
| 391 | } |
| 392 | |
| 393 | LogicalResult replaceSlice(tosa::SliceOp op, tosa::SliceOpAdaptor adaptor, |
| 394 | Type type, ConversionPatternRewriter &rewriter) { |
| 395 | rewriter.replaceOpWithNewOp<spirv::TosaSliceOp>( |
| 396 | op, args&: type, args: adaptor.getInput1(), args: adaptor.getStart(), args: adaptor.getSize()); |
| 397 | return success(); |
| 398 | } |
| 399 | |
| 400 | LogicalResult replaceTile(tosa::TileOp op, tosa::TileOpAdaptor adaptor, |
| 401 | Type type, ConversionPatternRewriter &rewriter) { |
| 402 | rewriter.replaceOpWithNewOp<spirv::TosaTileOp>(op, args&: type, args: adaptor.getInput1(), |
| 403 | args: adaptor.getMultiples()); |
| 404 | return success(); |
| 405 | } |
| 406 | |
| 407 | LogicalResult replaceTranspose(tosa::TransposeOp op, |
| 408 | tosa::TransposeOpAdaptor adaptor, Type type, |
| 409 | ConversionPatternRewriter &rewriter) { |
| 410 | DenseIntElementsAttr perms = |
| 411 | getI32TensorArmAttr(values: adaptor.getPerms(), rewriter); |
| 412 | rewriter.replaceOpWithNewOp<spirv::TosaTransposeOp>(op, args&: type, args&: perms, |
| 413 | args: adaptor.getInput1()); |
| 414 | return success(); |
| 415 | } |
| 416 | |
| 417 | LogicalResult replaceGather(tosa::GatherOp op, tosa::GatherOpAdaptor adaptor, |
| 418 | Type type, ConversionPatternRewriter &rewriter) { |
| 419 | rewriter.replaceOpWithNewOp<spirv::TosaGatherOp>( |
| 420 | op, args&: type, args: adaptor.getValues(), args: adaptor.getIndices()); |
| 421 | return success(); |
| 422 | } |
| 423 | |
| 424 | LogicalResult replaceScatter(tosa::ScatterOp op, tosa::ScatterOpAdaptor adaptor, |
| 425 | Type type, ConversionPatternRewriter &rewriter) { |
| 426 | rewriter.replaceOpWithNewOp<spirv::TosaScatterOp>( |
| 427 | op, args&: type, args: adaptor.getValuesIn(), args: adaptor.getIndices(), |
| 428 | args: adaptor.getInput()); |
| 429 | return success(); |
| 430 | } |
| 431 | |
| 432 | LogicalResult replaceResize(tosa::ResizeOp op, tosa::ResizeOpAdaptor adaptor, |
| 433 | Type type, ConversionPatternRewriter &rewriter) { |
| 434 | rewriter.replaceOpWithNewOp<spirv::TosaResizeOp>( |
| 435 | op, args&: type, args: getResizeMode(adaptor), args: adaptor.getInput(), args: adaptor.getScale(), |
| 436 | args: adaptor.getOffset(), args: adaptor.getBorder()); |
| 437 | return success(); |
| 438 | } |
| 439 | |
| 440 | LogicalResult replaceRescale(tosa::RescaleOp op, tosa::RescaleOpAdaptor adaptor, |
| 441 | Type type, ConversionPatternRewriter &rewriter) { |
| 442 | rewriter.replaceOpWithNewOp<spirv::TosaRescaleOp>( |
| 443 | op, args&: type, args: adaptor.getScale32(), args: getRoundingMode(adaptor), |
| 444 | args: adaptor.getPerChannel(), args: adaptor.getInputUnsigned(), |
| 445 | args: adaptor.getOutputUnsigned(), args: adaptor.getInput(), args: adaptor.getMultiplier(), |
| 446 | args: adaptor.getShift(), args: adaptor.getInputZp(), args: adaptor.getOutputZp()); |
| 447 | return success(); |
| 448 | } |
| 449 | |
| 450 | template <typename SourceOp> |
| 451 | LogicalResult replaceConstant(SourceOp op, typename SourceOp::Adaptor adaptor, |
| 452 | Type type, ConversionPatternRewriter &rewriter) { |
| 453 | if (auto graphConstantId = op->template getAttrOfType<IntegerAttr>( |
| 454 | graphARMGraphConstantIdAttrName)) { |
| 455 | rewriter.replaceOpWithNewOp<spirv::GraphConstantARMOp>(op, type, |
| 456 | graphConstantId); |
| 457 | return success(); |
| 458 | } |
| 459 | |
| 460 | auto convertedType = dyn_cast<ShapedType>(Val&: type); |
| 461 | auto values = dyn_cast<DenseElementsAttr>(adaptor.getValues()); |
| 462 | if (!convertedType || !values) |
| 463 | return failure(); |
| 464 | |
| 465 | FailureOr<DenseElementsAttr> convertedValues = |
| 466 | convertDenseElementsAttr(values, convertedType); |
| 467 | if (failed(Result: convertedValues)) |
| 468 | return failure(); |
| 469 | |
| 470 | rewriter.replaceOpWithNewOp<spirv::ConstantOp>(op, type, *convertedValues); |
| 471 | return success(); |
| 472 | } |
| 473 | |
| 474 | LogicalResult replaceIdentity(tosa::IdentityOp op, |
| 475 | tosa::IdentityOpAdaptor adaptor, Type type, |
| 476 | ConversionPatternRewriter &rewriter) { |
| 477 | rewriter.replaceOp(op, newValues: adaptor.getInput1()); |
| 478 | return success(); |
| 479 | } |
| 480 | |
| 481 | } // namespace |
| 482 | |
| 483 | void populateTosaToSPIRVTosaOpsConversionPatterns( |
| 484 | SPIRVTypeConverter &typeConverter, RewritePatternSet &patterns) { |
| 485 | patterns.add< |
| 486 | TosaOpConvert<tosa::ArgMaxOp, replaceNanModeReduction< |
| 487 | tosa::ArgMaxOp, spirv::TosaArgMaxOp>>, |
| 488 | TosaOpConvert<tosa::AvgPool2dOp, replaceAvgPool2d>, |
| 489 | TosaOpConvert<tosa::Conv2DOp, |
| 490 | replaceConvolution<tosa::Conv2DOp, spirv::TosaConv2DOp>>, |
| 491 | TosaOpConvert<tosa::Conv3DOp, |
| 492 | replaceConvolution<tosa::Conv3DOp, spirv::TosaConv3DOp>>, |
| 493 | TosaOpConvert<tosa::DepthwiseConv2DOp, |
| 494 | replaceConvolution<tosa::DepthwiseConv2DOp, |
| 495 | spirv::TosaDepthwiseConv2DOp>>, |
| 496 | TosaMultiResultOpConvert<tosa::FFT2dOp, replaceFFT2d>, |
| 497 | TosaOpConvert<tosa::MatMulOp, replaceMatMul>, |
| 498 | TosaOpConvert<tosa::MaxPool2dOp, replaceMaxPool2d>, |
| 499 | TosaMultiResultOpConvert<tosa::RFFT2dOp, replaceRFFT2d>, |
| 500 | TosaOpConvert<tosa::TransposeConv2DOp, replaceTransposeConv2d>, |
| 501 | TosaOpConvert<tosa::ClampOp, replaceClamp>, |
| 502 | TosaOpConvert<tosa::ErfOp, |
| 503 | replaceUnaryInput<tosa::ErfOp, spirv::TosaErfOp>>, |
| 504 | TosaOpConvert<tosa::SigmoidOp, |
| 505 | replaceUnaryInput<tosa::SigmoidOp, spirv::TosaSigmoidOp>>, |
| 506 | TosaOpConvert<tosa::TanhOp, |
| 507 | replaceUnaryInput<tosa::TanhOp, spirv::TosaTanhOp>>, |
| 508 | TosaOpConvert<tosa::AddOp, |
| 509 | replaceBinaryElementwise<tosa::AddOp, spirv::TosaAddOp>>, |
| 510 | TosaOpConvert<tosa::ArithmeticRightShiftOp, replaceArithmeticRightShift>, |
| 511 | TosaOpConvert<tosa::BitwiseAndOp, |
| 512 | replaceBinaryElementwise<tosa::BitwiseAndOp, |
| 513 | spirv::TosaBitwiseAndOp>>, |
| 514 | TosaOpConvert< |
| 515 | tosa::BitwiseOrOp, |
| 516 | replaceBinaryElementwise<tosa::BitwiseOrOp, spirv::TosaBitwiseOrOp>>, |
| 517 | TosaOpConvert<tosa::BitwiseXorOp, |
| 518 | replaceBinaryElementwise<tosa::BitwiseXorOp, |
| 519 | spirv::TosaBitwiseXorOp>>, |
| 520 | TosaOpConvert<tosa::IntDivOp, replaceBinaryElementwise< |
| 521 | tosa::IntDivOp, spirv::TosaIntDivOp>>, |
| 522 | TosaOpConvert<tosa::LogicalAndOp, |
| 523 | replaceBinaryElementwise<tosa::LogicalAndOp, |
| 524 | spirv::TosaLogicalAndOp>>, |
| 525 | TosaOpConvert<tosa::LogicalLeftShiftOp, |
| 526 | replaceBinaryElementwise<tosa::LogicalLeftShiftOp, |
| 527 | spirv::TosaLogicalLeftShiftOp>>, |
| 528 | TosaOpConvert<tosa::LogicalRightShiftOp, |
| 529 | replaceBinaryElementwise<tosa::LogicalRightShiftOp, |
| 530 | spirv::TosaLogicalRightShiftOp>>, |
| 531 | TosaOpConvert< |
| 532 | tosa::LogicalOrOp, |
| 533 | replaceBinaryElementwise<tosa::LogicalOrOp, spirv::TosaLogicalOrOp>>, |
| 534 | TosaOpConvert<tosa::LogicalXorOp, |
| 535 | replaceBinaryElementwise<tosa::LogicalXorOp, |
| 536 | spirv::TosaLogicalXorOp>>, |
| 537 | TosaOpConvert<tosa::MaximumOp, |
| 538 | replaceBinaryNanModeElementwise<tosa::MaximumOp, |
| 539 | spirv::TosaMaximumOp>>, |
| 540 | TosaOpConvert<tosa::MinimumOp, |
| 541 | replaceBinaryNanModeElementwise<tosa::MinimumOp, |
| 542 | spirv::TosaMinimumOp>>, |
| 543 | TosaOpConvert<tosa::MulOp, replaceMul>, |
| 544 | TosaOpConvert<tosa::PowOp, |
| 545 | replaceBinaryElementwise<tosa::PowOp, spirv::TosaPowOp>>, |
| 546 | TosaOpConvert<tosa::SubOp, |
| 547 | replaceBinaryElementwise<tosa::SubOp, spirv::TosaSubOp>>, |
| 548 | TosaOpConvert<tosa::TableOp, replaceTable>, |
| 549 | TosaOpConvert<tosa::AbsOp, |
| 550 | replaceUnaryInput1<tosa::AbsOp, spirv::TosaAbsOp>>, |
| 551 | TosaOpConvert< |
| 552 | tosa::BitwiseNotOp, |
| 553 | replaceUnaryInput1<tosa::BitwiseNotOp, spirv::TosaBitwiseNotOp>>, |
| 554 | TosaOpConvert<tosa::CeilOp, |
| 555 | replaceUnaryInput1<tosa::CeilOp, spirv::TosaCeilOp>>, |
| 556 | TosaOpConvert<tosa::ClzOp, |
| 557 | replaceUnaryInput1<tosa::ClzOp, spirv::TosaClzOp>>, |
| 558 | TosaOpConvert<tosa::CosOp, |
| 559 | replaceUnaryInput1<tosa::CosOp, spirv::TosaCosOp>>, |
| 560 | TosaOpConvert<tosa::ExpOp, |
| 561 | replaceUnaryInput1<tosa::ExpOp, spirv::TosaExpOp>>, |
| 562 | TosaOpConvert<tosa::FloorOp, |
| 563 | replaceUnaryInput1<tosa::FloorOp, spirv::TosaFloorOp>>, |
| 564 | TosaOpConvert<tosa::LogOp, |
| 565 | replaceUnaryInput1<tosa::LogOp, spirv::TosaLogOp>>, |
| 566 | TosaOpConvert< |
| 567 | tosa::LogicalNotOp, |
| 568 | replaceUnaryInput1<tosa::LogicalNotOp, spirv::TosaLogicalNotOp>>, |
| 569 | TosaOpConvert<tosa::NegateOp, replaceNegate>, |
| 570 | TosaOpConvert< |
| 571 | tosa::ReciprocalOp, |
| 572 | replaceUnaryInput1<tosa::ReciprocalOp, spirv::TosaReciprocalOp>>, |
| 573 | TosaOpConvert<tosa::RsqrtOp, |
| 574 | replaceUnaryInput1<tosa::RsqrtOp, spirv::TosaRsqrtOp>>, |
| 575 | TosaOpConvert<tosa::SinOp, |
| 576 | replaceUnaryInput1<tosa::SinOp, spirv::TosaSinOp>>, |
| 577 | TosaOpConvert<tosa::SelectOp, replaceSelect>, |
| 578 | TosaOpConvert<tosa::EqualOp, replaceBinaryElementwise< |
| 579 | tosa::EqualOp, spirv::TosaEqualOp>>, |
| 580 | TosaOpConvert< |
| 581 | tosa::GreaterOp, |
| 582 | replaceBinaryElementwise<tosa::GreaterOp, spirv::TosaGreaterOp>>, |
| 583 | TosaOpConvert<tosa::GreaterEqualOp, |
| 584 | replaceBinaryElementwise<tosa::GreaterEqualOp, |
| 585 | spirv::TosaGreaterEqualOp>>, |
| 586 | TosaOpConvert< |
| 587 | tosa::ReduceAllOp, |
| 588 | replaceReduction<tosa::ReduceAllOp, spirv::TosaReduceAllOp>>, |
| 589 | TosaOpConvert< |
| 590 | tosa::ReduceAnyOp, |
| 591 | replaceReduction<tosa::ReduceAnyOp, spirv::TosaReduceAnyOp>>, |
| 592 | TosaOpConvert< |
| 593 | tosa::ReduceMaxOp, |
| 594 | replaceNanModeReduction<tosa::ReduceMaxOp, spirv::TosaReduceMaxOp>>, |
| 595 | TosaOpConvert< |
| 596 | tosa::ReduceMinOp, |
| 597 | replaceNanModeReduction<tosa::ReduceMinOp, spirv::TosaReduceMinOp>>, |
| 598 | TosaOpConvert< |
| 599 | tosa::ReduceProductOp, |
| 600 | replaceReduction<tosa::ReduceProductOp, spirv::TosaReduceProductOp>>, |
| 601 | TosaOpConvert< |
| 602 | tosa::ReduceSumOp, |
| 603 | replaceReduction<tosa::ReduceSumOp, spirv::TosaReduceSumOp>>, |
| 604 | TosaOpConvert<tosa::ConcatOp, replaceConcat>, |
| 605 | TosaOpConvert<tosa::PadOp, replacePad>, |
| 606 | TosaOpConvert<tosa::ReshapeOp, replaceReshape>, |
| 607 | TosaOpConvert<tosa::ReverseOp, replaceReverse>, |
| 608 | TosaOpConvert<tosa::SliceOp, replaceSlice>, |
| 609 | TosaOpConvert<tosa::TileOp, replaceTile>, |
| 610 | TosaOpConvert<tosa::TransposeOp, replaceTranspose>, |
| 611 | TosaOpConvert<tosa::GatherOp, replaceGather>, |
| 612 | TosaOpConvert<tosa::ScatterOp, replaceScatter>, |
| 613 | TosaOpConvert<tosa::ResizeOp, replaceResize>, |
| 614 | TosaOpConvert<tosa::CastOp, |
| 615 | replaceUnaryInput<tosa::CastOp, spirv::TosaCastOp>>, |
| 616 | TosaOpConvert<tosa::RescaleOp, replaceRescale>, |
| 617 | TosaOpConvert<tosa::ConstOp, replaceConstant<tosa::ConstOp>>, |
| 618 | TosaOpConvert<tosa::ConstShapeOp, replaceConstant<tosa::ConstShapeOp>>, |
| 619 | TosaOpConvert<tosa::IdentityOp, replaceIdentity>>(arg&: typeConverter, |
| 620 | args: patterns.getContext()); |
| 621 | } |
| 622 | |
| 623 | } // namespace mlir::tosa |
| 624 | |