1//===- TosaDecomposeDepthwise.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Decompose TOSA Depthwise operation to a series of TOSA Ops specifically
10// (1) Convert a 1x1 Depthwise to Reshape -> Mul -> Reshape -> Add
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/Tosa/IR/TosaOps.h"
15#include "mlir/Dialect/Tosa/Transforms/Passes.h"
16#include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
17#include "mlir/IR/BuiltinTypes.h"
18
19using namespace mlir;
20using namespace mlir::tosa;
21
22namespace {
23
24struct DepthwiseConv2DIsMul : public OpRewritePattern<tosa::DepthwiseConv2DOp> {
25 explicit DepthwiseConv2DIsMul(MLIRContext *context)
26 : OpRewritePattern(context) {}
27
28 LogicalResult matchAndRewrite(tosa::DepthwiseConv2DOp op,
29 PatternRewriter &rewriter) const override {
30 Value input = op.getInput();
31 Value weight = op.getWeight();
32 ShapedType inputType = cast<ShapedType>(Val: input.getType());
33 ShapedType weightType = cast<ShapedType>(Val: weight.getType());
34 ShapedType resultType = cast<ShapedType>(Val: op.getOutput().getType());
35
36 // Any dimensions other than batchSize cannot be dynamic for input/output
37 for (unsigned int i = 1; i < 4; ++i) {
38 if (inputType.isDynamicDim(idx: i) || resultType.isDynamicDim(idx: i))
39 return failure();
40 }
41
42 if (!weightType.hasStaticShape()) {
43 return failure();
44 }
45
46 if (!llvm::all_of(Range: op.getStride(), P: [](int64_t v) { return v == 1; }))
47 return failure();
48
49 // Only works for a 1x1 kernel.
50 ArrayRef<int64_t> weightShape = weightType.getShape();
51 if (weightShape[0] != 1 || weightShape[1] != 1) {
52 return failure();
53 }
54
55 Type inputETy = inputType.getElementType();
56 Type weightETy = weightType.getElementType();
57 if (!inputETy.isIntOrFloat() || !weightETy.isIntOrFloat())
58 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported type");
59
60 // Get and verify zero points.
61 FailureOr<int64_t> maybeIZp = op.getInputZeroPoint();
62 if (failed(Result: maybeIZp))
63 return rewriter.notifyMatchFailure(
64 arg&: op, msg: "input zero point cannot be statically determined");
65
66 FailureOr<int64_t> maybeWZp = op.getWeightZeroPoint();
67 if (failed(Result: maybeWZp))
68 return rewriter.notifyMatchFailure(
69 arg&: op, msg: "weight zero point cannot be statically determined");
70
71 int64_t iZp = *maybeIZp;
72 int64_t wZp = *maybeWZp;
73 if (op.verifyInputZeroPoint(zp: iZp).failed())
74 return rewriter.notifyMatchFailure(
75 arg&: op, msg: "input zero point must be zero for non-int8 integer types");
76 if (op.verifyWeightZeroPoint(zp: wZp).failed())
77 return rewriter.notifyMatchFailure(
78 arg&: op, msg: "weight zero point must be zero for non-int8 integer types");
79
80 // Reshape input to [N, H, W, C] -> [N, H, W, C, 1].
81 ArrayRef<int64_t> inputShape = inputType.getShape();
82 llvm::SmallVector<int64_t, 2> revisedInputShape{
83 inputShape[0], inputShape[1], inputShape[2], inputShape[3], 1};
84 inputType = RankedTensorType::get(
85 shape: revisedInputShape,
86 elementType: dyn_cast<RankedTensorType>(Val: input.getType()).getElementType());
87 auto revisedInputShapeValue =
88 getTosaConstShape(rewriter, loc: op.getLoc(), shape: revisedInputShape);
89 input = tosa::ReshapeOp::create(builder&: rewriter, location: op.getLoc(), output: inputType, input1: input,
90 shape: revisedInputShapeValue)
91 .getResult();
92
93 Type resultETy = resultType.getElementType();
94 if (inputETy != resultETy) {
95 inputType = inputType.clone(elementType: resultETy);
96 input = tosa::CastOp::create(builder&: rewriter, location: op.getLoc(), output: inputType, input,
97 /*input_unsigned*/ false);
98 }
99
100 if (weightETy != resultETy) {
101 weightType = weightType.clone(elementType: resultETy);
102 weight = tosa::CastOp::create(builder&: rewriter, location: op.getLoc(), output: weightType, input: weight,
103 /*input_unsigned*/ false);
104 }
105
106 if (iZp != 0 || wZp != 0) {
107
108 auto applyZp = [&](Value val, int64_t zp) -> Value {
109 if (zp == 0)
110 return val;
111 auto ety = cast<ShapedType>(Val: val.getType()).getElementType();
112 std::vector<int64_t> shape(cast<ShapedType>(Val: val.getType()).getRank(),
113 1);
114 auto zpTy = RankedTensorType::get(shape, elementType: ety);
115 auto zpAttr =
116 DenseElementsAttr::get(type: zpTy, values: rewriter.getIntegerAttr(type: ety, value: zp));
117 auto zpVal = tosa::ConstOp::create(builder&: rewriter, location: op.getLoc(), output: zpTy, values: zpAttr);
118 return tosa::SubOp::create(builder&: rewriter, location: op.getLoc(), output: val.getType(), input1: val,
119 input2: zpVal);
120 };
121
122 input = applyZp(input, iZp);
123 weight = applyZp(weight, wZp);
124 }
125
126 ArrayRef<int64_t> padAttr = op.getPad();
127 llvm::SmallVector<int64_t> pad(10, 0);
128 for (const auto &it : llvm::enumerate(First&: padAttr))
129 pad[it.index() + 2] = it.value();
130
131 if (llvm::any_of(Range&: pad, P: [](int64_t p) { return p != 0; })) {
132 Attribute zeroAttr = rewriter.getZeroAttr(type: inputETy);
133
134 llvm::SmallVector<int64_t> newShape(inputType.getShape());
135 for (int i = 0, s = pad.size(); i < s; ++i) {
136 if (newShape[i / 2] != ShapedType::kDynamic) {
137 newShape[i / 2] += pad[i];
138 }
139 }
140
141 Value padSizeVal = getTosaConstShape(rewriter, loc: op->getLoc(), shape: pad);
142
143 auto padTy = RankedTensorType::get(shape: {1}, elementType: inputETy);
144 auto padAttr = DenseElementsAttr::get(type: padTy, values: zeroAttr);
145 Value padVal =
146 tosa::ConstOp::create(builder&: rewriter, location: op->getLoc(), output: padTy, values: padAttr);
147 inputType = RankedTensorType::get(shape: newShape, elementType: inputETy);
148 input = tosa::PadOp::create(builder&: rewriter, location: op->getLoc(), output: inputType, input1: input,
149 padding: padSizeVal, pad_const: padVal);
150 }
151
152 // Perform an elementwise mul over the reshaped input and weight.
153 llvm::SmallVector<int64_t, 2> mulShape{
154 inputType.getDimSize(idx: 0), inputType.getDimSize(idx: 1),
155 inputType.getDimSize(idx: 2), inputType.getDimSize(idx: 3), weightShape[3]};
156 auto mulShapeType = RankedTensorType::get(
157 shape: mulShape,
158 elementType: dyn_cast<RankedTensorType>(Val: weight.getType()).getElementType());
159
160 if (EqualizeRanks(rewriter, loc: op.getLoc(), input1&: input, input2&: weight).failed()) {
161 return failure();
162 }
163
164 auto shiftElementType = IntegerType::get(context: rewriter.getContext(), width: 8);
165 auto shiftType = RankedTensorType::get(shape: {1}, elementType: shiftElementType);
166 auto shiftZeroAttr = DenseElementsAttr::get(
167 type: shiftType, values: rewriter.getIntegerAttr(type: shiftElementType, value: 0));
168 Value constZero =
169 tosa::ConstOp::create(builder&: rewriter, location: op.getLoc(), output: shiftType, values: shiftZeroAttr);
170 Value mulValue = tosa::MulOp::create(builder&: rewriter, location: op.getLoc(), output: mulShapeType,
171 input1: input, input2: weight, shift: constZero)
172 .getResult();
173
174 // Reshape output to [N, H, W, C * M].
175 auto outputShape = cast<ShapedType>(Val: op.getOutput().getType()).getShape();
176 auto outputShapeType = RankedTensorType::get(
177 shape: outputShape,
178 elementType: dyn_cast<RankedTensorType>(Val: input.getType()).getElementType());
179 auto outputShapeValue =
180 getTosaConstShape(rewriter, loc: op->getLoc(), shape: outputShape);
181 Value outputValue = tosa::ReshapeOp::create(
182 builder&: rewriter, location: op.getLoc(), output: outputShapeType, input1: mulValue, shape: outputShapeValue);
183
184 Value bias = op.getBias();
185 if (EqualizeRanks(rewriter, loc: op.getLoc(), input1&: outputValue, input2&: bias).failed()) {
186 return failure();
187 }
188
189 // Add in the bias.
190 rewriter
191 .replaceOpWithNewOp<tosa::AddOp>(op, args&: outputShapeType, args&: outputValue, args&: bias)
192 .getResult();
193 return success();
194 }
195};
196
197} // namespace
198
199void mlir::tosa::populateTosaDecomposeDepthwise(MLIRContext *ctx,
200 RewritePatternSet &patterns) {
201 patterns.add<DepthwiseConv2DIsMul>(arg&: ctx);
202}
203