1//===- TosaToArith.cpp - Lowering Tosa to Arith Dialect -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These rewriters lower from the Tosa to the Arith dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Conversion/TosaToArith/TosaToArith.h"
14#include "mlir/Dialect/Arith/IR/Arith.h"
15#include "mlir/Dialect/Tosa/IR/TosaOps.h"
16#include "mlir/IR/PatternMatch.h"
17#include "mlir/IR/TypeUtilities.h"
18
19using namespace mlir;
20using namespace tosa;
21
22namespace {
23
24class ConstOpConverter : public OpRewritePattern<tosa::ConstOp> {
25public:
26 using Base::Base;
27
28 LogicalResult matchAndRewrite(tosa::ConstOp op,
29 PatternRewriter &rewriter) const final {
30 rewriter.replaceOpWithNewOp<arith::ConstantOp>(op, args: op.getValues());
31 return success();
32 }
33};
34
35Type matchContainerType(Type element, Type container) {
36 if (auto shapedTy = dyn_cast<ShapedType>(Val&: container))
37 return shapedTy.clone(elementType: element);
38
39 return element;
40}
41
42TypedAttr getConstantAttr(Type type, int64_t value, PatternRewriter &rewriter) {
43 if (auto shapedTy = dyn_cast<ShapedType>(Val&: type)) {
44 Type eTy = shapedTy.getElementType();
45 APInt valueInt(eTy.getIntOrFloatBitWidth(), value, /*isSigned=*/true);
46 return DenseIntElementsAttr::get(type: shapedTy, arg&: valueInt);
47 }
48
49 return rewriter.getIntegerAttr(type, value);
50}
51
52Value getConstantValue(Location loc, Type type, int64_t value,
53 PatternRewriter &rewriter) {
54 return arith::ConstantOp::create(builder&: rewriter, location: loc,
55 value: getConstantAttr(type, value, rewriter));
56}
57
58// This converts the TOSA ApplyScale operator to a set of arithmetic ops,
59// using 64-bit operations to perform the necessary multiply, bias, and shift.
60class ApplyScaleGenericOpConverter
61 : public OpRewritePattern<tosa::ApplyScaleOp> {
62public:
63 using Base::Base;
64
65 LogicalResult matchAndRewrite(tosa::ApplyScaleOp op,
66 PatternRewriter &rewriter) const final {
67 RoundingMode roundingMode = op.getRoundingMode();
68 if (roundingMode != RoundingMode::DOUBLE_ROUND &&
69 roundingMode != RoundingMode::SINGLE_ROUND) {
70 return failure();
71 }
72
73 Location loc = op.getLoc();
74 Value value = op.getValue();
75 Value multiplier32 = op.getMultiplier();
76
77 Type resultTy = op.getType();
78 Type valueTy = value.getType();
79 Type i32Ty = matchContainerType(element: rewriter.getI32Type(), container: resultTy);
80 Type i64Ty = matchContainerType(element: rewriter.getI64Type(), container: resultTy);
81
82 Value zero = getConstantValue(loc, type: valueTy, value: 0, rewriter);
83 Value one64 = getConstantValue(loc, type: i64Ty, value: 1, rewriter);
84 Value thirtyOne32 = getConstantValue(loc, type: i32Ty, value: 31, rewriter);
85
86 Value shift32 = arith::ExtUIOp::create(builder&: rewriter, location: loc, out: i32Ty, in: op.getShift());
87
88 // Compute the multiplication in 64-bits then select the high / low parts.
89 Value value64 = value;
90 if (getElementTypeOrSelf(type: valueTy) != rewriter.getI64Type())
91 value64 = arith::ExtSIOp::create(builder&: rewriter, location: loc, out: i64Ty, in: value);
92 Value multiplier64 =
93 arith::ExtSIOp::create(builder&: rewriter, location: loc, out: i64Ty, in: multiplier32);
94 Value multiply64 =
95 arith::MulIOp::create(builder&: rewriter, location: loc, lhs: value64, rhs: multiplier64);
96
97 // Apply normal rounding.
98 Value shift64 = arith::ExtUIOp::create(builder&: rewriter, location: loc, out: i64Ty, in: shift32);
99 Value round = arith::ShLIOp::create(builder&: rewriter, location: loc, lhs: one64, rhs: shift64);
100 round = arith::ShRUIOp::create(builder&: rewriter, location: loc, lhs: round, rhs: one64);
101 multiply64 = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: multiply64, rhs: round);
102
103 // Apply double rounding if necessary.
104 if (op.getRoundingMode() == RoundingMode::DOUBLE_ROUND) {
105 int64_t roundInt = 1 << 30;
106 Value roundUp = getConstantValue(loc, type: i64Ty, value: roundInt, rewriter);
107 Value roundDown = getConstantValue(loc, type: i64Ty, value: -roundInt, rewriter);
108 Value positive = arith::CmpIOp::create(
109 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::sge, lhs: value, rhs: zero);
110 Value dir =
111 arith::SelectOp::create(builder&: rewriter, location: loc, condition: positive, true_value: roundUp, false_value: roundDown);
112 Value val = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: dir, rhs: multiply64);
113 Value valid = arith::CmpIOp::create(
114 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::sgt, lhs: shift32, rhs: thirtyOne32);
115 multiply64 =
116 arith::SelectOp::create(builder&: rewriter, location: loc, condition: valid, true_value: val, false_value: multiply64);
117 }
118
119 Value result64 = arith::ShRSIOp::create(builder&: rewriter, location: loc, lhs: multiply64, rhs: shift64);
120 Value result32 = arith::TruncIOp::create(builder&: rewriter, location: loc, out: i32Ty, in: result64);
121
122 rewriter.replaceOp(op, newValues: result32);
123 return success();
124 }
125};
126
127class ApplyScale32BitOpConverter : public OpRewritePattern<tosa::ApplyScaleOp> {
128public:
129 using Base::Base;
130
131 LogicalResult matchAndRewrite(tosa::ApplyScaleOp op,
132 PatternRewriter &rewriter) const final {
133 RoundingMode roundingMode = op.getRoundingMode();
134 if (roundingMode != RoundingMode::DOUBLE_ROUND &&
135 roundingMode != RoundingMode::SINGLE_ROUND) {
136 return failure();
137 }
138
139 Location loc = op.getLoc();
140
141 Type resultTy = op.getType();
142 Type i32Ty = matchContainerType(element: rewriter.getI32Type(), container: resultTy);
143
144 Value value = op.getValue();
145 if (getElementTypeOrSelf(type: value.getType()).getIntOrFloatBitWidth() > 32) {
146 return failure();
147 }
148
149 Value value32 = op.getValue();
150 Value multiplier32 = op.getMultiplier();
151 Value shift32 = arith::ExtUIOp::create(builder&: rewriter, location: loc, out: i32Ty, in: op.getShift());
152
153 // Constants used during the scaling operation.
154 Value zero32 = getConstantValue(loc, type: i32Ty, value: 0, rewriter);
155 Value one32 = getConstantValue(loc, type: i32Ty, value: 1, rewriter);
156 Value two32 = getConstantValue(loc, type: i32Ty, value: 2, rewriter);
157 Value thirty32 = getConstantValue(loc, type: i32Ty, value: 30, rewriter);
158 Value thirtyTwo32 = getConstantValue(loc, type: i32Ty, value: 32, rewriter);
159
160 // Compute the multiplication in 64-bits then select the high / low parts.
161 // Grab out the high/low of the computation
162 auto value64 =
163 arith::MulSIExtendedOp::create(builder&: rewriter, location: loc, lhs: value32, rhs: multiplier32);
164 Value low32 = value64.getLow();
165 Value high32 = value64.getHigh();
166
167 // Determine the direction and amount to shift the high bits.
168 Value shiftOver32 = arith::CmpIOp::create(
169 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::sge, lhs: shift32, rhs: thirtyTwo32);
170 Value roundHighBits = arith::CmpIOp::create(
171 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::sgt, lhs: shift32, rhs: thirtyTwo32);
172
173 Value shiftHighL =
174 arith::SubIOp::create(builder&: rewriter, location: loc, lhs: thirtyTwo32, rhs: shift32);
175 Value shiftHighR =
176 arith::SubIOp::create(builder&: rewriter, location: loc, lhs: shift32, rhs: thirtyTwo32);
177
178 shiftHighL =
179 arith::SelectOp::create(builder&: rewriter, location: loc, condition: shiftOver32, true_value: zero32, false_value: shiftHighL);
180 shiftHighR =
181 arith::SelectOp::create(builder&: rewriter, location: loc, condition: shiftOver32, true_value: shiftHighR, false_value: zero32);
182
183 // Conditionally perform our double round.
184 if (op.getRoundingMode() == RoundingMode::DOUBLE_ROUND) {
185 Value negOne32 = getConstantValue(loc, type: i32Ty, value: -1, rewriter);
186 Value valuePositive = arith::CmpIOp::create(
187 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::sge, lhs: value32, rhs: zero32);
188
189 Value roundDir = arith::SelectOp::create(builder&: rewriter, location: loc, condition: valuePositive,
190 true_value: one32, false_value: negOne32);
191 roundDir =
192 arith::SelectOp::create(builder&: rewriter, location: loc, condition: shiftOver32, true_value: roundDir, false_value: zero32);
193
194 Value shiftLow = arith::ShRUIOp::create(builder&: rewriter, location: loc, lhs: low32, rhs: thirty32);
195 Value rounded = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: shiftLow, rhs: roundDir);
196 Value carry = arith::ShRSIOp::create(builder&: rewriter, location: loc, lhs: rounded, rhs: two32);
197
198 Value shiftRound =
199 arith::ShLIOp::create(builder&: rewriter, location: loc, lhs: roundDir, rhs: thirty32);
200
201 low32 = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: low32, rhs: shiftRound);
202 high32 = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: high32, rhs: carry);
203 }
204
205 // Conditionally apply rounding in the low bits.
206 {
207 Value shiftSubOne = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: shift32, rhs: one32);
208 Value roundBit = arith::ShLIOp::create(builder&: rewriter, location: loc, lhs: one32, rhs: shiftSubOne);
209 roundBit = arith::SelectOp::create(builder&: rewriter, location: loc, condition: roundHighBits, true_value: zero32,
210 false_value: roundBit);
211
212 Value newLow32 = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: low32, rhs: roundBit);
213 Value wasRounded = arith::CmpIOp::create(
214 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::ugt, lhs: low32, rhs: newLow32);
215 low32 = newLow32;
216
217 Value rounded32 =
218 arith::ExtUIOp::create(builder&: rewriter, location: loc, out: i32Ty, in: wasRounded);
219 high32 = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: high32, rhs: rounded32);
220 }
221
222 // Conditionally apply rounding in the high bits.
223 {
224 Value shiftSubOne =
225 arith::SubIOp::create(builder&: rewriter, location: loc, lhs: shiftHighR, rhs: one32);
226 Value roundBit = arith::ShLIOp::create(builder&: rewriter, location: loc, lhs: one32, rhs: shiftSubOne);
227 roundBit = arith::SelectOp::create(builder&: rewriter, location: loc, condition: roundHighBits, true_value: roundBit,
228 false_value: zero32);
229 high32 = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: high32, rhs: roundBit);
230 }
231
232 // Combine the correct high/low bits into the final rescale result.
233 high32 = arith::ShLIOp::create(builder&: rewriter, location: loc, lhs: high32, rhs: shiftHighL);
234 high32 = arith::ShRSIOp::create(builder&: rewriter, location: loc, lhs: high32, rhs: shiftHighR);
235 low32 = arith::ShRUIOp::create(builder&: rewriter, location: loc, lhs: low32, rhs: shift32);
236 low32 = arith::SelectOp::create(builder&: rewriter, location: loc, condition: shiftOver32, true_value: zero32, false_value: low32);
237
238 // Apply the rounding behavior and shift to the final alignment.
239 Value result = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: low32, rhs: high32);
240
241 // Truncate if necessary.
242 if (!getElementTypeOrSelf(type: resultTy).isInteger(width: 32)) {
243 result = arith::TruncIOp::create(builder&: rewriter, location: loc, out: resultTy, in: result);
244 }
245
246 rewriter.replaceOp(op, newValues: result);
247 return success();
248 }
249};
250
251} // namespace
252
253void mlir::tosa::populateTosaToArithConversionPatterns(
254 RewritePatternSet *patterns) {
255 patterns->add<ConstOpConverter>(arg: patterns->getContext());
256}
257
258void mlir::tosa::populateTosaRescaleToArithConversionPatterns(
259 RewritePatternSet *patterns, bool include32Bit) {
260 patterns->add<ApplyScaleGenericOpConverter>(arg: patterns->getContext(), args: 100);
261 if (include32Bit) {
262 patterns->add<ApplyScale32BitOpConverter>(arg: patterns->getContext(), args: 200);
263 }
264}
265