1//===- TosaDowngrade1_1To1_0.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Rewrites constructs which are only compatible in TOSA specification 1.1 and
10// above to their TOSA 1.0 counterparts where possible. Downgrading is
11// best-effort and validation should be performed afterwards to ensure
12// compatibility with the TOSA 1.0 specification.
13//
14//===----------------------------------------------------------------------===//
15
16#include "mlir/Dialect/Tosa/Transforms/Passes.h"
17
18#include "mlir/Dialect/Func/IR/FuncOps.h"
19#include "mlir/Dialect/Tosa/IR/TosaOps.h"
20#include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
21#include "mlir/IR/PatternMatch.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23
24namespace mlir {
25namespace tosa {
26#define GEN_PASS_DEF_TOSADOWNGRADE1P1TO1P0PASS
27#include "mlir/Dialect/Tosa/Transforms/Passes.h.inc"
28} // namespace tosa
29} // namespace mlir
30
31using namespace mlir;
32using namespace mlir::tosa;
33
34namespace {
35
36class BoolFp32CastRewrite : public OpRewritePattern<tosa::CastOp> {
37public:
38 using Base::Base;
39
40 LogicalResult matchAndRewrite(tosa::CastOp op,
41 PatternRewriter &rewriter) const override {
42 const Value input = op.getInput();
43
44 const Type i1Type = rewriter.getI1Type();
45 const Type f32Type = rewriter.getF32Type();
46
47 const Type inputElemType = getElementTypeOrSelf(type: input.getType());
48 const Type outputElemType = getElementTypeOrSelf(type: op.getType());
49 const bool isFp32ToBool =
50 inputElemType == f32Type && outputElemType == i1Type;
51 const bool isBoolToFp32 =
52 inputElemType == i1Type && outputElemType == f32Type;
53
54 if (!isFp32ToBool && !isBoolToFp32)
55 return rewriter.notifyMatchFailure(arg&: op,
56 msg: "expected cast between bool and f32");
57
58 const Type outputType = op.getType();
59 const Type i8Type = rewriter.getI8Type();
60 const Type intermediateType = cast<TensorType>(Val: outputType).clone(elementType: i8Type);
61
62 auto inner = tosa::CastOp::create(builder&: rewriter, location: op.getLoc(), output: intermediateType,
63 input, /*input_unsigned*/ false);
64 auto outer =
65 tosa::CastOp::create(builder&: rewriter, location: op.getLoc(), output: outputType,
66 input: inner.getOutput(), /*input_unsigned*/ false);
67 rewriter.replaceOp(op, newValues: outer.getOutput());
68 return success();
69 }
70};
71
72class BoolGatherRewrite : public OpRewritePattern<tosa::GatherOp> {
73public:
74 using Base::Base;
75
76 LogicalResult matchAndRewrite(tosa::GatherOp op,
77 PatternRewriter &rewriter) const override {
78 const Value values = op.getValues();
79 const Value indices = op.getIndices();
80
81 const Type valuesType = values.getType();
82 const Type resultType = op.getType();
83
84 const Type i1Type = rewriter.getI1Type();
85 const Type i32Type = rewriter.getI32Type();
86 if (getElementTypeOrSelf(type: valuesType) != i1Type ||
87 getElementTypeOrSelf(type: indices.getType()) != i32Type)
88 return rewriter.notifyMatchFailure(
89 arg&: op, msg: "expected values of bool type and indices of i32 type");
90
91 const Type i8Type = rewriter.getI8Type();
92 const Type valuesI8Type = cast<TensorType>(Val: valuesType).clone(elementType: i8Type);
93 const Type resultI8Type = cast<TensorType>(Val: resultType).clone(elementType: i8Type);
94
95 auto valuesToI8 = tosa::CastOp::create(builder&: rewriter, location: op.getLoc(), output: valuesI8Type,
96 input: values, /*input_unsigned*/ false);
97 auto gatherI8 = tosa::GatherOp::create(builder&: rewriter, location: op.getLoc(), output: resultI8Type,
98 values: valuesToI8.getOutput(), indices);
99 auto i8ToBool =
100 tosa::CastOp::create(builder&: rewriter, location: op.getLoc(), output: resultType,
101 input: gatherI8.getOutput(), /*input_unsigned*/ false);
102 rewriter.replaceOp(op, newValues: i8ToBool.getOutput());
103 return success();
104 }
105};
106
107class BoolScatterRewrite : public OpRewritePattern<tosa::ScatterOp> {
108public:
109 using Base::Base;
110
111 LogicalResult matchAndRewrite(tosa::ScatterOp op,
112 PatternRewriter &rewriter) const override {
113 const Value valuesIn = op.getValuesIn();
114 const Value indices = op.getIndices();
115
116 const Type valuesInType = valuesIn.getType();
117 const Type i1Type = rewriter.getI1Type();
118 const Type i32Type = rewriter.getI32Type();
119 if (getElementTypeOrSelf(type: valuesInType) != i1Type ||
120 getElementTypeOrSelf(type: indices.getType()) != i32Type)
121 return rewriter.notifyMatchFailure(
122 arg&: op, msg: "expected values of bool type and indices of i32 type");
123
124 const Value input = op.getInput();
125 const Type inputType = input.getType();
126 const Type resultType = op.getType();
127
128 const Type i8Type = rewriter.getI8Type();
129 const Type valuesInI8Type = cast<TensorType>(Val: valuesInType).clone(elementType: i8Type);
130 const Type inputI8Type = cast<TensorType>(Val: inputType).clone(elementType: i8Type);
131 const Type resultI8Type = cast<TensorType>(Val: resultType).clone(elementType: i8Type);
132
133 auto valuesInToI8 =
134 tosa::CastOp::create(builder&: rewriter, location: op.getLoc(), output: valuesInI8Type, input: valuesIn,
135 /*input_unsigned*/ false);
136 auto inputToI8 = tosa::CastOp::create(builder&: rewriter, location: op.getLoc(), output: inputI8Type,
137 input, /*input_unsigned*/ false);
138 auto scatterI8 = tosa::ScatterOp::create(
139 builder&: rewriter, location: op.getLoc(), values_out: resultI8Type, values_in: valuesInToI8.getOutput(), indices,
140 input: inputToI8.getOutput());
141 auto i8ToBool = tosa::CastOp::create(builder&: rewriter, location: op.getLoc(), output: resultType,
142 input: scatterI8.getValuesOut(),
143 /*input_unsigned*/ false);
144 rewriter.replaceOp(op, newValues: i8ToBool.getOutput());
145 return success();
146 }
147};
148
149static LogicalResult isMatMulTTypeCompatibleForDowngrade(tosa::MatMulTOp op) {
150 const Type aElementType = getStorageElementTypeOrSelf(type: op.getA().getType());
151 const Type bElementType = getStorageElementTypeOrSelf(type: op.getB().getType());
152 const Type outputElementType =
153 getStorageElementTypeOrSelf(type: op.getOutput().getType());
154
155 if (aElementType != bElementType)
156 return failure();
157
158 if (isa<BlockScaledType>(Val: aElementType) || isa<BlockScaledType>(Val: bElementType))
159 return failure();
160
161 if ((aElementType.isF16() && outputElementType.isF16()) ||
162 (aElementType.isF16() && outputElementType.isF32()) ||
163 (aElementType.isF32() && outputElementType.isF32()) ||
164 (aElementType.isBF16() && outputElementType.isF32()) ||
165 (aElementType.isInteger(width: 8) && outputElementType.isInteger(width: 32)) ||
166 (aElementType.isInteger(width: 16) && outputElementType.isInteger(width: 48)) ||
167 (isa<Float8E5M2Type>(Val: aElementType) && outputElementType.isF16()) ||
168 (isa<Float8E4M3FNType>(Val: aElementType) && outputElementType.isF16()))
169 return success();
170
171 return failure();
172}
173
174class MatMulTRewrite : public OpRewritePattern<tosa::MatMulTOp> {
175public:
176 using Base::Base;
177
178 LogicalResult matchAndRewrite(tosa::MatMulTOp op,
179 PatternRewriter &rewriter) const override {
180 if (failed(Result: isMatMulTTypeCompatibleForDowngrade(op)))
181 return rewriter.notifyMatchFailure(
182 arg&: op, msg: "expected 1.0-compatible matmul_t element types");
183
184 const Type aType = op.getA().getType();
185 const Type bType = op.getB().getType();
186 const ShapeAdaptor aShape(aType);
187 const ShapeAdaptor bShape(bType);
188 if (!aShape.hasRank() || !bShape.hasRank())
189 return rewriter.notifyMatchFailure(arg&: op, msg: "expected ranked A and B tensors");
190
191 // TOSA 1.0 MATMUL requires rank 3 inputs. MATMUL_T operations with other
192 // ranks require separate rank and batch broadcast legalization.
193 if (aShape.getRank() != 3 || bShape.getRank() != 3)
194 return rewriter.notifyMatchFailure(arg&: op, msg: "expected rank 3 A and B tensors");
195
196 const int64_t dSize = bShape.getDimSize(index: 0);
197 const int64_t nSize = aShape.getDimSize(index: 0);
198
199 // To convert broadcasting behaviour to TOSA 1.0, we're required to tile the
200 // input. TOSA 1.0 does not support shape expressions, so the batch size
201 // must be known at compile time.
202 if (ShapedType::isDynamic(dValue: dSize) ||
203 (dSize == 1 && ShapedType::isDynamic(dValue: nSize)))
204 return rewriter.notifyMatchFailure(
205 arg&: op, msg: "expected known batch size for broadcast");
206
207 // TOSA 1.0 MATMUL does not broadcast batch dimensions. The lowering below
208 // handles broadcasting B by tiling it, but does not implement tiling A.
209 if (dSize != 1 && (nSize == 1 || ShapedType::isDynamic(dValue: nSize)))
210 return rewriter.notifyMatchFailure(
211 arg&: op, msg: "cannot downgrade A batch broadcasting");
212
213 const int64_t wSize = bShape.getDimSize(index: 1);
214 const int64_t cSize = bShape.getDimSize(index: 2);
215 const Location loc = op.getLoc();
216 const RankedTensorType transposedBType =
217 cast<RankedTensorType>(Val: bType).clone(shape: {dSize, cSize, wSize});
218 auto transpose =
219 tosa::TransposeOp::create(builder&: rewriter, location: loc, output: transposedBType, input1: op.getB(),
220 perms: rewriter.getDenseI32ArrayAttr(values: {0, 2, 1}));
221 Value matMulB = transpose.getOutput();
222
223 // Matmul does not support broadcasting, so tile b if required
224 if (dSize == 1 && nSize != 1) {
225 const RankedTensorType tiledBType =
226 cast<RankedTensorType>(Val: bType).clone(shape: {nSize, cSize, wSize});
227 const Value multiples = getTosaConstShape(rewriter, loc, shape: {nSize, 1, 1});
228 auto tile =
229 tosa::TileOp::create(builder&: rewriter, location: loc, output: tiledBType, input1: matMulB, multiples);
230 matMulB = tile.getOutput();
231 }
232
233 auto matmul = tosa::MatMulOp::create(builder&: rewriter, location: loc, output: op.getType(), a: op.getA(),
234 b: matMulB, a_zp: op.getAZp(), b_zp: op.getBZp());
235 rewriter.replaceOp(op, newValues: matmul.getOutput());
236 return success();
237 }
238};
239
240struct TosaDowngrade1p1To1p0Pass
241 : public tosa::impl::TosaDowngrade1p1To1p0PassBase<
242 TosaDowngrade1p1To1p0Pass> {
243 using Base::Base;
244
245 void runOnOperation() override {
246 MLIRContext &context = getContext();
247 func::FuncOp func = getOperation();
248
249 RewritePatternSet patterns(&context);
250 patterns.add<BoolFp32CastRewrite, BoolGatherRewrite, BoolScatterRewrite,
251 MatMulTRewrite>(arg: &context);
252 FrozenRewritePatternSet frozenPatterns(std::move(patterns));
253
254 if (failed(Result: applyPatternsGreedily(op: func, patterns: frozenPatterns)))
255 return signalPassFailure();
256 }
257};
258
259} // namespace
260