1//===- OuterProductFusion.cpp - Fuse 'arm_sme.outerproduct' ops -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements rewrites that fuse 'arm_sme.outerproduct' operations
10// into the 2-way or 4-way widening outerproduct operations.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/ArmSME/IR/ArmSME.h"
15#include "mlir/Dialect/ArmSME/Transforms/Passes.h"
16#include "mlir/Dialect/ArmSME/Transforms/Transforms.h"
17#include "mlir/Dialect/Func/IR/FuncOps.h"
18#include "mlir/IR/PatternMatch.h"
19#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
20#include "llvm/ADT/TypeSwitch.h"
21
22#define DEBUG_TYPE "arm-sme-outerproduct-fusion"
23
24namespace mlir::arm_sme {
25#define GEN_PASS_DEF_OUTERPRODUCTFUSION
26#include "mlir/Dialect/ArmSME/Transforms/Passes.h.inc"
27} // namespace mlir::arm_sme
28
29using namespace mlir;
30using namespace mlir::arm_sme;
31
32namespace {
33
34// Common match failure reasons.
35static constexpr StringLiteral
36 kMatchFailureNoAccumulator("no accumulator operand");
37static constexpr StringLiteral kMatchFailureExpectedOuterProductDefOp(
38 "defining op of accumulator must be 'arm_sme.outerproduct'");
39static constexpr StringLiteral kMatchFailureInconsistentCombiningKind(
40 "combining kind (add or sub) of outer products must match");
41static constexpr StringLiteral kMatchFailureInconsistentMasking(
42 "unsupported masking, either both outerproducts are masked "
43 "or neither");
44static constexpr StringLiteral kMatchFailureOuterProductNotSingleUse(
45 "outer product(s) not single use and cannot be removed, no benefit to "
46 "fusing");
47
48// An outer product is compatible if all of the following are true:
49// - the result type matches `resultType`.
50// - the defining operation of LHS is of the type `LhsExtOp`.
51// - the defining operation of RHS is of the type `RhsExtOp`.
52// - the input types of the defining operations are identical and match
53// `inputType`.
54template <typename LhsExtOp, typename RhsExtOp = LhsExtOp>
55static LogicalResult isCompatible(PatternRewriter &rewriter,
56 arm_sme::OuterProductOp op,
57 VectorType resultType, VectorType inputType) {
58 if (op.getResultType() != resultType)
59 return rewriter.notifyMatchFailure(op.getLoc(), [&](Diagnostic &diag) {
60 diag << "unsupported result type, expected " << resultType;
61 });
62
63 auto lhsDefOp = op.getLhs().getDefiningOp<LhsExtOp>();
64 auto rhsDefOp = op.getRhs().getDefiningOp<RhsExtOp>();
65
66 if (!lhsDefOp || !rhsDefOp)
67 return rewriter.notifyMatchFailure(
68 arg&: op, msg: "defining op of outerproduct operands must be one of: "
69 "'arith.extf' or 'arith.extsi' or 'arith.extui'");
70
71 auto lhsInType = cast<VectorType>(lhsDefOp.getIn().getType());
72 auto rhsInType = cast<VectorType>(rhsDefOp.getIn().getType());
73
74 if (lhsInType != inputType || rhsInType != inputType)
75 return rewriter.notifyMatchFailure(op.getLoc(), [&](Diagnostic &diag) {
76 diag << "unsupported input type, expected " << inputType;
77 });
78
79 return success();
80}
81
82// Fuse two 'arm_sme.outerproduct' operations that are chained via the
83// accumulator into 2-way outer product operation.
84//
85// For example:
86//
87// %a0_ext = arith.extf %a0 : vector<[4]xf16> to vector<[4]xf32>
88// %b0_ext = arith.extf %b0 : vector<[4]xf16> to vector<[4]xf32>
89// %0 = arm_sme.outerproduct %a0_ext, %b0_ext : vector<[4]xf32>,
90// vector<[4]xf32>
91//
92// %a1_ext = arith.extf %a1 : vector<[4]xf16> to vector<[4]xf32>
93// %b1_ext = arith.extf %b1 : vector<[4]xf16> to vector<[4]xf32>
94// %1 = arm_sme.outerproduct %a1_ext, %b1_ext, %0 : vector<[4]xf32>,
95// vector<[4]xf32>
96//
97// Becomes:
98//
99// %a_packed = vector.interleave %a0, %a1 : vector<[4]xf16> -> vector<[8]xf16>
100// %b_packed = vector.interleave %b0, %b1 : vector<[4]xf16> -> vector<[8]xf16>
101// %0 = arm_sme.fmopa_2way %a_packed, %b_packed
102// : vector<[8]xf16>, vector<[8]xf16> into vector<[4]x[4]xf32>
103class OuterProductFusion2Way
104 : public OpRewritePattern<arm_sme::OuterProductOp> {
105public:
106 using Base::Base;
107
108 LogicalResult matchAndRewrite(arm_sme::OuterProductOp op,
109 PatternRewriter &rewriter) const override {
110 Value acc = op.getAcc();
111 if (!acc)
112 return rewriter.notifyMatchFailure(arg&: op, msg: kMatchFailureNoAccumulator);
113
114 arm_sme::OuterProductOp op1 = acc.getDefiningOp<arm_sme::OuterProductOp>();
115 arm_sme::OuterProductOp op2 = op;
116 if (!op1)
117 return rewriter.notifyMatchFailure(
118 arg&: op, msg: kMatchFailureExpectedOuterProductDefOp);
119
120 if (op1.getKind() != op2.getKind())
121 return rewriter.notifyMatchFailure(
122 arg&: op, msg: kMatchFailureInconsistentCombiningKind);
123
124 if (!op1->hasOneUse()) {
125 // If the first outer product has uses other than as the input to another
126 // outer product, it can't be erased after fusion.
127 return rewriter.notifyMatchFailure(arg&: op,
128 msg: kMatchFailureOuterProductNotSingleUse);
129 }
130
131 if (bool(op1.getLhsMask()) != bool(op2.getLhsMask()))
132 return rewriter.notifyMatchFailure(arg&: op, msg: kMatchFailureInconsistentMasking);
133
134 if (failed(Result: canFuseOuterProducts(rewriter, op1, op2)))
135 return failure();
136
137 auto loc = op.getLoc();
138 auto packInputs = [&](Value lhs, Value rhs) {
139 return vector::InterleaveOp::create(builder&: rewriter, location: loc, lhs, rhs);
140 };
141
142 auto lhs = packInputs(op1.getLhs().getDefiningOp()->getOperand(idx: 0),
143 op2.getLhs().getDefiningOp()->getOperand(idx: 0));
144 auto rhs = packInputs(op1.getRhs().getDefiningOp()->getOperand(idx: 0),
145 op2.getRhs().getDefiningOp()->getOperand(idx: 0));
146
147 Value lhsMask, rhsMask;
148 if (op1.getLhsMask() || op2.getLhsMask()) {
149 lhsMask = packInputs(op1.getLhsMask(), op2.getLhsMask());
150 rhsMask = packInputs(op1.getRhsMask(), op2.getRhsMask());
151 }
152
153 auto *extOp = op.getLhs().getDefiningOp();
154
155 arm_sme::CombiningKind kind = op.getKind();
156 if (kind == arm_sme::CombiningKind::Add) {
157 TypeSwitch<Operation *>(extOp)
158 .Case(caseFn: [&](arith::ExtFOp) {
159 rewriter.replaceOpWithNewOp<arm_sme::FMopa2WayOp>(
160 op: op2, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask,
161 args: op1.getAcc());
162 })
163 .Case(caseFn: [&](arith::ExtSIOp) {
164 rewriter.replaceOpWithNewOp<arm_sme::SMopa2WayOp>(
165 op: op2, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask,
166 args: op1.getAcc());
167 })
168 .Case(caseFn: [&](arith::ExtUIOp) {
169 rewriter.replaceOpWithNewOp<arm_sme::UMopa2WayOp>(
170 op: op2, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask,
171 args: op1.getAcc());
172 })
173 .DefaultUnreachable(message: "unexpected extend op!");
174 } else if (kind == arm_sme::CombiningKind::Sub) {
175 TypeSwitch<Operation *>(extOp)
176 .Case(caseFn: [&](arith::ExtFOp) {
177 rewriter.replaceOpWithNewOp<arm_sme::FMops2WayOp>(
178 op: op2, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask,
179 args: op1.getAcc());
180 })
181 .Case(caseFn: [&](arith::ExtSIOp) {
182 rewriter.replaceOpWithNewOp<arm_sme::SMops2WayOp>(
183 op: op2, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask,
184 args: op1.getAcc());
185 })
186 .Case(caseFn: [&](arith::ExtUIOp) {
187 rewriter.replaceOpWithNewOp<arm_sme::UMops2WayOp>(
188 op: op2, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask,
189 args: op1.getAcc());
190 })
191 .DefaultUnreachable(message: "unexpected extend op!");
192 } else {
193 llvm_unreachable("unexpected arm_sme::CombiningKind!");
194 }
195
196 return success();
197 }
198
199private:
200 // A pair of outer product can be fused if all of the following are true:
201 // - input and result types match.
202 // - the defining operations of the inputs are identical extensions,
203 // specifically either:
204 // - a signed or unsigned extension for integer types.
205 // - a floating-point extension for floating-point types.
206 // - the types and extension are supported, i.e. there's a 2-way operation
207 // they can be fused into.
208 LogicalResult canFuseOuterProducts(PatternRewriter &rewriter,
209 arm_sme::OuterProductOp op1,
210 arm_sme::OuterProductOp op2) const {
211 // Supported result types.
212 auto nxnxv4i32 =
213 VectorType::get(shape: {4, 4}, elementType: rewriter.getI32Type(), scalableDims: {true, true});
214 auto nxnxv4f32 =
215 VectorType::get(shape: {4, 4}, elementType: rewriter.getF32Type(), scalableDims: {true, true});
216 // Supported input types.
217 // Note: this is before packing so these have half the number of elements
218 // of the input vector types of the 2-way operations.
219 auto nxv4i16 = VectorType::get(shape: {4}, elementType: rewriter.getI16Type(), scalableDims: true);
220 auto nxv4f16 = VectorType::get(shape: {4}, elementType: rewriter.getF16Type(), scalableDims: true);
221 auto nxv4bf16 = VectorType::get(shape: {4}, elementType: rewriter.getBF16Type(), scalableDims: true);
222 const bool isF16Compatible =
223 succeeded(
224 Result: isCompatible<arith::ExtFOp>(rewriter, op: op1, resultType: nxnxv4f32, inputType: nxv4f16)) &&
225 succeeded(
226 Result: isCompatible<arith::ExtFOp>(rewriter, op: op2, resultType: nxnxv4f32, inputType: nxv4f16));
227 const bool isBF16Compatible =
228 succeeded(
229 Result: isCompatible<arith::ExtFOp>(rewriter, op: op1, resultType: nxnxv4f32, inputType: nxv4bf16)) &&
230 succeeded(
231 Result: isCompatible<arith::ExtFOp>(rewriter, op: op2, resultType: nxnxv4f32, inputType: nxv4bf16));
232 const bool isI16Compatible =
233 succeeded(
234 Result: isCompatible<arith::ExtSIOp>(rewriter, op: op1, resultType: nxnxv4i32, inputType: nxv4i16)) &&
235 succeeded(
236 Result: isCompatible<arith::ExtSIOp>(rewriter, op: op2, resultType: nxnxv4i32, inputType: nxv4i16));
237 const bool isUI16Compatible =
238 succeeded(
239 Result: isCompatible<arith::ExtUIOp>(rewriter, op: op1, resultType: nxnxv4i32, inputType: nxv4i16)) &&
240 succeeded(
241 Result: isCompatible<arith::ExtUIOp>(rewriter, op: op2, resultType: nxnxv4i32, inputType: nxv4i16));
242
243 return (isF16Compatible || isBF16Compatible || isI16Compatible ||
244 isUI16Compatible)
245 ? success()
246 : failure();
247 }
248};
249
250// Fuse four 'arm_sme.outerproduct' operations that are chained via the
251// accumulator into 4-way outer product operation.
252class OuterProductFusion4Way
253 : public OpRewritePattern<arm_sme::OuterProductOp> {
254public:
255 using Base::Base;
256
257 LogicalResult matchAndRewrite(arm_sme::OuterProductOp op,
258 PatternRewriter &rewriter) const override {
259 SmallVector<arm_sme::OuterProductOp, 4> outerProductChain;
260 outerProductChain.push_back(Elt: op);
261
262 for (int i = 0; i < 3; ++i) {
263 auto currentOp = outerProductChain.back();
264 auto acc = currentOp.getAcc();
265 if (!acc)
266 return rewriter.notifyMatchFailure(arg&: op, msg: kMatchFailureNoAccumulator);
267 auto previousOp = acc.getDefiningOp<arm_sme::OuterProductOp>();
268 if (!previousOp)
269 return rewriter.notifyMatchFailure(
270 arg&: op, msg: kMatchFailureExpectedOuterProductDefOp);
271 if (!previousOp->hasOneUse())
272 return rewriter.notifyMatchFailure(
273 arg&: op, msg: kMatchFailureOuterProductNotSingleUse);
274 if (previousOp.getKind() != currentOp.getKind())
275 return rewriter.notifyMatchFailure(
276 arg&: op, msg: kMatchFailureInconsistentCombiningKind);
277 if (bool(previousOp.getLhsMask()) != bool(currentOp.getLhsMask()))
278 return rewriter.notifyMatchFailure(
279 arg&: op, msg: kMatchFailureInconsistentCombiningKind);
280 outerProductChain.push_back(Elt: previousOp);
281 }
282
283 if (failed(Result: canFuseOuterProducts(rewriter, ops: outerProductChain)))
284 return failure();
285
286 arm_sme::OuterProductOp op1 = outerProductChain[3];
287 arm_sme::OuterProductOp op2 = outerProductChain[2];
288 arm_sme::OuterProductOp op3 = outerProductChain[1];
289 arm_sme::OuterProductOp op4 = outerProductChain[0];
290
291 auto loc = op.getLoc();
292 auto packInputs = [&](Value lhs, Value rhs) {
293 return vector::InterleaveOp::create(builder&: rewriter, location: loc, lhs, rhs);
294 };
295
296 auto lhs0 = packInputs(op1.getLhs().getDefiningOp()->getOperand(idx: 0),
297 op3.getLhs().getDefiningOp()->getOperand(idx: 0));
298 auto lhs1 = packInputs(op2.getLhs().getDefiningOp()->getOperand(idx: 0),
299 op4.getLhs().getDefiningOp()->getOperand(idx: 0));
300 auto lhs = packInputs(lhs0, lhs1);
301
302 auto rhs0 = packInputs(op1.getRhs().getDefiningOp()->getOperand(idx: 0),
303 op3.getRhs().getDefiningOp()->getOperand(idx: 0));
304 auto rhs1 = packInputs(op2.getRhs().getDefiningOp()->getOperand(idx: 0),
305 op4.getRhs().getDefiningOp()->getOperand(idx: 0));
306 auto rhs = packInputs(rhs0, rhs1);
307
308 Value lhsMask, rhsMask;
309 if (op1.getLhsMask() || op2.getLhsMask() || op3.getLhsMask() ||
310 op4.getLhsMask()) {
311 auto lhs0Mask = packInputs(op1.getLhsMask(), op3.getLhsMask());
312 auto lhs1Mask = packInputs(op2.getLhsMask(), op4.getLhsMask());
313 lhsMask = packInputs(lhs0Mask, lhs1Mask);
314
315 auto rhs0Mask = packInputs(op1.getRhsMask(), op3.getRhsMask());
316 auto rhs1Mask = packInputs(op2.getRhsMask(), op4.getRhsMask());
317 rhsMask = packInputs(rhs0Mask, rhs1Mask);
318 }
319
320 auto *lhsExtOp = op.getLhs().getDefiningOp();
321 auto *rhsExtOp = op.getRhs().getDefiningOp();
322
323 arm_sme::CombiningKind kind = op.getKind();
324 if (kind == arm_sme::CombiningKind::Add) {
325 if (isa<arith::ExtSIOp>(Val: lhsExtOp) && isa<arith::ExtSIOp>(Val: rhsExtOp)) {
326 // signed
327 rewriter.replaceOpWithNewOp<arm_sme::SMopa4WayOp>(
328 op: op4, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask, args: op1.getAcc());
329 } else if (isa<arith::ExtUIOp>(Val: lhsExtOp) &&
330 isa<arith::ExtUIOp>(Val: rhsExtOp)) {
331 // unsigned
332 rewriter.replaceOpWithNewOp<arm_sme::UMopa4WayOp>(
333 op: op4, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask, args: op1.getAcc());
334 } else if (isa<arith::ExtSIOp>(Val: lhsExtOp) &&
335 isa<arith::ExtUIOp>(Val: rhsExtOp)) {
336 // signed by unsigned
337 rewriter.replaceOpWithNewOp<arm_sme::SuMopa4WayOp>(
338 op: op4, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask, args: op1.getAcc());
339 } else if (isa<arith::ExtUIOp>(Val: lhsExtOp) &&
340 isa<arith::ExtSIOp>(Val: rhsExtOp)) {
341 // unsigned by signed
342 rewriter.replaceOpWithNewOp<arm_sme::UsMopa4WayOp>(
343 op: op4, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask, args: op1.getAcc());
344 } else {
345 llvm_unreachable("unexpected extend op!");
346 }
347 } else if (kind == arm_sme::CombiningKind::Sub) {
348 if (isa<arith::ExtSIOp>(Val: lhsExtOp) && isa<arith::ExtSIOp>(Val: rhsExtOp)) {
349 // signed
350 rewriter.replaceOpWithNewOp<arm_sme::SMops4WayOp>(
351 op: op4, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask, args: op1.getAcc());
352 } else if (isa<arith::ExtUIOp>(Val: lhsExtOp) &&
353 isa<arith::ExtUIOp>(Val: rhsExtOp)) {
354 // unsigned
355 rewriter.replaceOpWithNewOp<arm_sme::UMops4WayOp>(
356 op: op4, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask, args: op1.getAcc());
357 } else if (isa<arith::ExtSIOp>(Val: lhsExtOp) &&
358 isa<arith::ExtUIOp>(Val: rhsExtOp)) {
359 // signed by unsigned
360 rewriter.replaceOpWithNewOp<arm_sme::SuMops4WayOp>(
361 op: op4, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask, args: op1.getAcc());
362 } else if (isa<arith::ExtUIOp>(Val: lhsExtOp) &&
363 isa<arith::ExtSIOp>(Val: rhsExtOp)) {
364 // unsigned by signed
365 rewriter.replaceOpWithNewOp<arm_sme::UsMops4WayOp>(
366 op: op4, args: op.getResultType(), args&: lhs, args&: rhs, args&: lhsMask, args&: rhsMask, args: op1.getAcc());
367 } else {
368 llvm_unreachable("unexpected extend op!");
369 }
370 } else {
371 llvm_unreachable("unexpected arm_sme::CombiningKind!");
372 }
373
374 return success();
375 }
376
377private:
378 // Four outer products can be fused if all of the following are true:
379 // - input and result types match.
380 // - the defining operations of the inputs are identical extensions,
381 // specifically either:
382 // - a signed or unsigned extension for integer types.
383 // - a floating-point extension for floating-point types.
384 // - the types and extension are supported, i.e. there's a 4-way operation
385 // they can be fused into.
386 LogicalResult
387 canFuseOuterProducts(PatternRewriter &rewriter,
388 ArrayRef<arm_sme::OuterProductOp> ops) const {
389 // Supported result types.
390 auto nxnxv4i32 =
391 VectorType::get(shape: {4, 4}, elementType: rewriter.getI32Type(), scalableDims: {true, true});
392 auto nxnxv2i64 =
393 VectorType::get(shape: {2, 2}, elementType: rewriter.getI64Type(), scalableDims: {true, true});
394
395 // Supported input types.
396 // Note: this is before packing so these have 1/4 the number of elements
397 // of the input vector types of the 4-way operations.
398 auto nxv4i8 = VectorType::get(shape: {4}, elementType: rewriter.getI8Type(), scalableDims: true);
399 auto nxv2i16 = VectorType::get(shape: {2}, elementType: rewriter.getI16Type(), scalableDims: true);
400
401 auto failedToMatch = [&](VectorType resultType, VectorType inputType,
402 auto lhsExtendOp, auto rhsExtendOp) {
403 using LhsExtendOpTy = decltype(lhsExtendOp);
404 using RhsExtendOpTy = decltype(rhsExtendOp);
405 for (auto op : ops) {
406 if (failed(isCompatible<LhsExtendOpTy, RhsExtendOpTy>(
407 rewriter, op, resultType, inputType)))
408 return true;
409 }
410 return false;
411 };
412
413 if (failedToMatch(nxnxv4i32, nxv4i8, arith::ExtSIOp{}, arith::ExtSIOp{}) &&
414 failedToMatch(nxnxv4i32, nxv4i8, arith::ExtUIOp{}, arith::ExtUIOp{}) &&
415 failedToMatch(nxnxv4i32, nxv4i8, arith::ExtSIOp{}, arith::ExtUIOp{}) &&
416 failedToMatch(nxnxv4i32, nxv4i8, arith::ExtUIOp{}, arith::ExtSIOp{}) &&
417 failedToMatch(nxnxv2i64, nxv2i16, arith::ExtSIOp{}, arith::ExtSIOp{}) &&
418 failedToMatch(nxnxv2i64, nxv2i16, arith::ExtUIOp{}, arith::ExtUIOp{}) &&
419 failedToMatch(nxnxv2i64, nxv2i16, arith::ExtSIOp{}, arith::ExtUIOp{}) &&
420 failedToMatch(nxnxv2i64, nxv2i16, arith::ExtUIOp{}, arith::ExtSIOp{}))
421 return failure();
422
423 return success();
424 }
425};
426
427// Rewrites: vector.extract(arith.extend) -> arith.extend(vector.extract).
428//
429// This transforms IR like:
430// %0 = arith.extsi %src : vector<4x[8]xi8> to vector<4x[8]xi32>
431// %1 = vector.extract %0[0] : vector<[8]xi32> from vector<4x[8]xi32>
432// Into:
433// %0 = vector.extract %src[0] : vector<[8]xi8> from vector<4x[8]xi8>
434// %1 = arith.extsi %0 : vector<[8]xi8> to vector<[8]xi32>
435//
436// This enables outer product fusion in the `-arm-sme-outer-product-fusion`
437// pass when the result is the input to an outer product.
438struct SwapVectorExtractOfArithExtend
439 : public OpRewritePattern<vector::ExtractOp> {
440 using Base::Base;
441
442 LogicalResult matchAndRewrite(vector::ExtractOp extractOp,
443 PatternRewriter &rewriter) const override {
444 VectorType resultType = llvm::dyn_cast<VectorType>(Val: extractOp.getType());
445 if (!resultType)
446 return rewriter.notifyMatchFailure(arg&: extractOp,
447 msg: "extracted type is not a vector type");
448
449 auto numScalableDims = resultType.getNumScalableDims();
450 if (numScalableDims != 1)
451 return rewriter.notifyMatchFailure(
452 arg&: extractOp, msg: "extracted type is not a 1-D scalable vector type");
453
454 auto *extendOp = extractOp.getSource().getDefiningOp();
455 if (!isa_and_present<arith::ExtSIOp, arith::ExtUIOp, arith::ExtFOp>(
456 Val: extendOp))
457 return rewriter.notifyMatchFailure(arg&: extractOp,
458 msg: "extract not from extend op");
459
460 auto loc = extractOp.getLoc();
461 StringAttr extendOpName = extendOp->getName().getIdentifier();
462 Value extendSource = extendOp->getOperand(idx: 0);
463
464 // Create new extract from source of extend.
465 Value newExtract = vector::ExtractOp::create(builder&: rewriter, location: loc, source: extendSource,
466 position: extractOp.getMixedPosition());
467
468 // Extend new extract to original result type.
469 Operation *newExtend =
470 rewriter.create(loc, opName: extendOpName, operands: Value(newExtract), types: resultType);
471
472 rewriter.replaceOp(op: extractOp, newOp: newExtend);
473
474 return success();
475 }
476};
477
478// Same as above, but for vector.scalable.extract.
479//
480// This transforms IR like:
481// %0 = arith.extsi %src : vector<[8]xi8> to vector<[8]xi32>
482// %1 = vector.scalable.extract %0[0] : vector<[4]xi32> from vector<[8]xi32>
483// Into:
484// %0 = vector.scalable.extract %src[0] : vector<[4]xi8> from vector<[8]xi8>
485// %1 = arith.extsi %0 : vector<[4]xi8> to vector<[4]xi32>
486//
487// This enables outer product fusion in the `-arm-sme-outer-product-fusion`
488// pass when the result is the input to an outer product.
489struct SwapVectorScalableExtractOfArithExtend
490 : public OpRewritePattern<vector::ScalableExtractOp> {
491 using Base::Base;
492
493 LogicalResult matchAndRewrite(vector::ScalableExtractOp extractOp,
494 PatternRewriter &rewriter) const override {
495 auto *extendOp = extractOp.getSource().getDefiningOp();
496 if (!isa_and_present<arith::ExtSIOp, arith::ExtUIOp, arith::ExtFOp>(
497 Val: extendOp))
498 return rewriter.notifyMatchFailure(arg&: extractOp,
499 msg: "extract not from extend op");
500
501 auto loc = extractOp.getLoc();
502 VectorType resultType = extractOp.getResultVectorType();
503
504 Value extendSource = extendOp->getOperand(idx: 0);
505 StringAttr extendOpName = extendOp->getName().getIdentifier();
506 VectorType extendSourceVectorType =
507 cast<VectorType>(Val: extendSource.getType());
508
509 // Create new extract from source of extend.
510 VectorType extractResultVectorType =
511 resultType.clone(elementType: extendSourceVectorType.getElementType());
512 Value newExtract = vector::ScalableExtractOp::create(
513 builder&: rewriter, location: loc, result: extractResultVectorType, source: extendSource,
514 pos: extractOp.getPos());
515
516 // Extend new extract to original result type.
517 Operation *newExtend =
518 rewriter.create(loc, opName: extendOpName, operands: Value(newExtract), types: resultType);
519
520 rewriter.replaceOp(op: extractOp, newOp: newExtend);
521
522 return success();
523 }
524};
525
526struct OuterProductFusionPass
527 : public arm_sme::impl::OuterProductFusionBase<OuterProductFusionPass> {
528
529 void runOnOperation() override {
530 RewritePatternSet patterns(&getContext());
531 populateOuterProductFusionPatterns(patterns);
532
533 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns))))
534 signalPassFailure();
535 }
536};
537
538} // namespace
539
540void mlir::arm_sme::populateOuterProductFusionPatterns(
541 RewritePatternSet &patterns) {
542 MLIRContext *context = patterns.getContext();
543 // Note: High benefit to ensure extract(extend) are swapped first.
544 patterns.add<SwapVectorExtractOfArithExtend,
545 SwapVectorScalableExtractOfArithExtend>(arg&: context, args: 1024);
546 patterns.add<OuterProductFusion2Way, OuterProductFusion4Way>(arg&: context);
547}
548
549std::unique_ptr<Pass> mlir::arm_sme::createOuterProductFusionPass() {
550 return std::make_unique<OuterProductFusionPass>();
551}
552