1//===- LowerContractToSVEPatterns.cpp - Contract to I8MM/BF16 ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements lowering patterns from vector.contract to operations
10// that map to instructions from the SVE ISA, including matrix-multiplication
11// extensions: FEAT_I8MM and FEAT_BF16.
12//
13// TODO: There may be opportunities to unify this with a similar pattern
14// for Neon. See:
15// https://github.com/llvm/llvm-project/issues/145559
16// LowerContractToNeonPatterns.cpp
17//
18//===----------------------------------------------------------------------===//
19
20#include "mlir/Dialect/Arith/IR/Arith.h"
21#include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
22#include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
23#include "mlir/Dialect/Func/IR/FuncOps.h"
24#include "mlir/Dialect/LLVMIR/LLVMVectorIntrinsicOps.h"
25#include "mlir/Dialect/UB/IR/UBOps.h"
26#include "mlir/Dialect/Vector/IR/VectorOps.h"
27#include "mlir/IR/AffineMap.h"
28#include "mlir/IR/PatternMatch.h"
29
30#include <cassert>
31#include <numeric>
32
33#define DEBUG_TYPE "lower-contract-to-arm-sve-i8mm"
34
35using namespace mlir;
36using namespace mlir::arm_sve;
37
38namespace {
39// Get the operand of a `vector.contract`. This function is intended to abstract
40// away from the particular way a value is extended before feeding it into the
41// `vector.contract` - via zero-extend or an explicit or implicit sign-extend
42// (for implicit sign-extension see `vector.contract` documentation).
43//
44// The template parameter `Op` indicates the extension operation (explicit or
45// implicit) for which we are checking.
46//
47// Return success only for extensions from `i8` to `i32`.
48template <typename Op>
49std::optional<Value> getExtOperand(Value v) {
50
51 static_assert(llvm::is_one_of<Op, arith::ExtSIOp, arith::ExtUIOp>::value,
52 "Must be instantiated with either sign- or zero- extension op");
53
54 // If the operand is not defined by an explicit extend operation of the
55 // accepted operation type allow for an implicit sign-extension.
56 auto extOp = v.getDefiningOp<Op>();
57 if (!extOp) {
58 if constexpr (std::is_same<Op, arith::ExtSIOp>::value) {
59 auto vTy = cast<VectorType>(Val: v.getType());
60 if (!vTy.getElementType().isSignlessInteger(width: 8))
61 return {};
62 return v;
63 }
64 return {};
65 }
66
67 // If the operand is defined by an explicit extend operation of the accepted
68 // operation type, check it's extended from `i8` to `i32`.
69 auto inOp = extOp.getIn();
70 auto inTy = dyn_cast<VectorType>(inOp.getType());
71 if (!inTy || !inTy.getElementType().isSignlessInteger(8))
72 return {};
73
74 auto outTy = dyn_cast<VectorType>(extOp.getType());
75 if (!outTy || !outTy.getElementType().isSignlessInteger(32))
76 return {};
77
78 return inOp;
79}
80
81/// This class encapsulates the algorithm and parametrisation (in terms of types
82/// and dimensions) of lowering a `vector.contract` to "primitive" matrix
83/// multiplication operations of the SVE dialect (here "primitive" would mean
84/// corresponding to a single target instruction).
85///
86/// Supported are lowering to FEAT_I8MM `smmla`, `ummla`, and `usmmla`, and to
87/// FEAT_BF16 `bfmmla`. All the transformations are very similar to each other
88/// for concreteness the description below is given for `smmla`.
89///
90/// The lowering triggers for a contraction operation that performs a matrix
91/// multiply of two 8-bit integer matrix tiles with logical dimensions
92/// <Mx8> and <8x[N]> for the left-hand side (LHS) and the right-hand side
93/// (RHS), respectively, added to a 32-bit integer accumulator operand (ACC)
94/// with dimensions <Mx[N]>, yielding a <Mx[N]> 32-bit integer result (OUT).
95///
96/// The operands' shapes are such that the operands can be evenly split into
97/// sub-tiles with dimensions as expected by the targeted FEAT_I8MM
98/// instructions. The intent is that M and N are chosen (by higher level
99/// transforms) in such a way as to maximise register usage. The main use case
100/// we envision as of now is MMT4D, thus the RHS operand is expected
101/// pre-transposed.
102///
103/// The matrix multiplication is performed by unrolling the usual tiled matrix
104/// multiplication algorithm using sub-tiles with dimensions <2x8> for the
105/// LHS, <8x[2]> for the RHS, and <2x[2]> for the result and the input
106/// accumulator.
107///
108/// One way to illustrate the operation is as follows:
109///
110/// RHS<8x[N]>: <8x[2]> <8x[2]> ... <8x[2]>
111/// +-----------------------------
112/// LHS<Mx8>: <2x8> | <2x[2]> <2x[2]> ... <2x[2]>
113/// <2x8> | <2x[2]> <2x[2]> ... <2x[2]>
114/// ... | ... ... ... ...
115/// <2x8> | <2x[2]> <2x[2]> ... <2x[2]>
116///
117/// The RHS operand is unpacked into N/2 values, each representing a sequence
118/// of VSCALE number of sub-tiles with dimensions <8x2>.
119/// The LHS operand is initially unpacked into M/2 values, each representing a
120/// sub-tile with dimensions <2x8>, and then each such sub-tile is replicated
121/// VSCALE times. Multiplying thus replicated LHS sub-tile by the corresponding
122/// RHS sub-tile correctly computes an entire result sub-tile.
123/// The 2x2 sub-tiles of the ACC and OUT have rows that are not adjacent
124/// (in memory or when imposing a row-major layout on the 2D vector value).
125/// Reading the ACC is implemented as reading two consecutive rows and
126/// interleaving the by pairs to obtain a vector having length twice the length
127/// of an ACC row. This vector now is a sequence of one-dimensional tiles with
128/// the exact layout needed by the `smmla`/`bfmmla`/etc instructions, which
129/// tiles are extracted one by one. For illustration, if we have an 2x4 ACC tile
130/// a0 a1 b0 b1
131/// a2 a3 b2 b3
132/// we read the two rows as separate values and then interleave by pairs
133/// to obtain
134/// a0 a1 a2 a3 b0 b1 b2 b3
135/// from which we extract `a0 a1 a2 a3` and `b0 b1 b2 b3`.
136///
137/// Writing the OUT tile is done by the reverse of the above procedure,
138/// concatenate two "flattened" sub-tiles into
139/// c0 c1 c2 c3 d0 d1 d2 d3
140/// deinterleave by pairs to obtain as separate values
141/// c0 c1 d0 d1
142/// c2 c3 d2 d3
143/// which are then inserted into the final result.
144///
145/// Multiplication of a signed LHS by an unsigned LHS is performed by
146/// swapping the order of the operands and emitting an `usmmla` (since there
147/// isn't an `summla` instruction). Therefore each ACC sub-tile needs
148/// to be transposed before the addition and the sum, an OUT sub-tile,
149/// needs to be transposed before insertion into the final result.
150/// This is done very elegantly by a modification of the above to
151/// interleave/deinterleave not by pairs, but by individual elements, e.g.
152/// after ordinary interleave we obtain
153/// a0 a2 a1 a3 b0 b2 b1 b3
154/// which is exactly the desired layout of having each individual 2x2 tile
155/// transposed.
156///
157/// All of the above readily applies to FEAT_BF16 `bfmmla` with the
158/// difference that the shapes of the LHS, RHS are <Mx4>, <4x[M]>, and
159/// respectively, that is the "K" dimension is fixed to 4, instead of 8 (like
160/// for the integer case).
161class VectorContractRewriter {
162protected:
163 // Designate the operation (resp. instruction) used to do sub-tile matrix
164 // multiplications.
165 enum class MMLA {
166 Nop,
167 SignedInt, // smmla
168 UnsignedInt, // ummla
169 MixedInt, // usmmla
170 Bfloat // bfmmla
171 };
172
173 // Lower-level operation to be emitted.
174 MMLA mmlaOp = MMLA::Nop;
175
176 // Indicate if the operands for the ArmSVE dialect operation need to be
177 // swapped. Currently this is needed in order to emulate an "summla"
178 // operation.
179 bool swapOperands = false;
180
181 // The operand tiles. These are not necessarily the operends of
182 // `vector.contract`, for example they could be operands to `arith.extsi`
183 // that is in turn fed into `vector.contract`.
184 Value lhs;
185 Value rhs;
186 Value acc;
187
188 // Conventional names for matrix dimensions.
189 int64_t m = 0;
190 int64_t n = 0;
191 int64_t k = 0;
192
193 // Create the matrix mulitply and accumulate operation according to
194 // `mmlaOp`.
195 Value createMMLA(PatternRewriter &rewriter, Location loc, Value acc,
196 Value lhs, Value rhs);
197
198 // Check general preconditions for applying the transformation, common to the
199 // integer and the bfloat16 case.
200 LogicalResult match(vector::ContractionOp op, PatternRewriter &rewriter);
201
202public:
203 VectorContractRewriter() = default;
204
205 // Do the actuall rewrite. This member function is shared by both integer and
206 // bfloat16 rewrites.
207 Value lower(vector::ContractionOp op, PatternRewriter &rewriter);
208};
209
210Value VectorContractRewriter::createMMLA(PatternRewriter &rewriter,
211 Location loc, Value acc, Value lhs,
212 Value rhs) {
213
214 Type resTy = acc.getType();
215 if (swapOperands)
216 std::swap(a&: lhs, b&: rhs);
217
218 switch (mmlaOp) {
219 case MMLA::SignedInt:
220 return arm_sve::SmmlaOp::create(builder&: rewriter, location: loc, dst: resTy, acc, src1: lhs, src2: rhs);
221 case MMLA::UnsignedInt:
222 return arm_sve::UmmlaOp::create(builder&: rewriter, location: loc, dst: resTy, acc, src1: lhs, src2: rhs);
223 case MMLA::MixedInt:
224 return arm_sve::UsmmlaOp::create(builder&: rewriter, location: loc, dst: resTy, acc, src1: lhs, src2: rhs);
225 case MMLA::Bfloat:
226 return arm_sve::BfmmlaOp::create(builder&: rewriter, location: loc, res: resTy, acc, src1: lhs, src2: rhs);
227 default:
228 llvm_unreachable("Uninitialized operation kind");
229 }
230}
231
232LogicalResult VectorContractRewriter::match(vector::ContractionOp op,
233 PatternRewriter &rewriter) {
234 // Check iterator types for matrix multiplication.
235 auto itTypes = op.getIteratorTypesArray();
236 if (itTypes.size() != 3 || itTypes[0] != vector::IteratorType::parallel ||
237 itTypes[1] != vector::IteratorType::parallel ||
238 itTypes[2] != vector::IteratorType::reduction)
239 return rewriter.notifyMatchFailure(
240 arg&: op, msg: "iterator types do not correspond to matrix multiplication");
241
242 // Check permutation maps. For now only accept
243 // lhs: (d0, d1, d2) -> (d0, d2)
244 // rhs: (d0, d1, d2) -> (d1, d2)
245 // acc: (d0, d1, d2) -> (d0, d1)
246 // This corresponds to matrix multiplication with transposed RHS.
247 if (op.getIndexingMapsArray()[0] !=
248 AffineMap::getMultiDimMapWithTargets(numDims: 3, targets: ArrayRef{0u, 2u},
249 context: op.getContext()) ||
250 op.getIndexingMapsArray()[1] !=
251 AffineMap::getMultiDimMapWithTargets(numDims: 3, targets: ArrayRef{1u, 2u},
252 context: op.getContext()) ||
253 op.getIndexingMapsArray()[2] != AffineMap::getMultiDimMapWithTargets(
254 numDims: 3, targets: ArrayRef{0u, 1u}, context: op.getContext()))
255 return rewriter.notifyMatchFailure(arg&: op, msg: "non-matching permutation maps");
256
257 // Check the combining kind is addition.
258 if (op.getKind() != vector::CombiningKind::ADD)
259 return rewriter.notifyMatchFailure(arg&: op, msg: "combining kind is not an addition");
260
261 return success();
262}
263
264Value VectorContractRewriter::lower(vector::ContractionOp op,
265 PatternRewriter &rewriter) {
266
267 // Initialize some helper types.
268 Type operandEltType = cast<VectorType>(Val: lhs.getType()).getElementType();
269 Type resultEltType = cast<VectorType>(Val: op.getResultType()).getElementType();
270
271 const int64_t numOperandSubTileElts =
272 128 / operandEltType.getIntOrFloatBitWidth();
273
274 assert(resultEltType.getIntOrFloatBitWidth() == 32 &&
275 "Only implemented for i32 or f32 output");
276 const int64_t numResultSubTileElts = 4;
277
278 // Single-dimensional vector types for the operands of the ArmSVE dialect
279 // op.
280 auto flatLhsType =
281 VectorType::get(/*shape=*/numOperandSubTileElts, elementType: operandEltType,
282 /*scalableDims=*/{true});
283 auto flatRhsType =
284 VectorType::get(/*shape=*/numOperandSubTileElts, elementType: operandEltType,
285 /*scalableDims=*/{true});
286 auto flatAccType =
287 VectorType::get(/*shape=*/numResultSubTileElts, elementType: resultEltType,
288 /*scalableDims=*/{true});
289
290 // Single-dimension vector type for the entire RHS tile.
291
292 auto flatRhsTileType = VectorType::get(/*shape=*/k * n, elementType: operandEltType,
293 /*scalableDims=*/{true});
294
295 // Vector type having the same number of elements as a row in the
296 // accumulator/output tile and the same element type.
297 auto accRowTy = VectorType::get(/*shape=*/n, elementType: resultEltType,
298 /*scalableDims=*/{true});
299
300 // Vector type having twice the number of elements as a row in the
301 // accumulator/output tile the same element type.
302 auto accRowX2Ty = VectorType::get(/*shape=*/2 * n, elementType: resultEltType,
303 /*scalableDims=*/{true});
304 // Vector type having half the number of elements as a row in the
305 // accumulator/output tile and an integer element type with twice the bit
306 // width.
307 auto accRow64Ty = VectorType::get(/*shape=*/n / 2, elementType: rewriter.getI64Type(),
308 /*scalableDims=*/{true});
309 // Vector type having the same the number of elements as a row in the
310 // accumulator/output tile and an integer element type with twice the bit
311 // width.
312 auto accRowX264Ty = VectorType::get(/*shape=*/n, elementType: rewriter.getI64Type(),
313 /*scalableDims=*/{true});
314
315 Location loc = op.getLoc();
316
317 // Extract LHS sub-tiles with logical shape <2xK>.
318 SmallVector<Value> lhsTile;
319 for (int64_t i = 0; i < m; i += 2) {
320 // Extract two consecutive rows of the LHS tile.
321 auto r0 =
322 vector::ExtractOp::create(builder&: rewriter, location: loc, source: lhs, position: ArrayRef<int64_t>{i});
323 auto r1 =
324 vector::ExtractOp::create(builder&: rewriter, location: loc, source: lhs, position: ArrayRef<int64_t>{i + 1});
325 // Concatenate to obtain a 2 x K x <input-type> flattened sub-tile.
326 SmallVector<int64_t> shuffleIdx(2 * k);
327 std::iota(first: shuffleIdx.begin(), last: shuffleIdx.end(), value: 0);
328 auto t = vector::ShuffleOp::create(builder&: rewriter, location: loc, v1: r0, v2: r1, mask: shuffleIdx);
329 // Turn it into a scalable vector.
330 auto s = vector::ScalableInsertOp::create(
331 builder&: rewriter, location: loc, valueToStore: t, dest: ub::PoisonOp::create(builder&: rewriter, location: loc, result: flatLhsType), pos: 0);
332 // Replicate the sub-tile VSCALE times to fill the entire vector.
333 auto r = arm_sve::DupQLaneOp::create(builder&: rewriter, location: loc, src: s, lane: 0);
334 lhsTile.push_back(Elt: r);
335 }
336
337 // "Flatten" the RHS tile from <[N]xK> to <[N*K]>.
338 auto rhs = vector::ShapeCastOp::create(builder&: rewriter, location: this->rhs.getLoc(),
339 result: flatRhsTileType, source: this->rhs);
340
341 // Extract the RHS sub-tiles with logical shape <Kx[2]>.
342 SmallVector<Value> rhsTile;
343 for (int64_t j = 0; j < n; j += 2)
344 rhsTile.push_back(Elt: vector::ScalableExtractOp::create(
345 builder&: rewriter, location: loc, result: flatRhsType, source: rhs, pos: j * k));
346
347 // Extract and pack the ACC sub-tiles.
348 SmallVector<Value> accTile;
349 for (int64_t i = 0; i < m; i += 2) {
350 // Extract two consecutive rows of the accumulator tile.
351 auto r0 = vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getAcc(),
352 position: ArrayRef<int64_t>{i});
353 auto r1 = vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getAcc(),
354 position: ArrayRef<int64_t>{i + 1});
355 Value accTileVec;
356 if (swapOperands) {
357 // We are performing the operation with swapped LHS and RHS we need to
358 // transpose each individual 2x2 tile of the accumulator and (later) the
359 // final result.
360 accTileVec = vector::InterleaveOp::create(builder&: rewriter, location: loc, lhs: r0, rhs: r1);
361 } else {
362 // Bitcast accumulator rows to double-width integer elements, so
363 // subsequent interleave/deinterleave work on pairs of elements.
364 auto r0I64 = vector::BitCastOp::create(builder&: rewriter, location: loc, result: accRow64Ty, source: r0);
365 auto r1I64 = vector::BitCastOp::create(builder&: rewriter, location: loc, result: accRow64Ty, source: r1);
366
367 // Interleave the rows, effectively flattening each 2x2 tile into 4
368 // consecutive elements.
369 auto intrI64 = vector::InterleaveOp::create(builder&: rewriter, location: loc, lhs: r0I64, rhs: r1I64);
370
371 // Bitcast back to original element type.
372 accTileVec =
373 vector::BitCastOp::create(builder&: rewriter, location: loc, result: accRowX2Ty, source: intrI64);
374 }
375 // Extract ACC sub-tiles.
376 for (int64_t j = 0; j < n; j += 2)
377 accTile.push_back(Elt: vector::ScalableExtractOp::create(
378 builder&: rewriter, location: loc, result: flatAccType, source: accTileVec, pos: j * 2));
379 }
380
381 // Emit sub-tile matrix multiplications.
382 SmallVector<Value> outTile;
383 for (int64_t i = 0; i < m / 2; ++i)
384 for (int64_t j = 0; j < n / 2; ++j) {
385 Value mmla = createMMLA(rewriter, loc, acc: accTile[i * n / 2 + j], lhs: lhsTile[i],
386 rhs: rhsTile[j]);
387 outTile.push_back(Elt: mmla);
388 }
389
390 // Unpack the OUT sub-tiles and insert into the result.
391 Value result = ub::PoisonOp::create(builder&: rewriter, location: loc, result: op.getResultType());
392 for (int64_t i = 0; i < m / 2; ++i) {
393 // Collect a number of sub-tiles in a row.
394 Value row = ub::PoisonOp::create(builder&: rewriter, location: loc, result: accRowX2Ty);
395 for (int64_t j = 0; j < n / 2; ++j)
396 row = vector::ScalableInsertOp::create(
397 builder&: rewriter, location: loc, valueToStore: outTile[i * n / 2 + j], dest: row, pos: j * 4);
398
399 // Unpack the row to obtain two rows of the output. If we have the out
400 // sub-tiles transposed we obtain two consecutive output rows by
401 // separating even and odd elements, i.e. a simple deinterleave.
402 // Otherwise, the interleave is by pairs.
403 Value out0, out1;
404 if (swapOperands) {
405 auto tmp = vector::DeinterleaveOp::create(builder&: rewriter, location: loc, source: row);
406 out0 = tmp.getResultEven();
407 out1 = tmp.getResultOdd();
408 } else {
409 // Deinterleave by pairs.
410 auto row64 = vector::BitCastOp::create(builder&: rewriter, location: loc, result: accRowX264Ty, source: row);
411 auto deintr64 = vector::DeinterleaveOp::create(builder&: rewriter, location: loc, source: row64);
412
413 // Bitcast back into original element type and insert into the result.
414 out0 = vector::BitCastOp::create(builder&: rewriter, location: loc, result: accRowTy,
415 source: deintr64.getResultEven());
416 out1 = vector::BitCastOp::create(builder&: rewriter, location: loc, result: accRowTy,
417 source: deintr64.getResultOdd());
418 }
419 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: out0, dest: result, position: i * 2);
420 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: out1, dest: result, position: i * 2 + 1);
421 }
422
423 return result;
424}
425
426class VectorContractRewriterI8MM : public VectorContractRewriter {
427public:
428 // Check the specific preconditions for the integer case. Initialise
429 // parametrisation types and dimensions.
430 LogicalResult matchAndInit(vector::ContractionOp op,
431 PatternRewriter &rewriter) {
432 if (failed(Result: match(op, rewriter)))
433 return failure();
434
435 VectorType lhsType = op.getLhsType();
436 VectorType rhsType = op.getRhsType();
437
438 m = lhsType.getDimSize(idx: 0);
439 n = rhsType.getDimSize(idx: 0);
440 k = rhsType.getDimSize(idx: 1);
441
442 // Check the operands have the expected shape:
443 // * for LHS: fixed vector MxK
444 // * for RHS: scalable vector [N]xK
445 // * K == 8
446 // * M and N even and at least 2
447 if (lhsType.isScalable() || !rhsType.getScalableDims()[0] ||
448 rhsType.getScalableDims()[1] || lhsType.getDimSize(idx: 1) != k || k != 8 ||
449 m < 2 || m % 2 != 0 || n < 2 || n % 2 != 0)
450 return rewriter.notifyMatchFailure(arg&: op, msg: "non-matching operand shape");
451
452 // Check the output is a vector of i32 elements.
453 auto outTy = dyn_cast<VectorType>(Val: op.getResultType());
454 if (!outTy || outTy.getElementType() != rewriter.getI32Type())
455 return rewriter.notifyMatchFailure(arg&: op,
456 msg: "output type is not a vector of i32");
457
458 // Check inputs are sign-/zero- extensions from i8 to i32. Get the values
459 // before the extension. All four signed/unsigned combinations for input
460 // operands are supported, but they are lowered to different operations.
461 // Determine which is the appropriate operation to lower to.
462 mmlaOp = MMLA::SignedInt;
463 swapOperands = false;
464 auto maybeLhs = getExtOperand<arith::ExtSIOp>(v: op.getLhs());
465 if (!maybeLhs) {
466 mmlaOp = MMLA::UnsignedInt;
467 maybeLhs = getExtOperand<arith::ExtUIOp>(v: op.getLhs());
468 }
469 if (!maybeLhs)
470 return rewriter.notifyMatchFailure(
471 arg&: op, msg: "LHS is not a sign- or zero- extended i8");
472
473 auto maybeRhs = getExtOperand<arith::ExtSIOp>(v: op.getRhs());
474 if (maybeRhs) {
475 if (mmlaOp == MMLA::UnsignedInt)
476 mmlaOp = MMLA::MixedInt;
477 } else {
478 if (mmlaOp == MMLA::SignedInt) {
479 mmlaOp = MMLA::MixedInt;
480 swapOperands = true;
481 }
482 maybeRhs = getExtOperand<arith::ExtUIOp>(v: op.getRhs());
483 }
484 if (!maybeRhs)
485 return rewriter.notifyMatchFailure(
486 arg&: op, msg: "RHS is not a sign- or zero- extended i8");
487
488 // Initialise algorithm parameters.
489 lhs = *maybeLhs;
490 rhs = *maybeRhs;
491 acc = op.getAcc();
492
493 return success();
494 }
495};
496
497class VectorContractRewriterBfloat : public VectorContractRewriter {
498public:
499 // Check the specific preconditions for the bfloat16 case. Initialise
500 // parametrisation types and dimensions.
501 LogicalResult matchAndInit(vector::ContractionOp op,
502 PatternRewriter &rewriter) {
503 if (failed(Result: match(op, rewriter)))
504 return failure();
505
506 VectorType lhsType = op.getLhsType();
507 VectorType rhsType = op.getRhsType();
508
509 m = lhsType.getDimSize(idx: 0);
510 n = rhsType.getDimSize(idx: 0);
511 k = rhsType.getDimSize(idx: 1);
512
513 // Check the operands have the expected shape:
514 // * for LHS: fixed vector MxK
515 // * for RHS: scalable vector [N]xK
516 // * K == 4
517 // * M and N even and at least 2
518 if (lhsType.isScalable() || !rhsType.getScalableDims()[0] ||
519 rhsType.getScalableDims()[1] || lhsType.getDimSize(idx: 1) != k || k != 4 ||
520 m < 2 || m % 2 != 0 || n < 2 || n % 2 != 0)
521 return rewriter.notifyMatchFailure(arg&: op, msg: "non-matching operand shape");
522
523 // Check the output is a vector of Float32 elements.
524 auto outTy = dyn_cast<VectorType>(Val: op.getResultType());
525 if (!outTy || outTy.getElementType() != rewriter.getF32Type())
526 return rewriter.notifyMatchFailure(arg&: op,
527 msg: "output type is not a vector of f32");
528
529 // Check the inputs are vectors of BFloat16 elements.
530 if (lhsType.getElementType() != rewriter.getBF16Type())
531 return rewriter.notifyMatchFailure(arg&: op,
532 msg: "input type is not a vector of bf16");
533
534 // Initialise algorithm parameters.
535 mmlaOp = MMLA::Bfloat;
536 swapOperands = false;
537 lhs = op.getLhs();
538 rhs = op.getRhs();
539 acc = op.getAcc();
540
541 return success();
542 }
543};
544
545class LowerContractionToSVEI8MMPattern
546 : public OpRewritePattern<vector::ContractionOp> {
547public:
548 using Base::Base;
549 LogicalResult matchAndRewrite(vector::ContractionOp op,
550 PatternRewriter &rewriter) const override {
551
552 // Match i8xi8 -> i32 matrix multiply and accumulate.
553 VectorContractRewriterI8MM vcr;
554 if (failed(Result: vcr.matchAndInit(op, rewriter)))
555 return failure();
556
557 Value result = vcr.lower(op, rewriter);
558 rewriter.replaceOp(op, newValues: result);
559
560 return success();
561 }
562};
563
564class LowerContractionToSVEBFMMLAPattern
565 : public OpRewritePattern<vector::ContractionOp> {
566public:
567 using Base::Base;
568 LogicalResult matchAndRewrite(vector::ContractionOp op,
569 PatternRewriter &rewriter) const override {
570
571 // Match bf16xbf16 -> f32 matrix multiply and accumulate.
572 VectorContractRewriterBfloat vcr;
573 if (failed(Result: vcr.matchAndInit(op, rewriter)))
574 return failure();
575
576 Value result = vcr.lower(op, rewriter);
577 rewriter.replaceOp(op, newValues: result);
578
579 return success();
580 }
581};
582
583/// A helper for LowerContractionToSVEIndexedFMLAPatterns to check the
584/// preconditions. Check the pattern for an overview of the conditions to
585/// check.
586static LogicalResult matchIndexed(vector::ContractionOp op,
587 PatternRewriter &rewriter) {
588 // Check iterator types for matrix multiplication.
589 // TODO: Extract to a helper hook.
590 auto itTypes = op.getIteratorTypesArray();
591 if (itTypes.size() != 3 || itTypes[0] != vector::IteratorType::parallel ||
592 itTypes[1] != vector::IteratorType::parallel ||
593 itTypes[2] != vector::IteratorType::reduction)
594 return rewriter.notifyMatchFailure(
595 arg&: op, msg: "iterator types do not correspond to matrix multiplication");
596
597 // Verify that the permutation maps represent a plain matmul:
598 // lhs: (d0, d1, d2) -> (d0, d2)
599 // rhs: (d0, d1, d2) -> (d1, d2)
600 // acc: (d0, d1, d2) -> (d0, d1)
601 // TODO: Extract to a helper hook.
602 if (op.getIndexingMapsArray()[0] !=
603 AffineMap::getMultiDimMapWithTargets(numDims: 3, targets: ArrayRef{0u, 2u},
604 context: op.getContext()) ||
605 op.getIndexingMapsArray()[1] !=
606 AffineMap::getMultiDimMapWithTargets(numDims: 3, targets: ArrayRef{2u, 1u},
607 context: op.getContext()) ||
608 op.getIndexingMapsArray()[2] != AffineMap::getMultiDimMapWithTargets(
609 numDims: 3, targets: ArrayRef{0u, 1u}, context: op.getContext()))
610 return rewriter.notifyMatchFailure(arg&: op, msg: "non-matching permutation maps");
611
612 // Check the combining kind is addition.
613 // TODO: Extract to a helper hook.
614 if (op.getKind() != vector::CombiningKind::ADD)
615 return rewriter.notifyMatchFailure(arg&: op, msg: "combining kind is not an addition");
616
617 // Check the operands have the supported shape:
618 // * LHS: <1 x 4>
619 // * RHS: <4 x [8]>
620 // * ACC: <1 x [8]>
621 VectorType lhsType = op.getLhsType();
622 VectorType rhsType = op.getRhsType();
623 auto m = lhsType.getDimSize(idx: 0);
624 auto n = rhsType.getDimSize(idx: 1);
625 auto k = rhsType.getDimSize(idx: 0);
626
627 if (rhsType.getScalableDims()[0] || !rhsType.getScalableDims()[1] ||
628 lhsType.isScalable() || lhsType.getDimSize(idx: 1) != k || k != 4 || m != 1 ||
629 n != 8)
630 return rewriter.notifyMatchFailure(arg&: op, msg: "non-matching operand shape");
631
632 // Check that output element type is Float32.
633 auto outTy = dyn_cast<VectorType>(Val: op.getResultType());
634 if (!outTy || outTy.getElementType() != rewriter.getF32Type())
635 return rewriter.notifyMatchFailure(arg&: op,
636 msg: "output type is not a vector of f32");
637
638 // Check that the input element type is Float32.
639 if (lhsType.getElementType() != rewriter.getF32Type())
640 return rewriter.notifyMatchFailure(arg&: op,
641 msg: "input type is not a vector of bf16");
642
643 return success();
644}
645
646/// Lowers a matching contraction operation to SVE's "indexed" FMLAs (FP32)
647///
648/// The input contraction must represent a plain matrix-multiplication with the
649/// following dimensions:
650///
651/// vector.contract #matmul_trait %lhs, %rhs, %acc
652/// : vector<1x4xf32>, vector<4x[8]xf32> into vector<1x[8]xf32>
653///
654/// It is lowered to a sequence of indexed FMLAs, one per reduction dimension
655/// (pseudo-code illustration):
656///
657/// %acc = fmla %acc, %rhs, %lhs[0]
658/// %acc = fmla %acc, %rhs, %lhs[1]
659/// %acc = fmla %acc, %rhs, %lhs[2]
660/// %acc = fmla %acc, %rhs, %lhs[3]
661///
662/// Note that the relative order of `%rhs` and `%lhs` is reversed to match the
663/// SVE Op semantics.
664///
665/// The actual output is a bit more complex - input vectors
666/// (e.g. vector<[8]xf32>) are broken down into valid SVE vectors (e.g.
667/// vector<[4]xf32>) that the FMLA Op expects.
668class LowerContractionToSVEIndexedFMLAPatterns
669 : public OpRewritePattern<vector::ContractionOp> {
670 using Base::Base;
671
672public:
673 LogicalResult matchAndRewrite(vector::ContractionOp op,
674 PatternRewriter &rewriter) const override {
675 // Check the specific preconditions.
676 if (failed(Result: matchIndexed(op, rewriter)))
677 return failure();
678
679 auto loc = op.getLoc();
680 auto acc = op.getAcc();
681
682 // Legal type for indexed FMLA:
683 // * vector<[4]xf32> rather than the input vector<[8]xf32>
684 mlir::VectorType legalType =
685 VectorType::get(shape: {4}, elementType: rewriter.getF32Type(), scalableDims: {true});
686
687 // Prepare LHS for indexed FMLA (2D -> 1D):
688 // * vector<1x4xf32> -> vector<[4]xf32>
689 auto lhs1D = vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getLhs(), position: 0);
690 auto repeatedTy =
691 VectorType::get(shape: {4}, elementType: rewriter.getF32Type(), /*scalableDims=*/{true});
692 Value lhsScalable =
693 LLVM::vector_repeat::create(builder&: rewriter, location: loc, res: repeatedTy, vec: lhs1D)
694 .getResult();
695
696 // Prepare ACC for indexed FMLA (split into legal vectors):
697 // vector<1x[8]xf32> -> [vector<[4]xf32>, vector<[4]xf32>]
698 auto accRow = vector::ExtractOp::create(builder&: rewriter, location: loc, source: acc, position: 0);
699 auto acc0 =
700 vector::ScalableExtractOp::create(builder&: rewriter, location: loc, result: legalType, source: accRow, pos: 0)
701 .getResult();
702 auto acc1 =
703 vector::ScalableExtractOp::create(builder&: rewriter, location: loc, result: legalType, source: accRow, pos: 4)
704 .getResult();
705
706 // Loop over the reduction lanes
707 for (size_t laneIdx = 0; laneIdx < 4; ++laneIdx) {
708 auto lane = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: laneIdx, width: 32);
709
710 // Prepare RHS for indexed FMLA (select one row and split):
711 // vector<4x[8]xf32> -> [vector<[4]xf32>, vector<[4]xf32>]
712 auto rhsRow =
713 vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getRhs(), position: laneIdx);
714 auto rhs0 = vector::ScalableExtractOp::create(builder&: rewriter, location: loc, result: legalType,
715 source: rhsRow, pos: 0);
716 auto rhs1 = vector::ScalableExtractOp::create(builder&: rewriter, location: loc, result: legalType,
717 source: rhsRow, pos: 4);
718
719 // FMLAs - note that the LHS and RHS operands from vector.contract are
720 // passed in the reverse order:
721 // %acc = fmla %acc, %rhs, %lhs[0]
722 acc0 = arm_sve::FmlaOp::create(builder&: rewriter, location: loc, dst: legalType, acc: acc0, src1: rhs0,
723 src2: lhsScalable, lane)
724 .getResult();
725 acc1 = arm_sve::FmlaOp::create(builder&: rewriter, location: loc, dst: legalType, acc: acc1, src1: rhs1,
726 src2: lhsScalable, lane)
727 .getResult();
728 }
729
730 // Store the result back
731 acc = vector::ScalableInsertOp::create(builder&: rewriter, location: loc, valueToStore: acc0, dest: accRow, pos: 0);
732 acc = vector::ScalableInsertOp::create(builder&: rewriter, location: loc, valueToStore: acc1, dest: acc, pos: 4);
733
734 rewriter.replaceOp(op, newOp: vector::ShapeCastOp::create(
735 builder&: rewriter, location: loc, result: op.getResultType(), source: acc));
736
737 return success();
738 }
739};
740} // namespace
741
742void mlir::arm_sve::populateLowerContractionToSVEI8MMPatterns(
743 RewritePatternSet &patterns) {
744 MLIRContext *context = patterns.getContext();
745 patterns.add<LowerContractionToSVEI8MMPattern>(arg&: context, /*benefit=*/args: 2);
746}
747
748void mlir::arm_sve::populateLowerContractionToSVEBFMMLAPatterns(
749 RewritePatternSet &patterns) {
750 MLIRContext *context = patterns.getContext();
751 patterns.add<LowerContractionToSVEBFMMLAPattern>(arg&: context, /*benefit=*/args: 2);
752}
753
754void mlir::arm_sve::populateLowerContractionToSVEIndexedFMLAPatterns(
755 RewritePatternSet &patterns) {
756 MLIRContext *context = patterns.getContext();
757 patterns.add<LowerContractionToSVEIndexedFMLAPatterns>(arg&: context,
758 /*benefit=*/args: 2);
759}
760