1//===- LowerContractToNeonPatterns.cpp - Contract to I8MM/BF16 --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements lowering patterns from vector.contract to operations
10// that map to instructions from the Neon FEAT_I8MM extension.
11//
12// TODO: There may be opportunities to unify this with a similar pattern
13// for SVE. See:
14// https://github.com/llvm/llvm-project/issues/145559
15// LowerContractToSVEPatterns.cpp
16//
17//===----------------------------------------------------------------------===//
18
19#include "mlir/Dialect/Arith/IR/Arith.h"
20#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
21#include "mlir/Dialect/ArmNeon/Transforms.h"
22#include "mlir/Dialect/Func/IR/FuncOps.h"
23#include "mlir/Dialect/Utils/IndexingUtils.h"
24#include "mlir/Dialect/Vector/IR/VectorOps.h"
25#include "mlir/IR/AffineMap.h"
26#include "mlir/IR/PatternMatch.h"
27
28#define DEBUG_TYPE "lower-contract-to-arm-neon"
29
30using namespace mlir;
31using namespace mlir::arm_neon;
32
33namespace {
34/// Get the operand of a `vector.contract`. This function is intended to
35/// abstract away from the particular way a value is extended before feeding it
36/// into the `vector.contract` - via zero-extend or an explicit or implicit
37/// sign-extend (for implicit sign-extension see `vector.contract`
38/// documentation).
39///
40/// The template parameter `Op` indicates the extension operation (explicit or
41/// implicit) for which we are checking.
42///
43// Return success only for extensions from `iN` (N <= 8) to `i32`.
44template <typename Op>
45std::optional<Value> getExtOperand(Value v) {
46
47 static_assert(llvm::is_one_of<Op, arith::ExtSIOp, arith::ExtUIOp>::value,
48 "Must be instantiated with either sign- or zero- extension op");
49
50 // If the operand is not defined by an explicit extend operation of the
51 // accepted operation type allow for an implicit sign-extension.
52 auto extOp = v.getDefiningOp<Op>();
53 if (!extOp) {
54 if constexpr (std::is_same<Op, arith::ExtSIOp>::value) {
55 auto eltTy = cast<VectorType>(Val: v.getType()).getElementType();
56 if (!eltTy.isSignlessInteger() || eltTy.getIntOrFloatBitWidth() > 8)
57 return {};
58 return v;
59 }
60 return {};
61 }
62
63 // If the operand is defined by an explicit extend operation of the accepted
64 // operation type, check it's extended from `iN` (N <= 8) to `i32`.
65 auto inOp = extOp.getIn();
66 auto inTy = dyn_cast<VectorType>(inOp.getType());
67 if (!inTy)
68 return {};
69 auto inEltTy = inTy.getElementType();
70 if (!inEltTy.isSignlessInteger() || inEltTy.getIntOrFloatBitWidth() > 8)
71 return {};
72
73 auto outTy = dyn_cast<VectorType>(extOp.getType());
74 if (!(outTy && outTy.getElementType().isSignlessInteger(32)))
75 return {};
76
77 return inOp;
78}
79
80/// Helper function to extend a vector with elements iN, N < 8 to
81/// a vector of i8. Do sign extension if the parameter `signExt` is true,
82/// zero extension otherwise.
83Value extendSmallIntVector(Location loc, VectorType srcTy, Value val,
84 bool signExt, PatternRewriter &rewriter) {
85 Type targetTy = srcTy.clone(elementType: rewriter.getI8Type());
86 return signExt ? rewriter.createOrFold<arith::ExtSIOp>(location: loc, args&: targetTy, args&: val)
87 : rewriter.createOrFold<arith::ExtUIOp>(location: loc, args&: targetTy, args&: val);
88}
89
90class VectorContractRewriter {
91protected:
92 // Designate the operation (resp. instruction) used to do sub-tile matrix
93 // multiplications.
94 enum class MMLA {
95 Nop,
96 SignedInt, // smmla
97 UnsignedInt, // ummla
98 MixedInt, // usmmla
99 Bfloat // bfmmla
100 };
101
102 // Lower-level operation to be emitted.
103 MMLA mmlaOp = MMLA::Nop;
104
105 // Indicate if the operands for the ArmNeon dialect operation need to be
106 // swapped. Currently this is needed in order to emulate an "summla"
107 // operation.
108 bool swapOperands = false;
109
110 // The operand tiles. These are not necessarily the operands of
111 // `vector.contract`, for example they could be operands to `arith.extsi`
112 // that is in turn fed into `vector.contract`.
113 Value lhs;
114 Value rhs;
115 Value acc;
116
117 // The dimensions logically corresponding to matrix multiplication of
118 // MxK * KxN -> MxN. The operands and the result do not necessarily have these
119 // shapes, for example RHS could be NxK with a transposing indexing map.
120 int64_t dimM = 0;
121 int64_t dimN = 0;
122 int64_t dimK = 0;
123
124 // Unroll iteration bounds. See documentaiton for `StaticTileOffsetRange`.
125 SmallVector<int64_t> iterationBounds;
126
127 // Sub-tile shape. The algorithm handles operand shapes, which are multiples
128 // of this shape.
129 SmallVector<int64_t> subTileShape;
130
131 // Create the matrix multiply and accumulate operation according to `mmlaOp`.
132 Value createMMLA(PatternRewriter &rewriter, Location loc, Value acc,
133 Value lhs, Value rhs) {
134
135 if (swapOperands)
136 std::swap(a&: lhs, b&: rhs);
137 switch (mmlaOp) {
138 case MMLA::SignedInt:
139 return rewriter.createOrFold<arm_neon::SmmlaOp>(location: loc, args: acc.getType(), args&: acc,
140 args&: lhs, args&: rhs);
141 case MMLA::UnsignedInt:
142 return rewriter.createOrFold<arm_neon::UmmlaOp>(location: loc, args: acc.getType(), args&: acc,
143 args&: lhs, args&: rhs);
144 case MMLA::MixedInt:
145 return rewriter.createOrFold<arm_neon::UsmmlaOp>(location: loc, args: acc.getType(), args&: acc,
146 args&: lhs, args&: rhs);
147 case MMLA::Bfloat:
148 return arm_neon::BfmmlaOp::create(builder&: rewriter, location: loc, res: acc.getType(), acc, src1: lhs,
149 src2: rhs);
150 case MMLA::Nop:
151 llvm_unreachable("Uninitialized operation type");
152 }
153 llvm_unreachable("Unknown MMLA");
154 }
155
156 // Check common preconditions for applying the patterns and initialize
157 // logical dimensions.
158 LogicalResult matchAndInit(vector::ContractionOp op,
159 PatternRewriter &rewriter) {
160 // Check iterator types for matrix multiplication.
161 SmallVector<vector::IteratorType> itTypes = op.getIteratorTypesArray();
162 if ((itTypes.size() != 3 || itTypes[0] != vector::IteratorType::parallel ||
163 itTypes[1] != vector::IteratorType::parallel ||
164 itTypes[2] != vector::IteratorType::reduction) &&
165 (itTypes.size() != 2 || itTypes[0] != vector::IteratorType::parallel ||
166 itTypes[1] != vector::IteratorType::reduction))
167 return rewriter.notifyMatchFailure(
168 arg&: op, msg: "iterator types do not correspond to matrix multiplication");
169
170 // Avoid 0-D vectors and 1-D rhs:
171 VectorType lhsType = op.getLhsType();
172 VectorType rhsType = op.getRhsType();
173 if (!lhsType.hasRank() || !rhsType.hasRank() || lhsType.getRank() > 2 ||
174 rhsType.getRank() != 2)
175 return rewriter.notifyMatchFailure(arg&: op, msg: "Invalid operand rank");
176
177 // This codegen does not work for scalable vectors. Return failure so this
178 // pattern is not accidentally chosen over patterns that lower to ArmSVE.
179 if (lhsType.isScalable() || rhsType.isScalable())
180 return rewriter.notifyMatchFailure(arg&: op,
181 msg: "Not applicable to scalable vectors");
182
183 // Initialize dimensions and check for a matching K dimension.
184 dimM = lhsType.getDimSize(idx: 0);
185 dimN = rhsType.getDimSize(idx: 0);
186 dimK = rhsType.getDimSize(idx: 1);
187
188 int64_t lhsDimK;
189 if (lhsType.getRank() == 1) {
190 dimM = 1;
191 lhsDimK = lhsType.getDimSize(idx: 0);
192 } else {
193 lhsDimK = lhsType.getDimSize(idx: 1);
194 }
195
196 if (lhsDimK != dimK)
197 return rewriter.notifyMatchFailure(arg&: op, msg: "Dimensions mismatch");
198
199 return success();
200 }
201
202public:
203 void lower(vector::ContractionOp op, PatternRewriter &rewriter) {
204 // Create some convenience types.
205 auto inputElementType = cast<ShapedType>(Val: lhs.getType()).getElementType();
206 auto accElementType = cast<ShapedType>(Val: acc.getType()).getElementType();
207 auto inputExpandedType =
208 VectorType::get(shape: {2, subTileShape.back()}, elementType: inputElementType);
209 auto outputExpandedType = VectorType::get(shape: {2, 2}, elementType: accElementType);
210
211 // One-dimensional representation of logical sub-tiles as required by the
212 // ArmNeon ops.
213 auto collapsedInputType =
214 VectorType::get(shape: inputExpandedType.getNumElements(), elementType: inputElementType);
215 auto collapsedOutputType =
216 VectorType::get(shape: outputExpandedType.getNumElements(), elementType: accElementType);
217
218 // Get indexing maps for a more concise/convenient access.
219 auto indexingMaps = op.getIndexingMapsArray();
220 AffineMap &lhsPermutationMap = indexingMaps[0];
221 AffineMap &rhsPermutationMap = indexingMaps[1];
222 AffineMap &accPermutationMap = indexingMaps[2];
223
224 Location loc = op.getLoc();
225
226 // Initial accumulator for the final result. This is the un-tiled result if
227 // tiling is done.
228 Value result =
229 arith::ConstantOp::create(builder&: rewriter, location: loc, result: op.getResultType(),
230 value: rewriter.getZeroAttr(type: op.getResultType()));
231
232 SmallVector<int64_t, 3> loopOrder = {0, 1};
233 if (iterationBounds.size() == 3)
234 loopOrder.push_back(Elt: 2);
235
236 // Keep track of the previous accumulator when tiling over K.
237 Value kAcc;
238 for (SmallVector<int64_t> offsets :
239 StaticTileOffsetRange(iterationBounds, subTileShape, loopOrder)) {
240 // Helper to compute the new shape of each operand and extract the slice.
241 auto extractOperand = [&](Value operand, AffineMap permutationMap,
242 ArrayRef<int64_t> operandOffsets) {
243 SmallVector<int64_t> operandShape = applyPermutationMap(
244 map: permutationMap, source: ArrayRef<int64_t>(subTileShape));
245 SmallVector<int64_t> operandStrides(operandOffsets.size(), 1);
246 return rewriter.createOrFold<vector::ExtractStridedSliceOp>(
247 location: loc, args&: operand, args&: operandOffsets, args&: operandShape, args&: operandStrides);
248 };
249
250 // Extract tiled lhs, rhs, and acc
251 SmallVector<int64_t> lhsOffsets =
252 applyPermutationMap(map: lhsPermutationMap, source: ArrayRef<int64_t>(offsets));
253 Value tiledLhs = extractOperand(lhs, lhsPermutationMap, lhsOffsets);
254 SmallVector<int64_t> rhsOffsets =
255 applyPermutationMap(map: rhsPermutationMap, source: ArrayRef<int64_t>(offsets));
256 Value tiledRhs = extractOperand(rhs, rhsPermutationMap, rhsOffsets);
257 SmallVector<int64_t> accOffsets =
258 applyPermutationMap(map: accPermutationMap, source: ArrayRef<int64_t>(offsets));
259 Value tiledAcc = extractOperand(acc, accPermutationMap, accOffsets);
260
261 // With vecmat, tiled LHS and ACC will contain only one of 2 necessary
262 // rows along dimM. Expand their shapes to match the ArmNeon op.
263 if (dimM == 1) {
264 auto expandRowVector = [&](Value tiledOperand,
265 VectorType expandedTypeType) {
266 auto emptyOperand =
267 arith::ConstantOp::create(builder&: rewriter, location: loc, result: expandedTypeType,
268 value: rewriter.getZeroAttr(type: expandedTypeType));
269 SmallVector<int64_t> offsets(
270 cast<ShapedType>(Val: emptyOperand.getType()).getRank(), 0);
271 SmallVector<int64_t> strides(
272 cast<ShapedType>(Val: tiledOperand.getType()).getRank(), 1);
273 return rewriter.createOrFold<vector::InsertStridedSliceOp>(
274 location: loc, args&: tiledOperand, args&: emptyOperand, args&: offsets, args&: strides);
275 };
276 tiledLhs = expandRowVector(tiledLhs, inputExpandedType);
277 tiledAcc = expandRowVector(tiledAcc, outputExpandedType);
278 }
279
280 // Transpose ACC if doing signed by unsigned multiplication, because we're
281 // using the instruction for unsigned by signed multiplication with
282 // reversed operands.
283 if (swapOperands)
284 tiledAcc = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: tiledAcc,
285 permutation: ArrayRef<int64_t>({1, 0}));
286
287 // Collapse tiled operands to 1D vectors required by the ArmNeon ops
288 auto collapsedLhs = rewriter.createOrFold<vector::ShapeCastOp>(
289 location: tiledLhs.getLoc(), args&: collapsedInputType, args&: tiledLhs);
290 auto collapsedRhs = rewriter.createOrFold<vector::ShapeCastOp>(
291 location: tiledRhs.getLoc(), args&: collapsedInputType, args&: tiledRhs);
292
293 bool initialKAcc = offsets.back() == 0;
294 Value collapsedRes;
295 if (!initialKAcc) {
296 collapsedRes = kAcc;
297 } else {
298 collapsedRes = rewriter.createOrFold<vector::ShapeCastOp>(
299 location: tiledAcc.getLoc(), args&: collapsedOutputType, args&: tiledAcc);
300 }
301
302 // Insert contract op
303 kAcc =
304 createMMLA(rewriter, loc, acc: collapsedRes, lhs: collapsedLhs, rhs: collapsedRhs);
305
306 // Reshape output back to 2D
307 Value tiledRes = rewriter.createOrFold<vector::ShapeCastOp>(
308 location: kAcc.getLoc(), args: tiledAcc.getType(), args&: kAcc);
309
310 // Because of the reversed operands the result is obtained transposed.
311 // Transpose it back,
312 if (swapOperands)
313 tiledRes = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: tiledRes,
314 permutation: ArrayRef<int64_t>({1, 0}));
315
316 // With vecmat, only one row of tiled ACC can be inserted into the final
317 // result
318 if (dimM == 1)
319 tiledRes = rewriter.createOrFold<vector::ExtractOp>(location: loc, args&: tiledRes, args: 0);
320
321 // Insert the tiled result back into the non tiled result of the
322 // contract op.
323 SmallVector<int64_t> strides(
324 cast<ShapedType>(Val: tiledRes.getType()).getRank(), 1);
325 result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
326 location: loc, args&: tiledRes, args&: result, args&: accOffsets, args&: strides);
327 }
328
329 rewriter.replaceOp(op, newValues: result);
330 }
331};
332
333class VectorContractRewriterI8MM : public VectorContractRewriter {
334public:
335 LogicalResult matchAndInit(vector::ContractionOp op,
336 PatternRewriter &rewriter) {
337 if (failed(Result: VectorContractRewriter::matchAndInit(op, rewriter)))
338 return failure();
339
340 // Unrolling patterns can handle any [2, 2, 8] shaped multiple of inputs for
341 // tiling.
342 if ((dimM != 1 && dimM % 2 != 0) || dimN % 2 != 0 || dimK % 8 != 0)
343 return rewriter.notifyMatchFailure(arg&: op, msg: "Unsupported operand shapes");
344
345 // Check inputs are sign-/zero- extensions from iN (N <= 8) to i32. Get the
346 // values before the extension. All four signed/unsigned combinations for
347 // input operands are supported, but they are lowered to different
348 // operations. Determine which is the appropriate operation to lower to.
349 mmlaOp = MMLA::SignedInt;
350 auto maybeLhs = getExtOperand<arith::ExtSIOp>(v: op.getLhs());
351 if (!maybeLhs) {
352 mmlaOp = MMLA::UnsignedInt;
353 maybeLhs = getExtOperand<arith::ExtUIOp>(v: op.getLhs());
354 }
355 if (!maybeLhs)
356 return rewriter.notifyMatchFailure(
357 arg&: op, msg: "LHS is not a sign- or zero- extended iN, N <= 8");
358
359 auto maybeRhs = getExtOperand<arith::ExtSIOp>(v: op.getRhs());
360 if (maybeRhs) {
361 if (mmlaOp == MMLA::UnsignedInt)
362 mmlaOp = MMLA::MixedInt;
363 } else {
364 if (mmlaOp == MMLA::SignedInt) {
365 mmlaOp = MMLA::MixedInt;
366 swapOperands = true;
367 }
368 maybeRhs = getExtOperand<arith::ExtUIOp>(v: op.getRhs());
369 }
370
371 if (!maybeRhs)
372 return rewriter.notifyMatchFailure(
373 arg&: op, msg: "RHS is not a sign- or zero- extended iN, N <= 8");
374
375 lhs = *maybeLhs;
376 rhs = *maybeRhs;
377 acc = op.getAcc();
378
379 // Extend inputs from iN, N < 8 to i8.
380 Location loc = op.getLoc();
381 auto lhsExtInType = cast<VectorType>(Val: lhs.getType());
382 if (lhsExtInType.getElementTypeBitWidth() < 8)
383 lhs = extendSmallIntVector(loc, srcTy: lhsExtInType, val: lhs,
384 /* signExt */
385 (mmlaOp == MMLA::SignedInt ||
386 (mmlaOp == MMLA::MixedInt && !swapOperands)),
387 rewriter);
388
389 auto rhsExtInType = cast<VectorType>(Val: rhs.getType());
390 if (rhsExtInType.getElementTypeBitWidth() < 8)
391 rhs = extendSmallIntVector(loc, srcTy: rhsExtInType, val: rhs,
392 /* signExt */
393 (mmlaOp == MMLA::SignedInt ||
394 (mmlaOp == MMLA::MixedInt && swapOperands)),
395 rewriter);
396
397 // Initialize parameters for unrolling.
398 iterationBounds = *op.getIterationShapeToUnroll();
399 if (iterationBounds.size() == 3)
400 subTileShape = SmallVector<int64_t>({dimM == 1 ? 1 : 2, 2, 8});
401 else
402 subTileShape = SmallVector<int64_t>({2, 8});
403
404 return success();
405 }
406};
407
408class VectorContractRewriterBFMMLA : public VectorContractRewriter {
409public:
410 LogicalResult matchAndInit(vector::ContractionOp op,
411 PatternRewriter &rewriter) {
412
413 if (failed(Result: VectorContractRewriter::matchAndInit(op, rewriter)))
414 return failure();
415
416 // Unrolling patterns can handle any [2, 2, 4] shaped multiple of inputs for
417 // tiling.
418 if ((dimM != 1 && dimM % 2 != 0) || dimN % 2 != 0 || dimK % 4 != 0)
419 return rewriter.notifyMatchFailure(arg&: op, msg: "Unsupported operand shapes");
420
421 // Check the output is a vector of Float32 elements.
422 auto outTy = dyn_cast<VectorType>(Val: op.getResultType());
423 if (!outTy || outTy.getElementType() != rewriter.getF32Type())
424 return rewriter.notifyMatchFailure(arg&: op,
425 msg: "output type is not a vector of f32");
426
427 // Check the inputs are vectors of BFloat16 elements.
428 if (op.getLhsType().getElementType() != rewriter.getBF16Type())
429 return rewriter.notifyMatchFailure(arg&: op,
430 msg: "input type is not a vector of bf16");
431
432 mmlaOp = MMLA::Bfloat;
433 swapOperands = false;
434 lhs = op.getLhs();
435 rhs = op.getRhs();
436 acc = op.getAcc();
437
438 // Initialize parameters for unrolling.
439 iterationBounds = *op.getIterationShapeToUnroll();
440 if (iterationBounds.size() == 3)
441 subTileShape = SmallVector<int64_t>({dimM == 1 ? 1 : 2, 2, 4});
442 else
443 subTileShape = SmallVector<int64_t>({2, 4});
444
445 return success();
446 }
447};
448
449/// Lowering from a vector::contractOp arm neon smmla intrinsic. This will tile
450/// any vector.contract into multiple smmla instructions with unrolling so long
451/// as [2,2,8] is a divisor of its shape. It can also process vecmats with dimM
452/// = 1 (either explicitly or inferred if LHS has only dimK) If no unrolling is
453/// necessary, a single smmla instruction is emitted.
454class LowerContractionToNeonI8MMPattern
455 : public OpRewritePattern<vector::ContractionOp> {
456public:
457 using Base::Base;
458 LogicalResult matchAndRewrite(vector::ContractionOp op,
459 PatternRewriter &rewriter) const override {
460 if (cast<vector::MaskableOpInterface>(Val: op.getOperation()).isMasked())
461 return rewriter.notifyMatchFailure(
462 arg&: op, msg: "masked contractions are not supported");
463
464 VectorContractRewriterI8MM vcr;
465 if (failed(Result: vcr.matchAndInit(op, rewriter)))
466 return failure();
467 vcr.lower(op, rewriter);
468
469 return success();
470 }
471};
472
473class LowerContractionToNeonBFMMLAPattern
474 : public OpRewritePattern<vector::ContractionOp> {
475public:
476 using Base::Base;
477 LogicalResult matchAndRewrite(vector::ContractionOp op,
478 PatternRewriter &rewriter) const override {
479 if (cast<vector::MaskableOpInterface>(Val: op.getOperation()).isMasked())
480 return rewriter.notifyMatchFailure(
481 arg&: op, msg: "masked contractions are not supported");
482
483 VectorContractRewriterBFMMLA vcr;
484 if (failed(Result: vcr.matchAndInit(op, rewriter)))
485 return failure();
486 vcr.lower(op, rewriter);
487
488 return success();
489 }
490};
491
492} // namespace
493
494void mlir::arm_neon::populateLowerContractionToNeonI8MMPatterns(
495 RewritePatternSet &patterns) {
496 MLIRContext *context = patterns.getContext();
497 patterns.add<LowerContractionToNeonI8MMPattern>(arg&: context, /*benefit=*/args: 2);
498}
499
500void mlir::arm_neon::populateLowerContractionToNeonBFMMLAPatterns(
501 RewritePatternSet &patterns) {
502 MLIRContext *context = patterns.getContext();
503 patterns.add<LowerContractionToNeonBFMMLAPattern>(arg&: context, /*benefit=*/args: 2);
504}
505