| 1 | //===- LowerContractToNeonPatterns.cpp - Contract to I8MM/BF16 --*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements lowering patterns from vector.contract to operations |
| 10 | // that map to instructions from the Neon FEAT_I8MM extension. |
| 11 | // |
| 12 | // TODO: There may be opportunities to unify this with a similar pattern |
| 13 | // for SVE. See: |
| 14 | // https://github.com/llvm/llvm-project/issues/145559 |
| 15 | // LowerContractToSVEPatterns.cpp |
| 16 | // |
| 17 | //===----------------------------------------------------------------------===// |
| 18 | |
| 19 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 20 | #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h" |
| 21 | #include "mlir/Dialect/ArmNeon/Transforms.h" |
| 22 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 23 | #include "mlir/Dialect/Utils/IndexingUtils.h" |
| 24 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 25 | #include "mlir/IR/AffineMap.h" |
| 26 | #include "mlir/IR/PatternMatch.h" |
| 27 | |
| 28 | #define DEBUG_TYPE "lower-contract-to-arm-neon" |
| 29 | |
| 30 | using namespace mlir; |
| 31 | using namespace mlir::arm_neon; |
| 32 | |
| 33 | namespace { |
| 34 | /// Get the operand of a `vector.contract`. This function is intended to |
| 35 | /// abstract away from the particular way a value is extended before feeding it |
| 36 | /// into the `vector.contract` - via zero-extend or an explicit or implicit |
| 37 | /// sign-extend (for implicit sign-extension see `vector.contract` |
| 38 | /// documentation). |
| 39 | /// |
| 40 | /// The template parameter `Op` indicates the extension operation (explicit or |
| 41 | /// implicit) for which we are checking. |
| 42 | /// |
| 43 | // Return success only for extensions from `iN` (N <= 8) to `i32`. |
| 44 | template <typename Op> |
| 45 | std::optional<Value> getExtOperand(Value v) { |
| 46 | |
| 47 | static_assert(llvm::is_one_of<Op, arith::ExtSIOp, arith::ExtUIOp>::value, |
| 48 | "Must be instantiated with either sign- or zero- extension op" ); |
| 49 | |
| 50 | // If the operand is not defined by an explicit extend operation of the |
| 51 | // accepted operation type allow for an implicit sign-extension. |
| 52 | auto extOp = v.getDefiningOp<Op>(); |
| 53 | if (!extOp) { |
| 54 | if constexpr (std::is_same<Op, arith::ExtSIOp>::value) { |
| 55 | auto eltTy = cast<VectorType>(Val: v.getType()).getElementType(); |
| 56 | if (!eltTy.isSignlessInteger() || eltTy.getIntOrFloatBitWidth() > 8) |
| 57 | return {}; |
| 58 | return v; |
| 59 | } |
| 60 | return {}; |
| 61 | } |
| 62 | |
| 63 | // If the operand is defined by an explicit extend operation of the accepted |
| 64 | // operation type, check it's extended from `iN` (N <= 8) to `i32`. |
| 65 | auto inOp = extOp.getIn(); |
| 66 | auto inTy = dyn_cast<VectorType>(inOp.getType()); |
| 67 | if (!inTy) |
| 68 | return {}; |
| 69 | auto inEltTy = inTy.getElementType(); |
| 70 | if (!inEltTy.isSignlessInteger() || inEltTy.getIntOrFloatBitWidth() > 8) |
| 71 | return {}; |
| 72 | |
| 73 | auto outTy = dyn_cast<VectorType>(extOp.getType()); |
| 74 | if (!(outTy && outTy.getElementType().isSignlessInteger(32))) |
| 75 | return {}; |
| 76 | |
| 77 | return inOp; |
| 78 | } |
| 79 | |
| 80 | /// Helper function to extend a vector with elements iN, N < 8 to |
| 81 | /// a vector of i8. Do sign extension if the parameter `signExt` is true, |
| 82 | /// zero extension otherwise. |
| 83 | Value extendSmallIntVector(Location loc, VectorType srcTy, Value val, |
| 84 | bool signExt, PatternRewriter &rewriter) { |
| 85 | Type targetTy = srcTy.clone(elementType: rewriter.getI8Type()); |
| 86 | return signExt ? rewriter.createOrFold<arith::ExtSIOp>(location: loc, args&: targetTy, args&: val) |
| 87 | : rewriter.createOrFold<arith::ExtUIOp>(location: loc, args&: targetTy, args&: val); |
| 88 | } |
| 89 | |
| 90 | class VectorContractRewriter { |
| 91 | protected: |
| 92 | // Designate the operation (resp. instruction) used to do sub-tile matrix |
| 93 | // multiplications. |
| 94 | enum class MMLA { |
| 95 | Nop, |
| 96 | SignedInt, // smmla |
| 97 | UnsignedInt, // ummla |
| 98 | MixedInt, // usmmla |
| 99 | Bfloat // bfmmla |
| 100 | }; |
| 101 | |
| 102 | // Lower-level operation to be emitted. |
| 103 | MMLA mmlaOp = MMLA::Nop; |
| 104 | |
| 105 | // Indicate if the operands for the ArmNeon dialect operation need to be |
| 106 | // swapped. Currently this is needed in order to emulate an "summla" |
| 107 | // operation. |
| 108 | bool swapOperands = false; |
| 109 | |
| 110 | // The operand tiles. These are not necessarily the operands of |
| 111 | // `vector.contract`, for example they could be operands to `arith.extsi` |
| 112 | // that is in turn fed into `vector.contract`. |
| 113 | Value lhs; |
| 114 | Value rhs; |
| 115 | Value acc; |
| 116 | |
| 117 | // The dimensions logically corresponding to matrix multiplication of |
| 118 | // MxK * KxN -> MxN. The operands and the result do not necessarily have these |
| 119 | // shapes, for example RHS could be NxK with a transposing indexing map. |
| 120 | int64_t dimM = 0; |
| 121 | int64_t dimN = 0; |
| 122 | int64_t dimK = 0; |
| 123 | |
| 124 | // Unroll iteration bounds. See documentaiton for `StaticTileOffsetRange`. |
| 125 | SmallVector<int64_t> iterationBounds; |
| 126 | |
| 127 | // Sub-tile shape. The algorithm handles operand shapes, which are multiples |
| 128 | // of this shape. |
| 129 | SmallVector<int64_t> subTileShape; |
| 130 | |
| 131 | // Create the matrix multiply and accumulate operation according to `mmlaOp`. |
| 132 | Value createMMLA(PatternRewriter &rewriter, Location loc, Value acc, |
| 133 | Value lhs, Value rhs) { |
| 134 | |
| 135 | if (swapOperands) |
| 136 | std::swap(a&: lhs, b&: rhs); |
| 137 | switch (mmlaOp) { |
| 138 | case MMLA::SignedInt: |
| 139 | return rewriter.createOrFold<arm_neon::SmmlaOp>(location: loc, args: acc.getType(), args&: acc, |
| 140 | args&: lhs, args&: rhs); |
| 141 | case MMLA::UnsignedInt: |
| 142 | return rewriter.createOrFold<arm_neon::UmmlaOp>(location: loc, args: acc.getType(), args&: acc, |
| 143 | args&: lhs, args&: rhs); |
| 144 | case MMLA::MixedInt: |
| 145 | return rewriter.createOrFold<arm_neon::UsmmlaOp>(location: loc, args: acc.getType(), args&: acc, |
| 146 | args&: lhs, args&: rhs); |
| 147 | case MMLA::Bfloat: |
| 148 | return arm_neon::BfmmlaOp::create(builder&: rewriter, location: loc, res: acc.getType(), acc, src1: lhs, |
| 149 | src2: rhs); |
| 150 | case MMLA::Nop: |
| 151 | llvm_unreachable("Uninitialized operation type" ); |
| 152 | } |
| 153 | llvm_unreachable("Unknown MMLA" ); |
| 154 | } |
| 155 | |
| 156 | // Check common preconditions for applying the patterns and initialize |
| 157 | // logical dimensions. |
| 158 | LogicalResult matchAndInit(vector::ContractionOp op, |
| 159 | PatternRewriter &rewriter) { |
| 160 | // Check iterator types for matrix multiplication. |
| 161 | SmallVector<vector::IteratorType> itTypes = op.getIteratorTypesArray(); |
| 162 | if ((itTypes.size() != 3 || itTypes[0] != vector::IteratorType::parallel || |
| 163 | itTypes[1] != vector::IteratorType::parallel || |
| 164 | itTypes[2] != vector::IteratorType::reduction) && |
| 165 | (itTypes.size() != 2 || itTypes[0] != vector::IteratorType::parallel || |
| 166 | itTypes[1] != vector::IteratorType::reduction)) |
| 167 | return rewriter.notifyMatchFailure( |
| 168 | arg&: op, msg: "iterator types do not correspond to matrix multiplication" ); |
| 169 | |
| 170 | // Avoid 0-D vectors and 1-D rhs: |
| 171 | VectorType lhsType = op.getLhsType(); |
| 172 | VectorType rhsType = op.getRhsType(); |
| 173 | if (!lhsType.hasRank() || !rhsType.hasRank() || lhsType.getRank() > 2 || |
| 174 | rhsType.getRank() != 2) |
| 175 | return rewriter.notifyMatchFailure(arg&: op, msg: "Invalid operand rank" ); |
| 176 | |
| 177 | // This codegen does not work for scalable vectors. Return failure so this |
| 178 | // pattern is not accidentally chosen over patterns that lower to ArmSVE. |
| 179 | if (lhsType.isScalable() || rhsType.isScalable()) |
| 180 | return rewriter.notifyMatchFailure(arg&: op, |
| 181 | msg: "Not applicable to scalable vectors" ); |
| 182 | |
| 183 | // Initialize dimensions and check for a matching K dimension. |
| 184 | dimM = lhsType.getDimSize(idx: 0); |
| 185 | dimN = rhsType.getDimSize(idx: 0); |
| 186 | dimK = rhsType.getDimSize(idx: 1); |
| 187 | |
| 188 | int64_t lhsDimK; |
| 189 | if (lhsType.getRank() == 1) { |
| 190 | dimM = 1; |
| 191 | lhsDimK = lhsType.getDimSize(idx: 0); |
| 192 | } else { |
| 193 | lhsDimK = lhsType.getDimSize(idx: 1); |
| 194 | } |
| 195 | |
| 196 | if (lhsDimK != dimK) |
| 197 | return rewriter.notifyMatchFailure(arg&: op, msg: "Dimensions mismatch" ); |
| 198 | |
| 199 | return success(); |
| 200 | } |
| 201 | |
| 202 | public: |
| 203 | void lower(vector::ContractionOp op, PatternRewriter &rewriter) { |
| 204 | // Create some convenience types. |
| 205 | auto inputElementType = cast<ShapedType>(Val: lhs.getType()).getElementType(); |
| 206 | auto accElementType = cast<ShapedType>(Val: acc.getType()).getElementType(); |
| 207 | auto inputExpandedType = |
| 208 | VectorType::get(shape: {2, subTileShape.back()}, elementType: inputElementType); |
| 209 | auto outputExpandedType = VectorType::get(shape: {2, 2}, elementType: accElementType); |
| 210 | |
| 211 | // One-dimensional representation of logical sub-tiles as required by the |
| 212 | // ArmNeon ops. |
| 213 | auto collapsedInputType = |
| 214 | VectorType::get(shape: inputExpandedType.getNumElements(), elementType: inputElementType); |
| 215 | auto collapsedOutputType = |
| 216 | VectorType::get(shape: outputExpandedType.getNumElements(), elementType: accElementType); |
| 217 | |
| 218 | // Get indexing maps for a more concise/convenient access. |
| 219 | auto indexingMaps = op.getIndexingMapsArray(); |
| 220 | AffineMap &lhsPermutationMap = indexingMaps[0]; |
| 221 | AffineMap &rhsPermutationMap = indexingMaps[1]; |
| 222 | AffineMap &accPermutationMap = indexingMaps[2]; |
| 223 | |
| 224 | Location loc = op.getLoc(); |
| 225 | |
| 226 | // Initial accumulator for the final result. This is the un-tiled result if |
| 227 | // tiling is done. |
| 228 | Value result = |
| 229 | arith::ConstantOp::create(builder&: rewriter, location: loc, result: op.getResultType(), |
| 230 | value: rewriter.getZeroAttr(type: op.getResultType())); |
| 231 | |
| 232 | SmallVector<int64_t, 3> loopOrder = {0, 1}; |
| 233 | if (iterationBounds.size() == 3) |
| 234 | loopOrder.push_back(Elt: 2); |
| 235 | |
| 236 | // Keep track of the previous accumulator when tiling over K. |
| 237 | Value kAcc; |
| 238 | for (SmallVector<int64_t> offsets : |
| 239 | StaticTileOffsetRange(iterationBounds, subTileShape, loopOrder)) { |
| 240 | // Helper to compute the new shape of each operand and extract the slice. |
| 241 | auto extractOperand = [&](Value operand, AffineMap permutationMap, |
| 242 | ArrayRef<int64_t> operandOffsets) { |
| 243 | SmallVector<int64_t> operandShape = applyPermutationMap( |
| 244 | map: permutationMap, source: ArrayRef<int64_t>(subTileShape)); |
| 245 | SmallVector<int64_t> operandStrides(operandOffsets.size(), 1); |
| 246 | return rewriter.createOrFold<vector::ExtractStridedSliceOp>( |
| 247 | location: loc, args&: operand, args&: operandOffsets, args&: operandShape, args&: operandStrides); |
| 248 | }; |
| 249 | |
| 250 | // Extract tiled lhs, rhs, and acc |
| 251 | SmallVector<int64_t> lhsOffsets = |
| 252 | applyPermutationMap(map: lhsPermutationMap, source: ArrayRef<int64_t>(offsets)); |
| 253 | Value tiledLhs = extractOperand(lhs, lhsPermutationMap, lhsOffsets); |
| 254 | SmallVector<int64_t> rhsOffsets = |
| 255 | applyPermutationMap(map: rhsPermutationMap, source: ArrayRef<int64_t>(offsets)); |
| 256 | Value tiledRhs = extractOperand(rhs, rhsPermutationMap, rhsOffsets); |
| 257 | SmallVector<int64_t> accOffsets = |
| 258 | applyPermutationMap(map: accPermutationMap, source: ArrayRef<int64_t>(offsets)); |
| 259 | Value tiledAcc = extractOperand(acc, accPermutationMap, accOffsets); |
| 260 | |
| 261 | // With vecmat, tiled LHS and ACC will contain only one of 2 necessary |
| 262 | // rows along dimM. Expand their shapes to match the ArmNeon op. |
| 263 | if (dimM == 1) { |
| 264 | auto expandRowVector = [&](Value tiledOperand, |
| 265 | VectorType expandedTypeType) { |
| 266 | auto emptyOperand = |
| 267 | arith::ConstantOp::create(builder&: rewriter, location: loc, result: expandedTypeType, |
| 268 | value: rewriter.getZeroAttr(type: expandedTypeType)); |
| 269 | SmallVector<int64_t> offsets( |
| 270 | cast<ShapedType>(Val: emptyOperand.getType()).getRank(), 0); |
| 271 | SmallVector<int64_t> strides( |
| 272 | cast<ShapedType>(Val: tiledOperand.getType()).getRank(), 1); |
| 273 | return rewriter.createOrFold<vector::InsertStridedSliceOp>( |
| 274 | location: loc, args&: tiledOperand, args&: emptyOperand, args&: offsets, args&: strides); |
| 275 | }; |
| 276 | tiledLhs = expandRowVector(tiledLhs, inputExpandedType); |
| 277 | tiledAcc = expandRowVector(tiledAcc, outputExpandedType); |
| 278 | } |
| 279 | |
| 280 | // Transpose ACC if doing signed by unsigned multiplication, because we're |
| 281 | // using the instruction for unsigned by signed multiplication with |
| 282 | // reversed operands. |
| 283 | if (swapOperands) |
| 284 | tiledAcc = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: tiledAcc, |
| 285 | permutation: ArrayRef<int64_t>({1, 0})); |
| 286 | |
| 287 | // Collapse tiled operands to 1D vectors required by the ArmNeon ops |
| 288 | auto collapsedLhs = rewriter.createOrFold<vector::ShapeCastOp>( |
| 289 | location: tiledLhs.getLoc(), args&: collapsedInputType, args&: tiledLhs); |
| 290 | auto collapsedRhs = rewriter.createOrFold<vector::ShapeCastOp>( |
| 291 | location: tiledRhs.getLoc(), args&: collapsedInputType, args&: tiledRhs); |
| 292 | |
| 293 | bool initialKAcc = offsets.back() == 0; |
| 294 | Value collapsedRes; |
| 295 | if (!initialKAcc) { |
| 296 | collapsedRes = kAcc; |
| 297 | } else { |
| 298 | collapsedRes = rewriter.createOrFold<vector::ShapeCastOp>( |
| 299 | location: tiledAcc.getLoc(), args&: collapsedOutputType, args&: tiledAcc); |
| 300 | } |
| 301 | |
| 302 | // Insert contract op |
| 303 | kAcc = |
| 304 | createMMLA(rewriter, loc, acc: collapsedRes, lhs: collapsedLhs, rhs: collapsedRhs); |
| 305 | |
| 306 | // Reshape output back to 2D |
| 307 | Value tiledRes = rewriter.createOrFold<vector::ShapeCastOp>( |
| 308 | location: kAcc.getLoc(), args: tiledAcc.getType(), args&: kAcc); |
| 309 | |
| 310 | // Because of the reversed operands the result is obtained transposed. |
| 311 | // Transpose it back, |
| 312 | if (swapOperands) |
| 313 | tiledRes = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: tiledRes, |
| 314 | permutation: ArrayRef<int64_t>({1, 0})); |
| 315 | |
| 316 | // With vecmat, only one row of tiled ACC can be inserted into the final |
| 317 | // result |
| 318 | if (dimM == 1) |
| 319 | tiledRes = rewriter.createOrFold<vector::ExtractOp>(location: loc, args&: tiledRes, args: 0); |
| 320 | |
| 321 | // Insert the tiled result back into the non tiled result of the |
| 322 | // contract op. |
| 323 | SmallVector<int64_t> strides( |
| 324 | cast<ShapedType>(Val: tiledRes.getType()).getRank(), 1); |
| 325 | result = rewriter.createOrFold<vector::InsertStridedSliceOp>( |
| 326 | location: loc, args&: tiledRes, args&: result, args&: accOffsets, args&: strides); |
| 327 | } |
| 328 | |
| 329 | rewriter.replaceOp(op, newValues: result); |
| 330 | } |
| 331 | }; |
| 332 | |
| 333 | class VectorContractRewriterI8MM : public VectorContractRewriter { |
| 334 | public: |
| 335 | LogicalResult matchAndInit(vector::ContractionOp op, |
| 336 | PatternRewriter &rewriter) { |
| 337 | if (failed(Result: VectorContractRewriter::matchAndInit(op, rewriter))) |
| 338 | return failure(); |
| 339 | |
| 340 | // Unrolling patterns can handle any [2, 2, 8] shaped multiple of inputs for |
| 341 | // tiling. |
| 342 | if ((dimM != 1 && dimM % 2 != 0) || dimN % 2 != 0 || dimK % 8 != 0) |
| 343 | return rewriter.notifyMatchFailure(arg&: op, msg: "Unsupported operand shapes" ); |
| 344 | |
| 345 | // Check inputs are sign-/zero- extensions from iN (N <= 8) to i32. Get the |
| 346 | // values before the extension. All four signed/unsigned combinations for |
| 347 | // input operands are supported, but they are lowered to different |
| 348 | // operations. Determine which is the appropriate operation to lower to. |
| 349 | mmlaOp = MMLA::SignedInt; |
| 350 | auto maybeLhs = getExtOperand<arith::ExtSIOp>(v: op.getLhs()); |
| 351 | if (!maybeLhs) { |
| 352 | mmlaOp = MMLA::UnsignedInt; |
| 353 | maybeLhs = getExtOperand<arith::ExtUIOp>(v: op.getLhs()); |
| 354 | } |
| 355 | if (!maybeLhs) |
| 356 | return rewriter.notifyMatchFailure( |
| 357 | arg&: op, msg: "LHS is not a sign- or zero- extended iN, N <= 8" ); |
| 358 | |
| 359 | auto maybeRhs = getExtOperand<arith::ExtSIOp>(v: op.getRhs()); |
| 360 | if (maybeRhs) { |
| 361 | if (mmlaOp == MMLA::UnsignedInt) |
| 362 | mmlaOp = MMLA::MixedInt; |
| 363 | } else { |
| 364 | if (mmlaOp == MMLA::SignedInt) { |
| 365 | mmlaOp = MMLA::MixedInt; |
| 366 | swapOperands = true; |
| 367 | } |
| 368 | maybeRhs = getExtOperand<arith::ExtUIOp>(v: op.getRhs()); |
| 369 | } |
| 370 | |
| 371 | if (!maybeRhs) |
| 372 | return rewriter.notifyMatchFailure( |
| 373 | arg&: op, msg: "RHS is not a sign- or zero- extended iN, N <= 8" ); |
| 374 | |
| 375 | lhs = *maybeLhs; |
| 376 | rhs = *maybeRhs; |
| 377 | acc = op.getAcc(); |
| 378 | |
| 379 | // Extend inputs from iN, N < 8 to i8. |
| 380 | Location loc = op.getLoc(); |
| 381 | auto lhsExtInType = cast<VectorType>(Val: lhs.getType()); |
| 382 | if (lhsExtInType.getElementTypeBitWidth() < 8) |
| 383 | lhs = extendSmallIntVector(loc, srcTy: lhsExtInType, val: lhs, |
| 384 | /* signExt */ |
| 385 | (mmlaOp == MMLA::SignedInt || |
| 386 | (mmlaOp == MMLA::MixedInt && !swapOperands)), |
| 387 | rewriter); |
| 388 | |
| 389 | auto rhsExtInType = cast<VectorType>(Val: rhs.getType()); |
| 390 | if (rhsExtInType.getElementTypeBitWidth() < 8) |
| 391 | rhs = extendSmallIntVector(loc, srcTy: rhsExtInType, val: rhs, |
| 392 | /* signExt */ |
| 393 | (mmlaOp == MMLA::SignedInt || |
| 394 | (mmlaOp == MMLA::MixedInt && swapOperands)), |
| 395 | rewriter); |
| 396 | |
| 397 | // Initialize parameters for unrolling. |
| 398 | iterationBounds = *op.getIterationShapeToUnroll(); |
| 399 | if (iterationBounds.size() == 3) |
| 400 | subTileShape = SmallVector<int64_t>({dimM == 1 ? 1 : 2, 2, 8}); |
| 401 | else |
| 402 | subTileShape = SmallVector<int64_t>({2, 8}); |
| 403 | |
| 404 | return success(); |
| 405 | } |
| 406 | }; |
| 407 | |
| 408 | class VectorContractRewriterBFMMLA : public VectorContractRewriter { |
| 409 | public: |
| 410 | LogicalResult matchAndInit(vector::ContractionOp op, |
| 411 | PatternRewriter &rewriter) { |
| 412 | |
| 413 | if (failed(Result: VectorContractRewriter::matchAndInit(op, rewriter))) |
| 414 | return failure(); |
| 415 | |
| 416 | // Unrolling patterns can handle any [2, 2, 4] shaped multiple of inputs for |
| 417 | // tiling. |
| 418 | if ((dimM != 1 && dimM % 2 != 0) || dimN % 2 != 0 || dimK % 4 != 0) |
| 419 | return rewriter.notifyMatchFailure(arg&: op, msg: "Unsupported operand shapes" ); |
| 420 | |
| 421 | // Check the output is a vector of Float32 elements. |
| 422 | auto outTy = dyn_cast<VectorType>(Val: op.getResultType()); |
| 423 | if (!outTy || outTy.getElementType() != rewriter.getF32Type()) |
| 424 | return rewriter.notifyMatchFailure(arg&: op, |
| 425 | msg: "output type is not a vector of f32" ); |
| 426 | |
| 427 | // Check the inputs are vectors of BFloat16 elements. |
| 428 | if (op.getLhsType().getElementType() != rewriter.getBF16Type()) |
| 429 | return rewriter.notifyMatchFailure(arg&: op, |
| 430 | msg: "input type is not a vector of bf16" ); |
| 431 | |
| 432 | mmlaOp = MMLA::Bfloat; |
| 433 | swapOperands = false; |
| 434 | lhs = op.getLhs(); |
| 435 | rhs = op.getRhs(); |
| 436 | acc = op.getAcc(); |
| 437 | |
| 438 | // Initialize parameters for unrolling. |
| 439 | iterationBounds = *op.getIterationShapeToUnroll(); |
| 440 | if (iterationBounds.size() == 3) |
| 441 | subTileShape = SmallVector<int64_t>({dimM == 1 ? 1 : 2, 2, 4}); |
| 442 | else |
| 443 | subTileShape = SmallVector<int64_t>({2, 4}); |
| 444 | |
| 445 | return success(); |
| 446 | } |
| 447 | }; |
| 448 | |
| 449 | /// Lowering from a vector::contractOp arm neon smmla intrinsic. This will tile |
| 450 | /// any vector.contract into multiple smmla instructions with unrolling so long |
| 451 | /// as [2,2,8] is a divisor of its shape. It can also process vecmats with dimM |
| 452 | /// = 1 (either explicitly or inferred if LHS has only dimK) If no unrolling is |
| 453 | /// necessary, a single smmla instruction is emitted. |
| 454 | class LowerContractionToNeonI8MMPattern |
| 455 | : public OpRewritePattern<vector::ContractionOp> { |
| 456 | public: |
| 457 | using Base::Base; |
| 458 | LogicalResult matchAndRewrite(vector::ContractionOp op, |
| 459 | PatternRewriter &rewriter) const override { |
| 460 | if (cast<vector::MaskableOpInterface>(Val: op.getOperation()).isMasked()) |
| 461 | return rewriter.notifyMatchFailure( |
| 462 | arg&: op, msg: "masked contractions are not supported" ); |
| 463 | |
| 464 | VectorContractRewriterI8MM vcr; |
| 465 | if (failed(Result: vcr.matchAndInit(op, rewriter))) |
| 466 | return failure(); |
| 467 | vcr.lower(op, rewriter); |
| 468 | |
| 469 | return success(); |
| 470 | } |
| 471 | }; |
| 472 | |
| 473 | class LowerContractionToNeonBFMMLAPattern |
| 474 | : public OpRewritePattern<vector::ContractionOp> { |
| 475 | public: |
| 476 | using Base::Base; |
| 477 | LogicalResult matchAndRewrite(vector::ContractionOp op, |
| 478 | PatternRewriter &rewriter) const override { |
| 479 | if (cast<vector::MaskableOpInterface>(Val: op.getOperation()).isMasked()) |
| 480 | return rewriter.notifyMatchFailure( |
| 481 | arg&: op, msg: "masked contractions are not supported" ); |
| 482 | |
| 483 | VectorContractRewriterBFMMLA vcr; |
| 484 | if (failed(Result: vcr.matchAndInit(op, rewriter))) |
| 485 | return failure(); |
| 486 | vcr.lower(op, rewriter); |
| 487 | |
| 488 | return success(); |
| 489 | } |
| 490 | }; |
| 491 | |
| 492 | } // namespace |
| 493 | |
| 494 | void mlir::arm_neon::populateLowerContractionToNeonI8MMPatterns( |
| 495 | RewritePatternSet &patterns) { |
| 496 | MLIRContext *context = patterns.getContext(); |
| 497 | patterns.add<LowerContractionToNeonI8MMPattern>(arg&: context, /*benefit=*/args: 2); |
| 498 | } |
| 499 | |
| 500 | void mlir::arm_neon::populateLowerContractionToNeonBFMMLAPatterns( |
| 501 | RewritePatternSet &patterns) { |
| 502 | MLIRContext *context = patterns.getContext(); |
| 503 | patterns.add<LowerContractionToNeonBFMMLAPattern>(arg&: context, /*benefit=*/args: 2); |
| 504 | } |
| 505 | |