| 1 | //===- VectorContractBF16ToFMA.cpp-----------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 10 | #include "mlir/Dialect/MemRef/Utils/MemRefUtils.h" |
| 11 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 12 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 13 | #include "mlir/Dialect/Vector/Utils/VectorUtils.h" |
| 14 | #include "mlir/Dialect/X86/Transforms.h" |
| 15 | #include "mlir/Dialect/X86/Utils/X86Utils.h" |
| 16 | #include "mlir/Dialect/X86/X86Dialect.h" |
| 17 | |
| 18 | #include "mlir/IR/BuiltinAttributes.h" |
| 19 | #include "mlir/IR/Dominance.h" |
| 20 | #include "mlir/IR/PatternMatch.h" |
| 21 | |
| 22 | #include "mlir/Pass/Pass.h" |
| 23 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 24 | #include "llvm/Support/Casting.h" |
| 25 | |
| 26 | using namespace mlir; |
| 27 | using namespace mlir::vector; |
| 28 | using namespace mlir::x86; |
| 29 | |
| 30 | // Verifies that the LHS and RHS operands of a vector.contract are load or |
| 31 | // vector.transfer_read operations on a memref source buffer, and checks |
| 32 | // their bounds, dimensions, offsets, and strides. |
| 33 | static bool validateVectorContractOperands(Value prodOp, bool isVnni) { |
| 34 | Operation *defOp = prodOp.getDefiningOp(); |
| 35 | if (!defOp) |
| 36 | return false; |
| 37 | |
| 38 | if (auto readOp = prodOp.getDefiningOp<mlir::vector::TransferReadOp>()) { |
| 39 | if (readOp.hasOutOfBoundsDim()) |
| 40 | return false; |
| 41 | |
| 42 | if (!readOp.getPermutationMap().isMinorIdentity()) |
| 43 | return false; |
| 44 | } |
| 45 | |
| 46 | Value srcBuff; |
| 47 | SmallVector<OpFoldResult> indexVals; |
| 48 | llvm::TypeSwitch<Operation *>(defOp).Case<TransferReadOp, LoadOp>( |
| 49 | caseFn: [&](auto readOp) { |
| 50 | srcBuff = readOp.getOperand(0); |
| 51 | indexVals = SmallVector<OpFoldResult>(readOp.getIndices().begin(), |
| 52 | readOp.getIndices().end()); |
| 53 | }); |
| 54 | |
| 55 | if (!srcBuff) |
| 56 | return false; |
| 57 | |
| 58 | // Return false, if the source is not a memref type |
| 59 | Type srcType = srcBuff.getType(); |
| 60 | if (!llvm::isa<MemRefType>(Val: srcType)) |
| 61 | return false; |
| 62 | |
| 63 | // Return false if the two innermost strides of the memref are not contiguous. |
| 64 | // The x86.avx.cvt.packed.even/odd.indexed_to_f32 operations require |
| 65 | // an eight-element tuple of bf16 values to be contiguous. |
| 66 | int dimsToCheck = isVnni ? 2 : 1; |
| 67 | if (!cast<mlir::MemRefType>(Val&: srcType).areTrailingDimsContiguous(n: dimsToCheck)) |
| 68 | return false; |
| 69 | |
| 70 | // Return false if the vnni offset of load or transfer_read is not zero. |
| 71 | if (isVnni && getConstantIntValue(ofr: indexVals.back()) != 0) |
| 72 | return false; |
| 73 | |
| 74 | return true; |
| 75 | } |
| 76 | |
| 77 | // This function retrieves the source operation of the load or transfer |
| 78 | // reads and creates subviews for the BF16 packed-operations to |
| 79 | // broadcast or load BF16 elements as F32 packed elements. |
| 80 | // |
| 81 | // Example(1) Unit Dim: |
| 82 | // ``` |
| 83 | // vector.load %arg0[%c0, %c0, %c0]:memref<4x1x2xbf16>,vector<1x1x2xbf16> |
| 84 | // ``` |
| 85 | // to |
| 86 | // ``` |
| 87 | // memref.subview %arg0[%c0,%c0,%c1]:memref<4x1x2xbf16> to memref<1x1x1xbf16> |
| 88 | // memref.subview %arg0[%c0,%c0,%c0]:memref<4x1x2xbf16> to memref<1x1x1xbf16> |
| 89 | // ``` |
| 90 | // |
| 91 | // Example(2) Non-unit Dim: |
| 92 | // ``` |
| 93 | // vector.load %arg1[%c0, %c0, %c0]:memref<1x32x2xbf16>,vector<1x8x2xbf16> |
| 94 | // ``` |
| 95 | // to |
| 96 | // ``` |
| 97 | // memref.subview %arg1[%c0,%c0,%c0]:memref<1x32x2xbf16> to memref<1x8x2xbf16> |
| 98 | // ``` |
| 99 | static SmallVector<memref::SubViewOp> |
| 100 | getSubviewFromVectorInput(Location loc, PatternRewriter &rewriter, Value prodOp, |
| 101 | ArrayRef<int64_t> nonUnitDimShape, bool isUnitDim, |
| 102 | bool isVNNI) { |
| 103 | |
| 104 | Operation *defOp = prodOp.getDefiningOp(); |
| 105 | |
| 106 | Value srcBuff; |
| 107 | SmallVector<OpFoldResult> indexVals; |
| 108 | llvm::TypeSwitch<Operation *>(defOp).Case<TransferReadOp, LoadOp>( |
| 109 | caseFn: [&](auto readOp) { |
| 110 | srcBuff = readOp.getOperand(0); |
| 111 | indexVals = SmallVector<OpFoldResult>(readOp.getIndices().begin(), |
| 112 | readOp.getIndices().end()); |
| 113 | }); |
| 114 | |
| 115 | int64_t mnDimSize = 1; |
| 116 | unsigned mnDimIdx = 0; |
| 117 | |
| 118 | if (!isUnitDim) { |
| 119 | for (auto it : llvm::enumerate(First&: nonUnitDimShape)) { |
| 120 | if (it.value() != 1) { |
| 121 | mnDimSize = it.value(); |
| 122 | mnDimIdx = it.index(); |
| 123 | break; |
| 124 | } |
| 125 | } |
| 126 | } |
| 127 | |
| 128 | auto one = rewriter.getIndexAttr(value: 1); |
| 129 | SmallVector<memref::SubViewOp> subviews; |
| 130 | |
| 131 | if (!isVNNI) { |
| 132 | SmallVector<OpFoldResult> strides(indexVals.size(), one); |
| 133 | SmallVector<OpFoldResult> sizes(indexVals.size(), one); |
| 134 | // Retrive twice the nonUnit dim BF16 element for both even and odd |
| 135 | // index elements. |
| 136 | if (!isUnitDim) |
| 137 | mnDimSize = 2 * mnDimSize; |
| 138 | sizes[mnDimIdx] = rewriter.getIndexAttr(value: mnDimSize); |
| 139 | auto subview = memref::SubViewOp::create(builder&: rewriter, location: loc, source: srcBuff, offsets: indexVals, |
| 140 | sizes, strides); |
| 141 | subviews.push_back(Elt: subview); |
| 142 | return subviews; |
| 143 | } |
| 144 | |
| 145 | int vnniDimSize = isUnitDim ? 1 : 2; |
| 146 | auto nonVNNIDimSize = indexVals.size() - 1; |
| 147 | // Create the size and stride offsets. |
| 148 | SmallVector<OpFoldResult> strides(indexVals.size(), one); |
| 149 | SmallVector<OpFoldResult> sizes(nonVNNIDimSize, one); |
| 150 | |
| 151 | sizes.push_back(Elt: rewriter.getIndexAttr(value: vnniDimSize)); |
| 152 | |
| 153 | // update the unit/nonUnit Dim size either it is A(LHS) or B(RHS). |
| 154 | sizes[mnDimIdx] = rewriter.getIndexAttr(value: mnDimSize); |
| 155 | |
| 156 | // for unitDim, first broadcast odd element, so index is set to 1. |
| 157 | if (isUnitDim) |
| 158 | indexVals[indexVals.size() - 1] = rewriter.getIndexAttr(value: 1); |
| 159 | |
| 160 | auto subview = memref::SubViewOp::create(builder&: rewriter, location: loc, source: srcBuff, offsets: indexVals, |
| 161 | sizes, strides); |
| 162 | subviews.push_back(Elt: subview); |
| 163 | |
| 164 | // For unit-dims, two subviews should be created for the odd and even |
| 165 | // element in the VNNI tuple (2xbf16) because x86.avx.bcst_to_f32.packed |
| 166 | // op loads and broadcast the first BF16 element into packed F32. It |
| 167 | // cannot distinguish between even and odd BF16 elements within a |
| 168 | // packed pair. |
| 169 | // |
| 170 | // Example: |
| 171 | // memref.subview %arg0[%c0,%c1]:memref<1x2xbf16> to memref<1x1xbf16> // Odd |
| 172 | // memref.subview %arg0[%c0,%c0]:memref<1x2xbf16> to memref<1x1xbf16> // Even |
| 173 | if (mnDimSize == 1) { |
| 174 | indexVals[indexVals.size() - 1] = rewriter.getIndexAttr(value: 0); |
| 175 | sizes[indexVals.size() - 1] = rewriter.getIndexAttr(value: 1); |
| 176 | |
| 177 | auto unitDimEvenIdxSubview = memref::SubViewOp::create( |
| 178 | builder&: rewriter, location: loc, source: srcBuff, offsets: indexVals, sizes, strides); |
| 179 | subviews.push_back(Elt: unitDimEvenIdxSubview); |
| 180 | } |
| 181 | |
| 182 | return subviews; |
| 183 | } |
| 184 | |
| 185 | // Implements outer product contraction as a sequence of BF16-packed |
| 186 | // operation even/odd loads and FMA operations. |
| 187 | // |
| 188 | // For example (VNNI packed): |
| 189 | // ``` |
| 190 | // %1 = vector.load from memref (%m1) -> vector<1x1x2xbf16> |
| 191 | // %2 = vector.load from memref (%m2) -> vector<1x8x2xbf16> |
| 192 | // return vector.contract %1, %2, %arg1 |
| 193 | // ``` |
| 194 | // to |
| 195 | // ``` |
| 196 | // %1 = x86.avx.bcst_to_f32.packed %m1[c1] -> vector<8xf32> |
| 197 | // %2 = x86.avx.cvt.packed.odd.indexed_to_f32 %m2 -> vector<8xf32> |
| 198 | // %3 = vector.fma %1, %2, %arg1 |
| 199 | // %4 = x86.avx.bcst_to_f32.packed %m1[c0] -> vector<8xf32> |
| 200 | // %5 = x86.avx.cvt.packed.even.indexed_to_f32 %m2 -> vector<8xf32> |
| 201 | // return vector.fma %4, %5, %3 |
| 202 | // ``` |
| 203 | // |
| 204 | // For example (Flat layout): |
| 205 | // ``` |
| 206 | // %1 = vector.load from memref (%m1) -> vector<1x1xbf16> |
| 207 | // %2 = vector.load from memref (%m2) -> vector<1x8xbf16> |
| 208 | // %3 = vector.contract %1, %2, %arg1 |
| 209 | // %4 = vector.load from memref (%m2) -> vector<1x8xbf16> |
| 210 | // %5 = vector.contract %1, %4, %arg2 |
| 211 | // scf.yield %3, %4 |
| 212 | // ``` |
| 213 | // to |
| 214 | // ``` |
| 215 | // %1 = x86.avx.bcst_to_f32.packed %m1[c0] -> vector<8xf32> |
| 216 | // %2 = x86.avx.cvt.packed.even.indexed_to_f32 %m2 -> vector<8xf32> |
| 217 | // %3 = vector.fma %1, %2, %arg1 |
| 218 | // %4 = x86.avx.cvt.packed.odd.indexed_to_f32 %m2 -> vector<8xf32> |
| 219 | // %5 = vector.fma %1, %4, %arg2 |
| 220 | // scf.yield %3, %5 |
| 221 | struct VectorContractBF16ToFMA : public AddContractionOpRewritePattern { |
| 222 | using AddContractionOpRewritePattern::AddContractionOpRewritePattern; |
| 223 | |
| 224 | LogicalResult |
| 225 | matchAndRewriteAddContractOp(vector::ContractionOp contractOp, |
| 226 | PatternRewriter &rewriter) const override { |
| 227 | |
| 228 | // TODO: Move this validation to a common utility folder. Planned to |
| 229 | // do once (code refactoring), all architecture specific nanokernel |
| 230 | // passes are merged into the repo. |
| 231 | VectorType lhsTy = contractOp.getLhsType(); |
| 232 | if (!lhsTy.getElementType().isBF16()) |
| 233 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 234 | msg: "Only BF16 lowering is supported." ); |
| 235 | |
| 236 | bool isVnni = isInVnniLayout(op: contractOp.getOperation(), |
| 237 | indexingMaps: contractOp.getIndexingMapsArray(), |
| 238 | /*blockingFactor=*/2); |
| 239 | |
| 240 | VectorType accTy = dyn_cast<VectorType>(Val: contractOp.getAccType()); |
| 241 | if (!accTy) |
| 242 | return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Wrong accmulator type." ); |
| 243 | |
| 244 | if (!accTy.getElementType().isF32()) |
| 245 | return rewriter.notifyMatchFailure( |
| 246 | arg&: contractOp, msg: "Only F32 acumulation supported for BF16 type." ); |
| 247 | |
| 248 | ArrayRef<int64_t> accShape = accTy.getShape(); |
| 249 | llvm::SmallVector<int64_t> nonUnitDimAcc; |
| 250 | llvm::copy_if(Range&: accShape, Out: std::back_inserter(x&: nonUnitDimAcc), |
| 251 | P: [](int64_t dim) { return dim != 1; }); |
| 252 | if (nonUnitDimAcc.size() != 1) |
| 253 | return rewriter.notifyMatchFailure( |
| 254 | arg&: contractOp, msg: "A or B should be a non-unit dim in acc." ); |
| 255 | |
| 256 | ArrayRef<int64_t> lhsShape = lhsTy.getShape(); |
| 257 | llvm::SmallVector<int64_t> nonUnitDimLhs; |
| 258 | llvm::copy_if(Range&: lhsShape, Out: std::back_inserter(x&: nonUnitDimLhs), |
| 259 | P: [](int64_t dim) { return dim != 1; }); |
| 260 | |
| 261 | VectorType rhsTy = contractOp.getRhsType(); |
| 262 | ArrayRef<int64_t> rhsShape = rhsTy.getShape(); |
| 263 | llvm::SmallVector<int64_t> nonUnitDimRhs; |
| 264 | llvm::copy_if(Range&: rhsShape, Out: std::back_inserter(x&: nonUnitDimRhs), |
| 265 | P: [](int64_t dim) { return dim != 1; }); |
| 266 | |
| 267 | if (isVnni && (nonUnitDimLhs.size() - 1) > 0 && |
| 268 | (nonUnitDimRhs.size() - 1) > 0) |
| 269 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 270 | msg: "Excepts unit dimensions for either " |
| 271 | "LHS or RHS shape other than VNNI." ); |
| 272 | |
| 273 | if (isVnni && (nonUnitDimLhs.size() - 1) != 1 && |
| 274 | (nonUnitDimRhs.size() - 1) != 1) |
| 275 | return rewriter.notifyMatchFailure( |
| 276 | arg&: contractOp, |
| 277 | msg: "Excepts a one non-unit A/B dimension for either LHS or RHS shape." ); |
| 278 | |
| 279 | if (!isVnni && nonUnitDimLhs.size() > 0 && nonUnitDimRhs.size() > 0) |
| 280 | return rewriter.notifyMatchFailure(arg&: contractOp, |
| 281 | msg: "Excepts unit dimensions for either " |
| 282 | "LHS or RHS shape." ); |
| 283 | |
| 284 | if (!isVnni && nonUnitDimLhs.size() != 1 && nonUnitDimRhs.size() != 1) |
| 285 | return rewriter.notifyMatchFailure( |
| 286 | arg&: contractOp, |
| 287 | msg: "Excepts a one non-unit A/B dimension for either LHS or RHS shape." ); |
| 288 | |
| 289 | // Non-unit dimensions should match the vector length of BF16. |
| 290 | unsigned int nonUnitDim = nonUnitDimAcc.front(); |
| 291 | if (nonUnitDim != 4 && nonUnitDim != 8) |
| 292 | return rewriter.notifyMatchFailure( |
| 293 | arg&: contractOp, msg: "BF16 packed load operation expects non-unit (LHR or " |
| 294 | "RHS) dim and acc dim of size 4/8." ); |
| 295 | |
| 296 | if (!validateVectorContractOperands(prodOp: contractOp.getLhs(), isVnni) || |
| 297 | !validateVectorContractOperands(prodOp: contractOp.getRhs(), isVnni)) { |
| 298 | return rewriter.notifyMatchFailure( |
| 299 | arg&: contractOp, msg: "The LHS or RHS is in an invalid format. Either it has " |
| 300 | "false in-bounds, " |
| 301 | "a non-identity permutation map, a non-zero VNNI offset, " |
| 302 | "a non-memref " |
| 303 | "source, or a non-unit VNNI stride" ); |
| 304 | } |
| 305 | |
| 306 | // Lower vector.contract to FMAs with help of BF16 packed ops. |
| 307 | auto loc = contractOp.getLoc(); |
| 308 | |
| 309 | // create the unit-dimension LHS or RHS subview and the |
| 310 | // corresponding non-unit dimension LHS or RHS subview on the other-side. |
| 311 | // For example, if LHS has type vector<1x1x2xbf16> and RHS has type |
| 312 | // vector<1x8x2xbf16>, we create two subview for the LHS and one subview |
| 313 | // for the RHS. In the opposite case (non-unit dimension on the LHS), we |
| 314 | // do vice-versa. |
| 315 | |
| 316 | bool rhsHasMultipleNonUnitDims = nonUnitDimRhs.size() > 0; |
| 317 | if (isVnni) { |
| 318 | rhsHasMultipleNonUnitDims = (nonUnitDimRhs.size() - 1) > 0; |
| 319 | } |
| 320 | |
| 321 | // Select which operand is "unit" and which is "non-unit". |
| 322 | Value unitSrc = |
| 323 | rhsHasMultipleNonUnitDims ? contractOp.getLhs() : contractOp.getRhs(); |
| 324 | Value nonUnitSrc = |
| 325 | rhsHasMultipleNonUnitDims ? contractOp.getRhs() : contractOp.getLhs(); |
| 326 | |
| 327 | ArrayRef<int64_t> nonUnitDimShape = |
| 328 | rhsHasMultipleNonUnitDims ? rhsShape : lhsShape; |
| 329 | |
| 330 | // Get the pair vector.contract operation. The pair is decided on: |
| 331 | // (1) - the unitDim operand Lhs or Rhs should be same, |
| 332 | // (2) - the defining source memref should be same for nonUnitDim |
| 333 | // operation, (3) - the nonUnit dim offset difference between the |
| 334 | // vector.contracts should be 8. |
| 335 | vector::ContractionOp pairContractOp; |
| 336 | if (!isVnni) { |
| 337 | Operation *nextOp = contractOp; |
| 338 | while ((nextOp = nextOp->getNextNode())) { |
| 339 | auto contOp = dyn_cast<vector::ContractionOp>(Val: nextOp); |
| 340 | |
| 341 | if (!contOp) |
| 342 | continue; |
| 343 | |
| 344 | if (validatePairVectorContract(contractOp, pairContOp: contOp, |
| 345 | rhsHasMultipleNonUnitDims, |
| 346 | nonUnitDimValue: nonUnitDimAcc.front())) { |
| 347 | pairContractOp = contOp; |
| 348 | break; |
| 349 | } |
| 350 | } |
| 351 | |
| 352 | if (!pairContractOp) |
| 353 | return failure(); |
| 354 | |
| 355 | Operation *accReadOp0 = |
| 356 | traceToVectorReadLikeParentOperation(v: contractOp.getAcc()); |
| 357 | Operation *accReadOp1 = |
| 358 | traceToVectorReadLikeParentOperation(v: pairContractOp.getAcc()); |
| 359 | |
| 360 | // Iterate down to find the users of contact operations until it is store |
| 361 | // or transfer_write. |
| 362 | Operation *resultWriteOp0 = |
| 363 | traceToVectorWriteLikeUserOperation(v: contractOp.getResult()); |
| 364 | Operation *resultWriteOp1 = |
| 365 | traceToVectorWriteLikeUserOperation(v: pairContractOp.getResult()); |
| 366 | |
| 367 | if (!accReadOp0 || !accReadOp1) |
| 368 | return rewriter.notifyMatchFailure( |
| 369 | arg&: contractOp, |
| 370 | msg: "Operand doesn't have load or transfer_read as its parent op" ); |
| 371 | |
| 372 | if (!resultWriteOp0 || !resultWriteOp1) |
| 373 | return rewriter.notifyMatchFailure( |
| 374 | arg&: contractOp, |
| 375 | msg: "The use of contract operations are neither vector.store " |
| 376 | "or transfer_write or has multiple users" ); |
| 377 | |
| 378 | if (contractOp->getBlock() == accReadOp1->getBlock() && |
| 379 | contractOp->isBeforeInBlock(other: accReadOp1)) |
| 380 | return rewriter.notifyMatchFailure( |
| 381 | arg&: contractOp, msg: "The load/read operation of pair contract operation is " |
| 382 | "after the contractOp" ); |
| 383 | |
| 384 | if (pairContractOp->getBlock() == resultWriteOp0->getBlock() && |
| 385 | resultWriteOp0->isBeforeInBlock(other: pairContractOp)) { |
| 386 | return rewriter.notifyMatchFailure( |
| 387 | arg&: contractOp, msg: "The store/write operation of contract operation is " |
| 388 | "before the pair contract operation" ); |
| 389 | } |
| 390 | } |
| 391 | |
| 392 | // Build subviews. |
| 393 | auto unitDimSubview = getSubviewFromVectorInput( |
| 394 | loc, rewriter, prodOp: unitSrc, nonUnitDimShape, isUnitDim: true, isVNNI: isVnni); |
| 395 | |
| 396 | auto nonUnitDimSubview = getSubviewFromVectorInput( |
| 397 | loc, rewriter, prodOp: nonUnitSrc, nonUnitDimShape, isUnitDim: false, isVNNI: isVnni); |
| 398 | |
| 399 | auto castAcc = vector::ShapeCastOp::create( |
| 400 | builder&: rewriter, location: loc, |
| 401 | result: VectorType::get(shape: nonUnitDimAcc.front(), elementType: accTy.getElementType()), |
| 402 | source: contractOp.getAcc()); |
| 403 | VectorType dstType = |
| 404 | VectorType::get(shape: nonUnitDimAcc.front(), elementType: rewriter.getF32Type()); |
| 405 | |
| 406 | if (!isVnni) { |
| 407 | |
| 408 | // Validate and shuffle the accumulator |
| 409 | Operation *accReadOp0 = |
| 410 | traceToVectorReadLikeParentOperation(v: contractOp.getAcc()); |
| 411 | Operation *accReadOp1 = |
| 412 | traceToVectorReadLikeParentOperation(v: pairContractOp.getAcc()); |
| 413 | |
| 414 | if (!isa<arith::ConstantOp>(Val: accReadOp0)) { |
| 415 | // Shuffle the accumulators of the contract operations. |
| 416 | LogicalResult readShuffle = |
| 417 | shuffleAfterReadLikeOp(rewriter, opA: accReadOp0, opB: accReadOp1, contractA: contractOp, |
| 418 | contractB: pairContractOp, nonUnitDimAcc: nonUnitDim, accTy); |
| 419 | |
| 420 | if (failed(Result: readShuffle)) |
| 421 | return rewriter.notifyMatchFailure( |
| 422 | arg&: contractOp, msg: "Accumulator read is not by transfer_read or load" ); |
| 423 | } |
| 424 | |
| 425 | // Shuffle the output of contract operations before its use. |
| 426 | LogicalResult writeShuffle = shuffleBeforeWriteLikeOp( |
| 427 | rewriter, contractARes: contractOp.getResult(), contractBRes: pairContractOp.getResult(), |
| 428 | nonUnitDimAcc: nonUnitDim, accTy); |
| 429 | |
| 430 | if (failed(Result: writeShuffle)) |
| 431 | return rewriter.notifyMatchFailure( |
| 432 | arg&: contractOp, |
| 433 | msg: "Write to accumulator is not by transfer_write or store" ); |
| 434 | |
| 435 | rewriter.setInsertionPoint(contractOp); |
| 436 | castAcc = vector::ShapeCastOp::create( |
| 437 | builder&: rewriter, location: loc, |
| 438 | result: VectorType::get(shape: nonUnitDimAcc.front(), elementType: accTy.getElementType()), |
| 439 | source: contractOp.getAcc()); |
| 440 | |
| 441 | auto loadBcstBF16ElementToF32 = x86::avx::BcstToPackedF32Op::create( |
| 442 | builder&: rewriter, location: loc, dst: dstType, a: unitDimSubview[0]); |
| 443 | auto loadEvenIdxElementF32 = |
| 444 | x86::avx::CvtPackedEvenIndexedToF32Op::create(builder&: rewriter, location: loc, dst: dstType, |
| 445 | a: nonUnitDimSubview[0]); |
| 446 | auto evenIdxFMA = |
| 447 | vector::FMAOp::create(builder&: rewriter, location: loc, lhs: loadBcstBF16ElementToF32, |
| 448 | rhs: loadEvenIdxElementF32, acc: castAcc); |
| 449 | auto castEvenFma = |
| 450 | vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: accTy, source: evenIdxFMA); |
| 451 | rewriter.replaceOp(op: contractOp, newOp: castEvenFma); |
| 452 | |
| 453 | rewriter.setInsertionPoint(pairContractOp); |
| 454 | auto pairContOpLoc = pairContractOp.getLoc(); |
| 455 | VectorType accTyPairCont = |
| 456 | dyn_cast<VectorType>(Val: pairContractOp.getAccType()); |
| 457 | auto castAccPairCont = vector::ShapeCastOp::create( |
| 458 | builder&: rewriter, location: pairContOpLoc, |
| 459 | result: VectorType::get(shape: nonUnitDimAcc.front(), |
| 460 | elementType: accTyPairCont.getElementType()), |
| 461 | source: pairContractOp.getAcc()); |
| 462 | |
| 463 | auto loadOddIdxElementF32 = x86::avx::CvtPackedOddIndexedToF32Op::create( |
| 464 | builder&: rewriter, location: pairContOpLoc, dst: dstType, a: nonUnitDimSubview[0]); |
| 465 | auto oddIdxFMA = vector::FMAOp::create( |
| 466 | builder&: rewriter, location: pairContOpLoc, lhs: loadBcstBF16ElementToF32, |
| 467 | rhs: loadOddIdxElementF32, acc: castAccPairCont); |
| 468 | auto castOddFma = vector::ShapeCastOp::create(builder&: rewriter, location: pairContOpLoc, |
| 469 | result: accTyPairCont, source: oddIdxFMA); |
| 470 | rewriter.replaceOp(op: pairContractOp, newOp: castOddFma); |
| 471 | |
| 472 | return success(); |
| 473 | } |
| 474 | |
| 475 | // Load, broadcast, and do FMA for odd indexed BF16 elements. |
| 476 | auto loadBcstOddIdxElementToF32 = x86::avx::BcstToPackedF32Op::create( |
| 477 | builder&: rewriter, location: loc, dst: dstType, a: unitDimSubview[0]); |
| 478 | auto loadOddIdxElementF32 = x86::avx::CvtPackedOddIndexedToF32Op::create( |
| 479 | builder&: rewriter, location: loc, dst: dstType, a: nonUnitDimSubview[0]); |
| 480 | auto oddIdxFMA = |
| 481 | vector::FMAOp::create(builder&: rewriter, location: loc, lhs: loadBcstOddIdxElementToF32, |
| 482 | rhs: loadOddIdxElementF32, acc: castAcc); |
| 483 | |
| 484 | // Load, broadcast, and do FMA for even indexed BF16 elements. |
| 485 | auto loadBcstEvenIdxElementToF32 = x86::avx::BcstToPackedF32Op::create( |
| 486 | builder&: rewriter, location: loc, dst: dstType, a: unitDimSubview[1]); |
| 487 | auto loadEvenIdxElementF32 = x86::avx::CvtPackedEvenIndexedToF32Op::create( |
| 488 | builder&: rewriter, location: loc, dst: dstType, a: nonUnitDimSubview[0]); |
| 489 | vector::FMAOp fma = |
| 490 | vector::FMAOp::create(builder&: rewriter, location: loc, lhs: loadBcstEvenIdxElementToF32, |
| 491 | rhs: loadEvenIdxElementF32, acc: oddIdxFMA); |
| 492 | |
| 493 | auto castFma = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: accTy, source: fma); |
| 494 | rewriter.replaceOp(op: contractOp, newOp: castFma); |
| 495 | return success(); |
| 496 | } |
| 497 | }; |
| 498 | |
| 499 | void x86::populateVectorContractBF16ToFMAPatterns(RewritePatternSet &patterns) { |
| 500 | patterns.add<VectorContractBF16ToFMA>(arg: patterns.getContext()); |
| 501 | } |
| 502 | |