1//===- VectorContractBF16ToFMA.cpp-----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/MemRef/IR/MemRef.h"
10#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
11#include "mlir/Dialect/SCF/IR/SCF.h"
12#include "mlir/Dialect/Vector/IR/VectorOps.h"
13#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
14#include "mlir/Dialect/X86/Transforms.h"
15#include "mlir/Dialect/X86/Utils/X86Utils.h"
16#include "mlir/Dialect/X86/X86Dialect.h"
17
18#include "mlir/IR/BuiltinAttributes.h"
19#include "mlir/IR/Dominance.h"
20#include "mlir/IR/PatternMatch.h"
21
22#include "mlir/Pass/Pass.h"
23#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
24#include "llvm/Support/Casting.h"
25
26using namespace mlir;
27using namespace mlir::vector;
28using namespace mlir::x86;
29
30// Verifies that the LHS and RHS operands of a vector.contract are load or
31// vector.transfer_read operations on a memref source buffer, and checks
32// their bounds, dimensions, offsets, and strides.
33static bool validateVectorContractOperands(Value prodOp, bool isVnni) {
34 Operation *defOp = prodOp.getDefiningOp();
35 if (!defOp)
36 return false;
37
38 if (auto readOp = prodOp.getDefiningOp<mlir::vector::TransferReadOp>()) {
39 if (readOp.hasOutOfBoundsDim())
40 return false;
41
42 if (!readOp.getPermutationMap().isMinorIdentity())
43 return false;
44 }
45
46 Value srcBuff;
47 SmallVector<OpFoldResult> indexVals;
48 llvm::TypeSwitch<Operation *>(defOp).Case<TransferReadOp, LoadOp>(
49 caseFn: [&](auto readOp) {
50 srcBuff = readOp.getOperand(0);
51 indexVals = SmallVector<OpFoldResult>(readOp.getIndices().begin(),
52 readOp.getIndices().end());
53 });
54
55 if (!srcBuff)
56 return false;
57
58 // Return false, if the source is not a memref type
59 Type srcType = srcBuff.getType();
60 if (!llvm::isa<MemRefType>(Val: srcType))
61 return false;
62
63 // Return false if the two innermost strides of the memref are not contiguous.
64 // The x86.avx.cvt.packed.even/odd.indexed_to_f32 operations require
65 // an eight-element tuple of bf16 values to be contiguous.
66 int dimsToCheck = isVnni ? 2 : 1;
67 if (!cast<mlir::MemRefType>(Val&: srcType).areTrailingDimsContiguous(n: dimsToCheck))
68 return false;
69
70 // Return false if the vnni offset of load or transfer_read is not zero.
71 if (isVnni && getConstantIntValue(ofr: indexVals.back()) != 0)
72 return false;
73
74 return true;
75}
76
77// This function retrieves the source operation of the load or transfer
78// reads and creates subviews for the BF16 packed-operations to
79// broadcast or load BF16 elements as F32 packed elements.
80//
81// Example(1) Unit Dim:
82// ```
83// vector.load %arg0[%c0, %c0, %c0]:memref<4x1x2xbf16>,vector<1x1x2xbf16>
84// ```
85// to
86// ```
87// memref.subview %arg0[%c0,%c0,%c1]:memref<4x1x2xbf16> to memref<1x1x1xbf16>
88// memref.subview %arg0[%c0,%c0,%c0]:memref<4x1x2xbf16> to memref<1x1x1xbf16>
89// ```
90//
91// Example(2) Non-unit Dim:
92// ```
93// vector.load %arg1[%c0, %c0, %c0]:memref<1x32x2xbf16>,vector<1x8x2xbf16>
94// ```
95// to
96// ```
97// memref.subview %arg1[%c0,%c0,%c0]:memref<1x32x2xbf16> to memref<1x8x2xbf16>
98// ```
99static SmallVector<memref::SubViewOp>
100getSubviewFromVectorInput(Location loc, PatternRewriter &rewriter, Value prodOp,
101 ArrayRef<int64_t> nonUnitDimShape, bool isUnitDim,
102 bool isVNNI) {
103
104 Operation *defOp = prodOp.getDefiningOp();
105
106 Value srcBuff;
107 SmallVector<OpFoldResult> indexVals;
108 llvm::TypeSwitch<Operation *>(defOp).Case<TransferReadOp, LoadOp>(
109 caseFn: [&](auto readOp) {
110 srcBuff = readOp.getOperand(0);
111 indexVals = SmallVector<OpFoldResult>(readOp.getIndices().begin(),
112 readOp.getIndices().end());
113 });
114
115 int64_t mnDimSize = 1;
116 unsigned mnDimIdx = 0;
117
118 if (!isUnitDim) {
119 for (auto it : llvm::enumerate(First&: nonUnitDimShape)) {
120 if (it.value() != 1) {
121 mnDimSize = it.value();
122 mnDimIdx = it.index();
123 break;
124 }
125 }
126 }
127
128 auto one = rewriter.getIndexAttr(value: 1);
129 SmallVector<memref::SubViewOp> subviews;
130
131 if (!isVNNI) {
132 SmallVector<OpFoldResult> strides(indexVals.size(), one);
133 SmallVector<OpFoldResult> sizes(indexVals.size(), one);
134 // Retrive twice the nonUnit dim BF16 element for both even and odd
135 // index elements.
136 if (!isUnitDim)
137 mnDimSize = 2 * mnDimSize;
138 sizes[mnDimIdx] = rewriter.getIndexAttr(value: mnDimSize);
139 auto subview = memref::SubViewOp::create(builder&: rewriter, location: loc, source: srcBuff, offsets: indexVals,
140 sizes, strides);
141 subviews.push_back(Elt: subview);
142 return subviews;
143 }
144
145 int vnniDimSize = isUnitDim ? 1 : 2;
146 auto nonVNNIDimSize = indexVals.size() - 1;
147 // Create the size and stride offsets.
148 SmallVector<OpFoldResult> strides(indexVals.size(), one);
149 SmallVector<OpFoldResult> sizes(nonVNNIDimSize, one);
150
151 sizes.push_back(Elt: rewriter.getIndexAttr(value: vnniDimSize));
152
153 // update the unit/nonUnit Dim size either it is A(LHS) or B(RHS).
154 sizes[mnDimIdx] = rewriter.getIndexAttr(value: mnDimSize);
155
156 // for unitDim, first broadcast odd element, so index is set to 1.
157 if (isUnitDim)
158 indexVals[indexVals.size() - 1] = rewriter.getIndexAttr(value: 1);
159
160 auto subview = memref::SubViewOp::create(builder&: rewriter, location: loc, source: srcBuff, offsets: indexVals,
161 sizes, strides);
162 subviews.push_back(Elt: subview);
163
164 // For unit-dims, two subviews should be created for the odd and even
165 // element in the VNNI tuple (2xbf16) because x86.avx.bcst_to_f32.packed
166 // op loads and broadcast the first BF16 element into packed F32. It
167 // cannot distinguish between even and odd BF16 elements within a
168 // packed pair.
169 //
170 // Example:
171 // memref.subview %arg0[%c0,%c1]:memref<1x2xbf16> to memref<1x1xbf16> // Odd
172 // memref.subview %arg0[%c0,%c0]:memref<1x2xbf16> to memref<1x1xbf16> // Even
173 if (mnDimSize == 1) {
174 indexVals[indexVals.size() - 1] = rewriter.getIndexAttr(value: 0);
175 sizes[indexVals.size() - 1] = rewriter.getIndexAttr(value: 1);
176
177 auto unitDimEvenIdxSubview = memref::SubViewOp::create(
178 builder&: rewriter, location: loc, source: srcBuff, offsets: indexVals, sizes, strides);
179 subviews.push_back(Elt: unitDimEvenIdxSubview);
180 }
181
182 return subviews;
183}
184
185// Implements outer product contraction as a sequence of BF16-packed
186// operation even/odd loads and FMA operations.
187//
188// For example (VNNI packed):
189// ```
190// %1 = vector.load from memref (%m1) -> vector<1x1x2xbf16>
191// %2 = vector.load from memref (%m2) -> vector<1x8x2xbf16>
192// return vector.contract %1, %2, %arg1
193// ```
194// to
195// ```
196// %1 = x86.avx.bcst_to_f32.packed %m1[c1] -> vector<8xf32>
197// %2 = x86.avx.cvt.packed.odd.indexed_to_f32 %m2 -> vector<8xf32>
198// %3 = vector.fma %1, %2, %arg1
199// %4 = x86.avx.bcst_to_f32.packed %m1[c0] -> vector<8xf32>
200// %5 = x86.avx.cvt.packed.even.indexed_to_f32 %m2 -> vector<8xf32>
201// return vector.fma %4, %5, %3
202// ```
203//
204// For example (Flat layout):
205// ```
206// %1 = vector.load from memref (%m1) -> vector<1x1xbf16>
207// %2 = vector.load from memref (%m2) -> vector<1x8xbf16>
208// %3 = vector.contract %1, %2, %arg1
209// %4 = vector.load from memref (%m2) -> vector<1x8xbf16>
210// %5 = vector.contract %1, %4, %arg2
211// scf.yield %3, %4
212// ```
213// to
214// ```
215// %1 = x86.avx.bcst_to_f32.packed %m1[c0] -> vector<8xf32>
216// %2 = x86.avx.cvt.packed.even.indexed_to_f32 %m2 -> vector<8xf32>
217// %3 = vector.fma %1, %2, %arg1
218// %4 = x86.avx.cvt.packed.odd.indexed_to_f32 %m2 -> vector<8xf32>
219// %5 = vector.fma %1, %4, %arg2
220// scf.yield %3, %5
221struct VectorContractBF16ToFMA : public AddContractionOpRewritePattern {
222 using AddContractionOpRewritePattern::AddContractionOpRewritePattern;
223
224 LogicalResult
225 matchAndRewriteAddContractOp(vector::ContractionOp contractOp,
226 PatternRewriter &rewriter) const override {
227
228 // TODO: Move this validation to a common utility folder. Planned to
229 // do once (code refactoring), all architecture specific nanokernel
230 // passes are merged into the repo.
231 VectorType lhsTy = contractOp.getLhsType();
232 if (!lhsTy.getElementType().isBF16())
233 return rewriter.notifyMatchFailure(arg&: contractOp,
234 msg: "Only BF16 lowering is supported.");
235
236 bool isVnni = isInVnniLayout(op: contractOp.getOperation(),
237 indexingMaps: contractOp.getIndexingMapsArray(),
238 /*blockingFactor=*/2);
239
240 VectorType accTy = dyn_cast<VectorType>(Val: contractOp.getAccType());
241 if (!accTy)
242 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Wrong accmulator type.");
243
244 if (!accTy.getElementType().isF32())
245 return rewriter.notifyMatchFailure(
246 arg&: contractOp, msg: "Only F32 acumulation supported for BF16 type.");
247
248 ArrayRef<int64_t> accShape = accTy.getShape();
249 llvm::SmallVector<int64_t> nonUnitDimAcc;
250 llvm::copy_if(Range&: accShape, Out: std::back_inserter(x&: nonUnitDimAcc),
251 P: [](int64_t dim) { return dim != 1; });
252 if (nonUnitDimAcc.size() != 1)
253 return rewriter.notifyMatchFailure(
254 arg&: contractOp, msg: "A or B should be a non-unit dim in acc.");
255
256 ArrayRef<int64_t> lhsShape = lhsTy.getShape();
257 llvm::SmallVector<int64_t> nonUnitDimLhs;
258 llvm::copy_if(Range&: lhsShape, Out: std::back_inserter(x&: nonUnitDimLhs),
259 P: [](int64_t dim) { return dim != 1; });
260
261 VectorType rhsTy = contractOp.getRhsType();
262 ArrayRef<int64_t> rhsShape = rhsTy.getShape();
263 llvm::SmallVector<int64_t> nonUnitDimRhs;
264 llvm::copy_if(Range&: rhsShape, Out: std::back_inserter(x&: nonUnitDimRhs),
265 P: [](int64_t dim) { return dim != 1; });
266
267 if (isVnni && (nonUnitDimLhs.size() - 1) > 0 &&
268 (nonUnitDimRhs.size() - 1) > 0)
269 return rewriter.notifyMatchFailure(arg&: contractOp,
270 msg: "Excepts unit dimensions for either "
271 "LHS or RHS shape other than VNNI.");
272
273 if (isVnni && (nonUnitDimLhs.size() - 1) != 1 &&
274 (nonUnitDimRhs.size() - 1) != 1)
275 return rewriter.notifyMatchFailure(
276 arg&: contractOp,
277 msg: "Excepts a one non-unit A/B dimension for either LHS or RHS shape.");
278
279 if (!isVnni && nonUnitDimLhs.size() > 0 && nonUnitDimRhs.size() > 0)
280 return rewriter.notifyMatchFailure(arg&: contractOp,
281 msg: "Excepts unit dimensions for either "
282 "LHS or RHS shape.");
283
284 if (!isVnni && nonUnitDimLhs.size() != 1 && nonUnitDimRhs.size() != 1)
285 return rewriter.notifyMatchFailure(
286 arg&: contractOp,
287 msg: "Excepts a one non-unit A/B dimension for either LHS or RHS shape.");
288
289 // Non-unit dimensions should match the vector length of BF16.
290 unsigned int nonUnitDim = nonUnitDimAcc.front();
291 if (nonUnitDim != 4 && nonUnitDim != 8)
292 return rewriter.notifyMatchFailure(
293 arg&: contractOp, msg: "BF16 packed load operation expects non-unit (LHR or "
294 "RHS) dim and acc dim of size 4/8.");
295
296 if (!validateVectorContractOperands(prodOp: contractOp.getLhs(), isVnni) ||
297 !validateVectorContractOperands(prodOp: contractOp.getRhs(), isVnni)) {
298 return rewriter.notifyMatchFailure(
299 arg&: contractOp, msg: "The LHS or RHS is in an invalid format. Either it has "
300 "false in-bounds, "
301 "a non-identity permutation map, a non-zero VNNI offset, "
302 "a non-memref "
303 "source, or a non-unit VNNI stride");
304 }
305
306 // Lower vector.contract to FMAs with help of BF16 packed ops.
307 auto loc = contractOp.getLoc();
308
309 // create the unit-dimension LHS or RHS subview and the
310 // corresponding non-unit dimension LHS or RHS subview on the other-side.
311 // For example, if LHS has type vector<1x1x2xbf16> and RHS has type
312 // vector<1x8x2xbf16>, we create two subview for the LHS and one subview
313 // for the RHS. In the opposite case (non-unit dimension on the LHS), we
314 // do vice-versa.
315
316 bool rhsHasMultipleNonUnitDims = nonUnitDimRhs.size() > 0;
317 if (isVnni) {
318 rhsHasMultipleNonUnitDims = (nonUnitDimRhs.size() - 1) > 0;
319 }
320
321 // Select which operand is "unit" and which is "non-unit".
322 Value unitSrc =
323 rhsHasMultipleNonUnitDims ? contractOp.getLhs() : contractOp.getRhs();
324 Value nonUnitSrc =
325 rhsHasMultipleNonUnitDims ? contractOp.getRhs() : contractOp.getLhs();
326
327 ArrayRef<int64_t> nonUnitDimShape =
328 rhsHasMultipleNonUnitDims ? rhsShape : lhsShape;
329
330 // Get the pair vector.contract operation. The pair is decided on:
331 // (1) - the unitDim operand Lhs or Rhs should be same,
332 // (2) - the defining source memref should be same for nonUnitDim
333 // operation, (3) - the nonUnit dim offset difference between the
334 // vector.contracts should be 8.
335 vector::ContractionOp pairContractOp;
336 if (!isVnni) {
337 Operation *nextOp = contractOp;
338 while ((nextOp = nextOp->getNextNode())) {
339 auto contOp = dyn_cast<vector::ContractionOp>(Val: nextOp);
340
341 if (!contOp)
342 continue;
343
344 if (validatePairVectorContract(contractOp, pairContOp: contOp,
345 rhsHasMultipleNonUnitDims,
346 nonUnitDimValue: nonUnitDimAcc.front())) {
347 pairContractOp = contOp;
348 break;
349 }
350 }
351
352 if (!pairContractOp)
353 return failure();
354
355 Operation *accReadOp0 =
356 traceToVectorReadLikeParentOperation(v: contractOp.getAcc());
357 Operation *accReadOp1 =
358 traceToVectorReadLikeParentOperation(v: pairContractOp.getAcc());
359
360 // Iterate down to find the users of contact operations until it is store
361 // or transfer_write.
362 Operation *resultWriteOp0 =
363 traceToVectorWriteLikeUserOperation(v: contractOp.getResult());
364 Operation *resultWriteOp1 =
365 traceToVectorWriteLikeUserOperation(v: pairContractOp.getResult());
366
367 if (!accReadOp0 || !accReadOp1)
368 return rewriter.notifyMatchFailure(
369 arg&: contractOp,
370 msg: "Operand doesn't have load or transfer_read as its parent op");
371
372 if (!resultWriteOp0 || !resultWriteOp1)
373 return rewriter.notifyMatchFailure(
374 arg&: contractOp,
375 msg: "The use of contract operations are neither vector.store "
376 "or transfer_write or has multiple users");
377
378 if (contractOp->getBlock() == accReadOp1->getBlock() &&
379 contractOp->isBeforeInBlock(other: accReadOp1))
380 return rewriter.notifyMatchFailure(
381 arg&: contractOp, msg: "The load/read operation of pair contract operation is "
382 "after the contractOp");
383
384 if (pairContractOp->getBlock() == resultWriteOp0->getBlock() &&
385 resultWriteOp0->isBeforeInBlock(other: pairContractOp)) {
386 return rewriter.notifyMatchFailure(
387 arg&: contractOp, msg: "The store/write operation of contract operation is "
388 "before the pair contract operation");
389 }
390 }
391
392 // Build subviews.
393 auto unitDimSubview = getSubviewFromVectorInput(
394 loc, rewriter, prodOp: unitSrc, nonUnitDimShape, isUnitDim: true, isVNNI: isVnni);
395
396 auto nonUnitDimSubview = getSubviewFromVectorInput(
397 loc, rewriter, prodOp: nonUnitSrc, nonUnitDimShape, isUnitDim: false, isVNNI: isVnni);
398
399 auto castAcc = vector::ShapeCastOp::create(
400 builder&: rewriter, location: loc,
401 result: VectorType::get(shape: nonUnitDimAcc.front(), elementType: accTy.getElementType()),
402 source: contractOp.getAcc());
403 VectorType dstType =
404 VectorType::get(shape: nonUnitDimAcc.front(), elementType: rewriter.getF32Type());
405
406 if (!isVnni) {
407
408 // Validate and shuffle the accumulator
409 Operation *accReadOp0 =
410 traceToVectorReadLikeParentOperation(v: contractOp.getAcc());
411 Operation *accReadOp1 =
412 traceToVectorReadLikeParentOperation(v: pairContractOp.getAcc());
413
414 if (!isa<arith::ConstantOp>(Val: accReadOp0)) {
415 // Shuffle the accumulators of the contract operations.
416 LogicalResult readShuffle =
417 shuffleAfterReadLikeOp(rewriter, opA: accReadOp0, opB: accReadOp1, contractA: contractOp,
418 contractB: pairContractOp, nonUnitDimAcc: nonUnitDim, accTy);
419
420 if (failed(Result: readShuffle))
421 return rewriter.notifyMatchFailure(
422 arg&: contractOp, msg: "Accumulator read is not by transfer_read or load");
423 }
424
425 // Shuffle the output of contract operations before its use.
426 LogicalResult writeShuffle = shuffleBeforeWriteLikeOp(
427 rewriter, contractARes: contractOp.getResult(), contractBRes: pairContractOp.getResult(),
428 nonUnitDimAcc: nonUnitDim, accTy);
429
430 if (failed(Result: writeShuffle))
431 return rewriter.notifyMatchFailure(
432 arg&: contractOp,
433 msg: "Write to accumulator is not by transfer_write or store");
434
435 rewriter.setInsertionPoint(contractOp);
436 castAcc = vector::ShapeCastOp::create(
437 builder&: rewriter, location: loc,
438 result: VectorType::get(shape: nonUnitDimAcc.front(), elementType: accTy.getElementType()),
439 source: contractOp.getAcc());
440
441 auto loadBcstBF16ElementToF32 = x86::avx::BcstToPackedF32Op::create(
442 builder&: rewriter, location: loc, dst: dstType, a: unitDimSubview[0]);
443 auto loadEvenIdxElementF32 =
444 x86::avx::CvtPackedEvenIndexedToF32Op::create(builder&: rewriter, location: loc, dst: dstType,
445 a: nonUnitDimSubview[0]);
446 auto evenIdxFMA =
447 vector::FMAOp::create(builder&: rewriter, location: loc, lhs: loadBcstBF16ElementToF32,
448 rhs: loadEvenIdxElementF32, acc: castAcc);
449 auto castEvenFma =
450 vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: accTy, source: evenIdxFMA);
451 rewriter.replaceOp(op: contractOp, newOp: castEvenFma);
452
453 rewriter.setInsertionPoint(pairContractOp);
454 auto pairContOpLoc = pairContractOp.getLoc();
455 VectorType accTyPairCont =
456 dyn_cast<VectorType>(Val: pairContractOp.getAccType());
457 auto castAccPairCont = vector::ShapeCastOp::create(
458 builder&: rewriter, location: pairContOpLoc,
459 result: VectorType::get(shape: nonUnitDimAcc.front(),
460 elementType: accTyPairCont.getElementType()),
461 source: pairContractOp.getAcc());
462
463 auto loadOddIdxElementF32 = x86::avx::CvtPackedOddIndexedToF32Op::create(
464 builder&: rewriter, location: pairContOpLoc, dst: dstType, a: nonUnitDimSubview[0]);
465 auto oddIdxFMA = vector::FMAOp::create(
466 builder&: rewriter, location: pairContOpLoc, lhs: loadBcstBF16ElementToF32,
467 rhs: loadOddIdxElementF32, acc: castAccPairCont);
468 auto castOddFma = vector::ShapeCastOp::create(builder&: rewriter, location: pairContOpLoc,
469 result: accTyPairCont, source: oddIdxFMA);
470 rewriter.replaceOp(op: pairContractOp, newOp: castOddFma);
471
472 return success();
473 }
474
475 // Load, broadcast, and do FMA for odd indexed BF16 elements.
476 auto loadBcstOddIdxElementToF32 = x86::avx::BcstToPackedF32Op::create(
477 builder&: rewriter, location: loc, dst: dstType, a: unitDimSubview[0]);
478 auto loadOddIdxElementF32 = x86::avx::CvtPackedOddIndexedToF32Op::create(
479 builder&: rewriter, location: loc, dst: dstType, a: nonUnitDimSubview[0]);
480 auto oddIdxFMA =
481 vector::FMAOp::create(builder&: rewriter, location: loc, lhs: loadBcstOddIdxElementToF32,
482 rhs: loadOddIdxElementF32, acc: castAcc);
483
484 // Load, broadcast, and do FMA for even indexed BF16 elements.
485 auto loadBcstEvenIdxElementToF32 = x86::avx::BcstToPackedF32Op::create(
486 builder&: rewriter, location: loc, dst: dstType, a: unitDimSubview[1]);
487 auto loadEvenIdxElementF32 = x86::avx::CvtPackedEvenIndexedToF32Op::create(
488 builder&: rewriter, location: loc, dst: dstType, a: nonUnitDimSubview[0]);
489 vector::FMAOp fma =
490 vector::FMAOp::create(builder&: rewriter, location: loc, lhs: loadBcstEvenIdxElementToF32,
491 rhs: loadEvenIdxElementF32, acc: oddIdxFMA);
492
493 auto castFma = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: accTy, source: fma);
494 rewriter.replaceOp(op: contractOp, newOp: castFma);
495 return success();
496 }
497};
498
499void x86::populateVectorContractBF16ToFMAPatterns(RewritePatternSet &patterns) {
500 patterns.add<VectorContractBF16ToFMA>(arg: patterns.getContext());
501}
502