1//===- VectorContractToAMXDotProduct.cpp ----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/SCF/IR/SCF.h"
10#include "mlir/Dialect/Vector/IR/VectorOps.h"
11#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
12#include "mlir/Dialect/X86/Transforms.h"
13#include "mlir/Dialect/X86/Utils/X86Utils.h"
14#include "mlir/Dialect/X86/X86Dialect.h"
15
16#include "mlir/IR/BuiltinAttributes.h"
17#include "mlir/IR/Dominance.h"
18#include "mlir/IR/PatternMatch.h"
19#include "llvm/Support/Casting.h"
20
21#include "mlir/Pass/Pass.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23
24using namespace mlir;
25using namespace mlir::vector;
26using namespace mlir::x86;
27
28namespace {
29
30// Return true if the operation is a constant dense element
31// attribute.
32static bool isZeroVectorConstant(Operation *op) {
33 auto constantOp = dyn_cast_if_present<arith::ConstantOp>(Val: op);
34 if (!constantOp)
35 return false;
36
37 if (!isa<VectorType>(Val: constantOp.getType()))
38 return false;
39
40 auto denseAttr = dyn_cast<DenseElementsAttr>(Val: constantOp.getValue());
41 if (!denseAttr || !denseAttr.isSplat())
42 return false;
43
44 return isZeroIntegerOrFloat(v: denseAttr.getSplatValue<Attribute>());
45}
46
47// Recursively follows single-use values through scf.yield operations
48// and returns the first non-yield user result in the contraction chain.
49static Value contractionUsersAfterYield(Value v) {
50 if (v.getNumUses() != 1)
51 return nullptr;
52
53 OpOperand &use = *v.use_begin();
54 Operation *user = use.getOwner();
55
56 if (!isa<scf::YieldOp>(Val: user))
57 return v;
58
59 auto yield = cast<scf::YieldOp>(Val: user);
60 Operation *parent = yield->getParentOp();
61 unsigned idx = use.getOperandNumber();
62
63 return contractionUsersAfterYield(v: parent->getResult(idx));
64}
65
66// Function to collapse the last two dimension (k or n, and vnni) to help the
67// amx.tile_load to correctly load the packed element type.
68static Value collapseInnerDims(OpBuilder &builder, Location loc, Value input) {
69 ShapedType inputType = cast<ShapedType>(Val: input.getType());
70 int64_t firstDimToCollapse = inputType.getRank() - 2;
71
72 if (inputType.getRank() == 1)
73 return input;
74
75 SmallVector<ReassociationIndices> reassociation;
76 for (int64_t i = 0; i < firstDimToCollapse; ++i)
77 reassociation.push_back(Elt: ReassociationIndices{i});
78
79 ReassociationIndices collapsedIndices;
80 for (int64_t i = firstDimToCollapse; i < inputType.getRank(); ++i)
81 collapsedIndices.push_back(Elt: i);
82
83 reassociation.push_back(Elt: collapsedIndices);
84 return memref::CollapseShapeOp::create(builder, location: loc, src: input, reassociation);
85}
86
87// Check if a vector.contract operand has a memref read source.
88static bool isReadSrcMemref(Value operand) {
89 Operation *defOp = operand.getDefiningOp();
90 if (!defOp)
91 return false;
92
93 Value srcBuff;
94 llvm::TypeSwitch<Operation *>(operand.getDefiningOp())
95 .Case<TransferReadOp, LoadOp>(
96 caseFn: [&](auto readOp) { srcBuff = readOp.getOperand(0); });
97
98 return srcBuff && isa<MemRefType>(Val: srcBuff.getType());
99}
100
101// Check if a vector.contract operand reads whole VNNI groups: the memref's
102// static innermost dim is the VNNI factor and the read starts at its origin.
103static bool isVnniOperandRead(Value operand, int64_t vnni) {
104 Operation *defOp = operand.getDefiningOp();
105 if (!defOp)
106 return false;
107
108 Value srcBuff;
109 OpFoldResult vnniIndex;
110 llvm::TypeSwitch<Operation *>(defOp).Case<TransferReadOp, LoadOp>(
111 caseFn: [&](auto readOp) {
112 srcBuff = readOp.getBase();
113 vnniIndex = readOp.getIndices().back();
114 });
115
116 if (!srcBuff)
117 return false;
118 auto srcType = dyn_cast<MemRefType>(Val: srcBuff.getType());
119 return srcType && srcType.getRank() > 0 &&
120 srcType.getShape().back() == vnni && isZeroInteger(v: vnniIndex);
121}
122
123// Replaces the indices of the two innermost dims of a VNNI operand by the
124// index of the dim they collapse into.
125static void collapseVnniIndices(OpBuilder &rewriter, Location loc,
126 SmallVectorImpl<Value> &indices, int64_t vnni) {
127 Value inner = indices.pop_back_val();
128 Value cVnni = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: vnni);
129 Value scaled = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: indices.back(), rhs: cVnni);
130 indices.back() = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: scaled, rhs: inner);
131}
132
133// Get the MemRef source and offset index for the operands of
134// vector.contract.
135static FailureOr<std::pair<Value, SmallVector<Value>>>
136getSrcIndxValue(OpBuilder &rewriter, Location loc, Value operand,
137 bool isNotAcc) {
138 Operation *defOp = operand.getDefiningOp();
139 if (!defOp)
140 return failure();
141
142 Value srcBuff;
143 SmallVector<OpFoldResult> indexVals;
144 llvm::TypeSwitch<Operation *>(operand.getDefiningOp())
145 .Case<TransferReadOp, LoadOp>(caseFn: [&](auto readOp) {
146 indexVals = SmallVector<OpFoldResult>(readOp.getIndices().begin(),
147 readOp.getIndices().end());
148 srcBuff = readOp.getOperand(0);
149 });
150
151 if (!srcBuff || !isa<MemRefType>(Val: srcBuff.getType()))
152 return failure();
153
154 SmallVector<Value> indices;
155 indices.reserve(N: indexVals.size());
156
157 for (OpFoldResult ofr : indexVals) {
158 indices.push_back(
159 Elt: mlir::getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr));
160 }
161
162 if (isNotAcc) {
163 collapseVnniIndices(rewriter, loc, indices,
164 vnni: cast<MemRefType>(Val: srcBuff.getType()).getShape().back());
165 srcBuff = collapseInnerDims(builder&: rewriter, loc, input: srcBuff);
166 }
167
168 return std::make_pair(x&: srcBuff, y&: indices);
169}
170
171// Function to validate the loop step value.
172static LogicalResult validateLoopStep(OpBuilder &rewriter, Value step,
173 int64_t value) {
174
175 auto cst = step.getDefiningOp<arith::ConstantIndexOp>();
176 if (!cst)
177 return failure();
178
179 if (cst.value() != value && cst.value() != 1)
180 return failure();
181
182 return success();
183}
184
185// Function to validate the vector.contract operation.
186static LogicalResult validateContractOps(OpBuilder &rewriter,
187 vector::ContractionOp contractOp,
188 unsigned int blockingFactor,
189 Value srcBuffLhs, Value srcBuffRhs,
190 bool srcValidate) {
191
192 if (srcValidate) {
193 // Get the MemRef buffer of LHS operand.
194 auto srcIndxLhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(),
195 operand: contractOp.getLhs(), isNotAcc: false);
196 if (failed(Result: srcIndxLhs))
197 return failure();
198 auto [buffLhs, indicesLhs] = *srcIndxLhs;
199
200 // Get the MemRef buffer of RHS operand.
201 auto srcIndxRhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(),
202 operand: contractOp.getRhs(), isNotAcc: false);
203 if (failed(Result: srcIndxRhs))
204 return failure();
205 auto [buffRhs, indicesRhs] = *srcIndxRhs;
206
207 // Return failure if the Memref buff didn't match.
208 if (buffLhs != srcBuffLhs)
209 return failure();
210
211 if (buffRhs != srcBuffRhs)
212 return failure();
213 }
214
215 if (!contractionUsersAfterYield(v: contractOp.getResult()))
216 return failure();
217
218 VectorType accTy = dyn_cast<VectorType>(Val: contractOp.getAccType());
219 if (!accTy)
220 return failure();
221
222 // The Accumulator dims should be 16 or 1. Like <1x16x16> or <16x16>.
223 ArrayRef<int64_t> accShape = accTy.getShape();
224 SmallVector<int64_t> nonUnitDimAcc;
225 llvm::copy_if(Range&: accShape, Out: std::back_inserter(x&: nonUnitDimAcc),
226 P: [](int64_t dim) { return (dim != 16 && dim != 1); });
227
228 if (nonUnitDimAcc.size() != 0)
229 return failure();
230
231 // The LHS dims should be 16 or vnni or 1. Like <1x16x16x2> or
232 // <16x16x4>. The vnni dims should be 2 or 4.
233 VectorType lhsTy = contractOp.getLhsType();
234 ArrayRef<int64_t> lhsShape = lhsTy.getShape();
235 SmallVector<int64_t> nonUnitDimLhs;
236 llvm::copy_if(Range&: lhsShape, Out: std::back_inserter(x&: nonUnitDimLhs),
237 P: [](int64_t dim) { return (dim != 16 && dim != 1); });
238
239 if (nonUnitDimLhs.size() != 1)
240 return failure();
241
242 if (nonUnitDimLhs[0] != blockingFactor)
243 return failure();
244
245 // The RHS dims should be 16 or vnni or 1. Like <1x16x16x2> or
246 // <16x16x4>. The vnni dims should be 2 or 4.
247 VectorType rhsTy = contractOp.getRhsType();
248 ArrayRef<int64_t> rhsShape = rhsTy.getShape();
249 llvm::SmallVector<int64_t> nonUnitDimRhs;
250 llvm::copy_if(Range&: rhsShape, Out: std::back_inserter(x&: nonUnitDimRhs),
251 P: [](int64_t dim) { return (dim != 16 && dim != 1); });
252
253 if (nonUnitDimRhs.size() != 1)
254 return failure();
255
256 if (nonUnitDimRhs[0] != blockingFactor)
257 return failure();
258
259 return success();
260}
261
262// Traces an accumulator back through the iter_args of the given loops and
263// returns the first value that is not one of them, e.g., the init value of the
264// outermost loop or a value computed within the loops.
265static Value getLoopNestInit(Value acc, ArrayRef<scf::ForOp> loops) {
266 while (auto arg = dyn_cast<BlockArgument>(Val&: acc)) {
267 auto loop = dyn_cast_if_present<scf::ForOp>(Val: arg.getOwner()->getParentOp());
268 if (!loop || !llvm::is_contained(Range&: loops, Element: loop))
269 break;
270 OpOperand *init = loop.getTiedLoopInit(bbArg: arg);
271 if (!init)
272 break;
273 acc = init->get();
274 }
275 return acc;
276}
277
278// Returns the loop index position to get mapped during the
279// MemRef type clone.
280static unsigned getIndexPosition(Value operand, scf::ForOp loop) {
281 Value iv = loop.getInductionVar();
282
283 Value srcBuff;
284 llvm::TypeSwitch<Operation *>(operand.getDefiningOp())
285 .Case<TransferReadOp, LoadOp>(
286 caseFn: [&](auto readOp) { srcBuff = readOp.getOperand(0); });
287
288 auto subview = srcBuff.getDefiningOp<memref::SubViewOp>();
289 if (!subview)
290 return 0;
291
292 auto offsets = subview.getOffsets();
293
294 for (auto it : llvm::enumerate(First&: offsets)) {
295 if (it.value() == iv)
296 return it.index();
297 }
298
299 return 0;
300}
301
302// Creates amx.tile_loads.
303static amx::TileLoadOp createTileLoads(OpBuilder &rewriter, Location loc,
304 Value operand, Value mat, Type ipType,
305 unsigned int offset, bool isVnni) {
306
307 auto srcIndx = getSrcIndxValue(rewriter, loc, operand, isNotAcc: false);
308 auto [srcBuff, indices] = *srcIndx;
309 if (isVnni)
310 collapseVnniIndices(rewriter, loc, indices,
311 vnni: cast<MemRefType>(Val: srcBuff.getType()).getShape().back());
312
313 amx::TileType tileType = amx::TileType::get(shape: {16, (16 * offset)}, elementType: ipType);
314 return amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: mat, indices);
315}
316
317static void performShuffle(OpBuilder &rewriter, Location loc, Value matB,
318 Type ipType, unsigned int offset, Value packedBuffer,
319 Value indxToStoreInBuffer) {
320
321 Value c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
322 Value c16 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 16);
323 SmallVector<Value> subviewOffset(
324 llvm::cast<MemRefType>(Val: matB.getType()).getRank(), c0);
325
326 Value cStep = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: offset);
327 Value cBound = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: (16 * offset));
328 Value offsetIndx =
329 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: (offset / 2));
330
331 scf::ForOp::create(
332 builder&: rewriter, location: loc, lowerBound: c0, upperBound: cBound, step: cStep, initArgs: ValueRange{},
333 odsArg4: [&](OpBuilder &nestedBuilder, Location loc, Value iv,
334 ValueRange iterArgs) {
335 subviewOffset[subviewOffset.size() - 2] = iv;
336
337 // Retrieve two rows of vector (32) for int8 and f8 type. For bf16,
338 // retrieve one row of vector (32).
339 auto vectorType = VectorType::get(shape: {2, (16 * (offset / 2))}, elementType: ipType);
340 if (ipType.isBF16())
341 vectorType = VectorType::get(shape: (16 * offset), elementType: ipType);
342
343 int64_t srcRank = (dyn_cast<ShapedType>(Val: matB.getType())).getRank();
344 Value padding = ub::PoisonOp::create(builder&: rewriter, location: loc, result: ipType);
345 auto map = AffineMap::getMinorIdentityMap(dims: srcRank, results: vectorType.getRank(),
346 context: rewriter.getContext());
347 SmallVector<bool> inBounds(vectorType.getRank(), true);
348 Value vec1 = vector::TransferReadOp::create(
349 builder&: rewriter, location: loc, vectorType, source: matB, indices: ValueRange(subviewOffset), padding,
350 permutationMap: map, inBounds);
351
352 if (!ipType.isBF16())
353 vec1 = vector::ShapeCastOp::create(
354 builder&: rewriter, location: loc, result: VectorType::get(shape: (16 * offset), elementType: ipType), source: vec1);
355
356 // Increment the iv by 1 or 2 based on the type to load the next 32/64
357 // elements
358 Value incIV = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: offsetIndx, rhs: iv);
359 subviewOffset[subviewOffset.size() - 2] = incIV;
360
361 Value vec2 = vector::TransferReadOp::create(
362 builder&: rewriter, location: loc, vectorType, source: matB, indices: ValueRange(subviewOffset), padding,
363 permutationMap: map, inBounds);
364 if (!ipType.isBF16())
365 vec2 = vector::ShapeCastOp::create(
366 builder&: rewriter, location: loc, result: VectorType::get(shape: (16 * offset), elementType: ipType), source: vec2);
367
368 vector::ShuffleOp writeVal1;
369 vector::ShuffleOp writeVal2;
370
371 if (ipType.isBF16()) {
372
373 writeVal1 = vector::ShuffleOp::create(
374 builder&: rewriter, location: loc, result: VectorType::get(shape: {(16 * offset)}, elementType: ipType), v1: vec1,
375 v2: vec2,
376 mask: ArrayRef<int64_t>{0, 32, 1, 33, 2, 34, 3, 35, 8, 40, 9,
377 41, 10, 42, 11, 43, 16, 48, 17, 49, 18, 50,
378 19, 51, 24, 56, 25, 57, 26, 58, 27, 59});
379
380 writeVal2 = vector::ShuffleOp::create(
381 builder&: rewriter, location: loc, result: VectorType::get(shape: {(16 * offset)}, elementType: ipType), v1: vec1,
382 v2: vec2,
383 mask: ArrayRef<int64_t>{4, 36, 5, 37, 6, 38, 7, 39, 12, 44, 13,
384 45, 14, 46, 15, 47, 20, 52, 21, 53, 22, 54,
385 23, 55, 28, 60, 29, 61, 30, 62, 31, 63});
386 }
387
388 if (ipType.isSignlessInteger(width: 8) || ipType.isF8E5M2() ||
389 ipType.isF8E4M3FN()) {
390
391 writeVal1 = vector::ShuffleOp::create(
392 builder&: rewriter, location: loc, result: VectorType::get(shape: {(16 * offset)}, elementType: ipType), v1: vec1,
393 v2: vec2,
394 mask: ArrayRef<int64_t>{
395 0, 32, 64, 96, 1, 33, 65, 97, 2, 34, 66, 98, 3,
396 35, 67, 99, 8, 40, 72, 104, 9, 41, 73, 105, 10, 42,
397 74, 106, 11, 43, 75, 107, 16, 48, 80, 112, 17, 49, 81,
398 113, 18, 50, 82, 114, 19, 51, 83, 115, 24, 56, 88, 120,
399 25, 57, 89, 121, 26, 58, 90, 122, 27, 59, 91, 123});
400
401 writeVal2 = vector::ShuffleOp::create(
402 builder&: rewriter, location: loc, result: VectorType::get(shape: {(16 * offset)}, elementType: ipType), v1: vec1,
403 v2: vec2,
404 mask: ArrayRef<int64_t>{
405 4, 36, 68, 100, 5, 37, 69, 101, 6, 38, 70, 102, 7, 39,
406 71, 103, 12, 44, 76, 108, 13, 45, 77, 109, 14, 46, 78, 110,
407 15, 47, 79, 111, 20, 52, 84, 116, 21, 53, 85, 117, 22, 54,
408 86, 118, 23, 55, 87, 119, 28, 60, 92, 124, 29, 61, 93, 125,
409 30, 62, 94, 126, 31, 63, 95, 127});
410 }
411
412 // iv to store the shuffled elements
413 Value ivShuff1 = arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: iv, rhs: cStep);
414 Value ivShuff2 = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: ivShuff1, rhs: c16);
415
416 vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal1, base: packedBuffer,
417 indices: ValueRange{indxToStoreInBuffer, ivShuff1, c0});
418 vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal2, base: packedBuffer,
419 indices: ValueRange{indxToStoreInBuffer, ivShuff2, c0});
420
421 scf::YieldOp::create(builder&: nestedBuilder, location: loc);
422 });
423}
424
425static llvm::DenseMap<Operation *, amx::TileLoadOp>
426packInputs(OpBuilder &rewriter, Location loc,
427 SmallVector<vector::ContractionOp> ops, Value matB, Type ipType,
428 unsigned int offset, Value packedBuffer, bool pack,
429 Value indxToStoreInBuffer, Value indxToLoadFromMatB) {
430
431 llvm::DenseMap<Operation *, amx::TileLoadOp> readsToTileLoads;
432 Value c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
433 Value c16 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 16);
434
435 for (size_t j = 0; j < ops.size(); j++) {
436 for (size_t i = 0; i < ops.size(); i++) {
437
438 if (i != j && validatePairVectorContract(contractOp: ops[j], pairContOp: ops[i], rhsHasMultipleNonUnitDims: true, nonUnitDimValue: 16)) {
439
440 Operation *readOpRhs = ops[j].getRhs().getDefiningOp();
441 auto itRhs = readsToTileLoads.find(Val: readOpRhs);
442 if (itRhs != readsToTileLoads.end()) {
443 continue;
444 }
445
446 if (pack) {
447 performShuffle(rewriter, loc, matB, ipType, offset, packedBuffer,
448 indxToStoreInBuffer);
449 }
450
451 amx::TileType tileType =
452 amx::TileType::get(shape: {16, (16 * offset)}, elementType: ipType);
453 auto loadRow1 =
454 amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: packedBuffer,
455 indices: ValueRange{indxToLoadFromMatB, c0, c0});
456
457 auto loadRow2 =
458 amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: packedBuffer,
459 indices: ValueRange{indxToLoadFromMatB, c16, c0});
460
461 readsToTileLoads.try_emplace(Key: readOpRhs, Args&: loadRow1);
462 readsToTileLoads.try_emplace(Key: ops[i].getRhs().getDefiningOp(), Args&: loadRow2);
463 }
464 }
465 }
466
467 return readsToTileLoads;
468}
469
470// Creates tiled amx dot-products.
471static SmallVector<Value>
472createTiledDp(OpBuilder &rewriter, Location loc,
473 SmallVector<vector::ContractionOp> ops, Value matA, Value matB,
474 Type ipType, Type opType, ValueRange accIterArgs,
475 unsigned int offset, bool isVnni, Value packedBuffer, bool pack,
476 Value indxToStoreInBuffer, Value indxToLoadFromMatB) {
477
478 if (isVnni) {
479 matA = collapseInnerDims(builder&: rewriter, loc, input: matA);
480 matB = collapseInnerDims(builder&: rewriter, loc, input: matB);
481 }
482
483 SmallVector<Value> accumulators;
484 // Stores the amx.tile_load operation vs it's equivalent vector tranfer_read
485 // or load operations.
486 llvm::DenseMap<Operation *, amx::TileLoadOp> readsToTileLoads;
487
488 // function call to online pack the input B matrix
489 if (!isVnni) {
490 readsToTileLoads =
491 packInputs(rewriter, loc, ops, matB, ipType, offset, packedBuffer, pack,
492 indxToStoreInBuffer, indxToLoadFromMatB);
493 }
494
495 // Iterate over the contraction operations and compute the tiled dot-product.
496 for (size_t i = 0; i < ops.size(); i++) {
497
498 Operation *readOpLhs = ops[i].getLhs().getDefiningOp();
499 amx::TileLoadOp tilesLhs;
500 auto itLhs = readsToTileLoads.find(Val: readOpLhs);
501 if (itLhs != readsToTileLoads.end()) {
502 tilesLhs = itLhs->second;
503 } else {
504 tilesLhs = createTileLoads(rewriter, loc, operand: ops[i].getLhs(), mat: matA, ipType,
505 offset, isVnni);
506 readsToTileLoads.try_emplace(Key: readOpLhs, Args&: tilesLhs);
507 }
508
509 Operation *readOpRhs = ops[i].getRhs().getDefiningOp();
510 amx::TileLoadOp tilesRhs;
511 auto itRhs = readsToTileLoads.find(Val: readOpRhs);
512 if (itRhs != readsToTileLoads.end()) {
513 tilesRhs = itRhs->second;
514 } else {
515 tilesRhs = createTileLoads(rewriter, loc, operand: ops[i].getRhs(), mat: matB, ipType,
516 offset, isVnni);
517 readsToTileLoads.try_emplace(Key: readOpRhs, Args&: tilesRhs);
518 }
519
520 auto accTileType = amx::TileType::get(shape: {16, 16}, elementType: opType);
521
522 Value dp;
523 if (ipType.isBF16() || ipType.isF8E5M2() || ipType.isF8E4M3FN())
524 dp = amx::TileMulFOp::create(builder&: rewriter, location: loc, res: accTileType, lhs: tilesLhs,
525 rhs: tilesRhs, acc: accIterArgs[i]);
526
527 if (ipType.isSignlessInteger(width: 8))
528 dp = amx::TileMulIOp::create(builder&: rewriter, location: loc, res: accTileType, lhs: tilesLhs,
529 rhs: tilesRhs, acc: accIterArgs[i]);
530
531 accumulators.push_back(Elt: dp);
532 }
533 return accumulators;
534}
535
536static SmallVector<Value> createTileZeros(OpBuilder &rewriter, Location loc,
537 Type opType, scf::ForOp outerLoop,
538 int64_t size) {
539 OpBuilder::InsertionGuard guard(rewriter);
540 rewriter.setInsertionPoint(outerLoop);
541
542 SmallVector<Value> loopItrArgs;
543 auto zeroTileType = amx::TileType::get(shape: {16, 16}, elementType: opType);
544
545 for (int i = 0; i < size; i++) {
546 auto zeroTile = amx::TileZeroOp::create(builder&: rewriter, location: loc, res: zeroTileType);
547 loopItrArgs.push_back(Elt: zeroTile);
548 }
549 return loopItrArgs;
550}
551
552// Find memref source and indices of the accumulator tile for an AMX
553// contraction. Returns nullopt if the access is not suitable for AMX tile
554// load/store.
555static std::optional<std::pair<Value, SmallVector<Value>>>
556getAccTileAccess(Operation *op) {
557 if (!op)
558 return std::nullopt;
559
560 Value base;
561 VectorType vecTy;
562 ValueRange indices;
563 if (auto transfer = dyn_cast<VectorTransferOpInterface>(Val: op)) {
564 if (transfer.getMask() || transfer.hasOutOfBoundsDim() ||
565 !transfer.getPermutationMap().isMinorIdentity())
566 return std::nullopt;
567 base = transfer.getBase();
568 vecTy = transfer.getVectorType();
569 indices = transfer.getIndices();
570 } else if (auto load = dyn_cast<vector::LoadOp>(Val: op)) {
571 base = load.getBase();
572 vecTy = load.getVectorType();
573 indices = load.getIndices();
574 } else if (auto store = dyn_cast<vector::StoreOp>(Val: op)) {
575 base = store.getBase();
576 vecTy = store.getVectorType();
577 indices = store.getIndices();
578 } else {
579 return std::nullopt;
580 }
581
582 auto memrefTy = dyn_cast<MemRefType>(Val: base.getType());
583 ArrayRef<int64_t> shape = vecTy.getShape();
584 if (!memrefTy || memrefTy.getRank() < 2 || !memrefTy.isLastDimUnitStride() ||
585 shape.size() < 2 || shape.end()[-1] != 16 || shape.end()[-2] != 16 ||
586 llvm::any_of(Range: shape.drop_back(N: 2), P: [](int64_t d) { return d != 1; }))
587 return std::nullopt;
588
589 return std::make_pair(x&: base, y: SmallVector<Value>(indices));
590}
591
592// Returns the accumulator read and result write for each contraction if the
593// accumulator can be updated in place through direct AMX tile accesses.
594// If writeOnly is true, only the result writes are considered, and the read
595// accesses are ignored.
596static std::pair<std::optional<SmallVector<Operation *>>,
597 std::optional<SmallVector<Operation *>>>
598getInPlaceAccAccesses(ArrayRef<vector::ContractionOp> ops, bool writeOnly) {
599 SmallVector<Operation *> reads, writes;
600 for (vector::ContractionOp op : ops) {
601 Value result = contractionUsersAfterYield(v: op.getResult());
602 if (!result || !result.hasOneUse())
603 return {std::nullopt, std::nullopt};
604 Operation *write = *result.user_begin();
605 if (!isa<vector::TransferWriteOp, vector::StoreOp>(Val: write) ||
606 write->getOperand(idx: 0) != result || !getAccTileAccess(op: write))
607 return {std::nullopt, std::nullopt};
608 writes.push_back(Elt: write);
609
610 if (writeOnly)
611 continue;
612
613 Operation *read = traceToVectorReadLikeParentOperation(v: op.getAcc());
614 if (!read || !getAccTileAccess(op: read))
615 return {std::nullopt, std::nullopt};
616 reads.push_back(Elt: read);
617 }
618
619 if (writeOnly)
620 return {std::nullopt, writes};
621 return std::make_pair(x&: reads, y&: writes);
622}
623
624// Loads the initial accumulator tiles before the given loop.
625static SmallVector<Value> createAccTileLoads(OpBuilder &rewriter, Location loc,
626 Type opType, scf::ForOp loop,
627 ArrayRef<Operation *> reads) {
628 OpBuilder::InsertionGuard guard(rewriter);
629 rewriter.setInsertionPoint(loop);
630 auto tileType = amx::TileType::get(shape: {16, 16}, elementType: opType);
631 SmallVector<Value> tiles;
632 for (Operation *read : reads) {
633 auto [base, indices] = *getAccTileAccess(op: read);
634 tiles.push_back(
635 Elt: amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base, indices));
636 }
637 return tiles;
638}
639
640static Value getIndxToLoadStoreFromPckBuffer(OpBuilder &rewriter, Location loc,
641 Value ivInnerLoop,
642 Value ivOuterLoop,
643 bool isInnerLoopUBHasOddQuot,
644 bool isInnerLoopUBLarger,
645 bool pack, Value blockStride) {
646
647 Value c2 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 2);
648
649 // `blockStride` is the reduction (K) loop step, i.e. the amount by which the
650 // induction variable advances for one K-block. Dividing the induction value
651 // by it yields the K-block index regardless of whether the loop counts
652 // K-elements (step == 16*blockingFactor) or pre-blocked K-tiles (step == 1).
653 Value quotientInnerLoop =
654 arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: ivInnerLoop, rhs: blockStride);
655 Value remInnerLoop = arith::RemUIOp::create(
656 builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: quotientInnerLoop, rhs: c2);
657
658 if (!isInnerLoopUBLarger && !pack) {
659 remInnerLoop = arith::RemUIOp::create(
660 builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: ivOuterLoop, rhs: c2);
661 }
662
663 if (isInnerLoopUBHasOddQuot) {
664 auto remOuterLoop = arith::RemUIOp::create(
665 builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: ivOuterLoop, rhs: c2);
666 auto remAdd = arith::AddIOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
667 lhs: remInnerLoop, rhs: remOuterLoop);
668 remInnerLoop = arith::RemUIOp::create(builder&: rewriter, location: loc,
669 result: rewriter.getIndexType(), lhs: remAdd, rhs: c2);
670 }
671
672 return remInnerLoop;
673}
674
675static scf::ForOp
676createLoops(OpBuilder &rewriter, Location loc, Value lowerBound,
677 Value upperBound, Value step, SmallVector<Value> loopItrArgs,
678 Type ipType, Type opType, unsigned int blockingFactor, bool isVnni,
679 Operation *vectorOpLhs, Operation *vectorOpRhs,
680 vector::ContractionOp contractOp, scf::ForOp outerLoop,
681 scf::ForOp innerLoop, SmallVector<vector::ContractionOp> ops,
682 Value ivOuterLoop, Value packedBuffer, bool pack,
683 arith::ConstantIndexOp innerLoopIndex, bool isInnerLoopUBLarger,
684 bool isInnerLoopUBHasOddQuot) {
685
686 Value c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
687 Value c1 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1);
688 Value c2 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 2);
689
690 int64_t offset = 16 * blockingFactor;
691 if (auto cst = step.getDefiningOp<arith::ConstantIndexOp>())
692 offset = cst.value();
693
694 auto newLoop = scf::ForOp::create(
695 builder&: rewriter, location: loc, lowerBound, upperBound, step, initArgs: loopItrArgs,
696 odsArg4: [&](OpBuilder &rewriterNewInnerLoop, Location locNewInnerLoop,
697 Value ivNewInnerLoop, ValueRange iterArgsNewInnerLoop) {
698 IRMapping mapping;
699 if (outerLoop)
700 mapping.map(from: vectorOpLhs->getOperand(
701 idx: getIndexPosition(operand: contractOp.getLhs(), loop: outerLoop) + 1),
702 to: ivOuterLoop);
703
704 mapping.map(from: vectorOpLhs->getOperand(
705 idx: getIndexPosition(operand: contractOp.getLhs(), loop: innerLoop) + 1),
706 to: ivNewInnerLoop);
707 auto lhsClone = rewriterNewInnerLoop.clone(op&: *vectorOpLhs, mapper&: mapping);
708
709 Value indxToStoreInBuffer = c0;
710 Value indxToLoadFromBuffer = c0;
711 if (!isVnni) {
712 if (outerLoop) {
713 if (innerLoopIndex.value() == 0) {
714 if (pack) {
715 ivNewInnerLoop = c0;
716 ivOuterLoop = arith::AddIOp::create(builder&: rewriter, location: locNewInnerLoop,
717 lhs: c1, rhs: ivOuterLoop);
718
719 if (!isInnerLoopUBLarger || isInnerLoopUBHasOddQuot) {
720 indxToStoreInBuffer = arith::RemUIOp::create(
721 builder&: rewriter, location: locNewInnerLoop, result: rewriter.getIndexType(),
722 lhs: ivOuterLoop, rhs: c2);
723 }
724
725 Value indxToLoadFromMatB = arith::AddIOp::create(
726 builder&: rewriter, location: loc, lhs: indxToStoreInBuffer, rhs: c1);
727 indxToLoadFromBuffer = arith::RemUIOp::create(
728 builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: indxToLoadFromMatB,
729 rhs: c2);
730 }
731
732 } else {
733 Value nLoadIndx = arith::ConstantIndexOp::create(
734 builder&: rewriter, location: locNewInnerLoop, value: offset);
735 ivNewInnerLoop = arith::AddIOp::create(builder&: rewriter, location: locNewInnerLoop,
736 lhs: nLoadIndx, rhs: ivNewInnerLoop);
737 indxToStoreInBuffer = getIndxToLoadStoreFromPckBuffer(
738 rewriter, loc, ivInnerLoop: ivNewInnerLoop, ivOuterLoop,
739 isInnerLoopUBHasOddQuot, isInnerLoopUBLarger, pack, blockStride: step);
740 Value indxToLoadFromMatB =
741 arith::AddIOp::create(builder&: rewriter, location: loc, lhs: indxToStoreInBuffer, rhs: c1);
742 indxToLoadFromBuffer =
743 arith::RemUIOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
744 lhs: indxToLoadFromMatB, rhs: c2);
745 }
746 } else {
747 if (pack) {
748 Value nLoadIndx = arith::ConstantIndexOp::create(
749 builder&: rewriter, location: locNewInnerLoop, value: offset);
750 ivNewInnerLoop = arith::AddIOp::create(builder&: rewriter, location: locNewInnerLoop,
751 lhs: nLoadIndx, rhs: ivNewInnerLoop);
752 Value quotient_K = arith::DivUIOp::create(
753 builder&: rewriter, location: loc, lhs: ivNewInnerLoop, rhs: nLoadIndx);
754 indxToStoreInBuffer = arith::RemUIOp::create(
755 builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: quotient_K, rhs: c2);
756
757 Value indxToLoadFromMatB =
758 arith::AddIOp::create(builder&: rewriter, location: loc, lhs: indxToStoreInBuffer, rhs: c1);
759 indxToLoadFromBuffer =
760 arith::RemUIOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
761 lhs: indxToLoadFromMatB, rhs: c2);
762 }
763 }
764 }
765 IRMapping rhsMapping;
766
767 Value matB;
768 Operation *rhsOp = vectorOpRhs;
769
770 // Clone for the subview type operations
771 if (rhsOp->getNumOperands() > 0) {
772
773 if (outerLoop) {
774 int64_t outerPos = getIndexPosition(operand: contractOp.getRhs(), loop: outerLoop);
775
776 if (outerPos >= 0) {
777 unsigned operandIdx = static_cast<unsigned>(outerPos + 1);
778
779 if (operandIdx < rhsOp->getNumOperands())
780 rhsMapping.map(from: rhsOp->getOperand(idx: operandIdx), to: ivOuterLoop);
781 }
782 }
783
784 int64_t innerPos = getIndexPosition(operand: contractOp.getRhs(), loop: innerLoop);
785
786 if (innerPos >= 0) {
787 unsigned operandIdx = static_cast<unsigned>(innerPos + 1);
788
789 if (operandIdx < rhsOp->getNumOperands())
790 rhsMapping.map(from: rhsOp->getOperand(idx: operandIdx), to: ivNewInnerLoop);
791 }
792
793 auto rhsClone = rewriterNewInnerLoop.clone(op&: *rhsOp, mapper&: rhsMapping);
794 matB = rhsClone->getResult(idx: 0);
795
796 } else {
797 // The mat B is of kind 'memref.get_global @__constant'
798 matB = rhsOp->getResult(idx: 0);
799 }
800
801 if (!isVnni) {
802 if (outerLoop) {
803 if (!pack) {
804 matB = Value();
805 indxToLoadFromBuffer = c0;
806 // Use the real spill-block induction value (== spillInnerLoop)
807 // together with the loop step so the computed ping-pong slot
808 // matches the prefetch store side for any number of register
809 // blocks, including odd counts (e.g. 96 = 3 blocks). Passing a
810 // constant here mis-parities the slot for odd block counts.
811 indxToLoadFromBuffer = getIndxToLoadStoreFromPckBuffer(
812 rewriter, loc, ivInnerLoop: ivNewInnerLoop, ivOuterLoop,
813 isInnerLoopUBHasOddQuot, isInnerLoopUBLarger, pack, blockStride: step);
814 }
815 } else {
816 if (!pack) {
817 Value nLoadIndx = arith::ConstantIndexOp::create(
818 builder&: rewriter, location: locNewInnerLoop, value: offset);
819 matB = Value();
820 Value quotient_K = arith::DivUIOp::create(
821 builder&: rewriter, location: loc, lhs: ivNewInnerLoop, rhs: nLoadIndx);
822 indxToLoadFromBuffer = arith::RemUIOp::create(
823 builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: quotient_K, rhs: c2);
824 }
825 }
826 }
827 // compute tiled dot-product
828 SmallVector<Value> accumulators = createTiledDp(
829 rewriter, loc: locNewInnerLoop, ops, matA: lhsClone->getResult(idx: 0), matB,
830 ipType, opType, accIterArgs: iterArgsNewInnerLoop, offset: blockingFactor, isVnni,
831 packedBuffer, pack, indxToStoreInBuffer, indxToLoadFromMatB: indxToLoadFromBuffer);
832
833 scf::YieldOp::create(builder&: rewriterNewInnerLoop, location: locNewInnerLoop,
834 results: accumulators);
835 });
836
837 return newLoop;
838}
839
840// Implements tiled dot-product operation for a vector.contract operation or a
841// sequence of vector.contracts inside the reduction loops.
842//
843// For example:
844// Case 1: register blocked vector.contract with prepacked input
845// ```
846// vector.transfer_read %arg0 {{.}*} : memref<16x32x4xi8>, vector<16x16x4xi8>
847// vector.transfer_read %arg1 {{.}*} : memref<16x32x4xi8>, vector<16x16x4xi8>
848// vector.contract <16x16x4xi8>, <16x16x4xi8> into <16x16xi32>
849// vector.transfer_write arg2 {{.}*} : vector<16x16xi32>, memref<32x32xi32>
850// ```
851// to
852// ```
853// amx.tile_load %arg0 {{.}*} : memref<16x32x4xi8> into !amx.tile<16x64xi8>
854// amx.tile_load %arg1 {{.}*} : memref<16x32x4xi8> into !amx.tile<16x64xi8>
855// amx.tile_muli !amx.tile<16x64xi8> -> !amx.tile<16x16xi32>
856// amx.tile_store %arg2{{.}*} : memref<32x32xi32>, !amx.tile<16x16xi32>
857// ```
858//
859//
860// Case2: vector.contract with register blocked
861//
862// Output IR with online packing (with s/w pipeline advantage):
863// s/w pipeline: load, pack to VNNI, and store the B sub matrix
864// of the 0th batch-reduce and K iteration.
865// scf.for (0 to 31) {
866// - load 0th and 1st vector<32xbf16>, pack into VNNI, store the
867// first shuffle in 0th and 2nd shuffle in 16th index of the
868// buffer.
869// }
870// scf.for (0 to br-2) { batch-reduce loop
871// scf.for (0 to k-2) { K loop
872// - load A matrix
873// - scf.loop for s/w pipeline: load, pack to VNNI, and store the B sub
874// matrix for the next K loop iteration (c) load VNNI pack B matrix of K
875// iteration from the buffer (d) compute the tiled dot-product
876// }
877// Last iteration of the the K Loop (k-1) {
878// - load A matrix
879// - scf.loop for s/w pipeline: load, pack to VNNI, and store the B sub
880// matrix for the next batch-reduce + K loop iteration (c) load VNNI pack B
881// matrix of K iteration from the buffer (d) compute the tiled dot-product
882// }
883// }
884// Last iteration of the batch-reduce loop (br-1) {
885// scf.for (0 to k-2) { K loop
886// - load A matrix
887// - scf.loop for s/w pipeline: load, pack to VNNI, and store the B sub
888// matrix for the next K loop iteration (c) load VNNI pack B matrix of K
889// iteration from the buffer (d) compute the tiled dot-product
890// }
891// Last iteration of the the K Loop (k-1) {
892// - load A matrix
893// - load VNNI pack B matrix of K iteration from the buffer
894// - compute the tiled dot-product
895// }
896// }
897//
898// scf.for (0 to M)
899// scf.for (0 to N)
900// - Load the ith and i+1th acc
901// - Shuffle them as we packed using vpunpack
902// - Load C matrix and do arith.add with the shuffle
903// - Store back into C matrix
904struct VectorContractToAMXDotProduct : public AddContractionOpRewritePattern {
905 using Base::Base;
906
907 LogicalResult
908 matchAndRewriteAddContractOp(vector::ContractionOp contractOp,
909 PatternRewriter &rewriter) const override {
910
911 unsigned int blockingFactor =
912 contractOp.getLhsType().getElementType().isBF16() ? 2 : 4;
913 bool isVnni =
914 isInVnniLayout(op: contractOp.getOperation(),
915 indexingMaps: contractOp.getIndexingMapsArray(), blockingFactor);
916
917 VectorType lhsTy = contractOp.getLhsType();
918 if (!lhsTy.getElementType().isBF16() &&
919 !lhsTy.getElementType().isSignlessInteger(width: 8) &&
920 !lhsTy.getElementType().isF8E4M3FN() &&
921 !lhsTy.getElementType().isF8E5M2())
922 return rewriter.notifyMatchFailure(
923 arg&: contractOp, msg: "Only BF16/Int8/F8 lowering is supported.");
924
925 if (lhsTy.getElementType() != contractOp.getRhsType().getElementType())
926 return rewriter.notifyMatchFailure(
927 arg&: contractOp, msg: "Contraction should have same lhs and rhs type.");
928
929 VectorType accTy = dyn_cast<VectorType>(Val: contractOp.getAccType());
930 if (!accTy)
931 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Wrong accmulator type.");
932
933 if (((lhsTy.getElementType().isBF16() ||
934 lhsTy.getElementType().isF8E4M3FN() ||
935 lhsTy.getElementType().isF8E5M2()) &&
936 !accTy.getElementType().isF32()) ||
937 (lhsTy.getElementType().isSignlessInteger(width: 8) &&
938 !accTy.getElementType().isSignlessInteger(width: 32)))
939 return rewriter.notifyMatchFailure(arg&: contractOp,
940 msg: "Only F32 for BF16 or Int32 for Int8 "
941 "accumulation type is supported.");
942
943 Operation *accReadOp =
944 traceToVectorReadLikeParentOperation(v: contractOp.getAcc());
945
946 // Only the contract result's first consumer is needed, not the final
947 // store. This keeps the lowering independent of the epilogue ops (truncf,
948 // bias add, ReLU, ...) that sit between the contraction and the write.
949 Value resultChainEnd = contractionUsersAfterYield(v: contractOp.getResult());
950
951 if (!accReadOp || !resultChainEnd)
952 return rewriter.notifyMatchFailure(
953 arg&: contractOp, msg: "The ACC operand of the vector.contract should be a "
954 "transfer_read or a load. And, the result should have a "
955 "single-use chain to its consumer.");
956
957 bool isAccZeroVectorConstant = isZeroVectorConstant(op: accReadOp);
958
959 Block *resultBlock = resultChainEnd.user_begin()->getBlock();
960
961 Type ipType = rewriter.getBF16Type();
962 Type opType = rewriter.getF32Type();
963
964 if (lhsTy.getElementType().isSignlessInteger(width: 8)) {
965 ipType = rewriter.getIntegerType(width: 8);
966 opType = rewriter.getIntegerType(width: 32);
967 }
968
969 if (lhsTy.getElementType().isF8E4M3FN())
970 ipType = rewriter.getF8E4M3FNType();
971
972 if (lhsTy.getElementType().isF8E5M2())
973 ipType = rewriter.getF8E5M2Type();
974
975 if (accReadOp->getBlock() == contractOp->getBlock() &&
976 resultBlock != contractOp->getBlock())
977 return rewriter.notifyMatchFailure(
978 arg&: contractOp, msg: "The accumulator store is in different block.");
979
980 if (accReadOp->getBlock() != contractOp->getBlock() &&
981 resultBlock == contractOp->getBlock())
982 return rewriter.notifyMatchFailure(
983 arg&: contractOp, msg: "The accumulator read is in different block.");
984
985 if (!(isReadSrcMemref(operand: contractOp.getLhs()) &&
986 isReadSrcMemref(operand: contractOp.getRhs())))
987 return rewriter.notifyMatchFailure(
988 arg&: contractOp, msg: "The LHS or RHS src is not a MemRef type.");
989
990 if (isVnni && !(isVnniOperandRead(operand: contractOp.getLhs(), vnni: blockingFactor) &&
991 isVnniOperandRead(operand: contractOp.getRhs(), vnni: blockingFactor)))
992 return rewriter.notifyMatchFailure(
993 arg&: contractOp, msg: "The LHS or RHS src does not read whole VNNI groups.");
994
995 unsigned int dimValue = blockingFactor;
996 if (!isVnni)
997 dimValue = 16 * blockingFactor;
998
999 // Case 1: For just one VC rewrite. Where all accumulator read/write
1000 // within the same block.
1001 if (accReadOp->getBlock() == contractOp->getBlock() &&
1002 resultBlock == contractOp->getBlock()) {
1003
1004 if (!isReadSrcMemref(operand: contractOp.getAcc()))
1005 return rewriter.notifyMatchFailure(arg&: contractOp,
1006 msg: "The ACC src is not a MemRef type.");
1007
1008 bool collapse = false;
1009 if (isVnni)
1010 collapse = true;
1011
1012 LogicalResult validate = validateContractOps(
1013 rewriter, contractOp, blockingFactor: dimValue, srcBuffLhs: Value(), srcBuffRhs: Value(), srcValidate: false);
1014
1015 if (failed(Result: validate))
1016 return rewriter.notifyMatchFailure(
1017 arg&: contractOp, msg: "The contract operation doesn't satisfy the operands "
1018 "dimensions. M, N, and vnni dims are 16, 16, and 2/4. "
1019 "The rest dims should be 1. Op should have one user.");
1020
1021 Location loc = contractOp.getLoc();
1022
1023 auto srcIndxLhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(),
1024 operand: contractOp.getLhs(), isNotAcc: collapse);
1025 if (failed(Result: srcIndxLhs))
1026 return rewriter.notifyMatchFailure(arg&: contractOp,
1027 msg: "Failed to get the LHS src.");
1028 auto [srcBuffLhs, indicesLhs] = *srcIndxLhs;
1029
1030 auto srcIndxRhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(),
1031 operand: contractOp.getRhs(), isNotAcc: collapse);
1032 if (failed(Result: srcIndxRhs))
1033 return rewriter.notifyMatchFailure(arg&: contractOp,
1034 msg: "Failed to get the RHS src.");
1035 auto rhsSrc = *srcIndxRhs;
1036 auto srcBuffRhs = rhsSrc.first;
1037 auto indicesRhs = rhsSrc.second;
1038
1039 auto srcIndxAcc = getSrcIndxValue(rewriter, loc: contractOp.getLoc(),
1040 operand: contractOp.getAcc(), isNotAcc: false);
1041 if (failed(Result: srcIndxAcc))
1042 return rewriter.notifyMatchFailure(arg&: contractOp,
1043 msg: "Failed to get the ACC src.");
1044 auto [srcBuffAcc, indicesAcc] = *srcIndxAcc;
1045
1046 Value c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
1047
1048 // amx.tile_loads
1049 auto tileType = amx::TileType::get(shape: {16, (16 * blockingFactor)}, elementType: ipType);
1050 auto loadLhs = amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType,
1051 base: srcBuffLhs, indices: indicesLhs);
1052
1053 // Create the subview and then load.
1054 amx::TileLoadOp loadRhs;
1055 if (!isVnni) {
1056 VectorType vecTy;
1057 SmallVector<OpFoldResult> indexVals;
1058 llvm::TypeSwitch<Operation *>(contractOp.getRhs().getDefiningOp())
1059 .Case<TransferReadOp, LoadOp>(caseFn: [&](auto readOp) {
1060 indexVals = SmallVector<OpFoldResult>(readOp.getIndices().begin(),
1061 readOp.getIndices().end());
1062 vecTy = readOp.getType();
1063 });
1064 auto one = rewriter.getIndexAttr(value: 1);
1065 SmallVector<OpFoldResult> strides(indexVals.size(), one);
1066 // A rank-reducing read drops leading unit dims. Append them back for
1067 // the subview.
1068 SmallVector<int64_t> subviewShape(indexVals.size() - vecTy.getRank(),
1069 1);
1070 llvm::append_range(C&: subviewShape, R: vecTy.getShape());
1071 SmallVector<OpFoldResult> sizes =
1072 getAsIndexOpFoldResult(ctx: contractOp->getContext(), values: subviewShape);
1073 auto subview = memref::SubViewOp::create(builder&: rewriter, location: loc, source: srcBuffRhs,
1074 offsets: indexVals, sizes, strides);
1075 auto bufferType = MemRefType::get(shape: {16, (16 * blockingFactor)}, elementType: ipType);
1076 auto packedBuffer = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: bufferType);
1077
1078 // Create a loop that does online packing.
1079 Value step =
1080 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: blockingFactor);
1081 Value uBound = arith::ConstantIndexOp::create(builder&: rewriter, location: loc,
1082 value: (blockingFactor * 16));
1083 Value nextLoadIndx =
1084 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: (blockingFactor / 2));
1085 Value nextStoreIndx = arith::ConstantIndexOp::create(
1086 builder&: rewriter, location: loc, value: 16 * (blockingFactor / 2));
1087
1088 scf::ForOp::create(
1089 builder&: rewriter, location: loc, lowerBound: c0, upperBound: uBound, step, initArgs: ValueRange{},
1090 odsArg4: [&](OpBuilder &nestedBuilder, Location loc, Value iv,
1091 ValueRange iterArgs) {
1092 Value i1_load =
1093 arith::AddIOp::create(builder&: rewriter, location: loc, lhs: nextLoadIndx, rhs: iv);
1094
1095 // The source subview already accounts for the leading unit
1096 // dimensions.
1097 size_t numOffsets = indicesRhs.size();
1098 SmallVector<Value> indices(numOffsets, c0);
1099 indices[numOffsets - 2] = iv;
1100 auto vec1 = vector::LoadOp::create(
1101 builder&: rewriter, location: loc,
1102 resultType: VectorType::get(shape: 16 * (blockingFactor / 2), elementType: ipType), base: subview,
1103 indices);
1104
1105 indices[numOffsets - 2] = i1_load;
1106 auto vec2 = vector::LoadOp::create(
1107 builder&: rewriter, location: loc,
1108 resultType: VectorType::get(shape: 16 * (blockingFactor / 2), elementType: ipType), base: subview,
1109 indices);
1110
1111 vector::ShuffleOp writeVal1;
1112 vector::ShuffleOp writeVal2;
1113
1114 if (blockingFactor == 2) {
1115
1116 writeVal1 = vector::ShuffleOp::create(
1117 builder&: rewriter, location: loc, result: VectorType::get(shape: {16}, elementType: ipType), v1: vec1, v2: vec2,
1118 mask: ArrayRef<int64_t>{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21,
1119 6, 22, 7, 23});
1120
1121 writeVal2 = vector::ShuffleOp::create(
1122 builder&: rewriter, location: loc, result: VectorType::get(shape: {16}, elementType: ipType), v1: vec1, v2: vec2,
1123 mask: ArrayRef<int64_t>{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13,
1124 29, 14, 30, 15, 31});
1125 }
1126
1127 if (blockingFactor == 4) {
1128 writeVal1 = vector::ShuffleOp::create(
1129 builder&: rewriter, location: loc, result: VectorType::get(shape: {32}, elementType: ipType), v1: vec1, v2: vec2,
1130 mask: ArrayRef<int64_t>{0, 16, 32, 48, 1, 17, 33, 49,
1131 2, 18, 34, 50, 3, 19, 35, 51,
1132 4, 20, 36, 52, 5, 21, 37, 53,
1133 6, 22, 38, 54, 7, 23, 39, 55});
1134
1135 writeVal2 = vector::ShuffleOp::create(
1136 builder&: rewriter, location: loc, result: VectorType::get(shape: {32}, elementType: ipType), v1: vec1, v2: vec2,
1137 mask: ArrayRef<int64_t>{8, 24, 40, 56, 9, 25, 41, 57,
1138 10, 26, 42, 58, 11, 27, 43, 59,
1139 12, 28, 44, 60, 13, 29, 45, 61,
1140 14, 30, 46, 62, 15, 31, 47, 63});
1141 }
1142
1143 auto rem = arith::DivUIOp::create(
1144 builder&: rewriter, location: loc, result: rewriter.getIndexType(), lhs: iv, rhs: step);
1145
1146 vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal1, base: packedBuffer,
1147 indices: ValueRange{rem, c0});
1148 vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal2, base: packedBuffer,
1149 indices: ValueRange{rem, nextStoreIndx});
1150
1151 scf::YieldOp::create(builder&: nestedBuilder, location: loc);
1152 });
1153 loadRhs = amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: packedBuffer,
1154 indices: ValueRange{c0, c0});
1155 } else {
1156
1157 loadRhs = amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileType, base: srcBuffRhs,
1158 indices: indicesRhs);
1159 }
1160
1161 auto tileTypeAcc = amx::TileType::get(shape: {16, 16}, elementType: opType);
1162 auto loadAcc = amx::TileLoadOp::create(builder&: rewriter, location: loc, res: tileTypeAcc,
1163 base: srcBuffAcc, indices: indicesAcc);
1164
1165 // Tiled dot-product.
1166 Value dp;
1167 if (ipType.isBF16() || ipType.isF8E5M2() || ipType.isF8E4M3FN())
1168 dp = amx::TileMulFOp::create(builder&: rewriter, location: loc, res: tileTypeAcc, lhs: loadLhs,
1169 rhs: loadRhs, acc: loadAcc);
1170
1171 if (ipType.isSignlessInteger(width: 8))
1172 dp = amx::TileMulIOp::create(builder&: rewriter, location: loc, res: tileTypeAcc, lhs: loadLhs,
1173 rhs: loadRhs, acc: loadAcc);
1174
1175 auto bufferType = MemRefType::get(shape: {16, 16}, elementType: opType);
1176 auto resultBuffer = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: bufferType);
1177
1178 amx::TileStoreOp::create(builder&: rewriter, location: loc, base: resultBuffer, indices: ValueRange{c0, c0},
1179 val: dp);
1180
1181 auto vectorType = VectorType::get(shape: {16, 16}, elementType: opType);
1182 int64_t srcRank =
1183 (dyn_cast<ShapedType>(Val: resultBuffer.getType())).getRank();
1184 Value padding = ub::PoisonOp::create(builder&: rewriter, location: loc, result: opType);
1185 auto map = AffineMap::getMinorIdentityMap(dims: srcRank, results: vectorType.getRank(),
1186 context: rewriter.getContext());
1187 SmallVector<bool> inBounds(vectorType.getRank(), true);
1188
1189 Value vecRow = vector::TransferReadOp::create(
1190 builder&: rewriter, location: loc, vectorType, source: resultBuffer, indices: ValueRange{c0, c0}, padding,
1191 permutationMap: map, inBounds);
1192
1193 Value resultOp = contractionUsersAfterYield(v: contractOp.getResult());
1194 if (auto vecType = dyn_cast<VectorType>(Val: resultOp.getType()))
1195 vecRow = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: vecType, source: vecRow);
1196
1197 rewriter.replaceAllUsesWith(from: resultOp, to: vecRow);
1198 return success();
1199 }
1200
1201 // Case 2: The acc are passed as iter args through the reduction loop.
1202 // We support, reduction loop depth until 2. TODO: Support for n-depth
1203 // reduction loop.
1204 // TODOs: Re-factor 2a and 2b.
1205 SmallVector<scf::ForOp> loopLists;
1206 Operation *current = contractOp;
1207 while (true) {
1208 Operation *parent = current->getParentOfType<scf::ForOp>();
1209
1210 if (!parent) {
1211 // The accumulator initialization can be hoisted above an enclosing
1212 // parallel region (scf.parallel/scf.forall) when the register tile
1213 // matches the problem size and the M/N register loops fold away. In
1214 // that case the reduction loop(s) collected so far are still valid to
1215 // rewrite, so stop climbing instead of bailing out.
1216 if (!loopLists.empty())
1217 break;
1218 return rewriter.notifyMatchFailure(
1219 arg&: contractOp,
1220 msg: "Accumulator read and contract op not within scf.for op");
1221 }
1222
1223 loopLists.push_back(Elt: dyn_cast<scf::ForOp>(Val: parent));
1224
1225 if (resultBlock == parent->getBlock()) {
1226 break;
1227 }
1228
1229 current = parent;
1230 }
1231
1232 if (loopLists.size() > 2 || loopLists.size() == 0)
1233 return rewriter.notifyMatchFailure(
1234 arg&: contractOp, msg: "Rewrite is supported until reduction loop depth of 2.");
1235
1236 auto srcIndxLhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(),
1237 operand: contractOp.getLhs(), isNotAcc: false);
1238 if (failed(Result: srcIndxLhs))
1239 return rewriter.notifyMatchFailure(arg&: contractOp,
1240 msg: "Failed to get the LHS src.");
1241 auto [srcBuffLhs, indicesLhs] = *srcIndxLhs;
1242
1243 auto srcIndxRhs = getSrcIndxValue(rewriter, loc: contractOp.getLoc(),
1244 operand: contractOp.getRhs(), isNotAcc: false);
1245 if (failed(Result: srcIndxRhs))
1246 return rewriter.notifyMatchFailure(arg&: contractOp,
1247 msg: "Failed to get the RHS src.");
1248 auto [srcBuffRhs, indicesRhs] = *srcIndxRhs;
1249 Operation *vectorOpLhs;
1250 llvm::TypeSwitch<Operation *>(contractOp.getLhs().getDefiningOp())
1251 .Case<TransferReadOp, LoadOp>(caseFn: [&](auto readOp) {
1252 vectorOpLhs = readOp.getBase().getDefiningOp();
1253 });
1254
1255 Operation *vectorOpRhs;
1256 llvm::TypeSwitch<Operation *>(contractOp.getRhs().getDefiningOp())
1257 .Case<TransferReadOp, LoadOp>(caseFn: [&](auto readOp) {
1258 vectorOpRhs = readOp.getBase().getDefiningOp();
1259 });
1260
1261 if (!vectorOpLhs || !vectorOpRhs)
1262 return rewriter.notifyMatchFailure(
1263 arg&: contractOp, msg: "Failed to find LHS or RHS read source operation");
1264
1265 // Retrive all the contaction operation within the loop.
1266 SmallVector<vector::ContractionOp> ops;
1267 // Accumulators whose initial value passed into the rewritten reduction
1268 // loops is neither read from memory nor zero, e.g. an iter_arg of an
1269 // enclosing loop. Paired with the index of their contraction in `ops`.
1270 SmallVector<std::pair<size_t, Value>> carriedAccs;
1271 for (Operation &op : loopLists[0].getBody()->getOperations()) {
1272
1273 if (auto contract = dyn_cast<vector::ContractionOp>(Val&: op)) {
1274 if (isVnni && !(isVnniOperandRead(operand: contract.getLhs(), vnni: blockingFactor) &&
1275 isVnniOperandRead(operand: contract.getRhs(), vnni: blockingFactor)))
1276 return rewriter.notifyMatchFailure(
1277 arg&: contractOp, msg: "The LHS or RHS src of an associated contract "
1278 "operation does not read whole VNNI groups.");
1279
1280 LogicalResult validate = validateContractOps(
1281 rewriter, contractOp: contract, blockingFactor: dimValue, srcBuffLhs, srcBuffRhs, srcValidate: true);
1282
1283 if (failed(Result: validate))
1284 return rewriter.notifyMatchFailure(
1285 arg&: contractOp,
1286 msg: "The associated contract operations doesn't satisfy "
1287 "the re-write conditions either the dimensions are "
1288 "wrong or MemRef source are different or many users.");
1289
1290 Operation *contractReadOp =
1291 traceToVectorReadLikeParentOperation(v: contract.getAcc());
1292
1293 if (isAccZeroVectorConstant != isZeroVectorConstant(op: contractReadOp))
1294 return rewriter.notifyMatchFailure(
1295 arg&: contractOp, msg: "The input acc should be a zero constant or "
1296 "transfer_read for all contracts");
1297
1298 Value accInit = getLoopNestInit(acc: contract.getAcc(), loops: loopLists);
1299 Operation *accInitOp = accInit.getDefiningOp();
1300 bool carried = !isa_and_present<TransferReadOp, LoadOp>(Val: accInitOp) &&
1301 !isZeroVectorConstant(op: accInitOp);
1302 if (carried) {
1303 if (loopLists.back().getRegion().isAncestor(
1304 other: accInit.getParentRegion()))
1305 return rewriter.notifyMatchFailure(
1306 arg&: contractOp, msg: "The accumulator should enter the reduction loops "
1307 "through their init args.");
1308 carriedAccs.emplace_back(Args: ops.size(), Args&: accInit);
1309 }
1310
1311 ops.push_back(Elt: contract);
1312 }
1313 }
1314
1315 // The accumulators read from memory are added back to all the results at
1316 // once, so they can't be mixed with carried ones.
1317 if (!isAccZeroVectorConstant && !carriedAccs.empty() &&
1318 carriedAccs.size() != ops.size())
1319 return rewriter.notifyMatchFailure(
1320 arg&: contractOp, msg: "The accumulators read from memory can't be mixed with "
1321 "carried in accumulators.");
1322
1323 if (!isVnni) {
1324 unsigned int pairCount = 0;
1325 for (size_t j = 0; j < ops.size(); j++) {
1326 for (size_t i = j; i < ops.size(); i++) {
1327 if (i != j && validatePairVectorContract(contractOp: ops[j], pairContOp: ops[i], rhsHasMultipleNonUnitDims: true, nonUnitDimValue: 16))
1328 pairCount = pairCount + 2;
1329 }
1330 }
1331
1332 if (pairCount != ops.size())
1333 return rewriter.notifyMatchFailure(
1334 arg&: contractOp, msg: "Coudn't find the pair vector contract ");
1335 }
1336
1337 scf::ForOp innerLoop;
1338 scf::ForOp outerLoop;
1339
1340 // For VNNI layout, accumulator tiles stay in their original layout.
1341 // Load and store them back in place when possible.
1342 // Otherwise, start from zero tile and accumulate only the final result.
1343 std::optional<SmallVector<Operation *>> inPlaceAccReads, inPlaceAccWrites;
1344 if (isVnni)
1345 std::tie(args&: inPlaceAccReads, args&: inPlaceAccWrites) =
1346 getInPlaceAccAccesses(ops, /*writeOnly=*/isAccZeroVectorConstant);
1347 auto createInitTiles = [&](scf::ForOp loop) {
1348 if (inPlaceAccReads)
1349 return createAccTileLoads(rewriter, loc: loop.getLoc(), opType, loop,
1350 reads: *inPlaceAccReads);
1351 return createTileZeros(rewriter, loc: loop.getLoc(), opType, outerLoop: loop, size: ops.size());
1352 };
1353
1354 scf::ForOp newLoop;
1355 // Case 2a: Reduction loop depth is 2.
1356 if (loopLists.size() == 2) {
1357 outerLoop = loopLists[1];
1358 innerLoop = loopLists[0];
1359
1360 LogicalResult validateOuterLoopStep =
1361 validateLoopStep(rewriter, step: outerLoop.getStep(), value: 1);
1362 if (failed(Result: validateOuterLoopStep))
1363 return rewriter.notifyMatchFailure(arg&: contractOp, msg: "Invalid loop step.");
1364
1365 int64_t stepValue = 16;
1366 if (!isVnni)
1367 stepValue = stepValue * blockingFactor;
1368 LogicalResult validateInnerLoopStep =
1369 validateLoopStep(rewriter, step: innerLoop.getStep(), value: stepValue);
1370 if (failed(Result: validateInnerLoopStep))
1371 return rewriter.notifyMatchFailure(
1372 arg&: contractOp, msg: "Invalid loop step. The step should be 32 for BF16 and "
1373 "64 for Int8/F8.");
1374
1375 SmallVector<Value> loopItrArgs = createInitTiles(outerLoop);
1376 rewriter.setInsertionPoint(outerLoop);
1377
1378 if (isVnni) {
1379 newLoop = scf::ForOp::create(
1380 builder&: rewriter, location: outerLoop.getLoc(), lowerBound: outerLoop.getLowerBound(),
1381 upperBound: outerLoop.getUpperBound(), step: outerLoop.getStep(), initArgs: loopItrArgs,
1382 odsArg4: [&](OpBuilder &rewriterOuterLoop, Location locOuterLoop,
1383 Value ivOuterLoop, ValueRange iterArgsOuterLoop) {
1384 auto newInnerLoop = createLoops(
1385 rewriter, loc: innerLoop.getLoc(), lowerBound: innerLoop.getLowerBound(),
1386 upperBound: innerLoop.getUpperBound(), step: innerLoop.getStep(),
1387 loopItrArgs: iterArgsOuterLoop, ipType, opType, blockingFactor, isVnni,
1388 vectorOpLhs, vectorOpRhs, contractOp, outerLoop, innerLoop,
1389 ops, ivOuterLoop, packedBuffer: nullptr, pack: true, innerLoopIndex: nullptr, isInnerLoopUBLarger: false, isInnerLoopUBHasOddQuot: false);
1390
1391 scf::YieldOp::create(builder&: rewriterOuterLoop, location: locOuterLoop,
1392 results: newInnerLoop.getResults());
1393 });
1394
1395 } else {
1396
1397 bool isInnerLoopUBLarger = false;
1398 bool isInnerLoopUBHasOddQuot = false;
1399
1400 int64_t ubVal = 16 * blockingFactor;
1401 Value ub = innerLoop.getUpperBound();
1402 if (auto constOp = ub.getDefiningOp<arith::ConstantOp>()) {
1403 if (auto intAttr = dyn_cast<IntegerAttr>(Val: constOp.getValue())) {
1404 ubVal = intAttr.getInt();
1405 }
1406 }
1407
1408 isInnerLoopUBLarger = ubVal > 16 * blockingFactor;
1409 isInnerLoopUBHasOddQuot =
1410 (((ubVal / (16 * blockingFactor)) % 2) == 1) && isInnerLoopUBLarger;
1411
1412 rewriter.setInsertionPoint(outerLoop);
1413
1414 auto c0 =
1415 arith::ConstantIndexOp::create(builder&: rewriter, location: outerLoop.getLoc(), value: 0);
1416 auto c1 =
1417 arith::ConstantIndexOp::create(builder&: rewriter, location: outerLoop.getLoc(), value: 1);
1418 auto spillLoopBound = arith::ConstantIndexOp::create(
1419 builder&: rewriter, location: outerLoop.getLoc(), value: 16 * blockingFactor);
1420
1421 Value spillOuterLoop = arith::SubIOp::create(
1422 builder&: rewriter, location: outerLoop.getLoc(), lhs: outerLoop.getUpperBound(), rhs: c1);
1423 Value spillInnerLoop =
1424 arith::SubIOp::create(builder&: rewriter, location: innerLoop.getLoc(),
1425 lhs: innerLoop.getUpperBound(), rhs: spillLoopBound);
1426 auto bufferType =
1427 MemRefType::get(shape: {2, 32, (blockingFactor * 16)}, elementType: ipType);
1428 auto packedBuffer =
1429 memref::AllocaOp::create(builder&: rewriter, location: outerLoop.getLoc(), memrefType: bufferType);
1430
1431 // First Shuffling outside the reduction loops
1432 IRMapping rhsMapping;
1433 rhsMapping.map(
1434 from: vectorOpRhs->getOperand(
1435 idx: getIndexPosition(operand: contractOp.getRhs(), loop: outerLoop) + 1),
1436 to: outerLoop.getLowerBound());
1437 rhsMapping.map(
1438 from: vectorOpRhs->getOperand(
1439 idx: getIndexPosition(operand: contractOp.getRhs(), loop: innerLoop) + 1),
1440 to: innerLoop.getLowerBound());
1441 auto rhsClone = rewriter.clone(op&: *vectorOpRhs, mapper&: rhsMapping);
1442
1443 Value quotient_batch = arith::DivUIOp::create(
1444 builder&: rewriter, location: outerLoop.getLoc(), lhs: outerLoop.getLowerBound(),
1445 rhs: outerLoop.getStep());
1446 Value quotient_k = arith::DivUIOp::create(builder&: rewriter, location: outerLoop.getLoc(),
1447 lhs: innerLoop.getLowerBound(),
1448 rhs: innerLoop.getStep());
1449
1450 Value quotient_add = arith::AddIOp::create(builder&: rewriter, location: outerLoop.getLoc(),
1451 lhs: quotient_batch, rhs: quotient_k);
1452 Value c2 =
1453 arith::ConstantIndexOp::create(builder&: rewriter, location: outerLoop.getLoc(), value: 2);
1454 Value rem = arith::RemUIOp::create(builder&: rewriter, location: outerLoop.getLoc(),
1455 lhs: quotient_add, rhs: c2);
1456
1457 performShuffle(rewriter, loc: outerLoop.getLoc(), matB: rhsClone->getResult(idx: 0),
1458 ipType, offset: blockingFactor, packedBuffer, indxToStoreInBuffer: rem);
1459
1460 // First Set of Loops
1461 auto newLoopNonSpill = scf::ForOp::create(
1462 builder&: rewriter, location: outerLoop.getLoc(), lowerBound: outerLoop.getLowerBound(),
1463 upperBound: spillOuterLoop, step: outerLoop.getStep(), initArgs: loopItrArgs,
1464 odsArg4: [&](OpBuilder &rewriterOuterLoop, Location locOuterLoop,
1465 Value ivOuterLoop, ValueRange iterArgsOuterLoop) {
1466 auto newInnerLoop1 = createLoops(
1467 rewriter, loc: innerLoop.getLoc(), lowerBound: innerLoop.getLowerBound(),
1468 upperBound: spillInnerLoop, step: innerLoop.getStep(), loopItrArgs: iterArgsOuterLoop,
1469 ipType, opType, blockingFactor, isVnni, vectorOpLhs,
1470 vectorOpRhs, contractOp, outerLoop, innerLoop, ops,
1471 ivOuterLoop, packedBuffer, pack: true, innerLoopIndex: spillLoopBound,
1472 isInnerLoopUBLarger, isInnerLoopUBHasOddQuot);
1473
1474 auto newInnerLoop = createLoops(
1475 rewriter, loc: innerLoop.getLoc(), lowerBound: spillInnerLoop,
1476 upperBound: innerLoop.getUpperBound(), step: innerLoop.getStep(),
1477 loopItrArgs: newInnerLoop1.getResults(), ipType, opType, blockingFactor,
1478 isVnni, vectorOpLhs, vectorOpRhs, contractOp, outerLoop,
1479 innerLoop, ops, ivOuterLoop, packedBuffer, pack: true, innerLoopIndex: c0,
1480 isInnerLoopUBLarger, isInnerLoopUBHasOddQuot);
1481
1482 scf::YieldOp::create(builder&: rewriterOuterLoop, location: locOuterLoop,
1483 results: newInnerLoop.getResults());
1484 });
1485
1486 // Last set of Loops
1487 newLoop = scf::ForOp::create(
1488 builder&: rewriter, location: outerLoop.getLoc(), lowerBound: spillOuterLoop,
1489 upperBound: outerLoop.getUpperBound(), step: outerLoop.getStep(),
1490 initArgs: newLoopNonSpill.getResults(),
1491 odsArg4: [&](OpBuilder &rewriterOuterLoop, Location locOuterLoop,
1492 Value ivOuterLoop, ValueRange iterArgsOuterLoop) {
1493 auto newInnerLoop1 = createLoops(
1494 rewriter, loc: innerLoop.getLoc(), lowerBound: innerLoop.getLowerBound(),
1495 upperBound: spillInnerLoop, step: innerLoop.getStep(), loopItrArgs: iterArgsOuterLoop,
1496 ipType, opType, blockingFactor, isVnni, vectorOpLhs,
1497 vectorOpRhs, contractOp, outerLoop, innerLoop, ops,
1498 ivOuterLoop, packedBuffer, pack: true, innerLoopIndex: spillLoopBound,
1499 isInnerLoopUBLarger, isInnerLoopUBHasOddQuot);
1500
1501 auto newInnerLoop = createLoops(
1502 rewriter, loc: innerLoop.getLoc(), lowerBound: spillInnerLoop,
1503 upperBound: innerLoop.getUpperBound(), step: innerLoop.getStep(),
1504 loopItrArgs: newInnerLoop1.getResults(), ipType, opType, blockingFactor,
1505 isVnni, vectorOpLhs, vectorOpRhs, contractOp, outerLoop,
1506 innerLoop, ops, ivOuterLoop, packedBuffer, pack: false, innerLoopIndex: c0,
1507 isInnerLoopUBLarger, isInnerLoopUBHasOddQuot);
1508
1509 scf::YieldOp::create(builder&: rewriterOuterLoop, location: locOuterLoop,
1510 results: newInnerLoop.getResults());
1511 });
1512 }
1513 }
1514
1515 // Case 2b: Reduction loop depth is 1.
1516 if (loopLists.size() == 1) {
1517
1518 innerLoop = loopLists[0];
1519 int64_t stepValue = 16;
1520 if (!isVnni)
1521 stepValue = stepValue * blockingFactor;
1522
1523 LogicalResult validateInnerLoopStep =
1524 validateLoopStep(rewriter, step: innerLoop.getStep(), value: stepValue);
1525 if (failed(Result: validateInnerLoopStep))
1526 return rewriter.notifyMatchFailure(
1527 arg&: contractOp,
1528 msg: "Invalid loop step. The step should be 32 for BF16 and "
1529 "64 for Int8/F8 or 1 if it is rduction loop other than K.");
1530
1531 SmallVector<Value> loopItrArgs = createInitTiles(innerLoop);
1532 rewriter.setInsertionPoint(innerLoop);
1533
1534 if (isVnni) {
1535 newLoop = createLoops(
1536 rewriter, loc: innerLoop.getLoc(), lowerBound: innerLoop.getLowerBound(),
1537 upperBound: innerLoop.getUpperBound(), step: innerLoop.getStep(), loopItrArgs, ipType,
1538 opType, blockingFactor, isVnni, vectorOpLhs, vectorOpRhs,
1539 contractOp, outerLoop: nullptr, innerLoop, ops, ivOuterLoop: nullptr, packedBuffer: nullptr, pack: true,
1540 innerLoopIndex: nullptr, isInnerLoopUBLarger: false, isInnerLoopUBHasOddQuot: false);
1541
1542 } else {
1543
1544 bool isInnerLoopUBLarger = false;
1545 bool isInnerLoopUBHasOddQuot = false;
1546
1547 int64_t ubVal = 16 * blockingFactor;
1548 Value ub = innerLoop.getUpperBound();
1549 if (auto constOp = ub.getDefiningOp<arith::ConstantOp>()) {
1550 if (auto intAttr = dyn_cast<IntegerAttr>(Val: constOp.getValue())) {
1551 ubVal = intAttr.getInt();
1552 }
1553 }
1554
1555 isInnerLoopUBLarger = ubVal > 16 * blockingFactor;
1556 isInnerLoopUBHasOddQuot =
1557 (((ubVal / (16 * blockingFactor)) % 2) == 1) && isInnerLoopUBLarger;
1558
1559 rewriter.setInsertionPoint(innerLoop);
1560
1561 auto c0 =
1562 arith::ConstantIndexOp::create(builder&: rewriter, location: innerLoop.getLoc(), value: 0);
1563 int64_t offset = 16 * blockingFactor;
1564 if (auto cst =
1565 innerLoop.getStep().getDefiningOp<arith::ConstantIndexOp>())
1566 offset = cst.value();
1567
1568 auto spillLoopBound = arith::ConstantIndexOp::create(
1569 builder&: rewriter, location: innerLoop.getLoc(), value: offset);
1570 Value spillInnerLoop =
1571 arith::SubIOp::create(builder&: rewriter, location: innerLoop.getLoc(),
1572 lhs: innerLoop.getUpperBound(), rhs: spillLoopBound);
1573
1574 auto bufferType =
1575 MemRefType::get(shape: {2, 32, (blockingFactor * 16)}, elementType: ipType);
1576 auto packedBuffer =
1577 memref::AllocaOp::create(builder&: rewriter, location: innerLoop.getLoc(), memrefType: bufferType);
1578
1579 // First Shuffling outside the reduction loops
1580 IRMapping rhsMapping;
1581 rhsMapping.map(
1582 from: vectorOpRhs->getOperand(
1583 idx: getIndexPosition(operand: contractOp.getRhs(), loop: innerLoop) + 1),
1584 to: innerLoop.getLowerBound());
1585 auto rhsClone = rewriter.clone(op&: *vectorOpRhs, mapper&: rhsMapping);
1586
1587 Value quotient_k = arith::DivUIOp::create(builder&: rewriter, location: innerLoop.getLoc(),
1588 lhs: innerLoop.getLowerBound(),
1589 rhs: innerLoop.getStep());
1590 Value c2 =
1591 arith::ConstantIndexOp::create(builder&: rewriter, location: innerLoop.getLoc(), value: 2);
1592 Value rem = arith::RemUIOp::create(builder&: rewriter, location: innerLoop.getLoc(),
1593 lhs: quotient_k, rhs: c2);
1594
1595 performShuffle(rewriter, loc: innerLoop.getLoc(), matB: rhsClone->getResult(idx: 0),
1596 ipType, offset: blockingFactor, packedBuffer, indxToStoreInBuffer: rem);
1597
1598 auto newLoopNonSpill = createLoops(
1599 rewriter, loc: innerLoop.getLoc(), lowerBound: innerLoop.getLowerBound(),
1600 upperBound: spillInnerLoop, step: innerLoop.getStep(), loopItrArgs, ipType, opType,
1601 blockingFactor, isVnni, vectorOpLhs, vectorOpRhs, contractOp,
1602 outerLoop: nullptr, innerLoop, ops, ivOuterLoop: nullptr, packedBuffer, pack: true,
1603 innerLoopIndex: spillLoopBound, isInnerLoopUBLarger, isInnerLoopUBHasOddQuot);
1604
1605 newLoop = createLoops(rewriter, loc: innerLoop.getLoc(), lowerBound: spillInnerLoop,
1606 upperBound: innerLoop.getUpperBound(), step: innerLoop.getStep(),
1607 loopItrArgs: newLoopNonSpill.getResults(), ipType, opType,
1608 blockingFactor, isVnni, vectorOpLhs, vectorOpRhs,
1609 contractOp, outerLoop: nullptr, innerLoop, ops, ivOuterLoop: nullptr,
1610 packedBuffer, pack: false, innerLoopIndex: c0, isInnerLoopUBLarger,
1611 isInnerLoopUBHasOddQuot);
1612 }
1613
1614 // This helps the final store back to the acc uses the same code for
1615 // the both reduction loop depth 1 or 2.
1616 outerLoop = innerLoop;
1617 }
1618
1619 // Copy the amx tile accumulation results to a MemRef buffer, add the
1620 // initial accumulation value, and store back to the C-Matrix
1621 Location loc = outerLoop.getLoc();
1622
1623 if (inPlaceAccWrites) {
1624 // The tiles already include the initial accumulator or were
1625 // zero-initialized: store each one where its contraction result was
1626 // written.
1627 for (auto [i, write] : llvm::enumerate(First&: *inPlaceAccWrites)) {
1628 auto [base, indices] = *getAccTileAccess(op: write);
1629 rewriter.setInsertionPoint(write);
1630 amx::TileStoreOp::create(builder&: rewriter, location: write->getLoc(), base, indices,
1631 val: newLoop.getResult(i));
1632 rewriter.eraseOp(op: write);
1633 }
1634 return success();
1635 }
1636
1637 // Carried-in accumulators are added to the results instead.
1638 bool addAccFromMemory = !isAccZeroVectorConstant && carriedAccs.empty();
1639
1640 // Direct tile stores weren't possible, so we need a temporary buffer to
1641 // move tile register contents through memory.
1642 unsigned int M, N;
1643 Value srcBuffAcc;
1644 SmallVector<Value> indicesAcc;
1645
1646 if (addAccFromMemory) {
1647 // We accumulate onto a register tile in memory, hence we let the
1648 // temporary buffer match its shape.
1649 llvm::TypeSwitch<Operation *>(accReadOp).Case<TransferReadOp, LoadOp>(
1650 caseFn: [&](auto readOp) {
1651 srcBuffAcc = readOp.getOperand(0);
1652
1653 auto indices = readOp.getIndices();
1654 indicesAcc.reserve(N: indices.size());
1655
1656 llvm::transform(indices, std::back_inserter(x&: indicesAcc),
1657 [&](OpFoldResult ofr) {
1658 return mlir::getValueOrCreateConstantIndexOp(
1659 b&: rewriter, loc, ofr);
1660 });
1661 });
1662 auto outputShapes = cast<MemRefType>(Val: srcBuffAcc.getType()).getShape();
1663 M = outputShapes[outputShapes.size() - 2];
1664 N = outputShapes[outputShapes.size() - 1];
1665 } else {
1666 // Otherwise we can just pick a suitable shape to hold all result tiles.
1667 // For the VNNI-packed case, we pass the result from each tile register to
1668 // memory individually. For the online packing case, we ensure two tiles
1669 // are neighboring in the N-dimension to facilitate the shuffling.
1670 M = isVnni ? ops.size() * 16 : ops.size() / 2 * 16;
1671 N = isVnni ? 16 : 2 * 16;
1672 }
1673
1674 SmallVector<Value> dps = newLoop.getResults();
1675 auto bufferType = MemRefType::get(shape: {M, N}, elementType: opType);
1676 auto resultBuffer = memref::AllocaOp::create(builder&: rewriter, location: loc, memrefType: bufferType);
1677
1678 // Store the amx tiled-dot product output into an MxN memref.
1679 for (unsigned int i = 0, k = 0; i < M; i = i + 16) {
1680 for (unsigned int j = 0; j < N; j = j + 16) {
1681 Value indexOp_i = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: i);
1682 Value indexOp_j = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: j);
1683 amx::TileStoreOp::create(builder&: rewriter, location: loc, base: resultBuffer,
1684 indices: ValueRange{indexOp_i, indexOp_j}, val: dps[k]);
1685 k++;
1686 }
1687 }
1688
1689 auto c0 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
1690 auto c16 = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 16);
1691 auto one = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1);
1692 auto mBound = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: M);
1693
1694 if (!isVnni || addAccFromMemory) {
1695 // Create a loop that iterates over the MxN memref, retrieves two rows +
1696 // shuffles them, adds up the C element values and stores them to the temp
1697 // buffer.
1698 scf::ForOp::create(
1699 builder&: rewriter, location: loc, lowerBound: c0, upperBound: mBound, step: one, initArgs: ValueRange{},
1700 odsArg4: [&](OpBuilder &nestedBuilder, Location loc, Value iv,
1701 ValueRange iterArgs) {
1702 auto row = vector::LoadOp::create(builder&: rewriter, location: loc,
1703 resultType: VectorType::get(shape: 16, elementType: opType),
1704 base: resultBuffer, indices: ValueRange{iv, c0});
1705
1706 auto row2 = vector::LoadOp::create(
1707 builder&: rewriter, location: loc, resultType: VectorType::get(shape: 16, elementType: opType), base: resultBuffer,
1708 indices: ValueRange{iv, c16});
1709
1710 Value writeVal1 = row;
1711 Value writeVal2 = row2;
1712
1713 if (!isVnni) {
1714 writeVal1 = vector::ShuffleOp::create(
1715 builder&: rewriter, location: loc, result: VectorType::get(shape: 16, elementType: opType), v1: row, v2: row2,
1716 mask: ArrayRef<int64_t>{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20,
1717 21, 22, 23});
1718
1719 writeVal2 = vector::ShuffleOp::create(
1720 builder&: rewriter, location: loc, result: VectorType::get(shape: 16, elementType: opType), v1: row, v2: row2,
1721 mask: ArrayRef<int64_t>{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14,
1722 15, 28, 29, 30, 31});
1723 }
1724
1725 if (addAccFromMemory) {
1726 indicesAcc[indicesAcc.size() - 2] = iv;
1727 indicesAcc[indicesAcc.size() - 1] = c0;
1728
1729 Value valueCRow1 = vector::LoadOp::create(
1730 builder&: rewriter, location: loc, resultType: VectorType::get(shape: 16, elementType: opType), base: srcBuffAcc,
1731 indices: indicesAcc);
1732 indicesAcc[indicesAcc.size() - 1] = c16;
1733
1734 Value valueCRow2 = vector::LoadOp::create(
1735 builder&: rewriter, location: loc, resultType: VectorType::get(shape: 16, elementType: opType), base: srcBuffAcc,
1736 indices: indicesAcc);
1737
1738 if (ipType.isBF16() || ipType.isF8E5M2() || ipType.isF8E4M3FN()) {
1739 writeVal1 =
1740 arith::AddFOp::create(builder&: rewriter, location: loc, lhs: writeVal1, rhs: valueCRow1);
1741
1742 writeVal2 =
1743 arith::AddFOp::create(builder&: rewriter, location: loc, lhs: writeVal2, rhs: valueCRow2);
1744 }
1745
1746 if (ipType.isSignlessInteger(width: 8)) {
1747 writeVal1 =
1748 arith::AddIOp::create(builder&: rewriter, location: loc, lhs: writeVal1, rhs: valueCRow1);
1749
1750 writeVal2 =
1751 arith::AddIOp::create(builder&: rewriter, location: loc, lhs: writeVal2, rhs: valueCRow2);
1752 }
1753 }
1754
1755 vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal1, base: resultBuffer,
1756 indices: ValueRange{iv, c0});
1757 vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: writeVal2, base: resultBuffer,
1758 indices: ValueRange{iv, c16});
1759
1760 scf::YieldOp::create(builder&: nestedBuilder, location: loc);
1761 });
1762 }
1763
1764 SmallVector<Value> writeResults;
1765 for (unsigned int i = 0; i < M; i = i + 16) {
1766 for (unsigned int j = 0; j < N; j = j + 16) {
1767 Value indexOp_i = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: i);
1768 Value indexOp_j = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: j);
1769
1770 auto vectorType = VectorType::get(shape: {16, 16}, elementType: opType);
1771
1772 int64_t srcRank =
1773 (dyn_cast<ShapedType>(Val: resultBuffer.getType())).getRank();
1774 Value padding = ub::PoisonOp::create(builder&: rewriter, location: loc, result: opType);
1775 auto map = AffineMap::getMinorIdentityMap(dims: srcRank, results: vectorType.getRank(),
1776 context: rewriter.getContext());
1777 SmallVector<bool> inBounds(vectorType.getRank(), true);
1778
1779 auto vec1 = vector::TransferReadOp::create(
1780 builder&: rewriter, location: loc, vectorType, source: resultBuffer,
1781 indices: ValueRange{indexOp_i, indexOp_j}, padding, permutationMap: map, inBounds);
1782 writeResults.push_back(Elt: vec1);
1783 }
1784 }
1785
1786 for (auto [idx, acc] : carriedAccs) {
1787 Type resultType = writeResults[idx].getType();
1788 if (acc.getType() != resultType)
1789 acc = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: resultType, source: acc);
1790 if (isa<FloatType>(Val: opType))
1791 writeResults[idx] =
1792 arith::AddFOp::create(builder&: rewriter, location: loc, lhs: writeResults[idx], rhs: acc);
1793 else
1794 writeResults[idx] =
1795 arith::AddIOp::create(builder&: rewriter, location: loc, lhs: writeResults[idx], rhs: acc);
1796 }
1797
1798 // Replace use of vector.contract with dot-products.
1799 for (size_t i = 0; i < ops.size(); i++) {
1800 vector::ContractionOp contOp = ops[i];
1801 Value vecRow = writeResults[i];
1802
1803 Value resultWriteOp = contractionUsersAfterYield(v: contOp.getResult());
1804 if (auto vecType = dyn_cast<VectorType>(Val: resultWriteOp.getType()))
1805 vecRow = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: vecType,
1806 source: writeResults[i]);
1807
1808 rewriter.replaceAllUsesWith(from: resultWriteOp, to: vecRow);
1809 }
1810
1811 return success();
1812 }
1813};
1814} // namespace
1815
1816void x86::populateVectorContractToAMXDotProductPatterns(
1817 RewritePatternSet &patterns) {
1818 patterns.add<VectorContractToAMXDotProduct>(arg: patterns.getContext());
1819}
1820