1//===- PrepareVectorContractLoops.cpp -------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Arith/IR/Arith.h"
10#include "mlir/Dialect/MemRef/IR/MemRef.h"
11#include "mlir/Dialect/SCF/IR/SCF.h"
12#include "mlir/Dialect/Utils/StaticValueUtils.h"
13#include "mlir/Dialect/Vector/IR/VectorOps.h"
14#include "mlir/Dialect/X86/Transforms.h"
15#include "mlir/IR/PatternMatch.h"
16#include "llvm/ADT/SmallVectorExtras.h"
17#include "llvm/Support/DebugLog.h"
18
19#define DEBUG_TYPE "x86-prepare-vector-contract-loops"
20
21using namespace mlir;
22
23static vector::ContractionOp getCandidate(scf::ForOp loop) {
24 if (loop.getNumRegionIterArgs() != 1) {
25 LDBG() << "reject loop at " << loop.getLoc() << ": expected one iter_arg";
26 return {};
27 }
28
29 auto yield = cast<scf::YieldOp>(Val: loop.getBody()->getTerminator());
30 for (Operation &op : loop.getBody()->without_terminator()) {
31 auto contract = dyn_cast<vector::ContractionOp>(Val: &op);
32 if (!contract)
33 continue;
34 if (contract.getKind() != vector::CombiningKind::ADD ||
35 contract.getAcc() != loop.getRegionIterArgs().front() ||
36 !contract.getAcc().hasOneUse() || !contract.getResult().hasOneUse() ||
37 yield->getOperand(idx: 0) != contract.getResult()) {
38 LDBG() << "reject contraction at " << contract.getLoc()
39 << ": not a single loop-carried accumulation";
40 continue;
41 }
42
43 auto iterators = contract.getIteratorTypesArray();
44 auto maps = contract.getIndexingMapsArray();
45 MLIRContext *ctx = contract.getContext();
46 auto m = getAffineDimExpr(position: 0, context: ctx);
47 auto n = getAffineDimExpr(position: 1, context: ctx);
48 auto k = getAffineDimExpr(position: 2, context: ctx);
49 auto par = vector::IteratorType::parallel;
50 auto red = vector::IteratorType::reduction;
51 if (llvm::equal(LRange&: iterators, RRange: ArrayRef{par, par, red}) &&
52 maps[0] == AffineMap::get(dimCount: 3, symbolCount: 0, results: {m, k}, context: ctx) &&
53 maps[1] == AffineMap::get(dimCount: 3, symbolCount: 0, results: {k, n}, context: ctx) &&
54 maps[2] == AffineMap::get(dimCount: 3, symbolCount: 0, results: {m, n}, context: ctx))
55 return contract;
56 LDBG() << "reject contraction at " << contract.getLoc()
57 << ": noncanonical matmul layout or iterators";
58 }
59 return {};
60}
61
62// Look through reshapes/broadcasts without changing the row-major element
63// order. A broadcast is safe to skip if it adds unit dimensions only.
64static Value skipReshapes(Value v) {
65 while (isa_and_present<vector::ShapeCastOp, vector::BroadcastOp>(
66 Val: v.getDefiningOp())) {
67 Value source = v.getDefiningOp()->getOperand(idx: 0);
68 auto sourceType = dyn_cast<VectorType>(Val: source.getType());
69 if (!sourceType || sourceType.getNumElements() !=
70 cast<VectorType>(Val: v.getType()).getNumElements())
71 break;
72 v = source;
73 }
74 return v;
75}
76
77// Find the source and VNNI lane j of a split: v[i] = src[factor * i + j].
78// A split step of width s keeps every s-th element. Match either form below
79// (unit dimensions may also be present):
80//
81// %even, %odd = vector.deinterleave %src // s = 2
82// : vector<4x8xbf16> -> vector<4x4xbf16>
83//
84// or:
85//
86// %packed = vector.shape_cast %src
87// : vector<4x8xbf16> to vector<4x4x2xbf16>
88// %split = vector.transpose %packed, [2, 0, 1]
89// : vector<4x4x2xbf16> to vector<2x4x4xbf16>
90// %even = vector.extract %split[0]
91// : vector<4x4xbf16> from vector<2x4x4xbf16>
92// %odd = vector.extract %split[1]
93// : vector<4x4xbf16> from vector<2x4x4xbf16>
94//
95// If s < factor, the split's input must itself come from splits of src
96// totaling factor / s. For example, for factor = 4, a 4-way transpose/extract
97// split of src matches, as do two levels of binary splits:
98//
99// %even, %odd = vector.deinterleave %src // lanes {0, 2} and {1, 3}
100// %l1, %l3 = vector.deinterleave %odd // %l3[i] = src[4 * i + 3]
101//
102// Splits closer to src give the low lane bits; result j of the split that
103// produces v gives the high bits.
104static std::optional<std::pair<Value, int64_t>> matchSplit(Value v,
105 int64_t factor) {
106 v = skipReshapes(v);
107 if (factor == 1)
108 return std::pair(v, int64_t(0));
109
110 auto matchSource =
111 [&](Value source, int64_t ways,
112 int64_t j) -> std::optional<std::pair<Value, int64_t>> {
113 if (factor % ways != 0)
114 return std::nullopt;
115 auto lane = matchSplit(v: source, factor: factor / ways);
116 if (!lane)
117 return std::nullopt;
118 lane->second += factor / ways * j;
119 return lane;
120 };
121 if (auto deinterleave = v.getDefiningOp<vector::DeinterleaveOp>())
122 return matchSource(deinterleave.getSource(), 2,
123 cast<OpResult>(Val&: v).getResultNumber());
124
125 auto extract = v.getDefiningOp<vector::ExtractOp>();
126 auto transpose =
127 extract ? extract.getSource().getDefiningOp<vector::TransposeOp>()
128 : vector::TransposeOp();
129 auto type = dyn_cast<VectorType>(Val: v.getType());
130 if (!transpose || !type || extract.hasDynamicPosition() ||
131 transpose.getSourceVectorType().isScalable())
132 return std::nullopt;
133
134 ArrayRef<int64_t> shape = transpose.getSourceVectorType().getShape();
135 auto isNonUnit = [&](int64_t dim) { return shape[dim] != 1; };
136 SmallVector<int64_t> order =
137 llvm::filter_to_vector(C: llvm::seq<int64_t>(Begin: 0, End: shape.size()), Pred&: isNonUnit);
138 if (order.empty())
139 return std::nullopt;
140 int64_t ways = shape[order.back()];
141
142 std::rotate(first: order.begin(), middle: std::prev(x: order.end()), last: order.end());
143 ArrayRef<int64_t> perm = transpose.getPermutation();
144 if (llvm::filter_to_vector(C&: perm, Pred&: isNonUnit) != order)
145 return std::nullopt;
146
147 // The split dimension is now first among the non-unit dimensions. Keeping
148 // exactly 1 / ways of the elements means the extract indexes that dimension;
149 // any other dimensions it indexes must be unit dimensions.
150 if (ways * type.getNumElements() !=
151 transpose.getSourceVectorType().getNumElements())
152 return std::nullopt;
153
154 int64_t j =
155 extract
156 .getStaticPosition()[llvm::find(Range&: perm, Val: order.front()) - perm.begin()];
157 if (j < 0 || j >= ways)
158 return std::nullopt;
159
160 return matchSource(transpose.getVector(), ways, j);
161}
162
163// Find the inputs in VNNI-lane order for a join. A join step of width s
164// interleaves s inputs. Match either form below:
165//
166// %joined = vector.interleave %even, %odd // s = 2
167// : vector<4x1xbf16> -> vector<4x2xbf16>
168//
169// or s slice inserts in any order (shown for s = 2):
170//
171// %poison = ub.poison : vector<4x2xbf16>
172// %partial = vector.insert_strided_slice %odd, %poison
173// offsets = [0, 1], strides = [1, 1]
174// : vector<4x1xbf16> into vector<4x2xbf16>
175// %joined = vector.insert_strided_slice %even, %partial
176// offsets = [0, 0], strides = [1, 1]
177// : vector<4x1xbf16> into vector<4x2xbf16>
178//
179// If s < factor, each input must itself be a join of width factor / s
180// (possibly built from smaller joins).
181static std::optional<SmallVector<Value, 4>> matchJoin(Value v, int64_t factor) {
182 v = skipReshapes(v);
183 if (factor == 1)
184 return SmallVector<Value, 4>{v};
185
186 SmallVector<Value, 4> parts;
187 if (auto interleave = v.getDefiningOp<vector::InterleaveOp>()) {
188 parts = {interleave.getLhs(), interleave.getRhs()};
189 } else if (auto insert = v.getDefiningOp<vector::InsertStridedSliceOp>()) {
190 int64_t ways = insert.getDestVectorType().getShape().back();
191 if (ways < 2 || factor % ways != 0)
192 return std::nullopt;
193 parts.resize(N: ways);
194 // All slices must be complete and disjoint so the original destination
195 // contributes no elements to the result.
196 for (int64_t i = 0; i < ways; ++i) {
197 auto insert = v.getDefiningOp<vector::InsertStridedSliceOp>();
198 if (!insert)
199 return std::nullopt;
200 ArrayRef<int64_t> shape = insert.getDestVectorType().getShape();
201 SmallVector<int64_t> slice(shape.drop_back());
202 slice.push_back(Elt: 1);
203 int64_t j =
204 extractFromIntegerArrayAttr<int64_t>(attr: insert.getOffsets()).back();
205 if (insert.getSourceVectorType().getShape() != ArrayRef(slice) ||
206 parts[j])
207 return std::nullopt;
208 parts[j] = insert.getValueToStore();
209 v = insert.getDest();
210 }
211 } else {
212 return std::nullopt;
213 }
214
215 int64_t ways = parts.size();
216 if (factor % ways != 0)
217 return std::nullopt;
218 SmallVector<Value, 4> lanes(factor);
219 for (auto [j, part] : llvm::enumerate(First&: parts)) {
220 auto inner = matchJoin(v: part, factor: factor / ways);
221 if (!inner)
222 return std::nullopt;
223 for (auto [i, lane] : llvm::enumerate(First&: *inner))
224 lanes[ways * i + j] = lane;
225 }
226 return lanes;
227}
228
229// Find the packed RHS so the contraction can use it without unpacking.
230// For a K x N RHS and VNNI factor f, require for each VNNI lane j:
231// rhs[f * r + j][c] = src[r][c][j],
232// where 0 <= j < f and src is viewed as K/f x N x f.
233// Match transpose/shape_cast unpacking (shown for f = 2):
234//
235// %packed = vector.shape_cast %src
236// : vector<4x6xbf16> to vector<4x3x2xbf16>
237// %transposed = vector.transpose %packed, [0, 2, 1]
238// : vector<4x3x2xbf16> to vector<4x2x3xbf16>
239// %rhs = vector.shape_cast %transposed
240// : vector<4x2x3xbf16> to vector<8x3xbf16>
241//
242// or split/transpose/join/transpose unpacking (shown for f = 2):
243//
244// %even, %odd = vector.deinterleave %src
245// : vector<4x6xbf16> -> vector<4x3xbf16>
246// %even_t = vector.transpose %even, [1, 0]
247// : vector<4x3xbf16> to vector<3x4xbf16>
248// %odd_t = vector.transpose %odd, [1, 0]
249// : vector<4x3xbf16> to vector<3x4xbf16>
250// %joined = vector.interleave %even_t, %odd_t
251// : vector<3x4xbf16> -> vector<3x8xbf16>
252// %rhs = vector.transpose %joined, [1, 0]
253// : vector<3x8xbf16> to vector<8x3xbf16>
254//
255// For f = 4, splits and joins may each be one 4-way transpose/extract or
256// slice-insert step, or two levels of binary steps. With binary steps, the
257// second-level splits produce VNNI lanes 0/2 and 1/3:
258//
259// %even, %odd = vector.deinterleave %src // lanes {0, 2} and {1, 3}
260// %l0, %l2 = vector.deinterleave %even
261// %l1, %l3 = vector.deinterleave %odd
262// // Transpose each lane %lj to %tj.
263// %even_t = vector.interleave %t0, %t2
264// %odd_t = vector.interleave %t1, %t3
265// %joined = vector.interleave %even_t, %odd_t
266//
267// Any binary split or join may instead use the transpose/extract or
268// slice-insert form.
269static Value matchUnpackedRhs(Value rhs, int64_t factor) {
270 auto type = cast<VectorType>(Val: rhs.getType());
271 int64_t k = type.getDimSize(idx: 0) / factor, n = type.getDimSize(idx: 1);
272 auto transpose = skipReshapes(v: rhs).getDefiningOp<vector::TransposeOp>();
273 if (!transpose)
274 return {};
275
276 ArrayRef<int64_t> shape = transpose.getSourceVectorType().getShape();
277 ArrayRef<int64_t> perm = transpose.getPermutation();
278 if (shape.size() >= 3) {
279 size_t leading = shape.size() - 3;
280 auto expected = llvm::to_vector(Range: llvm::seq<int64_t>(Begin: 0, End: shape.size()));
281 std::swap(a&: expected[leading + 1], b&: expected[leading + 2]);
282 bool isUnpacking =
283 llvm::all_of(Range: shape.take_front(N: leading),
284 P: [](int64_t dim) { return dim == 1; }) &&
285 shape.drop_front(N: leading) == ArrayRef<int64_t>{k, n, factor} &&
286 perm == ArrayRef(expected);
287 return isUnpacking ? skipReshapes(v: transpose.getVector()) : Value();
288 }
289
290 const int64_t swap[] = {1, 0};
291 if (perm != ArrayRef(swap) || shape != ArrayRef<int64_t>{n, factor * k})
292 return {};
293
294 auto lanes = matchJoin(v: transpose.getVector(), factor);
295 if (!lanes)
296 return {};
297
298 Value src;
299 for (auto [j, lane] : llvm::enumerate(First&: *lanes)) {
300 auto laneTranspose =
301 skipReshapes(v: lane).getDefiningOp<vector::TransposeOp>();
302 if (!laneTranspose || laneTranspose.getPermutation() != ArrayRef(swap) ||
303 laneTranspose.getSourceVectorType().getShape() !=
304 ArrayRef<int64_t>{k, n})
305 return {};
306
307 auto source = matchSplit(v: laneTranspose.getVector(), factor);
308 if (!source || source->second != int64_t(j) ||
309 (src && source->first != src))
310 return {};
311
312 src = source->first;
313 }
314 return src;
315}
316
317static bool isSimpleReadOutsideLoop(vector::TransferReadOp read,
318 scf::ForOp loop) {
319 return read && read->getParentOp() == loop &&
320 isa<MemRefType>(Val: read.getBase().getType()) &&
321 loop.isDefinedOutsideOfLoop(value: read.getBase()) &&
322 read.getPermutationMap().isMinorIdentity() && !read.getMask() &&
323 !read.hasOutOfBoundsDim();
324}
325
326namespace {
327struct ExpandedRead {
328 vector::TransferReadOp read;
329 MemRefType type;
330 SmallVector<ReassociationIndices> reassociation;
331 int64_t lastIndex;
332 // Static view of a dynamic last dimension, or null.
333 MemRefType view;
334};
335} // namespace
336
337static std::optional<ExpandedRead>
338getExpandableRead(vector::TransferReadOp read, unsigned factor,
339 scf::ForOp loop) {
340 if (!isSimpleReadOutsideLoop(read, loop))
341 return std::nullopt;
342 auto type = cast<MemRefType>(Val: read.getBase().getType());
343 if (type.getElementType() != read.getVectorType().getElementType() ||
344 type.getRank() < 2)
345 return std::nullopt;
346 auto lastIndex = getConstantIntValue(ofr: read.getIndices().back());
347 if (!lastIndex || *lastIndex < 0 || *lastIndex % factor != 0)
348 return std::nullopt;
349
350 MemRefType view;
351 if (type.isDynamicDim(idx: type.getRank() - 1)) {
352 SmallVector<int64_t> strides;
353 int64_t offset;
354 if (read.getVectorType().isScalable() ||
355 failed(Result: type.getStridesAndOffset(strides, offset)))
356 return std::nullopt;
357 // The full dimension has a dynamic size, but this in-bounds read ends at
358 // a known index. Create a view whose last dimension ends there and give
359 // the divisibility check below a static size to work with.
360 SmallVector<int64_t> shape(type.getShape());
361 shape.back() = *lastIndex + read.getVectorType().getShape().back();
362 view = type = MemRefType::get(
363 shape, elementType: type.getElementType(),
364 layout: StridedLayoutAttr::get(context: type.getContext(), offset, strides),
365 memorySpace: type.getMemorySpace());
366 }
367 if (type.getShape().back() % factor != 0)
368 return std::nullopt;
369
370 SmallVector<int64_t> shape(type.getShape());
371 shape.back() /= factor;
372 shape.push_back(Elt: factor);
373 SmallVector<ReassociationIndices> reassociation;
374 for (int64_t i = 0; i < type.getRank(); ++i)
375 reassociation.push_back(Elt: i == type.getRank() - 1
376 ? ReassociationIndices{i, i + 1}
377 : ReassociationIndices{i});
378 FailureOr<MemRefType> expanded =
379 memref::ExpandShapeOp::computeExpandedType(srcType: type, resultShape: shape, reassociation);
380 if (failed(Result: expanded))
381 return std::nullopt;
382 return ExpandedRead{.read: read, .type: *expanded, .reassociation: reassociation, .lastIndex: *lastIndex / factor,
383 .view: view};
384}
385
386static Value buildExpandedRead(PatternRewriter &rewriter, Location loc,
387 ExpandedRead &info, VectorType resultType) {
388 OpBuilder::InsertionGuard guard(rewriter);
389 rewriter.setInsertionPoint(info.read->getParentOp());
390 Value base = info.read.getBase();
391 if (info.view) {
392 // Unlike memref.subview, memref.reinterpret_cast is valid even when
393 // the loop runs zero times (which makes the view out of bounds).
394 auto metadata =
395 memref::ExtractStridedMetadataOp::create(builder&: rewriter, location: loc, source: base);
396 SmallVector<OpFoldResult> sizes = metadata.getConstifiedMixedSizes();
397 sizes.back() = rewriter.getIndexAttr(value: info.view.getShape().back());
398 base = memref::ReinterpretCastOp::create(
399 builder&: rewriter, location: loc, resultType: info.view, source: base, offset: metadata.getConstifiedMixedOffset(),
400 sizes, strides: metadata.getConstifiedMixedStrides());
401 }
402 Value expanded = memref::ExpandShapeOp::create(builder&: rewriter, location: loc, resultType: info.type, src: base,
403 reassociation: info.reassociation);
404 Value lastIndex =
405 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: info.lastIndex);
406 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
407 rewriter.setInsertionPoint(info.read);
408 SmallVector<Value> indices(info.read.getIndices());
409 indices.back() = lastIndex;
410 indices.push_back(Elt: zero);
411 auto packed = vector::TransferReadOp::create(
412 builder&: rewriter, location: loc, vectorType: resultType, source: expanded, indices, padding: info.read.getPadding());
413 packed.setInBoundsAttr(
414 rewriter.getBoolArrayAttr(values: SmallVector<bool>(resultType.getRank(), true)));
415 return packed;
416}
417
418namespace {
419// Fold RHS unpacking into a VNNI contraction:
420// - Match a packed RHS read unpacked for a flat matmul.
421// - Split the LHS memref's K dimension and reread it in VNNI layout.
422// A dynamic K dimension is first narrowed to the read's static extent.
423// - Contract the packed reads directly over K and the VNNI dimension.
424// For bf16, 32x32 LHS and RHS become 32x16x2 LHS and 16x32x2 RHS.
425struct FoldVNNIUnpacking : OpRewritePattern<vector::ContractionOp> {
426 using OpRewritePattern::OpRewritePattern;
427
428 LogicalResult matchAndRewrite(vector::ContractionOp contract,
429 PatternRewriter &rewriter) const override {
430 auto loop = dyn_cast<scf::ForOp>(Val: contract->getParentOp());
431 if (!loop)
432 return rewriter.notifyMatchFailure(arg&: contract,
433 msg: "not directly in a for loop");
434 if (getCandidate(loop) != contract)
435 return rewriter.notifyMatchFailure(
436 arg&: contract, msg: "not the loop's canonical contraction");
437
438 auto lhsType = contract.getLhsType();
439 auto rhsType = contract.getRhsType();
440 Type element = lhsType.getElementType();
441 assert(lhsType.getRank() == 2 && rhsType.getRank() == 2 &&
442 lhsType.getDimSize(1) == rhsType.getDimSize(0) &&
443 element == rhsType.getElementType() &&
444 "expected canonical matmul contraction");
445 if (lhsType.isScalable() || rhsType.isScalable())
446 return rewriter.notifyMatchFailure(arg&: contract, msg: "scalable operand vectors");
447 if (!(element.isBF16() || element.isInteger(width: 8)))
448 return rewriter.notifyMatchFailure(arg&: contract,
449 msg: "expected bf16 or i8 operands");
450 unsigned factor = 32 / element.getIntOrFloatBitWidth();
451 if (lhsType.getDimSize(idx: 1) % factor != 0)
452 return rewriter.notifyMatchFailure(
453 arg&: contract, msg: "K dimension is not divisible by VNNI factor");
454
455 Value packedSource = matchUnpackedRhs(rhs: contract.getRhs(), factor);
456 if (!packedSource)
457 return rewriter.notifyMatchFailure(arg&: contract,
458 msg: "RHS does not unpack a packed vector");
459 VectorType rhsPackedType =
460 VectorType::get(shape: {lhsType.getDimSize(idx: 1) / factor, rhsType.getDimSize(idx: 1),
461 int64_t(factor)},
462 elementType: element);
463 auto lhsRead = contract.getLhs().getDefiningOp<vector::TransferReadOp>();
464 auto lhs = getExpandableRead(read: lhsRead, factor, loop);
465 auto rhsRead = packedSource.getDefiningOp<vector::TransferReadOp>();
466 if (!lhs)
467 return rewriter.notifyMatchFailure(
468 arg&: contract, msg: "LHS read cannot be expanded into VNNI layout");
469 if (!isSimpleReadOutsideLoop(read: rhsRead, loop))
470 return rewriter.notifyMatchFailure(arg&: contract,
471 msg: "RHS is not a simple transfer read");
472 std::optional<ExpandedRead> splitRhs;
473 if (rhsRead.getVectorType() != rhsPackedType) {
474 auto flatType = rhsRead.getVectorType();
475 if (flatType.getRank() != 2 ||
476 flatType.getDimSize(idx: 0) != rhsPackedType.getDimSize(idx: 0) ||
477 flatType.getDimSize(idx: 1) % factor != 0 ||
478 flatType.getDimSize(idx: 1) / factor != rhsPackedType.getDimSize(idx: 1))
479 return rewriter.notifyMatchFailure(
480 arg&: contract, msg: "flat RHS shape is not VNNI compatible");
481 if (!(splitRhs = getExpandableRead(read: rhsRead, factor, loop)))
482 return rewriter.notifyMatchFailure(
483 arg&: contract, msg: "RHS read cannot be expanded into VNNI layout");
484 } else {
485 if (getConstantIntValue(ofr: rhsRead.getIndices().back()) != 0)
486 return rewriter.notifyMatchFailure(
487 arg&: contract, msg: "packed RHS has nonzero inner offset");
488 }
489
490 Location loc = contract.getLoc();
491 VectorType lhsPackedType =
492 VectorType::get(shape: {lhsType.getDimSize(idx: 0), lhsType.getDimSize(idx: 1) / factor,
493 int64_t(factor)},
494 elementType: element);
495 Value packedLhs = buildExpandedRead(rewriter, loc, info&: *lhs, resultType: lhsPackedType);
496 Value packedRhs =
497 splitRhs ? buildExpandedRead(rewriter, loc, info&: *splitRhs, resultType: rhsPackedType)
498 : rhsRead.getResult();
499 rewriter.setInsertionPoint(contract);
500 auto ctx = rewriter.getContext();
501 auto m = rewriter.getAffineDimExpr(position: 0);
502 auto n = rewriter.getAffineDimExpr(position: 1);
503 auto k = rewriter.getAffineDimExpr(position: 2);
504 auto v = rewriter.getAffineDimExpr(position: 3);
505 auto vnniMaps =
506 rewriter.getAffineMapArrayAttr(values: {AffineMap::get(dimCount: 4, symbolCount: 0, results: {m, k, v}, context: ctx),
507 AffineMap::get(dimCount: 4, symbolCount: 0, results: {k, n, v}, context: ctx),
508 AffineMap::get(dimCount: 4, symbolCount: 0, results: {m, n}, context: ctx)});
509 auto iterators = rewriter.getArrayAttr(
510 value: {vector::IteratorTypeAttr::get(context: ctx, value: vector::IteratorType::parallel),
511 vector::IteratorTypeAttr::get(context: ctx, value: vector::IteratorType::parallel),
512 vector::IteratorTypeAttr::get(context: ctx, value: vector::IteratorType::reduction),
513 vector::IteratorTypeAttr::get(context: ctx, value: vector::IteratorType::reduction)});
514 rewriter.replaceOpWithNewOp<vector::ContractionOp>(
515 op: contract, args&: packedLhs, args&: packedRhs, args: contract.getAcc(), args&: vnniMaps, args&: iterators,
516 args: contract.getKind(), args: contract.getFastmath());
517 LDBG() << "Replaced contract with VNNI-compatible contraction";
518 return success();
519 }
520};
521} // namespace
522
523void x86::populatePrepareVectorContractLoopsPatterns(
524 RewritePatternSet &patterns) {
525 patterns.add<FoldVNNIUnpacking>(arg: patterns.getContext());
526}
527