1//===- LowerVectorContract.cpp - Lower 'vector.contract' operation --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements target-independent rewrites and utilities to lower the
10// 'vector.contract' operation.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/Arith/IR/Arith.h"
15#include "mlir/Dialect/MemRef/IR/MemRef.h"
16#include "mlir/Dialect/Utils/IndexingUtils.h"
17#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
18#include "mlir/Dialect/Vector/IR/VectorOps.h"
19#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
20#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
21#include "mlir/IR/BuiltinTypes.h"
22#include "mlir/IR/Location.h"
23#include "mlir/IR/PatternMatch.h"
24#include "mlir/IR/TypeUtilities.h"
25
26#define DEBUG_TYPE "vector-contract-lowering"
27
28using namespace mlir;
29using namespace mlir::vector;
30
31//===----------------------------------------------------------------------===//
32// Helper functions
33//===----------------------------------------------------------------------===//
34// Helper to find an index in an affine map.
35static std::optional<int64_t> getResultIndex(AffineMap map, int64_t index) {
36 for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) {
37 int64_t idx = map.getDimPosition(idx: i);
38 if (idx == index)
39 return i;
40 }
41 return std::nullopt;
42}
43
44// Helper to construct iterator types with one index removed.
45static SmallVector<Attribute> adjustIter(ArrayAttr iteratorTypes,
46 int64_t index) {
47 SmallVector<Attribute> results;
48 for (const auto &it : llvm::enumerate(First&: iteratorTypes)) {
49 int64_t idx = it.index();
50 if (idx == index)
51 continue;
52 results.push_back(Elt: it.value());
53 }
54 return results;
55}
56
57// Helper to construct an affine map with one index removed.
58static AffineMap adjustMap(AffineMap map, int64_t index,
59 PatternRewriter &rewriter) {
60 auto *ctx = rewriter.getContext();
61 SmallVector<AffineExpr> results;
62 for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) {
63 int64_t idx = map.getDimPosition(idx: i);
64 if (idx == index)
65 continue;
66 // Re-insert remaining indices, but renamed when occurring
67 // after the removed index.
68 auto targetExpr = getAffineDimExpr(position: idx < index ? idx : idx - 1, context: ctx);
69 results.push_back(Elt: targetExpr);
70 }
71 return AffineMap::get(dimCount: map.getNumDims() - 1, symbolCount: 0, results, context: ctx);
72}
73
74/// Returns `val` with the dimension at position `index` dropped by indexing
75/// that dimension with `pos`.
76///
77/// If `index == -1`, returns `val` unchanged. If `index == 0`, the result is
78/// a single `vector.extract %val[pos]`.
79///
80/// Example (`index == 0`): extract the sub-vector at `pos` along the leading
81/// dimension.
82/// // val : vector<4x8xf32>, pos = 2
83/// %res = vector.extract %val[2] : vector<8xf32> from vector<4x8xf32>
84///
85/// For `index > 0`, recursively applies the same drop to each sub-vector of
86/// the leading dimension and reassembles the result.
87static Value reshapeLoad(Location loc, Value val, int64_t index, int64_t pos,
88 PatternRewriter &rewriter) {
89 if (index == -1)
90 return val;
91
92 // At extraction dimension?
93 if (index == 0)
94 return vector::ExtractOp::create(builder&: rewriter, location: loc, source: val, position: pos);
95
96 // Unroll leading dimensions.
97 VectorType type = cast<VectorType>(Val: val.getType());
98 VectorType resType = VectorType::Builder(type).dropDim(pos: index);
99 Value result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: resType,
100 value: rewriter.getZeroAttr(type: resType));
101 for (int64_t d = 0, e = resType.getDimSize(idx: 0); d < e; d++) {
102 Value ext = vector::ExtractOp::create(builder&: rewriter, location: loc, source: val, position: d);
103 Value load = reshapeLoad(loc, val: ext, index: index - 1, pos, rewriter);
104 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: load, dest: result, position: d);
105 }
106 return result;
107}
108
109/// Inserts `val` into `result` at position `pos` along dimension `index`.
110///
111/// This is the inverse of `reshapeLoad`. If `index == -1`, returns `val`. If
112/// `index == 0`, the result is a single `vector.insert %val, %result [pos]`.
113///
114/// Example (`index == 0`): insert `val` at `pos` along the leading dimension.
115/// // val : vector<4xf32>, acc : vector<2x4xf32>, pos = 1
116/// %res = vector.insert %val, %acc [1] : vector<4xf32> into vector<2x4xf32>
117///
118/// For `index > 0`, recursively applies the same insertion to each sub-vector
119/// of the leading dimension and reassembles the result.
120static Value reshapeStore(Location loc, Value val, Value result, int64_t index,
121 int64_t pos, PatternRewriter &rewriter) {
122 // Unmodified?
123 if (index == -1)
124 return val;
125 // At insertion dimension?
126 if (index == 0)
127 return vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: val, dest: result, position: pos);
128
129 // Unroll leading dimensions.
130 VectorType type = cast<VectorType>(Val: result.getType());
131 for (int64_t d = 0, e = type.getDimSize(idx: 0); d < e; d++) {
132 Value ext = vector::ExtractOp::create(builder&: rewriter, location: loc, source: result, position: d);
133 Value ins = vector::ExtractOp::create(builder&: rewriter, location: loc, source: val, position: d);
134 Value sto = reshapeStore(loc, val: ins, result: ext, index: index - 1, pos, rewriter);
135 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: sto, dest: result, position: d);
136 }
137 return result;
138}
139
140/// Helper to create arithmetic operation associated with a kind of contraction.
141static std::optional<Value>
142createContractArithOp(Location loc, Value x, Value y, Value acc,
143 vector::CombiningKind kind, PatternRewriter &rewriter,
144 bool isInt, Value mask = Value(),
145 arith::FastMathFlagsAttr fmf = {}) {
146 using vector::CombiningKind;
147 Value mul;
148
149 if (isInt) {
150 if (kind == CombiningKind::MINNUMF || kind == CombiningKind::MAXNUMF ||
151 kind == CombiningKind::MINIMUMF || kind == CombiningKind::MAXIMUMF ||
152 kind == CombiningKind::MINIMUMNUMF ||
153 kind == CombiningKind::MAXIMUMNUMF)
154 // Only valid for floating point types.
155 return std::nullopt;
156 mul = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: x, rhs: y);
157 } else {
158 // Float case.
159 if (kind == CombiningKind::AND || kind == CombiningKind::MINUI ||
160 kind == CombiningKind::MINSI || kind == CombiningKind::MAXUI ||
161 kind == CombiningKind::MAXSI || kind == CombiningKind::OR ||
162 kind == CombiningKind::XOR)
163 // Only valid for integer types.
164 return std::nullopt;
165 // Special case for fused multiply-add.
166 if (acc && isa<VectorType>(Val: acc.getType()) && kind == CombiningKind::ADD) {
167 Value fma = vector::FMAOp::create(builder&: rewriter, location: loc, lhs: x, rhs: y, acc);
168 if (mask)
169 // The fma op doesn't need explicit masking. However, fma ops used in
170 // reductions must preserve previous 'acc' values for masked-out lanes.
171 fma = selectPassthru(builder&: rewriter, mask, newValue: fma, passthru: acc);
172 return fma;
173 }
174 mul = arith::MulFOp::create(builder&: rewriter, location: loc, lhs: x, rhs: y, fastmath: fmf);
175 }
176
177 if (!acc)
178 return std::optional<Value>(mul);
179
180 return makeArithReduction(b&: rewriter, loc, kind, v1: mul, acc, fastmath: fmf, mask);
181}
182
183/// Return the positions of the reductions in the given map.
184static SmallVector<int64_t> getReductionIndex(AffineMap map,
185 ArrayAttr iteratorTypes) {
186 SmallVector<int64_t> dimsIdx;
187 for (unsigned i = 0, e = map.getNumResults(); i < e; i++) {
188 if (isReductionIterator(attr: iteratorTypes[map.getDimPosition(idx: i)]))
189 dimsIdx.push_back(Elt: i);
190 }
191 return dimsIdx;
192}
193
194/// Look for a given dimension in an affine map and return its position. Return
195/// std::nullopt if the dimension is not in the map results.
196static std::optional<unsigned> getDimPosition(AffineMap map, unsigned dim) {
197 for (unsigned i = 0, e = map.getNumResults(); i < e; i++) {
198 if (map.getDimPosition(idx: i) == dim)
199 return i;
200 }
201 return std::nullopt;
202}
203
204/// Creates an AddIOp if `isInt` is true otherwise create an arith::AddFOp using
205/// operands `x` and `y`.
206static Value createAdd(Location loc, Value x, Value y, bool isInt,
207 PatternRewriter &rewriter,
208 arith::FastMathFlagsAttr fmf = {}) {
209 if (isInt)
210 return arith::AddIOp::create(builder&: rewriter, location: loc, lhs: x, rhs: y);
211 return arith::AddFOp::create(builder&: rewriter, location: loc, lhs: x, rhs: y, fastmath: fmf);
212}
213
214/// Creates a MulIOp if `isInt` is true otherwise create an MulFOp using
215/// operands `x and `y`.
216static Value createMul(Location loc, Value x, Value y, bool isInt,
217 PatternRewriter &rewriter,
218 arith::FastMathFlagsAttr fmf = {}) {
219 if (isInt)
220 return arith::MulIOp::create(builder&: rewriter, location: loc, lhs: x, rhs: y);
221 return arith::MulFOp::create(builder&: rewriter, location: loc, lhs: x, rhs: y, fastmath: fmf);
222}
223
224namespace {
225
226/// Progressive lowering of a `vector.contract %a, %b, %c` with row-major matmul
227/// semantics to a reduction_size-unrolled sequence:
228/// ```
229/// %at = vector.transpose %a, [1, 0]
230/// %bRow0 = vector.extract %b[0]
231/// %atRow0 = vector.extract %at[0]
232/// %c0 = vector.outerproduct %atRow0, %bRow0, %c
233/// ...
234/// %bRowK = vector.extract %b[K]
235/// %atRowK = vector.extract %at[K]
236/// %cK = vector.outerproduct %atRowK, %bRowK, %cK-1
237/// ```
238///
239/// This only kicks in when vectorContractLowering is set to OuterProduct and
240/// the vector.contract op is a row-major matrix multiply.
241class ContractionOpToOuterProductOpLowering
242 : public MaskableOpRewritePattern<vector::ContractionOp> {
243public:
244 using MaskableOpRewritePattern::MaskableOpRewritePattern;
245
246 using FilterConstraintType =
247 std::function<LogicalResult(vector::ContractionOp op)>;
248
249 static LogicalResult defaultFilter(vector::ContractionOp op) {
250 return success();
251 }
252
253 ContractionOpToOuterProductOpLowering(
254 vector::VectorContractLowering vectorContractLowering,
255 MLIRContext *context, PatternBenefit benefit = 1,
256 FilterConstraintType constraint = defaultFilter)
257 : MaskableOpRewritePattern<vector::ContractionOp>(context, benefit),
258 vectorContractLowering(vectorContractLowering),
259 filter(std::move(constraint)) {}
260
261 FailureOr<Value>
262 matchAndRewriteMaskableOp(vector::ContractionOp op, MaskingOpInterface maskOp,
263 PatternRewriter &rewriter) const override;
264
265private:
266 /// Options to control the vector patterns.
267 vector::VectorContractLowering vectorContractLowering;
268 FilterConstraintType filter;
269};
270
271/// Progressive lowering of a `vector.contract %a, %b, %c` with row-major matmul
272/// semantics to an output-size-unrolled sequence:
273/// ```
274/// %out = arith.constant ... : vector<MxNxelt_type>
275/// %bt = vector.transpose %b, [1, 0]
276/// %aRow0 = vector.extract %a[0]
277/// %btRow0 = vector.extract %bt[0]
278/// %c00 = vector.reduction %atRow0, %bRow0
279/// %out00 = vector.insert %c00, %out[0, 0]
280/// ...
281/// %aRowLast = vector.extract %at[M-1]
282/// %btRowLast = vector.extract %b[N-1]
283/// %cLastLast = vector.reduction %atRowLast, %bRowLast
284/// %outcLastLast = vector.insert %cLastLast, %out[M-1, N-1]
285/// ```
286///
287/// This only kicks in when VectorTransformsOptions is set to Dot and
288/// the vector.contract op is a row-major matmul or matvec.
289class ContractionOpToDotLowering
290 : public MaskableOpRewritePattern<vector::ContractionOp> {
291public:
292 using MaskableOpRewritePattern::MaskableOpRewritePattern;
293
294 using FilterConstraintType =
295 std::function<LogicalResult(vector::ContractionOp op)>;
296
297 static LogicalResult defaultFilter(vector::ContractionOp op) {
298 return success();
299 }
300
301 ContractionOpToDotLowering(
302 vector::VectorContractLowering vectorContractLowering,
303 MLIRContext *context, PatternBenefit benefit = 1,
304 const FilterConstraintType &constraint = defaultFilter)
305 : MaskableOpRewritePattern<vector::ContractionOp>(context, benefit),
306 vectorContractLowering(vectorContractLowering), filter(defaultFilter) {}
307
308 FailureOr<Value>
309 matchAndRewriteMaskableOp(vector::ContractionOp op, MaskingOpInterface maskOp,
310 PatternRewriter &rewriter) const override;
311
312private:
313 /// Options to control the vector patterns.
314 vector::VectorContractLowering vectorContractLowering;
315 FilterConstraintType filter;
316};
317
318/// Progressive lowering of ContractionOp.
319///
320/// One:
321/// %x = vector.contract with at least one free/batch dimension
322/// is replaced by:
323/// %a = vector.contract with one less free/batch dimension
324/// %b = vector.contract with one less free/batch dimension
325/// ..
326/// %x = combine %a %b ..
327/// until a pure contraction is reached (no free/batch dimensions),
328/// which is replaced by a dot-product.
329///
330/// This only kicks in when either VectorTransformsOptions is set
331/// to Dot or when other contraction patterns fail.
332class ContractionOpLowering
333 : public MaskableOpRewritePattern<vector::ContractionOp> {
334public:
335 using MaskableOpRewritePattern::MaskableOpRewritePattern;
336 using FilterConstraintType =
337 std::function<LogicalResult(vector::ContractionOp op)>;
338
339 static LogicalResult defaultFilter(vector::ContractionOp op) {
340 return success();
341 }
342
343 ContractionOpLowering(
344 vector::VectorContractLowering vectorContractLoweringOption,
345 MLIRContext *context, PatternBenefit benefit = 1,
346 FilterConstraintType constraint = defaultFilter)
347 : MaskableOpRewritePattern<vector::ContractionOp>(context, benefit),
348 vectorContractLoweringOption(vectorContractLoweringOption),
349 filter(std::move(constraint)) {}
350
351 FailureOr<Value>
352 matchAndRewriteMaskableOp(vector::ContractionOp op, MaskingOpInterface maskOp,
353 PatternRewriter &rewriter) const override;
354
355private:
356 /// Options to control the vector patterns.
357 vector::VectorContractLowering vectorContractLoweringOption;
358 FilterConstraintType filter;
359 // Lower one parallel dimension.
360 FailureOr<Value> lowerParallel(PatternRewriter &rewriter,
361 vector::ContractionOp op, int64_t lhsIndex,
362 int64_t rhsIndex, Value mask) const;
363 // Lower one contracting dimension, given by its LHS and RHS indices.
364 FailureOr<Value> lowerReduction(PatternRewriter &rewriter,
365 vector::ContractionOp op, int64_t lhsIndex,
366 int64_t rhsIndex, Value mask) const;
367};
368
369/// Generate a vector implementation for matmat, matvec and tmatvec.
370/// This unrolls outer-products along the reduction dimension.
371struct UnrolledOuterProductGenerator
372 : public StructuredGenerator<vector::ContractionOp, vector::IteratorType> {
373 UnrolledOuterProductGenerator(RewriterBase &b, vector::ContractionOp op)
374 : StructuredGenerator<vector::ContractionOp, vector::IteratorType>(b, op),
375 kind(op.getKind()), lhs(op.getLhs()), rhs(op.getRhs()),
376 res(op.getAcc()), lhsType(op.getLhsType()) {
377 auto maskableOp = cast<MaskableOpInterface>(Val: op.getOperation());
378 if (maskableOp.isMasked())
379 mask = maskableOp.getMaskingOp().getMask();
380 }
381
382 Value t(Value v, ArrayRef<int64_t> perm = {1, 0}) {
383 if (!v)
384 return v;
385 return vector::TransposeOp::create(builder&: rewriter, location: loc, vector: v, permutation: perm);
386 }
387
388 Value promote(Value v, Type dstElementType) {
389 Type elementType = v.getType();
390 auto vecType = dyn_cast<VectorType>(Val&: elementType);
391 if (vecType)
392 elementType = vecType.getElementType();
393 if (elementType == dstElementType)
394 return v;
395 Type promotedType = dstElementType;
396 if (vecType)
397 promotedType = vecType.clone(elementType: promotedType);
398 if (isa<FloatType>(Val: dstElementType))
399 return arith::ExtFOp::create(builder&: rewriter, location: loc, out: promotedType, in: v,
400 /*fastmath=*/{});
401 return arith::ExtSIOp::create(builder&: rewriter, location: loc, out: promotedType, in: v);
402 }
403
404 FailureOr<Value> outerProd(Value lhs, Value rhs, Value res,
405 VectorType lhsType, int reductionSize,
406 std::optional<Value> maybeMask = std::nullopt) {
407 // Incremental support for masking.
408 if (mask && !maybeMask.has_value())
409 return failure();
410
411 Type resElementType = cast<VectorType>(Val: res.getType()).getElementType();
412 for (int64_t k = 0; k < reductionSize; ++k) {
413 Value extractA = vector::ExtractOp::create(builder&: rewriter, location: loc, source: lhs, position: k);
414 Value extractB = vector::ExtractOp::create(builder&: rewriter, location: loc, source: rhs, position: k);
415 extractA = promote(v: extractA, dstElementType: resElementType);
416 extractB = promote(v: extractB, dstElementType: resElementType);
417 Value extractMask;
418 if (maybeMask.has_value() && maybeMask.value())
419 extractMask =
420 vector::ExtractOp::create(builder&: rewriter, location: loc, source: maybeMask.value(), position: k);
421
422 Operation *outerProdOp = vector::OuterProductOp::create(
423 builder&: rewriter, location: loc, resultType0: res.getType(), lhs: extractA, rhs: extractB, acc: res, kind);
424 res = maskOperation(builder&: rewriter, maskableOp: outerProdOp, mask: extractMask)->getResult(idx: 0);
425 }
426 return res;
427 }
428
429 /// Helper function for `matmat`, `matvec`, `tmatvec`. Returns the size of
430 /// dimension `reductionDim`. If the dimension is a scalable dimension,
431 /// returns "nullopt".
432 std::optional<int64_t> getReductionSize(VectorType vecType,
433 int64_t reductionDim) {
434 // Cannot unroll scalable dimension.
435 if (vecType.getScalableDims()[reductionDim])
436 return std::nullopt;
437 int64_t reductionSize = vecType.getDimSize(idx: reductionDim);
438 assert(reductionSize > 0 &&
439 "Reduction dim must be a known static size to allow unrolling");
440 return reductionSize;
441 }
442
443 /// Two outer parallel, one inner reduction (matmat flavor).
444 FailureOr<Value> matmat() {
445 if (!iters(its: {Par(), Par(), Red()}))
446 return failure();
447 // Set up the parallel/reduction structure in the right form.
448 AffineExpr m, n, k;
449 bindDims(ctx: rewriter.getContext(), exprs&: m, exprs&: n, exprs&: k);
450
451 // Classical row-major matmul: Just permute the lhs.
452 if (layout(l: {{m, k}, {k, n}, {m, n}})) {
453 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 1)) {
454 // Note: `t` creates new IR. It must be nested within this `if` check
455 // so that no IR is created when then pattern returns "failure".
456 Value tLhs = t(v: lhs);
457 Value tMask = t(v: mask, perm: {2, 0, 1});
458 return outerProd(lhs: tLhs, rhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
459 }
460 }
461 // TODO: may be better to fail and use some vector<k> -> scalar reduction.
462 if (layout(l: {{m, k}, {n, k}, {m, n}})) {
463 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 1)) {
464 Value tLhs = t(v: lhs);
465 Value tRhs = t(v: rhs);
466 Value tMask = t(v: mask, perm: {2, 0, 1});
467 return outerProd(lhs: tLhs, rhs: tRhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
468 }
469 }
470 // No need to permute anything.
471 if (layout(l: {{k, m}, {k, n}, {m, n}})) {
472 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 0)) {
473 Value tMask = t(v: mask, perm: {2, 0, 1});
474 return outerProd(lhs, rhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
475 }
476 }
477 // Just permute the rhs.
478 if (layout(l: {{k, m}, {n, k}, {m, n}})) {
479 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 0)) {
480 Value tRhs = t(v: rhs);
481 Value tMask = t(v: mask, perm: {2, 0, 1});
482 return outerProd(lhs, rhs: tRhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
483 }
484 }
485 // Transposed output: swap RHS and LHS.
486 // Classical row-major matmul: permute the lhs.
487 if (layout(l: {{m, k}, {k, n}, {n, m}})) {
488 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 1)) {
489 Value tLhs = t(v: lhs);
490 Value tMask = t(v: mask, perm: {2, 0, 1});
491 return outerProd(lhs: rhs, rhs: tLhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
492 }
493 }
494 // TODO: may be better to fail and use some vector<k> -> scalar reduction.
495 if (layout(l: {{m, k}, {n, k}, {n, m}})) {
496 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 1)) {
497 Value tRhs = t(v: rhs);
498 Value tLhs = t(v: lhs);
499 Value tMask = t(v: mask, perm: {2, 0, 1});
500 return outerProd(lhs: tRhs, rhs: tLhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
501 }
502 }
503 if (layout(l: {{k, m}, {k, n}, {n, m}})) {
504 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 0)) {
505 Value tMask = t(v: mask, perm: {2, 0, 1});
506 return outerProd(lhs: rhs, rhs: lhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
507 }
508 }
509 if (layout(l: {{k, m}, {n, k}, {n, m}})) {
510 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 0)) {
511 Value tRhs = t(v: rhs);
512 Value tMask = t(v: mask, perm: {2, 0, 1});
513 return outerProd(lhs: tRhs, rhs: lhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
514 }
515 }
516 return failure();
517 }
518
519 //
520 // One outer parallel, one inner reduction (matvec flavor).
521 // Mask needs to be transposed everywhere to turn the reduction dimension
522 // outermost as required by outerproduct.
523 //
524 FailureOr<Value> matvec() {
525 if (!iters(its: {Par(), Red()}))
526 return failure();
527 AffineExpr m, k;
528 bindDims(ctx: rewriter.getContext(), exprs&: m, exprs&: k);
529
530 // Case mat-vec: transpose.
531 if (layout(l: {{m, k}, {k}, {m}})) {
532 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 1)) {
533 Value tLhs = t(v: lhs);
534 Value tMask = t(v: mask);
535 return outerProd(lhs: tLhs, rhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
536 }
537 }
538 // Case mat-trans-vec: ready to go.
539 if (layout(l: {{k, m}, {k}, {m}})) {
540 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 0)) {
541 Value tMask = t(v: mask);
542 return outerProd(lhs, rhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
543 }
544 }
545 // Case vec-mat: swap and transpose.
546 if (layout(l: {{k}, {m, k}, {m}})) {
547 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 0)) {
548 Value tRhs = t(v: rhs);
549 Value tMask = t(v: mask);
550 return outerProd(lhs: tRhs, rhs: lhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
551 }
552 }
553 // Case vec-mat-trans: swap and ready to go.
554 if (layout(l: {{k}, {k, m}, {m}})) {
555 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 0)) {
556 Value tMask = t(v: mask);
557 return outerProd(lhs: rhs, rhs: lhs, res, lhsType, reductionSize: *reductionSize, maybeMask: tMask);
558 }
559 }
560 return failure();
561 }
562
563 //
564 // One outer reduction, one inner parallel (tmatvec flavor).
565 // Mask already has the shape of the outer product.
566 //
567 FailureOr<Value> tmatvec() {
568 if (!iters(its: {Red(), Par()}))
569 return failure();
570 AffineExpr k, m;
571 bindDims(ctx: rewriter.getContext(), exprs&: k, exprs&: m);
572
573 // Case mat-vec: transpose.
574 if (layout(l: {{m, k}, {k}, {m}}))
575 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 1))
576 return outerProd(lhs: t(v: lhs), rhs, res, lhsType, reductionSize: *reductionSize, maybeMask: mask);
577 // Case mat-trans-vec: ready to go.
578 if (layout(l: {{k, m}, {k}, {m}}))
579 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 0))
580 return outerProd(lhs, rhs, res, lhsType, reductionSize: *reductionSize, maybeMask: mask);
581 // Case vec-mat: swap and transpose.
582 if (layout(l: {{k}, {m, k}, {m}}))
583 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 0))
584 return outerProd(lhs: t(v: rhs), rhs: lhs, res, lhsType, reductionSize: *reductionSize, maybeMask: mask);
585 // Case vec-mat-trans: swap and ready to go.
586 if (layout(l: {{k}, {k, m}, {m}}))
587 if (auto reductionSize = getReductionSize(vecType: lhsType, reductionDim: 0))
588 return outerProd(lhs: rhs, rhs: lhs, res, lhsType, reductionSize: *reductionSize, maybeMask: mask);
589 return failure();
590 }
591
592private:
593 vector::CombiningKind kind;
594 Value lhs, rhs, res, mask;
595 VectorType lhsType;
596};
597
598/// Progressively lower a `vector.contract %a, %b, %c` with row-major matmul
599/// semantics to a reduction_size-unrolled sequence:
600/// ```
601/// %at = vector.transpose %a, [1, 0]
602/// %bRow0 = vector.extract %b[0]
603/// %atRow0 = vector.extract %at[0]
604/// %c0 = vector.outerproduct %atRow0, %bRow0, %c
605/// ...
606/// %bRowK = vector.extract %b[K]
607/// %atRowK = vector.extract %at[K]
608/// %cK = vector.outerproduct %atRowK, %bRowK, %cK-1
609/// ```
610///
611/// This only kicks in when vectorContractLowering is set to OuterProduct but
612/// otherwise supports any layout permutation of the matrix-multiply.
613FailureOr<Value>
614ContractionOpToOuterProductOpLowering::matchAndRewriteMaskableOp(
615 vector::ContractionOp op, MaskingOpInterface maskOp,
616 PatternRewriter &rewriter) const {
617 if (vectorContractLowering != vector::VectorContractLowering::OuterProduct)
618 return failure();
619
620 if (failed(Result: filter(op)))
621 return failure();
622
623 UnrolledOuterProductGenerator e(rewriter, op);
624 FailureOr<Value> matmatRes = e.matmat();
625 if (succeeded(Result: matmatRes)) {
626 return matmatRes;
627 }
628 FailureOr<Value> matvecRes = e.matvec();
629 if (succeeded(Result: matvecRes)) {
630 return matvecRes;
631 }
632
633 FailureOr<Value> tmatvecRes = e.tmatvec();
634 return tmatvecRes;
635}
636
637FailureOr<Value> ContractionOpToDotLowering::matchAndRewriteMaskableOp(
638 vector::ContractionOp op, MaskingOpInterface maskOp,
639 PatternRewriter &rewriter) const {
640 // TODO: Support vector.mask.
641 if (maskOp)
642 return failure();
643
644 if (failed(Result: filter(op)))
645 return failure();
646
647 if (vectorContractLowering != vector::VectorContractLowering::Dot)
648 return failure();
649
650 // A scalable dst dim can't be unrolled with a static trip count; check
651 // before any IR is created below.
652 if (auto dstType = dyn_cast<VectorType>(Val: op.getResultType());
653 dstType && dstType.isScalable())
654 return rewriter.notifyMatchFailure(
655 arg&: op, msg: "cannot unroll a scalable dimension in dot-product lowering");
656
657 auto iteratorTypes = op.getIteratorTypes().getValue();
658 static constexpr std::array<int64_t, 2> perm = {1, 0};
659 Location loc = op.getLoc();
660 Value lhs = op.getLhs(), rhs = op.getRhs();
661
662 using MapList = ArrayRef<ArrayRef<AffineExpr>>;
663 auto infer = [&](MapList m) {
664 return AffineMap::inferFromExprList(exprsList: m, context: op.getContext());
665 };
666 AffineExpr m, n, k;
667 bindDims(ctx: rewriter.getContext(), exprs&: m, exprs&: n, exprs&: k);
668 SmallVector<AffineMap> maps = op.getIndexingMapsArray();
669 //
670 // In the following we wish to make the reduction dimension innermost so we
671 // can load vectors and just fmul + reduce into a scalar.
672 //
673 if (isParallelIterator(attr: iteratorTypes[0]) &&
674 isParallelIterator(attr: iteratorTypes[1]) &&
675 isReductionIterator(attr: iteratorTypes[2])) {
676 //
677 // Two outer parallel, one inner reduction (matmat flavor).
678 //
679 if (maps == infer({{m, k}, {k, n}, {m, n}})) {
680 rhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: rhs, permutation: perm);
681 } else if (maps == infer({{m, k}, {n, k}, {m, n}})) {
682 // No need to permute anything.
683 } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
684 lhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: lhs, permutation: perm);
685 rhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: rhs, permutation: perm);
686 } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
687 lhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: lhs, permutation: perm);
688 } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
689 // This is the classical row-major matmul. Just permute the lhs.
690 Value tmp = lhs;
691 lhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: rhs, permutation: perm);
692 rhs = tmp;
693 } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
694 std::swap(a&: lhs, b&: rhs);
695 } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
696 Value tmp = lhs;
697 lhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: rhs, permutation: perm);
698 rhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: tmp, permutation: perm);
699 } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
700 Value tmp = rhs;
701 rhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: lhs, permutation: perm);
702 lhs = tmp;
703 } else {
704 return failure();
705 }
706 } else if (isParallelIterator(attr: iteratorTypes[0]) &&
707 isReductionIterator(attr: iteratorTypes[1])) {
708 //
709 // One outer parallel, one inner reduction (matvec flavor)
710 //
711 if (maps == infer({{m, n}, {n}, {m}})) {
712 // No need to permute anything.
713 } else if (maps == infer({{n, m}, {n}, {m}})) {
714 lhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: lhs, permutation: perm);
715 } else if (maps == infer({{n}, {m, n}, {m}})) {
716 std::swap(a&: lhs, b&: rhs);
717 } else if (maps == infer({{n}, {n, m}, {m}})) {
718 std::swap(a&: lhs, b&: rhs);
719 lhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: lhs, permutation: perm);
720 } else {
721 return failure();
722 }
723 } else {
724 return failure();
725 }
726
727 VectorType dstType = cast<VectorType>(Val: op.getResultType());
728 assert(dstType.getRank() >= 1 && dstType.getRank() <= 2 &&
729 "Expected dst type of rank 1 or 2");
730
731 unsigned rank = dstType.getRank();
732 unsigned dstRows = dstType.getShape()[0];
733 unsigned dstColumns = rank == 1 ? 1 : dstType.getShape()[1];
734
735 // ExtractOp does not allow dynamic indexing, we must unroll explicitly.
736 Value res = arith::ConstantOp::create(builder&: rewriter, location: loc, result: dstType,
737 value: rewriter.getZeroAttr(type: dstType));
738 bool isInt = isa<IntegerType>(Val: dstType.getElementType());
739 arith::FastMathFlagsAttr fmf = op.getFastmathAttr();
740 llvm::SmallVector<Value> extractedCols;
741 extractedCols.reserve(N: dstColumns);
742 for (unsigned r = 0; r < dstRows; ++r) {
743 Value rowLhs = vector::ExtractOp::create(builder&: rewriter, location: op.getLoc(), source: lhs, position: r);
744 for (unsigned c = 0; c < dstColumns; ++c) {
745 // Extract each respective row and column of the LHS and RHS once to
746 // avoid having duplicate SSA values pointing to the same rows/columns.
747 if (r == 0) {
748 Value colRhs =
749 rank == 1
750 ? rhs
751 : vector::ExtractOp::create(builder&: rewriter, location: op.getLoc(), source: rhs, position: c);
752 extractedCols.push_back(Elt: colRhs);
753 }
754 Value extractedColRhs = extractedCols[c];
755 Value product =
756 createMul(loc: op.getLoc(), x: rowLhs, y: extractedColRhs, isInt, rewriter, fmf);
757 Value sum = vector::ReductionOp::create(builder&: rewriter, location: op.getLoc(),
758 kind: vector::CombiningKind::ADD,
759 vector: product, fastMathFlags: op.getFastmath());
760
761 SmallVector<int64_t, 2> pos = rank == 1 ? SmallVector<int64_t, 2>{r}
762 : SmallVector<int64_t, 2>{r, c};
763 res = vector::InsertOp::create(builder&: rewriter, location: op.getLoc(), valueToStore: sum, dest: res, position: pos);
764 }
765 }
766 if (auto acc = op.getAcc())
767 res = createAdd(loc: op.getLoc(), x: res, y: acc, isInt, rewriter, fmf);
768 return res;
769}
770
771/// Lower vector.contract with all size one reduction dimensions to
772/// elementwise ops when possible.
773struct ContractOpToElementwise
774 : public MaskableOpRewritePattern<vector::ContractionOp> {
775 using MaskableOpRewritePattern::MaskableOpRewritePattern;
776 using FilterConstraintType =
777 std::function<LogicalResult(vector::ContractionOp op)>;
778 static LogicalResult defaultFilter(vector::ContractionOp op) {
779 return success();
780 }
781 ContractOpToElementwise(
782 vector::VectorContractLowering vectorContractLowering,
783 MLIRContext *context, PatternBenefit benefit = 1,
784 const FilterConstraintType &constraint = defaultFilter)
785 : MaskableOpRewritePattern<vector::ContractionOp>(context, benefit),
786 vectorContractLowering(vectorContractLowering), filter(defaultFilter) {}
787
788 FailureOr<Value>
789 matchAndRewriteMaskableOp(vector::ContractionOp contractOp,
790 MaskingOpInterface maskOp,
791 PatternRewriter &rewriter) const override {
792 // TODO: Support vector.mask.
793 if (maskOp)
794 return failure();
795
796 if (failed(Result: filter(contractOp)))
797 return failure();
798
799 if (vectorContractLowering != vector::VectorContractLowering::ParallelArith)
800 return failure();
801
802 ArrayRef<int64_t> lhsShape = contractOp.getLhsType().getShape();
803 ArrayRef<int64_t> rhsShape = contractOp.getRhsType().getShape();
804 AffineMap lhsMap = contractOp.getIndexingMapsArray()[0];
805 AffineMap rhsMap = contractOp.getIndexingMapsArray()[1];
806 SmallVector<int64_t> lhsReductionDims =
807 getReductionIndex(map: lhsMap, iteratorTypes: contractOp.getIteratorTypes());
808 SmallVector<int64_t> rhsReductionDims =
809 getReductionIndex(map: rhsMap, iteratorTypes: contractOp.getIteratorTypes());
810 // All the reduction dimensions must be a size 1.
811 for (int64_t dim : lhsReductionDims) {
812 if (lhsShape[dim] != 1)
813 return failure();
814 }
815 for (int64_t dim : rhsReductionDims) {
816 if (rhsShape[dim] != 1)
817 return failure();
818 }
819 AffineMap accMap = contractOp.getIndexingMapsArray()[2];
820 unsigned numParallelDims = accMap.getNumResults();
821 unsigned numLhsDimToBroadcast =
822 numParallelDims - (lhsMap.getNumResults() - lhsReductionDims.size());
823 unsigned numRhsDimToBroadcast =
824 numParallelDims - (rhsMap.getNumResults() - rhsReductionDims.size());
825 SmallVector<int64_t> lhsDims;
826 SmallVector<int64_t> lhsTranspose;
827 SmallVector<int64_t> rhsDims;
828 SmallVector<int64_t> rhsTranspose;
829 for (int64_t dim : lhsReductionDims)
830 lhsTranspose.push_back(Elt: numLhsDimToBroadcast + dim);
831 for (int64_t dim : rhsReductionDims)
832 rhsTranspose.push_back(Elt: numRhsDimToBroadcast + dim);
833 // Loop through the parallel dimensions to calculate the dimensions to
834 // broadcast and to permute in order to extract only parallel dimensions.
835 for (unsigned i = 0; i < numParallelDims; i++) {
836 std::optional<unsigned> lhsDim =
837 getDimPosition(map: lhsMap, dim: accMap.getDimPosition(idx: i));
838 if (lhsDim) {
839 lhsTranspose.push_back(Elt: numLhsDimToBroadcast + *lhsDim);
840 } else {
841 // If the parallel dimension doesn't exist we will have to broadcast it.
842 lhsDims.push_back(
843 Elt: cast<VectorType>(Val: contractOp.getResultType()).getDimSize(idx: i));
844 lhsTranspose.push_back(Elt: lhsDims.size() - 1);
845 }
846 std::optional<unsigned> rhsDim =
847 getDimPosition(map: rhsMap, dim: accMap.getDimPosition(idx: i));
848 if (rhsDim) {
849 rhsTranspose.push_back(Elt: numRhsDimToBroadcast + *rhsDim);
850 } else {
851 // If the parallel dimension doesn't exist we will have to broadcast it.
852 rhsDims.push_back(
853 Elt: cast<VectorType>(Val: contractOp.getResultType()).getDimSize(idx: i));
854 rhsTranspose.push_back(Elt: rhsDims.size() - 1);
855 }
856 }
857 Value newLhs = contractOp.getLhs();
858 Value newRhs = contractOp.getRhs();
859 Location loc = contractOp.getLoc();
860 if (!lhsDims.empty()) {
861 lhsDims.append(in_start: lhsShape.begin(), in_end: lhsShape.end());
862 auto expandedType =
863 VectorType::get(shape: lhsDims, elementType: contractOp.getLhsType().getElementType());
864 newLhs = vector::BroadcastOp::create(builder&: rewriter, location: loc, result: expandedType, source: newLhs);
865 }
866 if (!rhsDims.empty()) {
867 rhsDims.append(in_start: rhsShape.begin(), in_end: rhsShape.end());
868 auto expandedType =
869 VectorType::get(shape: rhsDims, elementType: contractOp.getRhsType().getElementType());
870 newRhs = vector::BroadcastOp::create(builder&: rewriter, location: loc, result: expandedType, source: newRhs);
871 }
872 bool isInt = contractOp.getLhsType().getElementType().isIntOrIndex();
873 newLhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: newLhs, permutation: lhsTranspose);
874 newRhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: newRhs, permutation: rhsTranspose);
875 SmallVector<int64_t> lhsOffsets(lhsReductionDims.size(), 0);
876 SmallVector<int64_t> rhsOffsets(rhsReductionDims.size(), 0);
877 newLhs = vector::ExtractOp::create(builder&: rewriter, location: loc, source: newLhs, position: lhsOffsets);
878 newRhs = vector::ExtractOp::create(builder&: rewriter, location: loc, source: newRhs, position: rhsOffsets);
879 std::optional<Value> result =
880 createContractArithOp(loc, x: newLhs, y: newRhs, acc: contractOp.getAcc(),
881 kind: contractOp.getKind(), rewriter, isInt,
882 /*mask=*/Value(), fmf: contractOp.getFastmathAttr());
883 if (result)
884 return *result;
885
886 return failure();
887 }
888
889private:
890 /// Options to control the vector patterns.
891 vector::VectorContractLowering vectorContractLowering;
892 FilterConstraintType filter;
893};
894
895/// Progressive lowering of ContractionOp.
896/// One:
897/// %x = vector.contract with at least one free/batch dimension
898/// is replaced by:
899/// %a = vector.contract with one less free/batch dimension
900/// %b = vector.contract with one less free/batch dimension
901/// ..
902/// %x = combine %a %b ..
903/// until a pure contraction is reached (no free/batch dimensions),
904/// which is replaced by a dot-product.
905///
906/// This only kicks in when either vectorContractLoweringOption is set
907/// to DOT or when other contraction patterns fail.
908//
909// TODO: break down into transpose/reshape/cast ops
910// when they become available to avoid code dup
911// TODO: investigate lowering order impact on performance
912FailureOr<Value> ContractionOpLowering::matchAndRewriteMaskableOp(
913 vector::ContractionOp op, MaskingOpInterface maskOp,
914 PatternRewriter &rewriter) const {
915 if (failed(Result: filter(op)))
916 return failure();
917
918 // TODO: support mixed mode contract lowering.
919 if (op.getLhsType().getElementType() !=
920 getElementTypeOrSelf(type: op.getAccType()) ||
921 op.getRhsType().getElementType() != getElementTypeOrSelf(type: op.getAccType()))
922 return failure();
923
924 // TODO: the code below assumes the default contraction, make sure it supports
925 // other kinds before enabling this lowering.
926 if (!op.isAddKind()) {
927 return rewriter.notifyMatchFailure(
928 arg&: op, msg: "contractions other than 'add' not supported");
929 }
930
931 // TODO: implement benefits, cost models.
932 MLIRContext *ctx = op.getContext();
933
934 ContractionOpToOuterProductOpLowering pat1(vectorContractLoweringOption, ctx);
935 FailureOr<Value> newVal1 =
936 pat1.matchAndRewriteMaskableOp(op, maskOp, rewriter);
937 if (!failed(Result: newVal1))
938 return newVal1;
939
940 ContractionOpToDotLowering pat2(vectorContractLoweringOption, ctx);
941 FailureOr<Value> newVal2 =
942 pat2.matchAndRewriteMaskableOp(op, maskOp, rewriter);
943 if (!failed(Result: newVal2))
944 return newVal2;
945
946 ContractOpToElementwise pat4(vectorContractLoweringOption, ctx);
947 FailureOr<Value> newVal4 =
948 pat4.matchAndRewriteMaskableOp(contractOp: op, maskOp, rewriter);
949 if (!failed(Result: newVal4))
950 return newVal4;
951
952 // Vector mask setup.
953
954 Value mask;
955 if (maskOp)
956 mask = maskOp.getMask();
957 using DimPair = std::pair<int64_t, int64_t>;
958 // Find first batch dimension in LHS/RHS, and lower when found.
959 std::vector<DimPair> batchDimMap = op.getBatchDimMap();
960 if (!batchDimMap.empty()) {
961 int64_t lhsIndex = batchDimMap[0].first;
962 int64_t rhsIndex = batchDimMap[0].second;
963 auto newOp = lowerParallel(rewriter, op, lhsIndex, rhsIndex, mask);
964 if (failed(Result: newOp))
965 return failure();
966 return newOp;
967 }
968
969 std::vector<DimPair> freeDimMap = op.getFreeDimMap();
970
971 // Find first free dimension in LHS, and lower when found.
972 const DimPair *dimsToProcess = nullptr;
973 for (const DimPair &dims : freeDimMap) {
974 if (dims.first < 0)
975 continue;
976 if (!dimsToProcess || dims.first < dimsToProcess->first)
977 dimsToProcess = &dims;
978 }
979 if (dimsToProcess)
980 return lowerParallel(rewriter, op, lhsIndex: dimsToProcess->first,
981 rhsIndex: dimsToProcess->second, mask);
982
983 // Find first free dimension in RHS, and lower when found.
984 for (const DimPair &dims : freeDimMap) {
985 if (dims.second < 0)
986 continue;
987 if (!dimsToProcess || dims.second < dimsToProcess->second)
988 dimsToProcess = &dims;
989 }
990 if (dimsToProcess)
991 return lowerParallel(rewriter, op, lhsIndex: dimsToProcess->first,
992 rhsIndex: dimsToProcess->second, mask);
993
994 // Lower the first remaining contracting dimension.
995 std::vector<DimPair> contractingDimMap = op.getContractingDimMap();
996 if (!contractingDimMap.empty())
997 return lowerReduction(rewriter, op, lhsIndex: contractingDimMap[0].first,
998 rhsIndex: contractingDimMap[0].second, mask);
999
1000 return failure();
1001}
1002
1003// Lower one parallel dimension.
1004// TODO: consider reusing existing contract unrolling
1005FailureOr<Value> ContractionOpLowering::lowerParallel(PatternRewriter &rewriter,
1006 vector::ContractionOp op,
1007 int64_t lhsIndex,
1008 int64_t rhsIndex,
1009 Value mask) const {
1010 VectorType lhsType = op.getLhsType();
1011 VectorType rhsType = op.getRhsType();
1012 VectorType resType = cast<VectorType>(Val: op.getResultType());
1013 // Find the iterator type index and result index.
1014 SmallVector<AffineMap> iMap = op.getIndexingMapsArray();
1015 int64_t iterIndex = -1;
1016 int64_t dimSize = -1;
1017 if (lhsIndex >= 0) {
1018 iterIndex = iMap[0].getDimPosition(idx: lhsIndex);
1019 if (rhsIndex >= 0 && iterIndex != iMap[1].getDimPosition(idx: rhsIndex))
1020 return rewriter.notifyMatchFailure(op, reasonCallback: [&](Diagnostic &diag) {
1021 diag << "expected lhsIndex=" << lhsIndex << " and rhsIndex=" << rhsIndex
1022 << " to map to the same dimension";
1023 });
1024 if (lhsType.getScalableDims()[lhsIndex])
1025 return rewriter.notifyMatchFailure(op, reasonCallback: [&](Diagnostic &diag) {
1026 diag << "Unrolling scalable dimension (lhsIndex=" << lhsIndex
1027 << ") is not supported yet";
1028 });
1029 dimSize = lhsType.getDimSize(idx: lhsIndex);
1030 } else if (rhsIndex >= 0) {
1031 iterIndex = iMap[1].getDimPosition(idx: rhsIndex);
1032 if (rhsType.getScalableDims()[rhsIndex])
1033 return rewriter.notifyMatchFailure(op, reasonCallback: [&](Diagnostic &diag) {
1034 diag << "Unrolling scalable dimension (rhsIndex=" << rhsIndex
1035 << ") is not supported yet";
1036 });
1037 dimSize = rhsType.getDimSize(idx: rhsIndex);
1038 }
1039 if (iterIndex < 0)
1040 return rewriter.notifyMatchFailure(op, reasonCallback: [&](Diagnostic &diag) {
1041 diag << "expected either lhsIndex=" << lhsIndex
1042 << " or rhsIndex=" << rhsIndex << " to be nonnegative";
1043 });
1044 std::optional<int64_t> lookupRes = getResultIndex(map: iMap[2], index: iterIndex);
1045 if (!lookupRes.has_value())
1046 return rewriter.notifyMatchFailure(op, reasonCallback: [&](Diagnostic &diag) {
1047 diag << "expected the dimension for iterIndex=" << iterIndex
1048 << " to appear in the result map";
1049 });
1050 int64_t resIndex = *lookupRes;
1051
1052 // Construct new iterator types and affine map array attribute.
1053 std::array<AffineMap, 3> lowIndexingMaps = {
1054 adjustMap(map: iMap[0], index: iterIndex, rewriter),
1055 adjustMap(map: iMap[1], index: iterIndex, rewriter),
1056 adjustMap(map: iMap[2], index: iterIndex, rewriter)};
1057 auto lowAffine = rewriter.getAffineMapArrayAttr(values: lowIndexingMaps);
1058 auto lowIter =
1059 rewriter.getArrayAttr(value: adjustIter(iteratorTypes: op.getIteratorTypes(), index: iterIndex));
1060 // Unroll into a series of lower dimensional vector.contract ops.
1061 Location loc = op.getLoc();
1062 Value result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: resType,
1063 value: rewriter.getZeroAttr(type: resType));
1064
1065 for (int64_t d = 0; d < dimSize; ++d) {
1066 auto lhs = reshapeLoad(loc, val: op.getLhs(), index: lhsIndex, pos: d, rewriter);
1067 auto rhs = reshapeLoad(loc, val: op.getRhs(), index: rhsIndex, pos: d, rewriter);
1068 auto acc = reshapeLoad(loc, val: op.getAcc(), index: resIndex, pos: d, rewriter);
1069
1070 Value lowMask;
1071 if (mask)
1072 lowMask = reshapeLoad(loc, val: mask, index: iterIndex, pos: d, rewriter);
1073
1074 Operation *lowContract =
1075 vector::ContractionOp::create(builder&: rewriter, location: loc, lhs, rhs, acc, indexingMaps: lowAffine,
1076 iteratorTypes: lowIter, kind: op.getKind(), fastMathFlags: op.getFastmath());
1077 lowContract = maskOperation(builder&: rewriter, maskableOp: lowContract, mask: lowMask);
1078 result = reshapeStore(loc, val: lowContract->getResult(idx: 0), result, index: resIndex, pos: d,
1079 rewriter);
1080 }
1081 return result;
1082}
1083
1084// Lower one contracting dimension, given by its LHS and RHS indices.
1085FailureOr<Value> ContractionOpLowering::lowerReduction(
1086 PatternRewriter &rewriter, vector::ContractionOp op, int64_t lhsIndex,
1087 int64_t rhsIndex, Value mask) const {
1088 auto loc = op.getLoc();
1089 VectorType lhsType = op.getLhsType();
1090 VectorType rhsType = op.getRhsType();
1091 Type resType = op.getResultType();
1092 auto resVectorType = dyn_cast<VectorType>(Val&: resType);
1093 if (resVectorType && resVectorType.getRank() != 0)
1094 return rewriter.notifyMatchFailure(arg&: op,
1095 msg: "did not expect a VectorType result");
1096 bool isInt = isa<IntegerType>(Val: getElementTypeOrSelf(type: resType));
1097 SmallVector<AffineMap> iMap = op.getIndexingMapsArray();
1098 // The dimension must map to the same iterator and have the same size on both
1099 // operands.
1100 int64_t iterIndex = iMap[0].getDimPosition(idx: lhsIndex);
1101 if (iterIndex != iMap[1].getDimPosition(idx: rhsIndex))
1102 return rewriter.notifyMatchFailure(op, reasonCallback: [&](Diagnostic &diag) {
1103 diag << "expected lhsIndex=" << lhsIndex << " and rhsIndex=" << rhsIndex
1104 << " to map to the same dimension";
1105 });
1106 int64_t dimSize = lhsType.getDimSize(idx: lhsIndex);
1107 if (dimSize != rhsType.getDimSize(idx: rhsIndex))
1108 return rewriter.notifyMatchFailure(op, reasonCallback: [&](Diagnostic &diag) {
1109 diag << "expect LHS dimension " << lhsIndex
1110 << " to have the same size as RHS dimension " << rhsIndex;
1111 });
1112 // Base case.
1113 if (lhsType.getRank() == 1) {
1114 if (rhsType.getRank() != 1)
1115 return rewriter.notifyMatchFailure(
1116 arg&: op, msg: "When LHS has rank 1, expected also RHS to have rank 1");
1117 arith::FastMathFlagsAttr fmf = op.getFastmathAttr();
1118 Value m = createMul(loc, x: op.getLhs(), y: op.getRhs(), isInt, rewriter, fmf);
1119 auto kind = vector::CombiningKind::ADD;
1120
1121 Value acc = op.getAcc();
1122 // vector.reduction takes a scalar accumulator.
1123 if (resVectorType)
1124 acc = vector::ExtractOp::create(builder&: rewriter, location: loc, source: acc, position: ArrayRef<int64_t>{});
1125 Operation *reductionOp =
1126 acc ? vector::ReductionOp::create(builder&: rewriter, location: loc, kind, vector: m, acc,
1127 fastMathFlags: op.getFastmath())
1128 : vector::ReductionOp::create(builder&: rewriter, location: loc, kind, vector: m,
1129 fastMathFlags: op.getFastmath());
1130 Value reduced = maskOperation(builder&: rewriter, maskableOp: reductionOp, mask)->getResult(idx: 0);
1131 if (resVectorType)
1132 reduced =
1133 vector::BroadcastOp::create(builder&: rewriter, location: loc, result: resVectorType, source: reduced);
1134 return reduced;
1135 }
1136 // Construct new iterator types and affine map array attribute.
1137 std::array<AffineMap, 3> lowIndexingMaps = {
1138 adjustMap(map: iMap[0], index: iterIndex, rewriter),
1139 adjustMap(map: iMap[1], index: iterIndex, rewriter),
1140 adjustMap(map: iMap[2], index: iterIndex, rewriter)};
1141 auto lowAffine = rewriter.getAffineMapArrayAttr(values: lowIndexingMaps);
1142 auto lowIter =
1143 rewriter.getArrayAttr(value: adjustIter(iteratorTypes: op.getIteratorTypes(), index: iterIndex));
1144 // Unroll into a series of lower dimensional vector.contract ops.
1145 // By feeding the initial accumulator into the first contraction,
1146 // and the result of each contraction into the next, eventually
1147 // the sum of all reductions is computed.
1148 Value result = op.getAcc();
1149 for (int64_t d = 0; d < dimSize; ++d) {
1150 auto lhs = reshapeLoad(loc, val: op.getLhs(), index: lhsIndex, pos: d, rewriter);
1151 auto rhs = reshapeLoad(loc, val: op.getRhs(), index: rhsIndex, pos: d, rewriter);
1152 Value newMask;
1153 if (mask)
1154 newMask = reshapeLoad(loc, val: mask, index: iterIndex, pos: d, rewriter);
1155
1156 Operation *newContract = vector::ContractionOp::create(
1157 builder&: rewriter, location: loc, lhs, rhs, acc: result, indexingMaps: lowAffine, iteratorTypes: lowIter, kind: op.getKind(),
1158 fastMathFlags: op.getFastmath());
1159 result = maskOperation(builder&: rewriter, maskableOp: newContract, mask: newMask)->getResult(idx: 0);
1160 }
1161 return result;
1162}
1163
1164/// Progressive lowering of OuterProductOp.
1165/// One:
1166/// %x = vector.outerproduct %lhs, %rhs, %acc
1167/// is replaced by:
1168/// %z = zero-result
1169/// %0 = vector.extract %lhs[0]
1170/// %1 = vector.broadcast %0
1171/// %2 = vector.extract %acc[0]
1172/// %3 = vector.fma %1, %rhs, %2
1173/// %4 = vector.insert %3, %z[0]
1174/// ..
1175/// %x = vector.insert %.., %..[N-1]
1176///
1177class OuterProductOpLowering : public OpRewritePattern<vector::OuterProductOp> {
1178public:
1179 using Base::Base;
1180
1181 LogicalResult matchAndRewrite(vector::OuterProductOp op,
1182 PatternRewriter &rewriter) const override {
1183 VectorType resType = op.getResultVectorType();
1184 if ((resType.getShape().size() >= 2) && resType.allDimsScalable())
1185 return failure();
1186
1187 auto loc = op.getLoc();
1188
1189 VectorType lhsType = op.getOperandVectorTypeLHS();
1190 VectorType rhsType = dyn_cast<VectorType>(Val: op.getOperandTypeRHS());
1191 Type eltType = resType.getElementType();
1192 bool isInt = isa<IntegerType, IndexType>(Val: eltType);
1193 Value acc = op.getAcc();
1194 vector::CombiningKind kind = op.getKind();
1195
1196 // Vector mask setup.
1197 OpBuilder::InsertionGuard guard(rewriter);
1198 auto maskableOp = cast<vector::MaskableOpInterface>(Val: op.getOperation());
1199 Operation *rootOp;
1200 Value mask;
1201 if (maskableOp.isMasked()) {
1202 rewriter.setInsertionPoint(maskableOp.getMaskingOp());
1203 rootOp = maskableOp.getMaskingOp();
1204 mask = maskableOp.getMaskingOp().getMask();
1205 } else {
1206 rootOp = op;
1207 }
1208
1209 if (!rhsType) {
1210 // Special case: AXPY operation.
1211 Value b =
1212 vector::BroadcastOp::create(builder&: rewriter, location: loc, result: lhsType, source: op.getRhs());
1213 std::optional<Value> mult = createContractArithOp(
1214 loc, x: op.getLhs(), y: b, acc, kind, rewriter, isInt, mask);
1215 if (!mult.has_value())
1216 return failure();
1217 rewriter.replaceOp(op: rootOp, newValues: *mult);
1218 return success();
1219 }
1220
1221 Value result = arith::ConstantOp::create(builder&: rewriter, location: loc, result: resType,
1222 value: rewriter.getZeroAttr(type: resType));
1223 for (int64_t d = 0, e = resType.getDimSize(idx: 0); d < e; ++d) {
1224 Value x = vector::ExtractOp::create(builder&: rewriter, location: loc, source: op.getLhs(), position: d);
1225 Value a = vector::BroadcastOp::create(builder&: rewriter, location: loc, result: rhsType, source: x);
1226 Value r = nullptr;
1227 if (acc)
1228 r = vector::ExtractOp::create(builder&: rewriter, location: loc, source: acc, position: d);
1229 Value extrMask;
1230 if (mask)
1231 extrMask = vector::ExtractOp::create(builder&: rewriter, location: loc, source: mask, position: d);
1232
1233 std::optional<Value> m = createContractArithOp(
1234 loc, x: a, y: op.getRhs(), acc: r, kind, rewriter, isInt, mask: extrMask);
1235 if (!m.has_value())
1236 return failure();
1237 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: *m, dest: result, position: d);
1238 }
1239
1240 rewriter.replaceOp(op: rootOp, newValues: result);
1241 return success();
1242 }
1243};
1244
1245} // namespace
1246
1247void mlir::vector::populateVectorContractLoweringPatterns(
1248 RewritePatternSet &patterns,
1249 VectorContractLowering vectorContractLoweringOption, PatternBenefit benefit,
1250 bool disableOuterProductLowering) {
1251 if (!disableOuterProductLowering)
1252 patterns.add<OuterProductOpLowering>(arg: patterns.getContext(), args&: benefit);
1253 patterns.add<ContractionOpLowering, ContractionOpToOuterProductOpLowering>(
1254 arg&: vectorContractLoweringOption, args: patterns.getContext(), args&: benefit);
1255}
1256
1257void mlir::vector::populateVectorOuterProductLoweringPatterns(
1258 RewritePatternSet &patterns, PatternBenefit benefit) {
1259 patterns.add<OuterProductOpLowering>(arg: patterns.getContext(), args&: benefit);
1260}
1261