1//===- VectorToGPU.cpp - Convert vector to GPU dialect ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements lowering of vector operations to GPU dialect ops.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Conversion/VectorToGPU/VectorToGPU.h"
14
15#include "mlir/Analysis/SliceAnalysis.h"
16#include "mlir/Analysis/TopologicalSortUtils.h"
17#include "mlir/Conversion/VectorToGPUCommon/VectorToGPUUtils.h"
18#include "mlir/Dialect/Affine/IR/AffineOps.h"
19#include "mlir/Dialect/Arith/IR/Arith.h"
20#include "mlir/Dialect/GPU/IR/GPUDialect.h"
21#include "mlir/Dialect/MemRef/IR/MemRef.h"
22#include "mlir/Dialect/SCF/IR/SCF.h"
23#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
24#include "mlir/Dialect/Vector/IR/VectorOps.h"
25#include "mlir/IR/Builders.h"
26#include "mlir/IR/Region.h"
27#include "mlir/Pass/Pass.h"
28#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/TypeSwitch.h"
31#include "llvm/Support/DebugLog.h"
32
33#define DEBUG_TYPE "vector-to-gpu"
34
35namespace mlir {
36#define GEN_PASS_DEF_CONVERTVECTORTOGPUPASS
37#include "mlir/Conversion/Passes.h.inc"
38} // namespace mlir
39
40using namespace mlir;
41
42// Return true if the contract op can be convert to MMA matmul.
43static bool contractSupportsMMAMatrixType(vector::ContractionOp contract) {
44 // gpu.subgroup_mma_compute computes C += A * B, so only the `add` kind
45 // matches.
46 if (!contract.isAddKind())
47 return false;
48
49 using MapList = ArrayRef<ArrayRef<AffineExpr>>;
50 auto infer = [&](MapList m) {
51 return AffineMap::inferFromExprList(exprsList: m, context: contract.getContext());
52 };
53 AffineExpr m, n, k;
54 bindDims(ctx: contract.getContext(), exprs&: m, exprs&: n, exprs&: k);
55 auto iteratorTypes = contract.getIteratorTypes().getValue();
56 if (!(vector::isParallelIterator(attr: iteratorTypes[0]) &&
57 vector::isParallelIterator(attr: iteratorTypes[1]) &&
58 vector::isReductionIterator(attr: iteratorTypes[2])))
59 return false;
60
61 // The contract needs to represent a matmul to be able to convert to
62 // MMAMatrix matmul.
63 if (contract.getIndexingMapsArray() != infer({{m, k}, {k, n}, {m, n}}))
64 return false;
65
66 return true;
67}
68
69// Test whether the permutation map's first result corresponds to its last
70// dimension.
71//
72// In contexts where we only accept maps that have the last (most minor)
73// dimension as exactly one of the two results, this is sufficient to classify
74// whether it represents a transpose.
75static bool isFirstResultLastMapDimension(AffineMap permutationMap) {
76 MLIRContext *ctx = permutationMap.getContext();
77 const unsigned nDim = permutationMap.getNumDims();
78 if (0 == nDim || permutationMap.getResults().empty())
79 return false;
80 return permutationMap.getResult(idx: 0) == getAffineDimExpr(position: nDim - 1, context: ctx);
81}
82
83// Return the `leadDimension` (row stride) implied by |permutationMap| for
84// |type|, if |type| is a memref with a statically-known layout.
85//
86// The `leadDimension` is the stride (in elements) between consecutive rows in
87// the 2D view described by |permutationMap|. This helper supports the subset
88// of maps permitted by vector.transfer_read:
89// - Exactly 2 results.
90// - Each result is either an affine dimension or the constant 0 (broadcast).
91//
92// Constraints:
93// - Requires the most minor memref stride to be 1.
94//
95// Broadcast:
96// - If either result is constant 0, the implied `leadDimension` is 0.
97static std::optional<int64_t>
98getStaticallyKnownRowStride(ShapedType type, AffineMap permutationMap) {
99 auto memrefType = dyn_cast<MemRefType>(Val&: type);
100 if (!memrefType)
101 return std::nullopt;
102 // If the memref is 0 or 1D the horizontal stride is 0.
103 if (memrefType.getRank() < 2)
104 return 0;
105 int64_t offset = 0;
106 SmallVector<int64_t> strides;
107 if (failed(Result: memrefType.getStridesAndOffset(strides, offset)) ||
108 strides.back() != 1)
109 return std::nullopt;
110
111 if (permutationMap.getNumResults() != 2)
112 return std::nullopt;
113
114 unsigned strideIndex = strides.size();
115
116 for (AffineExpr result : permutationMap.getResults()) {
117 if (auto cst = dyn_cast<AffineConstantExpr>(Val&: result)) {
118 // Constant value must be zero.
119 if (0 != cst.getValue())
120 return std::nullopt;
121 // A broadcast result forces row stride to 0.
122 return 0;
123 }
124 auto dim = dyn_cast<AffineDimExpr>(Val&: result);
125 // Only Dim & Const results are supported.
126 if (!dim)
127 return std::nullopt;
128 strideIndex = std::min(a: strideIndex, b: dim.getPosition());
129 }
130
131 // Structural validity check: ensure that the map selects at least one
132 // dimension more major than the most minor dimension. This also excludes
133 // degenerate cases where both results map to the most minor dimension.
134 if (strideIndex + 1 >= strides.size())
135 return std::nullopt;
136
137 const int64_t stride = strides[strideIndex];
138 if (stride == ShapedType::kDynamic)
139 return std::nullopt;
140 return stride;
141}
142
143// Return true if the transfer op can be converted to a MMA matrix load.
144static bool transferReadSupportsMMAMatrixType(vector::TransferReadOp readOp) {
145 if (readOp.getMask() || readOp.hasOutOfBoundsDim() ||
146 readOp.getVectorType().getRank() != 2)
147 return false;
148
149 AffineMap permutationMap = readOp.getPermutationMap();
150 if (!getStaticallyKnownRowStride(type: readOp.getShapedType(), permutationMap))
151 return false;
152
153 // Only allow integer types if the signedness can be inferred.
154 if (readOp.getVectorType().getElementType().isInteger(width: 8))
155 if (!readOp->hasOneUse() || (!isa<arith::ExtSIOp>(Val: *readOp->user_begin()) &&
156 !isa<arith::ExtUIOp>(Val: *readOp->user_begin())))
157 return false;
158
159 MLIRContext *ctx = readOp.getContext();
160 AffineExpr innerDim = getAffineDimExpr(position: permutationMap.getNumDims() - 1, context: ctx);
161 return llvm::is_contained(Range: permutationMap.getResults(), Element: innerDim);
162}
163
164// Return true if the transfer op can be converted to a MMA matrix store.
165static bool
166transferWriteSupportsMMAMatrixType(vector::TransferWriteOp writeOp) {
167 // TODO: support 0-d corner case.
168 if (writeOp.getTransferRank() == 0)
169 return false;
170
171 if (writeOp.getMask() || writeOp.hasOutOfBoundsDim() ||
172 writeOp.getVectorType().getRank() != 2)
173 return false;
174
175 AffineMap permutationMap = writeOp.getPermutationMap();
176 std::optional<int64_t> stride =
177 getStaticallyKnownRowStride(type: writeOp.getShapedType(), permutationMap);
178 // Stride of zero means broadcast which is not permitted for writes.
179 if (!stride.has_value() || stride.value() == 0)
180 return false;
181
182 MLIRContext *ctx = writeOp.getContext();
183 AffineExpr innerDim = getAffineDimExpr(position: permutationMap.getNumDims() - 1, context: ctx);
184 return llvm::is_contained(Range: permutationMap.getResults(), Element: innerDim);
185}
186
187static bool supportsMMaMatrixType(Operation *op) {
188 if (isa<scf::ForOp, scf::YieldOp>(Val: op))
189 return true;
190 // MMA ops cannot honor a mask. An op masked by an enclosing vector.mask has
191 // no mask operand, so the checks below would miss it; rewriting it in place
192 // would leave non-maskable ops inside the vector.mask region.
193 auto maskableOp = dyn_cast<vector::MaskableOpInterface>(Val: op);
194 if (maskableOp && maskableOp.isMasked())
195 return false;
196 if (auto transferRead = dyn_cast<vector::TransferReadOp>(Val: op))
197 return transferReadSupportsMMAMatrixType(readOp: transferRead);
198 if (auto transferWrite = dyn_cast<vector::TransferWriteOp>(Val: op))
199 return transferWriteSupportsMMAMatrixType(writeOp: transferWrite);
200 if (auto contract = dyn_cast<vector::ContractionOp>(Val: op))
201 return contractSupportsMMAMatrixType(contract);
202 if (auto constant = dyn_cast<arith::ConstantOp>(Val: op))
203 return constantSupportsMMAMatrixType(constantOp: constant);
204 if (auto broadcast = dyn_cast<vector::BroadcastOp>(Val: op))
205 return broadcastSupportsMMAMatrixType(broadcastOp: broadcast);
206 if (auto signedExtend = dyn_cast<arith::ExtSIOp>(Val: op))
207 return integerExtendSupportsMMAMatrixType<arith::ExtSIOp>(extOp: signedExtend);
208 if (auto unsignedExtend = dyn_cast<arith::ExtUIOp>(Val: op))
209 return integerExtendSupportsMMAMatrixType<arith::ExtUIOp>(extOp: unsignedExtend);
210 return elementwiseSupportsMMAMatrixType(op);
211}
212
213// Analyze slice of operations based on convert op to figure out if the whole
214// slice can be converted to MMA operations.
215static SetVector<Operation *> getOpToConvert(mlir::Operation *op) {
216 auto hasVectorDest = [](Operation *op) {
217 return llvm::any_of(Range: op->getResultTypes(), P: llvm::IsaPred<VectorType>);
218 };
219 BackwardSliceOptions backwardSliceOptions;
220 backwardSliceOptions.filter = hasVectorDest;
221
222 auto hasVectorSrc = [](Operation *op) {
223 return llvm::any_of(Range: op->getOperandTypes(), P: llvm::IsaPred<VectorType>);
224 };
225 ForwardSliceOptions forwardSliceOptions;
226 forwardSliceOptions.filter = hasVectorSrc;
227
228 DenseMap<Operation *, SmallVector<Operation *>> backwardSliceCache;
229 DenseMap<Operation *, SmallVector<Operation *>> forwardSliceCache;
230 DenseMap<Operation *, bool> supportsMMAMatrixTypeCache;
231
232 auto getCachedBackwardSlice =
233 [&](Operation *currentOp) -> ArrayRef<Operation *> {
234 auto [it, inserted] = backwardSliceCache.try_emplace(Key: currentOp);
235 if (!inserted)
236 return it->second;
237
238 SetVector<Operation *> backwardSlice;
239 LogicalResult result =
240 getBackwardSlice(op: currentOp, backwardSlice: &backwardSlice, options: backwardSliceOptions);
241 assert(result.succeeded() && "expected a backward slice");
242 (void)result;
243 it->second = backwardSlice.takeVector();
244 return it->second;
245 };
246
247 auto getCachedForwardSlice =
248 [&](Operation *currentOp) -> ArrayRef<Operation *> {
249 auto [it, inserted] = forwardSliceCache.try_emplace(Key: currentOp);
250 if (!inserted)
251 return it->second;
252
253 SetVector<Operation *> forwardSlice;
254 // Special case for ForOp, we don't want to include the whole region but
255 // only the value using the region arguments.
256 // TODO: We should refine this to only care about the region arguments being
257 // converted to matrix type.
258 if (auto forOp = dyn_cast<scf::ForOp>(Val: currentOp)) {
259 for (Value forOpResult : forOp.getResults())
260 getForwardSlice(root: forOpResult, forwardSlice: &forwardSlice, options: forwardSliceOptions);
261 for (BlockArgument &arg : forOp.getRegionIterArgs())
262 getForwardSlice(root: arg, forwardSlice: &forwardSlice, options: forwardSliceOptions);
263 } else {
264 getForwardSlice(op: currentOp, forwardSlice: &forwardSlice, options: forwardSliceOptions);
265 }
266 it->second = forwardSlice.takeVector();
267 return it->second;
268 };
269
270 auto cachedSupportsMMAMatrixType = [&](Operation *currentOp) {
271 auto [it, inserted] =
272 supportsMMAMatrixTypeCache.try_emplace(Key: currentOp, Args: false);
273 if (inserted)
274 it->second = supportsMMaMatrixType(op: currentOp);
275 return it->second;
276 };
277
278 SetVector<Operation *> opToConvert;
279 op->walk(callback: [&](Operation *nestedOp) {
280 if (!isa<vector::ContractionOp>(Val: nestedOp) &&
281 !elementwiseSupportsMMAMatrixType(op: nestedOp))
282 return;
283 if (backwardSliceCache.contains(Val: nestedOp))
284 return;
285
286 SetVector<Operation *> dependentOps;
287 dependentOps.insert(X: nestedOp);
288 unsigned currentIndex = 0;
289 while (currentIndex != dependentOps.size()) {
290 Operation *currentOp = dependentOps[currentIndex++];
291 dependentOps.insert_range(R: getCachedBackwardSlice(currentOp));
292 dependentOps.insert_range(R: getCachedForwardSlice(currentOp));
293 }
294 // If any instruction cannot use MMA matrix type drop the whole
295 // chain. MMA matrix are stored in an opaque type so they cannot be used
296 // by all operations.
297 if (llvm::any_of(Range&: dependentOps, P: [&](Operation *op) {
298 if (!cachedSupportsMMAMatrixType(op)) {
299 LDBG() << "cannot convert op: " << *op;
300 return true;
301 }
302 return false;
303 }))
304 return;
305
306 opToConvert.insert_range(R&: dependentOps);
307 });
308 // Sort the operations so that we can convert them in topological order.
309 return topologicalSort(toSort: opToConvert);
310}
311
312namespace {
313// Transform contract into (m, k)x(k, n)x(m, n) form so that it can be converted
314// to MMA matmul.
315struct PrepareContractToGPUMMA
316 : public OpRewritePattern<vector::ContractionOp> {
317 using Base::Base;
318
319 LogicalResult matchAndRewrite(vector::ContractionOp op,
320 PatternRewriter &rewriter) const override {
321 // The mask is not updated: the rewrite may create ops inside the
322 // vector.mask region or swap the m and n iteration dimensions.
323 if (op.isMasked())
324 return rewriter.notifyMatchFailure(arg&: op, msg: "masked contraction");
325
326 Location loc = op.getLoc();
327 Value lhs = op.getLhs(), rhs = op.getRhs(), res = op.getAcc();
328
329 // Set up the parallel/reduction structure in right form.
330 using MapList = ArrayRef<ArrayRef<AffineExpr>>;
331 auto infer = [&](MapList m) {
332 return AffineMap::inferFromExprList(exprsList: m, context: op.getContext());
333 };
334 AffineExpr m, n, k;
335 bindDims(ctx: rewriter.getContext(), exprs&: m, exprs&: n, exprs&: k);
336 static constexpr std::array<int64_t, 2> perm = {1, 0};
337 auto iteratorTypes = op.getIteratorTypes().getValue();
338 SmallVector<AffineMap, 4> maps = op.getIndexingMapsArray();
339 if (!(vector::isParallelIterator(attr: iteratorTypes[0]) &&
340 vector::isParallelIterator(attr: iteratorTypes[1]) &&
341 vector::isReductionIterator(attr: iteratorTypes[2])))
342 return rewriter.notifyMatchFailure(arg&: op, msg: "not a gemm contraction");
343 //
344 // Two outer parallel, one inner reduction (matmat flavor).
345 //
346 // This is the classical row-major matmul, nothing to do.
347 if (maps == infer({{m, k}, {k, n}, {m, n}}))
348 return rewriter.notifyMatchFailure(arg&: op, msg: "contraction already prepared");
349 if (maps == infer({{m, k}, {n, k}, {m, n}})) {
350 rhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: rhs, permutation: perm);
351 } else if (maps == infer({{k, m}, {k, n}, {m, n}})) {
352 lhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: lhs, permutation: perm);
353 } else if (maps == infer({{k, m}, {n, k}, {m, n}})) {
354 rhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: rhs, permutation: perm);
355 lhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: lhs, permutation: perm);
356 } else if (maps == infer({{m, k}, {k, n}, {n, m}})) {
357 std::swap(a&: rhs, b&: lhs);
358 rhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: rhs, permutation: perm);
359 lhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: lhs, permutation: perm);
360 } else if (maps == infer({{m, k}, {n, k}, {n, m}})) {
361 std::swap(a&: rhs, b&: lhs);
362 rhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: rhs, permutation: perm);
363 } else if (maps == infer({{k, m}, {k, n}, {n, m}})) {
364 std::swap(a&: lhs, b&: rhs);
365 lhs = vector::TransposeOp::create(builder&: rewriter, location: loc, vector: lhs, permutation: perm);
366 } else if (maps == infer({{k, m}, {n, k}, {n, m}})) {
367 std::swap(a&: lhs, b&: rhs);
368 } else {
369 // TODO: llvm_unreachable ?
370 return rewriter.notifyMatchFailure(arg&: op, msg: "unexpected contraction case");
371 }
372 rewriter.replaceOpWithNewOp<vector::ContractionOp>(
373 op, args&: lhs, args&: rhs, args&: res,
374 args: rewriter.getAffineMapArrayAttr(values: infer({{m, k}, {k, n}, {m, n}})),
375 args: op.getIteratorTypes(), args: op.getKind(), args: op.getFastmath());
376 return success();
377 }
378};
379} // namespace
380
381// MMA types have different layout based on how they are used in matmul ops.
382// Figure the right layout to use by looking at op uses.
383// TODO: Change the GPU dialect to abstract the layout at the this level and
384// only care about it during lowering to NVVM.
385static const char *inferFragType(Operation *op) {
386 // We can have arith.ext ops before reaching contract ops. See through them
387 // and other kinds of elementwise ops.
388 if (op->hasOneUse()) {
389 Operation *userOp = *op->user_begin();
390 if (userOp->hasTrait<OpTrait::Elementwise>())
391 return inferFragType(op: userOp);
392 }
393
394 for (auto contract :
395 llvm::make_isa_range<vector::ContractionOp>(Range: op->getUsers())) {
396 assert(op->getNumResults() == 1);
397 if (contract.getLhs() == op->getResult(idx: 0))
398 return "AOp";
399 if (contract.getRhs() == op->getResult(idx: 0))
400 return "BOp";
401 }
402 return "COp";
403}
404
405static LogicalResult
406convertTransferReadOp(RewriterBase &rewriter, vector::TransferReadOp op,
407 llvm::DenseMap<Value, Value> &valueMapping) {
408 OpBuilder::InsertionGuard g(rewriter);
409 rewriter.setInsertionPoint(op);
410
411 assert(op.getTransferRank() > 0 && "unexpected 0-d transfer");
412 assert(transferReadSupportsMMAMatrixType(op) &&
413 "expected convertible operation");
414
415 AffineMap permutationMap = op.getPermutationMap();
416 std::optional<int64_t> stride =
417 getStaticallyKnownRowStride(type: op.getShapedType(), permutationMap);
418 if (!stride.has_value()) {
419 LDBG() << "no stride";
420 return rewriter.notifyMatchFailure(arg&: op, msg: "no stride");
421 }
422
423 // transferReadSupportsMMAMatrixType ensures that either of the map results is
424 // the most minor dimension. Under this constraint, whether the map represents
425 // a transposed view can be inferred from whether the first result is the most
426 // minor memref dimension.
427 const bool isTranspose = isFirstResultLastMapDimension(permutationMap);
428
429 Value mappingResult = op.getResult();
430 auto elType = op.getVectorType().getElementType();
431 const char *fragType = inferFragType(op);
432 if (op->hasOneUse()) {
433 auto *user = *op->user_begin();
434 // Infer the signedness of the mma type from the integer extend.
435 if (isa<arith::ExtSIOp, arith::ExtUIOp>(Val: user)) {
436 elType = IntegerType::get(
437 context: op.getContext(), width: cast<IntegerType>(Val&: elType).getWidth(),
438 signedness: isa<arith::ExtSIOp>(Val: user) ? IntegerType::Signed
439 : IntegerType::Unsigned);
440 mappingResult = user->getResult(idx: 0);
441 }
442 }
443 gpu::MMAMatrixType type =
444 gpu::MMAMatrixType::get(shape: op.getVectorType().getShape(), elementType: elType, operand: fragType);
445 Value load = gpu::SubgroupMmaLoadMatrixOp::create(
446 builder&: rewriter, location: op.getLoc(), res: type, srcMemref: op.getBase(), indices: op.getIndices(),
447 leadDimension: rewriter.getIndexAttr(value: *stride),
448 transpose: isTranspose ? rewriter.getUnitAttr() : UnitAttr());
449 valueMapping[mappingResult] = load;
450
451 LDBG() << "transfer read to: " << load;
452 return success();
453}
454
455static LogicalResult
456convertTransferWriteOp(RewriterBase &rewriter, vector::TransferWriteOp op,
457 llvm::DenseMap<Value, Value> &valueMapping) {
458 OpBuilder::InsertionGuard g(rewriter);
459 rewriter.setInsertionPoint(op);
460
461 assert(transferWriteSupportsMMAMatrixType(op));
462 AffineMap permutationMap = op.getPermutationMap();
463 std::optional<int64_t> stride =
464 getStaticallyKnownRowStride(type: op.getShapedType(), permutationMap);
465 if (!stride.has_value()) {
466 LDBG() << "no stride";
467 return rewriter.notifyMatchFailure(arg&: op, msg: "no stride");
468 }
469
470 // As for transfer_read, transferWriteSupportsMMAMatrixType ensures that
471 // either of the map results is the most minor dimension, so the first result
472 // being that dimension means a transposed store.
473 const bool isTranspose = isFirstResultLastMapDimension(permutationMap);
474
475 auto it = valueMapping.find(Val: op.getVector());
476 if (it == valueMapping.end()) {
477 LDBG() << "no mapping";
478 return rewriter.notifyMatchFailure(arg&: op, msg: "no mapping");
479 }
480
481 Value matrix = it->second;
482 auto store = gpu::SubgroupMmaStoreMatrixOp::create(
483 builder&: rewriter, location: op.getLoc(), src: matrix, dstMemref: op.getBase(), indices: op.getIndices(),
484 leadDimension: rewriter.getIndexAttr(value: *stride),
485 transpose: isTranspose ? rewriter.getUnitAttr() : UnitAttr());
486 (void)store;
487
488 LDBG() << "transfer write to: " << store;
489
490 LDBG() << "erase: " << op;
491 rewriter.eraseOp(op);
492 return success();
493}
494
495static LogicalResult
496convertContractOp(RewriterBase &rewriter, vector::ContractionOp op,
497 llvm::DenseMap<Value, Value> &valueMapping) {
498 OpBuilder::InsertionGuard g(rewriter);
499 rewriter.setInsertionPoint(op);
500
501 auto itA = valueMapping.find(Val: op.getLhs());
502 auto itB = valueMapping.find(Val: op.getRhs());
503 auto itC = valueMapping.find(Val: op.getAcc());
504 if (itA == valueMapping.end() || itB == valueMapping.end() ||
505 itC == valueMapping.end())
506 return rewriter.notifyMatchFailure(arg&: op, msg: "no mapping");
507 Value opA = itA->second, opB = itB->second, opC = itC->second;
508 Value matmul = gpu::SubgroupMmaComputeOp::create(builder&: rewriter, location: op.getLoc(),
509 res: opC.getType(), opA, opB, opC,
510 /*a_transpose=*/UnitAttr(),
511 /*b_transpose=*/UnitAttr());
512 valueMapping[op.getResult()] = matmul;
513 return success();
514}
515
516/// Convert a 2D splat ConstantOp to a SubgroupMmaConstantMatrix op.
517static LogicalResult
518convertConstantOp(RewriterBase &rewriter, arith::ConstantOp op,
519 llvm::DenseMap<Value, Value> &valueMapping) {
520 OpBuilder::InsertionGuard g(rewriter);
521 rewriter.setInsertionPoint(op);
522
523 assert(constantSupportsMMAMatrixType(op));
524
525 auto splat =
526 cast<SplatElementsAttr>(Val: op.getValue()).getSplatValue<TypedAttr>();
527 auto scalarConstant =
528 arith::ConstantOp::create(builder&: rewriter, location: op.getLoc(), result: splat.getType(), value: splat);
529 const char *fragType = inferFragType(op);
530 auto vecType = cast<VectorType>(Val: op.getType());
531 gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
532 shape: vecType.getShape(), elementType: vecType.getElementType(), operand: llvm::StringRef(fragType));
533 auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(builder&: rewriter, location: op.getLoc(),
534 res: type, value: scalarConstant);
535 valueMapping[op.getResult()] = matrix;
536 return success();
537}
538
539/// Convert a vector.broadcast from scalar to a SubgroupMmaConstantMatrix op.
540static LogicalResult
541convertBroadcastOp(RewriterBase &rewriter, vector::BroadcastOp op,
542 llvm::DenseMap<Value, Value> &valueMapping) {
543 OpBuilder::InsertionGuard g(rewriter);
544 rewriter.setInsertionPoint(op);
545
546 assert(broadcastSupportsMMAMatrixType(op));
547
548 const char *fragType = inferFragType(op);
549 auto vecType = op.getResultVectorType();
550 gpu::MMAMatrixType type = gpu::MMAMatrixType::get(
551 shape: vecType.getShape(), elementType: vecType.getElementType(), operand: llvm::StringRef(fragType));
552 auto matrix = gpu::SubgroupMmaConstantMatrixOp::create(builder&: rewriter, location: op.getLoc(),
553 res: type, value: op.getSource());
554 valueMapping[op.getResult()] = matrix;
555 return success();
556}
557
558/// Convert an elementwise op to the equivalent elementwise op on MMA matrix.
559static LogicalResult
560convertElementwiseOp(RewriterBase &rewriter, Operation *op,
561 gpu::MMAElementwiseOp opType,
562 llvm::DenseMap<Value, Value> &valueMapping) {
563 OpBuilder::InsertionGuard g(rewriter);
564 rewriter.setInsertionPoint(op);
565
566 SmallVector<Value> matrixOperands;
567 for (Value operand : op->getOperands()) {
568 auto it = valueMapping.find(Val: operand);
569 if (it == valueMapping.end())
570 return rewriter.notifyMatchFailure(arg&: op, msg: "no mapping");
571 matrixOperands.push_back(Elt: it->second);
572 }
573 auto resultType = cast<gpu::MMAMatrixType>(Val: matrixOperands[0].getType());
574 if (opType == gpu::MMAElementwiseOp::EXTF ||
575 opType == gpu::MMAElementwiseOp::TRUNCF) {
576 // The floating point extension and truncation has a different result type.
577 auto vectorType = cast<VectorType>(Val: op->getResultTypes()[0]);
578 resultType = gpu::MMAMatrixType::get(shape: resultType.getShape(),
579 elementType: vectorType.getElementType(),
580 operand: resultType.getOperand());
581 }
582
583 Value newOp = gpu::SubgroupMmaElementwiseOp::create(
584 builder&: rewriter, location: op->getLoc(), res: resultType, args: matrixOperands, opType);
585 valueMapping[op->getResult(idx: 0)] = newOp;
586 return success();
587}
588
589void mlir::populatePrepareVectorToMMAPatterns(RewritePatternSet &patterns) {
590 patterns.add<PrepareContractToGPUMMA>(arg: patterns.getContext());
591 populateVectorTransferReadTransposePatterns(patterns);
592}
593
594LogicalResult mlir::convertVectorToMMAOps(RewriterBase &rewriter,
595 Operation *rootOp) {
596 SetVector<Operation *> ops = getOpToConvert(op: rootOp);
597 llvm::DenseMap<Value, Value> valueMapping;
598
599 auto globalRes = LogicalResult::success();
600 for (Operation *op : ops) {
601 LDBG() << "Process op: " << *op;
602 // Apparently callers do not want to early exit on failure here.
603 auto res = LogicalResult::success();
604 if (auto transferRead = dyn_cast<vector::TransferReadOp>(Val: op)) {
605 res = convertTransferReadOp(rewriter, op: transferRead, valueMapping);
606 } else if (auto transferWrite = dyn_cast<vector::TransferWriteOp>(Val: op)) {
607 res = convertTransferWriteOp(rewriter, op: transferWrite, valueMapping);
608 } else if (auto contractOp = dyn_cast<vector::ContractionOp>(Val: op)) {
609 res = convertContractOp(rewriter, op: contractOp, valueMapping);
610 } else if (auto constantOp = dyn_cast<arith::ConstantOp>(Val: op)) {
611 res = convertConstantOp(rewriter, op: constantOp, valueMapping);
612 } else if (auto broadcastOp = dyn_cast<vector::BroadcastOp>(Val: op)) {
613 res = convertBroadcastOp(rewriter, op: broadcastOp, valueMapping);
614 } else if (auto forOp = dyn_cast<scf::ForOp>(Val: op)) {
615 res = convertForOpForMMA(rewriter, op: forOp, valueMapping);
616 } else if (auto yieldOp = dyn_cast<scf::YieldOp>(Val: op)) {
617 res = convertYieldOpForMMA(rewriter, op: yieldOp, valueMapping);
618 } else if (auto elementwiseType = convertElementwiseOpToMMA(op)) {
619 res = convertElementwiseOp(rewriter, op, opType: *elementwiseType, valueMapping);
620 }
621 if (failed(Result: res))
622 globalRes = failure();
623 }
624 return globalRes;
625}
626
627namespace {
628struct ConvertVectorToGPUPass
629 : public impl::ConvertVectorToGPUPassBase<ConvertVectorToGPUPass> {
630
631 void runOnOperation() override {
632 RewritePatternSet patterns(&getContext());
633 populatePrepareVectorToMMAPatterns(patterns);
634 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns))))
635 return signalPassFailure();
636
637 IRRewriter rewriter(&getContext());
638 (void)convertVectorToMMAOps(rewriter, rootOp: getOperation());
639 }
640};
641} // namespace
642