1//===- VectorToGPUUtils.cpp - Shared vector-to-MMA helpers ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/VectorToGPUCommon/VectorToGPUUtils.h"
10#include "mlir/Dialect/Arith/IR/Arith.h"
11#include "mlir/Dialect/GPU/IR/GPUDialect.h"
12#include "mlir/Dialect/SCF/IR/SCF.h"
13#include "mlir/Dialect/Vector/IR/VectorOps.h"
14#include "mlir/IR/AffineMap.h"
15#include "mlir/IR/Builders.h"
16#include "mlir/IR/PatternMatch.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/TypeSwitch.h"
20#include "llvm/Support/DebugLog.h"
21#include <cstddef>
22#include <tuple>
23#include <utility>
24
25#define DEBUG_TYPE "vector-to-gpu-common"
26
27using namespace mlir;
28
29bool mlir::constantSupportsMMAMatrixType(arith::ConstantOp constantOp) {
30 auto vecType = dyn_cast<VectorType>(Val: constantOp.getType());
31 if (!vecType || vecType.getRank() != 2)
32 return false;
33 return isa<SplatElementsAttr>(Val: constantOp.getValue());
34}
35
36bool mlir::broadcastSupportsMMAMatrixType(vector::BroadcastOp broadcastOp) {
37 return broadcastOp.getResultVectorType().getRank() == 2;
38}
39
40template <typename ExtOpTy>
41bool mlir::integerExtendSupportsMMAMatrixType(ExtOpTy extOp) {
42 auto transferReadOp =
43 extOp.getOperand().template getDefiningOp<vector::TransferReadOp>();
44 if (!transferReadOp)
45 return false;
46 return llvm::all_of(extOp->getUsers(), llvm::IsaPred<vector::ContractionOp>);
47}
48
49template bool
50 mlir::integerExtendSupportsMMAMatrixType<arith::ExtSIOp>(arith::ExtSIOp);
51
52template bool
53 mlir::integerExtendSupportsMMAMatrixType<arith::ExtUIOp>(arith::ExtUIOp);
54
55std::optional<gpu::MMAElementwiseOp>
56mlir::convertElementwiseOpToMMA(Operation *op) {
57 using MMAEwO = gpu::MMAElementwiseOp;
58 return TypeSwitch<Operation *, std::optional<MMAEwO>>(op)
59 .Case(caseFn: [](arith::AddFOp) { return MMAEwO::ADDF; })
60 .Case(caseFn: [](arith::AddIOp) { return MMAEwO::ADDI; })
61 .Case(caseFn: [](arith::DivFOp) { return MMAEwO::DIVF; })
62 .Case(caseFn: [](arith::DivSIOp) { return MMAEwO::DIVS; })
63 .Case(caseFn: [](arith::DivUIOp) { return MMAEwO::DIVU; })
64 .Case(caseFn: [](arith::ExtFOp) { return MMAEwO::EXTF; })
65 .Case(caseFn: [](arith::MaximumFOp) { return MMAEwO::MAXF; })
66 .Case(caseFn: [](arith::MinimumFOp) { return MMAEwO::MINF; })
67 .Case(caseFn: [](arith::MulFOp) { return MMAEwO::MULF; })
68 .Case(caseFn: [](arith::MulIOp) { return MMAEwO::MULI; })
69 .Case(caseFn: [](arith::NegFOp) { return MMAEwO::NEGATEF; })
70 .Case(caseFn: [](arith::SubFOp) { return MMAEwO::SUBF; })
71 .Case(caseFn: [](arith::SubIOp) { return MMAEwO::SUBI; })
72 .Case(caseFn: [](arith::TruncFOp) { return MMAEwO::TRUNCF; })
73 .Default(std::nullopt);
74}
75
76bool mlir::elementwiseSupportsMMAMatrixType(Operation *op) {
77 return convertElementwiseOpToMMA(op).has_value();
78}
79
80// Replace ForOp with a new ForOp with extra operands. The YieldOp is not
81// updated and needs to be updated separately for the loop to be correct.
82static scf::ForOp replaceForOpWithNewSignature(RewriterBase &rewriter,
83 scf::ForOp loop,
84 ValueRange newInitArgs) {
85 OpBuilder::InsertionGuard g(rewriter);
86
87 // Create a new loop before the existing one, with the extra operands.
88 rewriter.setInsertionPoint(loop);
89 auto operands = llvm::to_vector<4>(Range: loop.getInitArgs());
90 llvm::append_range(C&: operands, R&: newInitArgs);
91 scf::ForOp newLoop =
92 scf::ForOp::create(builder&: rewriter, location: loop.getLoc(), lowerBound: loop.getLowerBound(),
93 upperBound: loop.getUpperBound(), step: loop.getStep(), initArgs: operands);
94 rewriter.eraseBlock(block: newLoop.getBody());
95
96 newLoop.getRegion().getBlocks().splice(
97 where: newLoop.getRegion().getBlocks().begin(), L2&: loop.getRegion().getBlocks());
98 for (Value operand : newInitArgs)
99 newLoop.getBody()->addArgument(type: operand.getType(), loc: operand.getLoc());
100
101 for (auto it : llvm::zip(t: loop.getResults(), u: newLoop.getResults().take_front(
102 n: loop.getNumResults())))
103 rewriter.replaceAllUsesWith(from: std::get<0>(t&: it), to: std::get<1>(t&: it));
104
105 LDBG() << "newLoop now: " << newLoop;
106 LDBG() << "stripped scf.for: " << loop;
107 LDBG() << "erase: " << loop;
108
109 rewriter.eraseOp(op: loop);
110 return newLoop;
111}
112
113LogicalResult
114mlir::convertForOpForMMA(RewriterBase &rewriter, scf::ForOp op,
115 llvm::DenseMap<Value, Value> &valueMapping) {
116 OpBuilder::InsertionGuard g(rewriter);
117 rewriter.setInsertionPoint(op);
118
119 SmallVector<Value> newOperands;
120 SmallVector<std::pair<size_t, size_t>> argMapping;
121 for (const auto &operand : llvm::enumerate(First: op.getInitArgs())) {
122 auto it = valueMapping.find(Val: operand.value());
123 if (it == valueMapping.end()) {
124 LDBG() << "no value mapping for: " << operand.value();
125 continue;
126 }
127 argMapping.push_back(Elt: std::make_pair(
128 x: operand.index(), y: op.getInitArgs().size() + newOperands.size()));
129 newOperands.push_back(Elt: it->second);
130 }
131
132 scf::ForOp newForOp = replaceForOpWithNewSignature(rewriter, loop: op, newInitArgs: newOperands);
133 Block &loopBody = *newForOp.getBody();
134 for (auto mapping : argMapping) {
135 valueMapping[newForOp.getResult(i: mapping.first)] =
136 newForOp.getResult(i: mapping.second);
137 valueMapping[loopBody.getArgument(i: mapping.first +
138 newForOp.getNumInductionVars())] =
139 loopBody.getArgument(i: mapping.second + newForOp.getNumInductionVars());
140 }
141
142 LDBG() << "scf.for to: " << newForOp;
143 return success();
144}
145
146LogicalResult
147mlir::convertYieldOpForMMA(RewriterBase &rewriter, scf::YieldOp op,
148 llvm::DenseMap<Value, Value> &valueMapping) {
149 OpBuilder::InsertionGuard g(rewriter);
150 rewriter.setInsertionPoint(op);
151
152 auto loop = cast<scf::ForOp>(Val: op->getParentOp());
153 auto yieldOperands = llvm::to_vector<4>(Range: op.getOperands());
154 for (const auto &operand : llvm::enumerate(First: op.getOperands())) {
155 auto it = valueMapping.find(Val: operand.value());
156 if (it == valueMapping.end())
157 continue;
158 // Replace the yield of old value with the for op argument to make it easier
159 // to remove the dead code.
160 yieldOperands[operand.index()] = loop.getInitArgs()[operand.index()];
161 yieldOperands.push_back(Elt: it->second);
162 }
163 scf::YieldOp::create(builder&: rewriter, location: op.getLoc(), results: yieldOperands);
164
165 LDBG() << "erase: " << op;
166 rewriter.eraseOp(op);
167 return success();
168}
169
170namespace {
171// Fold vector.transpose into vector.transfer_read, including through
172// arithmetic extension operations.
173struct CombineTransferReadOpTranspose final
174 : public OpRewritePattern<vector::TransposeOp> {
175 using Base::Base;
176
177 LogicalResult matchAndRewrite(vector::TransposeOp op,
178 PatternRewriter &rewriter) const override {
179 // Look through integer extend ops.
180 Value source = op.getVector();
181 Type resultType = op.getType();
182 Operation *extOp;
183 if ((extOp = source.getDefiningOp<arith::ExtSIOp>()) ||
184 (extOp = source.getDefiningOp<arith::ExtUIOp>()) ||
185 (extOp = source.getDefiningOp<arith::ExtFOp>())) {
186 source = extOp->getOperand(idx: 0);
187 resultType =
188 VectorType::get(shape: cast<VectorType>(Val&: resultType).getShape(),
189 elementType: cast<VectorType>(Val: source.getType()).getElementType());
190 }
191
192 auto transferReadOp = source.getDefiningOp<vector::TransferReadOp>();
193 if (!transferReadOp)
194 return rewriter.notifyMatchFailure(arg&: op, msg: "no transfer read");
195
196 // TODO: support 0-d corner case.
197 if (transferReadOp.getTransferRank() == 0)
198 return rewriter.notifyMatchFailure(arg&: op, msg: "0-D transfer read");
199
200 if (transferReadOp.getMask() || transferReadOp.hasOutOfBoundsDim())
201 return rewriter.notifyMatchFailure(arg&: op, msg: "not inbounds transfer read");
202
203 AffineMap permutationMap =
204 AffineMap::getPermutationMap(permutation: op.getPermutation(), context: op.getContext());
205 AffineMap newMap =
206 permutationMap.compose(map: transferReadOp.getPermutationMap());
207
208 auto loc = op.getLoc();
209 Value result = vector::TransferReadOp::create(
210 builder&: rewriter, location: loc, result: resultType, base: transferReadOp.getBase(),
211 indices: transferReadOp.getIndices(), permutation_map: AffineMapAttr::get(value: newMap),
212 padding: transferReadOp.getPadding(), mask: transferReadOp.getMask(),
213 in_bounds: transferReadOp.getInBoundsAttr())
214 .getResult();
215
216 // Fuse through the integer extend op.
217 if (extOp) {
218 if (isa<arith::ExtSIOp>(Val: extOp))
219 result = arith::ExtSIOp::create(builder&: rewriter, location: loc, out: op.getType(), in: result)
220 .getResult();
221 else if (isa<arith::ExtUIOp>(Val: extOp))
222 result = arith::ExtUIOp::create(builder&: rewriter, location: loc, out: op.getType(), in: result)
223 .getResult();
224 else
225 result = arith::ExtFOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{op.getType()},
226 operands: ValueRange{result},
227 properties: arith::ExtFOp::Properties{})
228 .getResult();
229 }
230
231 rewriter.replaceOp(op, newValues: result);
232 return success();
233 }
234};
235} // namespace
236
237void mlir::populateVectorTransferReadTransposePatterns(
238 RewritePatternSet &patterns) {
239 patterns.add<CombineTransferReadOpTranspose>(arg: patterns.getContext());
240}
241