| 1 | //===- VectorToGPUUtils.cpp - Shared vector-to-MMA helpers ----------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/VectorToGPUCommon/VectorToGPUUtils.h" |
| 10 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 11 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 12 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 13 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 14 | #include "mlir/IR/AffineMap.h" |
| 15 | #include "mlir/IR/Builders.h" |
| 16 | #include "mlir/IR/PatternMatch.h" |
| 17 | #include "llvm/ADT/STLExtras.h" |
| 18 | #include "llvm/ADT/SmallVector.h" |
| 19 | #include "llvm/ADT/TypeSwitch.h" |
| 20 | #include "llvm/Support/DebugLog.h" |
| 21 | #include <cstddef> |
| 22 | #include <tuple> |
| 23 | #include <utility> |
| 24 | |
| 25 | #define DEBUG_TYPE "vector-to-gpu-common" |
| 26 | |
| 27 | using namespace mlir; |
| 28 | |
| 29 | bool mlir::constantSupportsMMAMatrixType(arith::ConstantOp constantOp) { |
| 30 | auto vecType = dyn_cast<VectorType>(Val: constantOp.getType()); |
| 31 | if (!vecType || vecType.getRank() != 2) |
| 32 | return false; |
| 33 | return isa<SplatElementsAttr>(Val: constantOp.getValue()); |
| 34 | } |
| 35 | |
| 36 | bool mlir::broadcastSupportsMMAMatrixType(vector::BroadcastOp broadcastOp) { |
| 37 | return broadcastOp.getResultVectorType().getRank() == 2; |
| 38 | } |
| 39 | |
| 40 | template <typename ExtOpTy> |
| 41 | bool mlir::integerExtendSupportsMMAMatrixType(ExtOpTy extOp) { |
| 42 | auto transferReadOp = |
| 43 | extOp.getOperand().template getDefiningOp<vector::TransferReadOp>(); |
| 44 | if (!transferReadOp) |
| 45 | return false; |
| 46 | return llvm::all_of(extOp->getUsers(), llvm::IsaPred<vector::ContractionOp>); |
| 47 | } |
| 48 | |
| 49 | template bool |
| 50 | mlir::integerExtendSupportsMMAMatrixType<arith::ExtSIOp>(arith::ExtSIOp); |
| 51 | |
| 52 | template bool |
| 53 | mlir::integerExtendSupportsMMAMatrixType<arith::ExtUIOp>(arith::ExtUIOp); |
| 54 | |
| 55 | std::optional<gpu::MMAElementwiseOp> |
| 56 | mlir::convertElementwiseOpToMMA(Operation *op) { |
| 57 | using MMAEwO = gpu::MMAElementwiseOp; |
| 58 | return TypeSwitch<Operation *, std::optional<MMAEwO>>(op) |
| 59 | .Case(caseFn: [](arith::AddFOp) { return MMAEwO::ADDF; }) |
| 60 | .Case(caseFn: [](arith::AddIOp) { return MMAEwO::ADDI; }) |
| 61 | .Case(caseFn: [](arith::DivFOp) { return MMAEwO::DIVF; }) |
| 62 | .Case(caseFn: [](arith::DivSIOp) { return MMAEwO::DIVS; }) |
| 63 | .Case(caseFn: [](arith::DivUIOp) { return MMAEwO::DIVU; }) |
| 64 | .Case(caseFn: [](arith::ExtFOp) { return MMAEwO::EXTF; }) |
| 65 | .Case(caseFn: [](arith::MaximumFOp) { return MMAEwO::MAXF; }) |
| 66 | .Case(caseFn: [](arith::MinimumFOp) { return MMAEwO::MINF; }) |
| 67 | .Case(caseFn: [](arith::MulFOp) { return MMAEwO::MULF; }) |
| 68 | .Case(caseFn: [](arith::MulIOp) { return MMAEwO::MULI; }) |
| 69 | .Case(caseFn: [](arith::NegFOp) { return MMAEwO::NEGATEF; }) |
| 70 | .Case(caseFn: [](arith::SubFOp) { return MMAEwO::SUBF; }) |
| 71 | .Case(caseFn: [](arith::SubIOp) { return MMAEwO::SUBI; }) |
| 72 | .Case(caseFn: [](arith::TruncFOp) { return MMAEwO::TRUNCF; }) |
| 73 | .Default(std::nullopt); |
| 74 | } |
| 75 | |
| 76 | bool mlir::elementwiseSupportsMMAMatrixType(Operation *op) { |
| 77 | return convertElementwiseOpToMMA(op).has_value(); |
| 78 | } |
| 79 | |
| 80 | // Replace ForOp with a new ForOp with extra operands. The YieldOp is not |
| 81 | // updated and needs to be updated separately for the loop to be correct. |
| 82 | static scf::ForOp replaceForOpWithNewSignature(RewriterBase &rewriter, |
| 83 | scf::ForOp loop, |
| 84 | ValueRange newInitArgs) { |
| 85 | OpBuilder::InsertionGuard g(rewriter); |
| 86 | |
| 87 | // Create a new loop before the existing one, with the extra operands. |
| 88 | rewriter.setInsertionPoint(loop); |
| 89 | auto operands = llvm::to_vector<4>(Range: loop.getInitArgs()); |
| 90 | llvm::append_range(C&: operands, R&: newInitArgs); |
| 91 | scf::ForOp newLoop = |
| 92 | scf::ForOp::create(builder&: rewriter, location: loop.getLoc(), lowerBound: loop.getLowerBound(), |
| 93 | upperBound: loop.getUpperBound(), step: loop.getStep(), initArgs: operands); |
| 94 | rewriter.eraseBlock(block: newLoop.getBody()); |
| 95 | |
| 96 | newLoop.getRegion().getBlocks().splice( |
| 97 | where: newLoop.getRegion().getBlocks().begin(), L2&: loop.getRegion().getBlocks()); |
| 98 | for (Value operand : newInitArgs) |
| 99 | newLoop.getBody()->addArgument(type: operand.getType(), loc: operand.getLoc()); |
| 100 | |
| 101 | for (auto it : llvm::zip(t: loop.getResults(), u: newLoop.getResults().take_front( |
| 102 | n: loop.getNumResults()))) |
| 103 | rewriter.replaceAllUsesWith(from: std::get<0>(t&: it), to: std::get<1>(t&: it)); |
| 104 | |
| 105 | LDBG() << "newLoop now: " << newLoop; |
| 106 | LDBG() << "stripped scf.for: " << loop; |
| 107 | LDBG() << "erase: " << loop; |
| 108 | |
| 109 | rewriter.eraseOp(op: loop); |
| 110 | return newLoop; |
| 111 | } |
| 112 | |
| 113 | LogicalResult |
| 114 | mlir::convertForOpForMMA(RewriterBase &rewriter, scf::ForOp op, |
| 115 | llvm::DenseMap<Value, Value> &valueMapping) { |
| 116 | OpBuilder::InsertionGuard g(rewriter); |
| 117 | rewriter.setInsertionPoint(op); |
| 118 | |
| 119 | SmallVector<Value> newOperands; |
| 120 | SmallVector<std::pair<size_t, size_t>> argMapping; |
| 121 | for (const auto &operand : llvm::enumerate(First: op.getInitArgs())) { |
| 122 | auto it = valueMapping.find(Val: operand.value()); |
| 123 | if (it == valueMapping.end()) { |
| 124 | LDBG() << "no value mapping for: " << operand.value(); |
| 125 | continue; |
| 126 | } |
| 127 | argMapping.push_back(Elt: std::make_pair( |
| 128 | x: operand.index(), y: op.getInitArgs().size() + newOperands.size())); |
| 129 | newOperands.push_back(Elt: it->second); |
| 130 | } |
| 131 | |
| 132 | scf::ForOp newForOp = replaceForOpWithNewSignature(rewriter, loop: op, newInitArgs: newOperands); |
| 133 | Block &loopBody = *newForOp.getBody(); |
| 134 | for (auto mapping : argMapping) { |
| 135 | valueMapping[newForOp.getResult(i: mapping.first)] = |
| 136 | newForOp.getResult(i: mapping.second); |
| 137 | valueMapping[loopBody.getArgument(i: mapping.first + |
| 138 | newForOp.getNumInductionVars())] = |
| 139 | loopBody.getArgument(i: mapping.second + newForOp.getNumInductionVars()); |
| 140 | } |
| 141 | |
| 142 | LDBG() << "scf.for to: " << newForOp; |
| 143 | return success(); |
| 144 | } |
| 145 | |
| 146 | LogicalResult |
| 147 | mlir::convertYieldOpForMMA(RewriterBase &rewriter, scf::YieldOp op, |
| 148 | llvm::DenseMap<Value, Value> &valueMapping) { |
| 149 | OpBuilder::InsertionGuard g(rewriter); |
| 150 | rewriter.setInsertionPoint(op); |
| 151 | |
| 152 | auto loop = cast<scf::ForOp>(Val: op->getParentOp()); |
| 153 | auto yieldOperands = llvm::to_vector<4>(Range: op.getOperands()); |
| 154 | for (const auto &operand : llvm::enumerate(First: op.getOperands())) { |
| 155 | auto it = valueMapping.find(Val: operand.value()); |
| 156 | if (it == valueMapping.end()) |
| 157 | continue; |
| 158 | // Replace the yield of old value with the for op argument to make it easier |
| 159 | // to remove the dead code. |
| 160 | yieldOperands[operand.index()] = loop.getInitArgs()[operand.index()]; |
| 161 | yieldOperands.push_back(Elt: it->second); |
| 162 | } |
| 163 | scf::YieldOp::create(builder&: rewriter, location: op.getLoc(), results: yieldOperands); |
| 164 | |
| 165 | LDBG() << "erase: " << op; |
| 166 | rewriter.eraseOp(op); |
| 167 | return success(); |
| 168 | } |
| 169 | |
| 170 | namespace { |
| 171 | // Fold vector.transpose into vector.transfer_read, including through |
| 172 | // arithmetic extension operations. |
| 173 | struct CombineTransferReadOpTranspose final |
| 174 | : public OpRewritePattern<vector::TransposeOp> { |
| 175 | using Base::Base; |
| 176 | |
| 177 | LogicalResult matchAndRewrite(vector::TransposeOp op, |
| 178 | PatternRewriter &rewriter) const override { |
| 179 | // Look through integer extend ops. |
| 180 | Value source = op.getVector(); |
| 181 | Type resultType = op.getType(); |
| 182 | Operation *extOp; |
| 183 | if ((extOp = source.getDefiningOp<arith::ExtSIOp>()) || |
| 184 | (extOp = source.getDefiningOp<arith::ExtUIOp>()) || |
| 185 | (extOp = source.getDefiningOp<arith::ExtFOp>())) { |
| 186 | source = extOp->getOperand(idx: 0); |
| 187 | resultType = |
| 188 | VectorType::get(shape: cast<VectorType>(Val&: resultType).getShape(), |
| 189 | elementType: cast<VectorType>(Val: source.getType()).getElementType()); |
| 190 | } |
| 191 | |
| 192 | auto transferReadOp = source.getDefiningOp<vector::TransferReadOp>(); |
| 193 | if (!transferReadOp) |
| 194 | return rewriter.notifyMatchFailure(arg&: op, msg: "no transfer read" ); |
| 195 | |
| 196 | // TODO: support 0-d corner case. |
| 197 | if (transferReadOp.getTransferRank() == 0) |
| 198 | return rewriter.notifyMatchFailure(arg&: op, msg: "0-D transfer read" ); |
| 199 | |
| 200 | if (transferReadOp.getMask() || transferReadOp.hasOutOfBoundsDim()) |
| 201 | return rewriter.notifyMatchFailure(arg&: op, msg: "not inbounds transfer read" ); |
| 202 | |
| 203 | AffineMap permutationMap = |
| 204 | AffineMap::getPermutationMap(permutation: op.getPermutation(), context: op.getContext()); |
| 205 | AffineMap newMap = |
| 206 | permutationMap.compose(map: transferReadOp.getPermutationMap()); |
| 207 | |
| 208 | auto loc = op.getLoc(); |
| 209 | Value result = vector::TransferReadOp::create( |
| 210 | builder&: rewriter, location: loc, result: resultType, base: transferReadOp.getBase(), |
| 211 | indices: transferReadOp.getIndices(), permutation_map: AffineMapAttr::get(value: newMap), |
| 212 | padding: transferReadOp.getPadding(), mask: transferReadOp.getMask(), |
| 213 | in_bounds: transferReadOp.getInBoundsAttr()) |
| 214 | .getResult(); |
| 215 | |
| 216 | // Fuse through the integer extend op. |
| 217 | if (extOp) { |
| 218 | if (isa<arith::ExtSIOp>(Val: extOp)) |
| 219 | result = arith::ExtSIOp::create(builder&: rewriter, location: loc, out: op.getType(), in: result) |
| 220 | .getResult(); |
| 221 | else if (isa<arith::ExtUIOp>(Val: extOp)) |
| 222 | result = arith::ExtUIOp::create(builder&: rewriter, location: loc, out: op.getType(), in: result) |
| 223 | .getResult(); |
| 224 | else |
| 225 | result = arith::ExtFOp::create(builder&: rewriter, location: loc, resultTypes: TypeRange{op.getType()}, |
| 226 | operands: ValueRange{result}, |
| 227 | properties: arith::ExtFOp::Properties{}) |
| 228 | .getResult(); |
| 229 | } |
| 230 | |
| 231 | rewriter.replaceOp(op, newValues: result); |
| 232 | return success(); |
| 233 | } |
| 234 | }; |
| 235 | } // namespace |
| 236 | |
| 237 | void mlir::populateVectorTransferReadTransposePatterns( |
| 238 | RewritePatternSet &patterns) { |
| 239 | patterns.add<CombineTransferReadOpTranspose>(arg: patterns.getContext()); |
| 240 | } |
| 241 | |