| 1 | //===- IndexedAccessOpInterfaceImpl.cpp -----------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // Implement IndexedAccessOpInterface on vector dialect operations with |
| 9 | // %memref[%i, %j, ...] operands so generic memref-dialect passes can rewrite |
| 10 | // their base/index pairs. Redundant leading unit vector dimensions are omitted |
| 11 | // from the accessed shape and restored with vector.shape_casts when an alias |
| 12 | // rewrite drops those dimensions. Transfer ops keep their |
| 13 | // VectorTransferOpInterface patterns; gather/scatter have tensor-or-memref |
| 14 | // bases and index-vector operands that do not fit IndexedAccessOpInterface's |
| 15 | // rank-matched index contract. |
| 16 | //===----------------------------------------------------------------------===// |
| 17 | |
| 18 | #include "mlir/Dialect/Vector/Transforms/IndexedAccessOpInterfaceImpl.h" |
| 19 | |
| 20 | #include "mlir/Dialect/MemRef/IR/MemoryAccessOpInterfaces.h" |
| 21 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 22 | #include "mlir/IR/Dialect.h" |
| 23 | #include "mlir/IR/IRMapping.h" |
| 24 | #include "mlir/IR/Operation.h" |
| 25 | #include "mlir/IR/PatternMatch.h" |
| 26 | |
| 27 | #include <type_traits> |
| 28 | |
| 29 | using namespace mlir; |
| 30 | using namespace mlir::memref; |
| 31 | |
| 32 | /// Return true if this op has the memref semantics expected by this model. |
| 33 | template <typename LoadStoreOp> |
| 34 | static bool hasMemrefSemantics(Operation *op) { |
| 35 | return llvm::isa<MemRefType>(cast<LoadStoreOp>(op).getBase().getType()); |
| 36 | } |
| 37 | |
| 38 | /// Return true if this op supports rank-0 vector operands/results. |
| 39 | template <typename LoadStoreOp> |
| 40 | static constexpr bool supportsRankZeroVectorAccess() { |
| 41 | return std::is_same_v<LoadStoreOp, vector::LoadOp> || |
| 42 | std::is_same_v<LoadStoreOp, vector::StoreOp>; |
| 43 | } |
| 44 | |
| 45 | /// Return the number of leading static unit dimensions in `vecTy`. |
| 46 | static unsigned getNumLeadingUnitDims(VectorType vecTy) { |
| 47 | unsigned numLeadingUnitDims = 0; |
| 48 | for (auto [size, scalable] : |
| 49 | llvm::zip_equal(t: vecTy.getShape(), u: vecTy.getScalableDims())) { |
| 50 | if (size != 1 || scalable) |
| 51 | break; |
| 52 | ++numLeadingUnitDims; |
| 53 | } |
| 54 | return numLeadingUnitDims; |
| 55 | } |
| 56 | |
| 57 | /// Return the vector shape whose access strides must be preserved, omitting |
| 58 | /// redundant leading static unit dimensions and marking scalable dimensions as |
| 59 | /// dynamic. If the op cannot access rank-0 vectors, preserve one trailing unit |
| 60 | /// dimension instead of returning an empty shape. |
| 61 | static SmallVector<int64_t> getAccessedVectorShape(VectorType vecTy, |
| 62 | bool supportsRankZero) { |
| 63 | unsigned numLeadingUnitDims = getNumLeadingUnitDims(vecTy); |
| 64 | unsigned rank = static_cast<unsigned>(vecTy.getRank()); |
| 65 | if (!supportsRankZero && numLeadingUnitDims == rank) |
| 66 | --numLeadingUnitDims; |
| 67 | return llvm::map_to_vector( |
| 68 | C: llvm::zip_equal(t: vecTy.getShape().drop_front(N: numLeadingUnitDims), |
| 69 | u: vecTy.getScalableDims().drop_front(N: numLeadingUnitDims)), |
| 70 | F: [](auto dim) { |
| 71 | auto [size, scalable] = dim; |
| 72 | return scalable ? ShapedType::kDynamic : size; |
| 73 | }); |
| 74 | } |
| 75 | |
| 76 | /// Return `vecTy` with `numLeadingDims` dimensions dropped from the front. |
| 77 | static VectorType dropLeadingDims(VectorType vecTy, unsigned numLeadingDims) { |
| 78 | return VectorType::get(shape: vecTy.getShape().drop_front(N: numLeadingDims), |
| 79 | elementType: vecTy.getElementType(), |
| 80 | scalableDims: vecTy.getScalableDims().drop_front(N: numLeadingDims)); |
| 81 | } |
| 82 | |
| 83 | /// Return the shape-cast type for vector operands that match `vecTy`. |
| 84 | static std::optional<VectorType> |
| 85 | getShapeCastTypeForOperand(Value operand, VectorType vecTy, |
| 86 | unsigned numLeadingDims) { |
| 87 | auto operandTy = dyn_cast<VectorType>(Val: operand.getType()); |
| 88 | if (!operandTy || operandTy.getShape() != vecTy.getShape() || |
| 89 | operandTy.getScalableDims() != vecTy.getScalableDims()) |
| 90 | return std::nullopt; |
| 91 | return dropLeadingDims(vecTy: operandTy, numLeadingDims); |
| 92 | } |
| 93 | |
| 94 | namespace { |
| 95 | template <typename LoadStoreOp> |
| 96 | struct VectorLoadStoreLikeOpImpl final |
| 97 | : IndexedAccessOpInterface::ExternalModel< |
| 98 | VectorLoadStoreLikeOpImpl<LoadStoreOp>, LoadStoreOp> { |
| 99 | TypedValue<MemRefType> getAccessedMemref(Operation *op) const { |
| 100 | return cast<LoadStoreOp>(op).getBase(); |
| 101 | } |
| 102 | |
| 103 | Operation::operand_range getIndices(Operation *op) const { |
| 104 | return cast<LoadStoreOp>(op).getIndices(); |
| 105 | } |
| 106 | |
| 107 | SmallVector<int64_t> getAccessedShape(Operation *op) const { |
| 108 | assert(hasMemrefSemantics<LoadStoreOp>(op) && |
| 109 | "expected vector op with memref semantics" ); |
| 110 | return getAccessedVectorShape(cast<LoadStoreOp>(op).getVectorType(), |
| 111 | supportsRankZeroVectorAccess<LoadStoreOp>()); |
| 112 | } |
| 113 | |
| 114 | std::optional<SmallVector<Value>> |
| 115 | updateMemrefAndIndices(Operation *op, RewriterBase &rewriter, Value newMemref, |
| 116 | ValueRange newIndices) const { |
| 117 | assert(hasMemrefSemantics<LoadStoreOp>(op) && |
| 118 | "expected vector op with memref semantics" ); |
| 119 | assert(llvm::isa<MemRefType>(newMemref.getType()) && |
| 120 | "expected replacement memref" ); |
| 121 | |
| 122 | VectorType vecTy = cast<LoadStoreOp>(op).getVectorType(); |
| 123 | if (static_cast<int64_t>(newIndices.size()) >= vecTy.getRank()) { |
| 124 | rewriter.modifyOpInPlace(op, [&]() { |
| 125 | auto concreteOp = cast<LoadStoreOp>(op); |
| 126 | concreteOp.getBaseMutable().assign(newMemref); |
| 127 | concreteOp.getIndicesMutable().assign(newIndices); |
| 128 | }); |
| 129 | return std::nullopt; |
| 130 | } |
| 131 | |
| 132 | unsigned numLeadingDimsToDrop = static_cast<unsigned>( |
| 133 | vecTy.getRank() - static_cast<int64_t>(newIndices.size())); |
| 134 | assert(numLeadingDimsToDrop <= getNumLeadingUnitDims(vecTy) && |
| 135 | "expected only redundant leading unit dimensions to be dropped" ); |
| 136 | |
| 137 | IRMapping dropDimsMap; |
| 138 | for (Value operand : op->getOperands()) { |
| 139 | std::optional<VectorType> castTy = |
| 140 | getShapeCastTypeForOperand(operand, vecTy, numLeadingDims: numLeadingDimsToDrop); |
| 141 | if (!castTy || dropDimsMap.lookupOrNull(from: operand)) |
| 142 | continue; |
| 143 | Value castedOperand = vector::ShapeCastOp::create( |
| 144 | builder&: rewriter, location: operand.getLoc(), result: *castTy, source: operand); |
| 145 | dropDimsMap.map(from: operand, to: castedOperand); |
| 146 | } |
| 147 | |
| 148 | if (op->getNumResults() == 1) { |
| 149 | // Result types cannot be changed in place on the original op because the |
| 150 | // caller replaces it using the returned value. Clone at the lower rank, |
| 151 | // then cast the result back to the original vector type. |
| 152 | VectorType droppedDimsTy = dropLeadingDims(vecTy, numLeadingDims: numLeadingDimsToDrop); |
| 153 | Operation *newOp = rewriter.clone(op&: *op, mapper&: dropDimsMap); |
| 154 | rewriter.modifyOpInPlace(newOp, [&]() { |
| 155 | auto concreteOp = cast<LoadStoreOp>(newOp); |
| 156 | concreteOp.getBaseMutable().assign(newMemref); |
| 157 | concreteOp.getIndicesMutable().assign(newIndices); |
| 158 | newOp->getResult(idx: 0).setType(droppedDimsTy); |
| 159 | }); |
| 160 | Value castBack = vector::ShapeCastOp::create(builder&: rewriter, location: newOp->getLoc(), |
| 161 | result: vecTy, source: newOp->getResult(idx: 0)); |
| 162 | return {{castBack}}; |
| 163 | } |
| 164 | |
| 165 | // Store-like ops have no results to replace, so update their vector |
| 166 | // operands and base/index pair in place. |
| 167 | rewriter.modifyOpInPlace(op, [&]() { |
| 168 | auto concreteOp = cast<LoadStoreOp>(op); |
| 169 | concreteOp.getBaseMutable().assign(newMemref); |
| 170 | concreteOp.getIndicesMutable().assign(newIndices); |
| 171 | for (OpOperand &operand : op->getOpOperands()) { |
| 172 | if (Value replacement = dropDimsMap.lookupOrNull(from: operand.get())) |
| 173 | operand.set(replacement); |
| 174 | } |
| 175 | }); |
| 176 | return std::nullopt; |
| 177 | } |
| 178 | |
| 179 | // TODO: The various load and store operations, at the very least vector.load |
| 180 | // and vector.store, should be taught a starts-in-bounds attribute that would |
| 181 | // let us optimize index generation. |
| 182 | bool hasInboundsIndices(Operation *op) const { |
| 183 | assert(hasMemrefSemantics<LoadStoreOp>(op) && |
| 184 | "expected vector op with memref semantics" ); |
| 185 | return false; |
| 186 | } |
| 187 | }; |
| 188 | |
| 189 | template <typename... Ops> |
| 190 | static void attachLoadStoreLike(MLIRContext *ctx) { |
| 191 | (Ops::template attachInterface<VectorLoadStoreLikeOpImpl<Ops>>(*ctx), ...); |
| 192 | } |
| 193 | |
| 194 | } // namespace |
| 195 | |
| 196 | void mlir::vector::registerIndexedAccessOpInterfaceExternalModels( |
| 197 | DialectRegistry ®istry) { |
| 198 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, vector::VectorDialect *dialect) { |
| 199 | attachLoadStoreLike<vector::LoadOp, vector::StoreOp, vector::MaskedLoadOp, |
| 200 | vector::MaskedStoreOp, vector::ExpandLoadOp, |
| 201 | vector::CompressStoreOp>(ctx); |
| 202 | }); |
| 203 | } |
| 204 | |