1//===- IndexedAccessOpInterfaceImpl.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// Implement IndexedAccessOpInterface on vector dialect operations with
9// %memref[%i, %j, ...] operands so generic memref-dialect passes can rewrite
10// their base/index pairs. Redundant leading unit vector dimensions are omitted
11// from the accessed shape and restored with vector.shape_casts when an alias
12// rewrite drops those dimensions. Transfer ops keep their
13// VectorTransferOpInterface patterns; gather/scatter have tensor-or-memref
14// bases and index-vector operands that do not fit IndexedAccessOpInterface's
15// rank-matched index contract.
16//===----------------------------------------------------------------------===//
17
18#include "mlir/Dialect/Vector/Transforms/IndexedAccessOpInterfaceImpl.h"
19
20#include "mlir/Dialect/MemRef/IR/MemoryAccessOpInterfaces.h"
21#include "mlir/Dialect/Vector/IR/VectorOps.h"
22#include "mlir/IR/Dialect.h"
23#include "mlir/IR/IRMapping.h"
24#include "mlir/IR/Operation.h"
25#include "mlir/IR/PatternMatch.h"
26
27#include <type_traits>
28
29using namespace mlir;
30using namespace mlir::memref;
31
32/// Return true if this op has the memref semantics expected by this model.
33template <typename LoadStoreOp>
34static bool hasMemrefSemantics(Operation *op) {
35 return llvm::isa<MemRefType>(cast<LoadStoreOp>(op).getBase().getType());
36}
37
38/// Return true if this op supports rank-0 vector operands/results.
39template <typename LoadStoreOp>
40static constexpr bool supportsRankZeroVectorAccess() {
41 return std::is_same_v<LoadStoreOp, vector::LoadOp> ||
42 std::is_same_v<LoadStoreOp, vector::StoreOp>;
43}
44
45/// Return the number of leading static unit dimensions in `vecTy`.
46static unsigned getNumLeadingUnitDims(VectorType vecTy) {
47 unsigned numLeadingUnitDims = 0;
48 for (auto [size, scalable] :
49 llvm::zip_equal(t: vecTy.getShape(), u: vecTy.getScalableDims())) {
50 if (size != 1 || scalable)
51 break;
52 ++numLeadingUnitDims;
53 }
54 return numLeadingUnitDims;
55}
56
57/// Return the vector shape whose access strides must be preserved, omitting
58/// redundant leading static unit dimensions and marking scalable dimensions as
59/// dynamic. If the op cannot access rank-0 vectors, preserve one trailing unit
60/// dimension instead of returning an empty shape.
61static SmallVector<int64_t> getAccessedVectorShape(VectorType vecTy,
62 bool supportsRankZero) {
63 unsigned numLeadingUnitDims = getNumLeadingUnitDims(vecTy);
64 unsigned rank = static_cast<unsigned>(vecTy.getRank());
65 if (!supportsRankZero && numLeadingUnitDims == rank)
66 --numLeadingUnitDims;
67 return llvm::map_to_vector(
68 C: llvm::zip_equal(t: vecTy.getShape().drop_front(N: numLeadingUnitDims),
69 u: vecTy.getScalableDims().drop_front(N: numLeadingUnitDims)),
70 F: [](auto dim) {
71 auto [size, scalable] = dim;
72 return scalable ? ShapedType::kDynamic : size;
73 });
74}
75
76/// Return `vecTy` with `numLeadingDims` dimensions dropped from the front.
77static VectorType dropLeadingDims(VectorType vecTy, unsigned numLeadingDims) {
78 return VectorType::get(shape: vecTy.getShape().drop_front(N: numLeadingDims),
79 elementType: vecTy.getElementType(),
80 scalableDims: vecTy.getScalableDims().drop_front(N: numLeadingDims));
81}
82
83/// Return the shape-cast type for vector operands that match `vecTy`.
84static std::optional<VectorType>
85getShapeCastTypeForOperand(Value operand, VectorType vecTy,
86 unsigned numLeadingDims) {
87 auto operandTy = dyn_cast<VectorType>(Val: operand.getType());
88 if (!operandTy || operandTy.getShape() != vecTy.getShape() ||
89 operandTy.getScalableDims() != vecTy.getScalableDims())
90 return std::nullopt;
91 return dropLeadingDims(vecTy: operandTy, numLeadingDims);
92}
93
94namespace {
95template <typename LoadStoreOp>
96struct VectorLoadStoreLikeOpImpl final
97 : IndexedAccessOpInterface::ExternalModel<
98 VectorLoadStoreLikeOpImpl<LoadStoreOp>, LoadStoreOp> {
99 TypedValue<MemRefType> getAccessedMemref(Operation *op) const {
100 return cast<LoadStoreOp>(op).getBase();
101 }
102
103 Operation::operand_range getIndices(Operation *op) const {
104 return cast<LoadStoreOp>(op).getIndices();
105 }
106
107 SmallVector<int64_t> getAccessedShape(Operation *op) const {
108 assert(hasMemrefSemantics<LoadStoreOp>(op) &&
109 "expected vector op with memref semantics");
110 return getAccessedVectorShape(cast<LoadStoreOp>(op).getVectorType(),
111 supportsRankZeroVectorAccess<LoadStoreOp>());
112 }
113
114 std::optional<SmallVector<Value>>
115 updateMemrefAndIndices(Operation *op, RewriterBase &rewriter, Value newMemref,
116 ValueRange newIndices) const {
117 assert(hasMemrefSemantics<LoadStoreOp>(op) &&
118 "expected vector op with memref semantics");
119 assert(llvm::isa<MemRefType>(newMemref.getType()) &&
120 "expected replacement memref");
121
122 VectorType vecTy = cast<LoadStoreOp>(op).getVectorType();
123 if (static_cast<int64_t>(newIndices.size()) >= vecTy.getRank()) {
124 rewriter.modifyOpInPlace(op, [&]() {
125 auto concreteOp = cast<LoadStoreOp>(op);
126 concreteOp.getBaseMutable().assign(newMemref);
127 concreteOp.getIndicesMutable().assign(newIndices);
128 });
129 return std::nullopt;
130 }
131
132 unsigned numLeadingDimsToDrop = static_cast<unsigned>(
133 vecTy.getRank() - static_cast<int64_t>(newIndices.size()));
134 assert(numLeadingDimsToDrop <= getNumLeadingUnitDims(vecTy) &&
135 "expected only redundant leading unit dimensions to be dropped");
136
137 IRMapping dropDimsMap;
138 for (Value operand : op->getOperands()) {
139 std::optional<VectorType> castTy =
140 getShapeCastTypeForOperand(operand, vecTy, numLeadingDims: numLeadingDimsToDrop);
141 if (!castTy || dropDimsMap.lookupOrNull(from: operand))
142 continue;
143 Value castedOperand = vector::ShapeCastOp::create(
144 builder&: rewriter, location: operand.getLoc(), result: *castTy, source: operand);
145 dropDimsMap.map(from: operand, to: castedOperand);
146 }
147
148 if (op->getNumResults() == 1) {
149 // Result types cannot be changed in place on the original op because the
150 // caller replaces it using the returned value. Clone at the lower rank,
151 // then cast the result back to the original vector type.
152 VectorType droppedDimsTy = dropLeadingDims(vecTy, numLeadingDims: numLeadingDimsToDrop);
153 Operation *newOp = rewriter.clone(op&: *op, mapper&: dropDimsMap);
154 rewriter.modifyOpInPlace(newOp, [&]() {
155 auto concreteOp = cast<LoadStoreOp>(newOp);
156 concreteOp.getBaseMutable().assign(newMemref);
157 concreteOp.getIndicesMutable().assign(newIndices);
158 newOp->getResult(idx: 0).setType(droppedDimsTy);
159 });
160 Value castBack = vector::ShapeCastOp::create(builder&: rewriter, location: newOp->getLoc(),
161 result: vecTy, source: newOp->getResult(idx: 0));
162 return {{castBack}};
163 }
164
165 // Store-like ops have no results to replace, so update their vector
166 // operands and base/index pair in place.
167 rewriter.modifyOpInPlace(op, [&]() {
168 auto concreteOp = cast<LoadStoreOp>(op);
169 concreteOp.getBaseMutable().assign(newMemref);
170 concreteOp.getIndicesMutable().assign(newIndices);
171 for (OpOperand &operand : op->getOpOperands()) {
172 if (Value replacement = dropDimsMap.lookupOrNull(from: operand.get()))
173 operand.set(replacement);
174 }
175 });
176 return std::nullopt;
177 }
178
179 // TODO: The various load and store operations, at the very least vector.load
180 // and vector.store, should be taught a starts-in-bounds attribute that would
181 // let us optimize index generation.
182 bool hasInboundsIndices(Operation *op) const {
183 assert(hasMemrefSemantics<LoadStoreOp>(op) &&
184 "expected vector op with memref semantics");
185 return false;
186 }
187};
188
189template <typename... Ops>
190static void attachLoadStoreLike(MLIRContext *ctx) {
191 (Ops::template attachInterface<VectorLoadStoreLikeOpImpl<Ops>>(*ctx), ...);
192}
193
194} // namespace
195
196void mlir::vector::registerIndexedAccessOpInterfaceExternalModels(
197 DialectRegistry &registry) {
198 registry.addExtension(extensionFn: +[](MLIRContext *ctx, vector::VectorDialect *dialect) {
199 attachLoadStoreLike<vector::LoadOp, vector::StoreOp, vector::MaskedLoadOp,
200 vector::MaskedStoreOp, vector::ExpandLoadOp,
201 vector::CompressStoreOp>(ctx);
202 });
203}
204