1//===- BufferizableOpInterfaceImpl.cpp - Impl. of BufferizableOpInterface -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Bufferization/Transforms/BufferizableOpInterfaceImpl.h"
10
11#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
12#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
13#include "mlir/Dialect/MemRef/IR/MemRef.h"
14#include "mlir/IR/Dialect.h"
15#include "mlir/IR/Operation.h"
16
17using namespace mlir;
18using namespace mlir::bufferization;
19
20namespace mlir {
21namespace bufferization {
22namespace {
23
24struct AllocTensorOpInterface
25 : public BufferizableOpInterface::ExternalModel<AllocTensorOpInterface,
26 AllocTensorOp> {
27 bool bufferizesToAllocation(Operation *op, Value value) const { return true; }
28
29 bool resultBufferizesToMemoryWrite(Operation *op, OpResult opResult,
30 const AnalysisState &state) const {
31 // AllocTensorOps do not write unless they have a `copy` value.
32 return static_cast<bool>(cast<AllocTensorOp>(Val: op).getCopy());
33 }
34
35 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
36 const AnalysisState &state) const {
37 assert(&opOperand == &cast<AllocTensorOp>(op).getCopyMutable()[0] &&
38 "expected copy operand");
39 return true;
40 }
41
42 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
43 const AnalysisState &state) const {
44 assert(&opOperand == &cast<AllocTensorOp>(op).getCopyMutable()[0] &&
45 "expected copy operand");
46 return false;
47 }
48
49 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
50 const AnalysisState &state) const {
51 // This is a new allocation. It does not alias with any other buffer.
52 return {};
53 }
54
55 FailureOr<BufferLikeType>
56 getBufferType(Operation *op, Value value, const BufferizationOptions &options,
57 const BufferizationState &state,
58 SmallVector<Value> &invocationStack) const {
59 auto allocTensorOp = cast<AllocTensorOp>(Val: op);
60 assert(value == allocTensorOp.getResult() && "invalid value");
61
62 FailureOr<BufferLikeType> copyBufferType;
63 if (allocTensorOp.getCopy()) {
64 copyBufferType = bufferization::getBufferType(
65 value: allocTensorOp.getCopy(), options, state, invocationStack);
66 if (failed(Result: copyBufferType))
67 return failure();
68 }
69
70 // Compute memory space of this allocation.
71 Attribute memorySpace;
72 if (allocTensorOp.getMemorySpace().has_value()) {
73 memorySpace = *allocTensorOp.getMemorySpace();
74 } else if (allocTensorOp.getCopy()) {
75 // A buffer that is not a memref has no memory space to inherit.
76 if (auto copyMemRefType = dyn_cast<BaseMemRefType>(Val&: *copyBufferType))
77 memorySpace = copyMemRefType.getMemorySpace();
78 } else if (auto ms = options.defaultMemorySpaceFn(
79 cast<TensorLikeType>(Val: allocTensorOp.getType()))) {
80 memorySpace = *ms;
81 } else {
82 return op->emitError(message: "could not infer memory space");
83 }
84
85 // A copy has the shape of the memref it copies, so the sizes `bufferize`
86 // reads from that memref are the copy's dynamic sizes.
87 if (allocTensorOp.getCopy())
88 if (auto copyMemRefType = dyn_cast<MemRefType>(Val&: *copyBufferType))
89 return cast<BufferLikeType>(Val: MemRefType::get(
90 shape: copyMemRefType.getShape(), elementType: copyMemRefType.getElementType(),
91 layout: MemRefLayoutAttrInterface{}, memorySpace));
92 // Any other result's buffer type is the options' choice.
93 if (auto tensorType = dyn_cast<RankedTensorType>(Val: allocTensorOp.getType()))
94 return cast<BufferLikeType>(
95 Val: getMemRefTypeWithStaticIdentityLayout(tensorType, memorySpace));
96 return options.unknownTypeConverterFn(allocTensorOp.getType(), memorySpace,
97 options);
98 }
99
100 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
101 const BufferizationOptions &options,
102 BufferizationState &state) const {
103 auto allocTensorOp = cast<AllocTensorOp>(Val: op);
104 OpBuilder::InsertionGuard g(rewriter);
105 Location loc = allocTensorOp.getLoc();
106
107 // Nothing to do for dead AllocTensorOps.
108 if (op->getUses().empty()) {
109 rewriter.eraseOp(op);
110 return success();
111 }
112
113 // Get "copy" buffer.
114 Value copyBuffer;
115 if (allocTensorOp.getCopy()) {
116 FailureOr<Value> maybeCopyBuffer = bufferization::getBuffer(
117 rewriter, value: allocTensorOp.getCopy(), options, state);
118 if (failed(Result: maybeCopyBuffer))
119 return failure();
120 copyBuffer = *maybeCopyBuffer;
121 }
122
123 // Create memory allocation.
124 auto allocType =
125 bufferization::getBufferType(value: allocTensorOp.getResult(), options, state);
126 if (failed(Result: allocType))
127 return failure();
128 // TODO: The allocation hook takes a MemRefType; a buffer type that is not
129 // a memref is diagnosed rather than allocated.
130 auto memRefType = dyn_cast<MemRefType>(Val&: *allocType);
131 if (!memRefType)
132 return op->emitError() << "allocating a buffer of type " << *allocType
133 << " is not supported";
134 SmallVector<Value> dynamicDims = allocTensorOp.getDynamicSizes();
135 if (allocTensorOp.getCopy()) {
136 assert(dynamicDims.empty() && "expected either `copy` or `dynamicDims`");
137 populateDynamicDimSizes(b&: rewriter, loc, shapedValue: copyBuffer, dynamicDims);
138 }
139 // TODO: A result that is not a ranked tensor takes no sizes (verifier), so
140 // a buffer type with a dynamic dimension is diagnosed rather than
141 // allocated unless the op copies a value.
142 if (memRefType.getNumDynamicDims() != dynamicDims.size())
143 return op->emitError()
144 << "allocating " << memRefType
145 << " needs a size for each dynamic dimension, which the op "
146 "takes only for a ranked tensor result";
147 FailureOr<Value> alloc = options.allocationFn(
148 rewriter, loc, memRefType, dynamicDims, options.bufferAlignment);
149 if (failed(Result: alloc))
150 return failure();
151
152 // Create memory copy (if any).
153 if (allocTensorOp.getCopy()) {
154 if (failed(Result: options.memCpyFn(rewriter, loc, copyBuffer, *alloc)))
155 return failure();
156 }
157
158 // Replace op.
159 replaceOpWithBufferizedValues(rewriter, op, values: *alloc);
160
161 return success();
162 }
163};
164
165struct DeallocTensorOpInterface
166 : public BufferizableOpInterface::ExternalModel<DeallocTensorOpInterface,
167 DeallocTensorOp> {
168 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
169 const AnalysisState &state) const {
170 return false;
171 }
172
173 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
174 const AnalysisState &state) const {
175 return false;
176 }
177
178 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
179 const AnalysisState &state) const {
180 return {};
181 }
182
183 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
184 const BufferizationOptions &options,
185 BufferizationState &state) const {
186 auto deallocTensorOp = cast<DeallocTensorOp>(Val: op);
187 FailureOr<Value> buffer = bufferization::getBuffer(
188 rewriter, value: deallocTensorOp.getTensor(), options, state);
189 if (failed(Result: buffer))
190 return failure();
191 memref::DeallocOp::create(builder&: rewriter, location: deallocTensorOp.getLoc(), memref: *buffer);
192 rewriter.eraseOp(op);
193 return success();
194 }
195};
196
197struct MaterializeInDestinationOpInterface
198 : public BufferizableOpInterface::ExternalModel<
199 MaterializeInDestinationOpInterface, MaterializeInDestinationOp> {
200 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
201 const AnalysisState &state) const {
202 return opOperand == cast<MaterializeInDestinationOp>(Val: op).getSourceMutable();
203 }
204
205 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
206 const AnalysisState &state) const {
207 auto materializeOp = cast<MaterializeInDestinationOp>(Val: op);
208 if (opOperand == materializeOp.getDestMutable()) {
209 assert(isa<TensorType>(materializeOp.getDest().getType()) &&
210 "expected tensor type");
211 return true;
212 }
213 return false;
214 }
215
216 bool mustBufferizeInPlace(Operation *op, OpOperand &opOperand,
217 const AnalysisState &state) const {
218 // The source is only read and not written, so it always bufferizes in-place
219 // by default. The destination is written and is forced to bufferize
220 // in-place (if it is a tensor).
221 return true;
222 }
223
224 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
225 const AnalysisState &state) const {
226 auto materializeOp = cast<MaterializeInDestinationOp>(Val: op);
227 if (opOperand == materializeOp.getDestMutable()) {
228 assert(isa<TensorType>(materializeOp.getDest().getType()) &&
229 "expected tensor type");
230 return {{op->getResult(idx: 0), BufferRelation::Equivalent}};
231 }
232 return {};
233 }
234
235 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
236 const BufferizationOptions &options,
237 BufferizationState &state) const {
238 auto materializeOp = cast<MaterializeInDestinationOp>(Val: op);
239 bool tensorDest = isa<TensorType>(Val: materializeOp.getDest().getType());
240 Value buffer;
241 if (tensorDest) {
242 FailureOr<Value> maybeBuffer = bufferization::getBuffer(
243 rewriter, value: materializeOp.getDest(), options, state);
244 if (failed(Result: maybeBuffer))
245 return failure();
246 buffer = *maybeBuffer;
247 } else {
248 assert(isa<BaseMemRefType>(materializeOp.getDest().getType()) &&
249 "expected memref type");
250 buffer = materializeOp.getDest();
251 }
252 auto srcBuffer = bufferization::getBuffer(
253 rewriter, value: materializeOp.getSource(), options, state);
254 if (failed(Result: srcBuffer))
255 return failure();
256 if (failed(Result: options.memCpyFn(rewriter, materializeOp.getLoc(), *srcBuffer,
257 buffer)))
258 return failure();
259 replaceOpWithBufferizedValues(
260 rewriter, op, values: tensorDest ? ValueRange(buffer) : ValueRange());
261 return success();
262 }
263
264 bool bufferizesToElementwiseAccess(Operation *op, const AnalysisState &state,
265 ArrayRef<OpOperand *> opOperands) const {
266 // As elements are copied from the "source" buffer to the "dest" buffer,
267 // already copied elements are not read a second time.
268 return true;
269 }
270
271 bool isWritable(Operation *op, Value value,
272 const AnalysisState &state) const {
273 auto materializeOp = cast<MaterializeInDestinationOp>(Val: op);
274 return isa<TensorType>(Val: materializeOp.getDest().getType())
275 ? true
276 : materializeOp.getWritable();
277 }
278};
279
280// Note: ToBufferOp / ToTensorOp are temporary ops that are inserted at the
281// bufferization boundary. When One-Shot bufferization is complete, there should
282// be no such ops left over. If `allowUnknownOps` (or after running a partial
283// bufferization pass), such ops may be part of the resulting IR, but such IR
284// may no longer be analyzable by One-Shot analysis.
285
286struct ToTensorOpInterface
287 : public BufferizableOpInterface::ExternalModel<ToTensorOpInterface,
288 ToTensorOp> {
289 bool isWritable(Operation *op, Value value,
290 const AnalysisState &state) const {
291 return cast<ToTensorOp>(Val: op).getWritable();
292 }
293
294 FailureOr<BufferLikeType>
295 getBufferType(Operation *op, Value value, const BufferizationOptions &options,
296 const BufferizationState &state,
297 SmallVector<Value> &invocationStack) const {
298 return cast<ToTensorOp>(Val: op).getBuffer().getType();
299 }
300
301 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
302 const BufferizationOptions &options,
303 BufferizationState &state) const {
304 // to_tensor/to_buffer pairs fold away after bufferization.
305 return success();
306 }
307};
308
309struct ToBufferOpInterface
310 : public BufferizableOpInterface::ExternalModel<ToBufferOpInterface,
311 ToBufferOp> {
312 bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand,
313 const AnalysisState &state) const {
314 // It is unknown whether the resulting memref will be read or not.
315 return true;
316 }
317
318 bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand,
319 const AnalysisState &state) const {
320 return !cast<ToBufferOp>(Val: op).getReadOnly();
321 }
322
323 AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand,
324 const AnalysisState &state) const {
325 return {};
326 }
327
328 LogicalResult bufferize(Operation *op, RewriterBase &rewriter,
329 const BufferizationOptions &options,
330 BufferizationState &state) const {
331 // Fold to_buffer(to_tensor(x)) to x. Insert a cast if necessary.
332 (void)foldToBufferToTensorPair(rewriter, toBuffer: cast<ToBufferOp>(Val: op), options);
333 // Note: The return value of `bufferize` indicates whether there was an
334 // error or not. (And not whether the pattern matched or not.)
335 return success();
336 }
337};
338
339} // namespace
340} // namespace bufferization
341} // namespace mlir
342
343void mlir::bufferization::registerBufferizableOpInterfaceExternalModels(
344 DialectRegistry &registry) {
345 registry.addExtension(extensionFn: +[](MLIRContext *ctx, BufferizationDialect *dialect) {
346 AllocTensorOp::attachInterface<AllocTensorOpInterface>(context&: *ctx);
347 DeallocTensorOp::attachInterface<DeallocTensorOpInterface>(context&: *ctx);
348 MaterializeInDestinationOp::attachInterface<
349 MaterializeInDestinationOpInterface>(context&: *ctx);
350 ToBufferOp::attachInterface<ToBufferOpInterface>(context&: *ctx);
351 ToTensorOp::attachInterface<ToTensorOpInterface>(context&: *ctx);
352 });
353}
354