| 1 | //===- BufferizableOpInterfaceImpl.cpp - Impl. of BufferizableOpInterface -===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Bufferization/Transforms/BufferizableOpInterfaceImpl.h" |
| 10 | |
| 11 | #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" |
| 12 | #include "mlir/Dialect/Bufferization/IR/Bufferization.h" |
| 13 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 14 | #include "mlir/IR/Dialect.h" |
| 15 | #include "mlir/IR/Operation.h" |
| 16 | |
| 17 | using namespace mlir; |
| 18 | using namespace mlir::bufferization; |
| 19 | |
| 20 | namespace mlir { |
| 21 | namespace bufferization { |
| 22 | namespace { |
| 23 | |
| 24 | struct AllocTensorOpInterface |
| 25 | : public BufferizableOpInterface::ExternalModel<AllocTensorOpInterface, |
| 26 | AllocTensorOp> { |
| 27 | bool bufferizesToAllocation(Operation *op, Value value) const { return true; } |
| 28 | |
| 29 | bool resultBufferizesToMemoryWrite(Operation *op, OpResult opResult, |
| 30 | const AnalysisState &state) const { |
| 31 | // AllocTensorOps do not write unless they have a `copy` value. |
| 32 | return static_cast<bool>(cast<AllocTensorOp>(Val: op).getCopy()); |
| 33 | } |
| 34 | |
| 35 | bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, |
| 36 | const AnalysisState &state) const { |
| 37 | assert(&opOperand == &cast<AllocTensorOp>(op).getCopyMutable()[0] && |
| 38 | "expected copy operand" ); |
| 39 | return true; |
| 40 | } |
| 41 | |
| 42 | bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, |
| 43 | const AnalysisState &state) const { |
| 44 | assert(&opOperand == &cast<AllocTensorOp>(op).getCopyMutable()[0] && |
| 45 | "expected copy operand" ); |
| 46 | return false; |
| 47 | } |
| 48 | |
| 49 | AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand, |
| 50 | const AnalysisState &state) const { |
| 51 | // This is a new allocation. It does not alias with any other buffer. |
| 52 | return {}; |
| 53 | } |
| 54 | |
| 55 | FailureOr<BufferLikeType> |
| 56 | getBufferType(Operation *op, Value value, const BufferizationOptions &options, |
| 57 | const BufferizationState &state, |
| 58 | SmallVector<Value> &invocationStack) const { |
| 59 | auto allocTensorOp = cast<AllocTensorOp>(Val: op); |
| 60 | assert(value == allocTensorOp.getResult() && "invalid value" ); |
| 61 | |
| 62 | FailureOr<BufferLikeType> copyBufferType; |
| 63 | if (allocTensorOp.getCopy()) { |
| 64 | copyBufferType = bufferization::getBufferType( |
| 65 | value: allocTensorOp.getCopy(), options, state, invocationStack); |
| 66 | if (failed(Result: copyBufferType)) |
| 67 | return failure(); |
| 68 | } |
| 69 | |
| 70 | // Compute memory space of this allocation. |
| 71 | Attribute memorySpace; |
| 72 | if (allocTensorOp.getMemorySpace().has_value()) { |
| 73 | memorySpace = *allocTensorOp.getMemorySpace(); |
| 74 | } else if (allocTensorOp.getCopy()) { |
| 75 | // A buffer that is not a memref has no memory space to inherit. |
| 76 | if (auto copyMemRefType = dyn_cast<BaseMemRefType>(Val&: *copyBufferType)) |
| 77 | memorySpace = copyMemRefType.getMemorySpace(); |
| 78 | } else if (auto ms = options.defaultMemorySpaceFn( |
| 79 | cast<TensorLikeType>(Val: allocTensorOp.getType()))) { |
| 80 | memorySpace = *ms; |
| 81 | } else { |
| 82 | return op->emitError(message: "could not infer memory space" ); |
| 83 | } |
| 84 | |
| 85 | // A copy has the shape of the memref it copies, so the sizes `bufferize` |
| 86 | // reads from that memref are the copy's dynamic sizes. |
| 87 | if (allocTensorOp.getCopy()) |
| 88 | if (auto copyMemRefType = dyn_cast<MemRefType>(Val&: *copyBufferType)) |
| 89 | return cast<BufferLikeType>(Val: MemRefType::get( |
| 90 | shape: copyMemRefType.getShape(), elementType: copyMemRefType.getElementType(), |
| 91 | layout: MemRefLayoutAttrInterface{}, memorySpace)); |
| 92 | // Any other result's buffer type is the options' choice. |
| 93 | if (auto tensorType = dyn_cast<RankedTensorType>(Val: allocTensorOp.getType())) |
| 94 | return cast<BufferLikeType>( |
| 95 | Val: getMemRefTypeWithStaticIdentityLayout(tensorType, memorySpace)); |
| 96 | return options.unknownTypeConverterFn(allocTensorOp.getType(), memorySpace, |
| 97 | options); |
| 98 | } |
| 99 | |
| 100 | LogicalResult bufferize(Operation *op, RewriterBase &rewriter, |
| 101 | const BufferizationOptions &options, |
| 102 | BufferizationState &state) const { |
| 103 | auto allocTensorOp = cast<AllocTensorOp>(Val: op); |
| 104 | OpBuilder::InsertionGuard g(rewriter); |
| 105 | Location loc = allocTensorOp.getLoc(); |
| 106 | |
| 107 | // Nothing to do for dead AllocTensorOps. |
| 108 | if (op->getUses().empty()) { |
| 109 | rewriter.eraseOp(op); |
| 110 | return success(); |
| 111 | } |
| 112 | |
| 113 | // Get "copy" buffer. |
| 114 | Value copyBuffer; |
| 115 | if (allocTensorOp.getCopy()) { |
| 116 | FailureOr<Value> maybeCopyBuffer = bufferization::getBuffer( |
| 117 | rewriter, value: allocTensorOp.getCopy(), options, state); |
| 118 | if (failed(Result: maybeCopyBuffer)) |
| 119 | return failure(); |
| 120 | copyBuffer = *maybeCopyBuffer; |
| 121 | } |
| 122 | |
| 123 | // Create memory allocation. |
| 124 | auto allocType = |
| 125 | bufferization::getBufferType(value: allocTensorOp.getResult(), options, state); |
| 126 | if (failed(Result: allocType)) |
| 127 | return failure(); |
| 128 | // TODO: The allocation hook takes a MemRefType; a buffer type that is not |
| 129 | // a memref is diagnosed rather than allocated. |
| 130 | auto memRefType = dyn_cast<MemRefType>(Val&: *allocType); |
| 131 | if (!memRefType) |
| 132 | return op->emitError() << "allocating a buffer of type " << *allocType |
| 133 | << " is not supported" ; |
| 134 | SmallVector<Value> dynamicDims = allocTensorOp.getDynamicSizes(); |
| 135 | if (allocTensorOp.getCopy()) { |
| 136 | assert(dynamicDims.empty() && "expected either `copy` or `dynamicDims`" ); |
| 137 | populateDynamicDimSizes(b&: rewriter, loc, shapedValue: copyBuffer, dynamicDims); |
| 138 | } |
| 139 | // TODO: A result that is not a ranked tensor takes no sizes (verifier), so |
| 140 | // a buffer type with a dynamic dimension is diagnosed rather than |
| 141 | // allocated unless the op copies a value. |
| 142 | if (memRefType.getNumDynamicDims() != dynamicDims.size()) |
| 143 | return op->emitError() |
| 144 | << "allocating " << memRefType |
| 145 | << " needs a size for each dynamic dimension, which the op " |
| 146 | "takes only for a ranked tensor result" ; |
| 147 | FailureOr<Value> alloc = options.allocationFn( |
| 148 | rewriter, loc, memRefType, dynamicDims, options.bufferAlignment); |
| 149 | if (failed(Result: alloc)) |
| 150 | return failure(); |
| 151 | |
| 152 | // Create memory copy (if any). |
| 153 | if (allocTensorOp.getCopy()) { |
| 154 | if (failed(Result: options.memCpyFn(rewriter, loc, copyBuffer, *alloc))) |
| 155 | return failure(); |
| 156 | } |
| 157 | |
| 158 | // Replace op. |
| 159 | replaceOpWithBufferizedValues(rewriter, op, values: *alloc); |
| 160 | |
| 161 | return success(); |
| 162 | } |
| 163 | }; |
| 164 | |
| 165 | struct DeallocTensorOpInterface |
| 166 | : public BufferizableOpInterface::ExternalModel<DeallocTensorOpInterface, |
| 167 | DeallocTensorOp> { |
| 168 | bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, |
| 169 | const AnalysisState &state) const { |
| 170 | return false; |
| 171 | } |
| 172 | |
| 173 | bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, |
| 174 | const AnalysisState &state) const { |
| 175 | return false; |
| 176 | } |
| 177 | |
| 178 | AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand, |
| 179 | const AnalysisState &state) const { |
| 180 | return {}; |
| 181 | } |
| 182 | |
| 183 | LogicalResult bufferize(Operation *op, RewriterBase &rewriter, |
| 184 | const BufferizationOptions &options, |
| 185 | BufferizationState &state) const { |
| 186 | auto deallocTensorOp = cast<DeallocTensorOp>(Val: op); |
| 187 | FailureOr<Value> buffer = bufferization::getBuffer( |
| 188 | rewriter, value: deallocTensorOp.getTensor(), options, state); |
| 189 | if (failed(Result: buffer)) |
| 190 | return failure(); |
| 191 | memref::DeallocOp::create(builder&: rewriter, location: deallocTensorOp.getLoc(), memref: *buffer); |
| 192 | rewriter.eraseOp(op); |
| 193 | return success(); |
| 194 | } |
| 195 | }; |
| 196 | |
| 197 | struct MaterializeInDestinationOpInterface |
| 198 | : public BufferizableOpInterface::ExternalModel< |
| 199 | MaterializeInDestinationOpInterface, MaterializeInDestinationOp> { |
| 200 | bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, |
| 201 | const AnalysisState &state) const { |
| 202 | return opOperand == cast<MaterializeInDestinationOp>(Val: op).getSourceMutable(); |
| 203 | } |
| 204 | |
| 205 | bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, |
| 206 | const AnalysisState &state) const { |
| 207 | auto materializeOp = cast<MaterializeInDestinationOp>(Val: op); |
| 208 | if (opOperand == materializeOp.getDestMutable()) { |
| 209 | assert(isa<TensorType>(materializeOp.getDest().getType()) && |
| 210 | "expected tensor type" ); |
| 211 | return true; |
| 212 | } |
| 213 | return false; |
| 214 | } |
| 215 | |
| 216 | bool mustBufferizeInPlace(Operation *op, OpOperand &opOperand, |
| 217 | const AnalysisState &state) const { |
| 218 | // The source is only read and not written, so it always bufferizes in-place |
| 219 | // by default. The destination is written and is forced to bufferize |
| 220 | // in-place (if it is a tensor). |
| 221 | return true; |
| 222 | } |
| 223 | |
| 224 | AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand, |
| 225 | const AnalysisState &state) const { |
| 226 | auto materializeOp = cast<MaterializeInDestinationOp>(Val: op); |
| 227 | if (opOperand == materializeOp.getDestMutable()) { |
| 228 | assert(isa<TensorType>(materializeOp.getDest().getType()) && |
| 229 | "expected tensor type" ); |
| 230 | return {{op->getResult(idx: 0), BufferRelation::Equivalent}}; |
| 231 | } |
| 232 | return {}; |
| 233 | } |
| 234 | |
| 235 | LogicalResult bufferize(Operation *op, RewriterBase &rewriter, |
| 236 | const BufferizationOptions &options, |
| 237 | BufferizationState &state) const { |
| 238 | auto materializeOp = cast<MaterializeInDestinationOp>(Val: op); |
| 239 | bool tensorDest = isa<TensorType>(Val: materializeOp.getDest().getType()); |
| 240 | Value buffer; |
| 241 | if (tensorDest) { |
| 242 | FailureOr<Value> maybeBuffer = bufferization::getBuffer( |
| 243 | rewriter, value: materializeOp.getDest(), options, state); |
| 244 | if (failed(Result: maybeBuffer)) |
| 245 | return failure(); |
| 246 | buffer = *maybeBuffer; |
| 247 | } else { |
| 248 | assert(isa<BaseMemRefType>(materializeOp.getDest().getType()) && |
| 249 | "expected memref type" ); |
| 250 | buffer = materializeOp.getDest(); |
| 251 | } |
| 252 | auto srcBuffer = bufferization::getBuffer( |
| 253 | rewriter, value: materializeOp.getSource(), options, state); |
| 254 | if (failed(Result: srcBuffer)) |
| 255 | return failure(); |
| 256 | if (failed(Result: options.memCpyFn(rewriter, materializeOp.getLoc(), *srcBuffer, |
| 257 | buffer))) |
| 258 | return failure(); |
| 259 | replaceOpWithBufferizedValues( |
| 260 | rewriter, op, values: tensorDest ? ValueRange(buffer) : ValueRange()); |
| 261 | return success(); |
| 262 | } |
| 263 | |
| 264 | bool bufferizesToElementwiseAccess(Operation *op, const AnalysisState &state, |
| 265 | ArrayRef<OpOperand *> opOperands) const { |
| 266 | // As elements are copied from the "source" buffer to the "dest" buffer, |
| 267 | // already copied elements are not read a second time. |
| 268 | return true; |
| 269 | } |
| 270 | |
| 271 | bool isWritable(Operation *op, Value value, |
| 272 | const AnalysisState &state) const { |
| 273 | auto materializeOp = cast<MaterializeInDestinationOp>(Val: op); |
| 274 | return isa<TensorType>(Val: materializeOp.getDest().getType()) |
| 275 | ? true |
| 276 | : materializeOp.getWritable(); |
| 277 | } |
| 278 | }; |
| 279 | |
| 280 | // Note: ToBufferOp / ToTensorOp are temporary ops that are inserted at the |
| 281 | // bufferization boundary. When One-Shot bufferization is complete, there should |
| 282 | // be no such ops left over. If `allowUnknownOps` (or after running a partial |
| 283 | // bufferization pass), such ops may be part of the resulting IR, but such IR |
| 284 | // may no longer be analyzable by One-Shot analysis. |
| 285 | |
| 286 | struct ToTensorOpInterface |
| 287 | : public BufferizableOpInterface::ExternalModel<ToTensorOpInterface, |
| 288 | ToTensorOp> { |
| 289 | bool isWritable(Operation *op, Value value, |
| 290 | const AnalysisState &state) const { |
| 291 | return cast<ToTensorOp>(Val: op).getWritable(); |
| 292 | } |
| 293 | |
| 294 | FailureOr<BufferLikeType> |
| 295 | getBufferType(Operation *op, Value value, const BufferizationOptions &options, |
| 296 | const BufferizationState &state, |
| 297 | SmallVector<Value> &invocationStack) const { |
| 298 | return cast<ToTensorOp>(Val: op).getBuffer().getType(); |
| 299 | } |
| 300 | |
| 301 | LogicalResult bufferize(Operation *op, RewriterBase &rewriter, |
| 302 | const BufferizationOptions &options, |
| 303 | BufferizationState &state) const { |
| 304 | // to_tensor/to_buffer pairs fold away after bufferization. |
| 305 | return success(); |
| 306 | } |
| 307 | }; |
| 308 | |
| 309 | struct ToBufferOpInterface |
| 310 | : public BufferizableOpInterface::ExternalModel<ToBufferOpInterface, |
| 311 | ToBufferOp> { |
| 312 | bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, |
| 313 | const AnalysisState &state) const { |
| 314 | // It is unknown whether the resulting memref will be read or not. |
| 315 | return true; |
| 316 | } |
| 317 | |
| 318 | bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, |
| 319 | const AnalysisState &state) const { |
| 320 | return !cast<ToBufferOp>(Val: op).getReadOnly(); |
| 321 | } |
| 322 | |
| 323 | AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand, |
| 324 | const AnalysisState &state) const { |
| 325 | return {}; |
| 326 | } |
| 327 | |
| 328 | LogicalResult bufferize(Operation *op, RewriterBase &rewriter, |
| 329 | const BufferizationOptions &options, |
| 330 | BufferizationState &state) const { |
| 331 | // Fold to_buffer(to_tensor(x)) to x. Insert a cast if necessary. |
| 332 | (void)foldToBufferToTensorPair(rewriter, toBuffer: cast<ToBufferOp>(Val: op), options); |
| 333 | // Note: The return value of `bufferize` indicates whether there was an |
| 334 | // error or not. (And not whether the pattern matched or not.) |
| 335 | return success(); |
| 336 | } |
| 337 | }; |
| 338 | |
| 339 | } // namespace |
| 340 | } // namespace bufferization |
| 341 | } // namespace mlir |
| 342 | |
| 343 | void mlir::bufferization::registerBufferizableOpInterfaceExternalModels( |
| 344 | DialectRegistry ®istry) { |
| 345 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, BufferizationDialect *dialect) { |
| 346 | AllocTensorOp::attachInterface<AllocTensorOpInterface>(context&: *ctx); |
| 347 | DeallocTensorOp::attachInterface<DeallocTensorOpInterface>(context&: *ctx); |
| 348 | MaterializeInDestinationOp::attachInterface< |
| 349 | MaterializeInDestinationOpInterface>(context&: *ctx); |
| 350 | ToBufferOp::attachInterface<ToBufferOpInterface>(context&: *ctx); |
| 351 | ToTensorOp::attachInterface<ToTensorOpInterface>(context&: *ctx); |
| 352 | }); |
| 353 | } |
| 354 | |