| 1 | //===----------------------------------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 10 | #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" |
| 11 | #include "mlir/Dialect/Bufferization/IR/Bufferization.h" |
| 12 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 13 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 14 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 15 | #include "mlir/Dialect/Utils/VerificationUtils.h" |
| 16 | #include "mlir/IR/Matchers.h" |
| 17 | #include "llvm/ADT/SmallVectorExtras.h" |
| 18 | #include <optional> |
| 19 | |
| 20 | using namespace mlir; |
| 21 | using namespace mlir::bufferization; |
| 22 | |
| 23 | //===----------------------------------------------------------------------===// |
| 24 | // Helper functions |
| 25 | //===----------------------------------------------------------------------===// |
| 26 | |
| 27 | FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue( |
| 28 | OpBuilder &b, Value value, MemRefType destType, |
| 29 | const BufferizationOptions &options) { |
| 30 | auto srcType = llvm::cast<MemRefType>(Val: value.getType()); |
| 31 | |
| 32 | // Element type and rank must match. |
| 33 | if (srcType.getElementType() != destType.getElementType()) |
| 34 | return failure(); |
| 35 | if (srcType.getRank() != destType.getRank()) |
| 36 | return failure(); |
| 37 | |
| 38 | // In case the affine maps are different, we may need to use a copy if we go |
| 39 | // from dynamic to static offset or stride (the canonicalization cannot know |
| 40 | // at this point that it is really cast compatible). |
| 41 | auto isGuaranteedCastCompatible = [](MemRefType source, MemRefType target) { |
| 42 | int64_t sourceOffset, targetOffset; |
| 43 | SmallVector<int64_t, 4> sourceStrides, targetStrides; |
| 44 | if (failed(Result: source.getStridesAndOffset(strides&: sourceStrides, offset&: sourceOffset)) || |
| 45 | failed(Result: target.getStridesAndOffset(strides&: targetStrides, offset&: targetOffset))) |
| 46 | return false; |
| 47 | auto dynamicToStatic = [](int64_t a, int64_t b) { |
| 48 | return ShapedType::isDynamic(dValue: a) && ShapedType::isStatic(dValue: b); |
| 49 | }; |
| 50 | if (dynamicToStatic(sourceOffset, targetOffset)) |
| 51 | return false; |
| 52 | for (auto it : zip(t&: sourceStrides, u&: targetStrides)) |
| 53 | if (dynamicToStatic(std::get<0>(t&: it), std::get<1>(t&: it))) |
| 54 | return false; |
| 55 | // A cast cannot safely zero out a non-zero offset. If the source has a |
| 56 | // non-identity layout (e.g., a subview with a dynamic offset) and the |
| 57 | // destination requires identity layout (zero offset, unit strides), a cast |
| 58 | // would silently strip the offset and cause incorrect loads in the callee. |
| 59 | // Fall through to the alloc+copy path instead. |
| 60 | if (!source.getLayout().isIdentity() && target.getLayout().isIdentity()) |
| 61 | return false; |
| 62 | return true; |
| 63 | }; |
| 64 | |
| 65 | // Note: If `areCastCompatible`, a cast is valid, but may fail at runtime. To |
| 66 | // ensure that we only generate casts that always succeed at runtime, we check |
| 67 | // a few extra conditions in `isGuaranteedCastCompatible`. |
| 68 | if (memref::CastOp::areCastCompatible(inputs: srcType, outputs: destType) && |
| 69 | isGuaranteedCastCompatible(srcType, destType)) { |
| 70 | Value casted = *options.castFn(b, value.getLoc(), destType, value); |
| 71 | return casted; |
| 72 | } |
| 73 | |
| 74 | auto loc = value.getLoc(); |
| 75 | SmallVector<Value, 4> dynamicOperands; |
| 76 | for (int i = 0; i < destType.getRank(); ++i) { |
| 77 | if (destType.getShape()[i] != ShapedType::kDynamic) |
| 78 | continue; |
| 79 | Value size = memref::DimOp::create(builder&: b, location: loc, source: value, index: i); |
| 80 | dynamicOperands.push_back(Elt: size); |
| 81 | } |
| 82 | |
| 83 | FailureOr<Value> copy = options.allocationFn( |
| 84 | b, loc, destType, dynamicOperands, options.bufferAlignment); |
| 85 | if (failed(Result: copy)) |
| 86 | return failure(); |
| 87 | if (failed(Result: options.memCpyFn(b, loc, value, *copy))) |
| 88 | return failure(); |
| 89 | return copy; |
| 90 | } |
| 91 | |
| 92 | /// Try to fold to_buffer(to_tensor(x)). If x's type and the result type of the |
| 93 | /// to_buffer op are different, a memref.cast is needed. |
| 94 | LogicalResult mlir::bufferization::foldToBufferToTensorPair( |
| 95 | RewriterBase &rewriter, ToBufferOp toBuffer, |
| 96 | const BufferizationOptions &options) { |
| 97 | auto bufferToTensor = toBuffer.getTensor().getDefiningOp<ToTensorOp>(); |
| 98 | if (!bufferToTensor) |
| 99 | return failure(); |
| 100 | |
| 101 | Type srcType = bufferToTensor.getBuffer().getType(); |
| 102 | Type destType = toBuffer.getType(); |
| 103 | |
| 104 | // Directly rewrite if the type did not change. |
| 105 | if (srcType == destType) { |
| 106 | rewriter.replaceOp(op: toBuffer, newValues: bufferToTensor.getBuffer()); |
| 107 | return success(); |
| 108 | } |
| 109 | |
| 110 | if (!llvm::isa<BaseMemRefType>(Val: srcType) || |
| 111 | !llvm::isa<BaseMemRefType>(Val: destType)) { |
| 112 | // Non-builtin case: the best is to try the user-provided cast. |
| 113 | auto replacement = |
| 114 | options.castFn(rewriter, bufferToTensor.getBuffer().getLoc(), destType, |
| 115 | bufferToTensor.getBuffer()); |
| 116 | if (failed(Result: replacement)) |
| 117 | return failure(); |
| 118 | rewriter.replaceOp(op: toBuffer, newValues: *replacement); |
| 119 | return success(); |
| 120 | } |
| 121 | |
| 122 | auto rankedSrcType = llvm::dyn_cast<MemRefType>(Val&: srcType); |
| 123 | auto rankedDestType = llvm::dyn_cast<MemRefType>(Val&: destType); |
| 124 | auto unrankedSrcType = llvm::dyn_cast<UnrankedMemRefType>(Val&: srcType); |
| 125 | |
| 126 | // Ranked memref -> Ranked memref cast. |
| 127 | if (rankedSrcType && rankedDestType) { |
| 128 | FailureOr<Value> replacement = castOrReallocMemRefValue( |
| 129 | b&: rewriter, value: bufferToTensor.getBuffer(), destType: rankedDestType, options); |
| 130 | if (failed(Result: replacement)) |
| 131 | return failure(); |
| 132 | |
| 133 | rewriter.replaceOp(op: toBuffer, newValues: *replacement); |
| 134 | return success(); |
| 135 | } |
| 136 | |
| 137 | // Unranked memref -> Ranked memref cast: May require a copy. |
| 138 | // TODO: Not implemented at the moment. |
| 139 | if (unrankedSrcType && rankedDestType) |
| 140 | return failure(); |
| 141 | |
| 142 | // Unranked/ranked memref -> unranked memref cast: No copy needed if the types |
| 143 | // are cast-compatible. |
| 144 | if (!memref::CastOp::areCastCompatible(inputs: srcType, outputs: destType)) |
| 145 | return failure(); |
| 146 | |
| 147 | rewriter.replaceOpWithNewOp<memref::CastOp>(op: toBuffer, args&: destType, |
| 148 | args: bufferToTensor.getBuffer()); |
| 149 | return success(); |
| 150 | } |
| 151 | |
| 152 | void mlir::bufferization::populateDynamicDimSizes( |
| 153 | OpBuilder &b, Location loc, Value shapedValue, |
| 154 | SmallVector<Value> &dynamicDims) { |
| 155 | auto shapedType = llvm::cast<ShapedType>(Val: shapedValue.getType()); |
| 156 | for (int64_t i = 0; i < shapedType.getRank(); ++i) { |
| 157 | if (shapedType.isDynamicDim(idx: i)) { |
| 158 | if (llvm::isa<MemRefType>(Val: shapedType)) { |
| 159 | dynamicDims.push_back(Elt: memref::DimOp::create(builder&: b, location: loc, source: shapedValue, index: i)); |
| 160 | } else { |
| 161 | assert(llvm::isa<RankedTensorType>(shapedType) && "expected tensor" ); |
| 162 | dynamicDims.push_back(Elt: tensor::DimOp::create(builder&: b, location: loc, source: shapedValue, index: i)); |
| 163 | } |
| 164 | } |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | //===----------------------------------------------------------------------===// |
| 169 | // AllocTensorOp |
| 170 | //===----------------------------------------------------------------------===// |
| 171 | |
| 172 | LogicalResult AllocTensorOp::verify() { |
| 173 | auto type = dyn_cast<RankedTensorType>(Val: getType()); |
| 174 | if (!type && (!getDynamicSizes().empty() || getSizeHint())) |
| 175 | return emitError(message: "takes dynamic sizes and a size hint only for a ranked " |
| 176 | "tensor result" ); |
| 177 | if (getCopy() && !getDynamicSizes().empty()) |
| 178 | return emitError(message: "dynamic sizes not needed when copying a tensor" ); |
| 179 | if (type && !getCopy() && |
| 180 | failed( |
| 181 | Result: verifyDynamicDimensionCount(op: getOperation(), type, dynamicSizes: getDynamicSizes()))) |
| 182 | return failure(); |
| 183 | if (getCopy() && getCopy().getType() != getType()) |
| 184 | return emitError(message: "expected that `copy` and return type match" ); |
| 185 | return success(); |
| 186 | } |
| 187 | |
| 188 | void AllocTensorOp::build(OpBuilder &builder, OperationState &result, |
| 189 | RankedTensorType type, ValueRange dynamicSizes) { |
| 190 | build(odsBuilder&: builder, odsState&: result, type: llvm::cast<TensorLikeType>(Val&: type), dynamicSizes, |
| 191 | /*copy=*/Value(), |
| 192 | /*size_hint=*/Value(), |
| 193 | /*memory_space=*/IntegerAttr()); |
| 194 | } |
| 195 | |
| 196 | void AllocTensorOp::build(OpBuilder &builder, OperationState &result, |
| 197 | RankedTensorType type, ValueRange dynamicSizes, |
| 198 | Value copy) { |
| 199 | build(odsBuilder&: builder, odsState&: result, type: llvm::cast<TensorLikeType>(Val&: type), dynamicSizes, copy); |
| 200 | } |
| 201 | |
| 202 | void AllocTensorOp::build(OpBuilder &builder, OperationState &result, |
| 203 | TensorType type, ValueRange dynamicSizes, Value copy, |
| 204 | IntegerAttr memorySpace) { |
| 205 | build(odsBuilder&: builder, odsState&: result, type: llvm::cast<TensorLikeType>(Val&: type), dynamicSizes, copy, |
| 206 | /*size_hint=*/Value(), memory_space: memorySpace); |
| 207 | } |
| 208 | |
| 209 | void AllocTensorOp::build(OpBuilder &builder, OperationState &result, |
| 210 | TensorLikeType type, ValueRange dynamicSizes, |
| 211 | Value copy) { |
| 212 | build(odsBuilder&: builder, odsState&: result, type, dynamicSizes, copy, /*size_hint=*/Value(), |
| 213 | /*memory_space=*/Attribute()); |
| 214 | } |
| 215 | |
| 216 | void AllocTensorOp::build(OpBuilder &builder, OperationState &result, |
| 217 | TensorLikeType type, ValueRange dynamicSizes, |
| 218 | Value copy, Value sizeHint, Attribute memorySpace) { |
| 219 | build(odsBuilder&: builder, odsState&: result, result: Type(type), dynamic_sizes: dynamicSizes, copy, size_hint: sizeHint, memory_space: memorySpace); |
| 220 | } |
| 221 | |
| 222 | namespace { |
| 223 | /// Change the type of the result of a `bufferization.alloc_tensor` by making |
| 224 | /// the result type statically sized along dimension that in the original |
| 225 | /// operation where defined as dynamic, but the size was defined using a |
| 226 | /// `constant` op. For example: |
| 227 | /// |
| 228 | /// %c5 = arith.constant 5: index |
| 229 | /// %0 = bufferization.alloc_tensor(%arg0, %c5) : tensor<?x?xf32> |
| 230 | /// |
| 231 | /// to |
| 232 | /// |
| 233 | /// %0 = bufferization.alloc_tensor(%arg0) : tensor<?x5xf32> |
| 234 | struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> { |
| 235 | using OpRewritePattern<AllocTensorOp>::OpRewritePattern; |
| 236 | |
| 237 | LogicalResult matchAndRewrite(AllocTensorOp op, |
| 238 | PatternRewriter &rewriter) const override { |
| 239 | // Dynamic sizes, and so this fold, belong to a ranked tensor result; any |
| 240 | // other result carries none (verifier). |
| 241 | auto type = dyn_cast<RankedTensorType>(Val: op.getType()); |
| 242 | if (!type || op.getCopy()) |
| 243 | return failure(); |
| 244 | SmallVector<int64_t> newShape = llvm::to_vector(Range: type.getShape()); |
| 245 | SmallVector<Value> newDynamicSizes; |
| 246 | unsigned int dynValCounter = 0; |
| 247 | for (int64_t i = 0; i < type.getRank(); ++i) { |
| 248 | if (!type.isDynamicDim(idx: i)) |
| 249 | continue; |
| 250 | Value value = op.getDynamicSizes()[dynValCounter++]; |
| 251 | APInt intVal; |
| 252 | if (matchPattern(value, pattern: m_ConstantInt(bind_value: &intVal))) { |
| 253 | int64_t dim = intVal.getSExtValue(); |
| 254 | if (dim >= 0) |
| 255 | newShape[i] = intVal.getSExtValue(); |
| 256 | else |
| 257 | newDynamicSizes.push_back(Elt: value); |
| 258 | } else { |
| 259 | newDynamicSizes.push_back(Elt: value); |
| 260 | } |
| 261 | } |
| 262 | RankedTensorType newType = RankedTensorType::get( |
| 263 | shape: newShape, elementType: type.getElementType(), encoding: type.getEncoding()); |
| 264 | if (newType == type) |
| 265 | return failure(); |
| 266 | auto newOp = AllocTensorOp::create(builder&: rewriter, location: op.getLoc(), type: newType, |
| 267 | dynamicSizes: newDynamicSizes, /*copy=*/Value()); |
| 268 | rewriter.replaceOpWithNewOp<tensor::CastOp>(op, args&: type, args&: newOp); |
| 269 | return success(); |
| 270 | } |
| 271 | }; |
| 272 | |
| 273 | struct FoldDimOfAllocTensorOp : public OpRewritePattern<tensor::DimOp> { |
| 274 | using OpRewritePattern<tensor::DimOp>::OpRewritePattern; |
| 275 | |
| 276 | LogicalResult matchAndRewrite(tensor::DimOp dimOp, |
| 277 | PatternRewriter &rewriter) const override { |
| 278 | std::optional<int64_t> maybeConstantIndex = dimOp.getConstantIndex(); |
| 279 | auto allocTensorOp = dimOp.getSource().getDefiningOp<AllocTensorOp>(); |
| 280 | if (!allocTensorOp || !maybeConstantIndex) |
| 281 | return failure(); |
| 282 | // Only a ranked tensor result has dynamic sizes to forward; any other |
| 283 | // result carries none (verifier). |
| 284 | auto type = dyn_cast<RankedTensorType>(Val: allocTensorOp.getType()); |
| 285 | if (!type || *maybeConstantIndex < 0 || |
| 286 | *maybeConstantIndex >= type.getRank()) |
| 287 | return failure(); |
| 288 | if (!type.isDynamicDim(idx: *maybeConstantIndex)) |
| 289 | return failure(); |
| 290 | rewriter.replaceOp( |
| 291 | op: dimOp, newValues: allocTensorOp.getDynamicSize(b&: rewriter, idx: *maybeConstantIndex)); |
| 292 | return success(); |
| 293 | } |
| 294 | }; |
| 295 | } // namespace |
| 296 | |
| 297 | void AllocTensorOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 298 | MLIRContext *ctx) { |
| 299 | results.add<FoldDimOfAllocTensorOp, ReplaceStaticShapeDims>(arg&: ctx); |
| 300 | } |
| 301 | |
| 302 | LogicalResult AllocTensorOp::reifyResultShapes( |
| 303 | OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedReturnShapes) { |
| 304 | // The extents of a result that is not a ranked tensor are known only from |
| 305 | // its buffer. |
| 306 | auto type = dyn_cast<RankedTensorType>(Val: getType()); |
| 307 | if (!type) |
| 308 | return failure(); |
| 309 | auto shapes = llvm::map_to_vector<4>( |
| 310 | C: llvm::seq<int64_t>(Begin: 0, End: type.getRank()), F: [&](int64_t dim) -> OpFoldResult { |
| 311 | if (type.isDynamicDim(idx: dim)) |
| 312 | return getDynamicSize(b&: builder, idx: dim); |
| 313 | return builder.getIndexAttr(value: type.getDimSize(idx: dim)); |
| 314 | }); |
| 315 | reifiedReturnShapes.emplace_back(Args: std::move(shapes)); |
| 316 | return success(); |
| 317 | } |
| 318 | |
| 319 | ParseResult AllocTensorOp::parse(OpAsmParser &parser, OperationState &result) { |
| 320 | SmallVector<OpAsmParser::UnresolvedOperand> dynamicSizesOperands; |
| 321 | if (parser.parseLParen() || parser.parseOperandList(result&: dynamicSizesOperands) || |
| 322 | parser.parseRParen()) |
| 323 | return failure(); |
| 324 | ParseResult copyKeyword = parser.parseOptionalKeyword(keyword: "copy" ); |
| 325 | OpAsmParser::UnresolvedOperand copyOperand; |
| 326 | if (copyKeyword.succeeded()) |
| 327 | if (parser.parseLParen() || parser.parseOperand(result&: copyOperand) || |
| 328 | parser.parseRParen()) |
| 329 | return failure(); |
| 330 | ParseResult sizeHintKeyword = parser.parseOptionalKeyword(keyword: "size_hint" ); |
| 331 | OpAsmParser::UnresolvedOperand sizeHintOperand; |
| 332 | if (sizeHintKeyword.succeeded()) |
| 333 | if (parser.parseEqual() || parser.parseOperand(result&: sizeHintOperand)) |
| 334 | return failure(); |
| 335 | |
| 336 | Attribute parsedProperties; |
| 337 | if (AllocTensorOp::genericParseProperties(parser, result&: parsedProperties)) |
| 338 | return failure(); |
| 339 | auto propertyDictionary = dyn_cast_or_null<DictionaryAttr>(Val&: parsedProperties); |
| 340 | if (parsedProperties && !propertyDictionary) |
| 341 | return parser.emitError(loc: parser.getNameLoc(), |
| 342 | message: "expected properties dictionary" ); |
| 343 | |
| 344 | auto attrsLoc = parser.getCurrentLocation(); |
| 345 | if (parser.parseOptionalAttrDict(result&: result.attributes)) |
| 346 | return failure(); |
| 347 | for (StringRef attrName : AllocTensorOp::getAttributeNames()) { |
| 348 | if (result.attributes.get(name: attrName)) |
| 349 | return parser.emitError(loc: attrsLoc) |
| 350 | << "inherent attribute '" << attrName |
| 351 | << "' cannot be parsed from attr-dict when strict properties in " |
| 352 | "assembly format is enabled" ; |
| 353 | } |
| 354 | if (parser.parseColon()) |
| 355 | return failure(); |
| 356 | |
| 357 | TensorLikeType type; |
| 358 | if (parser.parseCustomTypeWithFallback(result&: type)) |
| 359 | return failure(); |
| 360 | result.addTypes(newTypes: type); |
| 361 | |
| 362 | Type indexType = parser.getBuilder().getIndexType(); |
| 363 | if (parser.resolveOperands(operands&: dynamicSizesOperands, type: indexType, result&: result.operands)) |
| 364 | return failure(); |
| 365 | if (copyKeyword.succeeded()) |
| 366 | if (parser.resolveOperand(operand: copyOperand, type, result&: result.operands)) |
| 367 | return failure(); |
| 368 | if (sizeHintKeyword.succeeded()) |
| 369 | if (parser.resolveOperand(operand: sizeHintOperand, type: indexType, result&: result.operands)) |
| 370 | return failure(); |
| 371 | Builder &builder = parser.getBuilder(); |
| 372 | NamedAttrList properties(propertyDictionary ? propertyDictionary |
| 373 | : builder.getDictionaryAttr(value: {})); |
| 374 | properties.set(name: AllocTensorOp::getOperandSegmentSizeAttr(), |
| 375 | value: builder.getDenseI32ArrayAttr( |
| 376 | values: {static_cast<int32_t>(dynamicSizesOperands.size()), |
| 377 | static_cast<int32_t>(copyKeyword.succeeded()), |
| 378 | static_cast<int32_t>(sizeHintKeyword.succeeded())})); |
| 379 | propertyDictionary = properties.getDictionary(context: builder.getContext()); |
| 380 | auto emitError = [&]() { |
| 381 | return mlir::emitError(loc: result.location, message: "invalid properties " ) |
| 382 | << propertyDictionary << " for op " << result.name.getStringRef() |
| 383 | << ": " ; |
| 384 | }; |
| 385 | if (failed(Result: AllocTensorOp::setPropertiesFromParsedAttr( |
| 386 | prop&: result.getOrAddProperties<Properties>(), attr: propertyDictionary, |
| 387 | emitError))) |
| 388 | return failure(); |
| 389 | return success(); |
| 390 | } |
| 391 | |
| 392 | void AllocTensorOp::print(OpAsmPrinter &p) { |
| 393 | p << "(" << getDynamicSizes() << ")" ; |
| 394 | if (getCopy()) |
| 395 | p << " copy(" << getCopy() << ")" ; |
| 396 | if (getSizeHint()) |
| 397 | p << " size_hint=" << getSizeHint(); |
| 398 | AllocTensorOp::printProperties(ctx: getContext(), p, properties: getProperties(), |
| 399 | /*elidedProps=*/getOperandSegmentSizeAttr()); |
| 400 | p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue()); |
| 401 | p << " : " ; |
| 402 | auto type = getResult().getType(); |
| 403 | if (auto validType = llvm::dyn_cast<::mlir::TensorType>(Val&: type)) |
| 404 | p.printStrippedAttrOrType(attrOrType: validType); |
| 405 | else |
| 406 | p << type; |
| 407 | } |
| 408 | |
| 409 | Value AllocTensorOp::getDynamicSize(OpBuilder &b, unsigned idx) { |
| 410 | assert(isDynamicDim(idx) && "expected dynamic dim" ); |
| 411 | if (getCopy()) |
| 412 | return tensor::DimOp::create(builder&: b, location: getLoc(), source: getCopy(), index: idx); |
| 413 | return getOperand(i: getIndexOfDynamicSize(idx)); |
| 414 | } |
| 415 | |
| 416 | //===----------------------------------------------------------------------===// |
| 417 | // CloneOp |
| 418 | //===----------------------------------------------------------------------===// |
| 419 | |
| 420 | OpFoldResult CloneOp::fold(FoldAdaptor adaptor) { |
| 421 | return succeeded(Result: memref::foldMemRefCast(op: *this)) ? getResult() : Value(); |
| 422 | } |
| 423 | |
| 424 | namespace { |
| 425 | |
| 426 | /// Merge the clone and its source (by converting the clone to a cast) when |
| 427 | /// possible. |
| 428 | struct SimplifyClones : public OpRewritePattern<CloneOp> { |
| 429 | using OpRewritePattern<CloneOp>::OpRewritePattern; |
| 430 | |
| 431 | LogicalResult matchAndRewrite(CloneOp cloneOp, |
| 432 | PatternRewriter &rewriter) const override { |
| 433 | if (cloneOp.use_empty()) { |
| 434 | rewriter.eraseOp(op: cloneOp); |
| 435 | return success(); |
| 436 | } |
| 437 | |
| 438 | Value source = cloneOp.getInput(); |
| 439 | if (source.getType() != cloneOp.getType() && |
| 440 | !memref::CastOp::areCastCompatible(inputs: {source.getType()}, |
| 441 | outputs: {cloneOp.getType()})) |
| 442 | return failure(); |
| 443 | |
| 444 | // Aims to find the dealloc op for the canonical source |
| 445 | // which otherwise could prevent removal of unnecessary allocs. |
| 446 | Value canonicalSource = source; |
| 447 | while (auto iface = dyn_cast_or_null<ViewLikeOpInterface>( |
| 448 | Val: canonicalSource.getDefiningOp())) { |
| 449 | if (canonicalSource != iface.getViewDest()) { |
| 450 | break; |
| 451 | } |
| 452 | canonicalSource = iface.getViewSource(); |
| 453 | } |
| 454 | |
| 455 | std::optional<Operation *> maybeCloneDeallocOp = |
| 456 | memref::findDealloc(allocValue: cloneOp.getOutput()); |
| 457 | // Skip if either of them has > 1 deallocate operations. |
| 458 | if (!maybeCloneDeallocOp.has_value()) |
| 459 | return failure(); |
| 460 | std::optional<Operation *> maybeSourceDeallocOp = |
| 461 | memref::findDealloc(allocValue: canonicalSource); |
| 462 | if (!maybeSourceDeallocOp.has_value()) |
| 463 | return failure(); |
| 464 | Operation *cloneDeallocOp = *maybeCloneDeallocOp; |
| 465 | Operation *sourceDeallocOp = *maybeSourceDeallocOp; |
| 466 | |
| 467 | // If both are deallocated in the same block, their in-block lifetimes |
| 468 | // might not fully overlap, so we cannot decide which one to drop. |
| 469 | if (cloneDeallocOp && sourceDeallocOp && |
| 470 | cloneDeallocOp->getBlock() == sourceDeallocOp->getBlock()) |
| 471 | return failure(); |
| 472 | |
| 473 | Block *currentBlock = cloneOp->getBlock(); |
| 474 | Operation *redundantDealloc = nullptr; |
| 475 | if (cloneDeallocOp && cloneDeallocOp->getBlock() == currentBlock) { |
| 476 | redundantDealloc = cloneDeallocOp; |
| 477 | } else if (sourceDeallocOp && sourceDeallocOp->getBlock() == currentBlock) { |
| 478 | redundantDealloc = sourceDeallocOp; |
| 479 | } |
| 480 | |
| 481 | if (!redundantDealloc) |
| 482 | return failure(); |
| 483 | |
| 484 | // Safety check that there are no other deallocations inbetween |
| 485 | // cloneOp and redundantDealloc, as otherwise we might deallocate an alias |
| 486 | // of source before the uses of the clone. With alias information, we could |
| 487 | // restrict this to only fail of the dealloc's operand is an alias |
| 488 | // of the source. |
| 489 | for (Operation *pos = cloneOp->getNextNode(); pos != redundantDealloc; |
| 490 | pos = pos->getNextNode()) { |
| 491 | // Bail if we run out of operations while looking for a deallocation op. |
| 492 | if (!pos) |
| 493 | return failure(); |
| 494 | auto effectInterface = dyn_cast<MemoryEffectOpInterface>(Val: pos); |
| 495 | if (!effectInterface) |
| 496 | continue; |
| 497 | if (effectInterface.hasEffect<MemoryEffects::Free>()) |
| 498 | return failure(); |
| 499 | } |
| 500 | |
| 501 | if (source.getType() != cloneOp.getType()) |
| 502 | source = memref::CastOp::create(builder&: rewriter, location: cloneOp.getLoc(), |
| 503 | dest: cloneOp.getType(), source); |
| 504 | rewriter.replaceOp(op: cloneOp, newValues: source); |
| 505 | rewriter.eraseOp(op: redundantDealloc); |
| 506 | return success(); |
| 507 | } |
| 508 | }; |
| 509 | |
| 510 | } // namespace |
| 511 | |
| 512 | void CloneOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 513 | MLIRContext *context) { |
| 514 | results.add<SimplifyClones>(arg&: context); |
| 515 | } |
| 516 | |
| 517 | //===----------------------------------------------------------------------===// |
| 518 | // MaterializeInDestinationOp |
| 519 | //===----------------------------------------------------------------------===// |
| 520 | |
| 521 | LogicalResult MaterializeInDestinationOp::reifyResultShapes( |
| 522 | OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedReturnShapes) { |
| 523 | if (getOperation()->getNumResults() == 1) { |
| 524 | assert(isa<TensorType>(getDest().getType()) && "expected tensor type" ); |
| 525 | reifiedReturnShapes.resize(N: 1, |
| 526 | NV: SmallVector<OpFoldResult>(getType().getRank())); |
| 527 | reifiedReturnShapes[0] = |
| 528 | tensor::getMixedSizes(builder, loc: getLoc(), value: getDest()); |
| 529 | } |
| 530 | return success(); |
| 531 | } |
| 532 | |
| 533 | Value MaterializeInDestinationOp::(OpBuilder &builder, |
| 534 | Location loc) { |
| 535 | if (isa<TensorType>(Val: getDest().getType())) { |
| 536 | // The subset is the entire destination tensor. |
| 537 | return getDest(); |
| 538 | } |
| 539 | |
| 540 | // The "restrict" attribute is transferred from this op to the newly created |
| 541 | // to_tensor op. If this op does not the "restrict" attribute, the subset |
| 542 | // extraction cannot be built because there is no guarantee that there is no |
| 543 | // pre-existing "restrict" to_tensor op with the same/an aliasing destination. |
| 544 | if (!getRestrict()) |
| 545 | return {}; |
| 546 | |
| 547 | // Build a bufferization.to_tensor op. |
| 548 | assert(isa<BaseMemRefType>(getDest().getType()) && "expected memref type" ); |
| 549 | assert(getRestrict() && |
| 550 | "expected that ops with memrefs dest have 'restrict'" ); |
| 551 | setRestrict(false); |
| 552 | return ToTensorOp::create( |
| 553 | builder, location: loc, result: memref::getTensorTypeFromMemRefType(type: getDest().getType()), |
| 554 | buffer: getDest(), |
| 555 | /*restrict=*/true, writable: getWritable()); |
| 556 | } |
| 557 | |
| 558 | bool MaterializeInDestinationOp::isEquivalentSubset( |
| 559 | Value candidate, function_ref<bool(Value, Value)> equivalenceFn) { |
| 560 | return equivalenceFn(getDest(), candidate); |
| 561 | } |
| 562 | |
| 563 | SmallVector<Value> |
| 564 | MaterializeInDestinationOp::() { |
| 565 | return {getDest()}; |
| 566 | } |
| 567 | |
| 568 | OpOperand &MaterializeInDestinationOp::getSourceOperand() { |
| 569 | return getOperation()->getOpOperand(idx: 0) /*source*/; |
| 570 | } |
| 571 | |
| 572 | bool MaterializeInDestinationOp::operatesOnEquivalentSubset( |
| 573 | SubsetOpInterface subsetOp, |
| 574 | function_ref<bool(Value, Value)> equivalenceFn) { |
| 575 | return false; |
| 576 | } |
| 577 | |
| 578 | bool MaterializeInDestinationOp::operatesOnDisjointSubset( |
| 579 | SubsetOpInterface subsetOp, |
| 580 | function_ref<bool(Value, Value)> equivalenceFn) { |
| 581 | return false; |
| 582 | } |
| 583 | |
| 584 | LogicalResult MaterializeInDestinationOp::verify() { |
| 585 | if (!isa<TensorType, BaseMemRefType>(Val: getDest().getType())) |
| 586 | return emitOpError(message: "'dest' must be a tensor or a memref" ); |
| 587 | if (auto destType = dyn_cast<TensorType>(Val: getDest().getType())) { |
| 588 | if (getOperation()->getNumResults() != 1) |
| 589 | return emitOpError(message: "tensor 'dest' implies exactly one tensor result" ); |
| 590 | if (destType != getResult().getType()) |
| 591 | return emitOpError(message: "result and 'dest' types must match" ); |
| 592 | } |
| 593 | if (isa<BaseMemRefType>(Val: getDest().getType()) && |
| 594 | getOperation()->getNumResults() != 0) |
| 595 | return emitOpError(message: "memref 'dest' implies zero results" ); |
| 596 | if (getRestrict() && !isa<BaseMemRefType>(Val: getDest().getType())) |
| 597 | return emitOpError(message: "'restrict' is valid only for memref destinations" ); |
| 598 | if (getWritable() != isa<BaseMemRefType>(Val: getDest().getType())) |
| 599 | return emitOpError(message: "'writable' must be specified if and only if the " |
| 600 | "destination is of memref type" ); |
| 601 | TensorType srcType = getSource().getType(); |
| 602 | ShapedType destType = cast<ShapedType>(Val: getDest().getType()); |
| 603 | if (srcType.hasRank() != destType.hasRank()) |
| 604 | return emitOpError(message: "source/destination shapes are incompatible" ); |
| 605 | if (srcType.hasRank()) { |
| 606 | if (failed(Result: verifyRanksMatch(op: getOperation(), lhs: srcType, rhs: destType, lhsName: "source" , |
| 607 | rhsName: "destination" ))) |
| 608 | return failure(); |
| 609 | for (auto [src, dest] : |
| 610 | llvm::zip(t: srcType.getShape(), u: destType.getShape())) { |
| 611 | if (src == ShapedType::kDynamic || dest == ShapedType::kDynamic) { |
| 612 | // Cannot verify dynamic dimension size. Assume that that they match at |
| 613 | // runtime. |
| 614 | continue; |
| 615 | } |
| 616 | if (src != dest) |
| 617 | return emitOpError(message: "source/destination shapes are incompatible" ); |
| 618 | } |
| 619 | } |
| 620 | return success(); |
| 621 | } |
| 622 | |
| 623 | void MaterializeInDestinationOp::build(OpBuilder &builder, |
| 624 | OperationState &state, Value source, |
| 625 | Value dest) { |
| 626 | auto destTensorType = dyn_cast<TensorType>(Val: dest.getType()); |
| 627 | build(odsBuilder&: builder, odsState&: state, /*result=*/destTensorType ? destTensorType : Type(), |
| 628 | source, dest); |
| 629 | } |
| 630 | |
| 631 | MutableOperandRange MaterializeInDestinationOp::getDpsInitsMutable() { |
| 632 | return getDestMutable(); |
| 633 | } |
| 634 | |
| 635 | void MaterializeInDestinationOp::getEffects( |
| 636 | SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>> |
| 637 | &effects) { |
| 638 | if (isa<BaseMemRefType>(Val: getDest().getType())) |
| 639 | effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &getDestMutable(), |
| 640 | Args: SideEffects::DefaultResource::get()); |
| 641 | } |
| 642 | |
| 643 | //===----------------------------------------------------------------------===// |
| 644 | // ToTensorOp |
| 645 | //===----------------------------------------------------------------------===// |
| 646 | |
| 647 | OpFoldResult ToTensorOp::fold(FoldAdaptor) { |
| 648 | if (auto toBuffer = getBuffer().getDefiningOp<ToBufferOp>()) |
| 649 | // Approximate alias analysis by conservatively folding only when no there |
| 650 | // is no interleaved operation. |
| 651 | if (toBuffer->getBlock() == this->getOperation()->getBlock() && |
| 652 | toBuffer->getNextNode() == this->getOperation()) |
| 653 | return toBuffer.getTensor(); |
| 654 | return {}; |
| 655 | } |
| 656 | |
| 657 | namespace { |
| 658 | struct DimOfToTensorFolder : public OpRewritePattern<tensor::DimOp> { |
| 659 | using OpRewritePattern<tensor::DimOp>::OpRewritePattern; |
| 660 | |
| 661 | LogicalResult matchAndRewrite(tensor::DimOp dimOp, |
| 662 | PatternRewriter &rewriter) const override { |
| 663 | auto memrefToTensorOp = dimOp.getSource().getDefiningOp<ToTensorOp>(); |
| 664 | if (!memrefToTensorOp) |
| 665 | return failure(); |
| 666 | |
| 667 | rewriter.replaceOpWithNewOp<memref::DimOp>( |
| 668 | op: dimOp, args: memrefToTensorOp.getBuffer(), args: dimOp.getIndex()); |
| 669 | return success(); |
| 670 | } |
| 671 | }; |
| 672 | } // namespace |
| 673 | |
| 674 | void ToTensorOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 675 | MLIRContext *context) { |
| 676 | results.add<DimOfToTensorFolder>(arg&: context); |
| 677 | } |
| 678 | |
| 679 | //===----------------------------------------------------------------------===// |
| 680 | // ToBufferOp |
| 681 | //===----------------------------------------------------------------------===// |
| 682 | |
| 683 | OpFoldResult ToBufferOp::fold(FoldAdaptor) { |
| 684 | if (auto memrefToTensor = getTensor().getDefiningOp<ToTensorOp>()) |
| 685 | if (memrefToTensor.getBuffer().getType() == getType()) |
| 686 | return memrefToTensor.getBuffer(); |
| 687 | return {}; |
| 688 | } |
| 689 | |
| 690 | namespace { |
| 691 | |
| 692 | /// Replace tensor.cast + to_buffer by to_buffer + memref.cast. |
| 693 | struct ToBufferOfCast : public OpRewritePattern<ToBufferOp> { |
| 694 | using OpRewritePattern<ToBufferOp>::OpRewritePattern; |
| 695 | |
| 696 | LogicalResult matchAndRewrite(ToBufferOp toBuffer, |
| 697 | PatternRewriter &rewriter) const final { |
| 698 | auto tensorCastOperand = |
| 699 | toBuffer.getOperand().getDefiningOp<tensor::CastOp>(); |
| 700 | if (!tensorCastOperand) |
| 701 | return failure(); |
| 702 | auto srcTensorType = llvm::dyn_cast<RankedTensorType>( |
| 703 | Val: tensorCastOperand.getOperand().getType()); |
| 704 | if (!srcTensorType) |
| 705 | return failure(); |
| 706 | auto currentOutputMemRefType = |
| 707 | dyn_cast<BaseMemRefType>(Val: toBuffer.getResult().getType()); |
| 708 | if (!currentOutputMemRefType) |
| 709 | return failure(); |
| 710 | |
| 711 | auto memrefType = currentOutputMemRefType.cloneWith( |
| 712 | shape: srcTensorType.getShape(), elementType: srcTensorType.getElementType()); |
| 713 | Value memref = ToBufferOp::create(builder&: rewriter, location: toBuffer.getLoc(), buffer: memrefType, |
| 714 | tensor: tensorCastOperand.getOperand(), |
| 715 | read_only: toBuffer.getReadOnly()); |
| 716 | rewriter.replaceOpWithNewOp<memref::CastOp>(op: toBuffer, args: toBuffer.getType(), |
| 717 | args&: memref); |
| 718 | return success(); |
| 719 | } |
| 720 | }; |
| 721 | |
| 722 | /// Canonicalize bufferization.to_tensor + bufferization.to_buffer. Insert a |
| 723 | /// cast if necessary. |
| 724 | struct ToBufferToTensorFolding : public OpRewritePattern<ToBufferOp> { |
| 725 | using OpRewritePattern<ToBufferOp>::OpRewritePattern; |
| 726 | |
| 727 | LogicalResult matchAndRewrite(ToBufferOp toBuffer, |
| 728 | PatternRewriter &rewriter) const final { |
| 729 | BufferizationOptions options; |
| 730 | options.bufferAlignment = 0; |
| 731 | return foldToBufferToTensorPair(rewriter, toBuffer, options); |
| 732 | } |
| 733 | }; |
| 734 | |
| 735 | /// Fold a load on a to_buffer operation into an tensor.extract on the |
| 736 | /// corresponding tensor. |
| 737 | struct LoadOfToBuffer : public OpRewritePattern<memref::LoadOp> { |
| 738 | using OpRewritePattern<memref::LoadOp>::OpRewritePattern; |
| 739 | |
| 740 | LogicalResult matchAndRewrite(memref::LoadOp load, |
| 741 | PatternRewriter &rewriter) const override { |
| 742 | auto toBuffer = load.getMemref().getDefiningOp<ToBufferOp>(); |
| 743 | if (!toBuffer || !toBuffer.getReadOnly()) |
| 744 | return failure(); |
| 745 | |
| 746 | rewriter.replaceOpWithNewOp<tensor::ExtractOp>(op: load, args: toBuffer.getTensor(), |
| 747 | args: load.getIndices()); |
| 748 | return success(); |
| 749 | } |
| 750 | }; |
| 751 | |
| 752 | /// Fold dim of a to_buffer into the dim of the tensor. |
| 753 | struct DimOfCastOp : public OpRewritePattern<memref::DimOp> { |
| 754 | using OpRewritePattern<memref::DimOp>::OpRewritePattern; |
| 755 | |
| 756 | LogicalResult matchAndRewrite(memref::DimOp dimOp, |
| 757 | PatternRewriter &rewriter) const override { |
| 758 | auto castOp = dimOp.getSource().getDefiningOp<ToBufferOp>(); |
| 759 | if (!castOp) |
| 760 | return failure(); |
| 761 | Value newSource = castOp.getOperand(); |
| 762 | rewriter.replaceOpWithNewOp<tensor::DimOp>(op: dimOp, args&: newSource, |
| 763 | args: dimOp.getIndex()); |
| 764 | return success(); |
| 765 | } |
| 766 | }; |
| 767 | |
| 768 | } // namespace |
| 769 | |
| 770 | void ToBufferOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 771 | MLIRContext *context) { |
| 772 | results.add<DimOfCastOp, LoadOfToBuffer, ToBufferOfCast, |
| 773 | ToBufferToTensorFolding>(arg&: context); |
| 774 | } |
| 775 | |
| 776 | std::optional<Operation *> CloneOp::buildDealloc(OpBuilder &builder, |
| 777 | Value alloc) { |
| 778 | return memref::DeallocOp::create(builder, location: alloc.getLoc(), memref: alloc) |
| 779 | .getOperation(); |
| 780 | } |
| 781 | |
| 782 | std::optional<Value> CloneOp::buildClone(OpBuilder &builder, Value alloc) { |
| 783 | return CloneOp::create(builder, location: alloc.getLoc(), value: alloc).getResult(); |
| 784 | } |
| 785 | |
| 786 | //===----------------------------------------------------------------------===// |
| 787 | // DeallocOp |
| 788 | //===----------------------------------------------------------------------===// |
| 789 | |
| 790 | LogicalResult DeallocOp::inferReturnTypes( |
| 791 | MLIRContext *context, std::optional<::mlir::Location> location, |
| 792 | ValueRange operands, DictionaryAttr attributes, PropertyRef properties, |
| 793 | RegionRange regions, SmallVectorImpl<Type> &inferredReturnTypes) { |
| 794 | DeallocOpAdaptor adaptor(operands, attributes, properties, regions); |
| 795 | inferredReturnTypes = SmallVector<Type>(adaptor.getRetained().size(), |
| 796 | IntegerType::get(context, width: 1)); |
| 797 | return success(); |
| 798 | } |
| 799 | |
| 800 | LogicalResult DeallocOp::verify() { |
| 801 | if (getMemrefs().size() != getConditions().size()) |
| 802 | return emitOpError( |
| 803 | message: "must have the same number of conditions as memrefs to deallocate" ); |
| 804 | if (getRetained().size() != getUpdatedConditions().size()) |
| 805 | return emitOpError(message: "must have the same number of updated conditions " |
| 806 | "(results) as retained operands" ); |
| 807 | return success(); |
| 808 | } |
| 809 | |
| 810 | static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp, |
| 811 | ValueRange memrefs, |
| 812 | ValueRange conditions, |
| 813 | PatternRewriter &rewriter) { |
| 814 | if (deallocOp.getMemrefs() == memrefs && |
| 815 | deallocOp.getConditions() == conditions) |
| 816 | return failure(); |
| 817 | |
| 818 | rewriter.modifyOpInPlace(root: deallocOp, callable: [&]() { |
| 819 | deallocOp.getMemrefsMutable().assign(values: memrefs); |
| 820 | deallocOp.getConditionsMutable().assign(values: conditions); |
| 821 | }); |
| 822 | return success(); |
| 823 | } |
| 824 | |
| 825 | namespace { |
| 826 | |
| 827 | /// Remove duplicate values in the list of memrefs to be deallocated. We need to |
| 828 | /// make sure the corresponding condition value is updated accordingly since |
| 829 | /// their two conditions might not cover the same set of cases. In that case, we |
| 830 | /// have to combine them (by computing the disjunction of them). |
| 831 | /// Example: |
| 832 | /// ```mlir |
| 833 | /// bufferization.dealloc (%arg0, %arg0 : ...) if (%arg1, %arg2) |
| 834 | /// ``` |
| 835 | /// is canonicalized to |
| 836 | /// ```mlir |
| 837 | /// %0 = arith.ori %arg1, %arg2 : i1 |
| 838 | /// bufferization.dealloc (%arg0 : memref<2xi32>) if (%0) |
| 839 | /// ``` |
| 840 | struct DeallocRemoveDuplicateDeallocMemrefs |
| 841 | : public OpRewritePattern<DeallocOp> { |
| 842 | using OpRewritePattern<DeallocOp>::OpRewritePattern; |
| 843 | |
| 844 | LogicalResult matchAndRewrite(DeallocOp deallocOp, |
| 845 | PatternRewriter &rewriter) const override { |
| 846 | // Unique memrefs to be deallocated. |
| 847 | DenseMap<Value, unsigned> memrefToCondition; |
| 848 | SmallVector<Value> newMemrefs, newConditions; |
| 849 | for (auto [i, memref, cond] : |
| 850 | llvm::enumerate(First: deallocOp.getMemrefs(), Rest: deallocOp.getConditions())) { |
| 851 | if (memrefToCondition.count(Val: memref)) { |
| 852 | // If the dealloc conditions don't match, we need to make sure that the |
| 853 | // dealloc happens on the union of cases. |
| 854 | Value &newCond = newConditions[memrefToCondition[memref]]; |
| 855 | if (newCond != cond) |
| 856 | newCond = |
| 857 | arith::OrIOp::create(builder&: rewriter, location: deallocOp.getLoc(), lhs: newCond, rhs: cond); |
| 858 | } else { |
| 859 | memrefToCondition.insert(KV: {memref, newConditions.size()}); |
| 860 | newMemrefs.push_back(Elt: memref); |
| 861 | newConditions.push_back(Elt: cond); |
| 862 | } |
| 863 | } |
| 864 | |
| 865 | // Return failure if we don't change anything such that we don't run into an |
| 866 | // infinite loop of pattern applications. |
| 867 | return updateDeallocIfChanged(deallocOp, memrefs: newMemrefs, conditions: newConditions, |
| 868 | rewriter); |
| 869 | } |
| 870 | }; |
| 871 | |
| 872 | /// Remove duplicate values in the list of retained memrefs. We need to make |
| 873 | /// sure the corresponding result condition value is replaced properly. |
| 874 | /// Example: |
| 875 | /// ```mlir |
| 876 | /// %0:2 = bufferization.dealloc retain (%arg3, %arg3 : ...) |
| 877 | /// ``` |
| 878 | /// is canonicalized to |
| 879 | /// ```mlir |
| 880 | /// %0 = bufferization.dealloc retain (%arg3 : memref<2xi32>) |
| 881 | /// ``` |
| 882 | struct DeallocRemoveDuplicateRetainedMemrefs |
| 883 | : public OpRewritePattern<DeallocOp> { |
| 884 | using OpRewritePattern<DeallocOp>::OpRewritePattern; |
| 885 | |
| 886 | LogicalResult matchAndRewrite(DeallocOp deallocOp, |
| 887 | PatternRewriter &rewriter) const override { |
| 888 | // Unique retained values |
| 889 | DenseMap<Value, unsigned> seen; |
| 890 | SmallVector<Value> newRetained; |
| 891 | SmallVector<unsigned> resultReplacementIdx; |
| 892 | unsigned i = 0; |
| 893 | for (auto retained : deallocOp.getRetained()) { |
| 894 | if (seen.count(Val: retained)) { |
| 895 | resultReplacementIdx.push_back(Elt: seen[retained]); |
| 896 | continue; |
| 897 | } |
| 898 | |
| 899 | seen[retained] = i; |
| 900 | newRetained.push_back(Elt: retained); |
| 901 | resultReplacementIdx.push_back(Elt: i++); |
| 902 | } |
| 903 | |
| 904 | // Return failure if we don't change anything such that we don't run into an |
| 905 | // infinite loop of pattern applications. |
| 906 | if (newRetained.size() == deallocOp.getRetained().size()) |
| 907 | return failure(); |
| 908 | |
| 909 | // We need to create a new op because the number of results is always the |
| 910 | // same as the number of condition operands. |
| 911 | auto newDeallocOp = |
| 912 | DeallocOp::create(builder&: rewriter, location: deallocOp.getLoc(), memrefs: deallocOp.getMemrefs(), |
| 913 | conditions: deallocOp.getConditions(), retained: newRetained); |
| 914 | SmallVector<Value> replacements( |
| 915 | llvm::map_range(C&: resultReplacementIdx, F: [&](unsigned idx) { |
| 916 | return newDeallocOp.getUpdatedConditions()[idx]; |
| 917 | })); |
| 918 | rewriter.replaceOp(op: deallocOp, newValues: replacements); |
| 919 | return success(); |
| 920 | } |
| 921 | }; |
| 922 | |
| 923 | /// Erase deallocation operations where the variadic list of memrefs to |
| 924 | /// deallocate is empty. Example: |
| 925 | /// ```mlir |
| 926 | /// %0 = bufferization.dealloc retain (%arg0: memref<2xi32>) |
| 927 | /// ``` |
| 928 | struct EraseEmptyDealloc : public OpRewritePattern<DeallocOp> { |
| 929 | using OpRewritePattern<DeallocOp>::OpRewritePattern; |
| 930 | |
| 931 | LogicalResult matchAndRewrite(DeallocOp deallocOp, |
| 932 | PatternRewriter &rewriter) const override { |
| 933 | if (deallocOp.getMemrefs().empty()) { |
| 934 | Value constFalse = arith::ConstantOp::create(builder&: rewriter, location: deallocOp.getLoc(), |
| 935 | value: rewriter.getBoolAttr(value: false)); |
| 936 | rewriter.replaceOp( |
| 937 | op: deallocOp, newValues: SmallVector<Value>(deallocOp.getUpdatedConditions().size(), |
| 938 | constFalse)); |
| 939 | return success(); |
| 940 | } |
| 941 | return failure(); |
| 942 | } |
| 943 | }; |
| 944 | |
| 945 | /// Removes memrefs from the deallocation list if their associated condition is |
| 946 | /// always 'false'. |
| 947 | /// |
| 948 | /// Example: |
| 949 | /// ``` |
| 950 | /// bufferization.dealloc (%arg0, %arg1 : memref<2xi32>, memref<2xi32>) |
| 951 | /// if (%arg2, %false) |
| 952 | /// ``` |
| 953 | /// becomes |
| 954 | /// ``` |
| 955 | /// bufferization.dealloc (%arg0 : memref<2xi32>) if (%arg2) |
| 956 | /// ``` |
| 957 | struct EraseAlwaysFalseDealloc : public OpRewritePattern<DeallocOp> { |
| 958 | using OpRewritePattern<DeallocOp>::OpRewritePattern; |
| 959 | |
| 960 | LogicalResult matchAndRewrite(DeallocOp deallocOp, |
| 961 | PatternRewriter &rewriter) const override { |
| 962 | SmallVector<Value> newMemrefs, newConditions; |
| 963 | for (auto [memref, cond] : |
| 964 | llvm::zip(t: deallocOp.getMemrefs(), u: deallocOp.getConditions())) { |
| 965 | if (!matchPattern(value: cond, pattern: m_Zero())) { |
| 966 | newMemrefs.push_back(Elt: memref); |
| 967 | newConditions.push_back(Elt: cond); |
| 968 | } |
| 969 | } |
| 970 | |
| 971 | return updateDeallocIfChanged(deallocOp, memrefs: newMemrefs, conditions: newConditions, |
| 972 | rewriter); |
| 973 | } |
| 974 | }; |
| 975 | |
| 976 | /// The `memref.extract_strided_metadata` is often inserted to get the base |
| 977 | /// memref if the operand is not already guaranteed to be the result of a memref |
| 978 | /// allocation operation. This canonicalization pattern removes this extraction |
| 979 | /// operation if the operand is now produced by an allocation operation (e.g., |
| 980 | /// due to other canonicalizations simplifying the IR). |
| 981 | /// |
| 982 | /// Example: |
| 983 | /// ```mlir |
| 984 | /// %alloc = memref.alloc() : memref<2xi32> |
| 985 | /// %base_memref, %offset, %size, %stride = memref.extract_strided_metadata |
| 986 | /// %alloc : memref<2xi32> -> memref<i32>, index, index, index |
| 987 | /// bufferization.dealloc (%base_memref : memref<i32>) if (%cond) |
| 988 | /// ``` |
| 989 | /// is canonicalized to |
| 990 | /// ```mlir |
| 991 | /// %alloc = memref.alloc() : memref<2xi32> |
| 992 | /// bufferization.dealloc (%alloc : memref<2xi32>) if (%cond) |
| 993 | /// ``` |
| 994 | struct : public OpRewritePattern<DeallocOp> { |
| 995 | using OpRewritePattern<DeallocOp>::OpRewritePattern; |
| 996 | |
| 997 | LogicalResult matchAndRewrite(DeallocOp deallocOp, |
| 998 | PatternRewriter &rewriter) const override { |
| 999 | SmallVector<Value> newMemrefs( |
| 1000 | llvm::map_range(C: deallocOp.getMemrefs(), F: [&](Value memref) { |
| 1001 | auto = |
| 1002 | memref.getDefiningOp<memref::ExtractStridedMetadataOp>(); |
| 1003 | if (!extractStridedOp) |
| 1004 | return memref; |
| 1005 | Value allocMemref = extractStridedOp.getOperand(); |
| 1006 | auto allocOp = allocMemref.getDefiningOp<MemoryEffectOpInterface>(); |
| 1007 | if (!allocOp) |
| 1008 | return memref; |
| 1009 | if (allocOp.getEffectOnValue<MemoryEffects::Allocate>(value: allocMemref)) |
| 1010 | return allocMemref; |
| 1011 | return memref; |
| 1012 | })); |
| 1013 | |
| 1014 | return updateDeallocIfChanged(deallocOp, memrefs: newMemrefs, |
| 1015 | conditions: deallocOp.getConditions(), rewriter); |
| 1016 | } |
| 1017 | }; |
| 1018 | |
| 1019 | /// Removes pairs of `bufferization.dealloc` and alloc operations if there is no |
| 1020 | /// other user of the allocated value and the allocating operation can be safely |
| 1021 | /// removed. If the same value is present multiple times, this pattern relies on |
| 1022 | /// other canonicalization patterns to remove the duplicate first. |
| 1023 | /// |
| 1024 | /// Example: |
| 1025 | /// ```mlir |
| 1026 | /// %alloc = memref.alloc() : memref<2xi32> |
| 1027 | /// bufferization.dealloc (%alloc, %arg0, : ...) if (%true, %true) |
| 1028 | /// ``` |
| 1029 | /// is canonicalized to |
| 1030 | /// ```mlir |
| 1031 | /// bufferization.dealloc (%arg0 : ...) if (%true) |
| 1032 | /// ``` |
| 1033 | struct RemoveAllocDeallocPairWhenNoOtherUsers |
| 1034 | : public OpRewritePattern<DeallocOp> { |
| 1035 | using OpRewritePattern<DeallocOp>::OpRewritePattern; |
| 1036 | |
| 1037 | LogicalResult matchAndRewrite(DeallocOp deallocOp, |
| 1038 | PatternRewriter &rewriter) const override { |
| 1039 | SmallVector<Value> newMemrefs, newConditions; |
| 1040 | SmallVector<Operation *> toDelete; |
| 1041 | for (auto [memref, cond] : |
| 1042 | llvm::zip(t: deallocOp.getMemrefs(), u: deallocOp.getConditions())) { |
| 1043 | if (auto allocOp = memref.getDefiningOp<MemoryEffectOpInterface>()) { |
| 1044 | // Check that it is indeed an allocate effect, that the op has no other |
| 1045 | // side effects (which would not allow us to remove the op), and that |
| 1046 | // there are no other users. |
| 1047 | if (allocOp.getEffectOnValue<MemoryEffects::Allocate>(value: memref) && |
| 1048 | hasSingleEffect<MemoryEffects::Allocate>(op: allocOp, value: memref) && |
| 1049 | memref.hasOneUse()) { |
| 1050 | toDelete.push_back(Elt: allocOp); |
| 1051 | continue; |
| 1052 | } |
| 1053 | } |
| 1054 | |
| 1055 | newMemrefs.push_back(Elt: memref); |
| 1056 | newConditions.push_back(Elt: cond); |
| 1057 | } |
| 1058 | |
| 1059 | if (failed(Result: updateDeallocIfChanged(deallocOp, memrefs: newMemrefs, conditions: newConditions, |
| 1060 | rewriter))) |
| 1061 | return failure(); |
| 1062 | |
| 1063 | for (Operation *op : toDelete) |
| 1064 | rewriter.eraseOp(op); |
| 1065 | |
| 1066 | return success(); |
| 1067 | } |
| 1068 | }; |
| 1069 | |
| 1070 | } // anonymous namespace |
| 1071 | |
| 1072 | void DeallocOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 1073 | MLIRContext *context) { |
| 1074 | populateDeallocOpCanonicalizationPatterns(patterns&: results, context); |
| 1075 | } |
| 1076 | |
| 1077 | void bufferization::populateDeallocOpCanonicalizationPatterns( |
| 1078 | RewritePatternSet &patterns, MLIRContext *context) { |
| 1079 | patterns.add<DeallocRemoveDuplicateDeallocMemrefs, |
| 1080 | DeallocRemoveDuplicateRetainedMemrefs, EraseEmptyDealloc, |
| 1081 | EraseAlwaysFalseDealloc, SkipExtractMetadataOfAlloc, |
| 1082 | RemoveAllocDeallocPairWhenNoOtherUsers>(arg&: context); |
| 1083 | } |
| 1084 | |
| 1085 | //===----------------------------------------------------------------------===// |
| 1086 | // TableGen'd op method definitions |
| 1087 | //===----------------------------------------------------------------------===// |
| 1088 | |
| 1089 | #define GET_OP_CLASSES |
| 1090 | #include "mlir/Dialect/Bufferization/IR/BufferizationOps.cpp.inc" |
| 1091 | |