1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Arith/IR/Arith.h"
10#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
11#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
12#include "mlir/Dialect/Func/IR/FuncOps.h"
13#include "mlir/Dialect/MemRef/IR/MemRef.h"
14#include "mlir/Dialect/Tensor/IR/Tensor.h"
15#include "mlir/Dialect/Utils/VerificationUtils.h"
16#include "mlir/IR/Matchers.h"
17#include "llvm/ADT/SmallVectorExtras.h"
18#include <optional>
19
20using namespace mlir;
21using namespace mlir::bufferization;
22
23//===----------------------------------------------------------------------===//
24// Helper functions
25//===----------------------------------------------------------------------===//
26
27FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
28 OpBuilder &b, Value value, MemRefType destType,
29 const BufferizationOptions &options) {
30 auto srcType = llvm::cast<MemRefType>(Val: value.getType());
31
32 // Element type and rank must match.
33 if (srcType.getElementType() != destType.getElementType())
34 return failure();
35 if (srcType.getRank() != destType.getRank())
36 return failure();
37
38 // In case the affine maps are different, we may need to use a copy if we go
39 // from dynamic to static offset or stride (the canonicalization cannot know
40 // at this point that it is really cast compatible).
41 auto isGuaranteedCastCompatible = [](MemRefType source, MemRefType target) {
42 int64_t sourceOffset, targetOffset;
43 SmallVector<int64_t, 4> sourceStrides, targetStrides;
44 if (failed(Result: source.getStridesAndOffset(strides&: sourceStrides, offset&: sourceOffset)) ||
45 failed(Result: target.getStridesAndOffset(strides&: targetStrides, offset&: targetOffset)))
46 return false;
47 auto dynamicToStatic = [](int64_t a, int64_t b) {
48 return ShapedType::isDynamic(dValue: a) && ShapedType::isStatic(dValue: b);
49 };
50 if (dynamicToStatic(sourceOffset, targetOffset))
51 return false;
52 for (auto it : zip(t&: sourceStrides, u&: targetStrides))
53 if (dynamicToStatic(std::get<0>(t&: it), std::get<1>(t&: it)))
54 return false;
55 // A cast cannot safely zero out a non-zero offset. If the source has a
56 // non-identity layout (e.g., a subview with a dynamic offset) and the
57 // destination requires identity layout (zero offset, unit strides), a cast
58 // would silently strip the offset and cause incorrect loads in the callee.
59 // Fall through to the alloc+copy path instead.
60 if (!source.getLayout().isIdentity() && target.getLayout().isIdentity())
61 return false;
62 return true;
63 };
64
65 // Note: If `areCastCompatible`, a cast is valid, but may fail at runtime. To
66 // ensure that we only generate casts that always succeed at runtime, we check
67 // a few extra conditions in `isGuaranteedCastCompatible`.
68 if (memref::CastOp::areCastCompatible(inputs: srcType, outputs: destType) &&
69 isGuaranteedCastCompatible(srcType, destType)) {
70 Value casted = *options.castFn(b, value.getLoc(), destType, value);
71 return casted;
72 }
73
74 auto loc = value.getLoc();
75 SmallVector<Value, 4> dynamicOperands;
76 for (int i = 0; i < destType.getRank(); ++i) {
77 if (destType.getShape()[i] != ShapedType::kDynamic)
78 continue;
79 Value size = memref::DimOp::create(builder&: b, location: loc, source: value, index: i);
80 dynamicOperands.push_back(Elt: size);
81 }
82
83 FailureOr<Value> copy = options.allocationFn(
84 b, loc, destType, dynamicOperands, options.bufferAlignment);
85 if (failed(Result: copy))
86 return failure();
87 if (failed(Result: options.memCpyFn(b, loc, value, *copy)))
88 return failure();
89 return copy;
90}
91
92/// Try to fold to_buffer(to_tensor(x)). If x's type and the result type of the
93/// to_buffer op are different, a memref.cast is needed.
94LogicalResult mlir::bufferization::foldToBufferToTensorPair(
95 RewriterBase &rewriter, ToBufferOp toBuffer,
96 const BufferizationOptions &options) {
97 auto bufferToTensor = toBuffer.getTensor().getDefiningOp<ToTensorOp>();
98 if (!bufferToTensor)
99 return failure();
100
101 Type srcType = bufferToTensor.getBuffer().getType();
102 Type destType = toBuffer.getType();
103
104 // Directly rewrite if the type did not change.
105 if (srcType == destType) {
106 rewriter.replaceOp(op: toBuffer, newValues: bufferToTensor.getBuffer());
107 return success();
108 }
109
110 if (!llvm::isa<BaseMemRefType>(Val: srcType) ||
111 !llvm::isa<BaseMemRefType>(Val: destType)) {
112 // Non-builtin case: the best is to try the user-provided cast.
113 auto replacement =
114 options.castFn(rewriter, bufferToTensor.getBuffer().getLoc(), destType,
115 bufferToTensor.getBuffer());
116 if (failed(Result: replacement))
117 return failure();
118 rewriter.replaceOp(op: toBuffer, newValues: *replacement);
119 return success();
120 }
121
122 auto rankedSrcType = llvm::dyn_cast<MemRefType>(Val&: srcType);
123 auto rankedDestType = llvm::dyn_cast<MemRefType>(Val&: destType);
124 auto unrankedSrcType = llvm::dyn_cast<UnrankedMemRefType>(Val&: srcType);
125
126 // Ranked memref -> Ranked memref cast.
127 if (rankedSrcType && rankedDestType) {
128 FailureOr<Value> replacement = castOrReallocMemRefValue(
129 b&: rewriter, value: bufferToTensor.getBuffer(), destType: rankedDestType, options);
130 if (failed(Result: replacement))
131 return failure();
132
133 rewriter.replaceOp(op: toBuffer, newValues: *replacement);
134 return success();
135 }
136
137 // Unranked memref -> Ranked memref cast: May require a copy.
138 // TODO: Not implemented at the moment.
139 if (unrankedSrcType && rankedDestType)
140 return failure();
141
142 // Unranked/ranked memref -> unranked memref cast: No copy needed if the types
143 // are cast-compatible.
144 if (!memref::CastOp::areCastCompatible(inputs: srcType, outputs: destType))
145 return failure();
146
147 rewriter.replaceOpWithNewOp<memref::CastOp>(op: toBuffer, args&: destType,
148 args: bufferToTensor.getBuffer());
149 return success();
150}
151
152void mlir::bufferization::populateDynamicDimSizes(
153 OpBuilder &b, Location loc, Value shapedValue,
154 SmallVector<Value> &dynamicDims) {
155 auto shapedType = llvm::cast<ShapedType>(Val: shapedValue.getType());
156 for (int64_t i = 0; i < shapedType.getRank(); ++i) {
157 if (shapedType.isDynamicDim(idx: i)) {
158 if (llvm::isa<MemRefType>(Val: shapedType)) {
159 dynamicDims.push_back(Elt: memref::DimOp::create(builder&: b, location: loc, source: shapedValue, index: i));
160 } else {
161 assert(llvm::isa<RankedTensorType>(shapedType) && "expected tensor");
162 dynamicDims.push_back(Elt: tensor::DimOp::create(builder&: b, location: loc, source: shapedValue, index: i));
163 }
164 }
165 }
166}
167
168//===----------------------------------------------------------------------===//
169// AllocTensorOp
170//===----------------------------------------------------------------------===//
171
172LogicalResult AllocTensorOp::verify() {
173 auto type = dyn_cast<RankedTensorType>(Val: getType());
174 if (!type && (!getDynamicSizes().empty() || getSizeHint()))
175 return emitError(message: "takes dynamic sizes and a size hint only for a ranked "
176 "tensor result");
177 if (getCopy() && !getDynamicSizes().empty())
178 return emitError(message: "dynamic sizes not needed when copying a tensor");
179 if (type && !getCopy() &&
180 failed(
181 Result: verifyDynamicDimensionCount(op: getOperation(), type, dynamicSizes: getDynamicSizes())))
182 return failure();
183 if (getCopy() && getCopy().getType() != getType())
184 return emitError(message: "expected that `copy` and return type match");
185 return success();
186}
187
188void AllocTensorOp::build(OpBuilder &builder, OperationState &result,
189 RankedTensorType type, ValueRange dynamicSizes) {
190 build(odsBuilder&: builder, odsState&: result, type: llvm::cast<TensorLikeType>(Val&: type), dynamicSizes,
191 /*copy=*/Value(),
192 /*size_hint=*/Value(),
193 /*memory_space=*/IntegerAttr());
194}
195
196void AllocTensorOp::build(OpBuilder &builder, OperationState &result,
197 RankedTensorType type, ValueRange dynamicSizes,
198 Value copy) {
199 build(odsBuilder&: builder, odsState&: result, type: llvm::cast<TensorLikeType>(Val&: type), dynamicSizes, copy);
200}
201
202void AllocTensorOp::build(OpBuilder &builder, OperationState &result,
203 TensorType type, ValueRange dynamicSizes, Value copy,
204 IntegerAttr memorySpace) {
205 build(odsBuilder&: builder, odsState&: result, type: llvm::cast<TensorLikeType>(Val&: type), dynamicSizes, copy,
206 /*size_hint=*/Value(), memory_space: memorySpace);
207}
208
209void AllocTensorOp::build(OpBuilder &builder, OperationState &result,
210 TensorLikeType type, ValueRange dynamicSizes,
211 Value copy) {
212 build(odsBuilder&: builder, odsState&: result, type, dynamicSizes, copy, /*size_hint=*/Value(),
213 /*memory_space=*/Attribute());
214}
215
216void AllocTensorOp::build(OpBuilder &builder, OperationState &result,
217 TensorLikeType type, ValueRange dynamicSizes,
218 Value copy, Value sizeHint, Attribute memorySpace) {
219 build(odsBuilder&: builder, odsState&: result, result: Type(type), dynamic_sizes: dynamicSizes, copy, size_hint: sizeHint, memory_space: memorySpace);
220}
221
222namespace {
223/// Change the type of the result of a `bufferization.alloc_tensor` by making
224/// the result type statically sized along dimension that in the original
225/// operation where defined as dynamic, but the size was defined using a
226/// `constant` op. For example:
227///
228/// %c5 = arith.constant 5: index
229/// %0 = bufferization.alloc_tensor(%arg0, %c5) : tensor<?x?xf32>
230///
231/// to
232///
233/// %0 = bufferization.alloc_tensor(%arg0) : tensor<?x5xf32>
234struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> {
235 using OpRewritePattern<AllocTensorOp>::OpRewritePattern;
236
237 LogicalResult matchAndRewrite(AllocTensorOp op,
238 PatternRewriter &rewriter) const override {
239 // Dynamic sizes, and so this fold, belong to a ranked tensor result; any
240 // other result carries none (verifier).
241 auto type = dyn_cast<RankedTensorType>(Val: op.getType());
242 if (!type || op.getCopy())
243 return failure();
244 SmallVector<int64_t> newShape = llvm::to_vector(Range: type.getShape());
245 SmallVector<Value> newDynamicSizes;
246 unsigned int dynValCounter = 0;
247 for (int64_t i = 0; i < type.getRank(); ++i) {
248 if (!type.isDynamicDim(idx: i))
249 continue;
250 Value value = op.getDynamicSizes()[dynValCounter++];
251 APInt intVal;
252 if (matchPattern(value, pattern: m_ConstantInt(bind_value: &intVal))) {
253 int64_t dim = intVal.getSExtValue();
254 if (dim >= 0)
255 newShape[i] = intVal.getSExtValue();
256 else
257 newDynamicSizes.push_back(Elt: value);
258 } else {
259 newDynamicSizes.push_back(Elt: value);
260 }
261 }
262 RankedTensorType newType = RankedTensorType::get(
263 shape: newShape, elementType: type.getElementType(), encoding: type.getEncoding());
264 if (newType == type)
265 return failure();
266 auto newOp = AllocTensorOp::create(builder&: rewriter, location: op.getLoc(), type: newType,
267 dynamicSizes: newDynamicSizes, /*copy=*/Value());
268 rewriter.replaceOpWithNewOp<tensor::CastOp>(op, args&: type, args&: newOp);
269 return success();
270 }
271};
272
273struct FoldDimOfAllocTensorOp : public OpRewritePattern<tensor::DimOp> {
274 using OpRewritePattern<tensor::DimOp>::OpRewritePattern;
275
276 LogicalResult matchAndRewrite(tensor::DimOp dimOp,
277 PatternRewriter &rewriter) const override {
278 std::optional<int64_t> maybeConstantIndex = dimOp.getConstantIndex();
279 auto allocTensorOp = dimOp.getSource().getDefiningOp<AllocTensorOp>();
280 if (!allocTensorOp || !maybeConstantIndex)
281 return failure();
282 // Only a ranked tensor result has dynamic sizes to forward; any other
283 // result carries none (verifier).
284 auto type = dyn_cast<RankedTensorType>(Val: allocTensorOp.getType());
285 if (!type || *maybeConstantIndex < 0 ||
286 *maybeConstantIndex >= type.getRank())
287 return failure();
288 if (!type.isDynamicDim(idx: *maybeConstantIndex))
289 return failure();
290 rewriter.replaceOp(
291 op: dimOp, newValues: allocTensorOp.getDynamicSize(b&: rewriter, idx: *maybeConstantIndex));
292 return success();
293 }
294};
295} // namespace
296
297void AllocTensorOp::getCanonicalizationPatterns(RewritePatternSet &results,
298 MLIRContext *ctx) {
299 results.add<FoldDimOfAllocTensorOp, ReplaceStaticShapeDims>(arg&: ctx);
300}
301
302LogicalResult AllocTensorOp::reifyResultShapes(
303 OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
304 // The extents of a result that is not a ranked tensor are known only from
305 // its buffer.
306 auto type = dyn_cast<RankedTensorType>(Val: getType());
307 if (!type)
308 return failure();
309 auto shapes = llvm::map_to_vector<4>(
310 C: llvm::seq<int64_t>(Begin: 0, End: type.getRank()), F: [&](int64_t dim) -> OpFoldResult {
311 if (type.isDynamicDim(idx: dim))
312 return getDynamicSize(b&: builder, idx: dim);
313 return builder.getIndexAttr(value: type.getDimSize(idx: dim));
314 });
315 reifiedReturnShapes.emplace_back(Args: std::move(shapes));
316 return success();
317}
318
319ParseResult AllocTensorOp::parse(OpAsmParser &parser, OperationState &result) {
320 SmallVector<OpAsmParser::UnresolvedOperand> dynamicSizesOperands;
321 if (parser.parseLParen() || parser.parseOperandList(result&: dynamicSizesOperands) ||
322 parser.parseRParen())
323 return failure();
324 ParseResult copyKeyword = parser.parseOptionalKeyword(keyword: "copy");
325 OpAsmParser::UnresolvedOperand copyOperand;
326 if (copyKeyword.succeeded())
327 if (parser.parseLParen() || parser.parseOperand(result&: copyOperand) ||
328 parser.parseRParen())
329 return failure();
330 ParseResult sizeHintKeyword = parser.parseOptionalKeyword(keyword: "size_hint");
331 OpAsmParser::UnresolvedOperand sizeHintOperand;
332 if (sizeHintKeyword.succeeded())
333 if (parser.parseEqual() || parser.parseOperand(result&: sizeHintOperand))
334 return failure();
335
336 Attribute parsedProperties;
337 if (AllocTensorOp::genericParseProperties(parser, result&: parsedProperties))
338 return failure();
339 auto propertyDictionary = dyn_cast_or_null<DictionaryAttr>(Val&: parsedProperties);
340 if (parsedProperties && !propertyDictionary)
341 return parser.emitError(loc: parser.getNameLoc(),
342 message: "expected properties dictionary");
343
344 auto attrsLoc = parser.getCurrentLocation();
345 if (parser.parseOptionalAttrDict(result&: result.attributes))
346 return failure();
347 for (StringRef attrName : AllocTensorOp::getAttributeNames()) {
348 if (result.attributes.get(name: attrName))
349 return parser.emitError(loc: attrsLoc)
350 << "inherent attribute '" << attrName
351 << "' cannot be parsed from attr-dict when strict properties in "
352 "assembly format is enabled";
353 }
354 if (parser.parseColon())
355 return failure();
356
357 TensorLikeType type;
358 if (parser.parseCustomTypeWithFallback(result&: type))
359 return failure();
360 result.addTypes(newTypes: type);
361
362 Type indexType = parser.getBuilder().getIndexType();
363 if (parser.resolveOperands(operands&: dynamicSizesOperands, type: indexType, result&: result.operands))
364 return failure();
365 if (copyKeyword.succeeded())
366 if (parser.resolveOperand(operand: copyOperand, type, result&: result.operands))
367 return failure();
368 if (sizeHintKeyword.succeeded())
369 if (parser.resolveOperand(operand: sizeHintOperand, type: indexType, result&: result.operands))
370 return failure();
371 Builder &builder = parser.getBuilder();
372 NamedAttrList properties(propertyDictionary ? propertyDictionary
373 : builder.getDictionaryAttr(value: {}));
374 properties.set(name: AllocTensorOp::getOperandSegmentSizeAttr(),
375 value: builder.getDenseI32ArrayAttr(
376 values: {static_cast<int32_t>(dynamicSizesOperands.size()),
377 static_cast<int32_t>(copyKeyword.succeeded()),
378 static_cast<int32_t>(sizeHintKeyword.succeeded())}));
379 propertyDictionary = properties.getDictionary(context: builder.getContext());
380 auto emitError = [&]() {
381 return mlir::emitError(loc: result.location, message: "invalid properties ")
382 << propertyDictionary << " for op " << result.name.getStringRef()
383 << ": ";
384 };
385 if (failed(Result: AllocTensorOp::setPropertiesFromParsedAttr(
386 prop&: result.getOrAddProperties<Properties>(), attr: propertyDictionary,
387 emitError)))
388 return failure();
389 return success();
390}
391
392void AllocTensorOp::print(OpAsmPrinter &p) {
393 p << "(" << getDynamicSizes() << ")";
394 if (getCopy())
395 p << " copy(" << getCopy() << ")";
396 if (getSizeHint())
397 p << " size_hint=" << getSizeHint();
398 AllocTensorOp::printProperties(ctx: getContext(), p, properties: getProperties(),
399 /*elidedProps=*/getOperandSegmentSizeAttr());
400 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue());
401 p << " : ";
402 auto type = getResult().getType();
403 if (auto validType = llvm::dyn_cast<::mlir::TensorType>(Val&: type))
404 p.printStrippedAttrOrType(attrOrType: validType);
405 else
406 p << type;
407}
408
409Value AllocTensorOp::getDynamicSize(OpBuilder &b, unsigned idx) {
410 assert(isDynamicDim(idx) && "expected dynamic dim");
411 if (getCopy())
412 return tensor::DimOp::create(builder&: b, location: getLoc(), source: getCopy(), index: idx);
413 return getOperand(i: getIndexOfDynamicSize(idx));
414}
415
416//===----------------------------------------------------------------------===//
417// CloneOp
418//===----------------------------------------------------------------------===//
419
420OpFoldResult CloneOp::fold(FoldAdaptor adaptor) {
421 return succeeded(Result: memref::foldMemRefCast(op: *this)) ? getResult() : Value();
422}
423
424namespace {
425
426/// Merge the clone and its source (by converting the clone to a cast) when
427/// possible.
428struct SimplifyClones : public OpRewritePattern<CloneOp> {
429 using OpRewritePattern<CloneOp>::OpRewritePattern;
430
431 LogicalResult matchAndRewrite(CloneOp cloneOp,
432 PatternRewriter &rewriter) const override {
433 if (cloneOp.use_empty()) {
434 rewriter.eraseOp(op: cloneOp);
435 return success();
436 }
437
438 Value source = cloneOp.getInput();
439 if (source.getType() != cloneOp.getType() &&
440 !memref::CastOp::areCastCompatible(inputs: {source.getType()},
441 outputs: {cloneOp.getType()}))
442 return failure();
443
444 // Aims to find the dealloc op for the canonical source
445 // which otherwise could prevent removal of unnecessary allocs.
446 Value canonicalSource = source;
447 while (auto iface = dyn_cast_or_null<ViewLikeOpInterface>(
448 Val: canonicalSource.getDefiningOp())) {
449 if (canonicalSource != iface.getViewDest()) {
450 break;
451 }
452 canonicalSource = iface.getViewSource();
453 }
454
455 std::optional<Operation *> maybeCloneDeallocOp =
456 memref::findDealloc(allocValue: cloneOp.getOutput());
457 // Skip if either of them has > 1 deallocate operations.
458 if (!maybeCloneDeallocOp.has_value())
459 return failure();
460 std::optional<Operation *> maybeSourceDeallocOp =
461 memref::findDealloc(allocValue: canonicalSource);
462 if (!maybeSourceDeallocOp.has_value())
463 return failure();
464 Operation *cloneDeallocOp = *maybeCloneDeallocOp;
465 Operation *sourceDeallocOp = *maybeSourceDeallocOp;
466
467 // If both are deallocated in the same block, their in-block lifetimes
468 // might not fully overlap, so we cannot decide which one to drop.
469 if (cloneDeallocOp && sourceDeallocOp &&
470 cloneDeallocOp->getBlock() == sourceDeallocOp->getBlock())
471 return failure();
472
473 Block *currentBlock = cloneOp->getBlock();
474 Operation *redundantDealloc = nullptr;
475 if (cloneDeallocOp && cloneDeallocOp->getBlock() == currentBlock) {
476 redundantDealloc = cloneDeallocOp;
477 } else if (sourceDeallocOp && sourceDeallocOp->getBlock() == currentBlock) {
478 redundantDealloc = sourceDeallocOp;
479 }
480
481 if (!redundantDealloc)
482 return failure();
483
484 // Safety check that there are no other deallocations inbetween
485 // cloneOp and redundantDealloc, as otherwise we might deallocate an alias
486 // of source before the uses of the clone. With alias information, we could
487 // restrict this to only fail of the dealloc's operand is an alias
488 // of the source.
489 for (Operation *pos = cloneOp->getNextNode(); pos != redundantDealloc;
490 pos = pos->getNextNode()) {
491 // Bail if we run out of operations while looking for a deallocation op.
492 if (!pos)
493 return failure();
494 auto effectInterface = dyn_cast<MemoryEffectOpInterface>(Val: pos);
495 if (!effectInterface)
496 continue;
497 if (effectInterface.hasEffect<MemoryEffects::Free>())
498 return failure();
499 }
500
501 if (source.getType() != cloneOp.getType())
502 source = memref::CastOp::create(builder&: rewriter, location: cloneOp.getLoc(),
503 dest: cloneOp.getType(), source);
504 rewriter.replaceOp(op: cloneOp, newValues: source);
505 rewriter.eraseOp(op: redundantDealloc);
506 return success();
507 }
508};
509
510} // namespace
511
512void CloneOp::getCanonicalizationPatterns(RewritePatternSet &results,
513 MLIRContext *context) {
514 results.add<SimplifyClones>(arg&: context);
515}
516
517//===----------------------------------------------------------------------===//
518// MaterializeInDestinationOp
519//===----------------------------------------------------------------------===//
520
521LogicalResult MaterializeInDestinationOp::reifyResultShapes(
522 OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
523 if (getOperation()->getNumResults() == 1) {
524 assert(isa<TensorType>(getDest().getType()) && "expected tensor type");
525 reifiedReturnShapes.resize(N: 1,
526 NV: SmallVector<OpFoldResult>(getType().getRank()));
527 reifiedReturnShapes[0] =
528 tensor::getMixedSizes(builder, loc: getLoc(), value: getDest());
529 }
530 return success();
531}
532
533Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
534 Location loc) {
535 if (isa<TensorType>(Val: getDest().getType())) {
536 // The subset is the entire destination tensor.
537 return getDest();
538 }
539
540 // The "restrict" attribute is transferred from this op to the newly created
541 // to_tensor op. If this op does not the "restrict" attribute, the subset
542 // extraction cannot be built because there is no guarantee that there is no
543 // pre-existing "restrict" to_tensor op with the same/an aliasing destination.
544 if (!getRestrict())
545 return {};
546
547 // Build a bufferization.to_tensor op.
548 assert(isa<BaseMemRefType>(getDest().getType()) && "expected memref type");
549 assert(getRestrict() &&
550 "expected that ops with memrefs dest have 'restrict'");
551 setRestrict(false);
552 return ToTensorOp::create(
553 builder, location: loc, result: memref::getTensorTypeFromMemRefType(type: getDest().getType()),
554 buffer: getDest(),
555 /*restrict=*/true, writable: getWritable());
556}
557
558bool MaterializeInDestinationOp::isEquivalentSubset(
559 Value candidate, function_ref<bool(Value, Value)> equivalenceFn) {
560 return equivalenceFn(getDest(), candidate);
561}
562
563SmallVector<Value>
564MaterializeInDestinationOp::getValuesNeededToBuildSubsetExtraction() {
565 return {getDest()};
566}
567
568OpOperand &MaterializeInDestinationOp::getSourceOperand() {
569 return getOperation()->getOpOperand(idx: 0) /*source*/;
570}
571
572bool MaterializeInDestinationOp::operatesOnEquivalentSubset(
573 SubsetOpInterface subsetOp,
574 function_ref<bool(Value, Value)> equivalenceFn) {
575 return false;
576}
577
578bool MaterializeInDestinationOp::operatesOnDisjointSubset(
579 SubsetOpInterface subsetOp,
580 function_ref<bool(Value, Value)> equivalenceFn) {
581 return false;
582}
583
584LogicalResult MaterializeInDestinationOp::verify() {
585 if (!isa<TensorType, BaseMemRefType>(Val: getDest().getType()))
586 return emitOpError(message: "'dest' must be a tensor or a memref");
587 if (auto destType = dyn_cast<TensorType>(Val: getDest().getType())) {
588 if (getOperation()->getNumResults() != 1)
589 return emitOpError(message: "tensor 'dest' implies exactly one tensor result");
590 if (destType != getResult().getType())
591 return emitOpError(message: "result and 'dest' types must match");
592 }
593 if (isa<BaseMemRefType>(Val: getDest().getType()) &&
594 getOperation()->getNumResults() != 0)
595 return emitOpError(message: "memref 'dest' implies zero results");
596 if (getRestrict() && !isa<BaseMemRefType>(Val: getDest().getType()))
597 return emitOpError(message: "'restrict' is valid only for memref destinations");
598 if (getWritable() != isa<BaseMemRefType>(Val: getDest().getType()))
599 return emitOpError(message: "'writable' must be specified if and only if the "
600 "destination is of memref type");
601 TensorType srcType = getSource().getType();
602 ShapedType destType = cast<ShapedType>(Val: getDest().getType());
603 if (srcType.hasRank() != destType.hasRank())
604 return emitOpError(message: "source/destination shapes are incompatible");
605 if (srcType.hasRank()) {
606 if (failed(Result: verifyRanksMatch(op: getOperation(), lhs: srcType, rhs: destType, lhsName: "source",
607 rhsName: "destination")))
608 return failure();
609 for (auto [src, dest] :
610 llvm::zip(t: srcType.getShape(), u: destType.getShape())) {
611 if (src == ShapedType::kDynamic || dest == ShapedType::kDynamic) {
612 // Cannot verify dynamic dimension size. Assume that that they match at
613 // runtime.
614 continue;
615 }
616 if (src != dest)
617 return emitOpError(message: "source/destination shapes are incompatible");
618 }
619 }
620 return success();
621}
622
623void MaterializeInDestinationOp::build(OpBuilder &builder,
624 OperationState &state, Value source,
625 Value dest) {
626 auto destTensorType = dyn_cast<TensorType>(Val: dest.getType());
627 build(odsBuilder&: builder, odsState&: state, /*result=*/destTensorType ? destTensorType : Type(),
628 source, dest);
629}
630
631MutableOperandRange MaterializeInDestinationOp::getDpsInitsMutable() {
632 return getDestMutable();
633}
634
635void MaterializeInDestinationOp::getEffects(
636 SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
637 &effects) {
638 if (isa<BaseMemRefType>(Val: getDest().getType()))
639 effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &getDestMutable(),
640 Args: SideEffects::DefaultResource::get());
641}
642
643//===----------------------------------------------------------------------===//
644// ToTensorOp
645//===----------------------------------------------------------------------===//
646
647OpFoldResult ToTensorOp::fold(FoldAdaptor) {
648 if (auto toBuffer = getBuffer().getDefiningOp<ToBufferOp>())
649 // Approximate alias analysis by conservatively folding only when no there
650 // is no interleaved operation.
651 if (toBuffer->getBlock() == this->getOperation()->getBlock() &&
652 toBuffer->getNextNode() == this->getOperation())
653 return toBuffer.getTensor();
654 return {};
655}
656
657namespace {
658struct DimOfToTensorFolder : public OpRewritePattern<tensor::DimOp> {
659 using OpRewritePattern<tensor::DimOp>::OpRewritePattern;
660
661 LogicalResult matchAndRewrite(tensor::DimOp dimOp,
662 PatternRewriter &rewriter) const override {
663 auto memrefToTensorOp = dimOp.getSource().getDefiningOp<ToTensorOp>();
664 if (!memrefToTensorOp)
665 return failure();
666
667 rewriter.replaceOpWithNewOp<memref::DimOp>(
668 op: dimOp, args: memrefToTensorOp.getBuffer(), args: dimOp.getIndex());
669 return success();
670 }
671};
672} // namespace
673
674void ToTensorOp::getCanonicalizationPatterns(RewritePatternSet &results,
675 MLIRContext *context) {
676 results.add<DimOfToTensorFolder>(arg&: context);
677}
678
679//===----------------------------------------------------------------------===//
680// ToBufferOp
681//===----------------------------------------------------------------------===//
682
683OpFoldResult ToBufferOp::fold(FoldAdaptor) {
684 if (auto memrefToTensor = getTensor().getDefiningOp<ToTensorOp>())
685 if (memrefToTensor.getBuffer().getType() == getType())
686 return memrefToTensor.getBuffer();
687 return {};
688}
689
690namespace {
691
692/// Replace tensor.cast + to_buffer by to_buffer + memref.cast.
693struct ToBufferOfCast : public OpRewritePattern<ToBufferOp> {
694 using OpRewritePattern<ToBufferOp>::OpRewritePattern;
695
696 LogicalResult matchAndRewrite(ToBufferOp toBuffer,
697 PatternRewriter &rewriter) const final {
698 auto tensorCastOperand =
699 toBuffer.getOperand().getDefiningOp<tensor::CastOp>();
700 if (!tensorCastOperand)
701 return failure();
702 auto srcTensorType = llvm::dyn_cast<RankedTensorType>(
703 Val: tensorCastOperand.getOperand().getType());
704 if (!srcTensorType)
705 return failure();
706 auto currentOutputMemRefType =
707 dyn_cast<BaseMemRefType>(Val: toBuffer.getResult().getType());
708 if (!currentOutputMemRefType)
709 return failure();
710
711 auto memrefType = currentOutputMemRefType.cloneWith(
712 shape: srcTensorType.getShape(), elementType: srcTensorType.getElementType());
713 Value memref = ToBufferOp::create(builder&: rewriter, location: toBuffer.getLoc(), buffer: memrefType,
714 tensor: tensorCastOperand.getOperand(),
715 read_only: toBuffer.getReadOnly());
716 rewriter.replaceOpWithNewOp<memref::CastOp>(op: toBuffer, args: toBuffer.getType(),
717 args&: memref);
718 return success();
719 }
720};
721
722/// Canonicalize bufferization.to_tensor + bufferization.to_buffer. Insert a
723/// cast if necessary.
724struct ToBufferToTensorFolding : public OpRewritePattern<ToBufferOp> {
725 using OpRewritePattern<ToBufferOp>::OpRewritePattern;
726
727 LogicalResult matchAndRewrite(ToBufferOp toBuffer,
728 PatternRewriter &rewriter) const final {
729 BufferizationOptions options;
730 options.bufferAlignment = 0;
731 return foldToBufferToTensorPair(rewriter, toBuffer, options);
732 }
733};
734
735/// Fold a load on a to_buffer operation into an tensor.extract on the
736/// corresponding tensor.
737struct LoadOfToBuffer : public OpRewritePattern<memref::LoadOp> {
738 using OpRewritePattern<memref::LoadOp>::OpRewritePattern;
739
740 LogicalResult matchAndRewrite(memref::LoadOp load,
741 PatternRewriter &rewriter) const override {
742 auto toBuffer = load.getMemref().getDefiningOp<ToBufferOp>();
743 if (!toBuffer || !toBuffer.getReadOnly())
744 return failure();
745
746 rewriter.replaceOpWithNewOp<tensor::ExtractOp>(op: load, args: toBuffer.getTensor(),
747 args: load.getIndices());
748 return success();
749 }
750};
751
752/// Fold dim of a to_buffer into the dim of the tensor.
753struct DimOfCastOp : public OpRewritePattern<memref::DimOp> {
754 using OpRewritePattern<memref::DimOp>::OpRewritePattern;
755
756 LogicalResult matchAndRewrite(memref::DimOp dimOp,
757 PatternRewriter &rewriter) const override {
758 auto castOp = dimOp.getSource().getDefiningOp<ToBufferOp>();
759 if (!castOp)
760 return failure();
761 Value newSource = castOp.getOperand();
762 rewriter.replaceOpWithNewOp<tensor::DimOp>(op: dimOp, args&: newSource,
763 args: dimOp.getIndex());
764 return success();
765 }
766};
767
768} // namespace
769
770void ToBufferOp::getCanonicalizationPatterns(RewritePatternSet &results,
771 MLIRContext *context) {
772 results.add<DimOfCastOp, LoadOfToBuffer, ToBufferOfCast,
773 ToBufferToTensorFolding>(arg&: context);
774}
775
776std::optional<Operation *> CloneOp::buildDealloc(OpBuilder &builder,
777 Value alloc) {
778 return memref::DeallocOp::create(builder, location: alloc.getLoc(), memref: alloc)
779 .getOperation();
780}
781
782std::optional<Value> CloneOp::buildClone(OpBuilder &builder, Value alloc) {
783 return CloneOp::create(builder, location: alloc.getLoc(), value: alloc).getResult();
784}
785
786//===----------------------------------------------------------------------===//
787// DeallocOp
788//===----------------------------------------------------------------------===//
789
790LogicalResult DeallocOp::inferReturnTypes(
791 MLIRContext *context, std::optional<::mlir::Location> location,
792 ValueRange operands, DictionaryAttr attributes, PropertyRef properties,
793 RegionRange regions, SmallVectorImpl<Type> &inferredReturnTypes) {
794 DeallocOpAdaptor adaptor(operands, attributes, properties, regions);
795 inferredReturnTypes = SmallVector<Type>(adaptor.getRetained().size(),
796 IntegerType::get(context, width: 1));
797 return success();
798}
799
800LogicalResult DeallocOp::verify() {
801 if (getMemrefs().size() != getConditions().size())
802 return emitOpError(
803 message: "must have the same number of conditions as memrefs to deallocate");
804 if (getRetained().size() != getUpdatedConditions().size())
805 return emitOpError(message: "must have the same number of updated conditions "
806 "(results) as retained operands");
807 return success();
808}
809
810static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp,
811 ValueRange memrefs,
812 ValueRange conditions,
813 PatternRewriter &rewriter) {
814 if (deallocOp.getMemrefs() == memrefs &&
815 deallocOp.getConditions() == conditions)
816 return failure();
817
818 rewriter.modifyOpInPlace(root: deallocOp, callable: [&]() {
819 deallocOp.getMemrefsMutable().assign(values: memrefs);
820 deallocOp.getConditionsMutable().assign(values: conditions);
821 });
822 return success();
823}
824
825namespace {
826
827/// Remove duplicate values in the list of memrefs to be deallocated. We need to
828/// make sure the corresponding condition value is updated accordingly since
829/// their two conditions might not cover the same set of cases. In that case, we
830/// have to combine them (by computing the disjunction of them).
831/// Example:
832/// ```mlir
833/// bufferization.dealloc (%arg0, %arg0 : ...) if (%arg1, %arg2)
834/// ```
835/// is canonicalized to
836/// ```mlir
837/// %0 = arith.ori %arg1, %arg2 : i1
838/// bufferization.dealloc (%arg0 : memref<2xi32>) if (%0)
839/// ```
840struct DeallocRemoveDuplicateDeallocMemrefs
841 : public OpRewritePattern<DeallocOp> {
842 using OpRewritePattern<DeallocOp>::OpRewritePattern;
843
844 LogicalResult matchAndRewrite(DeallocOp deallocOp,
845 PatternRewriter &rewriter) const override {
846 // Unique memrefs to be deallocated.
847 DenseMap<Value, unsigned> memrefToCondition;
848 SmallVector<Value> newMemrefs, newConditions;
849 for (auto [i, memref, cond] :
850 llvm::enumerate(First: deallocOp.getMemrefs(), Rest: deallocOp.getConditions())) {
851 if (memrefToCondition.count(Val: memref)) {
852 // If the dealloc conditions don't match, we need to make sure that the
853 // dealloc happens on the union of cases.
854 Value &newCond = newConditions[memrefToCondition[memref]];
855 if (newCond != cond)
856 newCond =
857 arith::OrIOp::create(builder&: rewriter, location: deallocOp.getLoc(), lhs: newCond, rhs: cond);
858 } else {
859 memrefToCondition.insert(KV: {memref, newConditions.size()});
860 newMemrefs.push_back(Elt: memref);
861 newConditions.push_back(Elt: cond);
862 }
863 }
864
865 // Return failure if we don't change anything such that we don't run into an
866 // infinite loop of pattern applications.
867 return updateDeallocIfChanged(deallocOp, memrefs: newMemrefs, conditions: newConditions,
868 rewriter);
869 }
870};
871
872/// Remove duplicate values in the list of retained memrefs. We need to make
873/// sure the corresponding result condition value is replaced properly.
874/// Example:
875/// ```mlir
876/// %0:2 = bufferization.dealloc retain (%arg3, %arg3 : ...)
877/// ```
878/// is canonicalized to
879/// ```mlir
880/// %0 = bufferization.dealloc retain (%arg3 : memref<2xi32>)
881/// ```
882struct DeallocRemoveDuplicateRetainedMemrefs
883 : public OpRewritePattern<DeallocOp> {
884 using OpRewritePattern<DeallocOp>::OpRewritePattern;
885
886 LogicalResult matchAndRewrite(DeallocOp deallocOp,
887 PatternRewriter &rewriter) const override {
888 // Unique retained values
889 DenseMap<Value, unsigned> seen;
890 SmallVector<Value> newRetained;
891 SmallVector<unsigned> resultReplacementIdx;
892 unsigned i = 0;
893 for (auto retained : deallocOp.getRetained()) {
894 if (seen.count(Val: retained)) {
895 resultReplacementIdx.push_back(Elt: seen[retained]);
896 continue;
897 }
898
899 seen[retained] = i;
900 newRetained.push_back(Elt: retained);
901 resultReplacementIdx.push_back(Elt: i++);
902 }
903
904 // Return failure if we don't change anything such that we don't run into an
905 // infinite loop of pattern applications.
906 if (newRetained.size() == deallocOp.getRetained().size())
907 return failure();
908
909 // We need to create a new op because the number of results is always the
910 // same as the number of condition operands.
911 auto newDeallocOp =
912 DeallocOp::create(builder&: rewriter, location: deallocOp.getLoc(), memrefs: deallocOp.getMemrefs(),
913 conditions: deallocOp.getConditions(), retained: newRetained);
914 SmallVector<Value> replacements(
915 llvm::map_range(C&: resultReplacementIdx, F: [&](unsigned idx) {
916 return newDeallocOp.getUpdatedConditions()[idx];
917 }));
918 rewriter.replaceOp(op: deallocOp, newValues: replacements);
919 return success();
920 }
921};
922
923/// Erase deallocation operations where the variadic list of memrefs to
924/// deallocate is empty. Example:
925/// ```mlir
926/// %0 = bufferization.dealloc retain (%arg0: memref<2xi32>)
927/// ```
928struct EraseEmptyDealloc : public OpRewritePattern<DeallocOp> {
929 using OpRewritePattern<DeallocOp>::OpRewritePattern;
930
931 LogicalResult matchAndRewrite(DeallocOp deallocOp,
932 PatternRewriter &rewriter) const override {
933 if (deallocOp.getMemrefs().empty()) {
934 Value constFalse = arith::ConstantOp::create(builder&: rewriter, location: deallocOp.getLoc(),
935 value: rewriter.getBoolAttr(value: false));
936 rewriter.replaceOp(
937 op: deallocOp, newValues: SmallVector<Value>(deallocOp.getUpdatedConditions().size(),
938 constFalse));
939 return success();
940 }
941 return failure();
942 }
943};
944
945/// Removes memrefs from the deallocation list if their associated condition is
946/// always 'false'.
947///
948/// Example:
949/// ```
950/// bufferization.dealloc (%arg0, %arg1 : memref<2xi32>, memref<2xi32>)
951/// if (%arg2, %false)
952/// ```
953/// becomes
954/// ```
955/// bufferization.dealloc (%arg0 : memref<2xi32>) if (%arg2)
956/// ```
957struct EraseAlwaysFalseDealloc : public OpRewritePattern<DeallocOp> {
958 using OpRewritePattern<DeallocOp>::OpRewritePattern;
959
960 LogicalResult matchAndRewrite(DeallocOp deallocOp,
961 PatternRewriter &rewriter) const override {
962 SmallVector<Value> newMemrefs, newConditions;
963 for (auto [memref, cond] :
964 llvm::zip(t: deallocOp.getMemrefs(), u: deallocOp.getConditions())) {
965 if (!matchPattern(value: cond, pattern: m_Zero())) {
966 newMemrefs.push_back(Elt: memref);
967 newConditions.push_back(Elt: cond);
968 }
969 }
970
971 return updateDeallocIfChanged(deallocOp, memrefs: newMemrefs, conditions: newConditions,
972 rewriter);
973 }
974};
975
976/// The `memref.extract_strided_metadata` is often inserted to get the base
977/// memref if the operand is not already guaranteed to be the result of a memref
978/// allocation operation. This canonicalization pattern removes this extraction
979/// operation if the operand is now produced by an allocation operation (e.g.,
980/// due to other canonicalizations simplifying the IR).
981///
982/// Example:
983/// ```mlir
984/// %alloc = memref.alloc() : memref<2xi32>
985/// %base_memref, %offset, %size, %stride = memref.extract_strided_metadata
986/// %alloc : memref<2xi32> -> memref<i32>, index, index, index
987/// bufferization.dealloc (%base_memref : memref<i32>) if (%cond)
988/// ```
989/// is canonicalized to
990/// ```mlir
991/// %alloc = memref.alloc() : memref<2xi32>
992/// bufferization.dealloc (%alloc : memref<2xi32>) if (%cond)
993/// ```
994struct SkipExtractMetadataOfAlloc : public OpRewritePattern<DeallocOp> {
995 using OpRewritePattern<DeallocOp>::OpRewritePattern;
996
997 LogicalResult matchAndRewrite(DeallocOp deallocOp,
998 PatternRewriter &rewriter) const override {
999 SmallVector<Value> newMemrefs(
1000 llvm::map_range(C: deallocOp.getMemrefs(), F: [&](Value memref) {
1001 auto extractStridedOp =
1002 memref.getDefiningOp<memref::ExtractStridedMetadataOp>();
1003 if (!extractStridedOp)
1004 return memref;
1005 Value allocMemref = extractStridedOp.getOperand();
1006 auto allocOp = allocMemref.getDefiningOp<MemoryEffectOpInterface>();
1007 if (!allocOp)
1008 return memref;
1009 if (allocOp.getEffectOnValue<MemoryEffects::Allocate>(value: allocMemref))
1010 return allocMemref;
1011 return memref;
1012 }));
1013
1014 return updateDeallocIfChanged(deallocOp, memrefs: newMemrefs,
1015 conditions: deallocOp.getConditions(), rewriter);
1016 }
1017};
1018
1019/// Removes pairs of `bufferization.dealloc` and alloc operations if there is no
1020/// other user of the allocated value and the allocating operation can be safely
1021/// removed. If the same value is present multiple times, this pattern relies on
1022/// other canonicalization patterns to remove the duplicate first.
1023///
1024/// Example:
1025/// ```mlir
1026/// %alloc = memref.alloc() : memref<2xi32>
1027/// bufferization.dealloc (%alloc, %arg0, : ...) if (%true, %true)
1028/// ```
1029/// is canonicalized to
1030/// ```mlir
1031/// bufferization.dealloc (%arg0 : ...) if (%true)
1032/// ```
1033struct RemoveAllocDeallocPairWhenNoOtherUsers
1034 : public OpRewritePattern<DeallocOp> {
1035 using OpRewritePattern<DeallocOp>::OpRewritePattern;
1036
1037 LogicalResult matchAndRewrite(DeallocOp deallocOp,
1038 PatternRewriter &rewriter) const override {
1039 SmallVector<Value> newMemrefs, newConditions;
1040 SmallVector<Operation *> toDelete;
1041 for (auto [memref, cond] :
1042 llvm::zip(t: deallocOp.getMemrefs(), u: deallocOp.getConditions())) {
1043 if (auto allocOp = memref.getDefiningOp<MemoryEffectOpInterface>()) {
1044 // Check that it is indeed an allocate effect, that the op has no other
1045 // side effects (which would not allow us to remove the op), and that
1046 // there are no other users.
1047 if (allocOp.getEffectOnValue<MemoryEffects::Allocate>(value: memref) &&
1048 hasSingleEffect<MemoryEffects::Allocate>(op: allocOp, value: memref) &&
1049 memref.hasOneUse()) {
1050 toDelete.push_back(Elt: allocOp);
1051 continue;
1052 }
1053 }
1054
1055 newMemrefs.push_back(Elt: memref);
1056 newConditions.push_back(Elt: cond);
1057 }
1058
1059 if (failed(Result: updateDeallocIfChanged(deallocOp, memrefs: newMemrefs, conditions: newConditions,
1060 rewriter)))
1061 return failure();
1062
1063 for (Operation *op : toDelete)
1064 rewriter.eraseOp(op);
1065
1066 return success();
1067 }
1068};
1069
1070} // anonymous namespace
1071
1072void DeallocOp::getCanonicalizationPatterns(RewritePatternSet &results,
1073 MLIRContext *context) {
1074 populateDeallocOpCanonicalizationPatterns(patterns&: results, context);
1075}
1076
1077void bufferization::populateDeallocOpCanonicalizationPatterns(
1078 RewritePatternSet &patterns, MLIRContext *context) {
1079 patterns.add<DeallocRemoveDuplicateDeallocMemrefs,
1080 DeallocRemoveDuplicateRetainedMemrefs, EraseEmptyDealloc,
1081 EraseAlwaysFalseDealloc, SkipExtractMetadataOfAlloc,
1082 RemoveAllocDeallocPairWhenNoOtherUsers>(arg&: context);
1083}
1084
1085//===----------------------------------------------------------------------===//
1086// TableGen'd op method definitions
1087//===----------------------------------------------------------------------===//
1088
1089#define GET_OP_CLASSES
1090#include "mlir/Dialect/Bufferization/IR/BufferizationOps.cpp.inc"
1091