1//===- LowerDeallocations.cpp - Bufferization Deallocs to MemRef pass -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements patterns to convert `bufferization.dealloc` operations
10// to the MemRef dialect.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/Arith/IR/Arith.h"
15#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
16#include "mlir/Dialect/Bufferization/Transforms/Passes.h"
17#include "mlir/Dialect/Func/IR/FuncOps.h"
18#include "mlir/Dialect/MemRef/IR/MemRef.h"
19#include "mlir/Dialect/SCF/IR/SCF.h"
20#include "mlir/IR/BuiltinTypes.h"
21#include "mlir/Transforms/DialectConversion.h"
22
23namespace mlir {
24namespace bufferization {
25#define GEN_PASS_DEF_LOWERDEALLOCATIONSPASS
26#include "mlir/Dialect/Bufferization/Transforms/Passes.h.inc"
27} // namespace bufferization
28} // namespace mlir
29
30using namespace mlir;
31
32namespace {
33/// The DeallocOpConversion transforms all bufferization dealloc operations into
34/// memref dealloc operations potentially guarded by scf if operations.
35/// Additionally, memref extract_aligned_pointer_as_index and arith operations
36/// are inserted to compute the guard conditions. We distinguish multiple cases
37/// to provide an overall more efficient lowering. In the general case, a helper
38/// func is created to avoid quadratic code size explosion (relative to the
39/// number of operands of the dealloc operation). For examples of each case,
40/// refer to the documentation of the member functions of this class.
41class DeallocOpConversion
42 : public OpConversionPattern<bufferization::DeallocOp> {
43
44 /// Lower a simple case without any retained values and a single memref to
45 /// avoiding the helper function. Ideally, static analysis can provide enough
46 /// aliasing information to split the dealloc operations up into this simple
47 /// case as much as possible before running this pass.
48 ///
49 /// Example:
50 /// ```
51 /// bufferization.dealloc (%arg0 : memref<2xf32>) if (%arg1)
52 /// ```
53 /// is lowered to
54 /// ```
55 /// scf.if %arg1 {
56 /// memref.dealloc %arg0 : memref<2xf32>
57 /// }
58 /// ```
59 LogicalResult
60 rewriteOneMemrefNoRetainCase(bufferization::DeallocOp op, OpAdaptor adaptor,
61 ConversionPatternRewriter &rewriter) const {
62 assert(adaptor.getMemrefs().size() == 1 && "expected only one memref");
63 assert(adaptor.getRetained().empty() && "expected no retained memrefs");
64
65 rewriter.replaceOpWithNewOp<scf::IfOp>(
66 op, args: adaptor.getConditions()[0], args: [&](OpBuilder &builder, Location loc) {
67 memref::DeallocOp::create(builder, location: loc, memref: adaptor.getMemrefs()[0]);
68 scf::YieldOp::create(builder, location: loc);
69 });
70 return success();
71 }
72
73 /// A special case lowering for the deallocation operation with exactly one
74 /// memref, but arbitrary number of retained values. This avoids the helper
75 /// function that the general case needs and thus also avoids storing indices
76 /// to specifically allocated memrefs. The size of the code produced by this
77 /// lowering is linear to the number of retained values.
78 ///
79 /// Example:
80 /// ```mlir
81 /// %0:2 = bufferization.dealloc (%m : memref<2xf32>) if (%cond)
82 // retain (%r0, %r1 : memref<1xf32>, memref<2xf32>)
83 /// return %0#0, %0#1 : i1, i1
84 /// ```
85 /// ```mlir
86 /// %m_base_pointer = memref.extract_aligned_pointer_as_index %m
87 /// %r0_base_pointer = memref.extract_aligned_pointer_as_index %r0
88 /// %r0_does_not_alias = arith.cmpi ne, %m_base_pointer, %r0_base_pointer
89 /// %r1_base_pointer = memref.extract_aligned_pointer_as_index %r1
90 /// %r1_does_not_alias = arith.cmpi ne, %m_base_pointer, %r1_base_pointer
91 /// %not_retained = arith.andi %r0_does_not_alias, %r1_does_not_alias : i1
92 /// %should_dealloc = arith.andi %not_retained, %cond : i1
93 /// scf.if %should_dealloc {
94 /// memref.dealloc %m : memref<2xf32>
95 /// }
96 /// %true = arith.constant true
97 /// %r0_does_alias = arith.xori %r0_does_not_alias, %true : i1
98 /// %r0_ownership = arith.andi %r0_does_alias, %cond : i1
99 /// %r1_does_alias = arith.xori %r1_does_not_alias, %true : i1
100 /// %r1_ownership = arith.andi %r1_does_alias, %cond : i1
101 /// return %r0_ownership, %r1_ownership : i1, i1
102 /// ```
103 LogicalResult rewriteOneMemrefMultipleRetainCase(
104 bufferization::DeallocOp op, OpAdaptor adaptor,
105 ConversionPatternRewriter &rewriter) const {
106 assert(adaptor.getMemrefs().size() == 1 && "expected only one memref");
107
108 // Compute the base pointer indices, compare all retained indices to the
109 // memref index to check if they alias.
110 SmallVector<Value> doesNotAliasList;
111 Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
112 builder&: rewriter, location: op->getLoc(), source: adaptor.getMemrefs()[0]);
113 for (Value retained : adaptor.getRetained()) {
114 Value retainedAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
115 builder&: rewriter, location: op->getLoc(), source: retained);
116 Value doesNotAlias = arith::CmpIOp::create(builder&: rewriter, location: op->getLoc(),
117 predicate: arith::CmpIPredicate::ne,
118 lhs: memrefAsIdx, rhs: retainedAsIdx);
119 doesNotAliasList.push_back(Elt: doesNotAlias);
120 }
121
122 // AND-reduce the list of booleans from above.
123 Value prev = doesNotAliasList.front();
124 for (Value doesNotAlias : ArrayRef(doesNotAliasList).drop_front())
125 prev = arith::AndIOp::create(builder&: rewriter, location: op->getLoc(), lhs: prev, rhs: doesNotAlias);
126
127 // Also consider the condition given by the dealloc operation and perform a
128 // conditional deallocation guarded by that value.
129 Value shouldDealloc = arith::AndIOp::create(builder&: rewriter, location: op->getLoc(), lhs: prev,
130 rhs: adaptor.getConditions()[0]);
131
132 scf::IfOp::create(builder&: rewriter, location: op.getLoc(), cond: shouldDealloc,
133 thenBuilder: [&](OpBuilder &builder, Location loc) {
134 memref::DeallocOp::create(builder, location: loc,
135 memref: adaptor.getMemrefs()[0]);
136 scf::YieldOp::create(builder, location: loc);
137 });
138
139 // Compute the replacement values for the dealloc operation results. This
140 // inserts an already canonicalized form of
141 // `select(does_alias_with_memref(r), memref_cond, false)` for each retained
142 // value r.
143 SmallVector<Value> replacements;
144 Value trueVal = arith::ConstantOp::create(builder&: rewriter, location: op->getLoc(),
145 value: rewriter.getBoolAttr(value: true));
146 for (Value doesNotAlias : doesNotAliasList) {
147 Value aliases =
148 arith::XOrIOp::create(builder&: rewriter, location: op->getLoc(), lhs: doesNotAlias, rhs: trueVal);
149 Value result = arith::AndIOp::create(builder&: rewriter, location: op->getLoc(), lhs: aliases,
150 rhs: adaptor.getConditions()[0]);
151 replacements.push_back(Elt: result);
152 }
153
154 rewriter.replaceOp(op, newValues: replacements);
155
156 return success();
157 }
158
159 /// Lowering that supports all features the dealloc operation has to offer. It
160 /// computes the base pointer of each memref (as an index), stores it in a
161 /// new memref helper structure and passes it to the helper function generated
162 /// in 'buildDeallocationHelperFunction'. The results are stored in two lists
163 /// (represented as memrefs) of booleans passed as arguments. The first list
164 /// stores whether the corresponding condition should be deallocated, the
165 /// second list stores the ownership of the retained values which can be used
166 /// to replace the result values of the `bufferization.dealloc` operation.
167 ///
168 /// Example:
169 /// ```
170 /// %0:2 = bufferization.dealloc (%m0, %m1 : memref<2xf32>, memref<5xf32>)
171 /// if (%cond0, %cond1)
172 /// retain (%r0, %r1 : memref<1xf32>, memref<2xf32>)
173 /// ```
174 /// lowers to (simplified):
175 /// ```
176 /// %c0 = arith.constant 0 : index
177 /// %c1 = arith.constant 1 : index
178 /// %dealloc_base_pointer_list = memref.alloc() : memref<2xindex>
179 /// %cond_list = memref.alloc() : memref<2xi1>
180 /// %retain_base_pointer_list = memref.alloc() : memref<2xindex>
181 /// %m0_base_pointer = memref.extract_aligned_pointer_as_index %m0
182 /// memref.store %m0_base_pointer, %dealloc_base_pointer_list[%c0]
183 /// %m1_base_pointer = memref.extract_aligned_pointer_as_index %m1
184 /// memref.store %m1_base_pointer, %dealloc_base_pointer_list[%c1]
185 /// memref.store %cond0, %cond_list[%c0]
186 /// memref.store %cond1, %cond_list[%c1]
187 /// %r0_base_pointer = memref.extract_aligned_pointer_as_index %r0
188 /// memref.store %r0_base_pointer, %retain_base_pointer_list[%c0]
189 /// %r1_base_pointer = memref.extract_aligned_pointer_as_index %r1
190 /// memref.store %r1_base_pointer, %retain_base_pointer_list[%c1]
191 /// %dyn_dealloc_base_pointer_list = memref.cast %dealloc_base_pointer_list :
192 /// memref<2xindex> to memref<?xindex>
193 /// %dyn_cond_list = memref.cast %cond_list : memref<2xi1> to memref<?xi1>
194 /// %dyn_retain_base_pointer_list = memref.cast %retain_base_pointer_list :
195 /// memref<2xindex> to memref<?xindex>
196 /// %dealloc_cond_out = memref.alloc() : memref<2xi1>
197 /// %ownership_out = memref.alloc() : memref<2xi1>
198 /// %dyn_dealloc_cond_out = memref.cast %dealloc_cond_out :
199 /// memref<2xi1> to memref<?xi1>
200 /// %dyn_ownership_out = memref.cast %ownership_out :
201 /// memref<2xi1> to memref<?xi1>
202 /// call @dealloc_helper(%dyn_dealloc_base_pointer_list,
203 /// %dyn_retain_base_pointer_list,
204 /// %dyn_cond_list,
205 /// %dyn_dealloc_cond_out,
206 /// %dyn_ownership_out) : (...)
207 /// %m0_dealloc_cond = memref.load %dyn_dealloc_cond_out[%c0] : memref<2xi1>
208 /// scf.if %m0_dealloc_cond {
209 /// memref.dealloc %m0 : memref<2xf32>
210 /// }
211 /// %m1_dealloc_cond = memref.load %dyn_dealloc_cond_out[%c1] : memref<2xi1>
212 /// scf.if %m1_dealloc_cond {
213 /// memref.dealloc %m1 : memref<5xf32>
214 /// }
215 /// %r0_ownership = memref.load %dyn_ownership_out[%c0] : memref<2xi1>
216 /// %r1_ownership = memref.load %dyn_ownership_out[%c1] : memref<2xi1>
217 /// memref.dealloc %dealloc_base_pointer_list : memref<2xindex>
218 /// memref.dealloc %retain_base_pointer_list : memref<2xindex>
219 /// memref.dealloc %cond_list : memref<2xi1>
220 /// memref.dealloc %dealloc_cond_out : memref<2xi1>
221 /// memref.dealloc %ownership_out : memref<2xi1>
222 /// // replace %0#0 with %r0_ownership
223 /// // replace %0#1 with %r1_ownership
224 /// ```
225 LogicalResult rewriteGeneralCase(bufferization::DeallocOp op,
226 OpAdaptor adaptor,
227 ConversionPatternRewriter &rewriter) const {
228 // Allocate two memrefs holding the base pointer indices of the list of
229 // memrefs to be deallocated and the ones to be retained. These can then be
230 // passed to the helper function and the for-loops can iterate over them.
231 // Without storing them to memrefs, we could not use for-loops but only a
232 // completely unrolled version of it, potentially leading to code-size
233 // blow-up.
234 Value toDeallocMemref = memref::AllocOp::create(
235 builder&: rewriter, location: op.getLoc(),
236 memrefType: MemRefType::get(shape: {(int64_t)adaptor.getMemrefs().size()},
237 elementType: rewriter.getIndexType()));
238 Value conditionMemref = memref::AllocOp::create(
239 builder&: rewriter, location: op.getLoc(),
240 memrefType: MemRefType::get(shape: {(int64_t)adaptor.getConditions().size()},
241 elementType: rewriter.getI1Type()));
242 Value toRetainMemref = memref::AllocOp::create(
243 builder&: rewriter, location: op.getLoc(),
244 memrefType: MemRefType::get(shape: {(int64_t)adaptor.getRetained().size()},
245 elementType: rewriter.getIndexType()));
246
247 auto getConstValue = [&](uint64_t value) -> Value {
248 return arith::ConstantOp::create(builder&: rewriter, location: op.getLoc(),
249 value: rewriter.getIndexAttr(value));
250 };
251
252 // Extract the base pointers of the memrefs as indices to check for aliasing
253 // at runtime.
254 for (auto [i, toDealloc] : llvm::enumerate(First: adaptor.getMemrefs())) {
255 Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
256 builder&: rewriter, location: op.getLoc(), source: toDealloc);
257 memref::StoreOp::create(builder&: rewriter, location: op.getLoc(), valueToStore: memrefAsIdx,
258 memref: toDeallocMemref, indices: getConstValue(i));
259 }
260
261 for (auto [i, cond] : llvm::enumerate(First: adaptor.getConditions()))
262 memref::StoreOp::create(builder&: rewriter, location: op.getLoc(), valueToStore: cond, memref: conditionMemref,
263 indices: getConstValue(i));
264
265 for (auto [i, toRetain] : llvm::enumerate(First: adaptor.getRetained())) {
266 Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create(
267 builder&: rewriter, location: op.getLoc(), source: toRetain);
268 memref::StoreOp::create(builder&: rewriter, location: op.getLoc(), valueToStore: memrefAsIdx,
269 memref: toRetainMemref, indices: getConstValue(i));
270 }
271
272 // Cast the allocated memrefs to dynamic shape because we want only one
273 // helper function no matter how many operands the bufferization.dealloc
274 // has.
275 Value castedDeallocMemref = memref::CastOp::create(
276 builder&: rewriter, location: op->getLoc(),
277 dest: MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getIndexType()),
278 source: toDeallocMemref);
279 Value castedCondsMemref = memref::CastOp::create(
280 builder&: rewriter, location: op->getLoc(),
281 dest: MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getI1Type()),
282 source: conditionMemref);
283 Value castedRetainMemref = memref::CastOp::create(
284 builder&: rewriter, location: op->getLoc(),
285 dest: MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getIndexType()),
286 source: toRetainMemref);
287
288 Value deallocCondsMemref = memref::AllocOp::create(
289 builder&: rewriter, location: op.getLoc(),
290 memrefType: MemRefType::get(shape: {(int64_t)adaptor.getMemrefs().size()},
291 elementType: rewriter.getI1Type()));
292 Value retainCondsMemref = memref::AllocOp::create(
293 builder&: rewriter, location: op.getLoc(),
294 memrefType: MemRefType::get(shape: {(int64_t)adaptor.getRetained().size()},
295 elementType: rewriter.getI1Type()));
296
297 Value castedDeallocCondsMemref = memref::CastOp::create(
298 builder&: rewriter, location: op->getLoc(),
299 dest: MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getI1Type()),
300 source: deallocCondsMemref);
301 Value castedRetainCondsMemref = memref::CastOp::create(
302 builder&: rewriter, location: op->getLoc(),
303 dest: MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getI1Type()),
304 source: retainCondsMemref);
305
306 Operation *symtableOp = op->getParentWithTrait<OpTrait::SymbolTable>();
307 func::CallOp::create(
308 builder&: rewriter, location: op.getLoc(), callee: deallocHelperFuncMap.lookup(Val: symtableOp),
309 operands: SmallVector<Value>{castedDeallocMemref, castedRetainMemref,
310 castedCondsMemref, castedDeallocCondsMemref,
311 castedRetainCondsMemref});
312
313 for (unsigned i = 0, e = adaptor.getMemrefs().size(); i < e; ++i) {
314 Value idxValue = getConstValue(i);
315 Value shouldDealloc = memref::LoadOp::create(
316 builder&: rewriter, location: op.getLoc(), memref: deallocCondsMemref, indices: idxValue);
317 scf::IfOp::create(builder&: rewriter, location: op.getLoc(), cond: shouldDealloc,
318 thenBuilder: [&](OpBuilder &builder, Location loc) {
319 memref::DeallocOp::create(builder, location: loc,
320 memref: adaptor.getMemrefs()[i]);
321 scf::YieldOp::create(builder, location: loc);
322 });
323 }
324
325 SmallVector<Value> replacements;
326 for (unsigned i = 0, e = adaptor.getRetained().size(); i < e; ++i) {
327 Value idxValue = getConstValue(i);
328 Value ownership = memref::LoadOp::create(builder&: rewriter, location: op.getLoc(),
329 memref: retainCondsMemref, indices: idxValue);
330 replacements.push_back(Elt: ownership);
331 }
332
333 // Deallocate above allocated memrefs again to avoid memory leaks.
334 // Deallocation will not be run on code after this stage.
335 memref::DeallocOp::create(builder&: rewriter, location: op.getLoc(), memref: toDeallocMemref);
336 memref::DeallocOp::create(builder&: rewriter, location: op.getLoc(), memref: toRetainMemref);
337 memref::DeallocOp::create(builder&: rewriter, location: op.getLoc(), memref: conditionMemref);
338 memref::DeallocOp::create(builder&: rewriter, location: op.getLoc(), memref: deallocCondsMemref);
339 memref::DeallocOp::create(builder&: rewriter, location: op.getLoc(), memref: retainCondsMemref);
340
341 rewriter.replaceOp(op, newValues: replacements);
342 return success();
343 }
344
345public:
346 DeallocOpConversion(
347 MLIRContext *context,
348 const bufferization::DeallocHelperMap &deallocHelperFuncMap)
349 : OpConversionPattern<bufferization::DeallocOp>(context),
350 deallocHelperFuncMap(deallocHelperFuncMap) {}
351
352 LogicalResult
353 matchAndRewrite(bufferization::DeallocOp op, OpAdaptor adaptor,
354 ConversionPatternRewriter &rewriter) const override {
355 // Lower the trivial case.
356 if (adaptor.getMemrefs().empty()) {
357 Value falseVal = arith::ConstantOp::create(builder&: rewriter, location: op.getLoc(),
358 value: rewriter.getBoolAttr(value: false));
359 rewriter.replaceOp(
360 op, newValues: SmallVector<Value>(adaptor.getRetained().size(), falseVal));
361 return success();
362 }
363
364 if (adaptor.getMemrefs().size() == 1 && adaptor.getRetained().empty())
365 return rewriteOneMemrefNoRetainCase(op, adaptor, rewriter);
366
367 if (adaptor.getMemrefs().size() == 1)
368 return rewriteOneMemrefMultipleRetainCase(op, adaptor, rewriter);
369
370 Operation *symtableOp = op->getParentWithTrait<OpTrait::SymbolTable>();
371 if (!deallocHelperFuncMap.contains(Val: symtableOp))
372 return op->emitError(
373 message: "library function required for generic lowering, but cannot be "
374 "automatically inserted when operating on functions");
375
376 return rewriteGeneralCase(op, adaptor, rewriter);
377 }
378
379private:
380 const bufferization::DeallocHelperMap &deallocHelperFuncMap;
381};
382} // namespace
383
384namespace {
385struct LowerDeallocationsPass
386 : public bufferization::impl::LowerDeallocationsPassBase<
387 LowerDeallocationsPass> {
388 void runOnOperation() override {
389 if (!isa<ModuleOp, FunctionOpInterface>(Val: getOperation())) {
390 emitError(loc: getOperation()->getLoc(),
391 message: "root operation must be a builtin.module or a function");
392 signalPassFailure();
393 return;
394 }
395
396 bufferization::DeallocHelperMap deallocHelperFuncMap;
397 if (auto module = dyn_cast<ModuleOp>(Val: getOperation())) {
398 OpBuilder builder = OpBuilder::atBlockBegin(block: module.getBody());
399
400 // Build dealloc helper function if there are deallocs.
401 getOperation()->walk(callback: [&](bufferization::DeallocOp deallocOp) {
402 Operation *symtableOp =
403 deallocOp->getParentWithTrait<OpTrait::SymbolTable>();
404 if (deallocOp.getMemrefs().size() > 1 &&
405 !deallocHelperFuncMap.contains(Val: symtableOp)) {
406 SymbolTable symbolTable(symtableOp);
407 func::FuncOp helperFuncOp =
408 bufferization::buildDeallocationLibraryFunction(
409 builder, loc: getOperation()->getLoc(), symbolTable);
410 deallocHelperFuncMap[symtableOp] = helperFuncOp;
411 }
412 });
413 }
414
415 RewritePatternSet patterns(&getContext());
416 bufferization::populateBufferizationDeallocLoweringPattern(
417 patterns, deallocHelperFuncMap);
418
419 ConversionTarget target(getContext());
420 target.addLegalDialect<memref::MemRefDialect, arith::ArithDialect,
421 scf::SCFDialect, func::FuncDialect>();
422 target.addIllegalOp<bufferization::DeallocOp>();
423
424 if (failed(Result: applyPartialConversion(op: getOperation(), target,
425 patterns: std::move(patterns))))
426 signalPassFailure();
427 }
428};
429} // namespace
430
431func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction(
432 OpBuilder &builder, Location loc, SymbolTable &symbolTable) {
433 Type indexMemrefType =
434 MemRefType::get(shape: {ShapedType::kDynamic}, elementType: builder.getIndexType());
435 Type boolMemrefType =
436 MemRefType::get(shape: {ShapedType::kDynamic}, elementType: builder.getI1Type());
437 SmallVector<Type> argTypes{indexMemrefType, indexMemrefType, boolMemrefType,
438 boolMemrefType, boolMemrefType};
439 builder.clearInsertionPoint();
440
441 // Generate the func operation itself.
442 auto helperFuncOp = func::FuncOp::create(
443 location: loc, name: "dealloc_helper", type: builder.getFunctionType(inputs: argTypes, results: {}));
444 helperFuncOp.setVisibility(SymbolTable::Visibility::Private);
445 symbolTable.insert(symbol: helperFuncOp);
446 auto &block = helperFuncOp.getFunctionBody().emplaceBlock();
447 block.addArguments(types: argTypes, locs: SmallVector<Location>(argTypes.size(), loc));
448
449 builder.setInsertionPointToStart(&block);
450 Value toDeallocMemref = helperFuncOp.getArguments()[0];
451 Value toRetainMemref = helperFuncOp.getArguments()[1];
452 Value conditionMemref = helperFuncOp.getArguments()[2];
453 Value deallocCondsMemref = helperFuncOp.getArguments()[3];
454 Value retainCondsMemref = helperFuncOp.getArguments()[4];
455
456 // Insert some prerequisites.
457 Value c0 = arith::ConstantOp::create(builder, location: loc, value: builder.getIndexAttr(value: 0));
458 Value c1 = arith::ConstantOp::create(builder, location: loc, value: builder.getIndexAttr(value: 1));
459 Value trueValue =
460 arith::ConstantOp::create(builder, location: loc, value: builder.getBoolAttr(value: true));
461 Value falseValue =
462 arith::ConstantOp::create(builder, location: loc, value: builder.getBoolAttr(value: false));
463 Value toDeallocSize =
464 memref::DimOp::create(builder, location: loc, source: toDeallocMemref, index: c0);
465 Value toRetainSize = memref::DimOp::create(builder, location: loc, source: toRetainMemref, index: c0);
466
467 scf::ForOp::create(
468 builder, location: loc, lowerBound: c0, upperBound: toRetainSize, step: c1, initArgs: ValueRange(),
469 odsArg4: [&](OpBuilder &builder, Location loc, Value i, ValueRange iterArgs) {
470 memref::StoreOp::create(builder, location: loc, valueToStore: falseValue, memref: retainCondsMemref, indices: i);
471 scf::YieldOp::create(builder, location: loc);
472 });
473
474 scf::ForOp::create(
475 builder, location: loc, lowerBound: c0, upperBound: toDeallocSize, step: c1, initArgs: ValueRange(),
476 odsArg4: [&](OpBuilder &builder, Location loc, Value outerIter,
477 ValueRange iterArgs) {
478 Value toDealloc =
479 memref::LoadOp::create(builder, location: loc, memref: toDeallocMemref, indices: outerIter);
480 Value cond =
481 memref::LoadOp::create(builder, location: loc, memref: conditionMemref, indices: outerIter);
482
483 // Build the first for loop that computes aliasing with retained
484 // memrefs.
485 Value
486 noRetainAlias =
487 scf::ForOp::create(
488 builder, location: loc, lowerBound: c0, upperBound: toRetainSize, step: c1, initArgs: trueValue,
489 odsArg4: [&](OpBuilder &builder, Location loc, Value i,
490 ValueRange iterArgs) {
491 Value retainValue = memref::LoadOp::create(
492 builder, location: loc, memref: toRetainMemref, indices: i);
493 Value doesAlias = arith::CmpIOp::create(
494 builder, location: loc, predicate: arith::CmpIPredicate::eq, lhs: retainValue,
495 rhs: toDealloc);
496 scf::IfOp::create(
497 builder, location: loc, cond: doesAlias,
498 thenBuilder: [&](OpBuilder &builder, Location loc) {
499 Value retainCondValue = memref::LoadOp::create(
500 builder, location: loc, memref: retainCondsMemref, indices: i);
501 Value aggregatedRetainCond = arith::OrIOp::create(
502 builder, location: loc, lhs: retainCondValue, rhs: cond);
503 memref::StoreOp::create(builder, location: loc,
504 valueToStore: aggregatedRetainCond,
505 memref: retainCondsMemref, indices: i);
506 scf::YieldOp::create(builder, location: loc);
507 });
508 Value doesntAlias = arith::CmpIOp::create(
509 builder, location: loc, predicate: arith::CmpIPredicate::ne, lhs: retainValue,
510 rhs: toDealloc);
511 Value yieldValue = arith::AndIOp::create(
512 builder, location: loc, lhs: iterArgs[0], rhs: doesntAlias);
513 scf::YieldOp::create(builder, location: loc, results: yieldValue);
514 })
515 .getResult(i: 0);
516
517 // Build the second for loop that adds aliasing with previously
518 // deallocated memrefs.
519 Value
520 noAlias =
521 scf::ForOp::create(
522 builder, location: loc, lowerBound: c0, upperBound: outerIter, step: c1, initArgs: noRetainAlias,
523 odsArg4: [&](OpBuilder &builder, Location loc, Value i,
524 ValueRange iterArgs) {
525 Value prevDeallocValue = memref::LoadOp::create(
526 builder, location: loc, memref: toDeallocMemref, indices: i);
527 Value doesntAlias = arith::CmpIOp::create(
528 builder, location: loc, predicate: arith::CmpIPredicate::ne,
529 lhs: prevDeallocValue, rhs: toDealloc);
530 Value yieldValue = arith::AndIOp::create(
531 builder, location: loc, lhs: iterArgs[0], rhs: doesntAlias);
532 scf::YieldOp::create(builder, location: loc, results: yieldValue);
533 })
534 .getResult(i: 0);
535
536 Value shouldDealoc = arith::AndIOp::create(builder, location: loc, lhs: noAlias, rhs: cond);
537 memref::StoreOp::create(builder, location: loc, valueToStore: shouldDealoc, memref: deallocCondsMemref,
538 indices: outerIter);
539 scf::YieldOp::create(builder, location: loc);
540 });
541
542 func::ReturnOp::create(builder, location: loc);
543 return helperFuncOp;
544}
545
546void mlir::bufferization::populateBufferizationDeallocLoweringPattern(
547 RewritePatternSet &patterns,
548 const bufferization::DeallocHelperMap &deallocHelperFuncMap) {
549 patterns.add<DeallocOpConversion>(arg: patterns.getContext(),
550 args: deallocHelperFuncMap);
551}
552