| 1 | //===- LowerDeallocations.cpp - Bufferization Deallocs to MemRef pass -----===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements patterns to convert `bufferization.dealloc` operations |
| 10 | // to the MemRef dialect. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 15 | #include "mlir/Dialect/Bufferization/IR/Bufferization.h" |
| 16 | #include "mlir/Dialect/Bufferization/Transforms/Passes.h" |
| 17 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 18 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 19 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 20 | #include "mlir/IR/BuiltinTypes.h" |
| 21 | #include "mlir/Transforms/DialectConversion.h" |
| 22 | |
| 23 | namespace mlir { |
| 24 | namespace bufferization { |
| 25 | #define GEN_PASS_DEF_LOWERDEALLOCATIONSPASS |
| 26 | #include "mlir/Dialect/Bufferization/Transforms/Passes.h.inc" |
| 27 | } // namespace bufferization |
| 28 | } // namespace mlir |
| 29 | |
| 30 | using namespace mlir; |
| 31 | |
| 32 | namespace { |
| 33 | /// The DeallocOpConversion transforms all bufferization dealloc operations into |
| 34 | /// memref dealloc operations potentially guarded by scf if operations. |
| 35 | /// Additionally, memref extract_aligned_pointer_as_index and arith operations |
| 36 | /// are inserted to compute the guard conditions. We distinguish multiple cases |
| 37 | /// to provide an overall more efficient lowering. In the general case, a helper |
| 38 | /// func is created to avoid quadratic code size explosion (relative to the |
| 39 | /// number of operands of the dealloc operation). For examples of each case, |
| 40 | /// refer to the documentation of the member functions of this class. |
| 41 | class DeallocOpConversion |
| 42 | : public OpConversionPattern<bufferization::DeallocOp> { |
| 43 | |
| 44 | /// Lower a simple case without any retained values and a single memref to |
| 45 | /// avoiding the helper function. Ideally, static analysis can provide enough |
| 46 | /// aliasing information to split the dealloc operations up into this simple |
| 47 | /// case as much as possible before running this pass. |
| 48 | /// |
| 49 | /// Example: |
| 50 | /// ``` |
| 51 | /// bufferization.dealloc (%arg0 : memref<2xf32>) if (%arg1) |
| 52 | /// ``` |
| 53 | /// is lowered to |
| 54 | /// ``` |
| 55 | /// scf.if %arg1 { |
| 56 | /// memref.dealloc %arg0 : memref<2xf32> |
| 57 | /// } |
| 58 | /// ``` |
| 59 | LogicalResult |
| 60 | rewriteOneMemrefNoRetainCase(bufferization::DeallocOp op, OpAdaptor adaptor, |
| 61 | ConversionPatternRewriter &rewriter) const { |
| 62 | assert(adaptor.getMemrefs().size() == 1 && "expected only one memref" ); |
| 63 | assert(adaptor.getRetained().empty() && "expected no retained memrefs" ); |
| 64 | |
| 65 | rewriter.replaceOpWithNewOp<scf::IfOp>( |
| 66 | op, args: adaptor.getConditions()[0], args: [&](OpBuilder &builder, Location loc) { |
| 67 | memref::DeallocOp::create(builder, location: loc, memref: adaptor.getMemrefs()[0]); |
| 68 | scf::YieldOp::create(builder, location: loc); |
| 69 | }); |
| 70 | return success(); |
| 71 | } |
| 72 | |
| 73 | /// A special case lowering for the deallocation operation with exactly one |
| 74 | /// memref, but arbitrary number of retained values. This avoids the helper |
| 75 | /// function that the general case needs and thus also avoids storing indices |
| 76 | /// to specifically allocated memrefs. The size of the code produced by this |
| 77 | /// lowering is linear to the number of retained values. |
| 78 | /// |
| 79 | /// Example: |
| 80 | /// ```mlir |
| 81 | /// %0:2 = bufferization.dealloc (%m : memref<2xf32>) if (%cond) |
| 82 | // retain (%r0, %r1 : memref<1xf32>, memref<2xf32>) |
| 83 | /// return %0#0, %0#1 : i1, i1 |
| 84 | /// ``` |
| 85 | /// ```mlir |
| 86 | /// %m_base_pointer = memref.extract_aligned_pointer_as_index %m |
| 87 | /// %r0_base_pointer = memref.extract_aligned_pointer_as_index %r0 |
| 88 | /// %r0_does_not_alias = arith.cmpi ne, %m_base_pointer, %r0_base_pointer |
| 89 | /// %r1_base_pointer = memref.extract_aligned_pointer_as_index %r1 |
| 90 | /// %r1_does_not_alias = arith.cmpi ne, %m_base_pointer, %r1_base_pointer |
| 91 | /// %not_retained = arith.andi %r0_does_not_alias, %r1_does_not_alias : i1 |
| 92 | /// %should_dealloc = arith.andi %not_retained, %cond : i1 |
| 93 | /// scf.if %should_dealloc { |
| 94 | /// memref.dealloc %m : memref<2xf32> |
| 95 | /// } |
| 96 | /// %true = arith.constant true |
| 97 | /// %r0_does_alias = arith.xori %r0_does_not_alias, %true : i1 |
| 98 | /// %r0_ownership = arith.andi %r0_does_alias, %cond : i1 |
| 99 | /// %r1_does_alias = arith.xori %r1_does_not_alias, %true : i1 |
| 100 | /// %r1_ownership = arith.andi %r1_does_alias, %cond : i1 |
| 101 | /// return %r0_ownership, %r1_ownership : i1, i1 |
| 102 | /// ``` |
| 103 | LogicalResult rewriteOneMemrefMultipleRetainCase( |
| 104 | bufferization::DeallocOp op, OpAdaptor adaptor, |
| 105 | ConversionPatternRewriter &rewriter) const { |
| 106 | assert(adaptor.getMemrefs().size() == 1 && "expected only one memref" ); |
| 107 | |
| 108 | // Compute the base pointer indices, compare all retained indices to the |
| 109 | // memref index to check if they alias. |
| 110 | SmallVector<Value> doesNotAliasList; |
| 111 | Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create( |
| 112 | builder&: rewriter, location: op->getLoc(), source: adaptor.getMemrefs()[0]); |
| 113 | for (Value retained : adaptor.getRetained()) { |
| 114 | Value retainedAsIdx = memref::ExtractAlignedPointerAsIndexOp::create( |
| 115 | builder&: rewriter, location: op->getLoc(), source: retained); |
| 116 | Value doesNotAlias = arith::CmpIOp::create(builder&: rewriter, location: op->getLoc(), |
| 117 | predicate: arith::CmpIPredicate::ne, |
| 118 | lhs: memrefAsIdx, rhs: retainedAsIdx); |
| 119 | doesNotAliasList.push_back(Elt: doesNotAlias); |
| 120 | } |
| 121 | |
| 122 | // AND-reduce the list of booleans from above. |
| 123 | Value prev = doesNotAliasList.front(); |
| 124 | for (Value doesNotAlias : ArrayRef(doesNotAliasList).drop_front()) |
| 125 | prev = arith::AndIOp::create(builder&: rewriter, location: op->getLoc(), lhs: prev, rhs: doesNotAlias); |
| 126 | |
| 127 | // Also consider the condition given by the dealloc operation and perform a |
| 128 | // conditional deallocation guarded by that value. |
| 129 | Value shouldDealloc = arith::AndIOp::create(builder&: rewriter, location: op->getLoc(), lhs: prev, |
| 130 | rhs: adaptor.getConditions()[0]); |
| 131 | |
| 132 | scf::IfOp::create(builder&: rewriter, location: op.getLoc(), cond: shouldDealloc, |
| 133 | thenBuilder: [&](OpBuilder &builder, Location loc) { |
| 134 | memref::DeallocOp::create(builder, location: loc, |
| 135 | memref: adaptor.getMemrefs()[0]); |
| 136 | scf::YieldOp::create(builder, location: loc); |
| 137 | }); |
| 138 | |
| 139 | // Compute the replacement values for the dealloc operation results. This |
| 140 | // inserts an already canonicalized form of |
| 141 | // `select(does_alias_with_memref(r), memref_cond, false)` for each retained |
| 142 | // value r. |
| 143 | SmallVector<Value> replacements; |
| 144 | Value trueVal = arith::ConstantOp::create(builder&: rewriter, location: op->getLoc(), |
| 145 | value: rewriter.getBoolAttr(value: true)); |
| 146 | for (Value doesNotAlias : doesNotAliasList) { |
| 147 | Value aliases = |
| 148 | arith::XOrIOp::create(builder&: rewriter, location: op->getLoc(), lhs: doesNotAlias, rhs: trueVal); |
| 149 | Value result = arith::AndIOp::create(builder&: rewriter, location: op->getLoc(), lhs: aliases, |
| 150 | rhs: adaptor.getConditions()[0]); |
| 151 | replacements.push_back(Elt: result); |
| 152 | } |
| 153 | |
| 154 | rewriter.replaceOp(op, newValues: replacements); |
| 155 | |
| 156 | return success(); |
| 157 | } |
| 158 | |
| 159 | /// Lowering that supports all features the dealloc operation has to offer. It |
| 160 | /// computes the base pointer of each memref (as an index), stores it in a |
| 161 | /// new memref helper structure and passes it to the helper function generated |
| 162 | /// in 'buildDeallocationHelperFunction'. The results are stored in two lists |
| 163 | /// (represented as memrefs) of booleans passed as arguments. The first list |
| 164 | /// stores whether the corresponding condition should be deallocated, the |
| 165 | /// second list stores the ownership of the retained values which can be used |
| 166 | /// to replace the result values of the `bufferization.dealloc` operation. |
| 167 | /// |
| 168 | /// Example: |
| 169 | /// ``` |
| 170 | /// %0:2 = bufferization.dealloc (%m0, %m1 : memref<2xf32>, memref<5xf32>) |
| 171 | /// if (%cond0, %cond1) |
| 172 | /// retain (%r0, %r1 : memref<1xf32>, memref<2xf32>) |
| 173 | /// ``` |
| 174 | /// lowers to (simplified): |
| 175 | /// ``` |
| 176 | /// %c0 = arith.constant 0 : index |
| 177 | /// %c1 = arith.constant 1 : index |
| 178 | /// %dealloc_base_pointer_list = memref.alloc() : memref<2xindex> |
| 179 | /// %cond_list = memref.alloc() : memref<2xi1> |
| 180 | /// %retain_base_pointer_list = memref.alloc() : memref<2xindex> |
| 181 | /// %m0_base_pointer = memref.extract_aligned_pointer_as_index %m0 |
| 182 | /// memref.store %m0_base_pointer, %dealloc_base_pointer_list[%c0] |
| 183 | /// %m1_base_pointer = memref.extract_aligned_pointer_as_index %m1 |
| 184 | /// memref.store %m1_base_pointer, %dealloc_base_pointer_list[%c1] |
| 185 | /// memref.store %cond0, %cond_list[%c0] |
| 186 | /// memref.store %cond1, %cond_list[%c1] |
| 187 | /// %r0_base_pointer = memref.extract_aligned_pointer_as_index %r0 |
| 188 | /// memref.store %r0_base_pointer, %retain_base_pointer_list[%c0] |
| 189 | /// %r1_base_pointer = memref.extract_aligned_pointer_as_index %r1 |
| 190 | /// memref.store %r1_base_pointer, %retain_base_pointer_list[%c1] |
| 191 | /// %dyn_dealloc_base_pointer_list = memref.cast %dealloc_base_pointer_list : |
| 192 | /// memref<2xindex> to memref<?xindex> |
| 193 | /// %dyn_cond_list = memref.cast %cond_list : memref<2xi1> to memref<?xi1> |
| 194 | /// %dyn_retain_base_pointer_list = memref.cast %retain_base_pointer_list : |
| 195 | /// memref<2xindex> to memref<?xindex> |
| 196 | /// %dealloc_cond_out = memref.alloc() : memref<2xi1> |
| 197 | /// %ownership_out = memref.alloc() : memref<2xi1> |
| 198 | /// %dyn_dealloc_cond_out = memref.cast %dealloc_cond_out : |
| 199 | /// memref<2xi1> to memref<?xi1> |
| 200 | /// %dyn_ownership_out = memref.cast %ownership_out : |
| 201 | /// memref<2xi1> to memref<?xi1> |
| 202 | /// call @dealloc_helper(%dyn_dealloc_base_pointer_list, |
| 203 | /// %dyn_retain_base_pointer_list, |
| 204 | /// %dyn_cond_list, |
| 205 | /// %dyn_dealloc_cond_out, |
| 206 | /// %dyn_ownership_out) : (...) |
| 207 | /// %m0_dealloc_cond = memref.load %dyn_dealloc_cond_out[%c0] : memref<2xi1> |
| 208 | /// scf.if %m0_dealloc_cond { |
| 209 | /// memref.dealloc %m0 : memref<2xf32> |
| 210 | /// } |
| 211 | /// %m1_dealloc_cond = memref.load %dyn_dealloc_cond_out[%c1] : memref<2xi1> |
| 212 | /// scf.if %m1_dealloc_cond { |
| 213 | /// memref.dealloc %m1 : memref<5xf32> |
| 214 | /// } |
| 215 | /// %r0_ownership = memref.load %dyn_ownership_out[%c0] : memref<2xi1> |
| 216 | /// %r1_ownership = memref.load %dyn_ownership_out[%c1] : memref<2xi1> |
| 217 | /// memref.dealloc %dealloc_base_pointer_list : memref<2xindex> |
| 218 | /// memref.dealloc %retain_base_pointer_list : memref<2xindex> |
| 219 | /// memref.dealloc %cond_list : memref<2xi1> |
| 220 | /// memref.dealloc %dealloc_cond_out : memref<2xi1> |
| 221 | /// memref.dealloc %ownership_out : memref<2xi1> |
| 222 | /// // replace %0#0 with %r0_ownership |
| 223 | /// // replace %0#1 with %r1_ownership |
| 224 | /// ``` |
| 225 | LogicalResult rewriteGeneralCase(bufferization::DeallocOp op, |
| 226 | OpAdaptor adaptor, |
| 227 | ConversionPatternRewriter &rewriter) const { |
| 228 | // Allocate two memrefs holding the base pointer indices of the list of |
| 229 | // memrefs to be deallocated and the ones to be retained. These can then be |
| 230 | // passed to the helper function and the for-loops can iterate over them. |
| 231 | // Without storing them to memrefs, we could not use for-loops but only a |
| 232 | // completely unrolled version of it, potentially leading to code-size |
| 233 | // blow-up. |
| 234 | Value toDeallocMemref = memref::AllocOp::create( |
| 235 | builder&: rewriter, location: op.getLoc(), |
| 236 | memrefType: MemRefType::get(shape: {(int64_t)adaptor.getMemrefs().size()}, |
| 237 | elementType: rewriter.getIndexType())); |
| 238 | Value conditionMemref = memref::AllocOp::create( |
| 239 | builder&: rewriter, location: op.getLoc(), |
| 240 | memrefType: MemRefType::get(shape: {(int64_t)adaptor.getConditions().size()}, |
| 241 | elementType: rewriter.getI1Type())); |
| 242 | Value toRetainMemref = memref::AllocOp::create( |
| 243 | builder&: rewriter, location: op.getLoc(), |
| 244 | memrefType: MemRefType::get(shape: {(int64_t)adaptor.getRetained().size()}, |
| 245 | elementType: rewriter.getIndexType())); |
| 246 | |
| 247 | auto getConstValue = [&](uint64_t value) -> Value { |
| 248 | return arith::ConstantOp::create(builder&: rewriter, location: op.getLoc(), |
| 249 | value: rewriter.getIndexAttr(value)); |
| 250 | }; |
| 251 | |
| 252 | // Extract the base pointers of the memrefs as indices to check for aliasing |
| 253 | // at runtime. |
| 254 | for (auto [i, toDealloc] : llvm::enumerate(First: adaptor.getMemrefs())) { |
| 255 | Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create( |
| 256 | builder&: rewriter, location: op.getLoc(), source: toDealloc); |
| 257 | memref::StoreOp::create(builder&: rewriter, location: op.getLoc(), valueToStore: memrefAsIdx, |
| 258 | memref: toDeallocMemref, indices: getConstValue(i)); |
| 259 | } |
| 260 | |
| 261 | for (auto [i, cond] : llvm::enumerate(First: adaptor.getConditions())) |
| 262 | memref::StoreOp::create(builder&: rewriter, location: op.getLoc(), valueToStore: cond, memref: conditionMemref, |
| 263 | indices: getConstValue(i)); |
| 264 | |
| 265 | for (auto [i, toRetain] : llvm::enumerate(First: adaptor.getRetained())) { |
| 266 | Value memrefAsIdx = memref::ExtractAlignedPointerAsIndexOp::create( |
| 267 | builder&: rewriter, location: op.getLoc(), source: toRetain); |
| 268 | memref::StoreOp::create(builder&: rewriter, location: op.getLoc(), valueToStore: memrefAsIdx, |
| 269 | memref: toRetainMemref, indices: getConstValue(i)); |
| 270 | } |
| 271 | |
| 272 | // Cast the allocated memrefs to dynamic shape because we want only one |
| 273 | // helper function no matter how many operands the bufferization.dealloc |
| 274 | // has. |
| 275 | Value castedDeallocMemref = memref::CastOp::create( |
| 276 | builder&: rewriter, location: op->getLoc(), |
| 277 | dest: MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getIndexType()), |
| 278 | source: toDeallocMemref); |
| 279 | Value castedCondsMemref = memref::CastOp::create( |
| 280 | builder&: rewriter, location: op->getLoc(), |
| 281 | dest: MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getI1Type()), |
| 282 | source: conditionMemref); |
| 283 | Value castedRetainMemref = memref::CastOp::create( |
| 284 | builder&: rewriter, location: op->getLoc(), |
| 285 | dest: MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getIndexType()), |
| 286 | source: toRetainMemref); |
| 287 | |
| 288 | Value deallocCondsMemref = memref::AllocOp::create( |
| 289 | builder&: rewriter, location: op.getLoc(), |
| 290 | memrefType: MemRefType::get(shape: {(int64_t)adaptor.getMemrefs().size()}, |
| 291 | elementType: rewriter.getI1Type())); |
| 292 | Value retainCondsMemref = memref::AllocOp::create( |
| 293 | builder&: rewriter, location: op.getLoc(), |
| 294 | memrefType: MemRefType::get(shape: {(int64_t)adaptor.getRetained().size()}, |
| 295 | elementType: rewriter.getI1Type())); |
| 296 | |
| 297 | Value castedDeallocCondsMemref = memref::CastOp::create( |
| 298 | builder&: rewriter, location: op->getLoc(), |
| 299 | dest: MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getI1Type()), |
| 300 | source: deallocCondsMemref); |
| 301 | Value castedRetainCondsMemref = memref::CastOp::create( |
| 302 | builder&: rewriter, location: op->getLoc(), |
| 303 | dest: MemRefType::get(shape: {ShapedType::kDynamic}, elementType: rewriter.getI1Type()), |
| 304 | source: retainCondsMemref); |
| 305 | |
| 306 | Operation *symtableOp = op->getParentWithTrait<OpTrait::SymbolTable>(); |
| 307 | func::CallOp::create( |
| 308 | builder&: rewriter, location: op.getLoc(), callee: deallocHelperFuncMap.lookup(Val: symtableOp), |
| 309 | operands: SmallVector<Value>{castedDeallocMemref, castedRetainMemref, |
| 310 | castedCondsMemref, castedDeallocCondsMemref, |
| 311 | castedRetainCondsMemref}); |
| 312 | |
| 313 | for (unsigned i = 0, e = adaptor.getMemrefs().size(); i < e; ++i) { |
| 314 | Value idxValue = getConstValue(i); |
| 315 | Value shouldDealloc = memref::LoadOp::create( |
| 316 | builder&: rewriter, location: op.getLoc(), memref: deallocCondsMemref, indices: idxValue); |
| 317 | scf::IfOp::create(builder&: rewriter, location: op.getLoc(), cond: shouldDealloc, |
| 318 | thenBuilder: [&](OpBuilder &builder, Location loc) { |
| 319 | memref::DeallocOp::create(builder, location: loc, |
| 320 | memref: adaptor.getMemrefs()[i]); |
| 321 | scf::YieldOp::create(builder, location: loc); |
| 322 | }); |
| 323 | } |
| 324 | |
| 325 | SmallVector<Value> replacements; |
| 326 | for (unsigned i = 0, e = adaptor.getRetained().size(); i < e; ++i) { |
| 327 | Value idxValue = getConstValue(i); |
| 328 | Value ownership = memref::LoadOp::create(builder&: rewriter, location: op.getLoc(), |
| 329 | memref: retainCondsMemref, indices: idxValue); |
| 330 | replacements.push_back(Elt: ownership); |
| 331 | } |
| 332 | |
| 333 | // Deallocate above allocated memrefs again to avoid memory leaks. |
| 334 | // Deallocation will not be run on code after this stage. |
| 335 | memref::DeallocOp::create(builder&: rewriter, location: op.getLoc(), memref: toDeallocMemref); |
| 336 | memref::DeallocOp::create(builder&: rewriter, location: op.getLoc(), memref: toRetainMemref); |
| 337 | memref::DeallocOp::create(builder&: rewriter, location: op.getLoc(), memref: conditionMemref); |
| 338 | memref::DeallocOp::create(builder&: rewriter, location: op.getLoc(), memref: deallocCondsMemref); |
| 339 | memref::DeallocOp::create(builder&: rewriter, location: op.getLoc(), memref: retainCondsMemref); |
| 340 | |
| 341 | rewriter.replaceOp(op, newValues: replacements); |
| 342 | return success(); |
| 343 | } |
| 344 | |
| 345 | public: |
| 346 | DeallocOpConversion( |
| 347 | MLIRContext *context, |
| 348 | const bufferization::DeallocHelperMap &deallocHelperFuncMap) |
| 349 | : OpConversionPattern<bufferization::DeallocOp>(context), |
| 350 | deallocHelperFuncMap(deallocHelperFuncMap) {} |
| 351 | |
| 352 | LogicalResult |
| 353 | matchAndRewrite(bufferization::DeallocOp op, OpAdaptor adaptor, |
| 354 | ConversionPatternRewriter &rewriter) const override { |
| 355 | // Lower the trivial case. |
| 356 | if (adaptor.getMemrefs().empty()) { |
| 357 | Value falseVal = arith::ConstantOp::create(builder&: rewriter, location: op.getLoc(), |
| 358 | value: rewriter.getBoolAttr(value: false)); |
| 359 | rewriter.replaceOp( |
| 360 | op, newValues: SmallVector<Value>(adaptor.getRetained().size(), falseVal)); |
| 361 | return success(); |
| 362 | } |
| 363 | |
| 364 | if (adaptor.getMemrefs().size() == 1 && adaptor.getRetained().empty()) |
| 365 | return rewriteOneMemrefNoRetainCase(op, adaptor, rewriter); |
| 366 | |
| 367 | if (adaptor.getMemrefs().size() == 1) |
| 368 | return rewriteOneMemrefMultipleRetainCase(op, adaptor, rewriter); |
| 369 | |
| 370 | Operation *symtableOp = op->getParentWithTrait<OpTrait::SymbolTable>(); |
| 371 | if (!deallocHelperFuncMap.contains(Val: symtableOp)) |
| 372 | return op->emitError( |
| 373 | message: "library function required for generic lowering, but cannot be " |
| 374 | "automatically inserted when operating on functions" ); |
| 375 | |
| 376 | return rewriteGeneralCase(op, adaptor, rewriter); |
| 377 | } |
| 378 | |
| 379 | private: |
| 380 | const bufferization::DeallocHelperMap &deallocHelperFuncMap; |
| 381 | }; |
| 382 | } // namespace |
| 383 | |
| 384 | namespace { |
| 385 | struct LowerDeallocationsPass |
| 386 | : public bufferization::impl::LowerDeallocationsPassBase< |
| 387 | LowerDeallocationsPass> { |
| 388 | void runOnOperation() override { |
| 389 | if (!isa<ModuleOp, FunctionOpInterface>(Val: getOperation())) { |
| 390 | emitError(loc: getOperation()->getLoc(), |
| 391 | message: "root operation must be a builtin.module or a function" ); |
| 392 | signalPassFailure(); |
| 393 | return; |
| 394 | } |
| 395 | |
| 396 | bufferization::DeallocHelperMap deallocHelperFuncMap; |
| 397 | if (auto module = dyn_cast<ModuleOp>(Val: getOperation())) { |
| 398 | OpBuilder builder = OpBuilder::atBlockBegin(block: module.getBody()); |
| 399 | |
| 400 | // Build dealloc helper function if there are deallocs. |
| 401 | getOperation()->walk(callback: [&](bufferization::DeallocOp deallocOp) { |
| 402 | Operation *symtableOp = |
| 403 | deallocOp->getParentWithTrait<OpTrait::SymbolTable>(); |
| 404 | if (deallocOp.getMemrefs().size() > 1 && |
| 405 | !deallocHelperFuncMap.contains(Val: symtableOp)) { |
| 406 | SymbolTable symbolTable(symtableOp); |
| 407 | func::FuncOp helperFuncOp = |
| 408 | bufferization::buildDeallocationLibraryFunction( |
| 409 | builder, loc: getOperation()->getLoc(), symbolTable); |
| 410 | deallocHelperFuncMap[symtableOp] = helperFuncOp; |
| 411 | } |
| 412 | }); |
| 413 | } |
| 414 | |
| 415 | RewritePatternSet patterns(&getContext()); |
| 416 | bufferization::populateBufferizationDeallocLoweringPattern( |
| 417 | patterns, deallocHelperFuncMap); |
| 418 | |
| 419 | ConversionTarget target(getContext()); |
| 420 | target.addLegalDialect<memref::MemRefDialect, arith::ArithDialect, |
| 421 | scf::SCFDialect, func::FuncDialect>(); |
| 422 | target.addIllegalOp<bufferization::DeallocOp>(); |
| 423 | |
| 424 | if (failed(Result: applyPartialConversion(op: getOperation(), target, |
| 425 | patterns: std::move(patterns)))) |
| 426 | signalPassFailure(); |
| 427 | } |
| 428 | }; |
| 429 | } // namespace |
| 430 | |
| 431 | func::FuncOp mlir::bufferization::buildDeallocationLibraryFunction( |
| 432 | OpBuilder &builder, Location loc, SymbolTable &symbolTable) { |
| 433 | Type indexMemrefType = |
| 434 | MemRefType::get(shape: {ShapedType::kDynamic}, elementType: builder.getIndexType()); |
| 435 | Type boolMemrefType = |
| 436 | MemRefType::get(shape: {ShapedType::kDynamic}, elementType: builder.getI1Type()); |
| 437 | SmallVector<Type> argTypes{indexMemrefType, indexMemrefType, boolMemrefType, |
| 438 | boolMemrefType, boolMemrefType}; |
| 439 | builder.clearInsertionPoint(); |
| 440 | |
| 441 | // Generate the func operation itself. |
| 442 | auto helperFuncOp = func::FuncOp::create( |
| 443 | location: loc, name: "dealloc_helper" , type: builder.getFunctionType(inputs: argTypes, results: {})); |
| 444 | helperFuncOp.setVisibility(SymbolTable::Visibility::Private); |
| 445 | symbolTable.insert(symbol: helperFuncOp); |
| 446 | auto &block = helperFuncOp.getFunctionBody().emplaceBlock(); |
| 447 | block.addArguments(types: argTypes, locs: SmallVector<Location>(argTypes.size(), loc)); |
| 448 | |
| 449 | builder.setInsertionPointToStart(&block); |
| 450 | Value toDeallocMemref = helperFuncOp.getArguments()[0]; |
| 451 | Value toRetainMemref = helperFuncOp.getArguments()[1]; |
| 452 | Value conditionMemref = helperFuncOp.getArguments()[2]; |
| 453 | Value deallocCondsMemref = helperFuncOp.getArguments()[3]; |
| 454 | Value retainCondsMemref = helperFuncOp.getArguments()[4]; |
| 455 | |
| 456 | // Insert some prerequisites. |
| 457 | Value c0 = arith::ConstantOp::create(builder, location: loc, value: builder.getIndexAttr(value: 0)); |
| 458 | Value c1 = arith::ConstantOp::create(builder, location: loc, value: builder.getIndexAttr(value: 1)); |
| 459 | Value trueValue = |
| 460 | arith::ConstantOp::create(builder, location: loc, value: builder.getBoolAttr(value: true)); |
| 461 | Value falseValue = |
| 462 | arith::ConstantOp::create(builder, location: loc, value: builder.getBoolAttr(value: false)); |
| 463 | Value toDeallocSize = |
| 464 | memref::DimOp::create(builder, location: loc, source: toDeallocMemref, index: c0); |
| 465 | Value toRetainSize = memref::DimOp::create(builder, location: loc, source: toRetainMemref, index: c0); |
| 466 | |
| 467 | scf::ForOp::create( |
| 468 | builder, location: loc, lowerBound: c0, upperBound: toRetainSize, step: c1, initArgs: ValueRange(), |
| 469 | odsArg4: [&](OpBuilder &builder, Location loc, Value i, ValueRange iterArgs) { |
| 470 | memref::StoreOp::create(builder, location: loc, valueToStore: falseValue, memref: retainCondsMemref, indices: i); |
| 471 | scf::YieldOp::create(builder, location: loc); |
| 472 | }); |
| 473 | |
| 474 | scf::ForOp::create( |
| 475 | builder, location: loc, lowerBound: c0, upperBound: toDeallocSize, step: c1, initArgs: ValueRange(), |
| 476 | odsArg4: [&](OpBuilder &builder, Location loc, Value outerIter, |
| 477 | ValueRange iterArgs) { |
| 478 | Value toDealloc = |
| 479 | memref::LoadOp::create(builder, location: loc, memref: toDeallocMemref, indices: outerIter); |
| 480 | Value cond = |
| 481 | memref::LoadOp::create(builder, location: loc, memref: conditionMemref, indices: outerIter); |
| 482 | |
| 483 | // Build the first for loop that computes aliasing with retained |
| 484 | // memrefs. |
| 485 | Value |
| 486 | noRetainAlias = |
| 487 | scf::ForOp::create( |
| 488 | builder, location: loc, lowerBound: c0, upperBound: toRetainSize, step: c1, initArgs: trueValue, |
| 489 | odsArg4: [&](OpBuilder &builder, Location loc, Value i, |
| 490 | ValueRange iterArgs) { |
| 491 | Value retainValue = memref::LoadOp::create( |
| 492 | builder, location: loc, memref: toRetainMemref, indices: i); |
| 493 | Value doesAlias = arith::CmpIOp::create( |
| 494 | builder, location: loc, predicate: arith::CmpIPredicate::eq, lhs: retainValue, |
| 495 | rhs: toDealloc); |
| 496 | scf::IfOp::create( |
| 497 | builder, location: loc, cond: doesAlias, |
| 498 | thenBuilder: [&](OpBuilder &builder, Location loc) { |
| 499 | Value retainCondValue = memref::LoadOp::create( |
| 500 | builder, location: loc, memref: retainCondsMemref, indices: i); |
| 501 | Value aggregatedRetainCond = arith::OrIOp::create( |
| 502 | builder, location: loc, lhs: retainCondValue, rhs: cond); |
| 503 | memref::StoreOp::create(builder, location: loc, |
| 504 | valueToStore: aggregatedRetainCond, |
| 505 | memref: retainCondsMemref, indices: i); |
| 506 | scf::YieldOp::create(builder, location: loc); |
| 507 | }); |
| 508 | Value doesntAlias = arith::CmpIOp::create( |
| 509 | builder, location: loc, predicate: arith::CmpIPredicate::ne, lhs: retainValue, |
| 510 | rhs: toDealloc); |
| 511 | Value yieldValue = arith::AndIOp::create( |
| 512 | builder, location: loc, lhs: iterArgs[0], rhs: doesntAlias); |
| 513 | scf::YieldOp::create(builder, location: loc, results: yieldValue); |
| 514 | }) |
| 515 | .getResult(i: 0); |
| 516 | |
| 517 | // Build the second for loop that adds aliasing with previously |
| 518 | // deallocated memrefs. |
| 519 | Value |
| 520 | noAlias = |
| 521 | scf::ForOp::create( |
| 522 | builder, location: loc, lowerBound: c0, upperBound: outerIter, step: c1, initArgs: noRetainAlias, |
| 523 | odsArg4: [&](OpBuilder &builder, Location loc, Value i, |
| 524 | ValueRange iterArgs) { |
| 525 | Value prevDeallocValue = memref::LoadOp::create( |
| 526 | builder, location: loc, memref: toDeallocMemref, indices: i); |
| 527 | Value doesntAlias = arith::CmpIOp::create( |
| 528 | builder, location: loc, predicate: arith::CmpIPredicate::ne, |
| 529 | lhs: prevDeallocValue, rhs: toDealloc); |
| 530 | Value yieldValue = arith::AndIOp::create( |
| 531 | builder, location: loc, lhs: iterArgs[0], rhs: doesntAlias); |
| 532 | scf::YieldOp::create(builder, location: loc, results: yieldValue); |
| 533 | }) |
| 534 | .getResult(i: 0); |
| 535 | |
| 536 | Value shouldDealoc = arith::AndIOp::create(builder, location: loc, lhs: noAlias, rhs: cond); |
| 537 | memref::StoreOp::create(builder, location: loc, valueToStore: shouldDealoc, memref: deallocCondsMemref, |
| 538 | indices: outerIter); |
| 539 | scf::YieldOp::create(builder, location: loc); |
| 540 | }); |
| 541 | |
| 542 | func::ReturnOp::create(builder, location: loc); |
| 543 | return helperFuncOp; |
| 544 | } |
| 545 | |
| 546 | void mlir::bufferization::populateBufferizationDeallocLoweringPattern( |
| 547 | RewritePatternSet &patterns, |
| 548 | const bufferization::DeallocHelperMap &deallocHelperFuncMap) { |
| 549 | patterns.add<DeallocOpConversion>(arg: patterns.getContext(), |
| 550 | args: deallocHelperFuncMap); |
| 551 | } |
| 552 | |