| 1 | //===- BufferResultsToOutParams.cpp - Calling convention conversion -------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Bufferization/IR/AllocationOpInterface.h" |
| 10 | #include "mlir/Dialect/Bufferization/Transforms/Passes.h" |
| 11 | |
| 12 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 13 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 14 | #include "mlir/IR/Operation.h" |
| 15 | |
| 16 | namespace mlir { |
| 17 | namespace bufferization { |
| 18 | #define GEN_PASS_DEF_BUFFERRESULTSTOOUTPARAMSPASS |
| 19 | #include "mlir/Dialect/Bufferization/Transforms/Passes.h.inc" |
| 20 | } // namespace bufferization |
| 21 | } // namespace mlir |
| 22 | |
| 23 | using namespace mlir; |
| 24 | using AllocationFn = bufferization::BufferResultsToOutParamsOpts::AllocationFn; |
| 25 | using MemCpyFn = bufferization::BufferResultsToOutParamsOpts::MemCpyFn; |
| 26 | using AllocDynamicSizesMap = |
| 27 | llvm::DenseMap<func::FuncOp, SmallVector<SmallVector<Value>>>; |
| 28 | |
| 29 | /// Return `true` if the given MemRef type has a fully dynamic layout. |
| 30 | static bool hasFullyDynamicLayoutMap(MemRefType type) { |
| 31 | int64_t offset; |
| 32 | SmallVector<int64_t, 4> strides; |
| 33 | if (failed(Result: type.getStridesAndOffset(strides, offset))) |
| 34 | return false; |
| 35 | if (!llvm::all_of(Range&: strides, P: ShapedType::isDynamic)) |
| 36 | return false; |
| 37 | if (ShapedType::isStatic(dValue: offset)) |
| 38 | return false; |
| 39 | return true; |
| 40 | } |
| 41 | |
| 42 | /// Return `true` if the given MemRef type has a static identity layout (i.e., |
| 43 | /// no layout). |
| 44 | static bool hasStaticIdentityLayout(MemRefType type) { |
| 45 | return type.getLayout().isIdentity(); |
| 46 | } |
| 47 | |
| 48 | /// Return the dynamic shapes of the `memref` based on the defining op. If the |
| 49 | /// complete dynamic shape fails to be captured, return an empty value. |
| 50 | /// Currently, only function block arguments are supported for capturing. |
| 51 | static SmallVector<Value> getDynamicSize(Value memref, func::FuncOp funcOp) { |
| 52 | Operation *defOp = memref.getDefiningOp(); |
| 53 | if (!defOp) |
| 54 | return {}; |
| 55 | auto operands = defOp->getOperands(); |
| 56 | SmallVector<Value> dynamicSizes; |
| 57 | for (Value size : operands) { |
| 58 | if (!isa<IndexType>(Val: size.getType())) |
| 59 | continue; |
| 60 | |
| 61 | BlockArgument sizeSrc = dyn_cast<BlockArgument>(Val&: size); |
| 62 | if (!sizeSrc) |
| 63 | return {}; |
| 64 | auto arguments = funcOp.getArguments(); |
| 65 | auto iter = llvm::find(Range&: arguments, Val: sizeSrc); |
| 66 | if (iter == arguments.end()) |
| 67 | return {}; |
| 68 | dynamicSizes.push_back(Elt: *iter); |
| 69 | } |
| 70 | return dynamicSizes; |
| 71 | } |
| 72 | |
| 73 | /// Returns the dynamic sizes at the callee, through the call relationship |
| 74 | /// between the caller and callee. |
| 75 | static SmallVector<Value> mapDynamicSizeAtCaller(func::CallOp call, |
| 76 | func::FuncOp callee, |
| 77 | ValueRange dynamicSizes) { |
| 78 | SmallVector<Value> mappedDynamicSizes; |
| 79 | for (Value size : dynamicSizes) { |
| 80 | for (auto [src, dst] : |
| 81 | llvm::zip_first(t: call.getOperands(), u: callee.getArguments())) { |
| 82 | if (size != dst) |
| 83 | continue; |
| 84 | mappedDynamicSizes.push_back(Elt: src); |
| 85 | } |
| 86 | } |
| 87 | assert(mappedDynamicSizes.size() == dynamicSizes.size() && |
| 88 | "could not find all dynamic sizes" ); |
| 89 | return mappedDynamicSizes; |
| 90 | } |
| 91 | |
| 92 | // Updates the func op and entry block. |
| 93 | // |
| 94 | // Any args appended to the entry block are added to `appendedEntryArgs`. |
| 95 | // If `addResultAttribute` is true, adds the unit attribute `bufferize.result` |
| 96 | // to each newly created function argument. |
| 97 | static LogicalResult |
| 98 | updateFuncOp(func::FuncOp func, |
| 99 | SmallVectorImpl<BlockArgument> &appendedEntryArgs, |
| 100 | bool addResultAttribute) { |
| 101 | auto functionType = func.getFunctionType(); |
| 102 | |
| 103 | // Collect information about the results will become appended arguments. |
| 104 | SmallVector<Type, 6> erasedResultTypes; |
| 105 | BitVector erasedResultIndices(functionType.getNumResults()); |
| 106 | for (const auto &resultType : llvm::enumerate(First: functionType.getResults())) { |
| 107 | if (auto memrefType = dyn_cast<MemRefType>(Val: resultType.value())) { |
| 108 | if (!hasStaticIdentityLayout(type: memrefType) && |
| 109 | !hasFullyDynamicLayoutMap(type: memrefType)) { |
| 110 | // Only buffers with static identity layout can be allocated. These can |
| 111 | // be casted to memrefs with fully dynamic layout map. Other layout maps |
| 112 | // are not supported. |
| 113 | return func->emitError() |
| 114 | << "cannot create out param for result with unsupported layout" ; |
| 115 | } |
| 116 | erasedResultIndices.set(resultType.index()); |
| 117 | erasedResultTypes.push_back(Elt: memrefType); |
| 118 | } |
| 119 | } |
| 120 | |
| 121 | // Add the new arguments to the function type. |
| 122 | auto newArgTypes = llvm::to_vector<6>( |
| 123 | Range: llvm::concat<const Type>(Ranges: functionType.getInputs(), Ranges&: erasedResultTypes)); |
| 124 | auto newFunctionType = FunctionType::get(context: func.getContext(), inputs: newArgTypes, |
| 125 | results: functionType.getResults()); |
| 126 | func.setType(newFunctionType); |
| 127 | |
| 128 | // Transfer the result attributes to arg attributes. |
| 129 | auto erasedIndicesIt = erasedResultIndices.set_bits_begin(); |
| 130 | for (int i = 0, e = erasedResultTypes.size(); i < e; ++i, ++erasedIndicesIt) { |
| 131 | func.setArgAttrs(index: functionType.getNumInputs() + i, |
| 132 | attributes: func.getResultAttrs(index: *erasedIndicesIt)); |
| 133 | if (addResultAttribute) |
| 134 | func.setArgAttr(index: functionType.getNumInputs() + i, |
| 135 | name: StringAttr::get(context: func.getContext(), bytes: "bufferize.result" ), |
| 136 | value: UnitAttr::get(context: func.getContext())); |
| 137 | } |
| 138 | |
| 139 | // Erase the results. |
| 140 | if (failed(Result: func.eraseResults(resultIndices: erasedResultIndices))) |
| 141 | return failure(); |
| 142 | |
| 143 | // Add the new arguments to the entry block if the function is not external. |
| 144 | if (func.isExternal()) |
| 145 | return success(); |
| 146 | Location loc = func.getLoc(); |
| 147 | for (Type type : erasedResultTypes) |
| 148 | appendedEntryArgs.push_back(Elt: func.front().addArgument(type, loc)); |
| 149 | |
| 150 | return success(); |
| 151 | } |
| 152 | |
| 153 | // Updates all ReturnOps in the scope of the given func::FuncOp by either |
| 154 | // keeping them as return values or copying the associated buffer contents into |
| 155 | // the given out-params. |
| 156 | static LogicalResult |
| 157 | updateReturnOps(func::FuncOp func, ArrayRef<BlockArgument> appendedEntryArgs, |
| 158 | AllocDynamicSizesMap &map, |
| 159 | const bufferization::BufferResultsToOutParamsOpts &options) { |
| 160 | auto res = func.walk(callback: [&](func::ReturnOp op) { |
| 161 | SmallVector<Value, 6> copyIntoOutParams; |
| 162 | SmallVector<Value, 6> keepAsReturnOperands; |
| 163 | for (Value operand : op.getOperands()) { |
| 164 | if (isa<MemRefType>(Val: operand.getType())) |
| 165 | copyIntoOutParams.push_back(Elt: operand); |
| 166 | else |
| 167 | keepAsReturnOperands.push_back(Elt: operand); |
| 168 | } |
| 169 | OpBuilder builder(op); |
| 170 | SmallVector<SmallVector<Value>> dynamicSizes; |
| 171 | // Map each returned memref to its primary out parameter. |
| 172 | DenseMap<Value, BlockArgument> primaryArgs; |
| 173 | // Delay erasure until all return operands have been inspected because an |
| 174 | // allocation may be returned multiple times. |
| 175 | SetVector<Operation *> toErase; |
| 176 | for (auto [orig, arg] : llvm::zip(t&: copyIntoOutParams, u&: appendedEntryArgs)) { |
| 177 | bool hoistStaticAllocs = |
| 178 | options.hoistStaticAllocs && |
| 179 | cast<MemRefType>(Val: orig.getType()).hasStaticShape(); |
| 180 | bool hoistDynamicAllocs = |
| 181 | options.hoistDynamicAllocs && |
| 182 | !cast<MemRefType>(Val: orig.getType()).hasStaticShape(); |
| 183 | // Keep non-hoistable allocations such as memref.realloc and |
| 184 | // bufferization.clone. Erasing them would discard the data they preserve |
| 185 | // or reproduce in their results. |
| 186 | auto allocOp = dyn_cast_if_present<bufferization::AllocationOpInterface>( |
| 187 | Val: orig.getDefiningOp()); |
| 188 | if ((hoistStaticAllocs || hoistDynamicAllocs) && allocOp && |
| 189 | allocOp.getHoistingKind() != HoistingKind::None) { |
| 190 | if (hoistDynamicAllocs) { |
| 191 | SmallVector<Value> dynamicSize = getDynamicSize(memref: orig, funcOp: func); |
| 192 | dynamicSizes.push_back(Elt: dynamicSize); |
| 193 | } |
| 194 | |
| 195 | auto [it, inserted] = primaryArgs.try_emplace(Key: orig, Args: arg); |
| 196 | // Repeated returns of the same allocation reuse the primary out |
| 197 | // parameter; copy its contents into the out parameter for this |
| 198 | // occurrence. |
| 199 | if (!inserted) { |
| 200 | if (failed(Result: options.memCpyFn(builder, op->getLoc(), it->second, arg))) |
| 201 | return WalkResult::interrupt(); |
| 202 | continue; |
| 203 | } |
| 204 | orig.replaceAllUsesWith(newValue: arg); |
| 205 | toErase.insert(X: orig.getDefiningOp()); |
| 206 | } else if (orig != arg) { |
| 207 | if (failed(Result: options.memCpyFn(builder, op.getLoc(), orig, arg))) |
| 208 | return WalkResult::interrupt(); |
| 209 | } |
| 210 | } |
| 211 | |
| 212 | for (Operation *eraseOp : toErase) |
| 213 | eraseOp->erase(); |
| 214 | |
| 215 | func::ReturnOp::create(builder, location: op.getLoc(), operands: keepAsReturnOperands); |
| 216 | op.erase(); |
| 217 | auto dynamicSizePair = |
| 218 | std::pair<func::FuncOp, SmallVector<SmallVector<Value>>>(func, |
| 219 | dynamicSizes); |
| 220 | map.insert(KV: dynamicSizePair); |
| 221 | return WalkResult::advance(); |
| 222 | }); |
| 223 | return failure(IsFailure: res.wasInterrupted()); |
| 224 | } |
| 225 | |
| 226 | // Updates all CallOps in the scope of the given ModuleOp by allocating |
| 227 | // temporary buffers for newly introduced out params. |
| 228 | static LogicalResult |
| 229 | updateCalls(ModuleOp module, const AllocDynamicSizesMap &map, |
| 230 | const bufferization::BufferResultsToOutParamsOpts &options) { |
| 231 | bool didFail = false; |
| 232 | SymbolTable symtab(module); |
| 233 | module.walk(callback: [&](func::CallOp op) { |
| 234 | auto callee = symtab.lookup<func::FuncOp>(name: op.getCallee()); |
| 235 | if (!callee) { |
| 236 | op.emitError() << "cannot find callee '" << op.getCallee() << "' in " |
| 237 | << "symbol table" ; |
| 238 | didFail = true; |
| 239 | return; |
| 240 | } |
| 241 | if (!options.filterFn(&callee)) |
| 242 | return; |
| 243 | if (callee.isPublic() && !options.modifyPublicFunctions) |
| 244 | return; |
| 245 | if (callee.isExternal()) |
| 246 | return; |
| 247 | |
| 248 | SmallVector<Value, 6> replaceWithNewCallResults; |
| 249 | SmallVector<Value, 6> replaceWithOutParams; |
| 250 | for (OpResult result : op.getResults()) { |
| 251 | if (isa<MemRefType>(Val: result.getType())) |
| 252 | replaceWithOutParams.push_back(Elt: result); |
| 253 | else |
| 254 | replaceWithNewCallResults.push_back(Elt: result); |
| 255 | } |
| 256 | SmallVector<Value, 6> outParams; |
| 257 | OpBuilder builder(op); |
| 258 | SmallVector<SmallVector<Value>> dynamicSizes = map.lookup(Val: callee); |
| 259 | size_t dynamicSizesIndex = 0; |
| 260 | for (Value memref : replaceWithOutParams) { |
| 261 | SmallVector<Value> dynamicSize = dynamicSizes.size() > dynamicSizesIndex |
| 262 | ? dynamicSizes[dynamicSizesIndex] |
| 263 | : SmallVector<Value>(); |
| 264 | bool memrefStaticShape = |
| 265 | cast<MemRefType>(Val: memref.getType()).hasStaticShape(); |
| 266 | if (!memrefStaticShape && dynamicSize.empty()) { |
| 267 | op.emitError() |
| 268 | << "cannot create out param for dynamically shaped result" ; |
| 269 | didFail = true; |
| 270 | return; |
| 271 | } |
| 272 | auto memrefType = cast<MemRefType>(Val: memref.getType()); |
| 273 | auto allocType = |
| 274 | MemRefType::get(shape: memrefType.getShape(), elementType: memrefType.getElementType(), |
| 275 | map: AffineMap(), memorySpace: memrefType.getMemorySpace()); |
| 276 | |
| 277 | if (memrefStaticShape) { |
| 278 | dynamicSize = {}; |
| 279 | } else { |
| 280 | ++dynamicSizesIndex; |
| 281 | dynamicSize = mapDynamicSizeAtCaller(call: op, callee, dynamicSizes: dynamicSize); |
| 282 | } |
| 283 | auto maybeOutParam = |
| 284 | options.allocationFn(builder, op.getLoc(), allocType, dynamicSize); |
| 285 | if (failed(Result: maybeOutParam)) { |
| 286 | op.emitError() << "failed to create allocation op" ; |
| 287 | didFail = true; |
| 288 | return; |
| 289 | } |
| 290 | Value outParam = maybeOutParam.value(); |
| 291 | if (!hasStaticIdentityLayout(type: memrefType)) { |
| 292 | // Layout maps are already checked in `updateFuncOp`. |
| 293 | assert(hasFullyDynamicLayoutMap(memrefType) && |
| 294 | "layout map not supported" ); |
| 295 | outParam = |
| 296 | memref::CastOp::create(builder, location: op.getLoc(), dest: memrefType, source: outParam); |
| 297 | } |
| 298 | memref.replaceAllUsesWith(newValue: outParam); |
| 299 | outParams.push_back(Elt: outParam); |
| 300 | } |
| 301 | |
| 302 | auto newOperands = llvm::to_vector<6>(Range: op.getOperands()); |
| 303 | newOperands.append(in_start: outParams.begin(), in_end: outParams.end()); |
| 304 | auto newResultTypes = llvm::map_to_vector<6>( |
| 305 | C&: replaceWithNewCallResults, F: [](Value v) { return v.getType(); }); |
| 306 | auto newCall = func::CallOp::create( |
| 307 | builder, location: op.getLoc(), callee: op.getCalleeAttr(), results: newResultTypes, operands: newOperands); |
| 308 | for (auto t : llvm::zip(t&: replaceWithNewCallResults, u: newCall.getResults())) |
| 309 | std::get<0>(t&: t).replaceAllUsesWith(newValue: std::get<1>(t&: t)); |
| 310 | op.erase(); |
| 311 | }); |
| 312 | |
| 313 | return failure(IsFailure: didFail); |
| 314 | } |
| 315 | |
| 316 | LogicalResult mlir::bufferization::promoteBufferResultsToOutParams( |
| 317 | ModuleOp module, |
| 318 | const bufferization::BufferResultsToOutParamsOpts &options) { |
| 319 | // It maps the shape source of the dynamic shape memref returned by each |
| 320 | // function. |
| 321 | AllocDynamicSizesMap map; |
| 322 | for (auto func : module.getOps<func::FuncOp>()) { |
| 323 | if (func.isPublic() && !options.modifyPublicFunctions) |
| 324 | continue; |
| 325 | if (func.isExternal()) |
| 326 | continue; |
| 327 | if (!options.filterFn(&func)) |
| 328 | continue; |
| 329 | SmallVector<BlockArgument, 6> appendedEntryArgs; |
| 330 | if (failed( |
| 331 | Result: updateFuncOp(func, appendedEntryArgs, addResultAttribute: options.addResultAttribute))) |
| 332 | return failure(); |
| 333 | if (failed(Result: updateReturnOps(func, appendedEntryArgs, map, options))) { |
| 334 | return failure(); |
| 335 | } |
| 336 | } |
| 337 | if (failed(Result: updateCalls(module, map, options))) |
| 338 | return failure(); |
| 339 | return success(); |
| 340 | } |
| 341 | |
| 342 | namespace { |
| 343 | struct BufferResultsToOutParamsPass |
| 344 | : bufferization::impl::BufferResultsToOutParamsPassBase< |
| 345 | BufferResultsToOutParamsPass> { |
| 346 | using Base::Base; |
| 347 | |
| 348 | void runOnOperation() override { |
| 349 | // Convert from pass options in tablegen to BufferResultsToOutParamsOpts. |
| 350 | if (addResultAttribute) |
| 351 | options.addResultAttribute = true; |
| 352 | if (hoistStaticAllocs) |
| 353 | options.hoistStaticAllocs = true; |
| 354 | if (hoistDynamicAllocs) |
| 355 | options.hoistDynamicAllocs = true; |
| 356 | if (modifyPublicFunctions) |
| 357 | options.modifyPublicFunctions = true; |
| 358 | |
| 359 | if (failed(Result: bufferization::promoteBufferResultsToOutParams(module: getOperation(), |
| 360 | options))) |
| 361 | return signalPassFailure(); |
| 362 | } |
| 363 | |
| 364 | private: |
| 365 | bufferization::BufferResultsToOutParamsOpts options; |
| 366 | }; |
| 367 | } // namespace |
| 368 | |