| 1 | //===- OneShotModuleBufferize.cpp - Bufferization across Func. Boundaries |
| 2 | //----===// |
| 3 | // |
| 4 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 5 | // See https://llvm.org/LICENSE.txt for license information. |
| 6 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 7 | // |
| 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // Module Bufferization is an extension of One-Shot Bufferize that |
| 11 | // bufferizes function boundaries. It provides `BufferizableOpInterface` |
| 12 | // implementations for FuncOp, CallOp and ReturnOp. Although it is named |
| 13 | // Module Bufferization, it may operate on any SymbolTable. |
| 14 | // |
| 15 | // Module Bufferization is run via `runOneShotModuleBufferize(SymbolTableOp, |
| 16 | // ...)`. This function analyzes the given op and determines the order of |
| 17 | // analysis and bufferization: Functions that are called are processed before |
| 18 | // their respective callers. |
| 19 | // |
| 20 | // After analyzing a FuncOp, additional information about its bbArgs is |
| 21 | // gathered and stored in `FuncAnalysisState`. |
| 22 | // |
| 23 | // * `aliasingFuncOpBBArgsAnalysis` determines the equivalent/aliasing bbArgs |
| 24 | // for |
| 25 | // each tensor return value (if any). |
| 26 | // * `funcOpBbArgReadWriteAnalysis` determines whether or not a tensor bbArg is |
| 27 | // read/written. |
| 28 | // * `funcOpReturnValueWritabilityAnalysis` determines whether returned buffers |
| 29 | // are writable. |
| 30 | // |
| 31 | // Module Bufferization implements the following calling convention. |
| 32 | // |
| 33 | // * In the absence of conflicts within a FuncOp, the FuncOp's bbArgs may always |
| 34 | // be written to in-place. |
| 35 | // * If a tensor operand of a CallOp is read after the CallOp, the operand of |
| 36 | // the CallOp must bufferize out-of-place. |
| 37 | // |
| 38 | // Example: The tensor.insert op bufferizes in-place because it is allowed to |
| 39 | // modify the buffer of `%t1` directly. The CallOp in `caller` must bufferize |
| 40 | // out-of-place because `%t0` is modified by the callee but read by the |
| 41 | // tensor.extract op. The analysis of CallOps decides whether an OpOperand must |
| 42 | // bufferize out-of-place based on results of `funcOpBbArgReadWriteAnalysis`. |
| 43 | // ``` |
| 44 | // func @callee(%t1 : tensor<?xf32>) -> tensor<?xf32> { |
| 45 | // %f = ... : f32 |
| 46 | // %0 = tensor.insert %f into %t1[...] : tensor<?xf32> |
| 47 | // return %0 : tensor<?xf32> |
| 48 | // } |
| 49 | // |
| 50 | // func @caller() -> () { |
| 51 | // %t0 = ... : tensor<?xf32> |
| 52 | // %1 = call @callee(%t0) : (tensor<?xf32>) -> (tensor<?xf32>) |
| 53 | // %2 = tensor.extract %1[...] : tensor<?xf32> |
| 54 | // } |
| 55 | // ``` |
| 56 | // |
| 57 | // Note: If a function is external, `funcOpBbArgReadWriteAnalysis` cannot |
| 58 | // analyze the function body. In such a case, the CallOp analysis conservatively |
| 59 | // assumes that each tensor OpOperand is both read and written. |
| 60 | // |
| 61 | // TODO: Add FuncOp attributes so that bbArgs of external FuncOps can be marked |
| 62 | // as "not reading" and/or "not writing". |
| 63 | |
| 64 | #include "mlir/Dialect/Bufferization/Transforms/OneShotModuleBufferize.h" |
| 65 | #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" |
| 66 | #include "mlir/Dialect/Bufferization/IR/Bufferization.h" |
| 67 | #include "mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h" |
| 68 | #include "mlir/Dialect/Bufferization/Transforms/Bufferize.h" |
| 69 | #include "mlir/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.h" |
| 70 | #include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h" |
| 71 | #include "mlir/Dialect/Bufferization/Transforms/Transforms.h" |
| 72 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 73 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 74 | #include "mlir/IR/BuiltinTypes.h" |
| 75 | #include "mlir/IR/Operation.h" |
| 76 | #include "llvm/ADT/MapVector.h" |
| 77 | #include "llvm/ADT/SmallVectorExtras.h" |
| 78 | |
| 79 | using namespace mlir; |
| 80 | using namespace mlir::bufferization; |
| 81 | using namespace mlir::bufferization::func_ext; |
| 82 | |
| 83 | /// A mapping of FuncOps to their callers. |
| 84 | using FuncCallerMap = DenseMap<func::FuncOp, DenseSet<Operation *>>; |
| 85 | |
| 86 | /// Get or create FuncAnalysisState. |
| 87 | static FuncAnalysisState & |
| 88 | getOrCreateFuncAnalysisState(OneShotAnalysisState &state) { |
| 89 | auto *result = state.getExtension<FuncAnalysisState>(); |
| 90 | if (result) |
| 91 | return *result; |
| 92 | return state.addExtension<FuncAnalysisState>(); |
| 93 | } |
| 94 | |
| 95 | namespace { |
| 96 | |
| 97 | /// Annotate IR with the results of the analysis. For testing purposes only. |
| 98 | static void annotateEquivalentReturnBbArg(OpOperand &returnVal, |
| 99 | BlockArgument bbArg) { |
| 100 | const char *kEquivalentArgsAttr = "__equivalent_func_args__" ; |
| 101 | Operation *op = returnVal.getOwner(); |
| 102 | |
| 103 | SmallVector<int64_t> equivBbArgs; |
| 104 | if (op->hasDiscardableAttr(name: kEquivalentArgsAttr)) { |
| 105 | auto attr = cast<ArrayAttr>(Val: op->getDiscardableAttr(name: kEquivalentArgsAttr)); |
| 106 | equivBbArgs = llvm::map_to_vector<4>(C&: attr, F: [](Attribute a) { |
| 107 | return cast<IntegerAttr>(Val&: a).getValue().getSExtValue(); |
| 108 | }); |
| 109 | } else { |
| 110 | equivBbArgs.append(NumInputs: op->getNumOperands(), Elt: -1); |
| 111 | } |
| 112 | equivBbArgs[returnVal.getOperandNumber()] = bbArg.getArgNumber(); |
| 113 | |
| 114 | OpBuilder b(op->getContext()); |
| 115 | op->setDiscardableAttr(name: kEquivalentArgsAttr, value: b.getI64ArrayAttr(values: equivBbArgs)); |
| 116 | } |
| 117 | |
| 118 | /// Store function BlockArguments that are equivalent to/aliasing a returned |
| 119 | /// value in FuncAnalysisState. |
| 120 | static LogicalResult |
| 121 | aliasingFuncOpBBArgsAnalysis(FuncOp funcOp, OneShotAnalysisState &state, |
| 122 | FuncAnalysisState &funcState) { |
| 123 | if (funcOp.getBody().empty()) { |
| 124 | // No function body available. Conservatively assume that every tensor |
| 125 | // return value may alias with any tensor bbArg. |
| 126 | FunctionType type = funcOp.getFunctionType(); |
| 127 | for (const auto &inputIt : llvm::enumerate(First: type.getInputs())) { |
| 128 | if (!isa<TensorLikeType>(Val: inputIt.value())) |
| 129 | continue; |
| 130 | for (const auto &resultIt : llvm::enumerate(First: type.getResults())) { |
| 131 | if (!isa<TensorLikeType>(Val: resultIt.value())) |
| 132 | continue; |
| 133 | int64_t returnIdx = resultIt.index(); |
| 134 | int64_t bbArgIdx = inputIt.index(); |
| 135 | funcState.aliasingReturnVals[funcOp][bbArgIdx].push_back(Elt: returnIdx); |
| 136 | } |
| 137 | } |
| 138 | return success(); |
| 139 | } |
| 140 | |
| 141 | // Find all func.return ops. |
| 142 | SmallVector<func::ReturnOp> returnOps = getReturnOps(funcOp); |
| 143 | // TODO: throw error when there is any non-func.return op that has the |
| 144 | // ReturnLike trait |
| 145 | if (returnOps.empty()) { |
| 146 | return funcOp.emitError(message: "cannot bufferize func.func without func.return" ); |
| 147 | } |
| 148 | |
| 149 | // Build alias sets. Merge all aliases from all func.return ops. |
| 150 | for (BlockArgument bbArg : funcOp.getArguments()) { |
| 151 | if (isa<TensorLikeType>(Val: bbArg.getType())) { |
| 152 | int64_t bbArgIdx = bbArg.getArgNumber(); |
| 153 | // Store aliases in a set, so that we don't add the same alias twice. |
| 154 | SetVector<int64_t> aliases; |
| 155 | for (func::ReturnOp returnOp : returnOps) { |
| 156 | for (OpOperand &returnVal : returnOp->getOpOperands()) { |
| 157 | if (isa<TensorLikeType>(Val: returnVal.get().getType())) { |
| 158 | int64_t returnIdx = returnVal.getOperandNumber(); |
| 159 | if (state.areAliasingBufferizedValues(v1: returnVal.get(), v2: bbArg)) |
| 160 | aliases.insert(X: returnIdx); |
| 161 | } |
| 162 | } |
| 163 | } |
| 164 | for (int64_t alias : aliases) |
| 165 | funcState.aliasingReturnVals[funcOp][bbArgIdx].push_back(Elt: alias); |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | // Build equivalence sets. |
| 170 | // Helper function that finds an equivalent block argument index for the |
| 171 | // given OpOperand. Return std::nullopt if no equivalent block argument could |
| 172 | // be found. |
| 173 | auto findEquivalentBlockArgIdx = |
| 174 | [&](OpOperand &opOperand) -> std::optional<int64_t> { |
| 175 | Value v = opOperand.get(); |
| 176 | if (!isa<TensorLikeType>(Val: v.getType())) |
| 177 | return std::nullopt; |
| 178 | for (BlockArgument bbArg : funcOp.getArguments()) { |
| 179 | if (isa<TensorLikeType>(Val: bbArg.getType())) { |
| 180 | if (state.areEquivalentBufferizedValues(v1: v, v2: bbArg)) { |
| 181 | if (state.getOptions().testAnalysisOnly) |
| 182 | annotateEquivalentReturnBbArg(returnVal&: opOperand, bbArg); |
| 183 | return bbArg.getArgNumber(); |
| 184 | } |
| 185 | } |
| 186 | } |
| 187 | return std::nullopt; |
| 188 | }; |
| 189 | |
| 190 | int64_t numResults = returnOps.front()->getNumOperands(); |
| 191 | for (int64_t i = 0; i < numResults; ++i) { |
| 192 | // Find the equivalent block argument index for the i-th operand of the |
| 193 | // first func.return op. |
| 194 | std::optional<int64_t> maybeEquiv = |
| 195 | findEquivalentBlockArgIdx(returnOps.front()->getOpOperand(idx: i)); |
| 196 | if (!maybeEquiv.has_value()) |
| 197 | continue; |
| 198 | int64_t bbArgIdx = *maybeEquiv; |
| 199 | bool allEquiv = true; |
| 200 | |
| 201 | // Check if all other func.return ops have the same equivalent block |
| 202 | // argument for the i-th operand. In contrast to aliasing information, |
| 203 | // which is just "merged", equivalence information must match across all |
| 204 | // func.return ops. |
| 205 | for (func::ReturnOp returnOp : ArrayRef(returnOps).drop_front()) { |
| 206 | std::optional<int64_t> maybeEquiv = |
| 207 | findEquivalentBlockArgIdx(returnOp->getOpOperand(idx: i)); |
| 208 | if (maybeEquiv != bbArgIdx) { |
| 209 | allEquiv = false; |
| 210 | break; |
| 211 | } |
| 212 | } |
| 213 | |
| 214 | // All func.return ops have the same equivalent block argument for the i-th |
| 215 | // operand. |
| 216 | if (allEquiv) |
| 217 | funcState.equivalentFuncArgs[funcOp][i] = bbArgIdx; |
| 218 | } |
| 219 | |
| 220 | return success(); |
| 221 | } |
| 222 | |
| 223 | static void annotateFuncArgAccess(func::FuncOp funcOp, int64_t idx, bool isRead, |
| 224 | bool isWritten) { |
| 225 | OpBuilder b(funcOp.getContext()); |
| 226 | Attribute accessType; |
| 227 | if (isRead && isWritten) { |
| 228 | accessType = b.getStringAttr(bytes: "read-write" ); |
| 229 | } else if (isRead) { |
| 230 | accessType = b.getStringAttr(bytes: "read" ); |
| 231 | } else if (isWritten) { |
| 232 | accessType = b.getStringAttr(bytes: "write" ); |
| 233 | } else { |
| 234 | accessType = b.getStringAttr(bytes: "none" ); |
| 235 | } |
| 236 | funcOp.setArgAttr(index: idx, name: BufferizationDialect::kBufferAccessAttrName, |
| 237 | value: accessType); |
| 238 | } |
| 239 | |
| 240 | /// Determine which FuncOp bbArgs are read and which are written. When run on a |
| 241 | /// function with unknown ops, we conservatively assume that such ops bufferize |
| 242 | /// to a read + write. |
| 243 | static LogicalResult |
| 244 | funcOpBbArgReadWriteAnalysis(FuncOp funcOp, OneShotAnalysisState &state, |
| 245 | FuncAnalysisState &funcState) { |
| 246 | for (int64_t idx = 0, e = funcOp.getFunctionType().getNumInputs(); idx < e; |
| 247 | ++idx) { |
| 248 | // Skip non-tensor arguments. |
| 249 | if (!isa<TensorLikeType>(Val: funcOp.getFunctionType().getInput(i: idx))) |
| 250 | continue; |
| 251 | bool isRead; |
| 252 | bool isWritten; |
| 253 | if (auto accessAttr = funcOp.getArgAttrOfType<StringAttr>( |
| 254 | index: idx, name: BufferizationDialect::kBufferAccessAttrName)) { |
| 255 | // Buffer access behavior is specified on the function. Skip the analysis. |
| 256 | StringRef str = accessAttr.getValue(); |
| 257 | isRead = str == "read" || str == "read-write" ; |
| 258 | isWritten = str == "write" || str == "read-write" ; |
| 259 | } else if (funcOp.getBody().empty()) { |
| 260 | // If the function has no body, conservatively assume that all args are |
| 261 | // read + written. |
| 262 | isRead = true; |
| 263 | isWritten = true; |
| 264 | } else { |
| 265 | // Analyze the body of the function. |
| 266 | BlockArgument bbArg = funcOp.getArgument(idx); |
| 267 | isRead = state.isValueRead(value: bbArg); |
| 268 | isWritten = state.isValueWritten(value: bbArg); |
| 269 | } |
| 270 | |
| 271 | if (state.getOptions().testAnalysisOnly) |
| 272 | annotateFuncArgAccess(funcOp, idx, isRead, isWritten); |
| 273 | if (isRead) |
| 274 | funcState.readBbArgs[funcOp].insert(V: idx); |
| 275 | if (isWritten) |
| 276 | funcState.writtenBbArgs[funcOp].insert(V: idx); |
| 277 | } |
| 278 | |
| 279 | return success(); |
| 280 | } |
| 281 | |
| 282 | /// Determine which tensor return values are writable at every return site. |
| 283 | static void funcOpReturnValueWritabilityAnalysis(FuncOp funcOp, |
| 284 | OneShotAnalysisState &state, |
| 285 | FuncAnalysisState &funcState) { |
| 286 | // Without a body, none of the returned buffers are known to be writable. |
| 287 | if (funcOp.isExternal()) |
| 288 | return; |
| 289 | |
| 290 | SmallVector<func::ReturnOp> returnOps = getReturnOps(funcOp); |
| 291 | for (auto [index, type] : llvm::enumerate(First: funcOp.getResultTypes())) { |
| 292 | if (!isa<TensorLikeType>(Val: type)) |
| 293 | continue; |
| 294 | bool isWritable = true; |
| 295 | for (func::ReturnOp returnOp : returnOps) { |
| 296 | // A view or a forwarded value may itself be writable but alias a |
| 297 | // read-only buffer, such as an arith.constant or a callee result. |
| 298 | state.applyOnAliases(v: returnOp.getOperand(i: index), fun: [&](Value alias) { |
| 299 | isWritable &= state.isWritable(value: alias); |
| 300 | }); |
| 301 | } |
| 302 | if (isWritable) |
| 303 | funcState.writableReturnVals[funcOp].insert(V: index); |
| 304 | } |
| 305 | } |
| 306 | } // namespace |
| 307 | |
| 308 | /// Remove bufferization attributes on FuncOp arguments. |
| 309 | static void removeBufferizationAttributes(BlockArgument bbArg) { |
| 310 | auto funcOp = cast<func::FuncOp>(Val: bbArg.getOwner()->getParentOp()); |
| 311 | funcOp.removeArgAttr(index: bbArg.getArgNumber(), |
| 312 | name: BufferizationDialect::kBufferLayoutAttrName); |
| 313 | funcOp.removeArgAttr(index: bbArg.getArgNumber(), |
| 314 | name: BufferizationDialect::kWritableAttrName); |
| 315 | } |
| 316 | |
| 317 | /// Return the func::FuncOp called by `callOp`. |
| 318 | static func::FuncOp |
| 319 | getCalledFunction(func::CallOp callOp, |
| 320 | mlir::SymbolTableCollection &symbolTable) { |
| 321 | return dyn_cast_or_null<func::FuncOp>( |
| 322 | Val: callOp.resolveCallableInTable(symbolTable: &symbolTable)); |
| 323 | } |
| 324 | |
| 325 | /// Return "true" if the given function signature has tensor semantics. |
| 326 | static bool hasTensorSignature(func::FuncOp funcOp) { |
| 327 | return llvm::any_of(Range: funcOp.getFunctionType().getInputs(), |
| 328 | P: llvm::IsaPred<TensorLikeType>) || |
| 329 | llvm::any_of(Range: funcOp.getFunctionType().getResults(), |
| 330 | P: llvm::IsaPred<TensorLikeType>); |
| 331 | } |
| 332 | |
| 333 | /// Store all functions of the `moduleOp` in `orderedFuncOps`, sorted by |
| 334 | /// callee-caller order (i.e., callees without callers first). Store all |
| 335 | /// remaining functions (i.e., the ones that call each other recursively) in |
| 336 | /// `remainingFuncOps`. Does not traverse nested symbol tables. |
| 337 | /// |
| 338 | /// Store the map of FuncOp to all its callers in `callerMap`. |
| 339 | /// |
| 340 | /// Return `failure()` if we are unable to retrieve the called FuncOp from |
| 341 | /// any func::CallOp. |
| 342 | static LogicalResult getFuncOpsOrderedByCalls( |
| 343 | Operation *moduleOp, SmallVectorImpl<func::FuncOp> &orderedFuncOps, |
| 344 | SmallVectorImpl<func::FuncOp> &remainingFuncOps, FuncCallerMap &callerMap, |
| 345 | SymbolTableCollection &symbolTables) { |
| 346 | // For each FuncOp, the set of functions called by it (i.e. the union of |
| 347 | // symbols of all nested func::CallOp). |
| 348 | DenseMap<func::FuncOp, SetVector<func::FuncOp>> calledBy; |
| 349 | // For each FuncOp, the number of func::CallOp it contains. |
| 350 | llvm::MapVector<func::FuncOp, unsigned> numberCallOpsContainedInFuncOp; |
| 351 | for (mlir::Region ®ion : moduleOp->getRegions()) { |
| 352 | for (mlir::Block &block : region.getBlocks()) { |
| 353 | for (func::FuncOp funcOp : block.getOps<func::FuncOp>()) { |
| 354 | // Collect function calls and populate the caller map. |
| 355 | numberCallOpsContainedInFuncOp[funcOp] = 0; |
| 356 | WalkResult res = funcOp.walk(callback: [&](func::CallOp callOp) -> WalkResult { |
| 357 | func::FuncOp calledFunction = getCalledFunction(callOp, symbolTable&: symbolTables); |
| 358 | assert(calledFunction && "could not retrieved called func::FuncOp" ); |
| 359 | // If the called function does not have any tensors in its signature, |
| 360 | // then it is not necessary to bufferize the callee before the caller. |
| 361 | if (!hasTensorSignature(funcOp: calledFunction)) |
| 362 | return WalkResult::skip(); |
| 363 | |
| 364 | callerMap[calledFunction].insert(V: callOp); |
| 365 | if (calledBy[calledFunction].insert(X: funcOp)) { |
| 366 | numberCallOpsContainedInFuncOp[funcOp]++; |
| 367 | } |
| 368 | return WalkResult::advance(); |
| 369 | }); |
| 370 | if (res.wasInterrupted()) |
| 371 | return failure(); |
| 372 | } |
| 373 | } |
| 374 | } |
| 375 | |
| 376 | // Iteratively remove function operations that do not call any of the |
| 377 | // functions remaining in the callCounter map and add them to ordered list. |
| 378 | SmallVector<func::FuncOp> worklist; |
| 379 | |
| 380 | for (const auto &entry : numberCallOpsContainedInFuncOp) { |
| 381 | if (entry.second == 0) |
| 382 | worklist.push_back(Elt: entry.first); |
| 383 | } |
| 384 | |
| 385 | while (!worklist.empty()) { |
| 386 | func::FuncOp func = worklist.pop_back_val(); |
| 387 | orderedFuncOps.push_back(Elt: func); |
| 388 | |
| 389 | for (func::FuncOp caller : calledBy[func]) { |
| 390 | auto &count = numberCallOpsContainedInFuncOp[caller]; |
| 391 | |
| 392 | if (--count == 0) |
| 393 | worklist.push_back(Elt: caller); |
| 394 | } |
| 395 | |
| 396 | numberCallOpsContainedInFuncOp.erase(Key: func); |
| 397 | } |
| 398 | |
| 399 | // Put all other functions in the list of remaining functions. These are |
| 400 | // functions that call each other circularly. |
| 401 | for (auto it : numberCallOpsContainedInFuncOp) |
| 402 | remainingFuncOps.push_back(Elt: it.first); |
| 403 | |
| 404 | return success(); |
| 405 | } |
| 406 | |
| 407 | /// Helper function that extracts the source from a memref.cast. If the given |
| 408 | /// value is not a memref.cast result, simply returns the given value. |
| 409 | /// Only unpacks casts where the source is at least as specific as the result |
| 410 | /// (i.e., does not unpack casts from unranked to ranked memref, which would |
| 411 | /// downgrade the type). |
| 412 | static Value unpackCast(Value v) { |
| 413 | auto castOp = v.getDefiningOp<memref::CastOp>(); |
| 414 | if (!castOp) |
| 415 | return v; |
| 416 | // Do not unpack a cast from unranked to ranked memref: folding would |
| 417 | // downgrade the function return type from ranked to unranked. |
| 418 | if (isa<UnrankedMemRefType>(Val: castOp.getSource().getType()) && |
| 419 | isa<MemRefType>(Val: v.getType())) |
| 420 | return v; |
| 421 | return castOp.getSource(); |
| 422 | } |
| 423 | |
| 424 | /// Helper function that returns the return types (skipping casts) of the given |
| 425 | /// func.return ops. This function returns as many types as the return ops have |
| 426 | /// operands. If the i-th operand is not the same for all func.return ops, then |
| 427 | /// the i-th returned type is an "empty" type. |
| 428 | static SmallVector<Type> getReturnTypes(SmallVector<func::ReturnOp> returnOps) { |
| 429 | assert(!returnOps.empty() && "expected at least one ReturnOp" ); |
| 430 | int numOperands = returnOps.front()->getNumOperands(); |
| 431 | |
| 432 | // Helper function that unpacks memref.cast ops and returns the type. |
| 433 | auto getSourceType = [&](Value v) { return unpackCast(v).getType(); }; |
| 434 | |
| 435 | SmallVector<Type> result; |
| 436 | for (int i = 0; i < numOperands; ++i) { |
| 437 | // Get the type of the i-th operand of the first func.return ops. |
| 438 | Type t = getSourceType(returnOps.front()->getOperand(idx: i)); |
| 439 | |
| 440 | // Check if all other func.return ops have a matching operand type. |
| 441 | for (int j = 1; j < static_cast<int>(returnOps.size()); ++j) |
| 442 | if (getSourceType(returnOps[j]->getOperand(idx: i)) != t) |
| 443 | t = Type(); |
| 444 | |
| 445 | result.push_back(Elt: t); |
| 446 | } |
| 447 | |
| 448 | return result; |
| 449 | } |
| 450 | |
| 451 | /// Fold return values that are memref casts and update function return types. |
| 452 | /// |
| 453 | /// During FuncOp bufferization, the exact type of the returned memrefs (if any) |
| 454 | /// is not known yet. Therefore, the bufferization uses memref types with the |
| 455 | /// most generic layout map as function return types. After bufferizing the |
| 456 | /// entire function body, a more concise memref type can potentially be used for |
| 457 | /// the return type of the function. |
| 458 | static void foldMemRefCasts(func::FuncOp funcOp) { |
| 459 | // There is nothing to do for bodiless ops. |
| 460 | if (funcOp.getBody().empty()) |
| 461 | return; |
| 462 | |
| 463 | // Compute the common result types of all return ops. |
| 464 | SmallVector<func::ReturnOp> returnOps = getReturnOps(funcOp); |
| 465 | // There is nothing to fold if the function body does not end in a |
| 466 | // func::ReturnOp (e.g., the terminator is from a different dialect). Bail |
| 467 | // out gracefully instead of asserting inside `getReturnTypes`. |
| 468 | if (returnOps.empty()) |
| 469 | return; |
| 470 | SmallVector<Type> resultTypes = getReturnTypes(returnOps); |
| 471 | |
| 472 | // Remove direct casts. |
| 473 | for (func::ReturnOp returnOp : returnOps) { |
| 474 | for (OpOperand &operand : returnOp->getOpOperands()) { |
| 475 | // Bail if no common result type was found. |
| 476 | if (resultTypes[operand.getOperandNumber()]) { |
| 477 | operand.set(unpackCast(v: operand.get())); |
| 478 | } |
| 479 | } |
| 480 | } |
| 481 | |
| 482 | // Fill in the missing result types that were not the same among all |
| 483 | // func.return ops. |
| 484 | for (int i = 0; i < static_cast<int>(resultTypes.size()); ++i) { |
| 485 | if (resultTypes[i]) |
| 486 | continue; |
| 487 | resultTypes[i] = funcOp.getFunctionType().getResult(i); |
| 488 | } |
| 489 | |
| 490 | // Update the function type. |
| 491 | auto newFuncType = FunctionType::get( |
| 492 | context: funcOp.getContext(), inputs: funcOp.getFunctionType().getInputs(), results: resultTypes); |
| 493 | funcOp.setType(newFuncType); |
| 494 | } |
| 495 | |
| 496 | LogicalResult |
| 497 | mlir::bufferization::analyzeModuleOp(Operation *moduleOp, |
| 498 | OneShotAnalysisState &state, |
| 499 | BufferizationStatistics *statistics) { |
| 500 | assert(state.getOptions().bufferizeFunctionBoundaries && |
| 501 | "expected that function boundary bufferization is activated" ); |
| 502 | FuncAnalysisState &funcState = getOrCreateFuncAnalysisState(state); |
| 503 | |
| 504 | // A list of non-circular functions in the order in which they are analyzed |
| 505 | // and bufferized. |
| 506 | SmallVector<func::FuncOp> orderedFuncOps; |
| 507 | // A list of all other functions. I.e., functions that call each other |
| 508 | // recursively. For these, we analyze the function body but not the function |
| 509 | // boundary. |
| 510 | SmallVector<func::FuncOp> remainingFuncOps; |
| 511 | |
| 512 | // A mapping of FuncOps to their callers. |
| 513 | FuncCallerMap callerMap; |
| 514 | |
| 515 | if (failed(Result: getFuncOpsOrderedByCalls(moduleOp, orderedFuncOps, |
| 516 | remainingFuncOps, callerMap, |
| 517 | symbolTables&: funcState.symbolTables))) |
| 518 | return failure(); |
| 519 | |
| 520 | // Analyze functions in order. Starting with functions that are not calling |
| 521 | // any other functions. |
| 522 | for (func::FuncOp funcOp : orderedFuncOps) { |
| 523 | if (!state.getOptions().isOpAllowed(op: funcOp)) |
| 524 | continue; |
| 525 | |
| 526 | // Now analyzing function. |
| 527 | funcState.startFunctionAnalysis(funcOp); |
| 528 | |
| 529 | // Analyze funcOp. |
| 530 | if (failed(Result: analyzeOp(op: funcOp, state, statistics))) |
| 531 | return failure(); |
| 532 | |
| 533 | // Run some extra function analyses. |
| 534 | if (failed(Result: aliasingFuncOpBBArgsAnalysis(funcOp, state, funcState)) || |
| 535 | failed(Result: funcOpBbArgReadWriteAnalysis(funcOp, state, funcState))) |
| 536 | return failure(); |
| 537 | funcOpReturnValueWritabilityAnalysis(funcOp, state, funcState); |
| 538 | |
| 539 | // Mark op as fully analyzed. |
| 540 | funcState.analyzedFuncOps[funcOp] = FuncOpAnalysisState::Analyzed; |
| 541 | } |
| 542 | |
| 543 | // Analyze all other functions. All function boundary analyses are skipped. |
| 544 | for (func::FuncOp funcOp : remainingFuncOps) { |
| 545 | if (!state.getOptions().isOpAllowed(op: funcOp)) |
| 546 | continue; |
| 547 | |
| 548 | // Analyze funcOp. |
| 549 | if (failed(Result: analyzeOp(op: funcOp, state, statistics))) |
| 550 | return failure(); |
| 551 | |
| 552 | // TODO: We currently skip all function argument analyses for functions |
| 553 | // that call each other circularly. These analyses do not support recursive |
| 554 | // calls yet. The `BufferizableOpInterface` implementations of `func` |
| 555 | // dialect ops return conservative results in the absence of analysis |
| 556 | // information. |
| 557 | } |
| 558 | |
| 559 | return success(); |
| 560 | } |
| 561 | |
| 562 | void mlir::bufferization::removeBufferizationAttributesInModule( |
| 563 | Operation *moduleOp) { |
| 564 | for (mlir::Region ®ion : moduleOp->getRegions()) { |
| 565 | for (mlir::Block &block : region.getBlocks()) { |
| 566 | for (func::FuncOp funcOp : block.getOps<func::FuncOp>()) { |
| 567 | for (BlockArgument bbArg : funcOp.getArguments()) |
| 568 | removeBufferizationAttributes(bbArg); |
| 569 | } |
| 570 | } |
| 571 | } |
| 572 | } |
| 573 | |
| 574 | LogicalResult mlir::bufferization::bufferizeModuleOp( |
| 575 | Operation *moduleOp, const OneShotBufferizationOptions &options, |
| 576 | BufferizationState &state, BufferizationStatistics *statistics) { |
| 577 | assert(options.bufferizeFunctionBoundaries && |
| 578 | "expected that function boundary bufferization is activated" ); |
| 579 | IRRewriter rewriter(moduleOp->getContext()); |
| 580 | |
| 581 | // A list of non-circular functions in the order in which they are analyzed |
| 582 | // and bufferized. |
| 583 | SmallVector<func::FuncOp> orderedFuncOps; |
| 584 | // A list of all other functions. I.e., functions that call each other |
| 585 | // recursively. For these, we analyze the function body but not the function |
| 586 | // boundary. |
| 587 | SmallVector<func::FuncOp> remainingFuncOps; |
| 588 | |
| 589 | // A mapping of FuncOps to their callers. |
| 590 | FuncCallerMap callerMap; |
| 591 | |
| 592 | // Try to bufferize functions in calling order. I.e., first bufferize |
| 593 | // functions that do not call other functions. This allows us to infer |
| 594 | // accurate buffer types for function return values. Functions that call |
| 595 | // each other recursively are bufferized in an unspecified order at the end. |
| 596 | // We may use unnecessarily "complex" (in terms of layout map) buffer types. |
| 597 | if (failed(Result: getFuncOpsOrderedByCalls(moduleOp, orderedFuncOps, |
| 598 | remainingFuncOps, callerMap, |
| 599 | symbolTables&: state.getSymbolTables()))) |
| 600 | return failure(); |
| 601 | llvm::append_range(C&: orderedFuncOps, R&: remainingFuncOps); |
| 602 | |
| 603 | // Bufferize functions. |
| 604 | for (func::FuncOp funcOp : orderedFuncOps) { |
| 605 | // Note: It would be good to apply cleanups here but we cannot as aliasInfo |
| 606 | // would be invalidated. |
| 607 | |
| 608 | if (llvm::is_contained(Range: options.noAnalysisFuncFilter, Element: funcOp.getSymName())) { |
| 609 | // This function was not analyzed and RaW conflicts were not resolved. |
| 610 | // Buffer copies must be inserted before every write. |
| 611 | OneShotBufferizationOptions updatedOptions = options; |
| 612 | updatedOptions.copyBeforeWrite = true; |
| 613 | if (failed(Result: bufferizeOp(op: funcOp, options: updatedOptions, bufferizationState&: state, statistics))) |
| 614 | return failure(); |
| 615 | } else { |
| 616 | if (failed(Result: bufferizeOp(op: funcOp, options, bufferizationState&: state, statistics))) |
| 617 | return failure(); |
| 618 | } |
| 619 | |
| 620 | // Change buffer return types to more precise layout maps. |
| 621 | if (options.inferFunctionResultLayout) |
| 622 | foldMemRefCasts(funcOp); |
| 623 | } |
| 624 | |
| 625 | // Bufferize all other ops. |
| 626 | for (mlir::Region ®ion : moduleOp->getRegions()) { |
| 627 | for (mlir::Block &block : region.getBlocks()) { |
| 628 | for (mlir::Operation &op : |
| 629 | llvm::make_early_inc_range(Range&: block.getOperations())) { |
| 630 | // Functions were already bufferized. |
| 631 | if (isa<func::FuncOp>(Val: &op) || op.hasTrait<OpTrait::SymbolTable>()) |
| 632 | continue; |
| 633 | if (failed(Result: bufferizeOp(op: &op, options, bufferizationState&: state, statistics))) |
| 634 | return failure(); |
| 635 | } |
| 636 | } |
| 637 | } |
| 638 | |
| 639 | // Post-pass cleanup of function argument attributes. |
| 640 | removeBufferizationAttributesInModule(moduleOp); |
| 641 | |
| 642 | return success(); |
| 643 | } |
| 644 | |
| 645 | LogicalResult mlir::bufferization::runOneShotModuleBufferize( |
| 646 | Operation *moduleOp, const OneShotBufferizationOptions &options, |
| 647 | BufferizationState &state, BufferizationStatistics *statistics) { |
| 648 | assert(options.bufferizeFunctionBoundaries && |
| 649 | "expected that function boundary bufferization is activated" ); |
| 650 | assert(!(options.copyBeforeWrite && options.testAnalysisOnly) && |
| 651 | "invalid combination of bufferization flags" ); |
| 652 | if (!options.copyBeforeWrite) { |
| 653 | if (options.noAnalysisFuncFilter.empty()) { |
| 654 | if (failed(Result: insertTensorCopies(op: moduleOp, options, bufferizationState: state, statistics))) |
| 655 | return failure(); |
| 656 | } else { |
| 657 | // FuncOps whose names are specified in options.noAnalysisFuncFilter will |
| 658 | // not be analyzed. Ops in these FuncOps will not be analyzed as well. |
| 659 | OpFilter::Entry::FilterFn analysisFilterFn = [=](Operation *op) { |
| 660 | auto func = dyn_cast<func::FuncOp>(Val: op); |
| 661 | if (!func) |
| 662 | func = op->getParentOfType<func::FuncOp>(); |
| 663 | if (func) |
| 664 | return llvm::is_contained(Range: options.noAnalysisFuncFilter, |
| 665 | Element: func.getSymName()); |
| 666 | return false; |
| 667 | }; |
| 668 | OneShotBufferizationOptions updatedOptions(options); |
| 669 | updatedOptions.opFilter.denyOperation(fn: analysisFilterFn); |
| 670 | if (failed( |
| 671 | Result: insertTensorCopies(op: moduleOp, options: updatedOptions, bufferizationState: state, statistics))) |
| 672 | return failure(); |
| 673 | } |
| 674 | } |
| 675 | if (options.testAnalysisOnly) |
| 676 | return success(); |
| 677 | if (failed(Result: bufferizeModuleOp(moduleOp, options, state, statistics))) |
| 678 | return failure(); |
| 679 | return success(); |
| 680 | } |
| 681 | |