1//===- OneShotModuleBufferize.cpp - Bufferization across Func. Boundaries
2//----===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9//
10// Module Bufferization is an extension of One-Shot Bufferize that
11// bufferizes function boundaries. It provides `BufferizableOpInterface`
12// implementations for FuncOp, CallOp and ReturnOp. Although it is named
13// Module Bufferization, it may operate on any SymbolTable.
14//
15// Module Bufferization is run via `runOneShotModuleBufferize(SymbolTableOp,
16// ...)`. This function analyzes the given op and determines the order of
17// analysis and bufferization: Functions that are called are processed before
18// their respective callers.
19//
20// After analyzing a FuncOp, additional information about its bbArgs is
21// gathered and stored in `FuncAnalysisState`.
22//
23// * `aliasingFuncOpBBArgsAnalysis` determines the equivalent/aliasing bbArgs
24// for
25// each tensor return value (if any).
26// * `funcOpBbArgReadWriteAnalysis` determines whether or not a tensor bbArg is
27// read/written.
28// * `funcOpReturnValueWritabilityAnalysis` determines whether returned buffers
29// are writable.
30//
31// Module Bufferization implements the following calling convention.
32//
33// * In the absence of conflicts within a FuncOp, the FuncOp's bbArgs may always
34// be written to in-place.
35// * If a tensor operand of a CallOp is read after the CallOp, the operand of
36// the CallOp must bufferize out-of-place.
37//
38// Example: The tensor.insert op bufferizes in-place because it is allowed to
39// modify the buffer of `%t1` directly. The CallOp in `caller` must bufferize
40// out-of-place because `%t0` is modified by the callee but read by the
41// tensor.extract op. The analysis of CallOps decides whether an OpOperand must
42// bufferize out-of-place based on results of `funcOpBbArgReadWriteAnalysis`.
43// ```
44// func @callee(%t1 : tensor<?xf32>) -> tensor<?xf32> {
45// %f = ... : f32
46// %0 = tensor.insert %f into %t1[...] : tensor<?xf32>
47// return %0 : tensor<?xf32>
48// }
49//
50// func @caller() -> () {
51// %t0 = ... : tensor<?xf32>
52// %1 = call @callee(%t0) : (tensor<?xf32>) -> (tensor<?xf32>)
53// %2 = tensor.extract %1[...] : tensor<?xf32>
54// }
55// ```
56//
57// Note: If a function is external, `funcOpBbArgReadWriteAnalysis` cannot
58// analyze the function body. In such a case, the CallOp analysis conservatively
59// assumes that each tensor OpOperand is both read and written.
60//
61// TODO: Add FuncOp attributes so that bbArgs of external FuncOps can be marked
62// as "not reading" and/or "not writing".
63
64#include "mlir/Dialect/Bufferization/Transforms/OneShotModuleBufferize.h"
65#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
66#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
67#include "mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h"
68#include "mlir/Dialect/Bufferization/Transforms/Bufferize.h"
69#include "mlir/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.h"
70#include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
71#include "mlir/Dialect/Bufferization/Transforms/Transforms.h"
72#include "mlir/Dialect/Func/IR/FuncOps.h"
73#include "mlir/Dialect/MemRef/IR/MemRef.h"
74#include "mlir/IR/BuiltinTypes.h"
75#include "mlir/IR/Operation.h"
76#include "llvm/ADT/MapVector.h"
77#include "llvm/ADT/SmallVectorExtras.h"
78
79using namespace mlir;
80using namespace mlir::bufferization;
81using namespace mlir::bufferization::func_ext;
82
83/// A mapping of FuncOps to their callers.
84using FuncCallerMap = DenseMap<func::FuncOp, DenseSet<Operation *>>;
85
86/// Get or create FuncAnalysisState.
87static FuncAnalysisState &
88getOrCreateFuncAnalysisState(OneShotAnalysisState &state) {
89 auto *result = state.getExtension<FuncAnalysisState>();
90 if (result)
91 return *result;
92 return state.addExtension<FuncAnalysisState>();
93}
94
95namespace {
96
97/// Annotate IR with the results of the analysis. For testing purposes only.
98static void annotateEquivalentReturnBbArg(OpOperand &returnVal,
99 BlockArgument bbArg) {
100 const char *kEquivalentArgsAttr = "__equivalent_func_args__";
101 Operation *op = returnVal.getOwner();
102
103 SmallVector<int64_t> equivBbArgs;
104 if (op->hasDiscardableAttr(name: kEquivalentArgsAttr)) {
105 auto attr = cast<ArrayAttr>(Val: op->getDiscardableAttr(name: kEquivalentArgsAttr));
106 equivBbArgs = llvm::map_to_vector<4>(C&: attr, F: [](Attribute a) {
107 return cast<IntegerAttr>(Val&: a).getValue().getSExtValue();
108 });
109 } else {
110 equivBbArgs.append(NumInputs: op->getNumOperands(), Elt: -1);
111 }
112 equivBbArgs[returnVal.getOperandNumber()] = bbArg.getArgNumber();
113
114 OpBuilder b(op->getContext());
115 op->setDiscardableAttr(name: kEquivalentArgsAttr, value: b.getI64ArrayAttr(values: equivBbArgs));
116}
117
118/// Store function BlockArguments that are equivalent to/aliasing a returned
119/// value in FuncAnalysisState.
120static LogicalResult
121aliasingFuncOpBBArgsAnalysis(FuncOp funcOp, OneShotAnalysisState &state,
122 FuncAnalysisState &funcState) {
123 if (funcOp.getBody().empty()) {
124 // No function body available. Conservatively assume that every tensor
125 // return value may alias with any tensor bbArg.
126 FunctionType type = funcOp.getFunctionType();
127 for (const auto &inputIt : llvm::enumerate(First: type.getInputs())) {
128 if (!isa<TensorLikeType>(Val: inputIt.value()))
129 continue;
130 for (const auto &resultIt : llvm::enumerate(First: type.getResults())) {
131 if (!isa<TensorLikeType>(Val: resultIt.value()))
132 continue;
133 int64_t returnIdx = resultIt.index();
134 int64_t bbArgIdx = inputIt.index();
135 funcState.aliasingReturnVals[funcOp][bbArgIdx].push_back(Elt: returnIdx);
136 }
137 }
138 return success();
139 }
140
141 // Find all func.return ops.
142 SmallVector<func::ReturnOp> returnOps = getReturnOps(funcOp);
143 // TODO: throw error when there is any non-func.return op that has the
144 // ReturnLike trait
145 if (returnOps.empty()) {
146 return funcOp.emitError(message: "cannot bufferize func.func without func.return");
147 }
148
149 // Build alias sets. Merge all aliases from all func.return ops.
150 for (BlockArgument bbArg : funcOp.getArguments()) {
151 if (isa<TensorLikeType>(Val: bbArg.getType())) {
152 int64_t bbArgIdx = bbArg.getArgNumber();
153 // Store aliases in a set, so that we don't add the same alias twice.
154 SetVector<int64_t> aliases;
155 for (func::ReturnOp returnOp : returnOps) {
156 for (OpOperand &returnVal : returnOp->getOpOperands()) {
157 if (isa<TensorLikeType>(Val: returnVal.get().getType())) {
158 int64_t returnIdx = returnVal.getOperandNumber();
159 if (state.areAliasingBufferizedValues(v1: returnVal.get(), v2: bbArg))
160 aliases.insert(X: returnIdx);
161 }
162 }
163 }
164 for (int64_t alias : aliases)
165 funcState.aliasingReturnVals[funcOp][bbArgIdx].push_back(Elt: alias);
166 }
167 }
168
169 // Build equivalence sets.
170 // Helper function that finds an equivalent block argument index for the
171 // given OpOperand. Return std::nullopt if no equivalent block argument could
172 // be found.
173 auto findEquivalentBlockArgIdx =
174 [&](OpOperand &opOperand) -> std::optional<int64_t> {
175 Value v = opOperand.get();
176 if (!isa<TensorLikeType>(Val: v.getType()))
177 return std::nullopt;
178 for (BlockArgument bbArg : funcOp.getArguments()) {
179 if (isa<TensorLikeType>(Val: bbArg.getType())) {
180 if (state.areEquivalentBufferizedValues(v1: v, v2: bbArg)) {
181 if (state.getOptions().testAnalysisOnly)
182 annotateEquivalentReturnBbArg(returnVal&: opOperand, bbArg);
183 return bbArg.getArgNumber();
184 }
185 }
186 }
187 return std::nullopt;
188 };
189
190 int64_t numResults = returnOps.front()->getNumOperands();
191 for (int64_t i = 0; i < numResults; ++i) {
192 // Find the equivalent block argument index for the i-th operand of the
193 // first func.return op.
194 std::optional<int64_t> maybeEquiv =
195 findEquivalentBlockArgIdx(returnOps.front()->getOpOperand(idx: i));
196 if (!maybeEquiv.has_value())
197 continue;
198 int64_t bbArgIdx = *maybeEquiv;
199 bool allEquiv = true;
200
201 // Check if all other func.return ops have the same equivalent block
202 // argument for the i-th operand. In contrast to aliasing information,
203 // which is just "merged", equivalence information must match across all
204 // func.return ops.
205 for (func::ReturnOp returnOp : ArrayRef(returnOps).drop_front()) {
206 std::optional<int64_t> maybeEquiv =
207 findEquivalentBlockArgIdx(returnOp->getOpOperand(idx: i));
208 if (maybeEquiv != bbArgIdx) {
209 allEquiv = false;
210 break;
211 }
212 }
213
214 // All func.return ops have the same equivalent block argument for the i-th
215 // operand.
216 if (allEquiv)
217 funcState.equivalentFuncArgs[funcOp][i] = bbArgIdx;
218 }
219
220 return success();
221}
222
223static void annotateFuncArgAccess(func::FuncOp funcOp, int64_t idx, bool isRead,
224 bool isWritten) {
225 OpBuilder b(funcOp.getContext());
226 Attribute accessType;
227 if (isRead && isWritten) {
228 accessType = b.getStringAttr(bytes: "read-write");
229 } else if (isRead) {
230 accessType = b.getStringAttr(bytes: "read");
231 } else if (isWritten) {
232 accessType = b.getStringAttr(bytes: "write");
233 } else {
234 accessType = b.getStringAttr(bytes: "none");
235 }
236 funcOp.setArgAttr(index: idx, name: BufferizationDialect::kBufferAccessAttrName,
237 value: accessType);
238}
239
240/// Determine which FuncOp bbArgs are read and which are written. When run on a
241/// function with unknown ops, we conservatively assume that such ops bufferize
242/// to a read + write.
243static LogicalResult
244funcOpBbArgReadWriteAnalysis(FuncOp funcOp, OneShotAnalysisState &state,
245 FuncAnalysisState &funcState) {
246 for (int64_t idx = 0, e = funcOp.getFunctionType().getNumInputs(); idx < e;
247 ++idx) {
248 // Skip non-tensor arguments.
249 if (!isa<TensorLikeType>(Val: funcOp.getFunctionType().getInput(i: idx)))
250 continue;
251 bool isRead;
252 bool isWritten;
253 if (auto accessAttr = funcOp.getArgAttrOfType<StringAttr>(
254 index: idx, name: BufferizationDialect::kBufferAccessAttrName)) {
255 // Buffer access behavior is specified on the function. Skip the analysis.
256 StringRef str = accessAttr.getValue();
257 isRead = str == "read" || str == "read-write";
258 isWritten = str == "write" || str == "read-write";
259 } else if (funcOp.getBody().empty()) {
260 // If the function has no body, conservatively assume that all args are
261 // read + written.
262 isRead = true;
263 isWritten = true;
264 } else {
265 // Analyze the body of the function.
266 BlockArgument bbArg = funcOp.getArgument(idx);
267 isRead = state.isValueRead(value: bbArg);
268 isWritten = state.isValueWritten(value: bbArg);
269 }
270
271 if (state.getOptions().testAnalysisOnly)
272 annotateFuncArgAccess(funcOp, idx, isRead, isWritten);
273 if (isRead)
274 funcState.readBbArgs[funcOp].insert(V: idx);
275 if (isWritten)
276 funcState.writtenBbArgs[funcOp].insert(V: idx);
277 }
278
279 return success();
280}
281
282/// Determine which tensor return values are writable at every return site.
283static void funcOpReturnValueWritabilityAnalysis(FuncOp funcOp,
284 OneShotAnalysisState &state,
285 FuncAnalysisState &funcState) {
286 // Without a body, none of the returned buffers are known to be writable.
287 if (funcOp.isExternal())
288 return;
289
290 SmallVector<func::ReturnOp> returnOps = getReturnOps(funcOp);
291 for (auto [index, type] : llvm::enumerate(First: funcOp.getResultTypes())) {
292 if (!isa<TensorLikeType>(Val: type))
293 continue;
294 bool isWritable = true;
295 for (func::ReturnOp returnOp : returnOps) {
296 // A view or a forwarded value may itself be writable but alias a
297 // read-only buffer, such as an arith.constant or a callee result.
298 state.applyOnAliases(v: returnOp.getOperand(i: index), fun: [&](Value alias) {
299 isWritable &= state.isWritable(value: alias);
300 });
301 }
302 if (isWritable)
303 funcState.writableReturnVals[funcOp].insert(V: index);
304 }
305}
306} // namespace
307
308/// Remove bufferization attributes on FuncOp arguments.
309static void removeBufferizationAttributes(BlockArgument bbArg) {
310 auto funcOp = cast<func::FuncOp>(Val: bbArg.getOwner()->getParentOp());
311 funcOp.removeArgAttr(index: bbArg.getArgNumber(),
312 name: BufferizationDialect::kBufferLayoutAttrName);
313 funcOp.removeArgAttr(index: bbArg.getArgNumber(),
314 name: BufferizationDialect::kWritableAttrName);
315}
316
317/// Return the func::FuncOp called by `callOp`.
318static func::FuncOp
319getCalledFunction(func::CallOp callOp,
320 mlir::SymbolTableCollection &symbolTable) {
321 return dyn_cast_or_null<func::FuncOp>(
322 Val: callOp.resolveCallableInTable(symbolTable: &symbolTable));
323}
324
325/// Return "true" if the given function signature has tensor semantics.
326static bool hasTensorSignature(func::FuncOp funcOp) {
327 return llvm::any_of(Range: funcOp.getFunctionType().getInputs(),
328 P: llvm::IsaPred<TensorLikeType>) ||
329 llvm::any_of(Range: funcOp.getFunctionType().getResults(),
330 P: llvm::IsaPred<TensorLikeType>);
331}
332
333/// Store all functions of the `moduleOp` in `orderedFuncOps`, sorted by
334/// callee-caller order (i.e., callees without callers first). Store all
335/// remaining functions (i.e., the ones that call each other recursively) in
336/// `remainingFuncOps`. Does not traverse nested symbol tables.
337///
338/// Store the map of FuncOp to all its callers in `callerMap`.
339///
340/// Return `failure()` if we are unable to retrieve the called FuncOp from
341/// any func::CallOp.
342static LogicalResult getFuncOpsOrderedByCalls(
343 Operation *moduleOp, SmallVectorImpl<func::FuncOp> &orderedFuncOps,
344 SmallVectorImpl<func::FuncOp> &remainingFuncOps, FuncCallerMap &callerMap,
345 SymbolTableCollection &symbolTables) {
346 // For each FuncOp, the set of functions called by it (i.e. the union of
347 // symbols of all nested func::CallOp).
348 DenseMap<func::FuncOp, SetVector<func::FuncOp>> calledBy;
349 // For each FuncOp, the number of func::CallOp it contains.
350 llvm::MapVector<func::FuncOp, unsigned> numberCallOpsContainedInFuncOp;
351 for (mlir::Region &region : moduleOp->getRegions()) {
352 for (mlir::Block &block : region.getBlocks()) {
353 for (func::FuncOp funcOp : block.getOps<func::FuncOp>()) {
354 // Collect function calls and populate the caller map.
355 numberCallOpsContainedInFuncOp[funcOp] = 0;
356 WalkResult res = funcOp.walk(callback: [&](func::CallOp callOp) -> WalkResult {
357 func::FuncOp calledFunction = getCalledFunction(callOp, symbolTable&: symbolTables);
358 assert(calledFunction && "could not retrieved called func::FuncOp");
359 // If the called function does not have any tensors in its signature,
360 // then it is not necessary to bufferize the callee before the caller.
361 if (!hasTensorSignature(funcOp: calledFunction))
362 return WalkResult::skip();
363
364 callerMap[calledFunction].insert(V: callOp);
365 if (calledBy[calledFunction].insert(X: funcOp)) {
366 numberCallOpsContainedInFuncOp[funcOp]++;
367 }
368 return WalkResult::advance();
369 });
370 if (res.wasInterrupted())
371 return failure();
372 }
373 }
374 }
375
376 // Iteratively remove function operations that do not call any of the
377 // functions remaining in the callCounter map and add them to ordered list.
378 SmallVector<func::FuncOp> worklist;
379
380 for (const auto &entry : numberCallOpsContainedInFuncOp) {
381 if (entry.second == 0)
382 worklist.push_back(Elt: entry.first);
383 }
384
385 while (!worklist.empty()) {
386 func::FuncOp func = worklist.pop_back_val();
387 orderedFuncOps.push_back(Elt: func);
388
389 for (func::FuncOp caller : calledBy[func]) {
390 auto &count = numberCallOpsContainedInFuncOp[caller];
391
392 if (--count == 0)
393 worklist.push_back(Elt: caller);
394 }
395
396 numberCallOpsContainedInFuncOp.erase(Key: func);
397 }
398
399 // Put all other functions in the list of remaining functions. These are
400 // functions that call each other circularly.
401 for (auto it : numberCallOpsContainedInFuncOp)
402 remainingFuncOps.push_back(Elt: it.first);
403
404 return success();
405}
406
407/// Helper function that extracts the source from a memref.cast. If the given
408/// value is not a memref.cast result, simply returns the given value.
409/// Only unpacks casts where the source is at least as specific as the result
410/// (i.e., does not unpack casts from unranked to ranked memref, which would
411/// downgrade the type).
412static Value unpackCast(Value v) {
413 auto castOp = v.getDefiningOp<memref::CastOp>();
414 if (!castOp)
415 return v;
416 // Do not unpack a cast from unranked to ranked memref: folding would
417 // downgrade the function return type from ranked to unranked.
418 if (isa<UnrankedMemRefType>(Val: castOp.getSource().getType()) &&
419 isa<MemRefType>(Val: v.getType()))
420 return v;
421 return castOp.getSource();
422}
423
424/// Helper function that returns the return types (skipping casts) of the given
425/// func.return ops. This function returns as many types as the return ops have
426/// operands. If the i-th operand is not the same for all func.return ops, then
427/// the i-th returned type is an "empty" type.
428static SmallVector<Type> getReturnTypes(SmallVector<func::ReturnOp> returnOps) {
429 assert(!returnOps.empty() && "expected at least one ReturnOp");
430 int numOperands = returnOps.front()->getNumOperands();
431
432 // Helper function that unpacks memref.cast ops and returns the type.
433 auto getSourceType = [&](Value v) { return unpackCast(v).getType(); };
434
435 SmallVector<Type> result;
436 for (int i = 0; i < numOperands; ++i) {
437 // Get the type of the i-th operand of the first func.return ops.
438 Type t = getSourceType(returnOps.front()->getOperand(idx: i));
439
440 // Check if all other func.return ops have a matching operand type.
441 for (int j = 1; j < static_cast<int>(returnOps.size()); ++j)
442 if (getSourceType(returnOps[j]->getOperand(idx: i)) != t)
443 t = Type();
444
445 result.push_back(Elt: t);
446 }
447
448 return result;
449}
450
451/// Fold return values that are memref casts and update function return types.
452///
453/// During FuncOp bufferization, the exact type of the returned memrefs (if any)
454/// is not known yet. Therefore, the bufferization uses memref types with the
455/// most generic layout map as function return types. After bufferizing the
456/// entire function body, a more concise memref type can potentially be used for
457/// the return type of the function.
458static void foldMemRefCasts(func::FuncOp funcOp) {
459 // There is nothing to do for bodiless ops.
460 if (funcOp.getBody().empty())
461 return;
462
463 // Compute the common result types of all return ops.
464 SmallVector<func::ReturnOp> returnOps = getReturnOps(funcOp);
465 // There is nothing to fold if the function body does not end in a
466 // func::ReturnOp (e.g., the terminator is from a different dialect). Bail
467 // out gracefully instead of asserting inside `getReturnTypes`.
468 if (returnOps.empty())
469 return;
470 SmallVector<Type> resultTypes = getReturnTypes(returnOps);
471
472 // Remove direct casts.
473 for (func::ReturnOp returnOp : returnOps) {
474 for (OpOperand &operand : returnOp->getOpOperands()) {
475 // Bail if no common result type was found.
476 if (resultTypes[operand.getOperandNumber()]) {
477 operand.set(unpackCast(v: operand.get()));
478 }
479 }
480 }
481
482 // Fill in the missing result types that were not the same among all
483 // func.return ops.
484 for (int i = 0; i < static_cast<int>(resultTypes.size()); ++i) {
485 if (resultTypes[i])
486 continue;
487 resultTypes[i] = funcOp.getFunctionType().getResult(i);
488 }
489
490 // Update the function type.
491 auto newFuncType = FunctionType::get(
492 context: funcOp.getContext(), inputs: funcOp.getFunctionType().getInputs(), results: resultTypes);
493 funcOp.setType(newFuncType);
494}
495
496LogicalResult
497mlir::bufferization::analyzeModuleOp(Operation *moduleOp,
498 OneShotAnalysisState &state,
499 BufferizationStatistics *statistics) {
500 assert(state.getOptions().bufferizeFunctionBoundaries &&
501 "expected that function boundary bufferization is activated");
502 FuncAnalysisState &funcState = getOrCreateFuncAnalysisState(state);
503
504 // A list of non-circular functions in the order in which they are analyzed
505 // and bufferized.
506 SmallVector<func::FuncOp> orderedFuncOps;
507 // A list of all other functions. I.e., functions that call each other
508 // recursively. For these, we analyze the function body but not the function
509 // boundary.
510 SmallVector<func::FuncOp> remainingFuncOps;
511
512 // A mapping of FuncOps to their callers.
513 FuncCallerMap callerMap;
514
515 if (failed(Result: getFuncOpsOrderedByCalls(moduleOp, orderedFuncOps,
516 remainingFuncOps, callerMap,
517 symbolTables&: funcState.symbolTables)))
518 return failure();
519
520 // Analyze functions in order. Starting with functions that are not calling
521 // any other functions.
522 for (func::FuncOp funcOp : orderedFuncOps) {
523 if (!state.getOptions().isOpAllowed(op: funcOp))
524 continue;
525
526 // Now analyzing function.
527 funcState.startFunctionAnalysis(funcOp);
528
529 // Analyze funcOp.
530 if (failed(Result: analyzeOp(op: funcOp, state, statistics)))
531 return failure();
532
533 // Run some extra function analyses.
534 if (failed(Result: aliasingFuncOpBBArgsAnalysis(funcOp, state, funcState)) ||
535 failed(Result: funcOpBbArgReadWriteAnalysis(funcOp, state, funcState)))
536 return failure();
537 funcOpReturnValueWritabilityAnalysis(funcOp, state, funcState);
538
539 // Mark op as fully analyzed.
540 funcState.analyzedFuncOps[funcOp] = FuncOpAnalysisState::Analyzed;
541 }
542
543 // Analyze all other functions. All function boundary analyses are skipped.
544 for (func::FuncOp funcOp : remainingFuncOps) {
545 if (!state.getOptions().isOpAllowed(op: funcOp))
546 continue;
547
548 // Analyze funcOp.
549 if (failed(Result: analyzeOp(op: funcOp, state, statistics)))
550 return failure();
551
552 // TODO: We currently skip all function argument analyses for functions
553 // that call each other circularly. These analyses do not support recursive
554 // calls yet. The `BufferizableOpInterface` implementations of `func`
555 // dialect ops return conservative results in the absence of analysis
556 // information.
557 }
558
559 return success();
560}
561
562void mlir::bufferization::removeBufferizationAttributesInModule(
563 Operation *moduleOp) {
564 for (mlir::Region &region : moduleOp->getRegions()) {
565 for (mlir::Block &block : region.getBlocks()) {
566 for (func::FuncOp funcOp : block.getOps<func::FuncOp>()) {
567 for (BlockArgument bbArg : funcOp.getArguments())
568 removeBufferizationAttributes(bbArg);
569 }
570 }
571 }
572}
573
574LogicalResult mlir::bufferization::bufferizeModuleOp(
575 Operation *moduleOp, const OneShotBufferizationOptions &options,
576 BufferizationState &state, BufferizationStatistics *statistics) {
577 assert(options.bufferizeFunctionBoundaries &&
578 "expected that function boundary bufferization is activated");
579 IRRewriter rewriter(moduleOp->getContext());
580
581 // A list of non-circular functions in the order in which they are analyzed
582 // and bufferized.
583 SmallVector<func::FuncOp> orderedFuncOps;
584 // A list of all other functions. I.e., functions that call each other
585 // recursively. For these, we analyze the function body but not the function
586 // boundary.
587 SmallVector<func::FuncOp> remainingFuncOps;
588
589 // A mapping of FuncOps to their callers.
590 FuncCallerMap callerMap;
591
592 // Try to bufferize functions in calling order. I.e., first bufferize
593 // functions that do not call other functions. This allows us to infer
594 // accurate buffer types for function return values. Functions that call
595 // each other recursively are bufferized in an unspecified order at the end.
596 // We may use unnecessarily "complex" (in terms of layout map) buffer types.
597 if (failed(Result: getFuncOpsOrderedByCalls(moduleOp, orderedFuncOps,
598 remainingFuncOps, callerMap,
599 symbolTables&: state.getSymbolTables())))
600 return failure();
601 llvm::append_range(C&: orderedFuncOps, R&: remainingFuncOps);
602
603 // Bufferize functions.
604 for (func::FuncOp funcOp : orderedFuncOps) {
605 // Note: It would be good to apply cleanups here but we cannot as aliasInfo
606 // would be invalidated.
607
608 if (llvm::is_contained(Range: options.noAnalysisFuncFilter, Element: funcOp.getSymName())) {
609 // This function was not analyzed and RaW conflicts were not resolved.
610 // Buffer copies must be inserted before every write.
611 OneShotBufferizationOptions updatedOptions = options;
612 updatedOptions.copyBeforeWrite = true;
613 if (failed(Result: bufferizeOp(op: funcOp, options: updatedOptions, bufferizationState&: state, statistics)))
614 return failure();
615 } else {
616 if (failed(Result: bufferizeOp(op: funcOp, options, bufferizationState&: state, statistics)))
617 return failure();
618 }
619
620 // Change buffer return types to more precise layout maps.
621 if (options.inferFunctionResultLayout)
622 foldMemRefCasts(funcOp);
623 }
624
625 // Bufferize all other ops.
626 for (mlir::Region &region : moduleOp->getRegions()) {
627 for (mlir::Block &block : region.getBlocks()) {
628 for (mlir::Operation &op :
629 llvm::make_early_inc_range(Range&: block.getOperations())) {
630 // Functions were already bufferized.
631 if (isa<func::FuncOp>(Val: &op) || op.hasTrait<OpTrait::SymbolTable>())
632 continue;
633 if (failed(Result: bufferizeOp(op: &op, options, bufferizationState&: state, statistics)))
634 return failure();
635 }
636 }
637 }
638
639 // Post-pass cleanup of function argument attributes.
640 removeBufferizationAttributesInModule(moduleOp);
641
642 return success();
643}
644
645LogicalResult mlir::bufferization::runOneShotModuleBufferize(
646 Operation *moduleOp, const OneShotBufferizationOptions &options,
647 BufferizationState &state, BufferizationStatistics *statistics) {
648 assert(options.bufferizeFunctionBoundaries &&
649 "expected that function boundary bufferization is activated");
650 assert(!(options.copyBeforeWrite && options.testAnalysisOnly) &&
651 "invalid combination of bufferization flags");
652 if (!options.copyBeforeWrite) {
653 if (options.noAnalysisFuncFilter.empty()) {
654 if (failed(Result: insertTensorCopies(op: moduleOp, options, bufferizationState: state, statistics)))
655 return failure();
656 } else {
657 // FuncOps whose names are specified in options.noAnalysisFuncFilter will
658 // not be analyzed. Ops in these FuncOps will not be analyzed as well.
659 OpFilter::Entry::FilterFn analysisFilterFn = [=](Operation *op) {
660 auto func = dyn_cast<func::FuncOp>(Val: op);
661 if (!func)
662 func = op->getParentOfType<func::FuncOp>();
663 if (func)
664 return llvm::is_contained(Range: options.noAnalysisFuncFilter,
665 Element: func.getSymName());
666 return false;
667 };
668 OneShotBufferizationOptions updatedOptions(options);
669 updatedOptions.opFilter.denyOperation(fn: analysisFilterFn);
670 if (failed(
671 Result: insertTensorCopies(op: moduleOp, options: updatedOptions, bufferizationState: state, statistics)))
672 return failure();
673 }
674 }
675 if (options.testAnalysisOnly)
676 return success();
677 if (failed(Result: bufferizeModuleOp(moduleOp, options, state, statistics)))
678 return failure();
679 return success();
680}
681