1//===- BufferResultsToOutParams.cpp - Calling convention conversion -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Bufferization/IR/AllocationOpInterface.h"
10#include "mlir/Dialect/Bufferization/Transforms/Passes.h"
11
12#include "mlir/Dialect/Func/IR/FuncOps.h"
13#include "mlir/Dialect/MemRef/IR/MemRef.h"
14#include "mlir/IR/Operation.h"
15
16namespace mlir {
17namespace bufferization {
18#define GEN_PASS_DEF_BUFFERRESULTSTOOUTPARAMSPASS
19#include "mlir/Dialect/Bufferization/Transforms/Passes.h.inc"
20} // namespace bufferization
21} // namespace mlir
22
23using namespace mlir;
24using AllocationFn = bufferization::BufferResultsToOutParamsOpts::AllocationFn;
25using MemCpyFn = bufferization::BufferResultsToOutParamsOpts::MemCpyFn;
26using AllocDynamicSizesMap =
27 llvm::DenseMap<func::FuncOp, SmallVector<SmallVector<Value>>>;
28
29/// Return `true` if the given MemRef type has a fully dynamic layout.
30static bool hasFullyDynamicLayoutMap(MemRefType type) {
31 int64_t offset;
32 SmallVector<int64_t, 4> strides;
33 if (failed(Result: type.getStridesAndOffset(strides, offset)))
34 return false;
35 if (!llvm::all_of(Range&: strides, P: ShapedType::isDynamic))
36 return false;
37 if (ShapedType::isStatic(dValue: offset))
38 return false;
39 return true;
40}
41
42/// Return `true` if the given MemRef type has a static identity layout (i.e.,
43/// no layout).
44static bool hasStaticIdentityLayout(MemRefType type) {
45 return type.getLayout().isIdentity();
46}
47
48/// Return the dynamic shapes of the `memref` based on the defining op. If the
49/// complete dynamic shape fails to be captured, return an empty value.
50/// Currently, only function block arguments are supported for capturing.
51static SmallVector<Value> getDynamicSize(Value memref, func::FuncOp funcOp) {
52 Operation *defOp = memref.getDefiningOp();
53 if (!defOp)
54 return {};
55 auto operands = defOp->getOperands();
56 SmallVector<Value> dynamicSizes;
57 for (Value size : operands) {
58 if (!isa<IndexType>(Val: size.getType()))
59 continue;
60
61 BlockArgument sizeSrc = dyn_cast<BlockArgument>(Val&: size);
62 if (!sizeSrc)
63 return {};
64 auto arguments = funcOp.getArguments();
65 auto iter = llvm::find(Range&: arguments, Val: sizeSrc);
66 if (iter == arguments.end())
67 return {};
68 dynamicSizes.push_back(Elt: *iter);
69 }
70 return dynamicSizes;
71}
72
73/// Returns the dynamic sizes at the callee, through the call relationship
74/// between the caller and callee.
75static SmallVector<Value> mapDynamicSizeAtCaller(func::CallOp call,
76 func::FuncOp callee,
77 ValueRange dynamicSizes) {
78 SmallVector<Value> mappedDynamicSizes;
79 for (Value size : dynamicSizes) {
80 for (auto [src, dst] :
81 llvm::zip_first(t: call.getOperands(), u: callee.getArguments())) {
82 if (size != dst)
83 continue;
84 mappedDynamicSizes.push_back(Elt: src);
85 }
86 }
87 assert(mappedDynamicSizes.size() == dynamicSizes.size() &&
88 "could not find all dynamic sizes");
89 return mappedDynamicSizes;
90}
91
92// Updates the func op and entry block.
93//
94// Any args appended to the entry block are added to `appendedEntryArgs`.
95// If `addResultAttribute` is true, adds the unit attribute `bufferize.result`
96// to each newly created function argument.
97static LogicalResult
98updateFuncOp(func::FuncOp func,
99 SmallVectorImpl<BlockArgument> &appendedEntryArgs,
100 bool addResultAttribute) {
101 auto functionType = func.getFunctionType();
102
103 // Collect information about the results will become appended arguments.
104 SmallVector<Type, 6> erasedResultTypes;
105 BitVector erasedResultIndices(functionType.getNumResults());
106 for (const auto &resultType : llvm::enumerate(First: functionType.getResults())) {
107 if (auto memrefType = dyn_cast<MemRefType>(Val: resultType.value())) {
108 if (!hasStaticIdentityLayout(type: memrefType) &&
109 !hasFullyDynamicLayoutMap(type: memrefType)) {
110 // Only buffers with static identity layout can be allocated. These can
111 // be casted to memrefs with fully dynamic layout map. Other layout maps
112 // are not supported.
113 return func->emitError()
114 << "cannot create out param for result with unsupported layout";
115 }
116 erasedResultIndices.set(resultType.index());
117 erasedResultTypes.push_back(Elt: memrefType);
118 }
119 }
120
121 // Add the new arguments to the function type.
122 auto newArgTypes = llvm::to_vector<6>(
123 Range: llvm::concat<const Type>(Ranges: functionType.getInputs(), Ranges&: erasedResultTypes));
124 auto newFunctionType = FunctionType::get(context: func.getContext(), inputs: newArgTypes,
125 results: functionType.getResults());
126 func.setType(newFunctionType);
127
128 // Transfer the result attributes to arg attributes.
129 auto erasedIndicesIt = erasedResultIndices.set_bits_begin();
130 for (int i = 0, e = erasedResultTypes.size(); i < e; ++i, ++erasedIndicesIt) {
131 func.setArgAttrs(index: functionType.getNumInputs() + i,
132 attributes: func.getResultAttrs(index: *erasedIndicesIt));
133 if (addResultAttribute)
134 func.setArgAttr(index: functionType.getNumInputs() + i,
135 name: StringAttr::get(context: func.getContext(), bytes: "bufferize.result"),
136 value: UnitAttr::get(context: func.getContext()));
137 }
138
139 // Erase the results.
140 if (failed(Result: func.eraseResults(resultIndices: erasedResultIndices)))
141 return failure();
142
143 // Add the new arguments to the entry block if the function is not external.
144 if (func.isExternal())
145 return success();
146 Location loc = func.getLoc();
147 for (Type type : erasedResultTypes)
148 appendedEntryArgs.push_back(Elt: func.front().addArgument(type, loc));
149
150 return success();
151}
152
153// Updates all ReturnOps in the scope of the given func::FuncOp by either
154// keeping them as return values or copying the associated buffer contents into
155// the given out-params.
156static LogicalResult
157updateReturnOps(func::FuncOp func, ArrayRef<BlockArgument> appendedEntryArgs,
158 AllocDynamicSizesMap &map,
159 const bufferization::BufferResultsToOutParamsOpts &options) {
160 auto res = func.walk(callback: [&](func::ReturnOp op) {
161 SmallVector<Value, 6> copyIntoOutParams;
162 SmallVector<Value, 6> keepAsReturnOperands;
163 for (Value operand : op.getOperands()) {
164 if (isa<MemRefType>(Val: operand.getType()))
165 copyIntoOutParams.push_back(Elt: operand);
166 else
167 keepAsReturnOperands.push_back(Elt: operand);
168 }
169 OpBuilder builder(op);
170 SmallVector<SmallVector<Value>> dynamicSizes;
171 // Map each returned memref to its primary out parameter.
172 DenseMap<Value, BlockArgument> primaryArgs;
173 // Delay erasure until all return operands have been inspected because an
174 // allocation may be returned multiple times.
175 SetVector<Operation *> toErase;
176 for (auto [orig, arg] : llvm::zip(t&: copyIntoOutParams, u&: appendedEntryArgs)) {
177 bool hoistStaticAllocs =
178 options.hoistStaticAllocs &&
179 cast<MemRefType>(Val: orig.getType()).hasStaticShape();
180 bool hoistDynamicAllocs =
181 options.hoistDynamicAllocs &&
182 !cast<MemRefType>(Val: orig.getType()).hasStaticShape();
183 // Keep non-hoistable allocations such as memref.realloc and
184 // bufferization.clone. Erasing them would discard the data they preserve
185 // or reproduce in their results.
186 auto allocOp = dyn_cast_if_present<bufferization::AllocationOpInterface>(
187 Val: orig.getDefiningOp());
188 if ((hoistStaticAllocs || hoistDynamicAllocs) && allocOp &&
189 allocOp.getHoistingKind() != HoistingKind::None) {
190 if (hoistDynamicAllocs) {
191 SmallVector<Value> dynamicSize = getDynamicSize(memref: orig, funcOp: func);
192 dynamicSizes.push_back(Elt: dynamicSize);
193 }
194
195 auto [it, inserted] = primaryArgs.try_emplace(Key: orig, Args: arg);
196 // Repeated returns of the same allocation reuse the primary out
197 // parameter; copy its contents into the out parameter for this
198 // occurrence.
199 if (!inserted) {
200 if (failed(Result: options.memCpyFn(builder, op->getLoc(), it->second, arg)))
201 return WalkResult::interrupt();
202 continue;
203 }
204 orig.replaceAllUsesWith(newValue: arg);
205 toErase.insert(X: orig.getDefiningOp());
206 } else if (orig != arg) {
207 if (failed(Result: options.memCpyFn(builder, op.getLoc(), orig, arg)))
208 return WalkResult::interrupt();
209 }
210 }
211
212 for (Operation *eraseOp : toErase)
213 eraseOp->erase();
214
215 func::ReturnOp::create(builder, location: op.getLoc(), operands: keepAsReturnOperands);
216 op.erase();
217 auto dynamicSizePair =
218 std::pair<func::FuncOp, SmallVector<SmallVector<Value>>>(func,
219 dynamicSizes);
220 map.insert(KV: dynamicSizePair);
221 return WalkResult::advance();
222 });
223 return failure(IsFailure: res.wasInterrupted());
224}
225
226// Updates all CallOps in the scope of the given ModuleOp by allocating
227// temporary buffers for newly introduced out params.
228static LogicalResult
229updateCalls(ModuleOp module, const AllocDynamicSizesMap &map,
230 const bufferization::BufferResultsToOutParamsOpts &options) {
231 bool didFail = false;
232 SymbolTable symtab(module);
233 module.walk(callback: [&](func::CallOp op) {
234 auto callee = symtab.lookup<func::FuncOp>(name: op.getCallee());
235 if (!callee) {
236 op.emitError() << "cannot find callee '" << op.getCallee() << "' in "
237 << "symbol table";
238 didFail = true;
239 return;
240 }
241 if (!options.filterFn(&callee))
242 return;
243 if (callee.isPublic() && !options.modifyPublicFunctions)
244 return;
245 if (callee.isExternal())
246 return;
247
248 SmallVector<Value, 6> replaceWithNewCallResults;
249 SmallVector<Value, 6> replaceWithOutParams;
250 for (OpResult result : op.getResults()) {
251 if (isa<MemRefType>(Val: result.getType()))
252 replaceWithOutParams.push_back(Elt: result);
253 else
254 replaceWithNewCallResults.push_back(Elt: result);
255 }
256 SmallVector<Value, 6> outParams;
257 OpBuilder builder(op);
258 SmallVector<SmallVector<Value>> dynamicSizes = map.lookup(Val: callee);
259 size_t dynamicSizesIndex = 0;
260 for (Value memref : replaceWithOutParams) {
261 SmallVector<Value> dynamicSize = dynamicSizes.size() > dynamicSizesIndex
262 ? dynamicSizes[dynamicSizesIndex]
263 : SmallVector<Value>();
264 bool memrefStaticShape =
265 cast<MemRefType>(Val: memref.getType()).hasStaticShape();
266 if (!memrefStaticShape && dynamicSize.empty()) {
267 op.emitError()
268 << "cannot create out param for dynamically shaped result";
269 didFail = true;
270 return;
271 }
272 auto memrefType = cast<MemRefType>(Val: memref.getType());
273 auto allocType =
274 MemRefType::get(shape: memrefType.getShape(), elementType: memrefType.getElementType(),
275 map: AffineMap(), memorySpace: memrefType.getMemorySpace());
276
277 if (memrefStaticShape) {
278 dynamicSize = {};
279 } else {
280 ++dynamicSizesIndex;
281 dynamicSize = mapDynamicSizeAtCaller(call: op, callee, dynamicSizes: dynamicSize);
282 }
283 auto maybeOutParam =
284 options.allocationFn(builder, op.getLoc(), allocType, dynamicSize);
285 if (failed(Result: maybeOutParam)) {
286 op.emitError() << "failed to create allocation op";
287 didFail = true;
288 return;
289 }
290 Value outParam = maybeOutParam.value();
291 if (!hasStaticIdentityLayout(type: memrefType)) {
292 // Layout maps are already checked in `updateFuncOp`.
293 assert(hasFullyDynamicLayoutMap(memrefType) &&
294 "layout map not supported");
295 outParam =
296 memref::CastOp::create(builder, location: op.getLoc(), dest: memrefType, source: outParam);
297 }
298 memref.replaceAllUsesWith(newValue: outParam);
299 outParams.push_back(Elt: outParam);
300 }
301
302 auto newOperands = llvm::to_vector<6>(Range: op.getOperands());
303 newOperands.append(in_start: outParams.begin(), in_end: outParams.end());
304 auto newResultTypes = llvm::map_to_vector<6>(
305 C&: replaceWithNewCallResults, F: [](Value v) { return v.getType(); });
306 auto newCall = func::CallOp::create(
307 builder, location: op.getLoc(), callee: op.getCalleeAttr(), results: newResultTypes, operands: newOperands);
308 for (auto t : llvm::zip(t&: replaceWithNewCallResults, u: newCall.getResults()))
309 std::get<0>(t&: t).replaceAllUsesWith(newValue: std::get<1>(t&: t));
310 op.erase();
311 });
312
313 return failure(IsFailure: didFail);
314}
315
316LogicalResult mlir::bufferization::promoteBufferResultsToOutParams(
317 ModuleOp module,
318 const bufferization::BufferResultsToOutParamsOpts &options) {
319 // It maps the shape source of the dynamic shape memref returned by each
320 // function.
321 AllocDynamicSizesMap map;
322 for (auto func : module.getOps<func::FuncOp>()) {
323 if (func.isPublic() && !options.modifyPublicFunctions)
324 continue;
325 if (func.isExternal())
326 continue;
327 if (!options.filterFn(&func))
328 continue;
329 SmallVector<BlockArgument, 6> appendedEntryArgs;
330 if (failed(
331 Result: updateFuncOp(func, appendedEntryArgs, addResultAttribute: options.addResultAttribute)))
332 return failure();
333 if (failed(Result: updateReturnOps(func, appendedEntryArgs, map, options))) {
334 return failure();
335 }
336 }
337 if (failed(Result: updateCalls(module, map, options)))
338 return failure();
339 return success();
340}
341
342namespace {
343struct BufferResultsToOutParamsPass
344 : bufferization::impl::BufferResultsToOutParamsPassBase<
345 BufferResultsToOutParamsPass> {
346 using Base::Base;
347
348 void runOnOperation() override {
349 // Convert from pass options in tablegen to BufferResultsToOutParamsOpts.
350 if (addResultAttribute)
351 options.addResultAttribute = true;
352 if (hoistStaticAllocs)
353 options.hoistStaticAllocs = true;
354 if (hoistDynamicAllocs)
355 options.hoistDynamicAllocs = true;
356 if (modifyPublicFunctions)
357 options.modifyPublicFunctions = true;
358
359 if (failed(Result: bufferization::promoteBufferResultsToOutParams(module: getOperation(),
360 options)))
361 return signalPassFailure();
362 }
363
364private:
365 bufferization::BufferResultsToOutParamsOpts options;
366};
367} // namespace
368