| 1 | //===- BufferizableOpInterfaceImpl.cpp - Impl. of BufferizableOpInterface -===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.h" |
| 10 | #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" |
| 11 | #include "mlir/Dialect/Bufferization/IR/Bufferization.h" |
| 12 | #include "mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h" |
| 13 | #include "mlir/Dialect/Bufferization/Transforms/Bufferize.h" |
| 14 | #include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h" |
| 15 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 16 | #include "mlir/IR/Dialect.h" |
| 17 | #include "mlir/IR/Operation.h" |
| 18 | #include <optional> |
| 19 | |
| 20 | namespace mlir { |
| 21 | /// Return all func.return ops in the given function. |
| 22 | SmallVector<func::ReturnOp> bufferization::getReturnOps(func::FuncOp funcOp) { |
| 23 | SmallVector<func::ReturnOp> result; |
| 24 | for (Block &b : funcOp.getBody()) |
| 25 | if (auto returnOp = dyn_cast<func::ReturnOp>(Val: b.getTerminator())) |
| 26 | result.push_back(Elt: returnOp); |
| 27 | return result; |
| 28 | } |
| 29 | |
| 30 | namespace bufferization { |
| 31 | namespace func_ext { |
| 32 | |
| 33 | void FuncAnalysisState::startFunctionAnalysis(FuncOp funcOp) { |
| 34 | analyzedFuncOps[funcOp] = FuncOpAnalysisState::InProgress; |
| 35 | auto createdEquiv = equivalentFuncArgs.try_emplace(Key: funcOp, Args: IndexMapping()); |
| 36 | auto createdAliasingResults = |
| 37 | aliasingReturnVals.try_emplace(Key: funcOp, Args: IndexToIndexListMapping()); |
| 38 | auto createdRead = readBbArgs.try_emplace(Key: funcOp, Args: BbArgIndexSet()); |
| 39 | auto createdWritten = writtenBbArgs.try_emplace(Key: funcOp, Args: BbArgIndexSet()); |
| 40 | auto createdWritable = writableReturnVals.try_emplace(Key: funcOp); |
| 41 | (void)createdEquiv; |
| 42 | (void)createdAliasingResults; |
| 43 | (void)createdRead; |
| 44 | (void)createdWritten; |
| 45 | (void)createdWritable; |
| 46 | #ifndef NDEBUG |
| 47 | assert(createdEquiv.second && "equivalence info exists already" ); |
| 48 | assert(createdAliasingResults.second && "aliasing info exists already" ); |
| 49 | assert(createdRead.second && "bbarg access info exists already" ); |
| 50 | assert(createdWritten.second && "bbarg access info exists already" ); |
| 51 | assert(createdWritable.second && |
| 52 | "return value writability info exists already" ); |
| 53 | #endif // NDEBUG |
| 54 | } |
| 55 | |
| 56 | // Note: this is a local adaptor to unify TensorType and TensorLikeType code |
| 57 | // paths that both work with BufferizationOptions. |
| 58 | static mlir::Attribute |
| 59 | getDefaultMemorySpace(const BufferizationOptions &options, |
| 60 | TensorLikeType type) { |
| 61 | if (isa<TensorType>(Val: type)) { |
| 62 | return *options.defaultMemorySpaceFn(type); |
| 63 | } |
| 64 | return nullptr; |
| 65 | } |
| 66 | |
| 67 | /// Return the index-th bufferized function argument type. This assumes that the |
| 68 | /// specified argument is a tensor. If the tensor is ranked, a layout map may be |
| 69 | /// specified by the user (as per `options.functionArgTypeConverterFn`). |
| 70 | static BufferLikeType |
| 71 | getBufferizedFunctionArgType(FuncOp funcOp, int64_t index, |
| 72 | const BufferizationOptions &options) { |
| 73 | auto type = |
| 74 | dyn_cast<TensorLikeType>(Val: funcOp.getFunctionType().getInput(i: index)); |
| 75 | assert(type && "expected TensorLikeType" ); |
| 76 | |
| 77 | // Note: For builtin tensors there is additional logic related to layout. |
| 78 | if (auto tensorType = dyn_cast<TensorType>(Val&: type)) { |
| 79 | BufferLikeType memrefType = options.functionArgTypeConverterFn( |
| 80 | type, *options.defaultMemorySpaceFn(type), funcOp, options); |
| 81 | |
| 82 | auto layoutAttr = funcOp.getArgAttrOfType<MemRefLayoutAttrInterface>( |
| 83 | index, name: BufferizationDialect::kBufferLayoutAttrName); |
| 84 | if (!layoutAttr) |
| 85 | return memrefType; |
| 86 | |
| 87 | auto rankedMemrefType = dyn_cast<MemRefType>(Val&: memrefType); |
| 88 | assert(rankedMemrefType && |
| 89 | "buffer layout not supported on unranked tensors" ); |
| 90 | return cast<BufferLikeType>(Val: MemRefType::get( |
| 91 | shape: rankedMemrefType.getShape(), elementType: rankedMemrefType.getElementType(), |
| 92 | layout: layoutAttr, memorySpace: rankedMemrefType.getMemorySpace())); |
| 93 | } |
| 94 | |
| 95 | return options.functionArgTypeConverterFn(type, /*memSpace=*/nullptr, funcOp, |
| 96 | options); |
| 97 | } |
| 98 | |
| 99 | /// Return the FuncOp called by `callOp`. |
| 100 | static FuncOp getCalledFunction(CallOpInterface callOp, |
| 101 | SymbolTableCollection &symbolTables) { |
| 102 | return dyn_cast_or_null<FuncOp>(Val: callOp.resolveCallableInTable(symbolTable: &symbolTables)); |
| 103 | } |
| 104 | |
| 105 | /// Return the FuncOp called by `callOp`. |
| 106 | static FuncOp getCalledFunction(CallOpInterface callOp, |
| 107 | const AnalysisState &state) { |
| 108 | if (isa<OneShotAnalysisState>(Val: state)) { |
| 109 | auto &oneShotAnalysisState = |
| 110 | static_cast<const OneShotAnalysisState &>(state); |
| 111 | if (auto *funcAnalysisState = |
| 112 | oneShotAnalysisState.getExtension<FuncAnalysisState>()) { |
| 113 | // Use the cached symbol tables. |
| 114 | return getCalledFunction(callOp, symbolTables&: funcAnalysisState->symbolTables); |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | SymbolTableCollection symbolTables; |
| 119 | return getCalledFunction(callOp, symbolTables); |
| 120 | } |
| 121 | |
| 122 | /// Get FuncAnalysisState. |
| 123 | static const FuncAnalysisState & |
| 124 | getFuncAnalysisState(const AnalysisState &state) { |
| 125 | assert(isa<OneShotAnalysisState>(state) && "expected OneShotAnalysisState" ); |
| 126 | auto *result = static_cast<const OneShotAnalysisState &>(state) |
| 127 | .getExtension<FuncAnalysisState>(); |
| 128 | assert(result && "FuncAnalysisState does not exist" ); |
| 129 | return *result; |
| 130 | } |
| 131 | |
| 132 | /// Return the state (phase) of analysis of the FuncOp. |
| 133 | static FuncOpAnalysisState getFuncOpAnalysisState(const AnalysisState &state, |
| 134 | FuncOp funcOp) { |
| 135 | if (!isa<OneShotAnalysisState>(Val: state)) |
| 136 | return FuncOpAnalysisState::NotAnalyzed; |
| 137 | auto *funcState = static_cast<const OneShotAnalysisState &>(state) |
| 138 | .getExtension<FuncAnalysisState>(); |
| 139 | if (!funcState) |
| 140 | return FuncOpAnalysisState::NotAnalyzed; |
| 141 | const auto &analyzedFuncOps = funcState->analyzedFuncOps; |
| 142 | auto it = analyzedFuncOps.find(Val: funcOp); |
| 143 | if (it == analyzedFuncOps.end()) |
| 144 | return FuncOpAnalysisState::NotAnalyzed; |
| 145 | return it->second; |
| 146 | } |
| 147 | |
| 148 | /// Return the index of the bbArg in the given FuncOp that is equivalent to the |
| 149 | /// specified return value (if any). |
| 150 | static std::optional<int64_t> |
| 151 | getEquivalentFuncArgIdx(FuncOp funcOp, const FuncAnalysisState &state, |
| 152 | int64_t returnValIdx) { |
| 153 | auto funcOpIt = state.equivalentFuncArgs.find(Val: funcOp); |
| 154 | if (funcOpIt == state.equivalentFuncArgs.end()) |
| 155 | // No equivalence info stores for funcOp. |
| 156 | return std::nullopt; |
| 157 | |
| 158 | auto retValIt = funcOpIt->getSecond().find(Val: returnValIdx); |
| 159 | if (retValIt == funcOpIt->getSecond().end()) |
| 160 | // Return value has no equivalent bbArg. |
| 161 | return std::nullopt; |
| 162 | |
| 163 | return retValIt->getSecond(); |
| 164 | } |
| 165 | |
| 166 | struct CallOpInterface |
| 167 | : public BufferizableOpInterface::ExternalModel<CallOpInterface, |
| 168 | func::CallOp> { |
| 169 | bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, |
| 170 | const AnalysisState &state) const { |
| 171 | func::CallOp callOp = cast<func::CallOp>(Val: op); |
| 172 | FuncOp funcOp = getCalledFunction(callOp, state); |
| 173 | assert(funcOp && "expected CallOp to a FuncOp" ); |
| 174 | |
| 175 | if (getFuncOpAnalysisState(state, funcOp) != FuncOpAnalysisState::Analyzed) |
| 176 | // FuncOp not analyzed yet. Assume that OpOperand is read. |
| 177 | return true; |
| 178 | |
| 179 | const FuncAnalysisState &funcState = getFuncAnalysisState(state); |
| 180 | return funcState.readBbArgs.lookup(Val: funcOp).contains( |
| 181 | V: opOperand.getOperandNumber()); |
| 182 | } |
| 183 | |
| 184 | bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, |
| 185 | const AnalysisState &state) const { |
| 186 | func::CallOp callOp = cast<func::CallOp>(Val: op); |
| 187 | FuncOp funcOp = getCalledFunction(callOp, state); |
| 188 | assert(funcOp && "expected CallOp to a FuncOp" ); |
| 189 | |
| 190 | if (getFuncOpAnalysisState(state, funcOp) != FuncOpAnalysisState::Analyzed) |
| 191 | // FuncOp not analyzed yet. Assume that OpOperand is written. |
| 192 | return true; |
| 193 | |
| 194 | const FuncAnalysisState &funcState = getFuncAnalysisState(state); |
| 195 | return funcState.writtenBbArgs.lookup(Val: funcOp).contains( |
| 196 | V: opOperand.getOperandNumber()); |
| 197 | } |
| 198 | |
| 199 | AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand, |
| 200 | const AnalysisState &state) const { |
| 201 | func::CallOp callOp = cast<func::CallOp>(Val: op); |
| 202 | FuncOp funcOp = getCalledFunction(callOp, state); |
| 203 | assert(funcOp && "expected CallOp to a FuncOp" ); |
| 204 | if (getFuncOpAnalysisState(state, funcOp) != FuncOpAnalysisState::Analyzed) |
| 205 | // FuncOp not analyzed yet. Any OpResult may be aliasing. |
| 206 | return detail::unknownGetAliasingValues(opOperand); |
| 207 | |
| 208 | // Get aliasing results from state. |
| 209 | const FuncAnalysisState &funcState = getFuncAnalysisState(state); |
| 210 | auto aliasingReturnVals = |
| 211 | funcState.aliasingReturnVals.lookup(Val: funcOp).lookup( |
| 212 | Val: opOperand.getOperandNumber()); |
| 213 | |
| 214 | // Check if the aliasing OpResult is equivalent to the OpOperand. |
| 215 | std::optional<int64_t> equivalent = {}; |
| 216 | if (aliasingReturnVals.size() == 1) { |
| 217 | equivalent = getEquivalentFuncArgIdx(funcOp, state: funcState, |
| 218 | returnValIdx: aliasingReturnVals.front()); |
| 219 | assert((!equivalent.has_value() || |
| 220 | *equivalent == opOperand.getOperandNumber()) && |
| 221 | "inconsistent analysis state" ); |
| 222 | } |
| 223 | AliasingValueList result; |
| 224 | for (int64_t resultIdx : aliasingReturnVals) |
| 225 | result.addAlias(alias: {callOp->getOpResult(idx: resultIdx), |
| 226 | equivalent.has_value() ? BufferRelation::Equivalent |
| 227 | : BufferRelation::Unknown, |
| 228 | /*isDefinite=*/equivalent.has_value()}); |
| 229 | return result; |
| 230 | } |
| 231 | |
| 232 | bool isWritable(Operation *op, Value value, |
| 233 | const AnalysisState &state) const { |
| 234 | auto callOp = cast<func::CallOp>(Val: op); |
| 235 | FuncOp funcOp = getCalledFunction(callOp, state); |
| 236 | assert(funcOp && "expected CallOp to a FuncOp" ); |
| 237 | |
| 238 | // Unknown or not yet analyzed functions may return read-only buffers. |
| 239 | if (getFuncOpAnalysisState(state, funcOp) != FuncOpAnalysisState::Analyzed) |
| 240 | return false; |
| 241 | |
| 242 | const FuncAnalysisState &funcState = getFuncAnalysisState(state); |
| 243 | return funcState.writableReturnVals.lookup(Val: funcOp).contains( |
| 244 | V: cast<OpResult>(Val&: value).getResultNumber()); |
| 245 | } |
| 246 | |
| 247 | FailureOr<BufferLikeType> |
| 248 | getBufferType(Operation *op, Value value, const BufferizationOptions &options, |
| 249 | const BufferizationState &state, |
| 250 | SmallVector<Value> &invocationStack) const { |
| 251 | auto callOp = cast<func::CallOp>(Val: op); |
| 252 | |
| 253 | // Reuse the cached symbol tables from the bufferization state. |
| 254 | FuncOp funcOp = getCalledFunction(callOp, symbolTables&: state.getSymbolTables()); |
| 255 | assert(funcOp && "expected CallOp to a FuncOp" ); |
| 256 | |
| 257 | // If the callee was already bufferized, we can directly take the type from |
| 258 | // its signature. |
| 259 | FunctionType funcType = funcOp.getFunctionType(); |
| 260 | Type resultType = |
| 261 | funcType.getResult(i: cast<OpResult>(Val&: value).getResultNumber()); |
| 262 | if (auto bufferizedType = dyn_cast<BufferLikeType>(Val&: resultType)) |
| 263 | return bufferizedType; |
| 264 | |
| 265 | // Otherwise, call the type converter to compute the bufferized type. |
| 266 | auto tensorType = cast<TensorLikeType>(Val&: resultType); |
| 267 | return cast<BufferLikeType>(Val: options.functionArgTypeConverterFn( |
| 268 | tensorType, getDefaultMemorySpace(options, type: tensorType), funcOp, |
| 269 | options)); |
| 270 | } |
| 271 | |
| 272 | /// All function arguments are writable. It is the responsibility of the |
| 273 | /// CallOp to insert buffer copies where necessary. |
| 274 | LogicalResult bufferize(Operation *op, RewriterBase &rewriter, |
| 275 | const BufferizationOptions &options, |
| 276 | BufferizationState &state) const { |
| 277 | func::CallOp callOp = cast<func::CallOp>(Val: op); |
| 278 | |
| 279 | // 1. Compute the result types of the new CallOp. |
| 280 | SmallVector<Type> resultTypes; |
| 281 | for (Value result : callOp.getResults()) { |
| 282 | Type returnType = result.getType(); |
| 283 | if (!isa<TensorLikeType>(Val: returnType)) { |
| 284 | // Non-tensor values are returned. |
| 285 | resultTypes.push_back(Elt: returnType); |
| 286 | continue; |
| 287 | } |
| 288 | |
| 289 | // Returning a memref. |
| 290 | FailureOr<BufferLikeType> resultType = |
| 291 | bufferization::getBufferType(value: result, options, state); |
| 292 | if (failed(Result: resultType)) |
| 293 | return failure(); |
| 294 | resultTypes.push_back(Elt: *resultType); |
| 295 | } |
| 296 | |
| 297 | // 2. Rewrite tensor operands as memrefs based on type of the already |
| 298 | // bufferized callee. |
| 299 | SmallVector<Value> newOperands; |
| 300 | |
| 301 | FuncOp funcOp = getCalledFunction(callOp, symbolTables&: state.getSymbolTables()); |
| 302 | assert(funcOp && "expected CallOp to a FuncOp" ); |
| 303 | FunctionType funcType = funcOp.getFunctionType(); |
| 304 | |
| 305 | for (OpOperand &opOperand : callOp->getOpOperands()) { |
| 306 | // Non-tensor operands are just copied. |
| 307 | if (!isa<TensorLikeType>(Val: opOperand.get().getType())) { |
| 308 | newOperands.push_back(Elt: opOperand.get()); |
| 309 | continue; |
| 310 | } |
| 311 | |
| 312 | // Retrieve buffers for tensor operands. |
| 313 | FailureOr<Value> maybeBuffer = |
| 314 | getBuffer(rewriter, value: opOperand.get(), options, state); |
| 315 | if (failed(Result: maybeBuffer)) |
| 316 | return failure(); |
| 317 | Value buffer = *maybeBuffer; |
| 318 | |
| 319 | // Caller / callee type mismatch is handled with castOrReallocMemRefValue. |
| 320 | auto bufferType = funcType.getInput(i: opOperand.getOperandNumber()); |
| 321 | if (!isa<BufferLikeType>(Val: bufferType)) { |
| 322 | // The called function was not bufferized yet. This can happen when |
| 323 | // there cycles in the function call graph. Compute the bufferized |
| 324 | // result type. |
| 325 | FailureOr<BufferLikeType> maybeBufferType = |
| 326 | bufferization::getBufferType( |
| 327 | value: funcOp.getArgument(idx: opOperand.getOperandNumber()), options, |
| 328 | state); |
| 329 | if (failed(Result: maybeBufferType)) |
| 330 | return failure(); |
| 331 | bufferType = *maybeBufferType; |
| 332 | } |
| 333 | |
| 334 | // Since we don't yet have a clear layout story, to_buffer may |
| 335 | // conservatively turn tensors into more dynamic memref than necessary. |
| 336 | // If the memref type of the callee fails, introduce an extra memref.cast |
| 337 | // that will either canonicalize away or fail compilation until we can do |
| 338 | // something better. Insert a reallocation + copy if it cannot be |
| 339 | // statically guaranteed that a direct cast would be valid. |
| 340 | if (buffer.getType() != bufferType) { |
| 341 | auto memrefDstType = dyn_cast<MemRefType>(Val&: bufferType); |
| 342 | assert(memrefDstType && |
| 343 | "buffer layout not supported on unranked tensors" ); |
| 344 | FailureOr<Value> replacement = |
| 345 | bufferization::castOrReallocMemRefValue(b&: rewriter, value: buffer, |
| 346 | type: memrefDstType, options); |
| 347 | if (failed(Result: replacement)) |
| 348 | return failure(); |
| 349 | buffer = *replacement; |
| 350 | } |
| 351 | newOperands.push_back(Elt: buffer); |
| 352 | } |
| 353 | |
| 354 | // 3. Create the new CallOp. |
| 355 | func::CallOp newCallOp = |
| 356 | func::CallOp::create(builder&: rewriter, location: callOp.getLoc(), resultTypes, |
| 357 | operands: newOperands, properties: callOp.getProperties(), |
| 358 | discardableAttributes: callOp->getDiscardableAttrDictionary().getValue()); |
| 359 | |
| 360 | // 4. Replace the old op with the new op. |
| 361 | replaceOpWithBufferizedValues(rewriter, op: callOp, values: newCallOp->getResults()); |
| 362 | |
| 363 | return success(); |
| 364 | } |
| 365 | }; |
| 366 | |
| 367 | struct ReturnOpInterface |
| 368 | : public BufferizableOpInterface::ExternalModel<ReturnOpInterface, |
| 369 | func::ReturnOp> { |
| 370 | bool bufferizesToMemoryRead(Operation *op, OpOperand &opOperand, |
| 371 | const AnalysisState &state) const { |
| 372 | return true; |
| 373 | } |
| 374 | |
| 375 | bool bufferizesToMemoryWrite(Operation *op, OpOperand &opOperand, |
| 376 | const AnalysisState &state) const { |
| 377 | return false; |
| 378 | } |
| 379 | |
| 380 | AliasingValueList getAliasingValues(Operation *op, OpOperand &opOperand, |
| 381 | const AnalysisState &state) const { |
| 382 | return {}; |
| 383 | } |
| 384 | |
| 385 | LogicalResult bufferize(Operation *op, RewriterBase &rewriter, |
| 386 | const BufferizationOptions &options, |
| 387 | BufferizationState &state) const { |
| 388 | #ifndef NDEBUG |
| 389 | auto returnOp = cast<func::ReturnOp>(op); |
| 390 | assert(isa<FuncOp>(returnOp->getParentOp()) && |
| 391 | "only support FuncOp parent for ReturnOp" ); |
| 392 | #endif // NDEBUG |
| 393 | |
| 394 | // ReturnOps are bufferized as part of FuncOps. |
| 395 | return success(); |
| 396 | } |
| 397 | }; |
| 398 | |
| 399 | struct FuncOpInterface |
| 400 | : public OpWithUnstructuredControlFlowBufferizableOpInterfaceExternalModel< |
| 401 | FuncOpInterface, FuncOp> { |
| 402 | |
| 403 | static bool supportsUnstructuredControlFlow() { return true; } |
| 404 | |
| 405 | bool hasTensorSemantics(Operation *op) const { |
| 406 | auto isaTensor = llvm::IsaPred<TensorLikeType>; |
| 407 | |
| 408 | // A function has tensor semantics if it has tensor arguments/results. |
| 409 | auto funcOp = cast<FuncOp>(Val: op); |
| 410 | bool hasTensorArg = any_of(Range: funcOp.getArgumentTypes(), P: isaTensor); |
| 411 | bool hasTensorResult = any_of(Range: funcOp.getResultTypes(), P: isaTensor); |
| 412 | if (hasTensorArg || hasTensorResult) |
| 413 | return true; |
| 414 | |
| 415 | // It also has tensor semantics if it has tensor block arguments. |
| 416 | // TODO: Decouple bufferization of unstructured control flow from |
| 417 | // BufferizableOpInterface implementations. We should only care about |
| 418 | // region entry block arguments here (which are already covered by the |
| 419 | // argument types of the function). |
| 420 | for (Block &block : funcOp.getBody()) |
| 421 | if (any_of(Range: block.getArgumentTypes(), P: isaTensor)) |
| 422 | return true; |
| 423 | |
| 424 | return false; |
| 425 | } |
| 426 | |
| 427 | AliasingOpOperandList |
| 428 | getAliasingOpOperands(Operation *op, Value value, |
| 429 | const AnalysisState &state) const { |
| 430 | return getAliasingBranchOpOperands(op, bbArg: cast<BlockArgument>(Val&: value), state); |
| 431 | } |
| 432 | |
| 433 | FailureOr<BufferLikeType> |
| 434 | getBufferType(Operation *op, Value value, const BufferizationOptions &options, |
| 435 | const BufferizationState &state, |
| 436 | SmallVector<Value> &invocationStack) const { |
| 437 | auto funcOp = cast<FuncOp>(Val: op); |
| 438 | auto bbArg = cast<BlockArgument>(Val&: value); |
| 439 | |
| 440 | // Function arguments are special. |
| 441 | if (bbArg.getOwner() == &funcOp.getBody().front()) |
| 442 | return getBufferizedFunctionArgType(funcOp, index: bbArg.getArgNumber(), |
| 443 | options); |
| 444 | |
| 445 | return OpWithUnstructuredControlFlowBufferizableOpInterfaceExternalModel:: |
| 446 | getBufferType(op, value, options, state, invocationStack); |
| 447 | } |
| 448 | |
| 449 | /// Rewrite function bbArgs and return values into buffer form. This function |
| 450 | /// bufferizes the function signature and the ReturnOp. When the entire |
| 451 | /// function body has been bufferized, function return types can be switched |
| 452 | /// to more concise memref types as part of `foldMemRefCasts`. |
| 453 | /// |
| 454 | /// All function bbArgs are writable unless they are explicitly marked as |
| 455 | /// read-only. Callers must insert copies when needed. |
| 456 | LogicalResult bufferize(Operation *op, RewriterBase &rewriter, |
| 457 | const BufferizationOptions &options, |
| 458 | BufferizationState &state) const { |
| 459 | auto funcOp = cast<FuncOp>(Val: op); |
| 460 | FunctionType funcType = funcOp.getFunctionType(); |
| 461 | |
| 462 | // Compute the argument types. |
| 463 | SmallVector<Type> argTypes; |
| 464 | for (const auto &it : llvm::enumerate(First: funcType.getInputs())) { |
| 465 | Type argType = it.value(); |
| 466 | if (isa<TensorLikeType>(Val: argType)) { |
| 467 | argTypes.push_back( |
| 468 | Elt: getBufferizedFunctionArgType(funcOp, index: it.index(), options)); |
| 469 | continue; |
| 470 | } |
| 471 | argTypes.push_back(Elt: argType); |
| 472 | } |
| 473 | |
| 474 | // Compute the result types. |
| 475 | SmallVector<Type> retTypes; |
| 476 | for (Type resultType : funcType.getResults()) { |
| 477 | if (auto tensorType = dyn_cast<TensorLikeType>(Val&: resultType)) { |
| 478 | BufferLikeType resultType = options.functionArgTypeConverterFn( |
| 479 | tensorType, getDefaultMemorySpace(options, type: tensorType), funcOp, |
| 480 | options); |
| 481 | retTypes.push_back(Elt: resultType); |
| 482 | continue; |
| 483 | } |
| 484 | retTypes.push_back(Elt: resultType); |
| 485 | } |
| 486 | |
| 487 | // Compute the new function type. |
| 488 | auto newFuncType = FunctionType::get(context: op->getContext(), inputs: argTypes, results: retTypes); |
| 489 | |
| 490 | // If the function has no body, set the new function type and we are done. |
| 491 | if (funcOp.isExternal()) { |
| 492 | funcOp.setType(newFuncType); |
| 493 | return success(); |
| 494 | } |
| 495 | |
| 496 | // 1. Bufferize every block. |
| 497 | for (Block &block : funcOp.getBody()) |
| 498 | if (failed(Result: bufferization::bufferizeBlockSignature(block: &block, rewriter, |
| 499 | options, state))) |
| 500 | return failure(); |
| 501 | |
| 502 | // 2. Bufferize the operands of the all return op. |
| 503 | for (func::ReturnOp returnOp : getReturnOps(funcOp)) { |
| 504 | assert(returnOp->getNumOperands() == retTypes.size() && |
| 505 | "incorrect number of return values" ); |
| 506 | SmallVector<Value> returnValues; |
| 507 | for (auto [returnVal, bufferizedType] : |
| 508 | llvm::zip_equal(t: returnOp->getOperands(), u&: retTypes)) { |
| 509 | auto tensorType = dyn_cast<TensorLikeType>(Val: returnVal.getType()); |
| 510 | rewriter.setInsertionPoint(returnOp); |
| 511 | |
| 512 | // If not a tensor type just forward it. |
| 513 | if (!tensorType) { |
| 514 | returnValues.push_back(Elt: returnVal); |
| 515 | continue; |
| 516 | } |
| 517 | |
| 518 | // Note: If `inferFunctionResultLayout = true`, casts are later folded |
| 519 | // away. |
| 520 | Value toBufferOp = bufferization::ToBufferOp::create( |
| 521 | builder&: rewriter, location: returnOp.getLoc(), buffer: bufferizedType, tensor: returnVal); |
| 522 | returnValues.push_back(Elt: toBufferOp); |
| 523 | } |
| 524 | |
| 525 | returnOp.getOperandsMutable().assign(values: returnValues); |
| 526 | } |
| 527 | |
| 528 | // 3. Set the new function type. |
| 529 | funcOp.setType(newFuncType); |
| 530 | return success(); |
| 531 | } |
| 532 | |
| 533 | /// Return `true` if the given function argument is writable. |
| 534 | bool isWritable(Operation *op, Value value, |
| 535 | const AnalysisState &state) const { |
| 536 | auto funcOp = cast<FuncOp>(Val: op); |
| 537 | BlockArgument bbArg = dyn_cast<BlockArgument>(Val&: value); |
| 538 | assert(bbArg && "expected BlockArgument" ); |
| 539 | |
| 540 | // Non-entry block arguments are always writable. (They may alias with |
| 541 | // values that are not writable, which will turn them into read-only.) |
| 542 | if (bbArg.getOwner() != &funcOp.getBody().front()) |
| 543 | return true; |
| 544 | |
| 545 | // "bufferization.writable" overrides other writability decisions. This is |
| 546 | // currently used for testing only. |
| 547 | if (BoolAttr writable = funcOp.getArgAttrOfType<BoolAttr>( |
| 548 | index: bbArg.getArgNumber(), name: BufferizationDialect::kWritableAttrName)) |
| 549 | return writable.getValue(); |
| 550 | |
| 551 | // All function arguments are writable by default. |
| 552 | return true; |
| 553 | } |
| 554 | }; |
| 555 | |
| 556 | } // namespace func_ext |
| 557 | } // namespace bufferization |
| 558 | } // namespace mlir |
| 559 | |
| 560 | void mlir::bufferization::func_ext:: |
| 561 | registerBufferizableOpInterfaceExternalModels(DialectRegistry ®istry) { |
| 562 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, func::FuncDialect *dialect) { |
| 563 | func::CallOp::attachInterface<func_ext::CallOpInterface>(context&: *ctx); |
| 564 | func::FuncOp::attachInterface<func_ext::FuncOpInterface>(context&: *ctx); |
| 565 | func::ReturnOp::attachInterface<func_ext::ReturnOpInterface>(context&: *ctx); |
| 566 | }); |
| 567 | } |
| 568 | |