| 1 | //===- StackToShared.cpp -------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements transforms to swap stack allocations on the target |
| 10 | // device with device shared memory where applicable. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Dialect/OpenMP/Transforms/Passes.h" |
| 15 | |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMConversionOps.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 18 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 19 | #include "mlir/Dialect/OpenMP/OpenMPDialect.h" |
| 20 | #include "mlir/Dialect/OpenMP/Utils/Utils.h" |
| 21 | #include "mlir/Pass/Pass.h" |
| 22 | #include "llvm/ADT/STLExtras.h" |
| 23 | |
| 24 | namespace mlir { |
| 25 | namespace omp { |
| 26 | #define GEN_PASS_DEF_STACKTOSHAREDPASS |
| 27 | #include "mlir/Dialect/OpenMP/Transforms/Passes.h.inc" |
| 28 | } // namespace omp |
| 29 | } // namespace mlir |
| 30 | |
| 31 | using namespace mlir; |
| 32 | |
| 33 | /// Tell whether to replace an operation representing a stack allocation with a |
| 34 | /// device shared memory allocation/deallocation pair based on the location of |
| 35 | /// the allocation and its uses. |
| 36 | static bool shouldReplaceAllocaWithDeviceSharedMem(Operation &op) { |
| 37 | return omp::opInSharedDeviceContext(op) && |
| 38 | llvm::any_of(Range: op.getResults(), P: [&](Value result) { |
| 39 | return omp::allocaUsesRequireSharedMem(alloc: result); |
| 40 | }); |
| 41 | } |
| 42 | |
| 43 | /// Based on the location of the definition of the given value representing the |
| 44 | /// result of a device shared memory allocation, find the corresponding points |
| 45 | /// where its deallocation should be placed and introduce `omp.free_shared_mem` |
| 46 | /// ops at those points. |
| 47 | static void insertDeviceSharedMemDeallocation(OpBuilder &builder, |
| 48 | TypeAttr elemType, |
| 49 | Value arraySize, |
| 50 | IntegerAttr alignment, |
| 51 | Value allocVal) { |
| 52 | Block *allocaBlock = allocVal.getParentBlock(); |
| 53 | DominanceInfo domInfo; |
| 54 | for (Block &block : allocVal.getParentRegion()->getBlocks()) { |
| 55 | Operation *terminator = block.getTerminator(); |
| 56 | if (!terminator->hasSuccessors() && |
| 57 | domInfo.dominates(a: allocaBlock, b: &block)) { |
| 58 | builder.setInsertionPoint(terminator); |
| 59 | omp::FreeSharedMemOp::create(builder, location: allocVal.getLoc(), mem_elem_type: elemType, |
| 60 | mem_array_size: arraySize, mem_alignment: alignment, heapref: allocVal); |
| 61 | } |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | namespace { |
| 66 | class StackToSharedPass |
| 67 | : public omp::impl::StackToSharedPassBase<StackToSharedPass> { |
| 68 | public: |
| 69 | StackToSharedPass() = default; |
| 70 | |
| 71 | void runOnOperation() override { |
| 72 | MLIRContext *context = &getContext(); |
| 73 | OpBuilder builder(context); |
| 74 | |
| 75 | LLVM::LLVMFuncOp funcOp = getOperation(); |
| 76 | auto offloadIface = funcOp->getParentOfType<omp::OffloadModuleInterface>(); |
| 77 | if (!offloadIface || !offloadIface.getIsTargetDevice()) |
| 78 | return; |
| 79 | |
| 80 | llvm::SmallVector<Operation *> toBeDeleted; |
| 81 | funcOp->walk(callback: [&](LLVM::AllocaOp allocaOp) { |
| 82 | if (!shouldReplaceAllocaWithDeviceSharedMem(op&: *allocaOp)) |
| 83 | return; |
| 84 | // Replace llvm.alloca with omp.alloc_shared_mem. |
| 85 | Type resultType = allocaOp.getResult().getType(); |
| 86 | |
| 87 | // TODO: The handling of non-default address spaces might need to be |
| 88 | // improved. This currently only handles the case where an alloca to |
| 89 | // non-default address space is only used by a single addrspacecast to |
| 90 | // default address space. |
| 91 | bool nonDefaultAddrSpace = false; |
| 92 | if (auto llvmPtrType = dyn_cast<LLVM::LLVMPointerType>(Val&: resultType)) |
| 93 | nonDefaultAddrSpace = llvmPtrType.getAddressSpace() != 0; |
| 94 | |
| 95 | builder.setInsertionPoint(allocaOp); |
| 96 | auto sharedAllocOp = omp::AllocSharedMemOp::create( |
| 97 | builder, location: allocaOp->getLoc(), resultType0: LLVM::LLVMPointerType::get(context), |
| 98 | mem_elem_type: allocaOp.getElemTypeAttr(), mem_array_size: allocaOp.getArraySize(), |
| 99 | mem_alignment: allocaOp.getAlignmentAttr()); |
| 100 | if (nonDefaultAddrSpace) { |
| 101 | assert(allocaOp->hasOneUse() && " unsupported non-default address " |
| 102 | "space alloca with multiple uses" ); |
| 103 | auto asCastOp = |
| 104 | cast<LLVM::AddrSpaceCastOp>(Val: *allocaOp->getUsers().begin()); |
| 105 | asCastOp.replaceAllUsesWith(op: sharedAllocOp.getOperation()); |
| 106 | // Delete later because we can't delete the cast op before the top-level |
| 107 | // iteration visits it. Also, the alloca can't be deleted before because |
| 108 | // it's used by it. |
| 109 | toBeDeleted.push_back(Elt: asCastOp); |
| 110 | toBeDeleted.push_back(Elt: allocaOp); |
| 111 | } else { |
| 112 | allocaOp.replaceAllUsesWith(op: sharedAllocOp.getOperation()); |
| 113 | allocaOp.erase(); |
| 114 | } |
| 115 | |
| 116 | // Create a new omp.free_shared_mem for the allocated buffer prior to |
| 117 | // exiting the region. |
| 118 | insertDeviceSharedMemDeallocation( |
| 119 | builder, elemType: sharedAllocOp.getMemElemTypeAttr(), |
| 120 | arraySize: sharedAllocOp.getMemArraySize(), alignment: sharedAllocOp.getMemAlignmentAttr(), |
| 121 | allocVal: sharedAllocOp.getResult()); |
| 122 | }); |
| 123 | for (Operation *op : toBeDeleted) |
| 124 | op->erase(); |
| 125 | } |
| 126 | }; |
| 127 | } // namespace |
| 128 | |