1//===- StackToShared.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements transforms to swap stack allocations on the target
10// device with device shared memory where applicable.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/OpenMP/Transforms/Passes.h"
15
16#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
17#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
18#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
19#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
20#include "mlir/Dialect/OpenMP/Utils/Utils.h"
21#include "mlir/Pass/Pass.h"
22#include "llvm/ADT/STLExtras.h"
23
24namespace mlir {
25namespace omp {
26#define GEN_PASS_DEF_STACKTOSHAREDPASS
27#include "mlir/Dialect/OpenMP/Transforms/Passes.h.inc"
28} // namespace omp
29} // namespace mlir
30
31using namespace mlir;
32
33/// Tell whether to replace an operation representing a stack allocation with a
34/// device shared memory allocation/deallocation pair based on the location of
35/// the allocation and its uses.
36static bool shouldReplaceAllocaWithDeviceSharedMem(Operation &op) {
37 return omp::opInSharedDeviceContext(op) &&
38 llvm::any_of(Range: op.getResults(), P: [&](Value result) {
39 return omp::allocaUsesRequireSharedMem(alloc: result);
40 });
41}
42
43/// Based on the location of the definition of the given value representing the
44/// result of a device shared memory allocation, find the corresponding points
45/// where its deallocation should be placed and introduce `omp.free_shared_mem`
46/// ops at those points.
47static void insertDeviceSharedMemDeallocation(OpBuilder &builder,
48 TypeAttr elemType,
49 Value arraySize,
50 IntegerAttr alignment,
51 Value allocVal) {
52 Block *allocaBlock = allocVal.getParentBlock();
53 DominanceInfo domInfo;
54 for (Block &block : allocVal.getParentRegion()->getBlocks()) {
55 Operation *terminator = block.getTerminator();
56 if (!terminator->hasSuccessors() &&
57 domInfo.dominates(a: allocaBlock, b: &block)) {
58 builder.setInsertionPoint(terminator);
59 omp::FreeSharedMemOp::create(builder, location: allocVal.getLoc(), mem_elem_type: elemType,
60 mem_array_size: arraySize, mem_alignment: alignment, heapref: allocVal);
61 }
62 }
63}
64
65namespace {
66class StackToSharedPass
67 : public omp::impl::StackToSharedPassBase<StackToSharedPass> {
68public:
69 StackToSharedPass() = default;
70
71 void runOnOperation() override {
72 MLIRContext *context = &getContext();
73 OpBuilder builder(context);
74
75 LLVM::LLVMFuncOp funcOp = getOperation();
76 auto offloadIface = funcOp->getParentOfType<omp::OffloadModuleInterface>();
77 if (!offloadIface || !offloadIface.getIsTargetDevice())
78 return;
79
80 llvm::SmallVector<Operation *> toBeDeleted;
81 funcOp->walk(callback: [&](LLVM::AllocaOp allocaOp) {
82 if (!shouldReplaceAllocaWithDeviceSharedMem(op&: *allocaOp))
83 return;
84 // Replace llvm.alloca with omp.alloc_shared_mem.
85 Type resultType = allocaOp.getResult().getType();
86
87 // TODO: The handling of non-default address spaces might need to be
88 // improved. This currently only handles the case where an alloca to
89 // non-default address space is only used by a single addrspacecast to
90 // default address space.
91 bool nonDefaultAddrSpace = false;
92 if (auto llvmPtrType = dyn_cast<LLVM::LLVMPointerType>(Val&: resultType))
93 nonDefaultAddrSpace = llvmPtrType.getAddressSpace() != 0;
94
95 builder.setInsertionPoint(allocaOp);
96 auto sharedAllocOp = omp::AllocSharedMemOp::create(
97 builder, location: allocaOp->getLoc(), resultType0: LLVM::LLVMPointerType::get(context),
98 mem_elem_type: allocaOp.getElemTypeAttr(), mem_array_size: allocaOp.getArraySize(),
99 mem_alignment: allocaOp.getAlignmentAttr());
100 if (nonDefaultAddrSpace) {
101 assert(allocaOp->hasOneUse() && " unsupported non-default address "
102 "space alloca with multiple uses");
103 auto asCastOp =
104 cast<LLVM::AddrSpaceCastOp>(Val: *allocaOp->getUsers().begin());
105 asCastOp.replaceAllUsesWith(op: sharedAllocOp.getOperation());
106 // Delete later because we can't delete the cast op before the top-level
107 // iteration visits it. Also, the alloca can't be deleted before because
108 // it's used by it.
109 toBeDeleted.push_back(Elt: asCastOp);
110 toBeDeleted.push_back(Elt: allocaOp);
111 } else {
112 allocaOp.replaceAllUsesWith(op: sharedAllocOp.getOperation());
113 allocaOp.erase();
114 }
115
116 // Create a new omp.free_shared_mem for the allocated buffer prior to
117 // exiting the region.
118 insertDeviceSharedMemDeallocation(
119 builder, elemType: sharedAllocOp.getMemElemTypeAttr(),
120 arraySize: sharedAllocOp.getMemArraySize(), alignment: sharedAllocOp.getMemAlignmentAttr(),
121 allocVal: sharedAllocOp.getResult());
122 });
123 for (Operation *op : toBeDeleted)
124 op->erase();
125 }
126};
127} // namespace
128