1//===- OpenMPOffloadPrivatizationPrepare.cpp - Prepare OMP privatization --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Analysis/SliceAnalysis.h"
10#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h"
11#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
12#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
13#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
14#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
15#include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h"
16#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
17#include "mlir/IR/Builders.h"
18#include "mlir/IR/Dominance.h"
19#include "mlir/IR/IRMapping.h"
20#include "mlir/Pass/Pass.h"
21#include "mlir/Support/LLVM.h"
22#include "llvm/Support/DebugLog.h"
23#include "llvm/Support/FormatVariadic.h"
24#include <cstdint>
25#include <iterator>
26#include <utility>
27
28//===----------------------------------------------------------------------===//
29// A pass that prepares OpenMP code for translation of delayed privatization
30// in the context of deferred target tasks. Deferred target tasks are created
31// when the nowait clause is used on the target directive.
32//===----------------------------------------------------------------------===//
33
34#define DEBUG_TYPE "omp-prepare-for-offload-privatization"
35
36namespace mlir {
37namespace omp {
38
39#define GEN_PASS_DEF_PREPAREFOROMPOFFLOADPRIVATIZATIONPASS
40#include "mlir/Dialect/OpenMP/Transforms/Passes.h.inc"
41
42} // namespace omp
43} // namespace mlir
44
45using namespace mlir;
46namespace {
47
48//===----------------------------------------------------------------------===//
49// PrepareForOMPOffloadPrivatizationPass
50//===----------------------------------------------------------------------===//
51
52class PrepareForOMPOffloadPrivatizationPass
53 : public omp::impl::PrepareForOMPOffloadPrivatizationPassBase<
54 PrepareForOMPOffloadPrivatizationPass> {
55
56 void runOnOperation() override {
57 ModuleOp mod = getOperation();
58
59 // In this pass, we make host-allocated privatized variables persist for
60 // deferred target tasks by copying them to the heap. Once the target task
61 // is done, this heap memory is freed. Since all of this happens on the host
62 // we can skip device modules.
63 auto offloadModuleInterface =
64 dyn_cast<omp::OffloadModuleInterface>(Val: mod.getOperation());
65 if (offloadModuleInterface && offloadModuleInterface.getIsTargetDevice())
66 return;
67
68 getOperation()->walk(callback: [&](omp::TargetOp targetOp) {
69 if (!hasPrivateVars(targetOp) || !isTargetTaskDeferred(targetOp))
70 return;
71 IRRewriter rewriter(&getContext());
72 OperandRange privateVars = targetOp.getPrivateVars();
73 SmallVector<mlir::Value> newPrivVars;
74 Value fakeDependVar;
75 omp::TaskOp cleanupTaskOp;
76
77 newPrivVars.reserve(N: privateVars.size());
78 std::optional<ArrayAttr> privateSyms = targetOp.getPrivateSyms();
79 for (auto [privVarIdx, privVarSymPair] :
80 llvm::enumerate(First: llvm::zip_equal(t&: privateVars, u&: *privateSyms))) {
81 Value privVar = std::get<0>(t&: privVarSymPair);
82 Attribute privSym = std::get<1>(t&: privVarSymPair);
83
84 omp::PrivateClauseOp privatizer = findPrivatizer(op: targetOp, privSym);
85 if (!privatizer.needsMap()) {
86 newPrivVars.push_back(Elt: privVar);
87 continue;
88 }
89 bool isFirstPrivate = privatizer.getDataSharingType() ==
90 omp::DataSharingClauseType::FirstPrivate;
91
92 Value mappedValue = targetOp.getMappedValueForPrivateVar(privVarIdx);
93 auto mapInfoOp = cast<omp::MapInfoOp>(Val: mappedValue.getDefiningOp());
94
95 if (mapInfoOp.getMapCaptureType() == omp::VariableCaptureKind::ByCopy) {
96 newPrivVars.push_back(Elt: privVar);
97 continue;
98 }
99
100 // For deferred target tasks (!$omp target nowait), we need to keep
101 // a copy of the original, i.e. host variable being privatized so
102 // that it is available when the target task is eventually executed.
103 // We do this by first allocating as much heap memory as is needed by
104 // the original variable. Then, we use the init and copy regions of the
105 // privatizer, an instance of omp::PrivateClauseOp to set up the heap-
106 // allocated copy.
107 // After the target task is done, we need to use the dealloc region
108 // of the privatizer to clean up everything. We also need to free
109 // the heap memory we allocated. But due to the deferred nature
110 // of the target task, we cannot simply deallocate right after the
111 // omp.target operation else we may end up freeing memory before
112 // its eventual use by the target task. So, we create a dummy
113 // dependence between the target task and new omp.task. In the omp.task,
114 // we do all the cleanup. So, we end up with the following structure
115 //
116 // omp.target map_entries(..) ... nowait depend(out:fakeDependVar) {
117 // ...
118 // omp.terminator
119 // }
120 // omp.task depend(in: fakeDependVar) {
121 // /*cleanup_code*/
122 // omp.terminator
123 // }
124 // fakeDependVar is the address of the first heap-allocated copy of the
125 // host variable being privatized.
126
127 bool needsCleanupTask = !privatizer.getDeallocRegion().empty();
128
129 // Allocate heap memory that corresponds to the type of memory
130 // pointed to by varPtr
131 // For boxchars this won't be a pointer. But, MapsForPrivatizedSymbols
132 // should have mapped the pointer to the boxchar so use that as varPtr.
133 Value varPtr = mapInfoOp.getVarPtr();
134 Type varType = mapInfoOp.getVarPtrType();
135 bool isPrivatizedByValue =
136 !isa<LLVM::LLVMPointerType>(Val: privVar.getType());
137
138 assert(isa<LLVM::LLVMPointerType>(varPtr.getType()));
139 Value heapMem =
140 allocateHeapMem(targetOp, privVar: varPtr, varType, mod, rewriter);
141 if (!heapMem)
142 targetOp.emitError(
143 message: "Unable to allocate heap memory when trying to move "
144 "a private variable out of the stack and into the "
145 "heap for use by a deferred target task");
146
147 if (needsCleanupTask && !fakeDependVar)
148 fakeDependVar = heapMem;
149
150 // The types of private vars should match before and after the
151 // transformation. In particular, if the type is a pointer,
152 // simply record the newly allocated malloc location as the
153 // new private variable. If, however, the type is not a pointer
154 // then, we need to load the value from the newly allocated
155 // location. We'll insert that load later after we have updated
156 // the malloc'd location with the contents of the original
157 // variable.
158 if (!isPrivatizedByValue)
159 newPrivVars.push_back(Elt: heapMem);
160
161 // We now need to copy the original private variable into the newly
162 // allocated location in the heap.
163 // Find the earliest insertion point for the copy. This will be before
164 // the first in the list of omp::MapInfoOp instances that use varPtr.
165 // After the copy these omp::MapInfoOp instances will refer to heapMem
166 // instead.
167 Operation *varPtrDefiningOp = varPtr.getDefiningOp();
168 DenseSet<Operation *> users;
169 if (varPtrDefiningOp) {
170 users.insert(I: varPtrDefiningOp->user_begin(),
171 E: varPtrDefiningOp->user_end());
172 } else {
173 auto blockArg = cast<BlockArgument>(Val&: varPtr);
174 users.insert(I: blockArg.user_begin(), E: blockArg.user_end());
175 }
176 auto usesVarPtr = [&users](Operation *op) -> bool {
177 return users.count(V: op);
178 };
179
180 SmallVector<Operation *> chainOfOps;
181 chainOfOps.push_back(Elt: mapInfoOp);
182 for (auto member : mapInfoOp.getMembers()) {
183 omp::MapInfoOp memberMap =
184 cast<omp::MapInfoOp>(Val: member.getDefiningOp());
185 if (usesVarPtr(memberMap))
186 chainOfOps.push_back(Elt: memberMap);
187 if (memberMap.getVarPtrPtr()) {
188 Operation *defOp = memberMap.getVarPtrPtr().getDefiningOp();
189 if (defOp && usesVarPtr(defOp))
190 chainOfOps.push_back(Elt: defOp);
191 }
192 }
193
194 DominanceInfo dom;
195 llvm::sort(C&: chainOfOps, Comp: [&](Operation *l, Operation *r) {
196 if (l == r)
197 return false;
198 return dom.properlyDominates(a: l, b: r);
199 });
200
201 rewriter.setInsertionPoint(chainOfOps.front());
202
203 Operation *firstOp = chainOfOps.front();
204 Location loc = firstOp->getLoc();
205
206 // Create a llvm.func for 'region' that is marked always_inline and call
207 // it.
208 auto createAlwaysInlineFuncAndCallIt =
209 [&](Region &region, llvm::StringRef funcName,
210 llvm::ArrayRef<Value> args, bool returnsValue) -> Value {
211 assert(!region.empty() && "region cannot be empty");
212 LLVM::LLVMFuncOp func = createFuncOpForRegion(
213 loc, mod, srcRegion&: region, funcName, rewriter, returnsValue);
214 auto call = LLVM::CallOp::create(builder&: rewriter, location: loc, func, args);
215 return call.getResult();
216 };
217
218 Value moldArg, newArg;
219 if (isPrivatizedByValue) {
220 moldArg = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: varType, addr: varPtr);
221 newArg = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: varType, addr: heapMem);
222 } else {
223 moldArg = varPtr;
224 newArg = heapMem;
225 }
226
227 Value initializedVal;
228 if (!privatizer.getInitRegion().empty())
229 initializedVal = createAlwaysInlineFuncAndCallIt(
230 privatizer.getInitRegion(),
231 llvm::formatv(Fmt: "{0}_{1}", Vals: privatizer.getSymName(), Vals: "init").str(),
232 {moldArg, newArg}, /*returnsValue=*/true);
233 else
234 initializedVal = newArg;
235
236 if (isFirstPrivate && !privatizer.getCopyRegion().empty())
237 initializedVal = createAlwaysInlineFuncAndCallIt(
238 privatizer.getCopyRegion(),
239 llvm::formatv(Fmt: "{0}_{1}", Vals: privatizer.getSymName(), Vals: "copy").str(),
240 {moldArg, initializedVal}, /*returnsValue=*/true);
241
242 if (isPrivatizedByValue)
243 (void)LLVM::StoreOp::create(builder&: rewriter, location: loc, value: initializedVal, addr: heapMem);
244
245 // clone origOp, replace all uses of varPtr with heapMem and
246 // erase origOp.
247 auto cloneModifyAndErase = [&](Operation *origOp) -> Operation * {
248 Operation *clonedOp = rewriter.clone(op&: *origOp);
249 rewriter.replaceAllOpUsesWith(from: origOp, to: clonedOp);
250 rewriter.modifyOpInPlace(root: clonedOp, callable: [&]() {
251 clonedOp->replaceUsesOfWith(from: varPtr, to: heapMem);
252 });
253 rewriter.eraseOp(op: origOp);
254 return clonedOp;
255 };
256
257 // Now that we have set up the heap-allocated copy of the private
258 // variable, rewrite all the uses of the original variable with
259 // the heap-allocated variable.
260 rewriter.setInsertionPoint(targetOp);
261 mapInfoOp = cast<omp::MapInfoOp>(Val: cloneModifyAndErase(mapInfoOp));
262 rewriter.setInsertionPoint(mapInfoOp);
263
264 // Fix any members that may use varPtr to now use heapMem
265 for (auto member : mapInfoOp.getMembers()) {
266 auto memberMapInfoOp = cast<omp::MapInfoOp>(Val: member.getDefiningOp());
267 if (!usesVarPtr(memberMapInfoOp))
268 continue;
269 memberMapInfoOp =
270 cast<omp::MapInfoOp>(Val: cloneModifyAndErase(memberMapInfoOp));
271 rewriter.setInsertionPoint(memberMapInfoOp);
272
273 if (memberMapInfoOp.getVarPtrPtr()) {
274 Operation *varPtrPtrdefOp =
275 memberMapInfoOp.getVarPtrPtr().getDefiningOp();
276 rewriter.setInsertionPoint(cloneModifyAndErase(varPtrPtrdefOp));
277 }
278 }
279
280 // If the type of the private variable is not a pointer,
281 // which is typically the case with !fir.boxchar types, then
282 // we need to ensure that the new private variable is also
283 // not a pointer. Insert a load from heapMem right before
284 // targetOp.
285 if (isPrivatizedByValue) {
286 rewriter.setInsertionPoint(targetOp);
287 auto newPrivVar = LLVM::LoadOp::create(builder&: rewriter, location: mapInfoOp.getLoc(),
288 type: varType, addr: heapMem);
289 newPrivVars.push_back(Elt: newPrivVar);
290 }
291
292 // Deallocate
293 if (needsCleanupTask) {
294 if (!cleanupTaskOp) {
295 assert(fakeDependVar &&
296 "Need a valid value to set up a dependency");
297 rewriter.setInsertionPointAfter(targetOp);
298 omp::TaskOperands taskOperands;
299 auto inDepend = omp::ClauseTaskDependAttr::get(
300 context: rewriter.getContext(), value: omp::ClauseTaskDepend::taskdependin);
301 taskOperands.dependKinds.push_back(Elt: inDepend);
302 taskOperands.dependVars.push_back(Elt: fakeDependVar);
303 cleanupTaskOp = omp::TaskOp::create(builder&: rewriter, location: loc, clauses: taskOperands);
304 Block *taskBlock = rewriter.createBlock(parent: &cleanupTaskOp.getRegion());
305 rewriter.setInsertionPointToEnd(taskBlock);
306 omp::TerminatorOp::create(builder&: rewriter, location: cleanupTaskOp.getLoc());
307 }
308 rewriter.setInsertionPointToStart(
309 &*cleanupTaskOp.getRegion().getBlocks().begin());
310 (void)createAlwaysInlineFuncAndCallIt(
311 privatizer.getDeallocRegion(),
312 llvm::formatv(Fmt: "{0}_{1}", Vals: privatizer.getSymName(), Vals: "dealloc")
313 .str(),
314 {initializedVal}, /*returnsValue=*/false);
315 llvm::FailureOr<LLVM::LLVMFuncOp> freeFunc =
316 LLVM::lookupOrCreateFreeFn(b&: rewriter, moduleOp: mod);
317 assert(llvm::succeeded(freeFunc) &&
318 "Could not find free in the module");
319 (void)LLVM::CallOp::create(builder&: rewriter, location: loc, func: freeFunc.value(),
320 args: ValueRange{heapMem});
321 }
322 }
323 assert(newPrivVars.size() == privateVars.size() &&
324 "The number of private variables must match before and after "
325 "transformation");
326 if (fakeDependVar) {
327 omp::ClauseTaskDependAttr outDepend = omp::ClauseTaskDependAttr::get(
328 context: rewriter.getContext(), value: omp::ClauseTaskDepend::taskdependout);
329 SmallVector<Attribute> newDependKinds;
330 if (!targetOp.getDependVars().empty()) {
331 std::optional<ArrayAttr> dependKinds = targetOp.getDependKinds();
332 assert(dependKinds && "bad depend clause in omp::TargetOp");
333 llvm::copy(Range&: *dependKinds, Out: std::back_inserter(x&: newDependKinds));
334 }
335 newDependKinds.push_back(Elt: outDepend);
336 ArrayAttr newDependKindsAttr =
337 ArrayAttr::get(context: rewriter.getContext(), value: newDependKinds);
338 targetOp.getDependVarsMutable().append(values: fakeDependVar);
339 targetOp.setDependKindsAttr(newDependKindsAttr);
340 }
341 rewriter.setInsertionPoint(targetOp);
342 targetOp.getPrivateVarsMutable().clear();
343 targetOp.getPrivateVarsMutable().assign(values: newPrivVars);
344 });
345 }
346
347private:
348 bool hasPrivateVars(omp::TargetOp targetOp) const {
349 return !targetOp.getPrivateVars().empty();
350 }
351
352 bool isTargetTaskDeferred(omp::TargetOp targetOp) const {
353 return targetOp.getNowait();
354 }
355
356 template <typename OpTy>
357 omp::PrivateClauseOp findPrivatizer(OpTy op, Attribute privSym) const {
358 SymbolRefAttr privatizerName = llvm::cast<SymbolRefAttr>(Val&: privSym);
359 omp::PrivateClauseOp privatizer =
360 SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(
361 op, privatizerName);
362 return privatizer;
363 }
364
365 // Get the (compile-time constant) size of varType as per the
366 // given DataLayout dl.
367 std::int64_t getSizeInBytes(const DataLayout &dl, Type varType) const {
368 return dl.getTypeAllocSize(t: varType);
369 }
370
371 LLVM::LLVMFuncOp getMalloc(ModuleOp mod, IRRewriter &rewriter) const {
372 llvm::FailureOr<LLVM::LLVMFuncOp> mallocCall =
373 LLVM::lookupOrCreateMallocFn(b&: rewriter, moduleOp: mod, indexType: rewriter.getI64Type());
374 assert(llvm::succeeded(mallocCall) &&
375 "Could not find malloc in the module");
376 return mallocCall.value();
377 }
378
379 Value allocateHeapMem(omp::TargetOp targetOp, Value privVar, Type varType,
380 ModuleOp mod, IRRewriter &rewriter) const {
381 OpBuilder::InsertionGuard guard(rewriter);
382 Value varPtr = privVar;
383 Operation *definingOp = varPtr.getDefiningOp();
384 BlockArgument blockArg;
385 if (!definingOp) {
386 blockArg = mlir::dyn_cast<BlockArgument>(Val&: varPtr);
387 rewriter.setInsertionPointToStart(blockArg.getParentBlock());
388 } else {
389 rewriter.setInsertionPoint(definingOp);
390 }
391 Location loc = definingOp ? definingOp->getLoc() : blockArg.getLoc();
392 LLVM::LLVMFuncOp mallocFn = getMalloc(mod, rewriter);
393
394 assert(mod.getDataLayoutSpec() &&
395 "MLIR module with no datalayout spec not handled yet");
396
397 const DataLayout &dl = DataLayout(mod);
398 std::int64_t distance = getSizeInBytes(dl, varType);
399
400 Value sizeBytes = LLVM::ConstantOp::create(
401 builder&: rewriter, location: loc, type: mallocFn.getFunctionType().getParamType(i: 0), value: distance);
402
403 auto mallocCallOp =
404 LLVM::CallOp::create(builder&: rewriter, location: loc, func: mallocFn, args: ValueRange{sizeBytes});
405 return mallocCallOp.getResult();
406 }
407
408 // Create a function for srcRegion and attribute it to be always_inline.
409 // The big assumption here is that srcRegion is one of init, copy or dealloc
410 // regions of a omp::PrivateClauseop. Accordingly, the return type is assumed
411 // to either be the same as the types of the two arguments of the region (for
412 // init and copy regions) or void as would be the case for dealloc regions.
413 LLVM::LLVMFuncOp createFuncOpForRegion(Location loc, ModuleOp mod,
414 Region &srcRegion,
415 llvm::StringRef funcName,
416 IRRewriter &rewriter,
417 bool returnsValue = false) {
418
419 OpBuilder::InsertionGuard guard(rewriter);
420 rewriter.setInsertionPoint(block: mod.getBody(), insertPoint: mod.getBody()->end());
421 Region clonedRegion;
422 IRMapping mapper;
423 srcRegion.cloneInto(dest: &clonedRegion, mapper);
424
425 SmallVector<Type> paramTypes;
426 llvm::copy(Range: srcRegion.getArgumentTypes(), Out: std::back_inserter(x&: paramTypes));
427 Type resultType = returnsValue
428 ? srcRegion.getArgument(i: 0).getType()
429 : LLVM::LLVMVoidType::get(ctx: rewriter.getContext());
430 LLVM::LLVMFunctionType funcType =
431 LLVM::LLVMFunctionType::get(result: resultType, arguments: paramTypes);
432
433 LLVM::LLVMFuncOp func =
434 LLVM::LLVMFuncOp::create(builder&: rewriter, location: loc, name: funcName, type: funcType);
435 func.setAlwaysInline(true);
436 rewriter.inlineRegionBefore(region&: clonedRegion, parent&: func.getRegion(),
437 before: func.getRegion().end());
438 for (auto &block : func.getRegion().getBlocks()) {
439 if (isa<omp::YieldOp>(Val: block.getTerminator())) {
440 omp::YieldOp yieldOp = cast<omp::YieldOp>(Val: block.getTerminator());
441 rewriter.setInsertionPoint(yieldOp);
442 rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op: yieldOp, args: TypeRange(),
443 args: yieldOp.getOperands());
444 }
445 }
446 return func;
447 }
448};
449} // namespace
450