| 1 | //===- OpenMPOffloadPrivatizationPrepare.cpp - Prepare OMP privatization --===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Analysis/SliceAnalysis.h" |
| 10 | #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" |
| 11 | #include "mlir/Dialect/LLVMIR/LLVMCallOps.h" |
| 12 | #include "mlir/Dialect/LLVMIR/LLVMConstantOps.h" |
| 13 | #include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h" |
| 14 | #include "mlir/Dialect/LLVMIR/LLVMModuleOps.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMTerminatorOps.h" |
| 16 | #include "mlir/Dialect/OpenMP/OpenMPDialect.h" |
| 17 | #include "mlir/IR/Builders.h" |
| 18 | #include "mlir/IR/Dominance.h" |
| 19 | #include "mlir/IR/IRMapping.h" |
| 20 | #include "mlir/Pass/Pass.h" |
| 21 | #include "mlir/Support/LLVM.h" |
| 22 | #include "llvm/Support/DebugLog.h" |
| 23 | #include "llvm/Support/FormatVariadic.h" |
| 24 | #include <cstdint> |
| 25 | #include <iterator> |
| 26 | #include <utility> |
| 27 | |
| 28 | //===----------------------------------------------------------------------===// |
| 29 | // A pass that prepares OpenMP code for translation of delayed privatization |
| 30 | // in the context of deferred target tasks. Deferred target tasks are created |
| 31 | // when the nowait clause is used on the target directive. |
| 32 | //===----------------------------------------------------------------------===// |
| 33 | |
| 34 | #define DEBUG_TYPE "omp-prepare-for-offload-privatization" |
| 35 | |
| 36 | namespace mlir { |
| 37 | namespace omp { |
| 38 | |
| 39 | #define GEN_PASS_DEF_PREPAREFOROMPOFFLOADPRIVATIZATIONPASS |
| 40 | #include "mlir/Dialect/OpenMP/Transforms/Passes.h.inc" |
| 41 | |
| 42 | } // namespace omp |
| 43 | } // namespace mlir |
| 44 | |
| 45 | using namespace mlir; |
| 46 | namespace { |
| 47 | |
| 48 | //===----------------------------------------------------------------------===// |
| 49 | // PrepareForOMPOffloadPrivatizationPass |
| 50 | //===----------------------------------------------------------------------===// |
| 51 | |
| 52 | class PrepareForOMPOffloadPrivatizationPass |
| 53 | : public omp::impl::PrepareForOMPOffloadPrivatizationPassBase< |
| 54 | PrepareForOMPOffloadPrivatizationPass> { |
| 55 | |
| 56 | void runOnOperation() override { |
| 57 | ModuleOp mod = getOperation(); |
| 58 | |
| 59 | // In this pass, we make host-allocated privatized variables persist for |
| 60 | // deferred target tasks by copying them to the heap. Once the target task |
| 61 | // is done, this heap memory is freed. Since all of this happens on the host |
| 62 | // we can skip device modules. |
| 63 | auto offloadModuleInterface = |
| 64 | dyn_cast<omp::OffloadModuleInterface>(Val: mod.getOperation()); |
| 65 | if (offloadModuleInterface && offloadModuleInterface.getIsTargetDevice()) |
| 66 | return; |
| 67 | |
| 68 | getOperation()->walk(callback: [&](omp::TargetOp targetOp) { |
| 69 | if (!hasPrivateVars(targetOp) || !isTargetTaskDeferred(targetOp)) |
| 70 | return; |
| 71 | IRRewriter rewriter(&getContext()); |
| 72 | OperandRange privateVars = targetOp.getPrivateVars(); |
| 73 | SmallVector<mlir::Value> newPrivVars; |
| 74 | Value fakeDependVar; |
| 75 | omp::TaskOp cleanupTaskOp; |
| 76 | |
| 77 | newPrivVars.reserve(N: privateVars.size()); |
| 78 | std::optional<ArrayAttr> privateSyms = targetOp.getPrivateSyms(); |
| 79 | for (auto [privVarIdx, privVarSymPair] : |
| 80 | llvm::enumerate(First: llvm::zip_equal(t&: privateVars, u&: *privateSyms))) { |
| 81 | Value privVar = std::get<0>(t&: privVarSymPair); |
| 82 | Attribute privSym = std::get<1>(t&: privVarSymPair); |
| 83 | |
| 84 | omp::PrivateClauseOp privatizer = findPrivatizer(op: targetOp, privSym); |
| 85 | if (!privatizer.needsMap()) { |
| 86 | newPrivVars.push_back(Elt: privVar); |
| 87 | continue; |
| 88 | } |
| 89 | bool isFirstPrivate = privatizer.getDataSharingType() == |
| 90 | omp::DataSharingClauseType::FirstPrivate; |
| 91 | |
| 92 | Value mappedValue = targetOp.getMappedValueForPrivateVar(privVarIdx); |
| 93 | auto mapInfoOp = cast<omp::MapInfoOp>(Val: mappedValue.getDefiningOp()); |
| 94 | |
| 95 | if (mapInfoOp.getMapCaptureType() == omp::VariableCaptureKind::ByCopy) { |
| 96 | newPrivVars.push_back(Elt: privVar); |
| 97 | continue; |
| 98 | } |
| 99 | |
| 100 | // For deferred target tasks (!$omp target nowait), we need to keep |
| 101 | // a copy of the original, i.e. host variable being privatized so |
| 102 | // that it is available when the target task is eventually executed. |
| 103 | // We do this by first allocating as much heap memory as is needed by |
| 104 | // the original variable. Then, we use the init and copy regions of the |
| 105 | // privatizer, an instance of omp::PrivateClauseOp to set up the heap- |
| 106 | // allocated copy. |
| 107 | // After the target task is done, we need to use the dealloc region |
| 108 | // of the privatizer to clean up everything. We also need to free |
| 109 | // the heap memory we allocated. But due to the deferred nature |
| 110 | // of the target task, we cannot simply deallocate right after the |
| 111 | // omp.target operation else we may end up freeing memory before |
| 112 | // its eventual use by the target task. So, we create a dummy |
| 113 | // dependence between the target task and new omp.task. In the omp.task, |
| 114 | // we do all the cleanup. So, we end up with the following structure |
| 115 | // |
| 116 | // omp.target map_entries(..) ... nowait depend(out:fakeDependVar) { |
| 117 | // ... |
| 118 | // omp.terminator |
| 119 | // } |
| 120 | // omp.task depend(in: fakeDependVar) { |
| 121 | // /*cleanup_code*/ |
| 122 | // omp.terminator |
| 123 | // } |
| 124 | // fakeDependVar is the address of the first heap-allocated copy of the |
| 125 | // host variable being privatized. |
| 126 | |
| 127 | bool needsCleanupTask = !privatizer.getDeallocRegion().empty(); |
| 128 | |
| 129 | // Allocate heap memory that corresponds to the type of memory |
| 130 | // pointed to by varPtr |
| 131 | // For boxchars this won't be a pointer. But, MapsForPrivatizedSymbols |
| 132 | // should have mapped the pointer to the boxchar so use that as varPtr. |
| 133 | Value varPtr = mapInfoOp.getVarPtr(); |
| 134 | Type varType = mapInfoOp.getVarPtrType(); |
| 135 | bool isPrivatizedByValue = |
| 136 | !isa<LLVM::LLVMPointerType>(Val: privVar.getType()); |
| 137 | |
| 138 | assert(isa<LLVM::LLVMPointerType>(varPtr.getType())); |
| 139 | Value heapMem = |
| 140 | allocateHeapMem(targetOp, privVar: varPtr, varType, mod, rewriter); |
| 141 | if (!heapMem) |
| 142 | targetOp.emitError( |
| 143 | message: "Unable to allocate heap memory when trying to move " |
| 144 | "a private variable out of the stack and into the " |
| 145 | "heap for use by a deferred target task" ); |
| 146 | |
| 147 | if (needsCleanupTask && !fakeDependVar) |
| 148 | fakeDependVar = heapMem; |
| 149 | |
| 150 | // The types of private vars should match before and after the |
| 151 | // transformation. In particular, if the type is a pointer, |
| 152 | // simply record the newly allocated malloc location as the |
| 153 | // new private variable. If, however, the type is not a pointer |
| 154 | // then, we need to load the value from the newly allocated |
| 155 | // location. We'll insert that load later after we have updated |
| 156 | // the malloc'd location with the contents of the original |
| 157 | // variable. |
| 158 | if (!isPrivatizedByValue) |
| 159 | newPrivVars.push_back(Elt: heapMem); |
| 160 | |
| 161 | // We now need to copy the original private variable into the newly |
| 162 | // allocated location in the heap. |
| 163 | // Find the earliest insertion point for the copy. This will be before |
| 164 | // the first in the list of omp::MapInfoOp instances that use varPtr. |
| 165 | // After the copy these omp::MapInfoOp instances will refer to heapMem |
| 166 | // instead. |
| 167 | Operation *varPtrDefiningOp = varPtr.getDefiningOp(); |
| 168 | DenseSet<Operation *> users; |
| 169 | if (varPtrDefiningOp) { |
| 170 | users.insert(I: varPtrDefiningOp->user_begin(), |
| 171 | E: varPtrDefiningOp->user_end()); |
| 172 | } else { |
| 173 | auto blockArg = cast<BlockArgument>(Val&: varPtr); |
| 174 | users.insert(I: blockArg.user_begin(), E: blockArg.user_end()); |
| 175 | } |
| 176 | auto usesVarPtr = [&users](Operation *op) -> bool { |
| 177 | return users.count(V: op); |
| 178 | }; |
| 179 | |
| 180 | SmallVector<Operation *> chainOfOps; |
| 181 | chainOfOps.push_back(Elt: mapInfoOp); |
| 182 | for (auto member : mapInfoOp.getMembers()) { |
| 183 | omp::MapInfoOp memberMap = |
| 184 | cast<omp::MapInfoOp>(Val: member.getDefiningOp()); |
| 185 | if (usesVarPtr(memberMap)) |
| 186 | chainOfOps.push_back(Elt: memberMap); |
| 187 | if (memberMap.getVarPtrPtr()) { |
| 188 | Operation *defOp = memberMap.getVarPtrPtr().getDefiningOp(); |
| 189 | if (defOp && usesVarPtr(defOp)) |
| 190 | chainOfOps.push_back(Elt: defOp); |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | DominanceInfo dom; |
| 195 | llvm::sort(C&: chainOfOps, Comp: [&](Operation *l, Operation *r) { |
| 196 | if (l == r) |
| 197 | return false; |
| 198 | return dom.properlyDominates(a: l, b: r); |
| 199 | }); |
| 200 | |
| 201 | rewriter.setInsertionPoint(chainOfOps.front()); |
| 202 | |
| 203 | Operation *firstOp = chainOfOps.front(); |
| 204 | Location loc = firstOp->getLoc(); |
| 205 | |
| 206 | // Create a llvm.func for 'region' that is marked always_inline and call |
| 207 | // it. |
| 208 | auto createAlwaysInlineFuncAndCallIt = |
| 209 | [&](Region ®ion, llvm::StringRef funcName, |
| 210 | llvm::ArrayRef<Value> args, bool returnsValue) -> Value { |
| 211 | assert(!region.empty() && "region cannot be empty" ); |
| 212 | LLVM::LLVMFuncOp func = createFuncOpForRegion( |
| 213 | loc, mod, srcRegion&: region, funcName, rewriter, returnsValue); |
| 214 | auto call = LLVM::CallOp::create(builder&: rewriter, location: loc, func, args); |
| 215 | return call.getResult(); |
| 216 | }; |
| 217 | |
| 218 | Value moldArg, newArg; |
| 219 | if (isPrivatizedByValue) { |
| 220 | moldArg = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: varType, addr: varPtr); |
| 221 | newArg = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: varType, addr: heapMem); |
| 222 | } else { |
| 223 | moldArg = varPtr; |
| 224 | newArg = heapMem; |
| 225 | } |
| 226 | |
| 227 | Value initializedVal; |
| 228 | if (!privatizer.getInitRegion().empty()) |
| 229 | initializedVal = createAlwaysInlineFuncAndCallIt( |
| 230 | privatizer.getInitRegion(), |
| 231 | llvm::formatv(Fmt: "{0}_{1}" , Vals: privatizer.getSymName(), Vals: "init" ).str(), |
| 232 | {moldArg, newArg}, /*returnsValue=*/true); |
| 233 | else |
| 234 | initializedVal = newArg; |
| 235 | |
| 236 | if (isFirstPrivate && !privatizer.getCopyRegion().empty()) |
| 237 | initializedVal = createAlwaysInlineFuncAndCallIt( |
| 238 | privatizer.getCopyRegion(), |
| 239 | llvm::formatv(Fmt: "{0}_{1}" , Vals: privatizer.getSymName(), Vals: "copy" ).str(), |
| 240 | {moldArg, initializedVal}, /*returnsValue=*/true); |
| 241 | |
| 242 | if (isPrivatizedByValue) |
| 243 | (void)LLVM::StoreOp::create(builder&: rewriter, location: loc, value: initializedVal, addr: heapMem); |
| 244 | |
| 245 | // clone origOp, replace all uses of varPtr with heapMem and |
| 246 | // erase origOp. |
| 247 | auto cloneModifyAndErase = [&](Operation *origOp) -> Operation * { |
| 248 | Operation *clonedOp = rewriter.clone(op&: *origOp); |
| 249 | rewriter.replaceAllOpUsesWith(from: origOp, to: clonedOp); |
| 250 | rewriter.modifyOpInPlace(root: clonedOp, callable: [&]() { |
| 251 | clonedOp->replaceUsesOfWith(from: varPtr, to: heapMem); |
| 252 | }); |
| 253 | rewriter.eraseOp(op: origOp); |
| 254 | return clonedOp; |
| 255 | }; |
| 256 | |
| 257 | // Now that we have set up the heap-allocated copy of the private |
| 258 | // variable, rewrite all the uses of the original variable with |
| 259 | // the heap-allocated variable. |
| 260 | rewriter.setInsertionPoint(targetOp); |
| 261 | mapInfoOp = cast<omp::MapInfoOp>(Val: cloneModifyAndErase(mapInfoOp)); |
| 262 | rewriter.setInsertionPoint(mapInfoOp); |
| 263 | |
| 264 | // Fix any members that may use varPtr to now use heapMem |
| 265 | for (auto member : mapInfoOp.getMembers()) { |
| 266 | auto memberMapInfoOp = cast<omp::MapInfoOp>(Val: member.getDefiningOp()); |
| 267 | if (!usesVarPtr(memberMapInfoOp)) |
| 268 | continue; |
| 269 | memberMapInfoOp = |
| 270 | cast<omp::MapInfoOp>(Val: cloneModifyAndErase(memberMapInfoOp)); |
| 271 | rewriter.setInsertionPoint(memberMapInfoOp); |
| 272 | |
| 273 | if (memberMapInfoOp.getVarPtrPtr()) { |
| 274 | Operation *varPtrPtrdefOp = |
| 275 | memberMapInfoOp.getVarPtrPtr().getDefiningOp(); |
| 276 | rewriter.setInsertionPoint(cloneModifyAndErase(varPtrPtrdefOp)); |
| 277 | } |
| 278 | } |
| 279 | |
| 280 | // If the type of the private variable is not a pointer, |
| 281 | // which is typically the case with !fir.boxchar types, then |
| 282 | // we need to ensure that the new private variable is also |
| 283 | // not a pointer. Insert a load from heapMem right before |
| 284 | // targetOp. |
| 285 | if (isPrivatizedByValue) { |
| 286 | rewriter.setInsertionPoint(targetOp); |
| 287 | auto newPrivVar = LLVM::LoadOp::create(builder&: rewriter, location: mapInfoOp.getLoc(), |
| 288 | type: varType, addr: heapMem); |
| 289 | newPrivVars.push_back(Elt: newPrivVar); |
| 290 | } |
| 291 | |
| 292 | // Deallocate |
| 293 | if (needsCleanupTask) { |
| 294 | if (!cleanupTaskOp) { |
| 295 | assert(fakeDependVar && |
| 296 | "Need a valid value to set up a dependency" ); |
| 297 | rewriter.setInsertionPointAfter(targetOp); |
| 298 | omp::TaskOperands taskOperands; |
| 299 | auto inDepend = omp::ClauseTaskDependAttr::get( |
| 300 | context: rewriter.getContext(), value: omp::ClauseTaskDepend::taskdependin); |
| 301 | taskOperands.dependKinds.push_back(Elt: inDepend); |
| 302 | taskOperands.dependVars.push_back(Elt: fakeDependVar); |
| 303 | cleanupTaskOp = omp::TaskOp::create(builder&: rewriter, location: loc, clauses: taskOperands); |
| 304 | Block *taskBlock = rewriter.createBlock(parent: &cleanupTaskOp.getRegion()); |
| 305 | rewriter.setInsertionPointToEnd(taskBlock); |
| 306 | omp::TerminatorOp::create(builder&: rewriter, location: cleanupTaskOp.getLoc()); |
| 307 | } |
| 308 | rewriter.setInsertionPointToStart( |
| 309 | &*cleanupTaskOp.getRegion().getBlocks().begin()); |
| 310 | (void)createAlwaysInlineFuncAndCallIt( |
| 311 | privatizer.getDeallocRegion(), |
| 312 | llvm::formatv(Fmt: "{0}_{1}" , Vals: privatizer.getSymName(), Vals: "dealloc" ) |
| 313 | .str(), |
| 314 | {initializedVal}, /*returnsValue=*/false); |
| 315 | llvm::FailureOr<LLVM::LLVMFuncOp> freeFunc = |
| 316 | LLVM::lookupOrCreateFreeFn(b&: rewriter, moduleOp: mod); |
| 317 | assert(llvm::succeeded(freeFunc) && |
| 318 | "Could not find free in the module" ); |
| 319 | (void)LLVM::CallOp::create(builder&: rewriter, location: loc, func: freeFunc.value(), |
| 320 | args: ValueRange{heapMem}); |
| 321 | } |
| 322 | } |
| 323 | assert(newPrivVars.size() == privateVars.size() && |
| 324 | "The number of private variables must match before and after " |
| 325 | "transformation" ); |
| 326 | if (fakeDependVar) { |
| 327 | omp::ClauseTaskDependAttr outDepend = omp::ClauseTaskDependAttr::get( |
| 328 | context: rewriter.getContext(), value: omp::ClauseTaskDepend::taskdependout); |
| 329 | SmallVector<Attribute> newDependKinds; |
| 330 | if (!targetOp.getDependVars().empty()) { |
| 331 | std::optional<ArrayAttr> dependKinds = targetOp.getDependKinds(); |
| 332 | assert(dependKinds && "bad depend clause in omp::TargetOp" ); |
| 333 | llvm::copy(Range&: *dependKinds, Out: std::back_inserter(x&: newDependKinds)); |
| 334 | } |
| 335 | newDependKinds.push_back(Elt: outDepend); |
| 336 | ArrayAttr newDependKindsAttr = |
| 337 | ArrayAttr::get(context: rewriter.getContext(), value: newDependKinds); |
| 338 | targetOp.getDependVarsMutable().append(values: fakeDependVar); |
| 339 | targetOp.setDependKindsAttr(newDependKindsAttr); |
| 340 | } |
| 341 | rewriter.setInsertionPoint(targetOp); |
| 342 | targetOp.getPrivateVarsMutable().clear(); |
| 343 | targetOp.getPrivateVarsMutable().assign(values: newPrivVars); |
| 344 | }); |
| 345 | } |
| 346 | |
| 347 | private: |
| 348 | bool hasPrivateVars(omp::TargetOp targetOp) const { |
| 349 | return !targetOp.getPrivateVars().empty(); |
| 350 | } |
| 351 | |
| 352 | bool isTargetTaskDeferred(omp::TargetOp targetOp) const { |
| 353 | return targetOp.getNowait(); |
| 354 | } |
| 355 | |
| 356 | template <typename OpTy> |
| 357 | omp::PrivateClauseOp findPrivatizer(OpTy op, Attribute privSym) const { |
| 358 | SymbolRefAttr privatizerName = llvm::cast<SymbolRefAttr>(Val&: privSym); |
| 359 | omp::PrivateClauseOp privatizer = |
| 360 | SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>( |
| 361 | op, privatizerName); |
| 362 | return privatizer; |
| 363 | } |
| 364 | |
| 365 | // Get the (compile-time constant) size of varType as per the |
| 366 | // given DataLayout dl. |
| 367 | std::int64_t getSizeInBytes(const DataLayout &dl, Type varType) const { |
| 368 | return dl.getTypeAllocSize(t: varType); |
| 369 | } |
| 370 | |
| 371 | LLVM::LLVMFuncOp getMalloc(ModuleOp mod, IRRewriter &rewriter) const { |
| 372 | llvm::FailureOr<LLVM::LLVMFuncOp> mallocCall = |
| 373 | LLVM::lookupOrCreateMallocFn(b&: rewriter, moduleOp: mod, indexType: rewriter.getI64Type()); |
| 374 | assert(llvm::succeeded(mallocCall) && |
| 375 | "Could not find malloc in the module" ); |
| 376 | return mallocCall.value(); |
| 377 | } |
| 378 | |
| 379 | Value allocateHeapMem(omp::TargetOp targetOp, Value privVar, Type varType, |
| 380 | ModuleOp mod, IRRewriter &rewriter) const { |
| 381 | OpBuilder::InsertionGuard guard(rewriter); |
| 382 | Value varPtr = privVar; |
| 383 | Operation *definingOp = varPtr.getDefiningOp(); |
| 384 | BlockArgument blockArg; |
| 385 | if (!definingOp) { |
| 386 | blockArg = mlir::dyn_cast<BlockArgument>(Val&: varPtr); |
| 387 | rewriter.setInsertionPointToStart(blockArg.getParentBlock()); |
| 388 | } else { |
| 389 | rewriter.setInsertionPoint(definingOp); |
| 390 | } |
| 391 | Location loc = definingOp ? definingOp->getLoc() : blockArg.getLoc(); |
| 392 | LLVM::LLVMFuncOp mallocFn = getMalloc(mod, rewriter); |
| 393 | |
| 394 | assert(mod.getDataLayoutSpec() && |
| 395 | "MLIR module with no datalayout spec not handled yet" ); |
| 396 | |
| 397 | const DataLayout &dl = DataLayout(mod); |
| 398 | std::int64_t distance = getSizeInBytes(dl, varType); |
| 399 | |
| 400 | Value sizeBytes = LLVM::ConstantOp::create( |
| 401 | builder&: rewriter, location: loc, type: mallocFn.getFunctionType().getParamType(i: 0), value: distance); |
| 402 | |
| 403 | auto mallocCallOp = |
| 404 | LLVM::CallOp::create(builder&: rewriter, location: loc, func: mallocFn, args: ValueRange{sizeBytes}); |
| 405 | return mallocCallOp.getResult(); |
| 406 | } |
| 407 | |
| 408 | // Create a function for srcRegion and attribute it to be always_inline. |
| 409 | // The big assumption here is that srcRegion is one of init, copy or dealloc |
| 410 | // regions of a omp::PrivateClauseop. Accordingly, the return type is assumed |
| 411 | // to either be the same as the types of the two arguments of the region (for |
| 412 | // init and copy regions) or void as would be the case for dealloc regions. |
| 413 | LLVM::LLVMFuncOp createFuncOpForRegion(Location loc, ModuleOp mod, |
| 414 | Region &srcRegion, |
| 415 | llvm::StringRef funcName, |
| 416 | IRRewriter &rewriter, |
| 417 | bool returnsValue = false) { |
| 418 | |
| 419 | OpBuilder::InsertionGuard guard(rewriter); |
| 420 | rewriter.setInsertionPoint(block: mod.getBody(), insertPoint: mod.getBody()->end()); |
| 421 | Region clonedRegion; |
| 422 | IRMapping mapper; |
| 423 | srcRegion.cloneInto(dest: &clonedRegion, mapper); |
| 424 | |
| 425 | SmallVector<Type> paramTypes; |
| 426 | llvm::copy(Range: srcRegion.getArgumentTypes(), Out: std::back_inserter(x&: paramTypes)); |
| 427 | Type resultType = returnsValue |
| 428 | ? srcRegion.getArgument(i: 0).getType() |
| 429 | : LLVM::LLVMVoidType::get(ctx: rewriter.getContext()); |
| 430 | LLVM::LLVMFunctionType funcType = |
| 431 | LLVM::LLVMFunctionType::get(result: resultType, arguments: paramTypes); |
| 432 | |
| 433 | LLVM::LLVMFuncOp func = |
| 434 | LLVM::LLVMFuncOp::create(builder&: rewriter, location: loc, name: funcName, type: funcType); |
| 435 | func.setAlwaysInline(true); |
| 436 | rewriter.inlineRegionBefore(region&: clonedRegion, parent&: func.getRegion(), |
| 437 | before: func.getRegion().end()); |
| 438 | for (auto &block : func.getRegion().getBlocks()) { |
| 439 | if (isa<omp::YieldOp>(Val: block.getTerminator())) { |
| 440 | omp::YieldOp yieldOp = cast<omp::YieldOp>(Val: block.getTerminator()); |
| 441 | rewriter.setInsertionPoint(yieldOp); |
| 442 | rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op: yieldOp, args: TypeRange(), |
| 443 | args: yieldOp.getOperands()); |
| 444 | } |
| 445 | } |
| 446 | return func; |
| 447 | } |
| 448 | }; |
| 449 | } // namespace |
| 450 | |