| 1 | //===- OpenMPToLLVMIRTranslation.cpp - Translate OpenMP dialect to LLVM IR-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements a translation between the MLIR OpenMP dialect and LLVM |
| 10 | // IR. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | #include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h" |
| 14 | #include "mlir/Analysis/TopologicalSortUtils.h" |
| 15 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 16 | #include "mlir/Dialect/LLVMIR/LLVMDialect.h" |
| 17 | #include "mlir/Dialect/LLVMIR/LLVMTypes.h" |
| 18 | #include "mlir/Dialect/OpenMP/OpenMPDialect.h" |
| 19 | #include "mlir/Dialect/OpenMP/OpenMPInterfaces.h" |
| 20 | #include "mlir/Dialect/OpenMP/Utils/Utils.h" |
| 21 | #include "mlir/IR/Operation.h" |
| 22 | #include "mlir/Interfaces/SideEffectInterfaces.h" |
| 23 | #include "mlir/Support/LLVM.h" |
| 24 | #include "mlir/Target/LLVMIR/Dialect/OpenMPCommon.h" |
| 25 | #include "mlir/Target/LLVMIR/ModuleTranslation.h" |
| 26 | |
| 27 | #include "llvm/ADT/ArrayRef.h" |
| 28 | #include "llvm/ADT/SmallVector.h" |
| 29 | #include "llvm/ADT/TypeSwitch.h" |
| 30 | #include "llvm/Frontend/OpenMP/OMPConstants.h" |
| 31 | #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" |
| 32 | #include "llvm/IR/Constants.h" |
| 33 | #include "llvm/IR/DebugInfoMetadata.h" |
| 34 | #include "llvm/IR/DerivedTypes.h" |
| 35 | #include "llvm/IR/IRBuilder.h" |
| 36 | #include "llvm/IR/MDBuilder.h" |
| 37 | #include "llvm/IR/ReplaceConstant.h" |
| 38 | #include "llvm/Support/AMDGPUAddrSpace.h" |
| 39 | #include "llvm/Support/CommandLine.h" |
| 40 | #include "llvm/Support/FileSystem.h" |
| 41 | #include "llvm/Support/MathExtras.h" |
| 42 | #include "llvm/Support/NVPTXAddrSpace.h" |
| 43 | #include "llvm/Support/SaveAndRestore.h" |
| 44 | #include "llvm/Support/VirtualFileSystem.h" |
| 45 | #include "llvm/TargetParser/Triple.h" |
| 46 | #include "llvm/Transforms/Utils/ModuleUtils.h" |
| 47 | |
| 48 | #include <cstdint> |
| 49 | #include <iterator> |
| 50 | #include <numeric> |
| 51 | #include <optional> |
| 52 | #include <utility> |
| 53 | |
| 54 | using namespace mlir; |
| 55 | |
| 56 | static llvm::Expected<llvm::Value *> |
| 57 | getPrivateDataAddress(llvm::IRBuilderBase &builder, |
| 58 | LLVM::ModuleTranslation &moduleTranslation, |
| 59 | omp::PrivateClauseOp privatizer, llvm::Value *var); |
| 60 | |
| 61 | namespace { |
| 62 | /// Stores the option value so the OpenMP-to-LLVM IR translation can read it |
| 63 | /// outside registerOpenMPTranslationCLOptions(). The local cl::opt avoids a |
| 64 | /// global constructor in MLIR and writes to this variable through cl::location. |
| 65 | /// Without registration, false keeps Virtual enabled for eligible loops. |
| 66 | static bool disableVirtualCollapse = false; |
| 67 | |
| 68 | static llvm::omp::ScheduleKind |
| 69 | convertToScheduleKind(std::optional<omp::ClauseScheduleKind> schedKind) { |
| 70 | if (!schedKind.has_value()) |
| 71 | return llvm::omp::OMP_SCHEDULE_Default; |
| 72 | switch (schedKind.value()) { |
| 73 | case omp::ClauseScheduleKind::Static: |
| 74 | return llvm::omp::OMP_SCHEDULE_Static; |
| 75 | case omp::ClauseScheduleKind::Dynamic: |
| 76 | return llvm::omp::OMP_SCHEDULE_Dynamic; |
| 77 | case omp::ClauseScheduleKind::Guided: |
| 78 | return llvm::omp::OMP_SCHEDULE_Guided; |
| 79 | case omp::ClauseScheduleKind::Auto: |
| 80 | return llvm::omp::OMP_SCHEDULE_Auto; |
| 81 | case omp::ClauseScheduleKind::Runtime: |
| 82 | return llvm::omp::OMP_SCHEDULE_Runtime; |
| 83 | case omp::ClauseScheduleKind::Distribute: |
| 84 | return llvm::omp::OMP_SCHEDULE_Distribute; |
| 85 | } |
| 86 | llvm_unreachable("unhandled schedule clause argument" ); |
| 87 | } |
| 88 | |
| 89 | /// ModuleTranslation stack frame for OpenMP operations. This keeps track of the |
| 90 | /// blocks in which allocas are emitted. |
| 91 | class OpenMPAllocStackFrame |
| 92 | : public StateStackFrameBase<OpenMPAllocStackFrame> { |
| 93 | public: |
| 94 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPAllocStackFrame) |
| 95 | |
| 96 | explicit OpenMPAllocStackFrame( |
| 97 | llvm::BasicBlock *allocBlock, |
| 98 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) |
| 99 | : allocBlock(allocBlock), deallocBlocks(deallocBlocks) {} |
| 100 | llvm::BasicBlock *allocBlock; |
| 101 | llvm::SmallVector<llvm::BasicBlock *> deallocBlocks; |
| 102 | }; |
| 103 | |
| 104 | /// Stack frame to hold a \see llvm::CanonicalLoopInfo representing the |
| 105 | /// collapsed canonical loop information corresponding to an \c omp.loop_nest |
| 106 | /// operation. |
| 107 | class OpenMPLoopInfoStackFrame |
| 108 | : public StateStackFrameBase<OpenMPLoopInfoStackFrame> { |
| 109 | public: |
| 110 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPLoopInfoStackFrame) |
| 111 | llvm::CanonicalLoopInfo *loopInfo = nullptr; |
| 112 | }; |
| 113 | |
| 114 | /// OpenMP state that lives as long as the module translation: the |
| 115 | /// OpenMPIRBuilder associated with the LLVM module being constructed and the |
| 116 | /// not yet consumed canonical loop handles. |
| 117 | class OpenMPTranslationState : public LLVM::ModuleTranslation::Extension { |
| 118 | public: |
| 119 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPTranslationState) |
| 120 | |
| 121 | explicit OpenMPTranslationState(llvm::Module &module) : ompBuilder(module) { |
| 122 | // Flags represented as top-level OpenMP dialect attributes are set in |
| 123 | // `OpenMPDialectLLVMIRTranslationInterface::amendOperation()`. Here we set |
| 124 | // the default configuration. |
| 125 | llvm::OpenMPIRBuilderConfig config( |
| 126 | /* IsTargetDevice = */ false, /* IsGPU = */ false, |
| 127 | /* OpenMPOffloadMandatory = */ false, |
| 128 | /* HasRequiresReverseOffload = */ false, |
| 129 | /* HasRequiresUnifiedAddress = */ false, |
| 130 | /* HasRequiresUnifiedSharedMemory = */ false, |
| 131 | /* HasRequiresDynamicAllocators = */ false); |
| 132 | config.setDefaultTargetAS(module.getDataLayout().getProgramAddressSpace()); |
| 133 | config.setRuntimeCC(module.getTargetTriple().isSPIRV() |
| 134 | ? llvm::CallingConv::SPIR_FUNC |
| 135 | : llvm::CallingConv::C); |
| 136 | ompBuilder.setConfig(std::move(config)); |
| 137 | ompBuilder.initialize(); |
| 138 | } |
| 139 | |
| 140 | ~OpenMPTranslationState() override { |
| 141 | if (!ompBuilder.isFinalized()) |
| 142 | ompBuilder.finalize(); |
| 143 | } |
| 144 | |
| 145 | LogicalResult finalize() override { |
| 146 | ompBuilder.finalize(); |
| 147 | return success(); |
| 148 | } |
| 149 | |
| 150 | /// Builder for LLVM IR generation of OpenMP constructs. |
| 151 | llvm::OpenMPIRBuilder ompBuilder; |
| 152 | |
| 153 | /// List of not yet consumed MLIR loop handles (represented by an omp.new_cli |
| 154 | /// operation which creates a value of type CanonicalLoopInfoType) and their |
| 155 | /// LLVM-IR representation as CanonicalLoopInfo which is managed by the |
| 156 | /// OpenMPIRBuilder. |
| 157 | DenseMap<omp::NewCliOp, llvm::CanonicalLoopInfo *> loopMapping; |
| 158 | }; |
| 159 | } // namespace |
| 160 | |
| 161 | static OpenMPTranslationState & |
| 162 | getOpenMPState(LLVM::ModuleTranslation &moduleTranslation) { |
| 163 | return moduleTranslation.getOrCreateExtension<OpenMPTranslationState>( |
| 164 | args&: *moduleTranslation.getLLVMModule()); |
| 165 | } |
| 166 | |
| 167 | /// Returns the OpenMP IR builder associated with the LLVM IR module being |
| 168 | /// constructed. |
| 169 | static llvm::OpenMPIRBuilder * |
| 170 | getOpenMPBuilder(LLVM::ModuleTranslation &moduleTranslation) { |
| 171 | return &getOpenMPState(moduleTranslation).ompBuilder; |
| 172 | } |
| 173 | |
| 174 | /// Find the LLVM-IR loop that represents an MLIR loop. |
| 175 | static llvm::CanonicalLoopInfo * |
| 176 | lookupOMPLoop(LLVM::ModuleTranslation &moduleTranslation, Value mlir) { |
| 177 | auto cli = mlir.getDefiningOp<omp::NewCliOp>(); |
| 178 | llvm::CanonicalLoopInfo *result = |
| 179 | getOpenMPState(moduleTranslation).loopMapping.lookup(Val: cli); |
| 180 | assert(result && "attempt to get non-existing loop" ); |
| 181 | return result; |
| 182 | } |
| 183 | |
| 184 | /// Mark an OpenMP loop as having been consumed. |
| 185 | static void invalidateOmpLoop(LLVM::ModuleTranslation &moduleTranslation, |
| 186 | Value mlir) { |
| 187 | auto cli = mlir.getDefiningOp<omp::NewCliOp>(); |
| 188 | getOpenMPState(moduleTranslation).loopMapping.erase(Val: cli); |
| 189 | } |
| 190 | |
| 191 | /// Map an MLIR OpenMP dialect CanonicalLoopInfo to its lowered LLVM-IR |
| 192 | /// OpenMPIRBuilder CanonicalLoopInfo. |
| 193 | static void mapOmpLoop(LLVM::ModuleTranslation &moduleTranslation, Value mlir, |
| 194 | llvm::CanonicalLoopInfo *llvm) { |
| 195 | assert(llvm && "argument must be non-null" ); |
| 196 | auto cli = mlir.getDefiningOp<omp::NewCliOp>(); |
| 197 | llvm::CanonicalLoopInfo *&cur = |
| 198 | getOpenMPState(moduleTranslation).loopMapping[cli]; |
| 199 | assert(cur == nullptr && "attempting to map a loop that is already mapped" ); |
| 200 | cur = llvm; |
| 201 | } |
| 202 | |
| 203 | namespace { |
| 204 | /// Custom error class to signal translation errors that don't need reporting, |
| 205 | /// since encountering them will have already triggered relevant error messages. |
| 206 | /// |
| 207 | /// Its purpose is to serve as the glue between MLIR failures represented as |
| 208 | /// \see LogicalResult instances and \see llvm::Error instances used to |
| 209 | /// propagate errors through the \see llvm::OpenMPIRBuilder. Generally, when an |
| 210 | /// error of the first type is raised, a message is emitted directly (the \see |
| 211 | /// LogicalResult itself does not hold any information). If we need to forward |
| 212 | /// this error condition as an \see llvm::Error while avoiding triggering some |
| 213 | /// redundant error reporting later on, we need a custom \see llvm::ErrorInfo |
| 214 | /// class to just signal this situation has happened. |
| 215 | /// |
| 216 | /// For example, this class should be used to trigger errors from within |
| 217 | /// callbacks passed to the \see OpenMPIRBuilder when they were triggered by the |
| 218 | /// translation of their own regions. This unclutters the error log from |
| 219 | /// redundant messages. |
| 220 | class PreviouslyReportedError |
| 221 | : public llvm::ErrorInfo<PreviouslyReportedError> { |
| 222 | public: |
| 223 | void log(raw_ostream &) const override { |
| 224 | // Do not log anything. |
| 225 | } |
| 226 | |
| 227 | std::error_code convertToErrorCode() const override { |
| 228 | llvm_unreachable( |
| 229 | "PreviouslyReportedError doesn't support ECError conversion" ); |
| 230 | } |
| 231 | |
| 232 | // Used by ErrorInfo::classID. |
| 233 | static char ID; |
| 234 | }; |
| 235 | |
| 236 | char PreviouslyReportedError::ID = 0; |
| 237 | |
| 238 | /* |
| 239 | * Custom class for processing linear clause for omp.wsloop |
| 240 | * and omp.simd. Linear clause translation requires setup, |
| 241 | * initialization, update, and finalization at varying |
| 242 | * basic blocks in the IR. This class helps maintain |
| 243 | * internal state to allow consistent translation in |
| 244 | * each of these stages. |
| 245 | */ |
| 246 | |
| 247 | class LinearClauseProcessor { |
| 248 | |
| 249 | private: |
| 250 | SmallVector<llvm::Type *> linearVarTypes; |
| 251 | SmallVector<llvm::Value *> linearSteps; |
| 252 | // The `linear` privatizer, original and private copy of each linear var. |
| 253 | SmallVector<omp::PrivateClauseOp> linearPrivatizers; |
| 254 | SmallVector<llvm::Value *> linearOrigVars; |
| 255 | SmallVector<llvm::Value *> linearPrivateVars; |
| 256 | // The value of each linear var on entry to the loop. |
| 257 | SmallVector<llvm::Value *> linearStartVals; |
| 258 | llvm::BasicBlock *linearFinalizationBB; |
| 259 | llvm::BasicBlock *linearExitBB; |
| 260 | llvm::BasicBlock *linearLastIterExitBB; |
| 261 | std::optional<size_t> linearLoopIVIndex; |
| 262 | |
| 263 | // Address of the value of linear var `index` held by `var`. |
| 264 | llvm::Expected<llvm::Value *> |
| 265 | getAddress(llvm::IRBuilderBase &builder, |
| 266 | LLVM::ModuleTranslation &moduleTranslation, size_t index, |
| 267 | llvm::Value *var) { |
| 268 | return getPrivateDataAddress(builder, moduleTranslation, |
| 269 | privatizer: linearPrivatizers[index], var); |
| 270 | } |
| 271 | |
| 272 | // Copy the current value of each linear var back to the original. |
| 273 | llvm::Error storeLinearVars(llvm::IRBuilderBase &builder, |
| 274 | LLVM::ModuleTranslation &moduleTranslation) { |
| 275 | for (size_t index = 0; index < linearOrigVars.size(); index++) { |
| 276 | llvm::Expected<llvm::Value *> from = getAddress( |
| 277 | builder, moduleTranslation, index, var: linearPrivateVars[index]); |
| 278 | if (!from) |
| 279 | return from.takeError(); |
| 280 | llvm::LoadInst *linearVarTemp = |
| 281 | builder.CreateLoad(Ty: linearVarTypes[index], Ptr: *from); |
| 282 | llvm::Expected<llvm::Value *> to = |
| 283 | getAddress(builder, moduleTranslation, index, var: linearOrigVars[index]); |
| 284 | if (!to) |
| 285 | return to.takeError(); |
| 286 | builder.CreateStore(Val: linearVarTemp, Ptr: *to); |
| 287 | } |
| 288 | return llvm::Error::success(); |
| 289 | } |
| 290 | |
| 291 | public: |
| 292 | // Register type for the linear variables |
| 293 | void registerType(LLVM::ModuleTranslation &moduleTranslation, |
| 294 | mlir::Attribute &ty) { |
| 295 | linearVarTypes.push_back(Elt: moduleTranslation.convertType( |
| 296 | type: mlir::cast<mlir::TypeAttr>(Val&: ty).getValue())); |
| 297 | } |
| 298 | |
| 299 | // Register a linear var, allocating space for its start value at `allocaIP` |
| 300 | void createLinearVar(llvm::IRBuilderBase &builder, |
| 301 | const llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, |
| 302 | omp::PrivateClauseOp privatizer, llvm::Value *origVar, |
| 303 | llvm::Value *privateVar, int idx) { |
| 304 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 305 | builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator()); |
| 306 | linearStartVals.push_back( |
| 307 | Elt: builder.CreateAlloca(Ty: linearVarTypes[idx], ArraySize: nullptr, Name: ".linear_var" )); |
| 308 | linearPrivatizers.push_back(Elt: privatizer); |
| 309 | linearOrigVars.push_back(Elt: origVar); |
| 310 | linearPrivateVars.push_back(Elt: privateVar); |
| 311 | } |
| 312 | |
| 313 | // Initialize linear step |
| 314 | inline void initLinearStep(LLVM::ModuleTranslation &moduleTranslation, |
| 315 | mlir::Value &linearStep) { |
| 316 | linearSteps.push_back(Elt: moduleTranslation.lookupValue(value: linearStep)); |
| 317 | } |
| 318 | |
| 319 | // Emit IR for initialization of linear variables |
| 320 | llvm::Error initLinearVar(llvm::IRBuilderBase &builder, |
| 321 | LLVM::ModuleTranslation &moduleTranslation, |
| 322 | llvm::BasicBlock *) { |
| 323 | builder.SetInsertPoint(loopPreHeader->getTerminator()); |
| 324 | for (size_t index = 0; index < linearOrigVars.size(); index++) { |
| 325 | llvm::Expected<llvm::Value *> orig = |
| 326 | getAddress(builder, moduleTranslation, index, var: linearOrigVars[index]); |
| 327 | if (!orig) |
| 328 | return orig.takeError(); |
| 329 | llvm::LoadInst *linearVarLoad = |
| 330 | builder.CreateLoad(Ty: linearVarTypes[index], Ptr: *orig); |
| 331 | builder.CreateStore(Val: linearVarLoad, Ptr: linearStartVals[index]); |
| 332 | } |
| 333 | return llvm::Error::success(); |
| 334 | } |
| 335 | |
| 336 | // Find linear iteration variable and save it for later updates |
| 337 | LogicalResult initLinearIV(omp::SimdOp simdOp) { |
| 338 | auto loopOp = cast<omp::LoopNestOp>(Val: simdOp.getWrappedLoop()); |
| 339 | // NOTE iteration variables can only be linear in non-nested loops. |
| 340 | if (loopOp.getIVs().size() != 1) |
| 341 | return success(); |
| 342 | // Currently, frontends using `omp.simd` always generate a store from the |
| 343 | // `omp.loop_nest`'s IV to the corresponding iteration variable, possibly |
| 344 | // through the data address of its private copy. |
| 345 | // |
| 346 | // TODO Add an attribute to `omp.loop_nest` that explicitly lists the |
| 347 | // variables that correspond to the loop induction variables. |
| 348 | MutableArrayRef<BlockArgument> linearArgs = |
| 349 | cast<omp::BlockArgOpenMPOpInterface>(Val&: *simdOp).getLinearBlockArgs(); |
| 350 | SmallVector<Value> ivValues{loopOp.getIVs().front()}; |
| 351 | for (Operation *user : loopOp.getIVs().front().getUsers()) |
| 352 | if (isa<LLVM::SExtOp, LLVM::ZExtOp, LLVM::TruncOp>(Val: user)) |
| 353 | ivValues.push_back(Elt: user->getResult(idx: 0)); |
| 354 | |
| 355 | for (Value ivValue : ivValues) { |
| 356 | for (Operation *user : ivValue.getUsers()) { |
| 357 | auto storeOp = dyn_cast<LLVM::StoreOp>(Val: user); |
| 358 | if (!storeOp || storeOp.getValue() != ivValue) |
| 359 | continue; |
| 360 | // Look through loads, GEPs and copies of a private copy, e.g. of a |
| 361 | // descriptor, to find the linear block argument the IV is stored to. |
| 362 | Value addr = storeOp.getAddr(); |
| 363 | while (true) { |
| 364 | Operation *def = addr.getDefiningOp(); |
| 365 | if (auto load = dyn_cast_if_present<LLVM::LoadOp>(Val: def)) { |
| 366 | addr = load.getAddr(); |
| 367 | } else if (auto gep = dyn_cast_if_present<LLVM::GEPOp>(Val: def)) { |
| 368 | addr = gep.getBase(); |
| 369 | } else { |
| 370 | auto copy = llvm::find_if(Range: addr.getUsers(), P: [&](Operation *user) { |
| 371 | auto memcpy = dyn_cast<LLVM::MemcpyOp>(Val: user); |
| 372 | return memcpy && memcpy.getDst() == addr && |
| 373 | llvm::is_contained(Range&: linearArgs, Element: memcpy.getSrc()); |
| 374 | }); |
| 375 | if (copy == addr.getUsers().end()) |
| 376 | break; |
| 377 | addr = cast<LLVM::MemcpyOp>(Val: *copy).getSrc(); |
| 378 | } |
| 379 | } |
| 380 | auto *it = llvm::find(Range&: linearArgs, Val: addr); |
| 381 | if (it == linearArgs.end()) |
| 382 | continue; |
| 383 | size_t index = std::distance(first: linearArgs.begin(), last: it); |
| 384 | if (linearLoopIVIndex && *linearLoopIVIndex != index) |
| 385 | return simdOp.emitError( |
| 386 | message: "Could not determine the linear variable associated with the " |
| 387 | "loop nest induction variable" ); |
| 388 | linearLoopIVIndex = index; |
| 389 | } |
| 390 | } |
| 391 | return success(); |
| 392 | } |
| 393 | |
| 394 | // Emit IR for updating Linear variables |
| 395 | llvm::Error updateLinearVar(llvm::IRBuilderBase &builder, |
| 396 | LLVM::ModuleTranslation &moduleTranslation, |
| 397 | llvm::BasicBlock *loopBody, |
| 398 | llvm::Value *loopInductionVar) { |
| 399 | builder.SetInsertPoint(loopBody->getTerminator()); |
| 400 | for (size_t index = 0; index < linearStartVals.size(); index++) { |
| 401 | llvm::Type *linearVarType = linearVarTypes[index]; |
| 402 | llvm::Value *iv = loopInductionVar; |
| 403 | llvm::Value *step = linearSteps[index]; |
| 404 | llvm::Expected<llvm::Value *> result = getAddress( |
| 405 | builder, moduleTranslation, index, var: linearPrivateVars[index]); |
| 406 | if (!result) |
| 407 | return result.takeError(); |
| 408 | |
| 409 | if (!iv->getType()->isIntegerTy()) |
| 410 | llvm_unreachable("OpenMP loop induction variable must be an integer " |
| 411 | "type" ); |
| 412 | |
| 413 | if (linearVarType->isIntegerTy()) { |
| 414 | // Integer path: normalize all arithmetic to linearVarType |
| 415 | iv = builder.CreateSExtOrTrunc(V: iv, DestTy: linearVarType); |
| 416 | step = builder.CreateSExtOrTrunc(V: step, DestTy: linearVarType); |
| 417 | |
| 418 | llvm::LoadInst *linearVarStart = |
| 419 | builder.CreateLoad(Ty: linearVarType, Ptr: linearStartVals[index]); |
| 420 | llvm::Value *mulInst = builder.CreateMul(LHS: iv, RHS: step); |
| 421 | llvm::Value *addInst = builder.CreateAdd(LHS: linearVarStart, RHS: mulInst); |
| 422 | builder.CreateStore(Val: addInst, Ptr: *result); |
| 423 | } else if (linearVarType->isFloatingPointTy()) { |
| 424 | // Float path: perform multiply in integer, then convert to float |
| 425 | step = builder.CreateSExtOrTrunc(V: step, DestTy: iv->getType()); |
| 426 | llvm::Value *mulInst = builder.CreateMul(LHS: iv, RHS: step); |
| 427 | |
| 428 | llvm::LoadInst *linearVarStart = |
| 429 | builder.CreateLoad(Ty: linearVarType, Ptr: linearStartVals[index]); |
| 430 | llvm::Value *mulFp = builder.CreateSIToFP(V: mulInst, DestTy: linearVarType); |
| 431 | llvm::Value *addInst = builder.CreateFAdd(L: linearVarStart, R: mulFp); |
| 432 | builder.CreateStore(Val: addInst, Ptr: *result); |
| 433 | } else { |
| 434 | llvm_unreachable( |
| 435 | "Linear variable must be of integer or floating-point type" ); |
| 436 | } |
| 437 | } |
| 438 | return llvm::Error::success(); |
| 439 | } |
| 440 | |
| 441 | // Emit IR for updating linear iteration variables on loop exit |
| 442 | llvm::Error updateLinearIV(llvm::IRBuilderBase &builder, |
| 443 | LLVM::ModuleTranslation &moduleTranslation) { |
| 444 | if (!linearLoopIVIndex) |
| 445 | return llvm::Error::success(); |
| 446 | size_t index = *linearLoopIVIndex; |
| 447 | |
| 448 | // Add one more step to the linear iteration variable |
| 449 | llvm::Type *varType = linearVarTypes[index]; |
| 450 | if (!varType->isIntegerTy()) |
| 451 | llvm_unreachable("Linear iteration variable must be of integer type" ); |
| 452 | llvm::Expected<llvm::Value *> var = |
| 453 | getAddress(builder, moduleTranslation, index, var: linearPrivateVars[index]); |
| 454 | if (!var) |
| 455 | return var.takeError(); |
| 456 | llvm::Value *step = builder.CreateSExtOrTrunc(V: linearSteps[index], DestTy: varType); |
| 457 | llvm::Value *val = builder.CreateLoad(Ty: varType, Ptr: *var); |
| 458 | llvm::Value *addInst = builder.CreateAdd(LHS: val, RHS: step); |
| 459 | builder.CreateStore(Val: addInst, Ptr: *var); |
| 460 | return llvm::Error::success(); |
| 461 | } |
| 462 | |
| 463 | // Linear variable finalization is conditional on the last logical iteration. |
| 464 | // Create BB splits to manage the same. |
| 465 | void splitLinearFiniBB(llvm::IRBuilderBase &builder, |
| 466 | llvm::BasicBlock *loopExit) { |
| 467 | linearFinalizationBB = loopExit->splitBasicBlock( |
| 468 | I: loopExit->getTerminator(), BBName: "omp_loop.linear_finalization" ); |
| 469 | linearExitBB = linearFinalizationBB->splitBasicBlock( |
| 470 | I: linearFinalizationBB->getTerminator(), BBName: "omp_loop.linear_exit" ); |
| 471 | linearLastIterExitBB = linearFinalizationBB->splitBasicBlock( |
| 472 | I: linearFinalizationBB->getTerminator(), BBName: "omp_loop.linear_lastiter_exit" ); |
| 473 | } |
| 474 | |
| 475 | // Finalize the linear vars |
| 476 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy |
| 477 | finalizeLinearVar(llvm::IRBuilderBase &builder, |
| 478 | LLVM::ModuleTranslation &moduleTranslation, |
| 479 | llvm::Value *lastIter) { |
| 480 | // Emit condition to check whether last logical iteration is being executed |
| 481 | builder.SetInsertPoint(linearFinalizationBB->getTerminator()); |
| 482 | llvm::Value *loopLastIterLoad = builder.CreateLoad( |
| 483 | Ty: llvm::Type::getInt32Ty(C&: builder.getContext()), Ptr: lastIter); |
| 484 | llvm::Value *isLast = |
| 485 | builder.CreateCmp(Pred: llvm::CmpInst::ICMP_NE, LHS: loopLastIterLoad, |
| 486 | RHS: llvm::ConstantInt::get( |
| 487 | Ty: llvm::Type::getInt32Ty(C&: builder.getContext()), V: 0)); |
| 488 | // Store the linear variable values to original variables. |
| 489 | builder.SetInsertPoint(linearLastIterExitBB->getTerminator()); |
| 490 | if (llvm::Error err = storeLinearVars(builder, moduleTranslation)) |
| 491 | return std::move(err); |
| 492 | |
| 493 | // Create conditional branch such that the linear variable |
| 494 | // values are stored to original variables only at the |
| 495 | // last logical iteration |
| 496 | builder.SetInsertPoint(linearFinalizationBB->getTerminator()); |
| 497 | builder.CreateCondBr(Cond: isLast, True: linearLastIterExitBB, False: linearExitBB); |
| 498 | linearFinalizationBB->getTerminator()->eraseFromParent(); |
| 499 | // Emit barrier |
| 500 | builder.SetInsertPoint(linearExitBB->getTerminator()); |
| 501 | return getOpenMPBuilder(moduleTranslation) |
| 502 | ->createBarrier(Loc: builder, Kind: llvm::omp::OMPD_barrier); |
| 503 | } |
| 504 | |
| 505 | // Emit stores for linear variables. Useful in case of SIMD |
| 506 | // construct. |
| 507 | llvm::Error |
| 508 | emitStoresForLinearVar(llvm::IRBuilderBase &builder, |
| 509 | LLVM::ModuleTranslation &moduleTranslation) { |
| 510 | return storeLinearVars(builder, moduleTranslation); |
| 511 | } |
| 512 | }; |
| 513 | } // namespace |
| 514 | |
| 515 | /// Looks up from the operation from and returns the PrivateClauseOp with |
| 516 | /// name symbolName |
| 517 | static omp::PrivateClauseOp findPrivatizer(Operation *from, |
| 518 | SymbolRefAttr symbolName) { |
| 519 | omp::PrivateClauseOp privatizer = |
| 520 | SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(from, |
| 521 | symbol: symbolName); |
| 522 | assert(privatizer && "privatizer not found in the symbol table" ); |
| 523 | return privatizer; |
| 524 | } |
| 525 | |
| 526 | /// Check whether translation to LLVM IR for the given operation is currently |
| 527 | /// supported. If not, descriptive diagnostics will be emitted to let users know |
| 528 | /// this is a not-yet-implemented feature. |
| 529 | /// |
| 530 | /// \returns success if no unimplemented features are needed to translate the |
| 531 | /// given operation. |
| 532 | static LogicalResult checkImplementationStatus(Operation &op) { |
| 533 | auto todo = [&op](StringRef clauseName) { |
| 534 | return op.emitError() << "not yet implemented: Unhandled clause " |
| 535 | << clauseName << " in " << op.getName() |
| 536 | << " operation" ; |
| 537 | }; |
| 538 | |
| 539 | auto checkAllocate = [&todo](auto op, LogicalResult &result) { |
| 540 | if (!op.getAllocateVars().empty() || !op.getAllocatorVars().empty()) |
| 541 | result = todo("allocate" ); |
| 542 | }; |
| 543 | auto checkBare = [&todo](auto op, LogicalResult &result) { |
| 544 | if (op.getKernelType() == omp::TargetExecMode::bare) |
| 545 | result = todo("ompx_bare" ); |
| 546 | }; |
| 547 | auto checkDepend = [&todo](auto op, LogicalResult &result) { |
| 548 | if (!op.getDependVars().empty() || op.getDependKinds() || |
| 549 | !op.getDependIterated().empty() || op.getDependIteratedKinds()) |
| 550 | result = todo("depend" ); |
| 551 | }; |
| 552 | auto checkHint = [](auto op, LogicalResult &) { |
| 553 | if (op.getHint()) |
| 554 | op.emitWarning("hint clause discarded" ); |
| 555 | }; |
| 556 | auto checkInReduction = [&todo](auto op, LogicalResult &result) { |
| 557 | if (isa<omp::TargetOp, omp::TaskOp, omp::TaskloopContextOp>( |
| 558 | op.getOperation())) { |
| 559 | if (auto byrefAttr = op.getInReductionByref()) { |
| 560 | for (bool isByRef : *byrefAttr) { |
| 561 | if (isByRef) { |
| 562 | result = todo("in_reduction with byref modifier" ); |
| 563 | return; |
| 564 | } |
| 565 | } |
| 566 | } |
| 567 | if (isa<omp::TargetOp>(op.getOperation())) { |
| 568 | if (auto inReductionSyms = op.getInReductionSyms()) { |
| 569 | for (auto sym : |
| 570 | (*inReductionSyms).template getAsRange<SymbolRefAttr>()) { |
| 571 | auto decl = |
| 572 | SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>( |
| 573 | op, sym); |
| 574 | assert(decl && |
| 575 | "symbol resolution should be guaranteed by the op verifier" ); |
| 576 | if (decl.getInitializerRegion().front().getNumArguments() != 1) { |
| 577 | result = todo("in_reduction with two-argument initializer" ); |
| 578 | return; |
| 579 | } |
| 580 | if (!decl.getCleanupRegion().empty()) { |
| 581 | result = todo("in_reduction with cleanup region" ); |
| 582 | return; |
| 583 | } |
| 584 | } |
| 585 | } |
| 586 | } |
| 587 | } else if (!op.getInReductionVars().empty() || op.getInReductionByref() || |
| 588 | op.getInReductionSyms()) { |
| 589 | result = todo("in_reduction" ); |
| 590 | } |
| 591 | }; |
| 592 | auto checkNowait = [&todo](auto op, LogicalResult &result) { |
| 593 | if (op.getNowait()) |
| 594 | result = todo("nowait" ); |
| 595 | }; |
| 596 | auto checkOrder = [&todo](auto op, LogicalResult &result) { |
| 597 | if (op.getOrder() || op.getOrderMod()) |
| 598 | result = todo("order" ); |
| 599 | }; |
| 600 | auto checkPrivate = [&todo](auto op, LogicalResult &result) { |
| 601 | if (!op.getPrivateVars().empty() || op.getPrivateSyms()) |
| 602 | result = todo("privatization" ); |
| 603 | }; |
| 604 | auto checkReduction = [&todo](auto op, LogicalResult &result) { |
| 605 | if (isa<omp::TeamsOp>(op)) |
| 606 | if (!op.getReductionVars().empty() || op.getReductionByref() || |
| 607 | op.getReductionSyms()) |
| 608 | result = todo("reduction" ); |
| 609 | for (unsigned i = 0, e = op.getNumReductionVars(); i != e; ++i) { |
| 610 | omp::ReductionModifier mod = op.getReductionModifier(i); |
| 611 | if (mod == omp::ReductionModifier::defaultmod) |
| 612 | continue; |
| 613 | // Task modifiers are supported only on parallel and worksharing |
| 614 | // constructs, and only for non-byref reduction variables. |
| 615 | bool taskModifierSupported = |
| 616 | mod == omp::ReductionModifier::task && |
| 617 | isa<omp::ParallelOp, omp::WsloopOp, omp::SectionsOp>(op); |
| 618 | if (!taskModifierSupported) { |
| 619 | result = todo("reduction with modifier" ); |
| 620 | } else if (auto byref = op.getReductionByref()) { |
| 621 | if ((*byref)[i]) |
| 622 | result = todo("task reduction modifier with by-ref reduction" ); |
| 623 | } |
| 624 | } |
| 625 | }; |
| 626 | auto checkTaskReductionByref = [&todo](auto op, LogicalResult &result) { |
| 627 | if (auto byrefAttr = op.getTaskReductionByref()) |
| 628 | for (bool isByRef : *byrefAttr) |
| 629 | if (isByRef) { |
| 630 | result = todo("task_reduction with byref modifier" ); |
| 631 | return; |
| 632 | } |
| 633 | }; |
| 634 | auto checkReductionByref = [&todo](auto op, LogicalResult &result) { |
| 635 | if (auto byrefAttr = op.getReductionByref()) |
| 636 | for (bool isByRef : *byrefAttr) |
| 637 | if (isByRef) { |
| 638 | result = todo("reduction with byref modifier" ); |
| 639 | return; |
| 640 | } |
| 641 | }; |
| 642 | auto checkNumTeams = [&todo](auto op, LogicalResult &result) { |
| 643 | if (op.hasNumTeamsMultiDim()) |
| 644 | result = todo("num_teams with multi-dimensional values" ); |
| 645 | }; |
| 646 | auto checkNumThreads = [&todo](auto op, LogicalResult &result) { |
| 647 | if (op.hasNumThreadsMultiDim()) |
| 648 | result = todo("num_threads with multi-dimensional values" ); |
| 649 | }; |
| 650 | |
| 651 | auto checkThreadLimit = [&todo](auto op, LogicalResult &result) { |
| 652 | if (op.hasThreadLimitMultiDim()) |
| 653 | result = todo("thread_limit with multi-dimensional values" ); |
| 654 | }; |
| 655 | auto checkMap = [&todo](auto op, LogicalResult &result) { |
| 656 | if (!op.getMapIterated().empty()) |
| 657 | result = todo("map/motion clause with iterator modifier" ); |
| 658 | }; |
| 659 | |
| 660 | auto checkDynGroupprivate = [&todo](auto op, LogicalResult &result) { |
| 661 | if (op.getDynGroupprivateSize()) |
| 662 | result = todo("dyn_groupprivate" ); |
| 663 | }; |
| 664 | |
| 665 | // `declare_mapper.info` supports a single-dimension iterator whose region |
| 666 | // (which may itself span several blocks) yields one member-free omp.map.info |
| 667 | // for a locator described by var_ptr + bounds. Descriptor-indirect |
| 668 | // (var_ptr_ptr) locators aren't handled yet. Anything else falls back to the |
| 669 | // generic (not yet implemented) diagnostic above. |
| 670 | auto checkDeclareMapperMap = [&todo](omp::DeclareMapperInfoOp op, |
| 671 | LogicalResult &result) { |
| 672 | for (Value iterVal : op.getMapIterated()) { |
| 673 | auto itersOp = iterVal.getDefiningOp<omp::IteratorOp>(); |
| 674 | bool supported = itersOp && itersOp.getLoopLowerBounds().size() == 1; |
| 675 | if (supported) { |
| 676 | omp::YieldOp yieldOp = itersOp.getYieldOp(); |
| 677 | supported = yieldOp && yieldOp.getResults().size() == 1; |
| 678 | if (supported) { |
| 679 | auto mapInfoOp = |
| 680 | yieldOp.getResults()[0].getDefiningOp<omp::MapInfoOp>(); |
| 681 | supported = |
| 682 | mapInfoOp && !mapInfoOp.getVarPtrPtr() && |
| 683 | mapInfoOp.getMembers().empty() && |
| 684 | mapInfoOp.getMapCaptureType() == omp::VariableCaptureKind::ByRef; |
| 685 | } |
| 686 | } |
| 687 | if (!supported) { |
| 688 | result = todo("map/motion clause with iterator modifier" ); |
| 689 | return; |
| 690 | } |
| 691 | } |
| 692 | }; |
| 693 | |
| 694 | LogicalResult result = success(); |
| 695 | llvm::TypeSwitch<Operation &>(op) |
| 696 | .Case(caseFn: [&](omp::DistributeOp op) { |
| 697 | checkAllocate(op, result); |
| 698 | checkOrder(op, result); |
| 699 | }) |
| 700 | .Case(caseFn: [&](omp::SectionsOp op) { |
| 701 | checkAllocate(op, result); |
| 702 | checkPrivate(op, result); |
| 703 | checkReduction(op, result); |
| 704 | }) |
| 705 | .Case(caseFn: [&](omp::ScopeOp op) { checkReduction(op, result); }) |
| 706 | .Case(caseFn: [&](omp::SingleOp op) { |
| 707 | checkAllocate(op, result); |
| 708 | checkPrivate(op, result); |
| 709 | }) |
| 710 | .Case(caseFn: [&](omp::TeamsOp op) { |
| 711 | checkAllocate(op, result); |
| 712 | checkPrivate(op, result); |
| 713 | checkNumTeams(op, result); |
| 714 | checkThreadLimit(op, result); |
| 715 | checkDynGroupprivate(op, result); |
| 716 | }) |
| 717 | .Case(caseFn: [&](omp::TaskOp op) { |
| 718 | checkAllocate(op, result); |
| 719 | checkInReduction(op, result); |
| 720 | }) |
| 721 | .Case(caseFn: [&](omp::TaskgroupOp op) { |
| 722 | checkAllocate(op, result); |
| 723 | checkTaskReductionByref(op, result); |
| 724 | }) |
| 725 | .Case(caseFn: [&](omp::DispatchOp op) { |
| 726 | // OpenMP 5.1 dispatch creates an explicit task; nowait controls whether |
| 727 | // it is included. Diagnose unsupported asynchronous tasking before 5.2, |
| 728 | // where the nowait property has no effect on dispatch. |
| 729 | int64_t version = omp::getOpenMPVersionAttribute( |
| 730 | module: op->getParentOfType<ModuleOp>(), /*fallback=*/51); |
| 731 | if (version < 52) |
| 732 | checkNowait(op, result); |
| 733 | }) |
| 734 | .Case(caseFn: [&](omp::TaskloopContextOp op) { |
| 735 | checkAllocate(op, result); |
| 736 | checkInReduction(op, result); |
| 737 | checkReduction(op, result); |
| 738 | checkReductionByref(op, result); |
| 739 | }) |
| 740 | .Case(caseFn: [&](omp::WsloopOp op) { |
| 741 | checkAllocate(op, result); |
| 742 | checkOrder(op, result); |
| 743 | checkReduction(op, result); |
| 744 | }) |
| 745 | .Case(caseFn: [&](omp::ParallelOp op) { |
| 746 | checkReduction(op, result); |
| 747 | checkNumThreads(op, result); |
| 748 | }) |
| 749 | .Case(caseFn: [&](omp::SimdOp op) { checkReduction(op, result); }) |
| 750 | .Case<omp::AtomicReadOp, omp::AtomicWriteOp, omp::AtomicUpdateOp, |
| 751 | omp::AtomicCaptureOp>(caseFn: [&](auto op) { checkHint(op, result); }) |
| 752 | .Case(caseFn: [&](omp::AtomicCompareOp op) { |
| 753 | checkHint(op, result); |
| 754 | Region ®ion = op.getRegion(); |
| 755 | if (region.empty()) |
| 756 | return; |
| 757 | mlir::Type argType = region.front().getArgument(i: 0).getType(); |
| 758 | auto structTy = dyn_cast<LLVM::LLVMStructType>(Val&: argType); |
| 759 | if (!structTy) |
| 760 | return; |
| 761 | DataLayout dl = DataLayout(op->getParentOfType<ModuleOp>()); |
| 762 | unsigned totalBits = dl.getTypeSizeInBits(t: structTy); |
| 763 | if (totalBits > 128) |
| 764 | result = todo("compare for complex types wider than 128 bits" ); |
| 765 | }) |
| 766 | .Case<omp::TargetEnterDataOp, omp::TargetExitDataOp>(caseFn: [&](auto op) { |
| 767 | checkDepend(op, result); |
| 768 | checkMap(op, result); |
| 769 | }) |
| 770 | .Case(caseFn: [&](omp::TargetUpdateOp op) { |
| 771 | checkDepend(op, result); |
| 772 | checkMap(op, result); |
| 773 | }) |
| 774 | .Case(caseFn: [&](omp::TargetOp op) { |
| 775 | checkAllocate(op, result); |
| 776 | checkBare(op, result); |
| 777 | checkInReduction(op, result); |
| 778 | checkMap(op, result); |
| 779 | checkThreadLimit(op, result); |
| 780 | }) |
| 781 | .Case(caseFn: [&](omp::TargetDataOp op) { checkMap(op, result); }) |
| 782 | .Case(caseFn: [&](omp::DeclareMapperInfoOp op) { |
| 783 | checkDeclareMapperMap(op, result); |
| 784 | }) |
| 785 | .Default(defaultFn: [](Operation &) { |
| 786 | // Assume all clauses for an operation can be translated unless they are |
| 787 | // checked above. |
| 788 | }); |
| 789 | return result; |
| 790 | } |
| 791 | |
| 792 | static LogicalResult handleError(llvm::Error error, Operation &op) { |
| 793 | LogicalResult result = success(); |
| 794 | if (error) { |
| 795 | llvm::handleAllErrors( |
| 796 | E: std::move(error), |
| 797 | Handlers: [&](const PreviouslyReportedError &) { result = failure(); }, |
| 798 | Handlers: [&](const llvm::ErrorInfoBase &err) { |
| 799 | result = op.emitError(message: err.message()); |
| 800 | }); |
| 801 | } |
| 802 | return result; |
| 803 | } |
| 804 | |
| 805 | template <typename T> |
| 806 | static LogicalResult handleError(llvm::Expected<T> &result, Operation &op) { |
| 807 | if (!result) |
| 808 | return handleError(result.takeError(), op); |
| 809 | |
| 810 | return success(); |
| 811 | } |
| 812 | |
| 813 | /// Find the insertion point for allocas given the current insertion point for |
| 814 | /// normal operations in the builder. |
| 815 | static llvm::OpenMPIRBuilder::InsertPointTy findAllocInsertPoints( |
| 816 | llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, |
| 817 | llvm::SmallVectorImpl<llvm::BasicBlock *> *deallocBlocks = nullptr) { |
| 818 | // If there is an allocation insertion point on stack, i.e. we are in a nested |
| 819 | // operation and a specific point was provided by some surrounding operation, |
| 820 | // use it. |
| 821 | llvm::OpenMPIRBuilder::InsertPointTy allocInsertPoint; |
| 822 | llvm::ArrayRef<llvm::BasicBlock *> deallocInsertPoints; |
| 823 | WalkResult walkResult = moduleTranslation.stackWalk<OpenMPAllocStackFrame>( |
| 824 | callback: [&](OpenMPAllocStackFrame &frame) { |
| 825 | allocInsertPoint = frame.allocBlock->getFirstInsertionPt(); |
| 826 | deallocInsertPoints = frame.deallocBlocks; |
| 827 | return WalkResult::interrupt(); |
| 828 | }); |
| 829 | // In cases with multiple levels of outlining, the tree walk might find an |
| 830 | // insertion point that is inside the original function while the builder |
| 831 | // insertion point is inside the outlined function. We need to make sure that |
| 832 | // we do not use it in those cases. |
| 833 | if (walkResult.wasInterrupted() && |
| 834 | allocInsertPoint.getNodeParent()->getParent() == |
| 835 | builder.GetInsertBlock()->getParent()) { |
| 836 | if (deallocBlocks) |
| 837 | deallocBlocks->insert(I: deallocBlocks->end(), From: deallocInsertPoints.begin(), |
| 838 | To: deallocInsertPoints.end()); |
| 839 | return allocInsertPoint; |
| 840 | } |
| 841 | |
| 842 | // Otherwise, insert to the entry block of the surrounding function. |
| 843 | // If the current IRBuilder InsertPoint is the function's entry, it cannot |
| 844 | // also be used for alloca insertion which would result in insertion order |
| 845 | // confusion. Create a new BasicBlock for the Builder and use the entry block |
| 846 | // for the allocs. |
| 847 | // TODO: Create a dedicated alloca BasicBlock at function creation such that |
| 848 | // we do not need to move the current InsertPoint here. |
| 849 | if (builder.GetInsertBlock() == |
| 850 | &builder.GetInsertBlock()->getParent()->getEntryBlock()) { |
| 851 | assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() && |
| 852 | "Assuming end of basic block" ); |
| 853 | llvm::BasicBlock *entryBB = llvm::BasicBlock::Create( |
| 854 | Context&: builder.getContext(), Name: "entry" , Parent: builder.GetInsertBlock()->getParent(), |
| 855 | InsertBefore: builder.GetInsertBlock()->getNextNode()); |
| 856 | builder.CreateBr(Dest: entryBB); |
| 857 | builder.SetInsertPoint(entryBB); |
| 858 | } |
| 859 | |
| 860 | // Collect exit blocks, which is where explicit deallocations should happen in |
| 861 | // this case. |
| 862 | if (deallocBlocks) { |
| 863 | for (llvm::BasicBlock &block : *builder.GetInsertBlock()->getParent()) { |
| 864 | // TODO: This currently results in no blocks being added to the list when |
| 865 | // all exit blocks of the enclosing function have not been lowered before |
| 866 | // this is reached. |
| 867 | llvm::Instruction *terminator = block.getTerminatorOrNull(); |
| 868 | if (isa_and_present<llvm::ReturnInst>(Val: terminator)) |
| 869 | deallocBlocks->emplace_back(Args: &block); |
| 870 | } |
| 871 | } |
| 872 | |
| 873 | llvm::BasicBlock &funcEntryBlock = |
| 874 | builder.GetInsertBlock()->getParent()->getEntryBlock(); |
| 875 | return funcEntryBlock.getFirstInsertionPt(); |
| 876 | } |
| 877 | |
| 878 | /// Find the loop information structure for the loop nest being translated. It |
| 879 | /// will return a `null` value unless called from the translation function for |
| 880 | /// a loop wrapper operation after successfully translating its body. |
| 881 | static llvm::CanonicalLoopInfo * |
| 882 | findCurrentLoopInfo(LLVM::ModuleTranslation &moduleTranslation) { |
| 883 | llvm::CanonicalLoopInfo *loopInfo = nullptr; |
| 884 | moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>( |
| 885 | callback: [&](OpenMPLoopInfoStackFrame &frame) { |
| 886 | loopInfo = frame.loopInfo; |
| 887 | return WalkResult::interrupt(); |
| 888 | }); |
| 889 | return loopInfo; |
| 890 | } |
| 891 | |
| 892 | /// Converts the given region that appears within an OpenMP dialect operation to |
| 893 | /// LLVM IR, creating a branch from the `sourceBlock` to the entry block of the |
| 894 | /// region, and a branch from any block with an successor-less OpenMP terminator |
| 895 | /// to `continuationBlock`. Populates `continuationBlockPHIs` with the PHI nodes |
| 896 | /// of the continuation block if provided. |
| 897 | static llvm::Expected<llvm::BasicBlock *> convertOmpOpRegions( |
| 898 | Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, |
| 899 | LLVM::ModuleTranslation &moduleTranslation, |
| 900 | SmallVectorImpl<llvm::PHINode *> *continuationBlockPHIs = nullptr) { |
| 901 | bool isLoopWrapper = isa<omp::LoopWrapperInterface>(Val: region.getParentOp()); |
| 902 | |
| 903 | llvm::BasicBlock *continuationBlock = |
| 904 | splitBB(Builder&: builder, CreateBranch: true, Name: "omp.region.cont" ); |
| 905 | llvm::BasicBlock *sourceBlock = builder.GetInsertBlock(); |
| 906 | |
| 907 | llvm::LLVMContext &llvmContext = builder.getContext(); |
| 908 | for (Block &bb : region) { |
| 909 | llvm::BasicBlock *llvmBB = llvm::BasicBlock::Create( |
| 910 | Context&: llvmContext, Name: blockName, Parent: builder.GetInsertBlock()->getParent(), |
| 911 | InsertBefore: builder.GetInsertBlock()->getNextNode()); |
| 912 | moduleTranslation.mapBlock(mlir: &bb, llvm: llvmBB); |
| 913 | } |
| 914 | |
| 915 | llvm::Instruction *sourceTerminator = sourceBlock->getTerminator(); |
| 916 | |
| 917 | // Terminators (namely YieldOp) may be forwarding values to the region that |
| 918 | // need to be available in the continuation block. Collect the types of these |
| 919 | // operands in preparation of creating PHI nodes. This is skipped for loop |
| 920 | // wrapper operations, for which we know in advance they have no terminators. |
| 921 | SmallVector<llvm::Type *> continuationBlockPHITypes; |
| 922 | unsigned numYields = 0; |
| 923 | |
| 924 | if (!isLoopWrapper) { |
| 925 | bool operandsProcessed = false; |
| 926 | for (Block &bb : region.getBlocks()) { |
| 927 | if (omp::YieldOp yield = dyn_cast<omp::YieldOp>(Val: bb.getTerminator())) { |
| 928 | if (!operandsProcessed) { |
| 929 | for (unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) { |
| 930 | continuationBlockPHITypes.push_back( |
| 931 | Elt: moduleTranslation.convertType(type: yield->getOperand(idx: i).getType())); |
| 932 | } |
| 933 | operandsProcessed = true; |
| 934 | } else { |
| 935 | assert(continuationBlockPHITypes.size() == yield->getNumOperands() && |
| 936 | "mismatching number of values yielded from the region" ); |
| 937 | for (unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) { |
| 938 | llvm::Type *operandType = |
| 939 | moduleTranslation.convertType(type: yield->getOperand(idx: i).getType()); |
| 940 | (void)operandType; |
| 941 | assert(continuationBlockPHITypes[i] == operandType && |
| 942 | "values of mismatching types yielded from the region" ); |
| 943 | } |
| 944 | } |
| 945 | numYields++; |
| 946 | } |
| 947 | } |
| 948 | } |
| 949 | |
| 950 | // Insert PHI nodes in the continuation block for any values forwarded by the |
| 951 | // terminators in this region. |
| 952 | if (!continuationBlockPHITypes.empty()) |
| 953 | assert( |
| 954 | continuationBlockPHIs && |
| 955 | "expected continuation block PHIs if converted regions yield values" ); |
| 956 | if (continuationBlockPHIs) { |
| 957 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 958 | continuationBlockPHIs->reserve(N: continuationBlockPHITypes.size()); |
| 959 | builder.SetInsertPoint(continuationBlock->begin()); |
| 960 | for (llvm::Type *ty : continuationBlockPHITypes) |
| 961 | continuationBlockPHIs->push_back(Elt: builder.CreatePHI(Ty: ty, NumReservedValues: numYields)); |
| 962 | } |
| 963 | |
| 964 | // Convert blocks one by one in topological order to ensure |
| 965 | // defs are converted before uses. |
| 966 | SetVector<Block *> blocks = getBlocksSortedByDominance(region); |
| 967 | for (Block *bb : blocks) { |
| 968 | llvm::BasicBlock *llvmBB = moduleTranslation.lookupBlock(block: bb); |
| 969 | // Retarget the branch of the entry block to the entry block of the |
| 970 | // converted region (regions are single-entry). |
| 971 | if (bb->isEntryBlock()) { |
| 972 | assert(sourceTerminator->getNumSuccessors() == 1 && |
| 973 | "provided entry block has multiple successors" ); |
| 974 | assert(sourceTerminator->getSuccessor(0) == continuationBlock && |
| 975 | "ContinuationBlock is not the successor of the entry block" ); |
| 976 | sourceTerminator->setSuccessor(Idx: 0, BB: llvmBB); |
| 977 | } |
| 978 | |
| 979 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 980 | if (failed( |
| 981 | Result: moduleTranslation.convertBlock(bb&: *bb, ignoreArguments: bb->isEntryBlock(), builder))) |
| 982 | return llvm::make_error<PreviouslyReportedError>(); |
| 983 | |
| 984 | // Create a direct branch here for loop wrappers to prevent their lack of a |
| 985 | // terminator from causing a crash below. |
| 986 | if (isLoopWrapper) { |
| 987 | builder.CreateBr(Dest: continuationBlock); |
| 988 | continue; |
| 989 | } |
| 990 | |
| 991 | // Special handling for `omp.yield` and `omp.terminator` (we may have more |
| 992 | // than one): they return the control to the parent OpenMP dialect operation |
| 993 | // so replace them with the branch to the continuation block. We handle this |
| 994 | // here to avoid relying inter-function communication through the |
| 995 | // ModuleTranslation class to set up the correct insertion point. This is |
| 996 | // also consistent with MLIR's idiom of handling special region terminators |
| 997 | // in the same code that handles the region-owning operation. |
| 998 | Operation *terminator = bb->getTerminator(); |
| 999 | if (isa<omp::TerminatorOp, omp::YieldOp>(Val: terminator)) { |
| 1000 | builder.CreateBr(Dest: continuationBlock); |
| 1001 | |
| 1002 | for (unsigned i = 0, e = terminator->getNumOperands(); i < e; ++i) |
| 1003 | (*continuationBlockPHIs)[i]->addIncoming( |
| 1004 | V: moduleTranslation.lookupValue(value: terminator->getOperand(idx: i)), BB: llvmBB); |
| 1005 | } |
| 1006 | } |
| 1007 | // After all blocks have been traversed and values mapped, connect the PHI |
| 1008 | // nodes to the results of preceding blocks. |
| 1009 | LLVM::detail::connectPHINodes(region, state: moduleTranslation); |
| 1010 | |
| 1011 | // Remove the blocks and values defined in this region from the mapping since |
| 1012 | // they are not visible outside of this region. This allows the same region to |
| 1013 | // be converted several times, that is cloned, without clashes, and slightly |
| 1014 | // speeds up the lookups. |
| 1015 | moduleTranslation.forgetMapping(region); |
| 1016 | |
| 1017 | return continuationBlock; |
| 1018 | } |
| 1019 | |
| 1020 | /// Convert ProcBindKind from MLIR-generated enum to LLVM enum. |
| 1021 | static llvm::omp::ProcBindKind getProcBindKind(omp::ClauseProcBindKind kind) { |
| 1022 | switch (kind) { |
| 1023 | case omp::ClauseProcBindKind::Close: |
| 1024 | return llvm::omp::ProcBindKind::OMP_PROC_BIND_close; |
| 1025 | case omp::ClauseProcBindKind::Master: |
| 1026 | return llvm::omp::ProcBindKind::OMP_PROC_BIND_master; |
| 1027 | case omp::ClauseProcBindKind::Primary: |
| 1028 | return llvm::omp::ProcBindKind::OMP_PROC_BIND_primary; |
| 1029 | case omp::ClauseProcBindKind::Spread: |
| 1030 | return llvm::omp::ProcBindKind::OMP_PROC_BIND_spread; |
| 1031 | } |
| 1032 | llvm_unreachable("Unknown ClauseProcBindKind kind" ); |
| 1033 | } |
| 1034 | |
| 1035 | /// Convert 'dispatch' operation into LLVM IR. |
| 1036 | static LogicalResult |
| 1037 | convertOmpDispatch(Operation &opInst, llvm::IRBuilderBase &builder, |
| 1038 | LLVM::ModuleTranslation &moduleTranslation) { |
| 1039 | auto dispatchOp = cast<omp::DispatchOp>(Val&: opInst); |
| 1040 | |
| 1041 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 1042 | return failure(); |
| 1043 | |
| 1044 | auto ®ion = dispatchOp.getRegion(); |
| 1045 | auto result = convertOmpOpRegions(region, blockName: "omp.dispatch.region" , builder, |
| 1046 | moduleTranslation); |
| 1047 | if (!result) |
| 1048 | return handleError(error: result.takeError(), op&: opInst); |
| 1049 | builder.SetInsertPoint(*result); |
| 1050 | return success(); |
| 1051 | } |
| 1052 | |
| 1053 | /// Converts an OpenMP 'masked' operation into LLVM IR using OpenMPIRBuilder. |
| 1054 | static LogicalResult |
| 1055 | convertOmpMasked(Operation &opInst, llvm::IRBuilderBase &builder, |
| 1056 | LLVM::ModuleTranslation &moduleTranslation) { |
| 1057 | auto maskedOp = cast<omp::MaskedOp>(Val&: opInst); |
| 1058 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 1059 | |
| 1060 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 1061 | return failure(); |
| 1062 | |
| 1063 | auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP, |
| 1064 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) { |
| 1065 | // MaskedOp has only one region associated with it. |
| 1066 | auto ®ion = maskedOp.getRegion(); |
| 1067 | builder.restoreIP(IP: codeGenIP); |
| 1068 | return convertOmpOpRegions(region, blockName: "omp.masked.region" , builder, |
| 1069 | moduleTranslation) |
| 1070 | .takeError(); |
| 1071 | }; |
| 1072 | |
| 1073 | // TODO: Perform finalization actions for variables. This has to be |
| 1074 | // called for variables which have destructors/finalizers. |
| 1075 | auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; |
| 1076 | |
| 1077 | llvm::Value *filterVal = nullptr; |
| 1078 | if (auto filterVar = maskedOp.getFilteredThreadId()) { |
| 1079 | filterVal = moduleTranslation.lookupValue(value: filterVar); |
| 1080 | } else { |
| 1081 | llvm::LLVMContext &llvmContext = builder.getContext(); |
| 1082 | filterVal = |
| 1083 | llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: llvmContext), /*V=*/0); |
| 1084 | } |
| 1085 | assert(filterVal != nullptr); |
| 1086 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 1087 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 1088 | getOpenMPBuilder(moduleTranslation) |
| 1089 | ->createMasked(Loc: ompLoc, BodyGenCB: bodyGenCB, FiniCB: finiCB, Filter: filterVal); |
| 1090 | |
| 1091 | if (failed(Result: handleError(result&: afterIP, op&: opInst))) |
| 1092 | return failure(); |
| 1093 | |
| 1094 | builder.restoreIP(IP: *afterIP); |
| 1095 | return success(); |
| 1096 | } |
| 1097 | |
| 1098 | /// Converts an OpenMP 'master' operation into LLVM IR using OpenMPIRBuilder. |
| 1099 | static LogicalResult |
| 1100 | convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder, |
| 1101 | LLVM::ModuleTranslation &moduleTranslation) { |
| 1102 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 1103 | auto masterOp = cast<omp::MasterOp>(Val&: opInst); |
| 1104 | |
| 1105 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 1106 | return failure(); |
| 1107 | |
| 1108 | auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP, |
| 1109 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) { |
| 1110 | // MasterOp has only one region associated with it. |
| 1111 | auto ®ion = masterOp.getRegion(); |
| 1112 | builder.restoreIP(IP: codeGenIP); |
| 1113 | return convertOmpOpRegions(region, blockName: "omp.master.region" , builder, |
| 1114 | moduleTranslation) |
| 1115 | .takeError(); |
| 1116 | }; |
| 1117 | |
| 1118 | // TODO: Perform finalization actions for variables. This has to be |
| 1119 | // called for variables which have destructors/finalizers. |
| 1120 | auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; |
| 1121 | |
| 1122 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 1123 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 1124 | getOpenMPBuilder(moduleTranslation) |
| 1125 | ->createMaster(Loc: ompLoc, BodyGenCB: bodyGenCB, FiniCB: finiCB); |
| 1126 | |
| 1127 | if (failed(Result: handleError(result&: afterIP, op&: opInst))) |
| 1128 | return failure(); |
| 1129 | |
| 1130 | builder.restoreIP(IP: *afterIP); |
| 1131 | return success(); |
| 1132 | } |
| 1133 | |
| 1134 | /// Converts an OpenMP 'critical' operation into LLVM IR using OpenMPIRBuilder. |
| 1135 | static LogicalResult |
| 1136 | convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder, |
| 1137 | LLVM::ModuleTranslation &moduleTranslation) { |
| 1138 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 1139 | auto criticalOp = cast<omp::CriticalOp>(Val&: opInst); |
| 1140 | |
| 1141 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 1142 | return failure(); |
| 1143 | |
| 1144 | auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP, |
| 1145 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) { |
| 1146 | // CriticalOp has only one region associated with it. |
| 1147 | auto ®ion = cast<omp::CriticalOp>(Val&: opInst).getRegion(); |
| 1148 | builder.restoreIP(IP: codeGenIP); |
| 1149 | return convertOmpOpRegions(region, blockName: "omp.critical.region" , builder, |
| 1150 | moduleTranslation) |
| 1151 | .takeError(); |
| 1152 | }; |
| 1153 | |
| 1154 | // TODO: Perform finalization actions for variables. This has to be |
| 1155 | // called for variables which have destructors/finalizers. |
| 1156 | auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; |
| 1157 | |
| 1158 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 1159 | llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext(); |
| 1160 | llvm::Constant *hint = nullptr; |
| 1161 | |
| 1162 | // If it has a name, it probably has a hint too. |
| 1163 | if (criticalOp.getNameAttr()) { |
| 1164 | // The verifiers in OpenMP Dialect guarentee that all the pointers are |
| 1165 | // non-null |
| 1166 | auto symbolRef = cast<SymbolRefAttr>(Val: criticalOp.getNameAttr()); |
| 1167 | auto criticalDeclareOp = |
| 1168 | SymbolTable::lookupNearestSymbolFrom<omp::CriticalDeclareOp>(from: criticalOp, |
| 1169 | symbol: symbolRef); |
| 1170 | hint = |
| 1171 | llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: llvmContext), |
| 1172 | V: static_cast<int>(criticalDeclareOp.getHint())); |
| 1173 | } |
| 1174 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 1175 | getOpenMPBuilder(moduleTranslation) |
| 1176 | ->createCritical(Loc: ompLoc, BodyGenCB: bodyGenCB, FiniCB: finiCB, |
| 1177 | CriticalName: criticalOp.getName().value_or(u: "" ), HintInst: hint); |
| 1178 | |
| 1179 | if (failed(Result: handleError(result&: afterIP, op&: opInst))) |
| 1180 | return failure(); |
| 1181 | |
| 1182 | builder.restoreIP(IP: *afterIP); |
| 1183 | return success(); |
| 1184 | } |
| 1185 | |
| 1186 | namespace { |
| 1187 | /// A util to collect info needed to convert delayed privatizers from MLIR to |
| 1188 | /// LLVM. |
| 1189 | struct PrivateVarsInfo { |
| 1190 | /// Per-variable data for a single delayed privatizer. |
| 1191 | struct VarInfo { |
| 1192 | BlockArgument blockArg; |
| 1193 | mlir::Value mlirVar; |
| 1194 | omp::PrivateClauseOp privatizer; |
| 1195 | llvm::Value *llvmVar = nullptr; |
| 1196 | }; |
| 1197 | |
| 1198 | struct AllocatorInfo { |
| 1199 | llvm::Value *allocatedPtr; |
| 1200 | llvm::Value *allocator; |
| 1201 | }; |
| 1202 | |
| 1203 | template <typename OP> |
| 1204 | PrivateVarsInfo(OP op) { |
| 1205 | MutableArrayRef<BlockArgument> blockArgs = |
| 1206 | cast<omp::BlockArgOpenMPOpInterface>(*op).getPrivateBlockArgs(); |
| 1207 | SmallVector<omp::PrivateClauseOp> privatizers; |
| 1208 | |
| 1209 | collectPrivatizationDecls<OP>(op, op.getPrivateSyms(), privatizers); |
| 1210 | collectPrivatizationInfo(blockArgs, privateVars: op.getPrivateVars(), privatizers); |
| 1211 | numPrivateVars = vars.size(); |
| 1212 | } |
| 1213 | |
| 1214 | /// Adds the linear variables of `op`, which are privatized by their |
| 1215 | /// `linear` privatizers, after the private variables. |
| 1216 | template <typename OP> |
| 1217 | void addLinearVars(OP op) { |
| 1218 | MutableArrayRef<BlockArgument> blockArgs = |
| 1219 | cast<omp::BlockArgOpenMPOpInterface>(*op).getLinearBlockArgs(); |
| 1220 | SmallVector<omp::PrivateClauseOp> privatizers; |
| 1221 | |
| 1222 | collectPrivatizationDecls<OP>(op, op.getLinearSyms(), privatizers); |
| 1223 | collectPrivatizationInfo(blockArgs, privateVars: op.getLinearVars(), privatizers); |
| 1224 | } |
| 1225 | |
| 1226 | /// Number of private variables before the linear ones. |
| 1227 | size_t numPrivateVars = 0; |
| 1228 | SmallVector<VarInfo> vars; |
| 1229 | llvm::DenseMap<Value, llvm::Value *> convertedAllocators; |
| 1230 | SmallVector<AllocatorInfo> allocatorPrivates; |
| 1231 | |
| 1232 | SmallVector<llvm::Value *> getLlvmVars() const { |
| 1233 | return llvm::map_to_vector(C: vars, |
| 1234 | F: [](const VarInfo &v) { return v.llvmVar; }); |
| 1235 | } |
| 1236 | |
| 1237 | private: |
| 1238 | /// Adds the provided privatization information to `vars`. |
| 1239 | void collectPrivatizationInfo( |
| 1240 | MutableArrayRef<BlockArgument> blockArgs, OperandRange privateVars, |
| 1241 | const SmallVectorImpl<omp::PrivateClauseOp> &privatizers) { |
| 1242 | vars.reserve(N: vars.size() + blockArgs.size()); |
| 1243 | for (auto [blockArg, mlirVar, privatizer] : |
| 1244 | llvm::zip_equal(t&: blockArgs, u&: privateVars, args: privatizers)) |
| 1245 | vars.push_back(Elt: {.blockArg: blockArg, .mlirVar: mlirVar, .privatizer: privatizer}); |
| 1246 | } |
| 1247 | |
| 1248 | /// Populates `privatizers` with privatization declarations the provided |
| 1249 | /// `symsAttr` points to. |
| 1250 | template <class OP> |
| 1251 | void collectPrivatizationDecls( |
| 1252 | OP op, std::optional<ArrayAttr> symsAttr, |
| 1253 | SmallVectorImpl<omp::PrivateClauseOp> &privatizers) { |
| 1254 | if (!symsAttr) |
| 1255 | return; |
| 1256 | |
| 1257 | privatizers.reserve(N: privatizers.size() + symsAttr->size()); |
| 1258 | llvm::transform( |
| 1259 | symsAttr->getAsRange<SymbolRefAttr>(), std::back_inserter(x&: privatizers), |
| 1260 | [&](SymbolRefAttr symbolRef) { return findPrivatizer(op, symbolRef); }); |
| 1261 | } |
| 1262 | }; |
| 1263 | } // namespace |
| 1264 | |
| 1265 | /// Populates `reductions` with reduction declarations used in the given op. |
| 1266 | template <typename T> |
| 1267 | static void |
| 1268 | collectReductionDecls(T op, |
| 1269 | SmallVectorImpl<omp::DeclareReductionOp> &reductions) { |
| 1270 | std::optional<ArrayAttr> attr = op.getReductionSyms(); |
| 1271 | if (!attr) |
| 1272 | return; |
| 1273 | |
| 1274 | reductions.reserve(N: reductions.size() + op.getNumReductionVars()); |
| 1275 | for (auto symbolRef : attr->getAsRange<SymbolRefAttr>()) { |
| 1276 | reductions.push_back( |
| 1277 | Elt: SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>( |
| 1278 | op, symbolRef)); |
| 1279 | } |
| 1280 | } |
| 1281 | |
| 1282 | /// Look up and validate the declare_reduction ops referenced by a |
| 1283 | /// reduction-like clause on the omp.taskloop.context translation path. Only |
| 1284 | /// the non-byref, single-init-arg, no-cleanup form is supported in this |
| 1285 | /// initial cut; richer shapes are rejected here with a diagnostic. \p syms |
| 1286 | /// is the clause's symbol list (e.g. `getReductionSyms()` or |
| 1287 | /// `getInReductionSyms()`), \p opName is the textual op name used in |
| 1288 | /// diagnostics, and \p clauseName distinguishes "reduction" from |
| 1289 | /// "in_reduction" in those diagnostics. |
| 1290 | static LogicalResult collectAndValidateTaskloopRedDecls( |
| 1291 | Operation *contextOp, std::optional<ArrayAttr> syms, StringRef opName, |
| 1292 | StringRef clauseName, SmallVectorImpl<omp::DeclareReductionOp> &out) { |
| 1293 | if (!syms) |
| 1294 | return success(); |
| 1295 | out.reserve(N: out.size() + syms->size()); |
| 1296 | for (auto sym : syms->getAsRange<SymbolRefAttr>()) { |
| 1297 | auto decl = SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>( |
| 1298 | from: contextOp, symbol: sym); |
| 1299 | if (!decl) |
| 1300 | return contextOp->emitError() |
| 1301 | << "failed to resolve " << clauseName |
| 1302 | << " declare_reduction symbol " << sym.getRootReference() << " in " |
| 1303 | << opName; |
| 1304 | if (decl.getInitializerRegion().front().getNumArguments() != 1) |
| 1305 | return contextOp->emitError() |
| 1306 | << "not yet implemented: " << clauseName |
| 1307 | << " with two-argument initializer in " << opName; |
| 1308 | if (!decl.getCleanupRegion().empty()) |
| 1309 | return contextOp->emitError() << "not yet implemented: " << clauseName |
| 1310 | << " with cleanup region in " << opName; |
| 1311 | if (decl.getReductionRegion().empty()) |
| 1312 | return contextOp->emitError() |
| 1313 | << clauseName << " declare_reduction is missing a combiner region" ; |
| 1314 | out.push_back(Elt: decl); |
| 1315 | } |
| 1316 | return success(); |
| 1317 | } |
| 1318 | |
| 1319 | /// Translates the blocks contained in the given region and appends them to at |
| 1320 | /// the current insertion point of `builder`. The operations of the entry block |
| 1321 | /// are appended to the current insertion block. If set, `continuationBlockArgs` |
| 1322 | /// is populated with translated values that correspond to the values |
| 1323 | /// omp.yield'ed from the region. |
| 1324 | static LogicalResult inlineConvertOmpRegions( |
| 1325 | Region ®ion, StringRef blockName, llvm::IRBuilderBase &builder, |
| 1326 | LLVM::ModuleTranslation &moduleTranslation, |
| 1327 | SmallVectorImpl<llvm::Value *> *continuationBlockArgs = nullptr) { |
| 1328 | if (region.empty()) |
| 1329 | return success(); |
| 1330 | |
| 1331 | // Special case for single-block regions that don't create additional blocks: |
| 1332 | // insert operations without creating additional blocks. |
| 1333 | if (region.hasOneBlock()) { |
| 1334 | llvm::BasicBlock *insertBlock = builder.GetInsertBlock(); |
| 1335 | llvm::Instruction *potentialTerminator = |
| 1336 | insertBlock->empty() ? nullptr : &insertBlock->back(); |
| 1337 | |
| 1338 | if (potentialTerminator && potentialTerminator->isTerminator()) { |
| 1339 | assert(builder.GetInsertPoint() == potentialTerminator->getIterator() && |
| 1340 | "Insertion point should be the terminator" ); |
| 1341 | potentialTerminator->removeFromParent(); |
| 1342 | // Reset insertion point to the end of the block. |
| 1343 | builder.SetInsertPoint(insertBlock); |
| 1344 | } |
| 1345 | moduleTranslation.mapBlock(mlir: ®ion.front(), llvm: insertBlock); |
| 1346 | |
| 1347 | // convertBlock leaves the location of the region's last op current. Restore |
| 1348 | // the caller's, as convertOmpOpRegions does for every block it converts. |
| 1349 | llvm::DebugLoc loc = builder.getCurrentDebugLocation(); |
| 1350 | if (failed(Result: moduleTranslation.convertBlock( |
| 1351 | bb&: region.front(), /*ignoreArguments=*/true, builder))) |
| 1352 | return failure(); |
| 1353 | builder.SetCurrentDebugLocation(loc); |
| 1354 | |
| 1355 | // The continuation arguments are simply the translated terminator operands. |
| 1356 | if (continuationBlockArgs) |
| 1357 | llvm::append_range( |
| 1358 | C&: *continuationBlockArgs, |
| 1359 | R: moduleTranslation.lookupValues(values: region.front().back().getOperands())); |
| 1360 | |
| 1361 | // Drop the mapping that is no longer necessary so that the same region can |
| 1362 | // be processed multiple times. |
| 1363 | moduleTranslation.forgetMapping(region); |
| 1364 | |
| 1365 | if (potentialTerminator && potentialTerminator->isTerminator()) { |
| 1366 | llvm::BasicBlock *block = builder.GetInsertBlock(); |
| 1367 | if (block->empty()) { |
| 1368 | // this can happen for really simple reduction init regions e.g. |
| 1369 | // %0 = llvm.mlir.constant(0 : i32) : i32 |
| 1370 | // omp.yield(%0 : i32) |
| 1371 | // because the llvm.mlir.constant (MLIR op) isn't converted into any |
| 1372 | // llvm op |
| 1373 | potentialTerminator->insertInto(ParentBB: block, It: block->begin()); |
| 1374 | } else { |
| 1375 | potentialTerminator->insertAfter(InsertPos: &block->back()); |
| 1376 | } |
| 1377 | } |
| 1378 | |
| 1379 | return success(); |
| 1380 | } |
| 1381 | |
| 1382 | SmallVector<llvm::PHINode *> phis; |
| 1383 | llvm::Expected<llvm::BasicBlock *> continuationBlock = |
| 1384 | convertOmpOpRegions(region, blockName, builder, moduleTranslation, continuationBlockPHIs: &phis); |
| 1385 | |
| 1386 | if (failed(Result: handleError(result&: continuationBlock, op&: *region.getParentOp()))) |
| 1387 | return failure(); |
| 1388 | |
| 1389 | if (continuationBlockArgs) |
| 1390 | llvm::append_range(C&: *continuationBlockArgs, R&: phis); |
| 1391 | builder.SetInsertPoint((*continuationBlock)->getFirstInsertionPt()); |
| 1392 | return success(); |
| 1393 | } |
| 1394 | |
| 1395 | namespace { |
| 1396 | /// Owning equivalents of OpenMPIRBuilder::(Atomic)ReductionGen that are used to |
| 1397 | /// store lambdas with capture. |
| 1398 | using OwningReductionGen = |
| 1399 | std::function<llvm::OpenMPIRBuilder::InsertPointOrErrorTy( |
| 1400 | llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *, |
| 1401 | llvm::Value *&)>; |
| 1402 | using OwningAtomicReductionGen = |
| 1403 | std::function<llvm::OpenMPIRBuilder::InsertPointOrErrorTy( |
| 1404 | llvm::OpenMPIRBuilder::InsertPointTy, llvm::Type *, llvm::Value *, |
| 1405 | llvm::Value *)>; |
| 1406 | using OwningDataPtrPtrReductionGen = |
| 1407 | std::function<llvm::OpenMPIRBuilder::InsertPointOrErrorTy( |
| 1408 | llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *&)>; |
| 1409 | } // namespace |
| 1410 | |
| 1411 | /// Create an OpenMPIRBuilder-compatible reduction generator for the given |
| 1412 | /// reduction declaration. The generator uses `builder` but ignores its |
| 1413 | /// insertion point. |
| 1414 | static OwningReductionGen |
| 1415 | makeReductionGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder, |
| 1416 | LLVM::ModuleTranslation &moduleTranslation) { |
| 1417 | // The lambda is mutable because we need access to non-const methods of decl |
| 1418 | // (which aren't actually mutating it), and we must capture decl by-value to |
| 1419 | // avoid the dangling reference after the parent function returns. |
| 1420 | OwningReductionGen gen = |
| 1421 | [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, |
| 1422 | llvm::Value *lhs, llvm::Value *rhs, |
| 1423 | llvm::Value *&result) mutable |
| 1424 | -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { |
| 1425 | moduleTranslation.mapValue(mlir: decl.getReductionLhsArg(), llvm: lhs); |
| 1426 | moduleTranslation.mapValue(mlir: decl.getReductionRhsArg(), llvm: rhs); |
| 1427 | builder.restoreIP(IP: insertPoint); |
| 1428 | SmallVector<llvm::Value *> phis; |
| 1429 | if (failed(Result: inlineConvertOmpRegions(region&: decl.getReductionRegion(), |
| 1430 | blockName: "omp.reduction.nonatomic.body" , builder, |
| 1431 | moduleTranslation, continuationBlockArgs: &phis))) |
| 1432 | return llvm::createStringError( |
| 1433 | Fmt: "failed to inline `combiner` region of `omp.declare_reduction`" ); |
| 1434 | result = llvm::getSingleElement(C&: phis); |
| 1435 | return builder.saveIP(); |
| 1436 | }; |
| 1437 | return gen; |
| 1438 | } |
| 1439 | |
| 1440 | /// Create an OpenMPIRBuilder-compatible atomic reduction generator for the |
| 1441 | /// given reduction declaration. The generator uses `builder` but ignores its |
| 1442 | /// insertion point. Returns null if there is no atomic region available in the |
| 1443 | /// reduction declaration. |
| 1444 | static OwningAtomicReductionGen |
| 1445 | makeAtomicReductionGen(omp::DeclareReductionOp decl, |
| 1446 | llvm::IRBuilderBase &builder, |
| 1447 | LLVM::ModuleTranslation &moduleTranslation) { |
| 1448 | if (decl.getAtomicReductionRegion().empty()) |
| 1449 | return OwningAtomicReductionGen(); |
| 1450 | |
| 1451 | // The lambda is mutable because we need access to non-const methods of decl |
| 1452 | // (which aren't actually mutating it), and we must capture decl by-value to |
| 1453 | // avoid the dangling reference after the parent function returns. |
| 1454 | OwningAtomicReductionGen atomicGen = |
| 1455 | [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *, |
| 1456 | llvm::Value *lhs, llvm::Value *rhs) mutable |
| 1457 | -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { |
| 1458 | moduleTranslation.mapValue(mlir: decl.getAtomicReductionLhsArg(), llvm: lhs); |
| 1459 | moduleTranslation.mapValue(mlir: decl.getAtomicReductionRhsArg(), llvm: rhs); |
| 1460 | builder.restoreIP(IP: insertPoint); |
| 1461 | SmallVector<llvm::Value *> phis; |
| 1462 | if (failed(Result: inlineConvertOmpRegions(region&: decl.getAtomicReductionRegion(), |
| 1463 | blockName: "omp.reduction.atomic.body" , builder, |
| 1464 | moduleTranslation, continuationBlockArgs: &phis))) |
| 1465 | return llvm::createStringError( |
| 1466 | Fmt: "failed to inline `atomic` region of `omp.declare_reduction`" ); |
| 1467 | assert(phis.empty()); |
| 1468 | return builder.saveIP(); |
| 1469 | }; |
| 1470 | return atomicGen; |
| 1471 | } |
| 1472 | |
| 1473 | /// Create an OpenMPIRBuilder-compatible `data_ptr_ptr` reduction generator for |
| 1474 | /// the given reduction declaration. The generator uses `builder` but ignores |
| 1475 | /// its insertion point. Returns null if there is no `data_ptr_ptr` region |
| 1476 | /// available in the reduction declaration. |
| 1477 | static OwningDataPtrPtrReductionGen |
| 1478 | makeRefDataPtrGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder, |
| 1479 | LLVM::ModuleTranslation &moduleTranslation, bool isByRef) { |
| 1480 | if (!isByRef || decl.getDataPtrPtrRegion().empty()) |
| 1481 | return OwningDataPtrPtrReductionGen(); |
| 1482 | |
| 1483 | OwningDataPtrPtrReductionGen refDataPtrGen = |
| 1484 | [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, |
| 1485 | llvm::Value *byRefVal, llvm::Value *&result) mutable |
| 1486 | -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { |
| 1487 | moduleTranslation.mapValue(mlir: decl.getDataPtrPtrRegionArg(), llvm: byRefVal); |
| 1488 | builder.restoreIP(IP: insertPoint); |
| 1489 | SmallVector<llvm::Value *> phis; |
| 1490 | if (failed(Result: inlineConvertOmpRegions(region&: decl.getDataPtrPtrRegion(), |
| 1491 | blockName: "omp.data_ptr_ptr.body" , builder, |
| 1492 | moduleTranslation, continuationBlockArgs: &phis))) |
| 1493 | return llvm::createStringError( |
| 1494 | Fmt: "failed to inline `data_ptr_ptr` region of `omp.declare_reduction`" ); |
| 1495 | result = llvm::getSingleElement(C&: phis); |
| 1496 | return builder.saveIP(); |
| 1497 | }; |
| 1498 | |
| 1499 | return refDataPtrGen; |
| 1500 | } |
| 1501 | |
| 1502 | /// Converts an OpenMP 'ordered' operation into LLVM IR using OpenMPIRBuilder. |
| 1503 | static LogicalResult |
| 1504 | convertOmpOrdered(Operation &opInst, llvm::IRBuilderBase &builder, |
| 1505 | LLVM::ModuleTranslation &moduleTranslation) { |
| 1506 | auto orderedOp = cast<omp::OrderedOp>(Val&: opInst); |
| 1507 | |
| 1508 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 1509 | return failure(); |
| 1510 | |
| 1511 | omp::ClauseDepend dependType = *orderedOp.getDoacrossDependType(); |
| 1512 | bool isDependSource = dependType == omp::ClauseDepend::dependsource; |
| 1513 | unsigned numLoops = *orderedOp.getDoacrossNumLoops(); |
| 1514 | SmallVector<llvm::Value *> vecValues = |
| 1515 | moduleTranslation.lookupValues(values: orderedOp.getDoacrossDependVars()); |
| 1516 | |
| 1517 | size_t indexVecValues = 0; |
| 1518 | while (indexVecValues < vecValues.size()) { |
| 1519 | SmallVector<llvm::Value *> storeValues; |
| 1520 | storeValues.reserve(N: numLoops); |
| 1521 | for (unsigned i = 0; i < numLoops; i++) { |
| 1522 | storeValues.push_back(Elt: vecValues[indexVecValues]); |
| 1523 | indexVecValues++; |
| 1524 | } |
| 1525 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 1526 | findAllocInsertPoints(builder, moduleTranslation); |
| 1527 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 1528 | builder.restoreIP(IP: getOpenMPBuilder(moduleTranslation) |
| 1529 | ->createOrderedDepend(Loc: ompLoc, AllocaIP: allocaIP, NumLoops: numLoops, |
| 1530 | StoreValues: storeValues, Name: ".cnt.addr" , |
| 1531 | IsDependSource: isDependSource)); |
| 1532 | } |
| 1533 | return success(); |
| 1534 | } |
| 1535 | |
| 1536 | /// Converts an OpenMP 'ordered_region' operation into LLVM IR using |
| 1537 | /// OpenMPIRBuilder. |
| 1538 | static LogicalResult |
| 1539 | convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder, |
| 1540 | LLVM::ModuleTranslation &moduleTranslation) { |
| 1541 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 1542 | auto orderedRegionOp = cast<omp::OrderedRegionOp>(Val&: opInst); |
| 1543 | |
| 1544 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 1545 | return failure(); |
| 1546 | |
| 1547 | auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP, |
| 1548 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) { |
| 1549 | // OrderedOp has only one region associated with it. |
| 1550 | auto ®ion = cast<omp::OrderedRegionOp>(Val&: opInst).getRegion(); |
| 1551 | builder.restoreIP(IP: codeGenIP); |
| 1552 | return convertOmpOpRegions(region, blockName: "omp.ordered.region" , builder, |
| 1553 | moduleTranslation) |
| 1554 | .takeError(); |
| 1555 | }; |
| 1556 | |
| 1557 | // TODO: Perform finalization actions for variables. This has to be |
| 1558 | // called for variables which have destructors/finalizers. |
| 1559 | auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; |
| 1560 | |
| 1561 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 1562 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 1563 | getOpenMPBuilder(moduleTranslation) |
| 1564 | ->createOrderedThreadsSimd(Loc: ompLoc, BodyGenCB: bodyGenCB, FiniCB: finiCB, |
| 1565 | IsThreads: !orderedRegionOp.getParLevelSimd()); |
| 1566 | |
| 1567 | if (failed(Result: handleError(result&: afterIP, op&: opInst))) |
| 1568 | return failure(); |
| 1569 | |
| 1570 | builder.restoreIP(IP: *afterIP); |
| 1571 | return success(); |
| 1572 | } |
| 1573 | |
| 1574 | namespace { |
| 1575 | /// A util to collect info needed to translate the reductions of an OpenMP |
| 1576 | /// construct from MLIR to LLVM. Mirrors `PrivateVarsInfo` for delayed |
| 1577 | /// privatizers. |
| 1578 | struct ReductionVarsInfo { |
| 1579 | /// Per-variable data for a single reduction. |
| 1580 | struct VarInfo { |
| 1581 | BlockArgument blockArg; |
| 1582 | mlir::Value mlirVar; |
| 1583 | omp::DeclareReductionOp decl; |
| 1584 | bool byRef; |
| 1585 | llvm::Value *privateVar = nullptr; |
| 1586 | llvm::Value *storage = nullptr; |
| 1587 | }; |
| 1588 | |
| 1589 | /// Contains the arguments for an LLVM store operation. |
| 1590 | struct DeferredStore { |
| 1591 | DeferredStore(llvm::Value *value, llvm::Value *address) |
| 1592 | : value(value), address(address) {} |
| 1593 | |
| 1594 | llvm::Value *value; |
| 1595 | llvm::Value *address; |
| 1596 | }; |
| 1597 | |
| 1598 | template <typename OP> |
| 1599 | ReductionVarsInfo(OP op) { |
| 1600 | MutableArrayRef<BlockArgument> blockArgs = |
| 1601 | cast<omp::BlockArgOpenMPOpInterface>(*op).getReductionBlockArgs(); |
| 1602 | SmallVector<omp::DeclareReductionOp> decls; |
| 1603 | collectReductionDecls(op, decls); |
| 1604 | std::optional<ArrayRef<bool>> attr = op.getReductionByref(); |
| 1605 | ArrayRef<bool> byRefs = attr ? *attr : ArrayRef<bool>(); |
| 1606 | |
| 1607 | vars.reserve(N: blockArgs.size()); |
| 1608 | for (auto [blockArg, mlirVar, decl, byRef] : |
| 1609 | llvm::zip_equal(blockArgs, op.getReductionVars(), decls, byRefs)) |
| 1610 | vars.push_back(Elt: {blockArg, mlirVar, decl, byRef}); |
| 1611 | } |
| 1612 | |
| 1613 | SmallVector<VarInfo> vars; |
| 1614 | /// Stores deferred until after all allocas (see `allocReductionVars`). |
| 1615 | SmallVector<DeferredStore> deferredStores; |
| 1616 | |
| 1617 | size_t size() const { return vars.size(); } |
| 1618 | |
| 1619 | SmallVector<omp::DeclareReductionOp> getReductionDecls() const { |
| 1620 | return llvm::map_to_vector(C: vars, F: [](const VarInfo &v) { return v.decl; }); |
| 1621 | } |
| 1622 | SmallVector<llvm::Value *> getPrivateReductionVariables() const { |
| 1623 | return llvm::map_to_vector(C: vars, |
| 1624 | F: [](const VarInfo &v) { return v.privateVar; }); |
| 1625 | } |
| 1626 | SmallVector<bool> getIsByRef() const { |
| 1627 | return llvm::map_to_vector(C: vars, F: [](const VarInfo &v) { return v.byRef; }); |
| 1628 | } |
| 1629 | }; |
| 1630 | } // namespace |
| 1631 | |
| 1632 | /// Allocate space for privatized reduction variables. |
| 1633 | /// `deferredStores` contains information to create store operations which needs |
| 1634 | /// to be inserted after all allocas |
| 1635 | static LogicalResult |
| 1636 | allocReductionVars(Operation &op, llvm::IRBuilderBase &builder, |
| 1637 | LLVM::ModuleTranslation &moduleTranslation, |
| 1638 | const llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, |
| 1639 | ReductionVarsInfo &redInfo) { |
| 1640 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 1641 | builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator()); |
| 1642 | |
| 1643 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 1644 | bool useDeviceSharedMem = omp::opInSharedDeviceContext(op); |
| 1645 | |
| 1646 | // delay creating stores until after all allocas |
| 1647 | redInfo.deferredStores.reserve(N: redInfo.size()); |
| 1648 | |
| 1649 | for (ReductionVarsInfo::VarInfo &var : redInfo.vars) { |
| 1650 | Region &allocRegion = var.decl.getAllocRegion(); |
| 1651 | if (var.byRef) { |
| 1652 | if (allocRegion.empty()) |
| 1653 | continue; |
| 1654 | |
| 1655 | SmallVector<llvm::Value *, 1> phis; |
| 1656 | if (failed(Result: inlineConvertOmpRegions(region&: allocRegion, blockName: "omp.reduction.alloc" , |
| 1657 | builder, moduleTranslation, continuationBlockArgs: &phis))) |
| 1658 | return op.emitError( |
| 1659 | message: "failed to inline `alloc` region of `omp.declare_reduction`" ); |
| 1660 | |
| 1661 | assert(phis.size() == 1 && "expected one allocation to be yielded" ); |
| 1662 | builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator()); |
| 1663 | |
| 1664 | // Allocate reduction variable (which is a pointer to the real reduction |
| 1665 | // variable allocated in the inlined region) |
| 1666 | llvm::Type *ptrTy = builder.getPtrTy(); |
| 1667 | llvm::Type *varTy = moduleTranslation.convertType(type: var.decl.getType()); |
| 1668 | llvm::Value *llvmVar; |
| 1669 | if (useDeviceSharedMem) { |
| 1670 | llvmVar = ompBuilder->createOMPAllocShared(Loc: builder, VarType: varTy); |
| 1671 | } else { |
| 1672 | llvmVar = builder.CreateAlloca(Ty: varTy); |
| 1673 | llvmVar = builder.CreatePointerBitCastOrAddrSpaceCast(V: llvmVar, DestTy: ptrTy); |
| 1674 | } |
| 1675 | |
| 1676 | llvm::Value *castPhi = |
| 1677 | builder.CreatePointerBitCastOrAddrSpaceCast(V: phis[0], DestTy: ptrTy); |
| 1678 | |
| 1679 | redInfo.deferredStores.emplace_back(Args&: castPhi, Args&: llvmVar); |
| 1680 | |
| 1681 | var.privateVar = llvmVar; |
| 1682 | moduleTranslation.mapValue(mlir: var.blockArg, llvm: castPhi); |
| 1683 | var.storage = castPhi; |
| 1684 | } else { |
| 1685 | assert(allocRegion.empty() && |
| 1686 | "allocaction is implicit for by-val reduction" ); |
| 1687 | |
| 1688 | llvm::Type *ptrTy = builder.getPtrTy(); |
| 1689 | llvm::Type *varTy = moduleTranslation.convertType(type: var.decl.getType()); |
| 1690 | llvm::Value *llvmVar; |
| 1691 | if (useDeviceSharedMem) { |
| 1692 | llvmVar = ompBuilder->createOMPAllocShared(Loc: builder, VarType: varTy); |
| 1693 | } else { |
| 1694 | llvmVar = builder.CreateAlloca(Ty: varTy); |
| 1695 | llvmVar = builder.CreatePointerBitCastOrAddrSpaceCast(V: llvmVar, DestTy: ptrTy); |
| 1696 | } |
| 1697 | |
| 1698 | var.privateVar = llvmVar; |
| 1699 | moduleTranslation.mapValue(mlir: var.blockArg, llvm: llvmVar); |
| 1700 | var.storage = llvmVar; |
| 1701 | } |
| 1702 | } |
| 1703 | |
| 1704 | return success(); |
| 1705 | } |
| 1706 | |
| 1707 | /// Map input arguments to reduction initialization region |
| 1708 | static void |
| 1709 | mapInitializationArgs(LLVM::ModuleTranslation &moduleTranslation, |
| 1710 | llvm::IRBuilderBase &builder, |
| 1711 | const ReductionVarsInfo::VarInfo &redVarInfo) { |
| 1712 | // map input argument to the initialization region |
| 1713 | omp::DeclareReductionOp reduction = redVarInfo.decl; |
| 1714 | Region &initializerRegion = reduction.getInitializerRegion(); |
| 1715 | Block &entry = initializerRegion.front(); |
| 1716 | |
| 1717 | mlir::Value mlirSource = redVarInfo.mlirVar; |
| 1718 | llvm::Value *llvmSource = moduleTranslation.lookupValue(value: mlirSource); |
| 1719 | llvm::Value *origVal = llvmSource; |
| 1720 | // If a non-pointer value is expected, load the value from the source pointer. |
| 1721 | if (!isa<LLVM::LLVMPointerType>( |
| 1722 | Val: reduction.getInitializerMoldArg().getType()) && |
| 1723 | isa<LLVM::LLVMPointerType>(Val: mlirSource.getType())) { |
| 1724 | origVal = |
| 1725 | builder.CreateLoad(Ty: moduleTranslation.convertType( |
| 1726 | type: reduction.getInitializerMoldArg().getType()), |
| 1727 | Ptr: llvmSource, Name: "omp_orig" ); |
| 1728 | } |
| 1729 | moduleTranslation.mapValue(mlir: reduction.getInitializerMoldArg(), llvm: origVal); |
| 1730 | |
| 1731 | if (entry.getNumArguments() > 1) |
| 1732 | moduleTranslation.mapValue(mlir: reduction.getInitializerAllocArg(), |
| 1733 | llvm: redVarInfo.storage); |
| 1734 | } |
| 1735 | |
| 1736 | static void |
| 1737 | setInsertPointForPossiblyEmptyBlock(llvm::IRBuilderBase &builder, |
| 1738 | llvm::BasicBlock *block = nullptr) { |
| 1739 | if (block == nullptr) |
| 1740 | block = builder.GetInsertBlock(); |
| 1741 | |
| 1742 | if (!block->hasTerminator()) |
| 1743 | builder.SetInsertPoint(block); |
| 1744 | else |
| 1745 | builder.SetInsertPoint(block->getTerminator()); |
| 1746 | } |
| 1747 | |
| 1748 | /// Inline reductions' `init` regions. This functions assumes that the |
| 1749 | /// `builder`'s insertion point is where the user wants the `init` regions to be |
| 1750 | /// inlined; i.e. it does not try to find a proper insertion location for the |
| 1751 | /// `init` regions. It also leaves the `builder's insertions point in a state |
| 1752 | /// where the user can continue the code-gen directly afterwards. |
| 1753 | static LogicalResult |
| 1754 | initReductionVars(Operation &op, llvm::IRBuilderBase &builder, |
| 1755 | LLVM::ModuleTranslation &moduleTranslation, |
| 1756 | llvm::BasicBlock *latestAllocaBlock, |
| 1757 | ReductionVarsInfo &redInfo) { |
| 1758 | if (redInfo.size() == 0) |
| 1759 | return success(); |
| 1760 | |
| 1761 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 1762 | bool useDeviceSharedMem = omp::opInSharedDeviceContext(op); |
| 1763 | |
| 1764 | llvm::BasicBlock *initBlock = splitBB(Builder&: builder, CreateBranch: true, Name: "omp.reduction.init" ); |
| 1765 | SmallVector<llvm::Value *> byRefVars(redInfo.size()); |
| 1766 | { |
| 1767 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 1768 | auto allocaIP = latestAllocaBlock->getTerminator()->getIterator(); |
| 1769 | builder.restoreIP(IP: allocaIP); |
| 1770 | |
| 1771 | for (auto [var, byRefVar] : llvm::zip_equal(t&: redInfo.vars, u&: byRefVars)) { |
| 1772 | if (var.byRef) { |
| 1773 | if (!var.decl.getAllocRegion().empty()) |
| 1774 | continue; |
| 1775 | |
| 1776 | // TODO: remove after all users of by-ref are updated to use the alloc |
| 1777 | // region: Allocate reduction variable (which is a pointer to the real |
| 1778 | // reduciton variable allocated in the inlined region) |
| 1779 | llvm::Type *varTy = moduleTranslation.convertType(type: var.decl.getType()); |
| 1780 | if (useDeviceSharedMem) |
| 1781 | byRefVar = ompBuilder->createOMPAllocShared(Loc: builder, VarType: varTy); |
| 1782 | else |
| 1783 | byRefVar = builder.CreateAlloca(Ty: varTy); |
| 1784 | } |
| 1785 | } |
| 1786 | } |
| 1787 | |
| 1788 | setInsertPointForPossiblyEmptyBlock(builder, block: initBlock); |
| 1789 | |
| 1790 | // store result of the alloc region to the allocated pointer to the real |
| 1791 | // reduction variable |
| 1792 | for (auto [data, addr] : redInfo.deferredStores) |
| 1793 | builder.CreateStore(Val: data, Ptr: addr); |
| 1794 | |
| 1795 | // Before the loop, store the initial values of reductions into reduction |
| 1796 | // variables. Although this could be done after allocas, we don't want to mess |
| 1797 | // up with the alloca insertion point. |
| 1798 | for (auto [var, byRefVar] : llvm::zip_equal(t&: redInfo.vars, u&: byRefVars)) { |
| 1799 | SmallVector<llvm::Value *, 1> phis; |
| 1800 | |
| 1801 | // map block argument to initializer region |
| 1802 | mapInitializationArgs(moduleTranslation, builder, redVarInfo: var); |
| 1803 | |
| 1804 | // TODO In some cases (specially on the GPU), the init regions may |
| 1805 | // contains stack alloctaions. If the region is inlined in a loop, this is |
| 1806 | // problematic. Instead of just inlining the region, handle allocations by |
| 1807 | // hoisting fixed length allocations to the function entry and using |
| 1808 | // stacksave and restore for variable length ones. |
| 1809 | if (failed(Result: inlineConvertOmpRegions(region&: var.decl.getInitializerRegion(), |
| 1810 | blockName: "omp.reduction.neutral" , builder, |
| 1811 | moduleTranslation, continuationBlockArgs: &phis))) |
| 1812 | return failure(); |
| 1813 | |
| 1814 | assert(phis.size() == 1 && "expected one value to be yielded from the " |
| 1815 | "reduction neutral element declaration region" ); |
| 1816 | |
| 1817 | setInsertPointForPossiblyEmptyBlock(builder); |
| 1818 | |
| 1819 | if (var.byRef) { |
| 1820 | if (!var.decl.getAllocRegion().empty()) |
| 1821 | // done in allocReductionVars |
| 1822 | continue; |
| 1823 | |
| 1824 | // TODO: this path can be removed once all users of by-ref are updated to |
| 1825 | // use an alloc region |
| 1826 | |
| 1827 | // Store the result of the inlined region to the allocated reduction var |
| 1828 | // ptr |
| 1829 | builder.CreateStore(Val: phis[0], Ptr: byRefVar); |
| 1830 | var.privateVar = byRefVar; |
| 1831 | |
| 1832 | moduleTranslation.mapValue(mlir: var.blockArg, llvm: phis[0]); |
| 1833 | var.storage = phis[0]; |
| 1834 | } else { |
| 1835 | // for by-ref case the store is inside of the reduction region |
| 1836 | builder.CreateStore(Val: phis[0], Ptr: var.privateVar); |
| 1837 | // the rest was handled in allocByValReductionVars |
| 1838 | } |
| 1839 | |
| 1840 | // forget the mapping for the initializer region because we might need a |
| 1841 | // different mapping if this reduction declaration is re-used for a |
| 1842 | // different variable |
| 1843 | moduleTranslation.forgetMapping(region&: var.decl.getInitializerRegion()); |
| 1844 | } |
| 1845 | |
| 1846 | return success(); |
| 1847 | } |
| 1848 | |
| 1849 | /// Collect reduction info |
| 1850 | static void collectReductionInfo( |
| 1851 | llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, |
| 1852 | ReductionVarsInfo &redInfo, |
| 1853 | SmallVectorImpl<OwningReductionGen> &owningReductionGens, |
| 1854 | SmallVectorImpl<OwningAtomicReductionGen> &owningAtomicReductionGens, |
| 1855 | SmallVector<OwningDataPtrPtrReductionGen> &owningDataPtrPtrReductionGens, |
| 1856 | SmallVectorImpl<llvm::OpenMPIRBuilder::ReductionInfo> &reductionInfos) { |
| 1857 | unsigned numReductions = redInfo.size(); |
| 1858 | |
| 1859 | for (ReductionVarsInfo::VarInfo &var : redInfo.vars) { |
| 1860 | owningReductionGens.push_back( |
| 1861 | Elt: makeReductionGen(decl: var.decl, builder, moduleTranslation)); |
| 1862 | owningAtomicReductionGens.push_back( |
| 1863 | Elt: makeAtomicReductionGen(decl: var.decl, builder, moduleTranslation)); |
| 1864 | owningDataPtrPtrReductionGens.push_back( |
| 1865 | Elt: makeRefDataPtrGen(decl: var.decl, builder, moduleTranslation, isByRef: var.byRef)); |
| 1866 | } |
| 1867 | |
| 1868 | // Collect the reduction information. |
| 1869 | reductionInfos.reserve(N: numReductions); |
| 1870 | for (auto [var, redGen, atomicRedGen, dataPtrPtrRedGen] : llvm::zip_equal( |
| 1871 | t&: redInfo.vars, u&: owningReductionGens, args&: owningAtomicReductionGens, |
| 1872 | args&: owningDataPtrPtrReductionGens)) { |
| 1873 | llvm::OpenMPIRBuilder::ReductionGenAtomicCBTy atomicGen = nullptr; |
| 1874 | if (atomicRedGen) |
| 1875 | atomicGen = atomicRedGen; |
| 1876 | llvm::Value *variable = moduleTranslation.lookupValue(value: var.mlirVar); |
| 1877 | mlir::Type allocatedType; |
| 1878 | var.decl.getAllocRegion().walk(callback: [&](mlir::Operation *op) { |
| 1879 | if (auto alloca = mlir::dyn_cast<LLVM::AllocaOp>(Val: op)) { |
| 1880 | allocatedType = alloca.getElemType(); |
| 1881 | return mlir::WalkResult::interrupt(); |
| 1882 | } |
| 1883 | |
| 1884 | return mlir::WalkResult::advance(); |
| 1885 | }); |
| 1886 | |
| 1887 | reductionInfos.push_back( |
| 1888 | Elt: {moduleTranslation.convertType(type: var.decl.getType()), variable, |
| 1889 | var.privateVar, |
| 1890 | /*EvaluationKind=*/llvm::OpenMPIRBuilder::EvalKind::Scalar, redGen, |
| 1891 | /*ReductionGenClang=*/nullptr, atomicGen, dataPtrPtrRedGen, |
| 1892 | allocatedType ? moduleTranslation.convertType(type: allocatedType) : nullptr, |
| 1893 | var.decl.getByrefElementType() |
| 1894 | ? moduleTranslation.convertType(type: *var.decl.getByrefElementType()) |
| 1895 | : nullptr}); |
| 1896 | } |
| 1897 | } |
| 1898 | |
| 1899 | /// handling of DeclareReductionOp's cleanup region |
| 1900 | static LogicalResult |
| 1901 | inlineOmpRegionCleanup(llvm::SmallVectorImpl<Region *> &cleanupRegions, |
| 1902 | llvm::ArrayRef<llvm::Value *> privateVariables, |
| 1903 | LLVM::ModuleTranslation &moduleTranslation, |
| 1904 | llvm::IRBuilderBase &builder, StringRef regionName, |
| 1905 | bool shouldLoadCleanupRegionArg = true) { |
| 1906 | for (auto [i, cleanupRegion] : llvm::enumerate(First&: cleanupRegions)) { |
| 1907 | if (cleanupRegion->empty()) |
| 1908 | continue; |
| 1909 | |
| 1910 | // map the argument to the cleanup region |
| 1911 | Block &entry = cleanupRegion->front(); |
| 1912 | |
| 1913 | llvm::Instruction *potentialTerminator = |
| 1914 | builder.GetInsertBlock()->empty() ? nullptr |
| 1915 | : &builder.GetInsertBlock()->back(); |
| 1916 | if (potentialTerminator && potentialTerminator->isTerminator()) |
| 1917 | builder.SetInsertPoint(potentialTerminator); |
| 1918 | llvm::Value *privateVarValue = |
| 1919 | shouldLoadCleanupRegionArg |
| 1920 | ? builder.CreateLoad( |
| 1921 | Ty: moduleTranslation.convertType(type: entry.getArgument(i: 0).getType()), |
| 1922 | Ptr: privateVariables[i]) |
| 1923 | : privateVariables[i]; |
| 1924 | |
| 1925 | moduleTranslation.mapValue(mlir: entry.getArgument(i: 0), llvm: privateVarValue); |
| 1926 | |
| 1927 | if (failed(Result: inlineConvertOmpRegions(region&: *cleanupRegion, blockName: regionName, builder, |
| 1928 | moduleTranslation))) |
| 1929 | return failure(); |
| 1930 | |
| 1931 | // clear block argument mapping in case it needs to be re-created with a |
| 1932 | // different source for another use of the same reduction decl |
| 1933 | moduleTranslation.forgetMapping(region&: *cleanupRegion); |
| 1934 | } |
| 1935 | return success(); |
| 1936 | } |
| 1937 | |
| 1938 | // TODO: not used by ParallelOp |
| 1939 | static LogicalResult |
| 1940 | createReductionsAndCleanup(Operation &op, llvm::IRBuilderBase &builder, |
| 1941 | LLVM::ModuleTranslation &moduleTranslation, |
| 1942 | llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, |
| 1943 | ReductionVarsInfo &redInfo, bool isNowait = false, |
| 1944 | bool isTeamsReduction = false) { |
| 1945 | // Process the reductions if required. |
| 1946 | if (redInfo.size() == 0) |
| 1947 | return success(); |
| 1948 | |
| 1949 | SmallVector<OwningReductionGen> owningReductionGens; |
| 1950 | SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens; |
| 1951 | SmallVector<OwningDataPtrPtrReductionGen> owningReductionGenRefDataPtrGens; |
| 1952 | SmallVector<llvm::OpenMPIRBuilder::ReductionInfo, 2> reductionInfos; |
| 1953 | |
| 1954 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 1955 | |
| 1956 | // Create the reduction generators. We need to own them here because |
| 1957 | // ReductionInfo only accepts references to the generators. |
| 1958 | collectReductionInfo(builder, moduleTranslation, redInfo, owningReductionGens, |
| 1959 | owningAtomicReductionGens, |
| 1960 | owningDataPtrPtrReductionGens&: owningReductionGenRefDataPtrGens, reductionInfos); |
| 1961 | |
| 1962 | // The call to createReductions below expects the block to have a |
| 1963 | // terminator. Create an unreachable instruction to serve as terminator |
| 1964 | // and remove it later. |
| 1965 | llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable(); |
| 1966 | builder.SetInsertPoint(tempTerminator); |
| 1967 | llvm::DebugLoc reductionLoc = builder.getCurrentDebugLocation(); |
| 1968 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy contInsertPoint = |
| 1969 | ompBuilder->createReductions(Loc: builder, AllocaIP: allocaIP, ReductionInfos: reductionInfos, |
| 1970 | IsByRef: redInfo.getIsByRef(), IsNoWait: isNowait, |
| 1971 | IsTeamsReduction: isTeamsReduction); |
| 1972 | |
| 1973 | if (failed(Result: handleError(result&: contInsertPoint, op))) |
| 1974 | return failure(); |
| 1975 | |
| 1976 | if (!contInsertPoint->isValid()) |
| 1977 | return op.emitOpError() << "failed to convert reductions" ; |
| 1978 | |
| 1979 | llvm::OpenMPIRBuilder::InsertPointTy afterIP = *contInsertPoint; |
| 1980 | if (!isTeamsReduction && !isNowait) { |
| 1981 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy barrierIP = |
| 1982 | ompBuilder->createBarrier(Loc: {*contInsertPoint, reductionLoc}, |
| 1983 | Kind: llvm::omp::OMPD_for); |
| 1984 | |
| 1985 | if (failed(Result: handleError(result&: barrierIP, op))) |
| 1986 | return failure(); |
| 1987 | afterIP = *barrierIP; |
| 1988 | } |
| 1989 | |
| 1990 | tempTerminator->eraseFromParent(); |
| 1991 | builder.restoreIP(IP: afterIP); |
| 1992 | |
| 1993 | // after the construct, deallocate private reduction variables |
| 1994 | SmallVector<Region *> reductionRegions; |
| 1995 | llvm::transform(Range&: redInfo.vars, d_first: std::back_inserter(x&: reductionRegions), |
| 1996 | F: [](ReductionVarsInfo::VarInfo &var) { |
| 1997 | return &var.decl.getCleanupRegion(); |
| 1998 | }); |
| 1999 | LogicalResult result = inlineOmpRegionCleanup( |
| 2000 | cleanupRegions&: reductionRegions, privateVariables: redInfo.getPrivateReductionVariables(), |
| 2001 | moduleTranslation, builder, regionName: "omp.reduction.cleanup" ); |
| 2002 | |
| 2003 | bool useDeviceSharedMem = omp::opInSharedDeviceContext(op); |
| 2004 | if (useDeviceSharedMem) { |
| 2005 | for (ReductionVarsInfo::VarInfo &var : redInfo.vars) |
| 2006 | ompBuilder->createOMPFreeShared( |
| 2007 | Loc: builder, Addr: var.privateVar, |
| 2008 | VarType: moduleTranslation.convertType(type: var.decl.getType())); |
| 2009 | } |
| 2010 | |
| 2011 | return result; |
| 2012 | } |
| 2013 | |
| 2014 | // TODO: not used by omp.parallel |
| 2015 | static LogicalResult |
| 2016 | allocAndInitializeReductionVars(Operation &op, llvm::IRBuilderBase &builder, |
| 2017 | LLVM::ModuleTranslation &moduleTranslation, |
| 2018 | llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, |
| 2019 | ReductionVarsInfo &redInfo) { |
| 2020 | if (redInfo.size() == 0) |
| 2021 | return success(); |
| 2022 | |
| 2023 | if (failed(Result: allocReductionVars(op, builder, moduleTranslation, allocaIP, |
| 2024 | redInfo))) |
| 2025 | return failure(); |
| 2026 | |
| 2027 | return initReductionVars(op, builder, moduleTranslation, |
| 2028 | latestAllocaBlock: allocaIP.getNodeParent(), redInfo); |
| 2029 | } |
| 2030 | |
| 2031 | /// Return the llvm::Value * corresponding to the `privateVar` that |
| 2032 | /// is being privatized. It isn't always as simple as looking up |
| 2033 | /// moduleTranslation with privateVar. For instance, in case of |
| 2034 | /// an allocatable, the descriptor for the allocatable is privatized. |
| 2035 | /// This descriptor is mapped using an MapInfoOp. So, this function |
| 2036 | /// will return a pointer to the llvm::Value corresponding to the |
| 2037 | /// block argument for the mapped descriptor. |
| 2038 | static llvm::Value * |
| 2039 | findAssociatedValue(Value privateVar, llvm::IRBuilderBase &builder, |
| 2040 | LLVM::ModuleTranslation &moduleTranslation, |
| 2041 | llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) { |
| 2042 | if (mappedPrivateVars == nullptr || !mappedPrivateVars->contains(Val: privateVar)) |
| 2043 | return moduleTranslation.lookupValue(value: privateVar); |
| 2044 | |
| 2045 | Value blockArg = (*mappedPrivateVars)[privateVar]; |
| 2046 | Type privVarType = privateVar.getType(); |
| 2047 | Type blockArgType = blockArg.getType(); |
| 2048 | assert(isa<LLVM::LLVMPointerType>(blockArgType) && |
| 2049 | "A block argument corresponding to a mapped var should have " |
| 2050 | "!llvm.ptr type" ); |
| 2051 | |
| 2052 | if (privVarType == blockArgType) |
| 2053 | return moduleTranslation.lookupValue(value: blockArg); |
| 2054 | |
| 2055 | // This typically happens when the privatized type is lowered from |
| 2056 | // boxchar<KIND> and gets lowered to !llvm.struct<(ptr, i64)>. That is the |
| 2057 | // struct/pair is passed by value. But, mapped values are passed only as |
| 2058 | // pointers, so before we privatize, we must load the pointer. |
| 2059 | if (!isa<LLVM::LLVMPointerType>(Val: privVarType)) |
| 2060 | return builder.CreateLoad(Ty: moduleTranslation.convertType(type: privVarType), |
| 2061 | Ptr: moduleTranslation.lookupValue(value: blockArg)); |
| 2062 | |
| 2063 | return moduleTranslation.lookupValue(value: privateVar); |
| 2064 | } |
| 2065 | |
| 2066 | // Privatizer region arguments may be by-value even when the available LLVM |
| 2067 | // value is storage for that value, e.g. lowered Fortran boxchar descriptors in |
| 2068 | // task context structs. Materialize the value expected by the region argument |
| 2069 | // while preserving the existing pointer mapping for pointer arguments. |
| 2070 | static llvm::Value * |
| 2071 | materializeRegionArgValue(llvm::IRBuilderBase &builder, |
| 2072 | LLVM::ModuleTranslation &moduleTranslation, |
| 2073 | BlockArgument regionArg, llvm::Value *value) { |
| 2074 | if (!regionArg) |
| 2075 | return value; |
| 2076 | |
| 2077 | llvm::Type *regionArgType = |
| 2078 | moduleTranslation.convertType(type: regionArg.getType()); |
| 2079 | if (regionArgType->isPointerTy() || !value->getType()->isPointerTy()) |
| 2080 | return value; |
| 2081 | |
| 2082 | return builder.CreateLoad(Ty: regionArgType, Ptr: value); |
| 2083 | } |
| 2084 | |
| 2085 | /// Returns the address of the data in `var`, via the `data_address` region. |
| 2086 | static llvm::Expected<llvm::Value *> |
| 2087 | getPrivateDataAddress(llvm::IRBuilderBase &builder, |
| 2088 | LLVM::ModuleTranslation &moduleTranslation, |
| 2089 | omp::PrivateClauseOp privatizer, llvm::Value *var) { |
| 2090 | Region ®ion = privatizer.getDataAddressRegion(); |
| 2091 | if (region.empty()) |
| 2092 | return var; |
| 2093 | |
| 2094 | BlockArgument arg = privatizer.getDataAddressArg(); |
| 2095 | moduleTranslation.mapValue( |
| 2096 | mlir: arg, llvm: materializeRegionArgValue(builder, moduleTranslation, regionArg: arg, value: var)); |
| 2097 | SmallVector<llvm::Value *, 1> address; |
| 2098 | if (failed(Result: inlineConvertOmpRegions(region, blockName: "omp.private.data_address" , |
| 2099 | builder, moduleTranslation, continuationBlockArgs: &address))) |
| 2100 | return llvm::make_error<PreviouslyReportedError>(); |
| 2101 | setInsertPointForPossiblyEmptyBlock(builder); |
| 2102 | return address.front(); |
| 2103 | } |
| 2104 | |
| 2105 | /// Initialize a single (first)private variable. You probably want to use |
| 2106 | /// allocateAndInitPrivateVars instead of this. |
| 2107 | /// This returns the private variable which has been initialized. This |
| 2108 | /// variable should be mapped before constructing the body of the Op. |
| 2109 | static llvm::Expected<llvm::Value *> |
| 2110 | initPrivateVar(llvm::IRBuilderBase &builder, |
| 2111 | LLVM::ModuleTranslation &moduleTranslation, |
| 2112 | omp::PrivateClauseOp &privDecl, llvm::Value *nonPrivateVar, |
| 2113 | BlockArgument &blockArg, llvm::Value *llvmPrivateVar, |
| 2114 | llvm::BasicBlock *privInitBlock, |
| 2115 | llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) { |
| 2116 | Region &initRegion = privDecl.getInitRegion(); |
| 2117 | if (initRegion.empty()) |
| 2118 | return llvmPrivateVar; |
| 2119 | |
| 2120 | assert(nonPrivateVar); |
| 2121 | moduleTranslation.mapValue(mlir: privDecl.getInitMoldArg(), llvm: nonPrivateVar); |
| 2122 | moduleTranslation.mapValue(mlir: privDecl.getInitPrivateArg(), llvm: llvmPrivateVar); |
| 2123 | |
| 2124 | // in-place convert the private initialization region |
| 2125 | SmallVector<llvm::Value *, 1> phis; |
| 2126 | if (failed(Result: inlineConvertOmpRegions(region&: initRegion, blockName: "omp.private.init" , builder, |
| 2127 | moduleTranslation, continuationBlockArgs: &phis))) |
| 2128 | return llvm::createStringError( |
| 2129 | Fmt: "failed to inline `init` region of `omp.private`" ); |
| 2130 | |
| 2131 | assert(phis.size() == 1 && "expected one allocation to be yielded" ); |
| 2132 | |
| 2133 | // clear init region block argument mapping in case it needs to be |
| 2134 | // re-created with a different source for another use of the same |
| 2135 | // reduction decl |
| 2136 | moduleTranslation.forgetMapping(region&: initRegion); |
| 2137 | |
| 2138 | // Prefer the value yielded from the init region to the allocated private |
| 2139 | // variable in case the region is operating on arguments by-value (e.g. |
| 2140 | // Fortran character boxes). |
| 2141 | return phis[0]; |
| 2142 | } |
| 2143 | |
| 2144 | /// Version of initPrivateVar which looks up the nonPrivateVar from mlirPrivVar. |
| 2145 | static llvm::Expected<llvm::Value *> initPrivateVar( |
| 2146 | llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation, |
| 2147 | omp::PrivateClauseOp &privDecl, Value mlirPrivVar, BlockArgument &blockArg, |
| 2148 | llvm::Value *llvmPrivateVar, llvm::BasicBlock *privInitBlock, |
| 2149 | llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) { |
| 2150 | return initPrivateVar( |
| 2151 | builder, moduleTranslation, privDecl, |
| 2152 | nonPrivateVar: findAssociatedValue(privateVar: mlirPrivVar, builder, moduleTranslation, |
| 2153 | mappedPrivateVars), |
| 2154 | blockArg, llvmPrivateVar, privInitBlock, mappedPrivateVars); |
| 2155 | } |
| 2156 | |
| 2157 | static llvm::Error |
| 2158 | initPrivateVars(llvm::IRBuilderBase &builder, |
| 2159 | LLVM::ModuleTranslation &moduleTranslation, |
| 2160 | PrivateVarsInfo &privateVarsInfo, |
| 2161 | llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) { |
| 2162 | if (privateVarsInfo.vars.empty()) |
| 2163 | return llvm::Error::success(); |
| 2164 | |
| 2165 | llvm::BasicBlock *privInitBlock = splitBB(Builder&: builder, CreateBranch: true, Name: "omp.private.init" ); |
| 2166 | setInsertPointForPossiblyEmptyBlock(builder, block: privInitBlock); |
| 2167 | |
| 2168 | for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) { |
| 2169 | llvm::Expected<llvm::Value *> privVarOrErr = initPrivateVar( |
| 2170 | builder, moduleTranslation, privDecl&: var.privatizer, mlirPrivVar: var.mlirVar, blockArg&: var.blockArg, |
| 2171 | llvmPrivateVar: var.llvmVar, privInitBlock, mappedPrivateVars); |
| 2172 | |
| 2173 | if (!privVarOrErr) |
| 2174 | return privVarOrErr.takeError(); |
| 2175 | |
| 2176 | var.llvmVar = privVarOrErr.get(); |
| 2177 | moduleTranslation.mapValue(mlir: var.blockArg, llvm: var.llvmVar); |
| 2178 | |
| 2179 | setInsertPointForPossiblyEmptyBlock(builder); |
| 2180 | } |
| 2181 | |
| 2182 | return llvm::Error::success(); |
| 2183 | } |
| 2184 | |
| 2185 | static LogicalResult |
| 2186 | convertAllocatorVars(Operation &op, ValueRange allocatorVars, |
| 2187 | llvm::IRBuilderBase &builder, |
| 2188 | LLVM::ModuleTranslation &moduleTranslation, |
| 2189 | PrivateVarsInfo &privateVarsInfo) { |
| 2190 | for (Value allocatorVar : allocatorVars) { |
| 2191 | if (privateVarsInfo.convertedAllocators.contains(Val: allocatorVar)) |
| 2192 | continue; |
| 2193 | |
| 2194 | llvm::Value *allocator = moduleTranslation.lookupValue(value: allocatorVar); |
| 2195 | if (!allocator) |
| 2196 | return op.emitError(message: "failed to translate OpenMP allocator operand" ); |
| 2197 | if (allocator->getType()->isIntegerTy()) |
| 2198 | allocator = builder.CreateIntToPtr(V: allocator, DestTy: builder.getPtrTy()); |
| 2199 | else if (allocator->getType()->isPointerTy()) |
| 2200 | allocator = builder.CreatePointerBitCastOrAddrSpaceCast( |
| 2201 | V: allocator, DestTy: builder.getPtrTy()); |
| 2202 | else |
| 2203 | return op.emitError( |
| 2204 | message: "OpenMP allocator operand must have integer or pointer type" ); |
| 2205 | |
| 2206 | privateVarsInfo.convertedAllocators.try_emplace(Key: allocatorVar, Args&: allocator); |
| 2207 | } |
| 2208 | return success(); |
| 2209 | } |
| 2210 | |
| 2211 | /// Allocate and initialize delayed private variables. Returns the basic block |
| 2212 | /// which comes after all of these allocations. llvm::Value * for each of these |
| 2213 | /// private variables are populated in llvmPrivateVars. |
| 2214 | template <typename T> |
| 2215 | static llvm::Expected<llvm::BasicBlock *> allocatePrivateVars( |
| 2216 | T op, llvm::IRBuilderBase &builder, |
| 2217 | LLVM::ModuleTranslation &moduleTranslation, |
| 2218 | PrivateVarsInfo &privateVarsInfo, |
| 2219 | llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, |
| 2220 | llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr, |
| 2221 | std::optional<llvm::OpenMPIRBuilder::InsertPointTy> allocatorIP = |
| 2222 | std::nullopt) { |
| 2223 | // Allocate private vars |
| 2224 | // Save blocks before splits since allocaIP/allocatorIP iterators may follow |
| 2225 | // spliced instructions to the new block. |
| 2226 | llvm::BasicBlock *allocaBB = allocaIP.getNodeParent(); |
| 2227 | llvm::Instruction *allocaTerminator = allocaBB->getTerminator(); |
| 2228 | splitBB(IP: allocaTerminator->getIterator(), CreateBranch: true, |
| 2229 | DL: allocaTerminator->getDebugLoc(), Name: "omp.region.after_alloca" ); |
| 2230 | // Update the allocaTerminator since the alloca block was split above. |
| 2231 | allocaTerminator = allocaBB->getTerminator(); |
| 2232 | // The new terminator is an uncondition branch created by the splitBB above. |
| 2233 | assert(allocaTerminator->getNumSuccessors() == 1 && |
| 2234 | "This is an unconditional branch created by splitBB" ); |
| 2235 | allocaIP = allocaTerminator->getIterator(); |
| 2236 | |
| 2237 | llvm::Instruction *allocatorTerminator = nullptr; |
| 2238 | llvm::BasicBlock *afterAllocatorAllocations = nullptr; |
| 2239 | if (allocatorIP) { |
| 2240 | llvm::BasicBlock *allocatorBB = allocatorIP->getNodeParent(); |
| 2241 | allocatorTerminator = allocatorBB->getTerminator(); |
| 2242 | afterAllocatorAllocations = splitBB( |
| 2243 | IP: allocatorTerminator->getIterator(), CreateBranch: true, |
| 2244 | DL: allocatorTerminator->getDebugLoc(), Name: "omp.region.after_allocate" ); |
| 2245 | allocatorTerminator = allocatorBB->getTerminator(); |
| 2246 | assert(allocatorTerminator->getNumSuccessors() == 1 && |
| 2247 | "This is an unconditional branch created by splitBB" ); |
| 2248 | } |
| 2249 | |
| 2250 | std::optional<llvm::IRBuilderBase::InsertPointGuard> guard; |
| 2251 | if (!allocatorIP) |
| 2252 | guard.emplace(args&: builder); |
| 2253 | builder.SetInsertPoint(allocaTerminator); |
| 2254 | |
| 2255 | llvm::DataLayout dataLayout = builder.getDataLayout(); |
| 2256 | llvm::BasicBlock *afterAllocas = allocaTerminator->getSuccessor(Idx: 0); |
| 2257 | |
| 2258 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 2259 | bool mightUseDeviceSharedMem = omp::opInSharedDeviceContext(op&: *op); |
| 2260 | unsigned int allocaAS = |
| 2261 | moduleTranslation.getLLVMModule()->getDataLayout().getAllocaAddrSpace(); |
| 2262 | unsigned int defaultAS = moduleTranslation.getLLVMModule() |
| 2263 | ->getDataLayout() |
| 2264 | .getProgramAddressSpace(); |
| 2265 | |
| 2266 | SmallVector<int64_t> allocateItemForPrivate(privateVarsInfo.vars.size(), -1); |
| 2267 | ValueRange allocatorVars; |
| 2268 | DenseI64ArrayAttr allocateAlignments; |
| 2269 | if constexpr (std::is_same_v<T, omp::ParallelOp> || |
| 2270 | std::is_same_v<T, omp::ScopeOp>) { |
| 2271 | allocatorVars = op.getAllocatorVars(); |
| 2272 | allocateAlignments = op.getAllocateAlignmentsAttr(); |
| 2273 | if (auto privateIndices = op.getAllocatePrivateIndicesAttr()) |
| 2274 | for (auto [allocateIndex, privateIndex] : |
| 2275 | llvm::enumerate(privateIndices.asArrayRef())) |
| 2276 | allocateItemForPrivate[privateIndex] = allocateIndex; |
| 2277 | } |
| 2278 | |
| 2279 | for (auto [privateIndex, var] : llvm::enumerate(First&: privateVarsInfo.vars)) { |
| 2280 | llvm::Type *llvmAllocType = |
| 2281 | moduleTranslation.convertType(type: var.privatizer.getType()); |
| 2282 | llvm::Value *llvmPrivateVar = nullptr; |
| 2283 | int64_t allocateIndex = allocateItemForPrivate[privateIndex]; |
| 2284 | builder.SetInsertPoint(allocateIndex >= 0 && allocatorTerminator |
| 2285 | ? allocatorTerminator |
| 2286 | : allocaTerminator); |
| 2287 | if (allocateIndex >= 0) { |
| 2288 | if (ompBuilder->Config.isTargetDevice() || |
| 2289 | op->template getParentOfType<omp::TargetOp>()) |
| 2290 | return llvm::createStringError( |
| 2291 | Fmt: "allocate clause in an OpenMP device context is not supported" ); |
| 2292 | if (!llvmAllocType->isSized()) |
| 2293 | return llvm::createStringError( |
| 2294 | Fmt: "allocate clause private type must have a fixed size" ); |
| 2295 | llvm::TypeSize size = dataLayout.getTypeAllocSize(Ty: llvmAllocType); |
| 2296 | if (size.isScalable()) |
| 2297 | return llvm::createStringError( |
| 2298 | Fmt: "allocate clause private type must have a fixed size" ); |
| 2299 | llvm::IntegerType *sizeTy = |
| 2300 | moduleTranslation.getLLVMModule()->getDataLayout().getIntPtrType( |
| 2301 | C&: moduleTranslation.getLLVMModule()->getContext()); |
| 2302 | if (!llvm::isUIntN(N: sizeTy->getBitWidth(), x: size.getFixedValue())) |
| 2303 | return llvm::createStringError( |
| 2304 | Fmt: "OpenMP allocation size cannot be represented by the target size " |
| 2305 | "type" ); |
| 2306 | llvm::Value *sizeValue = |
| 2307 | llvm::ConstantInt::get(Ty: sizeTy, V: size.getFixedValue()); |
| 2308 | |
| 2309 | Value allocatorVar = allocatorVars[allocateIndex]; |
| 2310 | auto allocator = privateVarsInfo.convertedAllocators.find(Val: allocatorVar); |
| 2311 | if (allocator == privateVarsInfo.convertedAllocators.end()) |
| 2312 | return llvm::createStringError( |
| 2313 | Fmt: "failed to find converted OpenMP allocator operand" ); |
| 2314 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 2315 | int64_t alignment = |
| 2316 | allocateAlignments ? allocateAlignments[allocateIndex] : 0; |
| 2317 | if (alignment != 0) { |
| 2318 | // The allocation must be aligned to at least the maximum of the |
| 2319 | // requested alignment and the alignment the base language requires |
| 2320 | // for the type being allocated. |
| 2321 | uint64_t alignmentValue = std::max<uint64_t>( |
| 2322 | a: static_cast<uint64_t>(alignment), |
| 2323 | b: dataLayout.getABITypeAlign(Ty: llvmAllocType).value()); |
| 2324 | if (!llvm::isUIntN(N: sizeTy->getBitWidth(), x: alignmentValue)) |
| 2325 | return llvm::createStringError( |
| 2326 | Fmt: "OpenMP allocation alignment cannot be represented by the " |
| 2327 | "target size type" ); |
| 2328 | llvmPrivateVar = ompBuilder->createOMPAlignedAlloc( |
| 2329 | Loc: ompLoc, Align: llvm::ConstantInt::get(Ty: sizeTy, V: alignmentValue), Size: sizeValue, |
| 2330 | Allocator: allocator->second, Name: "omp.private.alloc" ); |
| 2331 | } else { |
| 2332 | llvmPrivateVar = ompBuilder->createOMPAlloc( |
| 2333 | Loc: ompLoc, Size: sizeValue, Allocator: allocator->second, Name: "omp.private.alloc" ); |
| 2334 | } |
| 2335 | if (!llvmPrivateVar) |
| 2336 | return llvm::createStringError( |
| 2337 | Fmt: "failed to create OpenMP private allocation" ); |
| 2338 | privateVarsInfo.allocatorPrivates.push_back( |
| 2339 | Elt: {.allocatedPtr: llvmPrivateVar, .allocator: allocator->second}); |
| 2340 | } else if (mightUseDeviceSharedMem && |
| 2341 | omp::allocaUsesRequireSharedMem(alloc: var.blockArg)) { |
| 2342 | llvmPrivateVar = ompBuilder->createOMPAllocShared(Loc: builder, VarType: llvmAllocType); |
| 2343 | } else { |
| 2344 | llvmPrivateVar = builder.CreateAlloca( |
| 2345 | Ty: llvmAllocType, /*ArraySize=*/nullptr, Name: "omp.private.alloc" ); |
| 2346 | if (allocaAS != defaultAS) |
| 2347 | llvmPrivateVar = builder.CreateAddrSpaceCast( |
| 2348 | V: llvmPrivateVar, DestTy: builder.getPtrTy(AddrSpace: defaultAS)); |
| 2349 | } |
| 2350 | |
| 2351 | var.llvmVar = llvmPrivateVar; |
| 2352 | } |
| 2353 | |
| 2354 | return afterAllocatorAllocations ? afterAllocatorAllocations : afterAllocas; |
| 2355 | } |
| 2356 | |
| 2357 | /// A compiler-generated barrier is unsafe when only part of the current team |
| 2358 | /// can reach it. This cannot always be determined statically (e.g. across a |
| 2359 | /// function call), but the enclosing OpenMP constructs can rule it out. |
| 2360 | static bool opMightBeSafeForBarriers(mlir::Operation *op) { |
| 2361 | for (mlir::Operation *parent = op; parent != nullptr; |
| 2362 | parent = parent->getParentOp()) { |
| 2363 | // An inner parallel construct creates a new team, even when it is nested |
| 2364 | // inside a construct that only some threads of the outer team encounter. |
| 2365 | if (mlir::isa<omp::ParallelOp>(Val: parent)) |
| 2366 | return true; |
| 2367 | |
| 2368 | if (mlir::isa<omp::SingleOp, omp::CriticalOp, omp::MaskedOp, omp::MasterOp, |
| 2369 | omp::SectionOp>(Val: parent)) |
| 2370 | return false; |
| 2371 | } |
| 2372 | return true; |
| 2373 | } |
| 2374 | |
| 2375 | static LogicalResult |
| 2376 | emitPrivatizationBarrier(mlir::Operation *op, llvm::IRBuilderBase &builder, |
| 2377 | LLVM::ModuleTranslation &moduleTranslation) { |
| 2378 | if (!opMightBeSafeForBarriers(op)) |
| 2379 | return success(); |
| 2380 | |
| 2381 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 2382 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy res = |
| 2383 | ompBuilder->createBarrier(Loc: builder, Kind: llvm::omp::OMPD_barrier); |
| 2384 | return handleError(result&: res, op&: *op); |
| 2385 | } |
| 2386 | |
| 2387 | /// Applies the firstprivate copy region for each variable, pairing the mold |
| 2388 | /// values with the private allocations and privatizer declarations from |
| 2389 | /// `privateVarsInfo`, and emits a barrier if requested. |
| 2390 | /// |
| 2391 | /// `moldVars` and `llvmPrivateVars` are optional overrides that, if empty, |
| 2392 | /// are read directly from `privateVarsInfo`. `mappedPrivateVars` is only |
| 2393 | /// consulted while materializing mold values from `privateVarsInfo` (i.e. empty |
| 2394 | /// `moldVars`). |
| 2395 | static LogicalResult |
| 2396 | completePrivateVars(mlir::Operation *op, llvm::IRBuilderBase &builder, |
| 2397 | LLVM::ModuleTranslation &moduleTranslation, |
| 2398 | const PrivateVarsInfo &privateVarsInfo, bool insertBarrier, |
| 2399 | ArrayRef<llvm::Value *> moldVars = {}, |
| 2400 | ArrayRef<llvm::Value *> llvmPrivateVars = {}, |
| 2401 | DenseMap<Value, Value> *mappedPrivateVars = nullptr) { |
| 2402 | assert((moldVars.empty() || moldVars.size() == privateVarsInfo.vars.size()) && |
| 2403 | "expected one mold value per variable when provided" ); |
| 2404 | assert((llvmPrivateVars.empty() || |
| 2405 | llvmPrivateVars.size() == privateVarsInfo.vars.size()) && |
| 2406 | "expected one private allocation per variable when provided" ); |
| 2407 | |
| 2408 | // Apply copy region for firstprivate. |
| 2409 | bool needsFirstprivate = llvm::any_of( |
| 2410 | Range: privateVarsInfo.vars, P: [](const PrivateVarsInfo::VarInfo &var) { |
| 2411 | // Copy the op handle so the non-const accessor can be called. |
| 2412 | omp::PrivateClauseOp privOp = var.privatizer; |
| 2413 | return privOp.getDataSharingType() == |
| 2414 | omp::DataSharingClauseType::FirstPrivate; |
| 2415 | }); |
| 2416 | |
| 2417 | if (!needsFirstprivate) |
| 2418 | return insertBarrier |
| 2419 | ? emitPrivatizationBarrier(op, builder, moduleTranslation) |
| 2420 | : success(); |
| 2421 | |
| 2422 | // Materialize the mold values from the MLIR variables when the caller didn't |
| 2423 | // supply explicit locations. |
| 2424 | SmallVector<llvm::Value *> materializedMoldVars; |
| 2425 | if (moldVars.empty()) { |
| 2426 | materializedMoldVars.reserve(N: privateVarsInfo.vars.size()); |
| 2427 | for (const PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) { |
| 2428 | // map copyRegion rhs arg |
| 2429 | llvm::Value *moldVar = findAssociatedValue( |
| 2430 | privateVar: var.mlirVar, builder, moduleTranslation, mappedPrivateVars); |
| 2431 | assert(moldVar); |
| 2432 | materializedMoldVars.push_back(Elt: moldVar); |
| 2433 | } |
| 2434 | moldVars = materializedMoldVars; |
| 2435 | } |
| 2436 | |
| 2437 | llvm::BasicBlock *copyBlock = |
| 2438 | splitBB(Builder&: builder, /*CreateBranch=*/true, Name: "omp.private.copy" ); |
| 2439 | setInsertPointForPossiblyEmptyBlock(builder, block: copyBlock); |
| 2440 | |
| 2441 | unsigned varIdx = 0; |
| 2442 | for (auto [var, moldVar] : llvm::zip_equal(t: privateVarsInfo.vars, u&: moldVars)) { |
| 2443 | // The private allocation defaults to the one tracked in `privateVarsInfo`, |
| 2444 | // unless the caller overrode it with an explicit location. |
| 2445 | llvm::Value *llvmVar = |
| 2446 | llvmPrivateVars.empty() ? var.llvmVar : llvmPrivateVars[varIdx]; |
| 2447 | ++varIdx; |
| 2448 | |
| 2449 | omp::PrivateClauseOp decl = var.privatizer; |
| 2450 | if (decl.getDataSharingType() != omp::DataSharingClauseType::FirstPrivate) |
| 2451 | continue; |
| 2452 | |
| 2453 | // copyRegion implements `lhs = rhs` |
| 2454 | Region ©Region = decl.getCopyRegion(); |
| 2455 | |
| 2456 | llvm::Value *copyMoldVar = materializeRegionArgValue( |
| 2457 | builder, moduleTranslation, regionArg: decl.getCopyMoldArg(), value: moldVar); |
| 2458 | llvm::Value *copyPrivateVar = materializeRegionArgValue( |
| 2459 | builder, moduleTranslation, regionArg: decl.getCopyPrivateArg(), value: llvmVar); |
| 2460 | |
| 2461 | moduleTranslation.mapValue(mlir: decl.getCopyMoldArg(), llvm: copyMoldVar); |
| 2462 | |
| 2463 | // map copyRegion lhs arg |
| 2464 | moduleTranslation.mapValue(mlir: decl.getCopyPrivateArg(), llvm: copyPrivateVar); |
| 2465 | |
| 2466 | // in-place convert copy region |
| 2467 | if (failed(Result: inlineConvertOmpRegions(region&: copyRegion, blockName: "omp.private.copy" , builder, |
| 2468 | moduleTranslation))) |
| 2469 | return decl.emitError(message: "failed to inline `copy` region of `omp.private`" ); |
| 2470 | |
| 2471 | setInsertPointForPossiblyEmptyBlock(builder); |
| 2472 | |
| 2473 | // ignore unused value yielded from copy region |
| 2474 | |
| 2475 | // clear copy region block argument mapping in case it needs to be |
| 2476 | // re-created with different sources for reuse of the same reduction |
| 2477 | // decl |
| 2478 | moduleTranslation.forgetMapping(region&: copyRegion); |
| 2479 | } |
| 2480 | |
| 2481 | return insertBarrier |
| 2482 | ? emitPrivatizationBarrier(op, builder, moduleTranslation) |
| 2483 | : success(); |
| 2484 | } |
| 2485 | |
| 2486 | template <typename T> |
| 2487 | static LogicalResult |
| 2488 | cleanupPrivateVars(T op, llvm::IRBuilderBase &builder, |
| 2489 | LLVM::ModuleTranslation &moduleTranslation, Location loc, |
| 2490 | PrivateVarsInfo &privateVarsInfo) { |
| 2491 | // private variable deallocation |
| 2492 | SmallVector<Region *> privateCleanupRegions; |
| 2493 | llvm::transform(privateVarsInfo.vars, |
| 2494 | std::back_inserter(x&: privateCleanupRegions), |
| 2495 | [](PrivateVarsInfo::VarInfo &var) { |
| 2496 | return &var.privatizer.getDeallocRegion(); |
| 2497 | }); |
| 2498 | |
| 2499 | SmallVector<llvm::Value *> llvmPrivVars = privateVarsInfo.getLlvmVars(); |
| 2500 | if (failed(Result: inlineOmpRegionCleanup(cleanupRegions&: privateCleanupRegions, privateVariables: llvmPrivVars, |
| 2501 | moduleTranslation, builder, |
| 2502 | regionName: "omp.private.dealloc" , |
| 2503 | /*shouldLoadCleanupRegionArg=*/false))) |
| 2504 | return mlir::emitError(loc, message: "failed to inline `dealloc` region of an " |
| 2505 | "`omp.private` op in" ); |
| 2506 | setInsertPointForPossiblyEmptyBlock(builder); |
| 2507 | |
| 2508 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 2509 | bool mightUseDeviceSharedMem = omp::opInSharedDeviceContext(op&: *op); |
| 2510 | for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) { |
| 2511 | if (mightUseDeviceSharedMem && |
| 2512 | omp::allocaUsesRequireSharedMem(alloc: var.blockArg)) { |
| 2513 | ompBuilder->createOMPFreeShared( |
| 2514 | Loc: builder, Addr: var.llvmVar, |
| 2515 | VarType: moduleTranslation.convertType(type: var.privatizer.getType())); |
| 2516 | } |
| 2517 | } |
| 2518 | |
| 2519 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 2520 | for (const PrivateVarsInfo::AllocatorInfo &allocation : |
| 2521 | llvm::reverse(C&: privateVarsInfo.allocatorPrivates)) |
| 2522 | ompBuilder->createOMPFree(Loc: ompLoc, Addr: allocation.allocatedPtr, |
| 2523 | Allocator: allocation.allocator); |
| 2524 | |
| 2525 | return success(); |
| 2526 | } |
| 2527 | |
| 2528 | /// Returns true if the construct contains omp.cancel or omp.cancellation_point |
| 2529 | static bool constructIsCancellable(Operation *op) { |
| 2530 | // omp.cancel and omp.cancellation_point must be "closely nested" so they will |
| 2531 | // be visible and not inside of function calls. This is enforced by the |
| 2532 | // verifier. |
| 2533 | return op |
| 2534 | ->walk(callback: [](Operation *child) { |
| 2535 | if (mlir::isa<omp::CancelOp, omp::CancellationPointOp>(Val: child)) |
| 2536 | return WalkResult::interrupt(); |
| 2537 | return WalkResult::advance(); |
| 2538 | }) |
| 2539 | .wasInterrupted(); |
| 2540 | } |
| 2541 | |
| 2542 | // Forward declarations for the task-reduction helpers defined alongside the |
| 2543 | // omp.taskgroup lowering further down in this file. These are shared by the |
| 2544 | // `reduction(task, ...)` modifier lowering on the parallel/worksharing |
| 2545 | // constructs and by the omp.taskgroup / omp.taskloop.context task_reduction |
| 2546 | // lowering. When \p isModifier is set, `__kmpc_taskred_modifier_init` is |
| 2547 | // emitted (opening a task-reduction scope) instead of `__kmpc_taskred_init`, |
| 2548 | // with \p isWorksharing selecting the runtime `is_ws` argument. |
| 2549 | static llvm::Value *emitTaskReductionInitCall( |
| 2550 | ArrayRef<omp::DeclareReductionOp> redDecls, |
| 2551 | ArrayRef<llvm::Value *> origPtrs, StringRef helperNamePrefix, |
| 2552 | llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder::InsertPointTy allocaIP, |
| 2553 | LLVM::ModuleTranslation &moduleTranslation, bool isModifier = false, |
| 2554 | bool isWorksharing = false); |
| 2555 | static void |
| 2556 | emitTaskReductionModifierFini(bool isWorksharing, llvm::IRBuilderBase &builder, |
| 2557 | LLVM::ModuleTranslation &moduleTranslation); |
| 2558 | |
| 2559 | /// Register only the task-modified items. All items still participate in the |
| 2560 | /// enclosing construct's ordinary reduction combination and cleanup. |
| 2561 | template <typename OpTy> |
| 2562 | static llvm::Value *emitTaskReductionModifierInitCall( |
| 2563 | OpTy op, ArrayRef<omp::DeclareReductionOp> reductionDecls, |
| 2564 | ArrayRef<llvm::Value *> privateReductionVariables, |
| 2565 | llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder::InsertPointTy allocaIP, |
| 2566 | LLVM::ModuleTranslation &moduleTranslation, bool isWorksharing) { |
| 2567 | SmallVector<omp::DeclareReductionOp> taskDecls; |
| 2568 | SmallVector<llvm::Value *> taskVariables; |
| 2569 | for (unsigned i = 0, e = op.getNumReductionVars(); i != e; ++i) { |
| 2570 | if (op.getReductionModifier(i) != omp::ReductionModifier::task) |
| 2571 | continue; |
| 2572 | taskDecls.push_back(Elt: reductionDecls[i]); |
| 2573 | taskVariables.push_back(Elt: privateReductionVariables[i]); |
| 2574 | } |
| 2575 | return emitTaskReductionInitCall( |
| 2576 | redDecls: taskDecls, origPtrs: taskVariables, helperNamePrefix: "__omp_taskred_mod_" , builder, allocaIP, |
| 2577 | moduleTranslation, /*isModifier=*/true, isWorksharing); |
| 2578 | } |
| 2579 | |
| 2580 | static LogicalResult |
| 2581 | convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder, |
| 2582 | LLVM::ModuleTranslation &moduleTranslation) { |
| 2583 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 2584 | using StorableBodyGenCallbackTy = |
| 2585 | llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy; |
| 2586 | |
| 2587 | auto sectionsOp = cast<omp::SectionsOp>(Val&: opInst); |
| 2588 | |
| 2589 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 2590 | return failure(); |
| 2591 | |
| 2592 | ReductionVarsInfo redInfo(sectionsOp); |
| 2593 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 2594 | findAllocInsertPoints(builder, moduleTranslation); |
| 2595 | |
| 2596 | if (failed(Result: allocAndInitializeReductionVars( |
| 2597 | op&: *sectionsOp, builder, moduleTranslation, allocaIP, redInfo))) |
| 2598 | return failure(); |
| 2599 | |
| 2600 | bool isTaskReductionMod = |
| 2601 | sectionsOp.hasReductionModifier(modifier: omp::ReductionModifier::task); |
| 2602 | |
| 2603 | SmallVector<StorableBodyGenCallbackTy> sectionCBs; |
| 2604 | |
| 2605 | for (Operation &op : *sectionsOp.getRegion().begin()) { |
| 2606 | auto sectionOp = dyn_cast<omp::SectionOp>(Val&: op); |
| 2607 | if (!sectionOp) // omp.terminator |
| 2608 | continue; |
| 2609 | |
| 2610 | Region ®ion = sectionOp.getRegion(); |
| 2611 | auto sectionCB = [§ionsOp, ®ion, &builder, &moduleTranslation]( |
| 2612 | InsertPointTy allocaIP, InsertPointTy codeGenIP, |
| 2613 | ArrayRef<llvm::BasicBlock *> deallocBlocks) { |
| 2614 | builder.restoreIP(IP: codeGenIP); |
| 2615 | |
| 2616 | // map the omp.section reduction block argument to the omp.sections block |
| 2617 | // arguments |
| 2618 | // TODO: this assumes that the only block arguments are reduction |
| 2619 | // variables |
| 2620 | assert(region.getNumArguments() == |
| 2621 | sectionsOp.getRegion().getNumArguments()); |
| 2622 | for (auto [sectionsArg, sectionArg] : llvm::zip_equal( |
| 2623 | t: sectionsOp.getRegion().getArguments(), u: region.getArguments())) { |
| 2624 | llvm::Value *llvmVal = moduleTranslation.lookupValue(value: sectionsArg); |
| 2625 | assert(llvmVal); |
| 2626 | moduleTranslation.mapValue(mlir: sectionArg, llvm: llvmVal); |
| 2627 | } |
| 2628 | |
| 2629 | return convertOmpOpRegions(region, blockName: "omp.section.region" , builder, |
| 2630 | moduleTranslation) |
| 2631 | .takeError(); |
| 2632 | }; |
| 2633 | sectionCBs.push_back(Elt: sectionCB); |
| 2634 | } |
| 2635 | |
| 2636 | // Empty sections still require an implicit barrier unless nowait is present. |
| 2637 | if (sectionCBs.empty()) { |
| 2638 | if (!sectionsOp.getNowait()) { |
| 2639 | auto afterIP = getOpenMPBuilder(moduleTranslation) |
| 2640 | ->createBarrier(Loc: builder, Kind: llvm::omp::OMPD_sections); |
| 2641 | if (failed(Result: handleError(result&: afterIP, op&: opInst))) |
| 2642 | return failure(); |
| 2643 | builder.restoreIP(IP: *afterIP); |
| 2644 | } |
| 2645 | return success(); |
| 2646 | } |
| 2647 | |
| 2648 | // For `reduction(task, ...)` open a task-reduction scope for the worksharing |
| 2649 | // region. Participating explicit tasks accumulate into the per-thread private |
| 2650 | // copies, which the worksharing reduction then combines across threads. This |
| 2651 | // is emitted only after the empty-sections early return above, so it stays |
| 2652 | // balanced with the matching fini emitted after the sections region. |
| 2653 | if (isTaskReductionMod && |
| 2654 | !emitTaskReductionModifierInitCall( |
| 2655 | op: sectionsOp, reductionDecls: redInfo.getReductionDecls(), |
| 2656 | privateReductionVariables: redInfo.getPrivateReductionVariables(), builder, allocaIP, |
| 2657 | moduleTranslation, /*isWorksharing=*/true)) |
| 2658 | return sectionsOp.emitError( |
| 2659 | message: "failed to emit task reduction modifier initialization" ); |
| 2660 | |
| 2661 | assert(isa<omp::SectionOp>(*sectionsOp.getRegion().op_begin())); |
| 2662 | |
| 2663 | // TODO: Perform appropriate actions according to the data-sharing |
| 2664 | // attribute (shared, private, firstprivate, ...) of variables. |
| 2665 | // Currently defaults to shared. |
| 2666 | auto privCB = [&](InsertPointTy, InsertPointTy codeGenIP, llvm::Value &, |
| 2667 | llvm::Value &vPtr, llvm::Value *&replacementValue) |
| 2668 | -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { |
| 2669 | replacementValue = &vPtr; |
| 2670 | return codeGenIP; |
| 2671 | }; |
| 2672 | |
| 2673 | // TODO: Perform finalization actions for variables. This has to be |
| 2674 | // called for variables which have destructors/finalizers. |
| 2675 | auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; |
| 2676 | |
| 2677 | allocaIP = findAllocInsertPoints(builder, moduleTranslation); |
| 2678 | bool isCancellable = constructIsCancellable(op: sectionsOp); |
| 2679 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 2680 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 2681 | getOpenMPBuilder(moduleTranslation) |
| 2682 | ->createSections(Loc: ompLoc, AllocaIP: allocaIP, SectionCBs: sectionCBs, PrivCB: privCB, FiniCB: finiCB, |
| 2683 | IsCancellable: isCancellable, IsNowait: sectionsOp.getNowait()); |
| 2684 | |
| 2685 | if (failed(Result: handleError(result&: afterIP, op&: opInst))) |
| 2686 | return failure(); |
| 2687 | |
| 2688 | builder.restoreIP(IP: *afterIP); |
| 2689 | |
| 2690 | // Close the task-reduction scope before combining the worksharing copies. |
| 2691 | if (isTaskReductionMod) |
| 2692 | emitTaskReductionModifierFini(/*isWorksharing=*/true, builder, |
| 2693 | moduleTranslation); |
| 2694 | |
| 2695 | // Process the reductions if required. |
| 2696 | return createReductionsAndCleanup(op&: *sectionsOp, builder, moduleTranslation, |
| 2697 | allocaIP, redInfo, isNowait: sectionsOp.getNowait()); |
| 2698 | } |
| 2699 | |
| 2700 | /// Converts an OpenMP scope construct into LLVM IR. |
| 2701 | static LogicalResult |
| 2702 | convertOmpScope(omp::ScopeOp &scopeOp, llvm::IRBuilderBase &builder, |
| 2703 | LLVM::ModuleTranslation &moduleTranslation) { |
| 2704 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 2705 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 2706 | |
| 2707 | if (failed(Result: checkImplementationStatus(op&: *scopeOp))) |
| 2708 | return failure(); |
| 2709 | |
| 2710 | PrivateVarsInfo privateVarsInfo(scopeOp); |
| 2711 | if (failed(Result: convertAllocatorVars(op&: *scopeOp, allocatorVars: scopeOp.getAllocatorVars(), builder, |
| 2712 | moduleTranslation, privateVarsInfo))) |
| 2713 | return failure(); |
| 2714 | |
| 2715 | ReductionVarsInfo reductionInfo(scopeOp); |
| 2716 | InsertPointTy privateAllocaIP = |
| 2717 | findAllocInsertPoints(builder, moduleTranslation); |
| 2718 | |
| 2719 | if (scopeOp.getAllocateVars().empty()) { |
| 2720 | llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars( |
| 2721 | op: scopeOp, builder, moduleTranslation, privateVarsInfo, allocaIP&: privateAllocaIP); |
| 2722 | if (failed(Result: handleError(result&: afterAllocas, op&: *scopeOp))) |
| 2723 | return failure(); |
| 2724 | } |
| 2725 | |
| 2726 | if (failed(Result: allocAndInitializeReductionVars(op&: *scopeOp, builder, |
| 2727 | moduleTranslation, allocaIP&: privateAllocaIP, |
| 2728 | redInfo&: reductionInfo))) |
| 2729 | return failure(); |
| 2730 | |
| 2731 | auto bodyCB = |
| 2732 | [&](InsertPointTy allocaIP, InsertPointTy codeGenIP, |
| 2733 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error { |
| 2734 | if (!scopeOp.getAllocateVars().empty()) { |
| 2735 | // Runtime storage must be allocated on each dynamic Scope entry. Ordinary |
| 2736 | // private allocas still use the enclosing alloca insertion point. |
| 2737 | llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars( |
| 2738 | op: scopeOp, builder, moduleTranslation, privateVarsInfo, allocaIP&: privateAllocaIP, |
| 2739 | /*mappedPrivateVars=*/nullptr, allocatorIP: codeGenIP); |
| 2740 | if (handleError(result&: afterAllocas, op&: *scopeOp).failed()) |
| 2741 | return llvm::make_error<PreviouslyReportedError>(); |
| 2742 | builder.SetInsertPoint(afterAllocas.get()->getTerminator()); |
| 2743 | } else { |
| 2744 | builder.restoreIP(IP: codeGenIP); |
| 2745 | } |
| 2746 | |
| 2747 | if (handleError( |
| 2748 | error: initPrivateVars(builder, moduleTranslation, privateVarsInfo), |
| 2749 | op&: *scopeOp) |
| 2750 | .failed()) |
| 2751 | return llvm::make_error<PreviouslyReportedError>(); |
| 2752 | |
| 2753 | if (failed(Result: completePrivateVars(op: scopeOp, builder, moduleTranslation, |
| 2754 | privateVarsInfo, |
| 2755 | insertBarrier: scopeOp.getPrivateNeedsBarrier()))) |
| 2756 | return llvm::make_error<PreviouslyReportedError>(); |
| 2757 | |
| 2758 | return convertOmpOpRegions(region&: scopeOp.getRegion(), blockName: "omp.scope.region" , builder, |
| 2759 | moduleTranslation) |
| 2760 | .takeError(); |
| 2761 | }; |
| 2762 | |
| 2763 | auto finiCB = [&](InsertPointTy codeGenIP) -> llvm::Error { |
| 2764 | InsertPointTy oldIP = builder.saveIP(); |
| 2765 | builder.restoreIP(IP: codeGenIP); |
| 2766 | if (failed(Result: cleanupPrivateVars(op: scopeOp, builder, moduleTranslation, |
| 2767 | loc: scopeOp.getLoc(), privateVarsInfo))) |
| 2768 | return llvm::make_error<PreviouslyReportedError>(); |
| 2769 | builder.restoreIP(IP: oldIP); |
| 2770 | return llvm::Error::success(); |
| 2771 | }; |
| 2772 | |
| 2773 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 2774 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 2775 | ompBuilder->createScope(Loc: ompLoc, BodyGenCB: bodyCB, FiniCB: finiCB, IsNowait: scopeOp.getNowait()); |
| 2776 | |
| 2777 | if (failed(Result: handleError(result&: afterIP, op&: *scopeOp))) |
| 2778 | return failure(); |
| 2779 | |
| 2780 | builder.restoreIP(IP: *afterIP); |
| 2781 | |
| 2782 | // Process the reductions if required. |
| 2783 | return createReductionsAndCleanup(op&: *scopeOp, builder, moduleTranslation, |
| 2784 | allocaIP&: privateAllocaIP, redInfo&: reductionInfo, |
| 2785 | isNowait: scopeOp.getNowait(), |
| 2786 | /*isTeamsReduction=*/false); |
| 2787 | } |
| 2788 | |
| 2789 | /// Converts an OpenMP single construct into LLVM IR using OpenMPIRBuilder. |
| 2790 | static LogicalResult |
| 2791 | convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder, |
| 2792 | LLVM::ModuleTranslation &moduleTranslation) { |
| 2793 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 2794 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 2795 | |
| 2796 | if (failed(Result: checkImplementationStatus(op&: *singleOp))) |
| 2797 | return failure(); |
| 2798 | |
| 2799 | auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP, |
| 2800 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) { |
| 2801 | builder.restoreIP(IP: codegenIP); |
| 2802 | return convertOmpOpRegions(region&: singleOp.getRegion(), blockName: "omp.single.region" , |
| 2803 | builder, moduleTranslation) |
| 2804 | .takeError(); |
| 2805 | }; |
| 2806 | auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); }; |
| 2807 | |
| 2808 | // Handle copyprivate |
| 2809 | Operation::operand_range cpVars = singleOp.getCopyprivateVars(); |
| 2810 | std::optional<ArrayAttr> cpFuncs = singleOp.getCopyprivateSyms(); |
| 2811 | llvm::SmallVector<llvm::Value *> llvmCPVars; |
| 2812 | llvm::SmallVector<llvm::Function *> llvmCPFuncs; |
| 2813 | for (size_t i = 0, e = cpVars.size(); i < e; ++i) { |
| 2814 | llvmCPVars.push_back(Elt: moduleTranslation.lookupValue(value: cpVars[i])); |
| 2815 | auto llvmFuncOp = SymbolTable::lookupNearestSymbolFrom<LLVM::LLVMFuncOp>( |
| 2816 | from: singleOp, symbol: cast<SymbolRefAttr>(Val: (*cpFuncs)[i])); |
| 2817 | llvmCPFuncs.push_back( |
| 2818 | Elt: moduleTranslation.lookupFunction(name: llvmFuncOp.getName())); |
| 2819 | } |
| 2820 | |
| 2821 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 2822 | getOpenMPBuilder(moduleTranslation) |
| 2823 | ->createSingle(Loc: ompLoc, BodyGenCB: bodyCB, FiniCB: finiCB, IsNowait: singleOp.getNowait(), |
| 2824 | CPVars: llvmCPVars, CPFuncs: llvmCPFuncs); |
| 2825 | |
| 2826 | if (failed(Result: handleError(result&: afterIP, op&: *singleOp))) |
| 2827 | return failure(); |
| 2828 | |
| 2829 | builder.restoreIP(IP: *afterIP); |
| 2830 | return success(); |
| 2831 | } |
| 2832 | |
| 2833 | static omp::DistributeOp |
| 2834 | getDistributeCapturingTeamsReduction(omp::TeamsOp teamsOp) { |
| 2835 | // Early return if we found more than one distribute op or if we can't find |
| 2836 | // any distribute op in the teams region. |
| 2837 | omp::DistributeOp distOp; |
| 2838 | WalkResult walk = teamsOp.getRegion().walk(callback: [&](omp::DistributeOp op) { |
| 2839 | if (distOp) |
| 2840 | return WalkResult::interrupt(); |
| 2841 | distOp = op; |
| 2842 | return WalkResult::skip(); |
| 2843 | }); |
| 2844 | if (walk.wasInterrupted() || !distOp) |
| 2845 | return {}; |
| 2846 | |
| 2847 | auto iface = |
| 2848 | llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(Val: teamsOp.getOperation()); |
| 2849 | // Check that all uses of the reduction block arg has the same distribute op |
| 2850 | // parent. |
| 2851 | llvm::SmallVector<mlir::Operation *> debugUses; |
| 2852 | for (auto ra : iface.getReductionBlockArgs()) |
| 2853 | for (auto &use : ra.getUses()) { |
| 2854 | auto *useOp = use.getOwner(); |
| 2855 | // Ignore debug uses. |
| 2856 | if (mlir::isa<LLVM::DbgDeclareOp, LLVM::DbgValueOp>(Val: useOp)) { |
| 2857 | debugUses.push_back(Elt: useOp); |
| 2858 | continue; |
| 2859 | } |
| 2860 | if (!distOp->isProperAncestor(other: useOp)) |
| 2861 | return {}; |
| 2862 | } |
| 2863 | |
| 2864 | // If we are going to use distribute reduction then remove any debug uses of |
| 2865 | // the reduction parameters in teamsOp. Otherwise they will be left without |
| 2866 | // any mapped value in moduleTranslation and will eventually error out. |
| 2867 | for (auto *use : debugUses) |
| 2868 | use->erase(); |
| 2869 | return distOp; |
| 2870 | } |
| 2871 | |
| 2872 | // Convert an OpenMP Teams construct to LLVM IR using OpenMPIRBuilder |
| 2873 | static LogicalResult |
| 2874 | convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder, |
| 2875 | LLVM::ModuleTranslation &moduleTranslation) { |
| 2876 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 2877 | if (failed(Result: checkImplementationStatus(op&: *op))) |
| 2878 | return failure(); |
| 2879 | |
| 2880 | std::optional<ReductionVarsInfo> teamsRedInfo; |
| 2881 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 2882 | findAllocInsertPoints(builder, moduleTranslation); |
| 2883 | |
| 2884 | // Only do teams reduction if there is no distribute op that captures the |
| 2885 | // reduction instead. |
| 2886 | if (!getDistributeCapturingTeamsReduction(teamsOp: op)) { |
| 2887 | teamsRedInfo.emplace(args&: op); |
| 2888 | |
| 2889 | if (failed(Result: allocAndInitializeReductionVars(op&: *op, builder, moduleTranslation, |
| 2890 | allocaIP, redInfo&: *teamsRedInfo))) |
| 2891 | return failure(); |
| 2892 | } |
| 2893 | |
| 2894 | auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP, |
| 2895 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) { |
| 2896 | LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame( |
| 2897 | moduleTranslation, allocaIP.getNodeParent(), deallocBlocks); |
| 2898 | builder.restoreIP(IP: codegenIP); |
| 2899 | return convertOmpOpRegions(region&: op.getRegion(), blockName: "omp.teams.region" , builder, |
| 2900 | moduleTranslation) |
| 2901 | .takeError(); |
| 2902 | }; |
| 2903 | |
| 2904 | llvm::Value *numTeamsLower = nullptr; |
| 2905 | if (Value numTeamsLowerVar = op.getNumTeamsLower()) |
| 2906 | numTeamsLower = moduleTranslation.lookupValue(value: numTeamsLowerVar); |
| 2907 | |
| 2908 | llvm::Value *numTeamsUpper = nullptr; |
| 2909 | if (!op.getNumTeamsUpperVars().empty()) |
| 2910 | numTeamsUpper = moduleTranslation.lookupValue(value: op.getNumTeams(dim: 0)); |
| 2911 | |
| 2912 | llvm::Value *threadLimit = nullptr; |
| 2913 | if (!op.getThreadLimitVars().empty()) |
| 2914 | threadLimit = moduleTranslation.lookupValue(value: op.getThreadLimit(dim: 0)); |
| 2915 | |
| 2916 | llvm::Value *ifExpr = nullptr; |
| 2917 | if (Value ifVar = op.getIfExpr()) |
| 2918 | ifExpr = moduleTranslation.lookupValue(value: ifVar); |
| 2919 | |
| 2920 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 2921 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 2922 | getOpenMPBuilder(moduleTranslation) |
| 2923 | ->createTeams(Loc: ompLoc, BodyGenCB: bodyCB, NumTeamsLower: numTeamsLower, NumTeamsUpper: numTeamsUpper, |
| 2924 | ThreadLimit: threadLimit, IfExpr: ifExpr); |
| 2925 | |
| 2926 | if (failed(Result: handleError(result&: afterIP, op&: *op))) |
| 2927 | return failure(); |
| 2928 | |
| 2929 | builder.restoreIP(IP: *afterIP); |
| 2930 | if (teamsRedInfo) { |
| 2931 | // Process the reductions if required. |
| 2932 | return createReductionsAndCleanup(op&: *op, builder, moduleTranslation, allocaIP, |
| 2933 | redInfo&: *teamsRedInfo, |
| 2934 | /*isNoWait=*/isNowait: false, |
| 2935 | /*isTeamsReduction=*/true); |
| 2936 | } |
| 2937 | return success(); |
| 2938 | } |
| 2939 | |
| 2940 | static llvm::omp::RTLDependenceKindTy |
| 2941 | convertDependKind(mlir::omp::ClauseTaskDepend kind) { |
| 2942 | switch (kind) { |
| 2943 | case mlir::omp::ClauseTaskDepend::taskdependin: |
| 2944 | return llvm::omp::RTLDependenceKindTy::DepIn; |
| 2945 | // The OpenMP runtime requires that the codegen for 'depend' clause for |
| 2946 | // 'out' dependency kind must be the same as codegen for 'depend' clause |
| 2947 | // with 'inout' dependency. |
| 2948 | case mlir::omp::ClauseTaskDepend::taskdependout: |
| 2949 | case mlir::omp::ClauseTaskDepend::taskdependinout: |
| 2950 | return llvm::omp::RTLDependenceKindTy::DepInOut; |
| 2951 | case mlir::omp::ClauseTaskDepend::taskdependmutexinoutset: |
| 2952 | return llvm::omp::RTLDependenceKindTy::DepMutexInOutSet; |
| 2953 | case mlir::omp::ClauseTaskDepend::taskdependinoutset: |
| 2954 | return llvm::omp::RTLDependenceKindTy::DepInOutSet; |
| 2955 | } |
| 2956 | llvm_unreachable("unhandled depend kind" ); |
| 2957 | } |
| 2958 | |
| 2959 | static void buildDependDataLocator( |
| 2960 | std::optional<ArrayAttr> dependKinds, OperandRange dependVars, |
| 2961 | LLVM::ModuleTranslation &moduleTranslation, |
| 2962 | SmallVectorImpl<llvm::OpenMPIRBuilder::DependData> &dds) { |
| 2963 | if (dependVars.empty()) |
| 2964 | return; |
| 2965 | for (auto dep : llvm::zip(t&: dependVars, u: dependKinds->getValue())) { |
| 2966 | auto kind = |
| 2967 | cast<mlir::omp::ClauseTaskDependAttr>(Val: std::get<1>(t&: dep)).getValue(); |
| 2968 | llvm::omp::RTLDependenceKindTy type = convertDependKind(kind); |
| 2969 | llvm::Value *depVal = moduleTranslation.lookupValue(value: std::get<0>(t&: dep)); |
| 2970 | llvm::OpenMPIRBuilder::DependData dd(type, depVal->getType(), depVal); |
| 2971 | dds.emplace_back(Args&: dd); |
| 2972 | } |
| 2973 | } |
| 2974 | |
| 2975 | /// Shared implementation of a callback which adds a termiator for the new block |
| 2976 | /// created for the branch taken when an openmp construct is cancelled. The |
| 2977 | /// terminator is saved in \p cancelTerminators. This callback is invoked only |
| 2978 | /// if there is cancellation inside of the taskgroup body. |
| 2979 | /// The terminator will need to be fixed to branch to the correct block to |
| 2980 | /// cleanup the construct. |
| 2981 | static void pushCancelFinalizationCB( |
| 2982 | SmallVectorImpl<llvm::UncondBrInst *> &cancelTerminators, |
| 2983 | llvm::IRBuilderBase &llvmBuilder, llvm::OpenMPIRBuilder &ompBuilder, |
| 2984 | mlir::Operation *op, llvm::omp::Directive cancelDirective) { |
| 2985 | auto finiCB = [&](llvm::OpenMPIRBuilder::InsertPointTy ip) -> llvm::Error { |
| 2986 | llvm::IRBuilderBase::InsertPointGuard guard(llvmBuilder); |
| 2987 | |
| 2988 | // ip is currently in the block branched to if cancellation occurred. |
| 2989 | // We need to create a branch to terminate that block. |
| 2990 | llvmBuilder.restoreIP(IP: ip); |
| 2991 | |
| 2992 | // We must still clean up the construct after cancelling it, so we need to |
| 2993 | // branch to the block that finalizes the taskgroup. |
| 2994 | // That block has not been created yet so use this block as a dummy for now |
| 2995 | // and fix this after creating the operation. |
| 2996 | cancelTerminators.push_back(Elt: llvmBuilder.CreateBr(Dest: ip.getNodeParent())); |
| 2997 | return llvm::Error::success(); |
| 2998 | }; |
| 2999 | // We have to add the cleanup to the OpenMPIRBuilder before the body gets |
| 3000 | // created in case the body contains omp.cancel (which will then expect to be |
| 3001 | // able to find this cleanup callback). |
| 3002 | ompBuilder.pushFinalizationCB( |
| 3003 | FI: {finiCB, cancelDirective, constructIsCancellable(op)}); |
| 3004 | } |
| 3005 | |
| 3006 | /// If we cancelled the construct, we should branch to the finalization block of |
| 3007 | /// that construct. OMPIRBuilder structures the CFG such that the cleanup block |
| 3008 | /// is immediately before the continuation block. Now this finalization has |
| 3009 | /// been created we can fix the branch. |
| 3010 | static void |
| 3011 | popCancelFinalizationCB(const ArrayRef<llvm::UncondBrInst *> cancelTerminators, |
| 3012 | llvm::OpenMPIRBuilder &ompBuilder, |
| 3013 | llvm::BasicBlock *afterBB) { |
| 3014 | ompBuilder.popFinalizationCB(); |
| 3015 | llvm::BasicBlock *constructFini = afterBB->getSinglePredecessor(); |
| 3016 | for (llvm::UncondBrInst *cancelBranch : cancelTerminators) |
| 3017 | cancelBranch->setSuccessor(constructFini); |
| 3018 | } |
| 3019 | |
| 3020 | namespace { |
| 3021 | /// TaskContextStructManager takes care of creating and freeing a structure |
| 3022 | /// containing information needed by the task body to execute. |
| 3023 | class TaskContextStructManager { |
| 3024 | public: |
| 3025 | TaskContextStructManager(llvm::IRBuilderBase &builder, |
| 3026 | LLVM::ModuleTranslation &moduleTranslation, |
| 3027 | const PrivateVarsInfo &privateVarsInfo) |
| 3028 | : builder{builder}, moduleTranslation{moduleTranslation}, |
| 3029 | privateVarsInfo{privateVarsInfo} {} |
| 3030 | |
| 3031 | /// Creates a heap allocated struct containing space for each private |
| 3032 | /// variable. Invariant: privateVarTypes, privateVarsInfo.vars, and the |
| 3033 | /// elements of the structure should all have the same order (although |
| 3034 | /// privatizers which do not read from the mold argument are skipped). |
| 3035 | void generateTaskContextStruct(); |
| 3036 | |
| 3037 | /// Create GEPs to access each member of the structure representing a private |
| 3038 | /// variable, adding them to llvmPrivateVars. Null values are added where |
| 3039 | /// private decls were skipped so that the ordering continues to match the |
| 3040 | /// private decls. |
| 3041 | void createGEPsToPrivateVars(); |
| 3042 | |
| 3043 | /// Given the address of the structure, return a GEP for each private variable |
| 3044 | /// in the structure. Null values are added where private decls were skipped |
| 3045 | /// so that the ordering continues to match the private decls. |
| 3046 | /// Must be called after generateTaskContextStruct(). |
| 3047 | SmallVector<llvm::Value *> |
| 3048 | createGEPsToPrivateVars(llvm::Value *altStructPtr) const; |
| 3049 | |
| 3050 | /// De-allocate the task context structure. |
| 3051 | void freeStructPtr(); |
| 3052 | |
| 3053 | MutableArrayRef<llvm::Value *> getLLVMPrivateVarGEPs() { |
| 3054 | return llvmPrivateVarGEPs; |
| 3055 | } |
| 3056 | |
| 3057 | llvm::Value *getStructPtr() { return structPtr; } |
| 3058 | |
| 3059 | private: |
| 3060 | llvm::IRBuilderBase &builder; |
| 3061 | LLVM::ModuleTranslation &moduleTranslation; |
| 3062 | const PrivateVarsInfo &privateVarsInfo; |
| 3063 | |
| 3064 | /// The type of each member of the structure, in order. |
| 3065 | SmallVector<llvm::Type *> privateVarTypes; |
| 3066 | |
| 3067 | /// LLVM values for each private variable, or null if that private variable is |
| 3068 | /// not included in the task context structure |
| 3069 | SmallVector<llvm::Value *> llvmPrivateVarGEPs; |
| 3070 | |
| 3071 | /// A pointer to the structure containing context for this task. |
| 3072 | llvm::Value *structPtr = nullptr; |
| 3073 | /// The type of the structure |
| 3074 | llvm::Type *structTy = nullptr; |
| 3075 | }; |
| 3076 | |
| 3077 | /// IteratorInfo extracts and prepares loop bounds information from an |
| 3078 | /// mlir::omp::IteratorOp for lowering to LLVM IR. |
| 3079 | /// |
| 3080 | /// It computes the per-dimension trip counts and the total linearized trip |
| 3081 | /// count as i64. Bounds and steps keep their declared type so that the |
| 3082 | /// physical induction variables can be reconstructed in that type inside the |
| 3083 | /// loop body. |
| 3084 | class IteratorInfo { |
| 3085 | private: |
| 3086 | llvm::SmallVector<llvm::Value *> lowerBounds; |
| 3087 | llvm::SmallVector<llvm::Value *> upperBounds; |
| 3088 | llvm::SmallVector<llvm::Value *> steps; |
| 3089 | llvm::SmallVector<llvm::Value *> trips; |
| 3090 | unsigned dims; |
| 3091 | llvm::Value *totalTrips; |
| 3092 | |
| 3093 | public: |
| 3094 | IteratorInfo(mlir::omp::IteratorOp itersOp, |
| 3095 | mlir::LLVM::ModuleTranslation &moduleTranslation, |
| 3096 | llvm::IRBuilderBase &builder) { |
| 3097 | dims = itersOp.getLoopLowerBounds().size(); |
| 3098 | lowerBounds.resize(N: dims); |
| 3099 | upperBounds.resize(N: dims); |
| 3100 | steps.resize(N: dims); |
| 3101 | trips.resize(N: dims); |
| 3102 | |
| 3103 | for (unsigned d = 0; d < dims; ++d) { |
| 3104 | llvm::Value *lb = |
| 3105 | moduleTranslation.lookupValue(value: itersOp.getLoopLowerBounds()[d]); |
| 3106 | llvm::Value *ub = |
| 3107 | moduleTranslation.lookupValue(value: itersOp.getLoopUpperBounds()[d]); |
| 3108 | llvm::Value *st = |
| 3109 | moduleTranslation.lookupValue(value: itersOp.getLoopSteps()[d]); |
| 3110 | assert(lb && ub && st && |
| 3111 | "Expect lowerBounds, upperBounds, and steps in IteratorOp" ); |
| 3112 | assert(lb->getType() == ub->getType() && lb->getType() == st->getType() && |
| 3113 | "Expect matching iterator range types" ); |
| 3114 | assert((!llvm::isa<llvm::ConstantInt>(st) || |
| 3115 | !llvm::cast<llvm::ConstantInt>(st)->isZero()) && |
| 3116 | "Expect non-zero step in IteratorOp" ); |
| 3117 | |
| 3118 | lowerBounds[d] = lb; |
| 3119 | upperBounds[d] = ub; |
| 3120 | steps[d] = st; |
| 3121 | |
| 3122 | // Use a direction-aware count so an empty range contributes no entries. |
| 3123 | // Widen by one bit so the span between the bounds cannot overflow. |
| 3124 | llvm::Type *countTy = |
| 3125 | builder.getIntNTy(N: lb->getType()->getIntegerBitWidth() + 1); |
| 3126 | llvm::Value *start = builder.CreateSExt(V: lb, DestTy: countTy); |
| 3127 | llvm::Value *stop = builder.CreateSExt(V: ub, DestTy: countTy); |
| 3128 | llvm::Value *step = builder.CreateSExt(V: st, DestTy: countTy); |
| 3129 | llvm::Value *count = |
| 3130 | getOpenMPBuilder(moduleTranslation) |
| 3131 | ->calculateCanonicalLoopTripCount( |
| 3132 | Loc: builder, Start: start, Stop: stop, Step: step, |
| 3133 | /*IsSigned=*/true, InclusiveStop: itersOp.getLoopInclusive(), Name: "iterator" ); |
| 3134 | trips[d] = builder.CreateZExtOrTrunc(V: count, DestTy: builder.getInt64Ty()); |
| 3135 | } |
| 3136 | |
| 3137 | totalTrips = llvm::ConstantInt::get(Ty: builder.getInt64Ty(), V: 1); |
| 3138 | for (unsigned d = 0; d < dims; ++d) |
| 3139 | totalTrips = builder.CreateMul(LHS: totalTrips, RHS: trips[d]); |
| 3140 | } |
| 3141 | |
| 3142 | unsigned getDims() const { return dims; } |
| 3143 | llvm::ArrayRef<llvm::Value *> getLowerBounds() const { return lowerBounds; } |
| 3144 | llvm::ArrayRef<llvm::Value *> getUpperBounds() const { return upperBounds; } |
| 3145 | llvm::ArrayRef<llvm::Value *> getSteps() const { return steps; } |
| 3146 | llvm::ArrayRef<llvm::Value *> getTrips() const { return trips; } |
| 3147 | llvm::Value *getTotalTrips() const { return totalTrips; } |
| 3148 | }; |
| 3149 | } // namespace |
| 3150 | |
| 3151 | void TaskContextStructManager::generateTaskContextStruct() { |
| 3152 | if (privateVarsInfo.vars.empty()) |
| 3153 | return; |
| 3154 | privateVarTypes.reserve(N: privateVarsInfo.vars.size()); |
| 3155 | |
| 3156 | for (const PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) { |
| 3157 | omp::PrivateClauseOp privOp = var.privatizer; |
| 3158 | // Skip private variables which can safely be allocated and initialised |
| 3159 | // inside of the task |
| 3160 | if (!privOp.readsFromMold()) |
| 3161 | continue; |
| 3162 | Type mlirType = privOp.getType(); |
| 3163 | privateVarTypes.push_back(Elt: moduleTranslation.convertType(type: mlirType)); |
| 3164 | } |
| 3165 | |
| 3166 | if (privateVarTypes.empty()) |
| 3167 | return; |
| 3168 | |
| 3169 | structTy = llvm::StructType::get(Context&: moduleTranslation.getLLVMContext(), |
| 3170 | Elements: privateVarTypes); |
| 3171 | |
| 3172 | llvm::DataLayout dataLayout = builder.getDataLayout(); |
| 3173 | llvm::Type *intPtrTy = builder.getIntPtrTy(DL: dataLayout); |
| 3174 | llvm::Value *allocSize = |
| 3175 | builder.CreateTypeSize(Ty: intPtrTy, Size: dataLayout.getTypeAllocSize(Ty: structTy)); |
| 3176 | |
| 3177 | // Heap allocate the structure |
| 3178 | structPtr = builder.CreateMalloc(IntPtrTy: intPtrTy, AllocSize: allocSize, |
| 3179 | /*ArraySize=*/nullptr, /*MallocF=*/nullptr, |
| 3180 | Name: "omp.task.context_ptr" ); |
| 3181 | } |
| 3182 | |
| 3183 | SmallVector<llvm::Value *> TaskContextStructManager::createGEPsToPrivateVars( |
| 3184 | llvm::Value *altStructPtr) const { |
| 3185 | SmallVector<llvm::Value *> ret; |
| 3186 | |
| 3187 | // Create GEPs for each struct member |
| 3188 | ret.reserve(N: privateVarsInfo.vars.size()); |
| 3189 | llvm::Value *zero = builder.getInt32(C: 0); |
| 3190 | unsigned i = 0; |
| 3191 | for (const PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) { |
| 3192 | omp::PrivateClauseOp privDecl = var.privatizer; |
| 3193 | if (!privDecl.readsFromMold()) { |
| 3194 | // Handle this inside of the task so we don't pass unnessecary vars in |
| 3195 | ret.push_back(Elt: nullptr); |
| 3196 | continue; |
| 3197 | } |
| 3198 | llvm::Value *iVal = builder.getInt32(C: i); |
| 3199 | llvm::Value *gep = builder.CreateGEP(Ty: structTy, Ptr: altStructPtr, IdxList: {zero, iVal}); |
| 3200 | ret.push_back(Elt: gep); |
| 3201 | i += 1; |
| 3202 | } |
| 3203 | return ret; |
| 3204 | } |
| 3205 | |
| 3206 | void TaskContextStructManager::createGEPsToPrivateVars() { |
| 3207 | if (!structPtr) |
| 3208 | assert(privateVarTypes.empty()); |
| 3209 | // Still need to run createGEPsToPrivateVars to populate llvmPrivateVarGEPs |
| 3210 | // with null values for skipped private decls |
| 3211 | |
| 3212 | llvmPrivateVarGEPs = createGEPsToPrivateVars(altStructPtr: structPtr); |
| 3213 | } |
| 3214 | |
| 3215 | void TaskContextStructManager::freeStructPtr() { |
| 3216 | if (!structPtr) |
| 3217 | return; |
| 3218 | |
| 3219 | llvm::IRBuilderBase::InsertPointGuard guard{builder}; |
| 3220 | // Ensure we don't put the call to free() after the terminator |
| 3221 | builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator()); |
| 3222 | builder.CreateFree(Source: structPtr); |
| 3223 | } |
| 3224 | |
| 3225 | static void storeAffinityEntry(llvm::IRBuilderBase &builder, |
| 3226 | llvm::OpenMPIRBuilder &ompBuilder, |
| 3227 | llvm::Value *affinityList, llvm::Value *index, |
| 3228 | llvm::Value *addr, llvm::Value *len) { |
| 3229 | llvm::StructType *kmpTaskAffinityInfoTy = |
| 3230 | ompBuilder.getKmpTaskAffinityInfoTy(); |
| 3231 | llvm::Value *entry = builder.CreateInBoundsGEP( |
| 3232 | Ty: kmpTaskAffinityInfoTy, Ptr: affinityList, IdxList: index, Name: "omp.affinity.entry" ); |
| 3233 | |
| 3234 | addr = builder.CreatePtrToInt(V: addr, DestTy: kmpTaskAffinityInfoTy->getElementType(N: 0)); |
| 3235 | len = builder.CreateIntCast(V: len, DestTy: kmpTaskAffinityInfoTy->getElementType(N: 1), |
| 3236 | /*isSigned=*/false); |
| 3237 | llvm::Value *flags = builder.getInt32(C: 0); |
| 3238 | |
| 3239 | builder.CreateStore(Val: addr, |
| 3240 | Ptr: builder.CreateStructGEP(Ty: kmpTaskAffinityInfoTy, Ptr: entry, Idx: 0)); |
| 3241 | builder.CreateStore(Val: len, |
| 3242 | Ptr: builder.CreateStructGEP(Ty: kmpTaskAffinityInfoTy, Ptr: entry, Idx: 1)); |
| 3243 | builder.CreateStore(Val: flags, |
| 3244 | Ptr: builder.CreateStructGEP(Ty: kmpTaskAffinityInfoTy, Ptr: entry, Idx: 2)); |
| 3245 | } |
| 3246 | |
| 3247 | static void fillAffinityLocators(Operation::operand_range affinityVars, |
| 3248 | llvm::IRBuilderBase &builder, |
| 3249 | LLVM::ModuleTranslation &moduleTranslation, |
| 3250 | llvm::Value *affinityList) { |
| 3251 | for (auto [i, affinityVar] : llvm::enumerate(First&: affinityVars)) { |
| 3252 | auto entryOp = affinityVar.getDefiningOp<mlir::omp::AffinityEntryOp>(); |
| 3253 | assert(entryOp && "affinity item must be omp.affinity_entry" ); |
| 3254 | |
| 3255 | llvm::Value *addr = moduleTranslation.lookupValue(value: entryOp.getAddr()); |
| 3256 | llvm::Value *len = moduleTranslation.lookupValue(value: entryOp.getLen()); |
| 3257 | assert(addr && len && "expect affinity addr and len to be non-null" ); |
| 3258 | storeAffinityEntry(builder, ompBuilder&: *getOpenMPBuilder(moduleTranslation), |
| 3259 | affinityList, index: builder.getInt64(C: i), addr, len); |
| 3260 | } |
| 3261 | } |
| 3262 | |
| 3263 | static mlir::LogicalResult convertIteratorRegion( |
| 3264 | llvm::Value *linearIV, IteratorInfo &iterInfo, mlir::Region &iteratorRegion, |
| 3265 | llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { |
| 3266 | mlir::Block &entryBlock = iteratorRegion.front(); |
| 3267 | llvm::Value *tmp = linearIV; |
| 3268 | for (int d = (int)iterInfo.getDims() - 1; d >= 0; --d) { |
| 3269 | llvm::Value *trip = iterInfo.getTrips()[d]; |
| 3270 | llvm::Value *zero = llvm::ConstantInt::get(Ty: trip->getType(), V: 0); |
| 3271 | llvm::Value *one = llvm::ConstantInt::get(Ty: trip->getType(), V: 1); |
| 3272 | // A zero trip count makes the linearized loop empty. Use one in its |
| 3273 | // unreachable IV decoder so the generated IR contains no division by zero. |
| 3274 | llvm::Value *divisor = |
| 3275 | builder.CreateSelect(C: builder.CreateICmpEQ(LHS: trip, RHS: zero), True: one, False: trip); |
| 3276 | // idx_d = tmp % divisor_d |
| 3277 | llvm::Value *idx = builder.CreateURem(LHS: tmp, RHS: divisor); |
| 3278 | // tmp = tmp / divisor_d |
| 3279 | tmp = builder.CreateUDiv(LHS: tmp, RHS: divisor); |
| 3280 | |
| 3281 | // physIV_d = lb_d + idx_d * step_d |
| 3282 | idx = |
| 3283 | builder.CreateZExtOrTrunc(V: idx, DestTy: iterInfo.getLowerBounds()[d]->getType()); |
| 3284 | llvm::Value *physIV = builder.CreateAdd( |
| 3285 | LHS: iterInfo.getLowerBounds()[d], |
| 3286 | RHS: builder.CreateMul(LHS: idx, RHS: iterInfo.getSteps()[d]), Name: "omp.it.phys_iv" ); |
| 3287 | |
| 3288 | moduleTranslation.mapValue(mlir: entryBlock.getArgument(i: d), llvm: physIV); |
| 3289 | } |
| 3290 | |
| 3291 | // Translate the iterator region into the loop body. |
| 3292 | if (iteratorRegion.hasOneBlock()) { |
| 3293 | moduleTranslation.mapBlock(mlir: &entryBlock, llvm: builder.GetInsertBlock()); |
| 3294 | return moduleTranslation.convertBlock(bb&: entryBlock, /*ignoreArguments=*/true, |
| 3295 | builder); |
| 3296 | } |
| 3297 | |
| 3298 | // Unlike convertOmpOpRegions, do not forward the yielded value through a PHI: |
| 3299 | // it may have no LLVM type (e.g. `!omp.affinity_entry_ty`). The unique |
| 3300 | // `omp.yield` block is the continuation's only predecessor, so callers can |
| 3301 | // look up the yielded value directly. |
| 3302 | llvm::BasicBlock *continuation = |
| 3303 | splitBB(Builder&: builder, CreateBranch: true, Name: "omp.iterator.region.cont" ); |
| 3304 | llvm::Instruction *sourceBranch = builder.GetInsertBlock()->getTerminator(); |
| 3305 | for (mlir::Block &block : iteratorRegion) |
| 3306 | moduleTranslation.mapBlock( |
| 3307 | mlir: &block, |
| 3308 | llvm: llvm::BasicBlock::Create(Context&: builder.getContext(), Name: "omp.iterator.region" , |
| 3309 | Parent: continuation->getParent(), InsertBefore: continuation)); |
| 3310 | sourceBranch->setSuccessor(Idx: 0, BB: moduleTranslation.lookupBlock(block: &entryBlock)); |
| 3311 | |
| 3312 | for (mlir::Block *block : getBlocksSortedByDominance(region&: iteratorRegion)) { |
| 3313 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 3314 | if (failed(Result: moduleTranslation.convertBlock(bb&: *block, ignoreArguments: block->isEntryBlock(), |
| 3315 | builder))) |
| 3316 | return mlir::failure(); |
| 3317 | if (isa<omp::YieldOp>(Val: block->getTerminator())) |
| 3318 | builder.CreateBr(Dest: continuation); |
| 3319 | } |
| 3320 | LLVM::detail::connectPHINodes(region&: iteratorRegion, state: moduleTranslation); |
| 3321 | |
| 3322 | builder.SetInsertPoint(continuation); |
| 3323 | return mlir::success(); |
| 3324 | } |
| 3325 | |
| 3326 | using IteratorStoreEntryTy = |
| 3327 | llvm::function_ref<void(llvm::Value *linearIV, mlir::omp::YieldOp yield)>; |
| 3328 | |
| 3329 | static mlir::LogicalResult |
| 3330 | fillIteratorLoop(mlir::omp::IteratorOp itersOp, llvm::IRBuilderBase &builder, |
| 3331 | mlir::LLVM::ModuleTranslation &moduleTranslation, |
| 3332 | IteratorInfo &iterInfo, llvm::StringRef loopName, |
| 3333 | IteratorStoreEntryTy genStoreEntry) { |
| 3334 | mlir::Region &itersRegion = itersOp.getRegion(); |
| 3335 | |
| 3336 | llvm::OpenMPIRBuilder::LocationDescription loc(builder); |
| 3337 | |
| 3338 | auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy bodyIP, |
| 3339 | llvm::Value *linearIV) -> llvm::Error { |
| 3340 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 3341 | builder.restoreIP(IP: bodyIP); |
| 3342 | |
| 3343 | if (failed(Result: convertIteratorRegion(linearIV, iterInfo, iteratorRegion&: itersRegion, builder, |
| 3344 | moduleTranslation))) { |
| 3345 | return llvm::make_error<llvm::StringError>( |
| 3346 | Args: "failed to convert iterator region" , Args: llvm::inconvertibleErrorCode()); |
| 3347 | } |
| 3348 | |
| 3349 | mlir::omp::YieldOp yield = itersOp.getYieldOp(); |
| 3350 | assert(yield && yield.getResults().size() == 1 && |
| 3351 | "expect omp.yield in iterator region to have one result" ); |
| 3352 | |
| 3353 | genStoreEntry(linearIV, yield); |
| 3354 | |
| 3355 | // Iterator-region block/value mappings are temporary for this conversion, |
| 3356 | // clear them to avoid stale entries in ModuleTranslation. |
| 3357 | moduleTranslation.forgetMapping(region&: itersRegion); |
| 3358 | |
| 3359 | return llvm::Error::success(); |
| 3360 | }; |
| 3361 | |
| 3362 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 3363 | getOpenMPBuilder(moduleTranslation) |
| 3364 | ->createIteratorLoop(Loc: loc, TripCount: iterInfo.getTotalTrips(), BodyGen: bodyGen, |
| 3365 | Name: loopName); |
| 3366 | if (failed(Result: handleError(result&: afterIP, op&: *itersOp))) |
| 3367 | return failure(); |
| 3368 | |
| 3369 | builder.restoreIP(IP: *afterIP); |
| 3370 | |
| 3371 | return mlir::success(); |
| 3372 | } |
| 3373 | |
| 3374 | static mlir::LogicalResult |
| 3375 | buildAffinityData(mlir::omp::TaskOp &taskOp, llvm::IRBuilderBase &builder, |
| 3376 | mlir::LLVM::ModuleTranslation &moduleTranslation, |
| 3377 | llvm::OpenMPIRBuilder::AffinityData &ad) { |
| 3378 | |
| 3379 | if (taskOp.getAffinityVars().empty() && taskOp.getIterated().empty()) { |
| 3380 | ad.Count = nullptr; |
| 3381 | ad.Info = nullptr; |
| 3382 | return mlir::success(); |
| 3383 | } |
| 3384 | |
| 3385 | llvm::SmallVector<llvm::OpenMPIRBuilder::AffinityData> ads; |
| 3386 | llvm::StructType *kmpTaskAffinityInfoTy = |
| 3387 | getOpenMPBuilder(moduleTranslation)->getKmpTaskAffinityInfoTy(); |
| 3388 | |
| 3389 | auto allocateAffinityList = [&](llvm::Value *count) -> llvm::Value * { |
| 3390 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 3391 | if (llvm::isa<llvm::Constant>(Val: count) || llvm::isa<llvm::Argument>(Val: count)) |
| 3392 | builder.restoreIP(IP: findAllocInsertPoints(builder, moduleTranslation)); |
| 3393 | return builder.CreateAlloca(Ty: kmpTaskAffinityInfoTy, ArraySize: count, |
| 3394 | Name: "omp.affinity_list" ); |
| 3395 | }; |
| 3396 | |
| 3397 | auto createAffinity = |
| 3398 | [&](llvm::Value *count, |
| 3399 | llvm::Value *info) -> llvm::OpenMPIRBuilder::AffinityData { |
| 3400 | llvm::OpenMPIRBuilder::AffinityData ad{}; |
| 3401 | ad.Count = builder.CreateTrunc(V: count, DestTy: builder.getInt32Ty()); |
| 3402 | ad.Info = |
| 3403 | builder.CreatePointerBitCastOrAddrSpaceCast(V: info, DestTy: builder.getPtrTy(AddrSpace: 0)); |
| 3404 | return ad; |
| 3405 | }; |
| 3406 | |
| 3407 | if (!taskOp.getAffinityVars().empty()) { |
| 3408 | llvm::Value *count = llvm::ConstantInt::get( |
| 3409 | Ty: builder.getInt64Ty(), V: taskOp.getAffinityVars().size()); |
| 3410 | llvm::Value *list = allocateAffinityList(count); |
| 3411 | fillAffinityLocators(affinityVars: taskOp.getAffinityVars(), builder, moduleTranslation, |
| 3412 | affinityList: list); |
| 3413 | ads.emplace_back(Args: createAffinity(count, list)); |
| 3414 | } |
| 3415 | |
| 3416 | if (!taskOp.getIterated().empty()) { |
| 3417 | for (auto [i, iter] : llvm::enumerate(First: taskOp.getIterated())) { |
| 3418 | auto itersOp = iter.getDefiningOp<omp::IteratorOp>(); |
| 3419 | assert(itersOp && "iterated value must be defined by omp.iterator" ); |
| 3420 | IteratorInfo iterInfo(itersOp, moduleTranslation, builder); |
| 3421 | llvm::Value *affList = allocateAffinityList(iterInfo.getTotalTrips()); |
| 3422 | if (failed(Result: fillIteratorLoop( |
| 3423 | itersOp, builder, moduleTranslation, iterInfo, loopName: "iterator" , |
| 3424 | genStoreEntry: [&](llvm::Value *linearIV, mlir::omp::YieldOp yield) { |
| 3425 | auto entryOp = yield.getResults()[0] |
| 3426 | .getDefiningOp<mlir::omp::AffinityEntryOp>(); |
| 3427 | assert(entryOp && "expect yield produce an affinity entry" ); |
| 3428 | llvm::Value *addr = |
| 3429 | moduleTranslation.lookupValue(value: entryOp.getAddr()); |
| 3430 | llvm::Value *len = |
| 3431 | moduleTranslation.lookupValue(value: entryOp.getLen()); |
| 3432 | storeAffinityEntry(builder, |
| 3433 | ompBuilder&: *getOpenMPBuilder(moduleTranslation), |
| 3434 | affinityList: affList, index: linearIV, addr, len); |
| 3435 | }))) |
| 3436 | return llvm::failure(); |
| 3437 | ads.emplace_back(Args: createAffinity(iterInfo.getTotalTrips(), affList)); |
| 3438 | } |
| 3439 | } |
| 3440 | |
| 3441 | llvm::Value *totalAffinityCount = builder.getInt32(C: 0); |
| 3442 | for (const auto &affinity : ads) |
| 3443 | totalAffinityCount = builder.CreateAdd( |
| 3444 | LHS: totalAffinityCount, |
| 3445 | RHS: builder.CreateIntCast(V: affinity.Count, DestTy: builder.getInt32Ty(), |
| 3446 | /*isSigned=*/false)); |
| 3447 | |
| 3448 | llvm::Value *affinityInfo = ads.front().Info; |
| 3449 | if (ads.size() > 1) { |
| 3450 | llvm::StructType *kmpTaskAffinityInfoTy = |
| 3451 | getOpenMPBuilder(moduleTranslation)->getKmpTaskAffinityInfoTy(); |
| 3452 | llvm::Value *affinityInfoElemSize = builder.getInt64( |
| 3453 | C: moduleTranslation.getLLVMModule()->getDataLayout().getTypeAllocSize( |
| 3454 | Ty: kmpTaskAffinityInfoTy)); |
| 3455 | |
| 3456 | llvm::Value *packedAffinityInfo = allocateAffinityList(totalAffinityCount); |
| 3457 | llvm::Value *packedAffinityInfoOffset = builder.getInt32(C: 0); |
| 3458 | for (const auto &affinity : ads) { |
| 3459 | llvm::Value *affinityCount = builder.CreateIntCast( |
| 3460 | V: affinity.Count, DestTy: builder.getInt32Ty(), /*isSigned=*/false); |
| 3461 | llvm::Value *affinityCountInt64 = builder.CreateIntCast( |
| 3462 | V: affinityCount, DestTy: builder.getInt64Ty(), /*isSigned=*/false); |
| 3463 | llvm::Value *affinityInfoSize = |
| 3464 | builder.CreateMul(LHS: affinityCountInt64, RHS: affinityInfoElemSize); |
| 3465 | |
| 3466 | llvm::Value *packedAffinityInfoIndex = builder.CreateIntCast( |
| 3467 | V: packedAffinityInfoOffset, DestTy: kmpTaskAffinityInfoTy->getElementType(N: 0), |
| 3468 | /*isSigned=*/false); |
| 3469 | packedAffinityInfoIndex = builder.CreateInBoundsGEP( |
| 3470 | Ty: kmpTaskAffinityInfoTy, Ptr: packedAffinityInfo, IdxList: packedAffinityInfoIndex); |
| 3471 | |
| 3472 | builder.CreateMemCpy( |
| 3473 | Dst: packedAffinityInfoIndex, DstAlign: llvm::Align(1), |
| 3474 | Src: builder.CreatePointerBitCastOrAddrSpaceCast( |
| 3475 | V: affinity.Info, DestTy: builder.getPtrTy(AddrSpace: packedAffinityInfoIndex->getType() |
| 3476 | ->getPointerAddressSpace())), |
| 3477 | SrcAlign: llvm::Align(1), Size: affinityInfoSize); |
| 3478 | |
| 3479 | packedAffinityInfoOffset = |
| 3480 | builder.CreateAdd(LHS: packedAffinityInfoOffset, RHS: affinityCount); |
| 3481 | } |
| 3482 | |
| 3483 | affinityInfo = packedAffinityInfo; |
| 3484 | } |
| 3485 | |
| 3486 | ad.Count = totalAffinityCount; |
| 3487 | ad.Info = affinityInfo; |
| 3488 | |
| 3489 | return mlir::success(); |
| 3490 | } |
| 3491 | |
| 3492 | // Allocates a single kmp_dep_info array sized to hold both locator |
| 3493 | // (non-iterated) and iterated entries, fills the locator entries first, then |
| 3494 | // runs an iterator loop for each iterator modifier object. |
| 3495 | static mlir::LogicalResult |
| 3496 | buildDependData(OperandRange dependVars, std::optional<ArrayAttr> dependKinds, |
| 3497 | OperandRange dependIterated, |
| 3498 | std::optional<ArrayAttr> dependIteratedKinds, |
| 3499 | llvm::IRBuilderBase &builder, |
| 3500 | mlir::LLVM::ModuleTranslation &moduleTranslation, |
| 3501 | llvm::OpenMPIRBuilder::DependenciesInfo &taskDeps) { |
| 3502 | if (dependIterated.empty()) { |
| 3503 | buildDependDataLocator(dependKinds, dependVars, moduleTranslation, |
| 3504 | dds&: taskDeps.Deps); |
| 3505 | return mlir::success(); |
| 3506 | } |
| 3507 | |
| 3508 | llvm::OpenMPIRBuilder &ompBuilder = *getOpenMPBuilder(moduleTranslation); |
| 3509 | llvm::Type *dependInfoTy = ompBuilder.DependInfo; |
| 3510 | unsigned numLocator = dependVars.size(); |
| 3511 | |
| 3512 | // Compute total count: locator deps + sum of iterator trip counts. |
| 3513 | llvm::Value *totalCount = |
| 3514 | llvm::ConstantInt::get(Ty: builder.getInt64Ty(), V: numLocator); |
| 3515 | |
| 3516 | llvm::SmallVector<IteratorInfo, 2> iterInfos; |
| 3517 | for (auto iter : dependIterated) { |
| 3518 | auto itersOp = iter.getDefiningOp<mlir::omp::IteratorOp>(); |
| 3519 | assert(itersOp && "depend_iterated value must be defined by omp.iterator" ); |
| 3520 | iterInfos.emplace_back(Args&: itersOp, Args&: moduleTranslation, Args&: builder); |
| 3521 | totalCount = |
| 3522 | builder.CreateAdd(LHS: totalCount, RHS: iterInfos.back().getTotalTrips()); |
| 3523 | } |
| 3524 | |
| 3525 | // Heap-allocate the kmp_depend_info array so we don't risk |
| 3526 | // dynamic-sized alloca outside the entry block (e.g. inside loops). |
| 3527 | llvm::DataLayout dataLayout = builder.getDataLayout(); |
| 3528 | llvm::Value *allocSize = builder.CreateTypeSize( |
| 3529 | Ty: ompBuilder.SizeTy, Size: dataLayout.getTypeAllocSize(Ty: dependInfoTy)); |
| 3530 | llvm::Value *depArray = |
| 3531 | builder.CreateMalloc(IntPtrTy: ompBuilder.SizeTy, AllocSize: allocSize, ArraySize: totalCount, |
| 3532 | /*MallocF=*/nullptr, Name: ".dep.arr.addr" ); |
| 3533 | |
| 3534 | // Fill non-iterated entries at indices [0, numLocator). |
| 3535 | if (numLocator > 0) { |
| 3536 | SmallVector<llvm::OpenMPIRBuilder::DependData> dds; |
| 3537 | buildDependDataLocator(dependKinds, dependVars, moduleTranslation, dds); |
| 3538 | for (auto [i, dd] : llvm::enumerate(First&: dds)) { |
| 3539 | llvm::Value *idx = llvm::ConstantInt::get(Ty: builder.getInt64Ty(), V: i); |
| 3540 | llvm::Value *entry = |
| 3541 | builder.CreateInBoundsGEP(Ty: dependInfoTy, Ptr: depArray, IdxList: idx); |
| 3542 | ompBuilder.emitTaskDependency(Builder&: builder, Entry: entry, Dep: dd); |
| 3543 | } |
| 3544 | } |
| 3545 | |
| 3546 | // Fill iterated entries starting at index numLocator. |
| 3547 | llvm::Value *offset = |
| 3548 | llvm::ConstantInt::get(Ty: builder.getInt64Ty(), V: numLocator); |
| 3549 | for (auto [i, iterInfo] : llvm::enumerate(First&: iterInfos)) { |
| 3550 | auto kindAttr = cast<mlir::omp::ClauseTaskDependAttr>( |
| 3551 | Val: dependIteratedKinds->getValue()[i]); |
| 3552 | llvm::omp::RTLDependenceKindTy rtlKind = |
| 3553 | convertDependKind(kind: kindAttr.getValue()); |
| 3554 | |
| 3555 | auto itersOp = dependIterated[i].getDefiningOp<mlir::omp::IteratorOp>(); |
| 3556 | if (failed(Result: fillIteratorLoop( |
| 3557 | itersOp, builder, moduleTranslation, iterInfo, loopName: "dep_iterator" , |
| 3558 | genStoreEntry: [&](llvm::Value *linearIV, mlir::omp::YieldOp yield) { |
| 3559 | llvm::Value *addr = |
| 3560 | moduleTranslation.lookupValue(value: yield.getResults()[0]); |
| 3561 | llvm::Value *idx = builder.CreateAdd(LHS: offset, RHS: linearIV); |
| 3562 | llvm::Value *entry = |
| 3563 | builder.CreateInBoundsGEP(Ty: dependInfoTy, Ptr: depArray, IdxList: idx); |
| 3564 | ompBuilder.emitTaskDependency( |
| 3565 | Builder&: builder, Entry: entry, |
| 3566 | Dep: llvm::OpenMPIRBuilder::DependData{rtlKind, addr->getType(), |
| 3567 | addr}); |
| 3568 | }))) |
| 3569 | return mlir::failure(); |
| 3570 | |
| 3571 | // Advance offset by the trip count of this iterator. |
| 3572 | offset = builder.CreateAdd(LHS: offset, RHS: iterInfo.getTotalTrips()); |
| 3573 | } |
| 3574 | |
| 3575 | taskDeps.DepArray = depArray; |
| 3576 | taskDeps.NumDeps = builder.CreateTrunc(V: totalCount, DestTy: builder.getInt32Ty()); |
| 3577 | return mlir::success(); |
| 3578 | } |
| 3579 | |
| 3580 | /// Converts an OpenMP task construct into LLVM IR using OpenMPIRBuilder. |
| 3581 | static LogicalResult |
| 3582 | convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder, |
| 3583 | LLVM::ModuleTranslation &moduleTranslation) { |
| 3584 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 3585 | if (failed(Result: checkImplementationStatus(op&: *taskOp))) |
| 3586 | return failure(); |
| 3587 | |
| 3588 | PrivateVarsInfo privateVarsInfo(taskOp); |
| 3589 | TaskContextStructManager taskStructMgr{builder, moduleTranslation, |
| 3590 | privateVarsInfo}; |
| 3591 | |
| 3592 | // Allocate and copy private variables before creating the task. This avoids |
| 3593 | // accessing invalid memory if (after this scope ends) the private variables |
| 3594 | // are initialized from host variables or if the variables are copied into |
| 3595 | // from host variables (firstprivate). The insertion point is just before |
| 3596 | // where the code for creating and scheduling the task will go. That puts this |
| 3597 | // code outside of the outlined task region, which is what we want because |
| 3598 | // this way the initialization and copy regions are executed immediately while |
| 3599 | // the host variable data are still live. |
| 3600 | llvm::SmallVector<llvm::BasicBlock *> deallocBlocks; |
| 3601 | InsertPointTy allocaIP = |
| 3602 | findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks); |
| 3603 | |
| 3604 | // Not using splitBB() because that requires the current block to have a |
| 3605 | // terminator. |
| 3606 | assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end()); |
| 3607 | llvm::BasicBlock *taskStartBlock = llvm::BasicBlock::Create( |
| 3608 | Context&: builder.getContext(), Name: "omp.task.start" , |
| 3609 | /*Parent=*/builder.GetInsertBlock()->getParent()); |
| 3610 | llvm::Instruction *branchToTaskStartBlock = builder.CreateBr(Dest: taskStartBlock); |
| 3611 | builder.SetInsertPoint(branchToTaskStartBlock); |
| 3612 | |
| 3613 | // Now do this again to make the initialization and copy blocks |
| 3614 | llvm::BasicBlock *copyBlock = |
| 3615 | splitBB(Builder&: builder, /*CreateBranch=*/true, Name: "omp.private.copy" ); |
| 3616 | llvm::BasicBlock *initBlock = |
| 3617 | splitBB(Builder&: builder, /*CreateBranch=*/true, Name: "omp.private.init" ); |
| 3618 | |
| 3619 | // Now the control flow graph should look like |
| 3620 | // starter_block: |
| 3621 | // <---- where we started when convertOmpTaskOp was called |
| 3622 | // br %omp.private.init |
| 3623 | // omp.private.init: |
| 3624 | // br %omp.private.copy |
| 3625 | // omp.private.copy: |
| 3626 | // br %omp.task.start |
| 3627 | // omp.task.start: |
| 3628 | // <---- where we want the insertion point to be when we call createTask() |
| 3629 | |
| 3630 | // Save the alloca insertion point on ModuleTranslation stack for use in |
| 3631 | // nested regions. |
| 3632 | LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame( |
| 3633 | moduleTranslation, allocaIP.getNodeParent(), deallocBlocks); |
| 3634 | |
| 3635 | // Allocate and initialize private variables |
| 3636 | builder.SetInsertPoint(initBlock->getTerminator()); |
| 3637 | |
| 3638 | // Create task variable structure |
| 3639 | taskStructMgr.generateTaskContextStruct(); |
| 3640 | // GEPs so that we can initialize the variables. Don't use these GEPs inside |
| 3641 | // of the body otherwise it will be the GEP not the struct which is fowarded |
| 3642 | // to the outlined function. GEPs forwarded in this way are passed in a |
| 3643 | // stack-allocated (by OpenMPIRBuilder) structure which is not safe for tasks |
| 3644 | // which may not be executed until after the current stack frame goes out of |
| 3645 | // scope. |
| 3646 | taskStructMgr.createGEPsToPrivateVars(); |
| 3647 | |
| 3648 | for (auto [var, llvmPrivateVarAlloc] : llvm::zip_equal( |
| 3649 | t&: privateVarsInfo.vars, u: taskStructMgr.getLLVMPrivateVarGEPs())) { |
| 3650 | // To be handled inside the task. |
| 3651 | if (!var.privatizer.readsFromMold()) |
| 3652 | continue; |
| 3653 | assert(llvmPrivateVarAlloc && |
| 3654 | "reads from mold so shouldn't have been skipped" ); |
| 3655 | |
| 3656 | llvm::Expected<llvm::Value *> privateVarOrErr = |
| 3657 | initPrivateVar(builder, moduleTranslation, privDecl&: var.privatizer, mlirPrivVar: var.mlirVar, |
| 3658 | blockArg&: var.blockArg, llvmPrivateVar: llvmPrivateVarAlloc, privInitBlock: initBlock); |
| 3659 | if (!privateVarOrErr) |
| 3660 | return handleError(result&: privateVarOrErr, op&: *taskOp.getOperation()); |
| 3661 | |
| 3662 | setInsertPointForPossiblyEmptyBlock(builder); |
| 3663 | |
| 3664 | // TODO: this is a bit of a hack for Fortran character boxes. |
| 3665 | // Character boxes are passed by value into the init region and then the |
| 3666 | // initialized character box is yielded by value. Here we need to store the |
| 3667 | // yielded value into the private allocation, and load the private |
| 3668 | // allocation to match the type expected by region block arguments. |
| 3669 | [[maybe_unused]] llvm::Value *llvmPrivateVar = llvmPrivateVarAlloc; |
| 3670 | if ((privateVarOrErr.get() != llvmPrivateVarAlloc) && |
| 3671 | !mlir::isa<LLVM::LLVMPointerType>(Val: var.blockArg.getType())) { |
| 3672 | builder.CreateStore(Val: privateVarOrErr.get(), Ptr: llvmPrivateVarAlloc); |
| 3673 | // Load it so we have the value pointed to by the GEP |
| 3674 | llvmPrivateVar = builder.CreateLoad(Ty: privateVarOrErr.get()->getType(), |
| 3675 | Ptr: llvmPrivateVarAlloc); |
| 3676 | } |
| 3677 | assert(llvmPrivateVar->getType() == |
| 3678 | moduleTranslation.convertType(var.blockArg.getType())); |
| 3679 | |
| 3680 | // Mapping blockArg -> llvmPrivateVarAlloc is done inside the body callback |
| 3681 | // so that OpenMPIRBuilder doesn't try to pass each GEP address through a |
| 3682 | // stack allocated structure. |
| 3683 | } |
| 3684 | |
| 3685 | // firstprivate copy region |
| 3686 | setInsertPointForPossiblyEmptyBlock(builder, block: copyBlock); |
| 3687 | if (failed(Result: completePrivateVars( |
| 3688 | op: taskOp, builder, moduleTranslation, privateVarsInfo, |
| 3689 | insertBarrier: taskOp.getPrivateNeedsBarrier(), |
| 3690 | /*moldVars=*/{}, llvmPrivateVars: taskStructMgr.getLLVMPrivateVarGEPs()))) |
| 3691 | return llvm::failure(); |
| 3692 | |
| 3693 | llvm::OpenMPIRBuilder::AffinityData ad; |
| 3694 | if (failed(Result: buildAffinityData(taskOp, builder, moduleTranslation, ad))) |
| 3695 | return llvm::failure(); |
| 3696 | |
| 3697 | // Resolve and validate in_reduction declarations. Byref in_reduction has |
| 3698 | // already been rejected by checkImplementationStatus; the helper rejects the |
| 3699 | // remaining richer declare_reduction shapes (two-argument initializer, |
| 3700 | // cleanup region, missing combiner). This is pure MLIR symbol-table work and |
| 3701 | // emits no IR. The matching task_reduction descriptor is registered by an |
| 3702 | // enclosing taskgroup; here we only look the per-task storage up at runtime. |
| 3703 | SmallVector<omp::DeclareReductionOp> inRedDecls; |
| 3704 | if (failed(Result: collectAndValidateTaskloopRedDecls( |
| 3705 | contextOp: taskOp.getOperation(), syms: taskOp.getInReductionSyms(), opName: "omp.task" , |
| 3706 | clauseName: "in_reduction" , out&: inRedDecls))) |
| 3707 | return failure(); |
| 3708 | SmallVector<llvm::Value *> inRedOrigPtrs; |
| 3709 | inRedOrigPtrs.reserve(N: inRedDecls.size()); |
| 3710 | for (Value v : taskOp.getInReductionVars()) |
| 3711 | inRedOrigPtrs.push_back(Elt: moduleTranslation.lookupValue(value: v)); |
| 3712 | |
| 3713 | // Set up for call to createTask() |
| 3714 | builder.SetInsertPoint(taskStartBlock); |
| 3715 | |
| 3716 | auto bodyCB = |
| 3717 | [&](InsertPointTy allocaIP, InsertPointTy codegenIP, |
| 3718 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error { |
| 3719 | // Save the alloca insertion point on ModuleTranslation stack for use in |
| 3720 | // nested regions. |
| 3721 | LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame( |
| 3722 | moduleTranslation, allocaIP.getNodeParent(), deallocBlocks); |
| 3723 | |
| 3724 | // translate the body of the task: |
| 3725 | builder.restoreIP(IP: codegenIP); |
| 3726 | |
| 3727 | llvm::BasicBlock *privInitBlock = nullptr; |
| 3728 | for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) { |
| 3729 | // This is handled before the task executes |
| 3730 | if (var.privatizer.readsFromMold()) |
| 3731 | continue; |
| 3732 | |
| 3733 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 3734 | llvm::Type *llvmAllocType = |
| 3735 | moduleTranslation.convertType(type: var.privatizer.getType()); |
| 3736 | builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator()); |
| 3737 | llvm::Value *llvmPrivateVar = builder.CreateAlloca( |
| 3738 | Ty: llvmAllocType, /*ArraySize=*/nullptr, Name: "omp.private.alloc" ); |
| 3739 | |
| 3740 | llvm::Expected<llvm::Value *> privateVarOrError = initPrivateVar( |
| 3741 | builder, moduleTranslation, privDecl&: var.privatizer, mlirPrivVar: var.mlirVar, blockArg&: var.blockArg, |
| 3742 | llvmPrivateVar, privInitBlock); |
| 3743 | if (!privateVarOrError) |
| 3744 | return privateVarOrError.takeError(); |
| 3745 | moduleTranslation.mapValue(mlir: var.blockArg, llvm: privateVarOrError.get()); |
| 3746 | var.llvmVar = privateVarOrError.get(); |
| 3747 | } |
| 3748 | |
| 3749 | taskStructMgr.createGEPsToPrivateVars(); |
| 3750 | for (auto [i, llvmPrivVar] : |
| 3751 | llvm::enumerate(First: taskStructMgr.getLLVMPrivateVarGEPs())) { |
| 3752 | if (!llvmPrivVar) { |
| 3753 | assert(privateVarsInfo.vars[i].llvmVar && |
| 3754 | "This is added in the loop above" ); |
| 3755 | continue; |
| 3756 | } |
| 3757 | privateVarsInfo.vars[i].llvmVar = llvmPrivVar; |
| 3758 | } |
| 3759 | |
| 3760 | // Find and map the addresses of each variable within the task context |
| 3761 | // structure |
| 3762 | for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) { |
| 3763 | // This was handled above. |
| 3764 | if (!var.privatizer.readsFromMold()) |
| 3765 | continue; |
| 3766 | // Fix broken pass-by-value case for Fortran character boxes |
| 3767 | if (!mlir::isa<LLVM::LLVMPointerType>(Val: var.blockArg.getType())) { |
| 3768 | var.llvmVar = builder.CreateLoad( |
| 3769 | Ty: moduleTranslation.convertType(type: var.blockArg.getType()), Ptr: var.llvmVar); |
| 3770 | } |
| 3771 | assert(var.llvmVar->getType() == |
| 3772 | moduleTranslation.convertType(var.blockArg.getType())); |
| 3773 | moduleTranslation.mapValue(mlir: var.blockArg, llvm: var.llvmVar); |
| 3774 | } |
| 3775 | |
| 3776 | // Map in_reduction block arguments to the per-task private storage returned |
| 3777 | // by __kmpc_task_reduction_get_th_data. This call must be emitted inside |
| 3778 | // the to-be-outlined task body so that it returns the *executing* thread's |
| 3779 | // gtid (not the encountering thread's). The descriptor is NULL: the runtime |
| 3780 | // walks up enclosing taskgroups to find the matching task_reduction |
| 3781 | // registration for `origPtr`. The original pointers are auto-captured into |
| 3782 | // the task shareds aggregate by CodeExtractor during |
| 3783 | // OpenMPIRBuilder::finalize. |
| 3784 | if (!inRedDecls.empty()) { |
| 3785 | auto iface = cast<omp::BlockArgOpenMPOpInterface>(Val: taskOp.getOperation()); |
| 3786 | llvm::OpenMPIRBuilder &ompB = *getOpenMPBuilder(moduleTranslation); |
| 3787 | llvm::Module *m = moduleTranslation.getLLVMModule(); |
| 3788 | llvm::LLVMContext &llvmCtx = m->getContext(); |
| 3789 | llvm::OpenMPIRBuilder::LocationDescription bodyLoc(builder); |
| 3790 | uint32_t srcLocSize; |
| 3791 | llvm::Constant *srcLocStr = |
| 3792 | ompB.getOrCreateSrcLocStr(Loc: bodyLoc, SrcLocStrSize&: srcLocSize); |
| 3793 | llvm::Value *bodyIdent = ompB.getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize); |
| 3794 | // Align OpenMPIRBuilder's internal IRBuilder with `builder` so the gtid |
| 3795 | // call lands inside the to-be-outlined task body. |
| 3796 | ompB.updateToLocation(Loc: bodyLoc); |
| 3797 | llvm::Value *bodyGtid = ompB.getOrCreateThreadID(Ident: bodyIdent); |
| 3798 | llvm::FunctionCallee getThData = ompB.getOrCreateRuntimeFunction( |
| 3799 | M&: *m, FnID: llvm::omp::OMPRTL___kmpc_task_reduction_get_th_data); |
| 3800 | llvm::Type *ptrTy = llvm::PointerType::getUnqual(C&: llvmCtx); |
| 3801 | llvm::Value *nullDesc = llvm::ConstantPointerNull::get(T: ptrTy); |
| 3802 | ArrayRef<BlockArgument> inRedBlockArgs = iface.getInReductionBlockArgs(); |
| 3803 | for (auto [blockArg, origPtr] : |
| 3804 | llvm::zip_equal(t&: inRedBlockArgs, u&: inRedOrigPtrs)) { |
| 3805 | // __kmpc_task_reduction_get_th_data takes and returns a generic, |
| 3806 | // default-address-space `ptr`. Normalize a non-default-address-space |
| 3807 | // original pointer to the generic address space before the call, and |
| 3808 | // cast the returned private pointer back to the block argument's |
| 3809 | // address space when it differs (mirrors the taskloop reduction |
| 3810 | // remapping in convertOmpTaskloopContextOp). |
| 3811 | llvm::Value *lookupPtr = origPtr; |
| 3812 | if (auto *origPtrTy = |
| 3813 | llvm::dyn_cast<llvm::PointerType>(Val: lookupPtr->getType()); |
| 3814 | origPtrTy && origPtrTy->getAddressSpace() != 0) |
| 3815 | lookupPtr = builder.CreateAddrSpaceCast(V: lookupPtr, DestTy: ptrTy); |
| 3816 | llvm::Value *priv = builder.CreateCall( |
| 3817 | Callee: getThData, Args: {bodyGtid, nullDesc, lookupPtr}, Name: "omp.inred.priv" ); |
| 3818 | if (auto *argPtrTy = llvm::dyn_cast<llvm::PointerType>( |
| 3819 | Val: moduleTranslation.convertType(type: blockArg.getType())); |
| 3820 | argPtrTy && argPtrTy->getAddressSpace() != 0) |
| 3821 | priv = builder.CreateAddrSpaceCast(V: priv, DestTy: argPtrTy); |
| 3822 | moduleTranslation.mapValue(mlir: blockArg, llvm: priv); |
| 3823 | } |
| 3824 | } |
| 3825 | |
| 3826 | auto continuationBlockOrError = convertOmpOpRegions( |
| 3827 | region&: taskOp.getRegion(), blockName: "omp.task.region" , builder, moduleTranslation); |
| 3828 | if (failed(Result: handleError(result&: continuationBlockOrError, op&: *taskOp))) |
| 3829 | return llvm::make_error<PreviouslyReportedError>(); |
| 3830 | |
| 3831 | builder.SetInsertPoint(continuationBlockOrError.get()->getTerminator()); |
| 3832 | |
| 3833 | if (failed(Result: cleanupPrivateVars(op: taskOp, builder, moduleTranslation, |
| 3834 | loc: taskOp.getLoc(), privateVarsInfo))) |
| 3835 | return llvm::make_error<PreviouslyReportedError>(); |
| 3836 | |
| 3837 | // Free heap allocated task context structure at the end of the task. |
| 3838 | taskStructMgr.freeStructPtr(); |
| 3839 | |
| 3840 | return llvm::Error::success(); |
| 3841 | }; |
| 3842 | |
| 3843 | llvm::OpenMPIRBuilder &ompBuilder = *getOpenMPBuilder(moduleTranslation); |
| 3844 | SmallVector<llvm::UncondBrInst *> cancelTerminators; |
| 3845 | // The directive to match here is OMPD_taskgroup because it is the taskgroup |
| 3846 | // which is canceled. This is handled here because it is the task's cleanup |
| 3847 | // block which should be branched to. |
| 3848 | pushCancelFinalizationCB(cancelTerminators, llvmBuilder&: builder, ompBuilder, op: taskOp, |
| 3849 | cancelDirective: llvm::omp::Directive::OMPD_taskgroup); |
| 3850 | |
| 3851 | llvm::OpenMPIRBuilder::DependenciesInfo dependencies; |
| 3852 | if (failed(Result: buildDependData(dependVars: taskOp.getDependVars(), dependKinds: taskOp.getDependKinds(), |
| 3853 | dependIterated: taskOp.getDependIterated(), |
| 3854 | dependIteratedKinds: taskOp.getDependIteratedKinds(), builder, |
| 3855 | moduleTranslation, taskDeps&: dependencies))) |
| 3856 | return failure(); |
| 3857 | |
| 3858 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 3859 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 3860 | getOpenMPBuilder(moduleTranslation) |
| 3861 | ->createTask(Loc: ompLoc, AllocaIP: allocaIP, DeallocBlocks: deallocBlocks, BodyGenCB: bodyCB, |
| 3862 | Tied: !taskOp.getUntied(), |
| 3863 | Final: moduleTranslation.lookupValue(value: taskOp.getFinal()), |
| 3864 | IfCondition: moduleTranslation.lookupValue(value: taskOp.getIfExpr()), |
| 3865 | Dependencies: dependencies, Affinities: ad, Mergeable: taskOp.getMergeable(), |
| 3866 | EventHandle: moduleTranslation.lookupValue(value: taskOp.getEventHandle()), |
| 3867 | Priority: moduleTranslation.lookupValue(value: taskOp.getPriority()), |
| 3868 | FreeAgent: taskOp.getThreadset() == omp::ThreadsetPolicy::omp_pool); |
| 3869 | |
| 3870 | if (failed(Result: handleError(result&: afterIP, op&: *taskOp))) |
| 3871 | return failure(); |
| 3872 | |
| 3873 | // Set the correct branch target for task cancellation |
| 3874 | popCancelFinalizationCB(cancelTerminators, ompBuilder, |
| 3875 | afterBB: afterIP->getNodeParent()); |
| 3876 | |
| 3877 | builder.restoreIP(IP: *afterIP); |
| 3878 | |
| 3879 | if (dependencies.DepArray) |
| 3880 | builder.CreateFree(Source: dependencies.DepArray); |
| 3881 | |
| 3882 | return success(); |
| 3883 | } |
| 3884 | |
| 3885 | /// The correct entry point is convertOmpTaskloopContextOp. This gets called |
| 3886 | /// whilst lowering the body of the taskloop context (i.e. the task function). |
| 3887 | static LogicalResult |
| 3888 | convertOmpTaskloopWrapperOp(omp::TaskloopWrapperOp loopWrapperOp, |
| 3889 | llvm::IRBuilderBase &builder, |
| 3890 | LLVM::ModuleTranslation &moduleTranslation) { |
| 3891 | mlir::Operation &opInst = *loopWrapperOp.getOperation(); |
| 3892 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 3893 | return failure(); |
| 3894 | |
| 3895 | // Recurse into the loop body. |
| 3896 | auto continuationBlockOrError = convertOmpOpRegions( |
| 3897 | region&: loopWrapperOp.getRegion(), blockName: "omp.taskloop.wrapper.region" , builder, |
| 3898 | moduleTranslation); |
| 3899 | |
| 3900 | if (failed(Result: handleError(result&: continuationBlockOrError, op&: opInst))) |
| 3901 | return failure(); |
| 3902 | |
| 3903 | builder.SetInsertPoint(continuationBlockOrError.get()); |
| 3904 | return success(); |
| 3905 | } |
| 3906 | |
| 3907 | /// Look up the given value in the mapping, and if it's not there, translate its |
| 3908 | /// defining operation at the current builder insertion point. Only pure, |
| 3909 | /// regionless operations are supported because the same operation will later be |
| 3910 | /// translated again when the taskloop body itself is lowered. |
| 3911 | static llvm::Expected<llvm::Value *> |
| 3912 | lookupOrTranslatePureValue(Value value, |
| 3913 | LLVM::ModuleTranslation &moduleTranslation, |
| 3914 | llvm::IRBuilderBase &builder) { |
| 3915 | if (llvm::Value *mapped = moduleTranslation.lookupValue(value)) |
| 3916 | return mapped; |
| 3917 | |
| 3918 | Operation *defOp = value.getDefiningOp(); |
| 3919 | if (!defOp) |
| 3920 | return llvm::make_error<llvm::StringError>( |
| 3921 | Args: "value is a block argument and is not mapped" , |
| 3922 | Args: llvm::inconvertibleErrorCode()); |
| 3923 | if (defOp->getNumRegions() != 0 || !isPure(op: defOp)) |
| 3924 | return llvm::make_error<llvm::StringError>( |
| 3925 | Args: "unsupported op defining taskloop loop bound" , |
| 3926 | Args: llvm::inconvertibleErrorCode()); |
| 3927 | |
| 3928 | SmallVector<Value> mappingsToRemove; |
| 3929 | mappingsToRemove.reserve(N: defOp->getNumOperands() + defOp->getNumResults()); |
| 3930 | for (Value operand : defOp->getOperands()) { |
| 3931 | if (moduleTranslation.lookupValue(value: operand)) |
| 3932 | continue; |
| 3933 | |
| 3934 | llvm::Expected<llvm::Value *> operandOrError = |
| 3935 | lookupOrTranslatePureValue(value: operand, moduleTranslation, builder); |
| 3936 | if (!operandOrError) |
| 3937 | return operandOrError.takeError(); |
| 3938 | moduleTranslation.mapValue(mlir: operand, llvm: *operandOrError); |
| 3939 | mappingsToRemove.push_back(Elt: operand); |
| 3940 | } |
| 3941 | |
| 3942 | if (failed(Result: moduleTranslation.convertOperation(op&: *defOp, builder))) |
| 3943 | return llvm::make_error<llvm::StringError>( |
| 3944 | Args: "failed to convert op defining taskloop loop bound" , |
| 3945 | Args: llvm::inconvertibleErrorCode()); |
| 3946 | |
| 3947 | llvm::Value *result = moduleTranslation.lookupValue(value); |
| 3948 | assert(result && "expected conversion of loop bound op to produce a value" ); |
| 3949 | |
| 3950 | for (Value resultValue : defOp->getResults()) { |
| 3951 | if (moduleTranslation.lookupValue(value: resultValue)) |
| 3952 | mappingsToRemove.push_back(Elt: resultValue); |
| 3953 | } |
| 3954 | for (Value mappedValue : mappingsToRemove) |
| 3955 | moduleTranslation.forgetMapping(value: mappedValue); |
| 3956 | |
| 3957 | return result; |
| 3958 | } |
| 3959 | |
| 3960 | static llvm::Error |
| 3961 | computeTaskloopBounds(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder, |
| 3962 | LLVM::ModuleTranslation &moduleTranslation, |
| 3963 | llvm::Value *&lbVal, llvm::Value *&ubVal, |
| 3964 | llvm::Value *&stepVal) { |
| 3965 | Operation::operand_range lowerBounds = loopOp.getLoopLowerBounds(); |
| 3966 | Operation::operand_range upperBounds = loopOp.getLoopUpperBounds(); |
| 3967 | Operation::operand_range steps = loopOp.getLoopSteps(); |
| 3968 | |
| 3969 | llvm::Expected<llvm::Value *> firstLbOrErr = |
| 3970 | lookupOrTranslatePureValue(value: lowerBounds[0], moduleTranslation, builder); |
| 3971 | if (!firstLbOrErr) |
| 3972 | return firstLbOrErr.takeError(); |
| 3973 | |
| 3974 | llvm::Type *boundType = (*firstLbOrErr)->getType(); |
| 3975 | ubVal = builder.getIntN(N: boundType->getIntegerBitWidth(), C: 1); |
| 3976 | if (loopOp.getCollapseNumLoops() > 1) { |
| 3977 | // In cases where Collapse is used with Taskloop, the upper bound of the |
| 3978 | // iteration space needs to be recalculated to cater for the collapsed loop. |
| 3979 | // The Collapsed Loop UpperBound is the product of all collapsed |
| 3980 | // loop's tripcount. |
| 3981 | // The LowerBound for collapsed loops is always 1. When the loops are |
| 3982 | // collapsed, it will reset the bounds and introduce processing to ensure |
| 3983 | // the index's are presented as expected. As this happens after creating |
| 3984 | // Taskloop, these bounds need predicting. Example: |
| 3985 | // !$omp taskloop collapse(2) |
| 3986 | // do i = 1, 10 |
| 3987 | // do j = 1, 5 |
| 3988 | // .. |
| 3989 | // end do |
| 3990 | // end do |
| 3991 | // This loop above has a total of 50 iterations, so the lb will be 1, and |
| 3992 | // the ub will be 50. collapseLoops in OMPIRBuilder then handles ensuring |
| 3993 | // that i and j are properly presented when used in the loop. |
| 3994 | for (uint64_t i = 0; i < loopOp.getCollapseNumLoops(); i++) { |
| 3995 | llvm::Expected<llvm::Value *> lbOrErr = |
| 3996 | i == 0 ? std::move(firstLbOrErr) |
| 3997 | : lookupOrTranslatePureValue(value: lowerBounds[i], moduleTranslation, |
| 3998 | builder); |
| 3999 | if (!lbOrErr) |
| 4000 | return lbOrErr.takeError(); |
| 4001 | llvm::Expected<llvm::Value *> ubOrErr = lookupOrTranslatePureValue( |
| 4002 | value: upperBounds[i], moduleTranslation, builder); |
| 4003 | if (!ubOrErr) |
| 4004 | return ubOrErr.takeError(); |
| 4005 | llvm::Expected<llvm::Value *> stepOrErr = |
| 4006 | lookupOrTranslatePureValue(value: steps[i], moduleTranslation, builder); |
| 4007 | if (!stepOrErr) |
| 4008 | return stepOrErr.takeError(); |
| 4009 | |
| 4010 | llvm::Value *loopLb = *lbOrErr; |
| 4011 | llvm::Value *loopUb = *ubOrErr; |
| 4012 | llvm::Value *loopStep = *stepOrErr; |
| 4013 | // In some cases, such as where the ub is less than the lb so the loop |
| 4014 | // steps down, the calculation for the loopTripCount is swapped. To ensure |
| 4015 | // the correct value is found, calculate both UB - LB and LB - UB then |
| 4016 | // select which value to use depending on how the loop has been |
| 4017 | // configured. |
| 4018 | llvm::Value *loopLbMinusOne = builder.CreateSub( |
| 4019 | LHS: loopLb, RHS: builder.getIntN(N: boundType->getIntegerBitWidth(), C: 1)); |
| 4020 | llvm::Value *loopUbMinusOne = builder.CreateSub( |
| 4021 | LHS: loopUb, RHS: builder.getIntN(N: boundType->getIntegerBitWidth(), C: 1)); |
| 4022 | llvm::Value *boundsCmp = builder.CreateICmpSLT(LHS: loopLb, RHS: loopUb); |
| 4023 | llvm::Value *ubMinusLb = builder.CreateSub(LHS: loopUb, RHS: loopLbMinusOne); |
| 4024 | llvm::Value *lbMinusUb = builder.CreateSub(LHS: loopLb, RHS: loopUbMinusOne); |
| 4025 | llvm::Value *loopTripCount = |
| 4026 | builder.CreateSelect(C: boundsCmp, True: ubMinusLb, False: lbMinusUb); |
| 4027 | loopTripCount = builder.CreateBinaryIntrinsic( |
| 4028 | ID: llvm::Intrinsic::abs, LHS: loopTripCount, RHS: builder.getFalse()); |
| 4029 | // For loops that have a step value not equal to 1, we need to adjust the |
| 4030 | // trip count to ensure the correct number of iterations for the loop is |
| 4031 | // captured. |
| 4032 | llvm::Value *loopTripCountDivStep = |
| 4033 | builder.CreateSDiv(LHS: loopTripCount, RHS: loopStep); |
| 4034 | loopTripCountDivStep = builder.CreateBinaryIntrinsic( |
| 4035 | ID: llvm::Intrinsic::abs, LHS: loopTripCountDivStep, RHS: builder.getFalse()); |
| 4036 | llvm::Value *loopTripCountRem = |
| 4037 | builder.CreateSRem(LHS: loopTripCount, RHS: loopStep); |
| 4038 | loopTripCountRem = builder.CreateBinaryIntrinsic( |
| 4039 | ID: llvm::Intrinsic::abs, LHS: loopTripCountRem, RHS: builder.getFalse()); |
| 4040 | llvm::Value *needsRoundUp = builder.CreateICmpNE( |
| 4041 | LHS: loopTripCountRem, |
| 4042 | RHS: builder.getIntN(N: loopTripCountRem->getType()->getIntegerBitWidth(), |
| 4043 | C: 0)); |
| 4044 | loopTripCount = |
| 4045 | builder.CreateAdd(LHS: loopTripCountDivStep, |
| 4046 | RHS: builder.CreateZExtOrTrunc( |
| 4047 | V: needsRoundUp, DestTy: loopTripCountDivStep->getType())); |
| 4048 | ubVal = builder.CreateMul(LHS: ubVal, RHS: loopTripCount); |
| 4049 | } |
| 4050 | lbVal = builder.getIntN(N: boundType->getIntegerBitWidth(), C: 1); |
| 4051 | stepVal = builder.getIntN(N: boundType->getIntegerBitWidth(), C: 1); |
| 4052 | } else { |
| 4053 | llvm::Expected<llvm::Value *> ubOrErr = |
| 4054 | lookupOrTranslatePureValue(value: upperBounds[0], moduleTranslation, builder); |
| 4055 | if (!ubOrErr) |
| 4056 | return ubOrErr.takeError(); |
| 4057 | llvm::Expected<llvm::Value *> stepOrErr = |
| 4058 | lookupOrTranslatePureValue(value: steps[0], moduleTranslation, builder); |
| 4059 | if (!stepOrErr) |
| 4060 | return stepOrErr.takeError(); |
| 4061 | lbVal = *firstLbOrErr; |
| 4062 | ubVal = *ubOrErr; |
| 4063 | stepVal = *stepOrErr; |
| 4064 | } |
| 4065 | |
| 4066 | assert(lbVal != nullptr && "Expected value for lbVal" ); |
| 4067 | assert(ubVal != nullptr && "Expected value for ubVal" ); |
| 4068 | assert(stepVal != nullptr && "Expected value for stepVal" ); |
| 4069 | return llvm::Error::success(); |
| 4070 | } |
| 4071 | |
| 4072 | // Converts an OpenMP taskloop construct into LLVM IR using OpenMPIRBuilder. |
| 4073 | static LogicalResult |
| 4074 | convertOmpTaskloopContextOp(omp::TaskloopContextOp contextOp, |
| 4075 | llvm::IRBuilderBase &builder, |
| 4076 | LLVM::ModuleTranslation &moduleTranslation) { |
| 4077 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 4078 | mlir::Operation &opInst = *contextOp.getOperation(); |
| 4079 | omp::TaskloopWrapperOp loopWrapperOp = contextOp.getLoopOp(); |
| 4080 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 4081 | return failure(); |
| 4082 | |
| 4083 | // It stores the pointer of allocated firstprivate copies, |
| 4084 | // which can be used later for freeing the allocated space. |
| 4085 | SmallVector<llvm::Value *> llvmFirstPrivateVars; |
| 4086 | PrivateVarsInfo privateVarsInfo(contextOp); |
| 4087 | TaskContextStructManager taskStructMgr{builder, moduleTranslation, |
| 4088 | privateVarsInfo}; |
| 4089 | |
| 4090 | llvm::SmallVector<llvm::BasicBlock *> deallocBlocks; |
| 4091 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 4092 | findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks); |
| 4093 | |
| 4094 | assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end()); |
| 4095 | llvm::BasicBlock *taskloopStartBlock = llvm::BasicBlock::Create( |
| 4096 | Context&: builder.getContext(), Name: "omp.taskloop.wrapper.start" , |
| 4097 | /*Parent=*/builder.GetInsertBlock()->getParent()); |
| 4098 | llvm::Instruction *branchToTaskloopStartBlock = |
| 4099 | builder.CreateBr(Dest: taskloopStartBlock); |
| 4100 | builder.SetInsertPoint(branchToTaskloopStartBlock); |
| 4101 | |
| 4102 | llvm::BasicBlock *copyBlock = |
| 4103 | splitBB(Builder&: builder, /*CreateBranch=*/true, Name: "omp.private.copy" ); |
| 4104 | llvm::BasicBlock *initBlock = |
| 4105 | splitBB(Builder&: builder, /*CreateBranch=*/true, Name: "omp.private.init" ); |
| 4106 | |
| 4107 | LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame( |
| 4108 | moduleTranslation, allocaIP.getNodeParent(), deallocBlocks); |
| 4109 | |
| 4110 | // Allocate and initialize private variables |
| 4111 | builder.SetInsertPoint(initBlock->getTerminator()); |
| 4112 | |
| 4113 | // TODO: don't allocate if the loop has zero iterations. |
| 4114 | taskStructMgr.generateTaskContextStruct(); |
| 4115 | taskStructMgr.createGEPsToPrivateVars(); |
| 4116 | |
| 4117 | llvmFirstPrivateVars.resize(N: privateVarsInfo.vars.size()); |
| 4118 | |
| 4119 | for (auto [i, zip] : llvm::enumerate(First: llvm::zip_equal( |
| 4120 | t&: privateVarsInfo.vars, u: taskStructMgr.getLLVMPrivateVarGEPs()))) { |
| 4121 | auto &[var, llvmPrivateVarAlloc] = zip; |
| 4122 | // To be handled inside the taskloop. |
| 4123 | if (!var.privatizer.readsFromMold()) |
| 4124 | continue; |
| 4125 | assert(llvmPrivateVarAlloc && |
| 4126 | "reads from mold so shouldn't have been skipped" ); |
| 4127 | |
| 4128 | llvm::Expected<llvm::Value *> privateVarOrErr = |
| 4129 | initPrivateVar(builder, moduleTranslation, privDecl&: var.privatizer, mlirPrivVar: var.mlirVar, |
| 4130 | blockArg&: var.blockArg, llvmPrivateVar: llvmPrivateVarAlloc, privInitBlock: initBlock); |
| 4131 | if (!privateVarOrErr) |
| 4132 | return handleError(result&: privateVarOrErr, op&: *contextOp.getOperation()); |
| 4133 | |
| 4134 | llvmFirstPrivateVars[i] = privateVarOrErr.get(); |
| 4135 | |
| 4136 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 4137 | builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator()); |
| 4138 | |
| 4139 | [[maybe_unused]] llvm::Value *llvmPrivateVar = llvmPrivateVarAlloc; |
| 4140 | if ((privateVarOrErr.get() != llvmPrivateVarAlloc) && |
| 4141 | !mlir::isa<LLVM::LLVMPointerType>(Val: var.blockArg.getType())) { |
| 4142 | builder.CreateStore(Val: privateVarOrErr.get(), Ptr: llvmPrivateVarAlloc); |
| 4143 | // Load it so we have the value pointed to by the GEP |
| 4144 | llvmPrivateVar = builder.CreateLoad(Ty: privateVarOrErr.get()->getType(), |
| 4145 | Ptr: llvmPrivateVarAlloc); |
| 4146 | } |
| 4147 | assert(llvmPrivateVar->getType() == |
| 4148 | moduleTranslation.convertType(var.blockArg.getType())); |
| 4149 | } |
| 4150 | |
| 4151 | // firstprivate copy region |
| 4152 | setInsertPointForPossiblyEmptyBlock(builder, block: copyBlock); |
| 4153 | if (failed(Result: completePrivateVars( |
| 4154 | op: contextOp, builder, moduleTranslation, privateVarsInfo, |
| 4155 | insertBarrier: contextOp.getPrivateNeedsBarrier(), |
| 4156 | /*moldVars=*/{}, llvmPrivateVars: taskStructMgr.getLLVMPrivateVarGEPs()))) |
| 4157 | return llvm::failure(); |
| 4158 | |
| 4159 | // Resolve and validate reduction / in_reduction declarations up front. |
| 4160 | // This is pure MLIR symbol-table work and does not emit IR, so do it |
| 4161 | // before moving the builder to the taskloop start block. Richer |
| 4162 | // declare_reduction shapes (byref) have been rejected already by |
| 4163 | // checkImplementationStatus; the rest (two-argument initializer, cleanup |
| 4164 | // region, missing combiner) are rejected by the helper. |
| 4165 | SmallVector<omp::DeclareReductionOp> redDecls; |
| 4166 | if (failed(Result: collectAndValidateTaskloopRedDecls( |
| 4167 | contextOp: contextOp.getOperation(), syms: contextOp.getReductionSyms(), |
| 4168 | opName: "omp.taskloop.context" , clauseName: "reduction" , out&: redDecls))) |
| 4169 | return failure(); |
| 4170 | SmallVector<omp::DeclareReductionOp> inRedDecls; |
| 4171 | if (failed(Result: collectAndValidateTaskloopRedDecls( |
| 4172 | contextOp: contextOp.getOperation(), syms: contextOp.getInReductionSyms(), |
| 4173 | opName: "omp.taskloop.context" , clauseName: "in_reduction" , out&: inRedDecls))) |
| 4174 | return failure(); |
| 4175 | |
| 4176 | // The op verifier rejects nogroup + reduction, so no check is needed here. |
| 4177 | |
| 4178 | SmallVector<llvm::Value *> redOrigPtrs; |
| 4179 | redOrigPtrs.reserve(N: redDecls.size()); |
| 4180 | for (Value v : contextOp.getReductionVars()) |
| 4181 | redOrigPtrs.push_back(Elt: moduleTranslation.lookupValue(value: v)); |
| 4182 | SmallVector<llvm::Value *> inRedOrigPtrs; |
| 4183 | inRedOrigPtrs.reserve(N: inRedDecls.size()); |
| 4184 | for (Value v : contextOp.getInReductionVars()) |
| 4185 | inRedOrigPtrs.push_back(Elt: moduleTranslation.lookupValue(value: v)); |
| 4186 | |
| 4187 | // Set up insertion point for emitting the implicit-taskgroup reduction |
| 4188 | // setup (if any) and for the subsequent call to createTaskloop(). |
| 4189 | builder.SetInsertPoint(taskloopStartBlock); |
| 4190 | |
| 4191 | llvm::OpenMPIRBuilder &ompBuilderRef = *getOpenMPBuilder(moduleTranslation); |
| 4192 | llvm::Module *module = moduleTranslation.getLLVMModule(); |
| 4193 | |
| 4194 | // If we have task_reduction items, we must emit our own implicit |
| 4195 | // __kmpc_taskgroup so that the descriptor returned by __kmpc_taskred_init |
| 4196 | // is associated with that taskgroup. We then force NoGroup=true so that |
| 4197 | // OpenMPIRBuilder::createTaskloop does not emit a second taskgroup. |
| 4198 | bool implicitTaskgroup = !redDecls.empty(); |
| 4199 | llvm::Value *redDesc = nullptr; |
| 4200 | if (implicitTaskgroup) { |
| 4201 | llvm::OpenMPIRBuilder::LocationDescription redLoc(builder); |
| 4202 | uint32_t srcLocSize; |
| 4203 | llvm::Constant *srcLocStr = |
| 4204 | ompBuilderRef.getOrCreateSrcLocStr(Loc: redLoc, SrcLocStrSize&: srcLocSize); |
| 4205 | llvm::Value *ident = ompBuilderRef.getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize); |
| 4206 | // Align OpenMPIRBuilder's internal IRBuilder with `builder` so the |
| 4207 | // gtid call lands at our insertion point. |
| 4208 | ompBuilderRef.updateToLocation(Loc: redLoc); |
| 4209 | llvm::Value *outerGtid = ompBuilderRef.getOrCreateThreadID(Ident: ident); |
| 4210 | llvm::FunctionCallee taskgroupFn = ompBuilderRef.getOrCreateRuntimeFunction( |
| 4211 | M&: *module, FnID: llvm::omp::OMPRTL___kmpc_taskgroup); |
| 4212 | builder.CreateCall(Callee: taskgroupFn, Args: {ident, outerGtid}); |
| 4213 | |
| 4214 | redDesc = emitTaskReductionInitCall(redDecls, origPtrs: redOrigPtrs, |
| 4215 | helperNamePrefix: "__omp_taskloop_taskred_" , builder, |
| 4216 | allocaIP, moduleTranslation); |
| 4217 | if (!redDesc) |
| 4218 | return failure(); |
| 4219 | } |
| 4220 | |
| 4221 | auto loopOp = cast<omp::LoopNestOp>(Val: loopWrapperOp.getWrappedLoop()); |
| 4222 | llvm::Value *lbVal = nullptr; |
| 4223 | llvm::Value *ubVal = nullptr; |
| 4224 | llvm::Value *stepVal = nullptr; |
| 4225 | if (llvm::Error err = computeTaskloopBounds( |
| 4226 | loopOp, builder, moduleTranslation, lbVal, ubVal, stepVal)) |
| 4227 | return handleError(error: std::move(err), op&: opInst); |
| 4228 | |
| 4229 | auto bodyCB = |
| 4230 | [&](InsertPointTy allocaIP, InsertPointTy codegenIP, |
| 4231 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error { |
| 4232 | // Save the alloca insertion point on ModuleTranslation stack for use in |
| 4233 | // nested regions. |
| 4234 | LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame( |
| 4235 | moduleTranslation, allocaIP.getNodeParent(), deallocBlocks); |
| 4236 | |
| 4237 | // translate the body of the taskloop: |
| 4238 | builder.restoreIP(IP: codegenIP); |
| 4239 | |
| 4240 | llvm::BasicBlock *privInitBlock = nullptr; |
| 4241 | for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) { |
| 4242 | // This is handled before the task executes |
| 4243 | if (var.privatizer.readsFromMold()) |
| 4244 | continue; |
| 4245 | |
| 4246 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 4247 | llvm::Type *llvmAllocType = |
| 4248 | moduleTranslation.convertType(type: var.privatizer.getType()); |
| 4249 | builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator()); |
| 4250 | llvm::Value *llvmPrivateVar = builder.CreateAlloca( |
| 4251 | Ty: llvmAllocType, /*ArraySize=*/nullptr, Name: "omp.private.alloc" ); |
| 4252 | |
| 4253 | llvm::Expected<llvm::Value *> privateVarOrError = initPrivateVar( |
| 4254 | builder, moduleTranslation, privDecl&: var.privatizer, mlirPrivVar: var.mlirVar, blockArg&: var.blockArg, |
| 4255 | llvmPrivateVar, privInitBlock); |
| 4256 | if (!privateVarOrError) |
| 4257 | return privateVarOrError.takeError(); |
| 4258 | moduleTranslation.mapValue(mlir: var.blockArg, llvm: privateVarOrError.get()); |
| 4259 | var.llvmVar = privateVarOrError.get(); |
| 4260 | } |
| 4261 | |
| 4262 | taskStructMgr.createGEPsToPrivateVars(); |
| 4263 | for (auto [i, llvmPrivVar] : |
| 4264 | llvm::enumerate(First: taskStructMgr.getLLVMPrivateVarGEPs())) { |
| 4265 | if (!llvmPrivVar) { |
| 4266 | assert(privateVarsInfo.vars[i].llvmVar && |
| 4267 | "This is added in the loop above" ); |
| 4268 | continue; |
| 4269 | } |
| 4270 | privateVarsInfo.vars[i].llvmVar = llvmPrivVar; |
| 4271 | } |
| 4272 | |
| 4273 | // Find and map the addresses of each variable within the taskloop context |
| 4274 | // structure |
| 4275 | for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) { |
| 4276 | // This was handled above. |
| 4277 | if (!var.privatizer.readsFromMold()) |
| 4278 | continue; |
| 4279 | // Fix broken pass-by-value case for Fortran character boxes |
| 4280 | if (!mlir::isa<LLVM::LLVMPointerType>(Val: var.blockArg.getType())) { |
| 4281 | var.llvmVar = builder.CreateLoad( |
| 4282 | Ty: moduleTranslation.convertType(type: var.blockArg.getType()), Ptr: var.llvmVar); |
| 4283 | } |
| 4284 | assert(var.llvmVar->getType() == |
| 4285 | moduleTranslation.convertType(var.blockArg.getType())); |
| 4286 | moduleTranslation.mapValue(mlir: var.blockArg, llvm: var.llvmVar); |
| 4287 | } |
| 4288 | |
| 4289 | // Map reduction and in_reduction block arguments to the per-task private |
| 4290 | // storage returned by __kmpc_task_reduction_get_th_data. This call must |
| 4291 | // be emitted inside the to-be-outlined task body so that it returns the |
| 4292 | // *executing* thread's gtid (not the encountering thread's). The |
| 4293 | // taskgroup descriptor `redDesc` is computed in the outer scope and is |
| 4294 | // auto-captured into the task shareds aggregate by CodeExtractor during |
| 4295 | // OpenMPIRBuilder::finalize. For in_reduction the descriptor is NULL: |
| 4296 | // the runtime walks up enclosing taskgroups to find the matching |
| 4297 | // task_reduction registration for `origPtr`. |
| 4298 | if (!redDecls.empty() || !inRedDecls.empty()) { |
| 4299 | auto iface = |
| 4300 | cast<omp::BlockArgOpenMPOpInterface>(Val: contextOp.getOperation()); |
| 4301 | llvm::OpenMPIRBuilder &ompB = *getOpenMPBuilder(moduleTranslation); |
| 4302 | llvm::Module *m = moduleTranslation.getLLVMModule(); |
| 4303 | llvm::LLVMContext &llvmCtx = m->getContext(); |
| 4304 | llvm::OpenMPIRBuilder::LocationDescription bodyLoc(builder); |
| 4305 | uint32_t srcLocSize; |
| 4306 | llvm::Constant *srcLocStr = |
| 4307 | ompB.getOrCreateSrcLocStr(Loc: bodyLoc, SrcLocStrSize&: srcLocSize); |
| 4308 | llvm::Value *bodyIdent = ompB.getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize); |
| 4309 | // Align OpenMPIRBuilder's internal IRBuilder with `builder` so the |
| 4310 | // gtid call lands inside the to-be-outlined task body. |
| 4311 | ompB.updateToLocation(Loc: bodyLoc); |
| 4312 | llvm::Value *bodyGtid = ompB.getOrCreateThreadID(Ident: bodyIdent); |
| 4313 | llvm::FunctionCallee getThData = ompB.getOrCreateRuntimeFunction( |
| 4314 | M&: *m, FnID: llvm::omp::OMPRTL___kmpc_task_reduction_get_th_data); |
| 4315 | llvm::Type *ptrTy = llvm::PointerType::getUnqual(C&: llvmCtx); |
| 4316 | |
| 4317 | // Emit one __kmpc_task_reduction_get_th_data lookup for a reduction / |
| 4318 | // in_reduction item and map its block argument to the per-task private |
| 4319 | // storage the runtime returns. The runtime entry point takes (and |
| 4320 | // returns) a generic, default-address-space `ptr`, so normalize a |
| 4321 | // non-default-address-space original pointer to the generic address |
| 4322 | // space before the call (mirroring the descriptor setup in |
| 4323 | // emitTaskReductionInitCall), and cast the returned private pointer back |
| 4324 | // to the block argument's address space when that differs. |
| 4325 | auto remapReductionArg = [&](BlockArgument blockArg, llvm::Value *desc, |
| 4326 | llvm::Value *origPtr, |
| 4327 | const llvm::Twine &name) { |
| 4328 | if (auto *origPtrTy = |
| 4329 | llvm::dyn_cast<llvm::PointerType>(Val: origPtr->getType()); |
| 4330 | origPtrTy && origPtrTy->getAddressSpace() != 0) |
| 4331 | origPtr = builder.CreateAddrSpaceCast(V: origPtr, DestTy: ptrTy); |
| 4332 | llvm::Value *priv = |
| 4333 | builder.CreateCall(Callee: getThData, Args: {bodyGtid, desc, origPtr}, Name: name); |
| 4334 | if (auto *argPtrTy = llvm::dyn_cast<llvm::PointerType>( |
| 4335 | Val: moduleTranslation.convertType(type: blockArg.getType())); |
| 4336 | argPtrTy && argPtrTy->getAddressSpace() != 0) |
| 4337 | priv = builder.CreateAddrSpaceCast(V: priv, DestTy: argPtrTy); |
| 4338 | moduleTranslation.mapValue(mlir: blockArg, llvm: priv); |
| 4339 | }; |
| 4340 | |
| 4341 | ArrayRef<BlockArgument> redBlockArgs = iface.getReductionBlockArgs(); |
| 4342 | for (auto [blockArg, origPtr] : |
| 4343 | llvm::zip_equal(t&: redBlockArgs, u&: redOrigPtrs)) |
| 4344 | remapReductionArg(blockArg, redDesc, origPtr, "omp.taskred.priv" ); |
| 4345 | ArrayRef<BlockArgument> inRedBlockArgs = iface.getInReductionBlockArgs(); |
| 4346 | llvm::Value *nullDesc = llvm::ConstantPointerNull::get(T: ptrTy); |
| 4347 | for (auto [blockArg, origPtr] : |
| 4348 | llvm::zip_equal(t&: inRedBlockArgs, u&: inRedOrigPtrs)) |
| 4349 | remapReductionArg(blockArg, nullDesc, origPtr, "omp.inred.priv" ); |
| 4350 | } |
| 4351 | |
| 4352 | // Lower the contents of the taskloop context region: this is the body of |
| 4353 | // the generated task, not the loop. |
| 4354 | auto continuationBlockOrError = convertOmpOpRegions( |
| 4355 | region&: contextOp.getRegion(), blockName: "omp.taskloop.context.region" , builder, |
| 4356 | moduleTranslation); |
| 4357 | |
| 4358 | if (failed(Result: handleError(result&: continuationBlockOrError, op&: opInst))) |
| 4359 | return llvm::make_error<PreviouslyReportedError>(); |
| 4360 | |
| 4361 | builder.SetInsertPoint(continuationBlockOrError.get()->getTerminator()); |
| 4362 | |
| 4363 | // This is freeing the private variables as mapped inside of the task: these |
| 4364 | // will be per-task private copies possibly after task duplication. This is |
| 4365 | // handled transparently by how these are passed to the structure passed |
| 4366 | // into the outlined function. When the task is duplicated, that structure |
| 4367 | // is duplicated too. |
| 4368 | if (failed(Result: cleanupPrivateVars(op: contextOp, builder, moduleTranslation, |
| 4369 | loc: contextOp.getLoc(), privateVarsInfo))) |
| 4370 | return llvm::make_error<PreviouslyReportedError>(); |
| 4371 | // Similarly, the task context structure freed inside the task is the |
| 4372 | // per-task copy after task duplication. |
| 4373 | taskStructMgr.freeStructPtr(); |
| 4374 | |
| 4375 | return llvm::Error::success(); |
| 4376 | }; |
| 4377 | |
| 4378 | // Taskloop divides into an appropriate number of tasks by repeatedly |
| 4379 | // duplicating the original task. Each time this is done, the task context |
| 4380 | // structure must be duplicated too. |
| 4381 | auto taskDupCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP, |
| 4382 | llvm::Value *destPtr, llvm::Value *srcPtr) |
| 4383 | -> llvm::Expected<llvm::IRBuilderBase::InsertPoint> { |
| 4384 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 4385 | builder.restoreIP(IP: codegenIP); |
| 4386 | |
| 4387 | llvm::Type *ptrTy = |
| 4388 | builder.getPtrTy(AddrSpace: srcPtr->getType()->getPointerAddressSpace()); |
| 4389 | llvm::Value *src = |
| 4390 | builder.CreateLoad(Ty: ptrTy, Ptr: srcPtr, Name: "omp.taskloop.context.src" ); |
| 4391 | |
| 4392 | TaskContextStructManager &srcStructMgr = taskStructMgr; |
| 4393 | TaskContextStructManager destStructMgr(builder, moduleTranslation, |
| 4394 | privateVarsInfo); |
| 4395 | destStructMgr.generateTaskContextStruct(); |
| 4396 | llvm::Value *dest = destStructMgr.getStructPtr(); |
| 4397 | dest->setName("omp.taskloop.context.dest" ); |
| 4398 | builder.CreateStore(Val: dest, Ptr: destPtr); |
| 4399 | |
| 4400 | llvm::SmallVector<llvm::Value *> srcGEPs = |
| 4401 | srcStructMgr.createGEPsToPrivateVars(altStructPtr: src); |
| 4402 | llvm::SmallVector<llvm::Value *> destGEPs = |
| 4403 | destStructMgr.createGEPsToPrivateVars(altStructPtr: dest); |
| 4404 | |
| 4405 | // Inline init regions. |
| 4406 | for (auto [var, mold, llvmPrivateVarAlloc] : |
| 4407 | llvm::zip_equal(t&: privateVarsInfo.vars, u&: srcGEPs, args&: destGEPs)) { |
| 4408 | // To be handled inside task body. |
| 4409 | if (!var.privatizer.readsFromMold()) |
| 4410 | continue; |
| 4411 | assert(llvmPrivateVarAlloc && |
| 4412 | "reads from mold so shouldn't have been skipped" ); |
| 4413 | |
| 4414 | llvm::Value *moldArg = materializeRegionArgValue( |
| 4415 | builder, moduleTranslation, regionArg: var.privatizer.getInitMoldArg(), value: mold); |
| 4416 | llvm::Expected<llvm::Value *> privateVarOrErr = initPrivateVar( |
| 4417 | builder, moduleTranslation, privDecl&: var.privatizer, nonPrivateVar: moldArg, blockArg&: var.blockArg, |
| 4418 | llvmPrivateVar: llvmPrivateVarAlloc, privInitBlock: builder.GetInsertBlock()); |
| 4419 | if (!privateVarOrErr) |
| 4420 | return privateVarOrErr.takeError(); |
| 4421 | |
| 4422 | setInsertPointForPossiblyEmptyBlock(builder); |
| 4423 | |
| 4424 | // TODO: this is a bit of a hack for Fortran character boxes. |
| 4425 | // Character boxes are passed by value into the init region and then the |
| 4426 | // initialized character box is yielded by value. Here we need to store |
| 4427 | // the yielded value into the private allocation, and load the private |
| 4428 | // allocation to match the type expected by region block arguments. |
| 4429 | [[maybe_unused]] llvm::Value *llvmPrivateVar = llvmPrivateVarAlloc; |
| 4430 | if ((privateVarOrErr.get() != llvmPrivateVarAlloc) && |
| 4431 | !mlir::isa<LLVM::LLVMPointerType>(Val: var.blockArg.getType())) { |
| 4432 | builder.CreateStore(Val: privateVarOrErr.get(), Ptr: llvmPrivateVarAlloc); |
| 4433 | // Load it so we have the value pointed to by the GEP |
| 4434 | llvmPrivateVar = builder.CreateLoad(Ty: privateVarOrErr.get()->getType(), |
| 4435 | Ptr: llvmPrivateVarAlloc); |
| 4436 | } |
| 4437 | assert(llvmPrivateVar->getType() == |
| 4438 | moduleTranslation.convertType(var.blockArg.getType())); |
| 4439 | |
| 4440 | // Mapping blockArg -> llvmPrivateVarAlloc is done inside the body |
| 4441 | // callback so that OpenMPIRBuilder doesn't try to pass each GEP address |
| 4442 | // through a stack allocated structure. |
| 4443 | } |
| 4444 | |
| 4445 | if (failed(Result: completePrivateVars(op: contextOp.getOperation(), builder, |
| 4446 | moduleTranslation, privateVarsInfo, |
| 4447 | insertBarrier: contextOp.getPrivateNeedsBarrier(), moldVars: srcGEPs, |
| 4448 | llvmPrivateVars: destGEPs))) |
| 4449 | return llvm::make_error<PreviouslyReportedError>(); |
| 4450 | |
| 4451 | return builder.saveIP(); |
| 4452 | }; |
| 4453 | |
| 4454 | auto loopInfo = [&]() -> llvm::Expected<llvm::CanonicalLoopInfo *> { |
| 4455 | llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation); |
| 4456 | return loopInfo; |
| 4457 | }; |
| 4458 | |
| 4459 | llvm::Value *ifCond = nullptr; |
| 4460 | llvm::Value *grainsize = nullptr; |
| 4461 | int sched = 0; // default |
| 4462 | mlir::Value grainsizeVal = contextOp.getGrainsize(); |
| 4463 | mlir::Value numTasksVal = contextOp.getNumTasks(); |
| 4464 | if (Value ifVar = contextOp.getIfExpr()) |
| 4465 | ifCond = moduleTranslation.lookupValue(value: ifVar); |
| 4466 | if (grainsizeVal) { |
| 4467 | grainsize = moduleTranslation.lookupValue(value: grainsizeVal); |
| 4468 | sched = 1; // grainsize |
| 4469 | } else if (numTasksVal) { |
| 4470 | grainsize = moduleTranslation.lookupValue(value: numTasksVal); |
| 4471 | sched = 2; // num_tasks |
| 4472 | } |
| 4473 | |
| 4474 | llvm::OpenMPIRBuilder::TaskDupCallbackTy taskDupOrNull = nullptr; |
| 4475 | if (taskStructMgr.getStructPtr()) |
| 4476 | taskDupOrNull = taskDupCB; |
| 4477 | |
| 4478 | llvm::OpenMPIRBuilder &ompBuilder = *getOpenMPBuilder(moduleTranslation); |
| 4479 | SmallVector<llvm::UncondBrInst *> cancelTerminators; |
| 4480 | // The directive to match here is OMPD_taskgroup because it is the |
| 4481 | // taskgroup which is canceled. This is handled here because it is the |
| 4482 | // task's cleanup block which should be branched to. It doesn't depend upon |
| 4483 | // nogroup because even in that case the taskloop might still be inside an |
| 4484 | // explicit taskgroup. |
| 4485 | pushCancelFinalizationCB(cancelTerminators, llvmBuilder&: builder, ompBuilder, op: contextOp, |
| 4486 | cancelDirective: llvm::omp::Directive::OMPD_taskgroup); |
| 4487 | |
| 4488 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 4489 | bool effectiveNoGroup = contextOp.getNogroup() || implicitTaskgroup; |
| 4490 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 4491 | getOpenMPBuilder(moduleTranslation) |
| 4492 | ->createTaskloop( |
| 4493 | Loc: ompLoc, AllocaIP: allocaIP, DeallocBlocks: deallocBlocks, BodyGenCB: bodyCB, LoopInfo: loopInfo, LBVal: lbVal, UBVal: ubVal, |
| 4494 | StepVal: stepVal, Untied: contextOp.getUntied(), IfCond: ifCond, GrainSize: grainsize, |
| 4495 | NoGroup: effectiveNoGroup, Sched: sched, |
| 4496 | Final: moduleTranslation.lookupValue(value: contextOp.getFinal()), |
| 4497 | Mergeable: contextOp.getMergeable(), |
| 4498 | Priority: moduleTranslation.lookupValue(value: contextOp.getPriority()), |
| 4499 | NumOfCollapseLoops: loopOp.getCollapseNumLoops(), DupCB: taskDupOrNull, |
| 4500 | TaskContextStructPtrVal: taskStructMgr.getStructPtr(), |
| 4501 | FreeAgent: contextOp.getThreadset() == omp::ThreadsetPolicy::omp_pool); |
| 4502 | |
| 4503 | if (failed(Result: handleError(result&: afterIP, op&: opInst))) |
| 4504 | return failure(); |
| 4505 | |
| 4506 | popCancelFinalizationCB(cancelTerminators, ompBuilder, |
| 4507 | afterBB: afterIP->getNodeParent()); |
| 4508 | |
| 4509 | builder.restoreIP(IP: *afterIP); |
| 4510 | |
| 4511 | // Close the implicit taskgroup we opened for task_reduction. The end call |
| 4512 | // must execute on the encountering thread, so use the outer-scope gtid. |
| 4513 | if (implicitTaskgroup) { |
| 4514 | llvm::OpenMPIRBuilder::LocationDescription endLoc(builder); |
| 4515 | uint32_t srcLocSize; |
| 4516 | llvm::Constant *srcLocStr = |
| 4517 | ompBuilder.getOrCreateSrcLocStr(Loc: endLoc, SrcLocStrSize&: srcLocSize); |
| 4518 | llvm::Value *ident = ompBuilder.getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize); |
| 4519 | // Align OpenMPIRBuilder's internal IRBuilder with `builder` so the |
| 4520 | // gtid call lands at our insertion point. |
| 4521 | ompBuilder.updateToLocation(Loc: endLoc); |
| 4522 | llvm::Value *outerGtid = ompBuilder.getOrCreateThreadID(Ident: ident); |
| 4523 | llvm::FunctionCallee endTgFn = ompBuilder.getOrCreateRuntimeFunction( |
| 4524 | M&: *moduleTranslation.getLLVMModule(), |
| 4525 | FnID: llvm::omp::OMPRTL___kmpc_end_taskgroup); |
| 4526 | builder.CreateCall(Callee: endTgFn, Args: {ident, outerGtid}); |
| 4527 | } |
| 4528 | return success(); |
| 4529 | } |
| 4530 | |
| 4531 | /// Build an outlined init helper for a task_reduction declare_reduction op. |
| 4532 | /// Signature: void(ptr %priv, ptr %orig). For non-byref reductions, the init |
| 4533 | /// region's mold argument is mapped following the same rule as the regular |
| 4534 | /// reduction path (`mapInitializationArgs`): a non-pointer mold loads the |
| 4535 | /// value from %orig, while a pointer-typed mold receives %orig directly. The |
| 4536 | /// yielded value is stored into %priv. |
| 4537 | static llvm::Function * |
| 4538 | emitTaskReductionInitFn(omp::DeclareReductionOp decl, StringRef baseName, |
| 4539 | LLVM::ModuleTranslation &moduleTranslation) { |
| 4540 | llvm::Module *llvmModule = moduleTranslation.getLLVMModule(); |
| 4541 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
| 4542 | llvm::Type *voidTy = llvm::Type::getVoidTy(C&: ctx); |
| 4543 | llvm::Type *ptrTy = llvm::PointerType::getUnqual(C&: ctx); |
| 4544 | llvm::FunctionType *fty = |
| 4545 | llvm::FunctionType::get(Result: voidTy, Params: {ptrTy, ptrTy}, isVarArg: false); |
| 4546 | llvm::Function *fn = |
| 4547 | llvm::Function::Create(Ty: fty, Linkage: llvm::GlobalValue::InternalLinkage, |
| 4548 | N: baseName + ".red.init" , M: llvmModule); |
| 4549 | fn->setDoesNotRecurse(); |
| 4550 | fn->getArg(i: 0)->setName("priv" ); |
| 4551 | fn->getArg(i: 1)->setName("orig" ); |
| 4552 | |
| 4553 | llvm::BasicBlock *entry = llvm::BasicBlock::Create(Context&: ctx, Name: "entry" , Parent: fn); |
| 4554 | llvm::IRBuilder<> b(entry); |
| 4555 | |
| 4556 | // Map the initializer's mold argument the same way the regular reduction |
| 4557 | // path does in `mapInitializationArgs`: only load the original value when a |
| 4558 | // non-pointer mold is expected. For a pointer-typed mold the storage pointer |
| 4559 | // (%orig) is passed through directly, so a mold-yielding initializer lowers |
| 4560 | // to `store ptr %orig, ptr %priv` rather than emitting a spurious load. |
| 4561 | Value moldArg = decl.getInitializerMoldArg(); |
| 4562 | llvm::Value *origVal = fn->getArg(i: 1); |
| 4563 | if (!isa<LLVM::LLVMPointerType>(Val: moldArg.getType())) |
| 4564 | origVal = b.CreateLoad(Ty: moduleTranslation.convertType(type: moldArg.getType()), |
| 4565 | Ptr: fn->getArg(i: 1), Name: "omp.orig" ); |
| 4566 | moduleTranslation.mapValue(mlir: moldArg, llvm: origVal); |
| 4567 | SmallVector<llvm::Value *, 1> phis; |
| 4568 | if (failed(Result: inlineConvertOmpRegions(region&: decl.getInitializerRegion(), |
| 4569 | blockName: "omp.taskred.init" , builder&: b, moduleTranslation, |
| 4570 | continuationBlockArgs: &phis))) { |
| 4571 | fn->eraseFromParent(); |
| 4572 | return nullptr; |
| 4573 | } |
| 4574 | assert(phis.size() == 1 && |
| 4575 | "expected one value yielded from reduction initializer" ); |
| 4576 | b.CreateStore(Val: phis[0], Ptr: fn->getArg(i: 0)); |
| 4577 | b.CreateRetVoid(); |
| 4578 | |
| 4579 | moduleTranslation.forgetMapping(region&: decl.getInitializerRegion()); |
| 4580 | return fn; |
| 4581 | } |
| 4582 | |
| 4583 | /// Build an outlined combiner helper for a task_reduction declare_reduction op. |
| 4584 | /// Signature: void(ptr %lhs, ptr %rhs). For non-byref reductions, the values |
| 4585 | /// at *%lhs and *%rhs are loaded, fed into the combiner region, and the |
| 4586 | /// yielded scalar is stored back into *%lhs. |
| 4587 | static llvm::Function * |
| 4588 | emitTaskReductionCombFn(omp::DeclareReductionOp decl, StringRef baseName, |
| 4589 | LLVM::ModuleTranslation &moduleTranslation) { |
| 4590 | llvm::Module *llvmModule = moduleTranslation.getLLVMModule(); |
| 4591 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
| 4592 | llvm::Type *voidTy = llvm::Type::getVoidTy(C&: ctx); |
| 4593 | llvm::Type *ptrTy = llvm::PointerType::getUnqual(C&: ctx); |
| 4594 | llvm::FunctionType *fty = |
| 4595 | llvm::FunctionType::get(Result: voidTy, Params: {ptrTy, ptrTy}, isVarArg: false); |
| 4596 | llvm::Function *fn = |
| 4597 | llvm::Function::Create(Ty: fty, Linkage: llvm::GlobalValue::InternalLinkage, |
| 4598 | N: baseName + ".red.comb" , M: llvmModule); |
| 4599 | fn->setDoesNotRecurse(); |
| 4600 | fn->getArg(i: 0)->setName("lhs" ); |
| 4601 | fn->getArg(i: 1)->setName("rhs" ); |
| 4602 | |
| 4603 | llvm::BasicBlock *entry = llvm::BasicBlock::Create(Context&: ctx, Name: "entry" , Parent: fn); |
| 4604 | llvm::IRBuilder<> b(entry); |
| 4605 | |
| 4606 | llvm::Type *elemTy = moduleTranslation.convertType(type: decl.getType()); |
| 4607 | Block &combBlock = decl.getReductionRegion().front(); |
| 4608 | assert(combBlock.getNumArguments() == 2 && |
| 4609 | "expected two arguments in declare_reduction combiner" ); |
| 4610 | llvm::Value *lhsVal = b.CreateLoad(Ty: elemTy, Ptr: fn->getArg(i: 0), Name: "omp.lhs" ); |
| 4611 | llvm::Value *rhsVal = b.CreateLoad(Ty: elemTy, Ptr: fn->getArg(i: 1), Name: "omp.rhs" ); |
| 4612 | moduleTranslation.mapValue(mlir: combBlock.getArgument(i: 0), llvm: lhsVal); |
| 4613 | moduleTranslation.mapValue(mlir: combBlock.getArgument(i: 1), llvm: rhsVal); |
| 4614 | |
| 4615 | SmallVector<llvm::Value *, 1> phis; |
| 4616 | if (failed(Result: inlineConvertOmpRegions(region&: decl.getReductionRegion(), |
| 4617 | blockName: "omp.taskred.comb" , builder&: b, moduleTranslation, |
| 4618 | continuationBlockArgs: &phis))) { |
| 4619 | fn->eraseFromParent(); |
| 4620 | return nullptr; |
| 4621 | } |
| 4622 | assert(phis.size() == 1 && |
| 4623 | "expected one value yielded from reduction combiner" ); |
| 4624 | b.CreateStore(Val: phis[0], Ptr: fn->getArg(i: 0)); |
| 4625 | b.CreateRetVoid(); |
| 4626 | |
| 4627 | moduleTranslation.forgetMapping(region&: decl.getReductionRegion()); |
| 4628 | return fn; |
| 4629 | } |
| 4630 | |
| 4631 | /// Emit the per-taskgroup task_reduction descriptor array and the |
| 4632 | /// `__kmpc_taskred_init` runtime call. \p origPtrs holds the LLVM values for |
| 4633 | /// the original (shared) variables, one per declaration in \p redDecls. |
| 4634 | /// `builder` must be set to the point at which the descriptor stores and the |
| 4635 | /// init call should be emitted; the descriptor array itself is allocated at |
| 4636 | /// \p allocaIP. \p helperNamePrefix is used to disambiguate the generated |
| 4637 | /// init/combiner helper symbol names between taskgroup and taskloop callers. |
| 4638 | /// |
| 4639 | /// When \p isModifier is false, emits `__kmpc_taskred_init` and returns the |
| 4640 | /// `ptr` value it produces (the taskgroup reduction handle). When \p isModifier |
| 4641 | /// is true, emits `__kmpc_taskred_modifier_init` instead to open a |
| 4642 | /// task-reduction scope for a parallel or worksharing construct, passing |
| 4643 | /// \p isWorksharing as the runtime `is_ws` argument. Returns null on failure. |
| 4644 | /// |
| 4645 | /// Only the non-byref form is handled here. Byref reductions have already |
| 4646 | /// been rejected by `checkImplementationStatus`. |
| 4647 | static llvm::Value *emitTaskReductionInitCall( |
| 4648 | ArrayRef<omp::DeclareReductionOp> redDecls, |
| 4649 | ArrayRef<llvm::Value *> origPtrs, StringRef helperNamePrefix, |
| 4650 | llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder::InsertPointTy allocaIP, |
| 4651 | LLVM::ModuleTranslation &moduleTranslation, bool isModifier, |
| 4652 | bool isWorksharing) { |
| 4653 | assert(redDecls.size() == origPtrs.size() && |
| 4654 | "expected one orig pointer per reduction decl" ); |
| 4655 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 4656 | llvm::Module *llvmModule = moduleTranslation.getLLVMModule(); |
| 4657 | llvm::LLVMContext &ctx = llvmModule->getContext(); |
| 4658 | const llvm::DataLayout &dl = llvmModule->getDataLayout(); |
| 4659 | |
| 4660 | llvm::Type *ptrTy = llvm::PointerType::getUnqual(C&: ctx); |
| 4661 | llvm::Type *i32Ty = llvm::Type::getInt32Ty(C&: ctx); |
| 4662 | llvm::Type *sizeTy = |
| 4663 | llvm::Type::getIntNTy(C&: ctx, N: dl.getPointerSizeInBits(/*AddrSpace=*/AS: 0)); |
| 4664 | |
| 4665 | // Identified `kmp_taskred_input_t` struct, matching the layout used by |
| 4666 | // Clang's CGOpenMPRuntime::emitTaskReductionInit. |
| 4667 | llvm::StructType *redInputTy = |
| 4668 | llvm::StructType::getTypeByName(C&: ctx, Name: "kmp_taskred_input_t" ); |
| 4669 | if (!redInputTy) |
| 4670 | redInputTy = llvm::StructType::create( |
| 4671 | Context&: ctx, Elements: {ptrTy, ptrTy, sizeTy, ptrTy, ptrTy, ptrTy, i32Ty}, |
| 4672 | Name: "kmp_taskred_input_t" ); |
| 4673 | |
| 4674 | unsigned n = redDecls.size(); |
| 4675 | llvm::ArrayType *arrTy = llvm::ArrayType::get(ElementType: redInputTy, NumElements: n); |
| 4676 | |
| 4677 | // Allocate the descriptor array in the enclosing function's alloca block. |
| 4678 | llvm::AllocaInst *arrAlloca; |
| 4679 | { |
| 4680 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 4681 | builder.restoreIP(IP: allocaIP); |
| 4682 | arrAlloca = |
| 4683 | builder.CreateAlloca(Ty: arrTy, /*ArraySize=*/nullptr, Name: ".taskred.input" ); |
| 4684 | } |
| 4685 | |
| 4686 | // Fill each descriptor entry at the current builder insertion point. |
| 4687 | llvm::Value *zero = builder.getInt32(C: 0); |
| 4688 | for (unsigned i = 0; i < n; ++i) { |
| 4689 | omp::DeclareReductionOp decl = redDecls[i]; |
| 4690 | llvm::Value *orig = origPtrs[i]; |
| 4691 | if (auto *origPtrTy = llvm::dyn_cast<llvm::PointerType>(Val: orig->getType()); |
| 4692 | origPtrTy && origPtrTy->getAddressSpace() != 0) |
| 4693 | orig = builder.CreateAddrSpaceCast(V: orig, DestTy: ptrTy); |
| 4694 | llvm::Type *elemTy = moduleTranslation.convertType(type: decl.getType()); |
| 4695 | uint64_t size = dl.getTypeAllocSize(Ty: elemTy).getFixedValue(); |
| 4696 | |
| 4697 | std::string baseName = |
| 4698 | (llvm::Twine(helperNamePrefix) + decl.getSymName()).str(); |
| 4699 | llvm::Function *initFn = |
| 4700 | emitTaskReductionInitFn(decl, baseName, moduleTranslation); |
| 4701 | llvm::Function *combFn = |
| 4702 | emitTaskReductionCombFn(decl, baseName, moduleTranslation); |
| 4703 | if (!initFn || !combFn) |
| 4704 | return nullptr; |
| 4705 | llvm::Value *elemPtr = builder.CreateInBoundsGEP( |
| 4706 | Ty: arrTy, Ptr: arrAlloca, IdxList: {zero, builder.getInt32(C: i)}, Name: ".taskred.elem" ); |
| 4707 | auto storeField = [&](unsigned fieldIdx, llvm::Value *val) { |
| 4708 | llvm::Value *fieldPtr = |
| 4709 | builder.CreateStructGEP(Ty: redInputTy, Ptr: elemPtr, Idx: fieldIdx); |
| 4710 | builder.CreateStore(Val: val, Ptr: fieldPtr); |
| 4711 | }; |
| 4712 | storeField(0, orig); // reduce_shar |
| 4713 | storeField(1, orig); // reduce_orig |
| 4714 | storeField(2, llvm::ConstantInt::get(Ty: sizeTy, V: size)); // reduce_size |
| 4715 | storeField(3, initFn); // reduce_init |
| 4716 | storeField(4, llvm::ConstantPointerNull::get(T: ptrTy)); // reduce_fini |
| 4717 | storeField(5, combFn); // reduce_comb |
| 4718 | storeField(6, llvm::ConstantInt::get(Ty: i32Ty, V: 0)); // flags |
| 4719 | } |
| 4720 | |
| 4721 | // Emit the runtime call that registers the task reduction data. |
| 4722 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 4723 | uint32_t srcLocSize; |
| 4724 | llvm::Constant *srcLocStr = |
| 4725 | ompBuilder->getOrCreateSrcLocStr(Loc: ompLoc, SrcLocStrSize&: srcLocSize); |
| 4726 | llvm::Value *ident = ompBuilder->getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize); |
| 4727 | ompBuilder->updateToLocation(Loc: ompLoc); |
| 4728 | llvm::Value *gtid = ompBuilder->getOrCreateThreadID(Ident: ident); |
| 4729 | if (isModifier) { |
| 4730 | // __kmpc_taskred_modifier_init(loc, gtid, is_ws, num, &arr) opens a |
| 4731 | // task-reduction scope for the enclosing parallel/worksharing region. |
| 4732 | llvm::FunctionCallee modInit = ompBuilder->getOrCreateRuntimeFunction( |
| 4733 | M&: *llvmModule, FnID: llvm::omp::OMPRTL___kmpc_taskred_modifier_init); |
| 4734 | return builder.CreateCall(Callee: modInit, |
| 4735 | Args: {ident, gtid, |
| 4736 | builder.getInt32(C: isWorksharing ? 1 : 0), |
| 4737 | builder.getInt32(C: n), arrAlloca}, |
| 4738 | Name: ".taskred.desc" ); |
| 4739 | } |
| 4740 | // __kmpc_taskred_init(gtid, num, &arr). |
| 4741 | llvm::FunctionCallee taskredInit = ompBuilder->getOrCreateRuntimeFunction( |
| 4742 | M&: *llvmModule, FnID: llvm::omp::OMPRTL___kmpc_taskred_init); |
| 4743 | return builder.CreateCall(Callee: taskredInit, Args: {gtid, builder.getInt32(C: n), arrAlloca}, |
| 4744 | Name: ".taskred.desc" ); |
| 4745 | } |
| 4746 | |
| 4747 | /// Emits `__kmpc_task_reduction_modifier_fini(loc, gtid, is_ws)` at the current |
| 4748 | /// builder insertion point, closing the task-reduction scope opened by the |
| 4749 | /// `task` reduction modifier on a parallel or worksharing construct. |
| 4750 | static void |
| 4751 | emitTaskReductionModifierFini(bool isWorksharing, llvm::IRBuilderBase &builder, |
| 4752 | LLVM::ModuleTranslation &moduleTranslation) { |
| 4753 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 4754 | llvm::Module *llvmModule = moduleTranslation.getLLVMModule(); |
| 4755 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 4756 | uint32_t srcLocSize; |
| 4757 | llvm::Constant *srcLocStr = |
| 4758 | ompBuilder->getOrCreateSrcLocStr(Loc: ompLoc, SrcLocStrSize&: srcLocSize); |
| 4759 | llvm::Value *ident = ompBuilder->getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize); |
| 4760 | ompBuilder->updateToLocation(Loc: ompLoc); |
| 4761 | llvm::Value *gtid = ompBuilder->getOrCreateThreadID(Ident: ident); |
| 4762 | llvm::FunctionCallee fini = ompBuilder->getOrCreateRuntimeFunction( |
| 4763 | M&: *llvmModule, FnID: llvm::omp::OMPRTL___kmpc_task_reduction_modifier_fini); |
| 4764 | builder.CreateCall(Callee: fini, |
| 4765 | Args: {ident, gtid, builder.getInt32(C: isWorksharing ? 1 : 0)}); |
| 4766 | } |
| 4767 | |
| 4768 | /// Converts an OpenMP taskgroup construct into LLVM IR using OpenMPIRBuilder. |
| 4769 | static LogicalResult |
| 4770 | convertOmpTaskgroupOp(omp::TaskgroupOp tgOp, llvm::IRBuilderBase &builder, |
| 4771 | LLVM::ModuleTranslation &moduleTranslation) { |
| 4772 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 4773 | if (failed(Result: checkImplementationStatus(op&: *tgOp))) |
| 4774 | return failure(); |
| 4775 | |
| 4776 | // Resolve and validate task_reduction declarations up front. We only handle |
| 4777 | // declare_reduction ops shaped like a non-byref scalar reduction in this |
| 4778 | // first cut; richer shapes (two-argument initializer, cleanup region, |
| 4779 | // missing combiner) require additional infrastructure. |
| 4780 | SmallVector<omp::DeclareReductionOp> redDecls; |
| 4781 | if (auto syms = tgOp.getTaskReductionSyms()) { |
| 4782 | redDecls.reserve(N: syms->size()); |
| 4783 | for (auto sym : syms->getAsRange<SymbolRefAttr>()) { |
| 4784 | auto decl = SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>( |
| 4785 | from: tgOp, symbol: sym); |
| 4786 | if (!decl) |
| 4787 | return tgOp.emitError() |
| 4788 | << "failed to resolve task_reduction declare_reduction symbol " |
| 4789 | << sym.getRootReference() << " in omp.taskgroup" ; |
| 4790 | if (decl.getInitializerRegion().front().getNumArguments() != 1) |
| 4791 | return tgOp.emitError(message: "not yet implemented: task_reduction with " |
| 4792 | "two-argument initializer in omp.taskgroup" ); |
| 4793 | if (!decl.getCleanupRegion().empty()) |
| 4794 | return tgOp.emitError(message: "not yet implemented: task_reduction with " |
| 4795 | "cleanup region in omp.taskgroup" ); |
| 4796 | if (decl.getReductionRegion().empty()) |
| 4797 | return tgOp.emitError(message: "task_reduction declare_reduction is missing a " |
| 4798 | "combiner region" ); |
| 4799 | redDecls.push_back(Elt: decl); |
| 4800 | } |
| 4801 | } |
| 4802 | |
| 4803 | auto bodyCB = |
| 4804 | [&](InsertPointTy allocaIP, InsertPointTy codegenIP, |
| 4805 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error { |
| 4806 | builder.restoreIP(IP: codegenIP); |
| 4807 | |
| 4808 | if (!redDecls.empty()) { |
| 4809 | SmallVector<llvm::Value *> origPtrs; |
| 4810 | origPtrs.reserve(N: redDecls.size()); |
| 4811 | for (Value v : tgOp.getTaskReductionVars()) |
| 4812 | origPtrs.push_back(Elt: moduleTranslation.lookupValue(value: v)); |
| 4813 | if (!emitTaskReductionInitCall(redDecls, origPtrs, helperNamePrefix: "__omp_taskred_" , |
| 4814 | builder, allocaIP, moduleTranslation)) |
| 4815 | return llvm::createStringError( |
| 4816 | EC: llvm::inconvertibleErrorCode(), |
| 4817 | S: "failed to emit task_reduction initialization for omp.taskgroup" ); |
| 4818 | } |
| 4819 | |
| 4820 | // Inside the taskgroup body, each task_reduction block argument refers to |
| 4821 | // the same shared/original storage that the runtime now knows about via |
| 4822 | // the descriptor array. Inner tasks that declare in_reduction look up |
| 4823 | // per-task private copies through the runtime; the taskgroup body itself |
| 4824 | // uses the original variable. |
| 4825 | for (auto [i, blockArg] : |
| 4826 | llvm::enumerate(First: tgOp.getRegion().getArguments())) { |
| 4827 | llvm::Value *orig = |
| 4828 | moduleTranslation.lookupValue(value: tgOp.getTaskReductionVars()[i]); |
| 4829 | moduleTranslation.mapValue(mlir: blockArg, llvm: orig); |
| 4830 | } |
| 4831 | |
| 4832 | return convertOmpOpRegions(region&: tgOp.getRegion(), blockName: "omp.taskgroup.region" , |
| 4833 | builder, moduleTranslation) |
| 4834 | .takeError(); |
| 4835 | }; |
| 4836 | |
| 4837 | llvm::SmallVector<llvm::BasicBlock *> deallocBlocks; |
| 4838 | InsertPointTy allocaIP = |
| 4839 | findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks); |
| 4840 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 4841 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 4842 | getOpenMPBuilder(moduleTranslation) |
| 4843 | ->createTaskgroup(Loc: ompLoc, AllocaIP: allocaIP, DeallocBlocks: deallocBlocks, BodyGenCB: bodyCB); |
| 4844 | |
| 4845 | if (failed(Result: handleError(result&: afterIP, op&: *tgOp))) |
| 4846 | return failure(); |
| 4847 | |
| 4848 | builder.restoreIP(IP: *afterIP); |
| 4849 | return success(); |
| 4850 | } |
| 4851 | |
| 4852 | static LogicalResult |
| 4853 | convertOmpInteropInitOp(omp::InteropInitOp initOp, llvm::IRBuilderBase &builder, |
| 4854 | LLVM::ModuleTranslation &moduleTranslation) { |
| 4855 | if (!initOp.getDependVars().empty() || initOp.getDependKinds() || |
| 4856 | !initOp.getDependIterated().empty() || initOp.getDependIteratedKinds()) |
| 4857 | return initOp.emitError() |
| 4858 | << "not yet implemented: Unhandled clause depend in " |
| 4859 | << omp::InteropInitOp::getOperationName() << " operation" ; |
| 4860 | |
| 4861 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 4862 | llvm::Value *interopVar = |
| 4863 | moduleTranslation.lookupValue(value: initOp.getInteropVar()); |
| 4864 | llvm::Value *device = initOp.getDevice() |
| 4865 | ? moduleTranslation.lookupValue(value: initOp.getDevice()) |
| 4866 | : nullptr; |
| 4867 | |
| 4868 | // TODO: Handle depend clauses when supported. |
| 4869 | llvm::Value *numDeps = llvm::ConstantInt::get(Ty: builder.getInt32Ty(), V: 0); |
| 4870 | llvm::Value *depArray = llvm::ConstantPointerNull::get(T: builder.getPtrTy()); |
| 4871 | bool hasNowait = initOp.getNowait(); |
| 4872 | |
| 4873 | // A single `init` clause may list both `target` and `targetsync`, but the |
| 4874 | // runtime init call takes a single interop-type. Collapse the set to one |
| 4875 | // value, matching Clang: if `target` is present use Target, otherwise |
| 4876 | // TargetSync. The offload runtime object model supports only one type per |
| 4877 | // object; representing both would require a runtime change. |
| 4878 | bool hasTarget = false, hasTargetSync = false; |
| 4879 | for (mlir::Attribute typeAttr : initOp.getInteropTypes()) { |
| 4880 | switch (cast<omp::InteropTypeAttr>(Val&: typeAttr).getValue()) { |
| 4881 | case omp::InteropType::target: |
| 4882 | hasTarget = true; |
| 4883 | break; |
| 4884 | case omp::InteropType::targetsync: |
| 4885 | hasTargetSync = true; |
| 4886 | break; |
| 4887 | } |
| 4888 | } |
| 4889 | llvm::omp::OMPInteropType interopType = |
| 4890 | (!hasTarget && hasTargetSync) ? llvm::omp::OMPInteropType::TargetSync |
| 4891 | : llvm::omp::OMPInteropType::Target; |
| 4892 | ompBuilder->createOMPInteropInit(Loc: builder, InteropVar: interopVar, InteropType: interopType, Device: device, |
| 4893 | NumDependences: numDeps, DependenceAddress: depArray, HaveNowaitClause: hasNowait); |
| 4894 | return success(); |
| 4895 | } |
| 4896 | |
| 4897 | static LogicalResult |
| 4898 | convertOmpInteropDestroyOp(omp::InteropDestroyOp destroyOp, |
| 4899 | llvm::IRBuilderBase &builder, |
| 4900 | LLVM::ModuleTranslation &moduleTranslation) { |
| 4901 | if (!destroyOp.getDependVars().empty() || destroyOp.getDependKinds() || |
| 4902 | !destroyOp.getDependIterated().empty() || |
| 4903 | destroyOp.getDependIteratedKinds()) |
| 4904 | return destroyOp.emitError() |
| 4905 | << "not yet implemented: Unhandled clause depend in " |
| 4906 | << omp::InteropDestroyOp::getOperationName() << " operation" ; |
| 4907 | |
| 4908 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 4909 | llvm::Value *interopVar = |
| 4910 | moduleTranslation.lookupValue(value: destroyOp.getInteropVar()); |
| 4911 | llvm::Value *device = |
| 4912 | destroyOp.getDevice() |
| 4913 | ? moduleTranslation.lookupValue(value: destroyOp.getDevice()) |
| 4914 | : nullptr; |
| 4915 | |
| 4916 | llvm::Value *numDeps = llvm::ConstantInt::get(Ty: builder.getInt32Ty(), V: 0); |
| 4917 | llvm::Value *depArray = llvm::ConstantPointerNull::get(T: builder.getPtrTy()); |
| 4918 | bool hasNowait = destroyOp.getNowait(); |
| 4919 | |
| 4920 | ompBuilder->createOMPInteropDestroy(Loc: builder, InteropVar: interopVar, Device: device, NumDependences: numDeps, |
| 4921 | DependenceAddress: depArray, HaveNowaitClause: hasNowait); |
| 4922 | return success(); |
| 4923 | } |
| 4924 | |
| 4925 | static LogicalResult |
| 4926 | convertOmpInteropUseOp(omp::InteropUseOp useOp, llvm::IRBuilderBase &builder, |
| 4927 | LLVM::ModuleTranslation &moduleTranslation) { |
| 4928 | if (!useOp.getDependVars().empty() || useOp.getDependKinds() || |
| 4929 | !useOp.getDependIterated().empty() || useOp.getDependIteratedKinds()) |
| 4930 | return useOp.emitError() |
| 4931 | << "not yet implemented: Unhandled clause depend in " |
| 4932 | << omp::InteropUseOp::getOperationName() << " operation" ; |
| 4933 | |
| 4934 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 4935 | llvm::Value *interopVar = |
| 4936 | moduleTranslation.lookupValue(value: useOp.getInteropVar()); |
| 4937 | llvm::Value *device = useOp.getDevice() |
| 4938 | ? moduleTranslation.lookupValue(value: useOp.getDevice()) |
| 4939 | : nullptr; |
| 4940 | |
| 4941 | llvm::Value *numDeps = llvm::ConstantInt::get(Ty: builder.getInt32Ty(), V: 0); |
| 4942 | llvm::Value *depArray = llvm::ConstantPointerNull::get(T: builder.getPtrTy()); |
| 4943 | bool hasNowait = useOp.getNowait(); |
| 4944 | |
| 4945 | ompBuilder->createOMPInteropUse(Loc: builder, InteropVar: interopVar, Device: device, NumDependences: numDeps, |
| 4946 | DependenceAddress: depArray, HaveNowaitClause: hasNowait); |
| 4947 | return success(); |
| 4948 | } |
| 4949 | |
| 4950 | static LogicalResult |
| 4951 | convertOmpTaskwaitOp(omp::TaskwaitOp twOp, llvm::IRBuilderBase &builder, |
| 4952 | LLVM::ModuleTranslation &moduleTranslation) { |
| 4953 | if (failed(Result: checkImplementationStatus(op&: *twOp))) |
| 4954 | return failure(); |
| 4955 | |
| 4956 | llvm::OpenMPIRBuilder::DependenciesInfo dds; |
| 4957 | if (failed(Result: buildDependData( |
| 4958 | dependVars: twOp.getDependVars(), dependKinds: twOp.getDependKinds(), dependIterated: twOp.getDependIterated(), |
| 4959 | dependIteratedKinds: twOp.getDependIteratedKinds(), builder, moduleTranslation, taskDeps&: dds))) { |
| 4960 | return failure(); |
| 4961 | } |
| 4962 | |
| 4963 | getOpenMPBuilder(moduleTranslation) |
| 4964 | ->createTaskwait(Loc: builder, Dependencies: dds, IsNowait: twOp.getNowait()); |
| 4965 | if (dds.DepArray) { |
| 4966 | builder.CreateFree(Source: dds.DepArray); |
| 4967 | } |
| 4968 | |
| 4969 | return success(); |
| 4970 | } |
| 4971 | |
| 4972 | /// Converts an OpenMP workshare loop into LLVM IR using OpenMPIRBuilder. |
| 4973 | static LogicalResult |
| 4974 | convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder, |
| 4975 | LLVM::ModuleTranslation &moduleTranslation) { |
| 4976 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 4977 | auto wsloopOp = cast<omp::WsloopOp>(Val&: opInst); |
| 4978 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 4979 | return failure(); |
| 4980 | |
| 4981 | auto loopOp = cast<omp::LoopNestOp>(Val: wsloopOp.getWrappedLoop()); |
| 4982 | |
| 4983 | // Static is the default. |
| 4984 | auto schedule = |
| 4985 | wsloopOp.getScheduleKind().value_or(u: omp::ClauseScheduleKind::Static); |
| 4986 | |
| 4987 | // Find the loop configuration. |
| 4988 | llvm::Value *step = moduleTranslation.lookupValue(value: loopOp.getLoopSteps()[0]); |
| 4989 | llvm::Type *ivType = step->getType(); |
| 4990 | llvm::Value *chunk = nullptr; |
| 4991 | if (wsloopOp.getScheduleChunk()) { |
| 4992 | llvm::Value *chunkVar = |
| 4993 | moduleTranslation.lookupValue(value: wsloopOp.getScheduleChunk()); |
| 4994 | chunk = builder.CreateSExtOrTrunc(V: chunkVar, DestTy: ivType); |
| 4995 | } |
| 4996 | |
| 4997 | omp::DistributeOp distributeOp = nullptr; |
| 4998 | llvm::Value *distScheduleChunk = nullptr; |
| 4999 | bool hasDistSchedule = false; |
| 5000 | if (llvm::isa_and_present<omp::DistributeOp>(Val: opInst.getParentOp())) { |
| 5001 | distributeOp = cast<omp::DistributeOp>(Val: opInst.getParentOp()); |
| 5002 | hasDistSchedule = distributeOp.getDistScheduleStatic(); |
| 5003 | if (distributeOp.getDistScheduleChunkSize()) { |
| 5004 | llvm::Value *chunkVar = moduleTranslation.lookupValue( |
| 5005 | value: distributeOp.getDistScheduleChunkSize()); |
| 5006 | distScheduleChunk = builder.CreateSExtOrTrunc(V: chunkVar, DestTy: ivType); |
| 5007 | } |
| 5008 | } |
| 5009 | |
| 5010 | PrivateVarsInfo privateVarsInfo(wsloopOp); |
| 5011 | privateVarsInfo.addLinearVars(op: wsloopOp); |
| 5012 | ReductionVarsInfo reductionInfo(wsloopOp); |
| 5013 | |
| 5014 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 5015 | findAllocInsertPoints(builder, moduleTranslation); |
| 5016 | |
| 5017 | llvm::BasicBlock *allocaBlock = allocaIP.getNodeParent(); |
| 5018 | llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars( |
| 5019 | op: wsloopOp, builder, moduleTranslation, privateVarsInfo, allocaIP); |
| 5020 | if (handleError(result&: afterAllocas, op&: opInst).failed()) |
| 5021 | return failure(); |
| 5022 | |
| 5023 | if (failed(Result: allocReductionVars(op&: *wsloopOp, builder, moduleTranslation, allocaIP, |
| 5024 | redInfo&: reductionInfo))) |
| 5025 | return failure(); |
| 5026 | |
| 5027 | if (handleError(error: initPrivateVars(builder, moduleTranslation, privateVarsInfo), |
| 5028 | op&: opInst) |
| 5029 | .failed()) |
| 5030 | return failure(); |
| 5031 | |
| 5032 | if (failed(Result: completePrivateVars(op: wsloopOp, builder, moduleTranslation, |
| 5033 | privateVarsInfo, |
| 5034 | insertBarrier: wsloopOp.getPrivateNeedsBarrier()))) |
| 5035 | return failure(); |
| 5036 | |
| 5037 | assert(afterAllocas.get()->getSinglePredecessor()); |
| 5038 | if (failed(Result: initReductionVars(op&: *wsloopOp, builder, moduleTranslation, |
| 5039 | latestAllocaBlock: afterAllocas.get()->getSinglePredecessor(), |
| 5040 | redInfo&: reductionInfo))) |
| 5041 | return failure(); |
| 5042 | |
| 5043 | // For `reduction(task, ...)` open a task-reduction scope for the worksharing |
| 5044 | // loop. Participating explicit tasks accumulate into the per-thread private |
| 5045 | // copies, which the worksharing reduction then combines across threads. |
| 5046 | bool isTaskReductionMod = |
| 5047 | wsloopOp.hasReductionModifier(modifier: omp::ReductionModifier::task); |
| 5048 | if (isTaskReductionMod && |
| 5049 | !emitTaskReductionModifierInitCall( |
| 5050 | op: wsloopOp, reductionDecls: reductionInfo.getReductionDecls(), |
| 5051 | privateReductionVariables: reductionInfo.getPrivateReductionVariables(), builder, allocaIP, |
| 5052 | moduleTranslation, /*isWorksharing=*/true)) |
| 5053 | return wsloopOp.emitError( |
| 5054 | message: "failed to emit task reduction modifier initialization" ); |
| 5055 | |
| 5056 | // TODO: Handle doacross loops when the ordered clause has a parameter. |
| 5057 | bool isOrdered = wsloopOp.getOrdered().has_value(); |
| 5058 | std::optional<omp::ScheduleModifier> scheduleMod = wsloopOp.getScheduleMod(); |
| 5059 | bool isSimd = wsloopOp.getScheduleSimd(); |
| 5060 | bool loopNeedsBarrier = !wsloopOp.getNowait(); |
| 5061 | |
| 5062 | // The only legal way for the direct parent to be omp.distribute is that this |
| 5063 | // represents 'distribute parallel do'. Otherwise, this is a regular |
| 5064 | // worksharing loop. |
| 5065 | llvm::omp::WorksharingLoopType workshareLoopType = |
| 5066 | llvm::isa_and_present<omp::DistributeOp>(Val: opInst.getParentOp()) |
| 5067 | ? llvm::omp::WorksharingLoopType::DistributeForStaticLoop |
| 5068 | : llvm::omp::WorksharingLoopType::ForStaticLoop; |
| 5069 | |
| 5070 | SmallVector<llvm::UncondBrInst *> cancelTerminators; |
| 5071 | pushCancelFinalizationCB(cancelTerminators, llvmBuilder&: builder, ompBuilder&: *ompBuilder, op: wsloopOp, |
| 5072 | cancelDirective: llvm::omp::Directive::OMPD_for); |
| 5073 | |
| 5074 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 5075 | |
| 5076 | // Initialize linear variables and linear step |
| 5077 | LinearClauseProcessor linearClauseProcessor; |
| 5078 | |
| 5079 | if (!wsloopOp.getLinearVars().empty()) { |
| 5080 | auto linearVarTypes = wsloopOp.getLinearVarTypes().value(); |
| 5081 | for (mlir::Attribute linearVarType : linearVarTypes) |
| 5082 | linearClauseProcessor.registerType(moduleTranslation, ty&: linearVarType); |
| 5083 | |
| 5084 | for (auto [idx, linearVar] : llvm::enumerate(First: wsloopOp.getLinearVars())) { |
| 5085 | size_t privIdx = privateVarsInfo.numPrivateVars + idx; |
| 5086 | linearClauseProcessor.createLinearVar( |
| 5087 | builder, allocaIP, privatizer: privateVarsInfo.vars[privIdx].privatizer, |
| 5088 | origVar: moduleTranslation.lookupValue(value: linearVar), |
| 5089 | privateVar: privateVarsInfo.vars[privIdx].llvmVar, idx); |
| 5090 | } |
| 5091 | for (mlir::Value linearStep : wsloopOp.getLinearStepVars()) |
| 5092 | linearClauseProcessor.initLinearStep(moduleTranslation, linearStep); |
| 5093 | } |
| 5094 | |
| 5095 | llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions( |
| 5096 | region&: wsloopOp.getRegion(), blockName: "omp.wsloop.region" , builder, moduleTranslation); |
| 5097 | |
| 5098 | if (failed(Result: handleError(result&: regionBlock, op&: opInst))) |
| 5099 | return failure(); |
| 5100 | |
| 5101 | llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation); |
| 5102 | |
| 5103 | // Emit Initialization and Update IR for linear variables |
| 5104 | if (!wsloopOp.getLinearVars().empty()) { |
| 5105 | if (failed(Result: handleError( |
| 5106 | error: linearClauseProcessor.initLinearVar(builder, moduleTranslation, |
| 5107 | loopPreHeader: loopInfo->getPreheader()), |
| 5108 | op&: opInst))) |
| 5109 | return failure(); |
| 5110 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP = |
| 5111 | getOpenMPBuilder(moduleTranslation) |
| 5112 | ->createBarrier(Loc: builder, Kind: llvm::omp::OMPD_barrier); |
| 5113 | if (failed(Result: handleError(result&: afterBarrierIP, op&: *loopOp))) |
| 5114 | return failure(); |
| 5115 | builder.restoreIP(IP: *afterBarrierIP); |
| 5116 | if (failed(Result: handleError(error: linearClauseProcessor.updateLinearVar( |
| 5117 | builder, moduleTranslation, loopBody: loopInfo->getBody(), |
| 5118 | loopInductionVar: loopInfo->getIndVar()), |
| 5119 | op&: opInst))) |
| 5120 | return failure(); |
| 5121 | linearClauseProcessor.splitLinearFiniBB(builder, loopExit: loopInfo->getExit()); |
| 5122 | } |
| 5123 | |
| 5124 | builder.SetInsertPoint((*regionBlock)->begin()); |
| 5125 | |
| 5126 | // Check if we can generate no-loop kernel |
| 5127 | bool noLoopMode = false; |
| 5128 | omp::TargetOp targetOp = wsloopOp->getParentOfType<mlir::omp::TargetOp>(); |
| 5129 | if (targetOp && |
| 5130 | targetOp.getKernelType() == omp::TargetExecMode::spmd_no_loop) { |
| 5131 | Operation *targetCapturedOp = |
| 5132 | cast<omp::ComposableOpInterface>(Val&: *targetOp).findCapturedOp(); |
| 5133 | // We need this check because, without it, noLoopMode would be set to true |
| 5134 | // for every omp.wsloop nested inside a no-loop SPMD target region, even if |
| 5135 | // that loop is not the top-level SPMD one. |
| 5136 | if (loopOp == targetCapturedOp) |
| 5137 | noLoopMode = true; |
| 5138 | } |
| 5139 | |
| 5140 | // Nested regions can move the instruction referenced by allocaIP. |
| 5141 | allocaIP = allocaBlock->getFirstInsertionPt(); |
| 5142 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP = |
| 5143 | ompBuilder->applyWorkshareLoop( |
| 5144 | DL: ompLoc.DL, CLI: loopInfo, AllocaIP: allocaIP, NeedsBarrier: loopNeedsBarrier, |
| 5145 | SchedKind: convertToScheduleKind(schedKind: schedule), ChunkSize: chunk, HasSimdModifier: isSimd, |
| 5146 | HasMonotonicModifier: scheduleMod == omp::ScheduleModifier::monotonic, |
| 5147 | HasNonmonotonicModifier: scheduleMod == omp::ScheduleModifier::nonmonotonic, HasOrderedClause: isOrdered, |
| 5148 | LoopType: workshareLoopType, NoLoop: noLoopMode, HasDistSchedule: hasDistSchedule, DistScheduleChunkSize: distScheduleChunk, |
| 5149 | NeedsLastIter: !wsloopOp.getLinearVars().empty()); |
| 5150 | |
| 5151 | if (failed(Result: handleError(result&: wsloopIP, op&: opInst))) |
| 5152 | return failure(); |
| 5153 | |
| 5154 | // Save the continuation block before linear var finalization, which may |
| 5155 | // invalidate the wsloopIP iterator. |
| 5156 | llvm::BasicBlock *wsloopContinuationBB = wsloopIP->getNodeParent(); |
| 5157 | |
| 5158 | // Emit finalization and in-place rewrites for linear vars. |
| 5159 | if (!wsloopOp.getLinearVars().empty()) { |
| 5160 | llvm::OpenMPIRBuilder::InsertPointTy oldIP = builder.saveIP(); |
| 5161 | assert(loopInfo->getLastIter() && |
| 5162 | "`lastiter` in CanonicalLoopInfo is nullptr" ); |
| 5163 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP = |
| 5164 | linearClauseProcessor.finalizeLinearVar(builder, moduleTranslation, |
| 5165 | lastIter: loopInfo->getLastIter()); |
| 5166 | if (failed(Result: handleError(result&: afterBarrierIP, op&: *loopOp))) |
| 5167 | return failure(); |
| 5168 | |
| 5169 | builder.restoreIP(IP: oldIP); |
| 5170 | } |
| 5171 | |
| 5172 | // Set the correct branch target for task cancellation |
| 5173 | popCancelFinalizationCB(cancelTerminators, ompBuilder&: *ompBuilder, afterBB: wsloopContinuationBB); |
| 5174 | |
| 5175 | // Close the task-reduction scope before the worksharing reduction combine. |
| 5176 | if (isTaskReductionMod) |
| 5177 | emitTaskReductionModifierFini(/*isWorksharing=*/true, builder, |
| 5178 | moduleTranslation); |
| 5179 | |
| 5180 | // Process the reductions if required. |
| 5181 | if (failed(Result: createReductionsAndCleanup(op&: *wsloopOp, builder, moduleTranslation, |
| 5182 | allocaIP, redInfo&: reductionInfo, |
| 5183 | isNowait: wsloopOp.getNowait(), |
| 5184 | /*isTeamsReduction=*/false))) |
| 5185 | return failure(); |
| 5186 | |
| 5187 | return cleanupPrivateVars(op: wsloopOp, builder, moduleTranslation, |
| 5188 | loc: wsloopOp.getLoc(), privateVarsInfo); |
| 5189 | } |
| 5190 | |
| 5191 | /// Converts the OpenMP parallel operation to LLVM IR. |
| 5192 | static LogicalResult |
| 5193 | convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, |
| 5194 | LLVM::ModuleTranslation &moduleTranslation) { |
| 5195 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 5196 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 5197 | bool isCancellable = constructIsCancellable(op: opInst); |
| 5198 | |
| 5199 | if (failed(Result: checkImplementationStatus(op&: *opInst))) |
| 5200 | return failure(); |
| 5201 | |
| 5202 | PrivateVarsInfo privateVarsInfo(opInst); |
| 5203 | if (failed(Result: convertAllocatorVars(op&: *opInst, allocatorVars: opInst.getAllocatorVars(), builder, |
| 5204 | moduleTranslation, privateVarsInfo))) |
| 5205 | return failure(); |
| 5206 | |
| 5207 | // Gather reduction info. |
| 5208 | ReductionVarsInfo reductionInfo(opInst); |
| 5209 | |
| 5210 | // Only open a task-reduction scope when the `task` modifier is present and |
| 5211 | // there are reduction variables to combine; otherwise the matching fini in |
| 5212 | // the reduction-combine path (guarded by getNumReductionVars() > 0) would be |
| 5213 | // skipped, leaving the modifier init unbalanced. |
| 5214 | bool isTaskReductionMod = |
| 5215 | opInst.hasReductionModifier(modifier: omp::ReductionModifier::task); |
| 5216 | |
| 5217 | auto bodyGenCB = |
| 5218 | [&](InsertPointTy allocaIP, InsertPointTy codeGenIP, |
| 5219 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error { |
| 5220 | llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars( |
| 5221 | op: opInst, builder, moduleTranslation, privateVarsInfo, allocaIP); |
| 5222 | if (handleError(result&: afterAllocas, op&: *opInst).failed()) |
| 5223 | return llvm::make_error<PreviouslyReportedError>(); |
| 5224 | |
| 5225 | // Allocate reduction vars |
| 5226 | allocaIP = allocaIP.getNodeParent()->getTerminator()->getIterator(); |
| 5227 | |
| 5228 | if (failed(Result: allocReductionVars(op&: *opInst, builder, moduleTranslation, allocaIP, |
| 5229 | redInfo&: reductionInfo))) |
| 5230 | return llvm::make_error<PreviouslyReportedError>(); |
| 5231 | |
| 5232 | assert(afterAllocas.get()->getSinglePredecessor()); |
| 5233 | builder.restoreIP(IP: codeGenIP); |
| 5234 | |
| 5235 | if (handleError( |
| 5236 | error: initPrivateVars(builder, moduleTranslation, privateVarsInfo), |
| 5237 | op&: *opInst) |
| 5238 | .failed()) |
| 5239 | return llvm::make_error<PreviouslyReportedError>(); |
| 5240 | |
| 5241 | if (failed(Result: completePrivateVars(op: opInst, builder, moduleTranslation, |
| 5242 | privateVarsInfo, |
| 5243 | insertBarrier: opInst.getPrivateNeedsBarrier()))) |
| 5244 | return llvm::make_error<PreviouslyReportedError>(); |
| 5245 | |
| 5246 | if (failed(Result: initReductionVars(op&: *opInst, builder, moduleTranslation, |
| 5247 | latestAllocaBlock: afterAllocas.get()->getSinglePredecessor(), |
| 5248 | redInfo&: reductionInfo))) |
| 5249 | return llvm::make_error<PreviouslyReportedError>(); |
| 5250 | |
| 5251 | // For `reduction(task, ...)` open a task-reduction scope so participating |
| 5252 | // explicit tasks accumulate into the per-thread private copies; the |
| 5253 | // parallel reduction then combines those copies across the team. |
| 5254 | if (isTaskReductionMod && |
| 5255 | !emitTaskReductionModifierInitCall( |
| 5256 | op: opInst, reductionDecls: reductionInfo.getReductionDecls(), |
| 5257 | privateReductionVariables: reductionInfo.getPrivateReductionVariables(), builder, allocaIP, |
| 5258 | moduleTranslation, /*isWorksharing=*/false)) |
| 5259 | return llvm::createStringError( |
| 5260 | Fmt: "failed to emit task reduction modifier initialization" ); |
| 5261 | |
| 5262 | // Save the alloca insertion point on ModuleTranslation stack for use in |
| 5263 | // nested regions. |
| 5264 | LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame( |
| 5265 | moduleTranslation, allocaIP.getNodeParent(), deallocBlocks); |
| 5266 | |
| 5267 | // ParallelOp has only one region associated with it. |
| 5268 | llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions( |
| 5269 | region&: opInst.getRegion(), blockName: "omp.par.region" , builder, moduleTranslation); |
| 5270 | if (!regionBlock) |
| 5271 | return regionBlock.takeError(); |
| 5272 | |
| 5273 | // Process the reductions if required. |
| 5274 | if (opInst.getNumReductionVars() > 0) { |
| 5275 | // Collect reduction info |
| 5276 | SmallVector<OwningReductionGen> owningReductionGens; |
| 5277 | SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens; |
| 5278 | SmallVector<OwningDataPtrPtrReductionGen> |
| 5279 | owningReductionGenRefDataPtrGens; |
| 5280 | SmallVector<llvm::OpenMPIRBuilder::ReductionInfo, 2> reductionInfos; |
| 5281 | collectReductionInfo(builder, moduleTranslation, redInfo&: reductionInfo, |
| 5282 | owningReductionGens, owningAtomicReductionGens, |
| 5283 | owningDataPtrPtrReductionGens&: owningReductionGenRefDataPtrGens, reductionInfos); |
| 5284 | |
| 5285 | // Move to region cont block |
| 5286 | builder.SetInsertPoint((*regionBlock)->getTerminator()); |
| 5287 | |
| 5288 | // Close the task-reduction scope before the per-thread reduction |
| 5289 | // contributions are combined across the team. |
| 5290 | if (isTaskReductionMod) |
| 5291 | emitTaskReductionModifierFini(/*isWorksharing=*/false, builder, |
| 5292 | moduleTranslation); |
| 5293 | |
| 5294 | // Generate reductions from info |
| 5295 | llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable(); |
| 5296 | builder.SetInsertPoint(tempTerminator); |
| 5297 | |
| 5298 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy contInsertPoint = |
| 5299 | ompBuilder->createReductions(Loc: builder, AllocaIP: allocaIP, ReductionInfos: reductionInfos, |
| 5300 | IsByRef: reductionInfo.getIsByRef(), |
| 5301 | /*IsNoWait=*/false, |
| 5302 | /*IsTeamsReduction=*/false); |
| 5303 | if (!contInsertPoint) |
| 5304 | return contInsertPoint.takeError(); |
| 5305 | |
| 5306 | if (!contInsertPoint->isValid()) |
| 5307 | return llvm::make_error<PreviouslyReportedError>(); |
| 5308 | |
| 5309 | tempTerminator->eraseFromParent(); |
| 5310 | builder.restoreIP(IP: *contInsertPoint); |
| 5311 | } |
| 5312 | |
| 5313 | return llvm::Error::success(); |
| 5314 | }; |
| 5315 | |
| 5316 | auto privCB = [](InsertPointTy allocaIP, InsertPointTy codeGenIP, |
| 5317 | llvm::Value &, llvm::Value &val, llvm::Value *&replVal) { |
| 5318 | // tell OpenMPIRBuilder not to do anything. We handled Privatisation in |
| 5319 | // bodyGenCB. |
| 5320 | replVal = &val; |
| 5321 | return codeGenIP; |
| 5322 | }; |
| 5323 | |
| 5324 | // TODO: Perform finalization actions for variables. This has to be |
| 5325 | // called for variables which have destructors/finalizers. |
| 5326 | auto finiCB = [&](InsertPointTy codeGenIP) -> llvm::Error { |
| 5327 | InsertPointTy oldIP = builder.saveIP(); |
| 5328 | builder.restoreIP(IP: codeGenIP); |
| 5329 | |
| 5330 | // if the reduction has a cleanup region, inline it here to finalize the |
| 5331 | // reduction variables |
| 5332 | SmallVector<Region *> reductionCleanupRegions; |
| 5333 | llvm::transform(Range&: reductionInfo.vars, |
| 5334 | d_first: std::back_inserter(x&: reductionCleanupRegions), |
| 5335 | F: [](ReductionVarsInfo::VarInfo &var) { |
| 5336 | return &var.decl.getCleanupRegion(); |
| 5337 | }); |
| 5338 | if (failed(Result: inlineOmpRegionCleanup( |
| 5339 | cleanupRegions&: reductionCleanupRegions, |
| 5340 | privateVariables: reductionInfo.getPrivateReductionVariables(), moduleTranslation, |
| 5341 | builder, regionName: "omp.reduction.cleanup" ))) |
| 5342 | return llvm::createStringError( |
| 5343 | Fmt: "failed to inline `cleanup` region of `omp.declare_reduction`" ); |
| 5344 | |
| 5345 | if (failed(Result: cleanupPrivateVars(op: opInst, builder, moduleTranslation, |
| 5346 | loc: opInst.getLoc(), privateVarsInfo))) |
| 5347 | return llvm::make_error<PreviouslyReportedError>(); |
| 5348 | |
| 5349 | // If we could be performing cancellation, add the cancellation barrier on |
| 5350 | // the way out of the outlined region. |
| 5351 | if (isCancellable) { |
| 5352 | auto IPOrErr = ompBuilder->createBarrier( |
| 5353 | Loc: llvm::OpenMPIRBuilder::LocationDescription(builder), |
| 5354 | Kind: llvm::omp::Directive::OMPD_unknown, |
| 5355 | /* ForceSimpleCall */ false, |
| 5356 | /* CheckCancelFlag */ false); |
| 5357 | if (!IPOrErr) |
| 5358 | return IPOrErr.takeError(); |
| 5359 | } |
| 5360 | |
| 5361 | builder.restoreIP(IP: oldIP); |
| 5362 | return llvm::Error::success(); |
| 5363 | }; |
| 5364 | |
| 5365 | llvm::Value *ifCond = nullptr; |
| 5366 | if (auto ifVar = opInst.getIfExpr()) |
| 5367 | ifCond = moduleTranslation.lookupValue(value: ifVar); |
| 5368 | llvm::Value *numThreads = nullptr; |
| 5369 | if (!opInst.getNumThreadsVars().empty()) |
| 5370 | numThreads = moduleTranslation.lookupValue(value: opInst.getNumThreads(dim: 0)); |
| 5371 | auto pbKind = llvm::omp::OMP_PROC_BIND_default; |
| 5372 | if (auto bind = opInst.getProcBindKind()) |
| 5373 | pbKind = getProcBindKind(kind: *bind); |
| 5374 | |
| 5375 | llvm::SmallVector<llvm::BasicBlock *> deallocBlocks; |
| 5376 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 5377 | findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks); |
| 5378 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 5379 | |
| 5380 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 5381 | ompBuilder->createParallel(Loc: ompLoc, AllocaIP: allocaIP, DeallocBlocks: deallocBlocks, BodyGenCB: bodyGenCB, |
| 5382 | PrivCB: privCB, FiniCB: finiCB, IfCondition: ifCond, NumThreads: numThreads, ProcBind: pbKind, |
| 5383 | IsCancellable: isCancellable); |
| 5384 | |
| 5385 | if (failed(Result: handleError(result&: afterIP, op&: *opInst))) |
| 5386 | return failure(); |
| 5387 | |
| 5388 | builder.restoreIP(IP: *afterIP); |
| 5389 | return success(); |
| 5390 | } |
| 5391 | |
| 5392 | /// Convert Order attribute to llvm::omp::OrderKind. |
| 5393 | static llvm::omp::OrderKind |
| 5394 | convertOrderKind(std::optional<omp::ClauseOrderKind> o) { |
| 5395 | if (!o) |
| 5396 | return llvm::omp::OrderKind::OMP_ORDER_unknown; |
| 5397 | switch (*o) { |
| 5398 | case omp::ClauseOrderKind::Concurrent: |
| 5399 | return llvm::omp::OrderKind::OMP_ORDER_concurrent; |
| 5400 | } |
| 5401 | llvm_unreachable("Unknown ClauseOrderKind kind" ); |
| 5402 | } |
| 5403 | |
| 5404 | /// Converts an OpenMP simd loop into LLVM IR using OpenMPIRBuilder. |
| 5405 | static LogicalResult |
| 5406 | convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder, |
| 5407 | LLVM::ModuleTranslation &moduleTranslation) { |
| 5408 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 5409 | auto simdOp = cast<omp::SimdOp>(Val&: opInst); |
| 5410 | |
| 5411 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 5412 | return failure(); |
| 5413 | |
| 5414 | PrivateVarsInfo privateVarsInfo(simdOp); |
| 5415 | privateVarsInfo.addLinearVars(op: simdOp); |
| 5416 | ReductionVarsInfo reductionInfo(simdOp); |
| 5417 | |
| 5418 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 5419 | findAllocInsertPoints(builder, moduleTranslation); |
| 5420 | |
| 5421 | llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars( |
| 5422 | op: simdOp, builder, moduleTranslation, privateVarsInfo, allocaIP); |
| 5423 | if (handleError(result&: afterAllocas, op&: opInst).failed()) |
| 5424 | return failure(); |
| 5425 | |
| 5426 | if (failed(Result: allocReductionVars(op&: *simdOp, builder, moduleTranslation, allocaIP, |
| 5427 | redInfo&: reductionInfo))) |
| 5428 | return failure(); |
| 5429 | |
| 5430 | if (handleError(error: initPrivateVars(builder, moduleTranslation, privateVarsInfo), |
| 5431 | op&: opInst) |
| 5432 | .failed()) |
| 5433 | return failure(); |
| 5434 | |
| 5435 | // No call to completePrivateVars because SIMD does not allow FIRSTPRIVATE and |
| 5436 | // is not a worksharing construct, so a privatization barrier here could |
| 5437 | // deadlock. |
| 5438 | |
| 5439 | // Initialize linear variables and linear step |
| 5440 | LinearClauseProcessor linearClauseProcessor; |
| 5441 | if (linearClauseProcessor.initLinearIV(simdOp).failed()) |
| 5442 | return failure(); |
| 5443 | |
| 5444 | if (!simdOp.getLinearVars().empty()) { |
| 5445 | auto linearVarTypes = simdOp.getLinearVarTypes().value(); |
| 5446 | for (mlir::Attribute linearVarType : linearVarTypes) |
| 5447 | linearClauseProcessor.registerType(moduleTranslation, ty&: linearVarType); |
| 5448 | for (auto [idx, linearVar] : llvm::enumerate(First: simdOp.getLinearVars())) { |
| 5449 | size_t privIdx = privateVarsInfo.numPrivateVars + idx; |
| 5450 | linearClauseProcessor.createLinearVar( |
| 5451 | builder, allocaIP, privatizer: privateVarsInfo.vars[privIdx].privatizer, |
| 5452 | origVar: moduleTranslation.lookupValue(value: linearVar), |
| 5453 | privateVar: privateVarsInfo.vars[privIdx].llvmVar, idx); |
| 5454 | } |
| 5455 | for (mlir::Value linearStep : simdOp.getLinearStepVars()) |
| 5456 | linearClauseProcessor.initLinearStep(moduleTranslation, linearStep); |
| 5457 | } |
| 5458 | |
| 5459 | assert(afterAllocas.get()->getSinglePredecessor()); |
| 5460 | if (failed(Result: initReductionVars(op&: *simdOp, builder, moduleTranslation, |
| 5461 | latestAllocaBlock: afterAllocas.get()->getSinglePredecessor(), |
| 5462 | redInfo&: reductionInfo))) |
| 5463 | return failure(); |
| 5464 | |
| 5465 | llvm::ConstantInt *simdlen = nullptr; |
| 5466 | if (std::optional<uint64_t> simdlenVar = simdOp.getSimdlen()) |
| 5467 | simdlen = builder.getInt64(C: simdlenVar.value()); |
| 5468 | |
| 5469 | llvm::ConstantInt *safelen = nullptr; |
| 5470 | if (std::optional<uint64_t> safelenVar = simdOp.getSafelen()) |
| 5471 | safelen = builder.getInt64(C: safelenVar.value()); |
| 5472 | |
| 5473 | llvm::MapVector<llvm::Value *, llvm::Value *> alignedVars; |
| 5474 | llvm::omp::OrderKind order = convertOrderKind(o: simdOp.getOrder()); |
| 5475 | |
| 5476 | llvm::BasicBlock *sourceBlock = builder.GetInsertBlock(); |
| 5477 | std::optional<ArrayAttr> alignmentValues = simdOp.getAlignments(); |
| 5478 | mlir::OperandRange operands = simdOp.getAlignedVars(); |
| 5479 | for (size_t i = 0; i < operands.size(); ++i) { |
| 5480 | llvm::Value *alignment = nullptr; |
| 5481 | llvm::Value *llvmVal = moduleTranslation.lookupValue(value: operands[i]); |
| 5482 | llvm::Type *ty = llvmVal->getType(); |
| 5483 | |
| 5484 | auto intAttr = cast<IntegerAttr>(Val: (*alignmentValues)[i]); |
| 5485 | alignment = builder.getInt64(C: intAttr.getInt()); |
| 5486 | assert(ty->isPointerTy() && "Invalid type for aligned variable" ); |
| 5487 | assert(alignment && "Invalid alignment value" ); |
| 5488 | |
| 5489 | // Check if the alignment value is not a power of 2. If so, skip emitting |
| 5490 | // alignment. |
| 5491 | if (!intAttr.getValue().isPowerOf2()) |
| 5492 | continue; |
| 5493 | |
| 5494 | auto curInsert = builder.saveIP(); |
| 5495 | builder.SetInsertPoint(sourceBlock); |
| 5496 | llvmVal = builder.CreateLoad(Ty: ty, Ptr: llvmVal); |
| 5497 | builder.restoreIP(IP: curInsert); |
| 5498 | alignedVars[llvmVal] = alignment; |
| 5499 | } |
| 5500 | |
| 5501 | llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions( |
| 5502 | region&: simdOp.getRegion(), blockName: "omp.simd.region" , builder, moduleTranslation); |
| 5503 | |
| 5504 | if (failed(Result: handleError(result&: regionBlock, op&: opInst))) |
| 5505 | return failure(); |
| 5506 | |
| 5507 | llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation); |
| 5508 | // Emit Initialization for linear variables |
| 5509 | if (simdOp.getLinearVars().size()) { |
| 5510 | if (failed(Result: handleError( |
| 5511 | error: linearClauseProcessor.initLinearVar(builder, moduleTranslation, |
| 5512 | loopPreHeader: loopInfo->getPreheader()), |
| 5513 | op&: opInst))) |
| 5514 | return failure(); |
| 5515 | |
| 5516 | if (failed(Result: handleError(error: linearClauseProcessor.updateLinearVar( |
| 5517 | builder, moduleTranslation, loopBody: loopInfo->getBody(), |
| 5518 | loopInductionVar: loopInfo->getIndVar()), |
| 5519 | op&: opInst))) |
| 5520 | return failure(); |
| 5521 | } |
| 5522 | builder.SetInsertPoint((*regionBlock)->begin()); |
| 5523 | |
| 5524 | ompBuilder->applySimd(Loop: loopInfo, AlignedVars: alignedVars, |
| 5525 | IfCond: simdOp.getIfExpr() |
| 5526 | ? moduleTranslation.lookupValue(value: simdOp.getIfExpr()) |
| 5527 | : nullptr, |
| 5528 | Order: order, Simdlen: simdlen, Safelen: safelen); |
| 5529 | |
| 5530 | if (failed(Result: handleError( |
| 5531 | error: linearClauseProcessor.updateLinearIV(builder, moduleTranslation), |
| 5532 | op&: opInst))) |
| 5533 | return failure(); |
| 5534 | if (failed(Result: handleError(error: linearClauseProcessor.emitStoresForLinearVar( |
| 5535 | builder, moduleTranslation), |
| 5536 | op&: opInst))) |
| 5537 | return failure(); |
| 5538 | |
| 5539 | // We now need to reduce the per-simd-lane reduction variable into the |
| 5540 | // original variable. This works a bit differently to other reductions (e.g. |
| 5541 | // wsloop) because we don't need to call into the OpenMP runtime to handle |
| 5542 | // threads: everything happened in this one thread. |
| 5543 | for (auto [i, var] : llvm::enumerate(First&: reductionInfo.vars)) { |
| 5544 | OwningReductionGen gen = |
| 5545 | makeReductionGen(decl: var.decl, builder, moduleTranslation); |
| 5546 | llvm::Value *originalVariable = moduleTranslation.lookupValue(value: var.mlirVar); |
| 5547 | llvm::Type *reductionType = |
| 5548 | moduleTranslation.convertType(type: var.decl.getType()); |
| 5549 | |
| 5550 | // We have one less load for by-ref case because that load is now inside of |
| 5551 | // the reduction region. |
| 5552 | llvm::Value *redValue = originalVariable; |
| 5553 | if (!var.byRef) |
| 5554 | redValue = |
| 5555 | builder.CreateLoad(Ty: reductionType, Ptr: redValue, Name: "red.value." + Twine(i)); |
| 5556 | llvm::Value *privateRedValue = builder.CreateLoad( |
| 5557 | Ty: reductionType, Ptr: var.privateVar, Name: "red.private.value." + Twine(i)); |
| 5558 | llvm::Value *reduced; |
| 5559 | |
| 5560 | auto res = gen(builder.saveIP(), redValue, privateRedValue, reduced); |
| 5561 | if (failed(Result: handleError(result&: res, op&: opInst))) |
| 5562 | return failure(); |
| 5563 | builder.restoreIP(IP: res.get()); |
| 5564 | |
| 5565 | // For by-ref case, the store is inside of the reduction region. |
| 5566 | if (!var.byRef) |
| 5567 | builder.CreateStore(Val: reduced, Ptr: originalVariable); |
| 5568 | } |
| 5569 | |
| 5570 | // After the construct, deallocate private reduction variables. |
| 5571 | SmallVector<Region *> reductionRegions; |
| 5572 | llvm::transform(Range&: reductionInfo.vars, d_first: std::back_inserter(x&: reductionRegions), |
| 5573 | F: [](ReductionVarsInfo::VarInfo &var) { |
| 5574 | return &var.decl.getCleanupRegion(); |
| 5575 | }); |
| 5576 | if (failed(Result: inlineOmpRegionCleanup( |
| 5577 | cleanupRegions&: reductionRegions, privateVariables: reductionInfo.getPrivateReductionVariables(), |
| 5578 | moduleTranslation, builder, regionName: "omp.reduction.cleanup" ))) |
| 5579 | return failure(); |
| 5580 | |
| 5581 | return cleanupPrivateVars(op: simdOp, builder, moduleTranslation, loc: simdOp.getLoc(), |
| 5582 | privateVarsInfo); |
| 5583 | } |
| 5584 | |
| 5585 | /// Select Virtual only for two constant-count host worksharing loops. |
| 5586 | static llvm::OpenMPIRBuilder::CollapseIVMapping |
| 5587 | selectCollapseIVMapping(omp::LoopNestOp loopOp, bool isTargetDevice, |
| 5588 | ArrayRef<llvm::CanonicalLoopInfo *> loops) { |
| 5589 | using Mapping = llvm::OpenMPIRBuilder::CollapseIVMapping; |
| 5590 | if (isTargetDevice || loopOp.getCollapseNumLoops() != 2 || |
| 5591 | disableVirtualCollapse) |
| 5592 | return Mapping::DivMod; |
| 5593 | auto wsloop = dyn_cast<omp::WsloopOp>(Val: loopOp->getParentOp()); |
| 5594 | if (!wsloop || isa<omp::LoopWrapperInterface>(Val: wsloop->getParentOp()) || |
| 5595 | wsloop.getScheduleKind().value_or(u: omp::ClauseScheduleKind::Static) != |
| 5596 | omp::ClauseScheduleKind::Static || |
| 5597 | wsloop.getScheduleChunk() || wsloop.getOrdered().has_value() || |
| 5598 | !wsloop.getLinearVars().empty()) |
| 5599 | return Mapping::DivMod; |
| 5600 | // Use the existing normalized counts; the builder also validates direct |
| 5601 | // API callers. |
| 5602 | if (llvm::any_of(Range&: loops, P: [](llvm::CanonicalLoopInfo *loop) { |
| 5603 | auto *count = llvm::dyn_cast<llvm::ConstantInt>(Val: loop->getTripCount()); |
| 5604 | return !count; |
| 5605 | })) |
| 5606 | return Mapping::DivMod; |
| 5607 | return Mapping::Virtual; |
| 5608 | } |
| 5609 | |
| 5610 | /// Converts an OpenMP loop nest into LLVM IR using OpenMPIRBuilder. |
| 5611 | static LogicalResult |
| 5612 | convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder, |
| 5613 | LLVM::ModuleTranslation &moduleTranslation) { |
| 5614 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 5615 | auto loopOp = cast<omp::LoopNestOp>(Val&: opInst); |
| 5616 | |
| 5617 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 5618 | return failure(); |
| 5619 | |
| 5620 | // Set up the source location value for OpenMP runtime. |
| 5621 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 5622 | |
| 5623 | // Generator of the canonical loop body. |
| 5624 | SmallVector<llvm::CanonicalLoopInfo *> loopInfos; |
| 5625 | SmallVector<llvm::OpenMPIRBuilder::InsertPointTy> bodyInsertPoints; |
| 5626 | auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip, |
| 5627 | llvm::Value *iv) -> llvm::Error { |
| 5628 | // Make sure further conversions know about the induction variable. |
| 5629 | moduleTranslation.mapValue( |
| 5630 | mlir: loopOp.getRegion().front().getArgument(i: loopInfos.size()), llvm: iv); |
| 5631 | |
| 5632 | // Capture the body insertion point for use in nested loops. BodyIP of the |
| 5633 | // CanonicalLoopInfo always points to the beginning of the entry block of |
| 5634 | // the body. |
| 5635 | bodyInsertPoints.push_back(Elt: ip); |
| 5636 | |
| 5637 | if (loopInfos.size() != loopOp.getNumLoops() - 1) |
| 5638 | return llvm::Error::success(); |
| 5639 | |
| 5640 | // Convert the body of the loop. |
| 5641 | builder.restoreIP(IP: ip); |
| 5642 | llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions( |
| 5643 | region&: loopOp.getRegion(), blockName: "omp.loop_nest.region" , builder, moduleTranslation); |
| 5644 | if (!regionBlock) |
| 5645 | return regionBlock.takeError(); |
| 5646 | |
| 5647 | builder.SetInsertPoint((*regionBlock)->begin()); |
| 5648 | return llvm::Error::success(); |
| 5649 | }; |
| 5650 | |
| 5651 | // Delegate actual loop construction to the OpenMP IRBuilder. |
| 5652 | // TODO: this currently assumes omp.loop_nest is semantically similar to SCF |
| 5653 | // loop, i.e. it has a positive step, uses signed integer semantics. |
| 5654 | // Reconsider this code when the nested loop operation clearly supports more |
| 5655 | // cases. |
| 5656 | for (unsigned i = 0, e = loopOp.getNumLoops(); i < e; ++i) { |
| 5657 | llvm::Value *lowerBound = |
| 5658 | moduleTranslation.lookupValue(value: loopOp.getLoopLowerBounds()[i]); |
| 5659 | llvm::Value *upperBound = |
| 5660 | moduleTranslation.lookupValue(value: loopOp.getLoopUpperBounds()[i]); |
| 5661 | llvm::Value *step = moduleTranslation.lookupValue(value: loopOp.getLoopSteps()[i]); |
| 5662 | |
| 5663 | // Make sure loop trip count are emitted in the preheader of the outermost |
| 5664 | // loop at the latest so that they are all available for the new collapsed |
| 5665 | // loop will be created below. |
| 5666 | llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc; |
| 5667 | llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP; |
| 5668 | if (i != 0) { |
| 5669 | loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back(), |
| 5670 | ompLoc.DL); |
| 5671 | computeIP = loopInfos.front()->getPreheaderIP(); |
| 5672 | } |
| 5673 | |
| 5674 | llvm::Expected<llvm::CanonicalLoopInfo *> loopResult = |
| 5675 | ompBuilder->createCanonicalLoop( |
| 5676 | Loc: loc, BodyGenCB: bodyGen, Start: lowerBound, Stop: upperBound, Step: step, |
| 5677 | /*IsSigned=*/true, InclusiveStop: loopOp.getLoopInclusive(), ComputeIP: computeIP); |
| 5678 | |
| 5679 | if (failed(Result: handleError(result&: loopResult, op&: *loopOp))) |
| 5680 | return failure(); |
| 5681 | |
| 5682 | loopInfos.push_back(Elt: *loopResult); |
| 5683 | } |
| 5684 | |
| 5685 | llvm::OpenMPIRBuilder::InsertPointTy afterIP = |
| 5686 | loopInfos.front()->getAfterIP(); |
| 5687 | |
| 5688 | // Do tiling. |
| 5689 | if (const auto &tiles = loopOp.getTileSizes()) { |
| 5690 | llvm::Type *ivType = loopInfos.front()->getIndVarType(); |
| 5691 | SmallVector<llvm::Value *> tileSizes; |
| 5692 | |
| 5693 | for (auto tile : tiles.value()) { |
| 5694 | llvm::Value *tileVal = llvm::ConstantInt::get(Ty: ivType, V: tile); |
| 5695 | tileSizes.push_back(Elt: tileVal); |
| 5696 | } |
| 5697 | |
| 5698 | std::vector<llvm::CanonicalLoopInfo *> newLoops = |
| 5699 | ompBuilder->tileLoops(DL: ompLoc.DL, Loops: loopInfos, TileSizes: tileSizes); |
| 5700 | |
| 5701 | // Update afterIP to get the correct insertion point after |
| 5702 | // tiling. |
| 5703 | llvm::BasicBlock *afterBB = newLoops.front()->getAfter(); |
| 5704 | llvm::BasicBlock *afterAfterBB = afterBB->getSingleSuccessor(); |
| 5705 | afterIP = afterAfterBB->begin(); |
| 5706 | |
| 5707 | // Update the loop infos. |
| 5708 | loopInfos.clear(); |
| 5709 | for (const auto &newLoop : newLoops) |
| 5710 | loopInfos.push_back(Elt: newLoop); |
| 5711 | } // Tiling done. |
| 5712 | |
| 5713 | // Do collapse. |
| 5714 | const auto &numCollapse = loopOp.getCollapseNumLoops(); |
| 5715 | SmallVector<llvm::CanonicalLoopInfo *> collapseLoopInfos( |
| 5716 | loopInfos.begin(), loopInfos.begin() + (numCollapse)); |
| 5717 | |
| 5718 | llvm::SaveAndRestore<llvm::OpenMPIRBuilder::CollapseIVMapping> mappingGuard( |
| 5719 | ompBuilder->Config.CollapseMapping, |
| 5720 | selectCollapseIVMapping(loopOp, isTargetDevice: ompBuilder->Config.isTargetDevice(), |
| 5721 | loops: collapseLoopInfos)); |
| 5722 | auto newTopLoopInfo = |
| 5723 | ompBuilder->collapseLoops(DL: ompLoc.DL, Loops: collapseLoopInfos, ComputeIP: {}); |
| 5724 | |
| 5725 | assert(newTopLoopInfo && "New top loop information is missing" ); |
| 5726 | moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>( |
| 5727 | callback: [&](OpenMPLoopInfoStackFrame &frame) { |
| 5728 | frame.loopInfo = newTopLoopInfo; |
| 5729 | return WalkResult::interrupt(); |
| 5730 | }); |
| 5731 | |
| 5732 | // Continue building IR after the loop. Note that the LoopInfo returned by |
| 5733 | // `collapseLoops` points inside the outermost loop and is intended for |
| 5734 | // potential further loop transformations. Use the insertion point stored |
| 5735 | // before collapsing loops instead. |
| 5736 | builder.restoreIP(IP: afterIP); |
| 5737 | return success(); |
| 5738 | } |
| 5739 | |
| 5740 | /// Convert an omp.canonical_loop to LLVM-IR |
| 5741 | static LogicalResult |
| 5742 | convertOmpCanonicalLoopOp(omp::CanonicalLoopOp op, llvm::IRBuilderBase &builder, |
| 5743 | LLVM::ModuleTranslation &moduleTranslation) { |
| 5744 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 5745 | |
| 5746 | llvm::OpenMPIRBuilder::LocationDescription loopLoc(builder); |
| 5747 | Value loopIV = op.getInductionVar(); |
| 5748 | Value loopTC = op.getTripCount(); |
| 5749 | |
| 5750 | llvm::Value *llvmTC = moduleTranslation.lookupValue(value: loopTC); |
| 5751 | |
| 5752 | llvm::Expected<llvm::CanonicalLoopInfo *> llvmOrError = |
| 5753 | ompBuilder->createCanonicalLoop( |
| 5754 | Loc: loopLoc, |
| 5755 | BodyGenCB: [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *llvmIV) { |
| 5756 | // Register the mapping of MLIR induction variable to LLVM-IR |
| 5757 | // induction variable |
| 5758 | moduleTranslation.mapValue(mlir: loopIV, llvm: llvmIV); |
| 5759 | |
| 5760 | builder.restoreIP(IP: ip); |
| 5761 | llvm::Expected<llvm::BasicBlock *> bodyGenStatus = |
| 5762 | convertOmpOpRegions(region&: op.getRegion(), blockName: "omp.loop.region" , builder, |
| 5763 | moduleTranslation); |
| 5764 | |
| 5765 | return bodyGenStatus.takeError(); |
| 5766 | }, |
| 5767 | TripCount: llvmTC, Name: "omp.loop" ); |
| 5768 | if (!llvmOrError) |
| 5769 | return op.emitError(message: llvm::toString(E: llvmOrError.takeError())); |
| 5770 | |
| 5771 | llvm::CanonicalLoopInfo *llvmCLI = *llvmOrError; |
| 5772 | llvm::IRBuilderBase::InsertPoint afterIP = llvmCLI->getAfterIP(); |
| 5773 | builder.restoreIP(IP: afterIP); |
| 5774 | |
| 5775 | // Register the mapping of MLIR loop to LLVM-IR OpenMPIRBuilder loop |
| 5776 | if (Value cli = op.getCli()) |
| 5777 | mapOmpLoop(moduleTranslation, mlir: cli, llvm: llvmCLI); |
| 5778 | |
| 5779 | return success(); |
| 5780 | } |
| 5781 | |
| 5782 | /// Apply a `#pragma omp unroll` / "!$omp unroll" transformation using the |
| 5783 | /// OpenMPIRBuilder. |
| 5784 | static LogicalResult |
| 5785 | applyUnrollHeuristic(omp::UnrollHeuristicOp op, llvm::IRBuilderBase &builder, |
| 5786 | LLVM::ModuleTranslation &moduleTranslation) { |
| 5787 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 5788 | |
| 5789 | Value applyee = op.getApplyee(); |
| 5790 | assert(applyee && "Loop to apply unrolling on required" ); |
| 5791 | |
| 5792 | llvm::CanonicalLoopInfo *consBuilderCLI = |
| 5793 | lookupOMPLoop(moduleTranslation, mlir: applyee); |
| 5794 | llvm::OpenMPIRBuilder::LocationDescription loc(builder); |
| 5795 | ompBuilder->unrollLoopHeuristic(DL: loc.DL, Loop: consBuilderCLI); |
| 5796 | |
| 5797 | invalidateOmpLoop(moduleTranslation, mlir: applyee); |
| 5798 | return success(); |
| 5799 | } |
| 5800 | |
| 5801 | /// Apply a `#pragma omp unroll full` / `!$omp unroll full` transformation |
| 5802 | /// using the OpenMPIRBuilder. |
| 5803 | static LogicalResult |
| 5804 | applyUnrollFull(omp::UnrollFullOp op, llvm::IRBuilderBase &builder, |
| 5805 | LLVM::ModuleTranslation &moduleTranslation) { |
| 5806 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 5807 | |
| 5808 | Value applyee = op.getApplyee(); |
| 5809 | assert(applyee && "Loop to apply unrolling on required" ); |
| 5810 | |
| 5811 | llvm::CanonicalLoopInfo *consBuilderCLI = |
| 5812 | lookupOMPLoop(moduleTranslation, mlir: applyee); |
| 5813 | llvm::OpenMPIRBuilder::LocationDescription loc(builder); |
| 5814 | ompBuilder->unrollLoopFull(DL: loc.DL, Loop: consBuilderCLI); |
| 5815 | |
| 5816 | invalidateOmpLoop(moduleTranslation, mlir: applyee); |
| 5817 | return success(); |
| 5818 | } |
| 5819 | |
| 5820 | /// Apply a `#pragma omp unroll partial` / `!$omp unroll partial` |
| 5821 | /// transformation using the OpenMPIRBuilder. |
| 5822 | static LogicalResult |
| 5823 | applyUnrollPartial(omp::UnrollPartialOp op, llvm::IRBuilderBase &builder, |
| 5824 | LLVM::ModuleTranslation &moduleTranslation) { |
| 5825 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 5826 | |
| 5827 | Value applyee = op.getApplyee(); |
| 5828 | assert(applyee && "Loop to apply unrolling on required" ); |
| 5829 | |
| 5830 | llvm::CanonicalLoopInfo *consBuilderCLI = |
| 5831 | lookupOMPLoop(moduleTranslation, mlir: applyee); |
| 5832 | llvm::OpenMPIRBuilder::LocationDescription loc(builder); |
| 5833 | |
| 5834 | // No generatee is supported yet, so the unrolled loop's CanonicalLoopInfo is |
| 5835 | // not requested and unrolling is deferred to LLVM's LoopUnroll pass. |
| 5836 | int32_t factor = static_cast<int32_t>(op.getUnrollFactor()); |
| 5837 | ompBuilder->unrollLoopPartial(DL: loc.DL, Loop: consBuilderCLI, Factor: factor, |
| 5838 | /*UnrolledCLI=*/nullptr); |
| 5839 | |
| 5840 | invalidateOmpLoop(moduleTranslation, mlir: applyee); |
| 5841 | return success(); |
| 5842 | } |
| 5843 | |
| 5844 | /// Apply a `#pragma omp tile` / `!$omp tile` transformation using the |
| 5845 | /// OpenMPIRBuilder. |
| 5846 | static LogicalResult applyTile(omp::TileOp op, llvm::IRBuilderBase &builder, |
| 5847 | LLVM::ModuleTranslation &moduleTranslation) { |
| 5848 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 5849 | llvm::OpenMPIRBuilder::LocationDescription loc(builder); |
| 5850 | |
| 5851 | SmallVector<llvm::CanonicalLoopInfo *> translatedLoops; |
| 5852 | SmallVector<llvm::Value *> translatedSizes; |
| 5853 | |
| 5854 | for (Value size : op.getSizes()) { |
| 5855 | llvm::Value *translatedSize = moduleTranslation.lookupValue(value: size); |
| 5856 | assert(translatedSize && |
| 5857 | "sizes clause arguments must already be translated" ); |
| 5858 | translatedSizes.push_back(Elt: translatedSize); |
| 5859 | } |
| 5860 | |
| 5861 | for (Value applyee : op.getApplyees()) { |
| 5862 | llvm::CanonicalLoopInfo *consBuilderCLI = |
| 5863 | lookupOMPLoop(moduleTranslation, mlir: applyee); |
| 5864 | assert(applyee && "Canonical loop must already been translated" ); |
| 5865 | translatedLoops.push_back(Elt: consBuilderCLI); |
| 5866 | } |
| 5867 | |
| 5868 | auto generatedLoops = |
| 5869 | ompBuilder->tileLoops(DL: loc.DL, Loops: translatedLoops, TileSizes: translatedSizes); |
| 5870 | if (!op.getGeneratees().empty()) { |
| 5871 | for (auto [mlirLoop, genLoop] : |
| 5872 | zip_equal(t: op.getGeneratees(), u&: generatedLoops)) |
| 5873 | mapOmpLoop(moduleTranslation, mlir: mlirLoop, llvm: genLoop); |
| 5874 | } |
| 5875 | |
| 5876 | // CLIs can only be consumed once |
| 5877 | for (Value applyee : op.getApplyees()) |
| 5878 | invalidateOmpLoop(moduleTranslation, mlir: applyee); |
| 5879 | |
| 5880 | return success(); |
| 5881 | } |
| 5882 | |
| 5883 | /// Apply a `#pragma omp fuse` / `!$omp fuse` transformation using the |
| 5884 | /// OpenMPIRBuilder. |
| 5885 | static LogicalResult applyFuse(omp::FuseOp op, llvm::IRBuilderBase &builder, |
| 5886 | LLVM::ModuleTranslation &moduleTranslation) { |
| 5887 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 5888 | llvm::OpenMPIRBuilder::LocationDescription loc(builder); |
| 5889 | |
| 5890 | // Select what CLIs are going to be fused |
| 5891 | SmallVector<llvm::CanonicalLoopInfo *> beforeFuse, toFuse, afterFuse; |
| 5892 | for (size_t i = 0; i < op.getApplyees().size(); i++) { |
| 5893 | Value applyee = op.getApplyees()[i]; |
| 5894 | llvm::CanonicalLoopInfo *consBuilderCLI = |
| 5895 | lookupOMPLoop(moduleTranslation, mlir: applyee); |
| 5896 | assert(applyee && "Canonical loop must already been translated" ); |
| 5897 | if (op.getFirst().has_value() && i < op.getFirst().value() - 1) |
| 5898 | beforeFuse.push_back(Elt: consBuilderCLI); |
| 5899 | else if (op.getCount().has_value() && |
| 5900 | i >= op.getFirst().value() + op.getCount().value() - 1) |
| 5901 | afterFuse.push_back(Elt: consBuilderCLI); |
| 5902 | else |
| 5903 | toFuse.push_back(Elt: consBuilderCLI); |
| 5904 | } |
| 5905 | assert( |
| 5906 | (op.getGeneratees().empty() || |
| 5907 | beforeFuse.size() + afterFuse.size() + 1 == op.getGeneratees().size()) && |
| 5908 | "Wrong number of generatees" ); |
| 5909 | |
| 5910 | // do the fuse |
| 5911 | auto generatedLoop = ompBuilder->fuseLoops(DL: loc.DL, Loops: toFuse); |
| 5912 | if (!op.getGeneratees().empty()) { |
| 5913 | size_t i = 0; |
| 5914 | for (; i < beforeFuse.size(); i++) |
| 5915 | mapOmpLoop(moduleTranslation, mlir: op.getGeneratees()[i], llvm: beforeFuse[i]); |
| 5916 | mapOmpLoop(moduleTranslation, mlir: op.getGeneratees()[i++], llvm: generatedLoop); |
| 5917 | for (; i < afterFuse.size(); i++) |
| 5918 | mapOmpLoop(moduleTranslation, mlir: op.getGeneratees()[i], llvm: afterFuse[i]); |
| 5919 | } |
| 5920 | |
| 5921 | // CLIs can only be consumed once |
| 5922 | for (Value applyee : op.getApplyees()) |
| 5923 | invalidateOmpLoop(moduleTranslation, mlir: applyee); |
| 5924 | |
| 5925 | return success(); |
| 5926 | } |
| 5927 | |
| 5928 | /// Convert an Atomic Ordering attribute to llvm::AtomicOrdering. |
| 5929 | static llvm::AtomicOrdering |
| 5930 | convertAtomicOrdering(std::optional<omp::ClauseMemoryOrderKind> ao) { |
| 5931 | if (!ao) |
| 5932 | return llvm::AtomicOrdering::Monotonic; // Default Memory Ordering |
| 5933 | |
| 5934 | switch (*ao) { |
| 5935 | case omp::ClauseMemoryOrderKind::Seq_cst: |
| 5936 | return llvm::AtomicOrdering::SequentiallyConsistent; |
| 5937 | case omp::ClauseMemoryOrderKind::Acq_rel: |
| 5938 | return llvm::AtomicOrdering::AcquireRelease; |
| 5939 | case omp::ClauseMemoryOrderKind::Acquire: |
| 5940 | return llvm::AtomicOrdering::Acquire; |
| 5941 | case omp::ClauseMemoryOrderKind::Release: |
| 5942 | return llvm::AtomicOrdering::Release; |
| 5943 | case omp::ClauseMemoryOrderKind::Relaxed: |
| 5944 | return llvm::AtomicOrdering::Monotonic; |
| 5945 | } |
| 5946 | llvm_unreachable("Unknown ClauseMemoryOrderKind kind" ); |
| 5947 | } |
| 5948 | |
| 5949 | /// Compute the cmpxchg failure ordering for an atomic compare op: use the |
| 5950 | /// `fail` clause ordering when present (the verifier guarantees it is a valid |
| 5951 | /// cmpxchg failure ordering), otherwise the strongest failure ordering derived |
| 5952 | /// from the success ordering (which matches the OpenMPIRBuilder default). |
| 5953 | static llvm::AtomicOrdering |
| 5954 | getAtomicCompareFailureOrdering(omp::AtomicCompareOp atomicCompareOp, |
| 5955 | llvm::AtomicOrdering atomicOrdering) { |
| 5956 | if (atomicCompareOp.getFailMemoryOrder()) |
| 5957 | return convertAtomicOrdering(ao: atomicCompareOp.getFailMemoryOrder()); |
| 5958 | return llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: atomicOrdering); |
| 5959 | } |
| 5960 | |
| 5961 | /// Convert omp.atomic.read operation to LLVM IR. |
| 5962 | static LogicalResult |
| 5963 | convertOmpAtomicRead(Operation &opInst, llvm::IRBuilderBase &builder, |
| 5964 | LLVM::ModuleTranslation &moduleTranslation) { |
| 5965 | auto readOp = cast<omp::AtomicReadOp>(Val&: opInst); |
| 5966 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 5967 | return failure(); |
| 5968 | |
| 5969 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 5970 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 5971 | findAllocInsertPoints(builder, moduleTranslation); |
| 5972 | |
| 5973 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 5974 | |
| 5975 | llvm::AtomicOrdering AO = convertAtomicOrdering(ao: readOp.getMemoryOrder()); |
| 5976 | llvm::Value *x = moduleTranslation.lookupValue(value: readOp.getX()); |
| 5977 | llvm::Value *v = moduleTranslation.lookupValue(value: readOp.getV()); |
| 5978 | |
| 5979 | llvm::Type *elementType = |
| 5980 | moduleTranslation.convertType(type: readOp.getElementType()); |
| 5981 | |
| 5982 | llvm::OpenMPIRBuilder::AtomicOpValue V = {.Var: v, .ElemTy: elementType, .IsSigned: false, .IsVolatile: false}; |
| 5983 | llvm::OpenMPIRBuilder::AtomicOpValue X = {.Var: x, .ElemTy: elementType, .IsSigned: false, .IsVolatile: false}; |
| 5984 | builder.restoreIP(IP: ompBuilder->createAtomicRead(Loc: ompLoc, X, V, AO, AllocaIP: allocaIP)); |
| 5985 | return success(); |
| 5986 | } |
| 5987 | |
| 5988 | /// Converts an omp.atomic.write operation to LLVM IR. |
| 5989 | static LogicalResult |
| 5990 | convertOmpAtomicWrite(Operation &opInst, llvm::IRBuilderBase &builder, |
| 5991 | LLVM::ModuleTranslation &moduleTranslation) { |
| 5992 | auto writeOp = cast<omp::AtomicWriteOp>(Val&: opInst); |
| 5993 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 5994 | return failure(); |
| 5995 | |
| 5996 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 5997 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 5998 | findAllocInsertPoints(builder, moduleTranslation); |
| 5999 | |
| 6000 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 6001 | llvm::AtomicOrdering ao = convertAtomicOrdering(ao: writeOp.getMemoryOrder()); |
| 6002 | llvm::Value *expr = moduleTranslation.lookupValue(value: writeOp.getExpr()); |
| 6003 | llvm::Value *dest = moduleTranslation.lookupValue(value: writeOp.getX()); |
| 6004 | llvm::Type *ty = moduleTranslation.convertType(type: writeOp.getExpr().getType()); |
| 6005 | llvm::OpenMPIRBuilder::AtomicOpValue x = {.Var: dest, .ElemTy: ty, /*isSigned=*/.IsSigned: false, |
| 6006 | /*isVolatile=*/.IsVolatile: false}; |
| 6007 | builder.restoreIP( |
| 6008 | IP: ompBuilder->createAtomicWrite(Loc: ompLoc, X&: x, Expr: expr, AO: ao, AllocaIP: allocaIP)); |
| 6009 | return success(); |
| 6010 | } |
| 6011 | |
| 6012 | /// Converts an LLVM dialect binary operation to the corresponding enum value |
| 6013 | /// for `atomicrmw` supported binary operation. |
| 6014 | static llvm::AtomicRMWInst::BinOp convertBinOpToAtomic(Operation &op) { |
| 6015 | return llvm::TypeSwitch<Operation *, llvm::AtomicRMWInst::BinOp>(&op) |
| 6016 | .Case(caseFn: [&](LLVM::AddOp) { return llvm::AtomicRMWInst::BinOp::Add; }) |
| 6017 | .Case(caseFn: [&](LLVM::SubOp) { return llvm::AtomicRMWInst::BinOp::Sub; }) |
| 6018 | .Case(caseFn: [&](LLVM::AndOp) { return llvm::AtomicRMWInst::BinOp::And; }) |
| 6019 | .Case(caseFn: [&](LLVM::OrOp) { return llvm::AtomicRMWInst::BinOp::Or; }) |
| 6020 | .Case(caseFn: [&](LLVM::XOrOp) { return llvm::AtomicRMWInst::BinOp::Xor; }) |
| 6021 | .Case(caseFn: [&](LLVM::SMaxOp) { return llvm::AtomicRMWInst::BinOp::Max; }) |
| 6022 | .Case(caseFn: [&](LLVM::SMinOp) { return llvm::AtomicRMWInst::BinOp::Min; }) |
| 6023 | .Case(caseFn: [&](LLVM::UMaxOp) { return llvm::AtomicRMWInst::BinOp::UMax; }) |
| 6024 | .Case(caseFn: [&](LLVM::UMinOp) { return llvm::AtomicRMWInst::BinOp::UMin; }) |
| 6025 | .Case(caseFn: [&](LLVM::FAddOp) { return llvm::AtomicRMWInst::BinOp::FAdd; }) |
| 6026 | .Case(caseFn: [&](LLVM::FSubOp) { return llvm::AtomicRMWInst::BinOp::FSub; }) |
| 6027 | .Case(caseFn: [&](LLVM::MaxNumOp) { return llvm::AtomicRMWInst::BinOp::FMax; }) |
| 6028 | .Case(caseFn: [&](LLVM::MinNumOp) { return llvm::AtomicRMWInst::BinOp::FMin; }) |
| 6029 | .Case( |
| 6030 | caseFn: [&](LLVM::MaximumOp) { return llvm::AtomicRMWInst::BinOp::FMaximum; }) |
| 6031 | .Case( |
| 6032 | caseFn: [&](LLVM::MinimumOp) { return llvm::AtomicRMWInst::BinOp::FMinimum; }) |
| 6033 | .Default(defaultResult: llvm::AtomicRMWInst::BinOp::BAD_BINOP); |
| 6034 | } |
| 6035 | |
| 6036 | static void (omp::AtomicUpdateOp atomicUpdateOp, |
| 6037 | bool &isIgnoreDenormalMode, |
| 6038 | bool &isFineGrainedMemory, |
| 6039 | bool &isRemoteMemory) { |
| 6040 | isIgnoreDenormalMode = false; |
| 6041 | isFineGrainedMemory = false; |
| 6042 | isRemoteMemory = false; |
| 6043 | if (atomicUpdateOp && atomicUpdateOp.getAtomicControlAttr()) { |
| 6044 | mlir::omp::AtomicControlAttr atomicControlAttr = |
| 6045 | atomicUpdateOp.getAtomicControlAttr(); |
| 6046 | isIgnoreDenormalMode = atomicControlAttr.getIgnoreDenormalMode(); |
| 6047 | isFineGrainedMemory = atomicControlAttr.getFineGrainedMemory(); |
| 6048 | isRemoteMemory = atomicControlAttr.getRemoteMemory(); |
| 6049 | } |
| 6050 | } |
| 6051 | |
| 6052 | /// Converts an OpenMP atomic update operation using OpenMPIRBuilder. |
| 6053 | static LogicalResult |
| 6054 | convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst, |
| 6055 | llvm::IRBuilderBase &builder, |
| 6056 | LLVM::ModuleTranslation &moduleTranslation) { |
| 6057 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 6058 | if (failed(Result: checkImplementationStatus(op&: *opInst))) |
| 6059 | return failure(); |
| 6060 | |
| 6061 | // Convert values and types. |
| 6062 | auto &innerOpList = opInst.getRegion().front().getOperations(); |
| 6063 | bool isXBinopExpr{false}; |
| 6064 | llvm::AtomicRMWInst::BinOp binop; |
| 6065 | mlir::Value mlirExpr; |
| 6066 | llvm::Value *llvmExpr = nullptr; |
| 6067 | llvm::Value *llvmX = nullptr; |
| 6068 | llvm::Type *llvmXElementType = nullptr; |
| 6069 | if (innerOpList.size() == 2) { |
| 6070 | // The two operations here are the update and the terminator. |
| 6071 | // Since we can identify the update operation, there is a possibility |
| 6072 | // that we can generate the atomicrmw instruction. |
| 6073 | mlir::Operation &innerOp = *opInst.getRegion().front().begin(); |
| 6074 | if (!llvm::is_contained(Range: innerOp.getOperands(), |
| 6075 | Element: opInst.getRegion().getArgument(i: 0))) { |
| 6076 | return opInst.emitError(message: "no atomic update operation with region argument" |
| 6077 | " as operand found inside atomic.update region" ); |
| 6078 | } |
| 6079 | binop = convertBinOpToAtomic(op&: innerOp); |
| 6080 | isXBinopExpr = innerOp.getOperand(idx: 0) == opInst.getRegion().getArgument(i: 0); |
| 6081 | mlirExpr = (isXBinopExpr ? innerOp.getOperand(idx: 1) : innerOp.getOperand(idx: 0)); |
| 6082 | llvmExpr = moduleTranslation.lookupValue(value: mlirExpr); |
| 6083 | } else { |
| 6084 | // Since the update region includes more than one operation |
| 6085 | // we will resort to generating a cmpxchg loop. |
| 6086 | binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP; |
| 6087 | } |
| 6088 | llvmX = moduleTranslation.lookupValue(value: opInst.getX()); |
| 6089 | llvmXElementType = moduleTranslation.convertType( |
| 6090 | type: opInst.getRegion().getArgument(i: 0).getType()); |
| 6091 | llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {.Var: llvmX, .ElemTy: llvmXElementType, |
| 6092 | /*isSigned=*/.IsSigned: false, |
| 6093 | /*isVolatile=*/.IsVolatile: false}; |
| 6094 | |
| 6095 | llvm::AtomicOrdering atomicOrdering = |
| 6096 | convertAtomicOrdering(ao: opInst.getMemoryOrder()); |
| 6097 | |
| 6098 | // Generate update code. |
| 6099 | auto updateFn = |
| 6100 | [&opInst, &moduleTranslation]( |
| 6101 | llvm::Value *atomicx, |
| 6102 | llvm::IRBuilder<> &builder) -> llvm::Expected<llvm::Value *> { |
| 6103 | Block &bb = *opInst.getRegion().begin(); |
| 6104 | moduleTranslation.mapValue(mlir: *opInst.getRegion().args_begin(), llvm: atomicx); |
| 6105 | moduleTranslation.mapBlock(mlir: &bb, llvm: builder.GetInsertBlock()); |
| 6106 | if (failed(Result: moduleTranslation.convertBlock(bb, ignoreArguments: true, builder))) |
| 6107 | return llvm::make_error<PreviouslyReportedError>(); |
| 6108 | |
| 6109 | omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(Val: bb.getTerminator()); |
| 6110 | assert(yieldop && yieldop.getResults().size() == 1 && |
| 6111 | "terminator must be omp.yield op and it must have exactly one " |
| 6112 | "argument" ); |
| 6113 | return moduleTranslation.lookupValue(value: yieldop.getResults()[0]); |
| 6114 | }; |
| 6115 | |
| 6116 | bool isIgnoreDenormalMode; |
| 6117 | bool isFineGrainedMemory; |
| 6118 | bool isRemoteMemory; |
| 6119 | extractAtomicControlFlags(atomicUpdateOp: opInst, isIgnoreDenormalMode, isFineGrainedMemory, |
| 6120 | isRemoteMemory); |
| 6121 | // Handle ambiguous alloca, if any. |
| 6122 | auto allocaIP = findAllocInsertPoints(builder, moduleTranslation); |
| 6123 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 6124 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 6125 | ompBuilder->createAtomicUpdate(Loc: ompLoc, AllocaIP: allocaIP, X&: llvmAtomicX, Expr: llvmExpr, |
| 6126 | AO: atomicOrdering, RMWOp: binop, UpdateOp: updateFn, |
| 6127 | IsXBinopExpr: isXBinopExpr, IsIgnoreDenormalMode: isIgnoreDenormalMode, |
| 6128 | IsFineGrainedMemory: isFineGrainedMemory, IsRemoteMemory: isRemoteMemory); |
| 6129 | |
| 6130 | if (failed(Result: handleError(result&: afterIP, op&: *opInst))) |
| 6131 | return failure(); |
| 6132 | |
| 6133 | builder.restoreIP(IP: *afterIP); |
| 6134 | return success(); |
| 6135 | } |
| 6136 | |
| 6137 | /// Helper to extract the OMPAtomicCompareOp from an integer comparison |
| 6138 | /// predicate. Returns std::nullopt for unsupported predicates. |
| 6139 | static std::optional<llvm::omp::OMPAtomicCompareOp> |
| 6140 | convertICmpPredicateToAtomicCompareOp(LLVM::ICmpPredicate predicate) { |
| 6141 | switch (predicate) { |
| 6142 | case LLVM::ICmpPredicate::eq: |
| 6143 | return llvm::omp::OMPAtomicCompareOp::EQ; |
| 6144 | case LLVM::ICmpPredicate::slt: |
| 6145 | case LLVM::ICmpPredicate::ult: |
| 6146 | return llvm::omp::OMPAtomicCompareOp::MIN; |
| 6147 | case LLVM::ICmpPredicate::sgt: |
| 6148 | case LLVM::ICmpPredicate::ugt: |
| 6149 | return llvm::omp::OMPAtomicCompareOp::MAX; |
| 6150 | default: |
| 6151 | return std::nullopt; |
| 6152 | } |
| 6153 | } |
| 6154 | |
| 6155 | /// Helper to extract the OMPAtomicCompareOp from a floating-point comparison |
| 6156 | /// predicate. Returns std::nullopt for unsupported predicates. |
| 6157 | static std::optional<llvm::omp::OMPAtomicCompareOp> |
| 6158 | convertFCmpPredicateToAtomicCompareOp(LLVM::FCmpPredicate predicate) { |
| 6159 | switch (predicate) { |
| 6160 | case LLVM::FCmpPredicate::oeq: |
| 6161 | case LLVM::FCmpPredicate::ueq: |
| 6162 | return llvm::omp::OMPAtomicCompareOp::EQ; |
| 6163 | case LLVM::FCmpPredicate::olt: |
| 6164 | case LLVM::FCmpPredicate::ult: |
| 6165 | return llvm::omp::OMPAtomicCompareOp::MIN; |
| 6166 | case LLVM::FCmpPredicate::ogt: |
| 6167 | case LLVM::FCmpPredicate::ugt: |
| 6168 | return llvm::omp::OMPAtomicCompareOp::MAX; |
| 6169 | default: |
| 6170 | return std::nullopt; |
| 6171 | } |
| 6172 | } |
| 6173 | |
| 6174 | /// Result of matching the decomposed complex equality pattern inside an atomic |
| 6175 | /// compare region. |
| 6176 | struct ComplexComparePattern { |
| 6177 | bool isComplex = false; |
| 6178 | bool isNE = false; // `or` of the field compares => NE (unsupported). |
| 6179 | mlir::Value eAggregate; // The complex expected value (`e`). |
| 6180 | bool isXBinopExpr = false; // True if x is the first fcmp operand. |
| 6181 | }; |
| 6182 | |
| 6183 | /// Detect a decomposed complex equality comparison in an atomic compare region: |
| 6184 | /// %re_x = llvm.extractvalue %xval[0] |
| 6185 | /// %re_e = llvm.extractvalue %eStruct[0] |
| 6186 | /// %cmp_re = llvm.fcmp "oeq" %re_x, %re_e |
| 6187 | /// %im_x = llvm.extractvalue %xval[1] |
| 6188 | /// %im_e = llvm.extractvalue %eStruct[1] |
| 6189 | /// %cmp_im = llvm.fcmp "oeq" %im_x, %im_e |
| 6190 | /// %cmp = llvm.and %cmp_re, %cmp_im (llvm.or would be NE) |
| 6191 | /// It is recognised by an and/or whose operands are both fcmps operating on |
| 6192 | /// extractvalues, one chain rooted at the block argument (x) and the other at |
| 6193 | /// the expected complex value (e). |
| 6194 | static ComplexComparePattern detectComplexCompareEq(Block &block) { |
| 6195 | ComplexComparePattern result; |
| 6196 | auto traceToAggregate = [](mlir::Value v) -> mlir::Value { |
| 6197 | if (auto = v.getDefiningOp<LLVM::ExtractValueOp>()) |
| 6198 | return extractOp.getContainer(); |
| 6199 | return nullptr; |
| 6200 | }; |
| 6201 | for (Operation &op : block.getOperations()) { |
| 6202 | if (!isa<LLVM::AndOp, LLVM::OrOp>(Val: op)) |
| 6203 | continue; |
| 6204 | auto lhsFcmp = op.getOperand(idx: 0).getDefiningOp<LLVM::FCmpOp>(); |
| 6205 | auto rhsFcmp = op.getOperand(idx: 1).getDefiningOp<LLVM::FCmpOp>(); |
| 6206 | if (!lhsFcmp || !rhsFcmp) |
| 6207 | continue; |
| 6208 | mlir::Value lhsAgg0 = traceToAggregate(lhsFcmp.getOperand(i: 0)); |
| 6209 | mlir::Value lhsAgg1 = traceToAggregate(lhsFcmp.getOperand(i: 1)); |
| 6210 | bool lhsXIsOp0 = (lhsAgg0 == block.getArgument(i: 0)); |
| 6211 | bool lhsXIsOp1 = (lhsAgg1 == block.getArgument(i: 0)); |
| 6212 | if (!lhsXIsOp0 && !lhsXIsOp1) |
| 6213 | continue; |
| 6214 | mlir::Value eAggregate = lhsXIsOp0 ? lhsAgg1 : lhsAgg0; |
| 6215 | if (!eAggregate) |
| 6216 | continue; |
| 6217 | result.isComplex = true; |
| 6218 | result.isNE = isa<LLVM::OrOp>(Val: op); |
| 6219 | result.eAggregate = eAggregate; |
| 6220 | result.isXBinopExpr = lhsXIsOp0; |
| 6221 | break; |
| 6222 | } |
| 6223 | return result; |
| 6224 | } |
| 6225 | |
| 6226 | /// Emit an IEEE-754-correct `cmpxchg` for a complex (struct-typed) atomic |
| 6227 | /// compare with `fcmp oeq`. The old value of X is returned (as the complex |
| 6228 | /// struct type) in \p oldComplex and the success flag (i1) in \p cmpOk. |
| 6229 | /// \p failOrdering is the memory ordering used when the compare-exchange does |
| 6230 | /// not store; it must be a valid cmpxchg failure ordering. |
| 6231 | static void emitComplexAtomicCmpXchg(llvm::IRBuilderBase &builder, |
| 6232 | llvm::Value *llvmX, llvm::Type *complexTy, |
| 6233 | llvm::Value *eVal, llvm::Value *dVal, |
| 6234 | llvm::AtomicOrdering atomicOrdering, |
| 6235 | llvm::AtomicOrdering failOrdering, |
| 6236 | bool isWeak, llvm::Value *&oldComplex, |
| 6237 | llvm::Value *&cmpOk) { |
| 6238 | const llvm::DataLayout &DL = builder.getDataLayout(); |
| 6239 | unsigned totalBits = DL.getTypeStoreSizeInBits(Ty: complexTy).getFixedValue(); |
| 6240 | llvm::IntegerType *intTy = |
| 6241 | llvm::IntegerType::get(C&: builder.getContext(), NumBits: totalBits); |
| 6242 | llvm::Align complexAlign = DL.getABITypeAlign(Ty: complexTy); |
| 6243 | llvm::Align intAlign = DL.getABITypeAlign(Ty: intTy); |
| 6244 | llvm::Align maxAlign = std::max(a: complexAlign, b: intAlign); |
| 6245 | |
| 6246 | // Spill D to obtain its integer bit pattern for the swap value. |
| 6247 | llvm::AllocaInst *dAlloca = |
| 6248 | builder.CreateAlloca(Ty: complexTy, ArraySize: nullptr, Name: "cmplx.d" ); |
| 6249 | dAlloca->setAlignment(maxAlign); |
| 6250 | builder.CreateAlignedStore(Val: dVal, Ptr: dAlloca, Align: maxAlign); |
| 6251 | llvm::Value *dInt = |
| 6252 | builder.CreateAlignedLoad(Ty: intTy, Ptr: dAlloca, Align: maxAlign, Name: "cmplx.d.int" ); |
| 6253 | |
| 6254 | // Load X atomically and reinterpret as complex. Use the failure ordering: on |
| 6255 | // a failed component comparison we branch around the cmpxchg, so this load is |
| 6256 | // the only memory op on that path. |
| 6257 | llvm::LoadInst *xCurr = |
| 6258 | builder.CreateAlignedLoad(Ty: intTy, Ptr: llvmX, Align: maxAlign, Name: "cmplx.x.load" ); |
| 6259 | xCurr->setAtomic(Ordering: failOrdering); |
| 6260 | llvm::AllocaInst *xAlloca = |
| 6261 | builder.CreateAlloca(Ty: complexTy, ArraySize: nullptr, Name: "cmplx.x" ); |
| 6262 | xAlloca->setAlignment(maxAlign); |
| 6263 | builder.CreateAlignedStore(Val: xCurr, Ptr: xAlloca, Align: maxAlign); |
| 6264 | llvm::Value *xStruct = |
| 6265 | builder.CreateAlignedLoad(Ty: complexTy, Ptr: xAlloca, Align: maxAlign, Name: "cmplx.x.val" ); |
| 6266 | |
| 6267 | // Component-wise IEEE-754 equality: `fcmp oeq` yields false for NaN (so a |
| 6268 | // NaN component correctly makes the compare fail) and true for +0.0 vs -0.0 |
| 6269 | // (so a zero-sign difference does not spuriously fail the compare). |
| 6270 | llvm::Value *reX = builder.CreateExtractValue(Agg: xStruct, Idxs: 0); |
| 6271 | llvm::Value *imX = builder.CreateExtractValue(Agg: xStruct, Idxs: 1); |
| 6272 | llvm::Value *reE = builder.CreateExtractValue(Agg: eVal, Idxs: 0); |
| 6273 | llvm::Value *imE = builder.CreateExtractValue(Agg: eVal, Idxs: 1); |
| 6274 | llvm::Value *reEq = builder.CreateFCmpOEQ(LHS: reX, RHS: reE, Name: "cmplx.re.eq" ); |
| 6275 | llvm::Value *imEq = builder.CreateFCmpOEQ(LHS: imX, RHS: imE, Name: "cmplx.im.eq" ); |
| 6276 | llvm::Value *fpEqual = builder.CreateAnd(LHS: reEq, RHS: imEq, Name: "cmplx.eq" ); |
| 6277 | |
| 6278 | // When the components compare equal, attempt the swap using X's own loaded |
| 6279 | // bit pattern as the comparand; otherwise the compare fails and X is left |
| 6280 | // unchanged (the captured old value is the value just loaded). |
| 6281 | llvm::BasicBlock *curBB = builder.GetInsertBlock(); |
| 6282 | llvm::Function *fn = curBB->getParent(); |
| 6283 | llvm::BasicBlock *swapBB = |
| 6284 | llvm::BasicBlock::Create(Context&: builder.getContext(), Name: "cmplx.atomic.swap" , Parent: fn); |
| 6285 | llvm::BasicBlock *exitBB = |
| 6286 | llvm::BasicBlock::Create(Context&: builder.getContext(), Name: "cmplx.atomic.exit" , Parent: fn); |
| 6287 | builder.CreateCondBr(Cond: fpEqual, True: swapBB, False: exitBB); |
| 6288 | |
| 6289 | builder.SetInsertPoint(swapBB); |
| 6290 | llvm::AtomicCmpXchgInst *cmpXchg = builder.CreateAtomicCmpXchg( |
| 6291 | Ptr: llvmX, Cmp: xCurr, New: dInt, Align: maxAlign, SuccessOrdering: atomicOrdering, FailureOrdering: failOrdering); |
| 6292 | cmpXchg->setWeak(isWeak); |
| 6293 | llvm::Value *oldSwap = builder.CreateExtractValue(Agg: cmpXchg, Idxs: 0); |
| 6294 | llvm::Value *okSwap = builder.CreateExtractValue(Agg: cmpXchg, Idxs: 1); |
| 6295 | builder.CreateBr(Dest: exitBB); |
| 6296 | |
| 6297 | // Merge the swap and no-swap paths. |
| 6298 | builder.SetInsertPoint(exitBB); |
| 6299 | llvm::PHINode *oldIntPHI = builder.CreatePHI(Ty: intTy, NumReservedValues: 2, Name: "cmplx.old.int" ); |
| 6300 | oldIntPHI->addIncoming(V: oldSwap, BB: swapBB); |
| 6301 | oldIntPHI->addIncoming(V: xCurr, BB: curBB); |
| 6302 | llvm::PHINode *okPHI = builder.CreatePHI(Ty: builder.getInt1Ty(), NumReservedValues: 2, Name: "cmplx.ok" ); |
| 6303 | okPHI->addIncoming(V: okSwap, BB: swapBB); |
| 6304 | okPHI->addIncoming(V: builder.getFalse(), BB: curBB); |
| 6305 | |
| 6306 | // Reinterpret the old integer value as the complex struct via memory. |
| 6307 | llvm::AllocaInst *oldAlloca = |
| 6308 | builder.CreateAlloca(Ty: complexTy, ArraySize: nullptr, Name: "cmplx.old" ); |
| 6309 | oldAlloca->setAlignment(maxAlign); |
| 6310 | builder.CreateAlignedStore(Val: oldIntPHI, Ptr: oldAlloca, Align: maxAlign); |
| 6311 | oldComplex = builder.CreateAlignedLoad(Ty: complexTy, Ptr: oldAlloca, Align: maxAlign, |
| 6312 | Name: "cmplx.old.val" ); |
| 6313 | cmpOk = okPHI; |
| 6314 | } |
| 6315 | |
| 6316 | /// Holds the extracted comparison pattern information from an atomic compare |
| 6317 | /// region. |
| 6318 | struct AtomicComparePatternInfo { |
| 6319 | llvm::omp::OMPAtomicCompareOp compareOp = llvm::omp::OMPAtomicCompareOp::EQ; |
| 6320 | llvm::Value *eVal = nullptr; |
| 6321 | llvm::Value *dVal = nullptr; |
| 6322 | bool isXBinopExpr = false; |
| 6323 | bool isSigned = false; |
| 6324 | }; |
| 6325 | /// Extract comparison predicate, expected value (e), desired value (d), and |
| 6326 | /// related flags from an atomic compare region block by scanning for |
| 6327 | /// icmp/fcmp/select/min/max operations. |
| 6328 | static LogicalResult ( |
| 6329 | Block &block, |
| 6330 | llvm::function_ref<llvm::Value *(mlir::Value)> materializeValue, |
| 6331 | omp::AtomicCompareOp atomicCompareOp, AtomicComparePatternInfo &info) { |
| 6332 | // Complex equality is a decomposed per-field pattern (extractvalue + fcmp + |
| 6333 | // and) rather than a single scalar compare. Detect it first so the scalar |
| 6334 | // icmp/fcmp handling below does not mistake a real/imaginary field for the |
| 6335 | // whole expected value. |
| 6336 | if (ComplexComparePattern cplx = detectComplexCompareEq(block); |
| 6337 | cplx.isComplex) { |
| 6338 | if (cplx.isNE) |
| 6339 | return atomicCompareOp.emitError( |
| 6340 | message: "unsupported comparison predicate (NE) for complex atomic compare" ); |
| 6341 | info.compareOp = llvm::omp::OMPAtomicCompareOp::EQ; |
| 6342 | info.isXBinopExpr = cplx.isXBinopExpr; |
| 6343 | info.eVal = materializeValue(cplx.eAggregate); |
| 6344 | for (Operation &op : block.getOperations()) { |
| 6345 | if (auto selectOp = dyn_cast<LLVM::SelectOp>(Val&: op)) { |
| 6346 | info.dVal = materializeValue(selectOp.getTrueValue()); |
| 6347 | break; |
| 6348 | } |
| 6349 | } |
| 6350 | return success(); |
| 6351 | } |
| 6352 | |
| 6353 | for (Operation &op : block.getOperations()) { |
| 6354 | // Pre-filter: skip icmps that don't involve the block argument |
| 6355 | // (e.g., truthiness extractions from logical-to-integer conversion). |
| 6356 | if (auto icmpOp = dyn_cast<LLVM::ICmpOp>(Val&: op); |
| 6357 | icmpOp && icmpOp.getOperand(i: 0) != block.getArgument(i: 0) && |
| 6358 | icmpOp.getOperand(i: 1) != block.getArgument(i: 0)) |
| 6359 | continue; |
| 6360 | |
| 6361 | LogicalResult result = |
| 6362 | llvm::TypeSwitch<Operation *, LogicalResult>(&op) |
| 6363 | .Case<LLVM::ICmpOp>(caseFn: [&](LLVM::ICmpOp icmpOp) -> LogicalResult { |
| 6364 | auto maybeOp = |
| 6365 | convertICmpPredicateToAtomicCompareOp(predicate: icmpOp.getPredicate()); |
| 6366 | if (!maybeOp) |
| 6367 | return atomicCompareOp.emitError( |
| 6368 | message: "unsupported comparison predicate in atomic compare" ); |
| 6369 | info.compareOp = *maybeOp; |
| 6370 | LLVM::ICmpPredicate pred = icmpOp.getPredicate(); |
| 6371 | info.isSigned = (pred == LLVM::ICmpPredicate::slt || |
| 6372 | pred == LLVM::ICmpPredicate::sgt || |
| 6373 | pred == LLVM::ICmpPredicate::sle || |
| 6374 | pred == LLVM::ICmpPredicate::sge); |
| 6375 | info.isXBinopExpr = |
| 6376 | (icmpOp.getOperand(i: 0) == block.getArgument(i: 0)); |
| 6377 | mlir::Value eOperand = info.isXBinopExpr ? icmpOp.getOperand(i: 1) |
| 6378 | : icmpOp.getOperand(i: 0); |
| 6379 | info.eVal = materializeValue(eOperand); |
| 6380 | return success(); |
| 6381 | }) |
| 6382 | .Case<LLVM::FCmpOp>(caseFn: [&](LLVM::FCmpOp fcmpOp) -> LogicalResult { |
| 6383 | auto maybeOp = |
| 6384 | convertFCmpPredicateToAtomicCompareOp(predicate: fcmpOp.getPredicate()); |
| 6385 | if (!maybeOp) |
| 6386 | return atomicCompareOp.emitError( |
| 6387 | message: "unsupported comparison predicate in atomic compare" ); |
| 6388 | info.compareOp = *maybeOp; |
| 6389 | info.isXBinopExpr = |
| 6390 | (fcmpOp.getOperand(i: 0) == block.getArgument(i: 0)); |
| 6391 | mlir::Value eOperand = info.isXBinopExpr ? fcmpOp.getOperand(i: 1) |
| 6392 | : fcmpOp.getOperand(i: 0); |
| 6393 | info.eVal = materializeValue(eOperand); |
| 6394 | return success(); |
| 6395 | }) |
| 6396 | .Case<LLVM::SelectOp>(caseFn: [&](LLVM::SelectOp selectOp) { |
| 6397 | if (!info.dVal) |
| 6398 | info.dVal = materializeValue(selectOp.getTrueValue()); |
| 6399 | return success(); |
| 6400 | }) |
| 6401 | .Case<mlir::arith::MaxSIOp, mlir::arith::MinSIOp, |
| 6402 | mlir::arith::MaxUIOp, mlir::arith::MinUIOp, |
| 6403 | mlir::arith::MaximumFOp, mlir::arith::MinimumFOp, |
| 6404 | LLVM::SMaxOp, LLVM::SMinOp, LLVM::UMaxOp, LLVM::UMinOp, |
| 6405 | LLVM::MaxNumOp, LLVM::MinNumOp>(caseFn: [&](Operation *) { |
| 6406 | // Canonicalized min/max ops (arith or LLVM intrinsic form). |
| 6407 | // max(x,e) came from slt/ult/olt -> OMPAtomicCompareOp::MIN |
| 6408 | // min(x,e) came from sgt/ugt/ogt -> OMPAtomicCompareOp::MAX |
| 6409 | // (OMPIRBuilder inverts: MIN->atomicrmw max, MAX->atomicrmw min) |
| 6410 | bool isMax = isa<mlir::arith::MaxSIOp, mlir::arith::MaxUIOp, |
| 6411 | mlir::arith::MaximumFOp, LLVM::SMaxOp, |
| 6412 | LLVM::UMaxOp, LLVM::MaxNumOp>(Val: op); |
| 6413 | info.compareOp = isMax ? llvm::omp::OMPAtomicCompareOp::MIN |
| 6414 | : llvm::omp::OMPAtomicCompareOp::MAX; |
| 6415 | info.isSigned = isa<mlir::arith::MaxSIOp, mlir::arith::MinSIOp, |
| 6416 | LLVM::SMaxOp, LLVM::SMinOp>(Val: op); |
| 6417 | info.isXBinopExpr = (op.getOperand(idx: 0) == block.getArgument(i: 0)); |
| 6418 | mlir::Value eOperand = |
| 6419 | info.isXBinopExpr ? op.getOperand(idx: 1) : op.getOperand(idx: 0); |
| 6420 | info.eVal = materializeValue(eOperand); |
| 6421 | info.dVal = info.eVal; |
| 6422 | return success(); |
| 6423 | }) |
| 6424 | .Default(defaultFn: [](Operation *) { return success(); }); |
| 6425 | |
| 6426 | if (failed(Result: result)) |
| 6427 | return result; |
| 6428 | } |
| 6429 | return success(); |
| 6430 | } |
| 6431 | |
| 6432 | static LogicalResult |
| 6433 | convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp, |
| 6434 | llvm::IRBuilderBase &builder, |
| 6435 | LLVM::ModuleTranslation &moduleTranslation) { |
| 6436 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 6437 | if (failed(Result: checkImplementationStatus(op&: *atomicCaptureOp))) |
| 6438 | return failure(); |
| 6439 | |
| 6440 | omp::AtomicUpdateOp atomicUpdateOp = atomicCaptureOp.getAtomicUpdateOp(); |
| 6441 | omp::AtomicWriteOp atomicWriteOp = atomicCaptureOp.getAtomicWriteOp(); |
| 6442 | omp::AtomicCompareOp atomicCompareOp = atomicCaptureOp.getAtomicCompareOp(); |
| 6443 | |
| 6444 | // If the capture contains an atomic.compare, delegate to |
| 6445 | // createAtomicCompare with the capture variable (V) set. |
| 6446 | if (atomicCompareOp) { |
| 6447 | omp::AtomicReadOp atomicReadOp = atomicCaptureOp.getAtomicReadOp(); |
| 6448 | assert(atomicReadOp && "expected atomic.read in capture+compare" ); |
| 6449 | |
| 6450 | Region ®ion = atomicCompareOp.getRegion(); |
| 6451 | Block &block = region.front(); |
| 6452 | |
| 6453 | llvm::Type *llvmXElementType = |
| 6454 | moduleTranslation.convertType(type: block.getArgument(i: 0).getType()); |
| 6455 | llvm::Value *llvmX = moduleTranslation.lookupValue(value: atomicCompareOp.getX()); |
| 6456 | llvm::Value *llvmV = moduleTranslation.lookupValue(value: atomicReadOp.getV()); |
| 6457 | |
| 6458 | bool isSigned = false; |
| 6459 | llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = { |
| 6460 | .Var: llvmX, .ElemTy: llvmXElementType, .IsSigned: isSigned, /*IsVolatile=*/false}; |
| 6461 | llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicV = { |
| 6462 | .Var: llvmV, .ElemTy: llvmXElementType, /*isSigned=*/.IsSigned: false, /*IsVolatile=*/false}; |
| 6463 | llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicR = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false, |
| 6464 | .IsVolatile: false}; |
| 6465 | |
| 6466 | llvm::AtomicOrdering atomicOrdering = |
| 6467 | convertAtomicOrdering(ao: atomicCaptureOp.getMemoryOrder()); |
| 6468 | |
| 6469 | // Pre-translate non-pattern operations inside the compare region. |
| 6470 | auto isAtomicComparePatternOp = [](Operation &op) { |
| 6471 | return llvm::isa<LLVM::ICmpOp, LLVM::FCmpOp, LLVM::SelectOp, LLVM::AndOp, |
| 6472 | LLVM::OrOp, LLVM::SMaxOp, LLVM::SMinOp, LLVM::UMaxOp, |
| 6473 | LLVM::UMinOp, LLVM::MaxNumOp, LLVM::MinNumOp, |
| 6474 | mlir::arith::MaxSIOp, mlir::arith::MinSIOp, |
| 6475 | mlir::arith::MaxUIOp, mlir::arith::MinUIOp, |
| 6476 | mlir::arith::MaximumFOp, mlir::arith::MinimumFOp>(Val: op); |
| 6477 | }; |
| 6478 | for (Operation &op : block.without_terminator()) { |
| 6479 | if (isAtomicComparePatternOp(op)) |
| 6480 | continue; |
| 6481 | bool allOperandsMapped = |
| 6482 | llvm::all_of(Range: op.getOperands(), P: [&](mlir::Value v) { |
| 6483 | return moduleTranslation.lookupValue(value: v) != nullptr; |
| 6484 | }); |
| 6485 | if (!allOperandsMapped) |
| 6486 | continue; |
| 6487 | if (failed(Result: moduleTranslation.convertOperation(op, builder))) |
| 6488 | return atomicCompareOp.emitError( |
| 6489 | message: "failed to translate operation inside atomic compare region" ); |
| 6490 | } |
| 6491 | |
| 6492 | auto materializeValue = [&](mlir::Value val) -> llvm::Value * { |
| 6493 | if (llvm::Value *existing = moduleTranslation.lookupValue(value: val)) |
| 6494 | return existing; |
| 6495 | if (auto loadOp = val.getDefiningOp<LLVM::LoadOp>()) { |
| 6496 | if (loadOp->getParentRegion() == ®ion) { |
| 6497 | llvm::Value *loadAddr = |
| 6498 | moduleTranslation.lookupValue(value: loadOp.getAddr()); |
| 6499 | if (!loadAddr) |
| 6500 | return nullptr; |
| 6501 | llvm::Type *loadType = |
| 6502 | moduleTranslation.convertType(type: loadOp.getResult().getType()); |
| 6503 | return builder.CreateLoad(Ty: loadType, Ptr: loadAddr); |
| 6504 | } |
| 6505 | } |
| 6506 | return nullptr; |
| 6507 | }; |
| 6508 | |
| 6509 | // Extract comparison predicate, eVal, and dVal from the region. |
| 6510 | AtomicComparePatternInfo patternInfo; |
| 6511 | if (failed(Result: extractAtomicComparePattern(block, materializeValue, |
| 6512 | atomicCompareOp, info&: patternInfo))) |
| 6513 | return failure(); |
| 6514 | |
| 6515 | llvm::omp::OMPAtomicCompareOp compareOp = patternInfo.compareOp; |
| 6516 | llvm::Value *eVal = patternInfo.eVal; |
| 6517 | llvm::Value *dVal = patternInfo.dVal; |
| 6518 | bool isXBinopExpr = patternInfo.isXBinopExpr; |
| 6519 | isSigned = patternInfo.isSigned; |
| 6520 | |
| 6521 | if (!eVal) |
| 6522 | return atomicCompareOp.emitError( |
| 6523 | message: "failed to extract expected value (e) from atomic compare region" ); |
| 6524 | if (!dVal) { |
| 6525 | auto yieldOp = cast<omp::YieldOp>(Val: block.getTerminator()); |
| 6526 | if (yieldOp.getResults().empty()) |
| 6527 | return atomicCompareOp.emitError( |
| 6528 | message: "failed to extract desired value (d) from atomic compare region" ); |
| 6529 | dVal = materializeValue(yieldOp.getResults()[0]); |
| 6530 | } |
| 6531 | |
| 6532 | llvmAtomicX.IsSigned = isSigned; |
| 6533 | |
| 6534 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 6535 | bool isReadFirst = isa<omp::AtomicReadOp>(Val: atomicCaptureOp.getFirstOp()); |
| 6536 | bool isPostfixCapture = !isReadFirst; |
| 6537 | bool isFailOnly = atomicCaptureOp.getFailOnly(); |
| 6538 | |
| 6539 | // Complex equality capture: x is struct-typed, which the OMPIRBuilder |
| 6540 | // cannot handle, so emit an IEEE-754-correct cmpxchg (as in the non-capture |
| 6541 | // complex path) and reconstruct the captured value from its result. The |
| 6542 | // helper compares components with `fcmp oeq` so `-0.0 == +0.0` and `NaN` |
| 6543 | // are handled as in the scalar float path. Complex only supports the == |
| 6544 | // comparison. |
| 6545 | if (llvmXElementType->isStructTy()) { |
| 6546 | llvm::Value *oldComplex = nullptr; |
| 6547 | llvm::Value *cmpOk = nullptr; |
| 6548 | llvm::AtomicOrdering failOrdering = |
| 6549 | getAtomicCompareFailureOrdering(atomicCompareOp, atomicOrdering); |
| 6550 | emitComplexAtomicCmpXchg(builder, llvmX, complexTy: llvmXElementType, eVal, dVal, |
| 6551 | atomicOrdering, failOrdering, |
| 6552 | isWeak: atomicCompareOp.getWeak(), oldComplex, cmpOk); |
| 6553 | |
| 6554 | if (isFailOnly) { |
| 6555 | // v is written only when the compare fails (cmpOk == false). |
| 6556 | llvm::Value *cmpFailed = builder.CreateNot(V: cmpOk); |
| 6557 | llvm::BasicBlock *curBB = builder.GetInsertBlock(); |
| 6558 | llvm::Function *fn = curBB->getParent(); |
| 6559 | llvm::BasicBlock *contBB = llvm::BasicBlock::Create( |
| 6560 | Context&: builder.getContext(), Name: "omp.atomic.cont" , Parent: fn); |
| 6561 | llvm::BasicBlock *exitBB = llvm::BasicBlock::Create( |
| 6562 | Context&: builder.getContext(), Name: "omp.atomic.exit" , Parent: fn); |
| 6563 | builder.CreateCondBr(Cond: cmpFailed, True: contBB, False: exitBB); |
| 6564 | builder.SetInsertPoint(contBB); |
| 6565 | builder.CreateStore(Val: oldComplex, Ptr: llvmAtomicV.Var, |
| 6566 | isVolatile: llvmAtomicV.IsVolatile); |
| 6567 | builder.CreateBr(Dest: exitBB); |
| 6568 | builder.SetInsertPoint(exitBB); |
| 6569 | } else if (isPostfixCapture) { |
| 6570 | // v gets the new value of x: d on success, old x otherwise. |
| 6571 | llvm::Value *newComplex = builder.CreateSelect(C: cmpOk, True: dVal, False: oldComplex); |
| 6572 | builder.CreateStore(Val: newComplex, Ptr: llvmAtomicV.Var, |
| 6573 | isVolatile: llvmAtomicV.IsVolatile); |
| 6574 | } else { |
| 6575 | // Prefix: v gets the old value of x. |
| 6576 | builder.CreateStore(Val: oldComplex, Ptr: llvmAtomicV.Var, |
| 6577 | isVolatile: llvmAtomicV.IsVolatile); |
| 6578 | } |
| 6579 | |
| 6580 | // Emit flush after atomic compare if needed (release/acq_rel/seq_cst). |
| 6581 | if (atomicOrdering == llvm::AtomicOrdering::Release || |
| 6582 | atomicOrdering == llvm::AtomicOrdering::AcquireRelease || |
| 6583 | atomicOrdering == llvm::AtomicOrdering::SequentiallyConsistent) { |
| 6584 | llvm::OpenMPIRBuilder::LocationDescription flushLoc(builder); |
| 6585 | ompBuilder->createFlush(Loc: flushLoc); |
| 6586 | } |
| 6587 | return success(); |
| 6588 | } |
| 6589 | |
| 6590 | // Min/max (<, >) comparisons lower to an atomicrmw. The OMPIRBuilder has no |
| 6591 | // notion of a failed compare for an atomicrmw, so the fail-only capture |
| 6592 | // form (v written only when the compare fails) has no valid mapping and is |
| 6593 | // Min/max (<, >) comparisons lower to an atomicrmw. The OMPIRBuilder has no |
| 6594 | // notion of a failed compare for an atomicrmw (it asserts on IsFailOnly), |
| 6595 | // so for min/max the fail-only capture is reconstructed manually below. |
| 6596 | bool isMinMax = compareOp != llvm::omp::OMPAtomicCompareOp::EQ; |
| 6597 | |
| 6598 | llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicVForCall = llvmAtomicV; |
| 6599 | // Capture into V is reconstructed manually below for: |
| 6600 | // * postfix capture (v gets the new value of x): equality from the |
| 6601 | // cmpxchg result, min/max from the atomicrmw result; |
| 6602 | // * min/max fail-only capture (the OMPIRBuilder cannot express it). |
| 6603 | // Bypass V in the OMPIRBuilder for those cases so it does not also emit its |
| 6604 | // own (for min/max, incorrect or unsupported) capture store. |
| 6605 | bool minMaxManualCapture = isMinMax && (isPostfixCapture || isFailOnly); |
| 6606 | bool eqPostfixManualCapture = !isMinMax && isPostfixCapture && !isFailOnly; |
| 6607 | if (minMaxManualCapture || eqPostfixManualCapture) |
| 6608 | llvmAtomicVForCall = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false, .IsVolatile: false}; |
| 6609 | |
| 6610 | // The OMPIRBuilder only understands IsFailOnly for the equality (cmpxchg) |
| 6611 | // path; for min/max it would assert. Min/max fail-only is handled here. |
| 6612 | bool builderFailOnly = isFailOnly && !isMinMax; |
| 6613 | |
| 6614 | // IsPostfixUpdate selects which value the OMPIRBuilder captures into V: |
| 6615 | // * min/max prefix and equality prefix: a direct store of the old value |
| 6616 | // (IsPostfixUpdate=true). |
| 6617 | // * equality fail-only: a conditional store (IsPostfixUpdate=false). |
| 6618 | // Manually-reconstructed captures bypass V above. |
| 6619 | bool isPostfixUpdate = !builderFailOnly; |
| 6620 | |
| 6621 | bool isWeak = atomicCompareOp.getWeak(); |
| 6622 | bool savedHandleFPNegZero = ompBuilder->setHandleFPNegZero(true); |
| 6623 | llvm::AtomicOrdering failureOrdering = |
| 6624 | getAtomicCompareFailureOrdering(atomicCompareOp, atomicOrdering); |
| 6625 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 6626 | ompBuilder->createAtomicCompare( |
| 6627 | Loc: ompLoc, X&: llvmAtomicX, V&: llvmAtomicVForCall, R&: llvmAtomicR, E: eVal, D: dVal, |
| 6628 | AO: atomicOrdering, Op: compareOp, IsXBinopExpr: isXBinopExpr, IsPostfixUpdate: isPostfixUpdate, |
| 6629 | IsFailOnly: builderFailOnly, Failure: failureOrdering, IsWeak: isWeak); |
| 6630 | ompBuilder->setHandleFPNegZero(savedHandleFPNegZero); |
| 6631 | |
| 6632 | if (failed(Result: handleError(result&: afterIP, op&: *atomicCaptureOp))) |
| 6633 | return failure(); |
| 6634 | |
| 6635 | builder.restoreIP(IP: *afterIP); |
| 6636 | |
| 6637 | // Min/max capture is reconstructed from the atomicrmw the OMPIRBuilder |
| 6638 | // emits (its result is the old value of x). V was bypassed above. |
| 6639 | // * postfix: v gets the new value min/max(old, e); |
| 6640 | // * fail-only: v gets the old value, but only when the compare failed |
| 6641 | // (i.e. the atomicrmw did not change x). |
| 6642 | // (Prefix min/max captures the old value directly through V, so nothing |
| 6643 | // extra is needed there.) |
| 6644 | if (isMinMax && (isPostfixCapture || isFailOnly)) { |
| 6645 | llvm::BasicBlock *curBB = builder.GetInsertBlock(); |
| 6646 | llvm::AtomicRMWInst *rmw = nullptr; |
| 6647 | for (auto &inst : llvm::reverse(C&: *curBB)) { |
| 6648 | if (auto *r = dyn_cast<llvm::AtomicRMWInst>(Val: &inst)) { |
| 6649 | rmw = r; |
| 6650 | break; |
| 6651 | } |
| 6652 | } |
| 6653 | assert(rmw && "expected atomicrmw for min/max compare capture" ); |
| 6654 | llvm::Value *oldVal = rmw; |
| 6655 | llvm::Value *rhs = rmw->getValOperand(); |
| 6656 | |
| 6657 | if (isFailOnly) { |
| 6658 | // The compare "failed" (the else branch runs) exactly when the |
| 6659 | // atomicrmw did not change x. Recompute the original update condition |
| 6660 | // on the old value and negate it. v is stored only in that case. |
| 6661 | llvm::CmpInst::Predicate updatePred; |
| 6662 | switch (rmw->getOperation()) { |
| 6663 | case llvm::AtomicRMWInst::Min: |
| 6664 | updatePred = llvm::CmpInst::ICMP_SGT; |
| 6665 | break; |
| 6666 | case llvm::AtomicRMWInst::Max: |
| 6667 | updatePred = llvm::CmpInst::ICMP_SLT; |
| 6668 | break; |
| 6669 | case llvm::AtomicRMWInst::UMin: |
| 6670 | updatePred = llvm::CmpInst::ICMP_UGT; |
| 6671 | break; |
| 6672 | case llvm::AtomicRMWInst::UMax: |
| 6673 | updatePred = llvm::CmpInst::ICMP_ULT; |
| 6674 | break; |
| 6675 | case llvm::AtomicRMWInst::FMin: |
| 6676 | updatePred = llvm::CmpInst::FCMP_OGT; |
| 6677 | break; |
| 6678 | case llvm::AtomicRMWInst::FMax: |
| 6679 | updatePred = llvm::CmpInst::FCMP_OLT; |
| 6680 | break; |
| 6681 | default: |
| 6682 | llvm_unreachable( |
| 6683 | "unexpected atomicrmw op for min/max compare capture" ); |
| 6684 | } |
| 6685 | llvm::Value *updated = builder.CreateCmp(Pred: updatePred, LHS: oldVal, RHS: rhs); |
| 6686 | llvm::Value *failed = builder.CreateNot(V: updated); |
| 6687 | llvm::Function *fn = curBB->getParent(); |
| 6688 | llvm::BasicBlock *contBB = llvm::BasicBlock::Create( |
| 6689 | Context&: builder.getContext(), Name: "omp.atomic.cont" , Parent: fn); |
| 6690 | llvm::BasicBlock *exitBB = llvm::BasicBlock::Create( |
| 6691 | Context&: builder.getContext(), Name: "omp.atomic.exit" , Parent: fn); |
| 6692 | builder.CreateCondBr(Cond: failed, True: contBB, False: exitBB); |
| 6693 | builder.SetInsertPoint(contBB); |
| 6694 | builder.CreateStore(Val: oldVal, Ptr: llvmAtomicV.Var, isVolatile: llvmAtomicV.IsVolatile); |
| 6695 | builder.CreateBr(Dest: exitBB); |
| 6696 | builder.SetInsertPoint(exitBB); |
| 6697 | } else { |
| 6698 | llvm::Intrinsic::ID id; |
| 6699 | switch (rmw->getOperation()) { |
| 6700 | case llvm::AtomicRMWInst::Min: |
| 6701 | id = llvm::Intrinsic::smin; |
| 6702 | break; |
| 6703 | case llvm::AtomicRMWInst::Max: |
| 6704 | id = llvm::Intrinsic::smax; |
| 6705 | break; |
| 6706 | case llvm::AtomicRMWInst::UMin: |
| 6707 | id = llvm::Intrinsic::umin; |
| 6708 | break; |
| 6709 | case llvm::AtomicRMWInst::UMax: |
| 6710 | id = llvm::Intrinsic::umax; |
| 6711 | break; |
| 6712 | case llvm::AtomicRMWInst::FMin: |
| 6713 | id = llvm::Intrinsic::minnum; |
| 6714 | break; |
| 6715 | case llvm::AtomicRMWInst::FMax: |
| 6716 | id = llvm::Intrinsic::maxnum; |
| 6717 | break; |
| 6718 | default: |
| 6719 | llvm_unreachable( |
| 6720 | "unexpected atomicrmw op for min/max compare capture" ); |
| 6721 | } |
| 6722 | llvm::Value *newVal = builder.CreateBinaryIntrinsic(ID: id, LHS: oldVal, RHS: rhs); |
| 6723 | builder.CreateStore(Val: newVal, Ptr: llvmAtomicV.Var, isVolatile: llvmAtomicV.IsVolatile); |
| 6724 | } |
| 6725 | } |
| 6726 | |
| 6727 | // Equality postfix: v = select(success, D, old) — reconstructs the new |
| 6728 | // value of x from the cmpxchg result. |
| 6729 | if (!isMinMax && isPostfixCapture && !isFailOnly) { |
| 6730 | llvm::BasicBlock *curBB = builder.GetInsertBlock(); |
| 6731 | llvm::Value *oldVal = nullptr; |
| 6732 | llvm::Value *successVal = nullptr; |
| 6733 | |
| 6734 | // Integer path (and non-HandleFPNegZero FP path): a single cmpxchg |
| 6735 | // lives in the current block. |
| 6736 | for (auto &inst : llvm::reverse(C&: *curBB)) { |
| 6737 | if (isa<llvm::AtomicCmpXchgInst>(Val: &inst)) { |
| 6738 | oldVal = builder.CreateExtractValue(Agg: &inst, /*Idxs=*/0); |
| 6739 | successVal = builder.CreateExtractValue(Agg: &inst, /*Idxs=*/1); |
| 6740 | break; |
| 6741 | } |
| 6742 | } |
| 6743 | |
| 6744 | // FP HandleFPNegZero path: the OMPIRBuilder emits a multi-block |
| 6745 | // structure (NaN / ±0.0 handling) with cmpxchg in predecessor |
| 6746 | // blocks. Results are merged via PHI nodes in the current (exit) |
| 6747 | // block: an i1 PHI for success and a bitcast of an integer PHI |
| 6748 | // for the old FP value. |
| 6749 | if (!oldVal) { |
| 6750 | for (auto &inst : *curBB) { |
| 6751 | auto *phi = dyn_cast<llvm::PHINode>(Val: &inst); |
| 6752 | if (!phi) |
| 6753 | break; |
| 6754 | if (phi->getType()->isIntegerTy(BitWidth: 1)) |
| 6755 | successVal = phi; |
| 6756 | } |
| 6757 | for (auto &inst : *curBB) { |
| 6758 | if (auto *bc = dyn_cast<llvm::BitCastInst>(Val: &inst)) { |
| 6759 | oldVal = bc; |
| 6760 | break; |
| 6761 | } |
| 6762 | } |
| 6763 | } |
| 6764 | |
| 6765 | assert(oldVal && "expected cmpxchg or PHI+bitcast for compare capture" ); |
| 6766 | assert(successVal && "expected success flag for compare capture" ); |
| 6767 | llvm::Value *newVal = builder.CreateSelect(C: successVal, True: dVal, False: oldVal); |
| 6768 | builder.CreateStore(Val: newVal, Ptr: llvmAtomicV.Var, isVolatile: llvmAtomicV.IsVolatile); |
| 6769 | } |
| 6770 | |
| 6771 | return success(); |
| 6772 | } |
| 6773 | |
| 6774 | mlir::Value mlirExpr; |
| 6775 | bool isXBinopExpr = false, isPostfixUpdate = false; |
| 6776 | llvm::AtomicRMWInst::BinOp binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP; |
| 6777 | |
| 6778 | assert((atomicUpdateOp || atomicWriteOp) && |
| 6779 | "internal op must be an atomic.update or atomic.write op" ); |
| 6780 | |
| 6781 | if (atomicWriteOp) { |
| 6782 | isPostfixUpdate = true; |
| 6783 | mlirExpr = atomicWriteOp.getExpr(); |
| 6784 | } else { |
| 6785 | isPostfixUpdate = atomicCaptureOp.getSecondOp() == |
| 6786 | atomicCaptureOp.getAtomicUpdateOp().getOperation(); |
| 6787 | auto &innerOpList = atomicUpdateOp.getRegion().front().getOperations(); |
| 6788 | // Find the binary update operation that uses the region argument |
| 6789 | // and get the expression to update |
| 6790 | if (innerOpList.size() == 2) { |
| 6791 | mlir::Operation &innerOp = *atomicUpdateOp.getRegion().front().begin(); |
| 6792 | if (!llvm::is_contained(Range: innerOp.getOperands(), |
| 6793 | Element: atomicUpdateOp.getRegion().getArgument(i: 0))) { |
| 6794 | return atomicUpdateOp.emitError( |
| 6795 | message: "no atomic update operation with region argument" |
| 6796 | " as operand found inside atomic.update region" ); |
| 6797 | } |
| 6798 | binop = convertBinOpToAtomic(op&: innerOp); |
| 6799 | isXBinopExpr = |
| 6800 | innerOp.getOperand(idx: 0) == atomicUpdateOp.getRegion().getArgument(i: 0); |
| 6801 | mlirExpr = (isXBinopExpr ? innerOp.getOperand(idx: 1) : innerOp.getOperand(idx: 0)); |
| 6802 | } else { |
| 6803 | binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP; |
| 6804 | } |
| 6805 | } |
| 6806 | |
| 6807 | llvm::Value *llvmExpr = moduleTranslation.lookupValue(value: mlirExpr); |
| 6808 | llvm::Value *llvmX = |
| 6809 | moduleTranslation.lookupValue(value: atomicCaptureOp.getAtomicReadOp().getX()); |
| 6810 | llvm::Value *llvmV = |
| 6811 | moduleTranslation.lookupValue(value: atomicCaptureOp.getAtomicReadOp().getV()); |
| 6812 | llvm::Type *llvmXElementType = moduleTranslation.convertType( |
| 6813 | type: atomicCaptureOp.getAtomicReadOp().getElementType()); |
| 6814 | llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {.Var: llvmX, .ElemTy: llvmXElementType, |
| 6815 | /*isSigned=*/.IsSigned: false, |
| 6816 | /*isVolatile=*/.IsVolatile: false}; |
| 6817 | llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicV = {.Var: llvmV, .ElemTy: llvmXElementType, |
| 6818 | /*isSigned=*/.IsSigned: false, |
| 6819 | /*isVolatile=*/.IsVolatile: false}; |
| 6820 | |
| 6821 | llvm::AtomicOrdering atomicOrdering = |
| 6822 | convertAtomicOrdering(ao: atomicCaptureOp.getMemoryOrder()); |
| 6823 | |
| 6824 | auto updateFn = |
| 6825 | [&](llvm::Value *atomicx, |
| 6826 | llvm::IRBuilder<> &builder) -> llvm::Expected<llvm::Value *> { |
| 6827 | if (atomicWriteOp) |
| 6828 | return moduleTranslation.lookupValue(value: atomicWriteOp.getExpr()); |
| 6829 | Block &bb = *atomicUpdateOp.getRegion().begin(); |
| 6830 | moduleTranslation.mapValue(mlir: *atomicUpdateOp.getRegion().args_begin(), |
| 6831 | llvm: atomicx); |
| 6832 | moduleTranslation.mapBlock(mlir: &bb, llvm: builder.GetInsertBlock()); |
| 6833 | if (failed(Result: moduleTranslation.convertBlock(bb, ignoreArguments: true, builder))) |
| 6834 | return llvm::make_error<PreviouslyReportedError>(); |
| 6835 | |
| 6836 | omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(Val: bb.getTerminator()); |
| 6837 | assert(yieldop && yieldop.getResults().size() == 1 && |
| 6838 | "terminator must be omp.yield op and it must have exactly one " |
| 6839 | "argument" ); |
| 6840 | return moduleTranslation.lookupValue(value: yieldop.getResults()[0]); |
| 6841 | }; |
| 6842 | |
| 6843 | bool isIgnoreDenormalMode; |
| 6844 | bool isFineGrainedMemory; |
| 6845 | bool isRemoteMemory; |
| 6846 | extractAtomicControlFlags(atomicUpdateOp, isIgnoreDenormalMode, |
| 6847 | isFineGrainedMemory, isRemoteMemory); |
| 6848 | // Handle ambiguous alloca, if any. |
| 6849 | auto allocaIP = findAllocInsertPoints(builder, moduleTranslation); |
| 6850 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 6851 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 6852 | ompBuilder->createAtomicCapture( |
| 6853 | Loc: ompLoc, AllocaIP: allocaIP, X&: llvmAtomicX, V&: llvmAtomicV, Expr: llvmExpr, AO: atomicOrdering, |
| 6854 | RMWOp: binop, UpdateOp: updateFn, UpdateExpr: atomicUpdateOp, IsPostfixUpdate: isPostfixUpdate, IsXBinopExpr: isXBinopExpr, |
| 6855 | IsIgnoreDenormalMode: isIgnoreDenormalMode, IsFineGrainedMemory: isFineGrainedMemory, IsRemoteMemory: isRemoteMemory); |
| 6856 | |
| 6857 | if (failed(Result: handleError(result&: afterIP, op&: *atomicCaptureOp))) |
| 6858 | return failure(); |
| 6859 | |
| 6860 | builder.restoreIP(IP: *afterIP); |
| 6861 | return success(); |
| 6862 | } |
| 6863 | |
| 6864 | /// Converts an omp.atomic.compare operation to LLVM IR. |
| 6865 | /// |
| 6866 | /// if (x == e) x = d |
| 6867 | /// The region contains a comparison + select pattern: |
| 6868 | /// ^bb0(%xval: T): |
| 6869 | /// %cmp = llvm.icmp/fcmp <pred> %xval, %e : T |
| 6870 | /// %sel = llvm.select %cmp, %d, %xval : i1, T |
| 6871 | /// omp.yield(%sel : T) |
| 6872 | /// |
| 6873 | /// From MLIR extract: |
| 6874 | /// 1) comparison operator |
| 6875 | /// 2) expected value (e) |
| 6876 | /// 3) desired value (d) |
| 6877 | /// These are passed to OpenMPIRBuilder::createAtomicCompare which generates |
| 6878 | /// the actual cmpxchg / atomicrmw instruction. |
| 6879 | /// |
| 6880 | static LogicalResult |
| 6881 | convertOmpAtomicCompare(omp::AtomicCompareOp atomicCompareOp, |
| 6882 | llvm::IRBuilderBase &builder, |
| 6883 | LLVM::ModuleTranslation &moduleTranslation) { |
| 6884 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 6885 | if (failed(Result: checkImplementationStatus(op&: *atomicCompareOp))) |
| 6886 | return failure(); |
| 6887 | |
| 6888 | Region ®ion = atomicCompareOp.getRegion(); |
| 6889 | Block &block = region.front(); |
| 6890 | |
| 6891 | // Determine element type from the region block argument |
| 6892 | llvm::Type *llvmXElementType = |
| 6893 | moduleTranslation.convertType(type: block.getArgument(i: 0).getType()); |
| 6894 | if (!llvmXElementType) |
| 6895 | return atomicCompareOp.emitError( |
| 6896 | message: "unable to determine element type for atomic compare" ); |
| 6897 | |
| 6898 | llvm::Value *llvmX = moduleTranslation.lookupValue(value: atomicCompareOp.getX()); |
| 6899 | |
| 6900 | // IsSigned is determined from the comparison predicate in the region. |
| 6901 | // Signed ICmp predicates (slt/sgt) set this to true; unsigned (ult/ugt) |
| 6902 | // leave it false. For EQ and float comparisons, signedness is irrelevant. |
| 6903 | bool isSigned = false; |
| 6904 | llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {.Var: llvmX, .ElemTy: llvmXElementType, |
| 6905 | .IsSigned: isSigned, |
| 6906 | /*IsVolatile=*/false}; |
| 6907 | |
| 6908 | llvm::AtomicOrdering atomicOrdering = |
| 6909 | convertAtomicOrdering(ao: atomicCompareOp.getMemoryOrder()); |
| 6910 | |
| 6911 | auto isAtomicComparePatternOp = [](Operation &op) { |
| 6912 | return llvm::isa<LLVM::ICmpOp, LLVM::FCmpOp, LLVM::SelectOp, LLVM::AndOp, |
| 6913 | LLVM::OrOp, LLVM::SMaxOp, LLVM::SMinOp, LLVM::UMaxOp, |
| 6914 | LLVM::UMinOp, LLVM::MaxNumOp, LLVM::MinNumOp, |
| 6915 | mlir::arith::MaxSIOp, mlir::arith::MinSIOp, |
| 6916 | mlir::arith::MaxUIOp, mlir::arith::MinUIOp, |
| 6917 | mlir::arith::MaximumFOp, mlir::arith::MinimumFOp>(Val: op); |
| 6918 | }; |
| 6919 | |
| 6920 | // Pre-translate operations inside the region that compute e and d (e.g., |
| 6921 | // GEP, loads for dereferencing Fortran pointers) but are not part of the |
| 6922 | // atomic compare-and-swap pattern (icmp/fcmp, select, and/or). |
| 6923 | // |
| 6924 | // 1) Validity: The OpenMP spec requires e and d to be evaluated before the |
| 6925 | // atomic operation, so emitting their computation here is correct. |
| 6926 | // 2) Memory effects: These ops only depend on values defined outside the |
| 6927 | // region. They cannot observe the block argument (%xval), which is the |
| 6928 | // value loaded atomically by cmpxchg and does not exist yet. |
| 6929 | // 3) Invariant enforcement: The `allOperandsMapped` check below skips any |
| 6930 | // op whose operands include the unmapped block argument, guaranteeing |
| 6931 | // only region-external-dependent ops are pre-translated. |
| 6932 | for (Operation &op : block.without_terminator()) { |
| 6933 | // Skip operations that form the atomic compare pattern — these are |
| 6934 | // not emitted as individual instructions but are analyzed below to |
| 6935 | // extract the comparison predicate, expected value (e), and desired |
| 6936 | // value (d) for generating a single cmpxchg/atomicrmw. |
| 6937 | if (isAtomicComparePatternOp(op)) |
| 6938 | continue; |
| 6939 | |
| 6940 | // Avoid translating ops that depend on the unmapped block argument. |
| 6941 | bool allOperandsMapped = llvm::all_of(Range: op.getOperands(), P: [&](mlir::Value v) { |
| 6942 | return moduleTranslation.lookupValue(value: v) != nullptr; |
| 6943 | }); |
| 6944 | if (!allOperandsMapped) |
| 6945 | continue; |
| 6946 | |
| 6947 | if (failed(Result: moduleTranslation.convertOperation(op, builder))) |
| 6948 | return atomicCompareOp.emitError( |
| 6949 | message: "failed to translate operation inside atomic compare region" ); |
| 6950 | } |
| 6951 | |
| 6952 | // Look up a value that may have been pre-translated or defined outside the |
| 6953 | // region. |
| 6954 | auto materializeValue = [&](mlir::Value val) -> llvm::Value * { |
| 6955 | // Check if the value is already mapped (pre-translated or defined outside). |
| 6956 | if (llvm::Value *existing = moduleTranslation.lookupValue(value: val)) |
| 6957 | return existing; |
| 6958 | // Fallback for a single LoadOp whose address is mapped but whose result |
| 6959 | // was not pre-translated. |
| 6960 | if (auto loadOp = val.getDefiningOp<LLVM::LoadOp>()) { |
| 6961 | if (loadOp->getParentRegion() == ®ion) { |
| 6962 | llvm::Value *loadAddr = moduleTranslation.lookupValue(value: loadOp.getAddr()); |
| 6963 | if (!loadAddr) |
| 6964 | return nullptr; |
| 6965 | llvm::Type *loadType = |
| 6966 | moduleTranslation.convertType(type: loadOp.getResult().getType()); |
| 6967 | return builder.CreateLoad(Ty: loadType, Ptr: loadAddr); |
| 6968 | } |
| 6969 | } |
| 6970 | return nullptr; |
| 6971 | }; |
| 6972 | |
| 6973 | // Walk the region to extract comparison predicate, eVal, and dVal. |
| 6974 | // if (x == eVal) x = dVal |
| 6975 | llvm::omp::OMPAtomicCompareOp compareOp = llvm::omp::OMPAtomicCompareOp::EQ; |
| 6976 | llvm::Value *eVal = nullptr; |
| 6977 | llvm::Value *dVal = nullptr; |
| 6978 | bool isXBinopExpr = false; |
| 6979 | |
| 6980 | // Check for a decomposed complex comparison pattern (extractvalue + fcmp + |
| 6981 | // and/or of the real/imaginary fields). |
| 6982 | ComplexComparePattern cplx = detectComplexCompareEq(block); |
| 6983 | bool isComplexPattern = cplx.isComplex; |
| 6984 | if (isComplexPattern) { |
| 6985 | if (cplx.isNE) |
| 6986 | // OrOp corresponds to NE, which is not a valid atomic compare op. |
| 6987 | return atomicCompareOp.emitError( |
| 6988 | message: "unsupported comparison predicate (NE) for complex atomic compare" ); |
| 6989 | compareOp = llvm::omp::OMPAtomicCompareOp::EQ; |
| 6990 | isXBinopExpr = cplx.isXBinopExpr; |
| 6991 | eVal = materializeValue(cplx.eAggregate); |
| 6992 | } |
| 6993 | |
| 6994 | if (isComplexPattern) { |
| 6995 | // dVal from SelectOp or YieldOp. |
| 6996 | for (Operation &op : block.getOperations()) { |
| 6997 | if (auto selectOp = dyn_cast<LLVM::SelectOp>(Val&: op)) { |
| 6998 | dVal = materializeValue(selectOp.getTrueValue()); |
| 6999 | break; |
| 7000 | } |
| 7001 | } |
| 7002 | if (!dVal) { |
| 7003 | auto yieldOp = cast<omp::YieldOp>(Val: block.getTerminator()); |
| 7004 | if (yieldOp.getResults().empty()) |
| 7005 | return atomicCompareOp.emitError( |
| 7006 | message: "failed to extract desired value (d) from atomic compare region" ); |
| 7007 | dVal = materializeValue(yieldOp.getResults()[0]); |
| 7008 | } |
| 7009 | |
| 7010 | llvm::Value *oldComplex = nullptr; |
| 7011 | llvm::Value *cmpOk = nullptr; |
| 7012 | llvm::AtomicOrdering failOrdering = |
| 7013 | getAtomicCompareFailureOrdering(atomicCompareOp, atomicOrdering); |
| 7014 | emitComplexAtomicCmpXchg(builder, llvmX, complexTy: llvmXElementType, eVal, dVal, |
| 7015 | atomicOrdering, failOrdering, |
| 7016 | isWeak: atomicCompareOp.getWeak(), oldComplex, cmpOk); |
| 7017 | (void)oldComplex; |
| 7018 | (void)cmpOk; |
| 7019 | |
| 7020 | // Emit flush after atomic compare if needed (for release, acq_rel, |
| 7021 | // seq_cst orderings). |
| 7022 | if (atomicOrdering == llvm::AtomicOrdering::Release || |
| 7023 | atomicOrdering == llvm::AtomicOrdering::AcquireRelease || |
| 7024 | atomicOrdering == llvm::AtomicOrdering::SequentiallyConsistent) { |
| 7025 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 7026 | ompBuilder->createFlush(Loc: ompLoc); |
| 7027 | } |
| 7028 | return success(); |
| 7029 | } else { |
| 7030 | AtomicComparePatternInfo patternInfo; |
| 7031 | if (failed(Result: extractAtomicComparePattern(block, materializeValue, |
| 7032 | atomicCompareOp, info&: patternInfo))) |
| 7033 | return failure(); |
| 7034 | compareOp = patternInfo.compareOp; |
| 7035 | eVal = patternInfo.eVal; |
| 7036 | dVal = patternInfo.dVal; |
| 7037 | isXBinopExpr = patternInfo.isXBinopExpr; |
| 7038 | isSigned = patternInfo.isSigned; |
| 7039 | } |
| 7040 | |
| 7041 | if (!eVal) |
| 7042 | return atomicCompareOp.emitError( |
| 7043 | message: "failed to extract expected value (e) from atomic compare region" ); |
| 7044 | if (!dVal) { |
| 7045 | // Fall back to the yield operand. |
| 7046 | auto yieldOp = cast<omp::YieldOp>(Val: block.getTerminator()); |
| 7047 | if (yieldOp.getResults().empty()) |
| 7048 | return atomicCompareOp.emitError( |
| 7049 | message: "failed to extract desired value (d) from atomic compare region" ); |
| 7050 | dVal = materializeValue(yieldOp.getResults()[0]); |
| 7051 | } |
| 7052 | |
| 7053 | llvmAtomicX.IsSigned = isSigned; |
| 7054 | |
| 7055 | llvm::OpenMPIRBuilder::AtomicOpValue vOpVal = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false, |
| 7056 | .IsVolatile: false}; |
| 7057 | llvm::OpenMPIRBuilder::AtomicOpValue rOpVal = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false, |
| 7058 | .IsVolatile: false}; |
| 7059 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 7060 | |
| 7061 | bool isWeak = atomicCompareOp.getWeak(); |
| 7062 | |
| 7063 | bool savedHandleFPNegZero = ompBuilder->setHandleFPNegZero(true); |
| 7064 | llvm::AtomicOrdering failureOrdering = |
| 7065 | getAtomicCompareFailureOrdering(atomicCompareOp, atomicOrdering); |
| 7066 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 7067 | ompBuilder->createAtomicCompare( |
| 7068 | Loc: ompLoc, X&: llvmAtomicX, V&: vOpVal, R&: rOpVal, E: eVal, D: dVal, AO: atomicOrdering, |
| 7069 | Op: compareOp, IsXBinopExpr: isXBinopExpr, /*IsPostfixUpdate=*/false, |
| 7070 | /*IsFailOnly=*/false, Failure: failureOrdering, IsWeak: isWeak); |
| 7071 | ompBuilder->setHandleFPNegZero(savedHandleFPNegZero); |
| 7072 | |
| 7073 | if (failed(Result: handleError(result&: afterIP, op&: *atomicCompareOp))) |
| 7074 | return failure(); |
| 7075 | |
| 7076 | builder.restoreIP(IP: *afterIP); |
| 7077 | return success(); |
| 7078 | } |
| 7079 | |
| 7080 | static llvm::omp::Directive convertCancellationConstructType( |
| 7081 | omp::ClauseCancellationConstructType directive) { |
| 7082 | switch (directive) { |
| 7083 | case omp::ClauseCancellationConstructType::Loop: |
| 7084 | return llvm::omp::Directive::OMPD_for; |
| 7085 | case omp::ClauseCancellationConstructType::Parallel: |
| 7086 | return llvm::omp::Directive::OMPD_parallel; |
| 7087 | case omp::ClauseCancellationConstructType::Sections: |
| 7088 | return llvm::omp::Directive::OMPD_sections; |
| 7089 | case omp::ClauseCancellationConstructType::Taskgroup: |
| 7090 | return llvm::omp::Directive::OMPD_taskgroup; |
| 7091 | } |
| 7092 | llvm_unreachable("Unhandled cancellation construct type" ); |
| 7093 | } |
| 7094 | |
| 7095 | static LogicalResult |
| 7096 | convertOmpCancel(omp::CancelOp op, llvm::IRBuilderBase &builder, |
| 7097 | LLVM::ModuleTranslation &moduleTranslation) { |
| 7098 | if (failed(Result: checkImplementationStatus(op&: *op.getOperation()))) |
| 7099 | return failure(); |
| 7100 | |
| 7101 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 7102 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 7103 | |
| 7104 | llvm::Value *ifCond = nullptr; |
| 7105 | if (Value ifVar = op.getIfExpr()) |
| 7106 | ifCond = moduleTranslation.lookupValue(value: ifVar); |
| 7107 | |
| 7108 | llvm::omp::Directive cancelledDirective = |
| 7109 | convertCancellationConstructType(directive: op.getCancelDirective()); |
| 7110 | |
| 7111 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 7112 | ompBuilder->createCancel(Loc: ompLoc, IfCondition: ifCond, CanceledDirective: cancelledDirective); |
| 7113 | |
| 7114 | if (failed(Result: handleError(result&: afterIP, op&: *op.getOperation()))) |
| 7115 | return failure(); |
| 7116 | |
| 7117 | builder.restoreIP(IP: afterIP.get()); |
| 7118 | |
| 7119 | return success(); |
| 7120 | } |
| 7121 | |
| 7122 | static LogicalResult |
| 7123 | convertOmpCancellationPoint(omp::CancellationPointOp op, |
| 7124 | llvm::IRBuilderBase &builder, |
| 7125 | LLVM::ModuleTranslation &moduleTranslation) { |
| 7126 | if (failed(Result: checkImplementationStatus(op&: *op.getOperation()))) |
| 7127 | return failure(); |
| 7128 | |
| 7129 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 7130 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 7131 | |
| 7132 | llvm::omp::Directive cancelledDirective = |
| 7133 | convertCancellationConstructType(directive: op.getCancelDirective()); |
| 7134 | |
| 7135 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 7136 | ompBuilder->createCancellationPoint(Loc: ompLoc, CanceledDirective: cancelledDirective); |
| 7137 | |
| 7138 | if (failed(Result: handleError(result&: afterIP, op&: *op.getOperation()))) |
| 7139 | return failure(); |
| 7140 | |
| 7141 | builder.restoreIP(IP: afterIP.get()); |
| 7142 | |
| 7143 | return success(); |
| 7144 | } |
| 7145 | |
| 7146 | /// Converts an OpenMP Threadprivate operation into LLVM IR using |
| 7147 | /// OpenMPIRBuilder. |
| 7148 | static LogicalResult |
| 7149 | convertOmpThreadprivate(Operation &opInst, llvm::IRBuilderBase &builder, |
| 7150 | LLVM::ModuleTranslation &moduleTranslation) { |
| 7151 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 7152 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 7153 | auto threadprivateOp = cast<omp::ThreadprivateOp>(Val&: opInst); |
| 7154 | |
| 7155 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 7156 | return failure(); |
| 7157 | |
| 7158 | Value symAddr = threadprivateOp.getSymAddr(); |
| 7159 | auto *symOp = symAddr.getDefiningOp(); |
| 7160 | |
| 7161 | if (auto asCast = dyn_cast<LLVM::AddrSpaceCastOp>(Val: symOp)) |
| 7162 | symOp = asCast.getOperand().getDefiningOp(); |
| 7163 | |
| 7164 | if (!isa<LLVM::AddressOfOp>(Val: symOp)) |
| 7165 | return opInst.emitError(message: "Addressing symbol not found" ); |
| 7166 | LLVM::AddressOfOp addressOfOp = dyn_cast<LLVM::AddressOfOp>(Val: symOp); |
| 7167 | |
| 7168 | LLVM::GlobalOp global = |
| 7169 | addressOfOp.getGlobal(symbolTable&: moduleTranslation.symbolTable()); |
| 7170 | llvm::GlobalValue *globalValue = moduleTranslation.lookupGlobal(op: global); |
| 7171 | llvm::Type *type = globalValue->getValueType(); |
| 7172 | llvm::TypeSize typeSize = builder.getDataLayout().getTypeStoreSize(Ty: type); |
| 7173 | llvm::ConstantInt *size = builder.getInt64(C: typeSize.getFixedValue()); |
| 7174 | llvm::Value *callInst = ompBuilder->createCachedThreadPrivate( |
| 7175 | Loc: ompLoc, Pointer: globalValue, Size: size, Name: global.getSymName() + ".cache" ); |
| 7176 | moduleTranslation.mapValue(mlir: opInst.getResult(idx: 0), llvm: callInst); |
| 7177 | |
| 7178 | return success(); |
| 7179 | } |
| 7180 | |
| 7181 | static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind |
| 7182 | convertToDeviceClauseKind(mlir::omp::DeclareTargetDeviceType deviceClause) { |
| 7183 | switch (deviceClause) { |
| 7184 | case mlir::omp::DeclareTargetDeviceType::host: |
| 7185 | return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost; |
| 7186 | break; |
| 7187 | case mlir::omp::DeclareTargetDeviceType::nohost: |
| 7188 | return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost; |
| 7189 | break; |
| 7190 | case mlir::omp::DeclareTargetDeviceType::any: |
| 7191 | return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny; |
| 7192 | break; |
| 7193 | } |
| 7194 | llvm_unreachable("unhandled device clause" ); |
| 7195 | } |
| 7196 | |
| 7197 | static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind |
| 7198 | convertToCaptureClauseKind( |
| 7199 | mlir::omp::DeclareTargetCaptureClause captureClause) { |
| 7200 | switch (captureClause) { |
| 7201 | case mlir::omp::DeclareTargetCaptureClause::to: |
| 7202 | return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo; |
| 7203 | case mlir::omp::DeclareTargetCaptureClause::link: |
| 7204 | return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink; |
| 7205 | case mlir::omp::DeclareTargetCaptureClause::enter: |
| 7206 | return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter; |
| 7207 | case mlir::omp::DeclareTargetCaptureClause::none: |
| 7208 | return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone; |
| 7209 | } |
| 7210 | llvm_unreachable("unhandled capture clause" ); |
| 7211 | } |
| 7212 | |
| 7213 | static Operation *getGlobalOpFromValue(Value value) { |
| 7214 | Operation *op = value.getDefiningOp(); |
| 7215 | if (auto addrCast = dyn_cast_if_present<LLVM::AddrSpaceCastOp>(Val: op)) |
| 7216 | op = addrCast->getOperand(idx: 0).getDefiningOp(); |
| 7217 | if (auto addressOfOp = dyn_cast_if_present<LLVM::AddressOfOp>(Val: op)) { |
| 7218 | auto modOp = addressOfOp->getParentOfType<mlir::ModuleOp>(); |
| 7219 | return modOp.lookupSymbol(name: addressOfOp.getGlobalName()); |
| 7220 | } |
| 7221 | return nullptr; |
| 7222 | } |
| 7223 | |
| 7224 | static Value getBaseValueForTypeLookup(Value value) { |
| 7225 | while (Operation *op = value.getDefiningOp()) { |
| 7226 | if (auto addrCast = dyn_cast_if_present<LLVM::AddrSpaceCastOp>(Val: op)) |
| 7227 | value = addrCast.getOperand(); |
| 7228 | // Traces through hlfir.declare, fir.declare to reach the base address and |
| 7229 | // use for type lookup. |
| 7230 | else if (op->getName().getIdentifier() && |
| 7231 | (op->getName().getIdentifier().str() == "hlfir.declare" || |
| 7232 | op->getName().getIdentifier().str() == "fir.declare" )) { |
| 7233 | if (op->getNumOperands() > 0) |
| 7234 | value = op->getOperand(idx: 0); |
| 7235 | else |
| 7236 | break; |
| 7237 | } else { |
| 7238 | break; |
| 7239 | } |
| 7240 | } |
| 7241 | return value; |
| 7242 | } |
| 7243 | |
| 7244 | // Determine the LLVM type whose storage size should be allocated for an |
| 7245 | // OpenMP allocate directive list item. Opaque pointers lose element type, so |
| 7246 | // trace through declare wrappers to the underlying global or stack allocation. |
| 7247 | static llvm::Type * |
| 7248 | getAllocatedLlvmTypeForVariable(Value var, Value baseVar, |
| 7249 | LLVM::ModuleTranslation &moduleTranslation) { |
| 7250 | llvm::Type *llvmVarTy = moduleTranslation.convertType(type: var.getType()); |
| 7251 | if (!llvmVarTy->isPointerTy()) |
| 7252 | return llvmVarTy; |
| 7253 | |
| 7254 | if (Operation *globalOp = getGlobalOpFromValue(value: baseVar)) |
| 7255 | if (auto gop = dyn_cast<LLVM::GlobalOp>(Val: globalOp)) |
| 7256 | return moduleTranslation.convertType(type: gop.getGlobalType()); |
| 7257 | |
| 7258 | if (auto allocaOp = |
| 7259 | dyn_cast_if_present<LLVM::AllocaOp>(Val: baseVar.getDefiningOp())) |
| 7260 | return moduleTranslation.convertType(type: allocaOp.getElemType()); |
| 7261 | |
| 7262 | if (llvm::Value *baseLlvm = moduleTranslation.lookupValue(value: baseVar)) |
| 7263 | if (auto *allocaInst = dyn_cast<llvm::AllocaInst>(Val: baseLlvm)) |
| 7264 | return allocaInst->getAllocatedType(); |
| 7265 | |
| 7266 | return llvmVarTy; |
| 7267 | } |
| 7268 | |
| 7269 | // For dynamically-sized stack allocations, compute the allocation size from |
| 7270 | // the alloca's element count at runtime. |
| 7271 | static std::optional<llvm::Value *> getDynamicAllocatedSize( |
| 7272 | Value var, Value baseVar, LLVM::ModuleTranslation &moduleTranslation, |
| 7273 | llvm::IRBuilderBase &builder, const llvm::DataLayout &dataLayout) { |
| 7274 | if (auto allocaOp = |
| 7275 | dyn_cast_if_present<LLVM::AllocaOp>(Val: baseVar.getDefiningOp())) { |
| 7276 | if (Value arraySize = allocaOp.getArraySize()) { |
| 7277 | llvm::Type *elemTy = |
| 7278 | moduleTranslation.convertType(type: allocaOp.getElemType()); |
| 7279 | llvm::Value *numElems = moduleTranslation.lookupValue(value: arraySize); |
| 7280 | if (!numElems->getType()->isIntegerTy(BitWidth: 64)) |
| 7281 | numElems = builder.CreateZExt(V: numElems, DestTy: builder.getInt64Ty()); |
| 7282 | uint64_t elemSize = dataLayout.getTypeAllocSize(Ty: elemTy).getFixedValue(); |
| 7283 | return builder.CreateMul(LHS: numElems, RHS: builder.getInt64(C: elemSize)); |
| 7284 | } |
| 7285 | } |
| 7286 | if (llvm::Value *baseLlvm = moduleTranslation.lookupValue(value: baseVar)) { |
| 7287 | if (auto *allocaInst = dyn_cast<llvm::AllocaInst>(Val: baseLlvm)) { |
| 7288 | if (allocaInst->isArrayAllocation() && |
| 7289 | !llvm::isa<llvm::ArrayType>(Val: allocaInst->getAllocatedType())) { |
| 7290 | uint64_t elemSize = |
| 7291 | dataLayout.getTypeAllocSize(Ty: allocaInst->getAllocatedType()) |
| 7292 | .getFixedValue(); |
| 7293 | return builder.CreateMul(LHS: allocaInst->getArraySize(), |
| 7294 | RHS: builder.getInt64(C: elemSize)); |
| 7295 | } |
| 7296 | } |
| 7297 | } |
| 7298 | return std::nullopt; |
| 7299 | } |
| 7300 | |
| 7301 | static llvm::SmallString<64> |
| 7302 | getDeclareTargetRefPtrSuffix(LLVM::GlobalOp globalOp, |
| 7303 | llvm::OpenMPIRBuilder &ompBuilder, |
| 7304 | llvm::vfs::FileSystem &vfs) { |
| 7305 | llvm::SmallString<64> suffix; |
| 7306 | llvm::raw_svector_ostream os(suffix); |
| 7307 | if (globalOp.getVisibility() == mlir::SymbolTable::Visibility::Private) { |
| 7308 | auto loc = globalOp->getLoc()->findInstanceOf<FileLineColLoc>(); |
| 7309 | auto fileInfoCallBack = [&loc]() { |
| 7310 | return std::pair<std::string, uint64_t>( |
| 7311 | llvm::StringRef(loc.getFilename()), loc.getLine()); |
| 7312 | }; |
| 7313 | |
| 7314 | os << llvm::format( |
| 7315 | Fmt: "_%x" , |
| 7316 | Vals: ompBuilder.getTargetEntryUniqueInfo(CallBack: fileInfoCallBack, VFS&: vfs).FileID); |
| 7317 | } |
| 7318 | os << "_decl_tgt_ref_ptr" ; |
| 7319 | |
| 7320 | return suffix; |
| 7321 | } |
| 7322 | |
| 7323 | static bool isDeclareTargetLink(Value value) { |
| 7324 | if (auto declareTargetGlobal = |
| 7325 | dyn_cast_if_present<omp::DeclareTargetInterface>( |
| 7326 | Val: getGlobalOpFromValue(value))) { |
| 7327 | omp::DeclareTargetAttr declareTargetAttr = |
| 7328 | declareTargetGlobal.getDeclareTarget(); |
| 7329 | if (declareTargetAttr && declareTargetAttr.getCaptureClause() == |
| 7330 | omp::DeclareTargetCaptureClause::link) |
| 7331 | return true; |
| 7332 | } |
| 7333 | return false; |
| 7334 | } |
| 7335 | |
| 7336 | static bool isDeclareTargetTo(Value value) { |
| 7337 | if (auto declareTargetGlobal = |
| 7338 | dyn_cast_if_present<omp::DeclareTargetInterface>( |
| 7339 | Val: getGlobalOpFromValue(value))) { |
| 7340 | omp::DeclareTargetAttr declareTargetAttr = |
| 7341 | declareTargetGlobal.getDeclareTarget(); |
| 7342 | if (declareTargetAttr && (declareTargetAttr.getCaptureClause() == |
| 7343 | omp::DeclareTargetCaptureClause::to || |
| 7344 | declareTargetAttr.getCaptureClause() == |
| 7345 | omp::DeclareTargetCaptureClause::enter)) |
| 7346 | return true; |
| 7347 | } |
| 7348 | return false; |
| 7349 | } |
| 7350 | |
| 7351 | // Returns the reference pointer generated by the lowering of the declare |
| 7352 | // target operation in cases where the link clause is used or the to clause is |
| 7353 | // used in USM mode. |
| 7354 | static llvm::Value * |
| 7355 | getRefPtrIfDeclareTarget(Value value, |
| 7356 | LLVM::ModuleTranslation &moduleTranslation) { |
| 7357 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 7358 | if (auto gOp = |
| 7359 | dyn_cast_or_null<LLVM::GlobalOp>(Val: getGlobalOpFromValue(value))) { |
| 7360 | // In this case, we must utilise the reference pointer generated by |
| 7361 | // the declare target operation, similar to Clang |
| 7362 | if (isDeclareTargetLink(value) || |
| 7363 | (isDeclareTargetTo(value) && |
| 7364 | ompBuilder->Config.hasRequiresUnifiedSharedMemory())) { |
| 7365 | llvm::SmallString<64> suffix = getDeclareTargetRefPtrSuffix( |
| 7366 | globalOp: gOp, ompBuilder&: *ompBuilder, vfs&: moduleTranslation.getFileSystem()); |
| 7367 | |
| 7368 | if (gOp.getSymName().contains(Other: suffix)) |
| 7369 | return moduleTranslation.getLLVMModule()->getNamedValue( |
| 7370 | Name: gOp.getSymName()); |
| 7371 | |
| 7372 | return moduleTranslation.getLLVMModule()->getNamedValue( |
| 7373 | Name: (gOp.getSymName().str() + suffix.str()).str()); |
| 7374 | } |
| 7375 | } |
| 7376 | return nullptr; |
| 7377 | } |
| 7378 | |
| 7379 | namespace { |
| 7380 | // Append customMappers information to existing MapInfosTy |
| 7381 | struct MapInfosTy : llvm::OpenMPIRBuilder::MapInfosTy { |
| 7382 | SmallVector<Operation *, 4> Mappers; |
| 7383 | |
| 7384 | /// Append arrays in \a CurInfo. |
| 7385 | void append(MapInfosTy &curInfo) { |
| 7386 | Mappers.append(in_start: curInfo.Mappers.begin(), in_end: curInfo.Mappers.end()); |
| 7387 | llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo&: curInfo); |
| 7388 | } |
| 7389 | }; |
| 7390 | // A small helper structure to contain data gathered |
| 7391 | // for map lowering and coalese it into one area and |
| 7392 | // avoiding extra computations such as searches in the |
| 7393 | // llvm module for lowered mapped variables or checking |
| 7394 | // if something is declare target (and retrieving the |
| 7395 | // value) more than neccessary. |
| 7396 | struct MapInfoData : MapInfosTy { |
| 7397 | llvm::SmallVector<bool, 4> IsDeclareTarget; |
| 7398 | llvm::SmallVector<bool, 4> IsAMember; |
| 7399 | // Identify if mapping was added by mapClause or use_device clauses. |
| 7400 | llvm::SmallVector<bool, 4> IsAMapping; |
| 7401 | llvm::SmallVector<mlir::Operation *, 4> MapClause; |
| 7402 | llvm::SmallVector<llvm::Value *, 4> OriginalValue; |
| 7403 | // Stripped off array/pointer to get the underlying |
| 7404 | // element type |
| 7405 | llvm::SmallVector<llvm::Type *, 4> BaseType; |
| 7406 | |
| 7407 | /// Append arrays in \a CurInfo. |
| 7408 | void append(MapInfoData &CurInfo) { |
| 7409 | IsDeclareTarget.append(in_start: CurInfo.IsDeclareTarget.begin(), |
| 7410 | in_end: CurInfo.IsDeclareTarget.end()); |
| 7411 | MapClause.append(in_start: CurInfo.MapClause.begin(), in_end: CurInfo.MapClause.end()); |
| 7412 | OriginalValue.append(in_start: CurInfo.OriginalValue.begin(), |
| 7413 | in_end: CurInfo.OriginalValue.end()); |
| 7414 | BaseType.append(in_start: CurInfo.BaseType.begin(), in_end: CurInfo.BaseType.end()); |
| 7415 | MapInfosTy::append(curInfo&: CurInfo); |
| 7416 | } |
| 7417 | }; |
| 7418 | |
| 7419 | enum class TargetDirectiveEnumTy : uint32_t { |
| 7420 | None = 0, |
| 7421 | Target = 1, |
| 7422 | TargetData = 2, |
| 7423 | TargetEnterData = 3, |
| 7424 | TargetExitData = 4, |
| 7425 | TargetUpdate = 5 |
| 7426 | }; |
| 7427 | |
| 7428 | static TargetDirectiveEnumTy getTargetDirectiveEnumTyFromOp(Operation *op) { |
| 7429 | return llvm::TypeSwitch<Operation *, TargetDirectiveEnumTy>(op) |
| 7430 | .Case(caseFn: [](omp::TargetDataOp) { return TargetDirectiveEnumTy::TargetData; }) |
| 7431 | .Case(caseFn: [](omp::TargetEnterDataOp) { |
| 7432 | return TargetDirectiveEnumTy::TargetEnterData; |
| 7433 | }) |
| 7434 | .Case(caseFn: [&](omp::TargetExitDataOp) { |
| 7435 | return TargetDirectiveEnumTy::TargetExitData; |
| 7436 | }) |
| 7437 | .Case(caseFn: [&](omp::TargetUpdateOp) { |
| 7438 | return TargetDirectiveEnumTy::TargetUpdate; |
| 7439 | }) |
| 7440 | .Case(caseFn: [&](omp::TargetOp) { return TargetDirectiveEnumTy::Target; }) |
| 7441 | .Default(defaultFn: [&](Operation *op) { return TargetDirectiveEnumTy::None; }); |
| 7442 | } |
| 7443 | } // namespace |
| 7444 | |
| 7445 | static uint64_t getArrayElementSizeInBits(LLVM::LLVMArrayType arrTy, |
| 7446 | DataLayout &dl) { |
| 7447 | if (auto nestedArrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>( |
| 7448 | Val: arrTy.getElementType())) |
| 7449 | return getArrayElementSizeInBits(arrTy: nestedArrTy, dl); |
| 7450 | return dl.getTypeSizeInBits(t: arrTy.getElementType()); |
| 7451 | } |
| 7452 | |
| 7453 | // The intent is to verify if the mapped data being passed is a |
| 7454 | // pointer -> pointee that requires special handling in certain cases, |
| 7455 | // e.g. applying the OMP_MAP_PTR_AND_OBJ map type. |
| 7456 | // |
| 7457 | // There may be a better way to verify this, but unfortunately with |
| 7458 | // opaque pointers we lose the ability to easily check if something is |
| 7459 | // a pointer whilst maintaining access to the underlying type. |
| 7460 | static bool checkIfPointerMap(omp::MapInfoOp mapOp) { |
| 7461 | // If we have a varPtrPtr field assigned then the underlying type is a pointer |
| 7462 | if (mapOp.getVarPtrPtr()) |
| 7463 | return true; |
| 7464 | |
| 7465 | // If the map data is declare target with a link clause, then it's represented |
| 7466 | // as a pointer when we lower it to LLVM-IR even if at the MLIR level it has |
| 7467 | // no relation to pointers. |
| 7468 | if (isDeclareTargetLink(value: mapOp.getVarPtr())) |
| 7469 | return true; |
| 7470 | |
| 7471 | return false; |
| 7472 | } |
| 7473 | |
| 7474 | // A privatizeable attach map is a pointer/descriptor that is privatized and |
| 7475 | // passed directly as a kernel argument (target_param) rather than undergoing |
| 7476 | // the standard attach/parent mapping. These are handled specially in a couple |
| 7477 | // of places in map lowering. |
| 7478 | static bool isPrivatizeableAttachMap(omp::ClauseMapFlags mapType) { |
| 7479 | return bitEnumContainsAll(bits: mapType, bit: omp::ClauseMapFlags::priv | |
| 7480 | omp::ClauseMapFlags::target_param | |
| 7481 | omp::ClauseMapFlags::attach); |
| 7482 | } |
| 7483 | |
| 7484 | // This function calculates the size to be offloaded for a specified type, given |
| 7485 | // its associated map clause (which can contain bounds information which affects |
| 7486 | // the total size), this size is calculated based on the underlying element type |
| 7487 | // e.g. given a 1-D array of ints, we will calculate the size from the integer |
| 7488 | // type * number of elements in the array. This size can be used in other |
| 7489 | // calculations but is ultimately used as an argument to the OpenMP runtimes |
| 7490 | // kernel argument structure which is generated through the combinedInfo data |
| 7491 | // structures. |
| 7492 | // This function is somewhat equivalent to Clang's getExprTypeSize inside of |
| 7493 | // CGOpenMPRuntime.cpp. |
| 7494 | static llvm::Value *getSizeInBytes(DataLayout &dl, const mlir::Type &type, |
| 7495 | Operation *clauseOp, |
| 7496 | llvm::Value *basePointer, |
| 7497 | llvm::Type *baseType, |
| 7498 | llvm::IRBuilderBase &builder, |
| 7499 | LLVM::ModuleTranslation &moduleTranslation) { |
| 7500 | if (auto memberClause = |
| 7501 | mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(Val: clauseOp)) { |
| 7502 | // This calculates the size to transfer based on bounds and the underlying |
| 7503 | // element type, provided bounds have been specified (Fortran |
| 7504 | // pointers/allocatables/target and arrays that have sections specified fall |
| 7505 | // into this as well) |
| 7506 | if (!memberClause.getBounds().empty()) { |
| 7507 | llvm::Value *elementCount = builder.getInt64(C: 1); |
| 7508 | for (auto bounds : memberClause.getBounds()) { |
| 7509 | if (auto boundOp = mlir::dyn_cast_if_present<mlir::omp::MapBoundsOp>( |
| 7510 | Val: bounds.getDefiningOp())) { |
| 7511 | // The below calculation for the size to be mapped calculated from the |
| 7512 | // map.info's bounds is: (elemCount * [UB - LB] + 1), later we |
| 7513 | // multiply by the underlying element types byte size to get the full |
| 7514 | // size to be offloaded based on the bounds |
| 7515 | elementCount = builder.CreateMul( |
| 7516 | LHS: elementCount, |
| 7517 | RHS: builder.CreateAdd( |
| 7518 | LHS: builder.CreateSub( |
| 7519 | LHS: moduleTranslation.lookupValue(value: boundOp.getUpperBound()), |
| 7520 | RHS: moduleTranslation.lookupValue(value: boundOp.getLowerBound())), |
| 7521 | RHS: builder.getInt64(C: 1))); |
| 7522 | } |
| 7523 | } |
| 7524 | |
| 7525 | // utilising getTypeSizeInBits instead of getTypeSize as getTypeSize gives |
| 7526 | // the size in inconsistent byte or bit format. |
| 7527 | uint64_t underlyingTypeSzInBits = dl.getTypeSizeInBits(t: type); |
| 7528 | if (auto arrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>(Val: type)) |
| 7529 | underlyingTypeSzInBits = getArrayElementSizeInBits(arrTy, dl); |
| 7530 | |
| 7531 | // The size in bytes x number of elements, the sizeInBytes stored is |
| 7532 | // the underyling types size, e.g. if ptr<i32>, it'll be the i32's |
| 7533 | // size, so we do some on the fly runtime math to get the size in |
| 7534 | // bytes from the extent (ub - lb) * sizeInBytes. NOTE: This may need |
| 7535 | // some adjustment for members with more complex types. |
| 7536 | llvm::Value *sizeCalc = builder.CreateMul( |
| 7537 | LHS: elementCount, RHS: builder.getInt64(C: underlyingTypeSzInBits / 8), |
| 7538 | Name: "element_count" ); |
| 7539 | |
| 7540 | // This is a part of a "complicated" bit of size calculation logic that is |
| 7541 | // in place to handle a couple of scenarios, one specific to Fortran and |
| 7542 | // the other a more general OpenMP issue. The other piece of the |
| 7543 | // calculation can be found as the final size calculation within the |
| 7544 | // processIndividualMap function. Ideally we would move it here, but due |
| 7545 | // to the complexity of calculating the final base address of some |
| 7546 | // constructs (required for a nullary check), it's left as the final step. |
| 7547 | // So, in the below 2 cases, the nullary check is in processIndividualMap |
| 7548 | // and the size equality check is here. The cases this modifications help |
| 7549 | // cover are: |
| 7550 | // |
| 7551 | // 1) If an argument has a null base pointer, then the size must be set to |
| 7552 | // 0 to avoid the runtime exploding/complaining about an illegal |
| 7553 | // pointer map. The size returning non-zero is feasible in certain |
| 7554 | // cases if for example someone has specified there own bounds/range. |
| 7555 | // 2) We wish to support a very specific OpenMP Fortran edge-case where a |
| 7556 | // size zero array can be legally presence checked and found to be on |
| 7557 | // device when it has been mapped. In these rare occasions the |
| 7558 | // allocatable/pointer will have a size of 1 allocated for the |
| 7559 | // underlying data, but this wall not be represented within the size of |
| 7560 | // the descriptor, so we get a non-nullary pointer and a size of 0, |
| 7561 | // allowing us to specify a size of 1 in these cases registering it on |
| 7562 | // the device mapping table as present. |
| 7563 | // |
| 7564 | // The default fall through case is just returning the size calculation |
| 7565 | // above, if we are not nullary and the size we calculate is non-zero, |
| 7566 | // which is basically any pointer type that is allocated in someway |
| 7567 | // (providing you are not running on a rare system that allows malloc's of |
| 7568 | // size 0 with whatever caveats that may come with). |
| 7569 | // |
| 7570 | // Later in the nullary check in processIndividualMap it just devolves to |
| 7571 | // selecting a size of 0 if we are nullary, if we are not, we will return |
| 7572 | // either 1 or the calculated size, depending on the outcome of this |
| 7573 | // select. |
| 7574 | if (checkIfPointerMap(mapOp: memberClause)) { |
| 7575 | return builder.CreateSelect( |
| 7576 | C: builder.CreateICmpEQ(LHS: sizeCalc, RHS: builder.getInt64(C: 0)), |
| 7577 | True: builder.getInt64(C: 1), False: sizeCalc); |
| 7578 | } |
| 7579 | |
| 7580 | return sizeCalc; |
| 7581 | } |
| 7582 | } |
| 7583 | |
| 7584 | return builder.getInt64(C: dl.getTypeSizeInBits(t: type) / 8); |
| 7585 | } |
| 7586 | |
| 7587 | // Convert the MLIR map flag set to the runtime map flag set for embedding |
| 7588 | // in LLVM-IR. This is important as the two bit-flag lists do not correspond |
| 7589 | // 1-to-1 as there's flags the runtime doesn't care about and vice versa. |
| 7590 | // Certain flags are discarded here such as RefPtee and co. |
| 7591 | static llvm::omp::OpenMPOffloadMappingFlags |
| 7592 | convertClauseMapFlags(omp::ClauseMapFlags mlirFlags) { |
| 7593 | const bool hasExplicitMap = |
| 7594 | (mlirFlags & ~omp::ClauseMapFlags::is_device_ptr) != |
| 7595 | omp::ClauseMapFlags::none; |
| 7596 | |
| 7597 | llvm::omp::OpenMPOffloadMappingFlags mapType = |
| 7598 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE; |
| 7599 | |
| 7600 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::to)) |
| 7601 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO; |
| 7602 | |
| 7603 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::from)) |
| 7604 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM; |
| 7605 | |
| 7606 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::always)) |
| 7607 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS; |
| 7608 | |
| 7609 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::del)) |
| 7610 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE; |
| 7611 | |
| 7612 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::return_param)) |
| 7613 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; |
| 7614 | |
| 7615 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::priv)) |
| 7616 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE; |
| 7617 | |
| 7618 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::literal)) |
| 7619 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL; |
| 7620 | |
| 7621 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::implicit)) |
| 7622 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT; |
| 7623 | |
| 7624 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::close)) |
| 7625 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE; |
| 7626 | |
| 7627 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::present)) |
| 7628 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRESENT; |
| 7629 | |
| 7630 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::ompx_hold)) |
| 7631 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD; |
| 7632 | |
| 7633 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::attach)) |
| 7634 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH; |
| 7635 | |
| 7636 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::target_param)) |
| 7637 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; |
| 7638 | |
| 7639 | if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::is_device_ptr)) { |
| 7640 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; |
| 7641 | if (!hasExplicitMap) |
| 7642 | mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL; |
| 7643 | } |
| 7644 | |
| 7645 | return mapType; |
| 7646 | } |
| 7647 | |
| 7648 | static StringRef getMapClauseName(omp::MapInfoOp mapOp) { |
| 7649 | if (StringAttr name = mapOp.getNameAttr()) |
| 7650 | return name.getValue(); |
| 7651 | return {}; |
| 7652 | } |
| 7653 | |
| 7654 | static void collectMapDataFromMapOperands( |
| 7655 | MapInfoData &mapData, SmallVectorImpl<Value> &mapVars, |
| 7656 | LLVM::ModuleTranslation &moduleTranslation, DataLayout &dl, |
| 7657 | llvm::IRBuilderBase &builder, ArrayRef<Value> useDevPtrOperands = {}, |
| 7658 | ArrayRef<Value> useDevAddrOperands = {}, |
| 7659 | ArrayRef<Value> hasDevAddrOperands = {}) { |
| 7660 | |
| 7661 | auto checkRefPtrOrPteeMapWithAttach = [](omp::ClauseMapFlags mapType) { |
| 7662 | bool hasRefType = |
| 7663 | bitEnumContainsAll(bits: mapType, bit: omp::ClauseMapFlags::ref_ptr) || |
| 7664 | bitEnumContainsAll(bits: mapType, bit: omp::ClauseMapFlags::ref_ptee); |
| 7665 | return hasRefType && |
| 7666 | bitEnumContainsAll(bits: mapType, bit: omp::ClauseMapFlags::attach); |
| 7667 | }; |
| 7668 | |
| 7669 | auto checkIsAMember = [](const auto &mapVars, auto mapOp) { |
| 7670 | // Check if this is a member mapping and correctly assign that it is, if |
| 7671 | // it is a member of a larger object. |
| 7672 | // TODO: Need better handling of members, and distinguishing of members |
| 7673 | // that are implicitly allocated on device vs explicitly passed in as |
| 7674 | // arguments. |
| 7675 | // TODO: May require some further additions to support nested record |
| 7676 | // types, i.e. member maps that can have member maps. |
| 7677 | for (Value mapValue : mapVars) { |
| 7678 | auto map = cast<omp::MapInfoOp>(Val: mapValue.getDefiningOp()); |
| 7679 | for (auto member : map.getMembers()) |
| 7680 | if (member == mapOp) |
| 7681 | return true; |
| 7682 | } |
| 7683 | return false; |
| 7684 | }; |
| 7685 | |
| 7686 | // Process MapOperands |
| 7687 | for (Value mapValue : mapVars) { |
| 7688 | auto mapOp = cast<omp::MapInfoOp>(Val: mapValue.getDefiningOp()); |
| 7689 | bool isAttachStyleMap = |
| 7690 | checkRefPtrOrPteeMapWithAttach(mapOp.getMapType()) || |
| 7691 | isPrivatizeableAttachMap(mapType: mapOp.getMapType()); |
| 7692 | Value offloadPtr = (mapOp.getVarPtrPtr() && !isAttachStyleMap) |
| 7693 | ? mapOp.getVarPtrPtr() |
| 7694 | : mapOp.getVarPtr(); |
| 7695 | mapData.OriginalValue.push_back(Elt: moduleTranslation.lookupValue(value: offloadPtr)); |
| 7696 | mapData.Pointers.push_back( |
| 7697 | Elt: isAttachStyleMap ? moduleTranslation.lookupValue(value: mapOp.getVarPtrPtr()) |
| 7698 | : mapData.OriginalValue.back()); |
| 7699 | |
| 7700 | if (llvm::Value *refPtr = |
| 7701 | getRefPtrIfDeclareTarget(value: offloadPtr, moduleTranslation)) { |
| 7702 | mapData.IsDeclareTarget.push_back(Elt: true); |
| 7703 | mapData.BasePointers.push_back(Elt: refPtr); |
| 7704 | } else if (isDeclareTargetTo(value: offloadPtr)) { |
| 7705 | mapData.IsDeclareTarget.push_back(Elt: true); |
| 7706 | mapData.BasePointers.push_back(Elt: mapData.OriginalValue.back()); |
| 7707 | } else { // regular mapped variable |
| 7708 | mapData.IsDeclareTarget.push_back(Elt: false); |
| 7709 | mapData.BasePointers.push_back(Elt: mapData.OriginalValue.back()); |
| 7710 | } |
| 7711 | |
| 7712 | // In every situation we currently have if we have a varPtrPtr present |
| 7713 | // we wish to utilise it's type for the base type, main cases are |
| 7714 | // currently Fortran descriptor base address maps and attach maps. |
| 7715 | mapData.BaseType.push_back(Elt: moduleTranslation.convertType( |
| 7716 | type: mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtrType().value() |
| 7717 | : mapOp.getVarPtrType())); |
| 7718 | |
| 7719 | // For the attach map cases, it's a little odd, as we effectively have to |
| 7720 | // utilise the base address (including all bounds offsets) for the pointer |
| 7721 | // field, the pointer address for the base address field, and the pointer |
| 7722 | // not the data (base addresses) size. So we end up with a mix of base |
| 7723 | // types and sizes we wish to insert here. |
| 7724 | mlir::Type sizeType = (isAttachStyleMap || !mapOp.getVarPtrPtr()) |
| 7725 | ? mapOp.getVarPtrType() |
| 7726 | : mapOp.getVarPtrPtrType().value(); |
| 7727 | mapData.Sizes.push_back(Elt: getSizeInBytes( |
| 7728 | dl, type: sizeType, clauseOp: isAttachStyleMap ? nullptr : mapOp, |
| 7729 | basePointer: mapData.Pointers.back(), baseType: moduleTranslation.convertType(type: sizeType), |
| 7730 | builder, moduleTranslation)); |
| 7731 | mapData.MapClause.push_back(Elt: mapOp.getOperation()); |
| 7732 | mapData.Types.push_back(Elt: convertClauseMapFlags(mlirFlags: mapOp.getMapType())); |
| 7733 | // TODO: set HasAttachPtr from Flang for pointee-storage entries. |
| 7734 | mapData.HasAttachPtr.push_back(Elt: false); |
| 7735 | mapData.Names.push_back(Elt: LLVM::createMappingInformation( |
| 7736 | loc: mapOp.getLoc(), builder&: *getOpenMPBuilder(moduleTranslation), |
| 7737 | mapName: getMapClauseName(mapOp))); |
| 7738 | mapData.DevicePointers.push_back(Elt: llvm::OpenMPIRBuilder::DeviceInfoTy::None); |
| 7739 | if (mapOp.getMapperId()) |
| 7740 | mapData.Mappers.push_back( |
| 7741 | Elt: SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>( |
| 7742 | from: mapOp, symbol: mapOp.getMapperIdAttr())); |
| 7743 | else |
| 7744 | mapData.Mappers.push_back(Elt: nullptr); |
| 7745 | mapData.IsAMapping.push_back(Elt: true); |
| 7746 | mapData.IsAMember.push_back(Elt: checkIsAMember(mapVars, mapOp)); |
| 7747 | } |
| 7748 | |
| 7749 | auto findMapInfo = [&mapData](llvm::Value *val, |
| 7750 | llvm::OpenMPIRBuilder::DeviceInfoTy devInfoTy, |
| 7751 | size_t memberCount) { |
| 7752 | unsigned index = 0; |
| 7753 | bool found = false; |
| 7754 | for (llvm::Value *basePtr : mapData.OriginalValue) { |
| 7755 | auto mapOp = cast<omp::MapInfoOp>(Val: mapData.MapClause[index]); |
| 7756 | // TODO: Currently we define an equivalent mapping as |
| 7757 | // the same base pointer and an equivalent member count, but |
| 7758 | // that is a loose definition. We may have to extend to check |
| 7759 | // for other fields (varPtrPtr/individual members being mapped). |
| 7760 | // Note: Attach maps are not the same as a normal data transfer |
| 7761 | // they specify to the runtime to perform an attach map and they |
| 7762 | // (at least at the moment) are never something we would aim to |
| 7763 | // return in a use_dev_* clause, so they are skipped in terms of |
| 7764 | // duplicate maps. |
| 7765 | bool isAttachMap = |
| 7766 | (mapData.Types[index] & |
| 7767 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH) == |
| 7768 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH; |
| 7769 | if (!isAttachMap && basePtr == val && mapData.IsAMapping[index] && |
| 7770 | memberCount == mapOp.getMembers().size()) { |
| 7771 | found = true; |
| 7772 | mapData.Types[index] |= |
| 7773 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; |
| 7774 | mapData.DevicePointers[index] = devInfoTy; |
| 7775 | } |
| 7776 | index++; |
| 7777 | } |
| 7778 | return found; |
| 7779 | }; |
| 7780 | |
| 7781 | // Process useDevPtr(Addr)Operands |
| 7782 | auto addDevInfos = [&](const llvm::ArrayRef<Value> &useDevOperands, |
| 7783 | llvm::OpenMPIRBuilder::DeviceInfoTy devInfoTy) { |
| 7784 | for (Value mapValue : useDevOperands) { |
| 7785 | auto mapOp = cast<omp::MapInfoOp>(Val: mapValue.getDefiningOp()); |
| 7786 | Value offloadPtr = |
| 7787 | mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr() : mapOp.getVarPtr(); |
| 7788 | llvm::Value *origValue = moduleTranslation.lookupValue(value: offloadPtr); |
| 7789 | |
| 7790 | // Check if map info is already present for this entry. |
| 7791 | if (!findMapInfo(origValue, devInfoTy, mapOp.getMembers().size())) { |
| 7792 | mapData.OriginalValue.push_back(Elt: origValue); |
| 7793 | mapData.Pointers.push_back(Elt: mapData.OriginalValue.back()); |
| 7794 | mapData.IsDeclareTarget.push_back(Elt: false); |
| 7795 | mapData.BasePointers.push_back(Elt: mapData.OriginalValue.back()); |
| 7796 | mlir::Type baseTy = mapOp.getVarPtrPtr() |
| 7797 | ? mapOp.getVarPtrPtrType().value() |
| 7798 | : mapOp.getVarPtrType(); |
| 7799 | mapData.BaseType.push_back(Elt: moduleTranslation.convertType(type: baseTy)); |
| 7800 | mapData.Sizes.push_back(Elt: builder.getInt64(C: 0)); |
| 7801 | mapData.MapClause.push_back(Elt: mapOp.getOperation()); |
| 7802 | mapData.Types.push_back( |
| 7803 | Elt: llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM); |
| 7804 | // TODO: set HasAttachPtr from Flang for pointee-storage entries. |
| 7805 | mapData.HasAttachPtr.push_back(Elt: false); |
| 7806 | mapData.Names.push_back(Elt: LLVM::createMappingInformation( |
| 7807 | loc: mapOp.getLoc(), builder&: *getOpenMPBuilder(moduleTranslation), |
| 7808 | mapName: getMapClauseName(mapOp))); |
| 7809 | mapData.DevicePointers.push_back(Elt: devInfoTy); |
| 7810 | mapData.Mappers.push_back(Elt: nullptr); |
| 7811 | mapData.IsAMapping.push_back(Elt: false); |
| 7812 | mapData.IsAMember.push_back(Elt: checkIsAMember(useDevOperands, mapOp)); |
| 7813 | } |
| 7814 | } |
| 7815 | }; |
| 7816 | |
| 7817 | addDevInfos(useDevAddrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Address); |
| 7818 | addDevInfos(useDevPtrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer); |
| 7819 | |
| 7820 | for (Value mapValue : hasDevAddrOperands) { |
| 7821 | auto mapOp = cast<omp::MapInfoOp>(Val: mapValue.getDefiningOp()); |
| 7822 | Value offloadPtr = |
| 7823 | mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr() : mapOp.getVarPtr(); |
| 7824 | llvm::Value *origValue = moduleTranslation.lookupValue(value: offloadPtr); |
| 7825 | auto mapType = convertClauseMapFlags(mlirFlags: mapOp.getMapType()); |
| 7826 | auto mapTypeAlways = llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS; |
| 7827 | bool isDevicePtr = |
| 7828 | (mapOp.getMapType() & omp::ClauseMapFlags::is_device_ptr) != |
| 7829 | omp::ClauseMapFlags::none; |
| 7830 | |
| 7831 | mapData.OriginalValue.push_back(Elt: origValue); |
| 7832 | mapData.BasePointers.push_back(Elt: origValue); |
| 7833 | mapData.Pointers.push_back(Elt: origValue); |
| 7834 | mapData.IsDeclareTarget.push_back(Elt: false); |
| 7835 | |
| 7836 | mlir::Type baseTy = mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtrType().value() |
| 7837 | : mapOp.getVarPtrType(); |
| 7838 | mapData.BaseType.push_back(Elt: moduleTranslation.convertType(type: baseTy)); |
| 7839 | mapData.Sizes.push_back(Elt: builder.getInt64(C: dl.getTypeSize(t: baseTy))); |
| 7840 | |
| 7841 | mapData.MapClause.push_back(Elt: mapOp.getOperation()); |
| 7842 | if (llvm::to_underlying(E: mapType & mapTypeAlways)) { |
| 7843 | // Descriptors are mapped with the ALWAYS flag, since they can get |
| 7844 | // rematerialized, so the address of the decriptor for a given object |
| 7845 | // may change from one place to another. |
| 7846 | mapData.Types.push_back(Elt: mapType); |
| 7847 | // TODO: set HasAttachPtr from Flang for pointee-storage entries. |
| 7848 | mapData.HasAttachPtr.push_back(Elt: false); |
| 7849 | // Technically it's possible for a non-descriptor mapping to have |
| 7850 | // both has-device-addr and ALWAYS, so lookup the mapper in case it |
| 7851 | // exists. |
| 7852 | if (mapOp.getMapperId()) { |
| 7853 | mapData.Mappers.push_back( |
| 7854 | Elt: SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>( |
| 7855 | from: mapOp, symbol: mapOp.getMapperIdAttr())); |
| 7856 | } else { |
| 7857 | mapData.Mappers.push_back(Elt: nullptr); |
| 7858 | } |
| 7859 | } else { |
| 7860 | // For is_device_ptr we need the map type to propagate so the runtime |
| 7861 | // can materialize the device-side copy of the pointer container. |
| 7862 | mapData.Types.push_back( |
| 7863 | Elt: isDevicePtr ? mapType |
| 7864 | : llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL); |
| 7865 | // TODO: set HasAttachPtr from Flang for pointee-storage entries. |
| 7866 | mapData.HasAttachPtr.push_back(Elt: false); |
| 7867 | mapData.Mappers.push_back(Elt: nullptr); |
| 7868 | } |
| 7869 | mapData.Names.push_back(Elt: LLVM::createMappingInformation( |
| 7870 | loc: mapOp.getLoc(), builder&: *getOpenMPBuilder(moduleTranslation), |
| 7871 | mapName: getMapClauseName(mapOp))); |
| 7872 | mapData.DevicePointers.push_back( |
| 7873 | Elt: isDevicePtr ? llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer |
| 7874 | : llvm::OpenMPIRBuilder::DeviceInfoTy::Address); |
| 7875 | mapData.IsAMapping.push_back(Elt: false); |
| 7876 | mapData.IsAMember.push_back(Elt: checkIsAMember(hasDevAddrOperands, mapOp)); |
| 7877 | } |
| 7878 | } |
| 7879 | |
| 7880 | static int getMapDataMemberIdx(MapInfoData &mapData, omp::MapInfoOp memberOp) { |
| 7881 | auto *res = llvm::find(Range&: mapData.MapClause, Val: memberOp); |
| 7882 | assert(res != mapData.MapClause.end() && |
| 7883 | "MapInfoOp for member not found in MapData, cannot return index" ); |
| 7884 | return std::distance(first: mapData.MapClause.begin(), last: res); |
| 7885 | } |
| 7886 | |
| 7887 | static void sortMapIndices(llvm::SmallVectorImpl<size_t> &indices, |
| 7888 | omp::MapInfoOp mapInfo, bool first = true) { |
| 7889 | ArrayAttr indexAttr = mapInfo.getMembersIndexAttr(); |
| 7890 | llvm::SmallVector<size_t> occludedChildren; |
| 7891 | llvm::sort( |
| 7892 | Start: indices.begin(), End: indices.end(), Comp: [&](const size_t a, const size_t b) { |
| 7893 | // Bail early if we are asked to look at the same index. If we do not |
| 7894 | // bail early, we can end up mistakenly adding indices to |
| 7895 | // occludedChildren. This can occur with some types of libc++ hardening. |
| 7896 | if (a == b) |
| 7897 | return false; |
| 7898 | |
| 7899 | auto memberIndicesA = cast<ArrayAttr>(Val: indexAttr[a]); |
| 7900 | auto memberIndicesB = cast<ArrayAttr>(Val: indexAttr[b]); |
| 7901 | |
| 7902 | for (auto it : llvm::zip(t&: memberIndicesA, u&: memberIndicesB)) { |
| 7903 | int64_t aIndex = mlir::cast<IntegerAttr>(Val: std::get<0>(t&: it)).getInt(); |
| 7904 | int64_t bIndex = mlir::cast<IntegerAttr>(Val: std::get<1>(t&: it)).getInt(); |
| 7905 | |
| 7906 | if (aIndex == bIndex) |
| 7907 | continue; |
| 7908 | |
| 7909 | if (aIndex < bIndex) |
| 7910 | return first; |
| 7911 | |
| 7912 | if (aIndex > bIndex) |
| 7913 | return !first; |
| 7914 | } |
| 7915 | |
| 7916 | // Iterated up until the end of the smallest member and |
| 7917 | // they were found to be equal up to that point, so select |
| 7918 | // the member with the lowest index count, so the "parent" |
| 7919 | bool memberAParent = memberIndicesA.size() < memberIndicesB.size(); |
| 7920 | if (memberAParent) |
| 7921 | occludedChildren.push_back(Elt: b); |
| 7922 | else |
| 7923 | occludedChildren.push_back(Elt: a); |
| 7924 | return memberAParent; |
| 7925 | }); |
| 7926 | |
| 7927 | for (auto v : occludedChildren) |
| 7928 | indices.erase(CS: std::remove(first: indices.begin(), last: indices.end(), value: v), |
| 7929 | CE: indices.end()); |
| 7930 | } |
| 7931 | |
| 7932 | static omp::MapInfoOp getFirstOrLastMappedMemberPtr(omp::MapInfoOp mapInfo, |
| 7933 | bool first) { |
| 7934 | ArrayAttr indexAttr = mapInfo.getMembersIndexAttr(); |
| 7935 | // Only 1 member has been mapped, we can return it. |
| 7936 | if (indexAttr.size() == 1) |
| 7937 | return cast<omp::MapInfoOp>(Val: mapInfo.getMembers()[0].getDefiningOp()); |
| 7938 | llvm::SmallVector<size_t> indices(indexAttr.size()); |
| 7939 | std::iota(first: indices.begin(), last: indices.end(), value: 0); |
| 7940 | sortMapIndices(indices, mapInfo, first); |
| 7941 | return llvm::cast<omp::MapInfoOp>( |
| 7942 | Val: mapInfo.getMembers()[indices.front()].getDefiningOp()); |
| 7943 | } |
| 7944 | |
| 7945 | /// This function calculates the array/pointer offset for map data provided |
| 7946 | /// with bounds operations, e.g. when provided something like the following: |
| 7947 | /// |
| 7948 | /// Fortran |
| 7949 | /// map(tofrom: array(2:5, 3:2)) |
| 7950 | /// |
| 7951 | /// We must calculate the initial pointer offset to pass across, this function |
| 7952 | /// performs this using bounds. |
| 7953 | /// |
| 7954 | /// TODO/WARNING: This only supports Fortran's column major indexing currently |
| 7955 | /// as is noted in the note below and comments in the function, we must extend |
| 7956 | /// this function when we add a C++ frontend. |
| 7957 | /// NOTE: which while specified in row-major order it currently needs to be |
| 7958 | /// flipped for Fortran's column order array allocation and access (as |
| 7959 | /// opposed to C++'s row-major, hence the backwards processing where order is |
| 7960 | /// important). This is likely important to keep in mind for the future when |
| 7961 | /// we incorporate a C++ frontend, both frontends will need to agree on the |
| 7962 | /// ordering of generated bounds operations (one may have to flip them) to |
| 7963 | /// make the below lowering frontend agnostic. The offload size |
| 7964 | /// calcualtion may also have to be adjusted for C++. |
| 7965 | static std::vector<llvm::Value *> |
| 7966 | calculateBoundsOffset(LLVM::ModuleTranslation &moduleTranslation, |
| 7967 | llvm::IRBuilderBase &builder, bool isArrayTy, |
| 7968 | OperandRange bounds, bool &isByteOffset) { |
| 7969 | std::vector<llvm::Value *> idx; |
| 7970 | isByteOffset = false; |
| 7971 | // There's no bounds to calculate an offset from, we can safely |
| 7972 | // ignore and return no indices. |
| 7973 | if (bounds.empty()) |
| 7974 | return idx; |
| 7975 | |
| 7976 | // If we have an array type, then we have its type so can treat it as a |
| 7977 | // normal GEP instruction where the bounds operations are simply indexes |
| 7978 | // into the array. We currently do reverse order of the bounds, which |
| 7979 | // I believe leans more towards Fortran's column-major in memory. |
| 7980 | if (isArrayTy) { |
| 7981 | idx.push_back(x: builder.getInt64(C: 0)); |
| 7982 | for (int i = bounds.size() - 1; i >= 0; --i) { |
| 7983 | if (auto boundOp = dyn_cast_if_present<omp::MapBoundsOp>( |
| 7984 | Val: bounds[i].getDefiningOp())) { |
| 7985 | idx.push_back(x: moduleTranslation.lookupValue(value: boundOp.getLowerBound())); |
| 7986 | } |
| 7987 | } |
| 7988 | } else { |
| 7989 | // Check if Fortran descriptors provide strides in bytes |
| 7990 | llvm::Value *offset = builder.getInt64(C: 0); |
| 7991 | for (Value bound : bounds) { |
| 7992 | auto boundOp = |
| 7993 | dyn_cast_if_present<omp::MapBoundsOp>(Val: bound.getDefiningOp()); |
| 7994 | if (!boundOp || !boundOp.getStride()) |
| 7995 | continue; |
| 7996 | // Fortran descriptors provide strides in bytes; when that's the case the |
| 7997 | // accumulated offset is a byte offset and must be consumed with a byte |
| 7998 | // (i8) GEP at the call site to avoid double-scaling by the element size. |
| 7999 | isByteOffset = boundOp.getStrideInBytes(); |
| 8000 | } |
| 8001 | if (isByteOffset) { |
| 8002 | for (Value bound : bounds) { |
| 8003 | auto boundOp = |
| 8004 | dyn_cast_if_present<omp::MapBoundsOp>(Val: bound.getDefiningOp()); |
| 8005 | if (!boundOp || !boundOp.getStride()) |
| 8006 | continue; |
| 8007 | // If we do not have an array type, but we have bounds, then we're |
| 8008 | // dealing with a pointer that's being treated like an array |
| 8009 | // and we have the underlying type e.g. an i32, or f64 etc, |
| 8010 | // e.g. a fortran descriptor base address (pointer pointing to |
| 8011 | // the actual data) and we know that the offset stride is expressed |
| 8012 | // in bytes, so we must calculate the offset to the first mapped |
| 8013 | // element as a single linear index into that data. |
| 8014 | // |
| 8015 | // Each bound carries the stride between consecutive elements along its |
| 8016 | // dimension, so the linear offset to the section's origin is: |
| 8017 | // |
| 8018 | // offset += lower_bound[d] * stride[d] |
| 8019 | llvm::Value *lb = |
| 8020 | moduleTranslation.lookupValue(value: boundOp.getLowerBound()); |
| 8021 | llvm::Value *stride = |
| 8022 | moduleTranslation.lookupValue(value: boundOp.getStride()); |
| 8023 | offset = builder.CreateAdd(LHS: offset, RHS: builder.CreateMul(LHS: lb, RHS: stride)); |
| 8024 | } |
| 8025 | idx.push_back(x: offset); |
| 8026 | } else { |
| 8027 | for (int i = bounds.size() - 1; i >= 0; --i) { |
| 8028 | if (auto boundOp = dyn_cast_if_present<omp::MapBoundsOp>( |
| 8029 | Val: bounds[i].getDefiningOp())) { |
| 8030 | // If the stride in bytes is unknown, so we must calculate the |
| 8031 | // offset using a single index which the following loop attempts to |
| 8032 | // compute using the standard column-major algorithm e.g for a 3D |
| 8033 | // array: |
| 8034 | // |
| 8035 | // ((((c_idx * b_len) + b_idx) * a_len) + a_idx) |
| 8036 | // |
| 8037 | // It is of note that it's doing column-major rather than row-major at |
| 8038 | // the moment, but having a way for the frontend to indicate which |
| 8039 | // major format to use or standardizing/canonicalizing the order of |
| 8040 | // the bounds to compute the offset may be useful in the future when |
| 8041 | // there's other frontends with different formats. |
| 8042 | if (i == ((int)bounds.size() - 1)) |
| 8043 | idx.emplace_back( |
| 8044 | args: moduleTranslation.lookupValue(value: boundOp.getLowerBound())); |
| 8045 | else |
| 8046 | idx.back() = builder.CreateAdd( |
| 8047 | LHS: builder.CreateMul(LHS: idx.back(), RHS: moduleTranslation.lookupValue( |
| 8048 | value: boundOp.getExtent())), |
| 8049 | RHS: moduleTranslation.lookupValue(value: boundOp.getLowerBound())); |
| 8050 | } |
| 8051 | } |
| 8052 | } |
| 8053 | } |
| 8054 | |
| 8055 | return idx; |
| 8056 | } |
| 8057 | |
| 8058 | static void getAsIntegers(ArrayAttr values, llvm::SmallVector<int64_t> &ints) { |
| 8059 | llvm::transform(Range&: values, d_first: std::back_inserter(x&: ints), F: [](Attribute value) { |
| 8060 | return cast<IntegerAttr>(Val&: value).getInt(); |
| 8061 | }); |
| 8062 | } |
| 8063 | |
| 8064 | // Gathers members that are overlapping in the parent, excluding members that |
| 8065 | // themselves overlap, keeping the top-most (closest to parents level) map. |
| 8066 | static void |
| 8067 | getOverlappedMembers(llvm::SmallVectorImpl<size_t> &overlapMapDataIdxs, |
| 8068 | omp::MapInfoOp parentOp) { |
| 8069 | // No members mapped, no overlaps. |
| 8070 | if (parentOp.getMembers().empty()) |
| 8071 | return; |
| 8072 | |
| 8073 | // Single member, we can insert and return early. |
| 8074 | if (parentOp.getMembers().size() == 1) { |
| 8075 | overlapMapDataIdxs.push_back(Elt: 0); |
| 8076 | return; |
| 8077 | } |
| 8078 | |
| 8079 | ArrayAttr indexAttr = parentOp.getMembersIndexAttr(); |
| 8080 | size_t numMembers = indexAttr.size(); |
| 8081 | |
| 8082 | // Pre-convert all member indices to integer arrays for efficient comparison. |
| 8083 | llvm::SmallVector<llvm::SmallVector<int64_t>> memberIndices(numMembers); |
| 8084 | for (auto [i, indicesAttr] : llvm::enumerate(First&: indexAttr)) |
| 8085 | getAsIntegers(values: cast<ArrayAttr>(Val: indicesAttr), ints&: memberIndices[i]); |
| 8086 | |
| 8087 | // For each member, check if it's superseded by another (shorter prefix) |
| 8088 | // member. If member j's indices are a prefix of member i's indices, then |
| 8089 | // i is a child of j and should be skipped. e.g. if member [0] is mapped, |
| 8090 | // we skip members [0,1], [0,2], etc. |
| 8091 | llvm::SmallDenseSet<size_t> skipIndices; |
| 8092 | for (size_t i = 0; i < numMembers; ++i) { |
| 8093 | const auto &iIndices = memberIndices[i]; |
| 8094 | for (size_t j = 0; j < numMembers; ++j) { |
| 8095 | if (i == j) |
| 8096 | continue; |
| 8097 | const auto &jIndices = memberIndices[j]; |
| 8098 | // If j's indices are a strict prefix of i's indices, skip i |
| 8099 | if (jIndices.size() < iIndices.size() && |
| 8100 | std::equal(first1: jIndices.begin(), last1: jIndices.end(), first2: iIndices.begin())) { |
| 8101 | skipIndices.insert(V: i); |
| 8102 | break; // No need to check other potential parents |
| 8103 | } |
| 8104 | } |
| 8105 | } |
| 8106 | |
| 8107 | // Collect indices of members that are not superseded by a parent. |
| 8108 | for (size_t i = 0; i < numMembers; ++i) |
| 8109 | if (!skipIndices.contains(V: i)) |
| 8110 | overlapMapDataIdxs.push_back(Elt: i); |
| 8111 | } |
| 8112 | |
| 8113 | /// Describe a bounded update as a set of contiguous pieces. Bounds follow the |
| 8114 | /// dialect's normalized fastest-to-slowest varying order, which matches |
| 8115 | /// Fortran's column-major dimension order. Frontends for row-major languages |
| 8116 | /// such as C/C++ must normalize their source dimensions to this order. |
| 8117 | /// OpenMPIRBuilder reverses the vectors when emitting the runtime descriptor, |
| 8118 | /// which uses byte offsets and strides with the slowest varying dimension |
| 8119 | /// first. |
| 8120 | static void addNonContiguousUpdate(LLVM::ModuleTranslation &moduleTranslation, |
| 8121 | llvm::IRBuilderBase &builder, |
| 8122 | MapInfoData &mapData, size_t mapDataIdx, |
| 8123 | MapInfosTy &combinedInfo) { |
| 8124 | auto mapOp = cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIdx]); |
| 8125 | if (mapOp.getBounds().empty() || mapOp.getMapperId() || |
| 8126 | mapOp->getParentOfType<omp::DeclareMapperOp>() || |
| 8127 | bitEnumContainsAll(bits: mapOp.getMapType(), bit: omp::ClauseMapFlags::attach)) |
| 8128 | return; |
| 8129 | |
| 8130 | llvm::Type *elementType = mapData.BaseType[mapDataIdx]; |
| 8131 | // Bounds extents may be absent, and a scalar subscript has extent one. |
| 8132 | // Preserve the full source shape: for a(4,5), a(2,:) still needs a |
| 8133 | // second-dimension stride of four elements. |
| 8134 | SmallVector<uint64_t> arrayExtents; |
| 8135 | while (auto *arrayType = dyn_cast<llvm::ArrayType>(Val: elementType)) { |
| 8136 | arrayExtents.push_back(Elt: arrayType->getNumElements()); |
| 8137 | elementType = arrayType->getElementType(); |
| 8138 | } |
| 8139 | llvm::Value *elementSize = builder.getInt64( |
| 8140 | C: builder.GetInsertBlock()->getModule()->getDataLayout().getTypeAllocSize( |
| 8141 | Ty: elementType)); |
| 8142 | |
| 8143 | // Include a byte dimension so that a non-unit descriptor stride in the |
| 8144 | // innermost array dimension still transfers one element at a time. |
| 8145 | llvm::OpenMPIRBuilder::MapValuesArrayTy offsets{builder.getInt64(C: 0)}; |
| 8146 | llvm::OpenMPIRBuilder::MapValuesArrayTy counts{elementSize}; |
| 8147 | llvm::OpenMPIRBuilder::MapValuesArrayTy strides{builder.getInt64(C: 1)}; |
| 8148 | llvm::Value *sourceStride = elementSize; |
| 8149 | llvm::Value *contiguousSize = elementSize; |
| 8150 | // Unknown descriptor strides conservatively use a descriptor. The runtime |
| 8151 | // merges contiguous dimensions from their actual values. |
| 8152 | bool isContiguous = true; |
| 8153 | for (auto [dim, bound] : llvm::enumerate(First: mapOp.getBounds())) { |
| 8154 | auto bounds = cast<omp::MapBoundsOp>(Val: bound.getDefiningOp()); |
| 8155 | auto getInt64 = [&](Value value) { |
| 8156 | return builder.CreateSExtOrTrunc(V: moduleTranslation.lookupValue(value), |
| 8157 | DestTy: builder.getInt64Ty()); |
| 8158 | }; |
| 8159 | llvm::Value *lower = bounds.getLowerBound() |
| 8160 | ? getInt64(bounds.getLowerBound()) |
| 8161 | : builder.getInt64(C: 0); |
| 8162 | llvm::Value *count = builder.CreateAdd( |
| 8163 | LHS: builder.CreateSub(LHS: getInt64(bounds.getUpperBound()), RHS: lower), |
| 8164 | RHS: builder.getInt64(C: 1)); |
| 8165 | count = |
| 8166 | builder.CreateSelect(C: builder.CreateICmpSGT(LHS: count, RHS: builder.getInt64(C: 0)), |
| 8167 | True: count, False: builder.getInt64(C: 0)); |
| 8168 | llvm::Value *stride = sourceStride; |
| 8169 | if (bounds.getStrideInBytes() && bounds.getStride()) |
| 8170 | stride = getInt64(bounds.getStride()); |
| 8171 | else if (arrayExtents.empty() && bounds.getStride()) |
| 8172 | stride = builder.CreateMul(LHS: getInt64(bounds.getStride()), RHS: elementSize); |
| 8173 | offsets.push_back(Elt: builder.CreateMul(LHS: lower, RHS: stride)); |
| 8174 | counts.push_back(Elt: count); |
| 8175 | strides.push_back(Elt: stride); |
| 8176 | isContiguous &= stride == contiguousSize; |
| 8177 | contiguousSize = builder.CreateMul(LHS: contiguousSize, RHS: count); |
| 8178 | |
| 8179 | // Static array types carry the full source shape, including dimensions |
| 8180 | // selected with scalar subscripts. Otherwise the frontend supplies the |
| 8181 | // physical stride in elements (or descriptor bytes) for every dimension. |
| 8182 | if (dim < arrayExtents.size()) |
| 8183 | sourceStride = builder.CreateMul( |
| 8184 | LHS: sourceStride, |
| 8185 | RHS: builder.getInt64(C: arrayExtents[arrayExtents.size() - dim - 1])); |
| 8186 | else if (bounds.getExtent()) |
| 8187 | sourceStride = |
| 8188 | builder.CreateMul(LHS: sourceStride, RHS: getInt64(bounds.getExtent())); |
| 8189 | } |
| 8190 | if (isContiguous) |
| 8191 | return; |
| 8192 | |
| 8193 | llvm::Value *base = mapData.OriginalValue[mapDataIdx]; |
| 8194 | if (mapOp.getVarPtrPtr()) |
| 8195 | base = builder.CreateLoad(Ty: builder.getPtrTy(), Ptr: base); |
| 8196 | // Absent optional arrays and unallocated descriptors must not issue |
| 8197 | // transfers, including sections with explicit nonempty bounds. |
| 8198 | counts[1] = builder.CreateSelect(C: builder.CreateIsNull(Arg: base), |
| 8199 | True: builder.getInt64(C: 0), False: counts[1]); |
| 8200 | // The descriptor offsets already locate the first selected element. The |
| 8201 | // runtime needs the pointee base, not the descriptor or a section address. |
| 8202 | combinedInfo.BasePointers.back() = base; |
| 8203 | combinedInfo.Types.back() |= |
| 8204 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG; |
| 8205 | auto &nonContig = combinedInfo.NonContigInfo; |
| 8206 | nonContig.IsNonContiguous = true; |
| 8207 | nonContig.Dims.resize(N: combinedInfo.Types.size(), NV: 1); |
| 8208 | nonContig.Dims.back() = counts.size(); |
| 8209 | nonContig.Offsets.push_back(Elt: std::move(offsets)); |
| 8210 | nonContig.Counts.push_back(Elt: std::move(counts)); |
| 8211 | nonContig.Strides.push_back(Elt: std::move(strides)); |
| 8212 | } |
| 8213 | |
| 8214 | /// This function handles the insertion of a single item of map data from |
| 8215 | /// MapInfoData into the OMPIRBuilder's MapInfo list. Utilising this function |
| 8216 | /// means the map being inserted can be treated as a non-parent map entity, |
| 8217 | /// if the memberOfFlag is set then the map being inserted is treated as |
| 8218 | /// a member map of a larger entity. The insertion into the MapInfo list of |
| 8219 | /// the OMPIRBuilder can vary based on a number of factors, such as if it's |
| 8220 | /// a ref_ptr or ref_ptee map, if it's a member of a record, what construct |
| 8221 | /// the map belongs to and the various map type bit flags that are set for |
| 8222 | /// the map. |
| 8223 | static void processIndividualMap( |
| 8224 | LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, |
| 8225 | llvm::OpenMPIRBuilder &ompBuilder, MapInfoData &mapData, size_t mapDataIdx, |
| 8226 | MapInfosTy &combinedInfo, TargetDirectiveEnumTy targetDirective, |
| 8227 | llvm::omp::OpenMPOffloadMappingFlags memberOfFlag = |
| 8228 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE, |
| 8229 | bool isTargetParam = true, int mapDataParentIdx = -1) { |
| 8230 | auto mapFlag = mapData.Types[mapDataIdx]; |
| 8231 | auto mapInfoOp = llvm::cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIdx]); |
| 8232 | |
| 8233 | bool isPtrTy = checkIfPointerMap(mapOp: mapInfoOp); |
| 8234 | bool isAttachMap = ((convertClauseMapFlags(mlirFlags: mapInfoOp.getMapType()) & |
| 8235 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH) == |
| 8236 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH); |
| 8237 | |
| 8238 | // Declare target variables are not passed to the kernel, and for the moment |
| 8239 | // attach maps are not passed to the kernel. However, it is possible to create |
| 8240 | // attach maps that transfer data and thus can be kernel arguments, but our |
| 8241 | // existing frontend does not do this. |
| 8242 | if (isTargetParam && |
| 8243 | (targetDirective == TargetDirectiveEnumTy::Target && |
| 8244 | !mapData.IsDeclareTarget[mapDataIdx]) && |
| 8245 | !isAttachMap) |
| 8246 | mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM; |
| 8247 | |
| 8248 | if (mapInfoOp.getMapCaptureType() == omp::VariableCaptureKind::ByCopy && |
| 8249 | !isPtrTy) |
| 8250 | mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL; |
| 8251 | |
| 8252 | // If we have a pointer and it's part of a MEMBER_OF mapping we do not apply |
| 8253 | // MEMBER_OF, as the runtime currently has a work-around that utilises |
| 8254 | // MEMBER_OF to prevent reference updating in certain scenarios instead of |
| 8255 | // target_param. However, this causes a noticeable issue in cases where we |
| 8256 | // map some data (Fortran descriptor primarily at the moment), alter it on |
| 8257 | // the host, and then expect it to not be updated in a subsequent implicit map |
| 8258 | // (such as an implicit map on a target). |
| 8259 | if (memberOfFlag != llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE) { |
| 8260 | if (!isPtrTy && !isAttachMap) |
| 8261 | ompBuilder.setCorrectMemberOfFlag(Flags&: mapFlag, MemberOfFlag: memberOfFlag); |
| 8262 | |
| 8263 | // The return parameter should be the over-riding parent in cases where we |
| 8264 | // have a return parameter that is echoed to all members, the main case of |
| 8265 | // this currently is with fortran descriptors. It may need more finessing |
| 8266 | // for C/C++ in the future or descriptors that are members of derived |
| 8267 | // types. |
| 8268 | mapFlag &= ~llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; |
| 8269 | } |
| 8270 | |
| 8271 | // We apply MAP_PTR_AND_OBJ when within a declare mapper object as it enforces |
| 8272 | // MEMBER_OF mappings on maps that are passed the initial nesting depth, which |
| 8273 | // includes pointed to data and attach members, both of which are technically |
| 8274 | // not part of the main object. This has the side effect of causing early |
| 8275 | // map-backs in certain cases where an implicit declare mapper has been |
| 8276 | // emitted for a target region. Applying MAP_PTR_AND_OBJ in these situations |
| 8277 | // circumvents this. |
| 8278 | if (isPtrTy && !isAttachMap && mapData.IsDeclareTarget[mapDataIdx]) |
| 8279 | mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ; |
| 8280 | |
| 8281 | // if we're provided a mapDataParentIdx, then the data being mapped is |
| 8282 | // part of a larger object (in a parent <-> member mapping) and in this |
| 8283 | // case our BasePointer should be the parent. Except in the edge case |
| 8284 | // where we are mapping pointee data, where we try staying close to |
| 8285 | // what Clang currently does and utilise the regular base pointer of the |
| 8286 | // data. |
| 8287 | bool isRefPtee = |
| 8288 | !bitEnumContainsAll(bits: mapInfoOp.getMapType(), |
| 8289 | bit: omp::ClauseMapFlags::ref_ptr) && |
| 8290 | bitEnumContainsAll(bits: mapInfoOp.getMapType(), bit: omp::ClauseMapFlags::ref_ptee); |
| 8291 | bool isRefPtrPtee = bitEnumContainsAll(bits: mapInfoOp.getMapType(), |
| 8292 | bit: omp::ClauseMapFlags::ref_ptr | |
| 8293 | omp::ClauseMapFlags::ref_ptee); |
| 8294 | |
| 8295 | if (!mapInfoOp->getParentOfType<omp::DeclareMapperOp>() && |
| 8296 | mapDataParentIdx >= 0 && !(isRefPtee || (isRefPtrPtee && isPtrTy))) { |
| 8297 | combinedInfo.BasePointers.emplace_back( |
| 8298 | Args&: mapData.BasePointers[mapDataParentIdx]); |
| 8299 | } else { |
| 8300 | combinedInfo.BasePointers.emplace_back(Args&: mapData.BasePointers[mapDataIdx]); |
| 8301 | } |
| 8302 | |
| 8303 | combinedInfo.Pointers.emplace_back(Args&: mapData.Pointers[mapDataIdx]); |
| 8304 | combinedInfo.DevicePointers.emplace_back( |
| 8305 | Args: memberOfFlag != llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE |
| 8306 | ? llvm::OpenMPIRBuilder::DeviceInfoTy::None |
| 8307 | : mapData.DevicePointers[mapDataIdx]); |
| 8308 | combinedInfo.Mappers.emplace_back(Args&: mapData.Mappers[mapDataIdx]); |
| 8309 | combinedInfo.Names.emplace_back(Args&: mapData.Names[mapDataIdx]); |
| 8310 | combinedInfo.Types.emplace_back(Args&: mapFlag); |
| 8311 | // TODO: set HasAttachPtr from Flang for pointee-storage entries. |
| 8312 | combinedInfo.HasAttachPtr.emplace_back(Args: false); |
| 8313 | // A privatized attach map needs storage for the pointer or descriptor even |
| 8314 | // when its pointee is null. Its size describes that storage, not the pointee, |
| 8315 | // and is needed by the runtime for corresponding-pointer-initialization. |
| 8316 | combinedInfo.Sizes.emplace_back( |
| 8317 | Args: isPtrTy && !isPrivatizeableAttachMap(mapType: mapInfoOp.getMapType()) |
| 8318 | ? builder.CreateSelect( |
| 8319 | C: builder.CreateIsNull(Arg: mapData.Pointers[mapDataIdx]), |
| 8320 | True: builder.getInt64(C: 0), False: mapData.Sizes[mapDataIdx]) |
| 8321 | : mapData.Sizes[mapDataIdx]); |
| 8322 | if (targetDirective == TargetDirectiveEnumTy::TargetUpdate) |
| 8323 | addNonContiguousUpdate(moduleTranslation, builder, mapData, mapDataIdx, |
| 8324 | combinedInfo); |
| 8325 | } |
| 8326 | |
| 8327 | // This creates two insertions into the MapInfosTy data structure for the |
| 8328 | // "parent" of a set of members, (usually a container e.g. |
| 8329 | // class/structure/derived type) when subsequent members have also been |
| 8330 | // explicitly mapped on the same map clause. Certain types, such as Fortran |
| 8331 | // descriptors are mapped like this as well, however, the members are |
| 8332 | // implicit as far as a user is concerned, but we must explicitly map them |
| 8333 | // internally. |
| 8334 | // |
| 8335 | // This function also returns the memberOfFlag for this particular parent, |
| 8336 | // which is utilised in subsequent member mappings (by modifying there map type |
| 8337 | // with it) to indicate that a member is part of this parent and should be |
| 8338 | // treated by the runtime as such. Important to achieve the correct mapping. |
| 8339 | // |
| 8340 | // This function borrows a lot from Clang's emitCombinedEntry function |
| 8341 | // inside of CGOpenMPRuntime.cpp |
| 8342 | static void mapParentWithMembers( |
| 8343 | LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder, |
| 8344 | llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl, MapInfosTy &combinedInfo, |
| 8345 | MapInfoData &mapData, uint64_t mapDataIndex, |
| 8346 | llvm::omp::OpenMPOffloadMappingFlags memberOfFlag, |
| 8347 | TargetDirectiveEnumTy targetDirective) { |
| 8348 | using MapFlags = llvm::omp::OpenMPOffloadMappingFlags; |
| 8349 | assert(!ompBuilder.Config.isTargetDevice() && |
| 8350 | "function only supported for host device codegen" ); |
| 8351 | auto parentClause = |
| 8352 | llvm::cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]); |
| 8353 | auto *parentMapper = mapData.Mappers[mapDataIndex]; |
| 8354 | |
| 8355 | // Map the first segment of the parent. If a user-defined mapper is attached, |
| 8356 | // include the parent's to/from-style bits (and common modifiers) in this |
| 8357 | // base entry so the mapper receives correct copy semantics via its 'type' |
| 8358 | // parameter. Also keep TARGET_PARAM when required for kernel arguments. |
| 8359 | MapFlags baseFlag = (targetDirective == TargetDirectiveEnumTy::Target && |
| 8360 | !mapData.IsDeclareTarget[mapDataIndex]) |
| 8361 | ? MapFlags::OMP_MAP_TARGET_PARAM |
| 8362 | : MapFlags::OMP_MAP_NONE; |
| 8363 | |
| 8364 | if (parentMapper) { |
| 8365 | // Preserve relevant map-type bits from the parent clause. These include |
| 8366 | // the copy direction (TO/FROM), as well as commonly used modifiers that |
| 8367 | // should be visible to the mapper for correct behaviour. |
| 8368 | MapFlags parentFlags = mapData.Types[mapDataIndex]; |
| 8369 | MapFlags preserve = MapFlags::OMP_MAP_TO | MapFlags::OMP_MAP_FROM | |
| 8370 | MapFlags::OMP_MAP_ALWAYS | MapFlags::OMP_MAP_CLOSE | |
| 8371 | MapFlags::OMP_MAP_PRESENT | |
| 8372 | MapFlags::OMP_MAP_OMPX_HOLD | |
| 8373 | MapFlags::OMP_MAP_IMPLICIT; |
| 8374 | baseFlag |= (parentFlags & preserve); |
| 8375 | } else { |
| 8376 | MapFlags parentFlags = mapData.Types[mapDataIndex]; |
| 8377 | MapFlags preserve = MapFlags::OMP_MAP_TO | MapFlags::OMP_MAP_FROM | |
| 8378 | MapFlags::OMP_MAP_PRESENT | |
| 8379 | MapFlags::OMP_MAP_RETURN_PARAM | |
| 8380 | MapFlags::OMP_MAP_IMPLICIT; |
| 8381 | baseFlag |= (parentFlags & preserve); |
| 8382 | } |
| 8383 | |
| 8384 | combinedInfo.Types.emplace_back(Args&: baseFlag); |
| 8385 | // TODO: set HasAttachPtr from Flang for pointee-storage entries. |
| 8386 | combinedInfo.HasAttachPtr.emplace_back(Args: false); |
| 8387 | combinedInfo.DevicePointers.emplace_back( |
| 8388 | Args&: mapData.DevicePointers[mapDataIndex]); |
| 8389 | // Only attach the mapper to the base entry when we are mapping the whole |
| 8390 | // parent. Combined/segment entries must not carry a mapper; otherwise the |
| 8391 | // mapper can be invoked with a partial size, which is undefined behaviour. |
| 8392 | combinedInfo.Mappers.emplace_back( |
| 8393 | Args: parentMapper && !parentClause.getPartialMap() ? parentMapper : nullptr); |
| 8394 | combinedInfo.Names.emplace_back(Args: LLVM::createMappingInformation( |
| 8395 | loc: mapData.MapClause[mapDataIndex]->getLoc(), builder&: ompBuilder, |
| 8396 | mapName: getMapClauseName(mapOp: cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex])))); |
| 8397 | combinedInfo.BasePointers.emplace_back(Args&: mapData.BasePointers[mapDataIndex]); |
| 8398 | |
| 8399 | // Calculate size of the parent object being mapped based on the |
| 8400 | // addresses at runtime, highAddr - lowAddr = size. This of course |
| 8401 | // doesn't factor in allocated data like pointers, hence the further |
| 8402 | // processing of members specified by users, or in the case of |
| 8403 | // Fortran pointers and allocatables, the mapping of the pointed to |
| 8404 | // data by the descriptor (which itself, is a structure containing |
| 8405 | // runtime information on the dynamically allocated data). |
| 8406 | llvm::Value *lowAddr, *highAddr; |
| 8407 | if (!parentClause.getPartialMap()) { |
| 8408 | lowAddr = builder.CreatePointerCast(V: mapData.Pointers[mapDataIndex], |
| 8409 | DestTy: builder.getPtrTy()); |
| 8410 | highAddr = builder.CreatePointerCast( |
| 8411 | V: builder.CreateConstGEP1_32(Ty: mapData.BaseType[mapDataIndex], |
| 8412 | Ptr: mapData.Pointers[mapDataIndex], Idx0: 1), |
| 8413 | DestTy: builder.getPtrTy()); |
| 8414 | combinedInfo.Pointers.emplace_back(Args&: mapData.Pointers[mapDataIndex]); |
| 8415 | } else { |
| 8416 | auto mapOp = dyn_cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]); |
| 8417 | int firstMemberIdx = getMapDataMemberIdx( |
| 8418 | mapData, memberOp: getFirstOrLastMappedMemberPtr(mapInfo: mapOp, first: true)); |
| 8419 | lowAddr = builder.CreatePointerCast(V: mapData.BasePointers[firstMemberIdx], |
| 8420 | DestTy: builder.getPtrTy()); |
| 8421 | |
| 8422 | int lastMemberIdx = getMapDataMemberIdx( |
| 8423 | mapData, memberOp: getFirstOrLastMappedMemberPtr(mapInfo: mapOp, first: false)); |
| 8424 | auto lastMemberMapInfo = |
| 8425 | cast<omp::MapInfoOp>(Val: mapData.MapClause[lastMemberIdx]); |
| 8426 | |
| 8427 | // NOTE: Currently, for RefPtee the BaseType is set to the varPtrPtr field, |
| 8428 | // which is the pointer datas type and not the member within the structure |
| 8429 | // that it's part of, so we have to make sure we use the member type in this |
| 8430 | // case when calculating the parents size offsets. |
| 8431 | // TODO: May be good to extend MapInfoData to support tracking of both |
| 8432 | // VarPtr/VarPtrPtr BaseType's to better distinguish what's being used more |
| 8433 | // consistently. |
| 8434 | bool isRefPteeMap = bitEnumContainsAll(bits: lastMemberMapInfo.getMapType(), |
| 8435 | bit: omp::ClauseMapFlags::ref_ptee) && |
| 8436 | !bitEnumContainsAll(bits: lastMemberMapInfo.getMapType(), |
| 8437 | bit: omp::ClauseMapFlags::ref_ptr); |
| 8438 | llvm::Type *castType = mapData.BaseType[lastMemberIdx]; |
| 8439 | if (isRefPteeMap) |
| 8440 | castType = |
| 8441 | moduleTranslation.convertType(type: lastMemberMapInfo.getVarPtrType()); |
| 8442 | highAddr = builder.CreatePointerCast( |
| 8443 | V: builder.CreateGEP(Ty: castType, Ptr: mapData.BasePointers[lastMemberIdx], |
| 8444 | IdxList: builder.getInt64(C: 1)), |
| 8445 | DestTy: builder.getPtrTy()); |
| 8446 | combinedInfo.Pointers.emplace_back(Args&: mapData.BasePointers[firstMemberIdx]); |
| 8447 | } |
| 8448 | |
| 8449 | llvm::Value *size = builder.CreateIntCast( |
| 8450 | V: builder.CreatePtrDiff(ElemTy: builder.getInt8Ty(), LHS: highAddr, RHS: lowAddr), |
| 8451 | DestTy: builder.getInt64Ty(), |
| 8452 | /*isSigned=*/false); |
| 8453 | combinedInfo.Sizes.push_back(Elt: size); |
| 8454 | |
| 8455 | // This creates the initial MEMBER_OF mapping that consists of |
| 8456 | // the parent/top level container (same as above effectively, except |
| 8457 | // with a fixed initial compile time size and separate maptype which |
| 8458 | // indicates the true mape type (tofrom etc.). This parent mapping is |
| 8459 | // only relevant if the structure in its totality is being mapped, |
| 8460 | // otherwise the above suffices. |
| 8461 | if (!parentClause.getPartialMap()) { |
| 8462 | // TODO: This will need to be expanded to include the whole host of logic |
| 8463 | // for the map flags that Clang currently supports (e.g. it should do some |
| 8464 | // further case specific flag modifications). For the moment, it handles |
| 8465 | // what we support as expected. |
| 8466 | MapFlags mapFlag = mapData.Types[mapDataIndex]; |
| 8467 | bool hasMapClose = (MapFlags(mapFlag) & MapFlags::OMP_MAP_CLOSE) == |
| 8468 | MapFlags::OMP_MAP_CLOSE; |
| 8469 | ompBuilder.setCorrectMemberOfFlag(Flags&: mapFlag, MemberOfFlag: memberOfFlag); |
| 8470 | |
| 8471 | llvm::SmallVector<size_t> overlapIdxs; |
| 8472 | // Find all of the members that "overlap", i.e. occlude other members that |
| 8473 | // were mapped alongside the parent, e.g. member [0], occludes [0,1] and |
| 8474 | // [0,2], but not [1,0]. |
| 8475 | getOverlappedMembers(overlapMapDataIdxs&: overlapIdxs, parentOp: parentClause); |
| 8476 | |
| 8477 | // When we only have one overlap we skip the case that tries to segment the |
| 8478 | // mapping as best it can without creating holes, as the calculation is more |
| 8479 | // likely to have more overhead than anything we gain from mapping a smaller |
| 8480 | // chunk of data. This can be seen in cases where we are mapping Fortran |
| 8481 | // descriptors which are a special case of record type mapping. |
| 8482 | // |
| 8483 | // The cases for close and update are unique edge cases where the segmenting |
| 8484 | // does not play well with the runtime currently. |
| 8485 | if (targetDirective == TargetDirectiveEnumTy::TargetUpdate || hasMapClose || |
| 8486 | overlapIdxs.size() == 1) { |
| 8487 | combinedInfo.Types.emplace_back(Args&: mapFlag); |
| 8488 | // TODO: set HasAttachPtr from Flang for pointee-storage entries. |
| 8489 | combinedInfo.HasAttachPtr.emplace_back(Args: false); |
| 8490 | combinedInfo.DevicePointers.emplace_back( |
| 8491 | Args&: mapData.DevicePointers[mapDataIndex]); |
| 8492 | combinedInfo.Names.emplace_back(Args: LLVM::createMappingInformation( |
| 8493 | loc: mapData.MapClause[mapDataIndex]->getLoc(), builder&: ompBuilder, |
| 8494 | mapName: getMapClauseName( |
| 8495 | mapOp: cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex])))); |
| 8496 | combinedInfo.BasePointers.emplace_back( |
| 8497 | Args&: mapData.BasePointers[mapDataIndex]); |
| 8498 | combinedInfo.Pointers.emplace_back(Args&: mapData.Pointers[mapDataIndex]); |
| 8499 | combinedInfo.Sizes.emplace_back(Args&: mapData.Sizes[mapDataIndex]); |
| 8500 | combinedInfo.Mappers.emplace_back(Args: nullptr); |
| 8501 | } else { |
| 8502 | // We need to make sure the overlapped members are sorted in order of |
| 8503 | // lowest address to highest address. |
| 8504 | sortMapIndices(indices&: overlapIdxs, mapInfo: parentClause); |
| 8505 | |
| 8506 | lowAddr = builder.CreatePointerCast(V: mapData.Pointers[mapDataIndex], |
| 8507 | DestTy: builder.getPtrTy()); |
| 8508 | highAddr = builder.CreatePointerCast( |
| 8509 | V: builder.CreateConstGEP1_32(Ty: mapData.BaseType[mapDataIndex], |
| 8510 | Ptr: mapData.Pointers[mapDataIndex], Idx0: 1), |
| 8511 | DestTy: builder.getPtrTy()); |
| 8512 | |
| 8513 | // Currently, the return parameter should be the over-riding parent in |
| 8514 | // cases where we have a return parameter that is echoed to all members, |
| 8515 | // the main case of this currently is with fortran descriptors. It may |
| 8516 | // need more finessing for C/C++ in the future or descriptors that are |
| 8517 | // members of derived types. |
| 8518 | mapFlag &= ~llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM; |
| 8519 | |
| 8520 | // TODO: We may want to skip arrays/array sections in this as Clang does. |
| 8521 | // It appears to be an optimisation rather than a necessity though, |
| 8522 | // but this requires further investigation. However, we would have to make |
| 8523 | // sure to not exclude maps with bounds that ARE pointers, as these are |
| 8524 | // processed as separate components, i.e. pointer + data. |
| 8525 | for (auto v : overlapIdxs) { |
| 8526 | auto mapDataOverlapIdx = getMapDataMemberIdx( |
| 8527 | mapData, |
| 8528 | memberOp: cast<omp::MapInfoOp>(Val: parentClause.getMembers()[v].getDefiningOp())); |
| 8529 | auto isPtrMap = checkIfPointerMap( |
| 8530 | mapOp: llvm::cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataOverlapIdx])); |
| 8531 | combinedInfo.Types.emplace_back(Args&: mapFlag); |
| 8532 | // TODO: set HasAttachPtr from Flang for pointee-storage entries. |
| 8533 | combinedInfo.HasAttachPtr.emplace_back(Args: false); |
| 8534 | combinedInfo.DevicePointers.emplace_back( |
| 8535 | Args: llvm::OpenMPIRBuilder::DeviceInfoTy::None); |
| 8536 | combinedInfo.Names.emplace_back(Args: LLVM::createMappingInformation( |
| 8537 | loc: mapData.MapClause[mapDataIndex]->getLoc(), builder&: ompBuilder, |
| 8538 | mapName: getMapClauseName( |
| 8539 | mapOp: cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex])))); |
| 8540 | combinedInfo.BasePointers.emplace_back( |
| 8541 | Args&: mapData.BasePointers[mapDataIndex]); |
| 8542 | combinedInfo.Mappers.emplace_back(Args: nullptr); |
| 8543 | combinedInfo.Pointers.emplace_back(Args&: lowAddr); |
| 8544 | auto sizeCalc = builder.CreateIntCast( |
| 8545 | V: builder.CreatePtrDiff(ElemTy: builder.getInt8Ty(), |
| 8546 | LHS: mapData.OriginalValue[mapDataOverlapIdx], |
| 8547 | RHS: lowAddr), |
| 8548 | DestTy: builder.getInt64Ty(), /*isSigned=*/true); |
| 8549 | // In certain cases, we'll generate a size of 0 if we're not careful |
| 8550 | // (e.g. if lowAddr happens to be the first member), which isn't |
| 8551 | // correct, even if the runtimes is sometimes fine with it so, in these |
| 8552 | // scenarios we select the types size instead. |
| 8553 | llvm::DataLayout dataLayout = builder.getDataLayout(); |
| 8554 | auto sizeSel = builder.CreateSelect( |
| 8555 | C: builder.CreateICmpNE(LHS: builder.getInt64(C: 0), RHS: sizeCalc), True: sizeCalc, |
| 8556 | False: isPtrMap ? builder.getInt64(C: dataLayout.getPointerSize()) |
| 8557 | : mapData.Sizes[mapDataOverlapIdx]); |
| 8558 | combinedInfo.Sizes.emplace_back(Args&: sizeSel); |
| 8559 | lowAddr = builder.CreateConstGEP1_32( |
| 8560 | Ty: isPtrMap ? builder.getPtrTy() : mapData.BaseType[mapDataOverlapIdx], |
| 8561 | Ptr: mapData.BasePointers[mapDataOverlapIdx], Idx0: 1); |
| 8562 | } |
| 8563 | |
| 8564 | combinedInfo.Types.emplace_back(Args&: mapFlag); |
| 8565 | // TODO: set HasAttachPtr from Flang for pointee-storage entries. |
| 8566 | combinedInfo.HasAttachPtr.emplace_back(Args: false); |
| 8567 | combinedInfo.DevicePointers.emplace_back( |
| 8568 | Args: llvm::OpenMPIRBuilder::DeviceInfoTy::None); |
| 8569 | combinedInfo.Names.emplace_back(Args: LLVM::createMappingInformation( |
| 8570 | loc: mapData.MapClause[mapDataIndex]->getLoc(), builder&: ompBuilder, |
| 8571 | mapName: getMapClauseName( |
| 8572 | mapOp: cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex])))); |
| 8573 | combinedInfo.BasePointers.emplace_back( |
| 8574 | Args&: mapData.BasePointers[mapDataIndex]); |
| 8575 | combinedInfo.Mappers.emplace_back(Args: nullptr); |
| 8576 | combinedInfo.Pointers.emplace_back(Args&: lowAddr); |
| 8577 | combinedInfo.Sizes.emplace_back(Args: builder.CreateIntCast( |
| 8578 | V: builder.CreatePtrDiff(ElemTy: builder.getInt8Ty(), LHS: highAddr, RHS: lowAddr), |
| 8579 | DestTy: builder.getInt64Ty(), isSigned: true)); |
| 8580 | } |
| 8581 | } |
| 8582 | } |
| 8583 | |
| 8584 | static void processMapWithMembersOf(LLVM::ModuleTranslation &moduleTranslation, |
| 8585 | llvm::IRBuilderBase &builder, |
| 8586 | llvm::OpenMPIRBuilder &ompBuilder, |
| 8587 | DataLayout &dl, MapInfosTy &combinedInfo, |
| 8588 | MapInfoData &mapData, uint64_t mapDataIndex, |
| 8589 | TargetDirectiveEnumTy targetDirective) { |
| 8590 | assert(!ompBuilder.Config.isTargetDevice() && |
| 8591 | "function only supported for host device codegen" ); |
| 8592 | |
| 8593 | auto parentClause = |
| 8594 | llvm::cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]); |
| 8595 | |
| 8596 | // If we have a partial map (no parent referenced in the map clauses of the |
| 8597 | // directive, only members) and only a single member, we do not need to bind |
| 8598 | // the map of the member to the parent, we can pass the member separately. |
| 8599 | if (parentClause.getMembers().size() == 1 && parentClause.getPartialMap()) { |
| 8600 | auto memberClause = llvm::cast<omp::MapInfoOp>( |
| 8601 | Val: parentClause.getMembers()[0].getDefiningOp()); |
| 8602 | int memberDataIdx = getMapDataMemberIdx(mapData, memberOp: memberClause); |
| 8603 | // Note: Clang treats arrays with explicit bounds that fall into this |
| 8604 | // category as a parent with map case, however, it seems this isn't a |
| 8605 | // requirement, and processing them as an individual map is fine. So, |
| 8606 | // we will handle them as individual maps for the moment, as it's |
| 8607 | // difficult for us to check this as we always require bounds to be |
| 8608 | // specified currently and it's also marginally more optimal (single |
| 8609 | // map rather than two). The difference may come from the fact that |
| 8610 | // Clang maps array without bounds as pointers (which we do not |
| 8611 | // currently do), whereas we treat them as arrays in all cases |
| 8612 | // currently. |
| 8613 | processIndividualMap( |
| 8614 | moduleTranslation, builder, ompBuilder, mapData, mapDataIdx: memberDataIdx, |
| 8615 | combinedInfo, targetDirective, |
| 8616 | /*MemberOfFlag=*/memberOfFlag: llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE, |
| 8617 | /*isTargetParam=*/true, mapDataParentIdx: mapDataIndex); |
| 8618 | return; |
| 8619 | } |
| 8620 | |
| 8621 | auto collectMapInfoIdxs = |
| 8622 | [&](llvm::SmallVectorImpl<int64_t> &mapsAndInfoIdx) { |
| 8623 | auto parentClause = |
| 8624 | llvm::cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]); |
| 8625 | mapsAndInfoIdx.push_back(Elt: getMapDataMemberIdx(mapData, memberOp: parentClause)); |
| 8626 | for (auto member : parentClause.getMembers()) |
| 8627 | mapsAndInfoIdx.push_back(Elt: getMapDataMemberIdx( |
| 8628 | mapData, memberOp: llvm::cast<omp::MapInfoOp>(Val: member.getDefiningOp()))); |
| 8629 | }; |
| 8630 | |
| 8631 | llvm::SmallVector<int64_t> mapInfoIdx; |
| 8632 | collectMapInfoIdxs(mapInfoIdx); |
| 8633 | |
| 8634 | llvm::omp::OpenMPOffloadMappingFlags memberOfFlag = |
| 8635 | ompBuilder.getMemberOfFlag(Position: combinedInfo.Types.size()); |
| 8636 | |
| 8637 | // The first index is the parent map, the rest are its members. The parent |
| 8638 | // normally undergoes the standard parent-with-members mapping, contributing |
| 8639 | // the MEMBER_OF flag that binds each member to it. The one exception is a |
| 8640 | // privatizeable attach map (a privatized pointer/descriptor passed directly |
| 8641 | // as a kernel argument): here the parent is emitted as an individual map |
| 8642 | // instead, for the time being, as it's used only in pointer/allocatable to |
| 8643 | // array cases for the moment. This only ever applies to the parent, so it is |
| 8644 | // checked once here rather than inside the loop below. |
| 8645 | bool parentIsPrivatizeableAttach = |
| 8646 | isPrivatizeableAttachMap(mapType: parentClause.getMapType()); |
| 8647 | for (auto [i, idx] : llvm::enumerate(First&: mapInfoIdx)) { |
| 8648 | bool emitParentMap = i == 0 && !parentIsPrivatizeableAttach; |
| 8649 | if (emitParentMap) { |
| 8650 | mapParentWithMembers(moduleTranslation, builder, ompBuilder, dl, |
| 8651 | combinedInfo, mapData, mapDataIndex: idx, memberOfFlag, |
| 8652 | targetDirective); |
| 8653 | } else { |
| 8654 | processIndividualMap( |
| 8655 | moduleTranslation, builder, ompBuilder, mapData, mapDataIdx: idx, combinedInfo, |
| 8656 | targetDirective, |
| 8657 | memberOfFlag: parentIsPrivatizeableAttach |
| 8658 | ? llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE |
| 8659 | : memberOfFlag, |
| 8660 | /*isTargetParam=*/false, mapDataParentIdx: mapDataIndex); |
| 8661 | } |
| 8662 | } |
| 8663 | } |
| 8664 | |
| 8665 | // This is a variation on Clang's GenerateOpenMPCapturedVars, which |
| 8666 | // generates different operation (e.g. load/store) combinations for |
| 8667 | // arguments to the kernel, based on map capture kinds which are then |
| 8668 | // utilised in the combinedInfo in place of the original Map value. |
| 8669 | static void |
| 8670 | createAlteredByCaptureMap(MapInfoData &mapData, |
| 8671 | LLVM::ModuleTranslation &moduleTranslation, |
| 8672 | llvm::IRBuilderBase &builder) { |
| 8673 | assert(!getOpenMPBuilder(moduleTranslation)->Config.isTargetDevice() && |
| 8674 | "function only supported for host device codegen" ); |
| 8675 | for (size_t i = 0; i < mapData.MapClause.size(); ++i) { |
| 8676 | auto mapOp = cast<omp::MapInfoOp>(Val: mapData.MapClause[i]); |
| 8677 | bool isAttachMap = |
| 8678 | ((convertClauseMapFlags(mlirFlags: mapOp.getMapType()) & |
| 8679 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH) == |
| 8680 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH); |
| 8681 | |
| 8682 | // If it's declare target, skip it, it's handled separately. However, if |
| 8683 | // it's declare target, and an attach map, we want to calculate the exact |
| 8684 | // address offset so that we attach correctly. |
| 8685 | if (!mapData.IsDeclareTarget[i] || |
| 8686 | (mapData.IsDeclareTarget[i] && isAttachMap)) { |
| 8687 | omp::VariableCaptureKind captureKind = mapOp.getMapCaptureType(); |
| 8688 | bool isPtrTy = checkIfPointerMap(mapOp); |
| 8689 | |
| 8690 | // Currently handles array sectioning lowerbound case, but more |
| 8691 | // logic may be required in the future. Clang invokes EmitLValue, |
| 8692 | // which has specialised logic for special Clang types such as user |
| 8693 | // defines, so it is possible we will have to extend this for |
| 8694 | // structures or other complex types. As the general idea is that this |
| 8695 | // function mimics some of the logic from Clang that we require for |
| 8696 | // kernel argument passing from host -> device. |
| 8697 | switch (captureKind) { |
| 8698 | case omp::VariableCaptureKind::ByRef: { |
| 8699 | llvm::Value *newV = mapData.Pointers[i]; |
| 8700 | bool isByteOffset = false; |
| 8701 | std::vector<llvm::Value *> offsetIdx = calculateBoundsOffset( |
| 8702 | moduleTranslation, builder, isArrayTy: mapData.BaseType[i]->isArrayTy(), |
| 8703 | bounds: mapOp.getBounds(), isByteOffset); |
| 8704 | if (isPtrTy) |
| 8705 | newV = builder.CreateLoad(Ty: builder.getPtrTy(), Ptr: newV); |
| 8706 | |
| 8707 | if (!offsetIdx.empty()) |
| 8708 | newV = builder.CreateInBoundsGEP(Ty: isByteOffset ? builder.getInt8Ty() |
| 8709 | : mapData.BaseType[i], |
| 8710 | Ptr: newV, IdxList: offsetIdx, Name: "array_offset" ); |
| 8711 | mapData.Pointers[i] = newV; |
| 8712 | } break; |
| 8713 | case omp::VariableCaptureKind::ByCopy: { |
| 8714 | llvm::Type *type = mapData.BaseType[i]; |
| 8715 | llvm::Value *newV; |
| 8716 | if (mapData.Pointers[i]->getType()->isPointerTy()) |
| 8717 | newV = builder.CreateLoad(Ty: type, Ptr: mapData.Pointers[i]); |
| 8718 | else |
| 8719 | newV = mapData.Pointers[i]; |
| 8720 | |
| 8721 | if (!isPtrTy) { |
| 8722 | auto curInsert = builder.saveIP(); |
| 8723 | llvm::DebugLoc DbgLoc = builder.getCurrentDebugLocation(); |
| 8724 | builder.restoreIP(IP: findAllocInsertPoints(builder, moduleTranslation)); |
| 8725 | auto *memTempAlloc = |
| 8726 | builder.CreateAlloca(Ty: builder.getPtrTy(), ArraySize: nullptr, Name: ".casted" ); |
| 8727 | builder.SetCurrentDebugLocation(DbgLoc); |
| 8728 | builder.restoreIP(IP: curInsert); |
| 8729 | |
| 8730 | builder.CreateStore(Val: newV, Ptr: memTempAlloc); |
| 8731 | newV = builder.CreateLoad(Ty: builder.getPtrTy(), Ptr: memTempAlloc); |
| 8732 | } |
| 8733 | |
| 8734 | mapData.Pointers[i] = newV; |
| 8735 | mapData.BasePointers[i] = newV; |
| 8736 | } break; |
| 8737 | case omp::VariableCaptureKind::This: |
| 8738 | case omp::VariableCaptureKind::VLAType: |
| 8739 | mapData.MapClause[i]->emitOpError(message: "Unhandled capture kind" ); |
| 8740 | break; |
| 8741 | } |
| 8742 | } |
| 8743 | } |
| 8744 | } |
| 8745 | |
| 8746 | // Generate all map related information and fill the combinedInfo. |
| 8747 | static void genMapInfos(llvm::IRBuilderBase &builder, |
| 8748 | LLVM::ModuleTranslation &moduleTranslation, |
| 8749 | DataLayout &dl, MapInfosTy &combinedInfo, |
| 8750 | MapInfoData &mapData, |
| 8751 | TargetDirectiveEnumTy targetDirective) { |
| 8752 | assert(!getOpenMPBuilder(moduleTranslation)->Config.isTargetDevice() && |
| 8753 | "function only supported for host device codegen" ); |
| 8754 | // We wish to modify some of the methods in which arguments are |
| 8755 | // passed based on their capture type by the target region, this can |
| 8756 | // involve generating new loads and stores, which changes the |
| 8757 | // MLIR value to LLVM value mapping, however, we only wish to do this |
| 8758 | // locally for the current function/target and also avoid altering |
| 8759 | // ModuleTranslation, so we remap the base pointer or pointer stored |
| 8760 | // in the map infos corresponding MapInfoData, which is later accessed |
| 8761 | // by genMapInfos and createTarget to help generate the kernel and |
| 8762 | // kernel arg structure. It primarily becomes relevant in cases like |
| 8763 | // bycopy, or byref range'd arrays. In the default case, we simply |
| 8764 | // pass thee pointer byref as both basePointer and pointer. |
| 8765 | createAlteredByCaptureMap(mapData, moduleTranslation, builder); |
| 8766 | |
| 8767 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 8768 | |
| 8769 | // We operate under the assumption that all vectors that are |
| 8770 | // required in MapInfoData are of equal lengths (either filled with |
| 8771 | // default constructed data or appropiate information) so we can |
| 8772 | // utilise the size from any component of MapInfoData, if we can't |
| 8773 | // something is missing from the initial MapInfoData construction. |
| 8774 | for (size_t i = 0; i < mapData.MapClause.size(); ++i) { |
| 8775 | if (mapData.IsAMember[i]) |
| 8776 | continue; |
| 8777 | |
| 8778 | auto mapInfoOp = dyn_cast<omp::MapInfoOp>(Val: mapData.MapClause[i]); |
| 8779 | if (!mapInfoOp.getMembers().empty()) { |
| 8780 | processMapWithMembersOf(moduleTranslation, builder, ompBuilder&: *ompBuilder, dl, |
| 8781 | combinedInfo, mapData, mapDataIndex: i, targetDirective); |
| 8782 | continue; |
| 8783 | } |
| 8784 | |
| 8785 | processIndividualMap(moduleTranslation, builder, ompBuilder&: *ompBuilder, mapData, mapDataIdx: i, |
| 8786 | combinedInfo, targetDirective); |
| 8787 | } |
| 8788 | if (!combinedInfo.NonContigInfo.Dims.empty()) |
| 8789 | combinedInfo.NonContigInfo.Dims.resize(N: combinedInfo.Types.size(), NV: 1); |
| 8790 | } |
| 8791 | |
| 8792 | static llvm::Expected<llvm::Function *> |
| 8793 | emitUserDefinedMapper(Operation *declMapperOp, llvm::IRBuilderBase &builder, |
| 8794 | LLVM::ModuleTranslation &moduleTranslation, |
| 8795 | llvm::StringRef mapperFuncName, |
| 8796 | TargetDirectiveEnumTy targetDirective); |
| 8797 | |
| 8798 | static llvm::Expected<llvm::Function *> |
| 8799 | getOrCreateUserDefinedMapperFunc(Operation *op, llvm::IRBuilderBase &builder, |
| 8800 | LLVM::ModuleTranslation &moduleTranslation, |
| 8801 | TargetDirectiveEnumTy targetDirective) { |
| 8802 | assert(!getOpenMPBuilder(moduleTranslation)->Config.isTargetDevice() && |
| 8803 | "function only supported for host device codegen" ); |
| 8804 | auto declMapperOp = cast<omp::DeclareMapperOp>(Val: op); |
| 8805 | std::string mapperFuncName = |
| 8806 | getOpenMPBuilder(moduleTranslation) |
| 8807 | ->createPlatformSpecificName( |
| 8808 | Parts: {"omp_mapper" , declMapperOp.getSymName()}); |
| 8809 | |
| 8810 | if (auto *lookupFunc = moduleTranslation.lookupFunction(name: mapperFuncName)) |
| 8811 | return lookupFunc; |
| 8812 | |
| 8813 | // Recursive types can cause re-entrant mapper emission. The mapper function |
| 8814 | // is created by OpenMPIRBuilder before the callbacks run, so it may already |
| 8815 | // exist in the LLVM module even though it is not yet registered in the |
| 8816 | // ModuleTranslation mapping table. Reuse and register it to break the |
| 8817 | // recursion. |
| 8818 | if (llvm::Function *existingFunc = |
| 8819 | moduleTranslation.getLLVMModule()->getFunction(Name: mapperFuncName)) { |
| 8820 | moduleTranslation.mapFunction(name: mapperFuncName, func: existingFunc); |
| 8821 | return existingFunc; |
| 8822 | } |
| 8823 | |
| 8824 | return emitUserDefinedMapper(declMapperOp, builder, moduleTranslation, |
| 8825 | mapperFuncName, targetDirective); |
| 8826 | } |
| 8827 | |
| 8828 | /// The mapper body (and any iterator regions in it) is emitted inside the |
| 8829 | /// mapper's per-element loop, so temporaries Flang allocates in the mapper |
| 8830 | /// region (e.g. the storage for an iterator variable) would be allocated once |
| 8831 | /// per element and grow the stack with every iteration. Move constant-size |
| 8832 | /// allocas to the function's entry block, where a frontend would place them. |
| 8833 | static void hoistMapperAllocasToEntry(llvm::Function &mapperFn) { |
| 8834 | llvm::BasicBlock &entry = mapperFn.getEntryBlock(); |
| 8835 | llvm::SmallVector<llvm::AllocaInst *> toHoist; |
| 8836 | for (llvm::BasicBlock &bb : llvm::drop_begin(RangeOrContainer&: mapperFn)) |
| 8837 | for (llvm::Instruction &inst : bb) |
| 8838 | if (auto *alloca = dyn_cast<llvm::AllocaInst>(Val: &inst)) |
| 8839 | if (isa<llvm::ConstantInt>(Val: alloca->getArraySize())) |
| 8840 | toHoist.push_back(Elt: alloca); |
| 8841 | for (llvm::AllocaInst *alloca : toHoist) |
| 8842 | alloca->moveBefore(InsertPos: entry.getFirstNonPHIOrAlloca()); |
| 8843 | } |
| 8844 | |
| 8845 | static llvm::Expected<llvm::Function *> |
| 8846 | emitUserDefinedMapper(Operation *op, llvm::IRBuilderBase &builder, |
| 8847 | LLVM::ModuleTranslation &moduleTranslation, |
| 8848 | llvm::StringRef mapperFuncName, |
| 8849 | TargetDirectiveEnumTy targetDirective) { |
| 8850 | assert(!getOpenMPBuilder(moduleTranslation)->Config.isTargetDevice() && |
| 8851 | "function only supported for host device codegen" ); |
| 8852 | auto declMapperOp = cast<omp::DeclareMapperOp>(Val: op); |
| 8853 | auto declMapperInfoOp = declMapperOp.getDeclareMapperInfo(); |
| 8854 | if (failed(Result: checkImplementationStatus(op&: *declMapperInfoOp))) |
| 8855 | return llvm::make_error<PreviouslyReportedError>(); |
| 8856 | |
| 8857 | DataLayout dl = DataLayout(declMapperOp->getParentOfType<ModuleOp>()); |
| 8858 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 8859 | llvm::Type *varType = moduleTranslation.convertType(type: declMapperOp.getType()); |
| 8860 | SmallVector<Value> mapVars = declMapperInfoOp.getMapVars(); |
| 8861 | |
| 8862 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 8863 | |
| 8864 | // Fill up the arrays with all the mapped variables. |
| 8865 | MapInfosTy combinedInfo; |
| 8866 | auto genMapInfoCB = |
| 8867 | [&](InsertPointTy codeGenIP, llvm::Value *ptrPHI, |
| 8868 | llvm::Value *unused2) -> llvm::OpenMPIRBuilder::MapInfosOrErrorTy { |
| 8869 | builder.restoreIP(IP: codeGenIP); |
| 8870 | moduleTranslation.mapValue(mlir: declMapperOp.getSymVal(), llvm: ptrPHI); |
| 8871 | moduleTranslation.mapBlock(mlir: &declMapperOp.getRegion().front(), |
| 8872 | llvm: builder.GetInsertBlock()); |
| 8873 | if (failed(Result: moduleTranslation.convertBlock(bb&: declMapperOp.getRegion().front(), |
| 8874 | /*ignoreArguments=*/true, |
| 8875 | builder))) |
| 8876 | return llvm::make_error<PreviouslyReportedError>(); |
| 8877 | MapInfoData mapData; |
| 8878 | collectMapDataFromMapOperands(mapData, mapVars, moduleTranslation, dl, |
| 8879 | builder); |
| 8880 | |
| 8881 | // Components produced by an `iterator` modifier on the mapper's own map |
| 8882 | // clause: their count is only known at runtime, so they are recorded as |
| 8883 | // dynamic segments and pushed one at a time (via |
| 8884 | // __tgt_push_mapper_component) from a generated loop, mirroring how the |
| 8885 | // depend/affinity clauses already handle iterator-produced entries. Each |
| 8886 | // per-iteration locator keeps its array base in var_ptr with bounds |
| 8887 | // selecting the mapped element, the same shape as an ordinary |
| 8888 | // array-section map, so the existing calculateBoundsOffset helper |
| 8889 | // computes its address. |
| 8890 | // |
| 8891 | // Like statically-known members (see processMapWithMembersOf), the |
| 8892 | // iterated elements are grouped under a component for the mapper variable |
| 8893 | // via MEMBER_OF, so the runtime treats them as sub-ranges of one |
| 8894 | // allocation rather than independent objects. If the mapper also maps the |
| 8895 | // variable itself that component is reused. Otherwise a storage-only |
| 8896 | // parent is synthesized that, like the parent of explicit members, spans |
| 8897 | // exactly the elements the iterators expand to: its begin/end are the |
| 8898 | // runtime min/max over all iterations, so it never extends a mapping of |
| 8899 | // the component the user may have made separately, and it is skipped |
| 8900 | // altogether when every iterator has a zero trip count (an empty iterator |
| 8901 | // acts as if the list item were omitted). The parent must precede every |
| 8902 | // other component: the runtime maps components in order and rejects a |
| 8903 | // later one that extends an earlier mapping. |
| 8904 | struct IteratedEntryInfo { |
| 8905 | mlir::omp::IteratorOp itersOp; |
| 8906 | mlir::omp::MapInfoOp mapInfoOp; |
| 8907 | llvm::Type *baseLLVMTy; |
| 8908 | llvm::Value *elemSizeVal; |
| 8909 | IteratorInfo iterInfo; |
| 8910 | }; |
| 8911 | llvm::SmallVector<IteratedEntryInfo, 2> iteratedEntries; |
| 8912 | for (mlir::Value iterVal : declMapperInfoOp.getMapIterated()) { |
| 8913 | auto itersOp = iterVal.getDefiningOp<mlir::omp::IteratorOp>(); |
| 8914 | assert(itersOp && "map_iterated value must be defined by omp.iterator" ); |
| 8915 | // Lowering the locator's indices (e.g. involving LEN_TRIM) can |
| 8916 | // introduce control flow inside the iterator, so its `omp.yield` is |
| 8917 | // not necessarily in the region's first block. |
| 8918 | mlir::omp::YieldOp yieldOp = itersOp.getYieldOp(); |
| 8919 | assert(yieldOp && "expected exactly one omp.yield in the iterator body" ); |
| 8920 | auto mapInfoOp = |
| 8921 | yieldOp.getResults()[0].getDefiningOp<mlir::omp::MapInfoOp>(); |
| 8922 | assert(mapInfoOp && "expected an omp.map.info in the map_iterated body" ); |
| 8923 | |
| 8924 | mlir::Type varPtrMlirTy = mapInfoOp.getVarPtrType(); |
| 8925 | // Byte size of one element of the locator's innermost dimension; the |
| 8926 | // bounds supply the (runtime) element count per iteration. |
| 8927 | uint64_t elemSizeInBits = dl.getTypeSizeInBits(t: varPtrMlirTy); |
| 8928 | if (auto arrTy = mlir::dyn_cast<mlir::LLVM::LLVMArrayType>(Val&: varPtrMlirTy)) |
| 8929 | elemSizeInBits = getArrayElementSizeInBits(arrTy, dl); |
| 8930 | iteratedEntries.push_back( |
| 8931 | Elt: {.itersOp: itersOp, .mapInfoOp: mapInfoOp, .baseLLVMTy: moduleTranslation.convertType(type: varPtrMlirTy), |
| 8932 | .elemSizeVal: builder.getInt64(C: elemSizeInBits / 8), |
| 8933 | .iterInfo: IteratorInfo(itersOp, moduleTranslation, builder)}); |
| 8934 | } |
| 8935 | |
| 8936 | // Emits the (base, begin, size) of the entry for one iteration, given the |
| 8937 | // loop's linear induction variable. `info` is taken by value: it holds |
| 8938 | // the lb/ub/step/trip llvm::Value*s looked up while `declMapperOp`'s |
| 8939 | // region mapping was still valid, and this may run later (from within |
| 8940 | // OpenMPIRBuilder::emitUserDefinedMapper), after that mapping has been |
| 8941 | // dropped via forgetMapping. |
| 8942 | auto genIteratedEntry = [&moduleTranslation](IteratedEntryInfo info, |
| 8943 | llvm::IRBuilderBase &b, |
| 8944 | llvm::Value *linearIV) |
| 8945 | -> llvm::Expected<llvm::SmallVector<llvm::Value *, 3>> { |
| 8946 | if (failed(Result: convertIteratorRegion(linearIV, iterInfo&: info.iterInfo, |
| 8947 | iteratorRegion&: info.itersOp.getRegion(), builder&: b, |
| 8948 | moduleTranslation))) |
| 8949 | return llvm::make_error<PreviouslyReportedError>(); |
| 8950 | llvm::Value *base = |
| 8951 | moduleTranslation.lookupValue(value: info.mapInfoOp.getVarPtr()); |
| 8952 | bool isByteOffset = false; |
| 8953 | std::vector<llvm::Value *> offsetIdx = calculateBoundsOffset( |
| 8954 | moduleTranslation, builder&: b, isArrayTy: info.baseLLVMTy->isArrayTy(), |
| 8955 | bounds: info.mapInfoOp.getBounds(), isByteOffset); |
| 8956 | llvm::Value *addr = base; |
| 8957 | if (!offsetIdx.empty()) |
| 8958 | addr = b.CreateInBoundsGEP( |
| 8959 | Ty: isByteOffset ? b.getInt8Ty() : info.baseLLVMTy, Ptr: base, IdxList: offsetIdx); |
| 8960 | // Size = product over dimensions of (ub - lb + 1), times the element |
| 8961 | // size, the same calculation getSizeInBytes does for static entries. |
| 8962 | llvm::Value *elementCount = b.getInt64(C: 1); |
| 8963 | for (mlir::Value bound : info.mapInfoOp.getBounds()) { |
| 8964 | auto boundOp = bound.getDefiningOp<mlir::omp::MapBoundsOp>(); |
| 8965 | if (!boundOp) |
| 8966 | continue; |
| 8967 | llvm::Value *lb = |
| 8968 | moduleTranslation.lookupValue(value: boundOp.getLowerBound()); |
| 8969 | llvm::Value *ub = |
| 8970 | moduleTranslation.lookupValue(value: boundOp.getUpperBound()); |
| 8971 | elementCount = b.CreateMul( |
| 8972 | LHS: elementCount, RHS: b.CreateAdd(LHS: b.CreateSub(LHS: ub, RHS: lb), RHS: b.getInt64(C: 1))); |
| 8973 | } |
| 8974 | llvm::Value *size = b.CreateMul(LHS: elementCount, RHS: info.elemSizeVal); |
| 8975 | moduleTranslation.forgetMapping(region&: info.itersOp.getRegion()); |
| 8976 | // The array base is the component's base pointer, as for a |
| 8977 | // statically-known member; MEMBER_OF ties it to the parent entry. |
| 8978 | return llvm::SmallVector<llvm::Value *, 3>{base, addr, size}; |
| 8979 | }; |
| 8980 | |
| 8981 | bool hasIterated = !iteratedEntries.empty(); |
| 8982 | std::optional<unsigned> iteratedParentIdx; |
| 8983 | // Span of the iterated elements, computed when the mapper does not map |
| 8984 | // the variable whole; null otherwise. |
| 8985 | llvm::Value *anyTrips = nullptr; |
| 8986 | llvm::Value *spanBegin = nullptr; |
| 8987 | llvm::Value *spanEnd = nullptr; |
| 8988 | bool hasStaticParent = false; |
| 8989 | if (hasIterated) { |
| 8990 | bool mapsWholeVariable = false; |
| 8991 | for (Value mapVar : mapVars) { |
| 8992 | auto mapOp = cast<omp::MapInfoOp>(Val: mapVar.getDefiningOp()); |
| 8993 | if (mapOp.getVarPtr() != declMapperOp.getSymVal()) |
| 8994 | continue; |
| 8995 | // A partial map with a single member is emitted as an individual map |
| 8996 | // of that member (see processMapWithMembersOf), so there is no parent |
| 8997 | // component to attach the iterated elements to. |
| 8998 | if (mapOp.getPartialMap() && mapOp.getMembers().size() == 1) |
| 8999 | continue; |
| 9000 | hasStaticParent = true; |
| 9001 | if (!mapOp.getPartialMap() && mapOp.getBounds().empty()) |
| 9002 | mapsWholeVariable = true; |
| 9003 | } |
| 9004 | if (!mapsWholeVariable) { |
| 9005 | // Span of all iterated elements as [spanBegin, spanEnd) integers, |
| 9006 | // meaningful only when anyTrips. A pre-pass loop over each segment's |
| 9007 | // iterations (the same ones the mapper pushes later) accumulates the |
| 9008 | // min begin / max end in two slots; createIteratorLoop carries no |
| 9009 | // loop state, so memory is used rather than PHIs. The allocas end |
| 9010 | // up in the entry block via hoistMapperAllocasToEntry. |
| 9011 | llvm::Type *i64Ty = builder.getInt64Ty(); |
| 9012 | anyTrips = builder.getFalse(); |
| 9013 | spanBegin = builder.getInt64(C: 0); |
| 9014 | spanEnd = builder.getInt64(C: 0); |
| 9015 | llvm::Value *beginSlot = |
| 9016 | builder.CreateAlloca(Ty: i64Ty, ArraySize: nullptr, Name: "omp.iterparent.begin" ); |
| 9017 | llvm::Value *endSlot = |
| 9018 | builder.CreateAlloca(Ty: i64Ty, ArraySize: nullptr, Name: "omp.iterparent.end" ); |
| 9019 | for (IteratedEntryInfo &info : iteratedEntries) { |
| 9020 | llvm::Value *trips = info.iterInfo.getTotalTrips(); |
| 9021 | llvm::Value *hasTrips = |
| 9022 | builder.CreateICmpNE(LHS: trips, RHS: builder.getInt64(C: 0)); |
| 9023 | builder.CreateStore(Val: builder.getInt64(C: ~0ULL), Ptr: beginSlot); |
| 9024 | builder.CreateStore(Val: builder.getInt64(C: 0), Ptr: endSlot); |
| 9025 | auto spanBodyGen = [&](InsertPointTy bodyIP, |
| 9026 | llvm::Value *linearIV) -> llvm::Error { |
| 9027 | builder.restoreIP(IP: bodyIP); |
| 9028 | llvm::Expected<llvm::SmallVector<llvm::Value *, 3>> entryOrErr = |
| 9029 | genIteratedEntry(info, builder, linearIV); |
| 9030 | if (!entryOrErr) |
| 9031 | return entryOrErr.takeError(); |
| 9032 | llvm::Value *begin = |
| 9033 | builder.CreatePtrToInt(V: (*entryOrErr)[1], DestTy: i64Ty); |
| 9034 | llvm::Value *end = builder.CreateAdd(LHS: begin, RHS: (*entryOrErr)[2]); |
| 9035 | builder.CreateStore(Val: builder.CreateBinaryIntrinsic( |
| 9036 | ID: llvm::Intrinsic::umin, |
| 9037 | LHS: builder.CreateLoad(Ty: i64Ty, Ptr: beginSlot), |
| 9038 | RHS: begin), |
| 9039 | Ptr: beginSlot); |
| 9040 | builder.CreateStore(Val: builder.CreateBinaryIntrinsic( |
| 9041 | ID: llvm::Intrinsic::umax, |
| 9042 | LHS: builder.CreateLoad(Ty: i64Ty, Ptr: endSlot), RHS: end), |
| 9043 | Ptr: endSlot); |
| 9044 | return llvm::Error::success(); |
| 9045 | }; |
| 9046 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 9047 | ompBuilder->createIteratorLoop( |
| 9048 | Loc: llvm::OpenMPIRBuilder::LocationDescription(builder), TripCount: trips, |
| 9049 | BodyGen: spanBodyGen, Name: "mapper.iterspan" ); |
| 9050 | if (!afterIP) |
| 9051 | return afterIP.takeError(); |
| 9052 | builder.restoreIP(IP: *afterIP); |
| 9053 | llvm::Value *segBegin = builder.CreateLoad(Ty: i64Ty, Ptr: beginSlot); |
| 9054 | llvm::Value *segEnd = builder.CreateLoad(Ty: i64Ty, Ptr: endSlot); |
| 9055 | // Fold this segment into the overall span unless it is empty. |
| 9056 | llvm::Value *mergedBegin = builder.CreateSelect( |
| 9057 | C: anyTrips, |
| 9058 | True: builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::umin, LHS: spanBegin, |
| 9059 | RHS: segBegin), |
| 9060 | False: segBegin); |
| 9061 | llvm::Value *mergedEnd = |
| 9062 | builder.CreateSelect(C: anyTrips, |
| 9063 | True: builder.CreateBinaryIntrinsic( |
| 9064 | ID: llvm::Intrinsic::umax, LHS: spanEnd, RHS: segEnd), |
| 9065 | False: segEnd); |
| 9066 | spanBegin = builder.CreateSelect(C: hasTrips, True: mergedBegin, False: spanBegin); |
| 9067 | spanEnd = builder.CreateSelect(C: hasTrips, True: mergedEnd, False: spanEnd); |
| 9068 | anyTrips = builder.CreateOr(LHS: anyTrips, RHS: hasTrips); |
| 9069 | } |
| 9070 | // The pre-pass loops split the block emitUserDefinedMapper was |
| 9071 | // emitting into; move its builder to the continuation as well. |
| 9072 | ompBuilder->Builder.restoreIP(IP: builder.saveIP()); |
| 9073 | |
| 9074 | if (!hasStaticParent) { |
| 9075 | // No map clause covers the variable, so synthesize a storage-only |
| 9076 | // parent for the span, pushed only when there are elements. |
| 9077 | iteratedParentIdx = 0; |
| 9078 | combinedInfo.BasePointers.push_back(Elt: ptrPHI); |
| 9079 | combinedInfo.Pointers.push_back( |
| 9080 | Elt: builder.CreateIntToPtr(V: spanBegin, DestTy: builder.getPtrTy())); |
| 9081 | combinedInfo.Sizes.push_back(Elt: builder.CreateSub(LHS: spanEnd, RHS: spanBegin)); |
| 9082 | combinedInfo.Types.push_back( |
| 9083 | Elt: llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE); |
| 9084 | combinedInfo.HasAttachPtr.push_back(Elt: false); |
| 9085 | combinedInfo.DevicePointers.push_back( |
| 9086 | Elt: llvm::OpenMPIRBuilder::DeviceInfoTy::None); |
| 9087 | combinedInfo.Mappers.push_back(Elt: nullptr); |
| 9088 | combinedInfo.Names.push_back(Elt: LLVM::createMappingInformation( |
| 9089 | loc: declMapperOp.getLoc(), builder&: *ompBuilder, mapName: declMapperOp.getSymName())); |
| 9090 | combinedInfo.LeadingParentCond = anyTrips; |
| 9091 | } |
| 9092 | } |
| 9093 | } |
| 9094 | |
| 9095 | genMapInfos(builder, moduleTranslation, dl, combinedInfo, mapData, |
| 9096 | targetDirective); |
| 9097 | |
| 9098 | llvm::omp::OpenMPOffloadMappingFlags iteratedMemberOf = |
| 9099 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE; |
| 9100 | if (hasIterated) { |
| 9101 | if (!iteratedParentIdx) { |
| 9102 | // The component genMapInfos emitted for the variable: the whole |
| 9103 | // variable, or the partial parent of its statically mapped members. |
| 9104 | // Members also have the variable as base, but carry MEMBER_OF. |
| 9105 | for (unsigned i = 0; i < combinedInfo.BasePointers.size(); ++i) { |
| 9106 | if (combinedInfo.BasePointers[i] == ptrPHI && |
| 9107 | (combinedInfo.Types[i] & |
| 9108 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) == |
| 9109 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE) { |
| 9110 | iteratedParentIdx = i; |
| 9111 | break; |
| 9112 | } |
| 9113 | } |
| 9114 | } |
| 9115 | assert(iteratedParentIdx && "no parent component for iterated entries" ); |
| 9116 | if (iteratedParentIdx && spanBegin && hasStaticParent) { |
| 9117 | // The partial parent spans only the static members. The runtime |
| 9118 | // rejects a later entry that extends an earlier one, so rather than |
| 9119 | // adding a second parent, widen this one to also cover the iterated |
| 9120 | // elements (when there are any). |
| 9121 | unsigned idx = *iteratedParentIdx; |
| 9122 | llvm::Type *i64Ty = builder.getInt64Ty(); |
| 9123 | llvm::Value *parentBegin = |
| 9124 | builder.CreatePtrToInt(V: combinedInfo.Pointers[idx], DestTy: i64Ty); |
| 9125 | llvm::Value *parentEnd = |
| 9126 | builder.CreateAdd(LHS: parentBegin, RHS: combinedInfo.Sizes[idx]); |
| 9127 | llvm::Value *newBegin = builder.CreateSelect( |
| 9128 | C: anyTrips, |
| 9129 | True: builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::umin, LHS: parentBegin, |
| 9130 | RHS: spanBegin), |
| 9131 | False: parentBegin, Name: "omp.iterparent.begin.merged" ); |
| 9132 | llvm::Value *newEnd = |
| 9133 | builder.CreateSelect(C: anyTrips, |
| 9134 | True: builder.CreateBinaryIntrinsic( |
| 9135 | ID: llvm::Intrinsic::umax, LHS: parentEnd, RHS: spanEnd), |
| 9136 | False: parentEnd, Name: "omp.iterparent.end.merged" ); |
| 9137 | combinedInfo.Pointers[idx] = |
| 9138 | builder.CreateIntToPtr(V: newBegin, DestTy: builder.getPtrTy()); |
| 9139 | combinedInfo.Sizes[idx] = builder.CreateSub(LHS: newEnd, RHS: newBegin); |
| 9140 | } |
| 9141 | if (iteratedParentIdx) |
| 9142 | iteratedMemberOf = ompBuilder->getMemberOfFlag(Position: *iteratedParentIdx); |
| 9143 | } |
| 9144 | |
| 9145 | for (IteratedEntryInfo &info : iteratedEntries) { |
| 9146 | llvm::omp::OpenMPOffloadMappingFlags mapType = |
| 9147 | convertClauseMapFlags(mlirFlags: info.mapInfoOp.getMapType()); |
| 9148 | ompBuilder->setCorrectMemberOfFlag(Flags&: mapType, MemberOfFlag: iteratedMemberOf); |
| 9149 | bool hasAttachPtr = |
| 9150 | (mapType & llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH) != |
| 9151 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE; |
| 9152 | |
| 9153 | combinedInfo.DynamicSegments.push_back(Elt: MapInfosTy::DynamicSegment{ |
| 9154 | .Count: info.iterInfo.getTotalTrips(), .GenEntry: {}, .Type: mapType, .HasAttachPtr: hasAttachPtr}); |
| 9155 | MapInfosTy::DynamicSegment &seg = combinedInfo.DynamicSegments.back(); |
| 9156 | seg.GenEntry = [genIteratedEntry, info](llvm::IRBuilderBase &b, |
| 9157 | llvm::Value *linearIV) { |
| 9158 | return genIteratedEntry(info, b, linearIV); |
| 9159 | }; |
| 9160 | |
| 9161 | if (info.mapInfoOp.getMapperId()) { |
| 9162 | auto childMapperOp = |
| 9163 | SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>( |
| 9164 | from: info.mapInfoOp, symbol: info.mapInfoOp.getMapperIdAttr()); |
| 9165 | // Emitting the child mapper for the first time leaves builder's |
| 9166 | // insertion point inside the child function; restore it so that |
| 9167 | // later iterator segments are emitted into this (the parent) |
| 9168 | // mapper function instead. |
| 9169 | llvm::IRBuilderBase::InsertPointGuard childMapperIPGuard(builder); |
| 9170 | llvm::Expected<llvm::Function *> childMapperFn = |
| 9171 | getOrCreateUserDefinedMapperFunc( |
| 9172 | op: childMapperOp, builder, moduleTranslation, targetDirective); |
| 9173 | if (!childMapperFn) |
| 9174 | return childMapperFn.takeError(); |
| 9175 | seg.ChildMapperFn = *childMapperFn; |
| 9176 | } |
| 9177 | } |
| 9178 | |
| 9179 | return combinedInfo; |
| 9180 | }; |
| 9181 | |
| 9182 | auto customMapperCB = [&](unsigned i) -> llvm::Expected<llvm::Function *> { |
| 9183 | if (!combinedInfo.Mappers[i]) |
| 9184 | return nullptr; |
| 9185 | return getOrCreateUserDefinedMapperFunc(op: combinedInfo.Mappers[i], builder, |
| 9186 | moduleTranslation, targetDirective); |
| 9187 | }; |
| 9188 | |
| 9189 | llvm::Expected<llvm::Function *> newFn = ompBuilder->emitUserDefinedMapper( |
| 9190 | PrivAndGenMapInfoCB: genMapInfoCB, ElemTy: varType, FuncName: mapperFuncName, CustomMapperCB: customMapperCB, |
| 9191 | /*PreserveMemberOfFlags=*/true, /*PropagatePresentToPointeeStorage=*/true, |
| 9192 | /*PropagatePresentToNonPointeeStorage=*/true); |
| 9193 | // Only now that emitUserDefinedMapper has fully finished (including running |
| 9194 | // any iterator-produced DynamicSegment::GenEntry callbacks, which look up |
| 9195 | // values defined in declMapperOp's region) is it safe to drop the mapping, |
| 9196 | // so that the same region can be processed multiple times. |
| 9197 | moduleTranslation.forgetMapping(region&: declMapperOp.getRegion()); |
| 9198 | if (!newFn) |
| 9199 | return newFn.takeError(); |
| 9200 | hoistMapperAllocasToEntry(mapperFn&: **newFn); |
| 9201 | if ([[maybe_unused]] llvm::Function *mappedFunc = |
| 9202 | moduleTranslation.lookupFunction(name: mapperFuncName)) { |
| 9203 | assert(mappedFunc == *newFn && |
| 9204 | "mapper function mapping disagrees with emitted function" ); |
| 9205 | } else { |
| 9206 | moduleTranslation.mapFunction(name: mapperFuncName, func: *newFn); |
| 9207 | } |
| 9208 | return *newFn; |
| 9209 | } |
| 9210 | |
| 9211 | static llvm::Value *getSourceLocIdentFromOp(llvm::IRBuilderBase &builder, |
| 9212 | llvm::OpenMPIRBuilder &ompBuilder, |
| 9213 | Operation *op) { |
| 9214 | auto fileLoc = op->getLoc()->findInstanceOf<FileLineColLoc>(); |
| 9215 | if (!fileLoc) |
| 9216 | return nullptr; |
| 9217 | uint32_t strSize; |
| 9218 | llvm::Function *parentFn = builder.GetInsertBlock()->getParent(); |
| 9219 | llvm::StringRef fnName = parentFn ? parentFn->getName() : "" ; |
| 9220 | llvm::Constant *srcStr = LLVM::createSourceLocStrFromLocation( |
| 9221 | loc: fileLoc, builder&: ompBuilder, name: fnName, strLen&: strSize); |
| 9222 | return ompBuilder.getOrCreateIdent(SrcLocStr: srcStr, SrcLocStrSize: strSize); |
| 9223 | } |
| 9224 | |
| 9225 | static LogicalResult |
| 9226 | convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder, |
| 9227 | LLVM::ModuleTranslation &moduleTranslation) { |
| 9228 | llvm::Value *ifCond = nullptr; |
| 9229 | llvm::Value *deviceID = builder.getInt64(C: llvm::omp::OMP_DEVICEID_UNDEF); |
| 9230 | SmallVector<Value> mapVars; |
| 9231 | SmallVector<Value> useDevicePtrVars; |
| 9232 | SmallVector<Value> useDeviceAddrVars; |
| 9233 | llvm::omp::RuntimeFunction RTLFn; |
| 9234 | DataLayout DL = DataLayout(op->getParentOfType<ModuleOp>()); |
| 9235 | TargetDirectiveEnumTy targetDirective = getTargetDirectiveEnumTyFromOp(op); |
| 9236 | |
| 9237 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 9238 | llvm::OpenMPIRBuilder::TargetDataInfo info( |
| 9239 | /*RequiresDevicePointerInfo=*/true, |
| 9240 | /*SeparateBeginEndCalls=*/true); |
| 9241 | |
| 9242 | if (ompBuilder->Config.isTargetDevice()) |
| 9243 | return op->emitOpError() << "not allowed in a target device" ; |
| 9244 | |
| 9245 | bool isOffloadEntry = !ompBuilder->Config.TargetTriples.empty(); |
| 9246 | |
| 9247 | auto getDeviceID = [&](mlir::Value dev) -> llvm::Value * { |
| 9248 | llvm::Value *v = moduleTranslation.lookupValue(value: dev); |
| 9249 | return builder.CreateIntCast(V: v, DestTy: builder.getInt64Ty(), /*isSigned=*/true); |
| 9250 | }; |
| 9251 | |
| 9252 | LogicalResult result = |
| 9253 | llvm::TypeSwitch<Operation *, LogicalResult>(op) |
| 9254 | .Case(caseFn: [&](omp::TargetDataOp dataOp) { |
| 9255 | if (failed(Result: checkImplementationStatus(op&: *dataOp))) |
| 9256 | return failure(); |
| 9257 | |
| 9258 | if (auto ifVar = dataOp.getIfExpr()) |
| 9259 | ifCond = moduleTranslation.lookupValue(value: ifVar); |
| 9260 | |
| 9261 | if (mlir::Value devId = dataOp.getDevice()) |
| 9262 | deviceID = getDeviceID(devId); |
| 9263 | |
| 9264 | mapVars = dataOp.getMapVars(); |
| 9265 | useDevicePtrVars = dataOp.getUseDevicePtrVars(); |
| 9266 | useDeviceAddrVars = dataOp.getUseDeviceAddrVars(); |
| 9267 | return success(); |
| 9268 | }) |
| 9269 | .Case(caseFn: [&](omp::TargetEnterDataOp enterDataOp) -> LogicalResult { |
| 9270 | if (failed(Result: checkImplementationStatus(op&: *enterDataOp))) |
| 9271 | return failure(); |
| 9272 | |
| 9273 | if (auto ifVar = enterDataOp.getIfExpr()) |
| 9274 | ifCond = moduleTranslation.lookupValue(value: ifVar); |
| 9275 | |
| 9276 | if (mlir::Value devId = enterDataOp.getDevice()) |
| 9277 | deviceID = getDeviceID(devId); |
| 9278 | |
| 9279 | RTLFn = |
| 9280 | enterDataOp.getNowait() |
| 9281 | ? llvm::omp::OMPRTL___tgt_target_data_begin_nowait_mapper |
| 9282 | : llvm::omp::OMPRTL___tgt_target_data_begin_mapper; |
| 9283 | mapVars = enterDataOp.getMapVars(); |
| 9284 | info.HasNoWait = enterDataOp.getNowait(); |
| 9285 | return success(); |
| 9286 | }) |
| 9287 | .Case(caseFn: [&](omp::TargetExitDataOp exitDataOp) -> LogicalResult { |
| 9288 | if (failed(Result: checkImplementationStatus(op&: *exitDataOp))) |
| 9289 | return failure(); |
| 9290 | |
| 9291 | if (auto ifVar = exitDataOp.getIfExpr()) |
| 9292 | ifCond = moduleTranslation.lookupValue(value: ifVar); |
| 9293 | |
| 9294 | if (mlir::Value devId = exitDataOp.getDevice()) |
| 9295 | deviceID = getDeviceID(devId); |
| 9296 | |
| 9297 | RTLFn = exitDataOp.getNowait() |
| 9298 | ? llvm::omp::OMPRTL___tgt_target_data_end_nowait_mapper |
| 9299 | : llvm::omp::OMPRTL___tgt_target_data_end_mapper; |
| 9300 | mapVars = exitDataOp.getMapVars(); |
| 9301 | info.HasNoWait = exitDataOp.getNowait(); |
| 9302 | return success(); |
| 9303 | }) |
| 9304 | .Case(caseFn: [&](omp::TargetUpdateOp updateDataOp) -> LogicalResult { |
| 9305 | if (failed(Result: checkImplementationStatus(op&: *updateDataOp))) |
| 9306 | return failure(); |
| 9307 | |
| 9308 | if (auto ifVar = updateDataOp.getIfExpr()) |
| 9309 | ifCond = moduleTranslation.lookupValue(value: ifVar); |
| 9310 | |
| 9311 | if (mlir::Value devId = updateDataOp.getDevice()) |
| 9312 | deviceID = getDeviceID(devId); |
| 9313 | |
| 9314 | RTLFn = |
| 9315 | updateDataOp.getNowait() |
| 9316 | ? llvm::omp::OMPRTL___tgt_target_data_update_nowait_mapper |
| 9317 | : llvm::omp::OMPRTL___tgt_target_data_update_mapper; |
| 9318 | mapVars = updateDataOp.getMapVars(); |
| 9319 | info.HasNoWait = updateDataOp.getNowait(); |
| 9320 | return success(); |
| 9321 | }) |
| 9322 | .DefaultUnreachable(message: "unexpected operation" ); |
| 9323 | |
| 9324 | if (failed(Result: result)) |
| 9325 | return failure(); |
| 9326 | // Pretend we have IF(false) if we're not doing offload. |
| 9327 | if (!isOffloadEntry) |
| 9328 | ifCond = builder.getFalse(); |
| 9329 | |
| 9330 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 9331 | MapInfoData mapData; |
| 9332 | collectMapDataFromMapOperands(mapData, mapVars, moduleTranslation, dl&: DL, |
| 9333 | builder, useDevPtrOperands: useDevicePtrVars, useDevAddrOperands: useDeviceAddrVars); |
| 9334 | |
| 9335 | // Fill up the arrays with all the mapped variables. |
| 9336 | MapInfosTy combinedInfo; |
| 9337 | auto genMapInfoCB = [&](InsertPointTy codeGenIP) -> MapInfosTy & { |
| 9338 | builder.restoreIP(IP: codeGenIP); |
| 9339 | genMapInfos(builder, moduleTranslation, dl&: DL, combinedInfo, mapData, |
| 9340 | targetDirective); |
| 9341 | return combinedInfo; |
| 9342 | }; |
| 9343 | |
| 9344 | // Define a lambda to apply mappings between use_device_addr and |
| 9345 | // use_device_ptr base pointers, and their associated block arguments. |
| 9346 | auto mapUseDevice = |
| 9347 | [&moduleTranslation]( |
| 9348 | llvm::OpenMPIRBuilder::DeviceInfoTy type, |
| 9349 | llvm::ArrayRef<BlockArgument> blockArgs, |
| 9350 | llvm::SmallVectorImpl<Value> &useDeviceVars, MapInfoData &mapInfoData, |
| 9351 | llvm::function_ref<llvm::Value *(llvm::Value *)> mapper = nullptr) { |
| 9352 | for (auto [arg, useDevVar] : |
| 9353 | llvm::zip_equal(t&: blockArgs, u&: useDeviceVars)) { |
| 9354 | |
| 9355 | auto getMapBasePtr = [](omp::MapInfoOp mapInfoOp) { |
| 9356 | return mapInfoOp.getVarPtrPtr() ? mapInfoOp.getVarPtrPtr() |
| 9357 | : mapInfoOp.getVarPtr(); |
| 9358 | }; |
| 9359 | |
| 9360 | auto useDevMap = cast<omp::MapInfoOp>(Val: useDevVar.getDefiningOp()); |
| 9361 | for (auto [mapClause, devicePointer, basePointer] : llvm::zip_equal( |
| 9362 | t&: mapInfoData.MapClause, u&: mapInfoData.DevicePointers, |
| 9363 | args&: mapInfoData.BasePointers)) { |
| 9364 | auto mapOp = cast<omp::MapInfoOp>(Val: mapClause); |
| 9365 | if (getMapBasePtr(mapOp) != getMapBasePtr(useDevMap) || |
| 9366 | devicePointer != type) |
| 9367 | continue; |
| 9368 | |
| 9369 | if (llvm::Value *devPtrInfoMap = |
| 9370 | mapper ? mapper(basePointer) : basePointer) { |
| 9371 | moduleTranslation.mapValue(mlir: arg, llvm: devPtrInfoMap); |
| 9372 | break; |
| 9373 | } |
| 9374 | } |
| 9375 | } |
| 9376 | }; |
| 9377 | |
| 9378 | using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy; |
| 9379 | auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType) |
| 9380 | -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { |
| 9381 | // We must always restoreIP regardless of doing anything the caller |
| 9382 | // does not restore it, leading to incorrect (no) branch generation. |
| 9383 | builder.restoreIP(IP: codeGenIP); |
| 9384 | assert(isa<omp::TargetDataOp>(op) && |
| 9385 | "BodyGen requested for non TargetDataOp" ); |
| 9386 | auto blockArgIface = cast<omp::BlockArgOpenMPOpInterface>(Val: op); |
| 9387 | Region ®ion = cast<omp::TargetDataOp>(Val: op).getRegion(); |
| 9388 | switch (bodyGenType) { |
| 9389 | case BodyGenTy::Priv: |
| 9390 | // Check if any device ptr/addr info is available |
| 9391 | if (!info.DevicePtrInfoMap.empty()) { |
| 9392 | mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Address, |
| 9393 | blockArgIface.getUseDeviceAddrBlockArgs(), |
| 9394 | useDeviceAddrVars, mapData, |
| 9395 | [&](llvm::Value *basePointer) -> llvm::Value * { |
| 9396 | if (!info.DevicePtrInfoMap[basePointer].second) |
| 9397 | return nullptr; |
| 9398 | return builder.CreateLoad( |
| 9399 | Ty: builder.getPtrTy(), |
| 9400 | Ptr: info.DevicePtrInfoMap[basePointer].second); |
| 9401 | }); |
| 9402 | mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer, |
| 9403 | blockArgIface.getUseDevicePtrBlockArgs(), useDevicePtrVars, |
| 9404 | mapData, [&](llvm::Value *basePointer) { |
| 9405 | return info.DevicePtrInfoMap[basePointer].second; |
| 9406 | }); |
| 9407 | |
| 9408 | if (failed(Result: inlineConvertOmpRegions(region, blockName: "omp.data.region" , builder, |
| 9409 | moduleTranslation))) |
| 9410 | return llvm::make_error<PreviouslyReportedError>(); |
| 9411 | } |
| 9412 | break; |
| 9413 | case BodyGenTy::DupNoPriv: |
| 9414 | if (info.DevicePtrInfoMap.empty()) { |
| 9415 | // For host device we still need to do the mapping for codegen, |
| 9416 | // otherwise it may try to lookup a missing value. |
| 9417 | mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Address, |
| 9418 | blockArgIface.getUseDeviceAddrBlockArgs(), |
| 9419 | useDeviceAddrVars, mapData); |
| 9420 | mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer, |
| 9421 | blockArgIface.getUseDevicePtrBlockArgs(), useDevicePtrVars, |
| 9422 | mapData); |
| 9423 | } |
| 9424 | break; |
| 9425 | case BodyGenTy::NoPriv: |
| 9426 | // If device info is available then region has already been generated |
| 9427 | if (info.DevicePtrInfoMap.empty()) { |
| 9428 | if (failed(Result: inlineConvertOmpRegions(region, blockName: "omp.data.region" , builder, |
| 9429 | moduleTranslation))) |
| 9430 | return llvm::make_error<PreviouslyReportedError>(); |
| 9431 | } |
| 9432 | break; |
| 9433 | } |
| 9434 | return builder.saveIP(); |
| 9435 | }; |
| 9436 | |
| 9437 | auto customMapperCB = |
| 9438 | [&](unsigned int i) -> llvm::Expected<llvm::Function *> { |
| 9439 | if (!combinedInfo.Mappers[i]) |
| 9440 | return nullptr; |
| 9441 | info.HasMapper = true; |
| 9442 | return getOrCreateUserDefinedMapperFunc(op: combinedInfo.Mappers[i], builder, |
| 9443 | moduleTranslation, targetDirective); |
| 9444 | }; |
| 9445 | |
| 9446 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 9447 | llvm::SmallVector<llvm::BasicBlock *> deallocBlocks; |
| 9448 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 9449 | findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks); |
| 9450 | |
| 9451 | // Pass the region's source location to the runtime, taken from the op's own |
| 9452 | // location; only offloading entries emit the mapper calls that consume it. |
| 9453 | // No need to also guard on !isTargetDevice here: this function bails out |
| 9454 | // earlier for the target device, so isOffloadEntry alone is sufficient. |
| 9455 | llvm::Value *srcLocOverride = |
| 9456 | isOffloadEntry ? getSourceLocIdentFromOp(builder, ompBuilder&: *ompBuilder, op) |
| 9457 | : nullptr; |
| 9458 | |
| 9459 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = [&]() { |
| 9460 | if (isa<omp::TargetDataOp>(Val: op)) |
| 9461 | return ompBuilder->createTargetData( |
| 9462 | Loc: ompLoc, AllocaIP: allocaIP, CodeGenIP: builder.saveIP(), DeallocBlocks: deallocBlocks, DeviceID: deviceID, IfCond: ifCond, |
| 9463 | Info&: info, GenMapInfoCB: genMapInfoCB, CustomMapperCB: customMapperCB, |
| 9464 | /*MapperFunc=*/nullptr, BodyGenCB: bodyGenCB, |
| 9465 | /*DeviceAddrCB=*/nullptr, SrcLocInfo: srcLocOverride); |
| 9466 | return ompBuilder->createTargetData( |
| 9467 | Loc: ompLoc, AllocaIP: allocaIP, CodeGenIP: builder.saveIP(), DeallocBlocks: deallocBlocks, DeviceID: deviceID, IfCond: ifCond, |
| 9468 | Info&: info, GenMapInfoCB: genMapInfoCB, CustomMapperCB: customMapperCB, MapperFunc: &RTLFn, |
| 9469 | /*BodyGenCB=*/nullptr, |
| 9470 | /*DeviceAddrCB=*/nullptr, SrcLocInfo: srcLocOverride); |
| 9471 | }(); |
| 9472 | |
| 9473 | if (failed(Result: handleError(result&: afterIP, op&: *op))) |
| 9474 | return failure(); |
| 9475 | |
| 9476 | builder.restoreIP(IP: *afterIP); |
| 9477 | return success(); |
| 9478 | } |
| 9479 | |
| 9480 | static LogicalResult |
| 9481 | convertOmpDistribute(Operation &opInst, llvm::IRBuilderBase &builder, |
| 9482 | LLVM::ModuleTranslation &moduleTranslation) { |
| 9483 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 9484 | auto distributeOp = cast<omp::DistributeOp>(Val&: opInst); |
| 9485 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 9486 | return failure(); |
| 9487 | |
| 9488 | /// Process teams op reduction in distribute if the reduction is contained in |
| 9489 | /// this specific distribute op. |
| 9490 | omp::TeamsOp teamsOp = opInst.getParentOfType<omp::TeamsOp>(); |
| 9491 | std::optional<ReductionVarsInfo> distributeRedInfo; |
| 9492 | if (teamsOp && |
| 9493 | getDistributeCapturingTeamsReduction(teamsOp) == distributeOp) { |
| 9494 | distributeRedInfo.emplace(args&: teamsOp); |
| 9495 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 9496 | findAllocInsertPoints(builder, moduleTranslation); |
| 9497 | |
| 9498 | if (failed(Result: allocAndInitializeReductionVars(op&: *teamsOp, builder, |
| 9499 | moduleTranslation, allocaIP, |
| 9500 | redInfo&: *distributeRedInfo))) |
| 9501 | return failure(); |
| 9502 | } |
| 9503 | |
| 9504 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 9505 | auto bodyGenCB = |
| 9506 | [&](InsertPointTy allocaIP, InsertPointTy codeGenIP, |
| 9507 | llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error { |
| 9508 | llvm::BasicBlock *allocaBlock = allocaIP.getNodeParent(); |
| 9509 | // DistributeOp has only one region associated with it. |
| 9510 | builder.restoreIP(IP: codeGenIP); |
| 9511 | PrivateVarsInfo privVarsInfo(distributeOp); |
| 9512 | |
| 9513 | llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars( |
| 9514 | op: distributeOp, builder, moduleTranslation, privateVarsInfo&: privVarsInfo, allocaIP); |
| 9515 | if (handleError(result&: afterAllocas, op&: opInst).failed()) |
| 9516 | return llvm::make_error<PreviouslyReportedError>(); |
| 9517 | |
| 9518 | // Save the alloca insertion point on ModuleTranslation stack for use in |
| 9519 | // nested regions. This must be after allocatePrivateVars, which splits |
| 9520 | // the alloca block and updates allocaIP. |
| 9521 | LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame( |
| 9522 | moduleTranslation, allocaIP.getNodeParent(), deallocBlocks); |
| 9523 | |
| 9524 | if (handleError(error: initPrivateVars(builder, moduleTranslation, privateVarsInfo&: privVarsInfo), |
| 9525 | op&: opInst) |
| 9526 | .failed()) |
| 9527 | return llvm::make_error<PreviouslyReportedError>(); |
| 9528 | |
| 9529 | if (failed(Result: completePrivateVars(op: distributeOp, builder, moduleTranslation, |
| 9530 | privateVarsInfo: privVarsInfo, |
| 9531 | insertBarrier: distributeOp.getPrivateNeedsBarrier()))) |
| 9532 | return llvm::make_error<PreviouslyReportedError>(); |
| 9533 | |
| 9534 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 9535 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 9536 | llvm::Expected<llvm::BasicBlock *> regionBlock = |
| 9537 | convertOmpOpRegions(region&: distributeOp.getRegion(), blockName: "omp.distribute.region" , |
| 9538 | builder, moduleTranslation); |
| 9539 | if (!regionBlock) |
| 9540 | return regionBlock.takeError(); |
| 9541 | builder.SetInsertPoint((*regionBlock)->begin()); |
| 9542 | |
| 9543 | // Skip applying a workshare loop below when translating 'distribute |
| 9544 | // parallel do' (it's been already handled by this point while translating |
| 9545 | // the nested omp.wsloop). |
| 9546 | if (!isa_and_present<omp::WsloopOp>(Val: distributeOp.getNestedWrapper())) { |
| 9547 | // TODO: Add support for clauses which are valid for DISTRIBUTE |
| 9548 | // constructs. Static schedule is the default. |
| 9549 | bool hasDistSchedule = distributeOp.getDistScheduleStatic(); |
| 9550 | auto schedule = hasDistSchedule ? omp::ClauseScheduleKind::Distribute |
| 9551 | : omp::ClauseScheduleKind::Static; |
| 9552 | // dist_schedule clauses are ordered - otherise this should be false |
| 9553 | bool isOrdered = hasDistSchedule; |
| 9554 | std::optional<omp::ScheduleModifier> scheduleMod; |
| 9555 | bool isSimd = false; |
| 9556 | llvm::omp::WorksharingLoopType workshareLoopType = |
| 9557 | llvm::omp::WorksharingLoopType::DistributeStaticLoop; |
| 9558 | bool loopNeedsBarrier = false; |
| 9559 | llvm::Value *chunk = moduleTranslation.lookupValue( |
| 9560 | value: distributeOp.getDistScheduleChunkSize()); |
| 9561 | llvm::CanonicalLoopInfo *loopInfo = |
| 9562 | findCurrentLoopInfo(moduleTranslation); |
| 9563 | // Nested regions can move the instruction referenced by allocaIP. |
| 9564 | allocaIP = allocaBlock->getFirstInsertionPt(); |
| 9565 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP = |
| 9566 | ompBuilder->applyWorkshareLoop( |
| 9567 | DL: ompLoc.DL, CLI: loopInfo, AllocaIP: allocaIP, NeedsBarrier: loopNeedsBarrier, |
| 9568 | SchedKind: convertToScheduleKind(schedKind: schedule), ChunkSize: chunk, HasSimdModifier: isSimd, |
| 9569 | HasMonotonicModifier: scheduleMod == omp::ScheduleModifier::monotonic, |
| 9570 | HasNonmonotonicModifier: scheduleMod == omp::ScheduleModifier::nonmonotonic, HasOrderedClause: isOrdered, |
| 9571 | LoopType: workshareLoopType, NoLoop: false, HasDistSchedule: hasDistSchedule, DistScheduleChunkSize: chunk); |
| 9572 | |
| 9573 | if (!wsloopIP) |
| 9574 | return wsloopIP.takeError(); |
| 9575 | } |
| 9576 | if (failed(Result: cleanupPrivateVars(op: distributeOp, builder, moduleTranslation, |
| 9577 | loc: distributeOp.getLoc(), privateVarsInfo&: privVarsInfo))) |
| 9578 | return llvm::make_error<PreviouslyReportedError>(); |
| 9579 | |
| 9580 | return llvm::Error::success(); |
| 9581 | }; |
| 9582 | |
| 9583 | llvm::SmallVector<llvm::BasicBlock *> deallocBlocks; |
| 9584 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 9585 | findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks); |
| 9586 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 9587 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 9588 | ompBuilder->createDistribute(Loc: ompLoc, AllocaIP: allocaIP, DeallocBlocks: deallocBlocks, BodyGenCB: bodyGenCB); |
| 9589 | |
| 9590 | if (failed(Result: handleError(result&: afterIP, op&: opInst))) |
| 9591 | return failure(); |
| 9592 | |
| 9593 | builder.restoreIP(IP: *afterIP); |
| 9594 | |
| 9595 | if (distributeRedInfo) { |
| 9596 | // Process the reductions if required. |
| 9597 | return createReductionsAndCleanup(op&: *teamsOp, builder, moduleTranslation, |
| 9598 | allocaIP, redInfo&: *distributeRedInfo, |
| 9599 | /*isNoWait=*/isNowait: false, |
| 9600 | /*isTeamsReduction=*/true); |
| 9601 | } |
| 9602 | return success(); |
| 9603 | } |
| 9604 | |
| 9605 | /// Lowers the FlagsAttr which is applied to the module when offloading. This |
| 9606 | /// attribute contains OpenMP RTL globals that can be passed as flags to the |
| 9607 | /// frontend, otherwise they are set to default |
| 9608 | static LogicalResult |
| 9609 | convertFlagsAttr(Operation *op, mlir::omp::FlagsAttr attribute, |
| 9610 | LLVM::ModuleTranslation &moduleTranslation) { |
| 9611 | auto offloadMod = dyn_cast<omp::OffloadModuleInterface>(Val: op); |
| 9612 | if (!offloadMod) |
| 9613 | return op->emitOpError() << "omp flags attached to non offload module op" ; |
| 9614 | |
| 9615 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 9616 | |
| 9617 | if (offloadMod.getIsTargetDevice()) |
| 9618 | ompBuilder->M.addModuleFlag(Behavior: llvm::Module::Max, Key: "openmp-device" , |
| 9619 | Val: attribute.getOpenmpDeviceVersion()); |
| 9620 | |
| 9621 | // The flags below are only intended to be emitted for GPU offload targets. |
| 9622 | if (!offloadMod.getIsGPU()) |
| 9623 | return success(); |
| 9624 | |
| 9625 | if (attribute.getNoGpuLib()) |
| 9626 | return success(); |
| 9627 | |
| 9628 | ompBuilder->createGlobalFlag(Value: attribute.getDebugKind(), |
| 9629 | Name: "__omp_rtl_debug_kind" ); |
| 9630 | ompBuilder->createGlobalFlag(Value: attribute.getAssumeTeamsOversubscription(), |
| 9631 | Name: "__omp_rtl_assume_teams_oversubscription" ); |
| 9632 | ompBuilder->createGlobalFlag(Value: attribute.getAssumeThreadsOversubscription(), |
| 9633 | Name: "__omp_rtl_assume_threads_oversubscription" ); |
| 9634 | ompBuilder->createGlobalFlag(Value: attribute.getAssumeNoThreadState(), |
| 9635 | Name: "__omp_rtl_assume_no_thread_state" ); |
| 9636 | ompBuilder->createGlobalFlag(Value: attribute.getAssumeNoNestedParallelism(), |
| 9637 | Name: "__omp_rtl_assume_no_nested_parallelism" ); |
| 9638 | return success(); |
| 9639 | } |
| 9640 | |
| 9641 | static void getTargetEntryUniqueInfo(llvm::TargetRegionEntryInfo &targetInfo, |
| 9642 | omp::TargetOp targetOp, |
| 9643 | llvm::OpenMPIRBuilder &ompBuilder, |
| 9644 | llvm::vfs::FileSystem &vfs, |
| 9645 | llvm::StringRef parentName = "" ) { |
| 9646 | auto fileLoc = targetOp.getLoc()->findInstanceOf<FileLineColLoc>(); |
| 9647 | assert(fileLoc && "No file found from location" ); |
| 9648 | |
| 9649 | auto fileInfoCallBack = [&fileLoc]() { |
| 9650 | return std::pair<std::string, uint64_t>( |
| 9651 | llvm::StringRef(fileLoc.getFilename()), fileLoc.getLine()); |
| 9652 | }; |
| 9653 | |
| 9654 | targetInfo = |
| 9655 | ompBuilder.getTargetEntryUniqueInfo(CallBack: fileInfoCallBack, VFS&: vfs, ParentName: parentName); |
| 9656 | } |
| 9657 | |
| 9658 | // The createDeviceArgumentAccessor function generates |
| 9659 | // instructions for retrieving (acessing) kernel |
| 9660 | // arguments inside of the device kernel for use by |
| 9661 | // the kernel. This enables different semantics such as |
| 9662 | // the creation of temporary copies of data allowing |
| 9663 | // semantics like read-only/no host write back kernel |
| 9664 | // arguments. |
| 9665 | // |
| 9666 | // This currently implements a very light version of Clang's |
| 9667 | // EmitParmDecl's handling of direct argument handling as well |
| 9668 | // as a portion of the argument access generation based on |
| 9669 | // capture types found at the end of emitOutlinedFunctionPrologue |
| 9670 | // in Clang. The indirect path handling of EmitParmDecl's may be |
| 9671 | // required for future work, but a direct 1-to-1 copy doesn't seem |
| 9672 | // possible as the logic is rather scattered throughout Clang's |
| 9673 | // lowering and perhaps we wish to deviate slightly. |
| 9674 | // |
| 9675 | // \param mapData - A container containing vectors of information |
| 9676 | // corresponding to the input argument, which should have a |
| 9677 | // corresponding entry in the MapInfoData containers |
| 9678 | // OrigialValue's. |
| 9679 | // \param arg - This is the generated kernel function argument that |
| 9680 | // corresponds to the passed in input argument. We generated different |
| 9681 | // accesses of this Argument, based on capture type and other Input |
| 9682 | // related information. |
| 9683 | // \param input - This is the host side value that will be passed to |
| 9684 | // the kernel i.e. the kernel input, we rewrite all uses of this within |
| 9685 | // the kernel (as we generate the kernel body based on the target's region |
| 9686 | // which maintians references to the original input) to the retVal argument |
| 9687 | // apon exit of this function inside of the OMPIRBuilder. This interlinks |
| 9688 | // the kernel argument to future uses of it in the function providing |
| 9689 | // appropriate "glue" instructions inbetween. |
| 9690 | // \param retVal - This is the value that all uses of input inside of the |
| 9691 | // kernel will be re-written to, the goal of this function is to generate |
| 9692 | // an appropriate location for the kernel argument to be accessed from, |
| 9693 | // e.g. ByRef will result in a temporary allocation location and then |
| 9694 | // a store of the kernel argument into this allocated memory which |
| 9695 | // will then be loaded from, ByCopy will use the allocated memory |
| 9696 | // directly. |
| 9697 | static llvm::IRBuilderBase::InsertPoint createDeviceArgumentAccessor( |
| 9698 | omp::TargetOp targetOp, MapInfoData &mapData, llvm::Argument &arg, |
| 9699 | llvm::Value *input, llvm::Value *&retVal, llvm::IRBuilderBase &builder, |
| 9700 | llvm::OpenMPIRBuilder &ompBuilder, |
| 9701 | LLVM::ModuleTranslation &moduleTranslation, |
| 9702 | llvm::IRBuilderBase::InsertPoint allocaIP, |
| 9703 | llvm::IRBuilderBase::InsertPoint codeGenIP, |
| 9704 | llvm::ArrayRef<llvm::IRBuilderBase::InsertPoint> deallocIPs) { |
| 9705 | assert(ompBuilder.Config.isTargetDevice() && |
| 9706 | "function only supported for target device codegen" ); |
| 9707 | builder.restoreIP(IP: allocaIP); |
| 9708 | |
| 9709 | omp::VariableCaptureKind capture = omp::VariableCaptureKind::ByRef; |
| 9710 | LLVM::TypeToLLVMIRTranslator typeToLLVMIRTranslator( |
| 9711 | ompBuilder.M.getContext()); |
| 9712 | unsigned alignmentValue = 0; |
| 9713 | BlockArgument mlirArg; |
| 9714 | SmallVector<std::pair<Value, BlockArgument>> blockArgsPairs; |
| 9715 | cast<omp::BlockArgOpenMPOpInterface>(Val&: *targetOp).getBlockArgsPairs( |
| 9716 | pairs&: blockArgsPairs); |
| 9717 | // Find the associated MapInfoData entry for the current input |
| 9718 | for (size_t i = 0; i < mapData.MapClause.size(); ++i) { |
| 9719 | if (mapData.OriginalValue[i] == input) { |
| 9720 | auto mapOp = cast<omp::MapInfoOp>(Val: mapData.MapClause[i]); |
| 9721 | capture = mapOp.getMapCaptureType(); |
| 9722 | // Get information of alignment of mapped object |
| 9723 | alignmentValue = typeToLLVMIRTranslator.getPreferredAlignment( |
| 9724 | type: mapOp.getVarPtrType(), layout: ompBuilder.M.getDataLayout()); |
| 9725 | |
| 9726 | // Find the corresponding entry block argument, which can be associated to |
| 9727 | // a map, use_device* or has_device* clause. |
| 9728 | for (auto &[val, arg] : blockArgsPairs) { |
| 9729 | if (mapOp.getResult() == val) { |
| 9730 | mlirArg = arg; |
| 9731 | break; |
| 9732 | } |
| 9733 | } |
| 9734 | assert(mlirArg && "expected to find entry block argument for map clause" ); |
| 9735 | break; |
| 9736 | } |
| 9737 | } |
| 9738 | |
| 9739 | unsigned int allocaAS = ompBuilder.M.getDataLayout().getAllocaAddrSpace(); |
| 9740 | unsigned int defaultAS = |
| 9741 | ompBuilder.M.getDataLayout().getProgramAddressSpace(); |
| 9742 | |
| 9743 | // Create the allocation for the argument. |
| 9744 | llvm::Value *v = nullptr; |
| 9745 | if (omp::opInSharedDeviceContext(op&: *targetOp) && |
| 9746 | omp::allocaUsesRequireSharedMem(alloc: mlirArg)) { |
| 9747 | // Use the beginning of the codeGenIP rather than the usual allocation point |
| 9748 | // for shared memory allocations because otherwise these would be done prior |
| 9749 | // to the target initialization call. Also, the exit block (where the |
| 9750 | // deallocation is placed) is only executed if the initialization call |
| 9751 | // succeeds. |
| 9752 | builder.SetInsertPoint(codeGenIP.getNodeParent()->getFirstInsertionPt()); |
| 9753 | v = ompBuilder.createOMPAllocShared(Loc: builder, VarType: arg.getType()); |
| 9754 | |
| 9755 | // Create deallocations in all provided deallocation points and then restore |
| 9756 | // the insertion point to right after the new allocations. |
| 9757 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 9758 | for (auto deallocIP : deallocIPs) { |
| 9759 | builder.SetInsertPoint(deallocIP); |
| 9760 | ompBuilder.createOMPFreeShared(Loc: builder, Addr: v, VarType: arg.getType()); |
| 9761 | } |
| 9762 | } else { |
| 9763 | // Use the current point, which was previously set to allocaIP. |
| 9764 | v = builder.CreateAlloca(Ty: arg.getType(), AddrSpace: allocaAS); |
| 9765 | |
| 9766 | if (allocaAS != defaultAS && arg.getType()->isPointerTy()) |
| 9767 | v = builder.CreateAddrSpaceCast(V: v, DestTy: builder.getPtrTy(AddrSpace: defaultAS)); |
| 9768 | } |
| 9769 | |
| 9770 | builder.CreateStore(Val: &arg, Ptr: v); |
| 9771 | |
| 9772 | builder.restoreIP(IP: codeGenIP); |
| 9773 | |
| 9774 | switch (capture) { |
| 9775 | case omp::VariableCaptureKind::ByCopy: { |
| 9776 | retVal = v; |
| 9777 | break; |
| 9778 | } |
| 9779 | case omp::VariableCaptureKind::ByRef: { |
| 9780 | llvm::LoadInst *loadInst = builder.CreateAlignedLoad( |
| 9781 | Ty: v->getType(), Ptr: v, |
| 9782 | Align: ompBuilder.M.getDataLayout().getPrefTypeAlign(Ty: v->getType())); |
| 9783 | // CreateAlignedLoad function creates similar LLVM IR: |
| 9784 | // %res = load ptr, ptr %input, align 8 |
| 9785 | // This LLVM IR does not contain information about alignment |
| 9786 | // of the loaded value. We need to add !align metadata to unblock |
| 9787 | // optimizer. The existence of the !align metadata on the instruction |
| 9788 | // tells the optimizer that the value loaded is known to be aligned to |
| 9789 | // a boundary specified by the integer value in the metadata node. |
| 9790 | // Example: |
| 9791 | // %res = load ptr, ptr %input, align 8, !align !align_md_node |
| 9792 | // ^ ^ |
| 9793 | // | | |
| 9794 | // alignment of %input address | |
| 9795 | // | |
| 9796 | // alignment of %res object |
| 9797 | if (v->getType()->isPointerTy() && alignmentValue) { |
| 9798 | llvm::MDBuilder MDB(builder.getContext()); |
| 9799 | loadInst->setMetadata( |
| 9800 | KindID: llvm::LLVMContext::MD_align, |
| 9801 | Node: llvm::MDNode::get(Context&: builder.getContext(), |
| 9802 | MDs: MDB.createConstant(C: llvm::ConstantInt::get( |
| 9803 | Ty: llvm::Type::getInt64Ty(C&: builder.getContext()), |
| 9804 | V: alignmentValue)))); |
| 9805 | } |
| 9806 | retVal = loadInst; |
| 9807 | |
| 9808 | break; |
| 9809 | } |
| 9810 | case omp::VariableCaptureKind::This: |
| 9811 | case omp::VariableCaptureKind::VLAType: |
| 9812 | // TODO: Consider returning error to use standard reporting for |
| 9813 | // unimplemented features. |
| 9814 | assert(false && "Currently unsupported capture kind" ); |
| 9815 | break; |
| 9816 | } |
| 9817 | |
| 9818 | return builder.saveIP(); |
| 9819 | } |
| 9820 | |
| 9821 | /// Follow uses of `host_eval`-defined block arguments of the given `omp.target` |
| 9822 | /// operation and populate output variables with their corresponding host value |
| 9823 | /// (i.e. operand evaluated outside of the target region), based on their uses |
| 9824 | /// inside of the target region. |
| 9825 | /// |
| 9826 | /// Loop bounds and steps are only optionally populated, if output vectors are |
| 9827 | /// provided. |
| 9828 | static void |
| 9829 | (omp::TargetOp targetOp, Value &numThreads, |
| 9830 | Value &numTeamsLower, Value &numTeamsUpper, |
| 9831 | Value &threadLimit, |
| 9832 | llvm::SmallVectorImpl<Value> *lowerBounds = nullptr, |
| 9833 | llvm::SmallVectorImpl<Value> *upperBounds = nullptr, |
| 9834 | llvm::SmallVectorImpl<Value> *steps = nullptr) { |
| 9835 | auto blockArgIface = llvm::cast<omp::BlockArgOpenMPOpInterface>(Val&: *targetOp); |
| 9836 | for (auto item : llvm::zip_equal(t: targetOp.getHostEvalVars(), |
| 9837 | u: blockArgIface.getHostEvalBlockArgs())) { |
| 9838 | Value hostEvalVar = std::get<0>(t&: item), blockArg = std::get<1>(t&: item); |
| 9839 | |
| 9840 | for (Operation *user : blockArg.getUsers()) { |
| 9841 | llvm::TypeSwitch<Operation *>(user) |
| 9842 | .Case(caseFn: [&](omp::TeamsOp teamsOp) { |
| 9843 | if (teamsOp.getNumTeamsLower() == blockArg) |
| 9844 | numTeamsLower = hostEvalVar; |
| 9845 | else if (llvm::is_contained(Range: teamsOp.getNumTeamsUpperVars(), |
| 9846 | Element: blockArg)) |
| 9847 | numTeamsUpper = hostEvalVar; |
| 9848 | else if (!teamsOp.getThreadLimitVars().empty() && |
| 9849 | teamsOp.getThreadLimit(dim: 0) == blockArg) |
| 9850 | threadLimit = hostEvalVar; |
| 9851 | else |
| 9852 | llvm_unreachable("unsupported host_eval use" ); |
| 9853 | }) |
| 9854 | .Case(caseFn: [&](omp::ParallelOp parallelOp) { |
| 9855 | if (!parallelOp.getNumThreadsVars().empty() && |
| 9856 | parallelOp.getNumThreads(dim: 0) == blockArg) |
| 9857 | numThreads = hostEvalVar; |
| 9858 | else |
| 9859 | llvm_unreachable("unsupported host_eval use" ); |
| 9860 | }) |
| 9861 | .Case(caseFn: [&](omp::LoopNestOp loopOp) { |
| 9862 | auto processBounds = |
| 9863 | [&](OperandRange opBounds, |
| 9864 | llvm::SmallVectorImpl<Value> *outBounds) -> bool { |
| 9865 | bool found = false; |
| 9866 | for (auto [i, lb] : llvm::enumerate(First&: opBounds)) { |
| 9867 | if (lb == blockArg) { |
| 9868 | found = true; |
| 9869 | if (outBounds) |
| 9870 | (*outBounds)[i] = hostEvalVar; |
| 9871 | } |
| 9872 | } |
| 9873 | return found; |
| 9874 | }; |
| 9875 | bool found = |
| 9876 | processBounds(loopOp.getLoopLowerBounds(), lowerBounds); |
| 9877 | found = processBounds(loopOp.getLoopUpperBounds(), upperBounds) || |
| 9878 | found; |
| 9879 | found = processBounds(loopOp.getLoopSteps(), steps) || found; |
| 9880 | (void)found; |
| 9881 | assert(found && "unsupported host_eval use" ); |
| 9882 | }) |
| 9883 | .DefaultUnreachable(message: "unsupported host_eval use" ); |
| 9884 | } |
| 9885 | } |
| 9886 | } |
| 9887 | |
| 9888 | /// If \p op is of the given type parameter, return it casted to that type. |
| 9889 | /// Otherwise, if its immediate parent operation (or some other higher-level |
| 9890 | /// parent, if \p immediateParent is false) is of that type, return that parent |
| 9891 | /// casted to the given type. |
| 9892 | /// |
| 9893 | /// If \p op is \c null or neither it or its parent(s) are of the specified |
| 9894 | /// type, return a \c null operation. |
| 9895 | template <typename OpTy> |
| 9896 | static OpTy castOrGetParentOfType(Operation *op, bool immediateParent = false) { |
| 9897 | if (!op) |
| 9898 | return OpTy(); |
| 9899 | |
| 9900 | if (OpTy casted = dyn_cast<OpTy>(op)) |
| 9901 | return casted; |
| 9902 | |
| 9903 | if (immediateParent) |
| 9904 | return dyn_cast_if_present<OpTy>(op->getParentOp()); |
| 9905 | |
| 9906 | return op->getParentOfType<OpTy>(); |
| 9907 | } |
| 9908 | |
| 9909 | /// If the given \p value is defined by an \c llvm.mlir.constant operation and |
| 9910 | /// it is of an integer type, return its value. |
| 9911 | static std::optional<int64_t> (Value value) { |
| 9912 | if (!value) |
| 9913 | return std::nullopt; |
| 9914 | |
| 9915 | if (auto constOp = value.getDefiningOp<LLVM::ConstantOp>()) |
| 9916 | if (auto constAttr = dyn_cast<IntegerAttr>(Val: constOp.getValue())) |
| 9917 | return constAttr.getInt(); |
| 9918 | |
| 9919 | return std::nullopt; |
| 9920 | } |
| 9921 | |
| 9922 | static uint64_t getTypeByteSize(mlir::Type type, const DataLayout &dl) { |
| 9923 | uint64_t sizeInBits = dl.getTypeSizeInBits(t: type); |
| 9924 | uint64_t sizeInBytes = sizeInBits / 8; |
| 9925 | return sizeInBytes; |
| 9926 | } |
| 9927 | |
| 9928 | template <typename OpTy> |
| 9929 | static uint64_t getReductionDataSize(OpTy &op) { |
| 9930 | if (op.getNumReductionVars() > 0) { |
| 9931 | SmallVector<omp::DeclareReductionOp> reductions; |
| 9932 | collectReductionDecls(op, reductions); |
| 9933 | |
| 9934 | llvm::SmallVector<mlir::Type> members; |
| 9935 | members.reserve(N: reductions.size()); |
| 9936 | for (omp::DeclareReductionOp &red : reductions) { |
| 9937 | // For by-ref reductions, use the actual element type rather than the |
| 9938 | // pointer type so that the buffer size matches the access pattern in |
| 9939 | // the copy/reduce callbacks generated by OMPIRBuilder. |
| 9940 | if (red.getByrefElementType()) |
| 9941 | members.push_back(Elt: *red.getByrefElementType()); |
| 9942 | else |
| 9943 | members.push_back(Elt: red.getType()); |
| 9944 | } |
| 9945 | Operation *opp = op.getOperation(); |
| 9946 | auto structType = mlir::LLVM::LLVMStructType::getLiteral( |
| 9947 | context: opp->getContext(), types: members, /*isPacked=*/false); |
| 9948 | DataLayout dl = DataLayout(opp->getParentOfType<ModuleOp>()); |
| 9949 | return getTypeByteSize(type: structType, dl); |
| 9950 | } |
| 9951 | return 0; |
| 9952 | } |
| 9953 | |
| 9954 | /// Populate default `MinTeams`, `MaxTeams` and `MaxThreads` to their default |
| 9955 | /// values as stated by the corresponding clauses, if constant. |
| 9956 | /// |
| 9957 | /// These default values must be set before the creation of the outlined LLVM |
| 9958 | /// function for the target region, so that they can be used to initialize the |
| 9959 | /// corresponding global `ConfigurationEnvironmentTy` structure. |
| 9960 | static void |
| 9961 | initTargetDefaultAttrs(omp::TargetOp targetOp, Operation *capturedOp, |
| 9962 | llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &attrs, |
| 9963 | bool isTargetDevice, bool isGPU) { |
| 9964 | // TODO: Handle constant 'if' clauses. |
| 9965 | |
| 9966 | Value numThreads, numTeamsLower, numTeamsUpper, threadLimit; |
| 9967 | if (!isTargetDevice) { |
| 9968 | extractHostEvalClauses(targetOp, numThreads, numTeamsLower, numTeamsUpper, |
| 9969 | threadLimit); |
| 9970 | } else { |
| 9971 | // In the target device, values for these clauses are not passed as |
| 9972 | // host_eval, but instead evaluated prior to entry to the region. This |
| 9973 | // ensures values are mapped and available inside of the target region. |
| 9974 | if (auto teamsOp = castOrGetParentOfType<omp::TeamsOp>(op: capturedOp)) { |
| 9975 | numTeamsLower = teamsOp.getNumTeamsLower(); |
| 9976 | // Handle num_teams upper bounds (only first value for now) |
| 9977 | if (!teamsOp.getNumTeamsUpperVars().empty()) |
| 9978 | numTeamsUpper = teamsOp.getNumTeams(dim: 0); |
| 9979 | if (!teamsOp.getThreadLimitVars().empty()) |
| 9980 | threadLimit = teamsOp.getThreadLimit(dim: 0); |
| 9981 | } |
| 9982 | |
| 9983 | if (auto parallelOp = castOrGetParentOfType<omp::ParallelOp>(op: capturedOp)) { |
| 9984 | if (!parallelOp.getNumThreadsVars().empty()) |
| 9985 | numThreads = parallelOp.getNumThreads(dim: 0); |
| 9986 | } |
| 9987 | } |
| 9988 | |
| 9989 | // Handle clauses impacting the number of teams. |
| 9990 | |
| 9991 | int32_t minTeamsVal = 1, maxTeamsVal = -1; |
| 9992 | if (castOrGetParentOfType<omp::TeamsOp>(op: capturedOp)) { |
| 9993 | // TODO: Use `hostNumTeamsLower` to initialize `minTeamsVal`. For now, |
| 9994 | // match clang and set min and max to the same value. |
| 9995 | if (numTeamsUpper) { |
| 9996 | if (auto val = extractConstInteger(value: numTeamsUpper)) |
| 9997 | minTeamsVal = maxTeamsVal = *val; |
| 9998 | } else { |
| 9999 | minTeamsVal = maxTeamsVal = 0; |
| 10000 | } |
| 10001 | } else if (castOrGetParentOfType<omp::ParallelOp>(op: capturedOp, |
| 10002 | /*immediateParent=*/true) || |
| 10003 | castOrGetParentOfType<omp::SimdOp>(op: capturedOp, |
| 10004 | /*immediateParent=*/true)) { |
| 10005 | minTeamsVal = maxTeamsVal = 1; |
| 10006 | } else { |
| 10007 | minTeamsVal = maxTeamsVal = -1; |
| 10008 | } |
| 10009 | |
| 10010 | // Handle clauses impacting the number of threads. |
| 10011 | |
| 10012 | auto setMaxValueFromClause = [](Value clauseValue, int32_t &result) { |
| 10013 | if (!clauseValue) |
| 10014 | return; |
| 10015 | |
| 10016 | if (auto val = extractConstInteger(value: clauseValue)) |
| 10017 | result = *val; |
| 10018 | |
| 10019 | // Found an applicable clause, so it's not undefined. Mark as unknown |
| 10020 | // because it's not constant. |
| 10021 | if (result < 0) |
| 10022 | result = 0; |
| 10023 | }; |
| 10024 | |
| 10025 | // Extract 'thread_limit' clause from 'target' and 'teams' directives. |
| 10026 | int32_t targetThreadLimitVal = -1, teamsThreadLimitVal = -1; |
| 10027 | if (!targetOp.getThreadLimitVars().empty()) |
| 10028 | setMaxValueFromClause(targetOp.getThreadLimit(dim: 0), targetThreadLimitVal); |
| 10029 | setMaxValueFromClause(threadLimit, teamsThreadLimitVal); |
| 10030 | |
| 10031 | // Extract 'max_threads' clause from 'parallel' or set to 1 if it's SIMD. |
| 10032 | int32_t maxThreadsVal = -1; |
| 10033 | if (castOrGetParentOfType<omp::ParallelOp>(op: capturedOp)) |
| 10034 | setMaxValueFromClause(numThreads, maxThreadsVal); |
| 10035 | else if (castOrGetParentOfType<omp::SimdOp>(op: capturedOp, |
| 10036 | /*immediateParent=*/true)) |
| 10037 | maxThreadsVal = 1; |
| 10038 | |
| 10039 | // For max values, < 0 means unset, == 0 means set but unknown. Select the |
| 10040 | // minimum value between 'max_threads' and 'thread_limit' clauses that were |
| 10041 | // set. |
| 10042 | int32_t combinedMaxThreadsVal = targetThreadLimitVal; |
| 10043 | if (combinedMaxThreadsVal < 0 || |
| 10044 | (teamsThreadLimitVal >= 0 && teamsThreadLimitVal < combinedMaxThreadsVal)) |
| 10045 | combinedMaxThreadsVal = teamsThreadLimitVal; |
| 10046 | |
| 10047 | if (combinedMaxThreadsVal < 0 || |
| 10048 | (maxThreadsVal >= 0 && maxThreadsVal < combinedMaxThreadsVal)) |
| 10049 | combinedMaxThreadsVal = maxThreadsVal; |
| 10050 | |
| 10051 | int32_t reductionDataSize = 0; |
| 10052 | if (isGPU && capturedOp) { |
| 10053 | if (auto teamsOp = castOrGetParentOfType<omp::TeamsOp>(op: capturedOp)) |
| 10054 | reductionDataSize = getReductionDataSize(op&: teamsOp); |
| 10055 | } |
| 10056 | |
| 10057 | // Update kernel bounds structure for the `OpenMPIRBuilder` to use. |
| 10058 | // Use the kernel_type attribute set by the frontend instead of analyzing IR. |
| 10059 | omp::TargetExecMode execMode = targetOp.getKernelType(); |
| 10060 | switch (execMode) { |
| 10061 | case omp::TargetExecMode::bare: |
| 10062 | attrs.ExecFlags = llvm::omp::OMP_TGT_EXEC_MODE_BARE; |
| 10063 | break; |
| 10064 | case omp::TargetExecMode::generic: |
| 10065 | attrs.ExecFlags = llvm::omp::OMP_TGT_EXEC_MODE_GENERIC; |
| 10066 | break; |
| 10067 | case omp::TargetExecMode::spmd: |
| 10068 | attrs.ExecFlags = llvm::omp::OMP_TGT_EXEC_MODE_SPMD; |
| 10069 | break; |
| 10070 | case omp::TargetExecMode::spmd_no_loop: |
| 10071 | attrs.ExecFlags = llvm::omp::OMP_TGT_EXEC_MODE_SPMD_NO_LOOP; |
| 10072 | break; |
| 10073 | } |
| 10074 | attrs.MinTeams.front() = minTeamsVal; |
| 10075 | attrs.MaxTeams.front() = maxTeamsVal; |
| 10076 | attrs.MinThreads.front() = 1; |
| 10077 | attrs.MaxThreads.front() = combinedMaxThreadsVal; |
| 10078 | attrs.ReductionDataSize = reductionDataSize; |
| 10079 | } |
| 10080 | |
| 10081 | /// Gather LLVM runtime values for all clauses evaluated in the host that are |
| 10082 | /// passed to the kernel invocation. |
| 10083 | /// |
| 10084 | /// This function must be called only when compiling for the host. Also, it will |
| 10085 | /// only provide correct results if it's called after the body of \c targetOp |
| 10086 | /// has been fully generated. |
| 10087 | static void |
| 10088 | initTargetRuntimeAttrs(llvm::IRBuilderBase &builder, |
| 10089 | LLVM::ModuleTranslation &moduleTranslation, |
| 10090 | omp::TargetOp targetOp, Operation *capturedOp, |
| 10091 | llvm::OpenMPIRBuilder::TargetKernelRuntimeAttrs &attrs) { |
| 10092 | omp::LoopNestOp loopOp = castOrGetParentOfType<omp::LoopNestOp>(op: capturedOp); |
| 10093 | unsigned numLoops = loopOp ? loopOp.getNumLoops() : 0; |
| 10094 | |
| 10095 | Value numThreads, numTeamsLower, numTeamsUpper, teamsThreadLimit; |
| 10096 | llvm::SmallVector<Value> lowerBounds(numLoops), upperBounds(numLoops), |
| 10097 | steps(numLoops); |
| 10098 | extractHostEvalClauses(targetOp, numThreads, numTeamsLower, numTeamsUpper, |
| 10099 | threadLimit&: teamsThreadLimit, lowerBounds: &lowerBounds, upperBounds: &upperBounds, steps: &steps); |
| 10100 | |
| 10101 | // TODO: Handle constant 'if' clauses. |
| 10102 | if (!targetOp.getThreadLimitVars().empty()) { |
| 10103 | Value targetThreadLimit = targetOp.getThreadLimit(dim: 0); |
| 10104 | attrs.TargetThreadLimit.front() = |
| 10105 | moduleTranslation.lookupValue(value: targetThreadLimit); |
| 10106 | } |
| 10107 | |
| 10108 | // The __kmpc_push_num_teams_51 function expects int32 as the arguments. So, |
| 10109 | // truncate or sign extend lower and upper num_teams bounds as well as |
| 10110 | // thread_limit to match int32 ABI requirements for the OpenMP runtime. |
| 10111 | if (numTeamsLower) |
| 10112 | attrs.MinTeams.front() = builder.CreateSExtOrTrunc( |
| 10113 | V: moduleTranslation.lookupValue(value: numTeamsLower), DestTy: builder.getInt32Ty()); |
| 10114 | |
| 10115 | if (numTeamsUpper) |
| 10116 | attrs.MaxTeams.front() = builder.CreateSExtOrTrunc( |
| 10117 | V: moduleTranslation.lookupValue(value: numTeamsUpper), DestTy: builder.getInt32Ty()); |
| 10118 | |
| 10119 | if (teamsThreadLimit) |
| 10120 | attrs.TeamsThreadLimit.front() = builder.CreateSExtOrTrunc( |
| 10121 | V: moduleTranslation.lookupValue(value: teamsThreadLimit), DestTy: builder.getInt32Ty()); |
| 10122 | |
| 10123 | if (numThreads) |
| 10124 | attrs.MaxThreads.front() = moduleTranslation.lookupValue(value: numThreads); |
| 10125 | |
| 10126 | if (targetOp.hasHostEvalTripCount()) { |
| 10127 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 10128 | attrs.LoopTripCount = nullptr; |
| 10129 | |
| 10130 | // To calculate the trip count, we multiply together the trip counts of |
| 10131 | // every collapsed canonical loop. We don't need to create the loop nests |
| 10132 | // here, since we're only interested in the trip count. |
| 10133 | for (auto [loopLower, loopUpper, loopStep] : |
| 10134 | llvm::zip_equal(t&: lowerBounds, u&: upperBounds, args&: steps)) { |
| 10135 | llvm::Value *lowerBound = moduleTranslation.lookupValue(value: loopLower); |
| 10136 | llvm::Value *upperBound = moduleTranslation.lookupValue(value: loopUpper); |
| 10137 | llvm::Value *step = moduleTranslation.lookupValue(value: loopStep); |
| 10138 | |
| 10139 | if (!lowerBound || !upperBound || !step) { |
| 10140 | attrs.LoopTripCount = nullptr; |
| 10141 | break; |
| 10142 | } |
| 10143 | |
| 10144 | llvm::OpenMPIRBuilder::LocationDescription loc(builder); |
| 10145 | llvm::Value *tripCount = ompBuilder->calculateCanonicalLoopTripCount( |
| 10146 | Loc: loc, Start: lowerBound, Stop: upperBound, Step: step, /*IsSigned=*/true, |
| 10147 | InclusiveStop: loopOp.getLoopInclusive()); |
| 10148 | |
| 10149 | if (!attrs.LoopTripCount) { |
| 10150 | attrs.LoopTripCount = tripCount; |
| 10151 | continue; |
| 10152 | } |
| 10153 | |
| 10154 | // TODO: Enable UndefinedSanitizer to diagnose an overflow here. |
| 10155 | attrs.LoopTripCount = builder.CreateMul(LHS: attrs.LoopTripCount, RHS: tripCount, |
| 10156 | Name: {}, /*HasNUW=*/true); |
| 10157 | } |
| 10158 | } |
| 10159 | |
| 10160 | attrs.DeviceID = builder.getInt64(C: llvm::omp::OMP_DEVICEID_UNDEF); |
| 10161 | if (mlir::Value devId = targetOp.getDevice()) { |
| 10162 | attrs.DeviceID = moduleTranslation.lookupValue(value: devId); |
| 10163 | attrs.DeviceID = |
| 10164 | builder.CreateSExtOrTrunc(V: attrs.DeviceID, DestTy: builder.getInt64Ty()); |
| 10165 | } |
| 10166 | } |
| 10167 | |
| 10168 | static llvm::omp::OMPDynGroupprivateFallbackType |
| 10169 | getDynGroupprivateFallbackType(omp::FallbackModifierAttr fallbackAttr) { |
| 10170 | omp::FallbackModifier fb = fallbackAttr ? fallbackAttr.getValue() |
| 10171 | : omp::FallbackModifier::default_mem; |
| 10172 | switch (fb) { |
| 10173 | case omp::FallbackModifier::abort: |
| 10174 | return llvm::omp::OMPDynGroupprivateFallbackType::Abort; |
| 10175 | case omp::FallbackModifier::null: |
| 10176 | return llvm::omp::OMPDynGroupprivateFallbackType::Null; |
| 10177 | case omp::FallbackModifier::default_mem: |
| 10178 | return llvm::omp::OMPDynGroupprivateFallbackType::DefaultMem; |
| 10179 | } |
| 10180 | |
| 10181 | llvm_unreachable("unexpected dyn_groupprivate fallback type" ); |
| 10182 | } |
| 10183 | |
| 10184 | static LogicalResult |
| 10185 | convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, |
| 10186 | LLVM::ModuleTranslation &moduleTranslation) { |
| 10187 | auto targetOp = cast<omp::TargetOp>(Val&: opInst); |
| 10188 | |
| 10189 | // The current debug location already has the DISubprogram for the outlined |
| 10190 | // function that will be created for the target op. We save it here so that |
| 10191 | // we can set it on the outlined function. |
| 10192 | llvm::DebugLoc outlinedFnLoc = builder.getCurrentDebugLocation(); |
| 10193 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 10194 | return failure(); |
| 10195 | |
| 10196 | // During the handling of target op, we will generate instructions in the |
| 10197 | // parent function like call to the oulined function or branch to a new |
| 10198 | // BasicBlock. We set the debug location here to parent function so that those |
| 10199 | // get the correct debug locations. For outlined functions, the normal MLIR op |
| 10200 | // conversion will automatically pick the correct location. |
| 10201 | llvm::BasicBlock *parentBB = builder.GetInsertBlock(); |
| 10202 | assert(parentBB && "No insert block is set for the builder" ); |
| 10203 | llvm::Function *parentLLVMFn = parentBB->getParent(); |
| 10204 | assert(parentLLVMFn && "Parent Function must be valid" ); |
| 10205 | if (llvm::DISubprogram *SP = parentLLVMFn->getSubprogram()) |
| 10206 | builder.SetCurrentDebugLocation(llvm::DILocation::get( |
| 10207 | Context&: parentLLVMFn->getContext(), Line: outlinedFnLoc.getLine(), |
| 10208 | Column: outlinedFnLoc.getCol(), Scope: SP, InlinedAt: outlinedFnLoc.getInlinedAt())); |
| 10209 | |
| 10210 | // OMPIRBuilder emits runtime calls into the outlined function before bodyCB |
| 10211 | // below gets a chance to attach the subprogram to it, so it needs the |
| 10212 | // outlined function's location handed to it separately. Only pass it under |
| 10213 | // the same condition that decides whether the subprogram is attached at all: |
| 10214 | // a location may not be attached to an instruction in a function that has no |
| 10215 | // subprogram. |
| 10216 | llvm::DebugLoc outlinedFnDbgLoc; |
| 10217 | if (outlinedFnLoc && parentLLVMFn->getSubprogram()) |
| 10218 | outlinedFnDbgLoc = outlinedFnLoc; |
| 10219 | |
| 10220 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 10221 | bool isTargetDevice = ompBuilder->Config.isTargetDevice(); |
| 10222 | bool isGPU = ompBuilder->Config.isGPU(); |
| 10223 | |
| 10224 | auto parentFn = opInst.getParentOfType<LLVM::LLVMFuncOp>(); |
| 10225 | auto argIface = cast<omp::BlockArgOpenMPOpInterface>(Val&: opInst); |
| 10226 | auto &targetRegion = targetOp.getRegion(); |
| 10227 | // Holds the private vars that have been mapped along with the block |
| 10228 | // argument that corresponds to the MapInfoOp corresponding to the private |
| 10229 | // var in question. So, for instance: |
| 10230 | // |
| 10231 | // %10 = omp.map.info var_ptr(%6#0 : !fir.ref<!fir.box<!fir.heap<i32>>>, ..) |
| 10232 | // omp.target map_entries(%10 -> %arg0) private(@box.privatizer %6#0-> %arg1) |
| 10233 | // |
| 10234 | // Then, %10 has been created so that the descriptor can be used by the |
| 10235 | // privatizer @box.privatizer on the device side. Here we'd record {%6#0, |
| 10236 | // %arg0} in the mappedPrivateVars map. |
| 10237 | llvm::DenseMap<Value, Value> mappedPrivateVars; |
| 10238 | DataLayout dl = DataLayout(opInst.getParentOfType<ModuleOp>()); |
| 10239 | SmallVector<Value> mapVars = targetOp.getMapVars(); |
| 10240 | SmallVector<Value> hdaVars = targetOp.getHasDeviceAddrVars(); |
| 10241 | ArrayRef<BlockArgument> mapBlockArgs = argIface.getMapBlockArgs(); |
| 10242 | ArrayRef<BlockArgument> hdaBlockArgs = argIface.getHasDeviceAddrBlockArgs(); |
| 10243 | llvm::Function *llvmOutlinedFn = nullptr; |
| 10244 | TargetDirectiveEnumTy targetDirective = |
| 10245 | getTargetDirectiveEnumTyFromOp(op: &opInst); |
| 10246 | |
| 10247 | // TODO: It can also be false if a compile-time constant `false` IF clause is |
| 10248 | // specified. |
| 10249 | bool isOffloadEntry = |
| 10250 | isTargetDevice || !ompBuilder->Config.TargetTriples.empty(); |
| 10251 | |
| 10252 | // Resolve in_reduction clauses on omp.target for the host. From the target |
| 10253 | // device's perspective an in_reduction list item behaves as a regular |
| 10254 | // map(tofrom) variable, so no special handling is needed there; only the |
| 10255 | // host redirects the mapped value to the per-task reduction-private storage |
| 10256 | // returned by __kmpc_task_reduction_get_th_data (emitted inside the |
| 10257 | // to-be-outlined target task body). This applies to both offloading and |
| 10258 | // non-offloading host modules. |
| 10259 | // |
| 10260 | // The target body has no dedicated in_reduction block argument: each |
| 10261 | // in_reduction variable is accessed through its map_entries block argument. |
| 10262 | // So each in_reduction variable must also be captured by a matching |
| 10263 | // map_entries entry (guaranteed by the verifier); without one the outlined |
| 10264 | // body would reference a value defined in the host function. Record, for each |
| 10265 | // in_reduction variable, the position of that map entry so the corresponding |
| 10266 | // map block argument can be redirected inside the body. The in_reduction |
| 10267 | // operand itself is used as the `orig` argument of the runtime lookup. |
| 10268 | SmallVector<llvm::Value *> inRedOrigPtrs; |
| 10269 | SmallVector<unsigned> inRedMapArgIdx; |
| 10270 | if (!targetOp.getInReductionVars().empty() && !isTargetDevice) { |
| 10271 | inRedOrigPtrs.reserve(N: targetOp.getInReductionVars().size()); |
| 10272 | inRedMapArgIdx.reserve(N: targetOp.getInReductionVars().size()); |
| 10273 | for (Value v : targetOp.getInReductionVars()) { |
| 10274 | // Select the map_entries entry that captures this in_reduction operand. |
| 10275 | // The verifier guarantees at least one match exists; more than one |
| 10276 | // matching entry is a lowering ambiguity (the redirect cannot pick which |
| 10277 | // map argument to rebind). |
| 10278 | std::optional<unsigned> matchIdx; |
| 10279 | for (auto [idx, mapV] : llvm::enumerate(First: targetOp.getMapVars())) { |
| 10280 | auto mapInfo = mapV.getDefiningOp<omp::MapInfoOp>(); |
| 10281 | if (v != mapInfo.getVarPtr()) |
| 10282 | continue; |
| 10283 | if (matchIdx) |
| 10284 | return targetOp.emitError() |
| 10285 | << "in_reduction variable on omp.target has multiple matching " |
| 10286 | "map_entries entries; the redirect target is ambiguous" ; |
| 10287 | matchIdx = idx; |
| 10288 | } |
| 10289 | // The verifier requires a capturing map entry for every in_reduction |
| 10290 | // operand, so a match must exist here. |
| 10291 | assert(matchIdx && |
| 10292 | "TargetOp verifier guarantees a matching map_entries entry for " |
| 10293 | "each in_reduction variable" ); |
| 10294 | inRedMapArgIdx.push_back(Elt: *matchIdx); |
| 10295 | // The runtime `orig` pointer is the in_reduction operand itself, the |
| 10296 | // reduction variable the enclosing taskgroup registered. |
| 10297 | inRedOrigPtrs.push_back(Elt: moduleTranslation.lookupValue(value: v)); |
| 10298 | } |
| 10299 | } |
| 10300 | |
| 10301 | // For some private variables, the MapsForPrivatizedVariablesPass |
| 10302 | // creates MapInfoOp instances. Go through the private variables and |
| 10303 | // the mapped variables so that during codegeneration we are able |
| 10304 | // to quickly look up the corresponding map variable, if any for each |
| 10305 | // private variable. |
| 10306 | if (!targetOp.getPrivateVars().empty() && !targetOp.getMapVars().empty()) { |
| 10307 | OperandRange privateVars = targetOp.getPrivateVars(); |
| 10308 | std::optional<ArrayAttr> privateSyms = targetOp.getPrivateSyms(); |
| 10309 | std::optional<DenseI64ArrayAttr> privateMapIndices = |
| 10310 | targetOp.getPrivateMapsAttr(); |
| 10311 | |
| 10312 | for (auto [privVarIdx, privVarSymPair] : |
| 10313 | llvm::enumerate(First: llvm::zip_equal(t&: privateVars, u&: *privateSyms))) { |
| 10314 | auto privVar = std::get<0>(t&: privVarSymPair); |
| 10315 | auto privSym = std::get<1>(t&: privVarSymPair); |
| 10316 | |
| 10317 | SymbolRefAttr privatizerName = llvm::cast<SymbolRefAttr>(Val&: privSym); |
| 10318 | omp::PrivateClauseOp privatizer = |
| 10319 | findPrivatizer(from: targetOp, symbolName: privatizerName); |
| 10320 | |
| 10321 | if (!privatizer.needsMap()) |
| 10322 | continue; |
| 10323 | |
| 10324 | mlir::Value mappedValue = |
| 10325 | targetOp.getMappedValueForPrivateVar(privVarIdx); |
| 10326 | assert(mappedValue && "Expected to find mapped value for a privatized " |
| 10327 | "variable that needs mapping" ); |
| 10328 | |
| 10329 | // The MapInfoOp defining the map var isn't really needed later. |
| 10330 | // So, we don't store it in any datastructure. Instead, we just |
| 10331 | // do some sanity checks on it right now. |
| 10332 | auto mapInfoOp = mappedValue.getDefiningOp<omp::MapInfoOp>(); |
| 10333 | [[maybe_unused]] Type varType = mapInfoOp.getVarPtrType(); |
| 10334 | |
| 10335 | // Check #1: Check that the type of the private variable matches |
| 10336 | // the type of the variable being mapped. |
| 10337 | if (!isa<LLVM::LLVMPointerType>(Val: privVar.getType())) |
| 10338 | assert( |
| 10339 | varType == privVar.getType() && |
| 10340 | "Type of private var doesn't match the type of the mapped value" ); |
| 10341 | |
| 10342 | // Ok, only 1 sanity check for now. |
| 10343 | // Record the block argument corresponding to this mapvar. |
| 10344 | mappedPrivateVars.insert( |
| 10345 | KV: {privVar, |
| 10346 | targetRegion.getArgument(i: argIface.getMapBlockArgsStart() + |
| 10347 | (*privateMapIndices)[privVarIdx])}); |
| 10348 | } |
| 10349 | } |
| 10350 | |
| 10351 | using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy; |
| 10352 | auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP, |
| 10353 | ArrayRef<llvm::BasicBlock *> deallocBlocks) |
| 10354 | -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { |
| 10355 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 10356 | builder.SetCurrentDebugLocation(llvm::DebugLoc()); |
| 10357 | // Forward target-cpu and target-features function attributes from the |
| 10358 | // original function to the new outlined function. |
| 10359 | llvm::Function *llvmParentFn = |
| 10360 | moduleTranslation.lookupFunction(name: parentFn.getName()); |
| 10361 | llvmOutlinedFn = codeGenIP.getNodeParent()->getParent(); |
| 10362 | assert(llvmParentFn && llvmOutlinedFn && |
| 10363 | "Both parent and outlined functions must exist at this point" ); |
| 10364 | |
| 10365 | if (outlinedFnLoc && llvmParentFn->getSubprogram()) |
| 10366 | llvmOutlinedFn->setSubprogram(outlinedFnLoc->getScope()->getSubprogram()); |
| 10367 | |
| 10368 | if (auto attr = llvmParentFn->getFnAttribute(Kind: "target-cpu" ); |
| 10369 | attr.isStringAttribute()) |
| 10370 | llvmOutlinedFn->addFnAttr(Attr: attr); |
| 10371 | |
| 10372 | if (auto attr = llvmParentFn->getFnAttribute(Kind: "target-features" ); |
| 10373 | attr.isStringAttribute()) |
| 10374 | llvmOutlinedFn->addFnAttr(Attr: attr); |
| 10375 | |
| 10376 | for (auto [idx, arg] : llvm::enumerate(First&: mapBlockArgs)) { |
| 10377 | // in_reduction list items on omp.target are accessed through their |
| 10378 | // map_entries block argument, which is redirected below to the per-task |
| 10379 | // reduction-private storage returned by the runtime. Skip the default |
| 10380 | // host-value mapping for those block arguments so the write-once |
| 10381 | // mapValue mapping is free to be set to the private pointer. |
| 10382 | if (llvm::is_contained(Range&: inRedMapArgIdx, Element: idx)) |
| 10383 | continue; |
| 10384 | auto mapInfoOp = cast<omp::MapInfoOp>(Val: mapVars[idx].getDefiningOp()); |
| 10385 | llvm::Value *mapOpValue = |
| 10386 | moduleTranslation.lookupValue(value: mapInfoOp.getVarPtr()); |
| 10387 | moduleTranslation.mapValue(mlir: arg, llvm: mapOpValue); |
| 10388 | } |
| 10389 | for (auto [arg, mapOp] : llvm::zip_equal(t&: hdaBlockArgs, u&: hdaVars)) { |
| 10390 | auto mapInfoOp = cast<omp::MapInfoOp>(Val: mapOp.getDefiningOp()); |
| 10391 | llvm::Value *mapOpValue = |
| 10392 | moduleTranslation.lookupValue(value: mapInfoOp.getVarPtr()); |
| 10393 | moduleTranslation.mapValue(mlir: arg, llvm: mapOpValue); |
| 10394 | } |
| 10395 | |
| 10396 | // Do privatization after moduleTranslation has already recorded |
| 10397 | // mapped values. |
| 10398 | PrivateVarsInfo privateVarsInfo(targetOp); |
| 10399 | |
| 10400 | llvm::Expected<llvm::BasicBlock *> afterAllocas = |
| 10401 | allocatePrivateVars(op: targetOp, builder, moduleTranslation, |
| 10402 | privateVarsInfo, allocaIP, mappedPrivateVars: &mappedPrivateVars); |
| 10403 | |
| 10404 | if (failed(Result: handleError(result&: afterAllocas, op&: *targetOp))) |
| 10405 | return llvm::make_error<PreviouslyReportedError>(); |
| 10406 | |
| 10407 | builder.restoreIP(IP: codeGenIP); |
| 10408 | if (handleError(error: initPrivateVars(builder, moduleTranslation, privateVarsInfo, |
| 10409 | mappedPrivateVars: &mappedPrivateVars), |
| 10410 | op&: *targetOp) |
| 10411 | .failed()) |
| 10412 | return llvm::make_error<PreviouslyReportedError>(); |
| 10413 | |
| 10414 | if (failed(Result: completePrivateVars( |
| 10415 | op: targetOp, builder, moduleTranslation, privateVarsInfo, |
| 10416 | insertBarrier: targetOp.getPrivateNeedsBarrier(), /*moldVars=*/{}, |
| 10417 | /*llvmPrivateVars=*/{}, mappedPrivateVars: &mappedPrivateVars))) |
| 10418 | return llvm::make_error<PreviouslyReportedError>(); |
| 10419 | |
| 10420 | // The target body accesses each in_reduction variable through its |
| 10421 | // map_entries block argument. Redirect that block argument to the per-task |
| 10422 | // private storage returned by __kmpc_task_reduction_get_th_data so the body |
| 10423 | // accumulates into the reduction-private copy rather than the mapped |
| 10424 | // original. The lookup must run inside the target task body so the gtid |
| 10425 | // corresponds to the executing thread. The descriptor argument is NULL: the |
| 10426 | // runtime walks enclosing taskgroups to locate the matching task_reduction |
| 10427 | // registration for `origPtr`. Mirrors the in_reduction handling on |
| 10428 | // omp.taskloop.context. |
| 10429 | if (!inRedOrigPtrs.empty()) { |
| 10430 | // Collect, per item, the type the private pointer must have (the map |
| 10431 | // block argument's type), and, through the callback, rebind the map block |
| 10432 | // argument that stands in for each in_reduction list item to the per-task |
| 10433 | // reduction-private storage the runtime returns. |
| 10434 | SmallVector<llvm::Type *> inRedResultPtrTys; |
| 10435 | inRedResultPtrTys.reserve(N: inRedMapArgIdx.size()); |
| 10436 | for (unsigned mapArgIdx : inRedMapArgIdx) |
| 10437 | inRedResultPtrTys.push_back( |
| 10438 | Elt: moduleTranslation.convertType(type: mapBlockArgs[mapArgIdx].getType())); |
| 10439 | |
| 10440 | llvm::OpenMPIRBuilder::LocationDescription bodyLoc(builder); |
| 10441 | llvm::OpenMPIRBuilder::InsertPointTy redIP = |
| 10442 | ompBuilder->createTargetInReduction( |
| 10443 | Loc: bodyLoc, OrigPtrs: inRedOrigPtrs, ResultPtrTys: inRedResultPtrTys, |
| 10444 | MapPrivateCB: [&](unsigned idx, llvm::Value *priv) { |
| 10445 | moduleTranslation.mapValue(mlir: mapBlockArgs[inRedMapArgIdx[idx]], |
| 10446 | llvm: priv); |
| 10447 | }); |
| 10448 | builder.restoreIP(IP: redIP); |
| 10449 | } |
| 10450 | |
| 10451 | LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame( |
| 10452 | moduleTranslation, allocaIP.getNodeParent(), deallocBlocks); |
| 10453 | llvm::Expected<llvm::BasicBlock *> exitBlock = convertOmpOpRegions( |
| 10454 | region&: targetRegion, blockName: "omp.target" , builder, moduleTranslation); |
| 10455 | |
| 10456 | if (failed(Result: handleError(result&: exitBlock, op&: *targetOp))) |
| 10457 | return llvm::make_error<PreviouslyReportedError>(); |
| 10458 | |
| 10459 | builder.SetInsertPoint(exitBlock.get()->getTerminator()); |
| 10460 | |
| 10461 | if (failed(Result: cleanupPrivateVars(op: targetOp, builder, moduleTranslation, |
| 10462 | loc: targetOp.getLoc(), privateVarsInfo))) |
| 10463 | return llvm::make_error<PreviouslyReportedError>(); |
| 10464 | |
| 10465 | return builder.saveIP(); |
| 10466 | }; |
| 10467 | |
| 10468 | StringRef parentName = parentFn.getName(); |
| 10469 | |
| 10470 | llvm::TargetRegionEntryInfo entryInfo; |
| 10471 | |
| 10472 | getTargetEntryUniqueInfo(targetInfo&: entryInfo, targetOp, |
| 10473 | ompBuilder&: *getOpenMPBuilder(moduleTranslation), |
| 10474 | vfs&: moduleTranslation.getFileSystem(), parentName); |
| 10475 | |
| 10476 | MapInfoData mapData; |
| 10477 | collectMapDataFromMapOperands(mapData, mapVars, moduleTranslation, dl, |
| 10478 | builder, /*useDevPtrOperands=*/{}, |
| 10479 | /*useDevAddrOperands=*/{}, hasDevAddrOperands: hdaVars); |
| 10480 | |
| 10481 | MapInfosTy combinedInfos; |
| 10482 | auto genMapInfoCB = |
| 10483 | [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP) -> MapInfosTy & { |
| 10484 | builder.restoreIP(IP: codeGenIP); |
| 10485 | genMapInfos(builder, moduleTranslation, dl, combinedInfo&: combinedInfos, mapData, |
| 10486 | targetDirective); |
| 10487 | |
| 10488 | // Append a null entry for the implicit dyn_ptr argument so the argument |
| 10489 | // count sent to the runtime already includes it. |
| 10490 | auto *nullPtr = llvm::Constant::getNullValue(Ty: builder.getPtrTy()); |
| 10491 | combinedInfos.BasePointers.push_back(Elt: nullPtr); |
| 10492 | combinedInfos.Pointers.push_back(Elt: nullPtr); |
| 10493 | combinedInfos.DevicePointers.push_back( |
| 10494 | Elt: llvm::OpenMPIRBuilder::DeviceInfoTy::None); |
| 10495 | combinedInfos.Sizes.push_back(Elt: builder.getInt64(C: 0)); |
| 10496 | combinedInfos.Types.push_back( |
| 10497 | Elt: llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM | |
| 10498 | llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL); |
| 10499 | // TODO: set HasAttachPtr from Flang for pointee-storage entries. |
| 10500 | combinedInfos.HasAttachPtr.push_back(Elt: false); |
| 10501 | if (!combinedInfos.Names.empty()) |
| 10502 | combinedInfos.Names.push_back(Elt: nullPtr); |
| 10503 | combinedInfos.Mappers.push_back(Elt: nullptr); |
| 10504 | |
| 10505 | return combinedInfos; |
| 10506 | }; |
| 10507 | |
| 10508 | auto argAccessorCB = [&](llvm::Argument &arg, llvm::Value *input, |
| 10509 | llvm::Value *&retVal, InsertPointTy allocaIP, |
| 10510 | InsertPointTy codeGenIP, |
| 10511 | llvm::ArrayRef<InsertPointTy> deallocIPs) |
| 10512 | -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy { |
| 10513 | llvm::IRBuilderBase::InsertPointGuard guard(builder); |
| 10514 | builder.SetCurrentDebugLocation(llvm::DebugLoc()); |
| 10515 | // We just return the unaltered argument for the host function |
| 10516 | // for now, some alterations may be required in the future to |
| 10517 | // keep host fallback functions working identically to the device |
| 10518 | // version (e.g. pass ByCopy values should be treated as such on |
| 10519 | // host and device, currently not always the case) |
| 10520 | if (!isTargetDevice) { |
| 10521 | retVal = cast<llvm::Value>(Val: &arg); |
| 10522 | return codeGenIP; |
| 10523 | } |
| 10524 | |
| 10525 | return createDeviceArgumentAccessor(targetOp, mapData, arg, input, retVal, |
| 10526 | builder, ompBuilder&: *ompBuilder, moduleTranslation, |
| 10527 | allocaIP, codeGenIP, deallocIPs); |
| 10528 | }; |
| 10529 | |
| 10530 | llvm::OpenMPIRBuilder::TargetKernelRuntimeAttrs runtimeAttrs; |
| 10531 | llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs defaultAttrs; |
| 10532 | Operation *targetCapturedOp = |
| 10533 | cast<omp::ComposableOpInterface>(Val&: *targetOp).findCapturedOp(); |
| 10534 | initTargetDefaultAttrs(targetOp, capturedOp: targetCapturedOp, attrs&: defaultAttrs, |
| 10535 | isTargetDevice, isGPU); |
| 10536 | |
| 10537 | // Collect host-evaluated values needed to properly launch the kernel from the |
| 10538 | // host. |
| 10539 | if (!isTargetDevice) |
| 10540 | initTargetRuntimeAttrs(builder, moduleTranslation, targetOp, |
| 10541 | capturedOp: targetCapturedOp, attrs&: runtimeAttrs); |
| 10542 | |
| 10543 | // Pass host-evaluated values as parameters to the kernel / host fallback, |
| 10544 | // except if they are constants. In any case, map the MLIR block argument to |
| 10545 | // the corresponding LLVM values. |
| 10546 | llvm::SmallVector<llvm::Value *, 4> kernelInput; |
| 10547 | SmallVector<Value> hostEvalVars = targetOp.getHostEvalVars(); |
| 10548 | ArrayRef<BlockArgument> hostEvalBlockArgs = argIface.getHostEvalBlockArgs(); |
| 10549 | for (auto [arg, var] : llvm::zip_equal(t&: hostEvalBlockArgs, u&: hostEvalVars)) { |
| 10550 | llvm::Value *value = moduleTranslation.lookupValue(value: var); |
| 10551 | moduleTranslation.mapValue(mlir: arg, llvm: value); |
| 10552 | |
| 10553 | if (!llvm::isa<llvm::Constant>(Val: value)) |
| 10554 | kernelInput.push_back(Elt: value); |
| 10555 | } |
| 10556 | |
| 10557 | for (size_t i = 0, e = mapData.OriginalValue.size(); i != e; ++i) { |
| 10558 | // 1) Declare target arguments are not passed to kernels as arguments. |
| 10559 | // 2) Attach maps are not passed in as arguments to kernels, except for |
| 10560 | // private attach maps used for corresponding-pointer initialization. |
| 10561 | // 3) Children of record objects are not passed in as arguments. |
| 10562 | // TODO: We currently do not handle cases where a member is explicitly |
| 10563 | // passed in as an argument, this will likley need to be handled in |
| 10564 | // the near future, rather than using IsAMember, it may be better to |
| 10565 | // test if the relevant BlockArg is used within the target region and |
| 10566 | // then use that as a basis for exclusion in the kernel inputs. |
| 10567 | using MapFlags = llvm::omp::OpenMPOffloadMappingFlags; |
| 10568 | bool isAttachMap = (mapData.Types[i] & MapFlags::OMP_MAP_ATTACH) == |
| 10569 | MapFlags::OMP_MAP_ATTACH; |
| 10570 | bool isPrivateTargetParam = |
| 10571 | (mapData.Types[i] & |
| 10572 | (MapFlags::OMP_MAP_PRIVATE | MapFlags::OMP_MAP_TARGET_PARAM)) == |
| 10573 | (MapFlags::OMP_MAP_PRIVATE | MapFlags::OMP_MAP_TARGET_PARAM); |
| 10574 | |
| 10575 | if (!mapData.IsDeclareTarget[i] && !mapData.IsAMember[i] && |
| 10576 | (!isAttachMap || (isAttachMap && isPrivateTargetParam))) |
| 10577 | kernelInput.push_back(Elt: mapData.OriginalValue[i]); |
| 10578 | } |
| 10579 | |
| 10580 | llvm::SmallVector<llvm::BasicBlock *> deallocBlocks; |
| 10581 | llvm::OpenMPIRBuilder::InsertPointTy allocaIP = |
| 10582 | findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks); |
| 10583 | |
| 10584 | llvm::OpenMPIRBuilder::DependenciesInfo dds; |
| 10585 | if (failed(Result: buildDependData( |
| 10586 | dependVars: targetOp.getDependVars(), dependKinds: targetOp.getDependKinds(), |
| 10587 | dependIterated: targetOp.getDependIterated(), dependIteratedKinds: targetOp.getDependIteratedKinds(), |
| 10588 | builder, moduleTranslation, taskDeps&: dds))) |
| 10589 | return failure(); |
| 10590 | |
| 10591 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 10592 | |
| 10593 | llvm::OpenMPIRBuilder::TargetDataInfo info( |
| 10594 | /*RequiresDevicePointerInfo=*/false, |
| 10595 | /*SeparateBeginEndCalls=*/true); |
| 10596 | |
| 10597 | auto customMapperCB = |
| 10598 | [&](unsigned int i) -> llvm::Expected<llvm::Function *> { |
| 10599 | if (!combinedInfos.Mappers[i]) |
| 10600 | return nullptr; |
| 10601 | info.HasMapper = true; |
| 10602 | return getOrCreateUserDefinedMapperFunc(op: combinedInfos.Mappers[i], builder, |
| 10603 | moduleTranslation, targetDirective); |
| 10604 | }; |
| 10605 | |
| 10606 | llvm::Value *ifCond = nullptr; |
| 10607 | if (Value targetIfCond = targetOp.getIfExpr()) |
| 10608 | ifCond = moduleTranslation.lookupValue(value: targetIfCond); |
| 10609 | |
| 10610 | Value dynGroupPrivateSize = targetOp.getDynGroupprivateSize(); |
| 10611 | llvm::Value *dynSizeVal = nullptr; |
| 10612 | if (dynGroupPrivateSize) { |
| 10613 | dynSizeVal = moduleTranslation.lookupValue(value: dynGroupPrivateSize); |
| 10614 | dynSizeVal = builder.CreateIntCast(V: dynSizeVal, DestTy: builder.getInt32Ty(), |
| 10615 | /*isSigned=*/false); |
| 10616 | } |
| 10617 | |
| 10618 | llvm::omp::OMPDynGroupprivateFallbackType fallbackType = |
| 10619 | getDynGroupprivateFallbackType(fallbackAttr: targetOp.getDynGroupprivateFallbackAttr()); |
| 10620 | |
| 10621 | // Pass the target region's source location to the runtime, taken from the |
| 10622 | // op's own location. Restricted to the host offload path that actually emits |
| 10623 | // the kernel launch, to avoid creating an unused identifier on the device. |
| 10624 | llvm::Value *rtLocOverride = |
| 10625 | (!isTargetDevice && isOffloadEntry) |
| 10626 | ? getSourceLocIdentFromOp(builder, ompBuilder&: *ompBuilder, op: targetOp) |
| 10627 | : nullptr; |
| 10628 | |
| 10629 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 10630 | getOpenMPBuilder(moduleTranslation) |
| 10631 | ->createTarget(Loc: ompLoc, IsOffloadEntry: isOffloadEntry, AllocaIP: allocaIP, CodeGenIP: builder.saveIP(), |
| 10632 | DeallocBlocks: deallocBlocks, Info&: info, EntryInfo&: entryInfo, DefaultAttrs: defaultAttrs, |
| 10633 | RuntimeAttrs: runtimeAttrs, IfCond: ifCond, Inputs&: kernelInput, GenMapInfoCB: genMapInfoCB, |
| 10634 | BodyGenCB: bodyCB, ArgAccessorFuncCB: argAccessorCB, CustomMapperCB: customMapperCB, Dependencies: dds, |
| 10635 | HasNowait: targetOp.getNowait(), DynCGroupMem: dynSizeVal, DynCGroupMemFallback: fallbackType, |
| 10636 | OutlinedFnLoc: outlinedFnDbgLoc, RTLocOverride: rtLocOverride); |
| 10637 | |
| 10638 | if (failed(Result: handleError(result&: afterIP, op&: opInst))) |
| 10639 | return failure(); |
| 10640 | |
| 10641 | builder.restoreIP(IP: *afterIP); |
| 10642 | |
| 10643 | if (dds.DepArray) |
| 10644 | builder.CreateFree(Source: dds.DepArray); |
| 10645 | |
| 10646 | return success(); |
| 10647 | } |
| 10648 | |
| 10649 | static LogicalResult |
| 10650 | convertDeclareTargetAttr(Operation *op, mlir::omp::DeclareTargetAttr attribute, |
| 10651 | llvm::OpenMPIRBuilder *ompBuilder, |
| 10652 | LLVM::ModuleTranslation &moduleTranslation) { |
| 10653 | // Amend omp.declare_target by deleting the IR of the outlined functions |
| 10654 | // created for target regions. They cannot be filtered out from MLIR earlier |
| 10655 | // because the omp.target operation inside must be translated to LLVM, but |
| 10656 | // the wrapper functions themselves must not remain at the end of the |
| 10657 | // process. We know that functions where omp.declare_target does not match |
| 10658 | // omp.is_target_device at this stage can only be wrapper functions because |
| 10659 | // those that aren't are removed earlier as an MLIR transformation pass. |
| 10660 | if (FunctionOpInterface funcOp = dyn_cast<FunctionOpInterface>(Val: op)) { |
| 10661 | if (auto offloadMod = dyn_cast<omp::OffloadModuleInterface>( |
| 10662 | Val: op->getParentOfType<ModuleOp>().getOperation())) { |
| 10663 | if (!offloadMod.getIsTargetDevice()) |
| 10664 | return success(); |
| 10665 | |
| 10666 | omp::DeclareTargetDeviceType declareType = attribute.getDeviceType(); |
| 10667 | |
| 10668 | if (declareType == omp::DeclareTargetDeviceType::host) { |
| 10669 | llvm::Function *llvmFunc = |
| 10670 | moduleTranslation.lookupFunction(name: funcOp.getName()); |
| 10671 | llvmFunc->dropAllReferences(); |
| 10672 | llvmFunc->eraseFromParent(); |
| 10673 | |
| 10674 | // Invalidate the builder's current insertion point, as it now points to |
| 10675 | // a deleted block. |
| 10676 | ompBuilder->Builder.ClearInsertionPoint(); |
| 10677 | ompBuilder->Builder.SetCurrentDebugLocation(llvm::DebugLoc()); |
| 10678 | } else if (llvm::Function *llvmFunc = |
| 10679 | moduleTranslation.lookupFunction(name: funcOp.getName())) { |
| 10680 | // Device-side declare target functions are externally visible by |
| 10681 | // default so they can be referenced from other device translation |
| 10682 | // units. That also prevents the offload LTO from internalizing and |
| 10683 | // deleting them when they end up unused in the final device image. |
| 10684 | // Such dead functions can still reference internal LDS and trigger |
| 10685 | // spurious "local memory global used by non-kernel function" backend |
| 10686 | // warnings. Marking them hidden keeps the symbol usable within the |
| 10687 | // device image's linkage unit while letting LTO drop it when nothing |
| 10688 | // references it; symbols that must stay reachable (e.g. via an offload |
| 10689 | // entry that takes their address) are kept alive by that reference. |
| 10690 | if (!llvmFunc->isDeclaration() && llvmFunc->hasExternalLinkage() && |
| 10691 | llvmFunc->getVisibility() == llvm::GlobalValue::DefaultVisibility) |
| 10692 | llvmFunc->setVisibility(llvm::GlobalValue::HiddenVisibility); |
| 10693 | } |
| 10694 | } |
| 10695 | return success(); |
| 10696 | } |
| 10697 | |
| 10698 | if (LLVM::GlobalOp gOp = dyn_cast<LLVM::GlobalOp>(Val: op)) { |
| 10699 | llvm::Module *llvmModule = moduleTranslation.getLLVMModule(); |
| 10700 | if (auto *gVal = llvmModule->getNamedValue(Name: gOp.getSymName())) { |
| 10701 | auto *gVar = cast<llvm::GlobalVariable>(Val: gVal); |
| 10702 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 10703 | bool isDeclaration = gOp.isDeclaration(); |
| 10704 | bool isExternallyVisible = |
| 10705 | gOp.getVisibility() != mlir::SymbolTable::Visibility::Private; |
| 10706 | auto loc = op->getLoc()->findInstanceOf<FileLineColLoc>(); |
| 10707 | llvm::StringRef mangledName = gOp.getSymName(); |
| 10708 | mlir::omp::DeclareTargetCaptureClause captureClause = |
| 10709 | attribute.getCaptureClause(); |
| 10710 | auto captureClauseKind = convertToCaptureClauseKind(captureClause); |
| 10711 | auto deviceClause = convertToDeviceClauseKind(deviceClause: attribute.getDeviceType()); |
| 10712 | llvm::StringRef entryMangledName = mangledName; |
| 10713 | llvm::Constant *entryAddr = llvm::cast<llvm::Constant>(Val: gVal); |
| 10714 | std::function<llvm::GlobalValue::LinkageTypes()> variableLinkage; |
| 10715 | llvm::SmallString<128> entryNameStorage; |
| 10716 | bool requiresUSM = ompBuilder->Config.hasRequiresUnifiedSharedMemory(); |
| 10717 | bool isToOrEnter = |
| 10718 | captureClause == omp::DeclareTargetCaptureClause::to || |
| 10719 | captureClause == omp::DeclareTargetCaptureClause::enter; |
| 10720 | bool isHostOnly = |
| 10721 | attribute.getDeviceType() == omp::DeclareTargetDeviceType::host; |
| 10722 | |
| 10723 | // A to/enter declare-target variable needs a device-resident, |
| 10724 | // name-resolvable copy and a host offloading entry. A local-linkage |
| 10725 | // global provides neither, so we promote it to external. |
| 10726 | if (isToOrEnter && !isHostOnly && !requiresUSM && |
| 10727 | gVar->hasLocalLinkage()) { |
| 10728 | gVar->setLinkage(llvm::GlobalValue::ExternalLinkage); |
| 10729 | isExternallyVisible = true; |
| 10730 | |
| 10731 | // Clear the stale dso_local flag so it is referenced like a |
| 10732 | // module-scope declare target global. |
| 10733 | if (ompBuilder->Config.isTargetDevice()) |
| 10734 | gVar->setDSOLocal(false); |
| 10735 | } |
| 10736 | |
| 10737 | if (isToOrEnter && |
| 10738 | deviceClause == |
| 10739 | llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny && |
| 10740 | !requiresUSM && !isDeclaration && |
| 10741 | (gVal->hasLocalLinkage() || gVal->hasHiddenVisibility())) { |
| 10742 | // Keep the original symbol as-is for target code, but create a visible |
| 10743 | // alias for the offload entry so libomptarget can associate the host |
| 10744 | // global with the actual device global. |
| 10745 | entryNameStorage = (mangledName + llvm::Twine("_decl_tgt_entry" )).str(); |
| 10746 | entryMangledName = entryNameStorage; |
| 10747 | if (llvm::GlobalValue *existing = |
| 10748 | llvmModule->getNamedValue(Name: entryMangledName)) { |
| 10749 | entryAddr = llvm::cast<llvm::Constant>(Val: existing); |
| 10750 | } else { |
| 10751 | entryAddr = llvm::GlobalAlias::create( |
| 10752 | Ty: gVal->getValueType(), AddressSpace: gVal->getAddressSpace(), |
| 10753 | Linkage: llvm::GlobalValue::WeakAnyLinkage, Name: entryMangledName, Aliasee: entryAddr, |
| 10754 | Parent: llvmModule); |
| 10755 | llvm::cast<llvm::GlobalAlias>(Val: entryAddr)->setVisibility( |
| 10756 | llvm::GlobalValue::DefaultVisibility); |
| 10757 | } |
| 10758 | variableLinkage = [] { return llvm::GlobalValue::WeakAnyLinkage; }; |
| 10759 | } |
| 10760 | // unused for MLIR at the moment, required in Clang for book |
| 10761 | // keeping |
| 10762 | std::vector<llvm::GlobalVariable *> generatedRefs; |
| 10763 | |
| 10764 | std::vector<llvm::Triple> targetTriple; |
| 10765 | auto targetTripleAttr = dyn_cast_or_null<mlir::StringAttr>( |
| 10766 | Val: op->getParentOfType<mlir::ModuleOp>()->getDiscardableAttr( |
| 10767 | name: LLVM::LLVMDialect::getTargetTripleAttrName())); |
| 10768 | if (targetTripleAttr) |
| 10769 | targetTriple.emplace_back(args: targetTripleAttr.data()); |
| 10770 | |
| 10771 | auto fileInfoCallBack = [&loc]() { |
| 10772 | std::string filename = "" ; |
| 10773 | std::uint64_t lineNo = 0; |
| 10774 | |
| 10775 | if (loc) { |
| 10776 | filename = loc.getFilename().str(); |
| 10777 | lineNo = loc.getLine(); |
| 10778 | } |
| 10779 | |
| 10780 | return std::pair<std::string, std::uint64_t>(llvm::StringRef(filename), |
| 10781 | lineNo); |
| 10782 | }; |
| 10783 | |
| 10784 | llvm::vfs::FileSystem &vfs = moduleTranslation.getFileSystem(); |
| 10785 | ompBuilder->registerTargetGlobalVariable( |
| 10786 | CaptureClause: captureClauseKind, DeviceClause: deviceClause, IsDeclaration: isDeclaration, IsExternallyVisible: isExternallyVisible, |
| 10787 | EntryInfo: ompBuilder->getTargetEntryUniqueInfo(CallBack: fileInfoCallBack, VFS&: vfs), |
| 10788 | MangledName: entryMangledName, GeneratedRefs&: generatedRefs, /*OpenMPSimd*/ OpenMPSIMD: false, TargetTriple: targetTriple, |
| 10789 | /*GlobalInitializer*/ nullptr, VariableLinkage: variableLinkage, LlvmPtrTy: gVal->getType(), |
| 10790 | Addr: entryAddr); |
| 10791 | |
| 10792 | if (ompBuilder->Config.isTargetDevice() && |
| 10793 | (captureClause == omp::DeclareTargetCaptureClause::link || |
| 10794 | requiresUSM)) { |
| 10795 | // For USM and link we generate a global reference pointer in the |
| 10796 | // default address space (e.g address space 0), as opposed to the |
| 10797 | // globals original type and address space. |
| 10798 | llvm::Type *ptrTy = llvm::PointerType::get(C&: llvmModule->getContext(), AddressSpace: 0); |
| 10799 | llvm::Constant *refPtr = ompBuilder->getAddrOfDeclareTargetVar( |
| 10800 | CaptureClause: captureClauseKind, DeviceClause: deviceClause, IsDeclaration: isDeclaration, IsExternallyVisible: isExternallyVisible, |
| 10801 | EntryInfo: ompBuilder->getTargetEntryUniqueInfo(CallBack: fileInfoCallBack, VFS&: vfs), |
| 10802 | MangledName: mangledName, GeneratedRefs&: generatedRefs, /*OpenMPSimd*/ OpenMPSIMD: false, TargetTriple: targetTriple, |
| 10803 | LlvmPtrTy: ptrTy, /*GlobalInitializer*/ nullptr, |
| 10804 | /*VariableLinkage*/ nullptr); |
| 10805 | |
| 10806 | // For indirectly-accessed global pointers, we rely on "internal" |
| 10807 | // linkage to optimize out the unneeded full-variable storage later, |
| 10808 | // since we can't prevent the LLVM dialect from generating globals |
| 10809 | // without also breaking target lowering. However, We can only do |
| 10810 | // this for definiions, as global variable declarations must have |
| 10811 | // external or weak linkage. |
| 10812 | if (refPtr) { |
| 10813 | if (!gVar->isDeclaration()) |
| 10814 | gVar->setLinkage(llvm::GlobalValue::InternalLinkage); |
| 10815 | |
| 10816 | // Register the (original global, reference pointer) pair so that the |
| 10817 | // OpenMPIRBuilder can rewrite uses of the original global during |
| 10818 | // finalization. |
| 10819 | if (auto *newGV = |
| 10820 | dyn_cast<llvm::GlobalValue>(Val: refPtr->stripPointerCasts())) |
| 10821 | ompBuilder->registerDeclareTargetGlobalReplacement(Original: gVal, Replacement: newGV); |
| 10822 | } |
| 10823 | } |
| 10824 | |
| 10825 | // Mark 'device_type(host) enter(...)' variables as external in the device |
| 10826 | // since they're not supposed to have their own copy. This will cause |
| 10827 | // linker errors if accesses are attempted from the target device. |
| 10828 | if (ompBuilder->Config.isTargetDevice() && isHostOnly && isToOrEnter) { |
| 10829 | gVar->setLinkage(llvm::GlobalValue::ExternalLinkage); |
| 10830 | gVar->setInitializer(nullptr); |
| 10831 | } |
| 10832 | } |
| 10833 | } |
| 10834 | |
| 10835 | return success(); |
| 10836 | } |
| 10837 | |
| 10838 | namespace { |
| 10839 | |
| 10840 | /// Implementation of the dialect interface that converts operations belonging |
| 10841 | /// to the OpenMP dialect to LLVM IR. |
| 10842 | class OpenMPDialectLLVMIRTranslationInterface |
| 10843 | : public LLVMTranslationDialectInterface { |
| 10844 | public: |
| 10845 | using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface; |
| 10846 | |
| 10847 | /// Translates the given operation to LLVM IR using the provided IR builder |
| 10848 | /// and saving the state in `moduleTranslation`. |
| 10849 | LogicalResult |
| 10850 | convertOperation(Operation *op, llvm::IRBuilderBase &builder, |
| 10851 | LLVM::ModuleTranslation &moduleTranslation) const final; |
| 10852 | |
| 10853 | /// Given an OpenMP MLIR attribute, create the corresponding LLVM-IR, |
| 10854 | /// runtime calls, or operation amendments |
| 10855 | LogicalResult |
| 10856 | amendOperation(Operation *op, ArrayRef<llvm::Instruction *> instructions, |
| 10857 | NamedAttribute attribute, |
| 10858 | LLVM::ModuleTranslation &moduleTranslation) const final; |
| 10859 | |
| 10860 | /// Records the LLVM alloc pointer produced for an OMP ALLOCATE variable so |
| 10861 | /// that the paired omp.allocate_free op can generate the matching |
| 10862 | /// __kmpc_free call. |
| 10863 | void registerAllocatedPtr(Value var, llvm::Value *ptr) const { |
| 10864 | ompAllocatedPtrs[var] = ptr; |
| 10865 | } |
| 10866 | |
| 10867 | /// Returns the LLVM alloc pointer previously registered for var, or |
| 10868 | /// nullptr if no allocation was recorded. |
| 10869 | llvm::Value *lookupAllocatedPtr(Value var) const { |
| 10870 | auto it = ompAllocatedPtrs.find(Val: var); |
| 10871 | return it != ompAllocatedPtrs.end() ? it->second : nullptr; |
| 10872 | } |
| 10873 | |
| 10874 | private: |
| 10875 | /// Maps each MLIR variable value that appeared in an omp.allocate_dir op to |
| 10876 | /// the LLVM pointer returned by the corresponding __kmpc_alloc call. The |
| 10877 | /// paired omp.allocate_free op looks up these pointers to emit __kmpc_free. |
| 10878 | mutable DenseMap<Value, llvm::Value *> ompAllocatedPtrs; |
| 10879 | }; |
| 10880 | |
| 10881 | } // namespace |
| 10882 | |
| 10883 | LogicalResult OpenMPDialectLLVMIRTranslationInterface::amendOperation( |
| 10884 | Operation *op, ArrayRef<llvm::Instruction *> instructions, |
| 10885 | NamedAttribute attribute, |
| 10886 | LLVM::ModuleTranslation &moduleTranslation) const { |
| 10887 | return llvm::StringSwitch<llvm::function_ref<LogicalResult(Attribute)>>( |
| 10888 | attribute.getName()) |
| 10889 | .Case(S: "omp.is_target_device" , |
| 10890 | Value: [&](Attribute attr) { |
| 10891 | if (auto deviceAttr = dyn_cast<BoolAttr>(Val&: attr)) { |
| 10892 | llvm::OpenMPIRBuilderConfig &config = |
| 10893 | getOpenMPBuilder(moduleTranslation)->Config; |
| 10894 | config.setIsTargetDevice(deviceAttr.getValue()); |
| 10895 | return success(); |
| 10896 | } |
| 10897 | return failure(); |
| 10898 | }) |
| 10899 | .Case(S: "omp.is_gpu" , |
| 10900 | Value: [&](Attribute attr) { |
| 10901 | if (auto gpuAttr = dyn_cast<BoolAttr>(Val&: attr)) { |
| 10902 | llvm::OpenMPIRBuilderConfig &config = |
| 10903 | getOpenMPBuilder(moduleTranslation)->Config; |
| 10904 | config.setIsGPU(gpuAttr.getValue()); |
| 10905 | return success(); |
| 10906 | } |
| 10907 | return failure(); |
| 10908 | }) |
| 10909 | .Case(S: "omp.host_ir_filepath" , |
| 10910 | Value: [&](Attribute attr) { |
| 10911 | if (auto filepathAttr = dyn_cast<StringAttr>(Val&: attr)) { |
| 10912 | llvm::OpenMPIRBuilder *ompBuilder = |
| 10913 | getOpenMPBuilder(moduleTranslation); |
| 10914 | ompBuilder->loadOffloadInfoMetadata( |
| 10915 | VFS&: moduleTranslation.getFileSystem(), HostFilePath: filepathAttr.getValue()); |
| 10916 | return success(); |
| 10917 | } |
| 10918 | return failure(); |
| 10919 | }) |
| 10920 | .Case(S: "omp.flags" , |
| 10921 | Value: [&](Attribute attr) { |
| 10922 | if (auto rtlAttr = dyn_cast<omp::FlagsAttr>(Val&: attr)) |
| 10923 | return convertFlagsAttr(op, attribute: rtlAttr, moduleTranslation); |
| 10924 | return failure(); |
| 10925 | }) |
| 10926 | .Case(S: "omp.version" , |
| 10927 | Value: [&](Attribute attr) { |
| 10928 | if (auto versionAttr = dyn_cast<omp::VersionAttr>(Val&: attr)) { |
| 10929 | llvm::OpenMPIRBuilder *ompBuilder = |
| 10930 | getOpenMPBuilder(moduleTranslation); |
| 10931 | ompBuilder->M.addModuleFlag(Behavior: llvm::Module::Max, Key: "openmp" , |
| 10932 | Val: versionAttr.getVersion()); |
| 10933 | return success(); |
| 10934 | } |
| 10935 | return failure(); |
| 10936 | }) |
| 10937 | .Case(S: "omp.declare_target" , |
| 10938 | Value: [&](Attribute attr) { |
| 10939 | if (auto declareTargetAttr = |
| 10940 | dyn_cast<omp::DeclareTargetAttr>(Val&: attr)) { |
| 10941 | llvm::OpenMPIRBuilder *ompBuilder = |
| 10942 | getOpenMPBuilder(moduleTranslation); |
| 10943 | return convertDeclareTargetAttr(op, attribute: declareTargetAttr, |
| 10944 | ompBuilder, moduleTranslation); |
| 10945 | } |
| 10946 | return failure(); |
| 10947 | }) |
| 10948 | .Case(S: "omp.requires" , |
| 10949 | Value: [&](Attribute attr) { |
| 10950 | if (auto requiresAttr = dyn_cast<omp::ClauseRequiresAttr>(Val&: attr)) { |
| 10951 | using Requires = omp::ClauseRequires; |
| 10952 | Requires flags = requiresAttr.getValue(); |
| 10953 | llvm::OpenMPIRBuilderConfig &config = |
| 10954 | getOpenMPBuilder(moduleTranslation)->Config; |
| 10955 | config.setHasRequiresReverseOffload( |
| 10956 | bitEnumContainsAll(bits: flags, bit: Requires::reverse_offload)); |
| 10957 | config.setHasRequiresUnifiedAddress( |
| 10958 | bitEnumContainsAll(bits: flags, bit: Requires::unified_address)); |
| 10959 | config.setHasRequiresUnifiedSharedMemory( |
| 10960 | bitEnumContainsAll(bits: flags, bit: Requires::unified_shared_memory)); |
| 10961 | config.setHasRequiresDynamicAllocators( |
| 10962 | bitEnumContainsAll(bits: flags, bit: Requires::dynamic_allocators)); |
| 10963 | return success(); |
| 10964 | } |
| 10965 | return failure(); |
| 10966 | }) |
| 10967 | .Case(S: "omp.target_triples" , |
| 10968 | Value: [&](Attribute attr) { |
| 10969 | if (auto triplesAttr = dyn_cast<ArrayAttr>(Val&: attr)) { |
| 10970 | llvm::OpenMPIRBuilderConfig &config = |
| 10971 | getOpenMPBuilder(moduleTranslation)->Config; |
| 10972 | config.TargetTriples.clear(); |
| 10973 | config.TargetTriples.reserve(N: triplesAttr.size()); |
| 10974 | for (Attribute tripleAttr : triplesAttr) { |
| 10975 | if (auto tripleStrAttr = dyn_cast<StringAttr>(Val&: tripleAttr)) |
| 10976 | config.TargetTriples.emplace_back(Args: tripleStrAttr.getValue()); |
| 10977 | else |
| 10978 | return failure(); |
| 10979 | } |
| 10980 | return success(); |
| 10981 | } |
| 10982 | return failure(); |
| 10983 | }) |
| 10984 | .Case(S: "omp.integer_wrap_around" , |
| 10985 | Value: [&](Attribute attr) { |
| 10986 | if (auto wrapAttr = dyn_cast<omp::IntegerWrapAroundAttr>(Val&: attr)) { |
| 10987 | llvm::OpenMPIRBuilderConfig &config = |
| 10988 | getOpenMPBuilder(moduleTranslation)->Config; |
| 10989 | config.setNoSignedWrap(!wrapAttr.getIntegerWrapAround()); |
| 10990 | return success(); |
| 10991 | } |
| 10992 | return failure(); |
| 10993 | }) |
| 10994 | .Default(Value: [](Attribute) { |
| 10995 | // Fall through for omp attributes that do not require lowering. |
| 10996 | return success(); |
| 10997 | })(attribute.getValue()); |
| 10998 | |
| 10999 | return failure(); |
| 11000 | } |
| 11001 | |
| 11002 | // Returns true if the operation is not inside a TargetOp, it is part of a |
| 11003 | // function and that function is not declare target. |
| 11004 | static bool isHostDeviceOp(Operation *op) { |
| 11005 | // Assumes no reverse offloading |
| 11006 | if (op->getParentOfType<omp::TargetOp>()) |
| 11007 | return false; |
| 11008 | |
| 11009 | if (auto parentFn = op->getParentOfType<LLVM::LLVMFuncOp>()) { |
| 11010 | if (auto declareTargetIface = |
| 11011 | llvm::dyn_cast<mlir::omp::DeclareTargetInterface>( |
| 11012 | Val: parentFn.getOperation())) { |
| 11013 | omp::DeclareTargetAttr declareTargetAttr = |
| 11014 | declareTargetIface.getDeclareTarget(); |
| 11015 | if (declareTargetAttr && declareTargetAttr.getDeviceType() != |
| 11016 | mlir::omp::DeclareTargetDeviceType::host) |
| 11017 | return false; |
| 11018 | } |
| 11019 | |
| 11020 | return true; |
| 11021 | } |
| 11022 | |
| 11023 | return false; |
| 11024 | } |
| 11025 | |
| 11026 | static llvm::Function *getOmpTargetAlloc(llvm::IRBuilderBase &builder, |
| 11027 | llvm::Module *llvmModule) { |
| 11028 | llvm::Type *i64Ty = builder.getInt64Ty(); |
| 11029 | llvm::Type *i32Ty = builder.getInt32Ty(); |
| 11030 | llvm::Type *returnType = builder.getPtrTy(AddrSpace: 0); |
| 11031 | llvm::FunctionType *fnType = |
| 11032 | llvm::FunctionType::get(Result: returnType, Params: {i64Ty, i32Ty}, isVarArg: false); |
| 11033 | llvm::Function *func = cast<llvm::Function>( |
| 11034 | Val: llvmModule->getOrInsertFunction(Name: "omp_target_alloc" , T: fnType).getCallee()); |
| 11035 | return func; |
| 11036 | } |
| 11037 | |
| 11038 | template <typename T> |
| 11039 | static llvm::Value * |
| 11040 | getAllocationSize(llvm::IRBuilderBase &builder, |
| 11041 | LLVM::ModuleTranslation &moduleTranslation, T op) { |
| 11042 | llvm::DataLayout dataLayout = |
| 11043 | moduleTranslation.getLLVMModule()->getDataLayout(); |
| 11044 | llvm::Type *llvmHeapTy = |
| 11045 | moduleTranslation.convertType(type: op.getMemElemTypeAttr().getValue()); |
| 11046 | |
| 11047 | auto alignment = op.getMemAlignment(); |
| 11048 | llvm::TypeSize typeSize = llvm::alignTo( |
| 11049 | dataLayout.getTypeStoreSize(Ty: llvmHeapTy), |
| 11050 | alignment ? *alignment : dataLayout.getABITypeAlign(Ty: llvmHeapTy).value()); |
| 11051 | |
| 11052 | llvm::Value *allocSize = builder.getInt64(C: typeSize.getFixedValue()); |
| 11053 | return builder.CreateMul( |
| 11054 | LHS: allocSize, |
| 11055 | RHS: builder.CreateIntCast(moduleTranslation.lookupValue(value: op.getMemArraySize()), |
| 11056 | builder.getInt64Ty(), |
| 11057 | /*isSigned=*/false)); |
| 11058 | } |
| 11059 | |
| 11060 | template <> |
| 11061 | llvm::Value *getAllocationSize(llvm::IRBuilderBase &builder, |
| 11062 | LLVM::ModuleTranslation &moduleTranslation, |
| 11063 | omp::TargetAllocMemOp op) { |
| 11064 | llvm::DataLayout dataLayout = |
| 11065 | moduleTranslation.getLLVMModule()->getDataLayout(); |
| 11066 | llvm::Type *llvmHeapTy = moduleTranslation.convertType(type: op.getAllocatedType()); |
| 11067 | llvm::TypeSize typeSize = dataLayout.getTypeAllocSize(Ty: llvmHeapTy); |
| 11068 | llvm::Value *allocSize = builder.getInt64(C: typeSize.getFixedValue()); |
| 11069 | for (auto typeParam : op.getTypeparams()) { |
| 11070 | allocSize = builder.CreateMul( |
| 11071 | LHS: allocSize, |
| 11072 | RHS: builder.CreateIntCast(V: moduleTranslation.lookupValue(value: typeParam), |
| 11073 | DestTy: builder.getInt64Ty(), |
| 11074 | /*isSigned=*/false)); |
| 11075 | } |
| 11076 | return allocSize; |
| 11077 | } |
| 11078 | |
| 11079 | static LogicalResult |
| 11080 | convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder, |
| 11081 | LLVM::ModuleTranslation &moduleTranslation) { |
| 11082 | auto allocMemOp = cast<omp::TargetAllocMemOp>(Val&: opInst); |
| 11083 | if (!allocMemOp) |
| 11084 | return failure(); |
| 11085 | |
| 11086 | // Get "omp_target_alloc" function |
| 11087 | llvm::Module *llvmModule = moduleTranslation.getLLVMModule(); |
| 11088 | llvm::Function *ompTargetAllocFunc = getOmpTargetAlloc(builder, llvmModule); |
| 11089 | // Get the corresponding device value in llvm |
| 11090 | mlir::Value deviceNum = allocMemOp.getDevice(); |
| 11091 | llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(value: deviceNum); |
| 11092 | // Get the allocation size. |
| 11093 | llvm::Value *allocSize = |
| 11094 | getAllocationSize(builder, moduleTranslation, op: allocMemOp); |
| 11095 | // Create call to "omp_target_alloc" with the args as translated llvm values. |
| 11096 | llvm::CallInst *call = |
| 11097 | builder.CreateCall(Callee: ompTargetAllocFunc, Args: {allocSize, llvmDeviceNum}); |
| 11098 | llvm::Value *resultI64 = builder.CreatePtrToInt(V: call, DestTy: builder.getInt64Ty()); |
| 11099 | |
| 11100 | // Map the result |
| 11101 | moduleTranslation.mapValue(mlir: allocMemOp.getResult(), llvm: resultI64); |
| 11102 | return success(); |
| 11103 | } |
| 11104 | |
| 11105 | static LogicalResult |
| 11106 | convertAllocSharedMemOp(omp::AllocSharedMemOp allocMemOp, |
| 11107 | llvm::IRBuilderBase &builder, |
| 11108 | LLVM::ModuleTranslation &moduleTranslation) { |
| 11109 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 11110 | llvm::Value *size = getAllocationSize(builder, moduleTranslation, op: allocMemOp); |
| 11111 | moduleTranslation.mapValue(mlir: allocMemOp.getResult(), |
| 11112 | llvm: ompBuilder->createOMPAllocShared(Loc: builder, Size: size)); |
| 11113 | return success(); |
| 11114 | } |
| 11115 | |
| 11116 | static LogicalResult |
| 11117 | convertAllocateDirOp(Operation &opInst, llvm::IRBuilderBase &builder, |
| 11118 | LLVM::ModuleTranslation &moduleTranslation, |
| 11119 | const OpenMPDialectLLVMIRTranslationInterface &ompIface) { |
| 11120 | auto allocateDirOp = cast<omp::AllocateDirOp>(Val&: opInst); |
| 11121 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 11122 | |
| 11123 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 11124 | llvm::Module *llvmModule = moduleTranslation.getLLVMModule(); |
| 11125 | llvm::DataLayout dataLayout = llvmModule->getDataLayout(); |
| 11126 | SmallVector<Value> vars = allocateDirOp.getVarList(); |
| 11127 | std::optional<int64_t> alignAttr = allocateDirOp.getAlign(); |
| 11128 | |
| 11129 | llvm::Value *allocator; |
| 11130 | if (auto allocatorVar = allocateDirOp.getAllocator()) { |
| 11131 | allocator = moduleTranslation.lookupValue(value: allocatorVar); |
| 11132 | if (allocator->getType()->isIntegerTy()) |
| 11133 | allocator = builder.CreateIntToPtr(V: allocator, DestTy: builder.getPtrTy()); |
| 11134 | else if (allocator->getType()->isPointerTy()) |
| 11135 | allocator = builder.CreatePointerBitCastOrAddrSpaceCast( |
| 11136 | V: allocator, DestTy: builder.getPtrTy()); |
| 11137 | } else { |
| 11138 | allocator = llvm::ConstantPointerNull::get(T: builder.getPtrTy()); |
| 11139 | } |
| 11140 | |
| 11141 | for (Value var : vars) { |
| 11142 | Value baseVar = getBaseValueForTypeLookup(value: var); |
| 11143 | llvm::Type *typeToInspect = |
| 11144 | getAllocatedLlvmTypeForVariable(var, baseVar, moduleTranslation); |
| 11145 | |
| 11146 | llvm::Value *size; |
| 11147 | if (std::optional<llvm::Value *> dynamicSize = getDynamicAllocatedSize( |
| 11148 | var, baseVar, moduleTranslation, builder, dataLayout)) { |
| 11149 | size = *dynamicSize; |
| 11150 | } else if (typeToInspect->isArrayTy()) { |
| 11151 | size = builder.getInt64( |
| 11152 | C: dataLayout.getTypeAllocSize(Ty: typeToInspect).getFixedValue()); |
| 11153 | } else { |
| 11154 | size = builder.getInt64( |
| 11155 | C: dataLayout.getTypeAllocSize(Ty: typeToInspect).getFixedValue()); |
| 11156 | } |
| 11157 | |
| 11158 | uint64_t alignValue = |
| 11159 | alignAttr ? alignAttr.value() |
| 11160 | : dataLayout.getABITypeAlign(Ty: typeToInspect).value(); |
| 11161 | llvm::Value *alignConst = builder.getInt64(C: alignValue); |
| 11162 | // Align the size: ((size + align - 1) / align) * align |
| 11163 | size = builder.CreateAdd(LHS: size, RHS: builder.getInt64(C: alignValue - 1), Name: "" , HasNUW: true); |
| 11164 | size = builder.CreateUDiv(LHS: size, RHS: alignConst); |
| 11165 | size = builder.CreateMul(LHS: size, RHS: alignConst, Name: "" , HasNUW: true); |
| 11166 | |
| 11167 | std::string allocName = |
| 11168 | ompBuilder->createPlatformSpecificName(Parts: {".void.addr" }); |
| 11169 | llvm::CallInst *allocCall; |
| 11170 | if (alignAttr.has_value()) { |
| 11171 | allocCall = ompBuilder->createOMPAlignedAlloc( |
| 11172 | Loc: ompLoc, Align: builder.getInt64(C: alignAttr.value()), Size: size, Allocator: allocator, |
| 11173 | Name: allocName); |
| 11174 | } else { |
| 11175 | allocCall = |
| 11176 | ompBuilder->createOMPAlloc(Loc: ompLoc, Size: size, Allocator: allocator, Name: allocName); |
| 11177 | } |
| 11178 | // Record the alloc pointer keyed by the MLIR variable value. |
| 11179 | ompIface.registerAllocatedPtr(var, ptr: allocCall); |
| 11180 | |
| 11181 | if (llvm::Value *baseLlvm = moduleTranslation.lookupValue(value: baseVar)) { |
| 11182 | llvm::Value *boundPtr = builder.CreatePointerBitCastOrAddrSpaceCast( |
| 11183 | V: allocCall, DestTy: baseLlvm->getType()); |
| 11184 | moduleTranslation.remapAllValuesWith(oldValue: baseLlvm, newValue: boundPtr); |
| 11185 | } else if (llvm::Value *varLlvm = moduleTranslation.lookupValue(value: var)) { |
| 11186 | llvm::Value *boundPtr = builder.CreatePointerBitCastOrAddrSpaceCast( |
| 11187 | V: allocCall, DestTy: varLlvm->getType()); |
| 11188 | moduleTranslation.remapAllValuesWith(oldValue: varLlvm, newValue: boundPtr); |
| 11189 | } |
| 11190 | } |
| 11191 | |
| 11192 | return success(); |
| 11193 | } |
| 11194 | |
| 11195 | static LogicalResult |
| 11196 | convertAllocateFreeOp(Operation &opInst, llvm::IRBuilderBase &builder, |
| 11197 | LLVM::ModuleTranslation &moduleTranslation, |
| 11198 | const OpenMPDialectLLVMIRTranslationInterface &ompIface) { |
| 11199 | auto freeOp = cast<omp::AllocateFreeOp>(Val&: opInst); |
| 11200 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 11201 | llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder); |
| 11202 | |
| 11203 | llvm::Value *allocator; |
| 11204 | if (auto allocatorVar = freeOp.getAllocator()) { |
| 11205 | allocator = moduleTranslation.lookupValue(value: allocatorVar); |
| 11206 | if (allocator->getType()->isIntegerTy()) |
| 11207 | allocator = builder.CreateIntToPtr(V: allocator, DestTy: builder.getPtrTy()); |
| 11208 | else if (allocator->getType()->isPointerTy()) |
| 11209 | allocator = builder.CreatePointerBitCastOrAddrSpaceCast( |
| 11210 | V: allocator, DestTy: builder.getPtrTy()); |
| 11211 | } else { |
| 11212 | allocator = llvm::ConstantPointerNull::get(T: builder.getPtrTy()); |
| 11213 | } |
| 11214 | |
| 11215 | // Emit __kmpc_free for each variable in reverse allocation order. |
| 11216 | SmallVector<Value> vars = freeOp.getVarList(); |
| 11217 | for (Value var : llvm::reverse(C&: vars)) { |
| 11218 | llvm::Value *allocPtr = ompIface.lookupAllocatedPtr(var); |
| 11219 | if (!allocPtr) |
| 11220 | return opInst.emitError(message: "omp.allocate_free: no allocation recorded" ); |
| 11221 | ompBuilder->createOMPFree(Loc: ompLoc, Addr: allocPtr, Allocator: allocator, Name: "" ); |
| 11222 | } |
| 11223 | |
| 11224 | return success(); |
| 11225 | } |
| 11226 | |
| 11227 | static llvm::Function *getOmpTargetFree(llvm::IRBuilderBase &builder, |
| 11228 | llvm::Module *llvmModule) { |
| 11229 | llvm::Type *ptrTy = builder.getPtrTy(AddrSpace: 0); |
| 11230 | llvm::Type *i32Ty = builder.getInt32Ty(); |
| 11231 | llvm::Type *voidTy = builder.getVoidTy(); |
| 11232 | llvm::FunctionType *fnType = |
| 11233 | llvm::FunctionType::get(Result: voidTy, Params: {ptrTy, i32Ty}, isVarArg: false); |
| 11234 | llvm::Function *func = dyn_cast<llvm::Function>( |
| 11235 | Val: llvmModule->getOrInsertFunction(Name: "omp_target_free" , T: fnType).getCallee()); |
| 11236 | return func; |
| 11237 | } |
| 11238 | |
| 11239 | static LogicalResult |
| 11240 | convertTargetFreeMemOp(Operation &opInst, llvm::IRBuilderBase &builder, |
| 11241 | LLVM::ModuleTranslation &moduleTranslation) { |
| 11242 | auto freeMemOp = cast<omp::TargetFreeMemOp>(Val&: opInst); |
| 11243 | if (!freeMemOp) |
| 11244 | return failure(); |
| 11245 | |
| 11246 | // Get "omp_target_free" function |
| 11247 | llvm::Module *llvmModule = moduleTranslation.getLLVMModule(); |
| 11248 | llvm::Function *ompTragetFreeFunc = getOmpTargetFree(builder, llvmModule); |
| 11249 | // Get the corresponding device value in llvm |
| 11250 | mlir::Value deviceNum = freeMemOp.getDevice(); |
| 11251 | llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(value: deviceNum); |
| 11252 | // Get the corresponding heapref value in llvm |
| 11253 | mlir::Value heapref = freeMemOp.getHeapref(); |
| 11254 | llvm::Value *llvmHeapref = moduleTranslation.lookupValue(value: heapref); |
| 11255 | // Convert heapref int to ptr and call "omp_target_free" |
| 11256 | llvm::Value *intToPtr = |
| 11257 | builder.CreateIntToPtr(V: llvmHeapref, DestTy: builder.getPtrTy(AddrSpace: 0)); |
| 11258 | builder.CreateCall(Callee: ompTragetFreeFunc, Args: {intToPtr, llvmDeviceNum}); |
| 11259 | return success(); |
| 11260 | } |
| 11261 | |
| 11262 | static LogicalResult |
| 11263 | convertFreeSharedMemOp(omp::FreeSharedMemOp freeMemOp, |
| 11264 | llvm::IRBuilderBase &builder, |
| 11265 | LLVM::ModuleTranslation &moduleTranslation) { |
| 11266 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 11267 | llvm::Value *size = getAllocationSize(builder, moduleTranslation, op: freeMemOp); |
| 11268 | ompBuilder->createOMPFreeShared( |
| 11269 | Loc: builder, Addr: moduleTranslation.lookupValue(value: freeMemOp.getHeapref()), Size: size); |
| 11270 | return success(); |
| 11271 | } |
| 11272 | |
| 11273 | /// Converts an OpenMP groupprivate operation into LLVM IR. |
| 11274 | static LogicalResult |
| 11275 | convertOmpGroupprivate(Operation &opInst, llvm::IRBuilderBase &builder, |
| 11276 | LLVM::ModuleTranslation &moduleTranslation) { |
| 11277 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 11278 | auto groupprivateOp = cast<omp::GroupprivateOp>(Val&: opInst); |
| 11279 | |
| 11280 | if (failed(Result: checkImplementationStatus(op&: opInst))) |
| 11281 | return failure(); |
| 11282 | |
| 11283 | bool isTargetDevice = ompBuilder->Config.isTargetDevice(); |
| 11284 | |
| 11285 | // Determine whether group-private storage should be allocated based on |
| 11286 | // device_type. When not specified, default to 'any' (allocate on both). |
| 11287 | bool shouldAllocate = true; |
| 11288 | switch (groupprivateOp.getDeviceType().value_or( |
| 11289 | u: mlir::omp::DeclareTargetDeviceType::any)) { |
| 11290 | case mlir::omp::DeclareTargetDeviceType::host: |
| 11291 | shouldAllocate = !isTargetDevice; |
| 11292 | break; |
| 11293 | case mlir::omp::DeclareTargetDeviceType::nohost: |
| 11294 | shouldAllocate = isTargetDevice; |
| 11295 | break; |
| 11296 | case mlir::omp::DeclareTargetDeviceType::any: |
| 11297 | shouldAllocate = true; |
| 11298 | break; |
| 11299 | } |
| 11300 | |
| 11301 | // Look up the global variable directly by symbol name. |
| 11302 | LLVM::GlobalOp global = SymbolTable::lookupNearestSymbolFrom<LLVM::GlobalOp>( |
| 11303 | from: &opInst, symbol: groupprivateOp.getSymNameAttr()); |
| 11304 | if (!global) |
| 11305 | return opInst.emitError() |
| 11306 | << "expected symbol '" << groupprivateOp.getSymName() |
| 11307 | << "' to reference an LLVM global variable" ; |
| 11308 | |
| 11309 | llvm::GlobalValue *globalValue = moduleTranslation.lookupGlobal(op: global); |
| 11310 | llvm::Type *varType = moduleTranslation.convertType(type: global.getType()); |
| 11311 | std::string varName = globalValue->getName().str(); |
| 11312 | |
| 11313 | llvm::Value *resultPtr; |
| 11314 | if (shouldAllocate && isTargetDevice) { |
| 11315 | llvm::Module *llvmModule = moduleTranslation.getLLVMModule(); |
| 11316 | llvm::Triple targetTriple(llvmModule->getTargetTriple()); |
| 11317 | unsigned sharedAddressSpace; |
| 11318 | if (targetTriple.isAMDGCN()) |
| 11319 | sharedAddressSpace = llvm::AMDGPUAS::LOCAL_ADDRESS; |
| 11320 | else if (targetTriple.isNVPTX()) |
| 11321 | sharedAddressSpace = llvm::NVPTXAS::ADDRESS_SPACE_SHARED; |
| 11322 | else |
| 11323 | return opInst.emitError() << "groupprivate is not supported for target: " |
| 11324 | << targetTriple.str(); |
| 11325 | llvm::GlobalVariable *sharedVar = new llvm::GlobalVariable( |
| 11326 | *llvmModule, varType, /*isConstant=*/false, |
| 11327 | llvm::GlobalValue::InternalLinkage, llvm::PoisonValue::get(T: varType), |
| 11328 | varName, /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal, |
| 11329 | sharedAddressSpace, |
| 11330 | /*isExternallyInitialized=*/false); |
| 11331 | resultPtr = sharedVar; |
| 11332 | } else { |
| 11333 | if (shouldAllocate && !isTargetDevice) |
| 11334 | opInst.emitWarning(message: "groupprivate directive is currently ignored on the " |
| 11335 | "host, using original global" ); |
| 11336 | resultPtr = globalValue; |
| 11337 | } |
| 11338 | |
| 11339 | moduleTranslation.mapValue(mlir: opInst.getResult(idx: 0), llvm: resultPtr); |
| 11340 | return success(); |
| 11341 | } |
| 11342 | |
| 11343 | /// Given an OpenMP MLIR operation, create the corresponding LLVM IR (including |
| 11344 | /// OpenMP runtime calls). |
| 11345 | LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation( |
| 11346 | Operation *op, llvm::IRBuilderBase &builder, |
| 11347 | LLVM::ModuleTranslation &moduleTranslation) const { |
| 11348 | llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation); |
| 11349 | |
| 11350 | if (ompBuilder->Config.isTargetDevice() && |
| 11351 | !isa<omp::TargetOp, omp::MapInfoOp, omp::TerminatorOp, omp::YieldOp>( |
| 11352 | Val: op) && |
| 11353 | isHostDeviceOp(op)) |
| 11354 | return op->emitOpError() << "unsupported host op found in device" ; |
| 11355 | |
| 11356 | // For each loop, introduce one stack frame to hold loop information. Ensure |
| 11357 | // this is only done for the outermost loop wrapper to prevent introducing |
| 11358 | // multiple stack frames for a single loop. Initially set to null, the loop |
| 11359 | // information structure is initialized during translation of the nested |
| 11360 | // omp.loop_nest operation, making it available to translation of all loop |
| 11361 | // wrappers after their body has been successfully translated. |
| 11362 | bool isOutermostLoopWrapper = |
| 11363 | isa_and_present<omp::LoopWrapperInterface>(Val: op) && |
| 11364 | !dyn_cast_if_present<omp::LoopWrapperInterface>(Val: op->getParentOp()); |
| 11365 | |
| 11366 | // The TASKLOOP construct is implemented with an outer taskloop.context |
| 11367 | // operation which is not a loop wrapper, containing an inner taskloop |
| 11368 | // operation which is a loop wrapper. The stack frame should be pushed when |
| 11369 | // translating the outer taskloop.context and popped when translating the |
| 11370 | // inner taskloop which is a loop wrapper. We need access to the loop |
| 11371 | // information in the outer taskloop context so we need to create it and pop |
| 11372 | // it around the taskloop context not the inner loop wrapper. |
| 11373 | if (isa<omp::TaskloopContextOp>(Val: op)) |
| 11374 | isOutermostLoopWrapper = true; |
| 11375 | else if (isa<omp::TaskloopWrapperOp>(Val: op)) |
| 11376 | isOutermostLoopWrapper = false; |
| 11377 | |
| 11378 | if (isOutermostLoopWrapper) |
| 11379 | moduleTranslation.stackPush<OpenMPLoopInfoStackFrame>(); |
| 11380 | |
| 11381 | auto result = |
| 11382 | llvm::TypeSwitch<Operation *, LogicalResult>(op) |
| 11383 | .Case(caseFn: [&](omp::BarrierOp op) -> LogicalResult { |
| 11384 | if (failed(Result: checkImplementationStatus(op&: *op))) |
| 11385 | return failure(); |
| 11386 | |
| 11387 | llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = |
| 11388 | ompBuilder->createBarrier(Loc: builder, Kind: llvm::omp::OMPD_barrier); |
| 11389 | LogicalResult res = handleError(result&: afterIP, op&: *op); |
| 11390 | if (res.succeeded()) { |
| 11391 | // If the barrier generated a cancellation check, the insertion |
| 11392 | // point might now need to be changed to a new continuation block |
| 11393 | builder.restoreIP(IP: *afterIP); |
| 11394 | } |
| 11395 | return res; |
| 11396 | }) |
| 11397 | .Case(caseFn: [&](omp::TaskyieldOp op) { |
| 11398 | if (failed(Result: checkImplementationStatus(op&: *op))) |
| 11399 | return failure(); |
| 11400 | |
| 11401 | ompBuilder->createTaskyield(Loc: builder); |
| 11402 | return success(); |
| 11403 | }) |
| 11404 | .Case(caseFn: [&](omp::FlushOp op) { |
| 11405 | if (failed(Result: checkImplementationStatus(op&: *op))) |
| 11406 | return failure(); |
| 11407 | |
| 11408 | // No support in Openmp runtime function (__kmpc_flush) to accept |
| 11409 | // the argument list. |
| 11410 | // OpenMP standard states the following: |
| 11411 | // "An implementation may implement a flush with a list by ignoring |
| 11412 | // the list, and treating it the same as a flush without a list." |
| 11413 | // |
| 11414 | // The argument list is discarded so that, flush with a list is |
| 11415 | // treated same as a flush without a list. |
| 11416 | ompBuilder->createFlush(Loc: builder); |
| 11417 | return success(); |
| 11418 | }) |
| 11419 | .Case(caseFn: [&](omp::ErrorOp op) { |
| 11420 | if (failed(Result: checkImplementationStatus(op&: *op))) |
| 11421 | return failure(); |
| 11422 | |
| 11423 | llvm::Value *message = nullptr; |
| 11424 | if (mlir::Value messageExpr = op.getMessageExpr()) |
| 11425 | message = moduleTranslation.lookupValue(value: messageExpr); |
| 11426 | else if (std::optional<StringRef> msg = op.getMessage(); |
| 11427 | msg && !msg->empty()) |
| 11428 | message = builder.CreateGlobalString(Str: *msg); |
| 11429 | ompBuilder->createError( |
| 11430 | Loc: llvm::OpenMPIRBuilder::LocationDescription(builder), |
| 11431 | IsFatal: op.getSeverity() == omp::ClauseSeverity::fatal, Message: message); |
| 11432 | return success(); |
| 11433 | }) |
| 11434 | .Case(caseFn: [&](omp::ParallelOp op) { |
| 11435 | return convertOmpParallel(opInst: op, builder, moduleTranslation); |
| 11436 | }) |
| 11437 | .Case(caseFn: [&](omp::DispatchOp) { |
| 11438 | return convertOmpDispatch(opInst&: *op, builder, moduleTranslation); |
| 11439 | }) |
| 11440 | .Case(caseFn: [&](omp::MaskedOp) { |
| 11441 | return convertOmpMasked(opInst&: *op, builder, moduleTranslation); |
| 11442 | }) |
| 11443 | .Case(caseFn: [&](omp::MasterOp) { |
| 11444 | return convertOmpMaster(opInst&: *op, builder, moduleTranslation); |
| 11445 | }) |
| 11446 | .Case(caseFn: [&](omp::CriticalOp) { |
| 11447 | return convertOmpCritical(opInst&: *op, builder, moduleTranslation); |
| 11448 | }) |
| 11449 | .Case(caseFn: [&](omp::OrderedRegionOp) { |
| 11450 | return convertOmpOrderedRegion(opInst&: *op, builder, moduleTranslation); |
| 11451 | }) |
| 11452 | .Case(caseFn: [&](omp::OrderedOp) { |
| 11453 | return convertOmpOrdered(opInst&: *op, builder, moduleTranslation); |
| 11454 | }) |
| 11455 | .Case(caseFn: [&](omp::WsloopOp) { |
| 11456 | return convertOmpWsloop(opInst&: *op, builder, moduleTranslation); |
| 11457 | }) |
| 11458 | .Case(caseFn: [&](omp::SimdOp) { |
| 11459 | return convertOmpSimd(opInst&: *op, builder, moduleTranslation); |
| 11460 | }) |
| 11461 | .Case(caseFn: [&](omp::AtomicReadOp) { |
| 11462 | return convertOmpAtomicRead(opInst&: *op, builder, moduleTranslation); |
| 11463 | }) |
| 11464 | .Case(caseFn: [&](omp::AtomicWriteOp) { |
| 11465 | return convertOmpAtomicWrite(opInst&: *op, builder, moduleTranslation); |
| 11466 | }) |
| 11467 | .Case(caseFn: [&](omp::AtomicUpdateOp op) { |
| 11468 | return convertOmpAtomicUpdate(opInst&: op, builder, moduleTranslation); |
| 11469 | }) |
| 11470 | .Case(caseFn: [&](omp::AtomicCaptureOp op) { |
| 11471 | return convertOmpAtomicCapture(atomicCaptureOp: op, builder, moduleTranslation); |
| 11472 | }) |
| 11473 | .Case(caseFn: [&](omp::AtomicCompareOp op) { |
| 11474 | return convertOmpAtomicCompare(atomicCompareOp: op, builder, moduleTranslation); |
| 11475 | }) |
| 11476 | .Case(caseFn: [&](omp::CancelOp op) { |
| 11477 | return convertOmpCancel(op, builder, moduleTranslation); |
| 11478 | }) |
| 11479 | .Case(caseFn: [&](omp::CancellationPointOp op) { |
| 11480 | return convertOmpCancellationPoint(op, builder, moduleTranslation); |
| 11481 | }) |
| 11482 | .Case(caseFn: [&](omp::SectionsOp) { |
| 11483 | return convertOmpSections(opInst&: *op, builder, moduleTranslation); |
| 11484 | }) |
| 11485 | .Case(caseFn: [&](omp::ScopeOp op) { |
| 11486 | return convertOmpScope(scopeOp&: op, builder, moduleTranslation); |
| 11487 | }) |
| 11488 | .Case(caseFn: [&](omp::SingleOp op) { |
| 11489 | return convertOmpSingle(singleOp&: op, builder, moduleTranslation); |
| 11490 | }) |
| 11491 | .Case(caseFn: [&](omp::TeamsOp op) { |
| 11492 | return convertOmpTeams(op, builder, moduleTranslation); |
| 11493 | }) |
| 11494 | .Case(caseFn: [&](omp::TaskOp op) { |
| 11495 | return convertOmpTaskOp(taskOp: op, builder, moduleTranslation); |
| 11496 | }) |
| 11497 | .Case(caseFn: [&](omp::TaskloopWrapperOp op) { |
| 11498 | return convertOmpTaskloopWrapperOp(loopWrapperOp: op, builder, moduleTranslation); |
| 11499 | }) |
| 11500 | .Case(caseFn: [&](omp::TaskloopContextOp op) { |
| 11501 | return convertOmpTaskloopContextOp(contextOp: op, builder, moduleTranslation); |
| 11502 | }) |
| 11503 | .Case(caseFn: [&](omp::TaskgroupOp op) { |
| 11504 | return convertOmpTaskgroupOp(tgOp: op, builder, moduleTranslation); |
| 11505 | }) |
| 11506 | .Case(caseFn: [&](omp::TaskwaitOp op) { |
| 11507 | return convertOmpTaskwaitOp(twOp: op, builder, moduleTranslation); |
| 11508 | }) |
| 11509 | .Case(caseFn: [&](omp::InteropInitOp op) { |
| 11510 | return convertOmpInteropInitOp(initOp: op, builder, moduleTranslation); |
| 11511 | }) |
| 11512 | .Case(caseFn: [&](omp::InteropDestroyOp op) { |
| 11513 | return convertOmpInteropDestroyOp(destroyOp: op, builder, moduleTranslation); |
| 11514 | }) |
| 11515 | .Case(caseFn: [&](omp::InteropUseOp op) { |
| 11516 | return convertOmpInteropUseOp(useOp: op, builder, moduleTranslation); |
| 11517 | }) |
| 11518 | .Case<omp::YieldOp, omp::TerminatorOp, omp::DeclareMapperOp, |
| 11519 | omp::DeclareMapperInfoOp, omp::DeclareReductionOp, |
| 11520 | omp::CriticalDeclareOp>(caseFn: [](auto op) { |
| 11521 | // `yield` and `terminator` can be just omitted. The block structure |
| 11522 | // was created in the region that handles their parent operation. |
| 11523 | // `declare_reduction` will be used by reductions and is not |
| 11524 | // converted directly, skip it. |
| 11525 | // `declare_mapper` and `declare_mapper.info` are handled whenever |
| 11526 | // they are referred to through a `map` clause. |
| 11527 | // `critical.declare` is only used to declare names of critical |
| 11528 | // sections which will be used by `critical` ops and hence can be |
| 11529 | // ignored for lowering. The OpenMP IRBuilder will create unique |
| 11530 | // name for critical section names. |
| 11531 | return success(); |
| 11532 | }) |
| 11533 | .Case(caseFn: [&](omp::ThreadprivateOp) { |
| 11534 | return convertOmpThreadprivate(opInst&: *op, builder, moduleTranslation); |
| 11535 | }) |
| 11536 | .Case<omp::TargetDataOp, omp::TargetEnterDataOp, |
| 11537 | omp::TargetExitDataOp, omp::TargetUpdateOp>(caseFn: [&](auto op) { |
| 11538 | return convertOmpTargetData(op, builder, moduleTranslation); |
| 11539 | }) |
| 11540 | .Case(caseFn: [&](omp::TargetOp) { |
| 11541 | return convertOmpTarget(opInst&: *op, builder, moduleTranslation); |
| 11542 | }) |
| 11543 | .Case(caseFn: [&](omp::DistributeOp) { |
| 11544 | return convertOmpDistribute(opInst&: *op, builder, moduleTranslation); |
| 11545 | }) |
| 11546 | .Case(caseFn: [&](omp::LoopNestOp) { |
| 11547 | return convertOmpLoopNest(opInst&: *op, builder, moduleTranslation); |
| 11548 | }) |
| 11549 | .Case<omp::MapInfoOp, omp::MapBoundsOp, omp::PrivateClauseOp, |
| 11550 | omp::AffinityEntryOp, omp::IteratorOp>(caseFn: [&](auto op) { |
| 11551 | // No-op, should be handled by relevant owning operations e.g. |
| 11552 | // TargetOp, TargetEnterDataOp, TargetExitDataOp, TargetDataOp |
| 11553 | // etc. and then discarded |
| 11554 | return success(); |
| 11555 | }) |
| 11556 | .Case(caseFn: [&](omp::NewCliOp op) { |
| 11557 | // Meta-operation: Doesn't do anything by itself, but used to |
| 11558 | // identify a loop. |
| 11559 | return success(); |
| 11560 | }) |
| 11561 | .Case(caseFn: [&](omp::CanonicalLoopOp op) { |
| 11562 | return convertOmpCanonicalLoopOp(op, builder, moduleTranslation); |
| 11563 | }) |
| 11564 | .Case(caseFn: [&](omp::UnrollHeuristicOp op) { |
| 11565 | // FIXME: Handling omp.unroll_heuristic as an executable requires |
| 11566 | // that the generator (e.g. omp.canonical_loop) has been seen first. |
| 11567 | // For construct that require all codegen to occur inside a callback |
| 11568 | // (e.g. OpenMPIRBilder::createParallel), all codegen of that |
| 11569 | // contained region including their transformations must occur at |
| 11570 | // the omp.canonical_loop. |
| 11571 | return applyUnrollHeuristic(op, builder, moduleTranslation); |
| 11572 | }) |
| 11573 | .Case(caseFn: [&](omp::UnrollFullOp op) { |
| 11574 | return applyUnrollFull(op, builder, moduleTranslation); |
| 11575 | }) |
| 11576 | .Case(caseFn: [&](omp::UnrollPartialOp op) { |
| 11577 | return applyUnrollPartial(op, builder, moduleTranslation); |
| 11578 | }) |
| 11579 | .Case(caseFn: [&](omp::TileOp op) { |
| 11580 | return applyTile(op, builder, moduleTranslation); |
| 11581 | }) |
| 11582 | .Case(caseFn: [&](omp::FuseOp op) { |
| 11583 | return applyFuse(op, builder, moduleTranslation); |
| 11584 | }) |
| 11585 | .Case(caseFn: [&](omp::TargetAllocMemOp) { |
| 11586 | return convertTargetAllocMemOp(opInst&: *op, builder, moduleTranslation); |
| 11587 | }) |
| 11588 | .Case(caseFn: [&](omp::TargetFreeMemOp) { |
| 11589 | return convertTargetFreeMemOp(opInst&: *op, builder, moduleTranslation); |
| 11590 | }) |
| 11591 | .Case(caseFn: [&](omp::AllocateDirOp) { |
| 11592 | return convertAllocateDirOp(opInst&: *op, builder, moduleTranslation, ompIface: *this); |
| 11593 | }) |
| 11594 | .Case(caseFn: [&](omp::AllocateFreeOp) { |
| 11595 | return convertAllocateFreeOp(opInst&: *op, builder, moduleTranslation, |
| 11596 | ompIface: *this); |
| 11597 | }) |
| 11598 | .Case(caseFn: [&](omp::AllocSharedMemOp op) { |
| 11599 | return convertAllocSharedMemOp(allocMemOp: op, builder, moduleTranslation); |
| 11600 | }) |
| 11601 | .Case(caseFn: [&](omp::FreeSharedMemOp op) { |
| 11602 | return convertFreeSharedMemOp(freeMemOp: op, builder, moduleTranslation); |
| 11603 | }) |
| 11604 | .Case(caseFn: [&](omp::GroupprivateOp) { |
| 11605 | return convertOmpGroupprivate(opInst&: *op, builder, moduleTranslation); |
| 11606 | }) |
| 11607 | .Case(caseFn: [&](omp::DeclareSimdOp) { |
| 11608 | // TODO: Implement. |
| 11609 | // DECLARE_SIMD can be ignored. There is warning emitted in |
| 11610 | // semantic checks about the current situation, but not having an |
| 11611 | // implementation should not prevent compilation. |
| 11612 | return success(); |
| 11613 | }) |
| 11614 | .Default(defaultFn: [&](Operation *inst) { |
| 11615 | return inst->emitError() |
| 11616 | << "not yet implemented: " << inst->getName(); |
| 11617 | }); |
| 11618 | |
| 11619 | if (isOutermostLoopWrapper) |
| 11620 | moduleTranslation.stackPop(); |
| 11621 | |
| 11622 | return result; |
| 11623 | } |
| 11624 | |
| 11625 | void mlir::registerOpenMPTranslationCLOptions() { |
| 11626 | static llvm::cl::opt<bool, /*ExternalStorage=*/true> option( |
| 11627 | "openmp-disable-virtual-collapse" , llvm::cl::Hidden, |
| 11628 | llvm::cl::desc("Disable virtual collapse of eligible host loops" ), |
| 11629 | llvm::cl::location(L&: disableVirtualCollapse)); |
| 11630 | } |
| 11631 | |
| 11632 | void mlir::registerOpenMPDialectTranslation(DialectRegistry ®istry) { |
| 11633 | registry.insert<omp::OpenMPDialect>(); |
| 11634 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, omp::OpenMPDialect *dialect) { |
| 11635 | dialect->addInterfaces<OpenMPDialectLLVMIRTranslationInterface>(); |
| 11636 | }); |
| 11637 | } |
| 11638 | |
| 11639 | void mlir::registerOpenMPDialectTranslation(MLIRContext &context) { |
| 11640 | DialectRegistry registry; |
| 11641 | registerOpenMPDialectTranslation(registry); |
| 11642 | context.appendDialectRegistry(registry); |
| 11643 | } |
| 11644 | |