1//===- OpenMPToLLVMIRTranslation.cpp - Translate OpenMP dialect to LLVM IR-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a translation between the MLIR OpenMP dialect and LLVM
10// IR.
11//
12//===----------------------------------------------------------------------===//
13#include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h"
14#include "mlir/Analysis/TopologicalSortUtils.h"
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
17#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
18#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
19#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
20#include "mlir/Dialect/OpenMP/Utils/Utils.h"
21#include "mlir/IR/Operation.h"
22#include "mlir/Interfaces/SideEffectInterfaces.h"
23#include "mlir/Support/LLVM.h"
24#include "mlir/Target/LLVMIR/Dialect/OpenMPCommon.h"
25#include "mlir/Target/LLVMIR/ModuleTranslation.h"
26
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/SmallVector.h"
29#include "llvm/ADT/TypeSwitch.h"
30#include "llvm/Frontend/OpenMP/OMPConstants.h"
31#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DebugInfoMetadata.h"
34#include "llvm/IR/DerivedTypes.h"
35#include "llvm/IR/IRBuilder.h"
36#include "llvm/IR/MDBuilder.h"
37#include "llvm/IR/ReplaceConstant.h"
38#include "llvm/Support/AMDGPUAddrSpace.h"
39#include "llvm/Support/CommandLine.h"
40#include "llvm/Support/FileSystem.h"
41#include "llvm/Support/MathExtras.h"
42#include "llvm/Support/NVPTXAddrSpace.h"
43#include "llvm/Support/SaveAndRestore.h"
44#include "llvm/Support/VirtualFileSystem.h"
45#include "llvm/TargetParser/Triple.h"
46#include "llvm/Transforms/Utils/ModuleUtils.h"
47
48#include <cstdint>
49#include <iterator>
50#include <numeric>
51#include <optional>
52#include <utility>
53
54using namespace mlir;
55
56static llvm::Expected<llvm::Value *>
57getPrivateDataAddress(llvm::IRBuilderBase &builder,
58 LLVM::ModuleTranslation &moduleTranslation,
59 omp::PrivateClauseOp privatizer, llvm::Value *var);
60
61namespace {
62/// Stores the option value so the OpenMP-to-LLVM IR translation can read it
63/// outside registerOpenMPTranslationCLOptions(). The local cl::opt avoids a
64/// global constructor in MLIR and writes to this variable through cl::location.
65/// Without registration, false keeps Virtual enabled for eligible loops.
66static bool disableVirtualCollapse = false;
67
68static llvm::omp::ScheduleKind
69convertToScheduleKind(std::optional<omp::ClauseScheduleKind> schedKind) {
70 if (!schedKind.has_value())
71 return llvm::omp::OMP_SCHEDULE_Default;
72 switch (schedKind.value()) {
73 case omp::ClauseScheduleKind::Static:
74 return llvm::omp::OMP_SCHEDULE_Static;
75 case omp::ClauseScheduleKind::Dynamic:
76 return llvm::omp::OMP_SCHEDULE_Dynamic;
77 case omp::ClauseScheduleKind::Guided:
78 return llvm::omp::OMP_SCHEDULE_Guided;
79 case omp::ClauseScheduleKind::Auto:
80 return llvm::omp::OMP_SCHEDULE_Auto;
81 case omp::ClauseScheduleKind::Runtime:
82 return llvm::omp::OMP_SCHEDULE_Runtime;
83 case omp::ClauseScheduleKind::Distribute:
84 return llvm::omp::OMP_SCHEDULE_Distribute;
85 }
86 llvm_unreachable("unhandled schedule clause argument");
87}
88
89/// ModuleTranslation stack frame for OpenMP operations. This keeps track of the
90/// blocks in which allocas are emitted.
91class OpenMPAllocStackFrame
92 : public StateStackFrameBase<OpenMPAllocStackFrame> {
93public:
94 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPAllocStackFrame)
95
96 explicit OpenMPAllocStackFrame(
97 llvm::BasicBlock *allocBlock,
98 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks)
99 : allocBlock(allocBlock), deallocBlocks(deallocBlocks) {}
100 llvm::BasicBlock *allocBlock;
101 llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
102};
103
104/// Stack frame to hold a \see llvm::CanonicalLoopInfo representing the
105/// collapsed canonical loop information corresponding to an \c omp.loop_nest
106/// operation.
107class OpenMPLoopInfoStackFrame
108 : public StateStackFrameBase<OpenMPLoopInfoStackFrame> {
109public:
110 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPLoopInfoStackFrame)
111 llvm::CanonicalLoopInfo *loopInfo = nullptr;
112};
113
114/// OpenMP state that lives as long as the module translation: the
115/// OpenMPIRBuilder associated with the LLVM module being constructed and the
116/// not yet consumed canonical loop handles.
117class OpenMPTranslationState : public LLVM::ModuleTranslation::Extension {
118public:
119 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPTranslationState)
120
121 explicit OpenMPTranslationState(llvm::Module &module) : ompBuilder(module) {
122 // Flags represented as top-level OpenMP dialect attributes are set in
123 // `OpenMPDialectLLVMIRTranslationInterface::amendOperation()`. Here we set
124 // the default configuration.
125 llvm::OpenMPIRBuilderConfig config(
126 /* IsTargetDevice = */ false, /* IsGPU = */ false,
127 /* OpenMPOffloadMandatory = */ false,
128 /* HasRequiresReverseOffload = */ false,
129 /* HasRequiresUnifiedAddress = */ false,
130 /* HasRequiresUnifiedSharedMemory = */ false,
131 /* HasRequiresDynamicAllocators = */ false);
132 config.setDefaultTargetAS(module.getDataLayout().getProgramAddressSpace());
133 config.setRuntimeCC(module.getTargetTriple().isSPIRV()
134 ? llvm::CallingConv::SPIR_FUNC
135 : llvm::CallingConv::C);
136 ompBuilder.setConfig(std::move(config));
137 ompBuilder.initialize();
138 }
139
140 ~OpenMPTranslationState() override {
141 if (!ompBuilder.isFinalized())
142 ompBuilder.finalize();
143 }
144
145 LogicalResult finalize() override {
146 ompBuilder.finalize();
147 return success();
148 }
149
150 /// Builder for LLVM IR generation of OpenMP constructs.
151 llvm::OpenMPIRBuilder ompBuilder;
152
153 /// List of not yet consumed MLIR loop handles (represented by an omp.new_cli
154 /// operation which creates a value of type CanonicalLoopInfoType) and their
155 /// LLVM-IR representation as CanonicalLoopInfo which is managed by the
156 /// OpenMPIRBuilder.
157 DenseMap<omp::NewCliOp, llvm::CanonicalLoopInfo *> loopMapping;
158};
159} // namespace
160
161static OpenMPTranslationState &
162getOpenMPState(LLVM::ModuleTranslation &moduleTranslation) {
163 return moduleTranslation.getOrCreateExtension<OpenMPTranslationState>(
164 args&: *moduleTranslation.getLLVMModule());
165}
166
167/// Returns the OpenMP IR builder associated with the LLVM IR module being
168/// constructed.
169static llvm::OpenMPIRBuilder *
170getOpenMPBuilder(LLVM::ModuleTranslation &moduleTranslation) {
171 return &getOpenMPState(moduleTranslation).ompBuilder;
172}
173
174/// Find the LLVM-IR loop that represents an MLIR loop.
175static llvm::CanonicalLoopInfo *
176lookupOMPLoop(LLVM::ModuleTranslation &moduleTranslation, Value mlir) {
177 auto cli = mlir.getDefiningOp<omp::NewCliOp>();
178 llvm::CanonicalLoopInfo *result =
179 getOpenMPState(moduleTranslation).loopMapping.lookup(Val: cli);
180 assert(result && "attempt to get non-existing loop");
181 return result;
182}
183
184/// Mark an OpenMP loop as having been consumed.
185static void invalidateOmpLoop(LLVM::ModuleTranslation &moduleTranslation,
186 Value mlir) {
187 auto cli = mlir.getDefiningOp<omp::NewCliOp>();
188 getOpenMPState(moduleTranslation).loopMapping.erase(Val: cli);
189}
190
191/// Map an MLIR OpenMP dialect CanonicalLoopInfo to its lowered LLVM-IR
192/// OpenMPIRBuilder CanonicalLoopInfo.
193static void mapOmpLoop(LLVM::ModuleTranslation &moduleTranslation, Value mlir,
194 llvm::CanonicalLoopInfo *llvm) {
195 assert(llvm && "argument must be non-null");
196 auto cli = mlir.getDefiningOp<omp::NewCliOp>();
197 llvm::CanonicalLoopInfo *&cur =
198 getOpenMPState(moduleTranslation).loopMapping[cli];
199 assert(cur == nullptr && "attempting to map a loop that is already mapped");
200 cur = llvm;
201}
202
203namespace {
204/// Custom error class to signal translation errors that don't need reporting,
205/// since encountering them will have already triggered relevant error messages.
206///
207/// Its purpose is to serve as the glue between MLIR failures represented as
208/// \see LogicalResult instances and \see llvm::Error instances used to
209/// propagate errors through the \see llvm::OpenMPIRBuilder. Generally, when an
210/// error of the first type is raised, a message is emitted directly (the \see
211/// LogicalResult itself does not hold any information). If we need to forward
212/// this error condition as an \see llvm::Error while avoiding triggering some
213/// redundant error reporting later on, we need a custom \see llvm::ErrorInfo
214/// class to just signal this situation has happened.
215///
216/// For example, this class should be used to trigger errors from within
217/// callbacks passed to the \see OpenMPIRBuilder when they were triggered by the
218/// translation of their own regions. This unclutters the error log from
219/// redundant messages.
220class PreviouslyReportedError
221 : public llvm::ErrorInfo<PreviouslyReportedError> {
222public:
223 void log(raw_ostream &) const override {
224 // Do not log anything.
225 }
226
227 std::error_code convertToErrorCode() const override {
228 llvm_unreachable(
229 "PreviouslyReportedError doesn't support ECError conversion");
230 }
231
232 // Used by ErrorInfo::classID.
233 static char ID;
234};
235
236char PreviouslyReportedError::ID = 0;
237
238/*
239 * Custom class for processing linear clause for omp.wsloop
240 * and omp.simd. Linear clause translation requires setup,
241 * initialization, update, and finalization at varying
242 * basic blocks in the IR. This class helps maintain
243 * internal state to allow consistent translation in
244 * each of these stages.
245 */
246
247class LinearClauseProcessor {
248
249private:
250 SmallVector<llvm::Type *> linearVarTypes;
251 SmallVector<llvm::Value *> linearSteps;
252 // The `linear` privatizer, original and private copy of each linear var.
253 SmallVector<omp::PrivateClauseOp> linearPrivatizers;
254 SmallVector<llvm::Value *> linearOrigVars;
255 SmallVector<llvm::Value *> linearPrivateVars;
256 // The value of each linear var on entry to the loop.
257 SmallVector<llvm::Value *> linearStartVals;
258 llvm::BasicBlock *linearFinalizationBB;
259 llvm::BasicBlock *linearExitBB;
260 llvm::BasicBlock *linearLastIterExitBB;
261 std::optional<size_t> linearLoopIVIndex;
262
263 // Address of the value of linear var `index` held by `var`.
264 llvm::Expected<llvm::Value *>
265 getAddress(llvm::IRBuilderBase &builder,
266 LLVM::ModuleTranslation &moduleTranslation, size_t index,
267 llvm::Value *var) {
268 return getPrivateDataAddress(builder, moduleTranslation,
269 privatizer: linearPrivatizers[index], var);
270 }
271
272 // Copy the current value of each linear var back to the original.
273 llvm::Error storeLinearVars(llvm::IRBuilderBase &builder,
274 LLVM::ModuleTranslation &moduleTranslation) {
275 for (size_t index = 0; index < linearOrigVars.size(); index++) {
276 llvm::Expected<llvm::Value *> from = getAddress(
277 builder, moduleTranslation, index, var: linearPrivateVars[index]);
278 if (!from)
279 return from.takeError();
280 llvm::LoadInst *linearVarTemp =
281 builder.CreateLoad(Ty: linearVarTypes[index], Ptr: *from);
282 llvm::Expected<llvm::Value *> to =
283 getAddress(builder, moduleTranslation, index, var: linearOrigVars[index]);
284 if (!to)
285 return to.takeError();
286 builder.CreateStore(Val: linearVarTemp, Ptr: *to);
287 }
288 return llvm::Error::success();
289 }
290
291public:
292 // Register type for the linear variables
293 void registerType(LLVM::ModuleTranslation &moduleTranslation,
294 mlir::Attribute &ty) {
295 linearVarTypes.push_back(Elt: moduleTranslation.convertType(
296 type: mlir::cast<mlir::TypeAttr>(Val&: ty).getValue()));
297 }
298
299 // Register a linear var, allocating space for its start value at `allocaIP`
300 void createLinearVar(llvm::IRBuilderBase &builder,
301 const llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
302 omp::PrivateClauseOp privatizer, llvm::Value *origVar,
303 llvm::Value *privateVar, int idx) {
304 llvm::IRBuilderBase::InsertPointGuard guard(builder);
305 builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator());
306 linearStartVals.push_back(
307 Elt: builder.CreateAlloca(Ty: linearVarTypes[idx], ArraySize: nullptr, Name: ".linear_var"));
308 linearPrivatizers.push_back(Elt: privatizer);
309 linearOrigVars.push_back(Elt: origVar);
310 linearPrivateVars.push_back(Elt: privateVar);
311 }
312
313 // Initialize linear step
314 inline void initLinearStep(LLVM::ModuleTranslation &moduleTranslation,
315 mlir::Value &linearStep) {
316 linearSteps.push_back(Elt: moduleTranslation.lookupValue(value: linearStep));
317 }
318
319 // Emit IR for initialization of linear variables
320 llvm::Error initLinearVar(llvm::IRBuilderBase &builder,
321 LLVM::ModuleTranslation &moduleTranslation,
322 llvm::BasicBlock *loopPreHeader) {
323 builder.SetInsertPoint(loopPreHeader->getTerminator());
324 for (size_t index = 0; index < linearOrigVars.size(); index++) {
325 llvm::Expected<llvm::Value *> orig =
326 getAddress(builder, moduleTranslation, index, var: linearOrigVars[index]);
327 if (!orig)
328 return orig.takeError();
329 llvm::LoadInst *linearVarLoad =
330 builder.CreateLoad(Ty: linearVarTypes[index], Ptr: *orig);
331 builder.CreateStore(Val: linearVarLoad, Ptr: linearStartVals[index]);
332 }
333 return llvm::Error::success();
334 }
335
336 // Find linear iteration variable and save it for later updates
337 LogicalResult initLinearIV(omp::SimdOp simdOp) {
338 auto loopOp = cast<omp::LoopNestOp>(Val: simdOp.getWrappedLoop());
339 // NOTE iteration variables can only be linear in non-nested loops.
340 if (loopOp.getIVs().size() != 1)
341 return success();
342 // Currently, frontends using `omp.simd` always generate a store from the
343 // `omp.loop_nest`'s IV to the corresponding iteration variable, possibly
344 // through the data address of its private copy.
345 //
346 // TODO Add an attribute to `omp.loop_nest` that explicitly lists the
347 // variables that correspond to the loop induction variables.
348 MutableArrayRef<BlockArgument> linearArgs =
349 cast<omp::BlockArgOpenMPOpInterface>(Val&: *simdOp).getLinearBlockArgs();
350 SmallVector<Value> ivValues{loopOp.getIVs().front()};
351 for (Operation *user : loopOp.getIVs().front().getUsers())
352 if (isa<LLVM::SExtOp, LLVM::ZExtOp, LLVM::TruncOp>(Val: user))
353 ivValues.push_back(Elt: user->getResult(idx: 0));
354
355 for (Value ivValue : ivValues) {
356 for (Operation *user : ivValue.getUsers()) {
357 auto storeOp = dyn_cast<LLVM::StoreOp>(Val: user);
358 if (!storeOp || storeOp.getValue() != ivValue)
359 continue;
360 // Look through loads, GEPs and copies of a private copy, e.g. of a
361 // descriptor, to find the linear block argument the IV is stored to.
362 Value addr = storeOp.getAddr();
363 while (true) {
364 Operation *def = addr.getDefiningOp();
365 if (auto load = dyn_cast_if_present<LLVM::LoadOp>(Val: def)) {
366 addr = load.getAddr();
367 } else if (auto gep = dyn_cast_if_present<LLVM::GEPOp>(Val: def)) {
368 addr = gep.getBase();
369 } else {
370 auto copy = llvm::find_if(Range: addr.getUsers(), P: [&](Operation *user) {
371 auto memcpy = dyn_cast<LLVM::MemcpyOp>(Val: user);
372 return memcpy && memcpy.getDst() == addr &&
373 llvm::is_contained(Range&: linearArgs, Element: memcpy.getSrc());
374 });
375 if (copy == addr.getUsers().end())
376 break;
377 addr = cast<LLVM::MemcpyOp>(Val: *copy).getSrc();
378 }
379 }
380 auto *it = llvm::find(Range&: linearArgs, Val: addr);
381 if (it == linearArgs.end())
382 continue;
383 size_t index = std::distance(first: linearArgs.begin(), last: it);
384 if (linearLoopIVIndex && *linearLoopIVIndex != index)
385 return simdOp.emitError(
386 message: "Could not determine the linear variable associated with the "
387 "loop nest induction variable");
388 linearLoopIVIndex = index;
389 }
390 }
391 return success();
392 }
393
394 // Emit IR for updating Linear variables
395 llvm::Error updateLinearVar(llvm::IRBuilderBase &builder,
396 LLVM::ModuleTranslation &moduleTranslation,
397 llvm::BasicBlock *loopBody,
398 llvm::Value *loopInductionVar) {
399 builder.SetInsertPoint(loopBody->getTerminator());
400 for (size_t index = 0; index < linearStartVals.size(); index++) {
401 llvm::Type *linearVarType = linearVarTypes[index];
402 llvm::Value *iv = loopInductionVar;
403 llvm::Value *step = linearSteps[index];
404 llvm::Expected<llvm::Value *> result = getAddress(
405 builder, moduleTranslation, index, var: linearPrivateVars[index]);
406 if (!result)
407 return result.takeError();
408
409 if (!iv->getType()->isIntegerTy())
410 llvm_unreachable("OpenMP loop induction variable must be an integer "
411 "type");
412
413 if (linearVarType->isIntegerTy()) {
414 // Integer path: normalize all arithmetic to linearVarType
415 iv = builder.CreateSExtOrTrunc(V: iv, DestTy: linearVarType);
416 step = builder.CreateSExtOrTrunc(V: step, DestTy: linearVarType);
417
418 llvm::LoadInst *linearVarStart =
419 builder.CreateLoad(Ty: linearVarType, Ptr: linearStartVals[index]);
420 llvm::Value *mulInst = builder.CreateMul(LHS: iv, RHS: step);
421 llvm::Value *addInst = builder.CreateAdd(LHS: linearVarStart, RHS: mulInst);
422 builder.CreateStore(Val: addInst, Ptr: *result);
423 } else if (linearVarType->isFloatingPointTy()) {
424 // Float path: perform multiply in integer, then convert to float
425 step = builder.CreateSExtOrTrunc(V: step, DestTy: iv->getType());
426 llvm::Value *mulInst = builder.CreateMul(LHS: iv, RHS: step);
427
428 llvm::LoadInst *linearVarStart =
429 builder.CreateLoad(Ty: linearVarType, Ptr: linearStartVals[index]);
430 llvm::Value *mulFp = builder.CreateSIToFP(V: mulInst, DestTy: linearVarType);
431 llvm::Value *addInst = builder.CreateFAdd(L: linearVarStart, R: mulFp);
432 builder.CreateStore(Val: addInst, Ptr: *result);
433 } else {
434 llvm_unreachable(
435 "Linear variable must be of integer or floating-point type");
436 }
437 }
438 return llvm::Error::success();
439 }
440
441 // Emit IR for updating linear iteration variables on loop exit
442 llvm::Error updateLinearIV(llvm::IRBuilderBase &builder,
443 LLVM::ModuleTranslation &moduleTranslation) {
444 if (!linearLoopIVIndex)
445 return llvm::Error::success();
446 size_t index = *linearLoopIVIndex;
447
448 // Add one more step to the linear iteration variable
449 llvm::Type *varType = linearVarTypes[index];
450 if (!varType->isIntegerTy())
451 llvm_unreachable("Linear iteration variable must be of integer type");
452 llvm::Expected<llvm::Value *> var =
453 getAddress(builder, moduleTranslation, index, var: linearPrivateVars[index]);
454 if (!var)
455 return var.takeError();
456 llvm::Value *step = builder.CreateSExtOrTrunc(V: linearSteps[index], DestTy: varType);
457 llvm::Value *val = builder.CreateLoad(Ty: varType, Ptr: *var);
458 llvm::Value *addInst = builder.CreateAdd(LHS: val, RHS: step);
459 builder.CreateStore(Val: addInst, Ptr: *var);
460 return llvm::Error::success();
461 }
462
463 // Linear variable finalization is conditional on the last logical iteration.
464 // Create BB splits to manage the same.
465 void splitLinearFiniBB(llvm::IRBuilderBase &builder,
466 llvm::BasicBlock *loopExit) {
467 linearFinalizationBB = loopExit->splitBasicBlock(
468 I: loopExit->getTerminator(), BBName: "omp_loop.linear_finalization");
469 linearExitBB = linearFinalizationBB->splitBasicBlock(
470 I: linearFinalizationBB->getTerminator(), BBName: "omp_loop.linear_exit");
471 linearLastIterExitBB = linearFinalizationBB->splitBasicBlock(
472 I: linearFinalizationBB->getTerminator(), BBName: "omp_loop.linear_lastiter_exit");
473 }
474
475 // Finalize the linear vars
476 llvm::OpenMPIRBuilder::InsertPointOrErrorTy
477 finalizeLinearVar(llvm::IRBuilderBase &builder,
478 LLVM::ModuleTranslation &moduleTranslation,
479 llvm::Value *lastIter) {
480 // Emit condition to check whether last logical iteration is being executed
481 builder.SetInsertPoint(linearFinalizationBB->getTerminator());
482 llvm::Value *loopLastIterLoad = builder.CreateLoad(
483 Ty: llvm::Type::getInt32Ty(C&: builder.getContext()), Ptr: lastIter);
484 llvm::Value *isLast =
485 builder.CreateCmp(Pred: llvm::CmpInst::ICMP_NE, LHS: loopLastIterLoad,
486 RHS: llvm::ConstantInt::get(
487 Ty: llvm::Type::getInt32Ty(C&: builder.getContext()), V: 0));
488 // Store the linear variable values to original variables.
489 builder.SetInsertPoint(linearLastIterExitBB->getTerminator());
490 if (llvm::Error err = storeLinearVars(builder, moduleTranslation))
491 return std::move(err);
492
493 // Create conditional branch such that the linear variable
494 // values are stored to original variables only at the
495 // last logical iteration
496 builder.SetInsertPoint(linearFinalizationBB->getTerminator());
497 builder.CreateCondBr(Cond: isLast, True: linearLastIterExitBB, False: linearExitBB);
498 linearFinalizationBB->getTerminator()->eraseFromParent();
499 // Emit barrier
500 builder.SetInsertPoint(linearExitBB->getTerminator());
501 return getOpenMPBuilder(moduleTranslation)
502 ->createBarrier(Loc: builder, Kind: llvm::omp::OMPD_barrier);
503 }
504
505 // Emit stores for linear variables. Useful in case of SIMD
506 // construct.
507 llvm::Error
508 emitStoresForLinearVar(llvm::IRBuilderBase &builder,
509 LLVM::ModuleTranslation &moduleTranslation) {
510 return storeLinearVars(builder, moduleTranslation);
511 }
512};
513} // namespace
514
515/// Looks up from the operation from and returns the PrivateClauseOp with
516/// name symbolName
517static omp::PrivateClauseOp findPrivatizer(Operation *from,
518 SymbolRefAttr symbolName) {
519 omp::PrivateClauseOp privatizer =
520 SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(from,
521 symbol: symbolName);
522 assert(privatizer && "privatizer not found in the symbol table");
523 return privatizer;
524}
525
526/// Check whether translation to LLVM IR for the given operation is currently
527/// supported. If not, descriptive diagnostics will be emitted to let users know
528/// this is a not-yet-implemented feature.
529///
530/// \returns success if no unimplemented features are needed to translate the
531/// given operation.
532static LogicalResult checkImplementationStatus(Operation &op) {
533 auto todo = [&op](StringRef clauseName) {
534 return op.emitError() << "not yet implemented: Unhandled clause "
535 << clauseName << " in " << op.getName()
536 << " operation";
537 };
538
539 auto checkAllocate = [&todo](auto op, LogicalResult &result) {
540 if (!op.getAllocateVars().empty() || !op.getAllocatorVars().empty())
541 result = todo("allocate");
542 };
543 auto checkBare = [&todo](auto op, LogicalResult &result) {
544 if (op.getKernelType() == omp::TargetExecMode::bare)
545 result = todo("ompx_bare");
546 };
547 auto checkDepend = [&todo](auto op, LogicalResult &result) {
548 if (!op.getDependVars().empty() || op.getDependKinds() ||
549 !op.getDependIterated().empty() || op.getDependIteratedKinds())
550 result = todo("depend");
551 };
552 auto checkHint = [](auto op, LogicalResult &) {
553 if (op.getHint())
554 op.emitWarning("hint clause discarded");
555 };
556 auto checkInReduction = [&todo](auto op, LogicalResult &result) {
557 if (isa<omp::TargetOp, omp::TaskOp, omp::TaskloopContextOp>(
558 op.getOperation())) {
559 if (auto byrefAttr = op.getInReductionByref()) {
560 for (bool isByRef : *byrefAttr) {
561 if (isByRef) {
562 result = todo("in_reduction with byref modifier");
563 return;
564 }
565 }
566 }
567 if (isa<omp::TargetOp>(op.getOperation())) {
568 if (auto inReductionSyms = op.getInReductionSyms()) {
569 for (auto sym :
570 (*inReductionSyms).template getAsRange<SymbolRefAttr>()) {
571 auto decl =
572 SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>(
573 op, sym);
574 assert(decl &&
575 "symbol resolution should be guaranteed by the op verifier");
576 if (decl.getInitializerRegion().front().getNumArguments() != 1) {
577 result = todo("in_reduction with two-argument initializer");
578 return;
579 }
580 if (!decl.getCleanupRegion().empty()) {
581 result = todo("in_reduction with cleanup region");
582 return;
583 }
584 }
585 }
586 }
587 } else if (!op.getInReductionVars().empty() || op.getInReductionByref() ||
588 op.getInReductionSyms()) {
589 result = todo("in_reduction");
590 }
591 };
592 auto checkNowait = [&todo](auto op, LogicalResult &result) {
593 if (op.getNowait())
594 result = todo("nowait");
595 };
596 auto checkOrder = [&todo](auto op, LogicalResult &result) {
597 if (op.getOrder() || op.getOrderMod())
598 result = todo("order");
599 };
600 auto checkPrivate = [&todo](auto op, LogicalResult &result) {
601 if (!op.getPrivateVars().empty() || op.getPrivateSyms())
602 result = todo("privatization");
603 };
604 auto checkReduction = [&todo](auto op, LogicalResult &result) {
605 if (isa<omp::TeamsOp>(op))
606 if (!op.getReductionVars().empty() || op.getReductionByref() ||
607 op.getReductionSyms())
608 result = todo("reduction");
609 for (unsigned i = 0, e = op.getNumReductionVars(); i != e; ++i) {
610 omp::ReductionModifier mod = op.getReductionModifier(i);
611 if (mod == omp::ReductionModifier::defaultmod)
612 continue;
613 // Task modifiers are supported only on parallel and worksharing
614 // constructs, and only for non-byref reduction variables.
615 bool taskModifierSupported =
616 mod == omp::ReductionModifier::task &&
617 isa<omp::ParallelOp, omp::WsloopOp, omp::SectionsOp>(op);
618 if (!taskModifierSupported) {
619 result = todo("reduction with modifier");
620 } else if (auto byref = op.getReductionByref()) {
621 if ((*byref)[i])
622 result = todo("task reduction modifier with by-ref reduction");
623 }
624 }
625 };
626 auto checkTaskReductionByref = [&todo](auto op, LogicalResult &result) {
627 if (auto byrefAttr = op.getTaskReductionByref())
628 for (bool isByRef : *byrefAttr)
629 if (isByRef) {
630 result = todo("task_reduction with byref modifier");
631 return;
632 }
633 };
634 auto checkReductionByref = [&todo](auto op, LogicalResult &result) {
635 if (auto byrefAttr = op.getReductionByref())
636 for (bool isByRef : *byrefAttr)
637 if (isByRef) {
638 result = todo("reduction with byref modifier");
639 return;
640 }
641 };
642 auto checkNumTeams = [&todo](auto op, LogicalResult &result) {
643 if (op.hasNumTeamsMultiDim())
644 result = todo("num_teams with multi-dimensional values");
645 };
646 auto checkNumThreads = [&todo](auto op, LogicalResult &result) {
647 if (op.hasNumThreadsMultiDim())
648 result = todo("num_threads with multi-dimensional values");
649 };
650
651 auto checkThreadLimit = [&todo](auto op, LogicalResult &result) {
652 if (op.hasThreadLimitMultiDim())
653 result = todo("thread_limit with multi-dimensional values");
654 };
655 auto checkMap = [&todo](auto op, LogicalResult &result) {
656 if (!op.getMapIterated().empty())
657 result = todo("map/motion clause with iterator modifier");
658 };
659
660 auto checkDynGroupprivate = [&todo](auto op, LogicalResult &result) {
661 if (op.getDynGroupprivateSize())
662 result = todo("dyn_groupprivate");
663 };
664
665 // `declare_mapper.info` supports a single-dimension iterator whose region
666 // (which may itself span several blocks) yields one member-free omp.map.info
667 // for a locator described by var_ptr + bounds. Descriptor-indirect
668 // (var_ptr_ptr) locators aren't handled yet. Anything else falls back to the
669 // generic (not yet implemented) diagnostic above.
670 auto checkDeclareMapperMap = [&todo](omp::DeclareMapperInfoOp op,
671 LogicalResult &result) {
672 for (Value iterVal : op.getMapIterated()) {
673 auto itersOp = iterVal.getDefiningOp<omp::IteratorOp>();
674 bool supported = itersOp && itersOp.getLoopLowerBounds().size() == 1;
675 if (supported) {
676 omp::YieldOp yieldOp = itersOp.getYieldOp();
677 supported = yieldOp && yieldOp.getResults().size() == 1;
678 if (supported) {
679 auto mapInfoOp =
680 yieldOp.getResults()[0].getDefiningOp<omp::MapInfoOp>();
681 supported =
682 mapInfoOp && !mapInfoOp.getVarPtrPtr() &&
683 mapInfoOp.getMembers().empty() &&
684 mapInfoOp.getMapCaptureType() == omp::VariableCaptureKind::ByRef;
685 }
686 }
687 if (!supported) {
688 result = todo("map/motion clause with iterator modifier");
689 return;
690 }
691 }
692 };
693
694 LogicalResult result = success();
695 llvm::TypeSwitch<Operation &>(op)
696 .Case(caseFn: [&](omp::DistributeOp op) {
697 checkAllocate(op, result);
698 checkOrder(op, result);
699 })
700 .Case(caseFn: [&](omp::SectionsOp op) {
701 checkAllocate(op, result);
702 checkPrivate(op, result);
703 checkReduction(op, result);
704 })
705 .Case(caseFn: [&](omp::ScopeOp op) { checkReduction(op, result); })
706 .Case(caseFn: [&](omp::SingleOp op) {
707 checkAllocate(op, result);
708 checkPrivate(op, result);
709 })
710 .Case(caseFn: [&](omp::TeamsOp op) {
711 checkAllocate(op, result);
712 checkPrivate(op, result);
713 checkNumTeams(op, result);
714 checkThreadLimit(op, result);
715 checkDynGroupprivate(op, result);
716 })
717 .Case(caseFn: [&](omp::TaskOp op) {
718 checkAllocate(op, result);
719 checkInReduction(op, result);
720 })
721 .Case(caseFn: [&](omp::TaskgroupOp op) {
722 checkAllocate(op, result);
723 checkTaskReductionByref(op, result);
724 })
725 .Case(caseFn: [&](omp::DispatchOp op) {
726 // OpenMP 5.1 dispatch creates an explicit task; nowait controls whether
727 // it is included. Diagnose unsupported asynchronous tasking before 5.2,
728 // where the nowait property has no effect on dispatch.
729 int64_t version = omp::getOpenMPVersionAttribute(
730 module: op->getParentOfType<ModuleOp>(), /*fallback=*/51);
731 if (version < 52)
732 checkNowait(op, result);
733 })
734 .Case(caseFn: [&](omp::TaskloopContextOp op) {
735 checkAllocate(op, result);
736 checkInReduction(op, result);
737 checkReduction(op, result);
738 checkReductionByref(op, result);
739 })
740 .Case(caseFn: [&](omp::WsloopOp op) {
741 checkAllocate(op, result);
742 checkOrder(op, result);
743 checkReduction(op, result);
744 })
745 .Case(caseFn: [&](omp::ParallelOp op) {
746 checkReduction(op, result);
747 checkNumThreads(op, result);
748 })
749 .Case(caseFn: [&](omp::SimdOp op) { checkReduction(op, result); })
750 .Case<omp::AtomicReadOp, omp::AtomicWriteOp, omp::AtomicUpdateOp,
751 omp::AtomicCaptureOp>(caseFn: [&](auto op) { checkHint(op, result); })
752 .Case(caseFn: [&](omp::AtomicCompareOp op) {
753 checkHint(op, result);
754 Region &region = op.getRegion();
755 if (region.empty())
756 return;
757 mlir::Type argType = region.front().getArgument(i: 0).getType();
758 auto structTy = dyn_cast<LLVM::LLVMStructType>(Val&: argType);
759 if (!structTy)
760 return;
761 DataLayout dl = DataLayout(op->getParentOfType<ModuleOp>());
762 unsigned totalBits = dl.getTypeSizeInBits(t: structTy);
763 if (totalBits > 128)
764 result = todo("compare for complex types wider than 128 bits");
765 })
766 .Case<omp::TargetEnterDataOp, omp::TargetExitDataOp>(caseFn: [&](auto op) {
767 checkDepend(op, result);
768 checkMap(op, result);
769 })
770 .Case(caseFn: [&](omp::TargetUpdateOp op) {
771 checkDepend(op, result);
772 checkMap(op, result);
773 })
774 .Case(caseFn: [&](omp::TargetOp op) {
775 checkAllocate(op, result);
776 checkBare(op, result);
777 checkInReduction(op, result);
778 checkMap(op, result);
779 checkThreadLimit(op, result);
780 })
781 .Case(caseFn: [&](omp::TargetDataOp op) { checkMap(op, result); })
782 .Case(caseFn: [&](omp::DeclareMapperInfoOp op) {
783 checkDeclareMapperMap(op, result);
784 })
785 .Default(defaultFn: [](Operation &) {
786 // Assume all clauses for an operation can be translated unless they are
787 // checked above.
788 });
789 return result;
790}
791
792static LogicalResult handleError(llvm::Error error, Operation &op) {
793 LogicalResult result = success();
794 if (error) {
795 llvm::handleAllErrors(
796 E: std::move(error),
797 Handlers: [&](const PreviouslyReportedError &) { result = failure(); },
798 Handlers: [&](const llvm::ErrorInfoBase &err) {
799 result = op.emitError(message: err.message());
800 });
801 }
802 return result;
803}
804
805template <typename T>
806static LogicalResult handleError(llvm::Expected<T> &result, Operation &op) {
807 if (!result)
808 return handleError(result.takeError(), op);
809
810 return success();
811}
812
813/// Find the insertion point for allocas given the current insertion point for
814/// normal operations in the builder.
815static llvm::OpenMPIRBuilder::InsertPointTy findAllocInsertPoints(
816 llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation,
817 llvm::SmallVectorImpl<llvm::BasicBlock *> *deallocBlocks = nullptr) {
818 // If there is an allocation insertion point on stack, i.e. we are in a nested
819 // operation and a specific point was provided by some surrounding operation,
820 // use it.
821 llvm::OpenMPIRBuilder::InsertPointTy allocInsertPoint;
822 llvm::ArrayRef<llvm::BasicBlock *> deallocInsertPoints;
823 WalkResult walkResult = moduleTranslation.stackWalk<OpenMPAllocStackFrame>(
824 callback: [&](OpenMPAllocStackFrame &frame) {
825 allocInsertPoint = frame.allocBlock->getFirstInsertionPt();
826 deallocInsertPoints = frame.deallocBlocks;
827 return WalkResult::interrupt();
828 });
829 // In cases with multiple levels of outlining, the tree walk might find an
830 // insertion point that is inside the original function while the builder
831 // insertion point is inside the outlined function. We need to make sure that
832 // we do not use it in those cases.
833 if (walkResult.wasInterrupted() &&
834 allocInsertPoint.getNodeParent()->getParent() ==
835 builder.GetInsertBlock()->getParent()) {
836 if (deallocBlocks)
837 deallocBlocks->insert(I: deallocBlocks->end(), From: deallocInsertPoints.begin(),
838 To: deallocInsertPoints.end());
839 return allocInsertPoint;
840 }
841
842 // Otherwise, insert to the entry block of the surrounding function.
843 // If the current IRBuilder InsertPoint is the function's entry, it cannot
844 // also be used for alloca insertion which would result in insertion order
845 // confusion. Create a new BasicBlock for the Builder and use the entry block
846 // for the allocs.
847 // TODO: Create a dedicated alloca BasicBlock at function creation such that
848 // we do not need to move the current InsertPoint here.
849 if (builder.GetInsertBlock() ==
850 &builder.GetInsertBlock()->getParent()->getEntryBlock()) {
851 assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&
852 "Assuming end of basic block");
853 llvm::BasicBlock *entryBB = llvm::BasicBlock::Create(
854 Context&: builder.getContext(), Name: "entry", Parent: builder.GetInsertBlock()->getParent(),
855 InsertBefore: builder.GetInsertBlock()->getNextNode());
856 builder.CreateBr(Dest: entryBB);
857 builder.SetInsertPoint(entryBB);
858 }
859
860 // Collect exit blocks, which is where explicit deallocations should happen in
861 // this case.
862 if (deallocBlocks) {
863 for (llvm::BasicBlock &block : *builder.GetInsertBlock()->getParent()) {
864 // TODO: This currently results in no blocks being added to the list when
865 // all exit blocks of the enclosing function have not been lowered before
866 // this is reached.
867 llvm::Instruction *terminator = block.getTerminatorOrNull();
868 if (isa_and_present<llvm::ReturnInst>(Val: terminator))
869 deallocBlocks->emplace_back(Args: &block);
870 }
871 }
872
873 llvm::BasicBlock &funcEntryBlock =
874 builder.GetInsertBlock()->getParent()->getEntryBlock();
875 return funcEntryBlock.getFirstInsertionPt();
876}
877
878/// Find the loop information structure for the loop nest being translated. It
879/// will return a `null` value unless called from the translation function for
880/// a loop wrapper operation after successfully translating its body.
881static llvm::CanonicalLoopInfo *
882findCurrentLoopInfo(LLVM::ModuleTranslation &moduleTranslation) {
883 llvm::CanonicalLoopInfo *loopInfo = nullptr;
884 moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
885 callback: [&](OpenMPLoopInfoStackFrame &frame) {
886 loopInfo = frame.loopInfo;
887 return WalkResult::interrupt();
888 });
889 return loopInfo;
890}
891
892/// Converts the given region that appears within an OpenMP dialect operation to
893/// LLVM IR, creating a branch from the `sourceBlock` to the entry block of the
894/// region, and a branch from any block with an successor-less OpenMP terminator
895/// to `continuationBlock`. Populates `continuationBlockPHIs` with the PHI nodes
896/// of the continuation block if provided.
897static llvm::Expected<llvm::BasicBlock *> convertOmpOpRegions(
898 Region &region, StringRef blockName, llvm::IRBuilderBase &builder,
899 LLVM::ModuleTranslation &moduleTranslation,
900 SmallVectorImpl<llvm::PHINode *> *continuationBlockPHIs = nullptr) {
901 bool isLoopWrapper = isa<omp::LoopWrapperInterface>(Val: region.getParentOp());
902
903 llvm::BasicBlock *continuationBlock =
904 splitBB(Builder&: builder, CreateBranch: true, Name: "omp.region.cont");
905 llvm::BasicBlock *sourceBlock = builder.GetInsertBlock();
906
907 llvm::LLVMContext &llvmContext = builder.getContext();
908 for (Block &bb : region) {
909 llvm::BasicBlock *llvmBB = llvm::BasicBlock::Create(
910 Context&: llvmContext, Name: blockName, Parent: builder.GetInsertBlock()->getParent(),
911 InsertBefore: builder.GetInsertBlock()->getNextNode());
912 moduleTranslation.mapBlock(mlir: &bb, llvm: llvmBB);
913 }
914
915 llvm::Instruction *sourceTerminator = sourceBlock->getTerminator();
916
917 // Terminators (namely YieldOp) may be forwarding values to the region that
918 // need to be available in the continuation block. Collect the types of these
919 // operands in preparation of creating PHI nodes. This is skipped for loop
920 // wrapper operations, for which we know in advance they have no terminators.
921 SmallVector<llvm::Type *> continuationBlockPHITypes;
922 unsigned numYields = 0;
923
924 if (!isLoopWrapper) {
925 bool operandsProcessed = false;
926 for (Block &bb : region.getBlocks()) {
927 if (omp::YieldOp yield = dyn_cast<omp::YieldOp>(Val: bb.getTerminator())) {
928 if (!operandsProcessed) {
929 for (unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
930 continuationBlockPHITypes.push_back(
931 Elt: moduleTranslation.convertType(type: yield->getOperand(idx: i).getType()));
932 }
933 operandsProcessed = true;
934 } else {
935 assert(continuationBlockPHITypes.size() == yield->getNumOperands() &&
936 "mismatching number of values yielded from the region");
937 for (unsigned i = 0, e = yield->getNumOperands(); i < e; ++i) {
938 llvm::Type *operandType =
939 moduleTranslation.convertType(type: yield->getOperand(idx: i).getType());
940 (void)operandType;
941 assert(continuationBlockPHITypes[i] == operandType &&
942 "values of mismatching types yielded from the region");
943 }
944 }
945 numYields++;
946 }
947 }
948 }
949
950 // Insert PHI nodes in the continuation block for any values forwarded by the
951 // terminators in this region.
952 if (!continuationBlockPHITypes.empty())
953 assert(
954 continuationBlockPHIs &&
955 "expected continuation block PHIs if converted regions yield values");
956 if (continuationBlockPHIs) {
957 llvm::IRBuilderBase::InsertPointGuard guard(builder);
958 continuationBlockPHIs->reserve(N: continuationBlockPHITypes.size());
959 builder.SetInsertPoint(continuationBlock->begin());
960 for (llvm::Type *ty : continuationBlockPHITypes)
961 continuationBlockPHIs->push_back(Elt: builder.CreatePHI(Ty: ty, NumReservedValues: numYields));
962 }
963
964 // Convert blocks one by one in topological order to ensure
965 // defs are converted before uses.
966 SetVector<Block *> blocks = getBlocksSortedByDominance(region);
967 for (Block *bb : blocks) {
968 llvm::BasicBlock *llvmBB = moduleTranslation.lookupBlock(block: bb);
969 // Retarget the branch of the entry block to the entry block of the
970 // converted region (regions are single-entry).
971 if (bb->isEntryBlock()) {
972 assert(sourceTerminator->getNumSuccessors() == 1 &&
973 "provided entry block has multiple successors");
974 assert(sourceTerminator->getSuccessor(0) == continuationBlock &&
975 "ContinuationBlock is not the successor of the entry block");
976 sourceTerminator->setSuccessor(Idx: 0, BB: llvmBB);
977 }
978
979 llvm::IRBuilderBase::InsertPointGuard guard(builder);
980 if (failed(
981 Result: moduleTranslation.convertBlock(bb&: *bb, ignoreArguments: bb->isEntryBlock(), builder)))
982 return llvm::make_error<PreviouslyReportedError>();
983
984 // Create a direct branch here for loop wrappers to prevent their lack of a
985 // terminator from causing a crash below.
986 if (isLoopWrapper) {
987 builder.CreateBr(Dest: continuationBlock);
988 continue;
989 }
990
991 // Special handling for `omp.yield` and `omp.terminator` (we may have more
992 // than one): they return the control to the parent OpenMP dialect operation
993 // so replace them with the branch to the continuation block. We handle this
994 // here to avoid relying inter-function communication through the
995 // ModuleTranslation class to set up the correct insertion point. This is
996 // also consistent with MLIR's idiom of handling special region terminators
997 // in the same code that handles the region-owning operation.
998 Operation *terminator = bb->getTerminator();
999 if (isa<omp::TerminatorOp, omp::YieldOp>(Val: terminator)) {
1000 builder.CreateBr(Dest: continuationBlock);
1001
1002 for (unsigned i = 0, e = terminator->getNumOperands(); i < e; ++i)
1003 (*continuationBlockPHIs)[i]->addIncoming(
1004 V: moduleTranslation.lookupValue(value: terminator->getOperand(idx: i)), BB: llvmBB);
1005 }
1006 }
1007 // After all blocks have been traversed and values mapped, connect the PHI
1008 // nodes to the results of preceding blocks.
1009 LLVM::detail::connectPHINodes(region, state: moduleTranslation);
1010
1011 // Remove the blocks and values defined in this region from the mapping since
1012 // they are not visible outside of this region. This allows the same region to
1013 // be converted several times, that is cloned, without clashes, and slightly
1014 // speeds up the lookups.
1015 moduleTranslation.forgetMapping(region);
1016
1017 return continuationBlock;
1018}
1019
1020/// Convert ProcBindKind from MLIR-generated enum to LLVM enum.
1021static llvm::omp::ProcBindKind getProcBindKind(omp::ClauseProcBindKind kind) {
1022 switch (kind) {
1023 case omp::ClauseProcBindKind::Close:
1024 return llvm::omp::ProcBindKind::OMP_PROC_BIND_close;
1025 case omp::ClauseProcBindKind::Master:
1026 return llvm::omp::ProcBindKind::OMP_PROC_BIND_master;
1027 case omp::ClauseProcBindKind::Primary:
1028 return llvm::omp::ProcBindKind::OMP_PROC_BIND_primary;
1029 case omp::ClauseProcBindKind::Spread:
1030 return llvm::omp::ProcBindKind::OMP_PROC_BIND_spread;
1031 }
1032 llvm_unreachable("Unknown ClauseProcBindKind kind");
1033}
1034
1035/// Convert 'dispatch' operation into LLVM IR.
1036static LogicalResult
1037convertOmpDispatch(Operation &opInst, llvm::IRBuilderBase &builder,
1038 LLVM::ModuleTranslation &moduleTranslation) {
1039 auto dispatchOp = cast<omp::DispatchOp>(Val&: opInst);
1040
1041 if (failed(Result: checkImplementationStatus(op&: opInst)))
1042 return failure();
1043
1044 auto &region = dispatchOp.getRegion();
1045 auto result = convertOmpOpRegions(region, blockName: "omp.dispatch.region", builder,
1046 moduleTranslation);
1047 if (!result)
1048 return handleError(error: result.takeError(), op&: opInst);
1049 builder.SetInsertPoint(*result);
1050 return success();
1051}
1052
1053/// Converts an OpenMP 'masked' operation into LLVM IR using OpenMPIRBuilder.
1054static LogicalResult
1055convertOmpMasked(Operation &opInst, llvm::IRBuilderBase &builder,
1056 LLVM::ModuleTranslation &moduleTranslation) {
1057 auto maskedOp = cast<omp::MaskedOp>(Val&: opInst);
1058 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1059
1060 if (failed(Result: checkImplementationStatus(op&: opInst)))
1061 return failure();
1062
1063 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
1064 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
1065 // MaskedOp has only one region associated with it.
1066 auto &region = maskedOp.getRegion();
1067 builder.restoreIP(IP: codeGenIP);
1068 return convertOmpOpRegions(region, blockName: "omp.masked.region", builder,
1069 moduleTranslation)
1070 .takeError();
1071 };
1072
1073 // TODO: Perform finalization actions for variables. This has to be
1074 // called for variables which have destructors/finalizers.
1075 auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
1076
1077 llvm::Value *filterVal = nullptr;
1078 if (auto filterVar = maskedOp.getFilteredThreadId()) {
1079 filterVal = moduleTranslation.lookupValue(value: filterVar);
1080 } else {
1081 llvm::LLVMContext &llvmContext = builder.getContext();
1082 filterVal =
1083 llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: llvmContext), /*V=*/0);
1084 }
1085 assert(filterVal != nullptr);
1086 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1087 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
1088 getOpenMPBuilder(moduleTranslation)
1089 ->createMasked(Loc: ompLoc, BodyGenCB: bodyGenCB, FiniCB: finiCB, Filter: filterVal);
1090
1091 if (failed(Result: handleError(result&: afterIP, op&: opInst)))
1092 return failure();
1093
1094 builder.restoreIP(IP: *afterIP);
1095 return success();
1096}
1097
1098/// Converts an OpenMP 'master' operation into LLVM IR using OpenMPIRBuilder.
1099static LogicalResult
1100convertOmpMaster(Operation &opInst, llvm::IRBuilderBase &builder,
1101 LLVM::ModuleTranslation &moduleTranslation) {
1102 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1103 auto masterOp = cast<omp::MasterOp>(Val&: opInst);
1104
1105 if (failed(Result: checkImplementationStatus(op&: opInst)))
1106 return failure();
1107
1108 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
1109 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
1110 // MasterOp has only one region associated with it.
1111 auto &region = masterOp.getRegion();
1112 builder.restoreIP(IP: codeGenIP);
1113 return convertOmpOpRegions(region, blockName: "omp.master.region", builder,
1114 moduleTranslation)
1115 .takeError();
1116 };
1117
1118 // TODO: Perform finalization actions for variables. This has to be
1119 // called for variables which have destructors/finalizers.
1120 auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
1121
1122 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1123 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
1124 getOpenMPBuilder(moduleTranslation)
1125 ->createMaster(Loc: ompLoc, BodyGenCB: bodyGenCB, FiniCB: finiCB);
1126
1127 if (failed(Result: handleError(result&: afterIP, op&: opInst)))
1128 return failure();
1129
1130 builder.restoreIP(IP: *afterIP);
1131 return success();
1132}
1133
1134/// Converts an OpenMP 'critical' operation into LLVM IR using OpenMPIRBuilder.
1135static LogicalResult
1136convertOmpCritical(Operation &opInst, llvm::IRBuilderBase &builder,
1137 LLVM::ModuleTranslation &moduleTranslation) {
1138 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1139 auto criticalOp = cast<omp::CriticalOp>(Val&: opInst);
1140
1141 if (failed(Result: checkImplementationStatus(op&: opInst)))
1142 return failure();
1143
1144 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
1145 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
1146 // CriticalOp has only one region associated with it.
1147 auto &region = cast<omp::CriticalOp>(Val&: opInst).getRegion();
1148 builder.restoreIP(IP: codeGenIP);
1149 return convertOmpOpRegions(region, blockName: "omp.critical.region", builder,
1150 moduleTranslation)
1151 .takeError();
1152 };
1153
1154 // TODO: Perform finalization actions for variables. This has to be
1155 // called for variables which have destructors/finalizers.
1156 auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
1157
1158 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1159 llvm::LLVMContext &llvmContext = moduleTranslation.getLLVMContext();
1160 llvm::Constant *hint = nullptr;
1161
1162 // If it has a name, it probably has a hint too.
1163 if (criticalOp.getNameAttr()) {
1164 // The verifiers in OpenMP Dialect guarentee that all the pointers are
1165 // non-null
1166 auto symbolRef = cast<SymbolRefAttr>(Val: criticalOp.getNameAttr());
1167 auto criticalDeclareOp =
1168 SymbolTable::lookupNearestSymbolFrom<omp::CriticalDeclareOp>(from: criticalOp,
1169 symbol: symbolRef);
1170 hint =
1171 llvm::ConstantInt::get(Ty: llvm::Type::getInt32Ty(C&: llvmContext),
1172 V: static_cast<int>(criticalDeclareOp.getHint()));
1173 }
1174 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
1175 getOpenMPBuilder(moduleTranslation)
1176 ->createCritical(Loc: ompLoc, BodyGenCB: bodyGenCB, FiniCB: finiCB,
1177 CriticalName: criticalOp.getName().value_or(u: ""), HintInst: hint);
1178
1179 if (failed(Result: handleError(result&: afterIP, op&: opInst)))
1180 return failure();
1181
1182 builder.restoreIP(IP: *afterIP);
1183 return success();
1184}
1185
1186namespace {
1187/// A util to collect info needed to convert delayed privatizers from MLIR to
1188/// LLVM.
1189struct PrivateVarsInfo {
1190 /// Per-variable data for a single delayed privatizer.
1191 struct VarInfo {
1192 BlockArgument blockArg;
1193 mlir::Value mlirVar;
1194 omp::PrivateClauseOp privatizer;
1195 llvm::Value *llvmVar = nullptr;
1196 };
1197
1198 struct AllocatorInfo {
1199 llvm::Value *allocatedPtr;
1200 llvm::Value *allocator;
1201 };
1202
1203 template <typename OP>
1204 PrivateVarsInfo(OP op) {
1205 MutableArrayRef<BlockArgument> blockArgs =
1206 cast<omp::BlockArgOpenMPOpInterface>(*op).getPrivateBlockArgs();
1207 SmallVector<omp::PrivateClauseOp> privatizers;
1208
1209 collectPrivatizationDecls<OP>(op, op.getPrivateSyms(), privatizers);
1210 collectPrivatizationInfo(blockArgs, privateVars: op.getPrivateVars(), privatizers);
1211 numPrivateVars = vars.size();
1212 }
1213
1214 /// Adds the linear variables of `op`, which are privatized by their
1215 /// `linear` privatizers, after the private variables.
1216 template <typename OP>
1217 void addLinearVars(OP op) {
1218 MutableArrayRef<BlockArgument> blockArgs =
1219 cast<omp::BlockArgOpenMPOpInterface>(*op).getLinearBlockArgs();
1220 SmallVector<omp::PrivateClauseOp> privatizers;
1221
1222 collectPrivatizationDecls<OP>(op, op.getLinearSyms(), privatizers);
1223 collectPrivatizationInfo(blockArgs, privateVars: op.getLinearVars(), privatizers);
1224 }
1225
1226 /// Number of private variables before the linear ones.
1227 size_t numPrivateVars = 0;
1228 SmallVector<VarInfo> vars;
1229 llvm::DenseMap<Value, llvm::Value *> convertedAllocators;
1230 SmallVector<AllocatorInfo> allocatorPrivates;
1231
1232 SmallVector<llvm::Value *> getLlvmVars() const {
1233 return llvm::map_to_vector(C: vars,
1234 F: [](const VarInfo &v) { return v.llvmVar; });
1235 }
1236
1237private:
1238 /// Adds the provided privatization information to `vars`.
1239 void collectPrivatizationInfo(
1240 MutableArrayRef<BlockArgument> blockArgs, OperandRange privateVars,
1241 const SmallVectorImpl<omp::PrivateClauseOp> &privatizers) {
1242 vars.reserve(N: vars.size() + blockArgs.size());
1243 for (auto [blockArg, mlirVar, privatizer] :
1244 llvm::zip_equal(t&: blockArgs, u&: privateVars, args: privatizers))
1245 vars.push_back(Elt: {.blockArg: blockArg, .mlirVar: mlirVar, .privatizer: privatizer});
1246 }
1247
1248 /// Populates `privatizers` with privatization declarations the provided
1249 /// `symsAttr` points to.
1250 template <class OP>
1251 void collectPrivatizationDecls(
1252 OP op, std::optional<ArrayAttr> symsAttr,
1253 SmallVectorImpl<omp::PrivateClauseOp> &privatizers) {
1254 if (!symsAttr)
1255 return;
1256
1257 privatizers.reserve(N: privatizers.size() + symsAttr->size());
1258 llvm::transform(
1259 symsAttr->getAsRange<SymbolRefAttr>(), std::back_inserter(x&: privatizers),
1260 [&](SymbolRefAttr symbolRef) { return findPrivatizer(op, symbolRef); });
1261 }
1262};
1263} // namespace
1264
1265/// Populates `reductions` with reduction declarations used in the given op.
1266template <typename T>
1267static void
1268collectReductionDecls(T op,
1269 SmallVectorImpl<omp::DeclareReductionOp> &reductions) {
1270 std::optional<ArrayAttr> attr = op.getReductionSyms();
1271 if (!attr)
1272 return;
1273
1274 reductions.reserve(N: reductions.size() + op.getNumReductionVars());
1275 for (auto symbolRef : attr->getAsRange<SymbolRefAttr>()) {
1276 reductions.push_back(
1277 Elt: SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>(
1278 op, symbolRef));
1279 }
1280}
1281
1282/// Look up and validate the declare_reduction ops referenced by a
1283/// reduction-like clause on the omp.taskloop.context translation path. Only
1284/// the non-byref, single-init-arg, no-cleanup form is supported in this
1285/// initial cut; richer shapes are rejected here with a diagnostic. \p syms
1286/// is the clause's symbol list (e.g. `getReductionSyms()` or
1287/// `getInReductionSyms()`), \p opName is the textual op name used in
1288/// diagnostics, and \p clauseName distinguishes "reduction" from
1289/// "in_reduction" in those diagnostics.
1290static LogicalResult collectAndValidateTaskloopRedDecls(
1291 Operation *contextOp, std::optional<ArrayAttr> syms, StringRef opName,
1292 StringRef clauseName, SmallVectorImpl<omp::DeclareReductionOp> &out) {
1293 if (!syms)
1294 return success();
1295 out.reserve(N: out.size() + syms->size());
1296 for (auto sym : syms->getAsRange<SymbolRefAttr>()) {
1297 auto decl = SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>(
1298 from: contextOp, symbol: sym);
1299 if (!decl)
1300 return contextOp->emitError()
1301 << "failed to resolve " << clauseName
1302 << " declare_reduction symbol " << sym.getRootReference() << " in "
1303 << opName;
1304 if (decl.getInitializerRegion().front().getNumArguments() != 1)
1305 return contextOp->emitError()
1306 << "not yet implemented: " << clauseName
1307 << " with two-argument initializer in " << opName;
1308 if (!decl.getCleanupRegion().empty())
1309 return contextOp->emitError() << "not yet implemented: " << clauseName
1310 << " with cleanup region in " << opName;
1311 if (decl.getReductionRegion().empty())
1312 return contextOp->emitError()
1313 << clauseName << " declare_reduction is missing a combiner region";
1314 out.push_back(Elt: decl);
1315 }
1316 return success();
1317}
1318
1319/// Translates the blocks contained in the given region and appends them to at
1320/// the current insertion point of `builder`. The operations of the entry block
1321/// are appended to the current insertion block. If set, `continuationBlockArgs`
1322/// is populated with translated values that correspond to the values
1323/// omp.yield'ed from the region.
1324static LogicalResult inlineConvertOmpRegions(
1325 Region &region, StringRef blockName, llvm::IRBuilderBase &builder,
1326 LLVM::ModuleTranslation &moduleTranslation,
1327 SmallVectorImpl<llvm::Value *> *continuationBlockArgs = nullptr) {
1328 if (region.empty())
1329 return success();
1330
1331 // Special case for single-block regions that don't create additional blocks:
1332 // insert operations without creating additional blocks.
1333 if (region.hasOneBlock()) {
1334 llvm::BasicBlock *insertBlock = builder.GetInsertBlock();
1335 llvm::Instruction *potentialTerminator =
1336 insertBlock->empty() ? nullptr : &insertBlock->back();
1337
1338 if (potentialTerminator && potentialTerminator->isTerminator()) {
1339 assert(builder.GetInsertPoint() == potentialTerminator->getIterator() &&
1340 "Insertion point should be the terminator");
1341 potentialTerminator->removeFromParent();
1342 // Reset insertion point to the end of the block.
1343 builder.SetInsertPoint(insertBlock);
1344 }
1345 moduleTranslation.mapBlock(mlir: &region.front(), llvm: insertBlock);
1346
1347 // convertBlock leaves the location of the region's last op current. Restore
1348 // the caller's, as convertOmpOpRegions does for every block it converts.
1349 llvm::DebugLoc loc = builder.getCurrentDebugLocation();
1350 if (failed(Result: moduleTranslation.convertBlock(
1351 bb&: region.front(), /*ignoreArguments=*/true, builder)))
1352 return failure();
1353 builder.SetCurrentDebugLocation(loc);
1354
1355 // The continuation arguments are simply the translated terminator operands.
1356 if (continuationBlockArgs)
1357 llvm::append_range(
1358 C&: *continuationBlockArgs,
1359 R: moduleTranslation.lookupValues(values: region.front().back().getOperands()));
1360
1361 // Drop the mapping that is no longer necessary so that the same region can
1362 // be processed multiple times.
1363 moduleTranslation.forgetMapping(region);
1364
1365 if (potentialTerminator && potentialTerminator->isTerminator()) {
1366 llvm::BasicBlock *block = builder.GetInsertBlock();
1367 if (block->empty()) {
1368 // this can happen for really simple reduction init regions e.g.
1369 // %0 = llvm.mlir.constant(0 : i32) : i32
1370 // omp.yield(%0 : i32)
1371 // because the llvm.mlir.constant (MLIR op) isn't converted into any
1372 // llvm op
1373 potentialTerminator->insertInto(ParentBB: block, It: block->begin());
1374 } else {
1375 potentialTerminator->insertAfter(InsertPos: &block->back());
1376 }
1377 }
1378
1379 return success();
1380 }
1381
1382 SmallVector<llvm::PHINode *> phis;
1383 llvm::Expected<llvm::BasicBlock *> continuationBlock =
1384 convertOmpOpRegions(region, blockName, builder, moduleTranslation, continuationBlockPHIs: &phis);
1385
1386 if (failed(Result: handleError(result&: continuationBlock, op&: *region.getParentOp())))
1387 return failure();
1388
1389 if (continuationBlockArgs)
1390 llvm::append_range(C&: *continuationBlockArgs, R&: phis);
1391 builder.SetInsertPoint((*continuationBlock)->getFirstInsertionPt());
1392 return success();
1393}
1394
1395namespace {
1396/// Owning equivalents of OpenMPIRBuilder::(Atomic)ReductionGen that are used to
1397/// store lambdas with capture.
1398using OwningReductionGen =
1399 std::function<llvm::OpenMPIRBuilder::InsertPointOrErrorTy(
1400 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *,
1401 llvm::Value *&)>;
1402using OwningAtomicReductionGen =
1403 std::function<llvm::OpenMPIRBuilder::InsertPointOrErrorTy(
1404 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Type *, llvm::Value *,
1405 llvm::Value *)>;
1406using OwningDataPtrPtrReductionGen =
1407 std::function<llvm::OpenMPIRBuilder::InsertPointOrErrorTy(
1408 llvm::OpenMPIRBuilder::InsertPointTy, llvm::Value *, llvm::Value *&)>;
1409} // namespace
1410
1411/// Create an OpenMPIRBuilder-compatible reduction generator for the given
1412/// reduction declaration. The generator uses `builder` but ignores its
1413/// insertion point.
1414static OwningReductionGen
1415makeReductionGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder,
1416 LLVM::ModuleTranslation &moduleTranslation) {
1417 // The lambda is mutable because we need access to non-const methods of decl
1418 // (which aren't actually mutating it), and we must capture decl by-value to
1419 // avoid the dangling reference after the parent function returns.
1420 OwningReductionGen gen =
1421 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint,
1422 llvm::Value *lhs, llvm::Value *rhs,
1423 llvm::Value *&result) mutable
1424 -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
1425 moduleTranslation.mapValue(mlir: decl.getReductionLhsArg(), llvm: lhs);
1426 moduleTranslation.mapValue(mlir: decl.getReductionRhsArg(), llvm: rhs);
1427 builder.restoreIP(IP: insertPoint);
1428 SmallVector<llvm::Value *> phis;
1429 if (failed(Result: inlineConvertOmpRegions(region&: decl.getReductionRegion(),
1430 blockName: "omp.reduction.nonatomic.body", builder,
1431 moduleTranslation, continuationBlockArgs: &phis)))
1432 return llvm::createStringError(
1433 Fmt: "failed to inline `combiner` region of `omp.declare_reduction`");
1434 result = llvm::getSingleElement(C&: phis);
1435 return builder.saveIP();
1436 };
1437 return gen;
1438}
1439
1440/// Create an OpenMPIRBuilder-compatible atomic reduction generator for the
1441/// given reduction declaration. The generator uses `builder` but ignores its
1442/// insertion point. Returns null if there is no atomic region available in the
1443/// reduction declaration.
1444static OwningAtomicReductionGen
1445makeAtomicReductionGen(omp::DeclareReductionOp decl,
1446 llvm::IRBuilderBase &builder,
1447 LLVM::ModuleTranslation &moduleTranslation) {
1448 if (decl.getAtomicReductionRegion().empty())
1449 return OwningAtomicReductionGen();
1450
1451 // The lambda is mutable because we need access to non-const methods of decl
1452 // (which aren't actually mutating it), and we must capture decl by-value to
1453 // avoid the dangling reference after the parent function returns.
1454 OwningAtomicReductionGen atomicGen =
1455 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint, llvm::Type *,
1456 llvm::Value *lhs, llvm::Value *rhs) mutable
1457 -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
1458 moduleTranslation.mapValue(mlir: decl.getAtomicReductionLhsArg(), llvm: lhs);
1459 moduleTranslation.mapValue(mlir: decl.getAtomicReductionRhsArg(), llvm: rhs);
1460 builder.restoreIP(IP: insertPoint);
1461 SmallVector<llvm::Value *> phis;
1462 if (failed(Result: inlineConvertOmpRegions(region&: decl.getAtomicReductionRegion(),
1463 blockName: "omp.reduction.atomic.body", builder,
1464 moduleTranslation, continuationBlockArgs: &phis)))
1465 return llvm::createStringError(
1466 Fmt: "failed to inline `atomic` region of `omp.declare_reduction`");
1467 assert(phis.empty());
1468 return builder.saveIP();
1469 };
1470 return atomicGen;
1471}
1472
1473/// Create an OpenMPIRBuilder-compatible `data_ptr_ptr` reduction generator for
1474/// the given reduction declaration. The generator uses `builder` but ignores
1475/// its insertion point. Returns null if there is no `data_ptr_ptr` region
1476/// available in the reduction declaration.
1477static OwningDataPtrPtrReductionGen
1478makeRefDataPtrGen(omp::DeclareReductionOp decl, llvm::IRBuilderBase &builder,
1479 LLVM::ModuleTranslation &moduleTranslation, bool isByRef) {
1480 if (!isByRef || decl.getDataPtrPtrRegion().empty())
1481 return OwningDataPtrPtrReductionGen();
1482
1483 OwningDataPtrPtrReductionGen refDataPtrGen =
1484 [&, decl](llvm::OpenMPIRBuilder::InsertPointTy insertPoint,
1485 llvm::Value *byRefVal, llvm::Value *&result) mutable
1486 -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
1487 moduleTranslation.mapValue(mlir: decl.getDataPtrPtrRegionArg(), llvm: byRefVal);
1488 builder.restoreIP(IP: insertPoint);
1489 SmallVector<llvm::Value *> phis;
1490 if (failed(Result: inlineConvertOmpRegions(region&: decl.getDataPtrPtrRegion(),
1491 blockName: "omp.data_ptr_ptr.body", builder,
1492 moduleTranslation, continuationBlockArgs: &phis)))
1493 return llvm::createStringError(
1494 Fmt: "failed to inline `data_ptr_ptr` region of `omp.declare_reduction`");
1495 result = llvm::getSingleElement(C&: phis);
1496 return builder.saveIP();
1497 };
1498
1499 return refDataPtrGen;
1500}
1501
1502/// Converts an OpenMP 'ordered' operation into LLVM IR using OpenMPIRBuilder.
1503static LogicalResult
1504convertOmpOrdered(Operation &opInst, llvm::IRBuilderBase &builder,
1505 LLVM::ModuleTranslation &moduleTranslation) {
1506 auto orderedOp = cast<omp::OrderedOp>(Val&: opInst);
1507
1508 if (failed(Result: checkImplementationStatus(op&: opInst)))
1509 return failure();
1510
1511 omp::ClauseDepend dependType = *orderedOp.getDoacrossDependType();
1512 bool isDependSource = dependType == omp::ClauseDepend::dependsource;
1513 unsigned numLoops = *orderedOp.getDoacrossNumLoops();
1514 SmallVector<llvm::Value *> vecValues =
1515 moduleTranslation.lookupValues(values: orderedOp.getDoacrossDependVars());
1516
1517 size_t indexVecValues = 0;
1518 while (indexVecValues < vecValues.size()) {
1519 SmallVector<llvm::Value *> storeValues;
1520 storeValues.reserve(N: numLoops);
1521 for (unsigned i = 0; i < numLoops; i++) {
1522 storeValues.push_back(Elt: vecValues[indexVecValues]);
1523 indexVecValues++;
1524 }
1525 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
1526 findAllocInsertPoints(builder, moduleTranslation);
1527 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1528 builder.restoreIP(IP: getOpenMPBuilder(moduleTranslation)
1529 ->createOrderedDepend(Loc: ompLoc, AllocaIP: allocaIP, NumLoops: numLoops,
1530 StoreValues: storeValues, Name: ".cnt.addr",
1531 IsDependSource: isDependSource));
1532 }
1533 return success();
1534}
1535
1536/// Converts an OpenMP 'ordered_region' operation into LLVM IR using
1537/// OpenMPIRBuilder.
1538static LogicalResult
1539convertOmpOrderedRegion(Operation &opInst, llvm::IRBuilderBase &builder,
1540 LLVM::ModuleTranslation &moduleTranslation) {
1541 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
1542 auto orderedRegionOp = cast<omp::OrderedRegionOp>(Val&: opInst);
1543
1544 if (failed(Result: checkImplementationStatus(op&: opInst)))
1545 return failure();
1546
1547 auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
1548 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
1549 // OrderedOp has only one region associated with it.
1550 auto &region = cast<omp::OrderedRegionOp>(Val&: opInst).getRegion();
1551 builder.restoreIP(IP: codeGenIP);
1552 return convertOmpOpRegions(region, blockName: "omp.ordered.region", builder,
1553 moduleTranslation)
1554 .takeError();
1555 };
1556
1557 // TODO: Perform finalization actions for variables. This has to be
1558 // called for variables which have destructors/finalizers.
1559 auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
1560
1561 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
1562 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
1563 getOpenMPBuilder(moduleTranslation)
1564 ->createOrderedThreadsSimd(Loc: ompLoc, BodyGenCB: bodyGenCB, FiniCB: finiCB,
1565 IsThreads: !orderedRegionOp.getParLevelSimd());
1566
1567 if (failed(Result: handleError(result&: afterIP, op&: opInst)))
1568 return failure();
1569
1570 builder.restoreIP(IP: *afterIP);
1571 return success();
1572}
1573
1574namespace {
1575/// A util to collect info needed to translate the reductions of an OpenMP
1576/// construct from MLIR to LLVM. Mirrors `PrivateVarsInfo` for delayed
1577/// privatizers.
1578struct ReductionVarsInfo {
1579 /// Per-variable data for a single reduction.
1580 struct VarInfo {
1581 BlockArgument blockArg;
1582 mlir::Value mlirVar;
1583 omp::DeclareReductionOp decl;
1584 bool byRef;
1585 llvm::Value *privateVar = nullptr;
1586 llvm::Value *storage = nullptr;
1587 };
1588
1589 /// Contains the arguments for an LLVM store operation.
1590 struct DeferredStore {
1591 DeferredStore(llvm::Value *value, llvm::Value *address)
1592 : value(value), address(address) {}
1593
1594 llvm::Value *value;
1595 llvm::Value *address;
1596 };
1597
1598 template <typename OP>
1599 ReductionVarsInfo(OP op) {
1600 MutableArrayRef<BlockArgument> blockArgs =
1601 cast<omp::BlockArgOpenMPOpInterface>(*op).getReductionBlockArgs();
1602 SmallVector<omp::DeclareReductionOp> decls;
1603 collectReductionDecls(op, decls);
1604 std::optional<ArrayRef<bool>> attr = op.getReductionByref();
1605 ArrayRef<bool> byRefs = attr ? *attr : ArrayRef<bool>();
1606
1607 vars.reserve(N: blockArgs.size());
1608 for (auto [blockArg, mlirVar, decl, byRef] :
1609 llvm::zip_equal(blockArgs, op.getReductionVars(), decls, byRefs))
1610 vars.push_back(Elt: {blockArg, mlirVar, decl, byRef});
1611 }
1612
1613 SmallVector<VarInfo> vars;
1614 /// Stores deferred until after all allocas (see `allocReductionVars`).
1615 SmallVector<DeferredStore> deferredStores;
1616
1617 size_t size() const { return vars.size(); }
1618
1619 SmallVector<omp::DeclareReductionOp> getReductionDecls() const {
1620 return llvm::map_to_vector(C: vars, F: [](const VarInfo &v) { return v.decl; });
1621 }
1622 SmallVector<llvm::Value *> getPrivateReductionVariables() const {
1623 return llvm::map_to_vector(C: vars,
1624 F: [](const VarInfo &v) { return v.privateVar; });
1625 }
1626 SmallVector<bool> getIsByRef() const {
1627 return llvm::map_to_vector(C: vars, F: [](const VarInfo &v) { return v.byRef; });
1628 }
1629};
1630} // namespace
1631
1632/// Allocate space for privatized reduction variables.
1633/// `deferredStores` contains information to create store operations which needs
1634/// to be inserted after all allocas
1635static LogicalResult
1636allocReductionVars(Operation &op, llvm::IRBuilderBase &builder,
1637 LLVM::ModuleTranslation &moduleTranslation,
1638 const llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
1639 ReductionVarsInfo &redInfo) {
1640 llvm::IRBuilderBase::InsertPointGuard guard(builder);
1641 builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator());
1642
1643 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
1644 bool useDeviceSharedMem = omp::opInSharedDeviceContext(op);
1645
1646 // delay creating stores until after all allocas
1647 redInfo.deferredStores.reserve(N: redInfo.size());
1648
1649 for (ReductionVarsInfo::VarInfo &var : redInfo.vars) {
1650 Region &allocRegion = var.decl.getAllocRegion();
1651 if (var.byRef) {
1652 if (allocRegion.empty())
1653 continue;
1654
1655 SmallVector<llvm::Value *, 1> phis;
1656 if (failed(Result: inlineConvertOmpRegions(region&: allocRegion, blockName: "omp.reduction.alloc",
1657 builder, moduleTranslation, continuationBlockArgs: &phis)))
1658 return op.emitError(
1659 message: "failed to inline `alloc` region of `omp.declare_reduction`");
1660
1661 assert(phis.size() == 1 && "expected one allocation to be yielded");
1662 builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator());
1663
1664 // Allocate reduction variable (which is a pointer to the real reduction
1665 // variable allocated in the inlined region)
1666 llvm::Type *ptrTy = builder.getPtrTy();
1667 llvm::Type *varTy = moduleTranslation.convertType(type: var.decl.getType());
1668 llvm::Value *llvmVar;
1669 if (useDeviceSharedMem) {
1670 llvmVar = ompBuilder->createOMPAllocShared(Loc: builder, VarType: varTy);
1671 } else {
1672 llvmVar = builder.CreateAlloca(Ty: varTy);
1673 llvmVar = builder.CreatePointerBitCastOrAddrSpaceCast(V: llvmVar, DestTy: ptrTy);
1674 }
1675
1676 llvm::Value *castPhi =
1677 builder.CreatePointerBitCastOrAddrSpaceCast(V: phis[0], DestTy: ptrTy);
1678
1679 redInfo.deferredStores.emplace_back(Args&: castPhi, Args&: llvmVar);
1680
1681 var.privateVar = llvmVar;
1682 moduleTranslation.mapValue(mlir: var.blockArg, llvm: castPhi);
1683 var.storage = castPhi;
1684 } else {
1685 assert(allocRegion.empty() &&
1686 "allocaction is implicit for by-val reduction");
1687
1688 llvm::Type *ptrTy = builder.getPtrTy();
1689 llvm::Type *varTy = moduleTranslation.convertType(type: var.decl.getType());
1690 llvm::Value *llvmVar;
1691 if (useDeviceSharedMem) {
1692 llvmVar = ompBuilder->createOMPAllocShared(Loc: builder, VarType: varTy);
1693 } else {
1694 llvmVar = builder.CreateAlloca(Ty: varTy);
1695 llvmVar = builder.CreatePointerBitCastOrAddrSpaceCast(V: llvmVar, DestTy: ptrTy);
1696 }
1697
1698 var.privateVar = llvmVar;
1699 moduleTranslation.mapValue(mlir: var.blockArg, llvm: llvmVar);
1700 var.storage = llvmVar;
1701 }
1702 }
1703
1704 return success();
1705}
1706
1707/// Map input arguments to reduction initialization region
1708static void
1709mapInitializationArgs(LLVM::ModuleTranslation &moduleTranslation,
1710 llvm::IRBuilderBase &builder,
1711 const ReductionVarsInfo::VarInfo &redVarInfo) {
1712 // map input argument to the initialization region
1713 omp::DeclareReductionOp reduction = redVarInfo.decl;
1714 Region &initializerRegion = reduction.getInitializerRegion();
1715 Block &entry = initializerRegion.front();
1716
1717 mlir::Value mlirSource = redVarInfo.mlirVar;
1718 llvm::Value *llvmSource = moduleTranslation.lookupValue(value: mlirSource);
1719 llvm::Value *origVal = llvmSource;
1720 // If a non-pointer value is expected, load the value from the source pointer.
1721 if (!isa<LLVM::LLVMPointerType>(
1722 Val: reduction.getInitializerMoldArg().getType()) &&
1723 isa<LLVM::LLVMPointerType>(Val: mlirSource.getType())) {
1724 origVal =
1725 builder.CreateLoad(Ty: moduleTranslation.convertType(
1726 type: reduction.getInitializerMoldArg().getType()),
1727 Ptr: llvmSource, Name: "omp_orig");
1728 }
1729 moduleTranslation.mapValue(mlir: reduction.getInitializerMoldArg(), llvm: origVal);
1730
1731 if (entry.getNumArguments() > 1)
1732 moduleTranslation.mapValue(mlir: reduction.getInitializerAllocArg(),
1733 llvm: redVarInfo.storage);
1734}
1735
1736static void
1737setInsertPointForPossiblyEmptyBlock(llvm::IRBuilderBase &builder,
1738 llvm::BasicBlock *block = nullptr) {
1739 if (block == nullptr)
1740 block = builder.GetInsertBlock();
1741
1742 if (!block->hasTerminator())
1743 builder.SetInsertPoint(block);
1744 else
1745 builder.SetInsertPoint(block->getTerminator());
1746}
1747
1748/// Inline reductions' `init` regions. This functions assumes that the
1749/// `builder`'s insertion point is where the user wants the `init` regions to be
1750/// inlined; i.e. it does not try to find a proper insertion location for the
1751/// `init` regions. It also leaves the `builder's insertions point in a state
1752/// where the user can continue the code-gen directly afterwards.
1753static LogicalResult
1754initReductionVars(Operation &op, llvm::IRBuilderBase &builder,
1755 LLVM::ModuleTranslation &moduleTranslation,
1756 llvm::BasicBlock *latestAllocaBlock,
1757 ReductionVarsInfo &redInfo) {
1758 if (redInfo.size() == 0)
1759 return success();
1760
1761 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
1762 bool useDeviceSharedMem = omp::opInSharedDeviceContext(op);
1763
1764 llvm::BasicBlock *initBlock = splitBB(Builder&: builder, CreateBranch: true, Name: "omp.reduction.init");
1765 SmallVector<llvm::Value *> byRefVars(redInfo.size());
1766 {
1767 llvm::IRBuilderBase::InsertPointGuard guard(builder);
1768 auto allocaIP = latestAllocaBlock->getTerminator()->getIterator();
1769 builder.restoreIP(IP: allocaIP);
1770
1771 for (auto [var, byRefVar] : llvm::zip_equal(t&: redInfo.vars, u&: byRefVars)) {
1772 if (var.byRef) {
1773 if (!var.decl.getAllocRegion().empty())
1774 continue;
1775
1776 // TODO: remove after all users of by-ref are updated to use the alloc
1777 // region: Allocate reduction variable (which is a pointer to the real
1778 // reduciton variable allocated in the inlined region)
1779 llvm::Type *varTy = moduleTranslation.convertType(type: var.decl.getType());
1780 if (useDeviceSharedMem)
1781 byRefVar = ompBuilder->createOMPAllocShared(Loc: builder, VarType: varTy);
1782 else
1783 byRefVar = builder.CreateAlloca(Ty: varTy);
1784 }
1785 }
1786 }
1787
1788 setInsertPointForPossiblyEmptyBlock(builder, block: initBlock);
1789
1790 // store result of the alloc region to the allocated pointer to the real
1791 // reduction variable
1792 for (auto [data, addr] : redInfo.deferredStores)
1793 builder.CreateStore(Val: data, Ptr: addr);
1794
1795 // Before the loop, store the initial values of reductions into reduction
1796 // variables. Although this could be done after allocas, we don't want to mess
1797 // up with the alloca insertion point.
1798 for (auto [var, byRefVar] : llvm::zip_equal(t&: redInfo.vars, u&: byRefVars)) {
1799 SmallVector<llvm::Value *, 1> phis;
1800
1801 // map block argument to initializer region
1802 mapInitializationArgs(moduleTranslation, builder, redVarInfo: var);
1803
1804 // TODO In some cases (specially on the GPU), the init regions may
1805 // contains stack alloctaions. If the region is inlined in a loop, this is
1806 // problematic. Instead of just inlining the region, handle allocations by
1807 // hoisting fixed length allocations to the function entry and using
1808 // stacksave and restore for variable length ones.
1809 if (failed(Result: inlineConvertOmpRegions(region&: var.decl.getInitializerRegion(),
1810 blockName: "omp.reduction.neutral", builder,
1811 moduleTranslation, continuationBlockArgs: &phis)))
1812 return failure();
1813
1814 assert(phis.size() == 1 && "expected one value to be yielded from the "
1815 "reduction neutral element declaration region");
1816
1817 setInsertPointForPossiblyEmptyBlock(builder);
1818
1819 if (var.byRef) {
1820 if (!var.decl.getAllocRegion().empty())
1821 // done in allocReductionVars
1822 continue;
1823
1824 // TODO: this path can be removed once all users of by-ref are updated to
1825 // use an alloc region
1826
1827 // Store the result of the inlined region to the allocated reduction var
1828 // ptr
1829 builder.CreateStore(Val: phis[0], Ptr: byRefVar);
1830 var.privateVar = byRefVar;
1831
1832 moduleTranslation.mapValue(mlir: var.blockArg, llvm: phis[0]);
1833 var.storage = phis[0];
1834 } else {
1835 // for by-ref case the store is inside of the reduction region
1836 builder.CreateStore(Val: phis[0], Ptr: var.privateVar);
1837 // the rest was handled in allocByValReductionVars
1838 }
1839
1840 // forget the mapping for the initializer region because we might need a
1841 // different mapping if this reduction declaration is re-used for a
1842 // different variable
1843 moduleTranslation.forgetMapping(region&: var.decl.getInitializerRegion());
1844 }
1845
1846 return success();
1847}
1848
1849/// Collect reduction info
1850static void collectReductionInfo(
1851 llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation,
1852 ReductionVarsInfo &redInfo,
1853 SmallVectorImpl<OwningReductionGen> &owningReductionGens,
1854 SmallVectorImpl<OwningAtomicReductionGen> &owningAtomicReductionGens,
1855 SmallVector<OwningDataPtrPtrReductionGen> &owningDataPtrPtrReductionGens,
1856 SmallVectorImpl<llvm::OpenMPIRBuilder::ReductionInfo> &reductionInfos) {
1857 unsigned numReductions = redInfo.size();
1858
1859 for (ReductionVarsInfo::VarInfo &var : redInfo.vars) {
1860 owningReductionGens.push_back(
1861 Elt: makeReductionGen(decl: var.decl, builder, moduleTranslation));
1862 owningAtomicReductionGens.push_back(
1863 Elt: makeAtomicReductionGen(decl: var.decl, builder, moduleTranslation));
1864 owningDataPtrPtrReductionGens.push_back(
1865 Elt: makeRefDataPtrGen(decl: var.decl, builder, moduleTranslation, isByRef: var.byRef));
1866 }
1867
1868 // Collect the reduction information.
1869 reductionInfos.reserve(N: numReductions);
1870 for (auto [var, redGen, atomicRedGen, dataPtrPtrRedGen] : llvm::zip_equal(
1871 t&: redInfo.vars, u&: owningReductionGens, args&: owningAtomicReductionGens,
1872 args&: owningDataPtrPtrReductionGens)) {
1873 llvm::OpenMPIRBuilder::ReductionGenAtomicCBTy atomicGen = nullptr;
1874 if (atomicRedGen)
1875 atomicGen = atomicRedGen;
1876 llvm::Value *variable = moduleTranslation.lookupValue(value: var.mlirVar);
1877 mlir::Type allocatedType;
1878 var.decl.getAllocRegion().walk(callback: [&](mlir::Operation *op) {
1879 if (auto alloca = mlir::dyn_cast<LLVM::AllocaOp>(Val: op)) {
1880 allocatedType = alloca.getElemType();
1881 return mlir::WalkResult::interrupt();
1882 }
1883
1884 return mlir::WalkResult::advance();
1885 });
1886
1887 reductionInfos.push_back(
1888 Elt: {moduleTranslation.convertType(type: var.decl.getType()), variable,
1889 var.privateVar,
1890 /*EvaluationKind=*/llvm::OpenMPIRBuilder::EvalKind::Scalar, redGen,
1891 /*ReductionGenClang=*/nullptr, atomicGen, dataPtrPtrRedGen,
1892 allocatedType ? moduleTranslation.convertType(type: allocatedType) : nullptr,
1893 var.decl.getByrefElementType()
1894 ? moduleTranslation.convertType(type: *var.decl.getByrefElementType())
1895 : nullptr});
1896 }
1897}
1898
1899/// handling of DeclareReductionOp's cleanup region
1900static LogicalResult
1901inlineOmpRegionCleanup(llvm::SmallVectorImpl<Region *> &cleanupRegions,
1902 llvm::ArrayRef<llvm::Value *> privateVariables,
1903 LLVM::ModuleTranslation &moduleTranslation,
1904 llvm::IRBuilderBase &builder, StringRef regionName,
1905 bool shouldLoadCleanupRegionArg = true) {
1906 for (auto [i, cleanupRegion] : llvm::enumerate(First&: cleanupRegions)) {
1907 if (cleanupRegion->empty())
1908 continue;
1909
1910 // map the argument to the cleanup region
1911 Block &entry = cleanupRegion->front();
1912
1913 llvm::Instruction *potentialTerminator =
1914 builder.GetInsertBlock()->empty() ? nullptr
1915 : &builder.GetInsertBlock()->back();
1916 if (potentialTerminator && potentialTerminator->isTerminator())
1917 builder.SetInsertPoint(potentialTerminator);
1918 llvm::Value *privateVarValue =
1919 shouldLoadCleanupRegionArg
1920 ? builder.CreateLoad(
1921 Ty: moduleTranslation.convertType(type: entry.getArgument(i: 0).getType()),
1922 Ptr: privateVariables[i])
1923 : privateVariables[i];
1924
1925 moduleTranslation.mapValue(mlir: entry.getArgument(i: 0), llvm: privateVarValue);
1926
1927 if (failed(Result: inlineConvertOmpRegions(region&: *cleanupRegion, blockName: regionName, builder,
1928 moduleTranslation)))
1929 return failure();
1930
1931 // clear block argument mapping in case it needs to be re-created with a
1932 // different source for another use of the same reduction decl
1933 moduleTranslation.forgetMapping(region&: *cleanupRegion);
1934 }
1935 return success();
1936}
1937
1938// TODO: not used by ParallelOp
1939static LogicalResult
1940createReductionsAndCleanup(Operation &op, llvm::IRBuilderBase &builder,
1941 LLVM::ModuleTranslation &moduleTranslation,
1942 llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
1943 ReductionVarsInfo &redInfo, bool isNowait = false,
1944 bool isTeamsReduction = false) {
1945 // Process the reductions if required.
1946 if (redInfo.size() == 0)
1947 return success();
1948
1949 SmallVector<OwningReductionGen> owningReductionGens;
1950 SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens;
1951 SmallVector<OwningDataPtrPtrReductionGen> owningReductionGenRefDataPtrGens;
1952 SmallVector<llvm::OpenMPIRBuilder::ReductionInfo, 2> reductionInfos;
1953
1954 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
1955
1956 // Create the reduction generators. We need to own them here because
1957 // ReductionInfo only accepts references to the generators.
1958 collectReductionInfo(builder, moduleTranslation, redInfo, owningReductionGens,
1959 owningAtomicReductionGens,
1960 owningDataPtrPtrReductionGens&: owningReductionGenRefDataPtrGens, reductionInfos);
1961
1962 // The call to createReductions below expects the block to have a
1963 // terminator. Create an unreachable instruction to serve as terminator
1964 // and remove it later.
1965 llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
1966 builder.SetInsertPoint(tempTerminator);
1967 llvm::DebugLoc reductionLoc = builder.getCurrentDebugLocation();
1968 llvm::OpenMPIRBuilder::InsertPointOrErrorTy contInsertPoint =
1969 ompBuilder->createReductions(Loc: builder, AllocaIP: allocaIP, ReductionInfos: reductionInfos,
1970 IsByRef: redInfo.getIsByRef(), IsNoWait: isNowait,
1971 IsTeamsReduction: isTeamsReduction);
1972
1973 if (failed(Result: handleError(result&: contInsertPoint, op)))
1974 return failure();
1975
1976 if (!contInsertPoint->isValid())
1977 return op.emitOpError() << "failed to convert reductions";
1978
1979 llvm::OpenMPIRBuilder::InsertPointTy afterIP = *contInsertPoint;
1980 if (!isTeamsReduction && !isNowait) {
1981 llvm::OpenMPIRBuilder::InsertPointOrErrorTy barrierIP =
1982 ompBuilder->createBarrier(Loc: {*contInsertPoint, reductionLoc},
1983 Kind: llvm::omp::OMPD_for);
1984
1985 if (failed(Result: handleError(result&: barrierIP, op)))
1986 return failure();
1987 afterIP = *barrierIP;
1988 }
1989
1990 tempTerminator->eraseFromParent();
1991 builder.restoreIP(IP: afterIP);
1992
1993 // after the construct, deallocate private reduction variables
1994 SmallVector<Region *> reductionRegions;
1995 llvm::transform(Range&: redInfo.vars, d_first: std::back_inserter(x&: reductionRegions),
1996 F: [](ReductionVarsInfo::VarInfo &var) {
1997 return &var.decl.getCleanupRegion();
1998 });
1999 LogicalResult result = inlineOmpRegionCleanup(
2000 cleanupRegions&: reductionRegions, privateVariables: redInfo.getPrivateReductionVariables(),
2001 moduleTranslation, builder, regionName: "omp.reduction.cleanup");
2002
2003 bool useDeviceSharedMem = omp::opInSharedDeviceContext(op);
2004 if (useDeviceSharedMem) {
2005 for (ReductionVarsInfo::VarInfo &var : redInfo.vars)
2006 ompBuilder->createOMPFreeShared(
2007 Loc: builder, Addr: var.privateVar,
2008 VarType: moduleTranslation.convertType(type: var.decl.getType()));
2009 }
2010
2011 return result;
2012}
2013
2014// TODO: not used by omp.parallel
2015static LogicalResult
2016allocAndInitializeReductionVars(Operation &op, llvm::IRBuilderBase &builder,
2017 LLVM::ModuleTranslation &moduleTranslation,
2018 llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
2019 ReductionVarsInfo &redInfo) {
2020 if (redInfo.size() == 0)
2021 return success();
2022
2023 if (failed(Result: allocReductionVars(op, builder, moduleTranslation, allocaIP,
2024 redInfo)))
2025 return failure();
2026
2027 return initReductionVars(op, builder, moduleTranslation,
2028 latestAllocaBlock: allocaIP.getNodeParent(), redInfo);
2029}
2030
2031/// Return the llvm::Value * corresponding to the `privateVar` that
2032/// is being privatized. It isn't always as simple as looking up
2033/// moduleTranslation with privateVar. For instance, in case of
2034/// an allocatable, the descriptor for the allocatable is privatized.
2035/// This descriptor is mapped using an MapInfoOp. So, this function
2036/// will return a pointer to the llvm::Value corresponding to the
2037/// block argument for the mapped descriptor.
2038static llvm::Value *
2039findAssociatedValue(Value privateVar, llvm::IRBuilderBase &builder,
2040 LLVM::ModuleTranslation &moduleTranslation,
2041 llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) {
2042 if (mappedPrivateVars == nullptr || !mappedPrivateVars->contains(Val: privateVar))
2043 return moduleTranslation.lookupValue(value: privateVar);
2044
2045 Value blockArg = (*mappedPrivateVars)[privateVar];
2046 Type privVarType = privateVar.getType();
2047 Type blockArgType = blockArg.getType();
2048 assert(isa<LLVM::LLVMPointerType>(blockArgType) &&
2049 "A block argument corresponding to a mapped var should have "
2050 "!llvm.ptr type");
2051
2052 if (privVarType == blockArgType)
2053 return moduleTranslation.lookupValue(value: blockArg);
2054
2055 // This typically happens when the privatized type is lowered from
2056 // boxchar<KIND> and gets lowered to !llvm.struct<(ptr, i64)>. That is the
2057 // struct/pair is passed by value. But, mapped values are passed only as
2058 // pointers, so before we privatize, we must load the pointer.
2059 if (!isa<LLVM::LLVMPointerType>(Val: privVarType))
2060 return builder.CreateLoad(Ty: moduleTranslation.convertType(type: privVarType),
2061 Ptr: moduleTranslation.lookupValue(value: blockArg));
2062
2063 return moduleTranslation.lookupValue(value: privateVar);
2064}
2065
2066// Privatizer region arguments may be by-value even when the available LLVM
2067// value is storage for that value, e.g. lowered Fortran boxchar descriptors in
2068// task context structs. Materialize the value expected by the region argument
2069// while preserving the existing pointer mapping for pointer arguments.
2070static llvm::Value *
2071materializeRegionArgValue(llvm::IRBuilderBase &builder,
2072 LLVM::ModuleTranslation &moduleTranslation,
2073 BlockArgument regionArg, llvm::Value *value) {
2074 if (!regionArg)
2075 return value;
2076
2077 llvm::Type *regionArgType =
2078 moduleTranslation.convertType(type: regionArg.getType());
2079 if (regionArgType->isPointerTy() || !value->getType()->isPointerTy())
2080 return value;
2081
2082 return builder.CreateLoad(Ty: regionArgType, Ptr: value);
2083}
2084
2085/// Returns the address of the data in `var`, via the `data_address` region.
2086static llvm::Expected<llvm::Value *>
2087getPrivateDataAddress(llvm::IRBuilderBase &builder,
2088 LLVM::ModuleTranslation &moduleTranslation,
2089 omp::PrivateClauseOp privatizer, llvm::Value *var) {
2090 Region &region = privatizer.getDataAddressRegion();
2091 if (region.empty())
2092 return var;
2093
2094 BlockArgument arg = privatizer.getDataAddressArg();
2095 moduleTranslation.mapValue(
2096 mlir: arg, llvm: materializeRegionArgValue(builder, moduleTranslation, regionArg: arg, value: var));
2097 SmallVector<llvm::Value *, 1> address;
2098 if (failed(Result: inlineConvertOmpRegions(region, blockName: "omp.private.data_address",
2099 builder, moduleTranslation, continuationBlockArgs: &address)))
2100 return llvm::make_error<PreviouslyReportedError>();
2101 setInsertPointForPossiblyEmptyBlock(builder);
2102 return address.front();
2103}
2104
2105/// Initialize a single (first)private variable. You probably want to use
2106/// allocateAndInitPrivateVars instead of this.
2107/// This returns the private variable which has been initialized. This
2108/// variable should be mapped before constructing the body of the Op.
2109static llvm::Expected<llvm::Value *>
2110initPrivateVar(llvm::IRBuilderBase &builder,
2111 LLVM::ModuleTranslation &moduleTranslation,
2112 omp::PrivateClauseOp &privDecl, llvm::Value *nonPrivateVar,
2113 BlockArgument &blockArg, llvm::Value *llvmPrivateVar,
2114 llvm::BasicBlock *privInitBlock,
2115 llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) {
2116 Region &initRegion = privDecl.getInitRegion();
2117 if (initRegion.empty())
2118 return llvmPrivateVar;
2119
2120 assert(nonPrivateVar);
2121 moduleTranslation.mapValue(mlir: privDecl.getInitMoldArg(), llvm: nonPrivateVar);
2122 moduleTranslation.mapValue(mlir: privDecl.getInitPrivateArg(), llvm: llvmPrivateVar);
2123
2124 // in-place convert the private initialization region
2125 SmallVector<llvm::Value *, 1> phis;
2126 if (failed(Result: inlineConvertOmpRegions(region&: initRegion, blockName: "omp.private.init", builder,
2127 moduleTranslation, continuationBlockArgs: &phis)))
2128 return llvm::createStringError(
2129 Fmt: "failed to inline `init` region of `omp.private`");
2130
2131 assert(phis.size() == 1 && "expected one allocation to be yielded");
2132
2133 // clear init region block argument mapping in case it needs to be
2134 // re-created with a different source for another use of the same
2135 // reduction decl
2136 moduleTranslation.forgetMapping(region&: initRegion);
2137
2138 // Prefer the value yielded from the init region to the allocated private
2139 // variable in case the region is operating on arguments by-value (e.g.
2140 // Fortran character boxes).
2141 return phis[0];
2142}
2143
2144/// Version of initPrivateVar which looks up the nonPrivateVar from mlirPrivVar.
2145static llvm::Expected<llvm::Value *> initPrivateVar(
2146 llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation,
2147 omp::PrivateClauseOp &privDecl, Value mlirPrivVar, BlockArgument &blockArg,
2148 llvm::Value *llvmPrivateVar, llvm::BasicBlock *privInitBlock,
2149 llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) {
2150 return initPrivateVar(
2151 builder, moduleTranslation, privDecl,
2152 nonPrivateVar: findAssociatedValue(privateVar: mlirPrivVar, builder, moduleTranslation,
2153 mappedPrivateVars),
2154 blockArg, llvmPrivateVar, privInitBlock, mappedPrivateVars);
2155}
2156
2157static llvm::Error
2158initPrivateVars(llvm::IRBuilderBase &builder,
2159 LLVM::ModuleTranslation &moduleTranslation,
2160 PrivateVarsInfo &privateVarsInfo,
2161 llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr) {
2162 if (privateVarsInfo.vars.empty())
2163 return llvm::Error::success();
2164
2165 llvm::BasicBlock *privInitBlock = splitBB(Builder&: builder, CreateBranch: true, Name: "omp.private.init");
2166 setInsertPointForPossiblyEmptyBlock(builder, block: privInitBlock);
2167
2168 for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) {
2169 llvm::Expected<llvm::Value *> privVarOrErr = initPrivateVar(
2170 builder, moduleTranslation, privDecl&: var.privatizer, mlirPrivVar: var.mlirVar, blockArg&: var.blockArg,
2171 llvmPrivateVar: var.llvmVar, privInitBlock, mappedPrivateVars);
2172
2173 if (!privVarOrErr)
2174 return privVarOrErr.takeError();
2175
2176 var.llvmVar = privVarOrErr.get();
2177 moduleTranslation.mapValue(mlir: var.blockArg, llvm: var.llvmVar);
2178
2179 setInsertPointForPossiblyEmptyBlock(builder);
2180 }
2181
2182 return llvm::Error::success();
2183}
2184
2185static LogicalResult
2186convertAllocatorVars(Operation &op, ValueRange allocatorVars,
2187 llvm::IRBuilderBase &builder,
2188 LLVM::ModuleTranslation &moduleTranslation,
2189 PrivateVarsInfo &privateVarsInfo) {
2190 for (Value allocatorVar : allocatorVars) {
2191 if (privateVarsInfo.convertedAllocators.contains(Val: allocatorVar))
2192 continue;
2193
2194 llvm::Value *allocator = moduleTranslation.lookupValue(value: allocatorVar);
2195 if (!allocator)
2196 return op.emitError(message: "failed to translate OpenMP allocator operand");
2197 if (allocator->getType()->isIntegerTy())
2198 allocator = builder.CreateIntToPtr(V: allocator, DestTy: builder.getPtrTy());
2199 else if (allocator->getType()->isPointerTy())
2200 allocator = builder.CreatePointerBitCastOrAddrSpaceCast(
2201 V: allocator, DestTy: builder.getPtrTy());
2202 else
2203 return op.emitError(
2204 message: "OpenMP allocator operand must have integer or pointer type");
2205
2206 privateVarsInfo.convertedAllocators.try_emplace(Key: allocatorVar, Args&: allocator);
2207 }
2208 return success();
2209}
2210
2211/// Allocate and initialize delayed private variables. Returns the basic block
2212/// which comes after all of these allocations. llvm::Value * for each of these
2213/// private variables are populated in llvmPrivateVars.
2214template <typename T>
2215static llvm::Expected<llvm::BasicBlock *> allocatePrivateVars(
2216 T op, llvm::IRBuilderBase &builder,
2217 LLVM::ModuleTranslation &moduleTranslation,
2218 PrivateVarsInfo &privateVarsInfo,
2219 llvm::OpenMPIRBuilder::InsertPointTy &allocaIP,
2220 llvm::DenseMap<Value, Value> *mappedPrivateVars = nullptr,
2221 std::optional<llvm::OpenMPIRBuilder::InsertPointTy> allocatorIP =
2222 std::nullopt) {
2223 // Allocate private vars
2224 // Save blocks before splits since allocaIP/allocatorIP iterators may follow
2225 // spliced instructions to the new block.
2226 llvm::BasicBlock *allocaBB = allocaIP.getNodeParent();
2227 llvm::Instruction *allocaTerminator = allocaBB->getTerminator();
2228 splitBB(IP: allocaTerminator->getIterator(), CreateBranch: true,
2229 DL: allocaTerminator->getDebugLoc(), Name: "omp.region.after_alloca");
2230 // Update the allocaTerminator since the alloca block was split above.
2231 allocaTerminator = allocaBB->getTerminator();
2232 // The new terminator is an uncondition branch created by the splitBB above.
2233 assert(allocaTerminator->getNumSuccessors() == 1 &&
2234 "This is an unconditional branch created by splitBB");
2235 allocaIP = allocaTerminator->getIterator();
2236
2237 llvm::Instruction *allocatorTerminator = nullptr;
2238 llvm::BasicBlock *afterAllocatorAllocations = nullptr;
2239 if (allocatorIP) {
2240 llvm::BasicBlock *allocatorBB = allocatorIP->getNodeParent();
2241 allocatorTerminator = allocatorBB->getTerminator();
2242 afterAllocatorAllocations = splitBB(
2243 IP: allocatorTerminator->getIterator(), CreateBranch: true,
2244 DL: allocatorTerminator->getDebugLoc(), Name: "omp.region.after_allocate");
2245 allocatorTerminator = allocatorBB->getTerminator();
2246 assert(allocatorTerminator->getNumSuccessors() == 1 &&
2247 "This is an unconditional branch created by splitBB");
2248 }
2249
2250 std::optional<llvm::IRBuilderBase::InsertPointGuard> guard;
2251 if (!allocatorIP)
2252 guard.emplace(args&: builder);
2253 builder.SetInsertPoint(allocaTerminator);
2254
2255 llvm::DataLayout dataLayout = builder.getDataLayout();
2256 llvm::BasicBlock *afterAllocas = allocaTerminator->getSuccessor(Idx: 0);
2257
2258 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
2259 bool mightUseDeviceSharedMem = omp::opInSharedDeviceContext(op&: *op);
2260 unsigned int allocaAS =
2261 moduleTranslation.getLLVMModule()->getDataLayout().getAllocaAddrSpace();
2262 unsigned int defaultAS = moduleTranslation.getLLVMModule()
2263 ->getDataLayout()
2264 .getProgramAddressSpace();
2265
2266 SmallVector<int64_t> allocateItemForPrivate(privateVarsInfo.vars.size(), -1);
2267 ValueRange allocatorVars;
2268 DenseI64ArrayAttr allocateAlignments;
2269 if constexpr (std::is_same_v<T, omp::ParallelOp> ||
2270 std::is_same_v<T, omp::ScopeOp>) {
2271 allocatorVars = op.getAllocatorVars();
2272 allocateAlignments = op.getAllocateAlignmentsAttr();
2273 if (auto privateIndices = op.getAllocatePrivateIndicesAttr())
2274 for (auto [allocateIndex, privateIndex] :
2275 llvm::enumerate(privateIndices.asArrayRef()))
2276 allocateItemForPrivate[privateIndex] = allocateIndex;
2277 }
2278
2279 for (auto [privateIndex, var] : llvm::enumerate(First&: privateVarsInfo.vars)) {
2280 llvm::Type *llvmAllocType =
2281 moduleTranslation.convertType(type: var.privatizer.getType());
2282 llvm::Value *llvmPrivateVar = nullptr;
2283 int64_t allocateIndex = allocateItemForPrivate[privateIndex];
2284 builder.SetInsertPoint(allocateIndex >= 0 && allocatorTerminator
2285 ? allocatorTerminator
2286 : allocaTerminator);
2287 if (allocateIndex >= 0) {
2288 if (ompBuilder->Config.isTargetDevice() ||
2289 op->template getParentOfType<omp::TargetOp>())
2290 return llvm::createStringError(
2291 Fmt: "allocate clause in an OpenMP device context is not supported");
2292 if (!llvmAllocType->isSized())
2293 return llvm::createStringError(
2294 Fmt: "allocate clause private type must have a fixed size");
2295 llvm::TypeSize size = dataLayout.getTypeAllocSize(Ty: llvmAllocType);
2296 if (size.isScalable())
2297 return llvm::createStringError(
2298 Fmt: "allocate clause private type must have a fixed size");
2299 llvm::IntegerType *sizeTy =
2300 moduleTranslation.getLLVMModule()->getDataLayout().getIntPtrType(
2301 C&: moduleTranslation.getLLVMModule()->getContext());
2302 if (!llvm::isUIntN(N: sizeTy->getBitWidth(), x: size.getFixedValue()))
2303 return llvm::createStringError(
2304 Fmt: "OpenMP allocation size cannot be represented by the target size "
2305 "type");
2306 llvm::Value *sizeValue =
2307 llvm::ConstantInt::get(Ty: sizeTy, V: size.getFixedValue());
2308
2309 Value allocatorVar = allocatorVars[allocateIndex];
2310 auto allocator = privateVarsInfo.convertedAllocators.find(Val: allocatorVar);
2311 if (allocator == privateVarsInfo.convertedAllocators.end())
2312 return llvm::createStringError(
2313 Fmt: "failed to find converted OpenMP allocator operand");
2314 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
2315 int64_t alignment =
2316 allocateAlignments ? allocateAlignments[allocateIndex] : 0;
2317 if (alignment != 0) {
2318 // The allocation must be aligned to at least the maximum of the
2319 // requested alignment and the alignment the base language requires
2320 // for the type being allocated.
2321 uint64_t alignmentValue = std::max<uint64_t>(
2322 a: static_cast<uint64_t>(alignment),
2323 b: dataLayout.getABITypeAlign(Ty: llvmAllocType).value());
2324 if (!llvm::isUIntN(N: sizeTy->getBitWidth(), x: alignmentValue))
2325 return llvm::createStringError(
2326 Fmt: "OpenMP allocation alignment cannot be represented by the "
2327 "target size type");
2328 llvmPrivateVar = ompBuilder->createOMPAlignedAlloc(
2329 Loc: ompLoc, Align: llvm::ConstantInt::get(Ty: sizeTy, V: alignmentValue), Size: sizeValue,
2330 Allocator: allocator->second, Name: "omp.private.alloc");
2331 } else {
2332 llvmPrivateVar = ompBuilder->createOMPAlloc(
2333 Loc: ompLoc, Size: sizeValue, Allocator: allocator->second, Name: "omp.private.alloc");
2334 }
2335 if (!llvmPrivateVar)
2336 return llvm::createStringError(
2337 Fmt: "failed to create OpenMP private allocation");
2338 privateVarsInfo.allocatorPrivates.push_back(
2339 Elt: {.allocatedPtr: llvmPrivateVar, .allocator: allocator->second});
2340 } else if (mightUseDeviceSharedMem &&
2341 omp::allocaUsesRequireSharedMem(alloc: var.blockArg)) {
2342 llvmPrivateVar = ompBuilder->createOMPAllocShared(Loc: builder, VarType: llvmAllocType);
2343 } else {
2344 llvmPrivateVar = builder.CreateAlloca(
2345 Ty: llvmAllocType, /*ArraySize=*/nullptr, Name: "omp.private.alloc");
2346 if (allocaAS != defaultAS)
2347 llvmPrivateVar = builder.CreateAddrSpaceCast(
2348 V: llvmPrivateVar, DestTy: builder.getPtrTy(AddrSpace: defaultAS));
2349 }
2350
2351 var.llvmVar = llvmPrivateVar;
2352 }
2353
2354 return afterAllocatorAllocations ? afterAllocatorAllocations : afterAllocas;
2355}
2356
2357/// A compiler-generated barrier is unsafe when only part of the current team
2358/// can reach it. This cannot always be determined statically (e.g. across a
2359/// function call), but the enclosing OpenMP constructs can rule it out.
2360static bool opMightBeSafeForBarriers(mlir::Operation *op) {
2361 for (mlir::Operation *parent = op; parent != nullptr;
2362 parent = parent->getParentOp()) {
2363 // An inner parallel construct creates a new team, even when it is nested
2364 // inside a construct that only some threads of the outer team encounter.
2365 if (mlir::isa<omp::ParallelOp>(Val: parent))
2366 return true;
2367
2368 if (mlir::isa<omp::SingleOp, omp::CriticalOp, omp::MaskedOp, omp::MasterOp,
2369 omp::SectionOp>(Val: parent))
2370 return false;
2371 }
2372 return true;
2373}
2374
2375static LogicalResult
2376emitPrivatizationBarrier(mlir::Operation *op, llvm::IRBuilderBase &builder,
2377 LLVM::ModuleTranslation &moduleTranslation) {
2378 if (!opMightBeSafeForBarriers(op))
2379 return success();
2380
2381 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
2382 llvm::OpenMPIRBuilder::InsertPointOrErrorTy res =
2383 ompBuilder->createBarrier(Loc: builder, Kind: llvm::omp::OMPD_barrier);
2384 return handleError(result&: res, op&: *op);
2385}
2386
2387/// Applies the firstprivate copy region for each variable, pairing the mold
2388/// values with the private allocations and privatizer declarations from
2389/// `privateVarsInfo`, and emits a barrier if requested.
2390///
2391/// `moldVars` and `llvmPrivateVars` are optional overrides that, if empty,
2392/// are read directly from `privateVarsInfo`. `mappedPrivateVars` is only
2393/// consulted while materializing mold values from `privateVarsInfo` (i.e. empty
2394/// `moldVars`).
2395static LogicalResult
2396completePrivateVars(mlir::Operation *op, llvm::IRBuilderBase &builder,
2397 LLVM::ModuleTranslation &moduleTranslation,
2398 const PrivateVarsInfo &privateVarsInfo, bool insertBarrier,
2399 ArrayRef<llvm::Value *> moldVars = {},
2400 ArrayRef<llvm::Value *> llvmPrivateVars = {},
2401 DenseMap<Value, Value> *mappedPrivateVars = nullptr) {
2402 assert((moldVars.empty() || moldVars.size() == privateVarsInfo.vars.size()) &&
2403 "expected one mold value per variable when provided");
2404 assert((llvmPrivateVars.empty() ||
2405 llvmPrivateVars.size() == privateVarsInfo.vars.size()) &&
2406 "expected one private allocation per variable when provided");
2407
2408 // Apply copy region for firstprivate.
2409 bool needsFirstprivate = llvm::any_of(
2410 Range: privateVarsInfo.vars, P: [](const PrivateVarsInfo::VarInfo &var) {
2411 // Copy the op handle so the non-const accessor can be called.
2412 omp::PrivateClauseOp privOp = var.privatizer;
2413 return privOp.getDataSharingType() ==
2414 omp::DataSharingClauseType::FirstPrivate;
2415 });
2416
2417 if (!needsFirstprivate)
2418 return insertBarrier
2419 ? emitPrivatizationBarrier(op, builder, moduleTranslation)
2420 : success();
2421
2422 // Materialize the mold values from the MLIR variables when the caller didn't
2423 // supply explicit locations.
2424 SmallVector<llvm::Value *> materializedMoldVars;
2425 if (moldVars.empty()) {
2426 materializedMoldVars.reserve(N: privateVarsInfo.vars.size());
2427 for (const PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) {
2428 // map copyRegion rhs arg
2429 llvm::Value *moldVar = findAssociatedValue(
2430 privateVar: var.mlirVar, builder, moduleTranslation, mappedPrivateVars);
2431 assert(moldVar);
2432 materializedMoldVars.push_back(Elt: moldVar);
2433 }
2434 moldVars = materializedMoldVars;
2435 }
2436
2437 llvm::BasicBlock *copyBlock =
2438 splitBB(Builder&: builder, /*CreateBranch=*/true, Name: "omp.private.copy");
2439 setInsertPointForPossiblyEmptyBlock(builder, block: copyBlock);
2440
2441 unsigned varIdx = 0;
2442 for (auto [var, moldVar] : llvm::zip_equal(t: privateVarsInfo.vars, u&: moldVars)) {
2443 // The private allocation defaults to the one tracked in `privateVarsInfo`,
2444 // unless the caller overrode it with an explicit location.
2445 llvm::Value *llvmVar =
2446 llvmPrivateVars.empty() ? var.llvmVar : llvmPrivateVars[varIdx];
2447 ++varIdx;
2448
2449 omp::PrivateClauseOp decl = var.privatizer;
2450 if (decl.getDataSharingType() != omp::DataSharingClauseType::FirstPrivate)
2451 continue;
2452
2453 // copyRegion implements `lhs = rhs`
2454 Region &copyRegion = decl.getCopyRegion();
2455
2456 llvm::Value *copyMoldVar = materializeRegionArgValue(
2457 builder, moduleTranslation, regionArg: decl.getCopyMoldArg(), value: moldVar);
2458 llvm::Value *copyPrivateVar = materializeRegionArgValue(
2459 builder, moduleTranslation, regionArg: decl.getCopyPrivateArg(), value: llvmVar);
2460
2461 moduleTranslation.mapValue(mlir: decl.getCopyMoldArg(), llvm: copyMoldVar);
2462
2463 // map copyRegion lhs arg
2464 moduleTranslation.mapValue(mlir: decl.getCopyPrivateArg(), llvm: copyPrivateVar);
2465
2466 // in-place convert copy region
2467 if (failed(Result: inlineConvertOmpRegions(region&: copyRegion, blockName: "omp.private.copy", builder,
2468 moduleTranslation)))
2469 return decl.emitError(message: "failed to inline `copy` region of `omp.private`");
2470
2471 setInsertPointForPossiblyEmptyBlock(builder);
2472
2473 // ignore unused value yielded from copy region
2474
2475 // clear copy region block argument mapping in case it needs to be
2476 // re-created with different sources for reuse of the same reduction
2477 // decl
2478 moduleTranslation.forgetMapping(region&: copyRegion);
2479 }
2480
2481 return insertBarrier
2482 ? emitPrivatizationBarrier(op, builder, moduleTranslation)
2483 : success();
2484}
2485
2486template <typename T>
2487static LogicalResult
2488cleanupPrivateVars(T op, llvm::IRBuilderBase &builder,
2489 LLVM::ModuleTranslation &moduleTranslation, Location loc,
2490 PrivateVarsInfo &privateVarsInfo) {
2491 // private variable deallocation
2492 SmallVector<Region *> privateCleanupRegions;
2493 llvm::transform(privateVarsInfo.vars,
2494 std::back_inserter(x&: privateCleanupRegions),
2495 [](PrivateVarsInfo::VarInfo &var) {
2496 return &var.privatizer.getDeallocRegion();
2497 });
2498
2499 SmallVector<llvm::Value *> llvmPrivVars = privateVarsInfo.getLlvmVars();
2500 if (failed(Result: inlineOmpRegionCleanup(cleanupRegions&: privateCleanupRegions, privateVariables: llvmPrivVars,
2501 moduleTranslation, builder,
2502 regionName: "omp.private.dealloc",
2503 /*shouldLoadCleanupRegionArg=*/false)))
2504 return mlir::emitError(loc, message: "failed to inline `dealloc` region of an "
2505 "`omp.private` op in");
2506 setInsertPointForPossiblyEmptyBlock(builder);
2507
2508 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
2509 bool mightUseDeviceSharedMem = omp::opInSharedDeviceContext(op&: *op);
2510 for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) {
2511 if (mightUseDeviceSharedMem &&
2512 omp::allocaUsesRequireSharedMem(alloc: var.blockArg)) {
2513 ompBuilder->createOMPFreeShared(
2514 Loc: builder, Addr: var.llvmVar,
2515 VarType: moduleTranslation.convertType(type: var.privatizer.getType()));
2516 }
2517 }
2518
2519 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
2520 for (const PrivateVarsInfo::AllocatorInfo &allocation :
2521 llvm::reverse(C&: privateVarsInfo.allocatorPrivates))
2522 ompBuilder->createOMPFree(Loc: ompLoc, Addr: allocation.allocatedPtr,
2523 Allocator: allocation.allocator);
2524
2525 return success();
2526}
2527
2528/// Returns true if the construct contains omp.cancel or omp.cancellation_point
2529static bool constructIsCancellable(Operation *op) {
2530 // omp.cancel and omp.cancellation_point must be "closely nested" so they will
2531 // be visible and not inside of function calls. This is enforced by the
2532 // verifier.
2533 return op
2534 ->walk(callback: [](Operation *child) {
2535 if (mlir::isa<omp::CancelOp, omp::CancellationPointOp>(Val: child))
2536 return WalkResult::interrupt();
2537 return WalkResult::advance();
2538 })
2539 .wasInterrupted();
2540}
2541
2542// Forward declarations for the task-reduction helpers defined alongside the
2543// omp.taskgroup lowering further down in this file. These are shared by the
2544// `reduction(task, ...)` modifier lowering on the parallel/worksharing
2545// constructs and by the omp.taskgroup / omp.taskloop.context task_reduction
2546// lowering. When \p isModifier is set, `__kmpc_taskred_modifier_init` is
2547// emitted (opening a task-reduction scope) instead of `__kmpc_taskred_init`,
2548// with \p isWorksharing selecting the runtime `is_ws` argument.
2549static llvm::Value *emitTaskReductionInitCall(
2550 ArrayRef<omp::DeclareReductionOp> redDecls,
2551 ArrayRef<llvm::Value *> origPtrs, StringRef helperNamePrefix,
2552 llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder::InsertPointTy allocaIP,
2553 LLVM::ModuleTranslation &moduleTranslation, bool isModifier = false,
2554 bool isWorksharing = false);
2555static void
2556emitTaskReductionModifierFini(bool isWorksharing, llvm::IRBuilderBase &builder,
2557 LLVM::ModuleTranslation &moduleTranslation);
2558
2559/// Register only the task-modified items. All items still participate in the
2560/// enclosing construct's ordinary reduction combination and cleanup.
2561template <typename OpTy>
2562static llvm::Value *emitTaskReductionModifierInitCall(
2563 OpTy op, ArrayRef<omp::DeclareReductionOp> reductionDecls,
2564 ArrayRef<llvm::Value *> privateReductionVariables,
2565 llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder::InsertPointTy allocaIP,
2566 LLVM::ModuleTranslation &moduleTranslation, bool isWorksharing) {
2567 SmallVector<omp::DeclareReductionOp> taskDecls;
2568 SmallVector<llvm::Value *> taskVariables;
2569 for (unsigned i = 0, e = op.getNumReductionVars(); i != e; ++i) {
2570 if (op.getReductionModifier(i) != omp::ReductionModifier::task)
2571 continue;
2572 taskDecls.push_back(Elt: reductionDecls[i]);
2573 taskVariables.push_back(Elt: privateReductionVariables[i]);
2574 }
2575 return emitTaskReductionInitCall(
2576 redDecls: taskDecls, origPtrs: taskVariables, helperNamePrefix: "__omp_taskred_mod_", builder, allocaIP,
2577 moduleTranslation, /*isModifier=*/true, isWorksharing);
2578}
2579
2580static LogicalResult
2581convertOmpSections(Operation &opInst, llvm::IRBuilderBase &builder,
2582 LLVM::ModuleTranslation &moduleTranslation) {
2583 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2584 using StorableBodyGenCallbackTy =
2585 llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
2586
2587 auto sectionsOp = cast<omp::SectionsOp>(Val&: opInst);
2588
2589 if (failed(Result: checkImplementationStatus(op&: opInst)))
2590 return failure();
2591
2592 ReductionVarsInfo redInfo(sectionsOp);
2593 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
2594 findAllocInsertPoints(builder, moduleTranslation);
2595
2596 if (failed(Result: allocAndInitializeReductionVars(
2597 op&: *sectionsOp, builder, moduleTranslation, allocaIP, redInfo)))
2598 return failure();
2599
2600 bool isTaskReductionMod =
2601 sectionsOp.hasReductionModifier(modifier: omp::ReductionModifier::task);
2602
2603 SmallVector<StorableBodyGenCallbackTy> sectionCBs;
2604
2605 for (Operation &op : *sectionsOp.getRegion().begin()) {
2606 auto sectionOp = dyn_cast<omp::SectionOp>(Val&: op);
2607 if (!sectionOp) // omp.terminator
2608 continue;
2609
2610 Region &region = sectionOp.getRegion();
2611 auto sectionCB = [&sectionsOp, &region, &builder, &moduleTranslation](
2612 InsertPointTy allocaIP, InsertPointTy codeGenIP,
2613 ArrayRef<llvm::BasicBlock *> deallocBlocks) {
2614 builder.restoreIP(IP: codeGenIP);
2615
2616 // map the omp.section reduction block argument to the omp.sections block
2617 // arguments
2618 // TODO: this assumes that the only block arguments are reduction
2619 // variables
2620 assert(region.getNumArguments() ==
2621 sectionsOp.getRegion().getNumArguments());
2622 for (auto [sectionsArg, sectionArg] : llvm::zip_equal(
2623 t: sectionsOp.getRegion().getArguments(), u: region.getArguments())) {
2624 llvm::Value *llvmVal = moduleTranslation.lookupValue(value: sectionsArg);
2625 assert(llvmVal);
2626 moduleTranslation.mapValue(mlir: sectionArg, llvm: llvmVal);
2627 }
2628
2629 return convertOmpOpRegions(region, blockName: "omp.section.region", builder,
2630 moduleTranslation)
2631 .takeError();
2632 };
2633 sectionCBs.push_back(Elt: sectionCB);
2634 }
2635
2636 // Empty sections still require an implicit barrier unless nowait is present.
2637 if (sectionCBs.empty()) {
2638 if (!sectionsOp.getNowait()) {
2639 auto afterIP = getOpenMPBuilder(moduleTranslation)
2640 ->createBarrier(Loc: builder, Kind: llvm::omp::OMPD_sections);
2641 if (failed(Result: handleError(result&: afterIP, op&: opInst)))
2642 return failure();
2643 builder.restoreIP(IP: *afterIP);
2644 }
2645 return success();
2646 }
2647
2648 // For `reduction(task, ...)` open a task-reduction scope for the worksharing
2649 // region. Participating explicit tasks accumulate into the per-thread private
2650 // copies, which the worksharing reduction then combines across threads. This
2651 // is emitted only after the empty-sections early return above, so it stays
2652 // balanced with the matching fini emitted after the sections region.
2653 if (isTaskReductionMod &&
2654 !emitTaskReductionModifierInitCall(
2655 op: sectionsOp, reductionDecls: redInfo.getReductionDecls(),
2656 privateReductionVariables: redInfo.getPrivateReductionVariables(), builder, allocaIP,
2657 moduleTranslation, /*isWorksharing=*/true))
2658 return sectionsOp.emitError(
2659 message: "failed to emit task reduction modifier initialization");
2660
2661 assert(isa<omp::SectionOp>(*sectionsOp.getRegion().op_begin()));
2662
2663 // TODO: Perform appropriate actions according to the data-sharing
2664 // attribute (shared, private, firstprivate, ...) of variables.
2665 // Currently defaults to shared.
2666 auto privCB = [&](InsertPointTy, InsertPointTy codeGenIP, llvm::Value &,
2667 llvm::Value &vPtr, llvm::Value *&replacementValue)
2668 -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
2669 replacementValue = &vPtr;
2670 return codeGenIP;
2671 };
2672
2673 // TODO: Perform finalization actions for variables. This has to be
2674 // called for variables which have destructors/finalizers.
2675 auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
2676
2677 allocaIP = findAllocInsertPoints(builder, moduleTranslation);
2678 bool isCancellable = constructIsCancellable(op: sectionsOp);
2679 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
2680 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
2681 getOpenMPBuilder(moduleTranslation)
2682 ->createSections(Loc: ompLoc, AllocaIP: allocaIP, SectionCBs: sectionCBs, PrivCB: privCB, FiniCB: finiCB,
2683 IsCancellable: isCancellable, IsNowait: sectionsOp.getNowait());
2684
2685 if (failed(Result: handleError(result&: afterIP, op&: opInst)))
2686 return failure();
2687
2688 builder.restoreIP(IP: *afterIP);
2689
2690 // Close the task-reduction scope before combining the worksharing copies.
2691 if (isTaskReductionMod)
2692 emitTaskReductionModifierFini(/*isWorksharing=*/true, builder,
2693 moduleTranslation);
2694
2695 // Process the reductions if required.
2696 return createReductionsAndCleanup(op&: *sectionsOp, builder, moduleTranslation,
2697 allocaIP, redInfo, isNowait: sectionsOp.getNowait());
2698}
2699
2700/// Converts an OpenMP scope construct into LLVM IR.
2701static LogicalResult
2702convertOmpScope(omp::ScopeOp &scopeOp, llvm::IRBuilderBase &builder,
2703 LLVM::ModuleTranslation &moduleTranslation) {
2704 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2705 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
2706
2707 if (failed(Result: checkImplementationStatus(op&: *scopeOp)))
2708 return failure();
2709
2710 PrivateVarsInfo privateVarsInfo(scopeOp);
2711 if (failed(Result: convertAllocatorVars(op&: *scopeOp, allocatorVars: scopeOp.getAllocatorVars(), builder,
2712 moduleTranslation, privateVarsInfo)))
2713 return failure();
2714
2715 ReductionVarsInfo reductionInfo(scopeOp);
2716 InsertPointTy privateAllocaIP =
2717 findAllocInsertPoints(builder, moduleTranslation);
2718
2719 if (scopeOp.getAllocateVars().empty()) {
2720 llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(
2721 op: scopeOp, builder, moduleTranslation, privateVarsInfo, allocaIP&: privateAllocaIP);
2722 if (failed(Result: handleError(result&: afterAllocas, op&: *scopeOp)))
2723 return failure();
2724 }
2725
2726 if (failed(Result: allocAndInitializeReductionVars(op&: *scopeOp, builder,
2727 moduleTranslation, allocaIP&: privateAllocaIP,
2728 redInfo&: reductionInfo)))
2729 return failure();
2730
2731 auto bodyCB =
2732 [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
2733 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
2734 if (!scopeOp.getAllocateVars().empty()) {
2735 // Runtime storage must be allocated on each dynamic Scope entry. Ordinary
2736 // private allocas still use the enclosing alloca insertion point.
2737 llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(
2738 op: scopeOp, builder, moduleTranslation, privateVarsInfo, allocaIP&: privateAllocaIP,
2739 /*mappedPrivateVars=*/nullptr, allocatorIP: codeGenIP);
2740 if (handleError(result&: afterAllocas, op&: *scopeOp).failed())
2741 return llvm::make_error<PreviouslyReportedError>();
2742 builder.SetInsertPoint(afterAllocas.get()->getTerminator());
2743 } else {
2744 builder.restoreIP(IP: codeGenIP);
2745 }
2746
2747 if (handleError(
2748 error: initPrivateVars(builder, moduleTranslation, privateVarsInfo),
2749 op&: *scopeOp)
2750 .failed())
2751 return llvm::make_error<PreviouslyReportedError>();
2752
2753 if (failed(Result: completePrivateVars(op: scopeOp, builder, moduleTranslation,
2754 privateVarsInfo,
2755 insertBarrier: scopeOp.getPrivateNeedsBarrier())))
2756 return llvm::make_error<PreviouslyReportedError>();
2757
2758 return convertOmpOpRegions(region&: scopeOp.getRegion(), blockName: "omp.scope.region", builder,
2759 moduleTranslation)
2760 .takeError();
2761 };
2762
2763 auto finiCB = [&](InsertPointTy codeGenIP) -> llvm::Error {
2764 InsertPointTy oldIP = builder.saveIP();
2765 builder.restoreIP(IP: codeGenIP);
2766 if (failed(Result: cleanupPrivateVars(op: scopeOp, builder, moduleTranslation,
2767 loc: scopeOp.getLoc(), privateVarsInfo)))
2768 return llvm::make_error<PreviouslyReportedError>();
2769 builder.restoreIP(IP: oldIP);
2770 return llvm::Error::success();
2771 };
2772
2773 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
2774 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
2775 ompBuilder->createScope(Loc: ompLoc, BodyGenCB: bodyCB, FiniCB: finiCB, IsNowait: scopeOp.getNowait());
2776
2777 if (failed(Result: handleError(result&: afterIP, op&: *scopeOp)))
2778 return failure();
2779
2780 builder.restoreIP(IP: *afterIP);
2781
2782 // Process the reductions if required.
2783 return createReductionsAndCleanup(op&: *scopeOp, builder, moduleTranslation,
2784 allocaIP&: privateAllocaIP, redInfo&: reductionInfo,
2785 isNowait: scopeOp.getNowait(),
2786 /*isTeamsReduction=*/false);
2787}
2788
2789/// Converts an OpenMP single construct into LLVM IR using OpenMPIRBuilder.
2790static LogicalResult
2791convertOmpSingle(omp::SingleOp &singleOp, llvm::IRBuilderBase &builder,
2792 LLVM::ModuleTranslation &moduleTranslation) {
2793 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2794 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
2795
2796 if (failed(Result: checkImplementationStatus(op&: *singleOp)))
2797 return failure();
2798
2799 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP,
2800 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
2801 builder.restoreIP(IP: codegenIP);
2802 return convertOmpOpRegions(region&: singleOp.getRegion(), blockName: "omp.single.region",
2803 builder, moduleTranslation)
2804 .takeError();
2805 };
2806 auto finiCB = [&](InsertPointTy codeGenIP) { return llvm::Error::success(); };
2807
2808 // Handle copyprivate
2809 Operation::operand_range cpVars = singleOp.getCopyprivateVars();
2810 std::optional<ArrayAttr> cpFuncs = singleOp.getCopyprivateSyms();
2811 llvm::SmallVector<llvm::Value *> llvmCPVars;
2812 llvm::SmallVector<llvm::Function *> llvmCPFuncs;
2813 for (size_t i = 0, e = cpVars.size(); i < e; ++i) {
2814 llvmCPVars.push_back(Elt: moduleTranslation.lookupValue(value: cpVars[i]));
2815 auto llvmFuncOp = SymbolTable::lookupNearestSymbolFrom<LLVM::LLVMFuncOp>(
2816 from: singleOp, symbol: cast<SymbolRefAttr>(Val: (*cpFuncs)[i]));
2817 llvmCPFuncs.push_back(
2818 Elt: moduleTranslation.lookupFunction(name: llvmFuncOp.getName()));
2819 }
2820
2821 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
2822 getOpenMPBuilder(moduleTranslation)
2823 ->createSingle(Loc: ompLoc, BodyGenCB: bodyCB, FiniCB: finiCB, IsNowait: singleOp.getNowait(),
2824 CPVars: llvmCPVars, CPFuncs: llvmCPFuncs);
2825
2826 if (failed(Result: handleError(result&: afterIP, op&: *singleOp)))
2827 return failure();
2828
2829 builder.restoreIP(IP: *afterIP);
2830 return success();
2831}
2832
2833static omp::DistributeOp
2834getDistributeCapturingTeamsReduction(omp::TeamsOp teamsOp) {
2835 // Early return if we found more than one distribute op or if we can't find
2836 // any distribute op in the teams region.
2837 omp::DistributeOp distOp;
2838 WalkResult walk = teamsOp.getRegion().walk(callback: [&](omp::DistributeOp op) {
2839 if (distOp)
2840 return WalkResult::interrupt();
2841 distOp = op;
2842 return WalkResult::skip();
2843 });
2844 if (walk.wasInterrupted() || !distOp)
2845 return {};
2846
2847 auto iface =
2848 llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(Val: teamsOp.getOperation());
2849 // Check that all uses of the reduction block arg has the same distribute op
2850 // parent.
2851 llvm::SmallVector<mlir::Operation *> debugUses;
2852 for (auto ra : iface.getReductionBlockArgs())
2853 for (auto &use : ra.getUses()) {
2854 auto *useOp = use.getOwner();
2855 // Ignore debug uses.
2856 if (mlir::isa<LLVM::DbgDeclareOp, LLVM::DbgValueOp>(Val: useOp)) {
2857 debugUses.push_back(Elt: useOp);
2858 continue;
2859 }
2860 if (!distOp->isProperAncestor(other: useOp))
2861 return {};
2862 }
2863
2864 // If we are going to use distribute reduction then remove any debug uses of
2865 // the reduction parameters in teamsOp. Otherwise they will be left without
2866 // any mapped value in moduleTranslation and will eventually error out.
2867 for (auto *use : debugUses)
2868 use->erase();
2869 return distOp;
2870}
2871
2872// Convert an OpenMP Teams construct to LLVM IR using OpenMPIRBuilder
2873static LogicalResult
2874convertOmpTeams(omp::TeamsOp op, llvm::IRBuilderBase &builder,
2875 LLVM::ModuleTranslation &moduleTranslation) {
2876 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
2877 if (failed(Result: checkImplementationStatus(op&: *op)))
2878 return failure();
2879
2880 std::optional<ReductionVarsInfo> teamsRedInfo;
2881 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
2882 findAllocInsertPoints(builder, moduleTranslation);
2883
2884 // Only do teams reduction if there is no distribute op that captures the
2885 // reduction instead.
2886 if (!getDistributeCapturingTeamsReduction(teamsOp: op)) {
2887 teamsRedInfo.emplace(args&: op);
2888
2889 if (failed(Result: allocAndInitializeReductionVars(op&: *op, builder, moduleTranslation,
2890 allocaIP, redInfo&: *teamsRedInfo)))
2891 return failure();
2892 }
2893
2894 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP,
2895 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) {
2896 LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
2897 moduleTranslation, allocaIP.getNodeParent(), deallocBlocks);
2898 builder.restoreIP(IP: codegenIP);
2899 return convertOmpOpRegions(region&: op.getRegion(), blockName: "omp.teams.region", builder,
2900 moduleTranslation)
2901 .takeError();
2902 };
2903
2904 llvm::Value *numTeamsLower = nullptr;
2905 if (Value numTeamsLowerVar = op.getNumTeamsLower())
2906 numTeamsLower = moduleTranslation.lookupValue(value: numTeamsLowerVar);
2907
2908 llvm::Value *numTeamsUpper = nullptr;
2909 if (!op.getNumTeamsUpperVars().empty())
2910 numTeamsUpper = moduleTranslation.lookupValue(value: op.getNumTeams(dim: 0));
2911
2912 llvm::Value *threadLimit = nullptr;
2913 if (!op.getThreadLimitVars().empty())
2914 threadLimit = moduleTranslation.lookupValue(value: op.getThreadLimit(dim: 0));
2915
2916 llvm::Value *ifExpr = nullptr;
2917 if (Value ifVar = op.getIfExpr())
2918 ifExpr = moduleTranslation.lookupValue(value: ifVar);
2919
2920 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
2921 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
2922 getOpenMPBuilder(moduleTranslation)
2923 ->createTeams(Loc: ompLoc, BodyGenCB: bodyCB, NumTeamsLower: numTeamsLower, NumTeamsUpper: numTeamsUpper,
2924 ThreadLimit: threadLimit, IfExpr: ifExpr);
2925
2926 if (failed(Result: handleError(result&: afterIP, op&: *op)))
2927 return failure();
2928
2929 builder.restoreIP(IP: *afterIP);
2930 if (teamsRedInfo) {
2931 // Process the reductions if required.
2932 return createReductionsAndCleanup(op&: *op, builder, moduleTranslation, allocaIP,
2933 redInfo&: *teamsRedInfo,
2934 /*isNoWait=*/isNowait: false,
2935 /*isTeamsReduction=*/true);
2936 }
2937 return success();
2938}
2939
2940static llvm::omp::RTLDependenceKindTy
2941convertDependKind(mlir::omp::ClauseTaskDepend kind) {
2942 switch (kind) {
2943 case mlir::omp::ClauseTaskDepend::taskdependin:
2944 return llvm::omp::RTLDependenceKindTy::DepIn;
2945 // The OpenMP runtime requires that the codegen for 'depend' clause for
2946 // 'out' dependency kind must be the same as codegen for 'depend' clause
2947 // with 'inout' dependency.
2948 case mlir::omp::ClauseTaskDepend::taskdependout:
2949 case mlir::omp::ClauseTaskDepend::taskdependinout:
2950 return llvm::omp::RTLDependenceKindTy::DepInOut;
2951 case mlir::omp::ClauseTaskDepend::taskdependmutexinoutset:
2952 return llvm::omp::RTLDependenceKindTy::DepMutexInOutSet;
2953 case mlir::omp::ClauseTaskDepend::taskdependinoutset:
2954 return llvm::omp::RTLDependenceKindTy::DepInOutSet;
2955 }
2956 llvm_unreachable("unhandled depend kind");
2957}
2958
2959static void buildDependDataLocator(
2960 std::optional<ArrayAttr> dependKinds, OperandRange dependVars,
2961 LLVM::ModuleTranslation &moduleTranslation,
2962 SmallVectorImpl<llvm::OpenMPIRBuilder::DependData> &dds) {
2963 if (dependVars.empty())
2964 return;
2965 for (auto dep : llvm::zip(t&: dependVars, u: dependKinds->getValue())) {
2966 auto kind =
2967 cast<mlir::omp::ClauseTaskDependAttr>(Val: std::get<1>(t&: dep)).getValue();
2968 llvm::omp::RTLDependenceKindTy type = convertDependKind(kind);
2969 llvm::Value *depVal = moduleTranslation.lookupValue(value: std::get<0>(t&: dep));
2970 llvm::OpenMPIRBuilder::DependData dd(type, depVal->getType(), depVal);
2971 dds.emplace_back(Args&: dd);
2972 }
2973}
2974
2975/// Shared implementation of a callback which adds a termiator for the new block
2976/// created for the branch taken when an openmp construct is cancelled. The
2977/// terminator is saved in \p cancelTerminators. This callback is invoked only
2978/// if there is cancellation inside of the taskgroup body.
2979/// The terminator will need to be fixed to branch to the correct block to
2980/// cleanup the construct.
2981static void pushCancelFinalizationCB(
2982 SmallVectorImpl<llvm::UncondBrInst *> &cancelTerminators,
2983 llvm::IRBuilderBase &llvmBuilder, llvm::OpenMPIRBuilder &ompBuilder,
2984 mlir::Operation *op, llvm::omp::Directive cancelDirective) {
2985 auto finiCB = [&](llvm::OpenMPIRBuilder::InsertPointTy ip) -> llvm::Error {
2986 llvm::IRBuilderBase::InsertPointGuard guard(llvmBuilder);
2987
2988 // ip is currently in the block branched to if cancellation occurred.
2989 // We need to create a branch to terminate that block.
2990 llvmBuilder.restoreIP(IP: ip);
2991
2992 // We must still clean up the construct after cancelling it, so we need to
2993 // branch to the block that finalizes the taskgroup.
2994 // That block has not been created yet so use this block as a dummy for now
2995 // and fix this after creating the operation.
2996 cancelTerminators.push_back(Elt: llvmBuilder.CreateBr(Dest: ip.getNodeParent()));
2997 return llvm::Error::success();
2998 };
2999 // We have to add the cleanup to the OpenMPIRBuilder before the body gets
3000 // created in case the body contains omp.cancel (which will then expect to be
3001 // able to find this cleanup callback).
3002 ompBuilder.pushFinalizationCB(
3003 FI: {finiCB, cancelDirective, constructIsCancellable(op)});
3004}
3005
3006/// If we cancelled the construct, we should branch to the finalization block of
3007/// that construct. OMPIRBuilder structures the CFG such that the cleanup block
3008/// is immediately before the continuation block. Now this finalization has
3009/// been created we can fix the branch.
3010static void
3011popCancelFinalizationCB(const ArrayRef<llvm::UncondBrInst *> cancelTerminators,
3012 llvm::OpenMPIRBuilder &ompBuilder,
3013 llvm::BasicBlock *afterBB) {
3014 ompBuilder.popFinalizationCB();
3015 llvm::BasicBlock *constructFini = afterBB->getSinglePredecessor();
3016 for (llvm::UncondBrInst *cancelBranch : cancelTerminators)
3017 cancelBranch->setSuccessor(constructFini);
3018}
3019
3020namespace {
3021/// TaskContextStructManager takes care of creating and freeing a structure
3022/// containing information needed by the task body to execute.
3023class TaskContextStructManager {
3024public:
3025 TaskContextStructManager(llvm::IRBuilderBase &builder,
3026 LLVM::ModuleTranslation &moduleTranslation,
3027 const PrivateVarsInfo &privateVarsInfo)
3028 : builder{builder}, moduleTranslation{moduleTranslation},
3029 privateVarsInfo{privateVarsInfo} {}
3030
3031 /// Creates a heap allocated struct containing space for each private
3032 /// variable. Invariant: privateVarTypes, privateVarsInfo.vars, and the
3033 /// elements of the structure should all have the same order (although
3034 /// privatizers which do not read from the mold argument are skipped).
3035 void generateTaskContextStruct();
3036
3037 /// Create GEPs to access each member of the structure representing a private
3038 /// variable, adding them to llvmPrivateVars. Null values are added where
3039 /// private decls were skipped so that the ordering continues to match the
3040 /// private decls.
3041 void createGEPsToPrivateVars();
3042
3043 /// Given the address of the structure, return a GEP for each private variable
3044 /// in the structure. Null values are added where private decls were skipped
3045 /// so that the ordering continues to match the private decls.
3046 /// Must be called after generateTaskContextStruct().
3047 SmallVector<llvm::Value *>
3048 createGEPsToPrivateVars(llvm::Value *altStructPtr) const;
3049
3050 /// De-allocate the task context structure.
3051 void freeStructPtr();
3052
3053 MutableArrayRef<llvm::Value *> getLLVMPrivateVarGEPs() {
3054 return llvmPrivateVarGEPs;
3055 }
3056
3057 llvm::Value *getStructPtr() { return structPtr; }
3058
3059private:
3060 llvm::IRBuilderBase &builder;
3061 LLVM::ModuleTranslation &moduleTranslation;
3062 const PrivateVarsInfo &privateVarsInfo;
3063
3064 /// The type of each member of the structure, in order.
3065 SmallVector<llvm::Type *> privateVarTypes;
3066
3067 /// LLVM values for each private variable, or null if that private variable is
3068 /// not included in the task context structure
3069 SmallVector<llvm::Value *> llvmPrivateVarGEPs;
3070
3071 /// A pointer to the structure containing context for this task.
3072 llvm::Value *structPtr = nullptr;
3073 /// The type of the structure
3074 llvm::Type *structTy = nullptr;
3075};
3076
3077/// IteratorInfo extracts and prepares loop bounds information from an
3078/// mlir::omp::IteratorOp for lowering to LLVM IR.
3079///
3080/// It computes the per-dimension trip counts and the total linearized trip
3081/// count as i64. Bounds and steps keep their declared type so that the
3082/// physical induction variables can be reconstructed in that type inside the
3083/// loop body.
3084class IteratorInfo {
3085private:
3086 llvm::SmallVector<llvm::Value *> lowerBounds;
3087 llvm::SmallVector<llvm::Value *> upperBounds;
3088 llvm::SmallVector<llvm::Value *> steps;
3089 llvm::SmallVector<llvm::Value *> trips;
3090 unsigned dims;
3091 llvm::Value *totalTrips;
3092
3093public:
3094 IteratorInfo(mlir::omp::IteratorOp itersOp,
3095 mlir::LLVM::ModuleTranslation &moduleTranslation,
3096 llvm::IRBuilderBase &builder) {
3097 dims = itersOp.getLoopLowerBounds().size();
3098 lowerBounds.resize(N: dims);
3099 upperBounds.resize(N: dims);
3100 steps.resize(N: dims);
3101 trips.resize(N: dims);
3102
3103 for (unsigned d = 0; d < dims; ++d) {
3104 llvm::Value *lb =
3105 moduleTranslation.lookupValue(value: itersOp.getLoopLowerBounds()[d]);
3106 llvm::Value *ub =
3107 moduleTranslation.lookupValue(value: itersOp.getLoopUpperBounds()[d]);
3108 llvm::Value *st =
3109 moduleTranslation.lookupValue(value: itersOp.getLoopSteps()[d]);
3110 assert(lb && ub && st &&
3111 "Expect lowerBounds, upperBounds, and steps in IteratorOp");
3112 assert(lb->getType() == ub->getType() && lb->getType() == st->getType() &&
3113 "Expect matching iterator range types");
3114 assert((!llvm::isa<llvm::ConstantInt>(st) ||
3115 !llvm::cast<llvm::ConstantInt>(st)->isZero()) &&
3116 "Expect non-zero step in IteratorOp");
3117
3118 lowerBounds[d] = lb;
3119 upperBounds[d] = ub;
3120 steps[d] = st;
3121
3122 // Use a direction-aware count so an empty range contributes no entries.
3123 // Widen by one bit so the span between the bounds cannot overflow.
3124 llvm::Type *countTy =
3125 builder.getIntNTy(N: lb->getType()->getIntegerBitWidth() + 1);
3126 llvm::Value *start = builder.CreateSExt(V: lb, DestTy: countTy);
3127 llvm::Value *stop = builder.CreateSExt(V: ub, DestTy: countTy);
3128 llvm::Value *step = builder.CreateSExt(V: st, DestTy: countTy);
3129 llvm::Value *count =
3130 getOpenMPBuilder(moduleTranslation)
3131 ->calculateCanonicalLoopTripCount(
3132 Loc: builder, Start: start, Stop: stop, Step: step,
3133 /*IsSigned=*/true, InclusiveStop: itersOp.getLoopInclusive(), Name: "iterator");
3134 trips[d] = builder.CreateZExtOrTrunc(V: count, DestTy: builder.getInt64Ty());
3135 }
3136
3137 totalTrips = llvm::ConstantInt::get(Ty: builder.getInt64Ty(), V: 1);
3138 for (unsigned d = 0; d < dims; ++d)
3139 totalTrips = builder.CreateMul(LHS: totalTrips, RHS: trips[d]);
3140 }
3141
3142 unsigned getDims() const { return dims; }
3143 llvm::ArrayRef<llvm::Value *> getLowerBounds() const { return lowerBounds; }
3144 llvm::ArrayRef<llvm::Value *> getUpperBounds() const { return upperBounds; }
3145 llvm::ArrayRef<llvm::Value *> getSteps() const { return steps; }
3146 llvm::ArrayRef<llvm::Value *> getTrips() const { return trips; }
3147 llvm::Value *getTotalTrips() const { return totalTrips; }
3148};
3149} // namespace
3150
3151void TaskContextStructManager::generateTaskContextStruct() {
3152 if (privateVarsInfo.vars.empty())
3153 return;
3154 privateVarTypes.reserve(N: privateVarsInfo.vars.size());
3155
3156 for (const PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) {
3157 omp::PrivateClauseOp privOp = var.privatizer;
3158 // Skip private variables which can safely be allocated and initialised
3159 // inside of the task
3160 if (!privOp.readsFromMold())
3161 continue;
3162 Type mlirType = privOp.getType();
3163 privateVarTypes.push_back(Elt: moduleTranslation.convertType(type: mlirType));
3164 }
3165
3166 if (privateVarTypes.empty())
3167 return;
3168
3169 structTy = llvm::StructType::get(Context&: moduleTranslation.getLLVMContext(),
3170 Elements: privateVarTypes);
3171
3172 llvm::DataLayout dataLayout = builder.getDataLayout();
3173 llvm::Type *intPtrTy = builder.getIntPtrTy(DL: dataLayout);
3174 llvm::Value *allocSize =
3175 builder.CreateTypeSize(Ty: intPtrTy, Size: dataLayout.getTypeAllocSize(Ty: structTy));
3176
3177 // Heap allocate the structure
3178 structPtr = builder.CreateMalloc(IntPtrTy: intPtrTy, AllocSize: allocSize,
3179 /*ArraySize=*/nullptr, /*MallocF=*/nullptr,
3180 Name: "omp.task.context_ptr");
3181}
3182
3183SmallVector<llvm::Value *> TaskContextStructManager::createGEPsToPrivateVars(
3184 llvm::Value *altStructPtr) const {
3185 SmallVector<llvm::Value *> ret;
3186
3187 // Create GEPs for each struct member
3188 ret.reserve(N: privateVarsInfo.vars.size());
3189 llvm::Value *zero = builder.getInt32(C: 0);
3190 unsigned i = 0;
3191 for (const PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) {
3192 omp::PrivateClauseOp privDecl = var.privatizer;
3193 if (!privDecl.readsFromMold()) {
3194 // Handle this inside of the task so we don't pass unnessecary vars in
3195 ret.push_back(Elt: nullptr);
3196 continue;
3197 }
3198 llvm::Value *iVal = builder.getInt32(C: i);
3199 llvm::Value *gep = builder.CreateGEP(Ty: structTy, Ptr: altStructPtr, IdxList: {zero, iVal});
3200 ret.push_back(Elt: gep);
3201 i += 1;
3202 }
3203 return ret;
3204}
3205
3206void TaskContextStructManager::createGEPsToPrivateVars() {
3207 if (!structPtr)
3208 assert(privateVarTypes.empty());
3209 // Still need to run createGEPsToPrivateVars to populate llvmPrivateVarGEPs
3210 // with null values for skipped private decls
3211
3212 llvmPrivateVarGEPs = createGEPsToPrivateVars(altStructPtr: structPtr);
3213}
3214
3215void TaskContextStructManager::freeStructPtr() {
3216 if (!structPtr)
3217 return;
3218
3219 llvm::IRBuilderBase::InsertPointGuard guard{builder};
3220 // Ensure we don't put the call to free() after the terminator
3221 builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
3222 builder.CreateFree(Source: structPtr);
3223}
3224
3225static void storeAffinityEntry(llvm::IRBuilderBase &builder,
3226 llvm::OpenMPIRBuilder &ompBuilder,
3227 llvm::Value *affinityList, llvm::Value *index,
3228 llvm::Value *addr, llvm::Value *len) {
3229 llvm::StructType *kmpTaskAffinityInfoTy =
3230 ompBuilder.getKmpTaskAffinityInfoTy();
3231 llvm::Value *entry = builder.CreateInBoundsGEP(
3232 Ty: kmpTaskAffinityInfoTy, Ptr: affinityList, IdxList: index, Name: "omp.affinity.entry");
3233
3234 addr = builder.CreatePtrToInt(V: addr, DestTy: kmpTaskAffinityInfoTy->getElementType(N: 0));
3235 len = builder.CreateIntCast(V: len, DestTy: kmpTaskAffinityInfoTy->getElementType(N: 1),
3236 /*isSigned=*/false);
3237 llvm::Value *flags = builder.getInt32(C: 0);
3238
3239 builder.CreateStore(Val: addr,
3240 Ptr: builder.CreateStructGEP(Ty: kmpTaskAffinityInfoTy, Ptr: entry, Idx: 0));
3241 builder.CreateStore(Val: len,
3242 Ptr: builder.CreateStructGEP(Ty: kmpTaskAffinityInfoTy, Ptr: entry, Idx: 1));
3243 builder.CreateStore(Val: flags,
3244 Ptr: builder.CreateStructGEP(Ty: kmpTaskAffinityInfoTy, Ptr: entry, Idx: 2));
3245}
3246
3247static void fillAffinityLocators(Operation::operand_range affinityVars,
3248 llvm::IRBuilderBase &builder,
3249 LLVM::ModuleTranslation &moduleTranslation,
3250 llvm::Value *affinityList) {
3251 for (auto [i, affinityVar] : llvm::enumerate(First&: affinityVars)) {
3252 auto entryOp = affinityVar.getDefiningOp<mlir::omp::AffinityEntryOp>();
3253 assert(entryOp && "affinity item must be omp.affinity_entry");
3254
3255 llvm::Value *addr = moduleTranslation.lookupValue(value: entryOp.getAddr());
3256 llvm::Value *len = moduleTranslation.lookupValue(value: entryOp.getLen());
3257 assert(addr && len && "expect affinity addr and len to be non-null");
3258 storeAffinityEntry(builder, ompBuilder&: *getOpenMPBuilder(moduleTranslation),
3259 affinityList, index: builder.getInt64(C: i), addr, len);
3260 }
3261}
3262
3263static mlir::LogicalResult convertIteratorRegion(
3264 llvm::Value *linearIV, IteratorInfo &iterInfo, mlir::Region &iteratorRegion,
3265 llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) {
3266 mlir::Block &entryBlock = iteratorRegion.front();
3267 llvm::Value *tmp = linearIV;
3268 for (int d = (int)iterInfo.getDims() - 1; d >= 0; --d) {
3269 llvm::Value *trip = iterInfo.getTrips()[d];
3270 llvm::Value *zero = llvm::ConstantInt::get(Ty: trip->getType(), V: 0);
3271 llvm::Value *one = llvm::ConstantInt::get(Ty: trip->getType(), V: 1);
3272 // A zero trip count makes the linearized loop empty. Use one in its
3273 // unreachable IV decoder so the generated IR contains no division by zero.
3274 llvm::Value *divisor =
3275 builder.CreateSelect(C: builder.CreateICmpEQ(LHS: trip, RHS: zero), True: one, False: trip);
3276 // idx_d = tmp % divisor_d
3277 llvm::Value *idx = builder.CreateURem(LHS: tmp, RHS: divisor);
3278 // tmp = tmp / divisor_d
3279 tmp = builder.CreateUDiv(LHS: tmp, RHS: divisor);
3280
3281 // physIV_d = lb_d + idx_d * step_d
3282 idx =
3283 builder.CreateZExtOrTrunc(V: idx, DestTy: iterInfo.getLowerBounds()[d]->getType());
3284 llvm::Value *physIV = builder.CreateAdd(
3285 LHS: iterInfo.getLowerBounds()[d],
3286 RHS: builder.CreateMul(LHS: idx, RHS: iterInfo.getSteps()[d]), Name: "omp.it.phys_iv");
3287
3288 moduleTranslation.mapValue(mlir: entryBlock.getArgument(i: d), llvm: physIV);
3289 }
3290
3291 // Translate the iterator region into the loop body.
3292 if (iteratorRegion.hasOneBlock()) {
3293 moduleTranslation.mapBlock(mlir: &entryBlock, llvm: builder.GetInsertBlock());
3294 return moduleTranslation.convertBlock(bb&: entryBlock, /*ignoreArguments=*/true,
3295 builder);
3296 }
3297
3298 // Unlike convertOmpOpRegions, do not forward the yielded value through a PHI:
3299 // it may have no LLVM type (e.g. `!omp.affinity_entry_ty`). The unique
3300 // `omp.yield` block is the continuation's only predecessor, so callers can
3301 // look up the yielded value directly.
3302 llvm::BasicBlock *continuation =
3303 splitBB(Builder&: builder, CreateBranch: true, Name: "omp.iterator.region.cont");
3304 llvm::Instruction *sourceBranch = builder.GetInsertBlock()->getTerminator();
3305 for (mlir::Block &block : iteratorRegion)
3306 moduleTranslation.mapBlock(
3307 mlir: &block,
3308 llvm: llvm::BasicBlock::Create(Context&: builder.getContext(), Name: "omp.iterator.region",
3309 Parent: continuation->getParent(), InsertBefore: continuation));
3310 sourceBranch->setSuccessor(Idx: 0, BB: moduleTranslation.lookupBlock(block: &entryBlock));
3311
3312 for (mlir::Block *block : getBlocksSortedByDominance(region&: iteratorRegion)) {
3313 llvm::IRBuilderBase::InsertPointGuard guard(builder);
3314 if (failed(Result: moduleTranslation.convertBlock(bb&: *block, ignoreArguments: block->isEntryBlock(),
3315 builder)))
3316 return mlir::failure();
3317 if (isa<omp::YieldOp>(Val: block->getTerminator()))
3318 builder.CreateBr(Dest: continuation);
3319 }
3320 LLVM::detail::connectPHINodes(region&: iteratorRegion, state: moduleTranslation);
3321
3322 builder.SetInsertPoint(continuation);
3323 return mlir::success();
3324}
3325
3326using IteratorStoreEntryTy =
3327 llvm::function_ref<void(llvm::Value *linearIV, mlir::omp::YieldOp yield)>;
3328
3329static mlir::LogicalResult
3330fillIteratorLoop(mlir::omp::IteratorOp itersOp, llvm::IRBuilderBase &builder,
3331 mlir::LLVM::ModuleTranslation &moduleTranslation,
3332 IteratorInfo &iterInfo, llvm::StringRef loopName,
3333 IteratorStoreEntryTy genStoreEntry) {
3334 mlir::Region &itersRegion = itersOp.getRegion();
3335
3336 llvm::OpenMPIRBuilder::LocationDescription loc(builder);
3337
3338 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy bodyIP,
3339 llvm::Value *linearIV) -> llvm::Error {
3340 llvm::IRBuilderBase::InsertPointGuard guard(builder);
3341 builder.restoreIP(IP: bodyIP);
3342
3343 if (failed(Result: convertIteratorRegion(linearIV, iterInfo, iteratorRegion&: itersRegion, builder,
3344 moduleTranslation))) {
3345 return llvm::make_error<llvm::StringError>(
3346 Args: "failed to convert iterator region", Args: llvm::inconvertibleErrorCode());
3347 }
3348
3349 mlir::omp::YieldOp yield = itersOp.getYieldOp();
3350 assert(yield && yield.getResults().size() == 1 &&
3351 "expect omp.yield in iterator region to have one result");
3352
3353 genStoreEntry(linearIV, yield);
3354
3355 // Iterator-region block/value mappings are temporary for this conversion,
3356 // clear them to avoid stale entries in ModuleTranslation.
3357 moduleTranslation.forgetMapping(region&: itersRegion);
3358
3359 return llvm::Error::success();
3360 };
3361
3362 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
3363 getOpenMPBuilder(moduleTranslation)
3364 ->createIteratorLoop(Loc: loc, TripCount: iterInfo.getTotalTrips(), BodyGen: bodyGen,
3365 Name: loopName);
3366 if (failed(Result: handleError(result&: afterIP, op&: *itersOp)))
3367 return failure();
3368
3369 builder.restoreIP(IP: *afterIP);
3370
3371 return mlir::success();
3372}
3373
3374static mlir::LogicalResult
3375buildAffinityData(mlir::omp::TaskOp &taskOp, llvm::IRBuilderBase &builder,
3376 mlir::LLVM::ModuleTranslation &moduleTranslation,
3377 llvm::OpenMPIRBuilder::AffinityData &ad) {
3378
3379 if (taskOp.getAffinityVars().empty() && taskOp.getIterated().empty()) {
3380 ad.Count = nullptr;
3381 ad.Info = nullptr;
3382 return mlir::success();
3383 }
3384
3385 llvm::SmallVector<llvm::OpenMPIRBuilder::AffinityData> ads;
3386 llvm::StructType *kmpTaskAffinityInfoTy =
3387 getOpenMPBuilder(moduleTranslation)->getKmpTaskAffinityInfoTy();
3388
3389 auto allocateAffinityList = [&](llvm::Value *count) -> llvm::Value * {
3390 llvm::IRBuilderBase::InsertPointGuard guard(builder);
3391 if (llvm::isa<llvm::Constant>(Val: count) || llvm::isa<llvm::Argument>(Val: count))
3392 builder.restoreIP(IP: findAllocInsertPoints(builder, moduleTranslation));
3393 return builder.CreateAlloca(Ty: kmpTaskAffinityInfoTy, ArraySize: count,
3394 Name: "omp.affinity_list");
3395 };
3396
3397 auto createAffinity =
3398 [&](llvm::Value *count,
3399 llvm::Value *info) -> llvm::OpenMPIRBuilder::AffinityData {
3400 llvm::OpenMPIRBuilder::AffinityData ad{};
3401 ad.Count = builder.CreateTrunc(V: count, DestTy: builder.getInt32Ty());
3402 ad.Info =
3403 builder.CreatePointerBitCastOrAddrSpaceCast(V: info, DestTy: builder.getPtrTy(AddrSpace: 0));
3404 return ad;
3405 };
3406
3407 if (!taskOp.getAffinityVars().empty()) {
3408 llvm::Value *count = llvm::ConstantInt::get(
3409 Ty: builder.getInt64Ty(), V: taskOp.getAffinityVars().size());
3410 llvm::Value *list = allocateAffinityList(count);
3411 fillAffinityLocators(affinityVars: taskOp.getAffinityVars(), builder, moduleTranslation,
3412 affinityList: list);
3413 ads.emplace_back(Args: createAffinity(count, list));
3414 }
3415
3416 if (!taskOp.getIterated().empty()) {
3417 for (auto [i, iter] : llvm::enumerate(First: taskOp.getIterated())) {
3418 auto itersOp = iter.getDefiningOp<omp::IteratorOp>();
3419 assert(itersOp && "iterated value must be defined by omp.iterator");
3420 IteratorInfo iterInfo(itersOp, moduleTranslation, builder);
3421 llvm::Value *affList = allocateAffinityList(iterInfo.getTotalTrips());
3422 if (failed(Result: fillIteratorLoop(
3423 itersOp, builder, moduleTranslation, iterInfo, loopName: "iterator",
3424 genStoreEntry: [&](llvm::Value *linearIV, mlir::omp::YieldOp yield) {
3425 auto entryOp = yield.getResults()[0]
3426 .getDefiningOp<mlir::omp::AffinityEntryOp>();
3427 assert(entryOp && "expect yield produce an affinity entry");
3428 llvm::Value *addr =
3429 moduleTranslation.lookupValue(value: entryOp.getAddr());
3430 llvm::Value *len =
3431 moduleTranslation.lookupValue(value: entryOp.getLen());
3432 storeAffinityEntry(builder,
3433 ompBuilder&: *getOpenMPBuilder(moduleTranslation),
3434 affinityList: affList, index: linearIV, addr, len);
3435 })))
3436 return llvm::failure();
3437 ads.emplace_back(Args: createAffinity(iterInfo.getTotalTrips(), affList));
3438 }
3439 }
3440
3441 llvm::Value *totalAffinityCount = builder.getInt32(C: 0);
3442 for (const auto &affinity : ads)
3443 totalAffinityCount = builder.CreateAdd(
3444 LHS: totalAffinityCount,
3445 RHS: builder.CreateIntCast(V: affinity.Count, DestTy: builder.getInt32Ty(),
3446 /*isSigned=*/false));
3447
3448 llvm::Value *affinityInfo = ads.front().Info;
3449 if (ads.size() > 1) {
3450 llvm::StructType *kmpTaskAffinityInfoTy =
3451 getOpenMPBuilder(moduleTranslation)->getKmpTaskAffinityInfoTy();
3452 llvm::Value *affinityInfoElemSize = builder.getInt64(
3453 C: moduleTranslation.getLLVMModule()->getDataLayout().getTypeAllocSize(
3454 Ty: kmpTaskAffinityInfoTy));
3455
3456 llvm::Value *packedAffinityInfo = allocateAffinityList(totalAffinityCount);
3457 llvm::Value *packedAffinityInfoOffset = builder.getInt32(C: 0);
3458 for (const auto &affinity : ads) {
3459 llvm::Value *affinityCount = builder.CreateIntCast(
3460 V: affinity.Count, DestTy: builder.getInt32Ty(), /*isSigned=*/false);
3461 llvm::Value *affinityCountInt64 = builder.CreateIntCast(
3462 V: affinityCount, DestTy: builder.getInt64Ty(), /*isSigned=*/false);
3463 llvm::Value *affinityInfoSize =
3464 builder.CreateMul(LHS: affinityCountInt64, RHS: affinityInfoElemSize);
3465
3466 llvm::Value *packedAffinityInfoIndex = builder.CreateIntCast(
3467 V: packedAffinityInfoOffset, DestTy: kmpTaskAffinityInfoTy->getElementType(N: 0),
3468 /*isSigned=*/false);
3469 packedAffinityInfoIndex = builder.CreateInBoundsGEP(
3470 Ty: kmpTaskAffinityInfoTy, Ptr: packedAffinityInfo, IdxList: packedAffinityInfoIndex);
3471
3472 builder.CreateMemCpy(
3473 Dst: packedAffinityInfoIndex, DstAlign: llvm::Align(1),
3474 Src: builder.CreatePointerBitCastOrAddrSpaceCast(
3475 V: affinity.Info, DestTy: builder.getPtrTy(AddrSpace: packedAffinityInfoIndex->getType()
3476 ->getPointerAddressSpace())),
3477 SrcAlign: llvm::Align(1), Size: affinityInfoSize);
3478
3479 packedAffinityInfoOffset =
3480 builder.CreateAdd(LHS: packedAffinityInfoOffset, RHS: affinityCount);
3481 }
3482
3483 affinityInfo = packedAffinityInfo;
3484 }
3485
3486 ad.Count = totalAffinityCount;
3487 ad.Info = affinityInfo;
3488
3489 return mlir::success();
3490}
3491
3492// Allocates a single kmp_dep_info array sized to hold both locator
3493// (non-iterated) and iterated entries, fills the locator entries first, then
3494// runs an iterator loop for each iterator modifier object.
3495static mlir::LogicalResult
3496buildDependData(OperandRange dependVars, std::optional<ArrayAttr> dependKinds,
3497 OperandRange dependIterated,
3498 std::optional<ArrayAttr> dependIteratedKinds,
3499 llvm::IRBuilderBase &builder,
3500 mlir::LLVM::ModuleTranslation &moduleTranslation,
3501 llvm::OpenMPIRBuilder::DependenciesInfo &taskDeps) {
3502 if (dependIterated.empty()) {
3503 buildDependDataLocator(dependKinds, dependVars, moduleTranslation,
3504 dds&: taskDeps.Deps);
3505 return mlir::success();
3506 }
3507
3508 llvm::OpenMPIRBuilder &ompBuilder = *getOpenMPBuilder(moduleTranslation);
3509 llvm::Type *dependInfoTy = ompBuilder.DependInfo;
3510 unsigned numLocator = dependVars.size();
3511
3512 // Compute total count: locator deps + sum of iterator trip counts.
3513 llvm::Value *totalCount =
3514 llvm::ConstantInt::get(Ty: builder.getInt64Ty(), V: numLocator);
3515
3516 llvm::SmallVector<IteratorInfo, 2> iterInfos;
3517 for (auto iter : dependIterated) {
3518 auto itersOp = iter.getDefiningOp<mlir::omp::IteratorOp>();
3519 assert(itersOp && "depend_iterated value must be defined by omp.iterator");
3520 iterInfos.emplace_back(Args&: itersOp, Args&: moduleTranslation, Args&: builder);
3521 totalCount =
3522 builder.CreateAdd(LHS: totalCount, RHS: iterInfos.back().getTotalTrips());
3523 }
3524
3525 // Heap-allocate the kmp_depend_info array so we don't risk
3526 // dynamic-sized alloca outside the entry block (e.g. inside loops).
3527 llvm::DataLayout dataLayout = builder.getDataLayout();
3528 llvm::Value *allocSize = builder.CreateTypeSize(
3529 Ty: ompBuilder.SizeTy, Size: dataLayout.getTypeAllocSize(Ty: dependInfoTy));
3530 llvm::Value *depArray =
3531 builder.CreateMalloc(IntPtrTy: ompBuilder.SizeTy, AllocSize: allocSize, ArraySize: totalCount,
3532 /*MallocF=*/nullptr, Name: ".dep.arr.addr");
3533
3534 // Fill non-iterated entries at indices [0, numLocator).
3535 if (numLocator > 0) {
3536 SmallVector<llvm::OpenMPIRBuilder::DependData> dds;
3537 buildDependDataLocator(dependKinds, dependVars, moduleTranslation, dds);
3538 for (auto [i, dd] : llvm::enumerate(First&: dds)) {
3539 llvm::Value *idx = llvm::ConstantInt::get(Ty: builder.getInt64Ty(), V: i);
3540 llvm::Value *entry =
3541 builder.CreateInBoundsGEP(Ty: dependInfoTy, Ptr: depArray, IdxList: idx);
3542 ompBuilder.emitTaskDependency(Builder&: builder, Entry: entry, Dep: dd);
3543 }
3544 }
3545
3546 // Fill iterated entries starting at index numLocator.
3547 llvm::Value *offset =
3548 llvm::ConstantInt::get(Ty: builder.getInt64Ty(), V: numLocator);
3549 for (auto [i, iterInfo] : llvm::enumerate(First&: iterInfos)) {
3550 auto kindAttr = cast<mlir::omp::ClauseTaskDependAttr>(
3551 Val: dependIteratedKinds->getValue()[i]);
3552 llvm::omp::RTLDependenceKindTy rtlKind =
3553 convertDependKind(kind: kindAttr.getValue());
3554
3555 auto itersOp = dependIterated[i].getDefiningOp<mlir::omp::IteratorOp>();
3556 if (failed(Result: fillIteratorLoop(
3557 itersOp, builder, moduleTranslation, iterInfo, loopName: "dep_iterator",
3558 genStoreEntry: [&](llvm::Value *linearIV, mlir::omp::YieldOp yield) {
3559 llvm::Value *addr =
3560 moduleTranslation.lookupValue(value: yield.getResults()[0]);
3561 llvm::Value *idx = builder.CreateAdd(LHS: offset, RHS: linearIV);
3562 llvm::Value *entry =
3563 builder.CreateInBoundsGEP(Ty: dependInfoTy, Ptr: depArray, IdxList: idx);
3564 ompBuilder.emitTaskDependency(
3565 Builder&: builder, Entry: entry,
3566 Dep: llvm::OpenMPIRBuilder::DependData{rtlKind, addr->getType(),
3567 addr});
3568 })))
3569 return mlir::failure();
3570
3571 // Advance offset by the trip count of this iterator.
3572 offset = builder.CreateAdd(LHS: offset, RHS: iterInfo.getTotalTrips());
3573 }
3574
3575 taskDeps.DepArray = depArray;
3576 taskDeps.NumDeps = builder.CreateTrunc(V: totalCount, DestTy: builder.getInt32Ty());
3577 return mlir::success();
3578}
3579
3580/// Converts an OpenMP task construct into LLVM IR using OpenMPIRBuilder.
3581static LogicalResult
3582convertOmpTaskOp(omp::TaskOp taskOp, llvm::IRBuilderBase &builder,
3583 LLVM::ModuleTranslation &moduleTranslation) {
3584 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
3585 if (failed(Result: checkImplementationStatus(op&: *taskOp)))
3586 return failure();
3587
3588 PrivateVarsInfo privateVarsInfo(taskOp);
3589 TaskContextStructManager taskStructMgr{builder, moduleTranslation,
3590 privateVarsInfo};
3591
3592 // Allocate and copy private variables before creating the task. This avoids
3593 // accessing invalid memory if (after this scope ends) the private variables
3594 // are initialized from host variables or if the variables are copied into
3595 // from host variables (firstprivate). The insertion point is just before
3596 // where the code for creating and scheduling the task will go. That puts this
3597 // code outside of the outlined task region, which is what we want because
3598 // this way the initialization and copy regions are executed immediately while
3599 // the host variable data are still live.
3600 llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
3601 InsertPointTy allocaIP =
3602 findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks);
3603
3604 // Not using splitBB() because that requires the current block to have a
3605 // terminator.
3606 assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end());
3607 llvm::BasicBlock *taskStartBlock = llvm::BasicBlock::Create(
3608 Context&: builder.getContext(), Name: "omp.task.start",
3609 /*Parent=*/builder.GetInsertBlock()->getParent());
3610 llvm::Instruction *branchToTaskStartBlock = builder.CreateBr(Dest: taskStartBlock);
3611 builder.SetInsertPoint(branchToTaskStartBlock);
3612
3613 // Now do this again to make the initialization and copy blocks
3614 llvm::BasicBlock *copyBlock =
3615 splitBB(Builder&: builder, /*CreateBranch=*/true, Name: "omp.private.copy");
3616 llvm::BasicBlock *initBlock =
3617 splitBB(Builder&: builder, /*CreateBranch=*/true, Name: "omp.private.init");
3618
3619 // Now the control flow graph should look like
3620 // starter_block:
3621 // <---- where we started when convertOmpTaskOp was called
3622 // br %omp.private.init
3623 // omp.private.init:
3624 // br %omp.private.copy
3625 // omp.private.copy:
3626 // br %omp.task.start
3627 // omp.task.start:
3628 // <---- where we want the insertion point to be when we call createTask()
3629
3630 // Save the alloca insertion point on ModuleTranslation stack for use in
3631 // nested regions.
3632 LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
3633 moduleTranslation, allocaIP.getNodeParent(), deallocBlocks);
3634
3635 // Allocate and initialize private variables
3636 builder.SetInsertPoint(initBlock->getTerminator());
3637
3638 // Create task variable structure
3639 taskStructMgr.generateTaskContextStruct();
3640 // GEPs so that we can initialize the variables. Don't use these GEPs inside
3641 // of the body otherwise it will be the GEP not the struct which is fowarded
3642 // to the outlined function. GEPs forwarded in this way are passed in a
3643 // stack-allocated (by OpenMPIRBuilder) structure which is not safe for tasks
3644 // which may not be executed until after the current stack frame goes out of
3645 // scope.
3646 taskStructMgr.createGEPsToPrivateVars();
3647
3648 for (auto [var, llvmPrivateVarAlloc] : llvm::zip_equal(
3649 t&: privateVarsInfo.vars, u: taskStructMgr.getLLVMPrivateVarGEPs())) {
3650 // To be handled inside the task.
3651 if (!var.privatizer.readsFromMold())
3652 continue;
3653 assert(llvmPrivateVarAlloc &&
3654 "reads from mold so shouldn't have been skipped");
3655
3656 llvm::Expected<llvm::Value *> privateVarOrErr =
3657 initPrivateVar(builder, moduleTranslation, privDecl&: var.privatizer, mlirPrivVar: var.mlirVar,
3658 blockArg&: var.blockArg, llvmPrivateVar: llvmPrivateVarAlloc, privInitBlock: initBlock);
3659 if (!privateVarOrErr)
3660 return handleError(result&: privateVarOrErr, op&: *taskOp.getOperation());
3661
3662 setInsertPointForPossiblyEmptyBlock(builder);
3663
3664 // TODO: this is a bit of a hack for Fortran character boxes.
3665 // Character boxes are passed by value into the init region and then the
3666 // initialized character box is yielded by value. Here we need to store the
3667 // yielded value into the private allocation, and load the private
3668 // allocation to match the type expected by region block arguments.
3669 [[maybe_unused]] llvm::Value *llvmPrivateVar = llvmPrivateVarAlloc;
3670 if ((privateVarOrErr.get() != llvmPrivateVarAlloc) &&
3671 !mlir::isa<LLVM::LLVMPointerType>(Val: var.blockArg.getType())) {
3672 builder.CreateStore(Val: privateVarOrErr.get(), Ptr: llvmPrivateVarAlloc);
3673 // Load it so we have the value pointed to by the GEP
3674 llvmPrivateVar = builder.CreateLoad(Ty: privateVarOrErr.get()->getType(),
3675 Ptr: llvmPrivateVarAlloc);
3676 }
3677 assert(llvmPrivateVar->getType() ==
3678 moduleTranslation.convertType(var.blockArg.getType()));
3679
3680 // Mapping blockArg -> llvmPrivateVarAlloc is done inside the body callback
3681 // so that OpenMPIRBuilder doesn't try to pass each GEP address through a
3682 // stack allocated structure.
3683 }
3684
3685 // firstprivate copy region
3686 setInsertPointForPossiblyEmptyBlock(builder, block: copyBlock);
3687 if (failed(Result: completePrivateVars(
3688 op: taskOp, builder, moduleTranslation, privateVarsInfo,
3689 insertBarrier: taskOp.getPrivateNeedsBarrier(),
3690 /*moldVars=*/{}, llvmPrivateVars: taskStructMgr.getLLVMPrivateVarGEPs())))
3691 return llvm::failure();
3692
3693 llvm::OpenMPIRBuilder::AffinityData ad;
3694 if (failed(Result: buildAffinityData(taskOp, builder, moduleTranslation, ad)))
3695 return llvm::failure();
3696
3697 // Resolve and validate in_reduction declarations. Byref in_reduction has
3698 // already been rejected by checkImplementationStatus; the helper rejects the
3699 // remaining richer declare_reduction shapes (two-argument initializer,
3700 // cleanup region, missing combiner). This is pure MLIR symbol-table work and
3701 // emits no IR. The matching task_reduction descriptor is registered by an
3702 // enclosing taskgroup; here we only look the per-task storage up at runtime.
3703 SmallVector<omp::DeclareReductionOp> inRedDecls;
3704 if (failed(Result: collectAndValidateTaskloopRedDecls(
3705 contextOp: taskOp.getOperation(), syms: taskOp.getInReductionSyms(), opName: "omp.task",
3706 clauseName: "in_reduction", out&: inRedDecls)))
3707 return failure();
3708 SmallVector<llvm::Value *> inRedOrigPtrs;
3709 inRedOrigPtrs.reserve(N: inRedDecls.size());
3710 for (Value v : taskOp.getInReductionVars())
3711 inRedOrigPtrs.push_back(Elt: moduleTranslation.lookupValue(value: v));
3712
3713 // Set up for call to createTask()
3714 builder.SetInsertPoint(taskStartBlock);
3715
3716 auto bodyCB =
3717 [&](InsertPointTy allocaIP, InsertPointTy codegenIP,
3718 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
3719 // Save the alloca insertion point on ModuleTranslation stack for use in
3720 // nested regions.
3721 LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
3722 moduleTranslation, allocaIP.getNodeParent(), deallocBlocks);
3723
3724 // translate the body of the task:
3725 builder.restoreIP(IP: codegenIP);
3726
3727 llvm::BasicBlock *privInitBlock = nullptr;
3728 for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) {
3729 // This is handled before the task executes
3730 if (var.privatizer.readsFromMold())
3731 continue;
3732
3733 llvm::IRBuilderBase::InsertPointGuard guard(builder);
3734 llvm::Type *llvmAllocType =
3735 moduleTranslation.convertType(type: var.privatizer.getType());
3736 builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator());
3737 llvm::Value *llvmPrivateVar = builder.CreateAlloca(
3738 Ty: llvmAllocType, /*ArraySize=*/nullptr, Name: "omp.private.alloc");
3739
3740 llvm::Expected<llvm::Value *> privateVarOrError = initPrivateVar(
3741 builder, moduleTranslation, privDecl&: var.privatizer, mlirPrivVar: var.mlirVar, blockArg&: var.blockArg,
3742 llvmPrivateVar, privInitBlock);
3743 if (!privateVarOrError)
3744 return privateVarOrError.takeError();
3745 moduleTranslation.mapValue(mlir: var.blockArg, llvm: privateVarOrError.get());
3746 var.llvmVar = privateVarOrError.get();
3747 }
3748
3749 taskStructMgr.createGEPsToPrivateVars();
3750 for (auto [i, llvmPrivVar] :
3751 llvm::enumerate(First: taskStructMgr.getLLVMPrivateVarGEPs())) {
3752 if (!llvmPrivVar) {
3753 assert(privateVarsInfo.vars[i].llvmVar &&
3754 "This is added in the loop above");
3755 continue;
3756 }
3757 privateVarsInfo.vars[i].llvmVar = llvmPrivVar;
3758 }
3759
3760 // Find and map the addresses of each variable within the task context
3761 // structure
3762 for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) {
3763 // This was handled above.
3764 if (!var.privatizer.readsFromMold())
3765 continue;
3766 // Fix broken pass-by-value case for Fortran character boxes
3767 if (!mlir::isa<LLVM::LLVMPointerType>(Val: var.blockArg.getType())) {
3768 var.llvmVar = builder.CreateLoad(
3769 Ty: moduleTranslation.convertType(type: var.blockArg.getType()), Ptr: var.llvmVar);
3770 }
3771 assert(var.llvmVar->getType() ==
3772 moduleTranslation.convertType(var.blockArg.getType()));
3773 moduleTranslation.mapValue(mlir: var.blockArg, llvm: var.llvmVar);
3774 }
3775
3776 // Map in_reduction block arguments to the per-task private storage returned
3777 // by __kmpc_task_reduction_get_th_data. This call must be emitted inside
3778 // the to-be-outlined task body so that it returns the *executing* thread's
3779 // gtid (not the encountering thread's). The descriptor is NULL: the runtime
3780 // walks up enclosing taskgroups to find the matching task_reduction
3781 // registration for `origPtr`. The original pointers are auto-captured into
3782 // the task shareds aggregate by CodeExtractor during
3783 // OpenMPIRBuilder::finalize.
3784 if (!inRedDecls.empty()) {
3785 auto iface = cast<omp::BlockArgOpenMPOpInterface>(Val: taskOp.getOperation());
3786 llvm::OpenMPIRBuilder &ompB = *getOpenMPBuilder(moduleTranslation);
3787 llvm::Module *m = moduleTranslation.getLLVMModule();
3788 llvm::LLVMContext &llvmCtx = m->getContext();
3789 llvm::OpenMPIRBuilder::LocationDescription bodyLoc(builder);
3790 uint32_t srcLocSize;
3791 llvm::Constant *srcLocStr =
3792 ompB.getOrCreateSrcLocStr(Loc: bodyLoc, SrcLocStrSize&: srcLocSize);
3793 llvm::Value *bodyIdent = ompB.getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize);
3794 // Align OpenMPIRBuilder's internal IRBuilder with `builder` so the gtid
3795 // call lands inside the to-be-outlined task body.
3796 ompB.updateToLocation(Loc: bodyLoc);
3797 llvm::Value *bodyGtid = ompB.getOrCreateThreadID(Ident: bodyIdent);
3798 llvm::FunctionCallee getThData = ompB.getOrCreateRuntimeFunction(
3799 M&: *m, FnID: llvm::omp::OMPRTL___kmpc_task_reduction_get_th_data);
3800 llvm::Type *ptrTy = llvm::PointerType::getUnqual(C&: llvmCtx);
3801 llvm::Value *nullDesc = llvm::ConstantPointerNull::get(T: ptrTy);
3802 ArrayRef<BlockArgument> inRedBlockArgs = iface.getInReductionBlockArgs();
3803 for (auto [blockArg, origPtr] :
3804 llvm::zip_equal(t&: inRedBlockArgs, u&: inRedOrigPtrs)) {
3805 // __kmpc_task_reduction_get_th_data takes and returns a generic,
3806 // default-address-space `ptr`. Normalize a non-default-address-space
3807 // original pointer to the generic address space before the call, and
3808 // cast the returned private pointer back to the block argument's
3809 // address space when it differs (mirrors the taskloop reduction
3810 // remapping in convertOmpTaskloopContextOp).
3811 llvm::Value *lookupPtr = origPtr;
3812 if (auto *origPtrTy =
3813 llvm::dyn_cast<llvm::PointerType>(Val: lookupPtr->getType());
3814 origPtrTy && origPtrTy->getAddressSpace() != 0)
3815 lookupPtr = builder.CreateAddrSpaceCast(V: lookupPtr, DestTy: ptrTy);
3816 llvm::Value *priv = builder.CreateCall(
3817 Callee: getThData, Args: {bodyGtid, nullDesc, lookupPtr}, Name: "omp.inred.priv");
3818 if (auto *argPtrTy = llvm::dyn_cast<llvm::PointerType>(
3819 Val: moduleTranslation.convertType(type: blockArg.getType()));
3820 argPtrTy && argPtrTy->getAddressSpace() != 0)
3821 priv = builder.CreateAddrSpaceCast(V: priv, DestTy: argPtrTy);
3822 moduleTranslation.mapValue(mlir: blockArg, llvm: priv);
3823 }
3824 }
3825
3826 auto continuationBlockOrError = convertOmpOpRegions(
3827 region&: taskOp.getRegion(), blockName: "omp.task.region", builder, moduleTranslation);
3828 if (failed(Result: handleError(result&: continuationBlockOrError, op&: *taskOp)))
3829 return llvm::make_error<PreviouslyReportedError>();
3830
3831 builder.SetInsertPoint(continuationBlockOrError.get()->getTerminator());
3832
3833 if (failed(Result: cleanupPrivateVars(op: taskOp, builder, moduleTranslation,
3834 loc: taskOp.getLoc(), privateVarsInfo)))
3835 return llvm::make_error<PreviouslyReportedError>();
3836
3837 // Free heap allocated task context structure at the end of the task.
3838 taskStructMgr.freeStructPtr();
3839
3840 return llvm::Error::success();
3841 };
3842
3843 llvm::OpenMPIRBuilder &ompBuilder = *getOpenMPBuilder(moduleTranslation);
3844 SmallVector<llvm::UncondBrInst *> cancelTerminators;
3845 // The directive to match here is OMPD_taskgroup because it is the taskgroup
3846 // which is canceled. This is handled here because it is the task's cleanup
3847 // block which should be branched to.
3848 pushCancelFinalizationCB(cancelTerminators, llvmBuilder&: builder, ompBuilder, op: taskOp,
3849 cancelDirective: llvm::omp::Directive::OMPD_taskgroup);
3850
3851 llvm::OpenMPIRBuilder::DependenciesInfo dependencies;
3852 if (failed(Result: buildDependData(dependVars: taskOp.getDependVars(), dependKinds: taskOp.getDependKinds(),
3853 dependIterated: taskOp.getDependIterated(),
3854 dependIteratedKinds: taskOp.getDependIteratedKinds(), builder,
3855 moduleTranslation, taskDeps&: dependencies)))
3856 return failure();
3857
3858 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
3859 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
3860 getOpenMPBuilder(moduleTranslation)
3861 ->createTask(Loc: ompLoc, AllocaIP: allocaIP, DeallocBlocks: deallocBlocks, BodyGenCB: bodyCB,
3862 Tied: !taskOp.getUntied(),
3863 Final: moduleTranslation.lookupValue(value: taskOp.getFinal()),
3864 IfCondition: moduleTranslation.lookupValue(value: taskOp.getIfExpr()),
3865 Dependencies: dependencies, Affinities: ad, Mergeable: taskOp.getMergeable(),
3866 EventHandle: moduleTranslation.lookupValue(value: taskOp.getEventHandle()),
3867 Priority: moduleTranslation.lookupValue(value: taskOp.getPriority()),
3868 FreeAgent: taskOp.getThreadset() == omp::ThreadsetPolicy::omp_pool);
3869
3870 if (failed(Result: handleError(result&: afterIP, op&: *taskOp)))
3871 return failure();
3872
3873 // Set the correct branch target for task cancellation
3874 popCancelFinalizationCB(cancelTerminators, ompBuilder,
3875 afterBB: afterIP->getNodeParent());
3876
3877 builder.restoreIP(IP: *afterIP);
3878
3879 if (dependencies.DepArray)
3880 builder.CreateFree(Source: dependencies.DepArray);
3881
3882 return success();
3883}
3884
3885/// The correct entry point is convertOmpTaskloopContextOp. This gets called
3886/// whilst lowering the body of the taskloop context (i.e. the task function).
3887static LogicalResult
3888convertOmpTaskloopWrapperOp(omp::TaskloopWrapperOp loopWrapperOp,
3889 llvm::IRBuilderBase &builder,
3890 LLVM::ModuleTranslation &moduleTranslation) {
3891 mlir::Operation &opInst = *loopWrapperOp.getOperation();
3892 if (failed(Result: checkImplementationStatus(op&: opInst)))
3893 return failure();
3894
3895 // Recurse into the loop body.
3896 auto continuationBlockOrError = convertOmpOpRegions(
3897 region&: loopWrapperOp.getRegion(), blockName: "omp.taskloop.wrapper.region", builder,
3898 moduleTranslation);
3899
3900 if (failed(Result: handleError(result&: continuationBlockOrError, op&: opInst)))
3901 return failure();
3902
3903 builder.SetInsertPoint(continuationBlockOrError.get());
3904 return success();
3905}
3906
3907/// Look up the given value in the mapping, and if it's not there, translate its
3908/// defining operation at the current builder insertion point. Only pure,
3909/// regionless operations are supported because the same operation will later be
3910/// translated again when the taskloop body itself is lowered.
3911static llvm::Expected<llvm::Value *>
3912lookupOrTranslatePureValue(Value value,
3913 LLVM::ModuleTranslation &moduleTranslation,
3914 llvm::IRBuilderBase &builder) {
3915 if (llvm::Value *mapped = moduleTranslation.lookupValue(value))
3916 return mapped;
3917
3918 Operation *defOp = value.getDefiningOp();
3919 if (!defOp)
3920 return llvm::make_error<llvm::StringError>(
3921 Args: "value is a block argument and is not mapped",
3922 Args: llvm::inconvertibleErrorCode());
3923 if (defOp->getNumRegions() != 0 || !isPure(op: defOp))
3924 return llvm::make_error<llvm::StringError>(
3925 Args: "unsupported op defining taskloop loop bound",
3926 Args: llvm::inconvertibleErrorCode());
3927
3928 SmallVector<Value> mappingsToRemove;
3929 mappingsToRemove.reserve(N: defOp->getNumOperands() + defOp->getNumResults());
3930 for (Value operand : defOp->getOperands()) {
3931 if (moduleTranslation.lookupValue(value: operand))
3932 continue;
3933
3934 llvm::Expected<llvm::Value *> operandOrError =
3935 lookupOrTranslatePureValue(value: operand, moduleTranslation, builder);
3936 if (!operandOrError)
3937 return operandOrError.takeError();
3938 moduleTranslation.mapValue(mlir: operand, llvm: *operandOrError);
3939 mappingsToRemove.push_back(Elt: operand);
3940 }
3941
3942 if (failed(Result: moduleTranslation.convertOperation(op&: *defOp, builder)))
3943 return llvm::make_error<llvm::StringError>(
3944 Args: "failed to convert op defining taskloop loop bound",
3945 Args: llvm::inconvertibleErrorCode());
3946
3947 llvm::Value *result = moduleTranslation.lookupValue(value);
3948 assert(result && "expected conversion of loop bound op to produce a value");
3949
3950 for (Value resultValue : defOp->getResults()) {
3951 if (moduleTranslation.lookupValue(value: resultValue))
3952 mappingsToRemove.push_back(Elt: resultValue);
3953 }
3954 for (Value mappedValue : mappingsToRemove)
3955 moduleTranslation.forgetMapping(value: mappedValue);
3956
3957 return result;
3958}
3959
3960static llvm::Error
3961computeTaskloopBounds(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
3962 LLVM::ModuleTranslation &moduleTranslation,
3963 llvm::Value *&lbVal, llvm::Value *&ubVal,
3964 llvm::Value *&stepVal) {
3965 Operation::operand_range lowerBounds = loopOp.getLoopLowerBounds();
3966 Operation::operand_range upperBounds = loopOp.getLoopUpperBounds();
3967 Operation::operand_range steps = loopOp.getLoopSteps();
3968
3969 llvm::Expected<llvm::Value *> firstLbOrErr =
3970 lookupOrTranslatePureValue(value: lowerBounds[0], moduleTranslation, builder);
3971 if (!firstLbOrErr)
3972 return firstLbOrErr.takeError();
3973
3974 llvm::Type *boundType = (*firstLbOrErr)->getType();
3975 ubVal = builder.getIntN(N: boundType->getIntegerBitWidth(), C: 1);
3976 if (loopOp.getCollapseNumLoops() > 1) {
3977 // In cases where Collapse is used with Taskloop, the upper bound of the
3978 // iteration space needs to be recalculated to cater for the collapsed loop.
3979 // The Collapsed Loop UpperBound is the product of all collapsed
3980 // loop's tripcount.
3981 // The LowerBound for collapsed loops is always 1. When the loops are
3982 // collapsed, it will reset the bounds and introduce processing to ensure
3983 // the index's are presented as expected. As this happens after creating
3984 // Taskloop, these bounds need predicting. Example:
3985 // !$omp taskloop collapse(2)
3986 // do i = 1, 10
3987 // do j = 1, 5
3988 // ..
3989 // end do
3990 // end do
3991 // This loop above has a total of 50 iterations, so the lb will be 1, and
3992 // the ub will be 50. collapseLoops in OMPIRBuilder then handles ensuring
3993 // that i and j are properly presented when used in the loop.
3994 for (uint64_t i = 0; i < loopOp.getCollapseNumLoops(); i++) {
3995 llvm::Expected<llvm::Value *> lbOrErr =
3996 i == 0 ? std::move(firstLbOrErr)
3997 : lookupOrTranslatePureValue(value: lowerBounds[i], moduleTranslation,
3998 builder);
3999 if (!lbOrErr)
4000 return lbOrErr.takeError();
4001 llvm::Expected<llvm::Value *> ubOrErr = lookupOrTranslatePureValue(
4002 value: upperBounds[i], moduleTranslation, builder);
4003 if (!ubOrErr)
4004 return ubOrErr.takeError();
4005 llvm::Expected<llvm::Value *> stepOrErr =
4006 lookupOrTranslatePureValue(value: steps[i], moduleTranslation, builder);
4007 if (!stepOrErr)
4008 return stepOrErr.takeError();
4009
4010 llvm::Value *loopLb = *lbOrErr;
4011 llvm::Value *loopUb = *ubOrErr;
4012 llvm::Value *loopStep = *stepOrErr;
4013 // In some cases, such as where the ub is less than the lb so the loop
4014 // steps down, the calculation for the loopTripCount is swapped. To ensure
4015 // the correct value is found, calculate both UB - LB and LB - UB then
4016 // select which value to use depending on how the loop has been
4017 // configured.
4018 llvm::Value *loopLbMinusOne = builder.CreateSub(
4019 LHS: loopLb, RHS: builder.getIntN(N: boundType->getIntegerBitWidth(), C: 1));
4020 llvm::Value *loopUbMinusOne = builder.CreateSub(
4021 LHS: loopUb, RHS: builder.getIntN(N: boundType->getIntegerBitWidth(), C: 1));
4022 llvm::Value *boundsCmp = builder.CreateICmpSLT(LHS: loopLb, RHS: loopUb);
4023 llvm::Value *ubMinusLb = builder.CreateSub(LHS: loopUb, RHS: loopLbMinusOne);
4024 llvm::Value *lbMinusUb = builder.CreateSub(LHS: loopLb, RHS: loopUbMinusOne);
4025 llvm::Value *loopTripCount =
4026 builder.CreateSelect(C: boundsCmp, True: ubMinusLb, False: lbMinusUb);
4027 loopTripCount = builder.CreateBinaryIntrinsic(
4028 ID: llvm::Intrinsic::abs, LHS: loopTripCount, RHS: builder.getFalse());
4029 // For loops that have a step value not equal to 1, we need to adjust the
4030 // trip count to ensure the correct number of iterations for the loop is
4031 // captured.
4032 llvm::Value *loopTripCountDivStep =
4033 builder.CreateSDiv(LHS: loopTripCount, RHS: loopStep);
4034 loopTripCountDivStep = builder.CreateBinaryIntrinsic(
4035 ID: llvm::Intrinsic::abs, LHS: loopTripCountDivStep, RHS: builder.getFalse());
4036 llvm::Value *loopTripCountRem =
4037 builder.CreateSRem(LHS: loopTripCount, RHS: loopStep);
4038 loopTripCountRem = builder.CreateBinaryIntrinsic(
4039 ID: llvm::Intrinsic::abs, LHS: loopTripCountRem, RHS: builder.getFalse());
4040 llvm::Value *needsRoundUp = builder.CreateICmpNE(
4041 LHS: loopTripCountRem,
4042 RHS: builder.getIntN(N: loopTripCountRem->getType()->getIntegerBitWidth(),
4043 C: 0));
4044 loopTripCount =
4045 builder.CreateAdd(LHS: loopTripCountDivStep,
4046 RHS: builder.CreateZExtOrTrunc(
4047 V: needsRoundUp, DestTy: loopTripCountDivStep->getType()));
4048 ubVal = builder.CreateMul(LHS: ubVal, RHS: loopTripCount);
4049 }
4050 lbVal = builder.getIntN(N: boundType->getIntegerBitWidth(), C: 1);
4051 stepVal = builder.getIntN(N: boundType->getIntegerBitWidth(), C: 1);
4052 } else {
4053 llvm::Expected<llvm::Value *> ubOrErr =
4054 lookupOrTranslatePureValue(value: upperBounds[0], moduleTranslation, builder);
4055 if (!ubOrErr)
4056 return ubOrErr.takeError();
4057 llvm::Expected<llvm::Value *> stepOrErr =
4058 lookupOrTranslatePureValue(value: steps[0], moduleTranslation, builder);
4059 if (!stepOrErr)
4060 return stepOrErr.takeError();
4061 lbVal = *firstLbOrErr;
4062 ubVal = *ubOrErr;
4063 stepVal = *stepOrErr;
4064 }
4065
4066 assert(lbVal != nullptr && "Expected value for lbVal");
4067 assert(ubVal != nullptr && "Expected value for ubVal");
4068 assert(stepVal != nullptr && "Expected value for stepVal");
4069 return llvm::Error::success();
4070}
4071
4072// Converts an OpenMP taskloop construct into LLVM IR using OpenMPIRBuilder.
4073static LogicalResult
4074convertOmpTaskloopContextOp(omp::TaskloopContextOp contextOp,
4075 llvm::IRBuilderBase &builder,
4076 LLVM::ModuleTranslation &moduleTranslation) {
4077 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4078 mlir::Operation &opInst = *contextOp.getOperation();
4079 omp::TaskloopWrapperOp loopWrapperOp = contextOp.getLoopOp();
4080 if (failed(Result: checkImplementationStatus(op&: opInst)))
4081 return failure();
4082
4083 // It stores the pointer of allocated firstprivate copies,
4084 // which can be used later for freeing the allocated space.
4085 SmallVector<llvm::Value *> llvmFirstPrivateVars;
4086 PrivateVarsInfo privateVarsInfo(contextOp);
4087 TaskContextStructManager taskStructMgr{builder, moduleTranslation,
4088 privateVarsInfo};
4089
4090 llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
4091 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
4092 findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks);
4093
4094 assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end());
4095 llvm::BasicBlock *taskloopStartBlock = llvm::BasicBlock::Create(
4096 Context&: builder.getContext(), Name: "omp.taskloop.wrapper.start",
4097 /*Parent=*/builder.GetInsertBlock()->getParent());
4098 llvm::Instruction *branchToTaskloopStartBlock =
4099 builder.CreateBr(Dest: taskloopStartBlock);
4100 builder.SetInsertPoint(branchToTaskloopStartBlock);
4101
4102 llvm::BasicBlock *copyBlock =
4103 splitBB(Builder&: builder, /*CreateBranch=*/true, Name: "omp.private.copy");
4104 llvm::BasicBlock *initBlock =
4105 splitBB(Builder&: builder, /*CreateBranch=*/true, Name: "omp.private.init");
4106
4107 LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
4108 moduleTranslation, allocaIP.getNodeParent(), deallocBlocks);
4109
4110 // Allocate and initialize private variables
4111 builder.SetInsertPoint(initBlock->getTerminator());
4112
4113 // TODO: don't allocate if the loop has zero iterations.
4114 taskStructMgr.generateTaskContextStruct();
4115 taskStructMgr.createGEPsToPrivateVars();
4116
4117 llvmFirstPrivateVars.resize(N: privateVarsInfo.vars.size());
4118
4119 for (auto [i, zip] : llvm::enumerate(First: llvm::zip_equal(
4120 t&: privateVarsInfo.vars, u: taskStructMgr.getLLVMPrivateVarGEPs()))) {
4121 auto &[var, llvmPrivateVarAlloc] = zip;
4122 // To be handled inside the taskloop.
4123 if (!var.privatizer.readsFromMold())
4124 continue;
4125 assert(llvmPrivateVarAlloc &&
4126 "reads from mold so shouldn't have been skipped");
4127
4128 llvm::Expected<llvm::Value *> privateVarOrErr =
4129 initPrivateVar(builder, moduleTranslation, privDecl&: var.privatizer, mlirPrivVar: var.mlirVar,
4130 blockArg&: var.blockArg, llvmPrivateVar: llvmPrivateVarAlloc, privInitBlock: initBlock);
4131 if (!privateVarOrErr)
4132 return handleError(result&: privateVarOrErr, op&: *contextOp.getOperation());
4133
4134 llvmFirstPrivateVars[i] = privateVarOrErr.get();
4135
4136 llvm::IRBuilderBase::InsertPointGuard guard(builder);
4137 builder.SetInsertPoint(builder.GetInsertBlock()->getTerminator());
4138
4139 [[maybe_unused]] llvm::Value *llvmPrivateVar = llvmPrivateVarAlloc;
4140 if ((privateVarOrErr.get() != llvmPrivateVarAlloc) &&
4141 !mlir::isa<LLVM::LLVMPointerType>(Val: var.blockArg.getType())) {
4142 builder.CreateStore(Val: privateVarOrErr.get(), Ptr: llvmPrivateVarAlloc);
4143 // Load it so we have the value pointed to by the GEP
4144 llvmPrivateVar = builder.CreateLoad(Ty: privateVarOrErr.get()->getType(),
4145 Ptr: llvmPrivateVarAlloc);
4146 }
4147 assert(llvmPrivateVar->getType() ==
4148 moduleTranslation.convertType(var.blockArg.getType()));
4149 }
4150
4151 // firstprivate copy region
4152 setInsertPointForPossiblyEmptyBlock(builder, block: copyBlock);
4153 if (failed(Result: completePrivateVars(
4154 op: contextOp, builder, moduleTranslation, privateVarsInfo,
4155 insertBarrier: contextOp.getPrivateNeedsBarrier(),
4156 /*moldVars=*/{}, llvmPrivateVars: taskStructMgr.getLLVMPrivateVarGEPs())))
4157 return llvm::failure();
4158
4159 // Resolve and validate reduction / in_reduction declarations up front.
4160 // This is pure MLIR symbol-table work and does not emit IR, so do it
4161 // before moving the builder to the taskloop start block. Richer
4162 // declare_reduction shapes (byref) have been rejected already by
4163 // checkImplementationStatus; the rest (two-argument initializer, cleanup
4164 // region, missing combiner) are rejected by the helper.
4165 SmallVector<omp::DeclareReductionOp> redDecls;
4166 if (failed(Result: collectAndValidateTaskloopRedDecls(
4167 contextOp: contextOp.getOperation(), syms: contextOp.getReductionSyms(),
4168 opName: "omp.taskloop.context", clauseName: "reduction", out&: redDecls)))
4169 return failure();
4170 SmallVector<omp::DeclareReductionOp> inRedDecls;
4171 if (failed(Result: collectAndValidateTaskloopRedDecls(
4172 contextOp: contextOp.getOperation(), syms: contextOp.getInReductionSyms(),
4173 opName: "omp.taskloop.context", clauseName: "in_reduction", out&: inRedDecls)))
4174 return failure();
4175
4176 // The op verifier rejects nogroup + reduction, so no check is needed here.
4177
4178 SmallVector<llvm::Value *> redOrigPtrs;
4179 redOrigPtrs.reserve(N: redDecls.size());
4180 for (Value v : contextOp.getReductionVars())
4181 redOrigPtrs.push_back(Elt: moduleTranslation.lookupValue(value: v));
4182 SmallVector<llvm::Value *> inRedOrigPtrs;
4183 inRedOrigPtrs.reserve(N: inRedDecls.size());
4184 for (Value v : contextOp.getInReductionVars())
4185 inRedOrigPtrs.push_back(Elt: moduleTranslation.lookupValue(value: v));
4186
4187 // Set up insertion point for emitting the implicit-taskgroup reduction
4188 // setup (if any) and for the subsequent call to createTaskloop().
4189 builder.SetInsertPoint(taskloopStartBlock);
4190
4191 llvm::OpenMPIRBuilder &ompBuilderRef = *getOpenMPBuilder(moduleTranslation);
4192 llvm::Module *module = moduleTranslation.getLLVMModule();
4193
4194 // If we have task_reduction items, we must emit our own implicit
4195 // __kmpc_taskgroup so that the descriptor returned by __kmpc_taskred_init
4196 // is associated with that taskgroup. We then force NoGroup=true so that
4197 // OpenMPIRBuilder::createTaskloop does not emit a second taskgroup.
4198 bool implicitTaskgroup = !redDecls.empty();
4199 llvm::Value *redDesc = nullptr;
4200 if (implicitTaskgroup) {
4201 llvm::OpenMPIRBuilder::LocationDescription redLoc(builder);
4202 uint32_t srcLocSize;
4203 llvm::Constant *srcLocStr =
4204 ompBuilderRef.getOrCreateSrcLocStr(Loc: redLoc, SrcLocStrSize&: srcLocSize);
4205 llvm::Value *ident = ompBuilderRef.getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize);
4206 // Align OpenMPIRBuilder's internal IRBuilder with `builder` so the
4207 // gtid call lands at our insertion point.
4208 ompBuilderRef.updateToLocation(Loc: redLoc);
4209 llvm::Value *outerGtid = ompBuilderRef.getOrCreateThreadID(Ident: ident);
4210 llvm::FunctionCallee taskgroupFn = ompBuilderRef.getOrCreateRuntimeFunction(
4211 M&: *module, FnID: llvm::omp::OMPRTL___kmpc_taskgroup);
4212 builder.CreateCall(Callee: taskgroupFn, Args: {ident, outerGtid});
4213
4214 redDesc = emitTaskReductionInitCall(redDecls, origPtrs: redOrigPtrs,
4215 helperNamePrefix: "__omp_taskloop_taskred_", builder,
4216 allocaIP, moduleTranslation);
4217 if (!redDesc)
4218 return failure();
4219 }
4220
4221 auto loopOp = cast<omp::LoopNestOp>(Val: loopWrapperOp.getWrappedLoop());
4222 llvm::Value *lbVal = nullptr;
4223 llvm::Value *ubVal = nullptr;
4224 llvm::Value *stepVal = nullptr;
4225 if (llvm::Error err = computeTaskloopBounds(
4226 loopOp, builder, moduleTranslation, lbVal, ubVal, stepVal))
4227 return handleError(error: std::move(err), op&: opInst);
4228
4229 auto bodyCB =
4230 [&](InsertPointTy allocaIP, InsertPointTy codegenIP,
4231 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
4232 // Save the alloca insertion point on ModuleTranslation stack for use in
4233 // nested regions.
4234 LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
4235 moduleTranslation, allocaIP.getNodeParent(), deallocBlocks);
4236
4237 // translate the body of the taskloop:
4238 builder.restoreIP(IP: codegenIP);
4239
4240 llvm::BasicBlock *privInitBlock = nullptr;
4241 for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) {
4242 // This is handled before the task executes
4243 if (var.privatizer.readsFromMold())
4244 continue;
4245
4246 llvm::IRBuilderBase::InsertPointGuard guard(builder);
4247 llvm::Type *llvmAllocType =
4248 moduleTranslation.convertType(type: var.privatizer.getType());
4249 builder.SetInsertPoint(allocaIP.getNodeParent()->getTerminator());
4250 llvm::Value *llvmPrivateVar = builder.CreateAlloca(
4251 Ty: llvmAllocType, /*ArraySize=*/nullptr, Name: "omp.private.alloc");
4252
4253 llvm::Expected<llvm::Value *> privateVarOrError = initPrivateVar(
4254 builder, moduleTranslation, privDecl&: var.privatizer, mlirPrivVar: var.mlirVar, blockArg&: var.blockArg,
4255 llvmPrivateVar, privInitBlock);
4256 if (!privateVarOrError)
4257 return privateVarOrError.takeError();
4258 moduleTranslation.mapValue(mlir: var.blockArg, llvm: privateVarOrError.get());
4259 var.llvmVar = privateVarOrError.get();
4260 }
4261
4262 taskStructMgr.createGEPsToPrivateVars();
4263 for (auto [i, llvmPrivVar] :
4264 llvm::enumerate(First: taskStructMgr.getLLVMPrivateVarGEPs())) {
4265 if (!llvmPrivVar) {
4266 assert(privateVarsInfo.vars[i].llvmVar &&
4267 "This is added in the loop above");
4268 continue;
4269 }
4270 privateVarsInfo.vars[i].llvmVar = llvmPrivVar;
4271 }
4272
4273 // Find and map the addresses of each variable within the taskloop context
4274 // structure
4275 for (PrivateVarsInfo::VarInfo &var : privateVarsInfo.vars) {
4276 // This was handled above.
4277 if (!var.privatizer.readsFromMold())
4278 continue;
4279 // Fix broken pass-by-value case for Fortran character boxes
4280 if (!mlir::isa<LLVM::LLVMPointerType>(Val: var.blockArg.getType())) {
4281 var.llvmVar = builder.CreateLoad(
4282 Ty: moduleTranslation.convertType(type: var.blockArg.getType()), Ptr: var.llvmVar);
4283 }
4284 assert(var.llvmVar->getType() ==
4285 moduleTranslation.convertType(var.blockArg.getType()));
4286 moduleTranslation.mapValue(mlir: var.blockArg, llvm: var.llvmVar);
4287 }
4288
4289 // Map reduction and in_reduction block arguments to the per-task private
4290 // storage returned by __kmpc_task_reduction_get_th_data. This call must
4291 // be emitted inside the to-be-outlined task body so that it returns the
4292 // *executing* thread's gtid (not the encountering thread's). The
4293 // taskgroup descriptor `redDesc` is computed in the outer scope and is
4294 // auto-captured into the task shareds aggregate by CodeExtractor during
4295 // OpenMPIRBuilder::finalize. For in_reduction the descriptor is NULL:
4296 // the runtime walks up enclosing taskgroups to find the matching
4297 // task_reduction registration for `origPtr`.
4298 if (!redDecls.empty() || !inRedDecls.empty()) {
4299 auto iface =
4300 cast<omp::BlockArgOpenMPOpInterface>(Val: contextOp.getOperation());
4301 llvm::OpenMPIRBuilder &ompB = *getOpenMPBuilder(moduleTranslation);
4302 llvm::Module *m = moduleTranslation.getLLVMModule();
4303 llvm::LLVMContext &llvmCtx = m->getContext();
4304 llvm::OpenMPIRBuilder::LocationDescription bodyLoc(builder);
4305 uint32_t srcLocSize;
4306 llvm::Constant *srcLocStr =
4307 ompB.getOrCreateSrcLocStr(Loc: bodyLoc, SrcLocStrSize&: srcLocSize);
4308 llvm::Value *bodyIdent = ompB.getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize);
4309 // Align OpenMPIRBuilder's internal IRBuilder with `builder` so the
4310 // gtid call lands inside the to-be-outlined task body.
4311 ompB.updateToLocation(Loc: bodyLoc);
4312 llvm::Value *bodyGtid = ompB.getOrCreateThreadID(Ident: bodyIdent);
4313 llvm::FunctionCallee getThData = ompB.getOrCreateRuntimeFunction(
4314 M&: *m, FnID: llvm::omp::OMPRTL___kmpc_task_reduction_get_th_data);
4315 llvm::Type *ptrTy = llvm::PointerType::getUnqual(C&: llvmCtx);
4316
4317 // Emit one __kmpc_task_reduction_get_th_data lookup for a reduction /
4318 // in_reduction item and map its block argument to the per-task private
4319 // storage the runtime returns. The runtime entry point takes (and
4320 // returns) a generic, default-address-space `ptr`, so normalize a
4321 // non-default-address-space original pointer to the generic address
4322 // space before the call (mirroring the descriptor setup in
4323 // emitTaskReductionInitCall), and cast the returned private pointer back
4324 // to the block argument's address space when that differs.
4325 auto remapReductionArg = [&](BlockArgument blockArg, llvm::Value *desc,
4326 llvm::Value *origPtr,
4327 const llvm::Twine &name) {
4328 if (auto *origPtrTy =
4329 llvm::dyn_cast<llvm::PointerType>(Val: origPtr->getType());
4330 origPtrTy && origPtrTy->getAddressSpace() != 0)
4331 origPtr = builder.CreateAddrSpaceCast(V: origPtr, DestTy: ptrTy);
4332 llvm::Value *priv =
4333 builder.CreateCall(Callee: getThData, Args: {bodyGtid, desc, origPtr}, Name: name);
4334 if (auto *argPtrTy = llvm::dyn_cast<llvm::PointerType>(
4335 Val: moduleTranslation.convertType(type: blockArg.getType()));
4336 argPtrTy && argPtrTy->getAddressSpace() != 0)
4337 priv = builder.CreateAddrSpaceCast(V: priv, DestTy: argPtrTy);
4338 moduleTranslation.mapValue(mlir: blockArg, llvm: priv);
4339 };
4340
4341 ArrayRef<BlockArgument> redBlockArgs = iface.getReductionBlockArgs();
4342 for (auto [blockArg, origPtr] :
4343 llvm::zip_equal(t&: redBlockArgs, u&: redOrigPtrs))
4344 remapReductionArg(blockArg, redDesc, origPtr, "omp.taskred.priv");
4345 ArrayRef<BlockArgument> inRedBlockArgs = iface.getInReductionBlockArgs();
4346 llvm::Value *nullDesc = llvm::ConstantPointerNull::get(T: ptrTy);
4347 for (auto [blockArg, origPtr] :
4348 llvm::zip_equal(t&: inRedBlockArgs, u&: inRedOrigPtrs))
4349 remapReductionArg(blockArg, nullDesc, origPtr, "omp.inred.priv");
4350 }
4351
4352 // Lower the contents of the taskloop context region: this is the body of
4353 // the generated task, not the loop.
4354 auto continuationBlockOrError = convertOmpOpRegions(
4355 region&: contextOp.getRegion(), blockName: "omp.taskloop.context.region", builder,
4356 moduleTranslation);
4357
4358 if (failed(Result: handleError(result&: continuationBlockOrError, op&: opInst)))
4359 return llvm::make_error<PreviouslyReportedError>();
4360
4361 builder.SetInsertPoint(continuationBlockOrError.get()->getTerminator());
4362
4363 // This is freeing the private variables as mapped inside of the task: these
4364 // will be per-task private copies possibly after task duplication. This is
4365 // handled transparently by how these are passed to the structure passed
4366 // into the outlined function. When the task is duplicated, that structure
4367 // is duplicated too.
4368 if (failed(Result: cleanupPrivateVars(op: contextOp, builder, moduleTranslation,
4369 loc: contextOp.getLoc(), privateVarsInfo)))
4370 return llvm::make_error<PreviouslyReportedError>();
4371 // Similarly, the task context structure freed inside the task is the
4372 // per-task copy after task duplication.
4373 taskStructMgr.freeStructPtr();
4374
4375 return llvm::Error::success();
4376 };
4377
4378 // Taskloop divides into an appropriate number of tasks by repeatedly
4379 // duplicating the original task. Each time this is done, the task context
4380 // structure must be duplicated too.
4381 auto taskDupCB = [&](InsertPointTy allocaIP, InsertPointTy codegenIP,
4382 llvm::Value *destPtr, llvm::Value *srcPtr)
4383 -> llvm::Expected<llvm::IRBuilderBase::InsertPoint> {
4384 llvm::IRBuilderBase::InsertPointGuard guard(builder);
4385 builder.restoreIP(IP: codegenIP);
4386
4387 llvm::Type *ptrTy =
4388 builder.getPtrTy(AddrSpace: srcPtr->getType()->getPointerAddressSpace());
4389 llvm::Value *src =
4390 builder.CreateLoad(Ty: ptrTy, Ptr: srcPtr, Name: "omp.taskloop.context.src");
4391
4392 TaskContextStructManager &srcStructMgr = taskStructMgr;
4393 TaskContextStructManager destStructMgr(builder, moduleTranslation,
4394 privateVarsInfo);
4395 destStructMgr.generateTaskContextStruct();
4396 llvm::Value *dest = destStructMgr.getStructPtr();
4397 dest->setName("omp.taskloop.context.dest");
4398 builder.CreateStore(Val: dest, Ptr: destPtr);
4399
4400 llvm::SmallVector<llvm::Value *> srcGEPs =
4401 srcStructMgr.createGEPsToPrivateVars(altStructPtr: src);
4402 llvm::SmallVector<llvm::Value *> destGEPs =
4403 destStructMgr.createGEPsToPrivateVars(altStructPtr: dest);
4404
4405 // Inline init regions.
4406 for (auto [var, mold, llvmPrivateVarAlloc] :
4407 llvm::zip_equal(t&: privateVarsInfo.vars, u&: srcGEPs, args&: destGEPs)) {
4408 // To be handled inside task body.
4409 if (!var.privatizer.readsFromMold())
4410 continue;
4411 assert(llvmPrivateVarAlloc &&
4412 "reads from mold so shouldn't have been skipped");
4413
4414 llvm::Value *moldArg = materializeRegionArgValue(
4415 builder, moduleTranslation, regionArg: var.privatizer.getInitMoldArg(), value: mold);
4416 llvm::Expected<llvm::Value *> privateVarOrErr = initPrivateVar(
4417 builder, moduleTranslation, privDecl&: var.privatizer, nonPrivateVar: moldArg, blockArg&: var.blockArg,
4418 llvmPrivateVar: llvmPrivateVarAlloc, privInitBlock: builder.GetInsertBlock());
4419 if (!privateVarOrErr)
4420 return privateVarOrErr.takeError();
4421
4422 setInsertPointForPossiblyEmptyBlock(builder);
4423
4424 // TODO: this is a bit of a hack for Fortran character boxes.
4425 // Character boxes are passed by value into the init region and then the
4426 // initialized character box is yielded by value. Here we need to store
4427 // the yielded value into the private allocation, and load the private
4428 // allocation to match the type expected by region block arguments.
4429 [[maybe_unused]] llvm::Value *llvmPrivateVar = llvmPrivateVarAlloc;
4430 if ((privateVarOrErr.get() != llvmPrivateVarAlloc) &&
4431 !mlir::isa<LLVM::LLVMPointerType>(Val: var.blockArg.getType())) {
4432 builder.CreateStore(Val: privateVarOrErr.get(), Ptr: llvmPrivateVarAlloc);
4433 // Load it so we have the value pointed to by the GEP
4434 llvmPrivateVar = builder.CreateLoad(Ty: privateVarOrErr.get()->getType(),
4435 Ptr: llvmPrivateVarAlloc);
4436 }
4437 assert(llvmPrivateVar->getType() ==
4438 moduleTranslation.convertType(var.blockArg.getType()));
4439
4440 // Mapping blockArg -> llvmPrivateVarAlloc is done inside the body
4441 // callback so that OpenMPIRBuilder doesn't try to pass each GEP address
4442 // through a stack allocated structure.
4443 }
4444
4445 if (failed(Result: completePrivateVars(op: contextOp.getOperation(), builder,
4446 moduleTranslation, privateVarsInfo,
4447 insertBarrier: contextOp.getPrivateNeedsBarrier(), moldVars: srcGEPs,
4448 llvmPrivateVars: destGEPs)))
4449 return llvm::make_error<PreviouslyReportedError>();
4450
4451 return builder.saveIP();
4452 };
4453
4454 auto loopInfo = [&]() -> llvm::Expected<llvm::CanonicalLoopInfo *> {
4455 llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);
4456 return loopInfo;
4457 };
4458
4459 llvm::Value *ifCond = nullptr;
4460 llvm::Value *grainsize = nullptr;
4461 int sched = 0; // default
4462 mlir::Value grainsizeVal = contextOp.getGrainsize();
4463 mlir::Value numTasksVal = contextOp.getNumTasks();
4464 if (Value ifVar = contextOp.getIfExpr())
4465 ifCond = moduleTranslation.lookupValue(value: ifVar);
4466 if (grainsizeVal) {
4467 grainsize = moduleTranslation.lookupValue(value: grainsizeVal);
4468 sched = 1; // grainsize
4469 } else if (numTasksVal) {
4470 grainsize = moduleTranslation.lookupValue(value: numTasksVal);
4471 sched = 2; // num_tasks
4472 }
4473
4474 llvm::OpenMPIRBuilder::TaskDupCallbackTy taskDupOrNull = nullptr;
4475 if (taskStructMgr.getStructPtr())
4476 taskDupOrNull = taskDupCB;
4477
4478 llvm::OpenMPIRBuilder &ompBuilder = *getOpenMPBuilder(moduleTranslation);
4479 SmallVector<llvm::UncondBrInst *> cancelTerminators;
4480 // The directive to match here is OMPD_taskgroup because it is the
4481 // taskgroup which is canceled. This is handled here because it is the
4482 // task's cleanup block which should be branched to. It doesn't depend upon
4483 // nogroup because even in that case the taskloop might still be inside an
4484 // explicit taskgroup.
4485 pushCancelFinalizationCB(cancelTerminators, llvmBuilder&: builder, ompBuilder, op: contextOp,
4486 cancelDirective: llvm::omp::Directive::OMPD_taskgroup);
4487
4488 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
4489 bool effectiveNoGroup = contextOp.getNogroup() || implicitTaskgroup;
4490 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
4491 getOpenMPBuilder(moduleTranslation)
4492 ->createTaskloop(
4493 Loc: ompLoc, AllocaIP: allocaIP, DeallocBlocks: deallocBlocks, BodyGenCB: bodyCB, LoopInfo: loopInfo, LBVal: lbVal, UBVal: ubVal,
4494 StepVal: stepVal, Untied: contextOp.getUntied(), IfCond: ifCond, GrainSize: grainsize,
4495 NoGroup: effectiveNoGroup, Sched: sched,
4496 Final: moduleTranslation.lookupValue(value: contextOp.getFinal()),
4497 Mergeable: contextOp.getMergeable(),
4498 Priority: moduleTranslation.lookupValue(value: contextOp.getPriority()),
4499 NumOfCollapseLoops: loopOp.getCollapseNumLoops(), DupCB: taskDupOrNull,
4500 TaskContextStructPtrVal: taskStructMgr.getStructPtr(),
4501 FreeAgent: contextOp.getThreadset() == omp::ThreadsetPolicy::omp_pool);
4502
4503 if (failed(Result: handleError(result&: afterIP, op&: opInst)))
4504 return failure();
4505
4506 popCancelFinalizationCB(cancelTerminators, ompBuilder,
4507 afterBB: afterIP->getNodeParent());
4508
4509 builder.restoreIP(IP: *afterIP);
4510
4511 // Close the implicit taskgroup we opened for task_reduction. The end call
4512 // must execute on the encountering thread, so use the outer-scope gtid.
4513 if (implicitTaskgroup) {
4514 llvm::OpenMPIRBuilder::LocationDescription endLoc(builder);
4515 uint32_t srcLocSize;
4516 llvm::Constant *srcLocStr =
4517 ompBuilder.getOrCreateSrcLocStr(Loc: endLoc, SrcLocStrSize&: srcLocSize);
4518 llvm::Value *ident = ompBuilder.getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize);
4519 // Align OpenMPIRBuilder's internal IRBuilder with `builder` so the
4520 // gtid call lands at our insertion point.
4521 ompBuilder.updateToLocation(Loc: endLoc);
4522 llvm::Value *outerGtid = ompBuilder.getOrCreateThreadID(Ident: ident);
4523 llvm::FunctionCallee endTgFn = ompBuilder.getOrCreateRuntimeFunction(
4524 M&: *moduleTranslation.getLLVMModule(),
4525 FnID: llvm::omp::OMPRTL___kmpc_end_taskgroup);
4526 builder.CreateCall(Callee: endTgFn, Args: {ident, outerGtid});
4527 }
4528 return success();
4529}
4530
4531/// Build an outlined init helper for a task_reduction declare_reduction op.
4532/// Signature: void(ptr %priv, ptr %orig). For non-byref reductions, the init
4533/// region's mold argument is mapped following the same rule as the regular
4534/// reduction path (`mapInitializationArgs`): a non-pointer mold loads the
4535/// value from %orig, while a pointer-typed mold receives %orig directly. The
4536/// yielded value is stored into %priv.
4537static llvm::Function *
4538emitTaskReductionInitFn(omp::DeclareReductionOp decl, StringRef baseName,
4539 LLVM::ModuleTranslation &moduleTranslation) {
4540 llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
4541 llvm::LLVMContext &ctx = llvmModule->getContext();
4542 llvm::Type *voidTy = llvm::Type::getVoidTy(C&: ctx);
4543 llvm::Type *ptrTy = llvm::PointerType::getUnqual(C&: ctx);
4544 llvm::FunctionType *fty =
4545 llvm::FunctionType::get(Result: voidTy, Params: {ptrTy, ptrTy}, isVarArg: false);
4546 llvm::Function *fn =
4547 llvm::Function::Create(Ty: fty, Linkage: llvm::GlobalValue::InternalLinkage,
4548 N: baseName + ".red.init", M: llvmModule);
4549 fn->setDoesNotRecurse();
4550 fn->getArg(i: 0)->setName("priv");
4551 fn->getArg(i: 1)->setName("orig");
4552
4553 llvm::BasicBlock *entry = llvm::BasicBlock::Create(Context&: ctx, Name: "entry", Parent: fn);
4554 llvm::IRBuilder<> b(entry);
4555
4556 // Map the initializer's mold argument the same way the regular reduction
4557 // path does in `mapInitializationArgs`: only load the original value when a
4558 // non-pointer mold is expected. For a pointer-typed mold the storage pointer
4559 // (%orig) is passed through directly, so a mold-yielding initializer lowers
4560 // to `store ptr %orig, ptr %priv` rather than emitting a spurious load.
4561 Value moldArg = decl.getInitializerMoldArg();
4562 llvm::Value *origVal = fn->getArg(i: 1);
4563 if (!isa<LLVM::LLVMPointerType>(Val: moldArg.getType()))
4564 origVal = b.CreateLoad(Ty: moduleTranslation.convertType(type: moldArg.getType()),
4565 Ptr: fn->getArg(i: 1), Name: "omp.orig");
4566 moduleTranslation.mapValue(mlir: moldArg, llvm: origVal);
4567 SmallVector<llvm::Value *, 1> phis;
4568 if (failed(Result: inlineConvertOmpRegions(region&: decl.getInitializerRegion(),
4569 blockName: "omp.taskred.init", builder&: b, moduleTranslation,
4570 continuationBlockArgs: &phis))) {
4571 fn->eraseFromParent();
4572 return nullptr;
4573 }
4574 assert(phis.size() == 1 &&
4575 "expected one value yielded from reduction initializer");
4576 b.CreateStore(Val: phis[0], Ptr: fn->getArg(i: 0));
4577 b.CreateRetVoid();
4578
4579 moduleTranslation.forgetMapping(region&: decl.getInitializerRegion());
4580 return fn;
4581}
4582
4583/// Build an outlined combiner helper for a task_reduction declare_reduction op.
4584/// Signature: void(ptr %lhs, ptr %rhs). For non-byref reductions, the values
4585/// at *%lhs and *%rhs are loaded, fed into the combiner region, and the
4586/// yielded scalar is stored back into *%lhs.
4587static llvm::Function *
4588emitTaskReductionCombFn(omp::DeclareReductionOp decl, StringRef baseName,
4589 LLVM::ModuleTranslation &moduleTranslation) {
4590 llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
4591 llvm::LLVMContext &ctx = llvmModule->getContext();
4592 llvm::Type *voidTy = llvm::Type::getVoidTy(C&: ctx);
4593 llvm::Type *ptrTy = llvm::PointerType::getUnqual(C&: ctx);
4594 llvm::FunctionType *fty =
4595 llvm::FunctionType::get(Result: voidTy, Params: {ptrTy, ptrTy}, isVarArg: false);
4596 llvm::Function *fn =
4597 llvm::Function::Create(Ty: fty, Linkage: llvm::GlobalValue::InternalLinkage,
4598 N: baseName + ".red.comb", M: llvmModule);
4599 fn->setDoesNotRecurse();
4600 fn->getArg(i: 0)->setName("lhs");
4601 fn->getArg(i: 1)->setName("rhs");
4602
4603 llvm::BasicBlock *entry = llvm::BasicBlock::Create(Context&: ctx, Name: "entry", Parent: fn);
4604 llvm::IRBuilder<> b(entry);
4605
4606 llvm::Type *elemTy = moduleTranslation.convertType(type: decl.getType());
4607 Block &combBlock = decl.getReductionRegion().front();
4608 assert(combBlock.getNumArguments() == 2 &&
4609 "expected two arguments in declare_reduction combiner");
4610 llvm::Value *lhsVal = b.CreateLoad(Ty: elemTy, Ptr: fn->getArg(i: 0), Name: "omp.lhs");
4611 llvm::Value *rhsVal = b.CreateLoad(Ty: elemTy, Ptr: fn->getArg(i: 1), Name: "omp.rhs");
4612 moduleTranslation.mapValue(mlir: combBlock.getArgument(i: 0), llvm: lhsVal);
4613 moduleTranslation.mapValue(mlir: combBlock.getArgument(i: 1), llvm: rhsVal);
4614
4615 SmallVector<llvm::Value *, 1> phis;
4616 if (failed(Result: inlineConvertOmpRegions(region&: decl.getReductionRegion(),
4617 blockName: "omp.taskred.comb", builder&: b, moduleTranslation,
4618 continuationBlockArgs: &phis))) {
4619 fn->eraseFromParent();
4620 return nullptr;
4621 }
4622 assert(phis.size() == 1 &&
4623 "expected one value yielded from reduction combiner");
4624 b.CreateStore(Val: phis[0], Ptr: fn->getArg(i: 0));
4625 b.CreateRetVoid();
4626
4627 moduleTranslation.forgetMapping(region&: decl.getReductionRegion());
4628 return fn;
4629}
4630
4631/// Emit the per-taskgroup task_reduction descriptor array and the
4632/// `__kmpc_taskred_init` runtime call. \p origPtrs holds the LLVM values for
4633/// the original (shared) variables, one per declaration in \p redDecls.
4634/// `builder` must be set to the point at which the descriptor stores and the
4635/// init call should be emitted; the descriptor array itself is allocated at
4636/// \p allocaIP. \p helperNamePrefix is used to disambiguate the generated
4637/// init/combiner helper symbol names between taskgroup and taskloop callers.
4638///
4639/// When \p isModifier is false, emits `__kmpc_taskred_init` and returns the
4640/// `ptr` value it produces (the taskgroup reduction handle). When \p isModifier
4641/// is true, emits `__kmpc_taskred_modifier_init` instead to open a
4642/// task-reduction scope for a parallel or worksharing construct, passing
4643/// \p isWorksharing as the runtime `is_ws` argument. Returns null on failure.
4644///
4645/// Only the non-byref form is handled here. Byref reductions have already
4646/// been rejected by `checkImplementationStatus`.
4647static llvm::Value *emitTaskReductionInitCall(
4648 ArrayRef<omp::DeclareReductionOp> redDecls,
4649 ArrayRef<llvm::Value *> origPtrs, StringRef helperNamePrefix,
4650 llvm::IRBuilderBase &builder, llvm::OpenMPIRBuilder::InsertPointTy allocaIP,
4651 LLVM::ModuleTranslation &moduleTranslation, bool isModifier,
4652 bool isWorksharing) {
4653 assert(redDecls.size() == origPtrs.size() &&
4654 "expected one orig pointer per reduction decl");
4655 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
4656 llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
4657 llvm::LLVMContext &ctx = llvmModule->getContext();
4658 const llvm::DataLayout &dl = llvmModule->getDataLayout();
4659
4660 llvm::Type *ptrTy = llvm::PointerType::getUnqual(C&: ctx);
4661 llvm::Type *i32Ty = llvm::Type::getInt32Ty(C&: ctx);
4662 llvm::Type *sizeTy =
4663 llvm::Type::getIntNTy(C&: ctx, N: dl.getPointerSizeInBits(/*AddrSpace=*/AS: 0));
4664
4665 // Identified `kmp_taskred_input_t` struct, matching the layout used by
4666 // Clang's CGOpenMPRuntime::emitTaskReductionInit.
4667 llvm::StructType *redInputTy =
4668 llvm::StructType::getTypeByName(C&: ctx, Name: "kmp_taskred_input_t");
4669 if (!redInputTy)
4670 redInputTy = llvm::StructType::create(
4671 Context&: ctx, Elements: {ptrTy, ptrTy, sizeTy, ptrTy, ptrTy, ptrTy, i32Ty},
4672 Name: "kmp_taskred_input_t");
4673
4674 unsigned n = redDecls.size();
4675 llvm::ArrayType *arrTy = llvm::ArrayType::get(ElementType: redInputTy, NumElements: n);
4676
4677 // Allocate the descriptor array in the enclosing function's alloca block.
4678 llvm::AllocaInst *arrAlloca;
4679 {
4680 llvm::IRBuilderBase::InsertPointGuard guard(builder);
4681 builder.restoreIP(IP: allocaIP);
4682 arrAlloca =
4683 builder.CreateAlloca(Ty: arrTy, /*ArraySize=*/nullptr, Name: ".taskred.input");
4684 }
4685
4686 // Fill each descriptor entry at the current builder insertion point.
4687 llvm::Value *zero = builder.getInt32(C: 0);
4688 for (unsigned i = 0; i < n; ++i) {
4689 omp::DeclareReductionOp decl = redDecls[i];
4690 llvm::Value *orig = origPtrs[i];
4691 if (auto *origPtrTy = llvm::dyn_cast<llvm::PointerType>(Val: orig->getType());
4692 origPtrTy && origPtrTy->getAddressSpace() != 0)
4693 orig = builder.CreateAddrSpaceCast(V: orig, DestTy: ptrTy);
4694 llvm::Type *elemTy = moduleTranslation.convertType(type: decl.getType());
4695 uint64_t size = dl.getTypeAllocSize(Ty: elemTy).getFixedValue();
4696
4697 std::string baseName =
4698 (llvm::Twine(helperNamePrefix) + decl.getSymName()).str();
4699 llvm::Function *initFn =
4700 emitTaskReductionInitFn(decl, baseName, moduleTranslation);
4701 llvm::Function *combFn =
4702 emitTaskReductionCombFn(decl, baseName, moduleTranslation);
4703 if (!initFn || !combFn)
4704 return nullptr;
4705 llvm::Value *elemPtr = builder.CreateInBoundsGEP(
4706 Ty: arrTy, Ptr: arrAlloca, IdxList: {zero, builder.getInt32(C: i)}, Name: ".taskred.elem");
4707 auto storeField = [&](unsigned fieldIdx, llvm::Value *val) {
4708 llvm::Value *fieldPtr =
4709 builder.CreateStructGEP(Ty: redInputTy, Ptr: elemPtr, Idx: fieldIdx);
4710 builder.CreateStore(Val: val, Ptr: fieldPtr);
4711 };
4712 storeField(0, orig); // reduce_shar
4713 storeField(1, orig); // reduce_orig
4714 storeField(2, llvm::ConstantInt::get(Ty: sizeTy, V: size)); // reduce_size
4715 storeField(3, initFn); // reduce_init
4716 storeField(4, llvm::ConstantPointerNull::get(T: ptrTy)); // reduce_fini
4717 storeField(5, combFn); // reduce_comb
4718 storeField(6, llvm::ConstantInt::get(Ty: i32Ty, V: 0)); // flags
4719 }
4720
4721 // Emit the runtime call that registers the task reduction data.
4722 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
4723 uint32_t srcLocSize;
4724 llvm::Constant *srcLocStr =
4725 ompBuilder->getOrCreateSrcLocStr(Loc: ompLoc, SrcLocStrSize&: srcLocSize);
4726 llvm::Value *ident = ompBuilder->getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize);
4727 ompBuilder->updateToLocation(Loc: ompLoc);
4728 llvm::Value *gtid = ompBuilder->getOrCreateThreadID(Ident: ident);
4729 if (isModifier) {
4730 // __kmpc_taskred_modifier_init(loc, gtid, is_ws, num, &arr) opens a
4731 // task-reduction scope for the enclosing parallel/worksharing region.
4732 llvm::FunctionCallee modInit = ompBuilder->getOrCreateRuntimeFunction(
4733 M&: *llvmModule, FnID: llvm::omp::OMPRTL___kmpc_taskred_modifier_init);
4734 return builder.CreateCall(Callee: modInit,
4735 Args: {ident, gtid,
4736 builder.getInt32(C: isWorksharing ? 1 : 0),
4737 builder.getInt32(C: n), arrAlloca},
4738 Name: ".taskred.desc");
4739 }
4740 // __kmpc_taskred_init(gtid, num, &arr).
4741 llvm::FunctionCallee taskredInit = ompBuilder->getOrCreateRuntimeFunction(
4742 M&: *llvmModule, FnID: llvm::omp::OMPRTL___kmpc_taskred_init);
4743 return builder.CreateCall(Callee: taskredInit, Args: {gtid, builder.getInt32(C: n), arrAlloca},
4744 Name: ".taskred.desc");
4745}
4746
4747/// Emits `__kmpc_task_reduction_modifier_fini(loc, gtid, is_ws)` at the current
4748/// builder insertion point, closing the task-reduction scope opened by the
4749/// `task` reduction modifier on a parallel or worksharing construct.
4750static void
4751emitTaskReductionModifierFini(bool isWorksharing, llvm::IRBuilderBase &builder,
4752 LLVM::ModuleTranslation &moduleTranslation) {
4753 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
4754 llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
4755 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
4756 uint32_t srcLocSize;
4757 llvm::Constant *srcLocStr =
4758 ompBuilder->getOrCreateSrcLocStr(Loc: ompLoc, SrcLocStrSize&: srcLocSize);
4759 llvm::Value *ident = ompBuilder->getOrCreateIdent(SrcLocStr: srcLocStr, SrcLocStrSize: srcLocSize);
4760 ompBuilder->updateToLocation(Loc: ompLoc);
4761 llvm::Value *gtid = ompBuilder->getOrCreateThreadID(Ident: ident);
4762 llvm::FunctionCallee fini = ompBuilder->getOrCreateRuntimeFunction(
4763 M&: *llvmModule, FnID: llvm::omp::OMPRTL___kmpc_task_reduction_modifier_fini);
4764 builder.CreateCall(Callee: fini,
4765 Args: {ident, gtid, builder.getInt32(C: isWorksharing ? 1 : 0)});
4766}
4767
4768/// Converts an OpenMP taskgroup construct into LLVM IR using OpenMPIRBuilder.
4769static LogicalResult
4770convertOmpTaskgroupOp(omp::TaskgroupOp tgOp, llvm::IRBuilderBase &builder,
4771 LLVM::ModuleTranslation &moduleTranslation) {
4772 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
4773 if (failed(Result: checkImplementationStatus(op&: *tgOp)))
4774 return failure();
4775
4776 // Resolve and validate task_reduction declarations up front. We only handle
4777 // declare_reduction ops shaped like a non-byref scalar reduction in this
4778 // first cut; richer shapes (two-argument initializer, cleanup region,
4779 // missing combiner) require additional infrastructure.
4780 SmallVector<omp::DeclareReductionOp> redDecls;
4781 if (auto syms = tgOp.getTaskReductionSyms()) {
4782 redDecls.reserve(N: syms->size());
4783 for (auto sym : syms->getAsRange<SymbolRefAttr>()) {
4784 auto decl = SymbolTable::lookupNearestSymbolFrom<omp::DeclareReductionOp>(
4785 from: tgOp, symbol: sym);
4786 if (!decl)
4787 return tgOp.emitError()
4788 << "failed to resolve task_reduction declare_reduction symbol "
4789 << sym.getRootReference() << " in omp.taskgroup";
4790 if (decl.getInitializerRegion().front().getNumArguments() != 1)
4791 return tgOp.emitError(message: "not yet implemented: task_reduction with "
4792 "two-argument initializer in omp.taskgroup");
4793 if (!decl.getCleanupRegion().empty())
4794 return tgOp.emitError(message: "not yet implemented: task_reduction with "
4795 "cleanup region in omp.taskgroup");
4796 if (decl.getReductionRegion().empty())
4797 return tgOp.emitError(message: "task_reduction declare_reduction is missing a "
4798 "combiner region");
4799 redDecls.push_back(Elt: decl);
4800 }
4801 }
4802
4803 auto bodyCB =
4804 [&](InsertPointTy allocaIP, InsertPointTy codegenIP,
4805 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
4806 builder.restoreIP(IP: codegenIP);
4807
4808 if (!redDecls.empty()) {
4809 SmallVector<llvm::Value *> origPtrs;
4810 origPtrs.reserve(N: redDecls.size());
4811 for (Value v : tgOp.getTaskReductionVars())
4812 origPtrs.push_back(Elt: moduleTranslation.lookupValue(value: v));
4813 if (!emitTaskReductionInitCall(redDecls, origPtrs, helperNamePrefix: "__omp_taskred_",
4814 builder, allocaIP, moduleTranslation))
4815 return llvm::createStringError(
4816 EC: llvm::inconvertibleErrorCode(),
4817 S: "failed to emit task_reduction initialization for omp.taskgroup");
4818 }
4819
4820 // Inside the taskgroup body, each task_reduction block argument refers to
4821 // the same shared/original storage that the runtime now knows about via
4822 // the descriptor array. Inner tasks that declare in_reduction look up
4823 // per-task private copies through the runtime; the taskgroup body itself
4824 // uses the original variable.
4825 for (auto [i, blockArg] :
4826 llvm::enumerate(First: tgOp.getRegion().getArguments())) {
4827 llvm::Value *orig =
4828 moduleTranslation.lookupValue(value: tgOp.getTaskReductionVars()[i]);
4829 moduleTranslation.mapValue(mlir: blockArg, llvm: orig);
4830 }
4831
4832 return convertOmpOpRegions(region&: tgOp.getRegion(), blockName: "omp.taskgroup.region",
4833 builder, moduleTranslation)
4834 .takeError();
4835 };
4836
4837 llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
4838 InsertPointTy allocaIP =
4839 findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks);
4840 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
4841 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
4842 getOpenMPBuilder(moduleTranslation)
4843 ->createTaskgroup(Loc: ompLoc, AllocaIP: allocaIP, DeallocBlocks: deallocBlocks, BodyGenCB: bodyCB);
4844
4845 if (failed(Result: handleError(result&: afterIP, op&: *tgOp)))
4846 return failure();
4847
4848 builder.restoreIP(IP: *afterIP);
4849 return success();
4850}
4851
4852static LogicalResult
4853convertOmpInteropInitOp(omp::InteropInitOp initOp, llvm::IRBuilderBase &builder,
4854 LLVM::ModuleTranslation &moduleTranslation) {
4855 if (!initOp.getDependVars().empty() || initOp.getDependKinds() ||
4856 !initOp.getDependIterated().empty() || initOp.getDependIteratedKinds())
4857 return initOp.emitError()
4858 << "not yet implemented: Unhandled clause depend in "
4859 << omp::InteropInitOp::getOperationName() << " operation";
4860
4861 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
4862 llvm::Value *interopVar =
4863 moduleTranslation.lookupValue(value: initOp.getInteropVar());
4864 llvm::Value *device = initOp.getDevice()
4865 ? moduleTranslation.lookupValue(value: initOp.getDevice())
4866 : nullptr;
4867
4868 // TODO: Handle depend clauses when supported.
4869 llvm::Value *numDeps = llvm::ConstantInt::get(Ty: builder.getInt32Ty(), V: 0);
4870 llvm::Value *depArray = llvm::ConstantPointerNull::get(T: builder.getPtrTy());
4871 bool hasNowait = initOp.getNowait();
4872
4873 // A single `init` clause may list both `target` and `targetsync`, but the
4874 // runtime init call takes a single interop-type. Collapse the set to one
4875 // value, matching Clang: if `target` is present use Target, otherwise
4876 // TargetSync. The offload runtime object model supports only one type per
4877 // object; representing both would require a runtime change.
4878 bool hasTarget = false, hasTargetSync = false;
4879 for (mlir::Attribute typeAttr : initOp.getInteropTypes()) {
4880 switch (cast<omp::InteropTypeAttr>(Val&: typeAttr).getValue()) {
4881 case omp::InteropType::target:
4882 hasTarget = true;
4883 break;
4884 case omp::InteropType::targetsync:
4885 hasTargetSync = true;
4886 break;
4887 }
4888 }
4889 llvm::omp::OMPInteropType interopType =
4890 (!hasTarget && hasTargetSync) ? llvm::omp::OMPInteropType::TargetSync
4891 : llvm::omp::OMPInteropType::Target;
4892 ompBuilder->createOMPInteropInit(Loc: builder, InteropVar: interopVar, InteropType: interopType, Device: device,
4893 NumDependences: numDeps, DependenceAddress: depArray, HaveNowaitClause: hasNowait);
4894 return success();
4895}
4896
4897static LogicalResult
4898convertOmpInteropDestroyOp(omp::InteropDestroyOp destroyOp,
4899 llvm::IRBuilderBase &builder,
4900 LLVM::ModuleTranslation &moduleTranslation) {
4901 if (!destroyOp.getDependVars().empty() || destroyOp.getDependKinds() ||
4902 !destroyOp.getDependIterated().empty() ||
4903 destroyOp.getDependIteratedKinds())
4904 return destroyOp.emitError()
4905 << "not yet implemented: Unhandled clause depend in "
4906 << omp::InteropDestroyOp::getOperationName() << " operation";
4907
4908 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
4909 llvm::Value *interopVar =
4910 moduleTranslation.lookupValue(value: destroyOp.getInteropVar());
4911 llvm::Value *device =
4912 destroyOp.getDevice()
4913 ? moduleTranslation.lookupValue(value: destroyOp.getDevice())
4914 : nullptr;
4915
4916 llvm::Value *numDeps = llvm::ConstantInt::get(Ty: builder.getInt32Ty(), V: 0);
4917 llvm::Value *depArray = llvm::ConstantPointerNull::get(T: builder.getPtrTy());
4918 bool hasNowait = destroyOp.getNowait();
4919
4920 ompBuilder->createOMPInteropDestroy(Loc: builder, InteropVar: interopVar, Device: device, NumDependences: numDeps,
4921 DependenceAddress: depArray, HaveNowaitClause: hasNowait);
4922 return success();
4923}
4924
4925static LogicalResult
4926convertOmpInteropUseOp(omp::InteropUseOp useOp, llvm::IRBuilderBase &builder,
4927 LLVM::ModuleTranslation &moduleTranslation) {
4928 if (!useOp.getDependVars().empty() || useOp.getDependKinds() ||
4929 !useOp.getDependIterated().empty() || useOp.getDependIteratedKinds())
4930 return useOp.emitError()
4931 << "not yet implemented: Unhandled clause depend in "
4932 << omp::InteropUseOp::getOperationName() << " operation";
4933
4934 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
4935 llvm::Value *interopVar =
4936 moduleTranslation.lookupValue(value: useOp.getInteropVar());
4937 llvm::Value *device = useOp.getDevice()
4938 ? moduleTranslation.lookupValue(value: useOp.getDevice())
4939 : nullptr;
4940
4941 llvm::Value *numDeps = llvm::ConstantInt::get(Ty: builder.getInt32Ty(), V: 0);
4942 llvm::Value *depArray = llvm::ConstantPointerNull::get(T: builder.getPtrTy());
4943 bool hasNowait = useOp.getNowait();
4944
4945 ompBuilder->createOMPInteropUse(Loc: builder, InteropVar: interopVar, Device: device, NumDependences: numDeps,
4946 DependenceAddress: depArray, HaveNowaitClause: hasNowait);
4947 return success();
4948}
4949
4950static LogicalResult
4951convertOmpTaskwaitOp(omp::TaskwaitOp twOp, llvm::IRBuilderBase &builder,
4952 LLVM::ModuleTranslation &moduleTranslation) {
4953 if (failed(Result: checkImplementationStatus(op&: *twOp)))
4954 return failure();
4955
4956 llvm::OpenMPIRBuilder::DependenciesInfo dds;
4957 if (failed(Result: buildDependData(
4958 dependVars: twOp.getDependVars(), dependKinds: twOp.getDependKinds(), dependIterated: twOp.getDependIterated(),
4959 dependIteratedKinds: twOp.getDependIteratedKinds(), builder, moduleTranslation, taskDeps&: dds))) {
4960 return failure();
4961 }
4962
4963 getOpenMPBuilder(moduleTranslation)
4964 ->createTaskwait(Loc: builder, Dependencies: dds, IsNowait: twOp.getNowait());
4965 if (dds.DepArray) {
4966 builder.CreateFree(Source: dds.DepArray);
4967 }
4968
4969 return success();
4970}
4971
4972/// Converts an OpenMP workshare loop into LLVM IR using OpenMPIRBuilder.
4973static LogicalResult
4974convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
4975 LLVM::ModuleTranslation &moduleTranslation) {
4976 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
4977 auto wsloopOp = cast<omp::WsloopOp>(Val&: opInst);
4978 if (failed(Result: checkImplementationStatus(op&: opInst)))
4979 return failure();
4980
4981 auto loopOp = cast<omp::LoopNestOp>(Val: wsloopOp.getWrappedLoop());
4982
4983 // Static is the default.
4984 auto schedule =
4985 wsloopOp.getScheduleKind().value_or(u: omp::ClauseScheduleKind::Static);
4986
4987 // Find the loop configuration.
4988 llvm::Value *step = moduleTranslation.lookupValue(value: loopOp.getLoopSteps()[0]);
4989 llvm::Type *ivType = step->getType();
4990 llvm::Value *chunk = nullptr;
4991 if (wsloopOp.getScheduleChunk()) {
4992 llvm::Value *chunkVar =
4993 moduleTranslation.lookupValue(value: wsloopOp.getScheduleChunk());
4994 chunk = builder.CreateSExtOrTrunc(V: chunkVar, DestTy: ivType);
4995 }
4996
4997 omp::DistributeOp distributeOp = nullptr;
4998 llvm::Value *distScheduleChunk = nullptr;
4999 bool hasDistSchedule = false;
5000 if (llvm::isa_and_present<omp::DistributeOp>(Val: opInst.getParentOp())) {
5001 distributeOp = cast<omp::DistributeOp>(Val: opInst.getParentOp());
5002 hasDistSchedule = distributeOp.getDistScheduleStatic();
5003 if (distributeOp.getDistScheduleChunkSize()) {
5004 llvm::Value *chunkVar = moduleTranslation.lookupValue(
5005 value: distributeOp.getDistScheduleChunkSize());
5006 distScheduleChunk = builder.CreateSExtOrTrunc(V: chunkVar, DestTy: ivType);
5007 }
5008 }
5009
5010 PrivateVarsInfo privateVarsInfo(wsloopOp);
5011 privateVarsInfo.addLinearVars(op: wsloopOp);
5012 ReductionVarsInfo reductionInfo(wsloopOp);
5013
5014 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
5015 findAllocInsertPoints(builder, moduleTranslation);
5016
5017 llvm::BasicBlock *allocaBlock = allocaIP.getNodeParent();
5018 llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(
5019 op: wsloopOp, builder, moduleTranslation, privateVarsInfo, allocaIP);
5020 if (handleError(result&: afterAllocas, op&: opInst).failed())
5021 return failure();
5022
5023 if (failed(Result: allocReductionVars(op&: *wsloopOp, builder, moduleTranslation, allocaIP,
5024 redInfo&: reductionInfo)))
5025 return failure();
5026
5027 if (handleError(error: initPrivateVars(builder, moduleTranslation, privateVarsInfo),
5028 op&: opInst)
5029 .failed())
5030 return failure();
5031
5032 if (failed(Result: completePrivateVars(op: wsloopOp, builder, moduleTranslation,
5033 privateVarsInfo,
5034 insertBarrier: wsloopOp.getPrivateNeedsBarrier())))
5035 return failure();
5036
5037 assert(afterAllocas.get()->getSinglePredecessor());
5038 if (failed(Result: initReductionVars(op&: *wsloopOp, builder, moduleTranslation,
5039 latestAllocaBlock: afterAllocas.get()->getSinglePredecessor(),
5040 redInfo&: reductionInfo)))
5041 return failure();
5042
5043 // For `reduction(task, ...)` open a task-reduction scope for the worksharing
5044 // loop. Participating explicit tasks accumulate into the per-thread private
5045 // copies, which the worksharing reduction then combines across threads.
5046 bool isTaskReductionMod =
5047 wsloopOp.hasReductionModifier(modifier: omp::ReductionModifier::task);
5048 if (isTaskReductionMod &&
5049 !emitTaskReductionModifierInitCall(
5050 op: wsloopOp, reductionDecls: reductionInfo.getReductionDecls(),
5051 privateReductionVariables: reductionInfo.getPrivateReductionVariables(), builder, allocaIP,
5052 moduleTranslation, /*isWorksharing=*/true))
5053 return wsloopOp.emitError(
5054 message: "failed to emit task reduction modifier initialization");
5055
5056 // TODO: Handle doacross loops when the ordered clause has a parameter.
5057 bool isOrdered = wsloopOp.getOrdered().has_value();
5058 std::optional<omp::ScheduleModifier> scheduleMod = wsloopOp.getScheduleMod();
5059 bool isSimd = wsloopOp.getScheduleSimd();
5060 bool loopNeedsBarrier = !wsloopOp.getNowait();
5061
5062 // The only legal way for the direct parent to be omp.distribute is that this
5063 // represents 'distribute parallel do'. Otherwise, this is a regular
5064 // worksharing loop.
5065 llvm::omp::WorksharingLoopType workshareLoopType =
5066 llvm::isa_and_present<omp::DistributeOp>(Val: opInst.getParentOp())
5067 ? llvm::omp::WorksharingLoopType::DistributeForStaticLoop
5068 : llvm::omp::WorksharingLoopType::ForStaticLoop;
5069
5070 SmallVector<llvm::UncondBrInst *> cancelTerminators;
5071 pushCancelFinalizationCB(cancelTerminators, llvmBuilder&: builder, ompBuilder&: *ompBuilder, op: wsloopOp,
5072 cancelDirective: llvm::omp::Directive::OMPD_for);
5073
5074 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
5075
5076 // Initialize linear variables and linear step
5077 LinearClauseProcessor linearClauseProcessor;
5078
5079 if (!wsloopOp.getLinearVars().empty()) {
5080 auto linearVarTypes = wsloopOp.getLinearVarTypes().value();
5081 for (mlir::Attribute linearVarType : linearVarTypes)
5082 linearClauseProcessor.registerType(moduleTranslation, ty&: linearVarType);
5083
5084 for (auto [idx, linearVar] : llvm::enumerate(First: wsloopOp.getLinearVars())) {
5085 size_t privIdx = privateVarsInfo.numPrivateVars + idx;
5086 linearClauseProcessor.createLinearVar(
5087 builder, allocaIP, privatizer: privateVarsInfo.vars[privIdx].privatizer,
5088 origVar: moduleTranslation.lookupValue(value: linearVar),
5089 privateVar: privateVarsInfo.vars[privIdx].llvmVar, idx);
5090 }
5091 for (mlir::Value linearStep : wsloopOp.getLinearStepVars())
5092 linearClauseProcessor.initLinearStep(moduleTranslation, linearStep);
5093 }
5094
5095 llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(
5096 region&: wsloopOp.getRegion(), blockName: "omp.wsloop.region", builder, moduleTranslation);
5097
5098 if (failed(Result: handleError(result&: regionBlock, op&: opInst)))
5099 return failure();
5100
5101 llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);
5102
5103 // Emit Initialization and Update IR for linear variables
5104 if (!wsloopOp.getLinearVars().empty()) {
5105 if (failed(Result: handleError(
5106 error: linearClauseProcessor.initLinearVar(builder, moduleTranslation,
5107 loopPreHeader: loopInfo->getPreheader()),
5108 op&: opInst)))
5109 return failure();
5110 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
5111 getOpenMPBuilder(moduleTranslation)
5112 ->createBarrier(Loc: builder, Kind: llvm::omp::OMPD_barrier);
5113 if (failed(Result: handleError(result&: afterBarrierIP, op&: *loopOp)))
5114 return failure();
5115 builder.restoreIP(IP: *afterBarrierIP);
5116 if (failed(Result: handleError(error: linearClauseProcessor.updateLinearVar(
5117 builder, moduleTranslation, loopBody: loopInfo->getBody(),
5118 loopInductionVar: loopInfo->getIndVar()),
5119 op&: opInst)))
5120 return failure();
5121 linearClauseProcessor.splitLinearFiniBB(builder, loopExit: loopInfo->getExit());
5122 }
5123
5124 builder.SetInsertPoint((*regionBlock)->begin());
5125
5126 // Check if we can generate no-loop kernel
5127 bool noLoopMode = false;
5128 omp::TargetOp targetOp = wsloopOp->getParentOfType<mlir::omp::TargetOp>();
5129 if (targetOp &&
5130 targetOp.getKernelType() == omp::TargetExecMode::spmd_no_loop) {
5131 Operation *targetCapturedOp =
5132 cast<omp::ComposableOpInterface>(Val&: *targetOp).findCapturedOp();
5133 // We need this check because, without it, noLoopMode would be set to true
5134 // for every omp.wsloop nested inside a no-loop SPMD target region, even if
5135 // that loop is not the top-level SPMD one.
5136 if (loopOp == targetCapturedOp)
5137 noLoopMode = true;
5138 }
5139
5140 // Nested regions can move the instruction referenced by allocaIP.
5141 allocaIP = allocaBlock->getFirstInsertionPt();
5142 llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
5143 ompBuilder->applyWorkshareLoop(
5144 DL: ompLoc.DL, CLI: loopInfo, AllocaIP: allocaIP, NeedsBarrier: loopNeedsBarrier,
5145 SchedKind: convertToScheduleKind(schedKind: schedule), ChunkSize: chunk, HasSimdModifier: isSimd,
5146 HasMonotonicModifier: scheduleMod == omp::ScheduleModifier::monotonic,
5147 HasNonmonotonicModifier: scheduleMod == omp::ScheduleModifier::nonmonotonic, HasOrderedClause: isOrdered,
5148 LoopType: workshareLoopType, NoLoop: noLoopMode, HasDistSchedule: hasDistSchedule, DistScheduleChunkSize: distScheduleChunk,
5149 NeedsLastIter: !wsloopOp.getLinearVars().empty());
5150
5151 if (failed(Result: handleError(result&: wsloopIP, op&: opInst)))
5152 return failure();
5153
5154 // Save the continuation block before linear var finalization, which may
5155 // invalidate the wsloopIP iterator.
5156 llvm::BasicBlock *wsloopContinuationBB = wsloopIP->getNodeParent();
5157
5158 // Emit finalization and in-place rewrites for linear vars.
5159 if (!wsloopOp.getLinearVars().empty()) {
5160 llvm::OpenMPIRBuilder::InsertPointTy oldIP = builder.saveIP();
5161 assert(loopInfo->getLastIter() &&
5162 "`lastiter` in CanonicalLoopInfo is nullptr");
5163 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
5164 linearClauseProcessor.finalizeLinearVar(builder, moduleTranslation,
5165 lastIter: loopInfo->getLastIter());
5166 if (failed(Result: handleError(result&: afterBarrierIP, op&: *loopOp)))
5167 return failure();
5168
5169 builder.restoreIP(IP: oldIP);
5170 }
5171
5172 // Set the correct branch target for task cancellation
5173 popCancelFinalizationCB(cancelTerminators, ompBuilder&: *ompBuilder, afterBB: wsloopContinuationBB);
5174
5175 // Close the task-reduction scope before the worksharing reduction combine.
5176 if (isTaskReductionMod)
5177 emitTaskReductionModifierFini(/*isWorksharing=*/true, builder,
5178 moduleTranslation);
5179
5180 // Process the reductions if required.
5181 if (failed(Result: createReductionsAndCleanup(op&: *wsloopOp, builder, moduleTranslation,
5182 allocaIP, redInfo&: reductionInfo,
5183 isNowait: wsloopOp.getNowait(),
5184 /*isTeamsReduction=*/false)))
5185 return failure();
5186
5187 return cleanupPrivateVars(op: wsloopOp, builder, moduleTranslation,
5188 loc: wsloopOp.getLoc(), privateVarsInfo);
5189}
5190
5191/// Converts the OpenMP parallel operation to LLVM IR.
5192static LogicalResult
5193convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
5194 LLVM::ModuleTranslation &moduleTranslation) {
5195 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
5196 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
5197 bool isCancellable = constructIsCancellable(op: opInst);
5198
5199 if (failed(Result: checkImplementationStatus(op&: *opInst)))
5200 return failure();
5201
5202 PrivateVarsInfo privateVarsInfo(opInst);
5203 if (failed(Result: convertAllocatorVars(op&: *opInst, allocatorVars: opInst.getAllocatorVars(), builder,
5204 moduleTranslation, privateVarsInfo)))
5205 return failure();
5206
5207 // Gather reduction info.
5208 ReductionVarsInfo reductionInfo(opInst);
5209
5210 // Only open a task-reduction scope when the `task` modifier is present and
5211 // there are reduction variables to combine; otherwise the matching fini in
5212 // the reduction-combine path (guarded by getNumReductionVars() > 0) would be
5213 // skipped, leaving the modifier init unbalanced.
5214 bool isTaskReductionMod =
5215 opInst.hasReductionModifier(modifier: omp::ReductionModifier::task);
5216
5217 auto bodyGenCB =
5218 [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
5219 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
5220 llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(
5221 op: opInst, builder, moduleTranslation, privateVarsInfo, allocaIP);
5222 if (handleError(result&: afterAllocas, op&: *opInst).failed())
5223 return llvm::make_error<PreviouslyReportedError>();
5224
5225 // Allocate reduction vars
5226 allocaIP = allocaIP.getNodeParent()->getTerminator()->getIterator();
5227
5228 if (failed(Result: allocReductionVars(op&: *opInst, builder, moduleTranslation, allocaIP,
5229 redInfo&: reductionInfo)))
5230 return llvm::make_error<PreviouslyReportedError>();
5231
5232 assert(afterAllocas.get()->getSinglePredecessor());
5233 builder.restoreIP(IP: codeGenIP);
5234
5235 if (handleError(
5236 error: initPrivateVars(builder, moduleTranslation, privateVarsInfo),
5237 op&: *opInst)
5238 .failed())
5239 return llvm::make_error<PreviouslyReportedError>();
5240
5241 if (failed(Result: completePrivateVars(op: opInst, builder, moduleTranslation,
5242 privateVarsInfo,
5243 insertBarrier: opInst.getPrivateNeedsBarrier())))
5244 return llvm::make_error<PreviouslyReportedError>();
5245
5246 if (failed(Result: initReductionVars(op&: *opInst, builder, moduleTranslation,
5247 latestAllocaBlock: afterAllocas.get()->getSinglePredecessor(),
5248 redInfo&: reductionInfo)))
5249 return llvm::make_error<PreviouslyReportedError>();
5250
5251 // For `reduction(task, ...)` open a task-reduction scope so participating
5252 // explicit tasks accumulate into the per-thread private copies; the
5253 // parallel reduction then combines those copies across the team.
5254 if (isTaskReductionMod &&
5255 !emitTaskReductionModifierInitCall(
5256 op: opInst, reductionDecls: reductionInfo.getReductionDecls(),
5257 privateReductionVariables: reductionInfo.getPrivateReductionVariables(), builder, allocaIP,
5258 moduleTranslation, /*isWorksharing=*/false))
5259 return llvm::createStringError(
5260 Fmt: "failed to emit task reduction modifier initialization");
5261
5262 // Save the alloca insertion point on ModuleTranslation stack for use in
5263 // nested regions.
5264 LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
5265 moduleTranslation, allocaIP.getNodeParent(), deallocBlocks);
5266
5267 // ParallelOp has only one region associated with it.
5268 llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(
5269 region&: opInst.getRegion(), blockName: "omp.par.region", builder, moduleTranslation);
5270 if (!regionBlock)
5271 return regionBlock.takeError();
5272
5273 // Process the reductions if required.
5274 if (opInst.getNumReductionVars() > 0) {
5275 // Collect reduction info
5276 SmallVector<OwningReductionGen> owningReductionGens;
5277 SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens;
5278 SmallVector<OwningDataPtrPtrReductionGen>
5279 owningReductionGenRefDataPtrGens;
5280 SmallVector<llvm::OpenMPIRBuilder::ReductionInfo, 2> reductionInfos;
5281 collectReductionInfo(builder, moduleTranslation, redInfo&: reductionInfo,
5282 owningReductionGens, owningAtomicReductionGens,
5283 owningDataPtrPtrReductionGens&: owningReductionGenRefDataPtrGens, reductionInfos);
5284
5285 // Move to region cont block
5286 builder.SetInsertPoint((*regionBlock)->getTerminator());
5287
5288 // Close the task-reduction scope before the per-thread reduction
5289 // contributions are combined across the team.
5290 if (isTaskReductionMod)
5291 emitTaskReductionModifierFini(/*isWorksharing=*/false, builder,
5292 moduleTranslation);
5293
5294 // Generate reductions from info
5295 llvm::UnreachableInst *tempTerminator = builder.CreateUnreachable();
5296 builder.SetInsertPoint(tempTerminator);
5297
5298 llvm::OpenMPIRBuilder::InsertPointOrErrorTy contInsertPoint =
5299 ompBuilder->createReductions(Loc: builder, AllocaIP: allocaIP, ReductionInfos: reductionInfos,
5300 IsByRef: reductionInfo.getIsByRef(),
5301 /*IsNoWait=*/false,
5302 /*IsTeamsReduction=*/false);
5303 if (!contInsertPoint)
5304 return contInsertPoint.takeError();
5305
5306 if (!contInsertPoint->isValid())
5307 return llvm::make_error<PreviouslyReportedError>();
5308
5309 tempTerminator->eraseFromParent();
5310 builder.restoreIP(IP: *contInsertPoint);
5311 }
5312
5313 return llvm::Error::success();
5314 };
5315
5316 auto privCB = [](InsertPointTy allocaIP, InsertPointTy codeGenIP,
5317 llvm::Value &, llvm::Value &val, llvm::Value *&replVal) {
5318 // tell OpenMPIRBuilder not to do anything. We handled Privatisation in
5319 // bodyGenCB.
5320 replVal = &val;
5321 return codeGenIP;
5322 };
5323
5324 // TODO: Perform finalization actions for variables. This has to be
5325 // called for variables which have destructors/finalizers.
5326 auto finiCB = [&](InsertPointTy codeGenIP) -> llvm::Error {
5327 InsertPointTy oldIP = builder.saveIP();
5328 builder.restoreIP(IP: codeGenIP);
5329
5330 // if the reduction has a cleanup region, inline it here to finalize the
5331 // reduction variables
5332 SmallVector<Region *> reductionCleanupRegions;
5333 llvm::transform(Range&: reductionInfo.vars,
5334 d_first: std::back_inserter(x&: reductionCleanupRegions),
5335 F: [](ReductionVarsInfo::VarInfo &var) {
5336 return &var.decl.getCleanupRegion();
5337 });
5338 if (failed(Result: inlineOmpRegionCleanup(
5339 cleanupRegions&: reductionCleanupRegions,
5340 privateVariables: reductionInfo.getPrivateReductionVariables(), moduleTranslation,
5341 builder, regionName: "omp.reduction.cleanup")))
5342 return llvm::createStringError(
5343 Fmt: "failed to inline `cleanup` region of `omp.declare_reduction`");
5344
5345 if (failed(Result: cleanupPrivateVars(op: opInst, builder, moduleTranslation,
5346 loc: opInst.getLoc(), privateVarsInfo)))
5347 return llvm::make_error<PreviouslyReportedError>();
5348
5349 // If we could be performing cancellation, add the cancellation barrier on
5350 // the way out of the outlined region.
5351 if (isCancellable) {
5352 auto IPOrErr = ompBuilder->createBarrier(
5353 Loc: llvm::OpenMPIRBuilder::LocationDescription(builder),
5354 Kind: llvm::omp::Directive::OMPD_unknown,
5355 /* ForceSimpleCall */ false,
5356 /* CheckCancelFlag */ false);
5357 if (!IPOrErr)
5358 return IPOrErr.takeError();
5359 }
5360
5361 builder.restoreIP(IP: oldIP);
5362 return llvm::Error::success();
5363 };
5364
5365 llvm::Value *ifCond = nullptr;
5366 if (auto ifVar = opInst.getIfExpr())
5367 ifCond = moduleTranslation.lookupValue(value: ifVar);
5368 llvm::Value *numThreads = nullptr;
5369 if (!opInst.getNumThreadsVars().empty())
5370 numThreads = moduleTranslation.lookupValue(value: opInst.getNumThreads(dim: 0));
5371 auto pbKind = llvm::omp::OMP_PROC_BIND_default;
5372 if (auto bind = opInst.getProcBindKind())
5373 pbKind = getProcBindKind(kind: *bind);
5374
5375 llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
5376 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
5377 findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks);
5378 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
5379
5380 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
5381 ompBuilder->createParallel(Loc: ompLoc, AllocaIP: allocaIP, DeallocBlocks: deallocBlocks, BodyGenCB: bodyGenCB,
5382 PrivCB: privCB, FiniCB: finiCB, IfCondition: ifCond, NumThreads: numThreads, ProcBind: pbKind,
5383 IsCancellable: isCancellable);
5384
5385 if (failed(Result: handleError(result&: afterIP, op&: *opInst)))
5386 return failure();
5387
5388 builder.restoreIP(IP: *afterIP);
5389 return success();
5390}
5391
5392/// Convert Order attribute to llvm::omp::OrderKind.
5393static llvm::omp::OrderKind
5394convertOrderKind(std::optional<omp::ClauseOrderKind> o) {
5395 if (!o)
5396 return llvm::omp::OrderKind::OMP_ORDER_unknown;
5397 switch (*o) {
5398 case omp::ClauseOrderKind::Concurrent:
5399 return llvm::omp::OrderKind::OMP_ORDER_concurrent;
5400 }
5401 llvm_unreachable("Unknown ClauseOrderKind kind");
5402}
5403
5404/// Converts an OpenMP simd loop into LLVM IR using OpenMPIRBuilder.
5405static LogicalResult
5406convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
5407 LLVM::ModuleTranslation &moduleTranslation) {
5408 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
5409 auto simdOp = cast<omp::SimdOp>(Val&: opInst);
5410
5411 if (failed(Result: checkImplementationStatus(op&: opInst)))
5412 return failure();
5413
5414 PrivateVarsInfo privateVarsInfo(simdOp);
5415 privateVarsInfo.addLinearVars(op: simdOp);
5416 ReductionVarsInfo reductionInfo(simdOp);
5417
5418 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
5419 findAllocInsertPoints(builder, moduleTranslation);
5420
5421 llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(
5422 op: simdOp, builder, moduleTranslation, privateVarsInfo, allocaIP);
5423 if (handleError(result&: afterAllocas, op&: opInst).failed())
5424 return failure();
5425
5426 if (failed(Result: allocReductionVars(op&: *simdOp, builder, moduleTranslation, allocaIP,
5427 redInfo&: reductionInfo)))
5428 return failure();
5429
5430 if (handleError(error: initPrivateVars(builder, moduleTranslation, privateVarsInfo),
5431 op&: opInst)
5432 .failed())
5433 return failure();
5434
5435 // No call to completePrivateVars because SIMD does not allow FIRSTPRIVATE and
5436 // is not a worksharing construct, so a privatization barrier here could
5437 // deadlock.
5438
5439 // Initialize linear variables and linear step
5440 LinearClauseProcessor linearClauseProcessor;
5441 if (linearClauseProcessor.initLinearIV(simdOp).failed())
5442 return failure();
5443
5444 if (!simdOp.getLinearVars().empty()) {
5445 auto linearVarTypes = simdOp.getLinearVarTypes().value();
5446 for (mlir::Attribute linearVarType : linearVarTypes)
5447 linearClauseProcessor.registerType(moduleTranslation, ty&: linearVarType);
5448 for (auto [idx, linearVar] : llvm::enumerate(First: simdOp.getLinearVars())) {
5449 size_t privIdx = privateVarsInfo.numPrivateVars + idx;
5450 linearClauseProcessor.createLinearVar(
5451 builder, allocaIP, privatizer: privateVarsInfo.vars[privIdx].privatizer,
5452 origVar: moduleTranslation.lookupValue(value: linearVar),
5453 privateVar: privateVarsInfo.vars[privIdx].llvmVar, idx);
5454 }
5455 for (mlir::Value linearStep : simdOp.getLinearStepVars())
5456 linearClauseProcessor.initLinearStep(moduleTranslation, linearStep);
5457 }
5458
5459 assert(afterAllocas.get()->getSinglePredecessor());
5460 if (failed(Result: initReductionVars(op&: *simdOp, builder, moduleTranslation,
5461 latestAllocaBlock: afterAllocas.get()->getSinglePredecessor(),
5462 redInfo&: reductionInfo)))
5463 return failure();
5464
5465 llvm::ConstantInt *simdlen = nullptr;
5466 if (std::optional<uint64_t> simdlenVar = simdOp.getSimdlen())
5467 simdlen = builder.getInt64(C: simdlenVar.value());
5468
5469 llvm::ConstantInt *safelen = nullptr;
5470 if (std::optional<uint64_t> safelenVar = simdOp.getSafelen())
5471 safelen = builder.getInt64(C: safelenVar.value());
5472
5473 llvm::MapVector<llvm::Value *, llvm::Value *> alignedVars;
5474 llvm::omp::OrderKind order = convertOrderKind(o: simdOp.getOrder());
5475
5476 llvm::BasicBlock *sourceBlock = builder.GetInsertBlock();
5477 std::optional<ArrayAttr> alignmentValues = simdOp.getAlignments();
5478 mlir::OperandRange operands = simdOp.getAlignedVars();
5479 for (size_t i = 0; i < operands.size(); ++i) {
5480 llvm::Value *alignment = nullptr;
5481 llvm::Value *llvmVal = moduleTranslation.lookupValue(value: operands[i]);
5482 llvm::Type *ty = llvmVal->getType();
5483
5484 auto intAttr = cast<IntegerAttr>(Val: (*alignmentValues)[i]);
5485 alignment = builder.getInt64(C: intAttr.getInt());
5486 assert(ty->isPointerTy() && "Invalid type for aligned variable");
5487 assert(alignment && "Invalid alignment value");
5488
5489 // Check if the alignment value is not a power of 2. If so, skip emitting
5490 // alignment.
5491 if (!intAttr.getValue().isPowerOf2())
5492 continue;
5493
5494 auto curInsert = builder.saveIP();
5495 builder.SetInsertPoint(sourceBlock);
5496 llvmVal = builder.CreateLoad(Ty: ty, Ptr: llvmVal);
5497 builder.restoreIP(IP: curInsert);
5498 alignedVars[llvmVal] = alignment;
5499 }
5500
5501 llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(
5502 region&: simdOp.getRegion(), blockName: "omp.simd.region", builder, moduleTranslation);
5503
5504 if (failed(Result: handleError(result&: regionBlock, op&: opInst)))
5505 return failure();
5506
5507 llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);
5508 // Emit Initialization for linear variables
5509 if (simdOp.getLinearVars().size()) {
5510 if (failed(Result: handleError(
5511 error: linearClauseProcessor.initLinearVar(builder, moduleTranslation,
5512 loopPreHeader: loopInfo->getPreheader()),
5513 op&: opInst)))
5514 return failure();
5515
5516 if (failed(Result: handleError(error: linearClauseProcessor.updateLinearVar(
5517 builder, moduleTranslation, loopBody: loopInfo->getBody(),
5518 loopInductionVar: loopInfo->getIndVar()),
5519 op&: opInst)))
5520 return failure();
5521 }
5522 builder.SetInsertPoint((*regionBlock)->begin());
5523
5524 ompBuilder->applySimd(Loop: loopInfo, AlignedVars: alignedVars,
5525 IfCond: simdOp.getIfExpr()
5526 ? moduleTranslation.lookupValue(value: simdOp.getIfExpr())
5527 : nullptr,
5528 Order: order, Simdlen: simdlen, Safelen: safelen);
5529
5530 if (failed(Result: handleError(
5531 error: linearClauseProcessor.updateLinearIV(builder, moduleTranslation),
5532 op&: opInst)))
5533 return failure();
5534 if (failed(Result: handleError(error: linearClauseProcessor.emitStoresForLinearVar(
5535 builder, moduleTranslation),
5536 op&: opInst)))
5537 return failure();
5538
5539 // We now need to reduce the per-simd-lane reduction variable into the
5540 // original variable. This works a bit differently to other reductions (e.g.
5541 // wsloop) because we don't need to call into the OpenMP runtime to handle
5542 // threads: everything happened in this one thread.
5543 for (auto [i, var] : llvm::enumerate(First&: reductionInfo.vars)) {
5544 OwningReductionGen gen =
5545 makeReductionGen(decl: var.decl, builder, moduleTranslation);
5546 llvm::Value *originalVariable = moduleTranslation.lookupValue(value: var.mlirVar);
5547 llvm::Type *reductionType =
5548 moduleTranslation.convertType(type: var.decl.getType());
5549
5550 // We have one less load for by-ref case because that load is now inside of
5551 // the reduction region.
5552 llvm::Value *redValue = originalVariable;
5553 if (!var.byRef)
5554 redValue =
5555 builder.CreateLoad(Ty: reductionType, Ptr: redValue, Name: "red.value." + Twine(i));
5556 llvm::Value *privateRedValue = builder.CreateLoad(
5557 Ty: reductionType, Ptr: var.privateVar, Name: "red.private.value." + Twine(i));
5558 llvm::Value *reduced;
5559
5560 auto res = gen(builder.saveIP(), redValue, privateRedValue, reduced);
5561 if (failed(Result: handleError(result&: res, op&: opInst)))
5562 return failure();
5563 builder.restoreIP(IP: res.get());
5564
5565 // For by-ref case, the store is inside of the reduction region.
5566 if (!var.byRef)
5567 builder.CreateStore(Val: reduced, Ptr: originalVariable);
5568 }
5569
5570 // After the construct, deallocate private reduction variables.
5571 SmallVector<Region *> reductionRegions;
5572 llvm::transform(Range&: reductionInfo.vars, d_first: std::back_inserter(x&: reductionRegions),
5573 F: [](ReductionVarsInfo::VarInfo &var) {
5574 return &var.decl.getCleanupRegion();
5575 });
5576 if (failed(Result: inlineOmpRegionCleanup(
5577 cleanupRegions&: reductionRegions, privateVariables: reductionInfo.getPrivateReductionVariables(),
5578 moduleTranslation, builder, regionName: "omp.reduction.cleanup")))
5579 return failure();
5580
5581 return cleanupPrivateVars(op: simdOp, builder, moduleTranslation, loc: simdOp.getLoc(),
5582 privateVarsInfo);
5583}
5584
5585/// Select Virtual only for two constant-count host worksharing loops.
5586static llvm::OpenMPIRBuilder::CollapseIVMapping
5587selectCollapseIVMapping(omp::LoopNestOp loopOp, bool isTargetDevice,
5588 ArrayRef<llvm::CanonicalLoopInfo *> loops) {
5589 using Mapping = llvm::OpenMPIRBuilder::CollapseIVMapping;
5590 if (isTargetDevice || loopOp.getCollapseNumLoops() != 2 ||
5591 disableVirtualCollapse)
5592 return Mapping::DivMod;
5593 auto wsloop = dyn_cast<omp::WsloopOp>(Val: loopOp->getParentOp());
5594 if (!wsloop || isa<omp::LoopWrapperInterface>(Val: wsloop->getParentOp()) ||
5595 wsloop.getScheduleKind().value_or(u: omp::ClauseScheduleKind::Static) !=
5596 omp::ClauseScheduleKind::Static ||
5597 wsloop.getScheduleChunk() || wsloop.getOrdered().has_value() ||
5598 !wsloop.getLinearVars().empty())
5599 return Mapping::DivMod;
5600 // Use the existing normalized counts; the builder also validates direct
5601 // API callers.
5602 if (llvm::any_of(Range&: loops, P: [](llvm::CanonicalLoopInfo *loop) {
5603 auto *count = llvm::dyn_cast<llvm::ConstantInt>(Val: loop->getTripCount());
5604 return !count;
5605 }))
5606 return Mapping::DivMod;
5607 return Mapping::Virtual;
5608}
5609
5610/// Converts an OpenMP loop nest into LLVM IR using OpenMPIRBuilder.
5611static LogicalResult
5612convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
5613 LLVM::ModuleTranslation &moduleTranslation) {
5614 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
5615 auto loopOp = cast<omp::LoopNestOp>(Val&: opInst);
5616
5617 if (failed(Result: checkImplementationStatus(op&: opInst)))
5618 return failure();
5619
5620 // Set up the source location value for OpenMP runtime.
5621 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
5622
5623 // Generator of the canonical loop body.
5624 SmallVector<llvm::CanonicalLoopInfo *> loopInfos;
5625 SmallVector<llvm::OpenMPIRBuilder::InsertPointTy> bodyInsertPoints;
5626 auto bodyGen = [&](llvm::OpenMPIRBuilder::InsertPointTy ip,
5627 llvm::Value *iv) -> llvm::Error {
5628 // Make sure further conversions know about the induction variable.
5629 moduleTranslation.mapValue(
5630 mlir: loopOp.getRegion().front().getArgument(i: loopInfos.size()), llvm: iv);
5631
5632 // Capture the body insertion point for use in nested loops. BodyIP of the
5633 // CanonicalLoopInfo always points to the beginning of the entry block of
5634 // the body.
5635 bodyInsertPoints.push_back(Elt: ip);
5636
5637 if (loopInfos.size() != loopOp.getNumLoops() - 1)
5638 return llvm::Error::success();
5639
5640 // Convert the body of the loop.
5641 builder.restoreIP(IP: ip);
5642 llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(
5643 region&: loopOp.getRegion(), blockName: "omp.loop_nest.region", builder, moduleTranslation);
5644 if (!regionBlock)
5645 return regionBlock.takeError();
5646
5647 builder.SetInsertPoint((*regionBlock)->begin());
5648 return llvm::Error::success();
5649 };
5650
5651 // Delegate actual loop construction to the OpenMP IRBuilder.
5652 // TODO: this currently assumes omp.loop_nest is semantically similar to SCF
5653 // loop, i.e. it has a positive step, uses signed integer semantics.
5654 // Reconsider this code when the nested loop operation clearly supports more
5655 // cases.
5656 for (unsigned i = 0, e = loopOp.getNumLoops(); i < e; ++i) {
5657 llvm::Value *lowerBound =
5658 moduleTranslation.lookupValue(value: loopOp.getLoopLowerBounds()[i]);
5659 llvm::Value *upperBound =
5660 moduleTranslation.lookupValue(value: loopOp.getLoopUpperBounds()[i]);
5661 llvm::Value *step = moduleTranslation.lookupValue(value: loopOp.getLoopSteps()[i]);
5662
5663 // Make sure loop trip count are emitted in the preheader of the outermost
5664 // loop at the latest so that they are all available for the new collapsed
5665 // loop will be created below.
5666 llvm::OpenMPIRBuilder::LocationDescription loc = ompLoc;
5667 llvm::OpenMPIRBuilder::InsertPointTy computeIP = ompLoc.IP;
5668 if (i != 0) {
5669 loc = llvm::OpenMPIRBuilder::LocationDescription(bodyInsertPoints.back(),
5670 ompLoc.DL);
5671 computeIP = loopInfos.front()->getPreheaderIP();
5672 }
5673
5674 llvm::Expected<llvm::CanonicalLoopInfo *> loopResult =
5675 ompBuilder->createCanonicalLoop(
5676 Loc: loc, BodyGenCB: bodyGen, Start: lowerBound, Stop: upperBound, Step: step,
5677 /*IsSigned=*/true, InclusiveStop: loopOp.getLoopInclusive(), ComputeIP: computeIP);
5678
5679 if (failed(Result: handleError(result&: loopResult, op&: *loopOp)))
5680 return failure();
5681
5682 loopInfos.push_back(Elt: *loopResult);
5683 }
5684
5685 llvm::OpenMPIRBuilder::InsertPointTy afterIP =
5686 loopInfos.front()->getAfterIP();
5687
5688 // Do tiling.
5689 if (const auto &tiles = loopOp.getTileSizes()) {
5690 llvm::Type *ivType = loopInfos.front()->getIndVarType();
5691 SmallVector<llvm::Value *> tileSizes;
5692
5693 for (auto tile : tiles.value()) {
5694 llvm::Value *tileVal = llvm::ConstantInt::get(Ty: ivType, V: tile);
5695 tileSizes.push_back(Elt: tileVal);
5696 }
5697
5698 std::vector<llvm::CanonicalLoopInfo *> newLoops =
5699 ompBuilder->tileLoops(DL: ompLoc.DL, Loops: loopInfos, TileSizes: tileSizes);
5700
5701 // Update afterIP to get the correct insertion point after
5702 // tiling.
5703 llvm::BasicBlock *afterBB = newLoops.front()->getAfter();
5704 llvm::BasicBlock *afterAfterBB = afterBB->getSingleSuccessor();
5705 afterIP = afterAfterBB->begin();
5706
5707 // Update the loop infos.
5708 loopInfos.clear();
5709 for (const auto &newLoop : newLoops)
5710 loopInfos.push_back(Elt: newLoop);
5711 } // Tiling done.
5712
5713 // Do collapse.
5714 const auto &numCollapse = loopOp.getCollapseNumLoops();
5715 SmallVector<llvm::CanonicalLoopInfo *> collapseLoopInfos(
5716 loopInfos.begin(), loopInfos.begin() + (numCollapse));
5717
5718 llvm::SaveAndRestore<llvm::OpenMPIRBuilder::CollapseIVMapping> mappingGuard(
5719 ompBuilder->Config.CollapseMapping,
5720 selectCollapseIVMapping(loopOp, isTargetDevice: ompBuilder->Config.isTargetDevice(),
5721 loops: collapseLoopInfos));
5722 auto newTopLoopInfo =
5723 ompBuilder->collapseLoops(DL: ompLoc.DL, Loops: collapseLoopInfos, ComputeIP: {});
5724
5725 assert(newTopLoopInfo && "New top loop information is missing");
5726 moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
5727 callback: [&](OpenMPLoopInfoStackFrame &frame) {
5728 frame.loopInfo = newTopLoopInfo;
5729 return WalkResult::interrupt();
5730 });
5731
5732 // Continue building IR after the loop. Note that the LoopInfo returned by
5733 // `collapseLoops` points inside the outermost loop and is intended for
5734 // potential further loop transformations. Use the insertion point stored
5735 // before collapsing loops instead.
5736 builder.restoreIP(IP: afterIP);
5737 return success();
5738}
5739
5740/// Convert an omp.canonical_loop to LLVM-IR
5741static LogicalResult
5742convertOmpCanonicalLoopOp(omp::CanonicalLoopOp op, llvm::IRBuilderBase &builder,
5743 LLVM::ModuleTranslation &moduleTranslation) {
5744 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
5745
5746 llvm::OpenMPIRBuilder::LocationDescription loopLoc(builder);
5747 Value loopIV = op.getInductionVar();
5748 Value loopTC = op.getTripCount();
5749
5750 llvm::Value *llvmTC = moduleTranslation.lookupValue(value: loopTC);
5751
5752 llvm::Expected<llvm::CanonicalLoopInfo *> llvmOrError =
5753 ompBuilder->createCanonicalLoop(
5754 Loc: loopLoc,
5755 BodyGenCB: [&](llvm::OpenMPIRBuilder::InsertPointTy ip, llvm::Value *llvmIV) {
5756 // Register the mapping of MLIR induction variable to LLVM-IR
5757 // induction variable
5758 moduleTranslation.mapValue(mlir: loopIV, llvm: llvmIV);
5759
5760 builder.restoreIP(IP: ip);
5761 llvm::Expected<llvm::BasicBlock *> bodyGenStatus =
5762 convertOmpOpRegions(region&: op.getRegion(), blockName: "omp.loop.region", builder,
5763 moduleTranslation);
5764
5765 return bodyGenStatus.takeError();
5766 },
5767 TripCount: llvmTC, Name: "omp.loop");
5768 if (!llvmOrError)
5769 return op.emitError(message: llvm::toString(E: llvmOrError.takeError()));
5770
5771 llvm::CanonicalLoopInfo *llvmCLI = *llvmOrError;
5772 llvm::IRBuilderBase::InsertPoint afterIP = llvmCLI->getAfterIP();
5773 builder.restoreIP(IP: afterIP);
5774
5775 // Register the mapping of MLIR loop to LLVM-IR OpenMPIRBuilder loop
5776 if (Value cli = op.getCli())
5777 mapOmpLoop(moduleTranslation, mlir: cli, llvm: llvmCLI);
5778
5779 return success();
5780}
5781
5782/// Apply a `#pragma omp unroll` / "!$omp unroll" transformation using the
5783/// OpenMPIRBuilder.
5784static LogicalResult
5785applyUnrollHeuristic(omp::UnrollHeuristicOp op, llvm::IRBuilderBase &builder,
5786 LLVM::ModuleTranslation &moduleTranslation) {
5787 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
5788
5789 Value applyee = op.getApplyee();
5790 assert(applyee && "Loop to apply unrolling on required");
5791
5792 llvm::CanonicalLoopInfo *consBuilderCLI =
5793 lookupOMPLoop(moduleTranslation, mlir: applyee);
5794 llvm::OpenMPIRBuilder::LocationDescription loc(builder);
5795 ompBuilder->unrollLoopHeuristic(DL: loc.DL, Loop: consBuilderCLI);
5796
5797 invalidateOmpLoop(moduleTranslation, mlir: applyee);
5798 return success();
5799}
5800
5801/// Apply a `#pragma omp unroll full` / `!$omp unroll full` transformation
5802/// using the OpenMPIRBuilder.
5803static LogicalResult
5804applyUnrollFull(omp::UnrollFullOp op, llvm::IRBuilderBase &builder,
5805 LLVM::ModuleTranslation &moduleTranslation) {
5806 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
5807
5808 Value applyee = op.getApplyee();
5809 assert(applyee && "Loop to apply unrolling on required");
5810
5811 llvm::CanonicalLoopInfo *consBuilderCLI =
5812 lookupOMPLoop(moduleTranslation, mlir: applyee);
5813 llvm::OpenMPIRBuilder::LocationDescription loc(builder);
5814 ompBuilder->unrollLoopFull(DL: loc.DL, Loop: consBuilderCLI);
5815
5816 invalidateOmpLoop(moduleTranslation, mlir: applyee);
5817 return success();
5818}
5819
5820/// Apply a `#pragma omp unroll partial` / `!$omp unroll partial`
5821/// transformation using the OpenMPIRBuilder.
5822static LogicalResult
5823applyUnrollPartial(omp::UnrollPartialOp op, llvm::IRBuilderBase &builder,
5824 LLVM::ModuleTranslation &moduleTranslation) {
5825 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
5826
5827 Value applyee = op.getApplyee();
5828 assert(applyee && "Loop to apply unrolling on required");
5829
5830 llvm::CanonicalLoopInfo *consBuilderCLI =
5831 lookupOMPLoop(moduleTranslation, mlir: applyee);
5832 llvm::OpenMPIRBuilder::LocationDescription loc(builder);
5833
5834 // No generatee is supported yet, so the unrolled loop's CanonicalLoopInfo is
5835 // not requested and unrolling is deferred to LLVM's LoopUnroll pass.
5836 int32_t factor = static_cast<int32_t>(op.getUnrollFactor());
5837 ompBuilder->unrollLoopPartial(DL: loc.DL, Loop: consBuilderCLI, Factor: factor,
5838 /*UnrolledCLI=*/nullptr);
5839
5840 invalidateOmpLoop(moduleTranslation, mlir: applyee);
5841 return success();
5842}
5843
5844/// Apply a `#pragma omp tile` / `!$omp tile` transformation using the
5845/// OpenMPIRBuilder.
5846static LogicalResult applyTile(omp::TileOp op, llvm::IRBuilderBase &builder,
5847 LLVM::ModuleTranslation &moduleTranslation) {
5848 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
5849 llvm::OpenMPIRBuilder::LocationDescription loc(builder);
5850
5851 SmallVector<llvm::CanonicalLoopInfo *> translatedLoops;
5852 SmallVector<llvm::Value *> translatedSizes;
5853
5854 for (Value size : op.getSizes()) {
5855 llvm::Value *translatedSize = moduleTranslation.lookupValue(value: size);
5856 assert(translatedSize &&
5857 "sizes clause arguments must already be translated");
5858 translatedSizes.push_back(Elt: translatedSize);
5859 }
5860
5861 for (Value applyee : op.getApplyees()) {
5862 llvm::CanonicalLoopInfo *consBuilderCLI =
5863 lookupOMPLoop(moduleTranslation, mlir: applyee);
5864 assert(applyee && "Canonical loop must already been translated");
5865 translatedLoops.push_back(Elt: consBuilderCLI);
5866 }
5867
5868 auto generatedLoops =
5869 ompBuilder->tileLoops(DL: loc.DL, Loops: translatedLoops, TileSizes: translatedSizes);
5870 if (!op.getGeneratees().empty()) {
5871 for (auto [mlirLoop, genLoop] :
5872 zip_equal(t: op.getGeneratees(), u&: generatedLoops))
5873 mapOmpLoop(moduleTranslation, mlir: mlirLoop, llvm: genLoop);
5874 }
5875
5876 // CLIs can only be consumed once
5877 for (Value applyee : op.getApplyees())
5878 invalidateOmpLoop(moduleTranslation, mlir: applyee);
5879
5880 return success();
5881}
5882
5883/// Apply a `#pragma omp fuse` / `!$omp fuse` transformation using the
5884/// OpenMPIRBuilder.
5885static LogicalResult applyFuse(omp::FuseOp op, llvm::IRBuilderBase &builder,
5886 LLVM::ModuleTranslation &moduleTranslation) {
5887 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
5888 llvm::OpenMPIRBuilder::LocationDescription loc(builder);
5889
5890 // Select what CLIs are going to be fused
5891 SmallVector<llvm::CanonicalLoopInfo *> beforeFuse, toFuse, afterFuse;
5892 for (size_t i = 0; i < op.getApplyees().size(); i++) {
5893 Value applyee = op.getApplyees()[i];
5894 llvm::CanonicalLoopInfo *consBuilderCLI =
5895 lookupOMPLoop(moduleTranslation, mlir: applyee);
5896 assert(applyee && "Canonical loop must already been translated");
5897 if (op.getFirst().has_value() && i < op.getFirst().value() - 1)
5898 beforeFuse.push_back(Elt: consBuilderCLI);
5899 else if (op.getCount().has_value() &&
5900 i >= op.getFirst().value() + op.getCount().value() - 1)
5901 afterFuse.push_back(Elt: consBuilderCLI);
5902 else
5903 toFuse.push_back(Elt: consBuilderCLI);
5904 }
5905 assert(
5906 (op.getGeneratees().empty() ||
5907 beforeFuse.size() + afterFuse.size() + 1 == op.getGeneratees().size()) &&
5908 "Wrong number of generatees");
5909
5910 // do the fuse
5911 auto generatedLoop = ompBuilder->fuseLoops(DL: loc.DL, Loops: toFuse);
5912 if (!op.getGeneratees().empty()) {
5913 size_t i = 0;
5914 for (; i < beforeFuse.size(); i++)
5915 mapOmpLoop(moduleTranslation, mlir: op.getGeneratees()[i], llvm: beforeFuse[i]);
5916 mapOmpLoop(moduleTranslation, mlir: op.getGeneratees()[i++], llvm: generatedLoop);
5917 for (; i < afterFuse.size(); i++)
5918 mapOmpLoop(moduleTranslation, mlir: op.getGeneratees()[i], llvm: afterFuse[i]);
5919 }
5920
5921 // CLIs can only be consumed once
5922 for (Value applyee : op.getApplyees())
5923 invalidateOmpLoop(moduleTranslation, mlir: applyee);
5924
5925 return success();
5926}
5927
5928/// Convert an Atomic Ordering attribute to llvm::AtomicOrdering.
5929static llvm::AtomicOrdering
5930convertAtomicOrdering(std::optional<omp::ClauseMemoryOrderKind> ao) {
5931 if (!ao)
5932 return llvm::AtomicOrdering::Monotonic; // Default Memory Ordering
5933
5934 switch (*ao) {
5935 case omp::ClauseMemoryOrderKind::Seq_cst:
5936 return llvm::AtomicOrdering::SequentiallyConsistent;
5937 case omp::ClauseMemoryOrderKind::Acq_rel:
5938 return llvm::AtomicOrdering::AcquireRelease;
5939 case omp::ClauseMemoryOrderKind::Acquire:
5940 return llvm::AtomicOrdering::Acquire;
5941 case omp::ClauseMemoryOrderKind::Release:
5942 return llvm::AtomicOrdering::Release;
5943 case omp::ClauseMemoryOrderKind::Relaxed:
5944 return llvm::AtomicOrdering::Monotonic;
5945 }
5946 llvm_unreachable("Unknown ClauseMemoryOrderKind kind");
5947}
5948
5949/// Compute the cmpxchg failure ordering for an atomic compare op: use the
5950/// `fail` clause ordering when present (the verifier guarantees it is a valid
5951/// cmpxchg failure ordering), otherwise the strongest failure ordering derived
5952/// from the success ordering (which matches the OpenMPIRBuilder default).
5953static llvm::AtomicOrdering
5954getAtomicCompareFailureOrdering(omp::AtomicCompareOp atomicCompareOp,
5955 llvm::AtomicOrdering atomicOrdering) {
5956 if (atomicCompareOp.getFailMemoryOrder())
5957 return convertAtomicOrdering(ao: atomicCompareOp.getFailMemoryOrder());
5958 return llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: atomicOrdering);
5959}
5960
5961/// Convert omp.atomic.read operation to LLVM IR.
5962static LogicalResult
5963convertOmpAtomicRead(Operation &opInst, llvm::IRBuilderBase &builder,
5964 LLVM::ModuleTranslation &moduleTranslation) {
5965 auto readOp = cast<omp::AtomicReadOp>(Val&: opInst);
5966 if (failed(Result: checkImplementationStatus(op&: opInst)))
5967 return failure();
5968
5969 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
5970 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
5971 findAllocInsertPoints(builder, moduleTranslation);
5972
5973 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
5974
5975 llvm::AtomicOrdering AO = convertAtomicOrdering(ao: readOp.getMemoryOrder());
5976 llvm::Value *x = moduleTranslation.lookupValue(value: readOp.getX());
5977 llvm::Value *v = moduleTranslation.lookupValue(value: readOp.getV());
5978
5979 llvm::Type *elementType =
5980 moduleTranslation.convertType(type: readOp.getElementType());
5981
5982 llvm::OpenMPIRBuilder::AtomicOpValue V = {.Var: v, .ElemTy: elementType, .IsSigned: false, .IsVolatile: false};
5983 llvm::OpenMPIRBuilder::AtomicOpValue X = {.Var: x, .ElemTy: elementType, .IsSigned: false, .IsVolatile: false};
5984 builder.restoreIP(IP: ompBuilder->createAtomicRead(Loc: ompLoc, X, V, AO, AllocaIP: allocaIP));
5985 return success();
5986}
5987
5988/// Converts an omp.atomic.write operation to LLVM IR.
5989static LogicalResult
5990convertOmpAtomicWrite(Operation &opInst, llvm::IRBuilderBase &builder,
5991 LLVM::ModuleTranslation &moduleTranslation) {
5992 auto writeOp = cast<omp::AtomicWriteOp>(Val&: opInst);
5993 if (failed(Result: checkImplementationStatus(op&: opInst)))
5994 return failure();
5995
5996 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
5997 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
5998 findAllocInsertPoints(builder, moduleTranslation);
5999
6000 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
6001 llvm::AtomicOrdering ao = convertAtomicOrdering(ao: writeOp.getMemoryOrder());
6002 llvm::Value *expr = moduleTranslation.lookupValue(value: writeOp.getExpr());
6003 llvm::Value *dest = moduleTranslation.lookupValue(value: writeOp.getX());
6004 llvm::Type *ty = moduleTranslation.convertType(type: writeOp.getExpr().getType());
6005 llvm::OpenMPIRBuilder::AtomicOpValue x = {.Var: dest, .ElemTy: ty, /*isSigned=*/.IsSigned: false,
6006 /*isVolatile=*/.IsVolatile: false};
6007 builder.restoreIP(
6008 IP: ompBuilder->createAtomicWrite(Loc: ompLoc, X&: x, Expr: expr, AO: ao, AllocaIP: allocaIP));
6009 return success();
6010}
6011
6012/// Converts an LLVM dialect binary operation to the corresponding enum value
6013/// for `atomicrmw` supported binary operation.
6014static llvm::AtomicRMWInst::BinOp convertBinOpToAtomic(Operation &op) {
6015 return llvm::TypeSwitch<Operation *, llvm::AtomicRMWInst::BinOp>(&op)
6016 .Case(caseFn: [&](LLVM::AddOp) { return llvm::AtomicRMWInst::BinOp::Add; })
6017 .Case(caseFn: [&](LLVM::SubOp) { return llvm::AtomicRMWInst::BinOp::Sub; })
6018 .Case(caseFn: [&](LLVM::AndOp) { return llvm::AtomicRMWInst::BinOp::And; })
6019 .Case(caseFn: [&](LLVM::OrOp) { return llvm::AtomicRMWInst::BinOp::Or; })
6020 .Case(caseFn: [&](LLVM::XOrOp) { return llvm::AtomicRMWInst::BinOp::Xor; })
6021 .Case(caseFn: [&](LLVM::SMaxOp) { return llvm::AtomicRMWInst::BinOp::Max; })
6022 .Case(caseFn: [&](LLVM::SMinOp) { return llvm::AtomicRMWInst::BinOp::Min; })
6023 .Case(caseFn: [&](LLVM::UMaxOp) { return llvm::AtomicRMWInst::BinOp::UMax; })
6024 .Case(caseFn: [&](LLVM::UMinOp) { return llvm::AtomicRMWInst::BinOp::UMin; })
6025 .Case(caseFn: [&](LLVM::FAddOp) { return llvm::AtomicRMWInst::BinOp::FAdd; })
6026 .Case(caseFn: [&](LLVM::FSubOp) { return llvm::AtomicRMWInst::BinOp::FSub; })
6027 .Case(caseFn: [&](LLVM::MaxNumOp) { return llvm::AtomicRMWInst::BinOp::FMax; })
6028 .Case(caseFn: [&](LLVM::MinNumOp) { return llvm::AtomicRMWInst::BinOp::FMin; })
6029 .Case(
6030 caseFn: [&](LLVM::MaximumOp) { return llvm::AtomicRMWInst::BinOp::FMaximum; })
6031 .Case(
6032 caseFn: [&](LLVM::MinimumOp) { return llvm::AtomicRMWInst::BinOp::FMinimum; })
6033 .Default(defaultResult: llvm::AtomicRMWInst::BinOp::BAD_BINOP);
6034}
6035
6036static void extractAtomicControlFlags(omp::AtomicUpdateOp atomicUpdateOp,
6037 bool &isIgnoreDenormalMode,
6038 bool &isFineGrainedMemory,
6039 bool &isRemoteMemory) {
6040 isIgnoreDenormalMode = false;
6041 isFineGrainedMemory = false;
6042 isRemoteMemory = false;
6043 if (atomicUpdateOp && atomicUpdateOp.getAtomicControlAttr()) {
6044 mlir::omp::AtomicControlAttr atomicControlAttr =
6045 atomicUpdateOp.getAtomicControlAttr();
6046 isIgnoreDenormalMode = atomicControlAttr.getIgnoreDenormalMode();
6047 isFineGrainedMemory = atomicControlAttr.getFineGrainedMemory();
6048 isRemoteMemory = atomicControlAttr.getRemoteMemory();
6049 }
6050}
6051
6052/// Converts an OpenMP atomic update operation using OpenMPIRBuilder.
6053static LogicalResult
6054convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
6055 llvm::IRBuilderBase &builder,
6056 LLVM::ModuleTranslation &moduleTranslation) {
6057 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
6058 if (failed(Result: checkImplementationStatus(op&: *opInst)))
6059 return failure();
6060
6061 // Convert values and types.
6062 auto &innerOpList = opInst.getRegion().front().getOperations();
6063 bool isXBinopExpr{false};
6064 llvm::AtomicRMWInst::BinOp binop;
6065 mlir::Value mlirExpr;
6066 llvm::Value *llvmExpr = nullptr;
6067 llvm::Value *llvmX = nullptr;
6068 llvm::Type *llvmXElementType = nullptr;
6069 if (innerOpList.size() == 2) {
6070 // The two operations here are the update and the terminator.
6071 // Since we can identify the update operation, there is a possibility
6072 // that we can generate the atomicrmw instruction.
6073 mlir::Operation &innerOp = *opInst.getRegion().front().begin();
6074 if (!llvm::is_contained(Range: innerOp.getOperands(),
6075 Element: opInst.getRegion().getArgument(i: 0))) {
6076 return opInst.emitError(message: "no atomic update operation with region argument"
6077 " as operand found inside atomic.update region");
6078 }
6079 binop = convertBinOpToAtomic(op&: innerOp);
6080 isXBinopExpr = innerOp.getOperand(idx: 0) == opInst.getRegion().getArgument(i: 0);
6081 mlirExpr = (isXBinopExpr ? innerOp.getOperand(idx: 1) : innerOp.getOperand(idx: 0));
6082 llvmExpr = moduleTranslation.lookupValue(value: mlirExpr);
6083 } else {
6084 // Since the update region includes more than one operation
6085 // we will resort to generating a cmpxchg loop.
6086 binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
6087 }
6088 llvmX = moduleTranslation.lookupValue(value: opInst.getX());
6089 llvmXElementType = moduleTranslation.convertType(
6090 type: opInst.getRegion().getArgument(i: 0).getType());
6091 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {.Var: llvmX, .ElemTy: llvmXElementType,
6092 /*isSigned=*/.IsSigned: false,
6093 /*isVolatile=*/.IsVolatile: false};
6094
6095 llvm::AtomicOrdering atomicOrdering =
6096 convertAtomicOrdering(ao: opInst.getMemoryOrder());
6097
6098 // Generate update code.
6099 auto updateFn =
6100 [&opInst, &moduleTranslation](
6101 llvm::Value *atomicx,
6102 llvm::IRBuilder<> &builder) -> llvm::Expected<llvm::Value *> {
6103 Block &bb = *opInst.getRegion().begin();
6104 moduleTranslation.mapValue(mlir: *opInst.getRegion().args_begin(), llvm: atomicx);
6105 moduleTranslation.mapBlock(mlir: &bb, llvm: builder.GetInsertBlock());
6106 if (failed(Result: moduleTranslation.convertBlock(bb, ignoreArguments: true, builder)))
6107 return llvm::make_error<PreviouslyReportedError>();
6108
6109 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(Val: bb.getTerminator());
6110 assert(yieldop && yieldop.getResults().size() == 1 &&
6111 "terminator must be omp.yield op and it must have exactly one "
6112 "argument");
6113 return moduleTranslation.lookupValue(value: yieldop.getResults()[0]);
6114 };
6115
6116 bool isIgnoreDenormalMode;
6117 bool isFineGrainedMemory;
6118 bool isRemoteMemory;
6119 extractAtomicControlFlags(atomicUpdateOp: opInst, isIgnoreDenormalMode, isFineGrainedMemory,
6120 isRemoteMemory);
6121 // Handle ambiguous alloca, if any.
6122 auto allocaIP = findAllocInsertPoints(builder, moduleTranslation);
6123 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
6124 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
6125 ompBuilder->createAtomicUpdate(Loc: ompLoc, AllocaIP: allocaIP, X&: llvmAtomicX, Expr: llvmExpr,
6126 AO: atomicOrdering, RMWOp: binop, UpdateOp: updateFn,
6127 IsXBinopExpr: isXBinopExpr, IsIgnoreDenormalMode: isIgnoreDenormalMode,
6128 IsFineGrainedMemory: isFineGrainedMemory, IsRemoteMemory: isRemoteMemory);
6129
6130 if (failed(Result: handleError(result&: afterIP, op&: *opInst)))
6131 return failure();
6132
6133 builder.restoreIP(IP: *afterIP);
6134 return success();
6135}
6136
6137/// Helper to extract the OMPAtomicCompareOp from an integer comparison
6138/// predicate. Returns std::nullopt for unsupported predicates.
6139static std::optional<llvm::omp::OMPAtomicCompareOp>
6140convertICmpPredicateToAtomicCompareOp(LLVM::ICmpPredicate predicate) {
6141 switch (predicate) {
6142 case LLVM::ICmpPredicate::eq:
6143 return llvm::omp::OMPAtomicCompareOp::EQ;
6144 case LLVM::ICmpPredicate::slt:
6145 case LLVM::ICmpPredicate::ult:
6146 return llvm::omp::OMPAtomicCompareOp::MIN;
6147 case LLVM::ICmpPredicate::sgt:
6148 case LLVM::ICmpPredicate::ugt:
6149 return llvm::omp::OMPAtomicCompareOp::MAX;
6150 default:
6151 return std::nullopt;
6152 }
6153}
6154
6155/// Helper to extract the OMPAtomicCompareOp from a floating-point comparison
6156/// predicate. Returns std::nullopt for unsupported predicates.
6157static std::optional<llvm::omp::OMPAtomicCompareOp>
6158convertFCmpPredicateToAtomicCompareOp(LLVM::FCmpPredicate predicate) {
6159 switch (predicate) {
6160 case LLVM::FCmpPredicate::oeq:
6161 case LLVM::FCmpPredicate::ueq:
6162 return llvm::omp::OMPAtomicCompareOp::EQ;
6163 case LLVM::FCmpPredicate::olt:
6164 case LLVM::FCmpPredicate::ult:
6165 return llvm::omp::OMPAtomicCompareOp::MIN;
6166 case LLVM::FCmpPredicate::ogt:
6167 case LLVM::FCmpPredicate::ugt:
6168 return llvm::omp::OMPAtomicCompareOp::MAX;
6169 default:
6170 return std::nullopt;
6171 }
6172}
6173
6174/// Result of matching the decomposed complex equality pattern inside an atomic
6175/// compare region.
6176struct ComplexComparePattern {
6177 bool isComplex = false;
6178 bool isNE = false; // `or` of the field compares => NE (unsupported).
6179 mlir::Value eAggregate; // The complex expected value (`e`).
6180 bool isXBinopExpr = false; // True if x is the first fcmp operand.
6181};
6182
6183/// Detect a decomposed complex equality comparison in an atomic compare region:
6184/// %re_x = llvm.extractvalue %xval[0]
6185/// %re_e = llvm.extractvalue %eStruct[0]
6186/// %cmp_re = llvm.fcmp "oeq" %re_x, %re_e
6187/// %im_x = llvm.extractvalue %xval[1]
6188/// %im_e = llvm.extractvalue %eStruct[1]
6189/// %cmp_im = llvm.fcmp "oeq" %im_x, %im_e
6190/// %cmp = llvm.and %cmp_re, %cmp_im (llvm.or would be NE)
6191/// It is recognised by an and/or whose operands are both fcmps operating on
6192/// extractvalues, one chain rooted at the block argument (x) and the other at
6193/// the expected complex value (e).
6194static ComplexComparePattern detectComplexCompareEq(Block &block) {
6195 ComplexComparePattern result;
6196 auto traceToAggregate = [](mlir::Value v) -> mlir::Value {
6197 if (auto extractOp = v.getDefiningOp<LLVM::ExtractValueOp>())
6198 return extractOp.getContainer();
6199 return nullptr;
6200 };
6201 for (Operation &op : block.getOperations()) {
6202 if (!isa<LLVM::AndOp, LLVM::OrOp>(Val: op))
6203 continue;
6204 auto lhsFcmp = op.getOperand(idx: 0).getDefiningOp<LLVM::FCmpOp>();
6205 auto rhsFcmp = op.getOperand(idx: 1).getDefiningOp<LLVM::FCmpOp>();
6206 if (!lhsFcmp || !rhsFcmp)
6207 continue;
6208 mlir::Value lhsAgg0 = traceToAggregate(lhsFcmp.getOperand(i: 0));
6209 mlir::Value lhsAgg1 = traceToAggregate(lhsFcmp.getOperand(i: 1));
6210 bool lhsXIsOp0 = (lhsAgg0 == block.getArgument(i: 0));
6211 bool lhsXIsOp1 = (lhsAgg1 == block.getArgument(i: 0));
6212 if (!lhsXIsOp0 && !lhsXIsOp1)
6213 continue;
6214 mlir::Value eAggregate = lhsXIsOp0 ? lhsAgg1 : lhsAgg0;
6215 if (!eAggregate)
6216 continue;
6217 result.isComplex = true;
6218 result.isNE = isa<LLVM::OrOp>(Val: op);
6219 result.eAggregate = eAggregate;
6220 result.isXBinopExpr = lhsXIsOp0;
6221 break;
6222 }
6223 return result;
6224}
6225
6226/// Emit an IEEE-754-correct `cmpxchg` for a complex (struct-typed) atomic
6227/// compare with `fcmp oeq`. The old value of X is returned (as the complex
6228/// struct type) in \p oldComplex and the success flag (i1) in \p cmpOk.
6229/// \p failOrdering is the memory ordering used when the compare-exchange does
6230/// not store; it must be a valid cmpxchg failure ordering.
6231static void emitComplexAtomicCmpXchg(llvm::IRBuilderBase &builder,
6232 llvm::Value *llvmX, llvm::Type *complexTy,
6233 llvm::Value *eVal, llvm::Value *dVal,
6234 llvm::AtomicOrdering atomicOrdering,
6235 llvm::AtomicOrdering failOrdering,
6236 bool isWeak, llvm::Value *&oldComplex,
6237 llvm::Value *&cmpOk) {
6238 const llvm::DataLayout &DL = builder.getDataLayout();
6239 unsigned totalBits = DL.getTypeStoreSizeInBits(Ty: complexTy).getFixedValue();
6240 llvm::IntegerType *intTy =
6241 llvm::IntegerType::get(C&: builder.getContext(), NumBits: totalBits);
6242 llvm::Align complexAlign = DL.getABITypeAlign(Ty: complexTy);
6243 llvm::Align intAlign = DL.getABITypeAlign(Ty: intTy);
6244 llvm::Align maxAlign = std::max(a: complexAlign, b: intAlign);
6245
6246 // Spill D to obtain its integer bit pattern for the swap value.
6247 llvm::AllocaInst *dAlloca =
6248 builder.CreateAlloca(Ty: complexTy, ArraySize: nullptr, Name: "cmplx.d");
6249 dAlloca->setAlignment(maxAlign);
6250 builder.CreateAlignedStore(Val: dVal, Ptr: dAlloca, Align: maxAlign);
6251 llvm::Value *dInt =
6252 builder.CreateAlignedLoad(Ty: intTy, Ptr: dAlloca, Align: maxAlign, Name: "cmplx.d.int");
6253
6254 // Load X atomically and reinterpret as complex. Use the failure ordering: on
6255 // a failed component comparison we branch around the cmpxchg, so this load is
6256 // the only memory op on that path.
6257 llvm::LoadInst *xCurr =
6258 builder.CreateAlignedLoad(Ty: intTy, Ptr: llvmX, Align: maxAlign, Name: "cmplx.x.load");
6259 xCurr->setAtomic(Ordering: failOrdering);
6260 llvm::AllocaInst *xAlloca =
6261 builder.CreateAlloca(Ty: complexTy, ArraySize: nullptr, Name: "cmplx.x");
6262 xAlloca->setAlignment(maxAlign);
6263 builder.CreateAlignedStore(Val: xCurr, Ptr: xAlloca, Align: maxAlign);
6264 llvm::Value *xStruct =
6265 builder.CreateAlignedLoad(Ty: complexTy, Ptr: xAlloca, Align: maxAlign, Name: "cmplx.x.val");
6266
6267 // Component-wise IEEE-754 equality: `fcmp oeq` yields false for NaN (so a
6268 // NaN component correctly makes the compare fail) and true for +0.0 vs -0.0
6269 // (so a zero-sign difference does not spuriously fail the compare).
6270 llvm::Value *reX = builder.CreateExtractValue(Agg: xStruct, Idxs: 0);
6271 llvm::Value *imX = builder.CreateExtractValue(Agg: xStruct, Idxs: 1);
6272 llvm::Value *reE = builder.CreateExtractValue(Agg: eVal, Idxs: 0);
6273 llvm::Value *imE = builder.CreateExtractValue(Agg: eVal, Idxs: 1);
6274 llvm::Value *reEq = builder.CreateFCmpOEQ(LHS: reX, RHS: reE, Name: "cmplx.re.eq");
6275 llvm::Value *imEq = builder.CreateFCmpOEQ(LHS: imX, RHS: imE, Name: "cmplx.im.eq");
6276 llvm::Value *fpEqual = builder.CreateAnd(LHS: reEq, RHS: imEq, Name: "cmplx.eq");
6277
6278 // When the components compare equal, attempt the swap using X's own loaded
6279 // bit pattern as the comparand; otherwise the compare fails and X is left
6280 // unchanged (the captured old value is the value just loaded).
6281 llvm::BasicBlock *curBB = builder.GetInsertBlock();
6282 llvm::Function *fn = curBB->getParent();
6283 llvm::BasicBlock *swapBB =
6284 llvm::BasicBlock::Create(Context&: builder.getContext(), Name: "cmplx.atomic.swap", Parent: fn);
6285 llvm::BasicBlock *exitBB =
6286 llvm::BasicBlock::Create(Context&: builder.getContext(), Name: "cmplx.atomic.exit", Parent: fn);
6287 builder.CreateCondBr(Cond: fpEqual, True: swapBB, False: exitBB);
6288
6289 builder.SetInsertPoint(swapBB);
6290 llvm::AtomicCmpXchgInst *cmpXchg = builder.CreateAtomicCmpXchg(
6291 Ptr: llvmX, Cmp: xCurr, New: dInt, Align: maxAlign, SuccessOrdering: atomicOrdering, FailureOrdering: failOrdering);
6292 cmpXchg->setWeak(isWeak);
6293 llvm::Value *oldSwap = builder.CreateExtractValue(Agg: cmpXchg, Idxs: 0);
6294 llvm::Value *okSwap = builder.CreateExtractValue(Agg: cmpXchg, Idxs: 1);
6295 builder.CreateBr(Dest: exitBB);
6296
6297 // Merge the swap and no-swap paths.
6298 builder.SetInsertPoint(exitBB);
6299 llvm::PHINode *oldIntPHI = builder.CreatePHI(Ty: intTy, NumReservedValues: 2, Name: "cmplx.old.int");
6300 oldIntPHI->addIncoming(V: oldSwap, BB: swapBB);
6301 oldIntPHI->addIncoming(V: xCurr, BB: curBB);
6302 llvm::PHINode *okPHI = builder.CreatePHI(Ty: builder.getInt1Ty(), NumReservedValues: 2, Name: "cmplx.ok");
6303 okPHI->addIncoming(V: okSwap, BB: swapBB);
6304 okPHI->addIncoming(V: builder.getFalse(), BB: curBB);
6305
6306 // Reinterpret the old integer value as the complex struct via memory.
6307 llvm::AllocaInst *oldAlloca =
6308 builder.CreateAlloca(Ty: complexTy, ArraySize: nullptr, Name: "cmplx.old");
6309 oldAlloca->setAlignment(maxAlign);
6310 builder.CreateAlignedStore(Val: oldIntPHI, Ptr: oldAlloca, Align: maxAlign);
6311 oldComplex = builder.CreateAlignedLoad(Ty: complexTy, Ptr: oldAlloca, Align: maxAlign,
6312 Name: "cmplx.old.val");
6313 cmpOk = okPHI;
6314}
6315
6316/// Holds the extracted comparison pattern information from an atomic compare
6317/// region.
6318struct AtomicComparePatternInfo {
6319 llvm::omp::OMPAtomicCompareOp compareOp = llvm::omp::OMPAtomicCompareOp::EQ;
6320 llvm::Value *eVal = nullptr;
6321 llvm::Value *dVal = nullptr;
6322 bool isXBinopExpr = false;
6323 bool isSigned = false;
6324};
6325/// Extract comparison predicate, expected value (e), desired value (d), and
6326/// related flags from an atomic compare region block by scanning for
6327/// icmp/fcmp/select/min/max operations.
6328static LogicalResult extractAtomicComparePattern(
6329 Block &block,
6330 llvm::function_ref<llvm::Value *(mlir::Value)> materializeValue,
6331 omp::AtomicCompareOp atomicCompareOp, AtomicComparePatternInfo &info) {
6332 // Complex equality is a decomposed per-field pattern (extractvalue + fcmp +
6333 // and) rather than a single scalar compare. Detect it first so the scalar
6334 // icmp/fcmp handling below does not mistake a real/imaginary field for the
6335 // whole expected value.
6336 if (ComplexComparePattern cplx = detectComplexCompareEq(block);
6337 cplx.isComplex) {
6338 if (cplx.isNE)
6339 return atomicCompareOp.emitError(
6340 message: "unsupported comparison predicate (NE) for complex atomic compare");
6341 info.compareOp = llvm::omp::OMPAtomicCompareOp::EQ;
6342 info.isXBinopExpr = cplx.isXBinopExpr;
6343 info.eVal = materializeValue(cplx.eAggregate);
6344 for (Operation &op : block.getOperations()) {
6345 if (auto selectOp = dyn_cast<LLVM::SelectOp>(Val&: op)) {
6346 info.dVal = materializeValue(selectOp.getTrueValue());
6347 break;
6348 }
6349 }
6350 return success();
6351 }
6352
6353 for (Operation &op : block.getOperations()) {
6354 // Pre-filter: skip icmps that don't involve the block argument
6355 // (e.g., truthiness extractions from logical-to-integer conversion).
6356 if (auto icmpOp = dyn_cast<LLVM::ICmpOp>(Val&: op);
6357 icmpOp && icmpOp.getOperand(i: 0) != block.getArgument(i: 0) &&
6358 icmpOp.getOperand(i: 1) != block.getArgument(i: 0))
6359 continue;
6360
6361 LogicalResult result =
6362 llvm::TypeSwitch<Operation *, LogicalResult>(&op)
6363 .Case<LLVM::ICmpOp>(caseFn: [&](LLVM::ICmpOp icmpOp) -> LogicalResult {
6364 auto maybeOp =
6365 convertICmpPredicateToAtomicCompareOp(predicate: icmpOp.getPredicate());
6366 if (!maybeOp)
6367 return atomicCompareOp.emitError(
6368 message: "unsupported comparison predicate in atomic compare");
6369 info.compareOp = *maybeOp;
6370 LLVM::ICmpPredicate pred = icmpOp.getPredicate();
6371 info.isSigned = (pred == LLVM::ICmpPredicate::slt ||
6372 pred == LLVM::ICmpPredicate::sgt ||
6373 pred == LLVM::ICmpPredicate::sle ||
6374 pred == LLVM::ICmpPredicate::sge);
6375 info.isXBinopExpr =
6376 (icmpOp.getOperand(i: 0) == block.getArgument(i: 0));
6377 mlir::Value eOperand = info.isXBinopExpr ? icmpOp.getOperand(i: 1)
6378 : icmpOp.getOperand(i: 0);
6379 info.eVal = materializeValue(eOperand);
6380 return success();
6381 })
6382 .Case<LLVM::FCmpOp>(caseFn: [&](LLVM::FCmpOp fcmpOp) -> LogicalResult {
6383 auto maybeOp =
6384 convertFCmpPredicateToAtomicCompareOp(predicate: fcmpOp.getPredicate());
6385 if (!maybeOp)
6386 return atomicCompareOp.emitError(
6387 message: "unsupported comparison predicate in atomic compare");
6388 info.compareOp = *maybeOp;
6389 info.isXBinopExpr =
6390 (fcmpOp.getOperand(i: 0) == block.getArgument(i: 0));
6391 mlir::Value eOperand = info.isXBinopExpr ? fcmpOp.getOperand(i: 1)
6392 : fcmpOp.getOperand(i: 0);
6393 info.eVal = materializeValue(eOperand);
6394 return success();
6395 })
6396 .Case<LLVM::SelectOp>(caseFn: [&](LLVM::SelectOp selectOp) {
6397 if (!info.dVal)
6398 info.dVal = materializeValue(selectOp.getTrueValue());
6399 return success();
6400 })
6401 .Case<mlir::arith::MaxSIOp, mlir::arith::MinSIOp,
6402 mlir::arith::MaxUIOp, mlir::arith::MinUIOp,
6403 mlir::arith::MaximumFOp, mlir::arith::MinimumFOp,
6404 LLVM::SMaxOp, LLVM::SMinOp, LLVM::UMaxOp, LLVM::UMinOp,
6405 LLVM::MaxNumOp, LLVM::MinNumOp>(caseFn: [&](Operation *) {
6406 // Canonicalized min/max ops (arith or LLVM intrinsic form).
6407 // max(x,e) came from slt/ult/olt -> OMPAtomicCompareOp::MIN
6408 // min(x,e) came from sgt/ugt/ogt -> OMPAtomicCompareOp::MAX
6409 // (OMPIRBuilder inverts: MIN->atomicrmw max, MAX->atomicrmw min)
6410 bool isMax = isa<mlir::arith::MaxSIOp, mlir::arith::MaxUIOp,
6411 mlir::arith::MaximumFOp, LLVM::SMaxOp,
6412 LLVM::UMaxOp, LLVM::MaxNumOp>(Val: op);
6413 info.compareOp = isMax ? llvm::omp::OMPAtomicCompareOp::MIN
6414 : llvm::omp::OMPAtomicCompareOp::MAX;
6415 info.isSigned = isa<mlir::arith::MaxSIOp, mlir::arith::MinSIOp,
6416 LLVM::SMaxOp, LLVM::SMinOp>(Val: op);
6417 info.isXBinopExpr = (op.getOperand(idx: 0) == block.getArgument(i: 0));
6418 mlir::Value eOperand =
6419 info.isXBinopExpr ? op.getOperand(idx: 1) : op.getOperand(idx: 0);
6420 info.eVal = materializeValue(eOperand);
6421 info.dVal = info.eVal;
6422 return success();
6423 })
6424 .Default(defaultFn: [](Operation *) { return success(); });
6425
6426 if (failed(Result: result))
6427 return result;
6428 }
6429 return success();
6430}
6431
6432static LogicalResult
6433convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
6434 llvm::IRBuilderBase &builder,
6435 LLVM::ModuleTranslation &moduleTranslation) {
6436 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
6437 if (failed(Result: checkImplementationStatus(op&: *atomicCaptureOp)))
6438 return failure();
6439
6440 omp::AtomicUpdateOp atomicUpdateOp = atomicCaptureOp.getAtomicUpdateOp();
6441 omp::AtomicWriteOp atomicWriteOp = atomicCaptureOp.getAtomicWriteOp();
6442 omp::AtomicCompareOp atomicCompareOp = atomicCaptureOp.getAtomicCompareOp();
6443
6444 // If the capture contains an atomic.compare, delegate to
6445 // createAtomicCompare with the capture variable (V) set.
6446 if (atomicCompareOp) {
6447 omp::AtomicReadOp atomicReadOp = atomicCaptureOp.getAtomicReadOp();
6448 assert(atomicReadOp && "expected atomic.read in capture+compare");
6449
6450 Region &region = atomicCompareOp.getRegion();
6451 Block &block = region.front();
6452
6453 llvm::Type *llvmXElementType =
6454 moduleTranslation.convertType(type: block.getArgument(i: 0).getType());
6455 llvm::Value *llvmX = moduleTranslation.lookupValue(value: atomicCompareOp.getX());
6456 llvm::Value *llvmV = moduleTranslation.lookupValue(value: atomicReadOp.getV());
6457
6458 bool isSigned = false;
6459 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {
6460 .Var: llvmX, .ElemTy: llvmXElementType, .IsSigned: isSigned, /*IsVolatile=*/false};
6461 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicV = {
6462 .Var: llvmV, .ElemTy: llvmXElementType, /*isSigned=*/.IsSigned: false, /*IsVolatile=*/false};
6463 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicR = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false,
6464 .IsVolatile: false};
6465
6466 llvm::AtomicOrdering atomicOrdering =
6467 convertAtomicOrdering(ao: atomicCaptureOp.getMemoryOrder());
6468
6469 // Pre-translate non-pattern operations inside the compare region.
6470 auto isAtomicComparePatternOp = [](Operation &op) {
6471 return llvm::isa<LLVM::ICmpOp, LLVM::FCmpOp, LLVM::SelectOp, LLVM::AndOp,
6472 LLVM::OrOp, LLVM::SMaxOp, LLVM::SMinOp, LLVM::UMaxOp,
6473 LLVM::UMinOp, LLVM::MaxNumOp, LLVM::MinNumOp,
6474 mlir::arith::MaxSIOp, mlir::arith::MinSIOp,
6475 mlir::arith::MaxUIOp, mlir::arith::MinUIOp,
6476 mlir::arith::MaximumFOp, mlir::arith::MinimumFOp>(Val: op);
6477 };
6478 for (Operation &op : block.without_terminator()) {
6479 if (isAtomicComparePatternOp(op))
6480 continue;
6481 bool allOperandsMapped =
6482 llvm::all_of(Range: op.getOperands(), P: [&](mlir::Value v) {
6483 return moduleTranslation.lookupValue(value: v) != nullptr;
6484 });
6485 if (!allOperandsMapped)
6486 continue;
6487 if (failed(Result: moduleTranslation.convertOperation(op, builder)))
6488 return atomicCompareOp.emitError(
6489 message: "failed to translate operation inside atomic compare region");
6490 }
6491
6492 auto materializeValue = [&](mlir::Value val) -> llvm::Value * {
6493 if (llvm::Value *existing = moduleTranslation.lookupValue(value: val))
6494 return existing;
6495 if (auto loadOp = val.getDefiningOp<LLVM::LoadOp>()) {
6496 if (loadOp->getParentRegion() == &region) {
6497 llvm::Value *loadAddr =
6498 moduleTranslation.lookupValue(value: loadOp.getAddr());
6499 if (!loadAddr)
6500 return nullptr;
6501 llvm::Type *loadType =
6502 moduleTranslation.convertType(type: loadOp.getResult().getType());
6503 return builder.CreateLoad(Ty: loadType, Ptr: loadAddr);
6504 }
6505 }
6506 return nullptr;
6507 };
6508
6509 // Extract comparison predicate, eVal, and dVal from the region.
6510 AtomicComparePatternInfo patternInfo;
6511 if (failed(Result: extractAtomicComparePattern(block, materializeValue,
6512 atomicCompareOp, info&: patternInfo)))
6513 return failure();
6514
6515 llvm::omp::OMPAtomicCompareOp compareOp = patternInfo.compareOp;
6516 llvm::Value *eVal = patternInfo.eVal;
6517 llvm::Value *dVal = patternInfo.dVal;
6518 bool isXBinopExpr = patternInfo.isXBinopExpr;
6519 isSigned = patternInfo.isSigned;
6520
6521 if (!eVal)
6522 return atomicCompareOp.emitError(
6523 message: "failed to extract expected value (e) from atomic compare region");
6524 if (!dVal) {
6525 auto yieldOp = cast<omp::YieldOp>(Val: block.getTerminator());
6526 if (yieldOp.getResults().empty())
6527 return atomicCompareOp.emitError(
6528 message: "failed to extract desired value (d) from atomic compare region");
6529 dVal = materializeValue(yieldOp.getResults()[0]);
6530 }
6531
6532 llvmAtomicX.IsSigned = isSigned;
6533
6534 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
6535 bool isReadFirst = isa<omp::AtomicReadOp>(Val: atomicCaptureOp.getFirstOp());
6536 bool isPostfixCapture = !isReadFirst;
6537 bool isFailOnly = atomicCaptureOp.getFailOnly();
6538
6539 // Complex equality capture: x is struct-typed, which the OMPIRBuilder
6540 // cannot handle, so emit an IEEE-754-correct cmpxchg (as in the non-capture
6541 // complex path) and reconstruct the captured value from its result. The
6542 // helper compares components with `fcmp oeq` so `-0.0 == +0.0` and `NaN`
6543 // are handled as in the scalar float path. Complex only supports the ==
6544 // comparison.
6545 if (llvmXElementType->isStructTy()) {
6546 llvm::Value *oldComplex = nullptr;
6547 llvm::Value *cmpOk = nullptr;
6548 llvm::AtomicOrdering failOrdering =
6549 getAtomicCompareFailureOrdering(atomicCompareOp, atomicOrdering);
6550 emitComplexAtomicCmpXchg(builder, llvmX, complexTy: llvmXElementType, eVal, dVal,
6551 atomicOrdering, failOrdering,
6552 isWeak: atomicCompareOp.getWeak(), oldComplex, cmpOk);
6553
6554 if (isFailOnly) {
6555 // v is written only when the compare fails (cmpOk == false).
6556 llvm::Value *cmpFailed = builder.CreateNot(V: cmpOk);
6557 llvm::BasicBlock *curBB = builder.GetInsertBlock();
6558 llvm::Function *fn = curBB->getParent();
6559 llvm::BasicBlock *contBB = llvm::BasicBlock::Create(
6560 Context&: builder.getContext(), Name: "omp.atomic.cont", Parent: fn);
6561 llvm::BasicBlock *exitBB = llvm::BasicBlock::Create(
6562 Context&: builder.getContext(), Name: "omp.atomic.exit", Parent: fn);
6563 builder.CreateCondBr(Cond: cmpFailed, True: contBB, False: exitBB);
6564 builder.SetInsertPoint(contBB);
6565 builder.CreateStore(Val: oldComplex, Ptr: llvmAtomicV.Var,
6566 isVolatile: llvmAtomicV.IsVolatile);
6567 builder.CreateBr(Dest: exitBB);
6568 builder.SetInsertPoint(exitBB);
6569 } else if (isPostfixCapture) {
6570 // v gets the new value of x: d on success, old x otherwise.
6571 llvm::Value *newComplex = builder.CreateSelect(C: cmpOk, True: dVal, False: oldComplex);
6572 builder.CreateStore(Val: newComplex, Ptr: llvmAtomicV.Var,
6573 isVolatile: llvmAtomicV.IsVolatile);
6574 } else {
6575 // Prefix: v gets the old value of x.
6576 builder.CreateStore(Val: oldComplex, Ptr: llvmAtomicV.Var,
6577 isVolatile: llvmAtomicV.IsVolatile);
6578 }
6579
6580 // Emit flush after atomic compare if needed (release/acq_rel/seq_cst).
6581 if (atomicOrdering == llvm::AtomicOrdering::Release ||
6582 atomicOrdering == llvm::AtomicOrdering::AcquireRelease ||
6583 atomicOrdering == llvm::AtomicOrdering::SequentiallyConsistent) {
6584 llvm::OpenMPIRBuilder::LocationDescription flushLoc(builder);
6585 ompBuilder->createFlush(Loc: flushLoc);
6586 }
6587 return success();
6588 }
6589
6590 // Min/max (<, >) comparisons lower to an atomicrmw. The OMPIRBuilder has no
6591 // notion of a failed compare for an atomicrmw, so the fail-only capture
6592 // form (v written only when the compare fails) has no valid mapping and is
6593 // Min/max (<, >) comparisons lower to an atomicrmw. The OMPIRBuilder has no
6594 // notion of a failed compare for an atomicrmw (it asserts on IsFailOnly),
6595 // so for min/max the fail-only capture is reconstructed manually below.
6596 bool isMinMax = compareOp != llvm::omp::OMPAtomicCompareOp::EQ;
6597
6598 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicVForCall = llvmAtomicV;
6599 // Capture into V is reconstructed manually below for:
6600 // * postfix capture (v gets the new value of x): equality from the
6601 // cmpxchg result, min/max from the atomicrmw result;
6602 // * min/max fail-only capture (the OMPIRBuilder cannot express it).
6603 // Bypass V in the OMPIRBuilder for those cases so it does not also emit its
6604 // own (for min/max, incorrect or unsupported) capture store.
6605 bool minMaxManualCapture = isMinMax && (isPostfixCapture || isFailOnly);
6606 bool eqPostfixManualCapture = !isMinMax && isPostfixCapture && !isFailOnly;
6607 if (minMaxManualCapture || eqPostfixManualCapture)
6608 llvmAtomicVForCall = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false, .IsVolatile: false};
6609
6610 // The OMPIRBuilder only understands IsFailOnly for the equality (cmpxchg)
6611 // path; for min/max it would assert. Min/max fail-only is handled here.
6612 bool builderFailOnly = isFailOnly && !isMinMax;
6613
6614 // IsPostfixUpdate selects which value the OMPIRBuilder captures into V:
6615 // * min/max prefix and equality prefix: a direct store of the old value
6616 // (IsPostfixUpdate=true).
6617 // * equality fail-only: a conditional store (IsPostfixUpdate=false).
6618 // Manually-reconstructed captures bypass V above.
6619 bool isPostfixUpdate = !builderFailOnly;
6620
6621 bool isWeak = atomicCompareOp.getWeak();
6622 bool savedHandleFPNegZero = ompBuilder->setHandleFPNegZero(true);
6623 llvm::AtomicOrdering failureOrdering =
6624 getAtomicCompareFailureOrdering(atomicCompareOp, atomicOrdering);
6625 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
6626 ompBuilder->createAtomicCompare(
6627 Loc: ompLoc, X&: llvmAtomicX, V&: llvmAtomicVForCall, R&: llvmAtomicR, E: eVal, D: dVal,
6628 AO: atomicOrdering, Op: compareOp, IsXBinopExpr: isXBinopExpr, IsPostfixUpdate: isPostfixUpdate,
6629 IsFailOnly: builderFailOnly, Failure: failureOrdering, IsWeak: isWeak);
6630 ompBuilder->setHandleFPNegZero(savedHandleFPNegZero);
6631
6632 if (failed(Result: handleError(result&: afterIP, op&: *atomicCaptureOp)))
6633 return failure();
6634
6635 builder.restoreIP(IP: *afterIP);
6636
6637 // Min/max capture is reconstructed from the atomicrmw the OMPIRBuilder
6638 // emits (its result is the old value of x). V was bypassed above.
6639 // * postfix: v gets the new value min/max(old, e);
6640 // * fail-only: v gets the old value, but only when the compare failed
6641 // (i.e. the atomicrmw did not change x).
6642 // (Prefix min/max captures the old value directly through V, so nothing
6643 // extra is needed there.)
6644 if (isMinMax && (isPostfixCapture || isFailOnly)) {
6645 llvm::BasicBlock *curBB = builder.GetInsertBlock();
6646 llvm::AtomicRMWInst *rmw = nullptr;
6647 for (auto &inst : llvm::reverse(C&: *curBB)) {
6648 if (auto *r = dyn_cast<llvm::AtomicRMWInst>(Val: &inst)) {
6649 rmw = r;
6650 break;
6651 }
6652 }
6653 assert(rmw && "expected atomicrmw for min/max compare capture");
6654 llvm::Value *oldVal = rmw;
6655 llvm::Value *rhs = rmw->getValOperand();
6656
6657 if (isFailOnly) {
6658 // The compare "failed" (the else branch runs) exactly when the
6659 // atomicrmw did not change x. Recompute the original update condition
6660 // on the old value and negate it. v is stored only in that case.
6661 llvm::CmpInst::Predicate updatePred;
6662 switch (rmw->getOperation()) {
6663 case llvm::AtomicRMWInst::Min:
6664 updatePred = llvm::CmpInst::ICMP_SGT;
6665 break;
6666 case llvm::AtomicRMWInst::Max:
6667 updatePred = llvm::CmpInst::ICMP_SLT;
6668 break;
6669 case llvm::AtomicRMWInst::UMin:
6670 updatePred = llvm::CmpInst::ICMP_UGT;
6671 break;
6672 case llvm::AtomicRMWInst::UMax:
6673 updatePred = llvm::CmpInst::ICMP_ULT;
6674 break;
6675 case llvm::AtomicRMWInst::FMin:
6676 updatePred = llvm::CmpInst::FCMP_OGT;
6677 break;
6678 case llvm::AtomicRMWInst::FMax:
6679 updatePred = llvm::CmpInst::FCMP_OLT;
6680 break;
6681 default:
6682 llvm_unreachable(
6683 "unexpected atomicrmw op for min/max compare capture");
6684 }
6685 llvm::Value *updated = builder.CreateCmp(Pred: updatePred, LHS: oldVal, RHS: rhs);
6686 llvm::Value *failed = builder.CreateNot(V: updated);
6687 llvm::Function *fn = curBB->getParent();
6688 llvm::BasicBlock *contBB = llvm::BasicBlock::Create(
6689 Context&: builder.getContext(), Name: "omp.atomic.cont", Parent: fn);
6690 llvm::BasicBlock *exitBB = llvm::BasicBlock::Create(
6691 Context&: builder.getContext(), Name: "omp.atomic.exit", Parent: fn);
6692 builder.CreateCondBr(Cond: failed, True: contBB, False: exitBB);
6693 builder.SetInsertPoint(contBB);
6694 builder.CreateStore(Val: oldVal, Ptr: llvmAtomicV.Var, isVolatile: llvmAtomicV.IsVolatile);
6695 builder.CreateBr(Dest: exitBB);
6696 builder.SetInsertPoint(exitBB);
6697 } else {
6698 llvm::Intrinsic::ID id;
6699 switch (rmw->getOperation()) {
6700 case llvm::AtomicRMWInst::Min:
6701 id = llvm::Intrinsic::smin;
6702 break;
6703 case llvm::AtomicRMWInst::Max:
6704 id = llvm::Intrinsic::smax;
6705 break;
6706 case llvm::AtomicRMWInst::UMin:
6707 id = llvm::Intrinsic::umin;
6708 break;
6709 case llvm::AtomicRMWInst::UMax:
6710 id = llvm::Intrinsic::umax;
6711 break;
6712 case llvm::AtomicRMWInst::FMin:
6713 id = llvm::Intrinsic::minnum;
6714 break;
6715 case llvm::AtomicRMWInst::FMax:
6716 id = llvm::Intrinsic::maxnum;
6717 break;
6718 default:
6719 llvm_unreachable(
6720 "unexpected atomicrmw op for min/max compare capture");
6721 }
6722 llvm::Value *newVal = builder.CreateBinaryIntrinsic(ID: id, LHS: oldVal, RHS: rhs);
6723 builder.CreateStore(Val: newVal, Ptr: llvmAtomicV.Var, isVolatile: llvmAtomicV.IsVolatile);
6724 }
6725 }
6726
6727 // Equality postfix: v = select(success, D, old) — reconstructs the new
6728 // value of x from the cmpxchg result.
6729 if (!isMinMax && isPostfixCapture && !isFailOnly) {
6730 llvm::BasicBlock *curBB = builder.GetInsertBlock();
6731 llvm::Value *oldVal = nullptr;
6732 llvm::Value *successVal = nullptr;
6733
6734 // Integer path (and non-HandleFPNegZero FP path): a single cmpxchg
6735 // lives in the current block.
6736 for (auto &inst : llvm::reverse(C&: *curBB)) {
6737 if (isa<llvm::AtomicCmpXchgInst>(Val: &inst)) {
6738 oldVal = builder.CreateExtractValue(Agg: &inst, /*Idxs=*/0);
6739 successVal = builder.CreateExtractValue(Agg: &inst, /*Idxs=*/1);
6740 break;
6741 }
6742 }
6743
6744 // FP HandleFPNegZero path: the OMPIRBuilder emits a multi-block
6745 // structure (NaN / ±0.0 handling) with cmpxchg in predecessor
6746 // blocks. Results are merged via PHI nodes in the current (exit)
6747 // block: an i1 PHI for success and a bitcast of an integer PHI
6748 // for the old FP value.
6749 if (!oldVal) {
6750 for (auto &inst : *curBB) {
6751 auto *phi = dyn_cast<llvm::PHINode>(Val: &inst);
6752 if (!phi)
6753 break;
6754 if (phi->getType()->isIntegerTy(BitWidth: 1))
6755 successVal = phi;
6756 }
6757 for (auto &inst : *curBB) {
6758 if (auto *bc = dyn_cast<llvm::BitCastInst>(Val: &inst)) {
6759 oldVal = bc;
6760 break;
6761 }
6762 }
6763 }
6764
6765 assert(oldVal && "expected cmpxchg or PHI+bitcast for compare capture");
6766 assert(successVal && "expected success flag for compare capture");
6767 llvm::Value *newVal = builder.CreateSelect(C: successVal, True: dVal, False: oldVal);
6768 builder.CreateStore(Val: newVal, Ptr: llvmAtomicV.Var, isVolatile: llvmAtomicV.IsVolatile);
6769 }
6770
6771 return success();
6772 }
6773
6774 mlir::Value mlirExpr;
6775 bool isXBinopExpr = false, isPostfixUpdate = false;
6776 llvm::AtomicRMWInst::BinOp binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
6777
6778 assert((atomicUpdateOp || atomicWriteOp) &&
6779 "internal op must be an atomic.update or atomic.write op");
6780
6781 if (atomicWriteOp) {
6782 isPostfixUpdate = true;
6783 mlirExpr = atomicWriteOp.getExpr();
6784 } else {
6785 isPostfixUpdate = atomicCaptureOp.getSecondOp() ==
6786 atomicCaptureOp.getAtomicUpdateOp().getOperation();
6787 auto &innerOpList = atomicUpdateOp.getRegion().front().getOperations();
6788 // Find the binary update operation that uses the region argument
6789 // and get the expression to update
6790 if (innerOpList.size() == 2) {
6791 mlir::Operation &innerOp = *atomicUpdateOp.getRegion().front().begin();
6792 if (!llvm::is_contained(Range: innerOp.getOperands(),
6793 Element: atomicUpdateOp.getRegion().getArgument(i: 0))) {
6794 return atomicUpdateOp.emitError(
6795 message: "no atomic update operation with region argument"
6796 " as operand found inside atomic.update region");
6797 }
6798 binop = convertBinOpToAtomic(op&: innerOp);
6799 isXBinopExpr =
6800 innerOp.getOperand(idx: 0) == atomicUpdateOp.getRegion().getArgument(i: 0);
6801 mlirExpr = (isXBinopExpr ? innerOp.getOperand(idx: 1) : innerOp.getOperand(idx: 0));
6802 } else {
6803 binop = llvm::AtomicRMWInst::BinOp::BAD_BINOP;
6804 }
6805 }
6806
6807 llvm::Value *llvmExpr = moduleTranslation.lookupValue(value: mlirExpr);
6808 llvm::Value *llvmX =
6809 moduleTranslation.lookupValue(value: atomicCaptureOp.getAtomicReadOp().getX());
6810 llvm::Value *llvmV =
6811 moduleTranslation.lookupValue(value: atomicCaptureOp.getAtomicReadOp().getV());
6812 llvm::Type *llvmXElementType = moduleTranslation.convertType(
6813 type: atomicCaptureOp.getAtomicReadOp().getElementType());
6814 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {.Var: llvmX, .ElemTy: llvmXElementType,
6815 /*isSigned=*/.IsSigned: false,
6816 /*isVolatile=*/.IsVolatile: false};
6817 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicV = {.Var: llvmV, .ElemTy: llvmXElementType,
6818 /*isSigned=*/.IsSigned: false,
6819 /*isVolatile=*/.IsVolatile: false};
6820
6821 llvm::AtomicOrdering atomicOrdering =
6822 convertAtomicOrdering(ao: atomicCaptureOp.getMemoryOrder());
6823
6824 auto updateFn =
6825 [&](llvm::Value *atomicx,
6826 llvm::IRBuilder<> &builder) -> llvm::Expected<llvm::Value *> {
6827 if (atomicWriteOp)
6828 return moduleTranslation.lookupValue(value: atomicWriteOp.getExpr());
6829 Block &bb = *atomicUpdateOp.getRegion().begin();
6830 moduleTranslation.mapValue(mlir: *atomicUpdateOp.getRegion().args_begin(),
6831 llvm: atomicx);
6832 moduleTranslation.mapBlock(mlir: &bb, llvm: builder.GetInsertBlock());
6833 if (failed(Result: moduleTranslation.convertBlock(bb, ignoreArguments: true, builder)))
6834 return llvm::make_error<PreviouslyReportedError>();
6835
6836 omp::YieldOp yieldop = dyn_cast<omp::YieldOp>(Val: bb.getTerminator());
6837 assert(yieldop && yieldop.getResults().size() == 1 &&
6838 "terminator must be omp.yield op and it must have exactly one "
6839 "argument");
6840 return moduleTranslation.lookupValue(value: yieldop.getResults()[0]);
6841 };
6842
6843 bool isIgnoreDenormalMode;
6844 bool isFineGrainedMemory;
6845 bool isRemoteMemory;
6846 extractAtomicControlFlags(atomicUpdateOp, isIgnoreDenormalMode,
6847 isFineGrainedMemory, isRemoteMemory);
6848 // Handle ambiguous alloca, if any.
6849 auto allocaIP = findAllocInsertPoints(builder, moduleTranslation);
6850 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
6851 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
6852 ompBuilder->createAtomicCapture(
6853 Loc: ompLoc, AllocaIP: allocaIP, X&: llvmAtomicX, V&: llvmAtomicV, Expr: llvmExpr, AO: atomicOrdering,
6854 RMWOp: binop, UpdateOp: updateFn, UpdateExpr: atomicUpdateOp, IsPostfixUpdate: isPostfixUpdate, IsXBinopExpr: isXBinopExpr,
6855 IsIgnoreDenormalMode: isIgnoreDenormalMode, IsFineGrainedMemory: isFineGrainedMemory, IsRemoteMemory: isRemoteMemory);
6856
6857 if (failed(Result: handleError(result&: afterIP, op&: *atomicCaptureOp)))
6858 return failure();
6859
6860 builder.restoreIP(IP: *afterIP);
6861 return success();
6862}
6863
6864/// Converts an omp.atomic.compare operation to LLVM IR.
6865///
6866/// if (x == e) x = d
6867/// The region contains a comparison + select pattern:
6868/// ^bb0(%xval: T):
6869/// %cmp = llvm.icmp/fcmp <pred> %xval, %e : T
6870/// %sel = llvm.select %cmp, %d, %xval : i1, T
6871/// omp.yield(%sel : T)
6872///
6873/// From MLIR extract:
6874/// 1) comparison operator
6875/// 2) expected value (e)
6876/// 3) desired value (d)
6877/// These are passed to OpenMPIRBuilder::createAtomicCompare which generates
6878/// the actual cmpxchg / atomicrmw instruction.
6879///
6880static LogicalResult
6881convertOmpAtomicCompare(omp::AtomicCompareOp atomicCompareOp,
6882 llvm::IRBuilderBase &builder,
6883 LLVM::ModuleTranslation &moduleTranslation) {
6884 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
6885 if (failed(Result: checkImplementationStatus(op&: *atomicCompareOp)))
6886 return failure();
6887
6888 Region &region = atomicCompareOp.getRegion();
6889 Block &block = region.front();
6890
6891 // Determine element type from the region block argument
6892 llvm::Type *llvmXElementType =
6893 moduleTranslation.convertType(type: block.getArgument(i: 0).getType());
6894 if (!llvmXElementType)
6895 return atomicCompareOp.emitError(
6896 message: "unable to determine element type for atomic compare");
6897
6898 llvm::Value *llvmX = moduleTranslation.lookupValue(value: atomicCompareOp.getX());
6899
6900 // IsSigned is determined from the comparison predicate in the region.
6901 // Signed ICmp predicates (slt/sgt) set this to true; unsigned (ult/ugt)
6902 // leave it false. For EQ and float comparisons, signedness is irrelevant.
6903 bool isSigned = false;
6904 llvm::OpenMPIRBuilder::AtomicOpValue llvmAtomicX = {.Var: llvmX, .ElemTy: llvmXElementType,
6905 .IsSigned: isSigned,
6906 /*IsVolatile=*/false};
6907
6908 llvm::AtomicOrdering atomicOrdering =
6909 convertAtomicOrdering(ao: atomicCompareOp.getMemoryOrder());
6910
6911 auto isAtomicComparePatternOp = [](Operation &op) {
6912 return llvm::isa<LLVM::ICmpOp, LLVM::FCmpOp, LLVM::SelectOp, LLVM::AndOp,
6913 LLVM::OrOp, LLVM::SMaxOp, LLVM::SMinOp, LLVM::UMaxOp,
6914 LLVM::UMinOp, LLVM::MaxNumOp, LLVM::MinNumOp,
6915 mlir::arith::MaxSIOp, mlir::arith::MinSIOp,
6916 mlir::arith::MaxUIOp, mlir::arith::MinUIOp,
6917 mlir::arith::MaximumFOp, mlir::arith::MinimumFOp>(Val: op);
6918 };
6919
6920 // Pre-translate operations inside the region that compute e and d (e.g.,
6921 // GEP, loads for dereferencing Fortran pointers) but are not part of the
6922 // atomic compare-and-swap pattern (icmp/fcmp, select, and/or).
6923 //
6924 // 1) Validity: The OpenMP spec requires e and d to be evaluated before the
6925 // atomic operation, so emitting their computation here is correct.
6926 // 2) Memory effects: These ops only depend on values defined outside the
6927 // region. They cannot observe the block argument (%xval), which is the
6928 // value loaded atomically by cmpxchg and does not exist yet.
6929 // 3) Invariant enforcement: The `allOperandsMapped` check below skips any
6930 // op whose operands include the unmapped block argument, guaranteeing
6931 // only region-external-dependent ops are pre-translated.
6932 for (Operation &op : block.without_terminator()) {
6933 // Skip operations that form the atomic compare pattern — these are
6934 // not emitted as individual instructions but are analyzed below to
6935 // extract the comparison predicate, expected value (e), and desired
6936 // value (d) for generating a single cmpxchg/atomicrmw.
6937 if (isAtomicComparePatternOp(op))
6938 continue;
6939
6940 // Avoid translating ops that depend on the unmapped block argument.
6941 bool allOperandsMapped = llvm::all_of(Range: op.getOperands(), P: [&](mlir::Value v) {
6942 return moduleTranslation.lookupValue(value: v) != nullptr;
6943 });
6944 if (!allOperandsMapped)
6945 continue;
6946
6947 if (failed(Result: moduleTranslation.convertOperation(op, builder)))
6948 return atomicCompareOp.emitError(
6949 message: "failed to translate operation inside atomic compare region");
6950 }
6951
6952 // Look up a value that may have been pre-translated or defined outside the
6953 // region.
6954 auto materializeValue = [&](mlir::Value val) -> llvm::Value * {
6955 // Check if the value is already mapped (pre-translated or defined outside).
6956 if (llvm::Value *existing = moduleTranslation.lookupValue(value: val))
6957 return existing;
6958 // Fallback for a single LoadOp whose address is mapped but whose result
6959 // was not pre-translated.
6960 if (auto loadOp = val.getDefiningOp<LLVM::LoadOp>()) {
6961 if (loadOp->getParentRegion() == &region) {
6962 llvm::Value *loadAddr = moduleTranslation.lookupValue(value: loadOp.getAddr());
6963 if (!loadAddr)
6964 return nullptr;
6965 llvm::Type *loadType =
6966 moduleTranslation.convertType(type: loadOp.getResult().getType());
6967 return builder.CreateLoad(Ty: loadType, Ptr: loadAddr);
6968 }
6969 }
6970 return nullptr;
6971 };
6972
6973 // Walk the region to extract comparison predicate, eVal, and dVal.
6974 // if (x == eVal) x = dVal
6975 llvm::omp::OMPAtomicCompareOp compareOp = llvm::omp::OMPAtomicCompareOp::EQ;
6976 llvm::Value *eVal = nullptr;
6977 llvm::Value *dVal = nullptr;
6978 bool isXBinopExpr = false;
6979
6980 // Check for a decomposed complex comparison pattern (extractvalue + fcmp +
6981 // and/or of the real/imaginary fields).
6982 ComplexComparePattern cplx = detectComplexCompareEq(block);
6983 bool isComplexPattern = cplx.isComplex;
6984 if (isComplexPattern) {
6985 if (cplx.isNE)
6986 // OrOp corresponds to NE, which is not a valid atomic compare op.
6987 return atomicCompareOp.emitError(
6988 message: "unsupported comparison predicate (NE) for complex atomic compare");
6989 compareOp = llvm::omp::OMPAtomicCompareOp::EQ;
6990 isXBinopExpr = cplx.isXBinopExpr;
6991 eVal = materializeValue(cplx.eAggregate);
6992 }
6993
6994 if (isComplexPattern) {
6995 // dVal from SelectOp or YieldOp.
6996 for (Operation &op : block.getOperations()) {
6997 if (auto selectOp = dyn_cast<LLVM::SelectOp>(Val&: op)) {
6998 dVal = materializeValue(selectOp.getTrueValue());
6999 break;
7000 }
7001 }
7002 if (!dVal) {
7003 auto yieldOp = cast<omp::YieldOp>(Val: block.getTerminator());
7004 if (yieldOp.getResults().empty())
7005 return atomicCompareOp.emitError(
7006 message: "failed to extract desired value (d) from atomic compare region");
7007 dVal = materializeValue(yieldOp.getResults()[0]);
7008 }
7009
7010 llvm::Value *oldComplex = nullptr;
7011 llvm::Value *cmpOk = nullptr;
7012 llvm::AtomicOrdering failOrdering =
7013 getAtomicCompareFailureOrdering(atomicCompareOp, atomicOrdering);
7014 emitComplexAtomicCmpXchg(builder, llvmX, complexTy: llvmXElementType, eVal, dVal,
7015 atomicOrdering, failOrdering,
7016 isWeak: atomicCompareOp.getWeak(), oldComplex, cmpOk);
7017 (void)oldComplex;
7018 (void)cmpOk;
7019
7020 // Emit flush after atomic compare if needed (for release, acq_rel,
7021 // seq_cst orderings).
7022 if (atomicOrdering == llvm::AtomicOrdering::Release ||
7023 atomicOrdering == llvm::AtomicOrdering::AcquireRelease ||
7024 atomicOrdering == llvm::AtomicOrdering::SequentiallyConsistent) {
7025 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
7026 ompBuilder->createFlush(Loc: ompLoc);
7027 }
7028 return success();
7029 } else {
7030 AtomicComparePatternInfo patternInfo;
7031 if (failed(Result: extractAtomicComparePattern(block, materializeValue,
7032 atomicCompareOp, info&: patternInfo)))
7033 return failure();
7034 compareOp = patternInfo.compareOp;
7035 eVal = patternInfo.eVal;
7036 dVal = patternInfo.dVal;
7037 isXBinopExpr = patternInfo.isXBinopExpr;
7038 isSigned = patternInfo.isSigned;
7039 }
7040
7041 if (!eVal)
7042 return atomicCompareOp.emitError(
7043 message: "failed to extract expected value (e) from atomic compare region");
7044 if (!dVal) {
7045 // Fall back to the yield operand.
7046 auto yieldOp = cast<omp::YieldOp>(Val: block.getTerminator());
7047 if (yieldOp.getResults().empty())
7048 return atomicCompareOp.emitError(
7049 message: "failed to extract desired value (d) from atomic compare region");
7050 dVal = materializeValue(yieldOp.getResults()[0]);
7051 }
7052
7053 llvmAtomicX.IsSigned = isSigned;
7054
7055 llvm::OpenMPIRBuilder::AtomicOpValue vOpVal = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false,
7056 .IsVolatile: false};
7057 llvm::OpenMPIRBuilder::AtomicOpValue rOpVal = {.Var: nullptr, .ElemTy: nullptr, .IsSigned: false,
7058 .IsVolatile: false};
7059 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
7060
7061 bool isWeak = atomicCompareOp.getWeak();
7062
7063 bool savedHandleFPNegZero = ompBuilder->setHandleFPNegZero(true);
7064 llvm::AtomicOrdering failureOrdering =
7065 getAtomicCompareFailureOrdering(atomicCompareOp, atomicOrdering);
7066 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
7067 ompBuilder->createAtomicCompare(
7068 Loc: ompLoc, X&: llvmAtomicX, V&: vOpVal, R&: rOpVal, E: eVal, D: dVal, AO: atomicOrdering,
7069 Op: compareOp, IsXBinopExpr: isXBinopExpr, /*IsPostfixUpdate=*/false,
7070 /*IsFailOnly=*/false, Failure: failureOrdering, IsWeak: isWeak);
7071 ompBuilder->setHandleFPNegZero(savedHandleFPNegZero);
7072
7073 if (failed(Result: handleError(result&: afterIP, op&: *atomicCompareOp)))
7074 return failure();
7075
7076 builder.restoreIP(IP: *afterIP);
7077 return success();
7078}
7079
7080static llvm::omp::Directive convertCancellationConstructType(
7081 omp::ClauseCancellationConstructType directive) {
7082 switch (directive) {
7083 case omp::ClauseCancellationConstructType::Loop:
7084 return llvm::omp::Directive::OMPD_for;
7085 case omp::ClauseCancellationConstructType::Parallel:
7086 return llvm::omp::Directive::OMPD_parallel;
7087 case omp::ClauseCancellationConstructType::Sections:
7088 return llvm::omp::Directive::OMPD_sections;
7089 case omp::ClauseCancellationConstructType::Taskgroup:
7090 return llvm::omp::Directive::OMPD_taskgroup;
7091 }
7092 llvm_unreachable("Unhandled cancellation construct type");
7093}
7094
7095static LogicalResult
7096convertOmpCancel(omp::CancelOp op, llvm::IRBuilderBase &builder,
7097 LLVM::ModuleTranslation &moduleTranslation) {
7098 if (failed(Result: checkImplementationStatus(op&: *op.getOperation())))
7099 return failure();
7100
7101 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
7102 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
7103
7104 llvm::Value *ifCond = nullptr;
7105 if (Value ifVar = op.getIfExpr())
7106 ifCond = moduleTranslation.lookupValue(value: ifVar);
7107
7108 llvm::omp::Directive cancelledDirective =
7109 convertCancellationConstructType(directive: op.getCancelDirective());
7110
7111 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
7112 ompBuilder->createCancel(Loc: ompLoc, IfCondition: ifCond, CanceledDirective: cancelledDirective);
7113
7114 if (failed(Result: handleError(result&: afterIP, op&: *op.getOperation())))
7115 return failure();
7116
7117 builder.restoreIP(IP: afterIP.get());
7118
7119 return success();
7120}
7121
7122static LogicalResult
7123convertOmpCancellationPoint(omp::CancellationPointOp op,
7124 llvm::IRBuilderBase &builder,
7125 LLVM::ModuleTranslation &moduleTranslation) {
7126 if (failed(Result: checkImplementationStatus(op&: *op.getOperation())))
7127 return failure();
7128
7129 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
7130 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
7131
7132 llvm::omp::Directive cancelledDirective =
7133 convertCancellationConstructType(directive: op.getCancelDirective());
7134
7135 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
7136 ompBuilder->createCancellationPoint(Loc: ompLoc, CanceledDirective: cancelledDirective);
7137
7138 if (failed(Result: handleError(result&: afterIP, op&: *op.getOperation())))
7139 return failure();
7140
7141 builder.restoreIP(IP: afterIP.get());
7142
7143 return success();
7144}
7145
7146/// Converts an OpenMP Threadprivate operation into LLVM IR using
7147/// OpenMPIRBuilder.
7148static LogicalResult
7149convertOmpThreadprivate(Operation &opInst, llvm::IRBuilderBase &builder,
7150 LLVM::ModuleTranslation &moduleTranslation) {
7151 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
7152 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
7153 auto threadprivateOp = cast<omp::ThreadprivateOp>(Val&: opInst);
7154
7155 if (failed(Result: checkImplementationStatus(op&: opInst)))
7156 return failure();
7157
7158 Value symAddr = threadprivateOp.getSymAddr();
7159 auto *symOp = symAddr.getDefiningOp();
7160
7161 if (auto asCast = dyn_cast<LLVM::AddrSpaceCastOp>(Val: symOp))
7162 symOp = asCast.getOperand().getDefiningOp();
7163
7164 if (!isa<LLVM::AddressOfOp>(Val: symOp))
7165 return opInst.emitError(message: "Addressing symbol not found");
7166 LLVM::AddressOfOp addressOfOp = dyn_cast<LLVM::AddressOfOp>(Val: symOp);
7167
7168 LLVM::GlobalOp global =
7169 addressOfOp.getGlobal(symbolTable&: moduleTranslation.symbolTable());
7170 llvm::GlobalValue *globalValue = moduleTranslation.lookupGlobal(op: global);
7171 llvm::Type *type = globalValue->getValueType();
7172 llvm::TypeSize typeSize = builder.getDataLayout().getTypeStoreSize(Ty: type);
7173 llvm::ConstantInt *size = builder.getInt64(C: typeSize.getFixedValue());
7174 llvm::Value *callInst = ompBuilder->createCachedThreadPrivate(
7175 Loc: ompLoc, Pointer: globalValue, Size: size, Name: global.getSymName() + ".cache");
7176 moduleTranslation.mapValue(mlir: opInst.getResult(idx: 0), llvm: callInst);
7177
7178 return success();
7179}
7180
7181static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
7182convertToDeviceClauseKind(mlir::omp::DeclareTargetDeviceType deviceClause) {
7183 switch (deviceClause) {
7184 case mlir::omp::DeclareTargetDeviceType::host:
7185 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
7186 break;
7187 case mlir::omp::DeclareTargetDeviceType::nohost:
7188 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
7189 break;
7190 case mlir::omp::DeclareTargetDeviceType::any:
7191 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
7192 break;
7193 }
7194 llvm_unreachable("unhandled device clause");
7195}
7196
7197static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
7198convertToCaptureClauseKind(
7199 mlir::omp::DeclareTargetCaptureClause captureClause) {
7200 switch (captureClause) {
7201 case mlir::omp::DeclareTargetCaptureClause::to:
7202 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
7203 case mlir::omp::DeclareTargetCaptureClause::link:
7204 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
7205 case mlir::omp::DeclareTargetCaptureClause::enter:
7206 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
7207 case mlir::omp::DeclareTargetCaptureClause::none:
7208 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
7209 }
7210 llvm_unreachable("unhandled capture clause");
7211}
7212
7213static Operation *getGlobalOpFromValue(Value value) {
7214 Operation *op = value.getDefiningOp();
7215 if (auto addrCast = dyn_cast_if_present<LLVM::AddrSpaceCastOp>(Val: op))
7216 op = addrCast->getOperand(idx: 0).getDefiningOp();
7217 if (auto addressOfOp = dyn_cast_if_present<LLVM::AddressOfOp>(Val: op)) {
7218 auto modOp = addressOfOp->getParentOfType<mlir::ModuleOp>();
7219 return modOp.lookupSymbol(name: addressOfOp.getGlobalName());
7220 }
7221 return nullptr;
7222}
7223
7224static Value getBaseValueForTypeLookup(Value value) {
7225 while (Operation *op = value.getDefiningOp()) {
7226 if (auto addrCast = dyn_cast_if_present<LLVM::AddrSpaceCastOp>(Val: op))
7227 value = addrCast.getOperand();
7228 // Traces through hlfir.declare, fir.declare to reach the base address and
7229 // use for type lookup.
7230 else if (op->getName().getIdentifier() &&
7231 (op->getName().getIdentifier().str() == "hlfir.declare" ||
7232 op->getName().getIdentifier().str() == "fir.declare")) {
7233 if (op->getNumOperands() > 0)
7234 value = op->getOperand(idx: 0);
7235 else
7236 break;
7237 } else {
7238 break;
7239 }
7240 }
7241 return value;
7242}
7243
7244// Determine the LLVM type whose storage size should be allocated for an
7245// OpenMP allocate directive list item. Opaque pointers lose element type, so
7246// trace through declare wrappers to the underlying global or stack allocation.
7247static llvm::Type *
7248getAllocatedLlvmTypeForVariable(Value var, Value baseVar,
7249 LLVM::ModuleTranslation &moduleTranslation) {
7250 llvm::Type *llvmVarTy = moduleTranslation.convertType(type: var.getType());
7251 if (!llvmVarTy->isPointerTy())
7252 return llvmVarTy;
7253
7254 if (Operation *globalOp = getGlobalOpFromValue(value: baseVar))
7255 if (auto gop = dyn_cast<LLVM::GlobalOp>(Val: globalOp))
7256 return moduleTranslation.convertType(type: gop.getGlobalType());
7257
7258 if (auto allocaOp =
7259 dyn_cast_if_present<LLVM::AllocaOp>(Val: baseVar.getDefiningOp()))
7260 return moduleTranslation.convertType(type: allocaOp.getElemType());
7261
7262 if (llvm::Value *baseLlvm = moduleTranslation.lookupValue(value: baseVar))
7263 if (auto *allocaInst = dyn_cast<llvm::AllocaInst>(Val: baseLlvm))
7264 return allocaInst->getAllocatedType();
7265
7266 return llvmVarTy;
7267}
7268
7269// For dynamically-sized stack allocations, compute the allocation size from
7270// the alloca's element count at runtime.
7271static std::optional<llvm::Value *> getDynamicAllocatedSize(
7272 Value var, Value baseVar, LLVM::ModuleTranslation &moduleTranslation,
7273 llvm::IRBuilderBase &builder, const llvm::DataLayout &dataLayout) {
7274 if (auto allocaOp =
7275 dyn_cast_if_present<LLVM::AllocaOp>(Val: baseVar.getDefiningOp())) {
7276 if (Value arraySize = allocaOp.getArraySize()) {
7277 llvm::Type *elemTy =
7278 moduleTranslation.convertType(type: allocaOp.getElemType());
7279 llvm::Value *numElems = moduleTranslation.lookupValue(value: arraySize);
7280 if (!numElems->getType()->isIntegerTy(BitWidth: 64))
7281 numElems = builder.CreateZExt(V: numElems, DestTy: builder.getInt64Ty());
7282 uint64_t elemSize = dataLayout.getTypeAllocSize(Ty: elemTy).getFixedValue();
7283 return builder.CreateMul(LHS: numElems, RHS: builder.getInt64(C: elemSize));
7284 }
7285 }
7286 if (llvm::Value *baseLlvm = moduleTranslation.lookupValue(value: baseVar)) {
7287 if (auto *allocaInst = dyn_cast<llvm::AllocaInst>(Val: baseLlvm)) {
7288 if (allocaInst->isArrayAllocation() &&
7289 !llvm::isa<llvm::ArrayType>(Val: allocaInst->getAllocatedType())) {
7290 uint64_t elemSize =
7291 dataLayout.getTypeAllocSize(Ty: allocaInst->getAllocatedType())
7292 .getFixedValue();
7293 return builder.CreateMul(LHS: allocaInst->getArraySize(),
7294 RHS: builder.getInt64(C: elemSize));
7295 }
7296 }
7297 }
7298 return std::nullopt;
7299}
7300
7301static llvm::SmallString<64>
7302getDeclareTargetRefPtrSuffix(LLVM::GlobalOp globalOp,
7303 llvm::OpenMPIRBuilder &ompBuilder,
7304 llvm::vfs::FileSystem &vfs) {
7305 llvm::SmallString<64> suffix;
7306 llvm::raw_svector_ostream os(suffix);
7307 if (globalOp.getVisibility() == mlir::SymbolTable::Visibility::Private) {
7308 auto loc = globalOp->getLoc()->findInstanceOf<FileLineColLoc>();
7309 auto fileInfoCallBack = [&loc]() {
7310 return std::pair<std::string, uint64_t>(
7311 llvm::StringRef(loc.getFilename()), loc.getLine());
7312 };
7313
7314 os << llvm::format(
7315 Fmt: "_%x",
7316 Vals: ompBuilder.getTargetEntryUniqueInfo(CallBack: fileInfoCallBack, VFS&: vfs).FileID);
7317 }
7318 os << "_decl_tgt_ref_ptr";
7319
7320 return suffix;
7321}
7322
7323static bool isDeclareTargetLink(Value value) {
7324 if (auto declareTargetGlobal =
7325 dyn_cast_if_present<omp::DeclareTargetInterface>(
7326 Val: getGlobalOpFromValue(value))) {
7327 omp::DeclareTargetAttr declareTargetAttr =
7328 declareTargetGlobal.getDeclareTarget();
7329 if (declareTargetAttr && declareTargetAttr.getCaptureClause() ==
7330 omp::DeclareTargetCaptureClause::link)
7331 return true;
7332 }
7333 return false;
7334}
7335
7336static bool isDeclareTargetTo(Value value) {
7337 if (auto declareTargetGlobal =
7338 dyn_cast_if_present<omp::DeclareTargetInterface>(
7339 Val: getGlobalOpFromValue(value))) {
7340 omp::DeclareTargetAttr declareTargetAttr =
7341 declareTargetGlobal.getDeclareTarget();
7342 if (declareTargetAttr && (declareTargetAttr.getCaptureClause() ==
7343 omp::DeclareTargetCaptureClause::to ||
7344 declareTargetAttr.getCaptureClause() ==
7345 omp::DeclareTargetCaptureClause::enter))
7346 return true;
7347 }
7348 return false;
7349}
7350
7351// Returns the reference pointer generated by the lowering of the declare
7352// target operation in cases where the link clause is used or the to clause is
7353// used in USM mode.
7354static llvm::Value *
7355getRefPtrIfDeclareTarget(Value value,
7356 LLVM::ModuleTranslation &moduleTranslation) {
7357 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
7358 if (auto gOp =
7359 dyn_cast_or_null<LLVM::GlobalOp>(Val: getGlobalOpFromValue(value))) {
7360 // In this case, we must utilise the reference pointer generated by
7361 // the declare target operation, similar to Clang
7362 if (isDeclareTargetLink(value) ||
7363 (isDeclareTargetTo(value) &&
7364 ompBuilder->Config.hasRequiresUnifiedSharedMemory())) {
7365 llvm::SmallString<64> suffix = getDeclareTargetRefPtrSuffix(
7366 globalOp: gOp, ompBuilder&: *ompBuilder, vfs&: moduleTranslation.getFileSystem());
7367
7368 if (gOp.getSymName().contains(Other: suffix))
7369 return moduleTranslation.getLLVMModule()->getNamedValue(
7370 Name: gOp.getSymName());
7371
7372 return moduleTranslation.getLLVMModule()->getNamedValue(
7373 Name: (gOp.getSymName().str() + suffix.str()).str());
7374 }
7375 }
7376 return nullptr;
7377}
7378
7379namespace {
7380// Append customMappers information to existing MapInfosTy
7381struct MapInfosTy : llvm::OpenMPIRBuilder::MapInfosTy {
7382 SmallVector<Operation *, 4> Mappers;
7383
7384 /// Append arrays in \a CurInfo.
7385 void append(MapInfosTy &curInfo) {
7386 Mappers.append(in_start: curInfo.Mappers.begin(), in_end: curInfo.Mappers.end());
7387 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo&: curInfo);
7388 }
7389};
7390// A small helper structure to contain data gathered
7391// for map lowering and coalese it into one area and
7392// avoiding extra computations such as searches in the
7393// llvm module for lowered mapped variables or checking
7394// if something is declare target (and retrieving the
7395// value) more than neccessary.
7396struct MapInfoData : MapInfosTy {
7397 llvm::SmallVector<bool, 4> IsDeclareTarget;
7398 llvm::SmallVector<bool, 4> IsAMember;
7399 // Identify if mapping was added by mapClause or use_device clauses.
7400 llvm::SmallVector<bool, 4> IsAMapping;
7401 llvm::SmallVector<mlir::Operation *, 4> MapClause;
7402 llvm::SmallVector<llvm::Value *, 4> OriginalValue;
7403 // Stripped off array/pointer to get the underlying
7404 // element type
7405 llvm::SmallVector<llvm::Type *, 4> BaseType;
7406
7407 /// Append arrays in \a CurInfo.
7408 void append(MapInfoData &CurInfo) {
7409 IsDeclareTarget.append(in_start: CurInfo.IsDeclareTarget.begin(),
7410 in_end: CurInfo.IsDeclareTarget.end());
7411 MapClause.append(in_start: CurInfo.MapClause.begin(), in_end: CurInfo.MapClause.end());
7412 OriginalValue.append(in_start: CurInfo.OriginalValue.begin(),
7413 in_end: CurInfo.OriginalValue.end());
7414 BaseType.append(in_start: CurInfo.BaseType.begin(), in_end: CurInfo.BaseType.end());
7415 MapInfosTy::append(curInfo&: CurInfo);
7416 }
7417};
7418
7419enum class TargetDirectiveEnumTy : uint32_t {
7420 None = 0,
7421 Target = 1,
7422 TargetData = 2,
7423 TargetEnterData = 3,
7424 TargetExitData = 4,
7425 TargetUpdate = 5
7426};
7427
7428static TargetDirectiveEnumTy getTargetDirectiveEnumTyFromOp(Operation *op) {
7429 return llvm::TypeSwitch<Operation *, TargetDirectiveEnumTy>(op)
7430 .Case(caseFn: [](omp::TargetDataOp) { return TargetDirectiveEnumTy::TargetData; })
7431 .Case(caseFn: [](omp::TargetEnterDataOp) {
7432 return TargetDirectiveEnumTy::TargetEnterData;
7433 })
7434 .Case(caseFn: [&](omp::TargetExitDataOp) {
7435 return TargetDirectiveEnumTy::TargetExitData;
7436 })
7437 .Case(caseFn: [&](omp::TargetUpdateOp) {
7438 return TargetDirectiveEnumTy::TargetUpdate;
7439 })
7440 .Case(caseFn: [&](omp::TargetOp) { return TargetDirectiveEnumTy::Target; })
7441 .Default(defaultFn: [&](Operation *op) { return TargetDirectiveEnumTy::None; });
7442}
7443} // namespace
7444
7445static uint64_t getArrayElementSizeInBits(LLVM::LLVMArrayType arrTy,
7446 DataLayout &dl) {
7447 if (auto nestedArrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>(
7448 Val: arrTy.getElementType()))
7449 return getArrayElementSizeInBits(arrTy: nestedArrTy, dl);
7450 return dl.getTypeSizeInBits(t: arrTy.getElementType());
7451}
7452
7453// The intent is to verify if the mapped data being passed is a
7454// pointer -> pointee that requires special handling in certain cases,
7455// e.g. applying the OMP_MAP_PTR_AND_OBJ map type.
7456//
7457// There may be a better way to verify this, but unfortunately with
7458// opaque pointers we lose the ability to easily check if something is
7459// a pointer whilst maintaining access to the underlying type.
7460static bool checkIfPointerMap(omp::MapInfoOp mapOp) {
7461 // If we have a varPtrPtr field assigned then the underlying type is a pointer
7462 if (mapOp.getVarPtrPtr())
7463 return true;
7464
7465 // If the map data is declare target with a link clause, then it's represented
7466 // as a pointer when we lower it to LLVM-IR even if at the MLIR level it has
7467 // no relation to pointers.
7468 if (isDeclareTargetLink(value: mapOp.getVarPtr()))
7469 return true;
7470
7471 return false;
7472}
7473
7474// A privatizeable attach map is a pointer/descriptor that is privatized and
7475// passed directly as a kernel argument (target_param) rather than undergoing
7476// the standard attach/parent mapping. These are handled specially in a couple
7477// of places in map lowering.
7478static bool isPrivatizeableAttachMap(omp::ClauseMapFlags mapType) {
7479 return bitEnumContainsAll(bits: mapType, bit: omp::ClauseMapFlags::priv |
7480 omp::ClauseMapFlags::target_param |
7481 omp::ClauseMapFlags::attach);
7482}
7483
7484// This function calculates the size to be offloaded for a specified type, given
7485// its associated map clause (which can contain bounds information which affects
7486// the total size), this size is calculated based on the underlying element type
7487// e.g. given a 1-D array of ints, we will calculate the size from the integer
7488// type * number of elements in the array. This size can be used in other
7489// calculations but is ultimately used as an argument to the OpenMP runtimes
7490// kernel argument structure which is generated through the combinedInfo data
7491// structures.
7492// This function is somewhat equivalent to Clang's getExprTypeSize inside of
7493// CGOpenMPRuntime.cpp.
7494static llvm::Value *getSizeInBytes(DataLayout &dl, const mlir::Type &type,
7495 Operation *clauseOp,
7496 llvm::Value *basePointer,
7497 llvm::Type *baseType,
7498 llvm::IRBuilderBase &builder,
7499 LLVM::ModuleTranslation &moduleTranslation) {
7500 if (auto memberClause =
7501 mlir::dyn_cast_if_present<mlir::omp::MapInfoOp>(Val: clauseOp)) {
7502 // This calculates the size to transfer based on bounds and the underlying
7503 // element type, provided bounds have been specified (Fortran
7504 // pointers/allocatables/target and arrays that have sections specified fall
7505 // into this as well)
7506 if (!memberClause.getBounds().empty()) {
7507 llvm::Value *elementCount = builder.getInt64(C: 1);
7508 for (auto bounds : memberClause.getBounds()) {
7509 if (auto boundOp = mlir::dyn_cast_if_present<mlir::omp::MapBoundsOp>(
7510 Val: bounds.getDefiningOp())) {
7511 // The below calculation for the size to be mapped calculated from the
7512 // map.info's bounds is: (elemCount * [UB - LB] + 1), later we
7513 // multiply by the underlying element types byte size to get the full
7514 // size to be offloaded based on the bounds
7515 elementCount = builder.CreateMul(
7516 LHS: elementCount,
7517 RHS: builder.CreateAdd(
7518 LHS: builder.CreateSub(
7519 LHS: moduleTranslation.lookupValue(value: boundOp.getUpperBound()),
7520 RHS: moduleTranslation.lookupValue(value: boundOp.getLowerBound())),
7521 RHS: builder.getInt64(C: 1)));
7522 }
7523 }
7524
7525 // utilising getTypeSizeInBits instead of getTypeSize as getTypeSize gives
7526 // the size in inconsistent byte or bit format.
7527 uint64_t underlyingTypeSzInBits = dl.getTypeSizeInBits(t: type);
7528 if (auto arrTy = llvm::dyn_cast_if_present<LLVM::LLVMArrayType>(Val: type))
7529 underlyingTypeSzInBits = getArrayElementSizeInBits(arrTy, dl);
7530
7531 // The size in bytes x number of elements, the sizeInBytes stored is
7532 // the underyling types size, e.g. if ptr<i32>, it'll be the i32's
7533 // size, so we do some on the fly runtime math to get the size in
7534 // bytes from the extent (ub - lb) * sizeInBytes. NOTE: This may need
7535 // some adjustment for members with more complex types.
7536 llvm::Value *sizeCalc = builder.CreateMul(
7537 LHS: elementCount, RHS: builder.getInt64(C: underlyingTypeSzInBits / 8),
7538 Name: "element_count");
7539
7540 // This is a part of a "complicated" bit of size calculation logic that is
7541 // in place to handle a couple of scenarios, one specific to Fortran and
7542 // the other a more general OpenMP issue. The other piece of the
7543 // calculation can be found as the final size calculation within the
7544 // processIndividualMap function. Ideally we would move it here, but due
7545 // to the complexity of calculating the final base address of some
7546 // constructs (required for a nullary check), it's left as the final step.
7547 // So, in the below 2 cases, the nullary check is in processIndividualMap
7548 // and the size equality check is here. The cases this modifications help
7549 // cover are:
7550 //
7551 // 1) If an argument has a null base pointer, then the size must be set to
7552 // 0 to avoid the runtime exploding/complaining about an illegal
7553 // pointer map. The size returning non-zero is feasible in certain
7554 // cases if for example someone has specified there own bounds/range.
7555 // 2) We wish to support a very specific OpenMP Fortran edge-case where a
7556 // size zero array can be legally presence checked and found to be on
7557 // device when it has been mapped. In these rare occasions the
7558 // allocatable/pointer will have a size of 1 allocated for the
7559 // underlying data, but this wall not be represented within the size of
7560 // the descriptor, so we get a non-nullary pointer and a size of 0,
7561 // allowing us to specify a size of 1 in these cases registering it on
7562 // the device mapping table as present.
7563 //
7564 // The default fall through case is just returning the size calculation
7565 // above, if we are not nullary and the size we calculate is non-zero,
7566 // which is basically any pointer type that is allocated in someway
7567 // (providing you are not running on a rare system that allows malloc's of
7568 // size 0 with whatever caveats that may come with).
7569 //
7570 // Later in the nullary check in processIndividualMap it just devolves to
7571 // selecting a size of 0 if we are nullary, if we are not, we will return
7572 // either 1 or the calculated size, depending on the outcome of this
7573 // select.
7574 if (checkIfPointerMap(mapOp: memberClause)) {
7575 return builder.CreateSelect(
7576 C: builder.CreateICmpEQ(LHS: sizeCalc, RHS: builder.getInt64(C: 0)),
7577 True: builder.getInt64(C: 1), False: sizeCalc);
7578 }
7579
7580 return sizeCalc;
7581 }
7582 }
7583
7584 return builder.getInt64(C: dl.getTypeSizeInBits(t: type) / 8);
7585}
7586
7587// Convert the MLIR map flag set to the runtime map flag set for embedding
7588// in LLVM-IR. This is important as the two bit-flag lists do not correspond
7589// 1-to-1 as there's flags the runtime doesn't care about and vice versa.
7590// Certain flags are discarded here such as RefPtee and co.
7591static llvm::omp::OpenMPOffloadMappingFlags
7592convertClauseMapFlags(omp::ClauseMapFlags mlirFlags) {
7593 const bool hasExplicitMap =
7594 (mlirFlags & ~omp::ClauseMapFlags::is_device_ptr) !=
7595 omp::ClauseMapFlags::none;
7596
7597 llvm::omp::OpenMPOffloadMappingFlags mapType =
7598 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7599
7600 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::to))
7601 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
7602
7603 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::from))
7604 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7605
7606 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::always))
7607 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
7608
7609 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::del))
7610 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
7611
7612 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::return_param))
7613 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
7614
7615 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::priv))
7616 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE;
7617
7618 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::literal))
7619 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL;
7620
7621 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::implicit))
7622 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
7623
7624 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::close))
7625 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
7626
7627 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::present))
7628 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
7629
7630 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::ompx_hold))
7631 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
7632
7633 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::attach))
7634 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH;
7635
7636 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::target_param))
7637 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
7638
7639 if (bitEnumContainsAll(bits: mlirFlags, bit: omp::ClauseMapFlags::is_device_ptr)) {
7640 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
7641 if (!hasExplicitMap)
7642 mapType |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL;
7643 }
7644
7645 return mapType;
7646}
7647
7648static StringRef getMapClauseName(omp::MapInfoOp mapOp) {
7649 if (StringAttr name = mapOp.getNameAttr())
7650 return name.getValue();
7651 return {};
7652}
7653
7654static void collectMapDataFromMapOperands(
7655 MapInfoData &mapData, SmallVectorImpl<Value> &mapVars,
7656 LLVM::ModuleTranslation &moduleTranslation, DataLayout &dl,
7657 llvm::IRBuilderBase &builder, ArrayRef<Value> useDevPtrOperands = {},
7658 ArrayRef<Value> useDevAddrOperands = {},
7659 ArrayRef<Value> hasDevAddrOperands = {}) {
7660
7661 auto checkRefPtrOrPteeMapWithAttach = [](omp::ClauseMapFlags mapType) {
7662 bool hasRefType =
7663 bitEnumContainsAll(bits: mapType, bit: omp::ClauseMapFlags::ref_ptr) ||
7664 bitEnumContainsAll(bits: mapType, bit: omp::ClauseMapFlags::ref_ptee);
7665 return hasRefType &&
7666 bitEnumContainsAll(bits: mapType, bit: omp::ClauseMapFlags::attach);
7667 };
7668
7669 auto checkIsAMember = [](const auto &mapVars, auto mapOp) {
7670 // Check if this is a member mapping and correctly assign that it is, if
7671 // it is a member of a larger object.
7672 // TODO: Need better handling of members, and distinguishing of members
7673 // that are implicitly allocated on device vs explicitly passed in as
7674 // arguments.
7675 // TODO: May require some further additions to support nested record
7676 // types, i.e. member maps that can have member maps.
7677 for (Value mapValue : mapVars) {
7678 auto map = cast<omp::MapInfoOp>(Val: mapValue.getDefiningOp());
7679 for (auto member : map.getMembers())
7680 if (member == mapOp)
7681 return true;
7682 }
7683 return false;
7684 };
7685
7686 // Process MapOperands
7687 for (Value mapValue : mapVars) {
7688 auto mapOp = cast<omp::MapInfoOp>(Val: mapValue.getDefiningOp());
7689 bool isAttachStyleMap =
7690 checkRefPtrOrPteeMapWithAttach(mapOp.getMapType()) ||
7691 isPrivatizeableAttachMap(mapType: mapOp.getMapType());
7692 Value offloadPtr = (mapOp.getVarPtrPtr() && !isAttachStyleMap)
7693 ? mapOp.getVarPtrPtr()
7694 : mapOp.getVarPtr();
7695 mapData.OriginalValue.push_back(Elt: moduleTranslation.lookupValue(value: offloadPtr));
7696 mapData.Pointers.push_back(
7697 Elt: isAttachStyleMap ? moduleTranslation.lookupValue(value: mapOp.getVarPtrPtr())
7698 : mapData.OriginalValue.back());
7699
7700 if (llvm::Value *refPtr =
7701 getRefPtrIfDeclareTarget(value: offloadPtr, moduleTranslation)) {
7702 mapData.IsDeclareTarget.push_back(Elt: true);
7703 mapData.BasePointers.push_back(Elt: refPtr);
7704 } else if (isDeclareTargetTo(value: offloadPtr)) {
7705 mapData.IsDeclareTarget.push_back(Elt: true);
7706 mapData.BasePointers.push_back(Elt: mapData.OriginalValue.back());
7707 } else { // regular mapped variable
7708 mapData.IsDeclareTarget.push_back(Elt: false);
7709 mapData.BasePointers.push_back(Elt: mapData.OriginalValue.back());
7710 }
7711
7712 // In every situation we currently have if we have a varPtrPtr present
7713 // we wish to utilise it's type for the base type, main cases are
7714 // currently Fortran descriptor base address maps and attach maps.
7715 mapData.BaseType.push_back(Elt: moduleTranslation.convertType(
7716 type: mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtrType().value()
7717 : mapOp.getVarPtrType()));
7718
7719 // For the attach map cases, it's a little odd, as we effectively have to
7720 // utilise the base address (including all bounds offsets) for the pointer
7721 // field, the pointer address for the base address field, and the pointer
7722 // not the data (base addresses) size. So we end up with a mix of base
7723 // types and sizes we wish to insert here.
7724 mlir::Type sizeType = (isAttachStyleMap || !mapOp.getVarPtrPtr())
7725 ? mapOp.getVarPtrType()
7726 : mapOp.getVarPtrPtrType().value();
7727 mapData.Sizes.push_back(Elt: getSizeInBytes(
7728 dl, type: sizeType, clauseOp: isAttachStyleMap ? nullptr : mapOp,
7729 basePointer: mapData.Pointers.back(), baseType: moduleTranslation.convertType(type: sizeType),
7730 builder, moduleTranslation));
7731 mapData.MapClause.push_back(Elt: mapOp.getOperation());
7732 mapData.Types.push_back(Elt: convertClauseMapFlags(mlirFlags: mapOp.getMapType()));
7733 // TODO: set HasAttachPtr from Flang for pointee-storage entries.
7734 mapData.HasAttachPtr.push_back(Elt: false);
7735 mapData.Names.push_back(Elt: LLVM::createMappingInformation(
7736 loc: mapOp.getLoc(), builder&: *getOpenMPBuilder(moduleTranslation),
7737 mapName: getMapClauseName(mapOp)));
7738 mapData.DevicePointers.push_back(Elt: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
7739 if (mapOp.getMapperId())
7740 mapData.Mappers.push_back(
7741 Elt: SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>(
7742 from: mapOp, symbol: mapOp.getMapperIdAttr()));
7743 else
7744 mapData.Mappers.push_back(Elt: nullptr);
7745 mapData.IsAMapping.push_back(Elt: true);
7746 mapData.IsAMember.push_back(Elt: checkIsAMember(mapVars, mapOp));
7747 }
7748
7749 auto findMapInfo = [&mapData](llvm::Value *val,
7750 llvm::OpenMPIRBuilder::DeviceInfoTy devInfoTy,
7751 size_t memberCount) {
7752 unsigned index = 0;
7753 bool found = false;
7754 for (llvm::Value *basePtr : mapData.OriginalValue) {
7755 auto mapOp = cast<omp::MapInfoOp>(Val: mapData.MapClause[index]);
7756 // TODO: Currently we define an equivalent mapping as
7757 // the same base pointer and an equivalent member count, but
7758 // that is a loose definition. We may have to extend to check
7759 // for other fields (varPtrPtr/individual members being mapped).
7760 // Note: Attach maps are not the same as a normal data transfer
7761 // they specify to the runtime to perform an attach map and they
7762 // (at least at the moment) are never something we would aim to
7763 // return in a use_dev_* clause, so they are skipped in terms of
7764 // duplicate maps.
7765 bool isAttachMap =
7766 (mapData.Types[index] &
7767 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH) ==
7768 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH;
7769 if (!isAttachMap && basePtr == val && mapData.IsAMapping[index] &&
7770 memberCount == mapOp.getMembers().size()) {
7771 found = true;
7772 mapData.Types[index] |=
7773 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
7774 mapData.DevicePointers[index] = devInfoTy;
7775 }
7776 index++;
7777 }
7778 return found;
7779 };
7780
7781 // Process useDevPtr(Addr)Operands
7782 auto addDevInfos = [&](const llvm::ArrayRef<Value> &useDevOperands,
7783 llvm::OpenMPIRBuilder::DeviceInfoTy devInfoTy) {
7784 for (Value mapValue : useDevOperands) {
7785 auto mapOp = cast<omp::MapInfoOp>(Val: mapValue.getDefiningOp());
7786 Value offloadPtr =
7787 mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr() : mapOp.getVarPtr();
7788 llvm::Value *origValue = moduleTranslation.lookupValue(value: offloadPtr);
7789
7790 // Check if map info is already present for this entry.
7791 if (!findMapInfo(origValue, devInfoTy, mapOp.getMembers().size())) {
7792 mapData.OriginalValue.push_back(Elt: origValue);
7793 mapData.Pointers.push_back(Elt: mapData.OriginalValue.back());
7794 mapData.IsDeclareTarget.push_back(Elt: false);
7795 mapData.BasePointers.push_back(Elt: mapData.OriginalValue.back());
7796 mlir::Type baseTy = mapOp.getVarPtrPtr()
7797 ? mapOp.getVarPtrPtrType().value()
7798 : mapOp.getVarPtrType();
7799 mapData.BaseType.push_back(Elt: moduleTranslation.convertType(type: baseTy));
7800 mapData.Sizes.push_back(Elt: builder.getInt64(C: 0));
7801 mapData.MapClause.push_back(Elt: mapOp.getOperation());
7802 mapData.Types.push_back(
7803 Elt: llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);
7804 // TODO: set HasAttachPtr from Flang for pointee-storage entries.
7805 mapData.HasAttachPtr.push_back(Elt: false);
7806 mapData.Names.push_back(Elt: LLVM::createMappingInformation(
7807 loc: mapOp.getLoc(), builder&: *getOpenMPBuilder(moduleTranslation),
7808 mapName: getMapClauseName(mapOp)));
7809 mapData.DevicePointers.push_back(Elt: devInfoTy);
7810 mapData.Mappers.push_back(Elt: nullptr);
7811 mapData.IsAMapping.push_back(Elt: false);
7812 mapData.IsAMember.push_back(Elt: checkIsAMember(useDevOperands, mapOp));
7813 }
7814 }
7815 };
7816
7817 addDevInfos(useDevAddrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Address);
7818 addDevInfos(useDevPtrOperands, llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer);
7819
7820 for (Value mapValue : hasDevAddrOperands) {
7821 auto mapOp = cast<omp::MapInfoOp>(Val: mapValue.getDefiningOp());
7822 Value offloadPtr =
7823 mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtr() : mapOp.getVarPtr();
7824 llvm::Value *origValue = moduleTranslation.lookupValue(value: offloadPtr);
7825 auto mapType = convertClauseMapFlags(mlirFlags: mapOp.getMapType());
7826 auto mapTypeAlways = llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
7827 bool isDevicePtr =
7828 (mapOp.getMapType() & omp::ClauseMapFlags::is_device_ptr) !=
7829 omp::ClauseMapFlags::none;
7830
7831 mapData.OriginalValue.push_back(Elt: origValue);
7832 mapData.BasePointers.push_back(Elt: origValue);
7833 mapData.Pointers.push_back(Elt: origValue);
7834 mapData.IsDeclareTarget.push_back(Elt: false);
7835
7836 mlir::Type baseTy = mapOp.getVarPtrPtr() ? mapOp.getVarPtrPtrType().value()
7837 : mapOp.getVarPtrType();
7838 mapData.BaseType.push_back(Elt: moduleTranslation.convertType(type: baseTy));
7839 mapData.Sizes.push_back(Elt: builder.getInt64(C: dl.getTypeSize(t: baseTy)));
7840
7841 mapData.MapClause.push_back(Elt: mapOp.getOperation());
7842 if (llvm::to_underlying(E: mapType & mapTypeAlways)) {
7843 // Descriptors are mapped with the ALWAYS flag, since they can get
7844 // rematerialized, so the address of the decriptor for a given object
7845 // may change from one place to another.
7846 mapData.Types.push_back(Elt: mapType);
7847 // TODO: set HasAttachPtr from Flang for pointee-storage entries.
7848 mapData.HasAttachPtr.push_back(Elt: false);
7849 // Technically it's possible for a non-descriptor mapping to have
7850 // both has-device-addr and ALWAYS, so lookup the mapper in case it
7851 // exists.
7852 if (mapOp.getMapperId()) {
7853 mapData.Mappers.push_back(
7854 Elt: SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>(
7855 from: mapOp, symbol: mapOp.getMapperIdAttr()));
7856 } else {
7857 mapData.Mappers.push_back(Elt: nullptr);
7858 }
7859 } else {
7860 // For is_device_ptr we need the map type to propagate so the runtime
7861 // can materialize the device-side copy of the pointer container.
7862 mapData.Types.push_back(
7863 Elt: isDevicePtr ? mapType
7864 : llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
7865 // TODO: set HasAttachPtr from Flang for pointee-storage entries.
7866 mapData.HasAttachPtr.push_back(Elt: false);
7867 mapData.Mappers.push_back(Elt: nullptr);
7868 }
7869 mapData.Names.push_back(Elt: LLVM::createMappingInformation(
7870 loc: mapOp.getLoc(), builder&: *getOpenMPBuilder(moduleTranslation),
7871 mapName: getMapClauseName(mapOp)));
7872 mapData.DevicePointers.push_back(
7873 Elt: isDevicePtr ? llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer
7874 : llvm::OpenMPIRBuilder::DeviceInfoTy::Address);
7875 mapData.IsAMapping.push_back(Elt: false);
7876 mapData.IsAMember.push_back(Elt: checkIsAMember(hasDevAddrOperands, mapOp));
7877 }
7878}
7879
7880static int getMapDataMemberIdx(MapInfoData &mapData, omp::MapInfoOp memberOp) {
7881 auto *res = llvm::find(Range&: mapData.MapClause, Val: memberOp);
7882 assert(res != mapData.MapClause.end() &&
7883 "MapInfoOp for member not found in MapData, cannot return index");
7884 return std::distance(first: mapData.MapClause.begin(), last: res);
7885}
7886
7887static void sortMapIndices(llvm::SmallVectorImpl<size_t> &indices,
7888 omp::MapInfoOp mapInfo, bool first = true) {
7889 ArrayAttr indexAttr = mapInfo.getMembersIndexAttr();
7890 llvm::SmallVector<size_t> occludedChildren;
7891 llvm::sort(
7892 Start: indices.begin(), End: indices.end(), Comp: [&](const size_t a, const size_t b) {
7893 // Bail early if we are asked to look at the same index. If we do not
7894 // bail early, we can end up mistakenly adding indices to
7895 // occludedChildren. This can occur with some types of libc++ hardening.
7896 if (a == b)
7897 return false;
7898
7899 auto memberIndicesA = cast<ArrayAttr>(Val: indexAttr[a]);
7900 auto memberIndicesB = cast<ArrayAttr>(Val: indexAttr[b]);
7901
7902 for (auto it : llvm::zip(t&: memberIndicesA, u&: memberIndicesB)) {
7903 int64_t aIndex = mlir::cast<IntegerAttr>(Val: std::get<0>(t&: it)).getInt();
7904 int64_t bIndex = mlir::cast<IntegerAttr>(Val: std::get<1>(t&: it)).getInt();
7905
7906 if (aIndex == bIndex)
7907 continue;
7908
7909 if (aIndex < bIndex)
7910 return first;
7911
7912 if (aIndex > bIndex)
7913 return !first;
7914 }
7915
7916 // Iterated up until the end of the smallest member and
7917 // they were found to be equal up to that point, so select
7918 // the member with the lowest index count, so the "parent"
7919 bool memberAParent = memberIndicesA.size() < memberIndicesB.size();
7920 if (memberAParent)
7921 occludedChildren.push_back(Elt: b);
7922 else
7923 occludedChildren.push_back(Elt: a);
7924 return memberAParent;
7925 });
7926
7927 for (auto v : occludedChildren)
7928 indices.erase(CS: std::remove(first: indices.begin(), last: indices.end(), value: v),
7929 CE: indices.end());
7930}
7931
7932static omp::MapInfoOp getFirstOrLastMappedMemberPtr(omp::MapInfoOp mapInfo,
7933 bool first) {
7934 ArrayAttr indexAttr = mapInfo.getMembersIndexAttr();
7935 // Only 1 member has been mapped, we can return it.
7936 if (indexAttr.size() == 1)
7937 return cast<omp::MapInfoOp>(Val: mapInfo.getMembers()[0].getDefiningOp());
7938 llvm::SmallVector<size_t> indices(indexAttr.size());
7939 std::iota(first: indices.begin(), last: indices.end(), value: 0);
7940 sortMapIndices(indices, mapInfo, first);
7941 return llvm::cast<omp::MapInfoOp>(
7942 Val: mapInfo.getMembers()[indices.front()].getDefiningOp());
7943}
7944
7945/// This function calculates the array/pointer offset for map data provided
7946/// with bounds operations, e.g. when provided something like the following:
7947///
7948/// Fortran
7949/// map(tofrom: array(2:5, 3:2))
7950///
7951/// We must calculate the initial pointer offset to pass across, this function
7952/// performs this using bounds.
7953///
7954/// TODO/WARNING: This only supports Fortran's column major indexing currently
7955/// as is noted in the note below and comments in the function, we must extend
7956/// this function when we add a C++ frontend.
7957/// NOTE: which while specified in row-major order it currently needs to be
7958/// flipped for Fortran's column order array allocation and access (as
7959/// opposed to C++'s row-major, hence the backwards processing where order is
7960/// important). This is likely important to keep in mind for the future when
7961/// we incorporate a C++ frontend, both frontends will need to agree on the
7962/// ordering of generated bounds operations (one may have to flip them) to
7963/// make the below lowering frontend agnostic. The offload size
7964/// calcualtion may also have to be adjusted for C++.
7965static std::vector<llvm::Value *>
7966calculateBoundsOffset(LLVM::ModuleTranslation &moduleTranslation,
7967 llvm::IRBuilderBase &builder, bool isArrayTy,
7968 OperandRange bounds, bool &isByteOffset) {
7969 std::vector<llvm::Value *> idx;
7970 isByteOffset = false;
7971 // There's no bounds to calculate an offset from, we can safely
7972 // ignore and return no indices.
7973 if (bounds.empty())
7974 return idx;
7975
7976 // If we have an array type, then we have its type so can treat it as a
7977 // normal GEP instruction where the bounds operations are simply indexes
7978 // into the array. We currently do reverse order of the bounds, which
7979 // I believe leans more towards Fortran's column-major in memory.
7980 if (isArrayTy) {
7981 idx.push_back(x: builder.getInt64(C: 0));
7982 for (int i = bounds.size() - 1; i >= 0; --i) {
7983 if (auto boundOp = dyn_cast_if_present<omp::MapBoundsOp>(
7984 Val: bounds[i].getDefiningOp())) {
7985 idx.push_back(x: moduleTranslation.lookupValue(value: boundOp.getLowerBound()));
7986 }
7987 }
7988 } else {
7989 // Check if Fortran descriptors provide strides in bytes
7990 llvm::Value *offset = builder.getInt64(C: 0);
7991 for (Value bound : bounds) {
7992 auto boundOp =
7993 dyn_cast_if_present<omp::MapBoundsOp>(Val: bound.getDefiningOp());
7994 if (!boundOp || !boundOp.getStride())
7995 continue;
7996 // Fortran descriptors provide strides in bytes; when that's the case the
7997 // accumulated offset is a byte offset and must be consumed with a byte
7998 // (i8) GEP at the call site to avoid double-scaling by the element size.
7999 isByteOffset = boundOp.getStrideInBytes();
8000 }
8001 if (isByteOffset) {
8002 for (Value bound : bounds) {
8003 auto boundOp =
8004 dyn_cast_if_present<omp::MapBoundsOp>(Val: bound.getDefiningOp());
8005 if (!boundOp || !boundOp.getStride())
8006 continue;
8007 // If we do not have an array type, but we have bounds, then we're
8008 // dealing with a pointer that's being treated like an array
8009 // and we have the underlying type e.g. an i32, or f64 etc,
8010 // e.g. a fortran descriptor base address (pointer pointing to
8011 // the actual data) and we know that the offset stride is expressed
8012 // in bytes, so we must calculate the offset to the first mapped
8013 // element as a single linear index into that data.
8014 //
8015 // Each bound carries the stride between consecutive elements along its
8016 // dimension, so the linear offset to the section's origin is:
8017 //
8018 // offset += lower_bound[d] * stride[d]
8019 llvm::Value *lb =
8020 moduleTranslation.lookupValue(value: boundOp.getLowerBound());
8021 llvm::Value *stride =
8022 moduleTranslation.lookupValue(value: boundOp.getStride());
8023 offset = builder.CreateAdd(LHS: offset, RHS: builder.CreateMul(LHS: lb, RHS: stride));
8024 }
8025 idx.push_back(x: offset);
8026 } else {
8027 for (int i = bounds.size() - 1; i >= 0; --i) {
8028 if (auto boundOp = dyn_cast_if_present<omp::MapBoundsOp>(
8029 Val: bounds[i].getDefiningOp())) {
8030 // If the stride in bytes is unknown, so we must calculate the
8031 // offset using a single index which the following loop attempts to
8032 // compute using the standard column-major algorithm e.g for a 3D
8033 // array:
8034 //
8035 // ((((c_idx * b_len) + b_idx) * a_len) + a_idx)
8036 //
8037 // It is of note that it's doing column-major rather than row-major at
8038 // the moment, but having a way for the frontend to indicate which
8039 // major format to use or standardizing/canonicalizing the order of
8040 // the bounds to compute the offset may be useful in the future when
8041 // there's other frontends with different formats.
8042 if (i == ((int)bounds.size() - 1))
8043 idx.emplace_back(
8044 args: moduleTranslation.lookupValue(value: boundOp.getLowerBound()));
8045 else
8046 idx.back() = builder.CreateAdd(
8047 LHS: builder.CreateMul(LHS: idx.back(), RHS: moduleTranslation.lookupValue(
8048 value: boundOp.getExtent())),
8049 RHS: moduleTranslation.lookupValue(value: boundOp.getLowerBound()));
8050 }
8051 }
8052 }
8053 }
8054
8055 return idx;
8056}
8057
8058static void getAsIntegers(ArrayAttr values, llvm::SmallVector<int64_t> &ints) {
8059 llvm::transform(Range&: values, d_first: std::back_inserter(x&: ints), F: [](Attribute value) {
8060 return cast<IntegerAttr>(Val&: value).getInt();
8061 });
8062}
8063
8064// Gathers members that are overlapping in the parent, excluding members that
8065// themselves overlap, keeping the top-most (closest to parents level) map.
8066static void
8067getOverlappedMembers(llvm::SmallVectorImpl<size_t> &overlapMapDataIdxs,
8068 omp::MapInfoOp parentOp) {
8069 // No members mapped, no overlaps.
8070 if (parentOp.getMembers().empty())
8071 return;
8072
8073 // Single member, we can insert and return early.
8074 if (parentOp.getMembers().size() == 1) {
8075 overlapMapDataIdxs.push_back(Elt: 0);
8076 return;
8077 }
8078
8079 ArrayAttr indexAttr = parentOp.getMembersIndexAttr();
8080 size_t numMembers = indexAttr.size();
8081
8082 // Pre-convert all member indices to integer arrays for efficient comparison.
8083 llvm::SmallVector<llvm::SmallVector<int64_t>> memberIndices(numMembers);
8084 for (auto [i, indicesAttr] : llvm::enumerate(First&: indexAttr))
8085 getAsIntegers(values: cast<ArrayAttr>(Val: indicesAttr), ints&: memberIndices[i]);
8086
8087 // For each member, check if it's superseded by another (shorter prefix)
8088 // member. If member j's indices are a prefix of member i's indices, then
8089 // i is a child of j and should be skipped. e.g. if member [0] is mapped,
8090 // we skip members [0,1], [0,2], etc.
8091 llvm::SmallDenseSet<size_t> skipIndices;
8092 for (size_t i = 0; i < numMembers; ++i) {
8093 const auto &iIndices = memberIndices[i];
8094 for (size_t j = 0; j < numMembers; ++j) {
8095 if (i == j)
8096 continue;
8097 const auto &jIndices = memberIndices[j];
8098 // If j's indices are a strict prefix of i's indices, skip i
8099 if (jIndices.size() < iIndices.size() &&
8100 std::equal(first1: jIndices.begin(), last1: jIndices.end(), first2: iIndices.begin())) {
8101 skipIndices.insert(V: i);
8102 break; // No need to check other potential parents
8103 }
8104 }
8105 }
8106
8107 // Collect indices of members that are not superseded by a parent.
8108 for (size_t i = 0; i < numMembers; ++i)
8109 if (!skipIndices.contains(V: i))
8110 overlapMapDataIdxs.push_back(Elt: i);
8111}
8112
8113/// Describe a bounded update as a set of contiguous pieces. Bounds follow the
8114/// dialect's normalized fastest-to-slowest varying order, which matches
8115/// Fortran's column-major dimension order. Frontends for row-major languages
8116/// such as C/C++ must normalize their source dimensions to this order.
8117/// OpenMPIRBuilder reverses the vectors when emitting the runtime descriptor,
8118/// which uses byte offsets and strides with the slowest varying dimension
8119/// first.
8120static void addNonContiguousUpdate(LLVM::ModuleTranslation &moduleTranslation,
8121 llvm::IRBuilderBase &builder,
8122 MapInfoData &mapData, size_t mapDataIdx,
8123 MapInfosTy &combinedInfo) {
8124 auto mapOp = cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIdx]);
8125 if (mapOp.getBounds().empty() || mapOp.getMapperId() ||
8126 mapOp->getParentOfType<omp::DeclareMapperOp>() ||
8127 bitEnumContainsAll(bits: mapOp.getMapType(), bit: omp::ClauseMapFlags::attach))
8128 return;
8129
8130 llvm::Type *elementType = mapData.BaseType[mapDataIdx];
8131 // Bounds extents may be absent, and a scalar subscript has extent one.
8132 // Preserve the full source shape: for a(4,5), a(2,:) still needs a
8133 // second-dimension stride of four elements.
8134 SmallVector<uint64_t> arrayExtents;
8135 while (auto *arrayType = dyn_cast<llvm::ArrayType>(Val: elementType)) {
8136 arrayExtents.push_back(Elt: arrayType->getNumElements());
8137 elementType = arrayType->getElementType();
8138 }
8139 llvm::Value *elementSize = builder.getInt64(
8140 C: builder.GetInsertBlock()->getModule()->getDataLayout().getTypeAllocSize(
8141 Ty: elementType));
8142
8143 // Include a byte dimension so that a non-unit descriptor stride in the
8144 // innermost array dimension still transfers one element at a time.
8145 llvm::OpenMPIRBuilder::MapValuesArrayTy offsets{builder.getInt64(C: 0)};
8146 llvm::OpenMPIRBuilder::MapValuesArrayTy counts{elementSize};
8147 llvm::OpenMPIRBuilder::MapValuesArrayTy strides{builder.getInt64(C: 1)};
8148 llvm::Value *sourceStride = elementSize;
8149 llvm::Value *contiguousSize = elementSize;
8150 // Unknown descriptor strides conservatively use a descriptor. The runtime
8151 // merges contiguous dimensions from their actual values.
8152 bool isContiguous = true;
8153 for (auto [dim, bound] : llvm::enumerate(First: mapOp.getBounds())) {
8154 auto bounds = cast<omp::MapBoundsOp>(Val: bound.getDefiningOp());
8155 auto getInt64 = [&](Value value) {
8156 return builder.CreateSExtOrTrunc(V: moduleTranslation.lookupValue(value),
8157 DestTy: builder.getInt64Ty());
8158 };
8159 llvm::Value *lower = bounds.getLowerBound()
8160 ? getInt64(bounds.getLowerBound())
8161 : builder.getInt64(C: 0);
8162 llvm::Value *count = builder.CreateAdd(
8163 LHS: builder.CreateSub(LHS: getInt64(bounds.getUpperBound()), RHS: lower),
8164 RHS: builder.getInt64(C: 1));
8165 count =
8166 builder.CreateSelect(C: builder.CreateICmpSGT(LHS: count, RHS: builder.getInt64(C: 0)),
8167 True: count, False: builder.getInt64(C: 0));
8168 llvm::Value *stride = sourceStride;
8169 if (bounds.getStrideInBytes() && bounds.getStride())
8170 stride = getInt64(bounds.getStride());
8171 else if (arrayExtents.empty() && bounds.getStride())
8172 stride = builder.CreateMul(LHS: getInt64(bounds.getStride()), RHS: elementSize);
8173 offsets.push_back(Elt: builder.CreateMul(LHS: lower, RHS: stride));
8174 counts.push_back(Elt: count);
8175 strides.push_back(Elt: stride);
8176 isContiguous &= stride == contiguousSize;
8177 contiguousSize = builder.CreateMul(LHS: contiguousSize, RHS: count);
8178
8179 // Static array types carry the full source shape, including dimensions
8180 // selected with scalar subscripts. Otherwise the frontend supplies the
8181 // physical stride in elements (or descriptor bytes) for every dimension.
8182 if (dim < arrayExtents.size())
8183 sourceStride = builder.CreateMul(
8184 LHS: sourceStride,
8185 RHS: builder.getInt64(C: arrayExtents[arrayExtents.size() - dim - 1]));
8186 else if (bounds.getExtent())
8187 sourceStride =
8188 builder.CreateMul(LHS: sourceStride, RHS: getInt64(bounds.getExtent()));
8189 }
8190 if (isContiguous)
8191 return;
8192
8193 llvm::Value *base = mapData.OriginalValue[mapDataIdx];
8194 if (mapOp.getVarPtrPtr())
8195 base = builder.CreateLoad(Ty: builder.getPtrTy(), Ptr: base);
8196 // Absent optional arrays and unallocated descriptors must not issue
8197 // transfers, including sections with explicit nonempty bounds.
8198 counts[1] = builder.CreateSelect(C: builder.CreateIsNull(Arg: base),
8199 True: builder.getInt64(C: 0), False: counts[1]);
8200 // The descriptor offsets already locate the first selected element. The
8201 // runtime needs the pointee base, not the descriptor or a section address.
8202 combinedInfo.BasePointers.back() = base;
8203 combinedInfo.Types.back() |=
8204 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
8205 auto &nonContig = combinedInfo.NonContigInfo;
8206 nonContig.IsNonContiguous = true;
8207 nonContig.Dims.resize(N: combinedInfo.Types.size(), NV: 1);
8208 nonContig.Dims.back() = counts.size();
8209 nonContig.Offsets.push_back(Elt: std::move(offsets));
8210 nonContig.Counts.push_back(Elt: std::move(counts));
8211 nonContig.Strides.push_back(Elt: std::move(strides));
8212}
8213
8214/// This function handles the insertion of a single item of map data from
8215/// MapInfoData into the OMPIRBuilder's MapInfo list. Utilising this function
8216/// means the map being inserted can be treated as a non-parent map entity,
8217/// if the memberOfFlag is set then the map being inserted is treated as
8218/// a member map of a larger entity. The insertion into the MapInfo list of
8219/// the OMPIRBuilder can vary based on a number of factors, such as if it's
8220/// a ref_ptr or ref_ptee map, if it's a member of a record, what construct
8221/// the map belongs to and the various map type bit flags that are set for
8222/// the map.
8223static void processIndividualMap(
8224 LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder,
8225 llvm::OpenMPIRBuilder &ompBuilder, MapInfoData &mapData, size_t mapDataIdx,
8226 MapInfosTy &combinedInfo, TargetDirectiveEnumTy targetDirective,
8227 llvm::omp::OpenMPOffloadMappingFlags memberOfFlag =
8228 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE,
8229 bool isTargetParam = true, int mapDataParentIdx = -1) {
8230 auto mapFlag = mapData.Types[mapDataIdx];
8231 auto mapInfoOp = llvm::cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIdx]);
8232
8233 bool isPtrTy = checkIfPointerMap(mapOp: mapInfoOp);
8234 bool isAttachMap = ((convertClauseMapFlags(mlirFlags: mapInfoOp.getMapType()) &
8235 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH) ==
8236 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
8237
8238 // Declare target variables are not passed to the kernel, and for the moment
8239 // attach maps are not passed to the kernel. However, it is possible to create
8240 // attach maps that transfer data and thus can be kernel arguments, but our
8241 // existing frontend does not do this.
8242 if (isTargetParam &&
8243 (targetDirective == TargetDirectiveEnumTy::Target &&
8244 !mapData.IsDeclareTarget[mapDataIdx]) &&
8245 !isAttachMap)
8246 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
8247
8248 if (mapInfoOp.getMapCaptureType() == omp::VariableCaptureKind::ByCopy &&
8249 !isPtrTy)
8250 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL;
8251
8252 // If we have a pointer and it's part of a MEMBER_OF mapping we do not apply
8253 // MEMBER_OF, as the runtime currently has a work-around that utilises
8254 // MEMBER_OF to prevent reference updating in certain scenarios instead of
8255 // target_param. However, this causes a noticeable issue in cases where we
8256 // map some data (Fortran descriptor primarily at the moment), alter it on
8257 // the host, and then expect it to not be updated in a subsequent implicit map
8258 // (such as an implicit map on a target).
8259 if (memberOfFlag != llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE) {
8260 if (!isPtrTy && !isAttachMap)
8261 ompBuilder.setCorrectMemberOfFlag(Flags&: mapFlag, MemberOfFlag: memberOfFlag);
8262
8263 // The return parameter should be the over-riding parent in cases where we
8264 // have a return parameter that is echoed to all members, the main case of
8265 // this currently is with fortran descriptors. It may need more finessing
8266 // for C/C++ in the future or descriptors that are members of derived
8267 // types.
8268 mapFlag &= ~llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8269 }
8270
8271 // We apply MAP_PTR_AND_OBJ when within a declare mapper object as it enforces
8272 // MEMBER_OF mappings on maps that are passed the initial nesting depth, which
8273 // includes pointed to data and attach members, both of which are technically
8274 // not part of the main object. This has the side effect of causing early
8275 // map-backs in certain cases where an implicit declare mapper has been
8276 // emitted for a target region. Applying MAP_PTR_AND_OBJ in these situations
8277 // circumvents this.
8278 if (isPtrTy && !isAttachMap && mapData.IsDeclareTarget[mapDataIdx])
8279 mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
8280
8281 // if we're provided a mapDataParentIdx, then the data being mapped is
8282 // part of a larger object (in a parent <-> member mapping) and in this
8283 // case our BasePointer should be the parent. Except in the edge case
8284 // where we are mapping pointee data, where we try staying close to
8285 // what Clang currently does and utilise the regular base pointer of the
8286 // data.
8287 bool isRefPtee =
8288 !bitEnumContainsAll(bits: mapInfoOp.getMapType(),
8289 bit: omp::ClauseMapFlags::ref_ptr) &&
8290 bitEnumContainsAll(bits: mapInfoOp.getMapType(), bit: omp::ClauseMapFlags::ref_ptee);
8291 bool isRefPtrPtee = bitEnumContainsAll(bits: mapInfoOp.getMapType(),
8292 bit: omp::ClauseMapFlags::ref_ptr |
8293 omp::ClauseMapFlags::ref_ptee);
8294
8295 if (!mapInfoOp->getParentOfType<omp::DeclareMapperOp>() &&
8296 mapDataParentIdx >= 0 && !(isRefPtee || (isRefPtrPtee && isPtrTy))) {
8297 combinedInfo.BasePointers.emplace_back(
8298 Args&: mapData.BasePointers[mapDataParentIdx]);
8299 } else {
8300 combinedInfo.BasePointers.emplace_back(Args&: mapData.BasePointers[mapDataIdx]);
8301 }
8302
8303 combinedInfo.Pointers.emplace_back(Args&: mapData.Pointers[mapDataIdx]);
8304 combinedInfo.DevicePointers.emplace_back(
8305 Args: memberOfFlag != llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE
8306 ? llvm::OpenMPIRBuilder::DeviceInfoTy::None
8307 : mapData.DevicePointers[mapDataIdx]);
8308 combinedInfo.Mappers.emplace_back(Args&: mapData.Mappers[mapDataIdx]);
8309 combinedInfo.Names.emplace_back(Args&: mapData.Names[mapDataIdx]);
8310 combinedInfo.Types.emplace_back(Args&: mapFlag);
8311 // TODO: set HasAttachPtr from Flang for pointee-storage entries.
8312 combinedInfo.HasAttachPtr.emplace_back(Args: false);
8313 // A privatized attach map needs storage for the pointer or descriptor even
8314 // when its pointee is null. Its size describes that storage, not the pointee,
8315 // and is needed by the runtime for corresponding-pointer-initialization.
8316 combinedInfo.Sizes.emplace_back(
8317 Args: isPtrTy && !isPrivatizeableAttachMap(mapType: mapInfoOp.getMapType())
8318 ? builder.CreateSelect(
8319 C: builder.CreateIsNull(Arg: mapData.Pointers[mapDataIdx]),
8320 True: builder.getInt64(C: 0), False: mapData.Sizes[mapDataIdx])
8321 : mapData.Sizes[mapDataIdx]);
8322 if (targetDirective == TargetDirectiveEnumTy::TargetUpdate)
8323 addNonContiguousUpdate(moduleTranslation, builder, mapData, mapDataIdx,
8324 combinedInfo);
8325}
8326
8327// This creates two insertions into the MapInfosTy data structure for the
8328// "parent" of a set of members, (usually a container e.g.
8329// class/structure/derived type) when subsequent members have also been
8330// explicitly mapped on the same map clause. Certain types, such as Fortran
8331// descriptors are mapped like this as well, however, the members are
8332// implicit as far as a user is concerned, but we must explicitly map them
8333// internally.
8334//
8335// This function also returns the memberOfFlag for this particular parent,
8336// which is utilised in subsequent member mappings (by modifying there map type
8337// with it) to indicate that a member is part of this parent and should be
8338// treated by the runtime as such. Important to achieve the correct mapping.
8339//
8340// This function borrows a lot from Clang's emitCombinedEntry function
8341// inside of CGOpenMPRuntime.cpp
8342static void mapParentWithMembers(
8343 LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder,
8344 llvm::OpenMPIRBuilder &ompBuilder, DataLayout &dl, MapInfosTy &combinedInfo,
8345 MapInfoData &mapData, uint64_t mapDataIndex,
8346 llvm::omp::OpenMPOffloadMappingFlags memberOfFlag,
8347 TargetDirectiveEnumTy targetDirective) {
8348 using MapFlags = llvm::omp::OpenMPOffloadMappingFlags;
8349 assert(!ompBuilder.Config.isTargetDevice() &&
8350 "function only supported for host device codegen");
8351 auto parentClause =
8352 llvm::cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]);
8353 auto *parentMapper = mapData.Mappers[mapDataIndex];
8354
8355 // Map the first segment of the parent. If a user-defined mapper is attached,
8356 // include the parent's to/from-style bits (and common modifiers) in this
8357 // base entry so the mapper receives correct copy semantics via its 'type'
8358 // parameter. Also keep TARGET_PARAM when required for kernel arguments.
8359 MapFlags baseFlag = (targetDirective == TargetDirectiveEnumTy::Target &&
8360 !mapData.IsDeclareTarget[mapDataIndex])
8361 ? MapFlags::OMP_MAP_TARGET_PARAM
8362 : MapFlags::OMP_MAP_NONE;
8363
8364 if (parentMapper) {
8365 // Preserve relevant map-type bits from the parent clause. These include
8366 // the copy direction (TO/FROM), as well as commonly used modifiers that
8367 // should be visible to the mapper for correct behaviour.
8368 MapFlags parentFlags = mapData.Types[mapDataIndex];
8369 MapFlags preserve = MapFlags::OMP_MAP_TO | MapFlags::OMP_MAP_FROM |
8370 MapFlags::OMP_MAP_ALWAYS | MapFlags::OMP_MAP_CLOSE |
8371 MapFlags::OMP_MAP_PRESENT |
8372 MapFlags::OMP_MAP_OMPX_HOLD |
8373 MapFlags::OMP_MAP_IMPLICIT;
8374 baseFlag |= (parentFlags & preserve);
8375 } else {
8376 MapFlags parentFlags = mapData.Types[mapDataIndex];
8377 MapFlags preserve = MapFlags::OMP_MAP_TO | MapFlags::OMP_MAP_FROM |
8378 MapFlags::OMP_MAP_PRESENT |
8379 MapFlags::OMP_MAP_RETURN_PARAM |
8380 MapFlags::OMP_MAP_IMPLICIT;
8381 baseFlag |= (parentFlags & preserve);
8382 }
8383
8384 combinedInfo.Types.emplace_back(Args&: baseFlag);
8385 // TODO: set HasAttachPtr from Flang for pointee-storage entries.
8386 combinedInfo.HasAttachPtr.emplace_back(Args: false);
8387 combinedInfo.DevicePointers.emplace_back(
8388 Args&: mapData.DevicePointers[mapDataIndex]);
8389 // Only attach the mapper to the base entry when we are mapping the whole
8390 // parent. Combined/segment entries must not carry a mapper; otherwise the
8391 // mapper can be invoked with a partial size, which is undefined behaviour.
8392 combinedInfo.Mappers.emplace_back(
8393 Args: parentMapper && !parentClause.getPartialMap() ? parentMapper : nullptr);
8394 combinedInfo.Names.emplace_back(Args: LLVM::createMappingInformation(
8395 loc: mapData.MapClause[mapDataIndex]->getLoc(), builder&: ompBuilder,
8396 mapName: getMapClauseName(mapOp: cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]))));
8397 combinedInfo.BasePointers.emplace_back(Args&: mapData.BasePointers[mapDataIndex]);
8398
8399 // Calculate size of the parent object being mapped based on the
8400 // addresses at runtime, highAddr - lowAddr = size. This of course
8401 // doesn't factor in allocated data like pointers, hence the further
8402 // processing of members specified by users, or in the case of
8403 // Fortran pointers and allocatables, the mapping of the pointed to
8404 // data by the descriptor (which itself, is a structure containing
8405 // runtime information on the dynamically allocated data).
8406 llvm::Value *lowAddr, *highAddr;
8407 if (!parentClause.getPartialMap()) {
8408 lowAddr = builder.CreatePointerCast(V: mapData.Pointers[mapDataIndex],
8409 DestTy: builder.getPtrTy());
8410 highAddr = builder.CreatePointerCast(
8411 V: builder.CreateConstGEP1_32(Ty: mapData.BaseType[mapDataIndex],
8412 Ptr: mapData.Pointers[mapDataIndex], Idx0: 1),
8413 DestTy: builder.getPtrTy());
8414 combinedInfo.Pointers.emplace_back(Args&: mapData.Pointers[mapDataIndex]);
8415 } else {
8416 auto mapOp = dyn_cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]);
8417 int firstMemberIdx = getMapDataMemberIdx(
8418 mapData, memberOp: getFirstOrLastMappedMemberPtr(mapInfo: mapOp, first: true));
8419 lowAddr = builder.CreatePointerCast(V: mapData.BasePointers[firstMemberIdx],
8420 DestTy: builder.getPtrTy());
8421
8422 int lastMemberIdx = getMapDataMemberIdx(
8423 mapData, memberOp: getFirstOrLastMappedMemberPtr(mapInfo: mapOp, first: false));
8424 auto lastMemberMapInfo =
8425 cast<omp::MapInfoOp>(Val: mapData.MapClause[lastMemberIdx]);
8426
8427 // NOTE: Currently, for RefPtee the BaseType is set to the varPtrPtr field,
8428 // which is the pointer datas type and not the member within the structure
8429 // that it's part of, so we have to make sure we use the member type in this
8430 // case when calculating the parents size offsets.
8431 // TODO: May be good to extend MapInfoData to support tracking of both
8432 // VarPtr/VarPtrPtr BaseType's to better distinguish what's being used more
8433 // consistently.
8434 bool isRefPteeMap = bitEnumContainsAll(bits: lastMemberMapInfo.getMapType(),
8435 bit: omp::ClauseMapFlags::ref_ptee) &&
8436 !bitEnumContainsAll(bits: lastMemberMapInfo.getMapType(),
8437 bit: omp::ClauseMapFlags::ref_ptr);
8438 llvm::Type *castType = mapData.BaseType[lastMemberIdx];
8439 if (isRefPteeMap)
8440 castType =
8441 moduleTranslation.convertType(type: lastMemberMapInfo.getVarPtrType());
8442 highAddr = builder.CreatePointerCast(
8443 V: builder.CreateGEP(Ty: castType, Ptr: mapData.BasePointers[lastMemberIdx],
8444 IdxList: builder.getInt64(C: 1)),
8445 DestTy: builder.getPtrTy());
8446 combinedInfo.Pointers.emplace_back(Args&: mapData.BasePointers[firstMemberIdx]);
8447 }
8448
8449 llvm::Value *size = builder.CreateIntCast(
8450 V: builder.CreatePtrDiff(ElemTy: builder.getInt8Ty(), LHS: highAddr, RHS: lowAddr),
8451 DestTy: builder.getInt64Ty(),
8452 /*isSigned=*/false);
8453 combinedInfo.Sizes.push_back(Elt: size);
8454
8455 // This creates the initial MEMBER_OF mapping that consists of
8456 // the parent/top level container (same as above effectively, except
8457 // with a fixed initial compile time size and separate maptype which
8458 // indicates the true mape type (tofrom etc.). This parent mapping is
8459 // only relevant if the structure in its totality is being mapped,
8460 // otherwise the above suffices.
8461 if (!parentClause.getPartialMap()) {
8462 // TODO: This will need to be expanded to include the whole host of logic
8463 // for the map flags that Clang currently supports (e.g. it should do some
8464 // further case specific flag modifications). For the moment, it handles
8465 // what we support as expected.
8466 MapFlags mapFlag = mapData.Types[mapDataIndex];
8467 bool hasMapClose = (MapFlags(mapFlag) & MapFlags::OMP_MAP_CLOSE) ==
8468 MapFlags::OMP_MAP_CLOSE;
8469 ompBuilder.setCorrectMemberOfFlag(Flags&: mapFlag, MemberOfFlag: memberOfFlag);
8470
8471 llvm::SmallVector<size_t> overlapIdxs;
8472 // Find all of the members that "overlap", i.e. occlude other members that
8473 // were mapped alongside the parent, e.g. member [0], occludes [0,1] and
8474 // [0,2], but not [1,0].
8475 getOverlappedMembers(overlapMapDataIdxs&: overlapIdxs, parentOp: parentClause);
8476
8477 // When we only have one overlap we skip the case that tries to segment the
8478 // mapping as best it can without creating holes, as the calculation is more
8479 // likely to have more overhead than anything we gain from mapping a smaller
8480 // chunk of data. This can be seen in cases where we are mapping Fortran
8481 // descriptors which are a special case of record type mapping.
8482 //
8483 // The cases for close and update are unique edge cases where the segmenting
8484 // does not play well with the runtime currently.
8485 if (targetDirective == TargetDirectiveEnumTy::TargetUpdate || hasMapClose ||
8486 overlapIdxs.size() == 1) {
8487 combinedInfo.Types.emplace_back(Args&: mapFlag);
8488 // TODO: set HasAttachPtr from Flang for pointee-storage entries.
8489 combinedInfo.HasAttachPtr.emplace_back(Args: false);
8490 combinedInfo.DevicePointers.emplace_back(
8491 Args&: mapData.DevicePointers[mapDataIndex]);
8492 combinedInfo.Names.emplace_back(Args: LLVM::createMappingInformation(
8493 loc: mapData.MapClause[mapDataIndex]->getLoc(), builder&: ompBuilder,
8494 mapName: getMapClauseName(
8495 mapOp: cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]))));
8496 combinedInfo.BasePointers.emplace_back(
8497 Args&: mapData.BasePointers[mapDataIndex]);
8498 combinedInfo.Pointers.emplace_back(Args&: mapData.Pointers[mapDataIndex]);
8499 combinedInfo.Sizes.emplace_back(Args&: mapData.Sizes[mapDataIndex]);
8500 combinedInfo.Mappers.emplace_back(Args: nullptr);
8501 } else {
8502 // We need to make sure the overlapped members are sorted in order of
8503 // lowest address to highest address.
8504 sortMapIndices(indices&: overlapIdxs, mapInfo: parentClause);
8505
8506 lowAddr = builder.CreatePointerCast(V: mapData.Pointers[mapDataIndex],
8507 DestTy: builder.getPtrTy());
8508 highAddr = builder.CreatePointerCast(
8509 V: builder.CreateConstGEP1_32(Ty: mapData.BaseType[mapDataIndex],
8510 Ptr: mapData.Pointers[mapDataIndex], Idx0: 1),
8511 DestTy: builder.getPtrTy());
8512
8513 // Currently, the return parameter should be the over-riding parent in
8514 // cases where we have a return parameter that is echoed to all members,
8515 // the main case of this currently is with fortran descriptors. It may
8516 // need more finessing for C/C++ in the future or descriptors that are
8517 // members of derived types.
8518 mapFlag &= ~llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
8519
8520 // TODO: We may want to skip arrays/array sections in this as Clang does.
8521 // It appears to be an optimisation rather than a necessity though,
8522 // but this requires further investigation. However, we would have to make
8523 // sure to not exclude maps with bounds that ARE pointers, as these are
8524 // processed as separate components, i.e. pointer + data.
8525 for (auto v : overlapIdxs) {
8526 auto mapDataOverlapIdx = getMapDataMemberIdx(
8527 mapData,
8528 memberOp: cast<omp::MapInfoOp>(Val: parentClause.getMembers()[v].getDefiningOp()));
8529 auto isPtrMap = checkIfPointerMap(
8530 mapOp: llvm::cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataOverlapIdx]));
8531 combinedInfo.Types.emplace_back(Args&: mapFlag);
8532 // TODO: set HasAttachPtr from Flang for pointee-storage entries.
8533 combinedInfo.HasAttachPtr.emplace_back(Args: false);
8534 combinedInfo.DevicePointers.emplace_back(
8535 Args: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
8536 combinedInfo.Names.emplace_back(Args: LLVM::createMappingInformation(
8537 loc: mapData.MapClause[mapDataIndex]->getLoc(), builder&: ompBuilder,
8538 mapName: getMapClauseName(
8539 mapOp: cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]))));
8540 combinedInfo.BasePointers.emplace_back(
8541 Args&: mapData.BasePointers[mapDataIndex]);
8542 combinedInfo.Mappers.emplace_back(Args: nullptr);
8543 combinedInfo.Pointers.emplace_back(Args&: lowAddr);
8544 auto sizeCalc = builder.CreateIntCast(
8545 V: builder.CreatePtrDiff(ElemTy: builder.getInt8Ty(),
8546 LHS: mapData.OriginalValue[mapDataOverlapIdx],
8547 RHS: lowAddr),
8548 DestTy: builder.getInt64Ty(), /*isSigned=*/true);
8549 // In certain cases, we'll generate a size of 0 if we're not careful
8550 // (e.g. if lowAddr happens to be the first member), which isn't
8551 // correct, even if the runtimes is sometimes fine with it so, in these
8552 // scenarios we select the types size instead.
8553 llvm::DataLayout dataLayout = builder.getDataLayout();
8554 auto sizeSel = builder.CreateSelect(
8555 C: builder.CreateICmpNE(LHS: builder.getInt64(C: 0), RHS: sizeCalc), True: sizeCalc,
8556 False: isPtrMap ? builder.getInt64(C: dataLayout.getPointerSize())
8557 : mapData.Sizes[mapDataOverlapIdx]);
8558 combinedInfo.Sizes.emplace_back(Args&: sizeSel);
8559 lowAddr = builder.CreateConstGEP1_32(
8560 Ty: isPtrMap ? builder.getPtrTy() : mapData.BaseType[mapDataOverlapIdx],
8561 Ptr: mapData.BasePointers[mapDataOverlapIdx], Idx0: 1);
8562 }
8563
8564 combinedInfo.Types.emplace_back(Args&: mapFlag);
8565 // TODO: set HasAttachPtr from Flang for pointee-storage entries.
8566 combinedInfo.HasAttachPtr.emplace_back(Args: false);
8567 combinedInfo.DevicePointers.emplace_back(
8568 Args: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
8569 combinedInfo.Names.emplace_back(Args: LLVM::createMappingInformation(
8570 loc: mapData.MapClause[mapDataIndex]->getLoc(), builder&: ompBuilder,
8571 mapName: getMapClauseName(
8572 mapOp: cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]))));
8573 combinedInfo.BasePointers.emplace_back(
8574 Args&: mapData.BasePointers[mapDataIndex]);
8575 combinedInfo.Mappers.emplace_back(Args: nullptr);
8576 combinedInfo.Pointers.emplace_back(Args&: lowAddr);
8577 combinedInfo.Sizes.emplace_back(Args: builder.CreateIntCast(
8578 V: builder.CreatePtrDiff(ElemTy: builder.getInt8Ty(), LHS: highAddr, RHS: lowAddr),
8579 DestTy: builder.getInt64Ty(), isSigned: true));
8580 }
8581 }
8582}
8583
8584static void processMapWithMembersOf(LLVM::ModuleTranslation &moduleTranslation,
8585 llvm::IRBuilderBase &builder,
8586 llvm::OpenMPIRBuilder &ompBuilder,
8587 DataLayout &dl, MapInfosTy &combinedInfo,
8588 MapInfoData &mapData, uint64_t mapDataIndex,
8589 TargetDirectiveEnumTy targetDirective) {
8590 assert(!ompBuilder.Config.isTargetDevice() &&
8591 "function only supported for host device codegen");
8592
8593 auto parentClause =
8594 llvm::cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]);
8595
8596 // If we have a partial map (no parent referenced in the map clauses of the
8597 // directive, only members) and only a single member, we do not need to bind
8598 // the map of the member to the parent, we can pass the member separately.
8599 if (parentClause.getMembers().size() == 1 && parentClause.getPartialMap()) {
8600 auto memberClause = llvm::cast<omp::MapInfoOp>(
8601 Val: parentClause.getMembers()[0].getDefiningOp());
8602 int memberDataIdx = getMapDataMemberIdx(mapData, memberOp: memberClause);
8603 // Note: Clang treats arrays with explicit bounds that fall into this
8604 // category as a parent with map case, however, it seems this isn't a
8605 // requirement, and processing them as an individual map is fine. So,
8606 // we will handle them as individual maps for the moment, as it's
8607 // difficult for us to check this as we always require bounds to be
8608 // specified currently and it's also marginally more optimal (single
8609 // map rather than two). The difference may come from the fact that
8610 // Clang maps array without bounds as pointers (which we do not
8611 // currently do), whereas we treat them as arrays in all cases
8612 // currently.
8613 processIndividualMap(
8614 moduleTranslation, builder, ompBuilder, mapData, mapDataIdx: memberDataIdx,
8615 combinedInfo, targetDirective,
8616 /*MemberOfFlag=*/memberOfFlag: llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE,
8617 /*isTargetParam=*/true, mapDataParentIdx: mapDataIndex);
8618 return;
8619 }
8620
8621 auto collectMapInfoIdxs =
8622 [&](llvm::SmallVectorImpl<int64_t> &mapsAndInfoIdx) {
8623 auto parentClause =
8624 llvm::cast<omp::MapInfoOp>(Val: mapData.MapClause[mapDataIndex]);
8625 mapsAndInfoIdx.push_back(Elt: getMapDataMemberIdx(mapData, memberOp: parentClause));
8626 for (auto member : parentClause.getMembers())
8627 mapsAndInfoIdx.push_back(Elt: getMapDataMemberIdx(
8628 mapData, memberOp: llvm::cast<omp::MapInfoOp>(Val: member.getDefiningOp())));
8629 };
8630
8631 llvm::SmallVector<int64_t> mapInfoIdx;
8632 collectMapInfoIdxs(mapInfoIdx);
8633
8634 llvm::omp::OpenMPOffloadMappingFlags memberOfFlag =
8635 ompBuilder.getMemberOfFlag(Position: combinedInfo.Types.size());
8636
8637 // The first index is the parent map, the rest are its members. The parent
8638 // normally undergoes the standard parent-with-members mapping, contributing
8639 // the MEMBER_OF flag that binds each member to it. The one exception is a
8640 // privatizeable attach map (a privatized pointer/descriptor passed directly
8641 // as a kernel argument): here the parent is emitted as an individual map
8642 // instead, for the time being, as it's used only in pointer/allocatable to
8643 // array cases for the moment. This only ever applies to the parent, so it is
8644 // checked once here rather than inside the loop below.
8645 bool parentIsPrivatizeableAttach =
8646 isPrivatizeableAttachMap(mapType: parentClause.getMapType());
8647 for (auto [i, idx] : llvm::enumerate(First&: mapInfoIdx)) {
8648 bool emitParentMap = i == 0 && !parentIsPrivatizeableAttach;
8649 if (emitParentMap) {
8650 mapParentWithMembers(moduleTranslation, builder, ompBuilder, dl,
8651 combinedInfo, mapData, mapDataIndex: idx, memberOfFlag,
8652 targetDirective);
8653 } else {
8654 processIndividualMap(
8655 moduleTranslation, builder, ompBuilder, mapData, mapDataIdx: idx, combinedInfo,
8656 targetDirective,
8657 memberOfFlag: parentIsPrivatizeableAttach
8658 ? llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE
8659 : memberOfFlag,
8660 /*isTargetParam=*/false, mapDataParentIdx: mapDataIndex);
8661 }
8662 }
8663}
8664
8665// This is a variation on Clang's GenerateOpenMPCapturedVars, which
8666// generates different operation (e.g. load/store) combinations for
8667// arguments to the kernel, based on map capture kinds which are then
8668// utilised in the combinedInfo in place of the original Map value.
8669static void
8670createAlteredByCaptureMap(MapInfoData &mapData,
8671 LLVM::ModuleTranslation &moduleTranslation,
8672 llvm::IRBuilderBase &builder) {
8673 assert(!getOpenMPBuilder(moduleTranslation)->Config.isTargetDevice() &&
8674 "function only supported for host device codegen");
8675 for (size_t i = 0; i < mapData.MapClause.size(); ++i) {
8676 auto mapOp = cast<omp::MapInfoOp>(Val: mapData.MapClause[i]);
8677 bool isAttachMap =
8678 ((convertClauseMapFlags(mlirFlags: mapOp.getMapType()) &
8679 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH) ==
8680 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
8681
8682 // If it's declare target, skip it, it's handled separately. However, if
8683 // it's declare target, and an attach map, we want to calculate the exact
8684 // address offset so that we attach correctly.
8685 if (!mapData.IsDeclareTarget[i] ||
8686 (mapData.IsDeclareTarget[i] && isAttachMap)) {
8687 omp::VariableCaptureKind captureKind = mapOp.getMapCaptureType();
8688 bool isPtrTy = checkIfPointerMap(mapOp);
8689
8690 // Currently handles array sectioning lowerbound case, but more
8691 // logic may be required in the future. Clang invokes EmitLValue,
8692 // which has specialised logic for special Clang types such as user
8693 // defines, so it is possible we will have to extend this for
8694 // structures or other complex types. As the general idea is that this
8695 // function mimics some of the logic from Clang that we require for
8696 // kernel argument passing from host -> device.
8697 switch (captureKind) {
8698 case omp::VariableCaptureKind::ByRef: {
8699 llvm::Value *newV = mapData.Pointers[i];
8700 bool isByteOffset = false;
8701 std::vector<llvm::Value *> offsetIdx = calculateBoundsOffset(
8702 moduleTranslation, builder, isArrayTy: mapData.BaseType[i]->isArrayTy(),
8703 bounds: mapOp.getBounds(), isByteOffset);
8704 if (isPtrTy)
8705 newV = builder.CreateLoad(Ty: builder.getPtrTy(), Ptr: newV);
8706
8707 if (!offsetIdx.empty())
8708 newV = builder.CreateInBoundsGEP(Ty: isByteOffset ? builder.getInt8Ty()
8709 : mapData.BaseType[i],
8710 Ptr: newV, IdxList: offsetIdx, Name: "array_offset");
8711 mapData.Pointers[i] = newV;
8712 } break;
8713 case omp::VariableCaptureKind::ByCopy: {
8714 llvm::Type *type = mapData.BaseType[i];
8715 llvm::Value *newV;
8716 if (mapData.Pointers[i]->getType()->isPointerTy())
8717 newV = builder.CreateLoad(Ty: type, Ptr: mapData.Pointers[i]);
8718 else
8719 newV = mapData.Pointers[i];
8720
8721 if (!isPtrTy) {
8722 auto curInsert = builder.saveIP();
8723 llvm::DebugLoc DbgLoc = builder.getCurrentDebugLocation();
8724 builder.restoreIP(IP: findAllocInsertPoints(builder, moduleTranslation));
8725 auto *memTempAlloc =
8726 builder.CreateAlloca(Ty: builder.getPtrTy(), ArraySize: nullptr, Name: ".casted");
8727 builder.SetCurrentDebugLocation(DbgLoc);
8728 builder.restoreIP(IP: curInsert);
8729
8730 builder.CreateStore(Val: newV, Ptr: memTempAlloc);
8731 newV = builder.CreateLoad(Ty: builder.getPtrTy(), Ptr: memTempAlloc);
8732 }
8733
8734 mapData.Pointers[i] = newV;
8735 mapData.BasePointers[i] = newV;
8736 } break;
8737 case omp::VariableCaptureKind::This:
8738 case omp::VariableCaptureKind::VLAType:
8739 mapData.MapClause[i]->emitOpError(message: "Unhandled capture kind");
8740 break;
8741 }
8742 }
8743 }
8744}
8745
8746// Generate all map related information and fill the combinedInfo.
8747static void genMapInfos(llvm::IRBuilderBase &builder,
8748 LLVM::ModuleTranslation &moduleTranslation,
8749 DataLayout &dl, MapInfosTy &combinedInfo,
8750 MapInfoData &mapData,
8751 TargetDirectiveEnumTy targetDirective) {
8752 assert(!getOpenMPBuilder(moduleTranslation)->Config.isTargetDevice() &&
8753 "function only supported for host device codegen");
8754 // We wish to modify some of the methods in which arguments are
8755 // passed based on their capture type by the target region, this can
8756 // involve generating new loads and stores, which changes the
8757 // MLIR value to LLVM value mapping, however, we only wish to do this
8758 // locally for the current function/target and also avoid altering
8759 // ModuleTranslation, so we remap the base pointer or pointer stored
8760 // in the map infos corresponding MapInfoData, which is later accessed
8761 // by genMapInfos and createTarget to help generate the kernel and
8762 // kernel arg structure. It primarily becomes relevant in cases like
8763 // bycopy, or byref range'd arrays. In the default case, we simply
8764 // pass thee pointer byref as both basePointer and pointer.
8765 createAlteredByCaptureMap(mapData, moduleTranslation, builder);
8766
8767 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
8768
8769 // We operate under the assumption that all vectors that are
8770 // required in MapInfoData are of equal lengths (either filled with
8771 // default constructed data or appropiate information) so we can
8772 // utilise the size from any component of MapInfoData, if we can't
8773 // something is missing from the initial MapInfoData construction.
8774 for (size_t i = 0; i < mapData.MapClause.size(); ++i) {
8775 if (mapData.IsAMember[i])
8776 continue;
8777
8778 auto mapInfoOp = dyn_cast<omp::MapInfoOp>(Val: mapData.MapClause[i]);
8779 if (!mapInfoOp.getMembers().empty()) {
8780 processMapWithMembersOf(moduleTranslation, builder, ompBuilder&: *ompBuilder, dl,
8781 combinedInfo, mapData, mapDataIndex: i, targetDirective);
8782 continue;
8783 }
8784
8785 processIndividualMap(moduleTranslation, builder, ompBuilder&: *ompBuilder, mapData, mapDataIdx: i,
8786 combinedInfo, targetDirective);
8787 }
8788 if (!combinedInfo.NonContigInfo.Dims.empty())
8789 combinedInfo.NonContigInfo.Dims.resize(N: combinedInfo.Types.size(), NV: 1);
8790}
8791
8792static llvm::Expected<llvm::Function *>
8793emitUserDefinedMapper(Operation *declMapperOp, llvm::IRBuilderBase &builder,
8794 LLVM::ModuleTranslation &moduleTranslation,
8795 llvm::StringRef mapperFuncName,
8796 TargetDirectiveEnumTy targetDirective);
8797
8798static llvm::Expected<llvm::Function *>
8799getOrCreateUserDefinedMapperFunc(Operation *op, llvm::IRBuilderBase &builder,
8800 LLVM::ModuleTranslation &moduleTranslation,
8801 TargetDirectiveEnumTy targetDirective) {
8802 assert(!getOpenMPBuilder(moduleTranslation)->Config.isTargetDevice() &&
8803 "function only supported for host device codegen");
8804 auto declMapperOp = cast<omp::DeclareMapperOp>(Val: op);
8805 std::string mapperFuncName =
8806 getOpenMPBuilder(moduleTranslation)
8807 ->createPlatformSpecificName(
8808 Parts: {"omp_mapper", declMapperOp.getSymName()});
8809
8810 if (auto *lookupFunc = moduleTranslation.lookupFunction(name: mapperFuncName))
8811 return lookupFunc;
8812
8813 // Recursive types can cause re-entrant mapper emission. The mapper function
8814 // is created by OpenMPIRBuilder before the callbacks run, so it may already
8815 // exist in the LLVM module even though it is not yet registered in the
8816 // ModuleTranslation mapping table. Reuse and register it to break the
8817 // recursion.
8818 if (llvm::Function *existingFunc =
8819 moduleTranslation.getLLVMModule()->getFunction(Name: mapperFuncName)) {
8820 moduleTranslation.mapFunction(name: mapperFuncName, func: existingFunc);
8821 return existingFunc;
8822 }
8823
8824 return emitUserDefinedMapper(declMapperOp, builder, moduleTranslation,
8825 mapperFuncName, targetDirective);
8826}
8827
8828/// The mapper body (and any iterator regions in it) is emitted inside the
8829/// mapper's per-element loop, so temporaries Flang allocates in the mapper
8830/// region (e.g. the storage for an iterator variable) would be allocated once
8831/// per element and grow the stack with every iteration. Move constant-size
8832/// allocas to the function's entry block, where a frontend would place them.
8833static void hoistMapperAllocasToEntry(llvm::Function &mapperFn) {
8834 llvm::BasicBlock &entry = mapperFn.getEntryBlock();
8835 llvm::SmallVector<llvm::AllocaInst *> toHoist;
8836 for (llvm::BasicBlock &bb : llvm::drop_begin(RangeOrContainer&: mapperFn))
8837 for (llvm::Instruction &inst : bb)
8838 if (auto *alloca = dyn_cast<llvm::AllocaInst>(Val: &inst))
8839 if (isa<llvm::ConstantInt>(Val: alloca->getArraySize()))
8840 toHoist.push_back(Elt: alloca);
8841 for (llvm::AllocaInst *alloca : toHoist)
8842 alloca->moveBefore(InsertPos: entry.getFirstNonPHIOrAlloca());
8843}
8844
8845static llvm::Expected<llvm::Function *>
8846emitUserDefinedMapper(Operation *op, llvm::IRBuilderBase &builder,
8847 LLVM::ModuleTranslation &moduleTranslation,
8848 llvm::StringRef mapperFuncName,
8849 TargetDirectiveEnumTy targetDirective) {
8850 assert(!getOpenMPBuilder(moduleTranslation)->Config.isTargetDevice() &&
8851 "function only supported for host device codegen");
8852 auto declMapperOp = cast<omp::DeclareMapperOp>(Val: op);
8853 auto declMapperInfoOp = declMapperOp.getDeclareMapperInfo();
8854 if (failed(Result: checkImplementationStatus(op&: *declMapperInfoOp)))
8855 return llvm::make_error<PreviouslyReportedError>();
8856
8857 DataLayout dl = DataLayout(declMapperOp->getParentOfType<ModuleOp>());
8858 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
8859 llvm::Type *varType = moduleTranslation.convertType(type: declMapperOp.getType());
8860 SmallVector<Value> mapVars = declMapperInfoOp.getMapVars();
8861
8862 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
8863
8864 // Fill up the arrays with all the mapped variables.
8865 MapInfosTy combinedInfo;
8866 auto genMapInfoCB =
8867 [&](InsertPointTy codeGenIP, llvm::Value *ptrPHI,
8868 llvm::Value *unused2) -> llvm::OpenMPIRBuilder::MapInfosOrErrorTy {
8869 builder.restoreIP(IP: codeGenIP);
8870 moduleTranslation.mapValue(mlir: declMapperOp.getSymVal(), llvm: ptrPHI);
8871 moduleTranslation.mapBlock(mlir: &declMapperOp.getRegion().front(),
8872 llvm: builder.GetInsertBlock());
8873 if (failed(Result: moduleTranslation.convertBlock(bb&: declMapperOp.getRegion().front(),
8874 /*ignoreArguments=*/true,
8875 builder)))
8876 return llvm::make_error<PreviouslyReportedError>();
8877 MapInfoData mapData;
8878 collectMapDataFromMapOperands(mapData, mapVars, moduleTranslation, dl,
8879 builder);
8880
8881 // Components produced by an `iterator` modifier on the mapper's own map
8882 // clause: their count is only known at runtime, so they are recorded as
8883 // dynamic segments and pushed one at a time (via
8884 // __tgt_push_mapper_component) from a generated loop, mirroring how the
8885 // depend/affinity clauses already handle iterator-produced entries. Each
8886 // per-iteration locator keeps its array base in var_ptr with bounds
8887 // selecting the mapped element, the same shape as an ordinary
8888 // array-section map, so the existing calculateBoundsOffset helper
8889 // computes its address.
8890 //
8891 // Like statically-known members (see processMapWithMembersOf), the
8892 // iterated elements are grouped under a component for the mapper variable
8893 // via MEMBER_OF, so the runtime treats them as sub-ranges of one
8894 // allocation rather than independent objects. If the mapper also maps the
8895 // variable itself that component is reused. Otherwise a storage-only
8896 // parent is synthesized that, like the parent of explicit members, spans
8897 // exactly the elements the iterators expand to: its begin/end are the
8898 // runtime min/max over all iterations, so it never extends a mapping of
8899 // the component the user may have made separately, and it is skipped
8900 // altogether when every iterator has a zero trip count (an empty iterator
8901 // acts as if the list item were omitted). The parent must precede every
8902 // other component: the runtime maps components in order and rejects a
8903 // later one that extends an earlier mapping.
8904 struct IteratedEntryInfo {
8905 mlir::omp::IteratorOp itersOp;
8906 mlir::omp::MapInfoOp mapInfoOp;
8907 llvm::Type *baseLLVMTy;
8908 llvm::Value *elemSizeVal;
8909 IteratorInfo iterInfo;
8910 };
8911 llvm::SmallVector<IteratedEntryInfo, 2> iteratedEntries;
8912 for (mlir::Value iterVal : declMapperInfoOp.getMapIterated()) {
8913 auto itersOp = iterVal.getDefiningOp<mlir::omp::IteratorOp>();
8914 assert(itersOp && "map_iterated value must be defined by omp.iterator");
8915 // Lowering the locator's indices (e.g. involving LEN_TRIM) can
8916 // introduce control flow inside the iterator, so its `omp.yield` is
8917 // not necessarily in the region's first block.
8918 mlir::omp::YieldOp yieldOp = itersOp.getYieldOp();
8919 assert(yieldOp && "expected exactly one omp.yield in the iterator body");
8920 auto mapInfoOp =
8921 yieldOp.getResults()[0].getDefiningOp<mlir::omp::MapInfoOp>();
8922 assert(mapInfoOp && "expected an omp.map.info in the map_iterated body");
8923
8924 mlir::Type varPtrMlirTy = mapInfoOp.getVarPtrType();
8925 // Byte size of one element of the locator's innermost dimension; the
8926 // bounds supply the (runtime) element count per iteration.
8927 uint64_t elemSizeInBits = dl.getTypeSizeInBits(t: varPtrMlirTy);
8928 if (auto arrTy = mlir::dyn_cast<mlir::LLVM::LLVMArrayType>(Val&: varPtrMlirTy))
8929 elemSizeInBits = getArrayElementSizeInBits(arrTy, dl);
8930 iteratedEntries.push_back(
8931 Elt: {.itersOp: itersOp, .mapInfoOp: mapInfoOp, .baseLLVMTy: moduleTranslation.convertType(type: varPtrMlirTy),
8932 .elemSizeVal: builder.getInt64(C: elemSizeInBits / 8),
8933 .iterInfo: IteratorInfo(itersOp, moduleTranslation, builder)});
8934 }
8935
8936 // Emits the (base, begin, size) of the entry for one iteration, given the
8937 // loop's linear induction variable. `info` is taken by value: it holds
8938 // the lb/ub/step/trip llvm::Value*s looked up while `declMapperOp`'s
8939 // region mapping was still valid, and this may run later (from within
8940 // OpenMPIRBuilder::emitUserDefinedMapper), after that mapping has been
8941 // dropped via forgetMapping.
8942 auto genIteratedEntry = [&moduleTranslation](IteratedEntryInfo info,
8943 llvm::IRBuilderBase &b,
8944 llvm::Value *linearIV)
8945 -> llvm::Expected<llvm::SmallVector<llvm::Value *, 3>> {
8946 if (failed(Result: convertIteratorRegion(linearIV, iterInfo&: info.iterInfo,
8947 iteratorRegion&: info.itersOp.getRegion(), builder&: b,
8948 moduleTranslation)))
8949 return llvm::make_error<PreviouslyReportedError>();
8950 llvm::Value *base =
8951 moduleTranslation.lookupValue(value: info.mapInfoOp.getVarPtr());
8952 bool isByteOffset = false;
8953 std::vector<llvm::Value *> offsetIdx = calculateBoundsOffset(
8954 moduleTranslation, builder&: b, isArrayTy: info.baseLLVMTy->isArrayTy(),
8955 bounds: info.mapInfoOp.getBounds(), isByteOffset);
8956 llvm::Value *addr = base;
8957 if (!offsetIdx.empty())
8958 addr = b.CreateInBoundsGEP(
8959 Ty: isByteOffset ? b.getInt8Ty() : info.baseLLVMTy, Ptr: base, IdxList: offsetIdx);
8960 // Size = product over dimensions of (ub - lb + 1), times the element
8961 // size, the same calculation getSizeInBytes does for static entries.
8962 llvm::Value *elementCount = b.getInt64(C: 1);
8963 for (mlir::Value bound : info.mapInfoOp.getBounds()) {
8964 auto boundOp = bound.getDefiningOp<mlir::omp::MapBoundsOp>();
8965 if (!boundOp)
8966 continue;
8967 llvm::Value *lb =
8968 moduleTranslation.lookupValue(value: boundOp.getLowerBound());
8969 llvm::Value *ub =
8970 moduleTranslation.lookupValue(value: boundOp.getUpperBound());
8971 elementCount = b.CreateMul(
8972 LHS: elementCount, RHS: b.CreateAdd(LHS: b.CreateSub(LHS: ub, RHS: lb), RHS: b.getInt64(C: 1)));
8973 }
8974 llvm::Value *size = b.CreateMul(LHS: elementCount, RHS: info.elemSizeVal);
8975 moduleTranslation.forgetMapping(region&: info.itersOp.getRegion());
8976 // The array base is the component's base pointer, as for a
8977 // statically-known member; MEMBER_OF ties it to the parent entry.
8978 return llvm::SmallVector<llvm::Value *, 3>{base, addr, size};
8979 };
8980
8981 bool hasIterated = !iteratedEntries.empty();
8982 std::optional<unsigned> iteratedParentIdx;
8983 // Span of the iterated elements, computed when the mapper does not map
8984 // the variable whole; null otherwise.
8985 llvm::Value *anyTrips = nullptr;
8986 llvm::Value *spanBegin = nullptr;
8987 llvm::Value *spanEnd = nullptr;
8988 bool hasStaticParent = false;
8989 if (hasIterated) {
8990 bool mapsWholeVariable = false;
8991 for (Value mapVar : mapVars) {
8992 auto mapOp = cast<omp::MapInfoOp>(Val: mapVar.getDefiningOp());
8993 if (mapOp.getVarPtr() != declMapperOp.getSymVal())
8994 continue;
8995 // A partial map with a single member is emitted as an individual map
8996 // of that member (see processMapWithMembersOf), so there is no parent
8997 // component to attach the iterated elements to.
8998 if (mapOp.getPartialMap() && mapOp.getMembers().size() == 1)
8999 continue;
9000 hasStaticParent = true;
9001 if (!mapOp.getPartialMap() && mapOp.getBounds().empty())
9002 mapsWholeVariable = true;
9003 }
9004 if (!mapsWholeVariable) {
9005 // Span of all iterated elements as [spanBegin, spanEnd) integers,
9006 // meaningful only when anyTrips. A pre-pass loop over each segment's
9007 // iterations (the same ones the mapper pushes later) accumulates the
9008 // min begin / max end in two slots; createIteratorLoop carries no
9009 // loop state, so memory is used rather than PHIs. The allocas end
9010 // up in the entry block via hoistMapperAllocasToEntry.
9011 llvm::Type *i64Ty = builder.getInt64Ty();
9012 anyTrips = builder.getFalse();
9013 spanBegin = builder.getInt64(C: 0);
9014 spanEnd = builder.getInt64(C: 0);
9015 llvm::Value *beginSlot =
9016 builder.CreateAlloca(Ty: i64Ty, ArraySize: nullptr, Name: "omp.iterparent.begin");
9017 llvm::Value *endSlot =
9018 builder.CreateAlloca(Ty: i64Ty, ArraySize: nullptr, Name: "omp.iterparent.end");
9019 for (IteratedEntryInfo &info : iteratedEntries) {
9020 llvm::Value *trips = info.iterInfo.getTotalTrips();
9021 llvm::Value *hasTrips =
9022 builder.CreateICmpNE(LHS: trips, RHS: builder.getInt64(C: 0));
9023 builder.CreateStore(Val: builder.getInt64(C: ~0ULL), Ptr: beginSlot);
9024 builder.CreateStore(Val: builder.getInt64(C: 0), Ptr: endSlot);
9025 auto spanBodyGen = [&](InsertPointTy bodyIP,
9026 llvm::Value *linearIV) -> llvm::Error {
9027 builder.restoreIP(IP: bodyIP);
9028 llvm::Expected<llvm::SmallVector<llvm::Value *, 3>> entryOrErr =
9029 genIteratedEntry(info, builder, linearIV);
9030 if (!entryOrErr)
9031 return entryOrErr.takeError();
9032 llvm::Value *begin =
9033 builder.CreatePtrToInt(V: (*entryOrErr)[1], DestTy: i64Ty);
9034 llvm::Value *end = builder.CreateAdd(LHS: begin, RHS: (*entryOrErr)[2]);
9035 builder.CreateStore(Val: builder.CreateBinaryIntrinsic(
9036 ID: llvm::Intrinsic::umin,
9037 LHS: builder.CreateLoad(Ty: i64Ty, Ptr: beginSlot),
9038 RHS: begin),
9039 Ptr: beginSlot);
9040 builder.CreateStore(Val: builder.CreateBinaryIntrinsic(
9041 ID: llvm::Intrinsic::umax,
9042 LHS: builder.CreateLoad(Ty: i64Ty, Ptr: endSlot), RHS: end),
9043 Ptr: endSlot);
9044 return llvm::Error::success();
9045 };
9046 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
9047 ompBuilder->createIteratorLoop(
9048 Loc: llvm::OpenMPIRBuilder::LocationDescription(builder), TripCount: trips,
9049 BodyGen: spanBodyGen, Name: "mapper.iterspan");
9050 if (!afterIP)
9051 return afterIP.takeError();
9052 builder.restoreIP(IP: *afterIP);
9053 llvm::Value *segBegin = builder.CreateLoad(Ty: i64Ty, Ptr: beginSlot);
9054 llvm::Value *segEnd = builder.CreateLoad(Ty: i64Ty, Ptr: endSlot);
9055 // Fold this segment into the overall span unless it is empty.
9056 llvm::Value *mergedBegin = builder.CreateSelect(
9057 C: anyTrips,
9058 True: builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::umin, LHS: spanBegin,
9059 RHS: segBegin),
9060 False: segBegin);
9061 llvm::Value *mergedEnd =
9062 builder.CreateSelect(C: anyTrips,
9063 True: builder.CreateBinaryIntrinsic(
9064 ID: llvm::Intrinsic::umax, LHS: spanEnd, RHS: segEnd),
9065 False: segEnd);
9066 spanBegin = builder.CreateSelect(C: hasTrips, True: mergedBegin, False: spanBegin);
9067 spanEnd = builder.CreateSelect(C: hasTrips, True: mergedEnd, False: spanEnd);
9068 anyTrips = builder.CreateOr(LHS: anyTrips, RHS: hasTrips);
9069 }
9070 // The pre-pass loops split the block emitUserDefinedMapper was
9071 // emitting into; move its builder to the continuation as well.
9072 ompBuilder->Builder.restoreIP(IP: builder.saveIP());
9073
9074 if (!hasStaticParent) {
9075 // No map clause covers the variable, so synthesize a storage-only
9076 // parent for the span, pushed only when there are elements.
9077 iteratedParentIdx = 0;
9078 combinedInfo.BasePointers.push_back(Elt: ptrPHI);
9079 combinedInfo.Pointers.push_back(
9080 Elt: builder.CreateIntToPtr(V: spanBegin, DestTy: builder.getPtrTy()));
9081 combinedInfo.Sizes.push_back(Elt: builder.CreateSub(LHS: spanEnd, RHS: spanBegin));
9082 combinedInfo.Types.push_back(
9083 Elt: llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE);
9084 combinedInfo.HasAttachPtr.push_back(Elt: false);
9085 combinedInfo.DevicePointers.push_back(
9086 Elt: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
9087 combinedInfo.Mappers.push_back(Elt: nullptr);
9088 combinedInfo.Names.push_back(Elt: LLVM::createMappingInformation(
9089 loc: declMapperOp.getLoc(), builder&: *ompBuilder, mapName: declMapperOp.getSymName()));
9090 combinedInfo.LeadingParentCond = anyTrips;
9091 }
9092 }
9093 }
9094
9095 genMapInfos(builder, moduleTranslation, dl, combinedInfo, mapData,
9096 targetDirective);
9097
9098 llvm::omp::OpenMPOffloadMappingFlags iteratedMemberOf =
9099 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
9100 if (hasIterated) {
9101 if (!iteratedParentIdx) {
9102 // The component genMapInfos emitted for the variable: the whole
9103 // variable, or the partial parent of its statically mapped members.
9104 // Members also have the variable as base, but carry MEMBER_OF.
9105 for (unsigned i = 0; i < combinedInfo.BasePointers.size(); ++i) {
9106 if (combinedInfo.BasePointers[i] == ptrPHI &&
9107 (combinedInfo.Types[i] &
9108 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) ==
9109 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE) {
9110 iteratedParentIdx = i;
9111 break;
9112 }
9113 }
9114 }
9115 assert(iteratedParentIdx && "no parent component for iterated entries");
9116 if (iteratedParentIdx && spanBegin && hasStaticParent) {
9117 // The partial parent spans only the static members. The runtime
9118 // rejects a later entry that extends an earlier one, so rather than
9119 // adding a second parent, widen this one to also cover the iterated
9120 // elements (when there are any).
9121 unsigned idx = *iteratedParentIdx;
9122 llvm::Type *i64Ty = builder.getInt64Ty();
9123 llvm::Value *parentBegin =
9124 builder.CreatePtrToInt(V: combinedInfo.Pointers[idx], DestTy: i64Ty);
9125 llvm::Value *parentEnd =
9126 builder.CreateAdd(LHS: parentBegin, RHS: combinedInfo.Sizes[idx]);
9127 llvm::Value *newBegin = builder.CreateSelect(
9128 C: anyTrips,
9129 True: builder.CreateBinaryIntrinsic(ID: llvm::Intrinsic::umin, LHS: parentBegin,
9130 RHS: spanBegin),
9131 False: parentBegin, Name: "omp.iterparent.begin.merged");
9132 llvm::Value *newEnd =
9133 builder.CreateSelect(C: anyTrips,
9134 True: builder.CreateBinaryIntrinsic(
9135 ID: llvm::Intrinsic::umax, LHS: parentEnd, RHS: spanEnd),
9136 False: parentEnd, Name: "omp.iterparent.end.merged");
9137 combinedInfo.Pointers[idx] =
9138 builder.CreateIntToPtr(V: newBegin, DestTy: builder.getPtrTy());
9139 combinedInfo.Sizes[idx] = builder.CreateSub(LHS: newEnd, RHS: newBegin);
9140 }
9141 if (iteratedParentIdx)
9142 iteratedMemberOf = ompBuilder->getMemberOfFlag(Position: *iteratedParentIdx);
9143 }
9144
9145 for (IteratedEntryInfo &info : iteratedEntries) {
9146 llvm::omp::OpenMPOffloadMappingFlags mapType =
9147 convertClauseMapFlags(mlirFlags: info.mapInfoOp.getMapType());
9148 ompBuilder->setCorrectMemberOfFlag(Flags&: mapType, MemberOfFlag: iteratedMemberOf);
9149 bool hasAttachPtr =
9150 (mapType & llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ATTACH) !=
9151 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
9152
9153 combinedInfo.DynamicSegments.push_back(Elt: MapInfosTy::DynamicSegment{
9154 .Count: info.iterInfo.getTotalTrips(), .GenEntry: {}, .Type: mapType, .HasAttachPtr: hasAttachPtr});
9155 MapInfosTy::DynamicSegment &seg = combinedInfo.DynamicSegments.back();
9156 seg.GenEntry = [genIteratedEntry, info](llvm::IRBuilderBase &b,
9157 llvm::Value *linearIV) {
9158 return genIteratedEntry(info, b, linearIV);
9159 };
9160
9161 if (info.mapInfoOp.getMapperId()) {
9162 auto childMapperOp =
9163 SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>(
9164 from: info.mapInfoOp, symbol: info.mapInfoOp.getMapperIdAttr());
9165 // Emitting the child mapper for the first time leaves builder's
9166 // insertion point inside the child function; restore it so that
9167 // later iterator segments are emitted into this (the parent)
9168 // mapper function instead.
9169 llvm::IRBuilderBase::InsertPointGuard childMapperIPGuard(builder);
9170 llvm::Expected<llvm::Function *> childMapperFn =
9171 getOrCreateUserDefinedMapperFunc(
9172 op: childMapperOp, builder, moduleTranslation, targetDirective);
9173 if (!childMapperFn)
9174 return childMapperFn.takeError();
9175 seg.ChildMapperFn = *childMapperFn;
9176 }
9177 }
9178
9179 return combinedInfo;
9180 };
9181
9182 auto customMapperCB = [&](unsigned i) -> llvm::Expected<llvm::Function *> {
9183 if (!combinedInfo.Mappers[i])
9184 return nullptr;
9185 return getOrCreateUserDefinedMapperFunc(op: combinedInfo.Mappers[i], builder,
9186 moduleTranslation, targetDirective);
9187 };
9188
9189 llvm::Expected<llvm::Function *> newFn = ompBuilder->emitUserDefinedMapper(
9190 PrivAndGenMapInfoCB: genMapInfoCB, ElemTy: varType, FuncName: mapperFuncName, CustomMapperCB: customMapperCB,
9191 /*PreserveMemberOfFlags=*/true, /*PropagatePresentToPointeeStorage=*/true,
9192 /*PropagatePresentToNonPointeeStorage=*/true);
9193 // Only now that emitUserDefinedMapper has fully finished (including running
9194 // any iterator-produced DynamicSegment::GenEntry callbacks, which look up
9195 // values defined in declMapperOp's region) is it safe to drop the mapping,
9196 // so that the same region can be processed multiple times.
9197 moduleTranslation.forgetMapping(region&: declMapperOp.getRegion());
9198 if (!newFn)
9199 return newFn.takeError();
9200 hoistMapperAllocasToEntry(mapperFn&: **newFn);
9201 if ([[maybe_unused]] llvm::Function *mappedFunc =
9202 moduleTranslation.lookupFunction(name: mapperFuncName)) {
9203 assert(mappedFunc == *newFn &&
9204 "mapper function mapping disagrees with emitted function");
9205 } else {
9206 moduleTranslation.mapFunction(name: mapperFuncName, func: *newFn);
9207 }
9208 return *newFn;
9209}
9210
9211static llvm::Value *getSourceLocIdentFromOp(llvm::IRBuilderBase &builder,
9212 llvm::OpenMPIRBuilder &ompBuilder,
9213 Operation *op) {
9214 auto fileLoc = op->getLoc()->findInstanceOf<FileLineColLoc>();
9215 if (!fileLoc)
9216 return nullptr;
9217 uint32_t strSize;
9218 llvm::Function *parentFn = builder.GetInsertBlock()->getParent();
9219 llvm::StringRef fnName = parentFn ? parentFn->getName() : "";
9220 llvm::Constant *srcStr = LLVM::createSourceLocStrFromLocation(
9221 loc: fileLoc, builder&: ompBuilder, name: fnName, strLen&: strSize);
9222 return ompBuilder.getOrCreateIdent(SrcLocStr: srcStr, SrcLocStrSize: strSize);
9223}
9224
9225static LogicalResult
9226convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
9227 LLVM::ModuleTranslation &moduleTranslation) {
9228 llvm::Value *ifCond = nullptr;
9229 llvm::Value *deviceID = builder.getInt64(C: llvm::omp::OMP_DEVICEID_UNDEF);
9230 SmallVector<Value> mapVars;
9231 SmallVector<Value> useDevicePtrVars;
9232 SmallVector<Value> useDeviceAddrVars;
9233 llvm::omp::RuntimeFunction RTLFn;
9234 DataLayout DL = DataLayout(op->getParentOfType<ModuleOp>());
9235 TargetDirectiveEnumTy targetDirective = getTargetDirectiveEnumTyFromOp(op);
9236
9237 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
9238 llvm::OpenMPIRBuilder::TargetDataInfo info(
9239 /*RequiresDevicePointerInfo=*/true,
9240 /*SeparateBeginEndCalls=*/true);
9241
9242 if (ompBuilder->Config.isTargetDevice())
9243 return op->emitOpError() << "not allowed in a target device";
9244
9245 bool isOffloadEntry = !ompBuilder->Config.TargetTriples.empty();
9246
9247 auto getDeviceID = [&](mlir::Value dev) -> llvm::Value * {
9248 llvm::Value *v = moduleTranslation.lookupValue(value: dev);
9249 return builder.CreateIntCast(V: v, DestTy: builder.getInt64Ty(), /*isSigned=*/true);
9250 };
9251
9252 LogicalResult result =
9253 llvm::TypeSwitch<Operation *, LogicalResult>(op)
9254 .Case(caseFn: [&](omp::TargetDataOp dataOp) {
9255 if (failed(Result: checkImplementationStatus(op&: *dataOp)))
9256 return failure();
9257
9258 if (auto ifVar = dataOp.getIfExpr())
9259 ifCond = moduleTranslation.lookupValue(value: ifVar);
9260
9261 if (mlir::Value devId = dataOp.getDevice())
9262 deviceID = getDeviceID(devId);
9263
9264 mapVars = dataOp.getMapVars();
9265 useDevicePtrVars = dataOp.getUseDevicePtrVars();
9266 useDeviceAddrVars = dataOp.getUseDeviceAddrVars();
9267 return success();
9268 })
9269 .Case(caseFn: [&](omp::TargetEnterDataOp enterDataOp) -> LogicalResult {
9270 if (failed(Result: checkImplementationStatus(op&: *enterDataOp)))
9271 return failure();
9272
9273 if (auto ifVar = enterDataOp.getIfExpr())
9274 ifCond = moduleTranslation.lookupValue(value: ifVar);
9275
9276 if (mlir::Value devId = enterDataOp.getDevice())
9277 deviceID = getDeviceID(devId);
9278
9279 RTLFn =
9280 enterDataOp.getNowait()
9281 ? llvm::omp::OMPRTL___tgt_target_data_begin_nowait_mapper
9282 : llvm::omp::OMPRTL___tgt_target_data_begin_mapper;
9283 mapVars = enterDataOp.getMapVars();
9284 info.HasNoWait = enterDataOp.getNowait();
9285 return success();
9286 })
9287 .Case(caseFn: [&](omp::TargetExitDataOp exitDataOp) -> LogicalResult {
9288 if (failed(Result: checkImplementationStatus(op&: *exitDataOp)))
9289 return failure();
9290
9291 if (auto ifVar = exitDataOp.getIfExpr())
9292 ifCond = moduleTranslation.lookupValue(value: ifVar);
9293
9294 if (mlir::Value devId = exitDataOp.getDevice())
9295 deviceID = getDeviceID(devId);
9296
9297 RTLFn = exitDataOp.getNowait()
9298 ? llvm::omp::OMPRTL___tgt_target_data_end_nowait_mapper
9299 : llvm::omp::OMPRTL___tgt_target_data_end_mapper;
9300 mapVars = exitDataOp.getMapVars();
9301 info.HasNoWait = exitDataOp.getNowait();
9302 return success();
9303 })
9304 .Case(caseFn: [&](omp::TargetUpdateOp updateDataOp) -> LogicalResult {
9305 if (failed(Result: checkImplementationStatus(op&: *updateDataOp)))
9306 return failure();
9307
9308 if (auto ifVar = updateDataOp.getIfExpr())
9309 ifCond = moduleTranslation.lookupValue(value: ifVar);
9310
9311 if (mlir::Value devId = updateDataOp.getDevice())
9312 deviceID = getDeviceID(devId);
9313
9314 RTLFn =
9315 updateDataOp.getNowait()
9316 ? llvm::omp::OMPRTL___tgt_target_data_update_nowait_mapper
9317 : llvm::omp::OMPRTL___tgt_target_data_update_mapper;
9318 mapVars = updateDataOp.getMapVars();
9319 info.HasNoWait = updateDataOp.getNowait();
9320 return success();
9321 })
9322 .DefaultUnreachable(message: "unexpected operation");
9323
9324 if (failed(Result: result))
9325 return failure();
9326 // Pretend we have IF(false) if we're not doing offload.
9327 if (!isOffloadEntry)
9328 ifCond = builder.getFalse();
9329
9330 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
9331 MapInfoData mapData;
9332 collectMapDataFromMapOperands(mapData, mapVars, moduleTranslation, dl&: DL,
9333 builder, useDevPtrOperands: useDevicePtrVars, useDevAddrOperands: useDeviceAddrVars);
9334
9335 // Fill up the arrays with all the mapped variables.
9336 MapInfosTy combinedInfo;
9337 auto genMapInfoCB = [&](InsertPointTy codeGenIP) -> MapInfosTy & {
9338 builder.restoreIP(IP: codeGenIP);
9339 genMapInfos(builder, moduleTranslation, dl&: DL, combinedInfo, mapData,
9340 targetDirective);
9341 return combinedInfo;
9342 };
9343
9344 // Define a lambda to apply mappings between use_device_addr and
9345 // use_device_ptr base pointers, and their associated block arguments.
9346 auto mapUseDevice =
9347 [&moduleTranslation](
9348 llvm::OpenMPIRBuilder::DeviceInfoTy type,
9349 llvm::ArrayRef<BlockArgument> blockArgs,
9350 llvm::SmallVectorImpl<Value> &useDeviceVars, MapInfoData &mapInfoData,
9351 llvm::function_ref<llvm::Value *(llvm::Value *)> mapper = nullptr) {
9352 for (auto [arg, useDevVar] :
9353 llvm::zip_equal(t&: blockArgs, u&: useDeviceVars)) {
9354
9355 auto getMapBasePtr = [](omp::MapInfoOp mapInfoOp) {
9356 return mapInfoOp.getVarPtrPtr() ? mapInfoOp.getVarPtrPtr()
9357 : mapInfoOp.getVarPtr();
9358 };
9359
9360 auto useDevMap = cast<omp::MapInfoOp>(Val: useDevVar.getDefiningOp());
9361 for (auto [mapClause, devicePointer, basePointer] : llvm::zip_equal(
9362 t&: mapInfoData.MapClause, u&: mapInfoData.DevicePointers,
9363 args&: mapInfoData.BasePointers)) {
9364 auto mapOp = cast<omp::MapInfoOp>(Val: mapClause);
9365 if (getMapBasePtr(mapOp) != getMapBasePtr(useDevMap) ||
9366 devicePointer != type)
9367 continue;
9368
9369 if (llvm::Value *devPtrInfoMap =
9370 mapper ? mapper(basePointer) : basePointer) {
9371 moduleTranslation.mapValue(mlir: arg, llvm: devPtrInfoMap);
9372 break;
9373 }
9374 }
9375 }
9376 };
9377
9378 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
9379 auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType)
9380 -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
9381 // We must always restoreIP regardless of doing anything the caller
9382 // does not restore it, leading to incorrect (no) branch generation.
9383 builder.restoreIP(IP: codeGenIP);
9384 assert(isa<omp::TargetDataOp>(op) &&
9385 "BodyGen requested for non TargetDataOp");
9386 auto blockArgIface = cast<omp::BlockArgOpenMPOpInterface>(Val: op);
9387 Region &region = cast<omp::TargetDataOp>(Val: op).getRegion();
9388 switch (bodyGenType) {
9389 case BodyGenTy::Priv:
9390 // Check if any device ptr/addr info is available
9391 if (!info.DevicePtrInfoMap.empty()) {
9392 mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Address,
9393 blockArgIface.getUseDeviceAddrBlockArgs(),
9394 useDeviceAddrVars, mapData,
9395 [&](llvm::Value *basePointer) -> llvm::Value * {
9396 if (!info.DevicePtrInfoMap[basePointer].second)
9397 return nullptr;
9398 return builder.CreateLoad(
9399 Ty: builder.getPtrTy(),
9400 Ptr: info.DevicePtrInfoMap[basePointer].second);
9401 });
9402 mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer,
9403 blockArgIface.getUseDevicePtrBlockArgs(), useDevicePtrVars,
9404 mapData, [&](llvm::Value *basePointer) {
9405 return info.DevicePtrInfoMap[basePointer].second;
9406 });
9407
9408 if (failed(Result: inlineConvertOmpRegions(region, blockName: "omp.data.region", builder,
9409 moduleTranslation)))
9410 return llvm::make_error<PreviouslyReportedError>();
9411 }
9412 break;
9413 case BodyGenTy::DupNoPriv:
9414 if (info.DevicePtrInfoMap.empty()) {
9415 // For host device we still need to do the mapping for codegen,
9416 // otherwise it may try to lookup a missing value.
9417 mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Address,
9418 blockArgIface.getUseDeviceAddrBlockArgs(),
9419 useDeviceAddrVars, mapData);
9420 mapUseDevice(llvm::OpenMPIRBuilder::DeviceInfoTy::Pointer,
9421 blockArgIface.getUseDevicePtrBlockArgs(), useDevicePtrVars,
9422 mapData);
9423 }
9424 break;
9425 case BodyGenTy::NoPriv:
9426 // If device info is available then region has already been generated
9427 if (info.DevicePtrInfoMap.empty()) {
9428 if (failed(Result: inlineConvertOmpRegions(region, blockName: "omp.data.region", builder,
9429 moduleTranslation)))
9430 return llvm::make_error<PreviouslyReportedError>();
9431 }
9432 break;
9433 }
9434 return builder.saveIP();
9435 };
9436
9437 auto customMapperCB =
9438 [&](unsigned int i) -> llvm::Expected<llvm::Function *> {
9439 if (!combinedInfo.Mappers[i])
9440 return nullptr;
9441 info.HasMapper = true;
9442 return getOrCreateUserDefinedMapperFunc(op: combinedInfo.Mappers[i], builder,
9443 moduleTranslation, targetDirective);
9444 };
9445
9446 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
9447 llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
9448 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
9449 findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks);
9450
9451 // Pass the region's source location to the runtime, taken from the op's own
9452 // location; only offloading entries emit the mapper calls that consume it.
9453 // No need to also guard on !isTargetDevice here: this function bails out
9454 // earlier for the target device, so isOffloadEntry alone is sufficient.
9455 llvm::Value *srcLocOverride =
9456 isOffloadEntry ? getSourceLocIdentFromOp(builder, ompBuilder&: *ompBuilder, op)
9457 : nullptr;
9458
9459 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP = [&]() {
9460 if (isa<omp::TargetDataOp>(Val: op))
9461 return ompBuilder->createTargetData(
9462 Loc: ompLoc, AllocaIP: allocaIP, CodeGenIP: builder.saveIP(), DeallocBlocks: deallocBlocks, DeviceID: deviceID, IfCond: ifCond,
9463 Info&: info, GenMapInfoCB: genMapInfoCB, CustomMapperCB: customMapperCB,
9464 /*MapperFunc=*/nullptr, BodyGenCB: bodyGenCB,
9465 /*DeviceAddrCB=*/nullptr, SrcLocInfo: srcLocOverride);
9466 return ompBuilder->createTargetData(
9467 Loc: ompLoc, AllocaIP: allocaIP, CodeGenIP: builder.saveIP(), DeallocBlocks: deallocBlocks, DeviceID: deviceID, IfCond: ifCond,
9468 Info&: info, GenMapInfoCB: genMapInfoCB, CustomMapperCB: customMapperCB, MapperFunc: &RTLFn,
9469 /*BodyGenCB=*/nullptr,
9470 /*DeviceAddrCB=*/nullptr, SrcLocInfo: srcLocOverride);
9471 }();
9472
9473 if (failed(Result: handleError(result&: afterIP, op&: *op)))
9474 return failure();
9475
9476 builder.restoreIP(IP: *afterIP);
9477 return success();
9478}
9479
9480static LogicalResult
9481convertOmpDistribute(Operation &opInst, llvm::IRBuilderBase &builder,
9482 LLVM::ModuleTranslation &moduleTranslation) {
9483 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
9484 auto distributeOp = cast<omp::DistributeOp>(Val&: opInst);
9485 if (failed(Result: checkImplementationStatus(op&: opInst)))
9486 return failure();
9487
9488 /// Process teams op reduction in distribute if the reduction is contained in
9489 /// this specific distribute op.
9490 omp::TeamsOp teamsOp = opInst.getParentOfType<omp::TeamsOp>();
9491 std::optional<ReductionVarsInfo> distributeRedInfo;
9492 if (teamsOp &&
9493 getDistributeCapturingTeamsReduction(teamsOp) == distributeOp) {
9494 distributeRedInfo.emplace(args&: teamsOp);
9495 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
9496 findAllocInsertPoints(builder, moduleTranslation);
9497
9498 if (failed(Result: allocAndInitializeReductionVars(op&: *teamsOp, builder,
9499 moduleTranslation, allocaIP,
9500 redInfo&: *distributeRedInfo)))
9501 return failure();
9502 }
9503
9504 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
9505 auto bodyGenCB =
9506 [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
9507 llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
9508 llvm::BasicBlock *allocaBlock = allocaIP.getNodeParent();
9509 // DistributeOp has only one region associated with it.
9510 builder.restoreIP(IP: codeGenIP);
9511 PrivateVarsInfo privVarsInfo(distributeOp);
9512
9513 llvm::Expected<llvm::BasicBlock *> afterAllocas = allocatePrivateVars(
9514 op: distributeOp, builder, moduleTranslation, privateVarsInfo&: privVarsInfo, allocaIP);
9515 if (handleError(result&: afterAllocas, op&: opInst).failed())
9516 return llvm::make_error<PreviouslyReportedError>();
9517
9518 // Save the alloca insertion point on ModuleTranslation stack for use in
9519 // nested regions. This must be after allocatePrivateVars, which splits
9520 // the alloca block and updates allocaIP.
9521 LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
9522 moduleTranslation, allocaIP.getNodeParent(), deallocBlocks);
9523
9524 if (handleError(error: initPrivateVars(builder, moduleTranslation, privateVarsInfo&: privVarsInfo),
9525 op&: opInst)
9526 .failed())
9527 return llvm::make_error<PreviouslyReportedError>();
9528
9529 if (failed(Result: completePrivateVars(op: distributeOp, builder, moduleTranslation,
9530 privateVarsInfo: privVarsInfo,
9531 insertBarrier: distributeOp.getPrivateNeedsBarrier())))
9532 return llvm::make_error<PreviouslyReportedError>();
9533
9534 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
9535 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
9536 llvm::Expected<llvm::BasicBlock *> regionBlock =
9537 convertOmpOpRegions(region&: distributeOp.getRegion(), blockName: "omp.distribute.region",
9538 builder, moduleTranslation);
9539 if (!regionBlock)
9540 return regionBlock.takeError();
9541 builder.SetInsertPoint((*regionBlock)->begin());
9542
9543 // Skip applying a workshare loop below when translating 'distribute
9544 // parallel do' (it's been already handled by this point while translating
9545 // the nested omp.wsloop).
9546 if (!isa_and_present<omp::WsloopOp>(Val: distributeOp.getNestedWrapper())) {
9547 // TODO: Add support for clauses which are valid for DISTRIBUTE
9548 // constructs. Static schedule is the default.
9549 bool hasDistSchedule = distributeOp.getDistScheduleStatic();
9550 auto schedule = hasDistSchedule ? omp::ClauseScheduleKind::Distribute
9551 : omp::ClauseScheduleKind::Static;
9552 // dist_schedule clauses are ordered - otherise this should be false
9553 bool isOrdered = hasDistSchedule;
9554 std::optional<omp::ScheduleModifier> scheduleMod;
9555 bool isSimd = false;
9556 llvm::omp::WorksharingLoopType workshareLoopType =
9557 llvm::omp::WorksharingLoopType::DistributeStaticLoop;
9558 bool loopNeedsBarrier = false;
9559 llvm::Value *chunk = moduleTranslation.lookupValue(
9560 value: distributeOp.getDistScheduleChunkSize());
9561 llvm::CanonicalLoopInfo *loopInfo =
9562 findCurrentLoopInfo(moduleTranslation);
9563 // Nested regions can move the instruction referenced by allocaIP.
9564 allocaIP = allocaBlock->getFirstInsertionPt();
9565 llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
9566 ompBuilder->applyWorkshareLoop(
9567 DL: ompLoc.DL, CLI: loopInfo, AllocaIP: allocaIP, NeedsBarrier: loopNeedsBarrier,
9568 SchedKind: convertToScheduleKind(schedKind: schedule), ChunkSize: chunk, HasSimdModifier: isSimd,
9569 HasMonotonicModifier: scheduleMod == omp::ScheduleModifier::monotonic,
9570 HasNonmonotonicModifier: scheduleMod == omp::ScheduleModifier::nonmonotonic, HasOrderedClause: isOrdered,
9571 LoopType: workshareLoopType, NoLoop: false, HasDistSchedule: hasDistSchedule, DistScheduleChunkSize: chunk);
9572
9573 if (!wsloopIP)
9574 return wsloopIP.takeError();
9575 }
9576 if (failed(Result: cleanupPrivateVars(op: distributeOp, builder, moduleTranslation,
9577 loc: distributeOp.getLoc(), privateVarsInfo&: privVarsInfo)))
9578 return llvm::make_error<PreviouslyReportedError>();
9579
9580 return llvm::Error::success();
9581 };
9582
9583 llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
9584 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
9585 findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks);
9586 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
9587 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
9588 ompBuilder->createDistribute(Loc: ompLoc, AllocaIP: allocaIP, DeallocBlocks: deallocBlocks, BodyGenCB: bodyGenCB);
9589
9590 if (failed(Result: handleError(result&: afterIP, op&: opInst)))
9591 return failure();
9592
9593 builder.restoreIP(IP: *afterIP);
9594
9595 if (distributeRedInfo) {
9596 // Process the reductions if required.
9597 return createReductionsAndCleanup(op&: *teamsOp, builder, moduleTranslation,
9598 allocaIP, redInfo&: *distributeRedInfo,
9599 /*isNoWait=*/isNowait: false,
9600 /*isTeamsReduction=*/true);
9601 }
9602 return success();
9603}
9604
9605/// Lowers the FlagsAttr which is applied to the module when offloading. This
9606/// attribute contains OpenMP RTL globals that can be passed as flags to the
9607/// frontend, otherwise they are set to default
9608static LogicalResult
9609convertFlagsAttr(Operation *op, mlir::omp::FlagsAttr attribute,
9610 LLVM::ModuleTranslation &moduleTranslation) {
9611 auto offloadMod = dyn_cast<omp::OffloadModuleInterface>(Val: op);
9612 if (!offloadMod)
9613 return op->emitOpError() << "omp flags attached to non offload module op";
9614
9615 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
9616
9617 if (offloadMod.getIsTargetDevice())
9618 ompBuilder->M.addModuleFlag(Behavior: llvm::Module::Max, Key: "openmp-device",
9619 Val: attribute.getOpenmpDeviceVersion());
9620
9621 // The flags below are only intended to be emitted for GPU offload targets.
9622 if (!offloadMod.getIsGPU())
9623 return success();
9624
9625 if (attribute.getNoGpuLib())
9626 return success();
9627
9628 ompBuilder->createGlobalFlag(Value: attribute.getDebugKind(),
9629 Name: "__omp_rtl_debug_kind");
9630 ompBuilder->createGlobalFlag(Value: attribute.getAssumeTeamsOversubscription(),
9631 Name: "__omp_rtl_assume_teams_oversubscription");
9632 ompBuilder->createGlobalFlag(Value: attribute.getAssumeThreadsOversubscription(),
9633 Name: "__omp_rtl_assume_threads_oversubscription");
9634 ompBuilder->createGlobalFlag(Value: attribute.getAssumeNoThreadState(),
9635 Name: "__omp_rtl_assume_no_thread_state");
9636 ompBuilder->createGlobalFlag(Value: attribute.getAssumeNoNestedParallelism(),
9637 Name: "__omp_rtl_assume_no_nested_parallelism");
9638 return success();
9639}
9640
9641static void getTargetEntryUniqueInfo(llvm::TargetRegionEntryInfo &targetInfo,
9642 omp::TargetOp targetOp,
9643 llvm::OpenMPIRBuilder &ompBuilder,
9644 llvm::vfs::FileSystem &vfs,
9645 llvm::StringRef parentName = "") {
9646 auto fileLoc = targetOp.getLoc()->findInstanceOf<FileLineColLoc>();
9647 assert(fileLoc && "No file found from location");
9648
9649 auto fileInfoCallBack = [&fileLoc]() {
9650 return std::pair<std::string, uint64_t>(
9651 llvm::StringRef(fileLoc.getFilename()), fileLoc.getLine());
9652 };
9653
9654 targetInfo =
9655 ompBuilder.getTargetEntryUniqueInfo(CallBack: fileInfoCallBack, VFS&: vfs, ParentName: parentName);
9656}
9657
9658// The createDeviceArgumentAccessor function generates
9659// instructions for retrieving (acessing) kernel
9660// arguments inside of the device kernel for use by
9661// the kernel. This enables different semantics such as
9662// the creation of temporary copies of data allowing
9663// semantics like read-only/no host write back kernel
9664// arguments.
9665//
9666// This currently implements a very light version of Clang's
9667// EmitParmDecl's handling of direct argument handling as well
9668// as a portion of the argument access generation based on
9669// capture types found at the end of emitOutlinedFunctionPrologue
9670// in Clang. The indirect path handling of EmitParmDecl's may be
9671// required for future work, but a direct 1-to-1 copy doesn't seem
9672// possible as the logic is rather scattered throughout Clang's
9673// lowering and perhaps we wish to deviate slightly.
9674//
9675// \param mapData - A container containing vectors of information
9676// corresponding to the input argument, which should have a
9677// corresponding entry in the MapInfoData containers
9678// OrigialValue's.
9679// \param arg - This is the generated kernel function argument that
9680// corresponds to the passed in input argument. We generated different
9681// accesses of this Argument, based on capture type and other Input
9682// related information.
9683// \param input - This is the host side value that will be passed to
9684// the kernel i.e. the kernel input, we rewrite all uses of this within
9685// the kernel (as we generate the kernel body based on the target's region
9686// which maintians references to the original input) to the retVal argument
9687// apon exit of this function inside of the OMPIRBuilder. This interlinks
9688// the kernel argument to future uses of it in the function providing
9689// appropriate "glue" instructions inbetween.
9690// \param retVal - This is the value that all uses of input inside of the
9691// kernel will be re-written to, the goal of this function is to generate
9692// an appropriate location for the kernel argument to be accessed from,
9693// e.g. ByRef will result in a temporary allocation location and then
9694// a store of the kernel argument into this allocated memory which
9695// will then be loaded from, ByCopy will use the allocated memory
9696// directly.
9697static llvm::IRBuilderBase::InsertPoint createDeviceArgumentAccessor(
9698 omp::TargetOp targetOp, MapInfoData &mapData, llvm::Argument &arg,
9699 llvm::Value *input, llvm::Value *&retVal, llvm::IRBuilderBase &builder,
9700 llvm::OpenMPIRBuilder &ompBuilder,
9701 LLVM::ModuleTranslation &moduleTranslation,
9702 llvm::IRBuilderBase::InsertPoint allocaIP,
9703 llvm::IRBuilderBase::InsertPoint codeGenIP,
9704 llvm::ArrayRef<llvm::IRBuilderBase::InsertPoint> deallocIPs) {
9705 assert(ompBuilder.Config.isTargetDevice() &&
9706 "function only supported for target device codegen");
9707 builder.restoreIP(IP: allocaIP);
9708
9709 omp::VariableCaptureKind capture = omp::VariableCaptureKind::ByRef;
9710 LLVM::TypeToLLVMIRTranslator typeToLLVMIRTranslator(
9711 ompBuilder.M.getContext());
9712 unsigned alignmentValue = 0;
9713 BlockArgument mlirArg;
9714 SmallVector<std::pair<Value, BlockArgument>> blockArgsPairs;
9715 cast<omp::BlockArgOpenMPOpInterface>(Val&: *targetOp).getBlockArgsPairs(
9716 pairs&: blockArgsPairs);
9717 // Find the associated MapInfoData entry for the current input
9718 for (size_t i = 0; i < mapData.MapClause.size(); ++i) {
9719 if (mapData.OriginalValue[i] == input) {
9720 auto mapOp = cast<omp::MapInfoOp>(Val: mapData.MapClause[i]);
9721 capture = mapOp.getMapCaptureType();
9722 // Get information of alignment of mapped object
9723 alignmentValue = typeToLLVMIRTranslator.getPreferredAlignment(
9724 type: mapOp.getVarPtrType(), layout: ompBuilder.M.getDataLayout());
9725
9726 // Find the corresponding entry block argument, which can be associated to
9727 // a map, use_device* or has_device* clause.
9728 for (auto &[val, arg] : blockArgsPairs) {
9729 if (mapOp.getResult() == val) {
9730 mlirArg = arg;
9731 break;
9732 }
9733 }
9734 assert(mlirArg && "expected to find entry block argument for map clause");
9735 break;
9736 }
9737 }
9738
9739 unsigned int allocaAS = ompBuilder.M.getDataLayout().getAllocaAddrSpace();
9740 unsigned int defaultAS =
9741 ompBuilder.M.getDataLayout().getProgramAddressSpace();
9742
9743 // Create the allocation for the argument.
9744 llvm::Value *v = nullptr;
9745 if (omp::opInSharedDeviceContext(op&: *targetOp) &&
9746 omp::allocaUsesRequireSharedMem(alloc: mlirArg)) {
9747 // Use the beginning of the codeGenIP rather than the usual allocation point
9748 // for shared memory allocations because otherwise these would be done prior
9749 // to the target initialization call. Also, the exit block (where the
9750 // deallocation is placed) is only executed if the initialization call
9751 // succeeds.
9752 builder.SetInsertPoint(codeGenIP.getNodeParent()->getFirstInsertionPt());
9753 v = ompBuilder.createOMPAllocShared(Loc: builder, VarType: arg.getType());
9754
9755 // Create deallocations in all provided deallocation points and then restore
9756 // the insertion point to right after the new allocations.
9757 llvm::IRBuilderBase::InsertPointGuard guard(builder);
9758 for (auto deallocIP : deallocIPs) {
9759 builder.SetInsertPoint(deallocIP);
9760 ompBuilder.createOMPFreeShared(Loc: builder, Addr: v, VarType: arg.getType());
9761 }
9762 } else {
9763 // Use the current point, which was previously set to allocaIP.
9764 v = builder.CreateAlloca(Ty: arg.getType(), AddrSpace: allocaAS);
9765
9766 if (allocaAS != defaultAS && arg.getType()->isPointerTy())
9767 v = builder.CreateAddrSpaceCast(V: v, DestTy: builder.getPtrTy(AddrSpace: defaultAS));
9768 }
9769
9770 builder.CreateStore(Val: &arg, Ptr: v);
9771
9772 builder.restoreIP(IP: codeGenIP);
9773
9774 switch (capture) {
9775 case omp::VariableCaptureKind::ByCopy: {
9776 retVal = v;
9777 break;
9778 }
9779 case omp::VariableCaptureKind::ByRef: {
9780 llvm::LoadInst *loadInst = builder.CreateAlignedLoad(
9781 Ty: v->getType(), Ptr: v,
9782 Align: ompBuilder.M.getDataLayout().getPrefTypeAlign(Ty: v->getType()));
9783 // CreateAlignedLoad function creates similar LLVM IR:
9784 // %res = load ptr, ptr %input, align 8
9785 // This LLVM IR does not contain information about alignment
9786 // of the loaded value. We need to add !align metadata to unblock
9787 // optimizer. The existence of the !align metadata on the instruction
9788 // tells the optimizer that the value loaded is known to be aligned to
9789 // a boundary specified by the integer value in the metadata node.
9790 // Example:
9791 // %res = load ptr, ptr %input, align 8, !align !align_md_node
9792 // ^ ^
9793 // | |
9794 // alignment of %input address |
9795 // |
9796 // alignment of %res object
9797 if (v->getType()->isPointerTy() && alignmentValue) {
9798 llvm::MDBuilder MDB(builder.getContext());
9799 loadInst->setMetadata(
9800 KindID: llvm::LLVMContext::MD_align,
9801 Node: llvm::MDNode::get(Context&: builder.getContext(),
9802 MDs: MDB.createConstant(C: llvm::ConstantInt::get(
9803 Ty: llvm::Type::getInt64Ty(C&: builder.getContext()),
9804 V: alignmentValue))));
9805 }
9806 retVal = loadInst;
9807
9808 break;
9809 }
9810 case omp::VariableCaptureKind::This:
9811 case omp::VariableCaptureKind::VLAType:
9812 // TODO: Consider returning error to use standard reporting for
9813 // unimplemented features.
9814 assert(false && "Currently unsupported capture kind");
9815 break;
9816 }
9817
9818 return builder.saveIP();
9819}
9820
9821/// Follow uses of `host_eval`-defined block arguments of the given `omp.target`
9822/// operation and populate output variables with their corresponding host value
9823/// (i.e. operand evaluated outside of the target region), based on their uses
9824/// inside of the target region.
9825///
9826/// Loop bounds and steps are only optionally populated, if output vectors are
9827/// provided.
9828static void
9829extractHostEvalClauses(omp::TargetOp targetOp, Value &numThreads,
9830 Value &numTeamsLower, Value &numTeamsUpper,
9831 Value &threadLimit,
9832 llvm::SmallVectorImpl<Value> *lowerBounds = nullptr,
9833 llvm::SmallVectorImpl<Value> *upperBounds = nullptr,
9834 llvm::SmallVectorImpl<Value> *steps = nullptr) {
9835 auto blockArgIface = llvm::cast<omp::BlockArgOpenMPOpInterface>(Val&: *targetOp);
9836 for (auto item : llvm::zip_equal(t: targetOp.getHostEvalVars(),
9837 u: blockArgIface.getHostEvalBlockArgs())) {
9838 Value hostEvalVar = std::get<0>(t&: item), blockArg = std::get<1>(t&: item);
9839
9840 for (Operation *user : blockArg.getUsers()) {
9841 llvm::TypeSwitch<Operation *>(user)
9842 .Case(caseFn: [&](omp::TeamsOp teamsOp) {
9843 if (teamsOp.getNumTeamsLower() == blockArg)
9844 numTeamsLower = hostEvalVar;
9845 else if (llvm::is_contained(Range: teamsOp.getNumTeamsUpperVars(),
9846 Element: blockArg))
9847 numTeamsUpper = hostEvalVar;
9848 else if (!teamsOp.getThreadLimitVars().empty() &&
9849 teamsOp.getThreadLimit(dim: 0) == blockArg)
9850 threadLimit = hostEvalVar;
9851 else
9852 llvm_unreachable("unsupported host_eval use");
9853 })
9854 .Case(caseFn: [&](omp::ParallelOp parallelOp) {
9855 if (!parallelOp.getNumThreadsVars().empty() &&
9856 parallelOp.getNumThreads(dim: 0) == blockArg)
9857 numThreads = hostEvalVar;
9858 else
9859 llvm_unreachable("unsupported host_eval use");
9860 })
9861 .Case(caseFn: [&](omp::LoopNestOp loopOp) {
9862 auto processBounds =
9863 [&](OperandRange opBounds,
9864 llvm::SmallVectorImpl<Value> *outBounds) -> bool {
9865 bool found = false;
9866 for (auto [i, lb] : llvm::enumerate(First&: opBounds)) {
9867 if (lb == blockArg) {
9868 found = true;
9869 if (outBounds)
9870 (*outBounds)[i] = hostEvalVar;
9871 }
9872 }
9873 return found;
9874 };
9875 bool found =
9876 processBounds(loopOp.getLoopLowerBounds(), lowerBounds);
9877 found = processBounds(loopOp.getLoopUpperBounds(), upperBounds) ||
9878 found;
9879 found = processBounds(loopOp.getLoopSteps(), steps) || found;
9880 (void)found;
9881 assert(found && "unsupported host_eval use");
9882 })
9883 .DefaultUnreachable(message: "unsupported host_eval use");
9884 }
9885 }
9886}
9887
9888/// If \p op is of the given type parameter, return it casted to that type.
9889/// Otherwise, if its immediate parent operation (or some other higher-level
9890/// parent, if \p immediateParent is false) is of that type, return that parent
9891/// casted to the given type.
9892///
9893/// If \p op is \c null or neither it or its parent(s) are of the specified
9894/// type, return a \c null operation.
9895template <typename OpTy>
9896static OpTy castOrGetParentOfType(Operation *op, bool immediateParent = false) {
9897 if (!op)
9898 return OpTy();
9899
9900 if (OpTy casted = dyn_cast<OpTy>(op))
9901 return casted;
9902
9903 if (immediateParent)
9904 return dyn_cast_if_present<OpTy>(op->getParentOp());
9905
9906 return op->getParentOfType<OpTy>();
9907}
9908
9909/// If the given \p value is defined by an \c llvm.mlir.constant operation and
9910/// it is of an integer type, return its value.
9911static std::optional<int64_t> extractConstInteger(Value value) {
9912 if (!value)
9913 return std::nullopt;
9914
9915 if (auto constOp = value.getDefiningOp<LLVM::ConstantOp>())
9916 if (auto constAttr = dyn_cast<IntegerAttr>(Val: constOp.getValue()))
9917 return constAttr.getInt();
9918
9919 return std::nullopt;
9920}
9921
9922static uint64_t getTypeByteSize(mlir::Type type, const DataLayout &dl) {
9923 uint64_t sizeInBits = dl.getTypeSizeInBits(t: type);
9924 uint64_t sizeInBytes = sizeInBits / 8;
9925 return sizeInBytes;
9926}
9927
9928template <typename OpTy>
9929static uint64_t getReductionDataSize(OpTy &op) {
9930 if (op.getNumReductionVars() > 0) {
9931 SmallVector<omp::DeclareReductionOp> reductions;
9932 collectReductionDecls(op, reductions);
9933
9934 llvm::SmallVector<mlir::Type> members;
9935 members.reserve(N: reductions.size());
9936 for (omp::DeclareReductionOp &red : reductions) {
9937 // For by-ref reductions, use the actual element type rather than the
9938 // pointer type so that the buffer size matches the access pattern in
9939 // the copy/reduce callbacks generated by OMPIRBuilder.
9940 if (red.getByrefElementType())
9941 members.push_back(Elt: *red.getByrefElementType());
9942 else
9943 members.push_back(Elt: red.getType());
9944 }
9945 Operation *opp = op.getOperation();
9946 auto structType = mlir::LLVM::LLVMStructType::getLiteral(
9947 context: opp->getContext(), types: members, /*isPacked=*/false);
9948 DataLayout dl = DataLayout(opp->getParentOfType<ModuleOp>());
9949 return getTypeByteSize(type: structType, dl);
9950 }
9951 return 0;
9952}
9953
9954/// Populate default `MinTeams`, `MaxTeams` and `MaxThreads` to their default
9955/// values as stated by the corresponding clauses, if constant.
9956///
9957/// These default values must be set before the creation of the outlined LLVM
9958/// function for the target region, so that they can be used to initialize the
9959/// corresponding global `ConfigurationEnvironmentTy` structure.
9960static void
9961initTargetDefaultAttrs(omp::TargetOp targetOp, Operation *capturedOp,
9962 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &attrs,
9963 bool isTargetDevice, bool isGPU) {
9964 // TODO: Handle constant 'if' clauses.
9965
9966 Value numThreads, numTeamsLower, numTeamsUpper, threadLimit;
9967 if (!isTargetDevice) {
9968 extractHostEvalClauses(targetOp, numThreads, numTeamsLower, numTeamsUpper,
9969 threadLimit);
9970 } else {
9971 // In the target device, values for these clauses are not passed as
9972 // host_eval, but instead evaluated prior to entry to the region. This
9973 // ensures values are mapped and available inside of the target region.
9974 if (auto teamsOp = castOrGetParentOfType<omp::TeamsOp>(op: capturedOp)) {
9975 numTeamsLower = teamsOp.getNumTeamsLower();
9976 // Handle num_teams upper bounds (only first value for now)
9977 if (!teamsOp.getNumTeamsUpperVars().empty())
9978 numTeamsUpper = teamsOp.getNumTeams(dim: 0);
9979 if (!teamsOp.getThreadLimitVars().empty())
9980 threadLimit = teamsOp.getThreadLimit(dim: 0);
9981 }
9982
9983 if (auto parallelOp = castOrGetParentOfType<omp::ParallelOp>(op: capturedOp)) {
9984 if (!parallelOp.getNumThreadsVars().empty())
9985 numThreads = parallelOp.getNumThreads(dim: 0);
9986 }
9987 }
9988
9989 // Handle clauses impacting the number of teams.
9990
9991 int32_t minTeamsVal = 1, maxTeamsVal = -1;
9992 if (castOrGetParentOfType<omp::TeamsOp>(op: capturedOp)) {
9993 // TODO: Use `hostNumTeamsLower` to initialize `minTeamsVal`. For now,
9994 // match clang and set min and max to the same value.
9995 if (numTeamsUpper) {
9996 if (auto val = extractConstInteger(value: numTeamsUpper))
9997 minTeamsVal = maxTeamsVal = *val;
9998 } else {
9999 minTeamsVal = maxTeamsVal = 0;
10000 }
10001 } else if (castOrGetParentOfType<omp::ParallelOp>(op: capturedOp,
10002 /*immediateParent=*/true) ||
10003 castOrGetParentOfType<omp::SimdOp>(op: capturedOp,
10004 /*immediateParent=*/true)) {
10005 minTeamsVal = maxTeamsVal = 1;
10006 } else {
10007 minTeamsVal = maxTeamsVal = -1;
10008 }
10009
10010 // Handle clauses impacting the number of threads.
10011
10012 auto setMaxValueFromClause = [](Value clauseValue, int32_t &result) {
10013 if (!clauseValue)
10014 return;
10015
10016 if (auto val = extractConstInteger(value: clauseValue))
10017 result = *val;
10018
10019 // Found an applicable clause, so it's not undefined. Mark as unknown
10020 // because it's not constant.
10021 if (result < 0)
10022 result = 0;
10023 };
10024
10025 // Extract 'thread_limit' clause from 'target' and 'teams' directives.
10026 int32_t targetThreadLimitVal = -1, teamsThreadLimitVal = -1;
10027 if (!targetOp.getThreadLimitVars().empty())
10028 setMaxValueFromClause(targetOp.getThreadLimit(dim: 0), targetThreadLimitVal);
10029 setMaxValueFromClause(threadLimit, teamsThreadLimitVal);
10030
10031 // Extract 'max_threads' clause from 'parallel' or set to 1 if it's SIMD.
10032 int32_t maxThreadsVal = -1;
10033 if (castOrGetParentOfType<omp::ParallelOp>(op: capturedOp))
10034 setMaxValueFromClause(numThreads, maxThreadsVal);
10035 else if (castOrGetParentOfType<omp::SimdOp>(op: capturedOp,
10036 /*immediateParent=*/true))
10037 maxThreadsVal = 1;
10038
10039 // For max values, < 0 means unset, == 0 means set but unknown. Select the
10040 // minimum value between 'max_threads' and 'thread_limit' clauses that were
10041 // set.
10042 int32_t combinedMaxThreadsVal = targetThreadLimitVal;
10043 if (combinedMaxThreadsVal < 0 ||
10044 (teamsThreadLimitVal >= 0 && teamsThreadLimitVal < combinedMaxThreadsVal))
10045 combinedMaxThreadsVal = teamsThreadLimitVal;
10046
10047 if (combinedMaxThreadsVal < 0 ||
10048 (maxThreadsVal >= 0 && maxThreadsVal < combinedMaxThreadsVal))
10049 combinedMaxThreadsVal = maxThreadsVal;
10050
10051 int32_t reductionDataSize = 0;
10052 if (isGPU && capturedOp) {
10053 if (auto teamsOp = castOrGetParentOfType<omp::TeamsOp>(op: capturedOp))
10054 reductionDataSize = getReductionDataSize(op&: teamsOp);
10055 }
10056
10057 // Update kernel bounds structure for the `OpenMPIRBuilder` to use.
10058 // Use the kernel_type attribute set by the frontend instead of analyzing IR.
10059 omp::TargetExecMode execMode = targetOp.getKernelType();
10060 switch (execMode) {
10061 case omp::TargetExecMode::bare:
10062 attrs.ExecFlags = llvm::omp::OMP_TGT_EXEC_MODE_BARE;
10063 break;
10064 case omp::TargetExecMode::generic:
10065 attrs.ExecFlags = llvm::omp::OMP_TGT_EXEC_MODE_GENERIC;
10066 break;
10067 case omp::TargetExecMode::spmd:
10068 attrs.ExecFlags = llvm::omp::OMP_TGT_EXEC_MODE_SPMD;
10069 break;
10070 case omp::TargetExecMode::spmd_no_loop:
10071 attrs.ExecFlags = llvm::omp::OMP_TGT_EXEC_MODE_SPMD_NO_LOOP;
10072 break;
10073 }
10074 attrs.MinTeams.front() = minTeamsVal;
10075 attrs.MaxTeams.front() = maxTeamsVal;
10076 attrs.MinThreads.front() = 1;
10077 attrs.MaxThreads.front() = combinedMaxThreadsVal;
10078 attrs.ReductionDataSize = reductionDataSize;
10079}
10080
10081/// Gather LLVM runtime values for all clauses evaluated in the host that are
10082/// passed to the kernel invocation.
10083///
10084/// This function must be called only when compiling for the host. Also, it will
10085/// only provide correct results if it's called after the body of \c targetOp
10086/// has been fully generated.
10087static void
10088initTargetRuntimeAttrs(llvm::IRBuilderBase &builder,
10089 LLVM::ModuleTranslation &moduleTranslation,
10090 omp::TargetOp targetOp, Operation *capturedOp,
10091 llvm::OpenMPIRBuilder::TargetKernelRuntimeAttrs &attrs) {
10092 omp::LoopNestOp loopOp = castOrGetParentOfType<omp::LoopNestOp>(op: capturedOp);
10093 unsigned numLoops = loopOp ? loopOp.getNumLoops() : 0;
10094
10095 Value numThreads, numTeamsLower, numTeamsUpper, teamsThreadLimit;
10096 llvm::SmallVector<Value> lowerBounds(numLoops), upperBounds(numLoops),
10097 steps(numLoops);
10098 extractHostEvalClauses(targetOp, numThreads, numTeamsLower, numTeamsUpper,
10099 threadLimit&: teamsThreadLimit, lowerBounds: &lowerBounds, upperBounds: &upperBounds, steps: &steps);
10100
10101 // TODO: Handle constant 'if' clauses.
10102 if (!targetOp.getThreadLimitVars().empty()) {
10103 Value targetThreadLimit = targetOp.getThreadLimit(dim: 0);
10104 attrs.TargetThreadLimit.front() =
10105 moduleTranslation.lookupValue(value: targetThreadLimit);
10106 }
10107
10108 // The __kmpc_push_num_teams_51 function expects int32 as the arguments. So,
10109 // truncate or sign extend lower and upper num_teams bounds as well as
10110 // thread_limit to match int32 ABI requirements for the OpenMP runtime.
10111 if (numTeamsLower)
10112 attrs.MinTeams.front() = builder.CreateSExtOrTrunc(
10113 V: moduleTranslation.lookupValue(value: numTeamsLower), DestTy: builder.getInt32Ty());
10114
10115 if (numTeamsUpper)
10116 attrs.MaxTeams.front() = builder.CreateSExtOrTrunc(
10117 V: moduleTranslation.lookupValue(value: numTeamsUpper), DestTy: builder.getInt32Ty());
10118
10119 if (teamsThreadLimit)
10120 attrs.TeamsThreadLimit.front() = builder.CreateSExtOrTrunc(
10121 V: moduleTranslation.lookupValue(value: teamsThreadLimit), DestTy: builder.getInt32Ty());
10122
10123 if (numThreads)
10124 attrs.MaxThreads.front() = moduleTranslation.lookupValue(value: numThreads);
10125
10126 if (targetOp.hasHostEvalTripCount()) {
10127 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
10128 attrs.LoopTripCount = nullptr;
10129
10130 // To calculate the trip count, we multiply together the trip counts of
10131 // every collapsed canonical loop. We don't need to create the loop nests
10132 // here, since we're only interested in the trip count.
10133 for (auto [loopLower, loopUpper, loopStep] :
10134 llvm::zip_equal(t&: lowerBounds, u&: upperBounds, args&: steps)) {
10135 llvm::Value *lowerBound = moduleTranslation.lookupValue(value: loopLower);
10136 llvm::Value *upperBound = moduleTranslation.lookupValue(value: loopUpper);
10137 llvm::Value *step = moduleTranslation.lookupValue(value: loopStep);
10138
10139 if (!lowerBound || !upperBound || !step) {
10140 attrs.LoopTripCount = nullptr;
10141 break;
10142 }
10143
10144 llvm::OpenMPIRBuilder::LocationDescription loc(builder);
10145 llvm::Value *tripCount = ompBuilder->calculateCanonicalLoopTripCount(
10146 Loc: loc, Start: lowerBound, Stop: upperBound, Step: step, /*IsSigned=*/true,
10147 InclusiveStop: loopOp.getLoopInclusive());
10148
10149 if (!attrs.LoopTripCount) {
10150 attrs.LoopTripCount = tripCount;
10151 continue;
10152 }
10153
10154 // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
10155 attrs.LoopTripCount = builder.CreateMul(LHS: attrs.LoopTripCount, RHS: tripCount,
10156 Name: {}, /*HasNUW=*/true);
10157 }
10158 }
10159
10160 attrs.DeviceID = builder.getInt64(C: llvm::omp::OMP_DEVICEID_UNDEF);
10161 if (mlir::Value devId = targetOp.getDevice()) {
10162 attrs.DeviceID = moduleTranslation.lookupValue(value: devId);
10163 attrs.DeviceID =
10164 builder.CreateSExtOrTrunc(V: attrs.DeviceID, DestTy: builder.getInt64Ty());
10165 }
10166}
10167
10168static llvm::omp::OMPDynGroupprivateFallbackType
10169getDynGroupprivateFallbackType(omp::FallbackModifierAttr fallbackAttr) {
10170 omp::FallbackModifier fb = fallbackAttr ? fallbackAttr.getValue()
10171 : omp::FallbackModifier::default_mem;
10172 switch (fb) {
10173 case omp::FallbackModifier::abort:
10174 return llvm::omp::OMPDynGroupprivateFallbackType::Abort;
10175 case omp::FallbackModifier::null:
10176 return llvm::omp::OMPDynGroupprivateFallbackType::Null;
10177 case omp::FallbackModifier::default_mem:
10178 return llvm::omp::OMPDynGroupprivateFallbackType::DefaultMem;
10179 }
10180
10181 llvm_unreachable("unexpected dyn_groupprivate fallback type");
10182}
10183
10184static LogicalResult
10185convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder,
10186 LLVM::ModuleTranslation &moduleTranslation) {
10187 auto targetOp = cast<omp::TargetOp>(Val&: opInst);
10188
10189 // The current debug location already has the DISubprogram for the outlined
10190 // function that will be created for the target op. We save it here so that
10191 // we can set it on the outlined function.
10192 llvm::DebugLoc outlinedFnLoc = builder.getCurrentDebugLocation();
10193 if (failed(Result: checkImplementationStatus(op&: opInst)))
10194 return failure();
10195
10196 // During the handling of target op, we will generate instructions in the
10197 // parent function like call to the oulined function or branch to a new
10198 // BasicBlock. We set the debug location here to parent function so that those
10199 // get the correct debug locations. For outlined functions, the normal MLIR op
10200 // conversion will automatically pick the correct location.
10201 llvm::BasicBlock *parentBB = builder.GetInsertBlock();
10202 assert(parentBB && "No insert block is set for the builder");
10203 llvm::Function *parentLLVMFn = parentBB->getParent();
10204 assert(parentLLVMFn && "Parent Function must be valid");
10205 if (llvm::DISubprogram *SP = parentLLVMFn->getSubprogram())
10206 builder.SetCurrentDebugLocation(llvm::DILocation::get(
10207 Context&: parentLLVMFn->getContext(), Line: outlinedFnLoc.getLine(),
10208 Column: outlinedFnLoc.getCol(), Scope: SP, InlinedAt: outlinedFnLoc.getInlinedAt()));
10209
10210 // OMPIRBuilder emits runtime calls into the outlined function before bodyCB
10211 // below gets a chance to attach the subprogram to it, so it needs the
10212 // outlined function's location handed to it separately. Only pass it under
10213 // the same condition that decides whether the subprogram is attached at all:
10214 // a location may not be attached to an instruction in a function that has no
10215 // subprogram.
10216 llvm::DebugLoc outlinedFnDbgLoc;
10217 if (outlinedFnLoc && parentLLVMFn->getSubprogram())
10218 outlinedFnDbgLoc = outlinedFnLoc;
10219
10220 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
10221 bool isTargetDevice = ompBuilder->Config.isTargetDevice();
10222 bool isGPU = ompBuilder->Config.isGPU();
10223
10224 auto parentFn = opInst.getParentOfType<LLVM::LLVMFuncOp>();
10225 auto argIface = cast<omp::BlockArgOpenMPOpInterface>(Val&: opInst);
10226 auto &targetRegion = targetOp.getRegion();
10227 // Holds the private vars that have been mapped along with the block
10228 // argument that corresponds to the MapInfoOp corresponding to the private
10229 // var in question. So, for instance:
10230 //
10231 // %10 = omp.map.info var_ptr(%6#0 : !fir.ref<!fir.box<!fir.heap<i32>>>, ..)
10232 // omp.target map_entries(%10 -> %arg0) private(@box.privatizer %6#0-> %arg1)
10233 //
10234 // Then, %10 has been created so that the descriptor can be used by the
10235 // privatizer @box.privatizer on the device side. Here we'd record {%6#0,
10236 // %arg0} in the mappedPrivateVars map.
10237 llvm::DenseMap<Value, Value> mappedPrivateVars;
10238 DataLayout dl = DataLayout(opInst.getParentOfType<ModuleOp>());
10239 SmallVector<Value> mapVars = targetOp.getMapVars();
10240 SmallVector<Value> hdaVars = targetOp.getHasDeviceAddrVars();
10241 ArrayRef<BlockArgument> mapBlockArgs = argIface.getMapBlockArgs();
10242 ArrayRef<BlockArgument> hdaBlockArgs = argIface.getHasDeviceAddrBlockArgs();
10243 llvm::Function *llvmOutlinedFn = nullptr;
10244 TargetDirectiveEnumTy targetDirective =
10245 getTargetDirectiveEnumTyFromOp(op: &opInst);
10246
10247 // TODO: It can also be false if a compile-time constant `false` IF clause is
10248 // specified.
10249 bool isOffloadEntry =
10250 isTargetDevice || !ompBuilder->Config.TargetTriples.empty();
10251
10252 // Resolve in_reduction clauses on omp.target for the host. From the target
10253 // device's perspective an in_reduction list item behaves as a regular
10254 // map(tofrom) variable, so no special handling is needed there; only the
10255 // host redirects the mapped value to the per-task reduction-private storage
10256 // returned by __kmpc_task_reduction_get_th_data (emitted inside the
10257 // to-be-outlined target task body). This applies to both offloading and
10258 // non-offloading host modules.
10259 //
10260 // The target body has no dedicated in_reduction block argument: each
10261 // in_reduction variable is accessed through its map_entries block argument.
10262 // So each in_reduction variable must also be captured by a matching
10263 // map_entries entry (guaranteed by the verifier); without one the outlined
10264 // body would reference a value defined in the host function. Record, for each
10265 // in_reduction variable, the position of that map entry so the corresponding
10266 // map block argument can be redirected inside the body. The in_reduction
10267 // operand itself is used as the `orig` argument of the runtime lookup.
10268 SmallVector<llvm::Value *> inRedOrigPtrs;
10269 SmallVector<unsigned> inRedMapArgIdx;
10270 if (!targetOp.getInReductionVars().empty() && !isTargetDevice) {
10271 inRedOrigPtrs.reserve(N: targetOp.getInReductionVars().size());
10272 inRedMapArgIdx.reserve(N: targetOp.getInReductionVars().size());
10273 for (Value v : targetOp.getInReductionVars()) {
10274 // Select the map_entries entry that captures this in_reduction operand.
10275 // The verifier guarantees at least one match exists; more than one
10276 // matching entry is a lowering ambiguity (the redirect cannot pick which
10277 // map argument to rebind).
10278 std::optional<unsigned> matchIdx;
10279 for (auto [idx, mapV] : llvm::enumerate(First: targetOp.getMapVars())) {
10280 auto mapInfo = mapV.getDefiningOp<omp::MapInfoOp>();
10281 if (v != mapInfo.getVarPtr())
10282 continue;
10283 if (matchIdx)
10284 return targetOp.emitError()
10285 << "in_reduction variable on omp.target has multiple matching "
10286 "map_entries entries; the redirect target is ambiguous";
10287 matchIdx = idx;
10288 }
10289 // The verifier requires a capturing map entry for every in_reduction
10290 // operand, so a match must exist here.
10291 assert(matchIdx &&
10292 "TargetOp verifier guarantees a matching map_entries entry for "
10293 "each in_reduction variable");
10294 inRedMapArgIdx.push_back(Elt: *matchIdx);
10295 // The runtime `orig` pointer is the in_reduction operand itself, the
10296 // reduction variable the enclosing taskgroup registered.
10297 inRedOrigPtrs.push_back(Elt: moduleTranslation.lookupValue(value: v));
10298 }
10299 }
10300
10301 // For some private variables, the MapsForPrivatizedVariablesPass
10302 // creates MapInfoOp instances. Go through the private variables and
10303 // the mapped variables so that during codegeneration we are able
10304 // to quickly look up the corresponding map variable, if any for each
10305 // private variable.
10306 if (!targetOp.getPrivateVars().empty() && !targetOp.getMapVars().empty()) {
10307 OperandRange privateVars = targetOp.getPrivateVars();
10308 std::optional<ArrayAttr> privateSyms = targetOp.getPrivateSyms();
10309 std::optional<DenseI64ArrayAttr> privateMapIndices =
10310 targetOp.getPrivateMapsAttr();
10311
10312 for (auto [privVarIdx, privVarSymPair] :
10313 llvm::enumerate(First: llvm::zip_equal(t&: privateVars, u&: *privateSyms))) {
10314 auto privVar = std::get<0>(t&: privVarSymPair);
10315 auto privSym = std::get<1>(t&: privVarSymPair);
10316
10317 SymbolRefAttr privatizerName = llvm::cast<SymbolRefAttr>(Val&: privSym);
10318 omp::PrivateClauseOp privatizer =
10319 findPrivatizer(from: targetOp, symbolName: privatizerName);
10320
10321 if (!privatizer.needsMap())
10322 continue;
10323
10324 mlir::Value mappedValue =
10325 targetOp.getMappedValueForPrivateVar(privVarIdx);
10326 assert(mappedValue && "Expected to find mapped value for a privatized "
10327 "variable that needs mapping");
10328
10329 // The MapInfoOp defining the map var isn't really needed later.
10330 // So, we don't store it in any datastructure. Instead, we just
10331 // do some sanity checks on it right now.
10332 auto mapInfoOp = mappedValue.getDefiningOp<omp::MapInfoOp>();
10333 [[maybe_unused]] Type varType = mapInfoOp.getVarPtrType();
10334
10335 // Check #1: Check that the type of the private variable matches
10336 // the type of the variable being mapped.
10337 if (!isa<LLVM::LLVMPointerType>(Val: privVar.getType()))
10338 assert(
10339 varType == privVar.getType() &&
10340 "Type of private var doesn't match the type of the mapped value");
10341
10342 // Ok, only 1 sanity check for now.
10343 // Record the block argument corresponding to this mapvar.
10344 mappedPrivateVars.insert(
10345 KV: {privVar,
10346 targetRegion.getArgument(i: argIface.getMapBlockArgsStart() +
10347 (*privateMapIndices)[privVarIdx])});
10348 }
10349 }
10350
10351 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10352 auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
10353 ArrayRef<llvm::BasicBlock *> deallocBlocks)
10354 -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
10355 llvm::IRBuilderBase::InsertPointGuard guard(builder);
10356 builder.SetCurrentDebugLocation(llvm::DebugLoc());
10357 // Forward target-cpu and target-features function attributes from the
10358 // original function to the new outlined function.
10359 llvm::Function *llvmParentFn =
10360 moduleTranslation.lookupFunction(name: parentFn.getName());
10361 llvmOutlinedFn = codeGenIP.getNodeParent()->getParent();
10362 assert(llvmParentFn && llvmOutlinedFn &&
10363 "Both parent and outlined functions must exist at this point");
10364
10365 if (outlinedFnLoc && llvmParentFn->getSubprogram())
10366 llvmOutlinedFn->setSubprogram(outlinedFnLoc->getScope()->getSubprogram());
10367
10368 if (auto attr = llvmParentFn->getFnAttribute(Kind: "target-cpu");
10369 attr.isStringAttribute())
10370 llvmOutlinedFn->addFnAttr(Attr: attr);
10371
10372 if (auto attr = llvmParentFn->getFnAttribute(Kind: "target-features");
10373 attr.isStringAttribute())
10374 llvmOutlinedFn->addFnAttr(Attr: attr);
10375
10376 for (auto [idx, arg] : llvm::enumerate(First&: mapBlockArgs)) {
10377 // in_reduction list items on omp.target are accessed through their
10378 // map_entries block argument, which is redirected below to the per-task
10379 // reduction-private storage returned by the runtime. Skip the default
10380 // host-value mapping for those block arguments so the write-once
10381 // mapValue mapping is free to be set to the private pointer.
10382 if (llvm::is_contained(Range&: inRedMapArgIdx, Element: idx))
10383 continue;
10384 auto mapInfoOp = cast<omp::MapInfoOp>(Val: mapVars[idx].getDefiningOp());
10385 llvm::Value *mapOpValue =
10386 moduleTranslation.lookupValue(value: mapInfoOp.getVarPtr());
10387 moduleTranslation.mapValue(mlir: arg, llvm: mapOpValue);
10388 }
10389 for (auto [arg, mapOp] : llvm::zip_equal(t&: hdaBlockArgs, u&: hdaVars)) {
10390 auto mapInfoOp = cast<omp::MapInfoOp>(Val: mapOp.getDefiningOp());
10391 llvm::Value *mapOpValue =
10392 moduleTranslation.lookupValue(value: mapInfoOp.getVarPtr());
10393 moduleTranslation.mapValue(mlir: arg, llvm: mapOpValue);
10394 }
10395
10396 // Do privatization after moduleTranslation has already recorded
10397 // mapped values.
10398 PrivateVarsInfo privateVarsInfo(targetOp);
10399
10400 llvm::Expected<llvm::BasicBlock *> afterAllocas =
10401 allocatePrivateVars(op: targetOp, builder, moduleTranslation,
10402 privateVarsInfo, allocaIP, mappedPrivateVars: &mappedPrivateVars);
10403
10404 if (failed(Result: handleError(result&: afterAllocas, op&: *targetOp)))
10405 return llvm::make_error<PreviouslyReportedError>();
10406
10407 builder.restoreIP(IP: codeGenIP);
10408 if (handleError(error: initPrivateVars(builder, moduleTranslation, privateVarsInfo,
10409 mappedPrivateVars: &mappedPrivateVars),
10410 op&: *targetOp)
10411 .failed())
10412 return llvm::make_error<PreviouslyReportedError>();
10413
10414 if (failed(Result: completePrivateVars(
10415 op: targetOp, builder, moduleTranslation, privateVarsInfo,
10416 insertBarrier: targetOp.getPrivateNeedsBarrier(), /*moldVars=*/{},
10417 /*llvmPrivateVars=*/{}, mappedPrivateVars: &mappedPrivateVars)))
10418 return llvm::make_error<PreviouslyReportedError>();
10419
10420 // The target body accesses each in_reduction variable through its
10421 // map_entries block argument. Redirect that block argument to the per-task
10422 // private storage returned by __kmpc_task_reduction_get_th_data so the body
10423 // accumulates into the reduction-private copy rather than the mapped
10424 // original. The lookup must run inside the target task body so the gtid
10425 // corresponds to the executing thread. The descriptor argument is NULL: the
10426 // runtime walks enclosing taskgroups to locate the matching task_reduction
10427 // registration for `origPtr`. Mirrors the in_reduction handling on
10428 // omp.taskloop.context.
10429 if (!inRedOrigPtrs.empty()) {
10430 // Collect, per item, the type the private pointer must have (the map
10431 // block argument's type), and, through the callback, rebind the map block
10432 // argument that stands in for each in_reduction list item to the per-task
10433 // reduction-private storage the runtime returns.
10434 SmallVector<llvm::Type *> inRedResultPtrTys;
10435 inRedResultPtrTys.reserve(N: inRedMapArgIdx.size());
10436 for (unsigned mapArgIdx : inRedMapArgIdx)
10437 inRedResultPtrTys.push_back(
10438 Elt: moduleTranslation.convertType(type: mapBlockArgs[mapArgIdx].getType()));
10439
10440 llvm::OpenMPIRBuilder::LocationDescription bodyLoc(builder);
10441 llvm::OpenMPIRBuilder::InsertPointTy redIP =
10442 ompBuilder->createTargetInReduction(
10443 Loc: bodyLoc, OrigPtrs: inRedOrigPtrs, ResultPtrTys: inRedResultPtrTys,
10444 MapPrivateCB: [&](unsigned idx, llvm::Value *priv) {
10445 moduleTranslation.mapValue(mlir: mapBlockArgs[inRedMapArgIdx[idx]],
10446 llvm: priv);
10447 });
10448 builder.restoreIP(IP: redIP);
10449 }
10450
10451 LLVM::ModuleTranslation::SaveStack<OpenMPAllocStackFrame> frame(
10452 moduleTranslation, allocaIP.getNodeParent(), deallocBlocks);
10453 llvm::Expected<llvm::BasicBlock *> exitBlock = convertOmpOpRegions(
10454 region&: targetRegion, blockName: "omp.target", builder, moduleTranslation);
10455
10456 if (failed(Result: handleError(result&: exitBlock, op&: *targetOp)))
10457 return llvm::make_error<PreviouslyReportedError>();
10458
10459 builder.SetInsertPoint(exitBlock.get()->getTerminator());
10460
10461 if (failed(Result: cleanupPrivateVars(op: targetOp, builder, moduleTranslation,
10462 loc: targetOp.getLoc(), privateVarsInfo)))
10463 return llvm::make_error<PreviouslyReportedError>();
10464
10465 return builder.saveIP();
10466 };
10467
10468 StringRef parentName = parentFn.getName();
10469
10470 llvm::TargetRegionEntryInfo entryInfo;
10471
10472 getTargetEntryUniqueInfo(targetInfo&: entryInfo, targetOp,
10473 ompBuilder&: *getOpenMPBuilder(moduleTranslation),
10474 vfs&: moduleTranslation.getFileSystem(), parentName);
10475
10476 MapInfoData mapData;
10477 collectMapDataFromMapOperands(mapData, mapVars, moduleTranslation, dl,
10478 builder, /*useDevPtrOperands=*/{},
10479 /*useDevAddrOperands=*/{}, hasDevAddrOperands: hdaVars);
10480
10481 MapInfosTy combinedInfos;
10482 auto genMapInfoCB =
10483 [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP) -> MapInfosTy & {
10484 builder.restoreIP(IP: codeGenIP);
10485 genMapInfos(builder, moduleTranslation, dl, combinedInfo&: combinedInfos, mapData,
10486 targetDirective);
10487
10488 // Append a null entry for the implicit dyn_ptr argument so the argument
10489 // count sent to the runtime already includes it.
10490 auto *nullPtr = llvm::Constant::getNullValue(Ty: builder.getPtrTy());
10491 combinedInfos.BasePointers.push_back(Elt: nullPtr);
10492 combinedInfos.Pointers.push_back(Elt: nullPtr);
10493 combinedInfos.DevicePointers.push_back(
10494 Elt: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
10495 combinedInfos.Sizes.push_back(Elt: builder.getInt64(C: 0));
10496 combinedInfos.Types.push_back(
10497 Elt: llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
10498 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10499 // TODO: set HasAttachPtr from Flang for pointee-storage entries.
10500 combinedInfos.HasAttachPtr.push_back(Elt: false);
10501 if (!combinedInfos.Names.empty())
10502 combinedInfos.Names.push_back(Elt: nullPtr);
10503 combinedInfos.Mappers.push_back(Elt: nullptr);
10504
10505 return combinedInfos;
10506 };
10507
10508 auto argAccessorCB = [&](llvm::Argument &arg, llvm::Value *input,
10509 llvm::Value *&retVal, InsertPointTy allocaIP,
10510 InsertPointTy codeGenIP,
10511 llvm::ArrayRef<InsertPointTy> deallocIPs)
10512 -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
10513 llvm::IRBuilderBase::InsertPointGuard guard(builder);
10514 builder.SetCurrentDebugLocation(llvm::DebugLoc());
10515 // We just return the unaltered argument for the host function
10516 // for now, some alterations may be required in the future to
10517 // keep host fallback functions working identically to the device
10518 // version (e.g. pass ByCopy values should be treated as such on
10519 // host and device, currently not always the case)
10520 if (!isTargetDevice) {
10521 retVal = cast<llvm::Value>(Val: &arg);
10522 return codeGenIP;
10523 }
10524
10525 return createDeviceArgumentAccessor(targetOp, mapData, arg, input, retVal,
10526 builder, ompBuilder&: *ompBuilder, moduleTranslation,
10527 allocaIP, codeGenIP, deallocIPs);
10528 };
10529
10530 llvm::OpenMPIRBuilder::TargetKernelRuntimeAttrs runtimeAttrs;
10531 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs defaultAttrs;
10532 Operation *targetCapturedOp =
10533 cast<omp::ComposableOpInterface>(Val&: *targetOp).findCapturedOp();
10534 initTargetDefaultAttrs(targetOp, capturedOp: targetCapturedOp, attrs&: defaultAttrs,
10535 isTargetDevice, isGPU);
10536
10537 // Collect host-evaluated values needed to properly launch the kernel from the
10538 // host.
10539 if (!isTargetDevice)
10540 initTargetRuntimeAttrs(builder, moduleTranslation, targetOp,
10541 capturedOp: targetCapturedOp, attrs&: runtimeAttrs);
10542
10543 // Pass host-evaluated values as parameters to the kernel / host fallback,
10544 // except if they are constants. In any case, map the MLIR block argument to
10545 // the corresponding LLVM values.
10546 llvm::SmallVector<llvm::Value *, 4> kernelInput;
10547 SmallVector<Value> hostEvalVars = targetOp.getHostEvalVars();
10548 ArrayRef<BlockArgument> hostEvalBlockArgs = argIface.getHostEvalBlockArgs();
10549 for (auto [arg, var] : llvm::zip_equal(t&: hostEvalBlockArgs, u&: hostEvalVars)) {
10550 llvm::Value *value = moduleTranslation.lookupValue(value: var);
10551 moduleTranslation.mapValue(mlir: arg, llvm: value);
10552
10553 if (!llvm::isa<llvm::Constant>(Val: value))
10554 kernelInput.push_back(Elt: value);
10555 }
10556
10557 for (size_t i = 0, e = mapData.OriginalValue.size(); i != e; ++i) {
10558 // 1) Declare target arguments are not passed to kernels as arguments.
10559 // 2) Attach maps are not passed in as arguments to kernels, except for
10560 // private attach maps used for corresponding-pointer initialization.
10561 // 3) Children of record objects are not passed in as arguments.
10562 // TODO: We currently do not handle cases where a member is explicitly
10563 // passed in as an argument, this will likley need to be handled in
10564 // the near future, rather than using IsAMember, it may be better to
10565 // test if the relevant BlockArg is used within the target region and
10566 // then use that as a basis for exclusion in the kernel inputs.
10567 using MapFlags = llvm::omp::OpenMPOffloadMappingFlags;
10568 bool isAttachMap = (mapData.Types[i] & MapFlags::OMP_MAP_ATTACH) ==
10569 MapFlags::OMP_MAP_ATTACH;
10570 bool isPrivateTargetParam =
10571 (mapData.Types[i] &
10572 (MapFlags::OMP_MAP_PRIVATE | MapFlags::OMP_MAP_TARGET_PARAM)) ==
10573 (MapFlags::OMP_MAP_PRIVATE | MapFlags::OMP_MAP_TARGET_PARAM);
10574
10575 if (!mapData.IsDeclareTarget[i] && !mapData.IsAMember[i] &&
10576 (!isAttachMap || (isAttachMap && isPrivateTargetParam)))
10577 kernelInput.push_back(Elt: mapData.OriginalValue[i]);
10578 }
10579
10580 llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
10581 llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
10582 findAllocInsertPoints(builder, moduleTranslation, deallocBlocks: &deallocBlocks);
10583
10584 llvm::OpenMPIRBuilder::DependenciesInfo dds;
10585 if (failed(Result: buildDependData(
10586 dependVars: targetOp.getDependVars(), dependKinds: targetOp.getDependKinds(),
10587 dependIterated: targetOp.getDependIterated(), dependIteratedKinds: targetOp.getDependIteratedKinds(),
10588 builder, moduleTranslation, taskDeps&: dds)))
10589 return failure();
10590
10591 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
10592
10593 llvm::OpenMPIRBuilder::TargetDataInfo info(
10594 /*RequiresDevicePointerInfo=*/false,
10595 /*SeparateBeginEndCalls=*/true);
10596
10597 auto customMapperCB =
10598 [&](unsigned int i) -> llvm::Expected<llvm::Function *> {
10599 if (!combinedInfos.Mappers[i])
10600 return nullptr;
10601 info.HasMapper = true;
10602 return getOrCreateUserDefinedMapperFunc(op: combinedInfos.Mappers[i], builder,
10603 moduleTranslation, targetDirective);
10604 };
10605
10606 llvm::Value *ifCond = nullptr;
10607 if (Value targetIfCond = targetOp.getIfExpr())
10608 ifCond = moduleTranslation.lookupValue(value: targetIfCond);
10609
10610 Value dynGroupPrivateSize = targetOp.getDynGroupprivateSize();
10611 llvm::Value *dynSizeVal = nullptr;
10612 if (dynGroupPrivateSize) {
10613 dynSizeVal = moduleTranslation.lookupValue(value: dynGroupPrivateSize);
10614 dynSizeVal = builder.CreateIntCast(V: dynSizeVal, DestTy: builder.getInt32Ty(),
10615 /*isSigned=*/false);
10616 }
10617
10618 llvm::omp::OMPDynGroupprivateFallbackType fallbackType =
10619 getDynGroupprivateFallbackType(fallbackAttr: targetOp.getDynGroupprivateFallbackAttr());
10620
10621 // Pass the target region's source location to the runtime, taken from the
10622 // op's own location. Restricted to the host offload path that actually emits
10623 // the kernel launch, to avoid creating an unused identifier on the device.
10624 llvm::Value *rtLocOverride =
10625 (!isTargetDevice && isOffloadEntry)
10626 ? getSourceLocIdentFromOp(builder, ompBuilder&: *ompBuilder, op: targetOp)
10627 : nullptr;
10628
10629 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
10630 getOpenMPBuilder(moduleTranslation)
10631 ->createTarget(Loc: ompLoc, IsOffloadEntry: isOffloadEntry, AllocaIP: allocaIP, CodeGenIP: builder.saveIP(),
10632 DeallocBlocks: deallocBlocks, Info&: info, EntryInfo&: entryInfo, DefaultAttrs: defaultAttrs,
10633 RuntimeAttrs: runtimeAttrs, IfCond: ifCond, Inputs&: kernelInput, GenMapInfoCB: genMapInfoCB,
10634 BodyGenCB: bodyCB, ArgAccessorFuncCB: argAccessorCB, CustomMapperCB: customMapperCB, Dependencies: dds,
10635 HasNowait: targetOp.getNowait(), DynCGroupMem: dynSizeVal, DynCGroupMemFallback: fallbackType,
10636 OutlinedFnLoc: outlinedFnDbgLoc, RTLocOverride: rtLocOverride);
10637
10638 if (failed(Result: handleError(result&: afterIP, op&: opInst)))
10639 return failure();
10640
10641 builder.restoreIP(IP: *afterIP);
10642
10643 if (dds.DepArray)
10644 builder.CreateFree(Source: dds.DepArray);
10645
10646 return success();
10647}
10648
10649static LogicalResult
10650convertDeclareTargetAttr(Operation *op, mlir::omp::DeclareTargetAttr attribute,
10651 llvm::OpenMPIRBuilder *ompBuilder,
10652 LLVM::ModuleTranslation &moduleTranslation) {
10653 // Amend omp.declare_target by deleting the IR of the outlined functions
10654 // created for target regions. They cannot be filtered out from MLIR earlier
10655 // because the omp.target operation inside must be translated to LLVM, but
10656 // the wrapper functions themselves must not remain at the end of the
10657 // process. We know that functions where omp.declare_target does not match
10658 // omp.is_target_device at this stage can only be wrapper functions because
10659 // those that aren't are removed earlier as an MLIR transformation pass.
10660 if (FunctionOpInterface funcOp = dyn_cast<FunctionOpInterface>(Val: op)) {
10661 if (auto offloadMod = dyn_cast<omp::OffloadModuleInterface>(
10662 Val: op->getParentOfType<ModuleOp>().getOperation())) {
10663 if (!offloadMod.getIsTargetDevice())
10664 return success();
10665
10666 omp::DeclareTargetDeviceType declareType = attribute.getDeviceType();
10667
10668 if (declareType == omp::DeclareTargetDeviceType::host) {
10669 llvm::Function *llvmFunc =
10670 moduleTranslation.lookupFunction(name: funcOp.getName());
10671 llvmFunc->dropAllReferences();
10672 llvmFunc->eraseFromParent();
10673
10674 // Invalidate the builder's current insertion point, as it now points to
10675 // a deleted block.
10676 ompBuilder->Builder.ClearInsertionPoint();
10677 ompBuilder->Builder.SetCurrentDebugLocation(llvm::DebugLoc());
10678 } else if (llvm::Function *llvmFunc =
10679 moduleTranslation.lookupFunction(name: funcOp.getName())) {
10680 // Device-side declare target functions are externally visible by
10681 // default so they can be referenced from other device translation
10682 // units. That also prevents the offload LTO from internalizing and
10683 // deleting them when they end up unused in the final device image.
10684 // Such dead functions can still reference internal LDS and trigger
10685 // spurious "local memory global used by non-kernel function" backend
10686 // warnings. Marking them hidden keeps the symbol usable within the
10687 // device image's linkage unit while letting LTO drop it when nothing
10688 // references it; symbols that must stay reachable (e.g. via an offload
10689 // entry that takes their address) are kept alive by that reference.
10690 if (!llvmFunc->isDeclaration() && llvmFunc->hasExternalLinkage() &&
10691 llvmFunc->getVisibility() == llvm::GlobalValue::DefaultVisibility)
10692 llvmFunc->setVisibility(llvm::GlobalValue::HiddenVisibility);
10693 }
10694 }
10695 return success();
10696 }
10697
10698 if (LLVM::GlobalOp gOp = dyn_cast<LLVM::GlobalOp>(Val: op)) {
10699 llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
10700 if (auto *gVal = llvmModule->getNamedValue(Name: gOp.getSymName())) {
10701 auto *gVar = cast<llvm::GlobalVariable>(Val: gVal);
10702 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
10703 bool isDeclaration = gOp.isDeclaration();
10704 bool isExternallyVisible =
10705 gOp.getVisibility() != mlir::SymbolTable::Visibility::Private;
10706 auto loc = op->getLoc()->findInstanceOf<FileLineColLoc>();
10707 llvm::StringRef mangledName = gOp.getSymName();
10708 mlir::omp::DeclareTargetCaptureClause captureClause =
10709 attribute.getCaptureClause();
10710 auto captureClauseKind = convertToCaptureClauseKind(captureClause);
10711 auto deviceClause = convertToDeviceClauseKind(deviceClause: attribute.getDeviceType());
10712 llvm::StringRef entryMangledName = mangledName;
10713 llvm::Constant *entryAddr = llvm::cast<llvm::Constant>(Val: gVal);
10714 std::function<llvm::GlobalValue::LinkageTypes()> variableLinkage;
10715 llvm::SmallString<128> entryNameStorage;
10716 bool requiresUSM = ompBuilder->Config.hasRequiresUnifiedSharedMemory();
10717 bool isToOrEnter =
10718 captureClause == omp::DeclareTargetCaptureClause::to ||
10719 captureClause == omp::DeclareTargetCaptureClause::enter;
10720 bool isHostOnly =
10721 attribute.getDeviceType() == omp::DeclareTargetDeviceType::host;
10722
10723 // A to/enter declare-target variable needs a device-resident,
10724 // name-resolvable copy and a host offloading entry. A local-linkage
10725 // global provides neither, so we promote it to external.
10726 if (isToOrEnter && !isHostOnly && !requiresUSM &&
10727 gVar->hasLocalLinkage()) {
10728 gVar->setLinkage(llvm::GlobalValue::ExternalLinkage);
10729 isExternallyVisible = true;
10730
10731 // Clear the stale dso_local flag so it is referenced like a
10732 // module-scope declare target global.
10733 if (ompBuilder->Config.isTargetDevice())
10734 gVar->setDSOLocal(false);
10735 }
10736
10737 if (isToOrEnter &&
10738 deviceClause ==
10739 llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny &&
10740 !requiresUSM && !isDeclaration &&
10741 (gVal->hasLocalLinkage() || gVal->hasHiddenVisibility())) {
10742 // Keep the original symbol as-is for target code, but create a visible
10743 // alias for the offload entry so libomptarget can associate the host
10744 // global with the actual device global.
10745 entryNameStorage = (mangledName + llvm::Twine("_decl_tgt_entry")).str();
10746 entryMangledName = entryNameStorage;
10747 if (llvm::GlobalValue *existing =
10748 llvmModule->getNamedValue(Name: entryMangledName)) {
10749 entryAddr = llvm::cast<llvm::Constant>(Val: existing);
10750 } else {
10751 entryAddr = llvm::GlobalAlias::create(
10752 Ty: gVal->getValueType(), AddressSpace: gVal->getAddressSpace(),
10753 Linkage: llvm::GlobalValue::WeakAnyLinkage, Name: entryMangledName, Aliasee: entryAddr,
10754 Parent: llvmModule);
10755 llvm::cast<llvm::GlobalAlias>(Val: entryAddr)->setVisibility(
10756 llvm::GlobalValue::DefaultVisibility);
10757 }
10758 variableLinkage = [] { return llvm::GlobalValue::WeakAnyLinkage; };
10759 }
10760 // unused for MLIR at the moment, required in Clang for book
10761 // keeping
10762 std::vector<llvm::GlobalVariable *> generatedRefs;
10763
10764 std::vector<llvm::Triple> targetTriple;
10765 auto targetTripleAttr = dyn_cast_or_null<mlir::StringAttr>(
10766 Val: op->getParentOfType<mlir::ModuleOp>()->getDiscardableAttr(
10767 name: LLVM::LLVMDialect::getTargetTripleAttrName()));
10768 if (targetTripleAttr)
10769 targetTriple.emplace_back(args: targetTripleAttr.data());
10770
10771 auto fileInfoCallBack = [&loc]() {
10772 std::string filename = "";
10773 std::uint64_t lineNo = 0;
10774
10775 if (loc) {
10776 filename = loc.getFilename().str();
10777 lineNo = loc.getLine();
10778 }
10779
10780 return std::pair<std::string, std::uint64_t>(llvm::StringRef(filename),
10781 lineNo);
10782 };
10783
10784 llvm::vfs::FileSystem &vfs = moduleTranslation.getFileSystem();
10785 ompBuilder->registerTargetGlobalVariable(
10786 CaptureClause: captureClauseKind, DeviceClause: deviceClause, IsDeclaration: isDeclaration, IsExternallyVisible: isExternallyVisible,
10787 EntryInfo: ompBuilder->getTargetEntryUniqueInfo(CallBack: fileInfoCallBack, VFS&: vfs),
10788 MangledName: entryMangledName, GeneratedRefs&: generatedRefs, /*OpenMPSimd*/ OpenMPSIMD: false, TargetTriple: targetTriple,
10789 /*GlobalInitializer*/ nullptr, VariableLinkage: variableLinkage, LlvmPtrTy: gVal->getType(),
10790 Addr: entryAddr);
10791
10792 if (ompBuilder->Config.isTargetDevice() &&
10793 (captureClause == omp::DeclareTargetCaptureClause::link ||
10794 requiresUSM)) {
10795 // For USM and link we generate a global reference pointer in the
10796 // default address space (e.g address space 0), as opposed to the
10797 // globals original type and address space.
10798 llvm::Type *ptrTy = llvm::PointerType::get(C&: llvmModule->getContext(), AddressSpace: 0);
10799 llvm::Constant *refPtr = ompBuilder->getAddrOfDeclareTargetVar(
10800 CaptureClause: captureClauseKind, DeviceClause: deviceClause, IsDeclaration: isDeclaration, IsExternallyVisible: isExternallyVisible,
10801 EntryInfo: ompBuilder->getTargetEntryUniqueInfo(CallBack: fileInfoCallBack, VFS&: vfs),
10802 MangledName: mangledName, GeneratedRefs&: generatedRefs, /*OpenMPSimd*/ OpenMPSIMD: false, TargetTriple: targetTriple,
10803 LlvmPtrTy: ptrTy, /*GlobalInitializer*/ nullptr,
10804 /*VariableLinkage*/ nullptr);
10805
10806 // For indirectly-accessed global pointers, we rely on "internal"
10807 // linkage to optimize out the unneeded full-variable storage later,
10808 // since we can't prevent the LLVM dialect from generating globals
10809 // without also breaking target lowering. However, We can only do
10810 // this for definiions, as global variable declarations must have
10811 // external or weak linkage.
10812 if (refPtr) {
10813 if (!gVar->isDeclaration())
10814 gVar->setLinkage(llvm::GlobalValue::InternalLinkage);
10815
10816 // Register the (original global, reference pointer) pair so that the
10817 // OpenMPIRBuilder can rewrite uses of the original global during
10818 // finalization.
10819 if (auto *newGV =
10820 dyn_cast<llvm::GlobalValue>(Val: refPtr->stripPointerCasts()))
10821 ompBuilder->registerDeclareTargetGlobalReplacement(Original: gVal, Replacement: newGV);
10822 }
10823 }
10824
10825 // Mark 'device_type(host) enter(...)' variables as external in the device
10826 // since they're not supposed to have their own copy. This will cause
10827 // linker errors if accesses are attempted from the target device.
10828 if (ompBuilder->Config.isTargetDevice() && isHostOnly && isToOrEnter) {
10829 gVar->setLinkage(llvm::GlobalValue::ExternalLinkage);
10830 gVar->setInitializer(nullptr);
10831 }
10832 }
10833 }
10834
10835 return success();
10836}
10837
10838namespace {
10839
10840/// Implementation of the dialect interface that converts operations belonging
10841/// to the OpenMP dialect to LLVM IR.
10842class OpenMPDialectLLVMIRTranslationInterface
10843 : public LLVMTranslationDialectInterface {
10844public:
10845 using LLVMTranslationDialectInterface::LLVMTranslationDialectInterface;
10846
10847 /// Translates the given operation to LLVM IR using the provided IR builder
10848 /// and saving the state in `moduleTranslation`.
10849 LogicalResult
10850 convertOperation(Operation *op, llvm::IRBuilderBase &builder,
10851 LLVM::ModuleTranslation &moduleTranslation) const final;
10852
10853 /// Given an OpenMP MLIR attribute, create the corresponding LLVM-IR,
10854 /// runtime calls, or operation amendments
10855 LogicalResult
10856 amendOperation(Operation *op, ArrayRef<llvm::Instruction *> instructions,
10857 NamedAttribute attribute,
10858 LLVM::ModuleTranslation &moduleTranslation) const final;
10859
10860 /// Records the LLVM alloc pointer produced for an OMP ALLOCATE variable so
10861 /// that the paired omp.allocate_free op can generate the matching
10862 /// __kmpc_free call.
10863 void registerAllocatedPtr(Value var, llvm::Value *ptr) const {
10864 ompAllocatedPtrs[var] = ptr;
10865 }
10866
10867 /// Returns the LLVM alloc pointer previously registered for var, or
10868 /// nullptr if no allocation was recorded.
10869 llvm::Value *lookupAllocatedPtr(Value var) const {
10870 auto it = ompAllocatedPtrs.find(Val: var);
10871 return it != ompAllocatedPtrs.end() ? it->second : nullptr;
10872 }
10873
10874private:
10875 /// Maps each MLIR variable value that appeared in an omp.allocate_dir op to
10876 /// the LLVM pointer returned by the corresponding __kmpc_alloc call. The
10877 /// paired omp.allocate_free op looks up these pointers to emit __kmpc_free.
10878 mutable DenseMap<Value, llvm::Value *> ompAllocatedPtrs;
10879};
10880
10881} // namespace
10882
10883LogicalResult OpenMPDialectLLVMIRTranslationInterface::amendOperation(
10884 Operation *op, ArrayRef<llvm::Instruction *> instructions,
10885 NamedAttribute attribute,
10886 LLVM::ModuleTranslation &moduleTranslation) const {
10887 return llvm::StringSwitch<llvm::function_ref<LogicalResult(Attribute)>>(
10888 attribute.getName())
10889 .Case(S: "omp.is_target_device",
10890 Value: [&](Attribute attr) {
10891 if (auto deviceAttr = dyn_cast<BoolAttr>(Val&: attr)) {
10892 llvm::OpenMPIRBuilderConfig &config =
10893 getOpenMPBuilder(moduleTranslation)->Config;
10894 config.setIsTargetDevice(deviceAttr.getValue());
10895 return success();
10896 }
10897 return failure();
10898 })
10899 .Case(S: "omp.is_gpu",
10900 Value: [&](Attribute attr) {
10901 if (auto gpuAttr = dyn_cast<BoolAttr>(Val&: attr)) {
10902 llvm::OpenMPIRBuilderConfig &config =
10903 getOpenMPBuilder(moduleTranslation)->Config;
10904 config.setIsGPU(gpuAttr.getValue());
10905 return success();
10906 }
10907 return failure();
10908 })
10909 .Case(S: "omp.host_ir_filepath",
10910 Value: [&](Attribute attr) {
10911 if (auto filepathAttr = dyn_cast<StringAttr>(Val&: attr)) {
10912 llvm::OpenMPIRBuilder *ompBuilder =
10913 getOpenMPBuilder(moduleTranslation);
10914 ompBuilder->loadOffloadInfoMetadata(
10915 VFS&: moduleTranslation.getFileSystem(), HostFilePath: filepathAttr.getValue());
10916 return success();
10917 }
10918 return failure();
10919 })
10920 .Case(S: "omp.flags",
10921 Value: [&](Attribute attr) {
10922 if (auto rtlAttr = dyn_cast<omp::FlagsAttr>(Val&: attr))
10923 return convertFlagsAttr(op, attribute: rtlAttr, moduleTranslation);
10924 return failure();
10925 })
10926 .Case(S: "omp.version",
10927 Value: [&](Attribute attr) {
10928 if (auto versionAttr = dyn_cast<omp::VersionAttr>(Val&: attr)) {
10929 llvm::OpenMPIRBuilder *ompBuilder =
10930 getOpenMPBuilder(moduleTranslation);
10931 ompBuilder->M.addModuleFlag(Behavior: llvm::Module::Max, Key: "openmp",
10932 Val: versionAttr.getVersion());
10933 return success();
10934 }
10935 return failure();
10936 })
10937 .Case(S: "omp.declare_target",
10938 Value: [&](Attribute attr) {
10939 if (auto declareTargetAttr =
10940 dyn_cast<omp::DeclareTargetAttr>(Val&: attr)) {
10941 llvm::OpenMPIRBuilder *ompBuilder =
10942 getOpenMPBuilder(moduleTranslation);
10943 return convertDeclareTargetAttr(op, attribute: declareTargetAttr,
10944 ompBuilder, moduleTranslation);
10945 }
10946 return failure();
10947 })
10948 .Case(S: "omp.requires",
10949 Value: [&](Attribute attr) {
10950 if (auto requiresAttr = dyn_cast<omp::ClauseRequiresAttr>(Val&: attr)) {
10951 using Requires = omp::ClauseRequires;
10952 Requires flags = requiresAttr.getValue();
10953 llvm::OpenMPIRBuilderConfig &config =
10954 getOpenMPBuilder(moduleTranslation)->Config;
10955 config.setHasRequiresReverseOffload(
10956 bitEnumContainsAll(bits: flags, bit: Requires::reverse_offload));
10957 config.setHasRequiresUnifiedAddress(
10958 bitEnumContainsAll(bits: flags, bit: Requires::unified_address));
10959 config.setHasRequiresUnifiedSharedMemory(
10960 bitEnumContainsAll(bits: flags, bit: Requires::unified_shared_memory));
10961 config.setHasRequiresDynamicAllocators(
10962 bitEnumContainsAll(bits: flags, bit: Requires::dynamic_allocators));
10963 return success();
10964 }
10965 return failure();
10966 })
10967 .Case(S: "omp.target_triples",
10968 Value: [&](Attribute attr) {
10969 if (auto triplesAttr = dyn_cast<ArrayAttr>(Val&: attr)) {
10970 llvm::OpenMPIRBuilderConfig &config =
10971 getOpenMPBuilder(moduleTranslation)->Config;
10972 config.TargetTriples.clear();
10973 config.TargetTriples.reserve(N: triplesAttr.size());
10974 for (Attribute tripleAttr : triplesAttr) {
10975 if (auto tripleStrAttr = dyn_cast<StringAttr>(Val&: tripleAttr))
10976 config.TargetTriples.emplace_back(Args: tripleStrAttr.getValue());
10977 else
10978 return failure();
10979 }
10980 return success();
10981 }
10982 return failure();
10983 })
10984 .Case(S: "omp.integer_wrap_around",
10985 Value: [&](Attribute attr) {
10986 if (auto wrapAttr = dyn_cast<omp::IntegerWrapAroundAttr>(Val&: attr)) {
10987 llvm::OpenMPIRBuilderConfig &config =
10988 getOpenMPBuilder(moduleTranslation)->Config;
10989 config.setNoSignedWrap(!wrapAttr.getIntegerWrapAround());
10990 return success();
10991 }
10992 return failure();
10993 })
10994 .Default(Value: [](Attribute) {
10995 // Fall through for omp attributes that do not require lowering.
10996 return success();
10997 })(attribute.getValue());
10998
10999 return failure();
11000}
11001
11002// Returns true if the operation is not inside a TargetOp, it is part of a
11003// function and that function is not declare target.
11004static bool isHostDeviceOp(Operation *op) {
11005 // Assumes no reverse offloading
11006 if (op->getParentOfType<omp::TargetOp>())
11007 return false;
11008
11009 if (auto parentFn = op->getParentOfType<LLVM::LLVMFuncOp>()) {
11010 if (auto declareTargetIface =
11011 llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(
11012 Val: parentFn.getOperation())) {
11013 omp::DeclareTargetAttr declareTargetAttr =
11014 declareTargetIface.getDeclareTarget();
11015 if (declareTargetAttr && declareTargetAttr.getDeviceType() !=
11016 mlir::omp::DeclareTargetDeviceType::host)
11017 return false;
11018 }
11019
11020 return true;
11021 }
11022
11023 return false;
11024}
11025
11026static llvm::Function *getOmpTargetAlloc(llvm::IRBuilderBase &builder,
11027 llvm::Module *llvmModule) {
11028 llvm::Type *i64Ty = builder.getInt64Ty();
11029 llvm::Type *i32Ty = builder.getInt32Ty();
11030 llvm::Type *returnType = builder.getPtrTy(AddrSpace: 0);
11031 llvm::FunctionType *fnType =
11032 llvm::FunctionType::get(Result: returnType, Params: {i64Ty, i32Ty}, isVarArg: false);
11033 llvm::Function *func = cast<llvm::Function>(
11034 Val: llvmModule->getOrInsertFunction(Name: "omp_target_alloc", T: fnType).getCallee());
11035 return func;
11036}
11037
11038template <typename T>
11039static llvm::Value *
11040getAllocationSize(llvm::IRBuilderBase &builder,
11041 LLVM::ModuleTranslation &moduleTranslation, T op) {
11042 llvm::DataLayout dataLayout =
11043 moduleTranslation.getLLVMModule()->getDataLayout();
11044 llvm::Type *llvmHeapTy =
11045 moduleTranslation.convertType(type: op.getMemElemTypeAttr().getValue());
11046
11047 auto alignment = op.getMemAlignment();
11048 llvm::TypeSize typeSize = llvm::alignTo(
11049 dataLayout.getTypeStoreSize(Ty: llvmHeapTy),
11050 alignment ? *alignment : dataLayout.getABITypeAlign(Ty: llvmHeapTy).value());
11051
11052 llvm::Value *allocSize = builder.getInt64(C: typeSize.getFixedValue());
11053 return builder.CreateMul(
11054 LHS: allocSize,
11055 RHS: builder.CreateIntCast(moduleTranslation.lookupValue(value: op.getMemArraySize()),
11056 builder.getInt64Ty(),
11057 /*isSigned=*/false));
11058}
11059
11060template <>
11061llvm::Value *getAllocationSize(llvm::IRBuilderBase &builder,
11062 LLVM::ModuleTranslation &moduleTranslation,
11063 omp::TargetAllocMemOp op) {
11064 llvm::DataLayout dataLayout =
11065 moduleTranslation.getLLVMModule()->getDataLayout();
11066 llvm::Type *llvmHeapTy = moduleTranslation.convertType(type: op.getAllocatedType());
11067 llvm::TypeSize typeSize = dataLayout.getTypeAllocSize(Ty: llvmHeapTy);
11068 llvm::Value *allocSize = builder.getInt64(C: typeSize.getFixedValue());
11069 for (auto typeParam : op.getTypeparams()) {
11070 allocSize = builder.CreateMul(
11071 LHS: allocSize,
11072 RHS: builder.CreateIntCast(V: moduleTranslation.lookupValue(value: typeParam),
11073 DestTy: builder.getInt64Ty(),
11074 /*isSigned=*/false));
11075 }
11076 return allocSize;
11077}
11078
11079static LogicalResult
11080convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
11081 LLVM::ModuleTranslation &moduleTranslation) {
11082 auto allocMemOp = cast<omp::TargetAllocMemOp>(Val&: opInst);
11083 if (!allocMemOp)
11084 return failure();
11085
11086 // Get "omp_target_alloc" function
11087 llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
11088 llvm::Function *ompTargetAllocFunc = getOmpTargetAlloc(builder, llvmModule);
11089 // Get the corresponding device value in llvm
11090 mlir::Value deviceNum = allocMemOp.getDevice();
11091 llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(value: deviceNum);
11092 // Get the allocation size.
11093 llvm::Value *allocSize =
11094 getAllocationSize(builder, moduleTranslation, op: allocMemOp);
11095 // Create call to "omp_target_alloc" with the args as translated llvm values.
11096 llvm::CallInst *call =
11097 builder.CreateCall(Callee: ompTargetAllocFunc, Args: {allocSize, llvmDeviceNum});
11098 llvm::Value *resultI64 = builder.CreatePtrToInt(V: call, DestTy: builder.getInt64Ty());
11099
11100 // Map the result
11101 moduleTranslation.mapValue(mlir: allocMemOp.getResult(), llvm: resultI64);
11102 return success();
11103}
11104
11105static LogicalResult
11106convertAllocSharedMemOp(omp::AllocSharedMemOp allocMemOp,
11107 llvm::IRBuilderBase &builder,
11108 LLVM::ModuleTranslation &moduleTranslation) {
11109 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
11110 llvm::Value *size = getAllocationSize(builder, moduleTranslation, op: allocMemOp);
11111 moduleTranslation.mapValue(mlir: allocMemOp.getResult(),
11112 llvm: ompBuilder->createOMPAllocShared(Loc: builder, Size: size));
11113 return success();
11114}
11115
11116static LogicalResult
11117convertAllocateDirOp(Operation &opInst, llvm::IRBuilderBase &builder,
11118 LLVM::ModuleTranslation &moduleTranslation,
11119 const OpenMPDialectLLVMIRTranslationInterface &ompIface) {
11120 auto allocateDirOp = cast<omp::AllocateDirOp>(Val&: opInst);
11121 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
11122
11123 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
11124 llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
11125 llvm::DataLayout dataLayout = llvmModule->getDataLayout();
11126 SmallVector<Value> vars = allocateDirOp.getVarList();
11127 std::optional<int64_t> alignAttr = allocateDirOp.getAlign();
11128
11129 llvm::Value *allocator;
11130 if (auto allocatorVar = allocateDirOp.getAllocator()) {
11131 allocator = moduleTranslation.lookupValue(value: allocatorVar);
11132 if (allocator->getType()->isIntegerTy())
11133 allocator = builder.CreateIntToPtr(V: allocator, DestTy: builder.getPtrTy());
11134 else if (allocator->getType()->isPointerTy())
11135 allocator = builder.CreatePointerBitCastOrAddrSpaceCast(
11136 V: allocator, DestTy: builder.getPtrTy());
11137 } else {
11138 allocator = llvm::ConstantPointerNull::get(T: builder.getPtrTy());
11139 }
11140
11141 for (Value var : vars) {
11142 Value baseVar = getBaseValueForTypeLookup(value: var);
11143 llvm::Type *typeToInspect =
11144 getAllocatedLlvmTypeForVariable(var, baseVar, moduleTranslation);
11145
11146 llvm::Value *size;
11147 if (std::optional<llvm::Value *> dynamicSize = getDynamicAllocatedSize(
11148 var, baseVar, moduleTranslation, builder, dataLayout)) {
11149 size = *dynamicSize;
11150 } else if (typeToInspect->isArrayTy()) {
11151 size = builder.getInt64(
11152 C: dataLayout.getTypeAllocSize(Ty: typeToInspect).getFixedValue());
11153 } else {
11154 size = builder.getInt64(
11155 C: dataLayout.getTypeAllocSize(Ty: typeToInspect).getFixedValue());
11156 }
11157
11158 uint64_t alignValue =
11159 alignAttr ? alignAttr.value()
11160 : dataLayout.getABITypeAlign(Ty: typeToInspect).value();
11161 llvm::Value *alignConst = builder.getInt64(C: alignValue);
11162 // Align the size: ((size + align - 1) / align) * align
11163 size = builder.CreateAdd(LHS: size, RHS: builder.getInt64(C: alignValue - 1), Name: "", HasNUW: true);
11164 size = builder.CreateUDiv(LHS: size, RHS: alignConst);
11165 size = builder.CreateMul(LHS: size, RHS: alignConst, Name: "", HasNUW: true);
11166
11167 std::string allocName =
11168 ompBuilder->createPlatformSpecificName(Parts: {".void.addr"});
11169 llvm::CallInst *allocCall;
11170 if (alignAttr.has_value()) {
11171 allocCall = ompBuilder->createOMPAlignedAlloc(
11172 Loc: ompLoc, Align: builder.getInt64(C: alignAttr.value()), Size: size, Allocator: allocator,
11173 Name: allocName);
11174 } else {
11175 allocCall =
11176 ompBuilder->createOMPAlloc(Loc: ompLoc, Size: size, Allocator: allocator, Name: allocName);
11177 }
11178 // Record the alloc pointer keyed by the MLIR variable value.
11179 ompIface.registerAllocatedPtr(var, ptr: allocCall);
11180
11181 if (llvm::Value *baseLlvm = moduleTranslation.lookupValue(value: baseVar)) {
11182 llvm::Value *boundPtr = builder.CreatePointerBitCastOrAddrSpaceCast(
11183 V: allocCall, DestTy: baseLlvm->getType());
11184 moduleTranslation.remapAllValuesWith(oldValue: baseLlvm, newValue: boundPtr);
11185 } else if (llvm::Value *varLlvm = moduleTranslation.lookupValue(value: var)) {
11186 llvm::Value *boundPtr = builder.CreatePointerBitCastOrAddrSpaceCast(
11187 V: allocCall, DestTy: varLlvm->getType());
11188 moduleTranslation.remapAllValuesWith(oldValue: varLlvm, newValue: boundPtr);
11189 }
11190 }
11191
11192 return success();
11193}
11194
11195static LogicalResult
11196convertAllocateFreeOp(Operation &opInst, llvm::IRBuilderBase &builder,
11197 LLVM::ModuleTranslation &moduleTranslation,
11198 const OpenMPDialectLLVMIRTranslationInterface &ompIface) {
11199 auto freeOp = cast<omp::AllocateFreeOp>(Val&: opInst);
11200 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
11201 llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
11202
11203 llvm::Value *allocator;
11204 if (auto allocatorVar = freeOp.getAllocator()) {
11205 allocator = moduleTranslation.lookupValue(value: allocatorVar);
11206 if (allocator->getType()->isIntegerTy())
11207 allocator = builder.CreateIntToPtr(V: allocator, DestTy: builder.getPtrTy());
11208 else if (allocator->getType()->isPointerTy())
11209 allocator = builder.CreatePointerBitCastOrAddrSpaceCast(
11210 V: allocator, DestTy: builder.getPtrTy());
11211 } else {
11212 allocator = llvm::ConstantPointerNull::get(T: builder.getPtrTy());
11213 }
11214
11215 // Emit __kmpc_free for each variable in reverse allocation order.
11216 SmallVector<Value> vars = freeOp.getVarList();
11217 for (Value var : llvm::reverse(C&: vars)) {
11218 llvm::Value *allocPtr = ompIface.lookupAllocatedPtr(var);
11219 if (!allocPtr)
11220 return opInst.emitError(message: "omp.allocate_free: no allocation recorded");
11221 ompBuilder->createOMPFree(Loc: ompLoc, Addr: allocPtr, Allocator: allocator, Name: "");
11222 }
11223
11224 return success();
11225}
11226
11227static llvm::Function *getOmpTargetFree(llvm::IRBuilderBase &builder,
11228 llvm::Module *llvmModule) {
11229 llvm::Type *ptrTy = builder.getPtrTy(AddrSpace: 0);
11230 llvm::Type *i32Ty = builder.getInt32Ty();
11231 llvm::Type *voidTy = builder.getVoidTy();
11232 llvm::FunctionType *fnType =
11233 llvm::FunctionType::get(Result: voidTy, Params: {ptrTy, i32Ty}, isVarArg: false);
11234 llvm::Function *func = dyn_cast<llvm::Function>(
11235 Val: llvmModule->getOrInsertFunction(Name: "omp_target_free", T: fnType).getCallee());
11236 return func;
11237}
11238
11239static LogicalResult
11240convertTargetFreeMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
11241 LLVM::ModuleTranslation &moduleTranslation) {
11242 auto freeMemOp = cast<omp::TargetFreeMemOp>(Val&: opInst);
11243 if (!freeMemOp)
11244 return failure();
11245
11246 // Get "omp_target_free" function
11247 llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
11248 llvm::Function *ompTragetFreeFunc = getOmpTargetFree(builder, llvmModule);
11249 // Get the corresponding device value in llvm
11250 mlir::Value deviceNum = freeMemOp.getDevice();
11251 llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(value: deviceNum);
11252 // Get the corresponding heapref value in llvm
11253 mlir::Value heapref = freeMemOp.getHeapref();
11254 llvm::Value *llvmHeapref = moduleTranslation.lookupValue(value: heapref);
11255 // Convert heapref int to ptr and call "omp_target_free"
11256 llvm::Value *intToPtr =
11257 builder.CreateIntToPtr(V: llvmHeapref, DestTy: builder.getPtrTy(AddrSpace: 0));
11258 builder.CreateCall(Callee: ompTragetFreeFunc, Args: {intToPtr, llvmDeviceNum});
11259 return success();
11260}
11261
11262static LogicalResult
11263convertFreeSharedMemOp(omp::FreeSharedMemOp freeMemOp,
11264 llvm::IRBuilderBase &builder,
11265 LLVM::ModuleTranslation &moduleTranslation) {
11266 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
11267 llvm::Value *size = getAllocationSize(builder, moduleTranslation, op: freeMemOp);
11268 ompBuilder->createOMPFreeShared(
11269 Loc: builder, Addr: moduleTranslation.lookupValue(value: freeMemOp.getHeapref()), Size: size);
11270 return success();
11271}
11272
11273/// Converts an OpenMP groupprivate operation into LLVM IR.
11274static LogicalResult
11275convertOmpGroupprivate(Operation &opInst, llvm::IRBuilderBase &builder,
11276 LLVM::ModuleTranslation &moduleTranslation) {
11277 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
11278 auto groupprivateOp = cast<omp::GroupprivateOp>(Val&: opInst);
11279
11280 if (failed(Result: checkImplementationStatus(op&: opInst)))
11281 return failure();
11282
11283 bool isTargetDevice = ompBuilder->Config.isTargetDevice();
11284
11285 // Determine whether group-private storage should be allocated based on
11286 // device_type. When not specified, default to 'any' (allocate on both).
11287 bool shouldAllocate = true;
11288 switch (groupprivateOp.getDeviceType().value_or(
11289 u: mlir::omp::DeclareTargetDeviceType::any)) {
11290 case mlir::omp::DeclareTargetDeviceType::host:
11291 shouldAllocate = !isTargetDevice;
11292 break;
11293 case mlir::omp::DeclareTargetDeviceType::nohost:
11294 shouldAllocate = isTargetDevice;
11295 break;
11296 case mlir::omp::DeclareTargetDeviceType::any:
11297 shouldAllocate = true;
11298 break;
11299 }
11300
11301 // Look up the global variable directly by symbol name.
11302 LLVM::GlobalOp global = SymbolTable::lookupNearestSymbolFrom<LLVM::GlobalOp>(
11303 from: &opInst, symbol: groupprivateOp.getSymNameAttr());
11304 if (!global)
11305 return opInst.emitError()
11306 << "expected symbol '" << groupprivateOp.getSymName()
11307 << "' to reference an LLVM global variable";
11308
11309 llvm::GlobalValue *globalValue = moduleTranslation.lookupGlobal(op: global);
11310 llvm::Type *varType = moduleTranslation.convertType(type: global.getType());
11311 std::string varName = globalValue->getName().str();
11312
11313 llvm::Value *resultPtr;
11314 if (shouldAllocate && isTargetDevice) {
11315 llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
11316 llvm::Triple targetTriple(llvmModule->getTargetTriple());
11317 unsigned sharedAddressSpace;
11318 if (targetTriple.isAMDGCN())
11319 sharedAddressSpace = llvm::AMDGPUAS::LOCAL_ADDRESS;
11320 else if (targetTriple.isNVPTX())
11321 sharedAddressSpace = llvm::NVPTXAS::ADDRESS_SPACE_SHARED;
11322 else
11323 return opInst.emitError() << "groupprivate is not supported for target: "
11324 << targetTriple.str();
11325 llvm::GlobalVariable *sharedVar = new llvm::GlobalVariable(
11326 *llvmModule, varType, /*isConstant=*/false,
11327 llvm::GlobalValue::InternalLinkage, llvm::PoisonValue::get(T: varType),
11328 varName, /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
11329 sharedAddressSpace,
11330 /*isExternallyInitialized=*/false);
11331 resultPtr = sharedVar;
11332 } else {
11333 if (shouldAllocate && !isTargetDevice)
11334 opInst.emitWarning(message: "groupprivate directive is currently ignored on the "
11335 "host, using original global");
11336 resultPtr = globalValue;
11337 }
11338
11339 moduleTranslation.mapValue(mlir: opInst.getResult(idx: 0), llvm: resultPtr);
11340 return success();
11341}
11342
11343/// Given an OpenMP MLIR operation, create the corresponding LLVM IR (including
11344/// OpenMP runtime calls).
11345LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(
11346 Operation *op, llvm::IRBuilderBase &builder,
11347 LLVM::ModuleTranslation &moduleTranslation) const {
11348 llvm::OpenMPIRBuilder *ompBuilder = getOpenMPBuilder(moduleTranslation);
11349
11350 if (ompBuilder->Config.isTargetDevice() &&
11351 !isa<omp::TargetOp, omp::MapInfoOp, omp::TerminatorOp, omp::YieldOp>(
11352 Val: op) &&
11353 isHostDeviceOp(op))
11354 return op->emitOpError() << "unsupported host op found in device";
11355
11356 // For each loop, introduce one stack frame to hold loop information. Ensure
11357 // this is only done for the outermost loop wrapper to prevent introducing
11358 // multiple stack frames for a single loop. Initially set to null, the loop
11359 // information structure is initialized during translation of the nested
11360 // omp.loop_nest operation, making it available to translation of all loop
11361 // wrappers after their body has been successfully translated.
11362 bool isOutermostLoopWrapper =
11363 isa_and_present<omp::LoopWrapperInterface>(Val: op) &&
11364 !dyn_cast_if_present<omp::LoopWrapperInterface>(Val: op->getParentOp());
11365
11366 // The TASKLOOP construct is implemented with an outer taskloop.context
11367 // operation which is not a loop wrapper, containing an inner taskloop
11368 // operation which is a loop wrapper. The stack frame should be pushed when
11369 // translating the outer taskloop.context and popped when translating the
11370 // inner taskloop which is a loop wrapper. We need access to the loop
11371 // information in the outer taskloop context so we need to create it and pop
11372 // it around the taskloop context not the inner loop wrapper.
11373 if (isa<omp::TaskloopContextOp>(Val: op))
11374 isOutermostLoopWrapper = true;
11375 else if (isa<omp::TaskloopWrapperOp>(Val: op))
11376 isOutermostLoopWrapper = false;
11377
11378 if (isOutermostLoopWrapper)
11379 moduleTranslation.stackPush<OpenMPLoopInfoStackFrame>();
11380
11381 auto result =
11382 llvm::TypeSwitch<Operation *, LogicalResult>(op)
11383 .Case(caseFn: [&](omp::BarrierOp op) -> LogicalResult {
11384 if (failed(Result: checkImplementationStatus(op&: *op)))
11385 return failure();
11386
11387 llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
11388 ompBuilder->createBarrier(Loc: builder, Kind: llvm::omp::OMPD_barrier);
11389 LogicalResult res = handleError(result&: afterIP, op&: *op);
11390 if (res.succeeded()) {
11391 // If the barrier generated a cancellation check, the insertion
11392 // point might now need to be changed to a new continuation block
11393 builder.restoreIP(IP: *afterIP);
11394 }
11395 return res;
11396 })
11397 .Case(caseFn: [&](omp::TaskyieldOp op) {
11398 if (failed(Result: checkImplementationStatus(op&: *op)))
11399 return failure();
11400
11401 ompBuilder->createTaskyield(Loc: builder);
11402 return success();
11403 })
11404 .Case(caseFn: [&](omp::FlushOp op) {
11405 if (failed(Result: checkImplementationStatus(op&: *op)))
11406 return failure();
11407
11408 // No support in Openmp runtime function (__kmpc_flush) to accept
11409 // the argument list.
11410 // OpenMP standard states the following:
11411 // "An implementation may implement a flush with a list by ignoring
11412 // the list, and treating it the same as a flush without a list."
11413 //
11414 // The argument list is discarded so that, flush with a list is
11415 // treated same as a flush without a list.
11416 ompBuilder->createFlush(Loc: builder);
11417 return success();
11418 })
11419 .Case(caseFn: [&](omp::ErrorOp op) {
11420 if (failed(Result: checkImplementationStatus(op&: *op)))
11421 return failure();
11422
11423 llvm::Value *message = nullptr;
11424 if (mlir::Value messageExpr = op.getMessageExpr())
11425 message = moduleTranslation.lookupValue(value: messageExpr);
11426 else if (std::optional<StringRef> msg = op.getMessage();
11427 msg && !msg->empty())
11428 message = builder.CreateGlobalString(Str: *msg);
11429 ompBuilder->createError(
11430 Loc: llvm::OpenMPIRBuilder::LocationDescription(builder),
11431 IsFatal: op.getSeverity() == omp::ClauseSeverity::fatal, Message: message);
11432 return success();
11433 })
11434 .Case(caseFn: [&](omp::ParallelOp op) {
11435 return convertOmpParallel(opInst: op, builder, moduleTranslation);
11436 })
11437 .Case(caseFn: [&](omp::DispatchOp) {
11438 return convertOmpDispatch(opInst&: *op, builder, moduleTranslation);
11439 })
11440 .Case(caseFn: [&](omp::MaskedOp) {
11441 return convertOmpMasked(opInst&: *op, builder, moduleTranslation);
11442 })
11443 .Case(caseFn: [&](omp::MasterOp) {
11444 return convertOmpMaster(opInst&: *op, builder, moduleTranslation);
11445 })
11446 .Case(caseFn: [&](omp::CriticalOp) {
11447 return convertOmpCritical(opInst&: *op, builder, moduleTranslation);
11448 })
11449 .Case(caseFn: [&](omp::OrderedRegionOp) {
11450 return convertOmpOrderedRegion(opInst&: *op, builder, moduleTranslation);
11451 })
11452 .Case(caseFn: [&](omp::OrderedOp) {
11453 return convertOmpOrdered(opInst&: *op, builder, moduleTranslation);
11454 })
11455 .Case(caseFn: [&](omp::WsloopOp) {
11456 return convertOmpWsloop(opInst&: *op, builder, moduleTranslation);
11457 })
11458 .Case(caseFn: [&](omp::SimdOp) {
11459 return convertOmpSimd(opInst&: *op, builder, moduleTranslation);
11460 })
11461 .Case(caseFn: [&](omp::AtomicReadOp) {
11462 return convertOmpAtomicRead(opInst&: *op, builder, moduleTranslation);
11463 })
11464 .Case(caseFn: [&](omp::AtomicWriteOp) {
11465 return convertOmpAtomicWrite(opInst&: *op, builder, moduleTranslation);
11466 })
11467 .Case(caseFn: [&](omp::AtomicUpdateOp op) {
11468 return convertOmpAtomicUpdate(opInst&: op, builder, moduleTranslation);
11469 })
11470 .Case(caseFn: [&](omp::AtomicCaptureOp op) {
11471 return convertOmpAtomicCapture(atomicCaptureOp: op, builder, moduleTranslation);
11472 })
11473 .Case(caseFn: [&](omp::AtomicCompareOp op) {
11474 return convertOmpAtomicCompare(atomicCompareOp: op, builder, moduleTranslation);
11475 })
11476 .Case(caseFn: [&](omp::CancelOp op) {
11477 return convertOmpCancel(op, builder, moduleTranslation);
11478 })
11479 .Case(caseFn: [&](omp::CancellationPointOp op) {
11480 return convertOmpCancellationPoint(op, builder, moduleTranslation);
11481 })
11482 .Case(caseFn: [&](omp::SectionsOp) {
11483 return convertOmpSections(opInst&: *op, builder, moduleTranslation);
11484 })
11485 .Case(caseFn: [&](omp::ScopeOp op) {
11486 return convertOmpScope(scopeOp&: op, builder, moduleTranslation);
11487 })
11488 .Case(caseFn: [&](omp::SingleOp op) {
11489 return convertOmpSingle(singleOp&: op, builder, moduleTranslation);
11490 })
11491 .Case(caseFn: [&](omp::TeamsOp op) {
11492 return convertOmpTeams(op, builder, moduleTranslation);
11493 })
11494 .Case(caseFn: [&](omp::TaskOp op) {
11495 return convertOmpTaskOp(taskOp: op, builder, moduleTranslation);
11496 })
11497 .Case(caseFn: [&](omp::TaskloopWrapperOp op) {
11498 return convertOmpTaskloopWrapperOp(loopWrapperOp: op, builder, moduleTranslation);
11499 })
11500 .Case(caseFn: [&](omp::TaskloopContextOp op) {
11501 return convertOmpTaskloopContextOp(contextOp: op, builder, moduleTranslation);
11502 })
11503 .Case(caseFn: [&](omp::TaskgroupOp op) {
11504 return convertOmpTaskgroupOp(tgOp: op, builder, moduleTranslation);
11505 })
11506 .Case(caseFn: [&](omp::TaskwaitOp op) {
11507 return convertOmpTaskwaitOp(twOp: op, builder, moduleTranslation);
11508 })
11509 .Case(caseFn: [&](omp::InteropInitOp op) {
11510 return convertOmpInteropInitOp(initOp: op, builder, moduleTranslation);
11511 })
11512 .Case(caseFn: [&](omp::InteropDestroyOp op) {
11513 return convertOmpInteropDestroyOp(destroyOp: op, builder, moduleTranslation);
11514 })
11515 .Case(caseFn: [&](omp::InteropUseOp op) {
11516 return convertOmpInteropUseOp(useOp: op, builder, moduleTranslation);
11517 })
11518 .Case<omp::YieldOp, omp::TerminatorOp, omp::DeclareMapperOp,
11519 omp::DeclareMapperInfoOp, omp::DeclareReductionOp,
11520 omp::CriticalDeclareOp>(caseFn: [](auto op) {
11521 // `yield` and `terminator` can be just omitted. The block structure
11522 // was created in the region that handles their parent operation.
11523 // `declare_reduction` will be used by reductions and is not
11524 // converted directly, skip it.
11525 // `declare_mapper` and `declare_mapper.info` are handled whenever
11526 // they are referred to through a `map` clause.
11527 // `critical.declare` is only used to declare names of critical
11528 // sections which will be used by `critical` ops and hence can be
11529 // ignored for lowering. The OpenMP IRBuilder will create unique
11530 // name for critical section names.
11531 return success();
11532 })
11533 .Case(caseFn: [&](omp::ThreadprivateOp) {
11534 return convertOmpThreadprivate(opInst&: *op, builder, moduleTranslation);
11535 })
11536 .Case<omp::TargetDataOp, omp::TargetEnterDataOp,
11537 omp::TargetExitDataOp, omp::TargetUpdateOp>(caseFn: [&](auto op) {
11538 return convertOmpTargetData(op, builder, moduleTranslation);
11539 })
11540 .Case(caseFn: [&](omp::TargetOp) {
11541 return convertOmpTarget(opInst&: *op, builder, moduleTranslation);
11542 })
11543 .Case(caseFn: [&](omp::DistributeOp) {
11544 return convertOmpDistribute(opInst&: *op, builder, moduleTranslation);
11545 })
11546 .Case(caseFn: [&](omp::LoopNestOp) {
11547 return convertOmpLoopNest(opInst&: *op, builder, moduleTranslation);
11548 })
11549 .Case<omp::MapInfoOp, omp::MapBoundsOp, omp::PrivateClauseOp,
11550 omp::AffinityEntryOp, omp::IteratorOp>(caseFn: [&](auto op) {
11551 // No-op, should be handled by relevant owning operations e.g.
11552 // TargetOp, TargetEnterDataOp, TargetExitDataOp, TargetDataOp
11553 // etc. and then discarded
11554 return success();
11555 })
11556 .Case(caseFn: [&](omp::NewCliOp op) {
11557 // Meta-operation: Doesn't do anything by itself, but used to
11558 // identify a loop.
11559 return success();
11560 })
11561 .Case(caseFn: [&](omp::CanonicalLoopOp op) {
11562 return convertOmpCanonicalLoopOp(op, builder, moduleTranslation);
11563 })
11564 .Case(caseFn: [&](omp::UnrollHeuristicOp op) {
11565 // FIXME: Handling omp.unroll_heuristic as an executable requires
11566 // that the generator (e.g. omp.canonical_loop) has been seen first.
11567 // For construct that require all codegen to occur inside a callback
11568 // (e.g. OpenMPIRBilder::createParallel), all codegen of that
11569 // contained region including their transformations must occur at
11570 // the omp.canonical_loop.
11571 return applyUnrollHeuristic(op, builder, moduleTranslation);
11572 })
11573 .Case(caseFn: [&](omp::UnrollFullOp op) {
11574 return applyUnrollFull(op, builder, moduleTranslation);
11575 })
11576 .Case(caseFn: [&](omp::UnrollPartialOp op) {
11577 return applyUnrollPartial(op, builder, moduleTranslation);
11578 })
11579 .Case(caseFn: [&](omp::TileOp op) {
11580 return applyTile(op, builder, moduleTranslation);
11581 })
11582 .Case(caseFn: [&](omp::FuseOp op) {
11583 return applyFuse(op, builder, moduleTranslation);
11584 })
11585 .Case(caseFn: [&](omp::TargetAllocMemOp) {
11586 return convertTargetAllocMemOp(opInst&: *op, builder, moduleTranslation);
11587 })
11588 .Case(caseFn: [&](omp::TargetFreeMemOp) {
11589 return convertTargetFreeMemOp(opInst&: *op, builder, moduleTranslation);
11590 })
11591 .Case(caseFn: [&](omp::AllocateDirOp) {
11592 return convertAllocateDirOp(opInst&: *op, builder, moduleTranslation, ompIface: *this);
11593 })
11594 .Case(caseFn: [&](omp::AllocateFreeOp) {
11595 return convertAllocateFreeOp(opInst&: *op, builder, moduleTranslation,
11596 ompIface: *this);
11597 })
11598 .Case(caseFn: [&](omp::AllocSharedMemOp op) {
11599 return convertAllocSharedMemOp(allocMemOp: op, builder, moduleTranslation);
11600 })
11601 .Case(caseFn: [&](omp::FreeSharedMemOp op) {
11602 return convertFreeSharedMemOp(freeMemOp: op, builder, moduleTranslation);
11603 })
11604 .Case(caseFn: [&](omp::GroupprivateOp) {
11605 return convertOmpGroupprivate(opInst&: *op, builder, moduleTranslation);
11606 })
11607 .Case(caseFn: [&](omp::DeclareSimdOp) {
11608 // TODO: Implement.
11609 // DECLARE_SIMD can be ignored. There is warning emitted in
11610 // semantic checks about the current situation, but not having an
11611 // implementation should not prevent compilation.
11612 return success();
11613 })
11614 .Default(defaultFn: [&](Operation *inst) {
11615 return inst->emitError()
11616 << "not yet implemented: " << inst->getName();
11617 });
11618
11619 if (isOutermostLoopWrapper)
11620 moduleTranslation.stackPop();
11621
11622 return result;
11623}
11624
11625void mlir::registerOpenMPTranslationCLOptions() {
11626 static llvm::cl::opt<bool, /*ExternalStorage=*/true> option(
11627 "openmp-disable-virtual-collapse", llvm::cl::Hidden,
11628 llvm::cl::desc("Disable virtual collapse of eligible host loops"),
11629 llvm::cl::location(L&: disableVirtualCollapse));
11630}
11631
11632void mlir::registerOpenMPDialectTranslation(DialectRegistry &registry) {
11633 registry.insert<omp::OpenMPDialect>();
11634 registry.addExtension(extensionFn: +[](MLIRContext *ctx, omp::OpenMPDialect *dialect) {
11635 dialect->addInterfaces<OpenMPDialectLLVMIRTranslationInterface>();
11636 });
11637}
11638
11639void mlir::registerOpenMPDialectTranslation(MLIRContext &context) {
11640 DialectRegistry registry;
11641 registerOpenMPDialectTranslation(registry);
11642 context.appendDialectRegistry(registry);
11643}
11644