1//===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
14#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
15
16#include "CGValue.h"
17#include "clang/AST/DeclOpenMP.h"
18#include "clang/AST/GlobalDecl.h"
19#include "clang/AST/Type.h"
20#include "clang/Basic/OpenMPKinds.h"
21#include "clang/Basic/SourceLocation.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/PointerIntPair.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/Frontend/OpenMP/OMPConstants.h"
28#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/ValueHandle.h"
31#include "llvm/Support/AtomicOrdering.h"
32
33namespace llvm {
34class ArrayType;
35class Constant;
36class FunctionType;
37class GlobalVariable;
38class Type;
39class Value;
40class OpenMPIRBuilder;
41} // namespace llvm
42
43namespace clang {
44class Expr;
45class OMPDependClause;
46class OMPExecutableDirective;
47class OMPLoopDirective;
48class VarDecl;
49class OMPDeclareReductionDecl;
50
51namespace CodeGen {
52class Address;
53class CodeGenFunction;
54class CodeGenModule;
55
56/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
57/// region.
58class PrePostActionTy {
59public:
60 explicit PrePostActionTy() {}
61 virtual void Enter(CodeGenFunction &CGF) {}
62 virtual void Exit(CodeGenFunction &CGF) {}
63 virtual ~PrePostActionTy() {}
64};
65
66/// Class provides a way to call simple version of codegen for OpenMP region, or
67/// an advanced with possible pre|post-actions in codegen.
68class RegionCodeGenTy final {
69 intptr_t CodeGen;
70 typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &);
71 CodeGenTy Callback;
72 mutable PrePostActionTy *PrePostAction;
73 RegionCodeGenTy() = delete;
74 template <typename Callable>
75 static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF,
76 PrePostActionTy &Action) {
77 return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action);
78 }
79
80public:
81 template <typename Callable>
82 RegionCodeGenTy(
83 Callable &&CodeGen,
84 std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
85 RegionCodeGenTy>::value> * = nullptr)
86 : CodeGen(reinterpret_cast<intptr_t>(&CodeGen)),
87 Callback(CallbackFn<std::remove_reference_t<Callable>>),
88 PrePostAction(nullptr) {}
89 void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; }
90 void operator()(CodeGenFunction &CGF) const;
91};
92
93struct OMPTaskDataTy final {
94 SmallVector<const Expr *, 4> PrivateVars;
95 SmallVector<const Expr *, 4> PrivateCopies;
96 SmallVector<const Expr *, 4> FirstprivateVars;
97 SmallVector<const Expr *, 4> FirstprivateCopies;
98 SmallVector<const Expr *, 4> FirstprivateInits;
99 SmallVector<const Expr *, 4> LastprivateVars;
100 SmallVector<const Expr *, 4> LastprivateCopies;
101 SmallVector<const Expr *, 4> ReductionVars;
102 SmallVector<const Expr *, 4> ReductionOrigs;
103 SmallVector<const Expr *, 4> ReductionCopies;
104 SmallVector<const Expr *, 4> ReductionOps;
105 SmallVector<CanonicalDeclPtr<const VarDecl>, 4> PrivateLocals;
106 struct DependData {
107 OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown;
108 const Expr *IteratorExpr = nullptr;
109 SmallVector<const Expr *, 4> DepExprs;
110 explicit DependData() = default;
111 DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr)
112 : DepKind(DepKind), IteratorExpr(IteratorExpr) {}
113 };
114 SmallVector<DependData, 4> Dependences;
115 llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
116 llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule;
117 llvm::PointerIntPair<llvm::Value *, 1, bool> Priority;
118 llvm::Value *Reductions = nullptr;
119 unsigned NumberOfParts = 0;
120 bool Tied = true;
121 bool Nogroup = false;
122 bool IsReductionWithTaskMod = false;
123 bool IsWorksharingReduction = false;
124 bool HasNowaitClause = false;
125 bool HasModifier = false;
126};
127
128/// Class intended to support codegen of all kind of the reduction clauses.
129class ReductionCodeGen {
130private:
131 /// Data required for codegen of reduction clauses.
132 struct ReductionData {
133 /// Reference to the item shared between tasks to reduce into.
134 const Expr *Shared = nullptr;
135 /// Reference to the original item.
136 const Expr *Ref = nullptr;
137 /// Helper expression for generation of private copy.
138 const Expr *Private = nullptr;
139 /// Helper expression for generation reduction operation.
140 const Expr *ReductionOp = nullptr;
141 ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private,
142 const Expr *ReductionOp)
143 : Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) {
144 }
145 };
146 /// List of reduction-based clauses.
147 SmallVector<ReductionData, 4> ClausesData;
148
149 /// List of addresses of shared variables/expressions.
150 SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses;
151 /// List of addresses of original variables/expressions.
152 SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses;
153 /// Sizes of the reduction items in chars.
154 SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes;
155 /// Base declarations for the reduction items.
156 SmallVector<const VarDecl *, 4> BaseDecls;
157
158 /// Emits lvalue for shared expression.
159 LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E);
160 /// Emits upper bound for shared expression (if array section).
161 LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E);
162 /// Performs aggregate initialization.
163 /// \param N Number of reduction item in the common list.
164 /// \param PrivateAddr Address of the corresponding private item.
165 /// \param SharedAddr Address of the original shared variable.
166 /// \param DRD Declare reduction construct used for reduction item.
167 void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N,
168 Address PrivateAddr, Address SharedAddr,
169 const OMPDeclareReductionDecl *DRD);
170
171public:
172 ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs,
173 ArrayRef<const Expr *> Privates,
174 ArrayRef<const Expr *> ReductionOps);
175 /// Emits lvalue for the shared and original reduction item.
176 /// \param N Number of the reduction item.
177 void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N);
178 /// Emits the code for the variable-modified type, if required.
179 /// \param N Number of the reduction item.
180 void emitAggregateType(CodeGenFunction &CGF, unsigned N);
181 /// Emits the code for the variable-modified type, if required.
182 /// \param N Number of the reduction item.
183 /// \param Size Size of the type in chars.
184 void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size);
185 /// Performs initialization of the private copy for the reduction item.
186 /// \param N Number of the reduction item.
187 /// \param PrivateAddr Address of the corresponding private item.
188 /// \param DefaultInit Default initialization sequence that should be
189 /// performed if no reduction specific initialization is found.
190 /// \param SharedAddr Address of the original shared variable.
191 void
192 emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr,
193 Address SharedAddr,
194 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit);
195 /// Returns true if the private copy requires cleanups.
196 bool needCleanups(unsigned N);
197 /// Emits cleanup code for the reduction item.
198 /// \param N Number of the reduction item.
199 /// \param PrivateAddr Address of the corresponding private item.
200 void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr);
201 /// Adjusts \p PrivatedAddr for using instead of the original variable
202 /// address in normal operations.
203 /// \param N Number of the reduction item.
204 /// \param PrivateAddr Address of the corresponding private item.
205 Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
206 Address PrivateAddr);
207 /// Returns LValue for the reduction item.
208 LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; }
209 /// Returns LValue for the original reduction item.
210 LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; }
211 /// Returns the size of the reduction item (in chars and total number of
212 /// elements in the item), or nullptr, if the size is a constant.
213 std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const {
214 return Sizes[N];
215 }
216 /// Returns the base declaration of the reduction item.
217 const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; }
218 /// Returns the base declaration of the reduction item.
219 const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; }
220 /// Returns true if the initialization of the reduction item uses initializer
221 /// from declare reduction construct.
222 bool usesReductionInitializer(unsigned N) const;
223 /// Return the type of the private item.
224 QualType getPrivateType(unsigned N) const {
225 return cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl())
226 ->getType();
227 }
228};
229
230class CGOpenMPRuntime {
231public:
232 /// Allows to disable automatic handling of functions used in target regions
233 /// as those marked as `omp declare target`.
234 class DisableAutoDeclareTargetRAII {
235 CodeGenModule &CGM;
236 bool SavedShouldMarkAsGlobal = false;
237
238 public:
239 DisableAutoDeclareTargetRAII(CodeGenModule &CGM);
240 ~DisableAutoDeclareTargetRAII();
241 };
242
243 /// Manages list of nontemporal decls for the specified directive.
244 class NontemporalDeclsRAII {
245 CodeGenModule &CGM;
246 const bool NeedToPush;
247
248 public:
249 NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S);
250 ~NontemporalDeclsRAII();
251 };
252
253 /// Manages list of nontemporal decls for the specified directive.
254 class UntiedTaskLocalDeclsRAII {
255 CodeGenModule &CGM;
256 const bool NeedToPush;
257
258 public:
259 UntiedTaskLocalDeclsRAII(
260 CodeGenFunction &CGF,
261 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
262 std::pair<Address, Address>> &LocalVars);
263 ~UntiedTaskLocalDeclsRAII();
264 };
265
266 /// Maps the expression for the lastprivate variable to the global copy used
267 /// to store new value because original variables are not mapped in inner
268 /// parallel regions. Only private copies are captured but we need also to
269 /// store private copy in shared address.
270 /// Also, stores the expression for the private loop counter and it
271 /// threaprivate name.
272 struct LastprivateConditionalData {
273 llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>>
274 DeclToUniqueName;
275 LValue IVLVal;
276 llvm::Function *Fn = nullptr;
277 bool Disabled = false;
278 };
279 /// Manages list of lastprivate conditional decls for the specified directive.
280 class LastprivateConditionalRAII {
281 enum class ActionToDo {
282 DoNotPush,
283 PushAsLastprivateConditional,
284 DisableLastprivateConditional,
285 };
286 CodeGenModule &CGM;
287 ActionToDo Action = ActionToDo::DoNotPush;
288
289 /// Check and try to disable analysis of inner regions for changes in
290 /// lastprivate conditional.
291 void tryToDisableInnerAnalysis(const OMPExecutableDirective &S,
292 llvm::DenseSet<CanonicalDeclPtr<const Decl>>
293 &NeedToAddForLPCsAsDisabled) const;
294
295 LastprivateConditionalRAII(CodeGenFunction &CGF,
296 const OMPExecutableDirective &S);
297
298 public:
299 explicit LastprivateConditionalRAII(CodeGenFunction &CGF,
300 const OMPExecutableDirective &S,
301 LValue IVLVal);
302 static LastprivateConditionalRAII disable(CodeGenFunction &CGF,
303 const OMPExecutableDirective &S);
304 ~LastprivateConditionalRAII();
305 };
306
307 llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; }
308
309protected:
310 CodeGenModule &CGM;
311
312 /// An OpenMP-IR-Builder instance.
313 llvm::OpenMPIRBuilder OMPBuilder;
314
315 /// Helper to determine the min/max number of threads/teams for \p D.
316 void computeMinAndMaxThreadsAndTeams(
317 const OMPExecutableDirective &D, CodeGenFunction &CGF,
318 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs);
319
320 /// Helper to emit outlined function for 'target' directive.
321 /// \param D Directive to emit.
322 /// \param ParentName Name of the function that encloses the target region.
323 /// \param OutlinedFn Outlined function value to be defined by this call.
324 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
325 /// \param IsOffloadEntry True if the outlined function is an offload entry.
326 /// \param CodeGen Lambda codegen specific to an accelerator device.
327 /// An outlined function may not be an entry if, e.g. the if clause always
328 /// evaluates to false.
329 virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D,
330 StringRef ParentName,
331 llvm::Function *&OutlinedFn,
332 llvm::Constant *&OutlinedFnID,
333 bool IsOffloadEntry,
334 const RegionCodeGenTy &CodeGen);
335
336 /// Returns pointer to ident_t type.
337 llvm::Type *getIdentTyPointerTy();
338
339 /// Gets thread id value for the current thread.
340 ///
341 llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc);
342
343 /// Get the function name of an outlined region.
344 std::string getOutlinedHelperName(StringRef Name) const;
345 std::string getOutlinedHelperName(CodeGenFunction &CGF) const;
346
347 /// Get the function name of a reduction function.
348 std::string getReductionFuncName(StringRef Name) const;
349
350 /// Emits \p Callee function call with arguments \p Args with location \p Loc.
351 void emitCall(CodeGenFunction &CGF, SourceLocation Loc,
352 llvm::FunctionCallee Callee,
353 ArrayRef<llvm::Value *> Args = {}) const;
354
355 /// Emits address of the word in a memory where current thread id is
356 /// stored.
357 virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc);
358
359 void setLocThreadIdInsertPt(CodeGenFunction &CGF,
360 bool AtCurrentPoint = false);
361 void clearLocThreadIdInsertPt(CodeGenFunction &CGF);
362
363 /// Check if the default location must be constant.
364 /// Default is false to support OMPT/OMPD.
365 virtual bool isDefaultLocationConstant() const { return false; }
366
367 /// Returns additional flags that can be stored in reserved_2 field of the
368 /// default location.
369 virtual unsigned getDefaultLocationReserved2Flags() const { return 0; }
370
371 /// Returns default flags for the barriers depending on the directive, for
372 /// which this barier is going to be emitted.
373 static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind);
374
375 /// Get the LLVM type for the critical name.
376 llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;}
377
378 /// Returns corresponding lock object for the specified critical region
379 /// name. If the lock object does not exist it is created, otherwise the
380 /// reference to the existing copy is returned.
381 /// \param CriticalName Name of the critical region.
382 ///
383 llvm::Value *getCriticalRegionLock(StringRef CriticalName);
384
385protected:
386 /// Map for SourceLocation and OpenMP runtime library debug locations.
387 typedef llvm::DenseMap<SourceLocation, llvm::Value *> OpenMPDebugLocMapTy;
388 OpenMPDebugLocMapTy OpenMPDebugLocMap;
389 /// Stores debug location and ThreadID for the function.
390 struct DebugLocThreadIdTy {
391 llvm::Value *DebugLoc;
392 llvm::Value *ThreadID;
393 /// Insert point for the service instructions.
394 llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr;
395 };
396 /// Map of local debug location, ThreadId and functions.
397 typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy>
398 OpenMPLocThreadIDMapTy;
399 OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap;
400 /// Map of UDRs and corresponding combiner/initializer.
401 typedef llvm::DenseMap<const OMPDeclareReductionDecl *,
402 std::pair<llvm::Function *, llvm::Function *>>
403 UDRMapTy;
404 UDRMapTy UDRMap;
405 /// Map of functions and locally defined UDRs.
406 typedef llvm::DenseMap<llvm::Function *,
407 SmallVector<const OMPDeclareReductionDecl *, 4>>
408 FunctionUDRMapTy;
409 FunctionUDRMapTy FunctionUDRMap;
410 /// Map from the user-defined mapper declaration to its corresponding
411 /// functions.
412 llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap;
413 /// Map of functions and their local user-defined mappers.
414 using FunctionUDMMapTy =
415 llvm::DenseMap<llvm::Function *,
416 SmallVector<const OMPDeclareMapperDecl *, 4>>;
417 FunctionUDMMapTy FunctionUDMMap;
418 /// Maps local variables marked as lastprivate conditional to their internal
419 /// types.
420 llvm::DenseMap<llvm::Function *,
421 llvm::DenseMap<CanonicalDeclPtr<const Decl>,
422 std::tuple<QualType, const FieldDecl *,
423 const FieldDecl *, LValue>>>
424 LastprivateConditionalToTypes;
425 /// Maps function to the position of the untied task locals stack.
426 llvm::DenseMap<llvm::Function *, unsigned> FunctionToUntiedTaskStackMap;
427 /// Type kmp_critical_name, originally defined as typedef kmp_int32
428 /// kmp_critical_name[8];
429 llvm::ArrayType *KmpCriticalNameTy;
430 /// An ordered map of auto-generated variables to their unique names.
431 /// It stores variables with the following names: 1) ".gomp_critical_user_" +
432 /// <critical_section_name> + ".var" for "omp critical" directives; 2)
433 /// <mangled_name_for_global_var> + ".cache." for cache for threadprivate
434 /// variables.
435 llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>,
436 llvm::BumpPtrAllocator> InternalVars;
437 /// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);
438 llvm::Type *KmpRoutineEntryPtrTy = nullptr;
439 QualType KmpRoutineEntryPtrQTy;
440 /// Type typedef struct kmp_task {
441 /// void * shareds; /**< pointer to block of pointers to
442 /// shared vars */
443 /// kmp_routine_entry_t routine; /**< pointer to routine to call for
444 /// executing task */
445 /// kmp_int32 part_id; /**< part id for the task */
446 /// kmp_routine_entry_t destructors; /* pointer to function to invoke
447 /// deconstructors of firstprivate C++ objects */
448 /// } kmp_task_t;
449 QualType KmpTaskTQTy;
450 /// Saved kmp_task_t for task directive.
451 QualType SavedKmpTaskTQTy;
452 /// Saved kmp_task_t for taskloop-based directive.
453 QualType SavedKmpTaskloopTQTy;
454 /// Type typedef struct kmp_depend_info {
455 /// kmp_intptr_t base_addr;
456 /// size_t len;
457 /// struct {
458 /// bool in:1;
459 /// bool out:1;
460 /// } flags;
461 /// } kmp_depend_info_t;
462 QualType KmpDependInfoTy;
463 /// Type typedef struct kmp_task_affinity_info {
464 /// kmp_intptr_t base_addr;
465 /// size_t len;
466 /// struct {
467 /// bool flag1 : 1;
468 /// bool flag2 : 1;
469 /// kmp_int32 reserved : 30;
470 /// } flags;
471 /// } kmp_task_affinity_info_t;
472 QualType KmpTaskAffinityInfoTy;
473 /// struct kmp_dim { // loop bounds info casted to kmp_int64
474 /// kmp_int64 lo; // lower
475 /// kmp_int64 up; // upper
476 /// kmp_int64 st; // stride
477 /// };
478 QualType KmpDimTy;
479
480 bool ShouldMarkAsGlobal = true;
481 /// List of the emitted declarations.
482 llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls;
483 /// List of the global variables with their addresses that should not be
484 /// emitted for the target.
485 llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables;
486
487 /// List of variables that can become declare target implicitly and, thus,
488 /// must be emitted.
489 llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;
490
491 using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>;
492 /// Stack for list of declarations in current context marked as nontemporal.
493 /// The set is the union of all current stack elements.
494 llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack;
495
496 using UntiedLocalVarsAddressesMap =
497 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
498 std::pair<Address, Address>>;
499 llvm::SmallVector<UntiedLocalVarsAddressesMap, 4> UntiedLocalVarsStack;
500
501 /// Stack for list of addresses of declarations in current context marked as
502 /// lastprivate conditional. The set is the union of all current stack
503 /// elements.
504 llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack;
505
506 /// Flag for keeping track of weather a requires unified_shared_memory
507 /// directive is present.
508 bool HasRequiresUnifiedSharedMemory = false;
509
510 /// Atomic ordering from the omp requires directive.
511 llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
512
513 /// Flag for keeping track of weather a target region has been emitted.
514 bool HasEmittedTargetRegion = false;
515
516 /// Flag for keeping track of weather a device routine has been emitted.
517 /// Device routines are specific to the
518 bool HasEmittedDeclareTargetRegion = false;
519
520 /// Start scanning from statement \a S and emit all target regions
521 /// found along the way.
522 /// \param S Starting statement.
523 /// \param ParentName Name of the function declaration that is being scanned.
524 void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName);
525
526 /// Build type kmp_routine_entry_t (if not built yet).
527 void emitKmpRoutineEntryT(QualType KmpInt32Ty);
528
529 /// If the specified mangled name is not in the module, create and
530 /// return threadprivate cache object. This object is a pointer's worth of
531 /// storage that's reserved for use by the OpenMP runtime.
532 /// \param VD Threadprivate variable.
533 /// \return Cache variable for the specified threadprivate.
534 llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD);
535
536 /// Set of threadprivate variables with the generated initializer.
537 llvm::StringSet<> ThreadPrivateWithDefinition;
538
539 /// Set of declare target variables with the generated initializer.
540 llvm::StringSet<> DeclareTargetWithDefinition;
541
542 /// Emits initialization code for the threadprivate variables.
543 /// \param VDAddr Address of the global variable \a VD.
544 /// \param Ctor Pointer to a global init function for \a VD.
545 /// \param CopyCtor Pointer to a global copy function for \a VD.
546 /// \param Dtor Pointer to a global destructor function for \a VD.
547 /// \param Loc Location of threadprivate declaration.
548 void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr,
549 llvm::Value *Ctor, llvm::Value *CopyCtor,
550 llvm::Value *Dtor, SourceLocation Loc);
551
552 struct TaskResultTy {
553 llvm::Value *NewTask = nullptr;
554 llvm::Function *TaskEntry = nullptr;
555 llvm::Value *NewTaskNewTaskTTy = nullptr;
556 LValue TDBase;
557 const RecordDecl *KmpTaskTQTyRD = nullptr;
558 llvm::Value *TaskDupFn = nullptr;
559 };
560 /// Emit task region for the task directive. The task region is emitted in
561 /// several steps:
562 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
563 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
564 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
565 /// function:
566 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
567 /// TaskFunction(gtid, tt->part_id, tt->shareds);
568 /// return 0;
569 /// }
570 /// 2. Copy a list of shared variables to field shareds of the resulting
571 /// structure kmp_task_t returned by the previous call (if any).
572 /// 3. Copy a pointer to destructions function to field destructions of the
573 /// resulting structure kmp_task_t.
574 /// \param D Current task directive.
575 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
576 /// /*part_id*/, captured_struct */*__context*/);
577 /// \param SharedsTy A type which contains references the shared variables.
578 /// \param Shareds Context with the list of shared variables from the \p
579 /// TaskFunction.
580 /// \param Data Additional data for task generation like tiednsee, final
581 /// state, list of privates etc.
582 TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
583 const OMPExecutableDirective &D,
584 llvm::Function *TaskFunction, QualType SharedsTy,
585 Address Shareds, const OMPTaskDataTy &Data);
586
587 /// Emit update for lastprivate conditional data.
588 void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,
589 StringRef UniqueDeclName, LValue LVal,
590 SourceLocation Loc);
591
592 /// Returns the number of the elements and the address of the depobj
593 /// dependency array.
594 /// \return Number of elements in depobj array and the pointer to the array of
595 /// dependencies.
596 std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF,
597 LValue DepobjLVal,
598 SourceLocation Loc);
599
600 SmallVector<llvm::Value *, 4>
601 emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
602 const OMPTaskDataTy::DependData &Data);
603
604 void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
605 LValue PosLVal, const OMPTaskDataTy::DependData &Data,
606 Address DependenciesArray);
607
608 /// Keep track of VTable Declarations so we don't register duplicate VTable.
609 llvm::SmallDenseMap<CXXRecordDecl *, const VarDecl *> VTableDeclMap;
610
611public:
612 explicit CGOpenMPRuntime(CodeGenModule &CGM);
613 virtual ~CGOpenMPRuntime() {}
614 virtual void clear();
615
616 /// Emits object of ident_t type with info for source location.
617 /// \param Flags Flags for OpenMP location.
618 /// \param EmitLoc emit source location with debug-info is off.
619 ///
620 llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc,
621 unsigned Flags = 0, bool EmitLoc = false);
622
623 /// Emit the number of teams for a target directive. Inspect the num_teams
624 /// clause associated with a teams construct combined or closely nested
625 /// with the target directive.
626 ///
627 /// Emit a team of size one for directives such as 'target parallel' that
628 /// have no associated teams construct.
629 ///
630 /// Otherwise, return nullptr.
631 const Expr *getNumTeamsExprForTargetDirective(CodeGenFunction &CGF,
632 const OMPExecutableDirective &D,
633 int32_t &MinTeamsVal,
634 int32_t &MaxTeamsVal);
635 llvm::Value *emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
636 const OMPExecutableDirective &D);
637
638 /// Check for a number of threads upper bound constant value (stored in \p
639 /// UpperBound), or expression (returned). If the value is conditional (via an
640 /// if-clause), store the condition in \p CondExpr. Similarly, a potential
641 /// thread limit expression is stored in \p ThreadLimitExpr. If \p
642 /// UpperBoundOnly is true, no expression evaluation is perfomed.
643 const Expr *getNumThreadsExprForTargetDirective(
644 CodeGenFunction &CGF, const OMPExecutableDirective &D,
645 int32_t &UpperBound, bool UpperBoundOnly,
646 llvm::Value **CondExpr = nullptr, const Expr **ThreadLimitExpr = nullptr);
647
648 /// Emit an expression that denotes the number of threads a target region
649 /// shall use. Will generate "i32 0" to allow the runtime to choose.
650 llvm::Value *
651 emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
652 const OMPExecutableDirective &D);
653
654 /// Return the trip count of loops associated with constructs / 'target teams
655 /// distribute' and 'teams distribute parallel for'. \param SizeEmitter Emits
656 /// the int64 value for the number of iterations of the associated loop.
657 llvm::Value *emitTargetNumIterationsCall(
658 CodeGenFunction &CGF, const OMPExecutableDirective &D,
659 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
660 const OMPLoopDirective &D)>
661 SizeEmitter);
662
663 /// Returns true if the current target is a GPU.
664 virtual bool isGPU() const { return false; }
665
666 /// Check if the variable length declaration is delayed:
667 virtual bool isDelayedVariableLengthDecl(CodeGenFunction &CGF,
668 const VarDecl *VD) const {
669 return false;
670 };
671
672 /// Get call to __kmpc_alloc_shared
673 virtual std::pair<llvm::Value *, llvm::Value *>
674 getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) {
675 llvm_unreachable("not implemented");
676 }
677
678 /// Get call to __kmpc_free_shared
679 virtual void getKmpcFreeShared(
680 CodeGenFunction &CGF,
681 const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) {
682 llvm_unreachable("not implemented");
683 }
684
685 /// Emits code for OpenMP 'if' clause using specified \a CodeGen
686 /// function. Here is the logic:
687 /// if (Cond) {
688 /// ThenGen();
689 /// } else {
690 /// ElseGen();
691 /// }
692 void emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
693 const RegionCodeGenTy &ThenGen,
694 const RegionCodeGenTy &ElseGen);
695
696 /// Checks if the \p Body is the \a CompoundStmt and returns its child
697 /// statement iff there is only one that is not evaluatable at the compile
698 /// time.
699 static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body);
700
701 /// Get the platform-specific name separator.
702 std::string getName(ArrayRef<StringRef> Parts) const;
703
704 /// Emit code for the specified user defined reduction construct.
705 virtual void emitUserDefinedReduction(CodeGenFunction *CGF,
706 const OMPDeclareReductionDecl *D);
707 /// Get combiner/initializer for the specified user-defined reduction, if any.
708 virtual std::pair<llvm::Function *, llvm::Function *>
709 getUserDefinedReduction(const OMPDeclareReductionDecl *D);
710
711 /// Emit the function for the user defined mapper construct.
712 void emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
713 CodeGenFunction *CGF = nullptr);
714 /// Get the function for the specified user-defined mapper. If it does not
715 /// exist, create one.
716 llvm::Function *
717 getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D);
718
719 /// Emits outlined function for the specified OpenMP parallel directive
720 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
721 /// kmp_int32 BoundID, struct context_vars*).
722 /// \param CGF Reference to current CodeGenFunction.
723 /// \param D OpenMP directive.
724 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
725 /// \param InnermostKind Kind of innermost directive (for simple directives it
726 /// is a directive itself, for combined - its innermost directive).
727 /// \param CodeGen Code generation sequence for the \a D directive.
728 virtual llvm::Function *emitParallelOutlinedFunction(
729 CodeGenFunction &CGF, const OMPExecutableDirective &D,
730 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
731 const RegionCodeGenTy &CodeGen);
732
733 /// Emits outlined function for the specified OpenMP teams directive
734 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
735 /// kmp_int32 BoundID, struct context_vars*).
736 /// \param CGF Reference to current CodeGenFunction.
737 /// \param D OpenMP directive.
738 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
739 /// \param InnermostKind Kind of innermost directive (for simple directives it
740 /// is a directive itself, for combined - its innermost directive).
741 /// \param CodeGen Code generation sequence for the \a D directive.
742 virtual llvm::Function *emitTeamsOutlinedFunction(
743 CodeGenFunction &CGF, const OMPExecutableDirective &D,
744 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
745 const RegionCodeGenTy &CodeGen);
746
747 /// Emits outlined function for the OpenMP task directive \a D. This
748 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
749 /// TaskT).
750 /// \param D OpenMP directive.
751 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
752 /// \param PartIDVar Variable for partition id in the current OpenMP untied
753 /// task region.
754 /// \param TaskTVar Variable for task_t argument.
755 /// \param InnermostKind Kind of innermost directive (for simple directives it
756 /// is a directive itself, for combined - its innermost directive).
757 /// \param CodeGen Code generation sequence for the \a D directive.
758 /// \param Tied true if task is generated for tied task, false otherwise.
759 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
760 /// tasks.
761 ///
762 virtual llvm::Function *emitTaskOutlinedFunction(
763 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
764 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
765 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
766 bool Tied, unsigned &NumberOfParts);
767
768 /// Cleans up references to the objects in finished function.
769 ///
770 virtual void functionFinished(CodeGenFunction &CGF);
771
772 /// Emits code for parallel or serial call of the \a OutlinedFn with
773 /// variables captured in a record which address is stored in \a
774 /// CapturedStruct.
775 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
776 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
777 /// \param CapturedVars A pointer to the record with the references to
778 /// variables used in \a OutlinedFn function.
779 /// \param IfCond Condition in the associated 'if' clause, if it was
780 /// specified, nullptr otherwise.
781 /// \param NumThreads The value corresponding to the num_threads clause, if
782 /// any, or nullptr.
783 /// \param NumThreadsModifier The modifier of the num_threads clause, if
784 /// any, ignored otherwise.
785 /// \param Severity The severity corresponding to the num_threads clause, if
786 /// any, ignored otherwise.
787 /// \param Message The message string corresponding to the num_threads clause,
788 /// if any, or nullptr.
789 ///
790 virtual void
791 emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
792 llvm::Function *OutlinedFn,
793 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
794 llvm::Value *NumThreads,
795 OpenMPNumThreadsClauseModifier NumThreadsModifier =
796 OMPC_NUMTHREADS_unknown,
797 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
798 const Expr *Message = nullptr);
799
800 /// Emits a critical region.
801 /// \param CriticalName Name of the critical region.
802 /// \param CriticalOpGen Generator for the statement associated with the given
803 /// critical region.
804 /// \param Hint Value of the 'hint' clause (optional).
805 virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
806 const RegionCodeGenTy &CriticalOpGen,
807 SourceLocation Loc,
808 const Expr *Hint = nullptr);
809
810 /// Emits a master region.
811 /// \param MasterOpGen Generator for the statement associated with the given
812 /// master region.
813 virtual void emitMasterRegion(CodeGenFunction &CGF,
814 const RegionCodeGenTy &MasterOpGen,
815 SourceLocation Loc);
816
817 /// Emits a masked region.
818 /// \param MaskedOpGen Generator for the statement associated with the given
819 /// masked region.
820 virtual void emitMaskedRegion(CodeGenFunction &CGF,
821 const RegionCodeGenTy &MaskedOpGen,
822 SourceLocation Loc,
823 const Expr *Filter = nullptr);
824
825 /// Emits code for a taskyield directive.
826 virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);
827
828 /// Emit __kmpc_error call for error directive
829 /// extern void __kmpc_error(ident_t *loc, int severity, const char *message);
830 virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME,
831 bool IsFatal);
832
833 /// Emit a taskgroup region.
834 /// \param TaskgroupOpGen Generator for the statement associated with the
835 /// given taskgroup region.
836 virtual void emitTaskgroupRegion(CodeGenFunction &CGF,
837 const RegionCodeGenTy &TaskgroupOpGen,
838 SourceLocation Loc);
839
840 /// Emits a single region.
841 /// \param SingleOpGen Generator for the statement associated with the given
842 /// single region.
843 virtual void emitSingleRegion(CodeGenFunction &CGF,
844 const RegionCodeGenTy &SingleOpGen,
845 SourceLocation Loc,
846 ArrayRef<const Expr *> CopyprivateVars,
847 ArrayRef<const Expr *> DestExprs,
848 ArrayRef<const Expr *> SrcExprs,
849 ArrayRef<const Expr *> AssignmentOps);
850
851 /// Emit an ordered region.
852 /// \param OrderedOpGen Generator for the statement associated with the given
853 /// ordered region.
854 virtual void emitOrderedRegion(CodeGenFunction &CGF,
855 const RegionCodeGenTy &OrderedOpGen,
856 SourceLocation Loc, bool IsThreads);
857
858 /// Emit an implicit/explicit barrier for OpenMP threads.
859 /// \param Kind Directive for which this implicit barrier call must be
860 /// generated. Must be OMPD_barrier for explicit barrier generation.
861 /// \param EmitChecks true if need to emit checks for cancellation barriers.
862 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
863 /// runtime class decides which one to emit (simple or with cancellation
864 /// checks).
865 ///
866 virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
867 OpenMPDirectiveKind Kind,
868 bool EmitChecks = true,
869 bool ForceSimpleCall = false);
870
871 /// Check if the specified \a ScheduleKind is static non-chunked.
872 /// This kind of worksharing directive is emitted without outer loop.
873 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
874 /// \param Chunked True if chunk is specified in the clause.
875 ///
876 virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
877 bool Chunked) const;
878
879 /// Check if the specified \a ScheduleKind is static non-chunked.
880 /// This kind of distribute directive is emitted without outer loop.
881 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
882 /// \param Chunked True if chunk is specified in the clause.
883 ///
884 virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind,
885 bool Chunked) const;
886
887 /// Check if the specified \a ScheduleKind is static chunked.
888 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
889 /// \param Chunked True if chunk is specified in the clause.
890 ///
891 virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
892 bool Chunked) const;
893
894 /// Check if the specified \a ScheduleKind is static non-chunked.
895 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
896 /// \param Chunked True if chunk is specified in the clause.
897 ///
898 virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind,
899 bool Chunked) const;
900
901 /// Check if the specified \a ScheduleKind is dynamic.
902 /// This kind of worksharing directive is emitted without outer loop.
903 /// \param ScheduleKind Schedule Kind specified in the 'schedule' clause.
904 ///
905 virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const;
906
907 /// struct with the values to be passed to the dispatch runtime function
908 struct DispatchRTInput {
909 /// Loop lower bound
910 llvm::Value *LB = nullptr;
911 /// Loop upper bound
912 llvm::Value *UB = nullptr;
913 /// Chunk size specified using 'schedule' clause (nullptr if chunk
914 /// was not specified)
915 llvm::Value *Chunk = nullptr;
916 DispatchRTInput() = default;
917 DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk)
918 : LB(LB), UB(UB), Chunk(Chunk) {}
919 };
920
921 /// Call the appropriate runtime routine to initialize it before start
922 /// of loop.
923
924 /// This is used for non static scheduled types and when the ordered
925 /// clause is present on the loop construct.
926 /// Depending on the loop schedule, it is necessary to call some runtime
927 /// routine before start of the OpenMP loop to get the loop upper / lower
928 /// bounds \a LB and \a UB and stride \a ST.
929 ///
930 /// \param CGF Reference to current CodeGenFunction.
931 /// \param Loc Clang source location.
932 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
933 /// \param IVSize Size of the iteration variable in bits.
934 /// \param IVSigned Sign of the iteration variable.
935 /// \param Ordered true if loop is ordered, false otherwise.
936 /// \param DispatchValues struct containing llvm values for lower bound, upper
937 /// bound, and chunk expression.
938 /// For the default (nullptr) value, the chunk 1 will be used.
939 ///
940 virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
941 const OpenMPScheduleTy &ScheduleKind,
942 unsigned IVSize, bool IVSigned, bool Ordered,
943 const DispatchRTInput &DispatchValues);
944
945 /// This is used for non static scheduled types and when the ordered
946 /// clause is present on the loop construct.
947 ///
948 /// \param CGF Reference to current CodeGenFunction.
949 /// \param Loc Clang source location.
950 ///
951 virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc);
952
953 /// Struct with the values to be passed to the static runtime function
954 struct StaticRTInput {
955 /// Size of the iteration variable in bits.
956 unsigned IVSize = 0;
957 /// Sign of the iteration variable.
958 bool IVSigned = false;
959 /// true if loop is ordered, false otherwise.
960 bool Ordered = false;
961 /// Address of the output variable in which the flag of the last iteration
962 /// is returned.
963 Address IL = Address::invalid();
964 /// Address of the output variable in which the lower iteration number is
965 /// returned.
966 Address LB = Address::invalid();
967 /// Address of the output variable in which the upper iteration number is
968 /// returned.
969 Address UB = Address::invalid();
970 /// Address of the output variable in which the stride value is returned
971 /// necessary to generated the static_chunked scheduled loop.
972 Address ST = Address::invalid();
973 /// Value of the chunk for the static_chunked scheduled loop. For the
974 /// default (nullptr) value, the chunk 1 will be used.
975 llvm::Value *Chunk = nullptr;
976 StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL,
977 Address LB, Address UB, Address ST,
978 llvm::Value *Chunk = nullptr)
979 : IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB),
980 UB(UB), ST(ST), Chunk(Chunk) {}
981 };
982 /// Call the appropriate runtime routine to initialize it before start
983 /// of loop.
984 ///
985 /// This is used only in case of static schedule, when the user did not
986 /// specify a ordered clause on the loop construct.
987 /// Depending on the loop schedule, it is necessary to call some runtime
988 /// routine before start of the OpenMP loop to get the loop upper / lower
989 /// bounds LB and UB and stride ST.
990 ///
991 /// \param CGF Reference to current CodeGenFunction.
992 /// \param Loc Clang source location.
993 /// \param DKind Kind of the directive.
994 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
995 /// \param Values Input arguments for the construct.
996 ///
997 virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
998 OpenMPDirectiveKind DKind,
999 const OpenMPScheduleTy &ScheduleKind,
1000 const StaticRTInput &Values);
1001
1002 ///
1003 /// \param CGF Reference to current CodeGenFunction.
1004 /// \param Loc Clang source location.
1005 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1006 /// \param Values Input arguments for the construct.
1007 ///
1008 virtual void emitDistributeStaticInit(CodeGenFunction &CGF,
1009 SourceLocation Loc,
1010 OpenMPDistScheduleClauseKind SchedKind,
1011 const StaticRTInput &Values);
1012
1013 /// Call the appropriate runtime routine to notify that we finished
1014 /// iteration of the ordered loop with the dynamic scheduling.
1015 ///
1016 /// \param CGF Reference to current CodeGenFunction.
1017 /// \param Loc Clang source location.
1018 /// \param IVSize Size of the iteration variable in bits.
1019 /// \param IVSigned Sign of the iteration variable.
1020 ///
1021 virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF,
1022 SourceLocation Loc, unsigned IVSize,
1023 bool IVSigned);
1024
1025 /// Call the appropriate runtime routine to notify that we finished
1026 /// all the work with current loop.
1027 ///
1028 /// \param CGF Reference to current CodeGenFunction.
1029 /// \param Loc Clang source location.
1030 /// \param DKind Kind of the directive for which the static finish is emitted.
1031 ///
1032 virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1033 OpenMPDirectiveKind DKind);
1034
1035 /// Call __kmpc_dispatch_next(
1036 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1037 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1038 /// kmp_int[32|64] *p_stride);
1039 /// \param IVSize Size of the iteration variable in bits.
1040 /// \param IVSigned Sign of the iteration variable.
1041 /// \param IL Address of the output variable in which the flag of the
1042 /// last iteration is returned.
1043 /// \param LB Address of the output variable in which the lower iteration
1044 /// number is returned.
1045 /// \param UB Address of the output variable in which the upper iteration
1046 /// number is returned.
1047 /// \param ST Address of the output variable in which the stride value is
1048 /// returned.
1049 virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1050 unsigned IVSize, bool IVSigned,
1051 Address IL, Address LB,
1052 Address UB, Address ST);
1053
1054 virtual llvm::Value *emitMessageClause(CodeGenFunction &CGF,
1055 const Expr *Message,
1056 SourceLocation Loc);
1057
1058 virtual llvm::Value *emitSeverityClause(OpenMPSeverityClauseKind Severity,
1059 SourceLocation Loc);
1060
1061 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1062 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1063 /// clause.
1064 /// If the modifier 'strict' is given:
1065 /// Emits call to void __kmpc_push_num_threads_strict(ident_t *loc, kmp_int32
1066 /// global_tid, kmp_int32 num_threads, int severity, const char *message) to
1067 /// generate code for 'num_threads' clause with 'strict' modifier.
1068 /// \param NumThreads An integer value of threads.
1069 virtual void emitNumThreadsClause(
1070 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
1071 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown,
1072 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
1073 SourceLocation SeverityLoc = SourceLocation(),
1074 const Expr *Message = nullptr,
1075 SourceLocation MessageLoc = SourceLocation());
1076
1077 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1078 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1079 virtual void emitProcBindClause(CodeGenFunction &CGF,
1080 llvm::omp::ProcBindKind ProcBind,
1081 SourceLocation Loc);
1082
1083 /// Returns address of the threadprivate variable for the current
1084 /// thread.
1085 /// \param VD Threadprivate variable.
1086 /// \param VDAddr Address of the global variable \a VD.
1087 /// \param Loc Location of the reference to threadprivate var.
1088 /// \return Address of the threadprivate variable for the current thread.
1089 virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1090 const VarDecl *VD, Address VDAddr,
1091 SourceLocation Loc);
1092
1093 /// Returns the address of the variable marked as declare target with link
1094 /// clause OR as declare target with to clause and unified memory.
1095 virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD);
1096
1097 /// Emit a code for initialization of threadprivate variable. It emits
1098 /// a call to runtime library which adds initial value to the newly created
1099 /// threadprivate variable (if it is not constant) and registers destructor
1100 /// for the variable (if any).
1101 /// \param VD Threadprivate variable.
1102 /// \param VDAddr Address of the global variable \a VD.
1103 /// \param Loc Location of threadprivate declaration.
1104 /// \param PerformInit true if initialization expression is not constant.
1105 virtual llvm::Function *
1106 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1107 SourceLocation Loc, bool PerformInit,
1108 CodeGenFunction *CGF = nullptr);
1109
1110 /// Emit code for handling declare target functions in the runtime.
1111 /// \param FD Declare target function.
1112 /// \param Addr Address of the global \a FD.
1113 /// \param PerformInit true if initialization expression is not constant.
1114 virtual void emitDeclareTargetFunction(const FunctionDecl *FD,
1115 llvm::GlobalValue *GV);
1116
1117 /// Register VTable to OpenMP offload entry.
1118 /// \param VTable VTable of the C++ class.
1119 /// \param RD C++ class decl.
1120 virtual void registerVTableOffloadEntry(llvm::GlobalVariable *VTable,
1121 const VarDecl *VD);
1122 /// Emit code for registering vtable by scanning through map clause
1123 /// in OpenMP target region.
1124 /// \param D OpenMP target directive.
1125 virtual void registerVTable(const OMPExecutableDirective &D);
1126
1127 /// Emit and register VTable for the C++ class in OpenMP offload entry.
1128 /// \param CXXRecord C++ class decl.
1129 /// \param VD Variable decl which holds VTable.
1130 virtual void emitAndRegisterVTable(CodeGenModule &CGM,
1131 CXXRecordDecl *CXXRecord,
1132 const VarDecl *VD);
1133
1134 /// Creates artificial threadprivate variable with name \p Name and type \p
1135 /// VarType.
1136 /// \param VarType Type of the artificial threadprivate variable.
1137 /// \param Name Name of the artificial threadprivate variable.
1138 virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1139 QualType VarType,
1140 StringRef Name);
1141
1142 /// Emit flush of the variables specified in 'omp flush' directive.
1143 /// \param Vars List of variables to flush.
1144 virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1145 SourceLocation Loc, llvm::AtomicOrdering AO);
1146
1147 /// Emit task region for the task directive. The task region is
1148 /// emitted in several steps:
1149 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1150 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1151 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1152 /// function:
1153 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1154 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1155 /// return 0;
1156 /// }
1157 /// 2. Copy a list of shared variables to field shareds of the resulting
1158 /// structure kmp_task_t returned by the previous call (if any).
1159 /// 3. Copy a pointer to destructions function to field destructions of the
1160 /// resulting structure kmp_task_t.
1161 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1162 /// kmp_task_t *new_task), where new_task is a resulting structure from
1163 /// previous items.
1164 /// \param D Current task directive.
1165 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1166 /// /*part_id*/, captured_struct */*__context*/);
1167 /// \param SharedsTy A type which contains references the shared variables.
1168 /// \param Shareds Context with the list of shared variables from the \p
1169 /// TaskFunction.
1170 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1171 /// otherwise.
1172 /// \param Data Additional data for task generation like tiednsee, final
1173 /// state, list of privates etc.
1174 virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1175 const OMPExecutableDirective &D,
1176 llvm::Function *TaskFunction, QualType SharedsTy,
1177 Address Shareds, const Expr *IfCond,
1178 const OMPTaskDataTy &Data);
1179
1180 /// Emit task region for the taskloop directive. The taskloop region is
1181 /// emitted in several steps:
1182 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1183 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1184 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1185 /// function:
1186 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1187 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1188 /// return 0;
1189 /// }
1190 /// 2. Copy a list of shared variables to field shareds of the resulting
1191 /// structure kmp_task_t returned by the previous call (if any).
1192 /// 3. Copy a pointer to destructions function to field destructions of the
1193 /// resulting structure kmp_task_t.
1194 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
1195 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
1196 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
1197 /// is a resulting structure from
1198 /// previous items.
1199 /// \param D Current task directive.
1200 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1201 /// /*part_id*/, captured_struct */*__context*/);
1202 /// \param SharedsTy A type which contains references the shared variables.
1203 /// \param Shareds Context with the list of shared variables from the \p
1204 /// TaskFunction.
1205 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1206 /// otherwise.
1207 /// \param Data Additional data for task generation like tiednsee, final
1208 /// state, list of privates etc.
1209 virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
1210 const OMPLoopDirective &D,
1211 llvm::Function *TaskFunction,
1212 QualType SharedsTy, Address Shareds,
1213 const Expr *IfCond, const OMPTaskDataTy &Data);
1214
1215 /// Emit code for the directive that does not require outlining.
1216 ///
1217 /// \param InnermostKind Kind of innermost directive (for simple directives it
1218 /// is a directive itself, for combined - its innermost directive).
1219 /// \param CodeGen Code generation sequence for the \a D directive.
1220 /// \param HasCancel true if region has inner cancel directive, false
1221 /// otherwise.
1222 virtual void emitInlinedDirective(CodeGenFunction &CGF,
1223 OpenMPDirectiveKind InnermostKind,
1224 const RegionCodeGenTy &CodeGen,
1225 bool HasCancel = false);
1226
1227 /// Emits reduction function.
1228 /// \param ReducerName Name of the function calling the reduction.
1229 /// \param ArgsElemType Array type containing pointers to reduction variables.
1230 /// \param Privates List of private copies for original reduction arguments.
1231 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1232 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1233 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1234 /// or 'operator binop(LHS, RHS)'.
1235 llvm::Function *emitReductionFunction(
1236 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
1237 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
1238 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps);
1239
1240 /// Emits single reduction combiner
1241 void emitSingleReductionCombiner(CodeGenFunction &CGF,
1242 const Expr *ReductionOp,
1243 const Expr *PrivateRef,
1244 const DeclRefExpr *LHS,
1245 const DeclRefExpr *RHS);
1246
1247 struct ReductionOptionsTy {
1248 bool WithNowait;
1249 bool SimpleReduction;
1250 llvm::SmallVector<bool, 8> IsPrivateVarReduction;
1251 OpenMPDirectiveKind ReductionKind;
1252 };
1253
1254 /// Emits code for private variable reduction
1255 /// \param Privates List of private copies for original reduction arguments.
1256 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1257 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1258 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1259 /// or 'operator binop(LHS, RHS)'.
1260 void emitPrivateReduction(CodeGenFunction &CGF, SourceLocation Loc,
1261 const Expr *Privates, const Expr *LHSExprs,
1262 const Expr *RHSExprs, const Expr *ReductionOps);
1263
1264 /// Emit a code for reduction clause. Next code should be emitted for
1265 /// reduction:
1266 /// \code
1267 ///
1268 /// static kmp_critical_name lock = { 0 };
1269 ///
1270 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
1271 /// ...
1272 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
1273 /// ...
1274 /// }
1275 ///
1276 /// ...
1277 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
1278 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
1279 /// RedList, reduce_func, &<lock>)) {
1280 /// case 1:
1281 /// ...
1282 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
1283 /// ...
1284 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
1285 /// break;
1286 /// case 2:
1287 /// ...
1288 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
1289 /// ...
1290 /// break;
1291 /// default:;
1292 /// }
1293 /// \endcode
1294 ///
1295 /// \param Privates List of private copies for original reduction arguments.
1296 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1297 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1298 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1299 /// or 'operator binop(LHS, RHS)'.
1300 /// \param Options List of options for reduction codegen:
1301 /// WithNowait true if parent directive has also nowait clause, false
1302 /// otherwise.
1303 /// SimpleReduction Emit reduction operation only. Used for omp simd
1304 /// directive on the host.
1305 /// ReductionKind The kind of reduction to perform.
1306 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
1307 ArrayRef<const Expr *> Privates,
1308 ArrayRef<const Expr *> LHSExprs,
1309 ArrayRef<const Expr *> RHSExprs,
1310 ArrayRef<const Expr *> ReductionOps,
1311 ReductionOptionsTy Options);
1312
1313 /// Emit a code for initialization of task reduction clause. Next code
1314 /// should be emitted for reduction:
1315 /// \code
1316 ///
1317 /// _taskred_item_t red_data[n];
1318 /// ...
1319 /// red_data[i].shar = &shareds[i];
1320 /// red_data[i].orig = &origs[i];
1321 /// red_data[i].size = sizeof(origs[i]);
1322 /// red_data[i].f_init = (void*)RedInit<i>;
1323 /// red_data[i].f_fini = (void*)RedDest<i>;
1324 /// red_data[i].f_comb = (void*)RedOp<i>;
1325 /// red_data[i].flags = <Flag_i>;
1326 /// ...
1327 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
1328 /// \endcode
1329 /// For reduction clause with task modifier it emits the next call:
1330 /// \code
1331 ///
1332 /// _taskred_item_t red_data[n];
1333 /// ...
1334 /// red_data[i].shar = &shareds[i];
1335 /// red_data[i].orig = &origs[i];
1336 /// red_data[i].size = sizeof(origs[i]);
1337 /// red_data[i].f_init = (void*)RedInit<i>;
1338 /// red_data[i].f_fini = (void*)RedDest<i>;
1339 /// red_data[i].f_comb = (void*)RedOp<i>;
1340 /// red_data[i].flags = <Flag_i>;
1341 /// ...
1342 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
1343 /// red_data);
1344 /// \endcode
1345 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
1346 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
1347 /// \param Data Additional data for task generation like tiedness, final
1348 /// state, list of privates, reductions etc.
1349 virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF,
1350 SourceLocation Loc,
1351 ArrayRef<const Expr *> LHSExprs,
1352 ArrayRef<const Expr *> RHSExprs,
1353 const OMPTaskDataTy &Data);
1354
1355 /// Emits the following code for reduction clause with task modifier:
1356 /// \code
1357 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
1358 /// \endcode
1359 virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
1360 bool IsWorksharingReduction);
1361
1362 /// Required to resolve existing problems in the runtime. Emits threadprivate
1363 /// variables to store the size of the VLAs/array sections for
1364 /// initializer/combiner/finalizer functions.
1365 /// \param RCG Allows to reuse an existing data for the reductions.
1366 /// \param N Reduction item for which fixups must be emitted.
1367 virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
1368 ReductionCodeGen &RCG, unsigned N);
1369
1370 /// Get the address of `void *` type of the privatue copy of the reduction
1371 /// item specified by the \p SharedLVal.
1372 /// \param ReductionsPtr Pointer to the reduction data returned by the
1373 /// emitTaskReductionInit function.
1374 /// \param SharedLVal Address of the original reduction item.
1375 virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
1376 llvm::Value *ReductionsPtr,
1377 LValue SharedLVal);
1378
1379 /// Emit code for 'taskwait' directive.
1380 virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
1381 const OMPTaskDataTy &Data);
1382
1383 /// Emit code for 'cancellation point' construct.
1384 /// \param CancelRegion Region kind for which the cancellation point must be
1385 /// emitted.
1386 ///
1387 virtual void emitCancellationPointCall(CodeGenFunction &CGF,
1388 SourceLocation Loc,
1389 OpenMPDirectiveKind CancelRegion);
1390
1391 /// Emit code for 'cancel' construct.
1392 /// \param IfCond Condition in the associated 'if' clause, if it was
1393 /// specified, nullptr otherwise.
1394 /// \param CancelRegion Region kind for which the cancel must be emitted.
1395 ///
1396 virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
1397 const Expr *IfCond,
1398 OpenMPDirectiveKind CancelRegion);
1399
1400 /// Emit outilined function for 'target' directive.
1401 /// \param D Directive to emit.
1402 /// \param ParentName Name of the function that encloses the target region.
1403 /// \param OutlinedFn Outlined function value to be defined by this call.
1404 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
1405 /// \param IsOffloadEntry True if the outlined function is an offload entry.
1406 /// \param CodeGen Code generation sequence for the \a D directive.
1407 /// An outlined function may not be an entry if, e.g. the if clause always
1408 /// evaluates to false.
1409 virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
1410 StringRef ParentName,
1411 llvm::Function *&OutlinedFn,
1412 llvm::Constant *&OutlinedFnID,
1413 bool IsOffloadEntry,
1414 const RegionCodeGenTy &CodeGen);
1415
1416 /// Emit the target offloading code associated with \a D. The emitted
1417 /// code attempts offloading the execution to the device, an the event of
1418 /// a failure it executes the host version outlined in \a OutlinedFn.
1419 /// \param D Directive to emit.
1420 /// \param OutlinedFn Host version of the code to be offloaded.
1421 /// \param OutlinedFnID ID of host version of the code to be offloaded.
1422 /// \param IfCond Expression evaluated in if clause associated with the target
1423 /// directive, or null if no if clause is used.
1424 /// \param Device Expression evaluated in device clause associated with the
1425 /// target directive, or null if no device clause is used and device modifier.
1426 /// \param SizeEmitter Callback to emit number of iterations for loop-based
1427 /// directives.
1428 virtual void emitTargetCall(
1429 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1430 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
1431 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
1432 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
1433 const OMPLoopDirective &D)>
1434 SizeEmitter);
1435
1436 /// Emit the target regions enclosed in \a GD function definition or
1437 /// the function itself in case it is a valid device function. Returns true if
1438 /// \a GD was dealt with successfully.
1439 /// \param GD Function to scan.
1440 virtual bool emitTargetFunctions(GlobalDecl GD);
1441
1442 /// Emit the global variable if it is a valid device global variable.
1443 /// Returns true if \a GD was dealt with successfully.
1444 /// \param GD Variable declaration to emit.
1445 virtual bool emitTargetGlobalVariable(GlobalDecl GD);
1446
1447 /// Checks if the provided global decl \a GD is a declare target variable and
1448 /// registers it when emitting code for the host.
1449 virtual void registerTargetGlobalVariable(const VarDecl *VD,
1450 llvm::Constant *Addr);
1451
1452 /// Emit the global \a GD if it is meaningful for the target. Returns
1453 /// if it was emitted successfully.
1454 /// \param GD Global to scan.
1455 virtual bool emitTargetGlobal(GlobalDecl GD);
1456
1457 /// Creates all the offload entries in the current compilation unit
1458 /// along with the associated metadata.
1459 void createOffloadEntriesAndInfoMetadata();
1460
1461 /// Emits code for teams call of the \a OutlinedFn with
1462 /// variables captured in a record which address is stored in \a
1463 /// CapturedStruct.
1464 /// \param OutlinedFn Outlined function to be run by team masters. Type of
1465 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1466 /// \param CapturedVars A pointer to the record with the references to
1467 /// variables used in \a OutlinedFn function.
1468 ///
1469 virtual void emitTeamsCall(CodeGenFunction &CGF,
1470 const OMPExecutableDirective &D,
1471 SourceLocation Loc, llvm::Function *OutlinedFn,
1472 ArrayRef<llvm::Value *> CapturedVars);
1473
1474 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
1475 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
1476 /// for num_teams clause.
1477 /// \param NumTeams An integer expression of teams.
1478 /// \param ThreadLimit An integer expression of threads.
1479 virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
1480 const Expr *ThreadLimit, SourceLocation Loc);
1481
1482 /// Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32
1483 /// global_tid, kmp_int32 thread_limit) to generate code for
1484 /// thread_limit clause on target directive
1485 /// \param ThreadLimit An integer expression of threads.
1486 virtual void emitThreadLimitClause(CodeGenFunction &CGF,
1487 const Expr *ThreadLimit,
1488 SourceLocation Loc);
1489
1490 /// Struct that keeps all the relevant information that should be kept
1491 /// throughout a 'target data' region.
1492 class TargetDataInfo : public llvm::OpenMPIRBuilder::TargetDataInfo {
1493 public:
1494 explicit TargetDataInfo() : llvm::OpenMPIRBuilder::TargetDataInfo() {}
1495 explicit TargetDataInfo(bool RequiresDevicePointerInfo,
1496 bool SeparateBeginEndCalls)
1497 : llvm::OpenMPIRBuilder::TargetDataInfo(RequiresDevicePointerInfo,
1498 SeparateBeginEndCalls) {}
1499 /// Map between the a declaration of a capture and the corresponding new
1500 /// llvm address where the runtime returns the device pointers.
1501 llvm::DenseMap<const ValueDecl *, llvm::Value *> CaptureDeviceAddrMap;
1502 };
1503
1504 /// Emit the target data mapping code associated with \a D.
1505 /// \param D Directive to emit.
1506 /// \param IfCond Expression evaluated in if clause associated with the
1507 /// target directive, or null if no device clause is used.
1508 /// \param Device Expression evaluated in device clause associated with the
1509 /// target directive, or null if no device clause is used.
1510 /// \param Info A record used to store information that needs to be preserved
1511 /// until the region is closed.
1512 virtual void emitTargetDataCalls(CodeGenFunction &CGF,
1513 const OMPExecutableDirective &D,
1514 const Expr *IfCond, const Expr *Device,
1515 const RegionCodeGenTy &CodeGen,
1516 CGOpenMPRuntime::TargetDataInfo &Info);
1517
1518 /// Emit the data mapping/movement code associated with the directive
1519 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
1520 /// \param D Directive to emit.
1521 /// \param IfCond Expression evaluated in if clause associated with the target
1522 /// directive, or null if no if clause is used.
1523 /// \param Device Expression evaluated in device clause associated with the
1524 /// target directive, or null if no device clause is used.
1525 virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
1526 const OMPExecutableDirective &D,
1527 const Expr *IfCond,
1528 const Expr *Device);
1529
1530 /// Marks function \a Fn with properly mangled versions of vector functions.
1531 /// \param FD Function marked as 'declare simd'.
1532 /// \param Fn LLVM function that must be marked with 'declare simd'
1533 /// attributes.
1534 virtual void emitDeclareSimdFunction(const FunctionDecl *FD,
1535 llvm::Function *Fn);
1536
1537 /// Emit initialization for doacross loop nesting support.
1538 /// \param D Loop-based construct used in doacross nesting construct.
1539 virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
1540 ArrayRef<Expr *> NumIterations);
1541
1542 /// Emit code for doacross ordered directive with 'depend' clause.
1543 /// \param C 'depend' clause with 'sink|source' dependency kind.
1544 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1545 const OMPDependClause *C);
1546
1547 /// Emit code for doacross ordered directive with 'doacross' clause.
1548 /// \param C 'doacross' clause with 'sink|source' dependence type.
1549 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1550 const OMPDoacrossClause *C);
1551
1552 /// Translates the native parameter of outlined function if this is required
1553 /// for target.
1554 /// \param FD Field decl from captured record for the parameter.
1555 /// \param NativeParam Parameter itself.
1556 virtual const VarDecl *translateParameter(const FieldDecl *FD,
1557 const VarDecl *NativeParam) const {
1558 return NativeParam;
1559 }
1560
1561 /// Gets the address of the native argument basing on the address of the
1562 /// target-specific parameter.
1563 /// \param NativeParam Parameter itself.
1564 /// \param TargetParam Corresponding target-specific parameter.
1565 virtual Address getParameterAddress(CodeGenFunction &CGF,
1566 const VarDecl *NativeParam,
1567 const VarDecl *TargetParam) const;
1568
1569 /// Choose default schedule type and chunk value for the
1570 /// dist_schedule clause.
1571 virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
1572 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
1573 llvm::Value *&Chunk) const {}
1574
1575 /// Choose default schedule type and chunk value for the
1576 /// schedule clause.
1577 virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
1578 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
1579 const Expr *&ChunkExpr) const;
1580
1581 /// Emits call of the outlined function with the provided arguments,
1582 /// translating these arguments to correct target-specific arguments.
1583 virtual void
1584 emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
1585 llvm::FunctionCallee OutlinedFn,
1586 ArrayRef<llvm::Value *> Args = {}) const;
1587
1588 /// Emits OpenMP-specific function prolog.
1589 /// Required for device constructs.
1590 virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D);
1591
1592 /// Gets the OpenMP-specific address of the local variable.
1593 virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1594 const VarDecl *VD);
1595
1596 /// Marks the declaration as already emitted for the device code and returns
1597 /// true, if it was marked already, and false, otherwise.
1598 bool markAsGlobalTarget(GlobalDecl GD);
1599
1600 /// Emit deferred declare target variables marked for deferred emission.
1601 void emitDeferredTargetDecls() const;
1602
1603 /// Adjust some parameters for the target-based directives, like addresses of
1604 /// the variables captured by reference in lambdas.
1605 virtual void
1606 adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF,
1607 const OMPExecutableDirective &D) const;
1608
1609 /// Perform check on requires decl to ensure that target architecture
1610 /// supports unified addressing
1611 virtual void processRequiresDirective(const OMPRequiresDecl *D);
1612
1613 /// Gets default memory ordering as specified in requires directive.
1614 llvm::AtomicOrdering getDefaultMemoryOrdering() const;
1615
1616 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with
1617 /// the predefined allocator and translates it into the corresponding address
1618 /// space.
1619 virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS);
1620
1621 /// Return whether the unified_shared_memory has been specified.
1622 bool hasRequiresUnifiedSharedMemory() const;
1623
1624 /// Checks if the \p VD variable is marked as nontemporal declaration in
1625 /// current context.
1626 bool isNontemporalDecl(const ValueDecl *VD) const;
1627
1628 /// Create specialized alloca to handle lastprivate conditionals.
1629 Address emitLastprivateConditionalInit(CodeGenFunction &CGF,
1630 const VarDecl *VD);
1631
1632 /// Checks if the provided \p LVal is lastprivate conditional and emits the
1633 /// code to update the value of the original variable.
1634 /// \code
1635 /// lastprivate(conditional: a)
1636 /// ...
1637 /// <type> a;
1638 /// lp_a = ...;
1639 /// #pragma omp critical(a)
1640 /// if (last_iv_a <= iv) {
1641 /// last_iv_a = iv;
1642 /// global_a = lp_a;
1643 /// }
1644 /// \endcode
1645 virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
1646 const Expr *LHS);
1647
1648 /// Checks if the lastprivate conditional was updated in inner region and
1649 /// writes the value.
1650 /// \code
1651 /// lastprivate(conditional: a)
1652 /// ...
1653 /// <type> a;bool Fired = false;
1654 /// #pragma omp ... shared(a)
1655 /// {
1656 /// lp_a = ...;
1657 /// Fired = true;
1658 /// }
1659 /// if (Fired) {
1660 /// #pragma omp critical(a)
1661 /// if (last_iv_a <= iv) {
1662 /// last_iv_a = iv;
1663 /// global_a = lp_a;
1664 /// }
1665 /// Fired = false;
1666 /// }
1667 /// \endcode
1668 virtual void checkAndEmitSharedLastprivateConditional(
1669 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1670 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls);
1671
1672 /// Gets the address of the global copy used for lastprivate conditional
1673 /// update, if any.
1674 /// \param PrivLVal LValue for the private copy.
1675 /// \param VD Original lastprivate declaration.
1676 virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF,
1677 LValue PrivLVal,
1678 const VarDecl *VD,
1679 SourceLocation Loc);
1680
1681 /// Emits list of dependecies based on the provided data (array of
1682 /// dependence/expression pairs).
1683 /// \returns Pointer to the first element of the array casted to VoidPtr type.
1684 std::pair<llvm::Value *, Address>
1685 emitDependClause(CodeGenFunction &CGF,
1686 ArrayRef<OMPTaskDataTy::DependData> Dependencies,
1687 SourceLocation Loc);
1688
1689 /// Emits list of dependecies based on the provided data (array of
1690 /// dependence/expression pairs) for depobj construct. In this case, the
1691 /// variable is allocated in dynamically. \returns Pointer to the first
1692 /// element of the array casted to VoidPtr type.
1693 Address emitDepobjDependClause(CodeGenFunction &CGF,
1694 const OMPTaskDataTy::DependData &Dependencies,
1695 SourceLocation Loc);
1696
1697 /// Emits the code to destroy the dependency object provided in depobj
1698 /// directive.
1699 void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
1700 SourceLocation Loc);
1701
1702 /// Updates the dependency kind in the specified depobj object.
1703 /// \param DepobjLVal LValue for the main depobj object.
1704 /// \param NewDepKind New dependency kind.
1705 void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
1706 OpenMPDependClauseKind NewDepKind, SourceLocation Loc);
1707
1708 /// Initializes user defined allocators specified in the uses_allocators
1709 /// clauses.
1710 void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator,
1711 const Expr *AllocatorTraits);
1712
1713 /// Destroys user defined allocators specified in the uses_allocators clause.
1714 void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator);
1715
1716 /// Returns true if the variable is a local variable in untied task.
1717 bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const;
1718};
1719
1720/// Class supports emissionof SIMD-only code.
1721class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
1722public:
1723 explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {}
1724 ~CGOpenMPSIMDRuntime() override {}
1725
1726 /// Emits outlined function for the specified OpenMP parallel directive
1727 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1728 /// kmp_int32 BoundID, struct context_vars*).
1729 /// \param CGF Reference to current CodeGenFunction.
1730 /// \param D OpenMP directive.
1731 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1732 /// \param InnermostKind Kind of innermost directive (for simple directives it
1733 /// is a directive itself, for combined - its innermost directive).
1734 /// \param CodeGen Code generation sequence for the \a D directive.
1735 llvm::Function *emitParallelOutlinedFunction(
1736 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1737 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1738 const RegionCodeGenTy &CodeGen) override;
1739
1740 /// Emits outlined function for the specified OpenMP teams directive
1741 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1742 /// kmp_int32 BoundID, struct context_vars*).
1743 /// \param CGF Reference to current CodeGenFunction.
1744 /// \param D OpenMP directive.
1745 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1746 /// \param InnermostKind Kind of innermost directive (for simple directives it
1747 /// is a directive itself, for combined - its innermost directive).
1748 /// \param CodeGen Code generation sequence for the \a D directive.
1749 llvm::Function *emitTeamsOutlinedFunction(
1750 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1751 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1752 const RegionCodeGenTy &CodeGen) override;
1753
1754 /// Emits outlined function for the OpenMP task directive \a D. This
1755 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
1756 /// TaskT).
1757 /// \param D OpenMP directive.
1758 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1759 /// \param PartIDVar Variable for partition id in the current OpenMP untied
1760 /// task region.
1761 /// \param TaskTVar Variable for task_t argument.
1762 /// \param InnermostKind Kind of innermost directive (for simple directives it
1763 /// is a directive itself, for combined - its innermost directive).
1764 /// \param CodeGen Code generation sequence for the \a D directive.
1765 /// \param Tied true if task is generated for tied task, false otherwise.
1766 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
1767 /// tasks.
1768 ///
1769 llvm::Function *emitTaskOutlinedFunction(
1770 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1771 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1772 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1773 bool Tied, unsigned &NumberOfParts) override;
1774
1775 /// Emits code for parallel or serial call of the \a OutlinedFn with
1776 /// variables captured in a record which address is stored in \a
1777 /// CapturedStruct.
1778 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
1779 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1780 /// \param CapturedVars A pointer to the record with the references to
1781 /// variables used in \a OutlinedFn function.
1782 /// \param IfCond Condition in the associated 'if' clause, if it was
1783 /// specified, nullptr otherwise.
1784 /// \param NumThreads The value corresponding to the num_threads clause, if
1785 /// any, or nullptr.
1786 /// \param NumThreadsModifier The modifier of the num_threads clause, if
1787 /// any, ignored otherwise.
1788 /// \param Severity The severity corresponding to the num_threads clause, if
1789 /// any, ignored otherwise.
1790 /// \param Message The message string corresponding to the num_threads clause,
1791 /// if any, or nullptr.
1792 ///
1793 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1794 llvm::Function *OutlinedFn,
1795 ArrayRef<llvm::Value *> CapturedVars,
1796 const Expr *IfCond, llvm::Value *NumThreads,
1797 OpenMPNumThreadsClauseModifier NumThreadsModifier =
1798 OMPC_NUMTHREADS_unknown,
1799 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
1800 const Expr *Message = nullptr) override;
1801
1802 /// Emits a critical region.
1803 /// \param CriticalName Name of the critical region.
1804 /// \param CriticalOpGen Generator for the statement associated with the given
1805 /// critical region.
1806 /// \param Hint Value of the 'hint' clause (optional).
1807 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
1808 const RegionCodeGenTy &CriticalOpGen,
1809 SourceLocation Loc,
1810 const Expr *Hint = nullptr) override;
1811
1812 /// Emits a master region.
1813 /// \param MasterOpGen Generator for the statement associated with the given
1814 /// master region.
1815 void emitMasterRegion(CodeGenFunction &CGF,
1816 const RegionCodeGenTy &MasterOpGen,
1817 SourceLocation Loc) override;
1818
1819 /// Emits a masked region.
1820 /// \param MaskedOpGen Generator for the statement associated with the given
1821 /// masked region.
1822 void emitMaskedRegion(CodeGenFunction &CGF,
1823 const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc,
1824 const Expr *Filter = nullptr) override;
1825
1826 /// Emits a masked region.
1827 /// \param MaskedOpGen Generator for the statement associated with the given
1828 /// masked region.
1829
1830 /// Emits code for a taskyield directive.
1831 void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override;
1832
1833 /// Emit a taskgroup region.
1834 /// \param TaskgroupOpGen Generator for the statement associated with the
1835 /// given taskgroup region.
1836 void emitTaskgroupRegion(CodeGenFunction &CGF,
1837 const RegionCodeGenTy &TaskgroupOpGen,
1838 SourceLocation Loc) override;
1839
1840 /// Emits a single region.
1841 /// \param SingleOpGen Generator for the statement associated with the given
1842 /// single region.
1843 void emitSingleRegion(CodeGenFunction &CGF,
1844 const RegionCodeGenTy &SingleOpGen, SourceLocation Loc,
1845 ArrayRef<const Expr *> CopyprivateVars,
1846 ArrayRef<const Expr *> DestExprs,
1847 ArrayRef<const Expr *> SrcExprs,
1848 ArrayRef<const Expr *> AssignmentOps) override;
1849
1850 /// Emit an ordered region.
1851 /// \param OrderedOpGen Generator for the statement associated with the given
1852 /// ordered region.
1853 void emitOrderedRegion(CodeGenFunction &CGF,
1854 const RegionCodeGenTy &OrderedOpGen,
1855 SourceLocation Loc, bool IsThreads) override;
1856
1857 /// Emit an implicit/explicit barrier for OpenMP threads.
1858 /// \param Kind Directive for which this implicit barrier call must be
1859 /// generated. Must be OMPD_barrier for explicit barrier generation.
1860 /// \param EmitChecks true if need to emit checks for cancellation barriers.
1861 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
1862 /// runtime class decides which one to emit (simple or with cancellation
1863 /// checks).
1864 ///
1865 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
1866 OpenMPDirectiveKind Kind, bool EmitChecks = true,
1867 bool ForceSimpleCall = false) override;
1868
1869 /// This is used for non static scheduled types and when the ordered
1870 /// clause is present on the loop construct.
1871 /// Depending on the loop schedule, it is necessary to call some runtime
1872 /// routine before start of the OpenMP loop to get the loop upper / lower
1873 /// bounds \a LB and \a UB and stride \a ST.
1874 ///
1875 /// \param CGF Reference to current CodeGenFunction.
1876 /// \param Loc Clang source location.
1877 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1878 /// \param IVSize Size of the iteration variable in bits.
1879 /// \param IVSigned Sign of the iteration variable.
1880 /// \param Ordered true if loop is ordered, false otherwise.
1881 /// \param DispatchValues struct containing llvm values for lower bound, upper
1882 /// bound, and chunk expression.
1883 /// For the default (nullptr) value, the chunk 1 will be used.
1884 ///
1885 void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
1886 const OpenMPScheduleTy &ScheduleKind,
1887 unsigned IVSize, bool IVSigned, bool Ordered,
1888 const DispatchRTInput &DispatchValues) override;
1889
1890 /// This is used for non static scheduled types and when the ordered
1891 /// clause is present on the loop construct.
1892 ///
1893 /// \param CGF Reference to current CodeGenFunction.
1894 /// \param Loc Clang source location.
1895 ///
1896 void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override;
1897
1898 /// Call the appropriate runtime routine to initialize it before start
1899 /// of loop.
1900 ///
1901 /// This is used only in case of static schedule, when the user did not
1902 /// specify a ordered clause on the loop construct.
1903 /// Depending on the loop schedule, it is necessary to call some runtime
1904 /// routine before start of the OpenMP loop to get the loop upper / lower
1905 /// bounds LB and UB and stride ST.
1906 ///
1907 /// \param CGF Reference to current CodeGenFunction.
1908 /// \param Loc Clang source location.
1909 /// \param DKind Kind of the directive.
1910 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1911 /// \param Values Input arguments for the construct.
1912 ///
1913 void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1914 OpenMPDirectiveKind DKind,
1915 const OpenMPScheduleTy &ScheduleKind,
1916 const StaticRTInput &Values) override;
1917
1918 ///
1919 /// \param CGF Reference to current CodeGenFunction.
1920 /// \param Loc Clang source location.
1921 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1922 /// \param Values Input arguments for the construct.
1923 ///
1924 void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1925 OpenMPDistScheduleClauseKind SchedKind,
1926 const StaticRTInput &Values) override;
1927
1928 /// Call the appropriate runtime routine to notify that we finished
1929 /// iteration of the ordered loop with the dynamic scheduling.
1930 ///
1931 /// \param CGF Reference to current CodeGenFunction.
1932 /// \param Loc Clang source location.
1933 /// \param IVSize Size of the iteration variable in bits.
1934 /// \param IVSigned Sign of the iteration variable.
1935 ///
1936 void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc,
1937 unsigned IVSize, bool IVSigned) override;
1938
1939 /// Call the appropriate runtime routine to notify that we finished
1940 /// all the work with current loop.
1941 ///
1942 /// \param CGF Reference to current CodeGenFunction.
1943 /// \param Loc Clang source location.
1944 /// \param DKind Kind of the directive for which the static finish is emitted.
1945 ///
1946 void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1947 OpenMPDirectiveKind DKind) override;
1948
1949 /// Call __kmpc_dispatch_next(
1950 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1951 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1952 /// kmp_int[32|64] *p_stride);
1953 /// \param IVSize Size of the iteration variable in bits.
1954 /// \param IVSigned Sign of the iteration variable.
1955 /// \param IL Address of the output variable in which the flag of the
1956 /// last iteration is returned.
1957 /// \param LB Address of the output variable in which the lower iteration
1958 /// number is returned.
1959 /// \param UB Address of the output variable in which the upper iteration
1960 /// number is returned.
1961 /// \param ST Address of the output variable in which the stride value is
1962 /// returned.
1963 llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1964 unsigned IVSize, bool IVSigned, Address IL,
1965 Address LB, Address UB, Address ST) override;
1966
1967 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1968 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1969 /// clause.
1970 /// If the modifier 'strict' is given:
1971 /// Emits call to void __kmpc_push_num_threads_strict(ident_t *loc, kmp_int32
1972 /// global_tid, kmp_int32 num_threads, int severity, const char *message) to
1973 /// generate code for 'num_threads' clause with 'strict' modifier.
1974 /// \param NumThreads An integer value of threads.
1975 void emitNumThreadsClause(
1976 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
1977 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown,
1978 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
1979 SourceLocation SeverityLoc = SourceLocation(),
1980 const Expr *Message = nullptr,
1981 SourceLocation MessageLoc = SourceLocation()) override;
1982
1983 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1984 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1985 void emitProcBindClause(CodeGenFunction &CGF,
1986 llvm::omp::ProcBindKind ProcBind,
1987 SourceLocation Loc) override;
1988
1989 /// Returns address of the threadprivate variable for the current
1990 /// thread.
1991 /// \param VD Threadprivate variable.
1992 /// \param VDAddr Address of the global variable \a VD.
1993 /// \param Loc Location of the reference to threadprivate var.
1994 /// \return Address of the threadprivate variable for the current thread.
1995 Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD,
1996 Address VDAddr, SourceLocation Loc) override;
1997
1998 /// Emit a code for initialization of threadprivate variable. It emits
1999 /// a call to runtime library which adds initial value to the newly created
2000 /// threadprivate variable (if it is not constant) and registers destructor
2001 /// for the variable (if any).
2002 /// \param VD Threadprivate variable.
2003 /// \param VDAddr Address of the global variable \a VD.
2004 /// \param Loc Location of threadprivate declaration.
2005 /// \param PerformInit true if initialization expression is not constant.
2006 llvm::Function *
2007 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
2008 SourceLocation Loc, bool PerformInit,
2009 CodeGenFunction *CGF = nullptr) override;
2010
2011 /// Creates artificial threadprivate variable with name \p Name and type \p
2012 /// VarType.
2013 /// \param VarType Type of the artificial threadprivate variable.
2014 /// \param Name Name of the artificial threadprivate variable.
2015 Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2016 QualType VarType,
2017 StringRef Name) override;
2018
2019 /// Emit flush of the variables specified in 'omp flush' directive.
2020 /// \param Vars List of variables to flush.
2021 void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
2022 SourceLocation Loc, llvm::AtomicOrdering AO) override;
2023
2024 /// Emit task region for the task directive. The task region is
2025 /// emitted in several steps:
2026 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
2027 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2028 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
2029 /// function:
2030 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
2031 /// TaskFunction(gtid, tt->part_id, tt->shareds);
2032 /// return 0;
2033 /// }
2034 /// 2. Copy a list of shared variables to field shareds of the resulting
2035 /// structure kmp_task_t returned by the previous call (if any).
2036 /// 3. Copy a pointer to destructions function to field destructions of the
2037 /// resulting structure kmp_task_t.
2038 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
2039 /// kmp_task_t *new_task), where new_task is a resulting structure from
2040 /// previous items.
2041 /// \param D Current task directive.
2042 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
2043 /// /*part_id*/, captured_struct */*__context*/);
2044 /// \param SharedsTy A type which contains references the shared variables.
2045 /// \param Shareds Context with the list of shared variables from the \p
2046 /// TaskFunction.
2047 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
2048 /// otherwise.
2049 /// \param Data Additional data for task generation like tiednsee, final
2050 /// state, list of privates etc.
2051 void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
2052 const OMPExecutableDirective &D,
2053 llvm::Function *TaskFunction, QualType SharedsTy,
2054 Address Shareds, const Expr *IfCond,
2055 const OMPTaskDataTy &Data) override;
2056
2057 /// Emit task region for the taskloop directive. The taskloop region is
2058 /// emitted in several steps:
2059 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
2060 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2061 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
2062 /// function:
2063 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
2064 /// TaskFunction(gtid, tt->part_id, tt->shareds);
2065 /// return 0;
2066 /// }
2067 /// 2. Copy a list of shared variables to field shareds of the resulting
2068 /// structure kmp_task_t returned by the previous call (if any).
2069 /// 3. Copy a pointer to destructions function to field destructions of the
2070 /// resulting structure kmp_task_t.
2071 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
2072 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
2073 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
2074 /// is a resulting structure from
2075 /// previous items.
2076 /// \param D Current task directive.
2077 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
2078 /// /*part_id*/, captured_struct */*__context*/);
2079 /// \param SharedsTy A type which contains references the shared variables.
2080 /// \param Shareds Context with the list of shared variables from the \p
2081 /// TaskFunction.
2082 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
2083 /// otherwise.
2084 /// \param Data Additional data for task generation like tiednsee, final
2085 /// state, list of privates etc.
2086 void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
2087 const OMPLoopDirective &D, llvm::Function *TaskFunction,
2088 QualType SharedsTy, Address Shareds, const Expr *IfCond,
2089 const OMPTaskDataTy &Data) override;
2090
2091 /// Emit a code for reduction clause. Next code should be emitted for
2092 /// reduction:
2093 /// \code
2094 ///
2095 /// static kmp_critical_name lock = { 0 };
2096 ///
2097 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
2098 /// ...
2099 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
2100 /// ...
2101 /// }
2102 ///
2103 /// ...
2104 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
2105 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
2106 /// RedList, reduce_func, &<lock>)) {
2107 /// case 1:
2108 /// ...
2109 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2110 /// ...
2111 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2112 /// break;
2113 /// case 2:
2114 /// ...
2115 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
2116 /// ...
2117 /// break;
2118 /// default:;
2119 /// }
2120 /// \endcode
2121 ///
2122 /// \param Privates List of private copies for original reduction arguments.
2123 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
2124 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
2125 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
2126 /// or 'operator binop(LHS, RHS)'.
2127 /// \param Options List of options for reduction codegen:
2128 /// WithNowait true if parent directive has also nowait clause, false
2129 /// otherwise.
2130 /// SimpleReduction Emit reduction operation only. Used for omp simd
2131 /// directive on the host.
2132 /// ReductionKind The kind of reduction to perform.
2133 void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
2134 ArrayRef<const Expr *> Privates,
2135 ArrayRef<const Expr *> LHSExprs,
2136 ArrayRef<const Expr *> RHSExprs,
2137 ArrayRef<const Expr *> ReductionOps,
2138 ReductionOptionsTy Options) override;
2139
2140 /// Emit a code for initialization of task reduction clause. Next code
2141 /// should be emitted for reduction:
2142 /// \code
2143 ///
2144 /// _taskred_item_t red_data[n];
2145 /// ...
2146 /// red_data[i].shar = &shareds[i];
2147 /// red_data[i].orig = &origs[i];
2148 /// red_data[i].size = sizeof(origs[i]);
2149 /// red_data[i].f_init = (void*)RedInit<i>;
2150 /// red_data[i].f_fini = (void*)RedDest<i>;
2151 /// red_data[i].f_comb = (void*)RedOp<i>;
2152 /// red_data[i].flags = <Flag_i>;
2153 /// ...
2154 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
2155 /// \endcode
2156 /// For reduction clause with task modifier it emits the next call:
2157 /// \code
2158 ///
2159 /// _taskred_item_t red_data[n];
2160 /// ...
2161 /// red_data[i].shar = &shareds[i];
2162 /// red_data[i].orig = &origs[i];
2163 /// red_data[i].size = sizeof(origs[i]);
2164 /// red_data[i].f_init = (void*)RedInit<i>;
2165 /// red_data[i].f_fini = (void*)RedDest<i>;
2166 /// red_data[i].f_comb = (void*)RedOp<i>;
2167 /// red_data[i].flags = <Flag_i>;
2168 /// ...
2169 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
2170 /// red_data);
2171 /// \endcode
2172 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
2173 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
2174 /// \param Data Additional data for task generation like tiedness, final
2175 /// state, list of privates, reductions etc.
2176 llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc,
2177 ArrayRef<const Expr *> LHSExprs,
2178 ArrayRef<const Expr *> RHSExprs,
2179 const OMPTaskDataTy &Data) override;
2180
2181 /// Emits the following code for reduction clause with task modifier:
2182 /// \code
2183 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
2184 /// \endcode
2185 void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
2186 bool IsWorksharingReduction) override;
2187
2188 /// Required to resolve existing problems in the runtime. Emits threadprivate
2189 /// variables to store the size of the VLAs/array sections for
2190 /// initializer/combiner/finalizer functions + emits threadprivate variable to
2191 /// store the pointer to the original reduction item for the custom
2192 /// initializer defined by declare reduction construct.
2193 /// \param RCG Allows to reuse an existing data for the reductions.
2194 /// \param N Reduction item for which fixups must be emitted.
2195 void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
2196 ReductionCodeGen &RCG, unsigned N) override;
2197
2198 /// Get the address of `void *` type of the privatue copy of the reduction
2199 /// item specified by the \p SharedLVal.
2200 /// \param ReductionsPtr Pointer to the reduction data returned by the
2201 /// emitTaskReductionInit function.
2202 /// \param SharedLVal Address of the original reduction item.
2203 Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
2204 llvm::Value *ReductionsPtr,
2205 LValue SharedLVal) override;
2206
2207 /// Emit code for 'taskwait' directive.
2208 void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
2209 const OMPTaskDataTy &Data) override;
2210
2211 /// Emit code for 'cancellation point' construct.
2212 /// \param CancelRegion Region kind for which the cancellation point must be
2213 /// emitted.
2214 ///
2215 void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc,
2216 OpenMPDirectiveKind CancelRegion) override;
2217
2218 /// Emit code for 'cancel' construct.
2219 /// \param IfCond Condition in the associated 'if' clause, if it was
2220 /// specified, nullptr otherwise.
2221 /// \param CancelRegion Region kind for which the cancel must be emitted.
2222 ///
2223 void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
2224 const Expr *IfCond,
2225 OpenMPDirectiveKind CancelRegion) override;
2226
2227 /// Emit outilined function for 'target' directive.
2228 /// \param D Directive to emit.
2229 /// \param ParentName Name of the function that encloses the target region.
2230 /// \param OutlinedFn Outlined function value to be defined by this call.
2231 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
2232 /// \param IsOffloadEntry True if the outlined function is an offload entry.
2233 /// \param CodeGen Code generation sequence for the \a D directive.
2234 /// An outlined function may not be an entry if, e.g. the if clause always
2235 /// evaluates to false.
2236 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
2237 StringRef ParentName,
2238 llvm::Function *&OutlinedFn,
2239 llvm::Constant *&OutlinedFnID,
2240 bool IsOffloadEntry,
2241 const RegionCodeGenTy &CodeGen) override;
2242
2243 /// Emit the target offloading code associated with \a D. The emitted
2244 /// code attempts offloading the execution to the device, an the event of
2245 /// a failure it executes the host version outlined in \a OutlinedFn.
2246 /// \param D Directive to emit.
2247 /// \param OutlinedFn Host version of the code to be offloaded.
2248 /// \param OutlinedFnID ID of host version of the code to be offloaded.
2249 /// \param IfCond Expression evaluated in if clause associated with the target
2250 /// directive, or null if no if clause is used.
2251 /// \param Device Expression evaluated in device clause associated with the
2252 /// target directive, or null if no device clause is used and device modifier.
2253 void emitTargetCall(
2254 CodeGenFunction &CGF, const OMPExecutableDirective &D,
2255 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
2256 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
2257 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
2258 const OMPLoopDirective &D)>
2259 SizeEmitter) override;
2260
2261 /// Emit the target regions enclosed in \a GD function definition or
2262 /// the function itself in case it is a valid device function. Returns true if
2263 /// \a GD was dealt with successfully.
2264 /// \param GD Function to scan.
2265 bool emitTargetFunctions(GlobalDecl GD) override;
2266
2267 /// Emit the global variable if it is a valid device global variable.
2268 /// Returns true if \a GD was dealt with successfully.
2269 /// \param GD Variable declaration to emit.
2270 bool emitTargetGlobalVariable(GlobalDecl GD) override;
2271
2272 /// Emit the global \a GD if it is meaningful for the target. Returns
2273 /// if it was emitted successfully.
2274 /// \param GD Global to scan.
2275 bool emitTargetGlobal(GlobalDecl GD) override;
2276
2277 /// Emits code for teams call of the \a OutlinedFn with
2278 /// variables captured in a record which address is stored in \a
2279 /// CapturedStruct.
2280 /// \param OutlinedFn Outlined function to be run by team masters. Type of
2281 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
2282 /// \param CapturedVars A pointer to the record with the references to
2283 /// variables used in \a OutlinedFn function.
2284 ///
2285 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
2286 SourceLocation Loc, llvm::Function *OutlinedFn,
2287 ArrayRef<llvm::Value *> CapturedVars) override;
2288
2289 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
2290 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
2291 /// for num_teams clause.
2292 /// \param NumTeams An integer expression of teams.
2293 /// \param ThreadLimit An integer expression of threads.
2294 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
2295 const Expr *ThreadLimit, SourceLocation Loc) override;
2296
2297 /// Emit the target data mapping code associated with \a D.
2298 /// \param D Directive to emit.
2299 /// \param IfCond Expression evaluated in if clause associated with the
2300 /// target directive, or null if no device clause is used.
2301 /// \param Device Expression evaluated in device clause associated with the
2302 /// target directive, or null if no device clause is used.
2303 /// \param Info A record used to store information that needs to be preserved
2304 /// until the region is closed.
2305 void emitTargetDataCalls(CodeGenFunction &CGF,
2306 const OMPExecutableDirective &D, const Expr *IfCond,
2307 const Expr *Device, const RegionCodeGenTy &CodeGen,
2308 CGOpenMPRuntime::TargetDataInfo &Info) override;
2309
2310 /// Emit the data mapping/movement code associated with the directive
2311 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
2312 /// \param D Directive to emit.
2313 /// \param IfCond Expression evaluated in if clause associated with the target
2314 /// directive, or null if no if clause is used.
2315 /// \param Device Expression evaluated in device clause associated with the
2316 /// target directive, or null if no device clause is used.
2317 void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
2318 const OMPExecutableDirective &D,
2319 const Expr *IfCond,
2320 const Expr *Device) override;
2321
2322 /// Emit initialization for doacross loop nesting support.
2323 /// \param D Loop-based construct used in doacross nesting construct.
2324 void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
2325 ArrayRef<Expr *> NumIterations) override;
2326
2327 /// Emit code for doacross ordered directive with 'depend' clause.
2328 /// \param C 'depend' clause with 'sink|source' dependency kind.
2329 void emitDoacrossOrdered(CodeGenFunction &CGF,
2330 const OMPDependClause *C) override;
2331
2332 /// Emit code for doacross ordered directive with 'doacross' clause.
2333 /// \param C 'doacross' clause with 'sink|source' dependence type.
2334 void emitDoacrossOrdered(CodeGenFunction &CGF,
2335 const OMPDoacrossClause *C) override;
2336
2337 /// Translates the native parameter of outlined function if this is required
2338 /// for target.
2339 /// \param FD Field decl from captured record for the parameter.
2340 /// \param NativeParam Parameter itself.
2341 const VarDecl *translateParameter(const FieldDecl *FD,
2342 const VarDecl *NativeParam) const override;
2343
2344 /// Gets the address of the native argument basing on the address of the
2345 /// target-specific parameter.
2346 /// \param NativeParam Parameter itself.
2347 /// \param TargetParam Corresponding target-specific parameter.
2348 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
2349 const VarDecl *TargetParam) const override;
2350
2351 /// Gets the OpenMP-specific address of the local variable.
2352 Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2353 const VarDecl *VD) override {
2354 return Address::invalid();
2355 }
2356};
2357
2358} // namespace CodeGen
2359// Utility for openmp doacross clause kind
2360namespace {
2361template <typename T> class OMPDoacrossKind {
2362public:
2363 bool isSink(const T *) { return false; }
2364 bool isSource(const T *) { return false; }
2365};
2366template <> class OMPDoacrossKind<OMPDependClause> {
2367public:
2368 bool isSink(const OMPDependClause *C) {
2369 return C->getDependencyKind() == OMPC_DEPEND_sink;
2370 }
2371 bool isSource(const OMPDependClause *C) {
2372 return C->getDependencyKind() == OMPC_DEPEND_source;
2373 }
2374};
2375template <> class OMPDoacrossKind<OMPDoacrossClause> {
2376public:
2377 bool isSource(const OMPDoacrossClause *C) {
2378 return C->getDependenceType() == OMPC_DOACROSS_source ||
2379 C->getDependenceType() == OMPC_DOACROSS_source_omp_cur_iteration;
2380 }
2381 bool isSink(const OMPDoacrossClause *C) {
2382 return C->getDependenceType() == OMPC_DOACROSS_sink ||
2383 C->getDependenceType() == OMPC_DOACROSS_sink_omp_cur_iteration;
2384 }
2385};
2386} // namespace
2387} // namespace clang
2388
2389#endif
2390