1//===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
14#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
15
16#include "CGValue.h"
17#include "clang/AST/DeclOpenMP.h"
18#include "clang/AST/GlobalDecl.h"
19#include "clang/AST/Type.h"
20#include "clang/Basic/OpenMPKinds.h"
21#include "clang/Basic/SourceLocation.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/PointerIntPair.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/Frontend/OpenMP/OMPConstants.h"
28#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/ValueHandle.h"
31#include "llvm/Support/AtomicOrdering.h"
32
33namespace llvm {
34class ArrayType;
35class Constant;
36class FunctionType;
37class GlobalVariable;
38class Type;
39class Value;
40class OpenMPIRBuilder;
41} // namespace llvm
42
43namespace clang {
44class Expr;
45class OMPDependClause;
46class OMPExecutableDirective;
47class OMPLoopDirective;
48class VarDecl;
49class OMPDeclareReductionDecl;
50
51namespace CodeGen {
52class Address;
53class CodeGenFunction;
54class CodeGenModule;
55
56/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
57/// region.
58class PrePostActionTy {
59public:
60 explicit PrePostActionTy() {}
61 virtual void Enter(CodeGenFunction &CGF) {}
62 virtual void Exit(CodeGenFunction &CGF) {}
63 virtual ~PrePostActionTy() {}
64};
65
66/// Class provides a way to call simple version of codegen for OpenMP region, or
67/// an advanced with possible pre|post-actions in codegen.
68class RegionCodeGenTy final {
69 intptr_t CodeGen;
70 typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &);
71 CodeGenTy Callback;
72 mutable PrePostActionTy *PrePostAction;
73 RegionCodeGenTy() = delete;
74 template <typename Callable>
75 static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF,
76 PrePostActionTy &Action) {
77 return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action);
78 }
79
80public:
81 template <typename Callable>
82 RegionCodeGenTy(
83 Callable &&CodeGen,
84 std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
85 RegionCodeGenTy>::value> * = nullptr)
86 : CodeGen(reinterpret_cast<intptr_t>(&CodeGen)),
87 Callback(CallbackFn<std::remove_reference_t<Callable>>),
88 PrePostAction(nullptr) {}
89 void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; }
90 void operator()(CodeGenFunction &CGF) const;
91};
92
93struct OMPTaskDataTy final {
94 SmallVector<const Expr *, 4> PrivateVars;
95 SmallVector<const Expr *, 4> PrivateCopies;
96 SmallVector<const Expr *, 4> FirstprivateVars;
97 SmallVector<const Expr *, 4> FirstprivateCopies;
98 SmallVector<const Expr *, 4> FirstprivateInits;
99 SmallVector<const Expr *, 4> LastprivateVars;
100 SmallVector<const Expr *, 4> LastprivateCopies;
101 SmallVector<const Expr *, 4> ReductionVars;
102 SmallVector<const Expr *, 4> ReductionOrigs;
103 SmallVector<const Expr *, 4> ReductionCopies;
104 SmallVector<const Expr *, 4> ReductionOps;
105 SmallVector<CanonicalDeclPtr<const VarDecl>, 4> PrivateLocals;
106 struct DependData {
107 OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown;
108 const Expr *IteratorExpr = nullptr;
109 SmallVector<const Expr *, 4> DepExprs;
110 explicit DependData() = default;
111 DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr)
112 : DepKind(DepKind), IteratorExpr(IteratorExpr) {}
113 };
114 SmallVector<DependData, 4> Dependences;
115 llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
116 llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule;
117 llvm::PointerIntPair<llvm::Value *, 1, bool> Priority;
118 llvm::Value *Reductions = nullptr;
119 unsigned NumberOfParts = 0;
120 bool Tied = true;
121 bool Nogroup = false;
122 bool IsReductionWithTaskMod = false;
123 bool IsWorksharingReduction = false;
124 bool HasNowaitClause = false;
125 bool HasModifier = false;
126};
127
128/// Class intended to support codegen of all kind of the reduction clauses.
129class ReductionCodeGen {
130private:
131 /// Data required for codegen of reduction clauses.
132 struct ReductionData {
133 /// Reference to the item shared between tasks to reduce into.
134 const Expr *Shared = nullptr;
135 /// Reference to the original item.
136 const Expr *Ref = nullptr;
137 /// Helper expression for generation of private copy.
138 const Expr *Private = nullptr;
139 /// Helper expression for generation reduction operation.
140 const Expr *ReductionOp = nullptr;
141 ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private,
142 const Expr *ReductionOp)
143 : Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) {
144 }
145 };
146 /// List of reduction-based clauses.
147 SmallVector<ReductionData, 4> ClausesData;
148
149 /// List of addresses of shared variables/expressions.
150 SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses;
151 /// List of addresses of original variables/expressions.
152 SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses;
153 /// Sizes of the reduction items in chars.
154 SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes;
155 /// Base declarations for the reduction items.
156 SmallVector<const VarDecl *, 4> BaseDecls;
157
158 /// Emits lvalue for shared expression.
159 LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E);
160 /// Emits upper bound for shared expression (if array section).
161 LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E);
162 /// Performs aggregate initialization.
163 /// \param N Number of reduction item in the common list.
164 /// \param PrivateAddr Address of the corresponding private item.
165 /// \param SharedAddr Address of the original shared variable.
166 /// \param DRD Declare reduction construct used for reduction item.
167 void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N,
168 Address PrivateAddr, Address SharedAddr,
169 const OMPDeclareReductionDecl *DRD);
170
171public:
172 ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs,
173 ArrayRef<const Expr *> Privates,
174 ArrayRef<const Expr *> ReductionOps);
175 /// Emits lvalue for the shared and original reduction item.
176 /// \param N Number of the reduction item.
177 void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N);
178 /// Emits the code for the variable-modified type, if required.
179 /// \param N Number of the reduction item.
180 void emitAggregateType(CodeGenFunction &CGF, unsigned N);
181 /// Emits the code for the variable-modified type, if required.
182 /// \param N Number of the reduction item.
183 /// \param Size Size of the type in chars.
184 void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size);
185 /// Performs initialization of the private copy for the reduction item.
186 /// \param N Number of the reduction item.
187 /// \param PrivateAddr Address of the corresponding private item.
188 /// \param DefaultInit Default initialization sequence that should be
189 /// performed if no reduction specific initialization is found.
190 /// \param SharedAddr Address of the original shared variable.
191 void
192 emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr,
193 Address SharedAddr,
194 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit);
195 /// Returns true if the private copy requires cleanups.
196 bool needCleanups(unsigned N);
197 /// Emits cleanup code for the reduction item.
198 /// \param N Number of the reduction item.
199 /// \param PrivateAddr Address of the corresponding private item.
200 void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr);
201 /// Adjusts \p PrivatedAddr for using instead of the original variable
202 /// address in normal operations.
203 /// \param N Number of the reduction item.
204 /// \param PrivateAddr Address of the corresponding private item.
205 Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
206 Address PrivateAddr);
207 /// Returns LValue for the reduction item.
208 LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; }
209 /// Returns LValue for the original reduction item.
210 LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; }
211 /// Returns the size of the reduction item (in chars and total number of
212 /// elements in the item), or nullptr, if the size is a constant.
213 std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const {
214 return Sizes[N];
215 }
216 /// Returns the base declaration of the reduction item.
217 const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; }
218 /// Returns the base declaration of the reduction item.
219 const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; }
220 /// Returns true if the initialization of the reduction item uses initializer
221 /// from declare reduction construct.
222 bool usesReductionInitializer(unsigned N) const;
223 /// Return the type of the private item.
224 QualType getPrivateType(unsigned N) const {
225 return cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl())
226 ->getType();
227 }
228};
229
230class CGOpenMPRuntime {
231public:
232 /// Allows to disable automatic handling of functions used in target regions
233 /// as those marked as `omp declare target`.
234 class DisableAutoDeclareTargetRAII {
235 CodeGenModule &CGM;
236 bool SavedShouldMarkAsGlobal = false;
237
238 public:
239 DisableAutoDeclareTargetRAII(CodeGenModule &CGM);
240 ~DisableAutoDeclareTargetRAII();
241 };
242
243 /// Manages list of nontemporal decls for the specified directive.
244 class NontemporalDeclsRAII {
245 CodeGenModule &CGM;
246 const bool NeedToPush;
247
248 public:
249 NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S);
250 ~NontemporalDeclsRAII();
251 };
252
253 /// Manages list of nontemporal decls for the specified directive.
254 class UntiedTaskLocalDeclsRAII {
255 CodeGenModule &CGM;
256 const bool NeedToPush;
257
258 public:
259 UntiedTaskLocalDeclsRAII(
260 CodeGenFunction &CGF,
261 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
262 std::pair<Address, Address>> &LocalVars);
263 ~UntiedTaskLocalDeclsRAII();
264 };
265
266 /// Maps the expression for the lastprivate variable to the global copy used
267 /// to store new value because original variables are not mapped in inner
268 /// parallel regions. Only private copies are captured but we need also to
269 /// store private copy in shared address.
270 /// Also, stores the expression for the private loop counter and it
271 /// threaprivate name.
272 struct LastprivateConditionalData {
273 llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>>
274 DeclToUniqueName;
275 LValue IVLVal;
276 llvm::Function *Fn = nullptr;
277 bool Disabled = false;
278 };
279 /// Manages list of lastprivate conditional decls for the specified directive.
280 class LastprivateConditionalRAII {
281 enum class ActionToDo {
282 DoNotPush,
283 PushAsLastprivateConditional,
284 DisableLastprivateConditional,
285 };
286 CodeGenModule &CGM;
287 ActionToDo Action = ActionToDo::DoNotPush;
288
289 /// Check and try to disable analysis of inner regions for changes in
290 /// lastprivate conditional.
291 void tryToDisableInnerAnalysis(const OMPExecutableDirective &S,
292 llvm::DenseSet<CanonicalDeclPtr<const Decl>>
293 &NeedToAddForLPCsAsDisabled) const;
294
295 LastprivateConditionalRAII(CodeGenFunction &CGF,
296 const OMPExecutableDirective &S);
297
298 public:
299 explicit LastprivateConditionalRAII(CodeGenFunction &CGF,
300 const OMPExecutableDirective &S,
301 LValue IVLVal);
302 static LastprivateConditionalRAII disable(CodeGenFunction &CGF,
303 const OMPExecutableDirective &S);
304 ~LastprivateConditionalRAII();
305 };
306
307 llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; }
308
309protected:
310 CodeGenModule &CGM;
311
312 /// An OpenMP-IR-Builder instance.
313 llvm::OpenMPIRBuilder OMPBuilder;
314
315 /// Helper to determine the min/max number of threads/teams for \p D.
316 void computeMinAndMaxThreadsAndTeams(
317 const OMPExecutableDirective &D, CodeGenFunction &CGF,
318 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs);
319
320 /// Helper to emit outlined function for 'target' directive.
321 /// \param D Directive to emit.
322 /// \param ParentName Name of the function that encloses the target region.
323 /// \param OutlinedFn Outlined function value to be defined by this call.
324 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
325 /// \param IsOffloadEntry True if the outlined function is an offload entry.
326 /// \param CodeGen Lambda codegen specific to an accelerator device.
327 /// An outlined function may not be an entry if, e.g. the if clause always
328 /// evaluates to false.
329 virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D,
330 StringRef ParentName,
331 llvm::Function *&OutlinedFn,
332 llvm::Constant *&OutlinedFnID,
333 bool IsOffloadEntry,
334 const RegionCodeGenTy &CodeGen);
335
336 /// Returns pointer to ident_t type.
337 llvm::Type *getIdentTyPointerTy();
338
339 /// Gets thread id value for the current thread.
340 ///
341 llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc);
342
343 /// Get the function name of an outlined region.
344 std::string getOutlinedHelperName(StringRef Name) const;
345 std::string getOutlinedHelperName(CodeGenFunction &CGF) const;
346
347 /// Get the function name of a reduction function.
348 std::string getReductionFuncName(StringRef Name) const;
349
350 /// Emits \p Callee function call with arguments \p Args with location \p Loc.
351 void emitCall(CodeGenFunction &CGF, SourceLocation Loc,
352 llvm::FunctionCallee Callee,
353 ArrayRef<llvm::Value *> Args = {}) const;
354
355 /// Emits address of the word in a memory where current thread id is
356 /// stored.
357 virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc);
358
359 void setLocThreadIdInsertPt(CodeGenFunction &CGF,
360 bool AtCurrentPoint = false);
361 void clearLocThreadIdInsertPt(CodeGenFunction &CGF);
362
363 /// Check if the default location must be constant.
364 /// Default is false to support OMPT/OMPD.
365 virtual bool isDefaultLocationConstant() const { return false; }
366
367 /// Returns additional flags that can be stored in reserved_2 field of the
368 /// default location.
369 virtual unsigned getDefaultLocationReserved2Flags() const { return 0; }
370
371 /// Returns default flags for the barriers depending on the directive, for
372 /// which this barier is going to be emitted.
373 static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind);
374
375 /// Get the LLVM type for the critical name.
376 llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;}
377
378 /// Returns corresponding lock object for the specified critical region
379 /// name. If the lock object does not exist it is created, otherwise the
380 /// reference to the existing copy is returned.
381 /// \param CriticalName Name of the critical region.
382 ///
383 llvm::Value *getCriticalRegionLock(StringRef CriticalName);
384
385protected:
386 /// Map for SourceLocation and OpenMP runtime library debug locations.
387 typedef llvm::DenseMap<SourceLocation, llvm::Value *> OpenMPDebugLocMapTy;
388 OpenMPDebugLocMapTy OpenMPDebugLocMap;
389 /// Stores debug location and ThreadID for the function.
390 struct DebugLocThreadIdTy {
391 llvm::Value *DebugLoc;
392 llvm::Value *ThreadID;
393 /// Insert point for the service instructions.
394 llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr;
395 };
396 /// Map of local debug location, ThreadId and functions.
397 typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy>
398 OpenMPLocThreadIDMapTy;
399 OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap;
400 /// Map of UDRs and corresponding combiner/initializer.
401 typedef llvm::DenseMap<const OMPDeclareReductionDecl *,
402 std::pair<llvm::Function *, llvm::Function *>>
403 UDRMapTy;
404 UDRMapTy UDRMap;
405 /// Map of functions and locally defined UDRs.
406 typedef llvm::DenseMap<llvm::Function *,
407 SmallVector<const OMPDeclareReductionDecl *, 4>>
408 FunctionUDRMapTy;
409 FunctionUDRMapTy FunctionUDRMap;
410 /// Map from the user-defined mapper declaration to its corresponding
411 /// functions.
412 llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap;
413 /// Map of functions and their local user-defined mappers.
414 using FunctionUDMMapTy =
415 llvm::DenseMap<llvm::Function *,
416 SmallVector<const OMPDeclareMapperDecl *, 4>>;
417 FunctionUDMMapTy FunctionUDMMap;
418 /// Maps local variables marked as lastprivate conditional to their internal
419 /// types.
420 llvm::DenseMap<llvm::Function *,
421 llvm::DenseMap<CanonicalDeclPtr<const Decl>,
422 std::tuple<QualType, const FieldDecl *,
423 const FieldDecl *, LValue>>>
424 LastprivateConditionalToTypes;
425 /// Maps function to the position of the untied task locals stack.
426 llvm::DenseMap<llvm::Function *, unsigned> FunctionToUntiedTaskStackMap;
427 /// Type kmp_critical_name, originally defined as typedef kmp_int32
428 /// kmp_critical_name[8];
429 llvm::ArrayType *KmpCriticalNameTy;
430 /// An ordered map of auto-generated variables to their unique names.
431 /// It stores variables with the following names: 1) ".gomp_critical_user_" +
432 /// <critical_section_name> + ".var" for "omp critical" directives; 2)
433 /// <mangled_name_for_global_var> + ".cache." for cache for threadprivate
434 /// variables.
435 llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>,
436 llvm::BumpPtrAllocator> InternalVars;
437 /// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);
438 llvm::Type *KmpRoutineEntryPtrTy = nullptr;
439 QualType KmpRoutineEntryPtrQTy;
440 /// Type typedef struct kmp_task {
441 /// void * shareds; /**< pointer to block of pointers to
442 /// shared vars */
443 /// kmp_routine_entry_t routine; /**< pointer to routine to call for
444 /// executing task */
445 /// kmp_int32 part_id; /**< part id for the task */
446 /// kmp_routine_entry_t destructors; /* pointer to function to invoke
447 /// deconstructors of firstprivate C++ objects */
448 /// } kmp_task_t;
449 QualType KmpTaskTQTy;
450 /// Saved kmp_task_t for task directive.
451 QualType SavedKmpTaskTQTy;
452 /// Saved kmp_task_t for taskloop-based directive.
453 QualType SavedKmpTaskloopTQTy;
454 /// Type typedef struct kmp_depend_info {
455 /// kmp_intptr_t base_addr;
456 /// size_t len;
457 /// struct {
458 /// bool in:1;
459 /// bool out:1;
460 /// } flags;
461 /// } kmp_depend_info_t;
462 QualType KmpDependInfoTy;
463 /// Type typedef struct kmp_task_affinity_info {
464 /// kmp_intptr_t base_addr;
465 /// size_t len;
466 /// struct {
467 /// bool flag1 : 1;
468 /// bool flag2 : 1;
469 /// kmp_int32 reserved : 30;
470 /// } flags;
471 /// } kmp_task_affinity_info_t;
472 QualType KmpTaskAffinityInfoTy;
473 /// struct kmp_dim { // loop bounds info casted to kmp_int64
474 /// kmp_int64 lo; // lower
475 /// kmp_int64 up; // upper
476 /// kmp_int64 st; // stride
477 /// };
478 QualType KmpDimTy;
479
480 bool ShouldMarkAsGlobal = true;
481 /// List of the emitted declarations.
482 llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls;
483 /// List of the global variables with their addresses that should not be
484 /// emitted for the target.
485 llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables;
486
487 /// List of variables that can become declare target implicitly and, thus,
488 /// must be emitted.
489 llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;
490
491 using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>;
492 /// Stack for list of declarations in current context marked as nontemporal.
493 /// The set is the union of all current stack elements.
494 llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack;
495
496 using UntiedLocalVarsAddressesMap =
497 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
498 std::pair<Address, Address>>;
499 llvm::SmallVector<UntiedLocalVarsAddressesMap, 4> UntiedLocalVarsStack;
500
501 /// Stack for list of addresses of declarations in current context marked as
502 /// lastprivate conditional. The set is the union of all current stack
503 /// elements.
504 llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack;
505
506 /// Flag for keeping track of weather a requires unified_shared_memory
507 /// directive is present.
508 bool HasRequiresUnifiedSharedMemory = false;
509
510 /// Atomic ordering from the omp requires directive.
511 llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
512
513 /// Flag for keeping track of weather a target region has been emitted.
514 bool HasEmittedTargetRegion = false;
515
516 /// Flag for keeping track of weather a device routine has been emitted.
517 /// Device routines are specific to the
518 bool HasEmittedDeclareTargetRegion = false;
519
520 /// Start scanning from statement \a S and emit all target regions
521 /// found along the way.
522 /// \param S Starting statement.
523 /// \param ParentName Name of the function declaration that is being scanned.
524 void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName);
525
526 /// Build type kmp_routine_entry_t (if not built yet).
527 void emitKmpRoutineEntryT(QualType KmpInt32Ty);
528
529 /// If the specified mangled name is not in the module, create and
530 /// return threadprivate cache object. This object is a pointer's worth of
531 /// storage that's reserved for use by the OpenMP runtime.
532 /// \param VD Threadprivate variable.
533 /// \return Cache variable for the specified threadprivate.
534 llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD);
535
536 /// Set of threadprivate variables with the generated initializer.
537 llvm::StringSet<> ThreadPrivateWithDefinition;
538
539 /// Set of declare target variables with the generated initializer.
540 llvm::StringSet<> DeclareTargetWithDefinition;
541
542 /// Emits initialization code for the threadprivate variables.
543 /// \param VDAddr Address of the global variable \a VD.
544 /// \param Ctor Pointer to a global init function for \a VD.
545 /// \param CopyCtor Pointer to a global copy function for \a VD.
546 /// \param Dtor Pointer to a global destructor function for \a VD.
547 /// \param Loc Location of threadprivate declaration.
548 void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr,
549 llvm::Value *Ctor, llvm::Value *CopyCtor,
550 llvm::Value *Dtor, SourceLocation Loc);
551
552 struct TaskResultTy {
553 llvm::Value *NewTask = nullptr;
554 llvm::Function *TaskEntry = nullptr;
555 llvm::Value *NewTaskNewTaskTTy = nullptr;
556 LValue TDBase;
557 const RecordDecl *KmpTaskTQTyRD = nullptr;
558 llvm::Value *TaskDupFn = nullptr;
559 };
560 /// Emit task region for the task directive. The task region is emitted in
561 /// several steps:
562 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
563 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
564 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
565 /// function:
566 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
567 /// TaskFunction(gtid, tt->part_id, tt->shareds);
568 /// return 0;
569 /// }
570 /// 2. Copy a list of shared variables to field shareds of the resulting
571 /// structure kmp_task_t returned by the previous call (if any).
572 /// 3. Copy a pointer to destructions function to field destructions of the
573 /// resulting structure kmp_task_t.
574 /// \param D Current task directive.
575 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
576 /// /*part_id*/, captured_struct */*__context*/);
577 /// \param SharedsTy A type which contains references the shared variables.
578 /// \param Shareds Context with the list of shared variables from the \p
579 /// TaskFunction.
580 /// \param Data Additional data for task generation like tiednsee, final
581 /// state, list of privates etc.
582 TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
583 const OMPExecutableDirective &D,
584 llvm::Function *TaskFunction, QualType SharedsTy,
585 Address Shareds, const OMPTaskDataTy &Data);
586
587 /// Emit update for lastprivate conditional data.
588 void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,
589 StringRef UniqueDeclName, LValue LVal,
590 SourceLocation Loc);
591
592 /// Returns the number of the elements and the address of the depobj
593 /// dependency array.
594 /// \return Number of elements in depobj array and the pointer to the array of
595 /// dependencies.
596 std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF,
597 LValue DepobjLVal,
598 SourceLocation Loc);
599
600 SmallVector<llvm::Value *, 4>
601 emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
602 const OMPTaskDataTy::DependData &Data);
603
604 void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
605 LValue PosLVal, const OMPTaskDataTy::DependData &Data,
606 Address DependenciesArray);
607
608 /// Keep track of VTable Declarations so we don't register duplicate VTable.
609 llvm::SmallDenseMap<CXXRecordDecl *, const VarDecl *> VTableDeclMap;
610
611public:
612 explicit CGOpenMPRuntime(CodeGenModule &CGM);
613 virtual ~CGOpenMPRuntime() {}
614 virtual void clear();
615
616 /// Emits object of ident_t type with info for source location.
617 /// \param Flags Flags for OpenMP location.
618 /// \param EmitLoc emit source location with debug-info is off.
619 ///
620 llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc,
621 unsigned Flags = 0, bool EmitLoc = false);
622
623 /// Emit the number of teams for a target directive. Inspect the num_teams
624 /// clause associated with a teams construct combined or closely nested
625 /// with the target directive.
626 ///
627 /// Emit a team of size one for directives such as 'target parallel' that
628 /// have no associated teams construct.
629 ///
630 /// Otherwise, return nullptr.
631 const Expr *getNumTeamsExprForTargetDirective(CodeGenFunction &CGF,
632 const OMPExecutableDirective &D,
633 int32_t &MinTeamsVal,
634 int32_t &MaxTeamsVal);
635 llvm::Value *emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
636 const OMPExecutableDirective &D);
637
638 /// Check for a number of threads upper bound constant value (stored in \p
639 /// UpperBound), or expression (returned). If the value is conditional (via an
640 /// if-clause), store the condition in \p CondExpr. Similarly, a potential
641 /// thread limit expression is stored in \p ThreadLimitExpr. If \p
642 /// UpperBoundOnly is true, no expression evaluation is perfomed.
643 const Expr *getNumThreadsExprForTargetDirective(
644 CodeGenFunction &CGF, const OMPExecutableDirective &D,
645 int32_t &UpperBound, bool UpperBoundOnly,
646 llvm::Value **CondExpr = nullptr, const Expr **ThreadLimitExpr = nullptr);
647
648 /// Emit an expression that denotes the number of threads a target region
649 /// shall use. Will generate "i32 0" to allow the runtime to choose.
650 llvm::Value *
651 emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
652 const OMPExecutableDirective &D);
653
654 /// Return the trip count of loops associated with constructs / 'target teams
655 /// distribute' and 'teams distribute parallel for'. \param SizeEmitter Emits
656 /// the int64 value for the number of iterations of the associated loop.
657 llvm::Value *emitTargetNumIterationsCall(
658 CodeGenFunction &CGF, const OMPExecutableDirective &D,
659 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
660 const OMPLoopDirective &D)>
661 SizeEmitter);
662
663 /// Returns true if the current target is a GPU.
664 virtual bool isGPU() const { return false; }
665
666 /// Check if the variable length declaration is delayed:
667 virtual bool isDelayedVariableLengthDecl(CodeGenFunction &CGF,
668 const VarDecl *VD) const {
669 return false;
670 };
671
672 /// Get call to __kmpc_alloc_shared
673 virtual std::pair<llvm::Value *, llvm::Value *>
674 getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) {
675 llvm_unreachable("not implemented");
676 }
677
678 /// Get call to __kmpc_free_shared
679 virtual void getKmpcFreeShared(
680 CodeGenFunction &CGF,
681 const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) {
682 llvm_unreachable("not implemented");
683 }
684
685 /// Emits code for OpenMP 'if' clause using specified \a CodeGen
686 /// function. Here is the logic:
687 /// if (Cond) {
688 /// ThenGen();
689 /// } else {
690 /// ElseGen();
691 /// }
692 void emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
693 const RegionCodeGenTy &ThenGen,
694 const RegionCodeGenTy &ElseGen);
695
696 /// Checks if the \p Body is the \a CompoundStmt and returns its child
697 /// statement iff there is only one that is not evaluatable at the compile
698 /// time.
699 static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body);
700
701 /// Get the platform-specific name separator.
702 std::string getName(ArrayRef<StringRef> Parts) const;
703
704 /// Emit code for the specified user defined reduction construct.
705 virtual void emitUserDefinedReduction(CodeGenFunction *CGF,
706 const OMPDeclareReductionDecl *D);
707 /// Get combiner/initializer for the specified user-defined reduction, if any.
708 virtual std::pair<llvm::Function *, llvm::Function *>
709 getUserDefinedReduction(const OMPDeclareReductionDecl *D);
710
711 /// Emit the function for the user defined mapper construct.
712 void emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
713 CodeGenFunction *CGF = nullptr);
714 /// Get the function for the specified user-defined mapper. If it does not
715 /// exist, create one.
716 llvm::Function *
717 getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D);
718
719 /// Emits outlined function for the specified OpenMP parallel directive
720 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
721 /// kmp_int32 BoundID, struct context_vars*).
722 /// \param CGF Reference to current CodeGenFunction.
723 /// \param D OpenMP directive.
724 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
725 /// \param InnermostKind Kind of innermost directive (for simple directives it
726 /// is a directive itself, for combined - its innermost directive).
727 /// \param CodeGen Code generation sequence for the \a D directive.
728 virtual llvm::Function *emitParallelOutlinedFunction(
729 CodeGenFunction &CGF, const OMPExecutableDirective &D,
730 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
731 const RegionCodeGenTy &CodeGen);
732
733 /// Emits outlined function for the specified OpenMP teams directive
734 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
735 /// kmp_int32 BoundID, struct context_vars*).
736 /// \param CGF Reference to current CodeGenFunction.
737 /// \param D OpenMP directive.
738 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
739 /// \param InnermostKind Kind of innermost directive (for simple directives it
740 /// is a directive itself, for combined - its innermost directive).
741 /// \param CodeGen Code generation sequence for the \a D directive.
742 virtual llvm::Function *emitTeamsOutlinedFunction(
743 CodeGenFunction &CGF, const OMPExecutableDirective &D,
744 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
745 const RegionCodeGenTy &CodeGen);
746
747 /// Emits outlined function for the OpenMP task directive \a D. This
748 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
749 /// TaskT).
750 /// \param D OpenMP directive.
751 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
752 /// \param PartIDVar Variable for partition id in the current OpenMP untied
753 /// task region.
754 /// \param TaskTVar Variable for task_t argument.
755 /// \param InnermostKind Kind of innermost directive (for simple directives it
756 /// is a directive itself, for combined - its innermost directive).
757 /// \param CodeGen Code generation sequence for the \a D directive.
758 /// \param Tied true if task is generated for tied task, false otherwise.
759 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
760 /// tasks.
761 ///
762 virtual llvm::Function *emitTaskOutlinedFunction(
763 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
764 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
765 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
766 bool Tied, unsigned &NumberOfParts);
767
768 /// Cleans up references to the objects in finished function.
769 ///
770 virtual void functionFinished(CodeGenFunction &CGF);
771
772 /// Emits code for parallel or serial call of the \a OutlinedFn with
773 /// variables captured in a record which address is stored in \a
774 /// CapturedStruct.
775 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
776 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
777 /// \param CapturedVars A pointer to the record with the references to
778 /// variables used in \a OutlinedFn function.
779 /// \param IfCond Condition in the associated 'if' clause, if it was
780 /// specified, nullptr otherwise.
781 /// \param NumThreads The value corresponding to the num_threads clause, if
782 /// any, or nullptr.
783 /// \param NumThreadsModifier The modifier of the num_threads clause, if
784 /// any, ignored otherwise.
785 /// \param Severity The severity corresponding to the num_threads clause, if
786 /// any, ignored otherwise.
787 /// \param Message The message string corresponding to the num_threads clause,
788 /// if any, or nullptr.
789 ///
790 virtual void
791 emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
792 llvm::Function *OutlinedFn,
793 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
794 llvm::Value *NumThreads,
795 OpenMPNumThreadsClauseModifier NumThreadsModifier =
796 OMPC_NUMTHREADS_unknown,
797 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
798 const Expr *Message = nullptr);
799
800 /// Emits a critical region.
801 /// \param CriticalName Name of the critical region.
802 /// \param CriticalOpGen Generator for the statement associated with the given
803 /// critical region.
804 /// \param Hint Value of the 'hint' clause (optional).
805 virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
806 const RegionCodeGenTy &CriticalOpGen,
807 SourceLocation Loc,
808 const Expr *Hint = nullptr);
809
810 /// Emits a master region.
811 /// \param MasterOpGen Generator for the statement associated with the given
812 /// master region.
813 virtual void emitMasterRegion(CodeGenFunction &CGF,
814 const RegionCodeGenTy &MasterOpGen,
815 SourceLocation Loc);
816
817 /// Emits a masked region.
818 /// \param MaskedOpGen Generator for the statement associated with the given
819 /// masked region.
820 virtual void emitMaskedRegion(CodeGenFunction &CGF,
821 const RegionCodeGenTy &MaskedOpGen,
822 SourceLocation Loc,
823 const Expr *Filter = nullptr);
824
825 /// Emits code for a taskyield directive.
826 virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);
827
828 /// Emit __kmpc_error call for error directive
829 /// extern void __kmpc_error(ident_t *loc, int severity, const char *message);
830 virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME,
831 bool IsFatal);
832
833 /// Emit a taskgroup region.
834 /// \param TaskgroupOpGen Generator for the statement associated with the
835 /// given taskgroup region.
836 virtual void emitTaskgroupRegion(CodeGenFunction &CGF,
837 const RegionCodeGenTy &TaskgroupOpGen,
838 SourceLocation Loc);
839
840 /// Emits a single region.
841 /// \param SingleOpGen Generator for the statement associated with the given
842 /// single region.
843 virtual void emitSingleRegion(CodeGenFunction &CGF,
844 const RegionCodeGenTy &SingleOpGen,
845 SourceLocation Loc,
846 ArrayRef<const Expr *> CopyprivateVars,
847 ArrayRef<const Expr *> DestExprs,
848 ArrayRef<const Expr *> SrcExprs,
849 ArrayRef<const Expr *> AssignmentOps);
850
851 /// Emit an ordered region.
852 /// \param OrderedOpGen Generator for the statement associated with the given
853 /// ordered region.
854 virtual void emitOrderedRegion(CodeGenFunction &CGF,
855 const RegionCodeGenTy &OrderedOpGen,
856 SourceLocation Loc, bool IsThreads);
857
858 /// Emit an implicit/explicit barrier for OpenMP threads.
859 /// \param Kind Directive for which this implicit barrier call must be
860 /// generated. Must be OMPD_barrier for explicit barrier generation.
861 /// \param EmitChecks true if need to emit checks for cancellation barriers.
862 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
863 /// runtime class decides which one to emit (simple or with cancellation
864 /// checks).
865 ///
866 virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
867 OpenMPDirectiveKind Kind,
868 bool EmitChecks = true,
869 bool ForceSimpleCall = false);
870
871 /// Check if the specified \a ScheduleKind is static non-chunked.
872 /// This kind of worksharing directive is emitted without outer loop.
873 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
874 /// \param Chunked True if chunk is specified in the clause.
875 ///
876 virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
877 bool Chunked) const;
878
879 /// Check if the specified \a ScheduleKind is static non-chunked.
880 /// This kind of distribute directive is emitted without outer loop.
881 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
882 /// \param Chunked True if chunk is specified in the clause.
883 ///
884 virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind,
885 bool Chunked) const;
886
887 /// Check if the specified \a ScheduleKind is static chunked.
888 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
889 /// \param Chunked True if chunk is specified in the clause.
890 ///
891 virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
892 bool Chunked) const;
893
894 /// Check if the specified \a ScheduleKind is static non-chunked.
895 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
896 /// \param Chunked True if chunk is specified in the clause.
897 ///
898 virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind,
899 bool Chunked) const;
900
901 /// Check if the specified \a ScheduleKind is dynamic.
902 /// This kind of worksharing directive is emitted without outer loop.
903 /// \param ScheduleKind Schedule Kind specified in the 'schedule' clause.
904 ///
905 virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const;
906
907 /// struct with the values to be passed to the dispatch runtime function
908 struct DispatchRTInput {
909 /// Loop lower bound
910 llvm::Value *LB = nullptr;
911 /// Loop upper bound
912 llvm::Value *UB = nullptr;
913 /// Chunk size specified using 'schedule' clause (nullptr if chunk
914 /// was not specified)
915 llvm::Value *Chunk = nullptr;
916 DispatchRTInput() = default;
917 DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk)
918 : LB(LB), UB(UB), Chunk(Chunk) {}
919 };
920
921 /// Call the appropriate runtime routine to initialize it before start
922 /// of loop.
923
924 /// This is used for non static scheduled types and when the ordered
925 /// clause is present on the loop construct.
926 /// Depending on the loop schedule, it is necessary to call some runtime
927 /// routine before start of the OpenMP loop to get the loop upper / lower
928 /// bounds \a LB and \a UB and stride \a ST.
929 ///
930 /// \param CGF Reference to current CodeGenFunction.
931 /// \param Loc Clang source location.
932 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
933 /// \param IVSize Size of the iteration variable in bits.
934 /// \param IVSigned Sign of the iteration variable.
935 /// \param Ordered true if loop is ordered, false otherwise.
936 /// \param DispatchValues struct containing llvm values for lower bound, upper
937 /// bound, and chunk expression.
938 /// For the default (nullptr) value, the chunk 1 will be used.
939 ///
940 virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
941 const OpenMPScheduleTy &ScheduleKind,
942 unsigned IVSize, bool IVSigned, bool Ordered,
943 const DispatchRTInput &DispatchValues);
944
945 /// This is used for non static scheduled types and when the ordered
946 /// clause is present on the loop construct.
947 ///
948 /// \param CGF Reference to current CodeGenFunction.
949 /// \param Loc Clang source location.
950 ///
951 virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc);
952
953 /// Struct with the values to be passed to the static runtime function
954 struct StaticRTInput {
955 /// Size of the iteration variable in bits.
956 unsigned IVSize = 0;
957 /// Sign of the iteration variable.
958 bool IVSigned = false;
959 /// true if loop is ordered, false otherwise.
960 bool Ordered = false;
961 /// Address of the output variable in which the flag of the last iteration
962 /// is returned.
963 Address IL = Address::invalid();
964 /// Address of the output variable in which the lower iteration number is
965 /// returned.
966 Address LB = Address::invalid();
967 /// Address of the output variable in which the upper iteration number is
968 /// returned.
969 Address UB = Address::invalid();
970 /// Address of the output variable in which the stride value is returned
971 /// necessary to generated the static_chunked scheduled loop.
972 Address ST = Address::invalid();
973 /// Value of the chunk for the static_chunked scheduled loop. For the
974 /// default (nullptr) value, the chunk 1 will be used.
975 llvm::Value *Chunk = nullptr;
976 StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL,
977 Address LB, Address UB, Address ST,
978 llvm::Value *Chunk = nullptr)
979 : IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB),
980 UB(UB), ST(ST), Chunk(Chunk) {}
981 };
982 /// Call the appropriate runtime routine to initialize it before start
983 /// of loop.
984 ///
985 /// This is used only in case of static schedule, when the user did not
986 /// specify a ordered clause on the loop construct.
987 /// Depending on the loop schedule, it is necessary to call some runtime
988 /// routine before start of the OpenMP loop to get the loop upper / lower
989 /// bounds LB and UB and stride ST.
990 ///
991 /// \param CGF Reference to current CodeGenFunction.
992 /// \param Loc Clang source location.
993 /// \param DKind Kind of the directive.
994 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
995 /// \param Values Input arguments for the construct.
996 ///
997 virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
998 OpenMPDirectiveKind DKind,
999 const OpenMPScheduleTy &ScheduleKind,
1000 const StaticRTInput &Values);
1001
1002 ///
1003 /// \param CGF Reference to current CodeGenFunction.
1004 /// \param Loc Clang source location.
1005 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1006 /// \param Values Input arguments for the construct.
1007 ///
1008 virtual void emitDistributeStaticInit(CodeGenFunction &CGF,
1009 SourceLocation Loc,
1010 OpenMPDistScheduleClauseKind SchedKind,
1011 const StaticRTInput &Values);
1012
1013 /// Call the appropriate runtime routine to notify that we finished
1014 /// iteration of the ordered loop with the dynamic scheduling.
1015 ///
1016 /// \param CGF Reference to current CodeGenFunction.
1017 /// \param Loc Clang source location.
1018 /// \param IVSize Size of the iteration variable in bits.
1019 /// \param IVSigned Sign of the iteration variable.
1020 ///
1021 virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF,
1022 SourceLocation Loc, unsigned IVSize,
1023 bool IVSigned);
1024
1025 /// Call the appropriate runtime routine to notify that we finished
1026 /// all the work with current loop.
1027 ///
1028 /// \param CGF Reference to current CodeGenFunction.
1029 /// \param Loc Clang source location.
1030 /// \param DKind Kind of the directive for which the static finish is emitted.
1031 ///
1032 virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1033 OpenMPDirectiveKind DKind);
1034
1035 /// Call __kmpc_dispatch_next(
1036 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1037 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1038 /// kmp_int[32|64] *p_stride);
1039 /// \param IVSize Size of the iteration variable in bits.
1040 /// \param IVSigned Sign of the iteration variable.
1041 /// \param IL Address of the output variable in which the flag of the
1042 /// last iteration is returned.
1043 /// \param LB Address of the output variable in which the lower iteration
1044 /// number is returned.
1045 /// \param UB Address of the output variable in which the upper iteration
1046 /// number is returned.
1047 /// \param ST Address of the output variable in which the stride value is
1048 /// returned.
1049 virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1050 unsigned IVSize, bool IVSigned,
1051 Address IL, Address LB,
1052 Address UB, Address ST);
1053
1054 virtual llvm::Value *emitMessageClause(CodeGenFunction &CGF,
1055 const Expr *Message,
1056 SourceLocation Loc);
1057
1058 virtual llvm::Value *emitSeverityClause(OpenMPSeverityClauseKind Severity,
1059 SourceLocation Loc);
1060
1061 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1062 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1063 /// clause.
1064 /// If the modifier 'strict' is given:
1065 /// Emits call to void __kmpc_push_num_threads_strict(ident_t *loc, kmp_int32
1066 /// global_tid, kmp_int32 num_threads, int severity, const char *message) to
1067 /// generate code for 'num_threads' clause with 'strict' modifier.
1068 /// \param NumThreads An integer value of threads.
1069 virtual void emitNumThreadsClause(
1070 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
1071 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown,
1072 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
1073 SourceLocation SeverityLoc = SourceLocation(),
1074 const Expr *Message = nullptr,
1075 SourceLocation MessageLoc = SourceLocation());
1076
1077 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1078 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1079 virtual void emitProcBindClause(CodeGenFunction &CGF,
1080 llvm::omp::ProcBindKind ProcBind,
1081 SourceLocation Loc);
1082
1083 /// Returns address of the threadprivate variable for the current
1084 /// thread.
1085 /// \param VD Threadprivate variable.
1086 /// \param VDAddr Address of the global variable \a VD.
1087 /// \param Loc Location of the reference to threadprivate var.
1088 /// \return Address of the threadprivate variable for the current thread.
1089 virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1090 const VarDecl *VD, Address VDAddr,
1091 SourceLocation Loc);
1092
1093 /// Returns the address of the variable marked as declare target with link
1094 /// clause OR as declare target with to clause and unified memory.
1095 virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD);
1096
1097 /// Emit a code for initialization of threadprivate variable. It emits
1098 /// a call to runtime library which adds initial value to the newly created
1099 /// threadprivate variable (if it is not constant) and registers destructor
1100 /// for the variable (if any).
1101 /// \param VD Threadprivate variable.
1102 /// \param VDAddr Address of the global variable \a VD.
1103 /// \param Loc Location of threadprivate declaration.
1104 /// \param PerformInit true if initialization expression is not constant.
1105 virtual llvm::Function *
1106 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1107 SourceLocation Loc, bool PerformInit,
1108 CodeGenFunction *CGF = nullptr);
1109
1110 /// Emit code for handling declare target functions in the runtime.
1111 /// \param FD Declare target function.
1112 /// \param Addr Address of the global \a FD.
1113 /// \param PerformInit true if initialization expression is not constant.
1114 virtual void emitDeclareTargetFunction(const FunctionDecl *FD,
1115 llvm::GlobalValue *GV);
1116
1117 /// Register VTable to OpenMP offload entry.
1118 /// \param VTable VTable of the C++ class.
1119 /// \param RD C++ class decl.
1120 virtual void registerVTableOffloadEntry(llvm::GlobalVariable *VTable,
1121 const VarDecl *VD);
1122 /// Emit code for registering vtable by scanning through map clause
1123 /// in OpenMP target region.
1124 /// \param D OpenMP target directive.
1125 virtual void registerVTable(const OMPExecutableDirective &D);
1126
1127 /// Emit and register VTable for the C++ class in OpenMP offload entry.
1128 /// \param CXXRecord C++ class decl.
1129 /// \param VD Variable decl which holds VTable.
1130 virtual void emitAndRegisterVTable(CodeGenModule &CGM,
1131 CXXRecordDecl *CXXRecord,
1132 const VarDecl *VD);
1133
1134 /// Creates artificial threadprivate variable with name \p Name and type \p
1135 /// VarType.
1136 /// \param VarType Type of the artificial threadprivate variable.
1137 /// \param Name Name of the artificial threadprivate variable.
1138 virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1139 QualType VarType,
1140 StringRef Name);
1141
1142 /// Emit flush of the variables specified in 'omp flush' directive.
1143 /// \param Vars List of variables to flush.
1144 virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1145 SourceLocation Loc, llvm::AtomicOrdering AO);
1146
1147 /// Emit task region for the task directive. The task region is
1148 /// emitted in several steps:
1149 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1150 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1151 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1152 /// function:
1153 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1154 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1155 /// return 0;
1156 /// }
1157 /// 2. Copy a list of shared variables to field shareds of the resulting
1158 /// structure kmp_task_t returned by the previous call (if any).
1159 /// 3. Copy a pointer to destructions function to field destructions of the
1160 /// resulting structure kmp_task_t.
1161 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1162 /// kmp_task_t *new_task), where new_task is a resulting structure from
1163 /// previous items.
1164 /// \param D Current task directive.
1165 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1166 /// /*part_id*/, captured_struct */*__context*/);
1167 /// \param SharedsTy A type which contains references the shared variables.
1168 /// \param Shareds Context with the list of shared variables from the \p
1169 /// TaskFunction.
1170 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1171 /// otherwise.
1172 /// \param Data Additional data for task generation like tiednsee, final
1173 /// state, list of privates etc.
1174 virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1175 const OMPExecutableDirective &D,
1176 llvm::Function *TaskFunction, QualType SharedsTy,
1177 Address Shareds, const Expr *IfCond,
1178 const OMPTaskDataTy &Data);
1179
1180 /// Emit task region for the taskloop directive. The taskloop region is
1181 /// emitted in several steps:
1182 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1183 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1184 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1185 /// function:
1186 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1187 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1188 /// return 0;
1189 /// }
1190 /// 2. Copy a list of shared variables to field shareds of the resulting
1191 /// structure kmp_task_t returned by the previous call (if any).
1192 /// 3. Copy a pointer to destructions function to field destructions of the
1193 /// resulting structure kmp_task_t.
1194 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
1195 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
1196 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
1197 /// is a resulting structure from
1198 /// previous items.
1199 /// \param D Current task directive.
1200 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1201 /// /*part_id*/, captured_struct */*__context*/);
1202 /// \param SharedsTy A type which contains references the shared variables.
1203 /// \param Shareds Context with the list of shared variables from the \p
1204 /// TaskFunction.
1205 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1206 /// otherwise.
1207 /// \param Data Additional data for task generation like tiednsee, final
1208 /// state, list of privates etc.
1209 virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
1210 const OMPLoopDirective &D,
1211 llvm::Function *TaskFunction,
1212 QualType SharedsTy, Address Shareds,
1213 const Expr *IfCond, const OMPTaskDataTy &Data);
1214
1215 /// Emit code for the directive that does not require outlining.
1216 ///
1217 /// \param InnermostKind Kind of innermost directive (for simple directives it
1218 /// is a directive itself, for combined - its innermost directive).
1219 /// \param CodeGen Code generation sequence for the \a D directive.
1220 /// \param HasCancel true if region has inner cancel directive, false
1221 /// otherwise.
1222 virtual void emitInlinedDirective(CodeGenFunction &CGF,
1223 OpenMPDirectiveKind InnermostKind,
1224 const RegionCodeGenTy &CodeGen,
1225 bool HasCancel = false);
1226
1227 /// Emits reduction function.
1228 /// \param ReducerName Name of the function calling the reduction.
1229 /// \param ArgsElemType Array type containing pointers to reduction variables.
1230 /// \param Privates List of private copies for original reduction arguments.
1231 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1232 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1233 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1234 /// or 'operator binop(LHS, RHS)'.
1235 llvm::Function *emitReductionFunction(
1236 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
1237 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
1238 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps);
1239
1240 /// Emits single reduction combiner
1241 void emitSingleReductionCombiner(CodeGenFunction &CGF,
1242 const Expr *ReductionOp,
1243 const Expr *PrivateRef,
1244 const DeclRefExpr *LHS,
1245 const DeclRefExpr *RHS);
1246
1247 struct ReductionOptionsTy {
1248 bool WithNowait;
1249 bool SimpleReduction;
1250 llvm::SmallVector<bool, 8> IsPrivateVarReduction;
1251 OpenMPDirectiveKind ReductionKind;
1252 };
1253
1254 /// Emits code for private variable reduction
1255 /// \param Privates List of private copies for original reduction arguments.
1256 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1257 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1258 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1259 /// or 'operator binop(LHS, RHS)'.
1260 void emitPrivateReduction(CodeGenFunction &CGF, SourceLocation Loc,
1261 const Expr *Privates, const Expr *LHSExprs,
1262 const Expr *RHSExprs, const Expr *ReductionOps);
1263
1264 /// Emit a code for reduction clause. Next code should be emitted for
1265 /// reduction:
1266 /// \code
1267 ///
1268 /// static kmp_critical_name lock = { 0 };
1269 ///
1270 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
1271 /// ...
1272 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
1273 /// ...
1274 /// }
1275 ///
1276 /// ...
1277 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
1278 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
1279 /// RedList, reduce_func, &<lock>)) {
1280 /// case 1:
1281 /// ...
1282 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
1283 /// ...
1284 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
1285 /// break;
1286 /// case 2:
1287 /// ...
1288 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
1289 /// ...
1290 /// break;
1291 /// default:;
1292 /// }
1293 /// \endcode
1294 ///
1295 /// \param Privates List of private copies for original reduction arguments.
1296 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1297 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1298 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1299 /// or 'operator binop(LHS, RHS)'.
1300 /// \param Options List of options for reduction codegen:
1301 /// WithNowait true if parent directive has also nowait clause, false
1302 /// otherwise.
1303 /// SimpleReduction Emit reduction operation only. Used for omp simd
1304 /// directive on the host.
1305 /// ReductionKind The kind of reduction to perform.
1306 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
1307 ArrayRef<const Expr *> Privates,
1308 ArrayRef<const Expr *> LHSExprs,
1309 ArrayRef<const Expr *> RHSExprs,
1310 ArrayRef<const Expr *> ReductionOps,
1311 ReductionOptionsTy Options);
1312
1313 /// Emit a code for initialization of task reduction clause. Next code
1314 /// should be emitted for reduction:
1315 /// \code
1316 ///
1317 /// _taskred_item_t red_data[n];
1318 /// ...
1319 /// red_data[i].shar = &shareds[i];
1320 /// red_data[i].orig = &origs[i];
1321 /// red_data[i].size = sizeof(origs[i]);
1322 /// red_data[i].f_init = (void*)RedInit<i>;
1323 /// red_data[i].f_fini = (void*)RedDest<i>;
1324 /// red_data[i].f_comb = (void*)RedOp<i>;
1325 /// red_data[i].flags = <Flag_i>;
1326 /// ...
1327 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
1328 /// \endcode
1329 /// For reduction clause with task modifier it emits the next call:
1330 /// \code
1331 ///
1332 /// _taskred_item_t red_data[n];
1333 /// ...
1334 /// red_data[i].shar = &shareds[i];
1335 /// red_data[i].orig = &origs[i];
1336 /// red_data[i].size = sizeof(origs[i]);
1337 /// red_data[i].f_init = (void*)RedInit<i>;
1338 /// red_data[i].f_fini = (void*)RedDest<i>;
1339 /// red_data[i].f_comb = (void*)RedOp<i>;
1340 /// red_data[i].flags = <Flag_i>;
1341 /// ...
1342 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
1343 /// red_data);
1344 /// \endcode
1345 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
1346 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
1347 /// \param Data Additional data for task generation like tiedness, final
1348 /// state, list of privates, reductions etc.
1349 virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF,
1350 SourceLocation Loc,
1351 ArrayRef<const Expr *> LHSExprs,
1352 ArrayRef<const Expr *> RHSExprs,
1353 const OMPTaskDataTy &Data);
1354
1355 /// Emits the following code for reduction clause with task modifier:
1356 /// \code
1357 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
1358 /// \endcode
1359 virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
1360 bool IsWorksharingReduction);
1361
1362 /// Required to resolve existing problems in the runtime. Emits threadprivate
1363 /// variables to store the size of the VLAs/array sections for
1364 /// initializer/combiner/finalizer functions.
1365 /// \param RCG Allows to reuse an existing data for the reductions.
1366 /// \param N Reduction item for which fixups must be emitted.
1367 virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
1368 ReductionCodeGen &RCG, unsigned N);
1369
1370 /// Get the address of `void *` type of the privatue copy of the reduction
1371 /// item specified by the \p SharedLVal.
1372 /// \param ReductionsPtr Pointer to the reduction data returned by the
1373 /// emitTaskReductionInit function.
1374 /// \param SharedLVal Address of the original reduction item.
1375 virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
1376 llvm::Value *ReductionsPtr,
1377 LValue SharedLVal);
1378
1379 /// Emit code for 'taskwait' directive.
1380 virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
1381 const OMPTaskDataTy &Data);
1382
1383 /// Emit code for 'cancellation point' construct.
1384 /// \param CancelRegion Region kind for which the cancellation point must be
1385 /// emitted.
1386 ///
1387 virtual void emitCancellationPointCall(CodeGenFunction &CGF,
1388 SourceLocation Loc,
1389 OpenMPDirectiveKind CancelRegion);
1390
1391 /// Emit code for 'cancel' construct.
1392 /// \param IfCond Condition in the associated 'if' clause, if it was
1393 /// specified, nullptr otherwise.
1394 /// \param CancelRegion Region kind for which the cancel must be emitted.
1395 ///
1396 virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
1397 const Expr *IfCond,
1398 OpenMPDirectiveKind CancelRegion);
1399
1400 /// Emit outilined function for 'target' directive.
1401 /// \param D Directive to emit.
1402 /// \param ParentName Name of the function that encloses the target region.
1403 /// \param OutlinedFn Outlined function value to be defined by this call.
1404 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
1405 /// \param IsOffloadEntry True if the outlined function is an offload entry.
1406 /// \param CodeGen Code generation sequence for the \a D directive.
1407 /// An outlined function may not be an entry if, e.g. the if clause always
1408 /// evaluates to false.
1409 virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
1410 StringRef ParentName,
1411 llvm::Function *&OutlinedFn,
1412 llvm::Constant *&OutlinedFnID,
1413 bool IsOffloadEntry,
1414 const RegionCodeGenTy &CodeGen);
1415
1416 /// Emit the target offloading code associated with \a D. The emitted
1417 /// code attempts offloading the execution to the device, an the event of
1418 /// a failure it executes the host version outlined in \a OutlinedFn.
1419 /// \param D Directive to emit.
1420 /// \param OutlinedFn Host version of the code to be offloaded.
1421 /// \param OutlinedFnID ID of host version of the code to be offloaded.
1422 /// \param IfCond Expression evaluated in if clause associated with the target
1423 /// directive, or null if no if clause is used.
1424 /// \param Device Expression evaluated in device clause associated with the
1425 /// target directive, or null if no device clause is used and device modifier.
1426 /// \param SizeEmitter Callback to emit number of iterations for loop-based
1427 /// directives.
1428 virtual void emitTargetCall(
1429 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1430 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
1431 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
1432 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
1433 const OMPLoopDirective &D)>
1434 SizeEmitter);
1435
1436 /// Emit the target regions enclosed in \a GD function definition or
1437 /// the function itself in case it is a valid device function. Returns true if
1438 /// \a GD was dealt with successfully.
1439 /// \param GD Function to scan.
1440 virtual bool emitTargetFunctions(GlobalDecl GD);
1441
1442 /// Emit the global variable if it is a valid device global variable.
1443 /// Returns true if \a GD was dealt with successfully.
1444 /// \param GD Variable declaration to emit.
1445 virtual bool emitTargetGlobalVariable(GlobalDecl GD);
1446
1447 /// Checks if the provided global decl \a GD is a declare target variable and
1448 /// registers it when emitting code for the host.
1449 virtual void registerTargetGlobalVariable(const VarDecl *VD,
1450 llvm::Constant *Addr);
1451
1452 /// Emit the global \a GD if it is meaningful for the target. Returns
1453 /// if it was emitted successfully.
1454 /// \param GD Global to scan.
1455 virtual bool emitTargetGlobal(GlobalDecl GD);
1456
1457 /// Creates all the offload entries in the current compilation unit
1458 /// along with the associated metadata.
1459 void createOffloadEntriesAndInfoMetadata();
1460
1461 /// Emits code for teams call of the \a OutlinedFn with
1462 /// variables captured in a record which address is stored in \a
1463 /// CapturedStruct.
1464 /// \param OutlinedFn Outlined function to be run by team masters. Type of
1465 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1466 /// \param CapturedVars A pointer to the record with the references to
1467 /// variables used in \a OutlinedFn function.
1468 ///
1469 virtual void emitTeamsCall(CodeGenFunction &CGF,
1470 const OMPExecutableDirective &D,
1471 SourceLocation Loc, llvm::Function *OutlinedFn,
1472 ArrayRef<llvm::Value *> CapturedVars);
1473
1474 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
1475 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
1476 /// for num_teams clause.
1477 /// \param NumTeams An integer expression of teams.
1478 /// \param ThreadLimit An integer expression of threads.
1479 virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
1480 const Expr *ThreadLimit, SourceLocation Loc);
1481
1482 /// Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32
1483 /// global_tid, kmp_int32 thread_limit) to generate code for
1484 /// thread_limit clause on target directive
1485 /// \param ThreadLimit An integer expression of threads.
1486 virtual void emitThreadLimitClause(CodeGenFunction &CGF,
1487 const Expr *ThreadLimit,
1488 SourceLocation Loc);
1489
1490 /// Struct that keeps all the relevant information that should be kept
1491 /// throughout a 'target data' region.
1492 class TargetDataInfo : public llvm::OpenMPIRBuilder::TargetDataInfo {
1493 public:
1494 explicit TargetDataInfo() : llvm::OpenMPIRBuilder::TargetDataInfo() {}
1495 explicit TargetDataInfo(bool RequiresDevicePointerInfo,
1496 bool SeparateBeginEndCalls)
1497 : llvm::OpenMPIRBuilder::TargetDataInfo(RequiresDevicePointerInfo,
1498 SeparateBeginEndCalls) {}
1499 /// Map between the a declaration of a capture and the corresponding new
1500 /// llvm address where the runtime returns the device pointers.
1501 llvm::DenseMap<const ValueDecl *, llvm::Value *> CaptureDeviceAddrMap;
1502 };
1503
1504 /// Emit the target data mapping code associated with \a D.
1505 /// \param D Directive to emit.
1506 /// \param IfCond Expression evaluated in if clause associated with the
1507 /// target directive, or null if no device clause is used.
1508 /// \param Device Expression evaluated in device clause associated with the
1509 /// target directive, or null if no device clause is used.
1510 /// \param Info A record used to store information that needs to be preserved
1511 /// until the region is closed.
1512 virtual void emitTargetDataCalls(CodeGenFunction &CGF,
1513 const OMPExecutableDirective &D,
1514 const Expr *IfCond, const Expr *Device,
1515 const RegionCodeGenTy &CodeGen,
1516 CGOpenMPRuntime::TargetDataInfo &Info);
1517
1518 /// Emit the data mapping/movement code associated with the directive
1519 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
1520 /// \param D Directive to emit.
1521 /// \param IfCond Expression evaluated in if clause associated with the target
1522 /// directive, or null if no if clause is used.
1523 /// \param Device Expression evaluated in device clause associated with the
1524 /// target directive, or null if no device clause is used.
1525 virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
1526 const OMPExecutableDirective &D,
1527 const Expr *IfCond,
1528 const Expr *Device);
1529
1530 /// Marks function \a Fn with properly mangled versions of vector functions.
1531 /// \param FD Function marked as 'declare simd'.
1532 /// \param Fn LLVM function that must be marked with 'declare simd'
1533 /// attributes.
1534 virtual void emitDeclareSimdFunction(const FunctionDecl *FD,
1535 llvm::Function *Fn);
1536
1537 /// Emit initialization for doacross loop nesting support.
1538 /// \param D Loop-based construct used in doacross nesting construct.
1539 virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
1540 ArrayRef<Expr *> NumIterations);
1541
1542 /// Emit code for doacross ordered directive with 'depend' clause.
1543 /// \param C 'depend' clause with 'sink|source' dependency kind.
1544 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1545 const OMPDependClause *C);
1546
1547 /// Emit code for doacross ordered directive with 'doacross' clause.
1548 /// \param C 'doacross' clause with 'sink|source' dependence type.
1549 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1550 const OMPDoacrossClause *C);
1551
1552 /// Translates the native parameter of outlined function if this is required
1553 /// for target.
1554 /// \param FD Field decl from captured record for the parameter.
1555 /// \param NativeParam Parameter itself.
1556 virtual const VarDecl *translateParameter(const FieldDecl *FD,
1557 const VarDecl *NativeParam) const {
1558 return NativeParam;
1559 }
1560
1561 /// Gets the address of the native argument basing on the address of the
1562 /// target-specific parameter.
1563 /// \param NativeParam Parameter itself.
1564 /// \param TargetParam Corresponding target-specific parameter.
1565 virtual Address getParameterAddress(CodeGenFunction &CGF,
1566 const VarDecl *NativeParam,
1567 const VarDecl *TargetParam) const;
1568
1569 /// Choose default schedule type and chunk value for the
1570 /// dist_schedule clause.
1571 virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
1572 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
1573 llvm::Value *&Chunk) const {}
1574
1575 /// Choose default schedule type and chunk value for the
1576 /// schedule clause.
1577 virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
1578 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
1579 const Expr *&ChunkExpr) const;
1580
1581 /// Emits call of the outlined function with the provided arguments,
1582 /// translating these arguments to correct target-specific arguments.
1583 virtual void
1584 emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
1585 llvm::FunctionCallee OutlinedFn,
1586 ArrayRef<llvm::Value *> Args = {}) const;
1587
1588 /// Emits OpenMP-specific function prolog.
1589 /// Required for device constructs.
1590 virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D);
1591
1592 /// Gets the OpenMP-specific address of the local variable.
1593 virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1594 const VarDecl *VD);
1595
1596 /// Marks the declaration as already emitted for the device code and returns
1597 /// true, if it was marked already, and false, otherwise.
1598 bool markAsGlobalTarget(GlobalDecl GD);
1599
1600 /// Emit deferred declare target variables marked for deferred emission.
1601 void emitDeferredTargetDecls() const;
1602
1603 /// Adjust some parameters for the target-based directives, like addresses of
1604 /// the variables captured by reference in lambdas.
1605 virtual void
1606 adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF,
1607 const OMPExecutableDirective &D) const;
1608
1609 /// Perform check on requires decl to ensure that target architecture
1610 /// supports unified addressing
1611 virtual void processRequiresDirective(const OMPRequiresDecl *D);
1612
1613 /// Gets default memory ordering as specified in requires directive.
1614 llvm::AtomicOrdering getDefaultMemoryOrdering() const;
1615
1616 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with
1617 /// the predefined allocator and translates it into the corresponding address
1618 /// space.
1619 virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS);
1620
1621 /// Return whether the unified_shared_memory has been specified.
1622 bool hasRequiresUnifiedSharedMemory() const;
1623
1624 /// Checks if the \p VD variable is marked as nontemporal declaration in
1625 /// current context.
1626 bool isNontemporalDecl(const ValueDecl *VD) const;
1627
1628 /// Create specialized alloca to handle lastprivate conditionals.
1629 Address emitLastprivateConditionalInit(CodeGenFunction &CGF,
1630 const VarDecl *VD);
1631
1632 /// Checks if the provided \p LVal is lastprivate conditional and emits the
1633 /// code to update the value of the original variable.
1634 /// \code
1635 /// lastprivate(conditional: a)
1636 /// ...
1637 /// <type> a;
1638 /// lp_a = ...;
1639 /// #pragma omp critical(a)
1640 /// if (last_iv_a <= iv) {
1641 /// last_iv_a = iv;
1642 /// global_a = lp_a;
1643 /// }
1644 /// \endcode
1645 virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
1646 const Expr *LHS);
1647
1648 /// Checks if the lastprivate conditional was updated in inner region and
1649 /// writes the value.
1650 /// \code
1651 /// lastprivate(conditional: a)
1652 /// ...
1653 /// <type> a;bool Fired = false;
1654 /// #pragma omp ... shared(a)
1655 /// {
1656 /// lp_a = ...;
1657 /// Fired = true;
1658 /// }
1659 /// if (Fired) {
1660 /// #pragma omp critical(a)
1661 /// if (last_iv_a <= iv) {
1662 /// last_iv_a = iv;
1663 /// global_a = lp_a;
1664 /// }
1665 /// Fired = false;
1666 /// }
1667 /// \endcode
1668 virtual void checkAndEmitSharedLastprivateConditional(
1669 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1670 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls);
1671
1672 /// Gets the address of the global copy used for lastprivate conditional
1673 /// update, if any.
1674 /// \param PrivLVal LValue for the private copy.
1675 /// \param VD Original lastprivate declaration.
1676 virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF,
1677 LValue PrivLVal,
1678 const VarDecl *VD,
1679 SourceLocation Loc);
1680
1681 /// Emits list of dependecies based on the provided data (array of
1682 /// dependence/expression pairs).
1683 /// \returns Pointer to the first element of the array casted to VoidPtr type.
1684 std::pair<llvm::Value *, Address>
1685 emitDependClause(CodeGenFunction &CGF,
1686 ArrayRef<OMPTaskDataTy::DependData> Dependencies,
1687 SourceLocation Loc);
1688
1689 /// Emits list of dependecies based on the provided data (array of
1690 /// dependence/expression pairs) for depobj construct. In this case, the
1691 /// variable is allocated in dynamically. \returns Pointer to the first
1692 /// element of the array casted to VoidPtr type.
1693 Address emitDepobjDependClause(CodeGenFunction &CGF,
1694 const OMPTaskDataTy::DependData &Dependencies,
1695 SourceLocation Loc);
1696
1697 /// Emits the code to destroy the dependency object provided in depobj
1698 /// directive.
1699 void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
1700 SourceLocation Loc);
1701
1702 /// Updates the dependency kind in the specified depobj object.
1703 /// \param DepobjLVal LValue for the main depobj object.
1704 /// \param NewDepKind New dependency kind.
1705 void emitUpdateDependObjectsClause(CodeGenFunction &CGF, LValue DepobjLVal,
1706 OpenMPDependClauseKind NewDepKind,
1707 SourceLocation Loc);
1708
1709 /// Initializes user defined allocators specified in the uses_allocators
1710 /// clauses.
1711 void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator,
1712 const Expr *AllocatorTraits);
1713
1714 /// Destroys user defined allocators specified in the uses_allocators clause.
1715 void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator);
1716
1717 /// Returns true if the variable is a local variable in untied task.
1718 bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const;
1719};
1720
1721/// Class supports emissionof SIMD-only code.
1722class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
1723public:
1724 explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {}
1725 ~CGOpenMPSIMDRuntime() override {}
1726
1727 /// Emits outlined function for the specified OpenMP parallel directive
1728 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1729 /// kmp_int32 BoundID, struct context_vars*).
1730 /// \param CGF Reference to current CodeGenFunction.
1731 /// \param D OpenMP directive.
1732 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1733 /// \param InnermostKind Kind of innermost directive (for simple directives it
1734 /// is a directive itself, for combined - its innermost directive).
1735 /// \param CodeGen Code generation sequence for the \a D directive.
1736 llvm::Function *emitParallelOutlinedFunction(
1737 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1738 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1739 const RegionCodeGenTy &CodeGen) override;
1740
1741 /// Emits outlined function for the specified OpenMP teams directive
1742 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1743 /// kmp_int32 BoundID, struct context_vars*).
1744 /// \param CGF Reference to current CodeGenFunction.
1745 /// \param D OpenMP directive.
1746 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1747 /// \param InnermostKind Kind of innermost directive (for simple directives it
1748 /// is a directive itself, for combined - its innermost directive).
1749 /// \param CodeGen Code generation sequence for the \a D directive.
1750 llvm::Function *emitTeamsOutlinedFunction(
1751 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1752 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1753 const RegionCodeGenTy &CodeGen) override;
1754
1755 /// Emits outlined function for the OpenMP task directive \a D. This
1756 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
1757 /// TaskT).
1758 /// \param D OpenMP directive.
1759 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1760 /// \param PartIDVar Variable for partition id in the current OpenMP untied
1761 /// task region.
1762 /// \param TaskTVar Variable for task_t argument.
1763 /// \param InnermostKind Kind of innermost directive (for simple directives it
1764 /// is a directive itself, for combined - its innermost directive).
1765 /// \param CodeGen Code generation sequence for the \a D directive.
1766 /// \param Tied true if task is generated for tied task, false otherwise.
1767 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
1768 /// tasks.
1769 ///
1770 llvm::Function *emitTaskOutlinedFunction(
1771 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1772 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1773 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1774 bool Tied, unsigned &NumberOfParts) override;
1775
1776 /// Emits code for parallel or serial call of the \a OutlinedFn with
1777 /// variables captured in a record which address is stored in \a
1778 /// CapturedStruct.
1779 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
1780 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1781 /// \param CapturedVars A pointer to the record with the references to
1782 /// variables used in \a OutlinedFn function.
1783 /// \param IfCond Condition in the associated 'if' clause, if it was
1784 /// specified, nullptr otherwise.
1785 /// \param NumThreads The value corresponding to the num_threads clause, if
1786 /// any, or nullptr.
1787 /// \param NumThreadsModifier The modifier of the num_threads clause, if
1788 /// any, ignored otherwise.
1789 /// \param Severity The severity corresponding to the num_threads clause, if
1790 /// any, ignored otherwise.
1791 /// \param Message The message string corresponding to the num_threads clause,
1792 /// if any, or nullptr.
1793 ///
1794 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1795 llvm::Function *OutlinedFn,
1796 ArrayRef<llvm::Value *> CapturedVars,
1797 const Expr *IfCond, llvm::Value *NumThreads,
1798 OpenMPNumThreadsClauseModifier NumThreadsModifier =
1799 OMPC_NUMTHREADS_unknown,
1800 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
1801 const Expr *Message = nullptr) override;
1802
1803 /// Emits a critical region.
1804 /// \param CriticalName Name of the critical region.
1805 /// \param CriticalOpGen Generator for the statement associated with the given
1806 /// critical region.
1807 /// \param Hint Value of the 'hint' clause (optional).
1808 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
1809 const RegionCodeGenTy &CriticalOpGen,
1810 SourceLocation Loc,
1811 const Expr *Hint = nullptr) override;
1812
1813 /// Emits a master region.
1814 /// \param MasterOpGen Generator for the statement associated with the given
1815 /// master region.
1816 void emitMasterRegion(CodeGenFunction &CGF,
1817 const RegionCodeGenTy &MasterOpGen,
1818 SourceLocation Loc) override;
1819
1820 /// Emits a masked region.
1821 /// \param MaskedOpGen Generator for the statement associated with the given
1822 /// masked region.
1823 void emitMaskedRegion(CodeGenFunction &CGF,
1824 const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc,
1825 const Expr *Filter = nullptr) override;
1826
1827 /// Emits a masked region.
1828 /// \param MaskedOpGen Generator for the statement associated with the given
1829 /// masked region.
1830
1831 /// Emits code for a taskyield directive.
1832 void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override;
1833
1834 /// Emit a taskgroup region.
1835 /// \param TaskgroupOpGen Generator for the statement associated with the
1836 /// given taskgroup region.
1837 void emitTaskgroupRegion(CodeGenFunction &CGF,
1838 const RegionCodeGenTy &TaskgroupOpGen,
1839 SourceLocation Loc) override;
1840
1841 /// Emits a single region.
1842 /// \param SingleOpGen Generator for the statement associated with the given
1843 /// single region.
1844 void emitSingleRegion(CodeGenFunction &CGF,
1845 const RegionCodeGenTy &SingleOpGen, SourceLocation Loc,
1846 ArrayRef<const Expr *> CopyprivateVars,
1847 ArrayRef<const Expr *> DestExprs,
1848 ArrayRef<const Expr *> SrcExprs,
1849 ArrayRef<const Expr *> AssignmentOps) override;
1850
1851 /// Emit an ordered region.
1852 /// \param OrderedOpGen Generator for the statement associated with the given
1853 /// ordered region.
1854 void emitOrderedRegion(CodeGenFunction &CGF,
1855 const RegionCodeGenTy &OrderedOpGen,
1856 SourceLocation Loc, bool IsThreads) override;
1857
1858 /// Emit an implicit/explicit barrier for OpenMP threads.
1859 /// \param Kind Directive for which this implicit barrier call must be
1860 /// generated. Must be OMPD_barrier for explicit barrier generation.
1861 /// \param EmitChecks true if need to emit checks for cancellation barriers.
1862 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
1863 /// runtime class decides which one to emit (simple or with cancellation
1864 /// checks).
1865 ///
1866 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
1867 OpenMPDirectiveKind Kind, bool EmitChecks = true,
1868 bool ForceSimpleCall = false) override;
1869
1870 /// This is used for non static scheduled types and when the ordered
1871 /// clause is present on the loop construct.
1872 /// Depending on the loop schedule, it is necessary to call some runtime
1873 /// routine before start of the OpenMP loop to get the loop upper / lower
1874 /// bounds \a LB and \a UB and stride \a ST.
1875 ///
1876 /// \param CGF Reference to current CodeGenFunction.
1877 /// \param Loc Clang source location.
1878 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1879 /// \param IVSize Size of the iteration variable in bits.
1880 /// \param IVSigned Sign of the iteration variable.
1881 /// \param Ordered true if loop is ordered, false otherwise.
1882 /// \param DispatchValues struct containing llvm values for lower bound, upper
1883 /// bound, and chunk expression.
1884 /// For the default (nullptr) value, the chunk 1 will be used.
1885 ///
1886 void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
1887 const OpenMPScheduleTy &ScheduleKind,
1888 unsigned IVSize, bool IVSigned, bool Ordered,
1889 const DispatchRTInput &DispatchValues) override;
1890
1891 /// This is used for non static scheduled types and when the ordered
1892 /// clause is present on the loop construct.
1893 ///
1894 /// \param CGF Reference to current CodeGenFunction.
1895 /// \param Loc Clang source location.
1896 ///
1897 void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override;
1898
1899 /// Call the appropriate runtime routine to initialize it before start
1900 /// of loop.
1901 ///
1902 /// This is used only in case of static schedule, when the user did not
1903 /// specify a ordered clause on the loop construct.
1904 /// Depending on the loop schedule, it is necessary to call some runtime
1905 /// routine before start of the OpenMP loop to get the loop upper / lower
1906 /// bounds LB and UB and stride ST.
1907 ///
1908 /// \param CGF Reference to current CodeGenFunction.
1909 /// \param Loc Clang source location.
1910 /// \param DKind Kind of the directive.
1911 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1912 /// \param Values Input arguments for the construct.
1913 ///
1914 void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1915 OpenMPDirectiveKind DKind,
1916 const OpenMPScheduleTy &ScheduleKind,
1917 const StaticRTInput &Values) override;
1918
1919 ///
1920 /// \param CGF Reference to current CodeGenFunction.
1921 /// \param Loc Clang source location.
1922 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1923 /// \param Values Input arguments for the construct.
1924 ///
1925 void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1926 OpenMPDistScheduleClauseKind SchedKind,
1927 const StaticRTInput &Values) override;
1928
1929 /// Call the appropriate runtime routine to notify that we finished
1930 /// iteration of the ordered loop with the dynamic scheduling.
1931 ///
1932 /// \param CGF Reference to current CodeGenFunction.
1933 /// \param Loc Clang source location.
1934 /// \param IVSize Size of the iteration variable in bits.
1935 /// \param IVSigned Sign of the iteration variable.
1936 ///
1937 void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc,
1938 unsigned IVSize, bool IVSigned) override;
1939
1940 /// Call the appropriate runtime routine to notify that we finished
1941 /// all the work with current loop.
1942 ///
1943 /// \param CGF Reference to current CodeGenFunction.
1944 /// \param Loc Clang source location.
1945 /// \param DKind Kind of the directive for which the static finish is emitted.
1946 ///
1947 void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1948 OpenMPDirectiveKind DKind) override;
1949
1950 /// Call __kmpc_dispatch_next(
1951 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1952 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1953 /// kmp_int[32|64] *p_stride);
1954 /// \param IVSize Size of the iteration variable in bits.
1955 /// \param IVSigned Sign of the iteration variable.
1956 /// \param IL Address of the output variable in which the flag of the
1957 /// last iteration is returned.
1958 /// \param LB Address of the output variable in which the lower iteration
1959 /// number is returned.
1960 /// \param UB Address of the output variable in which the upper iteration
1961 /// number is returned.
1962 /// \param ST Address of the output variable in which the stride value is
1963 /// returned.
1964 llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1965 unsigned IVSize, bool IVSigned, Address IL,
1966 Address LB, Address UB, Address ST) override;
1967
1968 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1969 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1970 /// clause.
1971 /// If the modifier 'strict' is given:
1972 /// Emits call to void __kmpc_push_num_threads_strict(ident_t *loc, kmp_int32
1973 /// global_tid, kmp_int32 num_threads, int severity, const char *message) to
1974 /// generate code for 'num_threads' clause with 'strict' modifier.
1975 /// \param NumThreads An integer value of threads.
1976 void emitNumThreadsClause(
1977 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
1978 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown,
1979 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
1980 SourceLocation SeverityLoc = SourceLocation(),
1981 const Expr *Message = nullptr,
1982 SourceLocation MessageLoc = SourceLocation()) override;
1983
1984 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1985 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1986 void emitProcBindClause(CodeGenFunction &CGF,
1987 llvm::omp::ProcBindKind ProcBind,
1988 SourceLocation Loc) override;
1989
1990 /// Returns address of the threadprivate variable for the current
1991 /// thread.
1992 /// \param VD Threadprivate variable.
1993 /// \param VDAddr Address of the global variable \a VD.
1994 /// \param Loc Location of the reference to threadprivate var.
1995 /// \return Address of the threadprivate variable for the current thread.
1996 Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD,
1997 Address VDAddr, SourceLocation Loc) override;
1998
1999 /// Emit a code for initialization of threadprivate variable. It emits
2000 /// a call to runtime library which adds initial value to the newly created
2001 /// threadprivate variable (if it is not constant) and registers destructor
2002 /// for the variable (if any).
2003 /// \param VD Threadprivate variable.
2004 /// \param VDAddr Address of the global variable \a VD.
2005 /// \param Loc Location of threadprivate declaration.
2006 /// \param PerformInit true if initialization expression is not constant.
2007 llvm::Function *
2008 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
2009 SourceLocation Loc, bool PerformInit,
2010 CodeGenFunction *CGF = nullptr) override;
2011
2012 /// Creates artificial threadprivate variable with name \p Name and type \p
2013 /// VarType.
2014 /// \param VarType Type of the artificial threadprivate variable.
2015 /// \param Name Name of the artificial threadprivate variable.
2016 Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2017 QualType VarType,
2018 StringRef Name) override;
2019
2020 /// Emit flush of the variables specified in 'omp flush' directive.
2021 /// \param Vars List of variables to flush.
2022 void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
2023 SourceLocation Loc, llvm::AtomicOrdering AO) override;
2024
2025 /// Emit task region for the task directive. The task region is
2026 /// emitted in several steps:
2027 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
2028 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2029 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
2030 /// function:
2031 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
2032 /// TaskFunction(gtid, tt->part_id, tt->shareds);
2033 /// return 0;
2034 /// }
2035 /// 2. Copy a list of shared variables to field shareds of the resulting
2036 /// structure kmp_task_t returned by the previous call (if any).
2037 /// 3. Copy a pointer to destructions function to field destructions of the
2038 /// resulting structure kmp_task_t.
2039 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
2040 /// kmp_task_t *new_task), where new_task is a resulting structure from
2041 /// previous items.
2042 /// \param D Current task directive.
2043 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
2044 /// /*part_id*/, captured_struct */*__context*/);
2045 /// \param SharedsTy A type which contains references the shared variables.
2046 /// \param Shareds Context with the list of shared variables from the \p
2047 /// TaskFunction.
2048 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
2049 /// otherwise.
2050 /// \param Data Additional data for task generation like tiednsee, final
2051 /// state, list of privates etc.
2052 void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
2053 const OMPExecutableDirective &D,
2054 llvm::Function *TaskFunction, QualType SharedsTy,
2055 Address Shareds, const Expr *IfCond,
2056 const OMPTaskDataTy &Data) override;
2057
2058 /// Emit task region for the taskloop directive. The taskloop region is
2059 /// emitted in several steps:
2060 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
2061 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2062 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
2063 /// function:
2064 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
2065 /// TaskFunction(gtid, tt->part_id, tt->shareds);
2066 /// return 0;
2067 /// }
2068 /// 2. Copy a list of shared variables to field shareds of the resulting
2069 /// structure kmp_task_t returned by the previous call (if any).
2070 /// 3. Copy a pointer to destructions function to field destructions of the
2071 /// resulting structure kmp_task_t.
2072 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
2073 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
2074 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
2075 /// is a resulting structure from
2076 /// previous items.
2077 /// \param D Current task directive.
2078 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
2079 /// /*part_id*/, captured_struct */*__context*/);
2080 /// \param SharedsTy A type which contains references the shared variables.
2081 /// \param Shareds Context with the list of shared variables from the \p
2082 /// TaskFunction.
2083 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
2084 /// otherwise.
2085 /// \param Data Additional data for task generation like tiednsee, final
2086 /// state, list of privates etc.
2087 void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
2088 const OMPLoopDirective &D, llvm::Function *TaskFunction,
2089 QualType SharedsTy, Address Shareds, const Expr *IfCond,
2090 const OMPTaskDataTy &Data) override;
2091
2092 /// Emit a code for reduction clause. Next code should be emitted for
2093 /// reduction:
2094 /// \code
2095 ///
2096 /// static kmp_critical_name lock = { 0 };
2097 ///
2098 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
2099 /// ...
2100 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
2101 /// ...
2102 /// }
2103 ///
2104 /// ...
2105 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
2106 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
2107 /// RedList, reduce_func, &<lock>)) {
2108 /// case 1:
2109 /// ...
2110 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2111 /// ...
2112 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2113 /// break;
2114 /// case 2:
2115 /// ...
2116 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
2117 /// ...
2118 /// break;
2119 /// default:;
2120 /// }
2121 /// \endcode
2122 ///
2123 /// \param Privates List of private copies for original reduction arguments.
2124 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
2125 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
2126 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
2127 /// or 'operator binop(LHS, RHS)'.
2128 /// \param Options List of options for reduction codegen:
2129 /// WithNowait true if parent directive has also nowait clause, false
2130 /// otherwise.
2131 /// SimpleReduction Emit reduction operation only. Used for omp simd
2132 /// directive on the host.
2133 /// ReductionKind The kind of reduction to perform.
2134 void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
2135 ArrayRef<const Expr *> Privates,
2136 ArrayRef<const Expr *> LHSExprs,
2137 ArrayRef<const Expr *> RHSExprs,
2138 ArrayRef<const Expr *> ReductionOps,
2139 ReductionOptionsTy Options) override;
2140
2141 /// Emit a code for initialization of task reduction clause. Next code
2142 /// should be emitted for reduction:
2143 /// \code
2144 ///
2145 /// _taskred_item_t red_data[n];
2146 /// ...
2147 /// red_data[i].shar = &shareds[i];
2148 /// red_data[i].orig = &origs[i];
2149 /// red_data[i].size = sizeof(origs[i]);
2150 /// red_data[i].f_init = (void*)RedInit<i>;
2151 /// red_data[i].f_fini = (void*)RedDest<i>;
2152 /// red_data[i].f_comb = (void*)RedOp<i>;
2153 /// red_data[i].flags = <Flag_i>;
2154 /// ...
2155 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
2156 /// \endcode
2157 /// For reduction clause with task modifier it emits the next call:
2158 /// \code
2159 ///
2160 /// _taskred_item_t red_data[n];
2161 /// ...
2162 /// red_data[i].shar = &shareds[i];
2163 /// red_data[i].orig = &origs[i];
2164 /// red_data[i].size = sizeof(origs[i]);
2165 /// red_data[i].f_init = (void*)RedInit<i>;
2166 /// red_data[i].f_fini = (void*)RedDest<i>;
2167 /// red_data[i].f_comb = (void*)RedOp<i>;
2168 /// red_data[i].flags = <Flag_i>;
2169 /// ...
2170 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
2171 /// red_data);
2172 /// \endcode
2173 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
2174 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
2175 /// \param Data Additional data for task generation like tiedness, final
2176 /// state, list of privates, reductions etc.
2177 llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc,
2178 ArrayRef<const Expr *> LHSExprs,
2179 ArrayRef<const Expr *> RHSExprs,
2180 const OMPTaskDataTy &Data) override;
2181
2182 /// Emits the following code for reduction clause with task modifier:
2183 /// \code
2184 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
2185 /// \endcode
2186 void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
2187 bool IsWorksharingReduction) override;
2188
2189 /// Required to resolve existing problems in the runtime. Emits threadprivate
2190 /// variables to store the size of the VLAs/array sections for
2191 /// initializer/combiner/finalizer functions + emits threadprivate variable to
2192 /// store the pointer to the original reduction item for the custom
2193 /// initializer defined by declare reduction construct.
2194 /// \param RCG Allows to reuse an existing data for the reductions.
2195 /// \param N Reduction item for which fixups must be emitted.
2196 void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
2197 ReductionCodeGen &RCG, unsigned N) override;
2198
2199 /// Get the address of `void *` type of the privatue copy of the reduction
2200 /// item specified by the \p SharedLVal.
2201 /// \param ReductionsPtr Pointer to the reduction data returned by the
2202 /// emitTaskReductionInit function.
2203 /// \param SharedLVal Address of the original reduction item.
2204 Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
2205 llvm::Value *ReductionsPtr,
2206 LValue SharedLVal) override;
2207
2208 /// Emit code for 'taskwait' directive.
2209 void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
2210 const OMPTaskDataTy &Data) override;
2211
2212 /// Emit code for 'cancellation point' construct.
2213 /// \param CancelRegion Region kind for which the cancellation point must be
2214 /// emitted.
2215 ///
2216 void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc,
2217 OpenMPDirectiveKind CancelRegion) override;
2218
2219 /// Emit code for 'cancel' construct.
2220 /// \param IfCond Condition in the associated 'if' clause, if it was
2221 /// specified, nullptr otherwise.
2222 /// \param CancelRegion Region kind for which the cancel must be emitted.
2223 ///
2224 void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
2225 const Expr *IfCond,
2226 OpenMPDirectiveKind CancelRegion) override;
2227
2228 /// Emit outilined function for 'target' directive.
2229 /// \param D Directive to emit.
2230 /// \param ParentName Name of the function that encloses the target region.
2231 /// \param OutlinedFn Outlined function value to be defined by this call.
2232 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
2233 /// \param IsOffloadEntry True if the outlined function is an offload entry.
2234 /// \param CodeGen Code generation sequence for the \a D directive.
2235 /// An outlined function may not be an entry if, e.g. the if clause always
2236 /// evaluates to false.
2237 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
2238 StringRef ParentName,
2239 llvm::Function *&OutlinedFn,
2240 llvm::Constant *&OutlinedFnID,
2241 bool IsOffloadEntry,
2242 const RegionCodeGenTy &CodeGen) override;
2243
2244 /// Emit the target offloading code associated with \a D. The emitted
2245 /// code attempts offloading the execution to the device, an the event of
2246 /// a failure it executes the host version outlined in \a OutlinedFn.
2247 /// \param D Directive to emit.
2248 /// \param OutlinedFn Host version of the code to be offloaded.
2249 /// \param OutlinedFnID ID of host version of the code to be offloaded.
2250 /// \param IfCond Expression evaluated in if clause associated with the target
2251 /// directive, or null if no if clause is used.
2252 /// \param Device Expression evaluated in device clause associated with the
2253 /// target directive, or null if no device clause is used and device modifier.
2254 void emitTargetCall(
2255 CodeGenFunction &CGF, const OMPExecutableDirective &D,
2256 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
2257 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
2258 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
2259 const OMPLoopDirective &D)>
2260 SizeEmitter) override;
2261
2262 /// Emit the target regions enclosed in \a GD function definition or
2263 /// the function itself in case it is a valid device function. Returns true if
2264 /// \a GD was dealt with successfully.
2265 /// \param GD Function to scan.
2266 bool emitTargetFunctions(GlobalDecl GD) override;
2267
2268 /// Emit the global variable if it is a valid device global variable.
2269 /// Returns true if \a GD was dealt with successfully.
2270 /// \param GD Variable declaration to emit.
2271 bool emitTargetGlobalVariable(GlobalDecl GD) override;
2272
2273 /// Emit the global \a GD if it is meaningful for the target. Returns
2274 /// if it was emitted successfully.
2275 /// \param GD Global to scan.
2276 bool emitTargetGlobal(GlobalDecl GD) override;
2277
2278 /// Emits code for teams call of the \a OutlinedFn with
2279 /// variables captured in a record which address is stored in \a
2280 /// CapturedStruct.
2281 /// \param OutlinedFn Outlined function to be run by team masters. Type of
2282 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
2283 /// \param CapturedVars A pointer to the record with the references to
2284 /// variables used in \a OutlinedFn function.
2285 ///
2286 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
2287 SourceLocation Loc, llvm::Function *OutlinedFn,
2288 ArrayRef<llvm::Value *> CapturedVars) override;
2289
2290 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
2291 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
2292 /// for num_teams clause.
2293 /// \param NumTeams An integer expression of teams.
2294 /// \param ThreadLimit An integer expression of threads.
2295 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
2296 const Expr *ThreadLimit, SourceLocation Loc) override;
2297
2298 /// Emit the target data mapping code associated with \a D.
2299 /// \param D Directive to emit.
2300 /// \param IfCond Expression evaluated in if clause associated with the
2301 /// target directive, or null if no device clause is used.
2302 /// \param Device Expression evaluated in device clause associated with the
2303 /// target directive, or null if no device clause is used.
2304 /// \param Info A record used to store information that needs to be preserved
2305 /// until the region is closed.
2306 void emitTargetDataCalls(CodeGenFunction &CGF,
2307 const OMPExecutableDirective &D, const Expr *IfCond,
2308 const Expr *Device, const RegionCodeGenTy &CodeGen,
2309 CGOpenMPRuntime::TargetDataInfo &Info) override;
2310
2311 /// Emit the data mapping/movement code associated with the directive
2312 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
2313 /// \param D Directive to emit.
2314 /// \param IfCond Expression evaluated in if clause associated with the target
2315 /// directive, or null if no if clause is used.
2316 /// \param Device Expression evaluated in device clause associated with the
2317 /// target directive, or null if no device clause is used.
2318 void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
2319 const OMPExecutableDirective &D,
2320 const Expr *IfCond,
2321 const Expr *Device) override;
2322
2323 /// Emit initialization for doacross loop nesting support.
2324 /// \param D Loop-based construct used in doacross nesting construct.
2325 void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
2326 ArrayRef<Expr *> NumIterations) override;
2327
2328 /// Emit code for doacross ordered directive with 'depend' clause.
2329 /// \param C 'depend' clause with 'sink|source' dependency kind.
2330 void emitDoacrossOrdered(CodeGenFunction &CGF,
2331 const OMPDependClause *C) override;
2332
2333 /// Emit code for doacross ordered directive with 'doacross' clause.
2334 /// \param C 'doacross' clause with 'sink|source' dependence type.
2335 void emitDoacrossOrdered(CodeGenFunction &CGF,
2336 const OMPDoacrossClause *C) override;
2337
2338 /// Translates the native parameter of outlined function if this is required
2339 /// for target.
2340 /// \param FD Field decl from captured record for the parameter.
2341 /// \param NativeParam Parameter itself.
2342 const VarDecl *translateParameter(const FieldDecl *FD,
2343 const VarDecl *NativeParam) const override;
2344
2345 /// Gets the address of the native argument basing on the address of the
2346 /// target-specific parameter.
2347 /// \param NativeParam Parameter itself.
2348 /// \param TargetParam Corresponding target-specific parameter.
2349 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
2350 const VarDecl *TargetParam) const override;
2351
2352 /// Gets the OpenMP-specific address of the local variable.
2353 Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2354 const VarDecl *VD) override {
2355 return Address::invalid();
2356 }
2357};
2358
2359} // namespace CodeGen
2360// Utility for openmp doacross clause kind
2361namespace {
2362template <typename T> class OMPDoacrossKind {
2363public:
2364 bool isSink(const T *) { return false; }
2365 bool isSource(const T *) { return false; }
2366};
2367template <> class OMPDoacrossKind<OMPDependClause> {
2368public:
2369 bool isSink(const OMPDependClause *C) {
2370 return C->getDependencyKind() == OMPC_DEPEND_sink;
2371 }
2372 bool isSource(const OMPDependClause *C) {
2373 return C->getDependencyKind() == OMPC_DEPEND_source;
2374 }
2375};
2376template <> class OMPDoacrossKind<OMPDoacrossClause> {
2377public:
2378 bool isSource(const OMPDoacrossClause *C) {
2379 return C->getDependenceType() == OMPC_DOACROSS_source ||
2380 C->getDependenceType() == OMPC_DOACROSS_source_omp_cur_iteration;
2381 }
2382 bool isSink(const OMPDoacrossClause *C) {
2383 return C->getDependenceType() == OMPC_DOACROSS_sink ||
2384 C->getDependenceType() == OMPC_DOACROSS_sink_omp_cur_iteration;
2385 }
2386};
2387} // namespace
2388} // namespace clang
2389
2390#endif
2391