1//===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
14#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
15
16#include "CGValue.h"
17#include "clang/AST/DeclOpenMP.h"
18#include "clang/AST/GlobalDecl.h"
19#include "clang/AST/Type.h"
20#include "clang/Basic/OpenMPKinds.h"
21#include "clang/Basic/SourceLocation.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/PointerIntPair.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/Frontend/OpenMP/OMPConstants.h"
28#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/ValueHandle.h"
31#include "llvm/Support/AtomicOrdering.h"
32
33namespace llvm {
34class ArrayType;
35class Constant;
36class FunctionType;
37class GlobalVariable;
38class Type;
39class Value;
40class OpenMPIRBuilder;
41} // namespace llvm
42
43namespace clang {
44class Expr;
45class OMPDependClause;
46class OMPExecutableDirective;
47class OMPLoopDirective;
48class VarDecl;
49class OMPDeclareReductionDecl;
50
51namespace CodeGen {
52class Address;
53class CodeGenFunction;
54class CodeGenModule;
55
56/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
57/// region.
58class PrePostActionTy {
59public:
60 explicit PrePostActionTy() {}
61 virtual void Enter(CodeGenFunction &CGF) {}
62 virtual void Exit(CodeGenFunction &CGF) {}
63 virtual ~PrePostActionTy() {}
64};
65
66/// Class provides a way to call simple version of codegen for OpenMP region, or
67/// an advanced with possible pre|post-actions in codegen.
68class RegionCodeGenTy final {
69 intptr_t CodeGen;
70 typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &);
71 CodeGenTy Callback;
72 mutable PrePostActionTy *PrePostAction;
73 RegionCodeGenTy() = delete;
74 template <typename Callable>
75 static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF,
76 PrePostActionTy &Action) {
77 return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action);
78 }
79
80public:
81 template <typename Callable>
82 RegionCodeGenTy(
83 Callable &&CodeGen,
84 std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
85 RegionCodeGenTy>::value> * = nullptr)
86 : CodeGen(reinterpret_cast<intptr_t>(&CodeGen)),
87 Callback(CallbackFn<std::remove_reference_t<Callable>>),
88 PrePostAction(nullptr) {}
89 void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; }
90 void operator()(CodeGenFunction &CGF) const;
91};
92
93struct OMPTaskDataTy final {
94 SmallVector<const Expr *, 4> PrivateVars;
95 SmallVector<const Expr *, 4> PrivateCopies;
96 SmallVector<const Expr *, 4> FirstprivateVars;
97 SmallVector<const Expr *, 4> FirstprivateCopies;
98 SmallVector<const Expr *, 4> FirstprivateInits;
99 SmallVector<const Expr *, 4> LastprivateVars;
100 SmallVector<const Expr *, 4> LastprivateCopies;
101 SmallVector<const Expr *, 4> ReductionVars;
102 SmallVector<const Expr *, 4> ReductionOrigs;
103 SmallVector<const Expr *, 4> ReductionCopies;
104 SmallVector<const Expr *, 4> ReductionOps;
105 SmallVector<CanonicalDeclPtr<const VarDecl>, 4> PrivateLocals;
106 struct DependData {
107 OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown;
108 const Expr *IteratorExpr = nullptr;
109 SmallVector<const Expr *, 4> DepExprs;
110 explicit DependData() = default;
111 DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr)
112 : DepKind(DepKind), IteratorExpr(IteratorExpr) {}
113 };
114 SmallVector<DependData, 4> Dependences;
115 llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
116 llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule;
117 llvm::PointerIntPair<llvm::Value *, 1, bool> Priority;
118 llvm::Value *Reductions = nullptr;
119 unsigned NumberOfParts = 0;
120 bool Tied = true;
121 bool Nogroup = false;
122 bool IsReductionWithTaskMod = false;
123 bool IsWorksharingReduction = false;
124 bool HasNowaitClause = false;
125 bool HasModifier = false;
126};
127
128/// Class intended to support codegen of all kind of the reduction clauses.
129class ReductionCodeGen {
130private:
131 /// Data required for codegen of reduction clauses.
132 struct ReductionData {
133 /// Reference to the item shared between tasks to reduce into.
134 const Expr *Shared = nullptr;
135 /// Reference to the original item.
136 const Expr *Ref = nullptr;
137 /// Helper expression for generation of private copy.
138 const Expr *Private = nullptr;
139 /// Helper expression for generation reduction operation.
140 const Expr *ReductionOp = nullptr;
141 ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private,
142 const Expr *ReductionOp)
143 : Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) {
144 }
145 };
146 /// List of reduction-based clauses.
147 SmallVector<ReductionData, 4> ClausesData;
148
149 /// List of addresses of shared variables/expressions.
150 SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses;
151 /// List of addresses of original variables/expressions.
152 SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses;
153 /// Sizes of the reduction items in chars.
154 SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes;
155 /// Base declarations for the reduction items.
156 SmallVector<const VarDecl *, 4> BaseDecls;
157
158 /// Emits lvalue for shared expression.
159 LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E);
160 /// Emits upper bound for shared expression (if array section).
161 LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E);
162 /// Performs aggregate initialization.
163 /// \param N Number of reduction item in the common list.
164 /// \param PrivateAddr Address of the corresponding private item.
165 /// \param SharedAddr Address of the original shared variable.
166 /// \param DRD Declare reduction construct used for reduction item.
167 void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N,
168 Address PrivateAddr, Address SharedAddr,
169 const OMPDeclareReductionDecl *DRD);
170
171public:
172 ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs,
173 ArrayRef<const Expr *> Privates,
174 ArrayRef<const Expr *> ReductionOps);
175 /// Emits lvalue for the shared and original reduction item.
176 /// \param N Number of the reduction item.
177 void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N);
178 /// Emits the code for the variable-modified type, if required.
179 /// \param N Number of the reduction item.
180 void emitAggregateType(CodeGenFunction &CGF, unsigned N);
181 /// Emits the code for the variable-modified type, if required.
182 /// \param N Number of the reduction item.
183 /// \param Size Size of the type in chars.
184 void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size);
185 /// Performs initialization of the private copy for the reduction item.
186 /// \param N Number of the reduction item.
187 /// \param PrivateAddr Address of the corresponding private item.
188 /// \param DefaultInit Default initialization sequence that should be
189 /// performed if no reduction specific initialization is found.
190 /// \param SharedAddr Address of the original shared variable.
191 void
192 emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr,
193 Address SharedAddr,
194 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit);
195 /// Returns true if the private copy requires cleanups.
196 bool needCleanups(unsigned N);
197 /// Emits cleanup code for the reduction item.
198 /// \param N Number of the reduction item.
199 /// \param PrivateAddr Address of the corresponding private item.
200 void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr);
201 /// Adjusts \p PrivatedAddr for using instead of the original variable
202 /// address in normal operations.
203 /// \param N Number of the reduction item.
204 /// \param PrivateAddr Address of the corresponding private item.
205 Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
206 Address PrivateAddr);
207 /// Returns LValue for the reduction item.
208 LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; }
209 /// Returns LValue for the original reduction item.
210 LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; }
211 /// Returns the size of the reduction item (in chars and total number of
212 /// elements in the item), or nullptr, if the size is a constant.
213 std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const {
214 return Sizes[N];
215 }
216 /// Returns the base declaration of the reduction item.
217 const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; }
218 /// Returns the base declaration of the reduction item.
219 const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; }
220 /// Returns true if the initialization of the reduction item uses initializer
221 /// from declare reduction construct.
222 bool usesReductionInitializer(unsigned N) const;
223 /// Return the type of the private item.
224 QualType getPrivateType(unsigned N) const {
225 return cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl())
226 ->getType();
227 }
228};
229
230class CGOpenMPRuntime {
231public:
232 /// Allows to disable automatic handling of functions used in target regions
233 /// as those marked as `omp declare target`.
234 class DisableAutoDeclareTargetRAII {
235 CodeGenModule &CGM;
236 bool SavedShouldMarkAsGlobal = false;
237
238 public:
239 DisableAutoDeclareTargetRAII(CodeGenModule &CGM);
240 ~DisableAutoDeclareTargetRAII();
241 };
242
243 /// Manages list of nontemporal decls for the specified directive.
244 class NontemporalDeclsRAII {
245 CodeGenModule &CGM;
246 const bool NeedToPush;
247
248 public:
249 NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S);
250 ~NontemporalDeclsRAII();
251 };
252
253 /// Manages list of nontemporal decls for the specified directive.
254 class UntiedTaskLocalDeclsRAII {
255 CodeGenModule &CGM;
256 const bool NeedToPush;
257
258 public:
259 UntiedTaskLocalDeclsRAII(
260 CodeGenFunction &CGF,
261 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
262 std::pair<Address, Address>> &LocalVars);
263 ~UntiedTaskLocalDeclsRAII();
264 };
265
266 /// Maps the expression for the lastprivate variable to the global copy used
267 /// to store new value because original variables are not mapped in inner
268 /// parallel regions. Only private copies are captured but we need also to
269 /// store private copy in shared address.
270 /// Also, stores the expression for the private loop counter and it
271 /// threaprivate name.
272 struct LastprivateConditionalData {
273 llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>>
274 DeclToUniqueName;
275 LValue IVLVal;
276 llvm::Function *Fn = nullptr;
277 bool Disabled = false;
278 };
279 /// Manages list of lastprivate conditional decls for the specified directive.
280 class LastprivateConditionalRAII {
281 enum class ActionToDo {
282 DoNotPush,
283 PushAsLastprivateConditional,
284 DisableLastprivateConditional,
285 };
286 CodeGenModule &CGM;
287 ActionToDo Action = ActionToDo::DoNotPush;
288
289 /// Check and try to disable analysis of inner regions for changes in
290 /// lastprivate conditional.
291 void tryToDisableInnerAnalysis(const OMPExecutableDirective &S,
292 llvm::DenseSet<CanonicalDeclPtr<const Decl>>
293 &NeedToAddForLPCsAsDisabled) const;
294
295 LastprivateConditionalRAII(CodeGenFunction &CGF,
296 const OMPExecutableDirective &S);
297
298 public:
299 explicit LastprivateConditionalRAII(CodeGenFunction &CGF,
300 const OMPExecutableDirective &S,
301 LValue IVLVal);
302 static LastprivateConditionalRAII disable(CodeGenFunction &CGF,
303 const OMPExecutableDirective &S);
304 ~LastprivateConditionalRAII();
305 };
306
307 llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; }
308
309protected:
310 CodeGenModule &CGM;
311
312 /// An OpenMP-IR-Builder instance.
313 llvm::OpenMPIRBuilder OMPBuilder;
314
315 /// Helper to determine the min/max number of threads/teams for \p D.
316 void computeMinAndMaxThreadsAndTeams(
317 const OMPExecutableDirective &D, CodeGenFunction &CGF,
318 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs);
319
320 /// Helper to emit outlined function for 'target' directive.
321 /// \param D Directive to emit.
322 /// \param ParentName Name of the function that encloses the target region.
323 /// \param OutlinedFn Outlined function value to be defined by this call.
324 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
325 /// \param IsOffloadEntry True if the outlined function is an offload entry.
326 /// \param CodeGen Lambda codegen specific to an accelerator device.
327 /// An outlined function may not be an entry if, e.g. the if clause always
328 /// evaluates to false.
329 virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D,
330 StringRef ParentName,
331 llvm::Function *&OutlinedFn,
332 llvm::Constant *&OutlinedFnID,
333 bool IsOffloadEntry,
334 const RegionCodeGenTy &CodeGen);
335
336 /// Returns pointer to ident_t type.
337 llvm::Type *getIdentTyPointerTy();
338
339 /// Gets thread id value for the current thread.
340 ///
341 llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc);
342
343 /// Get the function name of an outlined region.
344 std::string getOutlinedHelperName(StringRef Name) const;
345 std::string getOutlinedHelperName(CodeGenFunction &CGF) const;
346
347 /// Get the function name of a reduction function.
348 std::string getReductionFuncName(StringRef Name) const;
349
350 /// Emits \p Callee function call with arguments \p Args with location \p Loc.
351 void emitCall(CodeGenFunction &CGF, SourceLocation Loc,
352 llvm::FunctionCallee Callee,
353 ArrayRef<llvm::Value *> Args = {}) const;
354
355 /// Emits address of the word in a memory where current thread id is
356 /// stored.
357 virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc);
358
359 void setLocThreadIdInsertPt(CodeGenFunction &CGF,
360 bool AtCurrentPoint = false);
361 void clearLocThreadIdInsertPt(CodeGenFunction &CGF);
362
363 /// Check if the default location must be constant.
364 /// Default is false to support OMPT/OMPD.
365 virtual bool isDefaultLocationConstant() const { return false; }
366
367 /// Returns additional flags that can be stored in reserved_2 field of the
368 /// default location.
369 virtual unsigned getDefaultLocationReserved2Flags() const { return 0; }
370
371 /// Returns default flags for the barriers depending on the directive, for
372 /// which this barier is going to be emitted.
373 static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind);
374
375 /// Get the LLVM type for the critical name.
376 llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;}
377
378 /// Returns corresponding lock object for the specified critical region
379 /// name. If the lock object does not exist it is created, otherwise the
380 /// reference to the existing copy is returned.
381 /// \param CriticalName Name of the critical region.
382 ///
383 llvm::Value *getCriticalRegionLock(StringRef CriticalName);
384
385protected:
386 /// Map for SourceLocation and OpenMP runtime library debug locations.
387 typedef llvm::DenseMap<SourceLocation, llvm::Value *> OpenMPDebugLocMapTy;
388 OpenMPDebugLocMapTy OpenMPDebugLocMap;
389 /// Stores debug location and ThreadID for the function.
390 struct DebugLocThreadIdTy {
391 llvm::Value *DebugLoc;
392 llvm::Value *ThreadID;
393 /// Insert point for the service instructions.
394 llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr;
395 };
396 /// Map of local debug location, ThreadId and functions.
397 typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy>
398 OpenMPLocThreadIDMapTy;
399 OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap;
400 /// Map of UDRs and corresponding combiner/initializer.
401 typedef llvm::DenseMap<const OMPDeclareReductionDecl *,
402 std::pair<llvm::Function *, llvm::Function *>>
403 UDRMapTy;
404 UDRMapTy UDRMap;
405 /// Map of functions and locally defined UDRs.
406 typedef llvm::DenseMap<llvm::Function *,
407 SmallVector<const OMPDeclareReductionDecl *, 4>>
408 FunctionUDRMapTy;
409 FunctionUDRMapTy FunctionUDRMap;
410 /// Map from the user-defined mapper declaration to its corresponding
411 /// functions.
412 llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap;
413 /// Map of functions and their local user-defined mappers.
414 using FunctionUDMMapTy =
415 llvm::DenseMap<llvm::Function *,
416 SmallVector<const OMPDeclareMapperDecl *, 4>>;
417 FunctionUDMMapTy FunctionUDMMap;
418 /// Maps local variables marked as lastprivate conditional to their internal
419 /// types.
420 llvm::DenseMap<llvm::Function *,
421 llvm::DenseMap<CanonicalDeclPtr<const Decl>,
422 std::tuple<QualType, const FieldDecl *,
423 const FieldDecl *, LValue>>>
424 LastprivateConditionalToTypes;
425 /// Maps function to the position of the untied task locals stack.
426 llvm::DenseMap<llvm::Function *, unsigned> FunctionToUntiedTaskStackMap;
427 /// Type kmp_critical_name, originally defined as typedef kmp_int32
428 /// kmp_critical_name[8];
429 llvm::ArrayType *KmpCriticalNameTy;
430 /// An ordered map of auto-generated variables to their unique names.
431 /// It stores variables with the following names: 1) ".gomp_critical_user_" +
432 /// <critical_section_name> + ".var" for "omp critical" directives; 2)
433 /// <mangled_name_for_global_var> + ".cache." for cache for threadprivate
434 /// variables.
435 llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>,
436 llvm::BumpPtrAllocator> InternalVars;
437 /// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);
438 llvm::Type *KmpRoutineEntryPtrTy = nullptr;
439 QualType KmpRoutineEntryPtrQTy;
440 /// Type typedef struct kmp_task {
441 /// void * shareds; /**< pointer to block of pointers to
442 /// shared vars */
443 /// kmp_routine_entry_t routine; /**< pointer to routine to call for
444 /// executing task */
445 /// kmp_int32 part_id; /**< part id for the task */
446 /// kmp_routine_entry_t destructors; /* pointer to function to invoke
447 /// deconstructors of firstprivate C++ objects */
448 /// } kmp_task_t;
449 QualType KmpTaskTQTy;
450 /// Saved kmp_task_t for task directive.
451 QualType SavedKmpTaskTQTy;
452 /// Saved kmp_task_t for taskloop-based directive.
453 QualType SavedKmpTaskloopTQTy;
454 /// Type typedef struct kmp_depend_info {
455 /// kmp_intptr_t base_addr;
456 /// size_t len;
457 /// struct {
458 /// bool in:1;
459 /// bool out:1;
460 /// } flags;
461 /// } kmp_depend_info_t;
462 QualType KmpDependInfoTy;
463 /// Type typedef struct kmp_task_affinity_info {
464 /// kmp_intptr_t base_addr;
465 /// size_t len;
466 /// struct {
467 /// bool flag1 : 1;
468 /// bool flag2 : 1;
469 /// kmp_int32 reserved : 30;
470 /// } flags;
471 /// } kmp_task_affinity_info_t;
472 QualType KmpTaskAffinityInfoTy;
473 /// struct kmp_dim { // loop bounds info casted to kmp_int64
474 /// kmp_int64 lo; // lower
475 /// kmp_int64 up; // upper
476 /// kmp_int64 st; // stride
477 /// };
478 QualType KmpDimTy;
479
480 bool ShouldMarkAsGlobal = true;
481 /// List of the emitted declarations.
482 llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls;
483 /// Implicitly-defined constructors already scanned for target regions; only
484 /// lambdas can hold such regions there and they compute their own parent.
485 llvm::SmallPtrSet<const CXXConstructorDecl *, 8> ScannedImplicitCtors;
486 /// List of the global variables with their addresses that should not be
487 /// emitted for the target.
488 llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables;
489
490 /// List of variables that can become declare target implicitly and, thus,
491 /// must be emitted.
492 llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;
493
494 using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>;
495 /// Stack for list of declarations in current context marked as nontemporal.
496 /// The set is the union of all current stack elements.
497 llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack;
498
499 using UntiedLocalVarsAddressesMap =
500 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
501 std::pair<Address, Address>>;
502 llvm::SmallVector<UntiedLocalVarsAddressesMap, 4> UntiedLocalVarsStack;
503
504 /// Stack for list of addresses of declarations in current context marked as
505 /// lastprivate conditional. The set is the union of all current stack
506 /// elements.
507 llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack;
508
509 /// Flag for keeping track of weather a requires unified_shared_memory
510 /// directive is present.
511 bool HasRequiresUnifiedSharedMemory = false;
512
513 /// Atomic ordering from the omp requires directive.
514 llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
515
516 /// Flag for keeping track of weather a target region has been emitted.
517 bool HasEmittedTargetRegion = false;
518
519 /// Flag for keeping track of weather a device routine has been emitted.
520 /// Device routines are specific to the
521 bool HasEmittedDeclareTargetRegion = false;
522
523 /// Start scanning from statement \a S and emit all target regions
524 /// found along the way.
525 /// \param S Starting statement.
526 /// \param ParentName Name of the function declaration that is being scanned.
527 void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName);
528
529 /// Scans the mem-initializers and the body of \a Ctor for target regions.
530 void scanConstructorForTargetRegions(const CXXConstructorDecl *Ctor,
531 StringRef ParentName);
532
533 /// Build type kmp_routine_entry_t (if not built yet).
534 void emitKmpRoutineEntryT(QualType KmpInt32Ty);
535
536 /// If the specified mangled name is not in the module, create and
537 /// return threadprivate cache object. This object is a pointer's worth of
538 /// storage that's reserved for use by the OpenMP runtime.
539 /// \param VD Threadprivate variable.
540 /// \return Cache variable for the specified threadprivate.
541 llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD);
542
543 /// Set of threadprivate variables with the generated initializer.
544 llvm::StringSet<> ThreadPrivateWithDefinition;
545
546 /// Set of declare target variables with the generated initializer.
547 llvm::StringSet<> DeclareTargetWithDefinition;
548
549 /// Emits initialization code for the threadprivate variables.
550 /// \param VDAddr Address of the global variable \a VD.
551 /// \param Ctor Pointer to a global init function for \a VD.
552 /// \param CopyCtor Pointer to a global copy function for \a VD.
553 /// \param Dtor Pointer to a global destructor function for \a VD.
554 /// \param Loc Location of threadprivate declaration.
555 void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr,
556 llvm::Value *Ctor, llvm::Value *CopyCtor,
557 llvm::Value *Dtor, SourceLocation Loc);
558
559 struct TaskResultTy {
560 llvm::Value *NewTask = nullptr;
561 llvm::Function *TaskEntry = nullptr;
562 llvm::Value *NewTaskNewTaskTTy = nullptr;
563 LValue TDBase;
564 const RecordDecl *KmpTaskTQTyRD = nullptr;
565 llvm::Value *TaskDupFn = nullptr;
566 };
567 /// Emit task region for the task directive. The task region is emitted in
568 /// several steps:
569 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
570 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
571 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
572 /// function:
573 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
574 /// TaskFunction(gtid, tt->part_id, tt->shareds);
575 /// return 0;
576 /// }
577 /// 2. Copy a list of shared variables to field shareds of the resulting
578 /// structure kmp_task_t returned by the previous call (if any).
579 /// 3. Copy a pointer to destructions function to field destructions of the
580 /// resulting structure kmp_task_t.
581 /// \param D Current task directive.
582 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
583 /// /*part_id*/, captured_struct */*__context*/);
584 /// \param SharedsTy A type which contains references the shared variables.
585 /// \param Shareds Context with the list of shared variables from the \p
586 /// TaskFunction.
587 /// \param Data Additional data for task generation like tiednsee, final
588 /// state, list of privates etc.
589 TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
590 const OMPExecutableDirective &D,
591 llvm::Function *TaskFunction, QualType SharedsTy,
592 Address Shareds, const OMPTaskDataTy &Data);
593
594 /// Emit update for lastprivate conditional data.
595 void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,
596 StringRef UniqueDeclName, LValue LVal,
597 SourceLocation Loc);
598
599 /// Returns the number of the elements and the address of the depobj
600 /// dependency array.
601 /// \return Number of elements in depobj array and the pointer to the array of
602 /// dependencies.
603 std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF,
604 LValue DepobjLVal,
605 SourceLocation Loc);
606
607 SmallVector<llvm::Value *, 4>
608 emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
609 const OMPTaskDataTy::DependData &Data);
610
611 void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
612 LValue PosLVal, const OMPTaskDataTy::DependData &Data,
613 Address DependenciesArray);
614
615 /// Keep track of VTable Declarations so we don't register duplicate VTable.
616 llvm::SmallDenseMap<CXXRecordDecl *, const VarDecl *> VTableDeclMap;
617
618public:
619 explicit CGOpenMPRuntime(CodeGenModule &CGM);
620 virtual ~CGOpenMPRuntime() {}
621 virtual void clear();
622
623 /// Emits object of ident_t type with info for source location.
624 /// \param Flags Flags for OpenMP location.
625 /// \param EmitLoc emit source location with debug-info is off.
626 ///
627 llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc,
628 unsigned Flags = 0, bool EmitLoc = false);
629
630 /// Emit the number of teams for a target directive. Inspect the num_teams
631 /// clause associated with a teams construct combined or closely nested
632 /// with the target directive.
633 ///
634 /// Emit a team of size one for directives such as 'target parallel' that
635 /// have no associated teams construct.
636 ///
637 /// Otherwise, return nullptr.
638 const Expr *getNumTeamsExprForTargetDirective(CodeGenFunction &CGF,
639 const OMPExecutableDirective &D,
640 int32_t &MinTeamsVal,
641 int32_t &MaxTeamsVal);
642 llvm::Value *emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
643 const OMPExecutableDirective &D);
644
645 /// Check for a number of threads upper bound constant value (stored in \p
646 /// UpperBound), or expression (returned). If the value is conditional (via an
647 /// if-clause), store the condition in \p CondExpr. Similarly, a potential
648 /// thread limit expression is stored in \p ThreadLimitExpr. If \p
649 /// UpperBoundOnly is true, no expression evaluation is perfomed.
650 const Expr *getNumThreadsExprForTargetDirective(
651 CodeGenFunction &CGF, const OMPExecutableDirective &D,
652 int32_t &UpperBound, bool UpperBoundOnly,
653 llvm::Value **CondExpr = nullptr, const Expr **ThreadLimitExpr = nullptr);
654
655 /// Emit an expression that denotes the number of threads a target region
656 /// shall use. Will generate "i32 0" to allow the runtime to choose.
657 llvm::Value *
658 emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
659 const OMPExecutableDirective &D);
660
661 /// Return the trip count of loops associated with constructs / 'target teams
662 /// distribute' and 'teams distribute parallel for'. \param SizeEmitter Emits
663 /// the int64 value for the number of iterations of the associated loop.
664 llvm::Value *emitTargetNumIterationsCall(
665 CodeGenFunction &CGF, const OMPExecutableDirective &D,
666 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
667 const OMPLoopDirective &D)>
668 SizeEmitter);
669
670 /// Returns true if the current target is a GPU.
671 virtual bool isGPU() const { return false; }
672
673 /// Check if the variable length declaration is delayed:
674 virtual bool isDelayedVariableLengthDecl(CodeGenFunction &CGF,
675 const VarDecl *VD) const {
676 return false;
677 };
678
679 /// Get call to __kmpc_alloc_shared
680 virtual std::pair<llvm::Value *, llvm::Value *>
681 getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) {
682 llvm_unreachable("not implemented");
683 }
684
685 /// Get call to __kmpc_free_shared
686 virtual void getKmpcFreeShared(
687 CodeGenFunction &CGF,
688 const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) {
689 llvm_unreachable("not implemented");
690 }
691
692 /// Emits code for OpenMP 'if' clause using specified \a CodeGen
693 /// function. Here is the logic:
694 /// if (Cond) {
695 /// ThenGen();
696 /// } else {
697 /// ElseGen();
698 /// }
699 void emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
700 const RegionCodeGenTy &ThenGen,
701 const RegionCodeGenTy &ElseGen);
702
703 /// Checks if the \p Body is the \a CompoundStmt and returns its child
704 /// statement iff there is only one that is not evaluatable at the compile
705 /// time.
706 static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body);
707
708 /// Get the platform-specific name separator.
709 std::string getName(ArrayRef<StringRef> Parts) const;
710
711 /// Emit code for the specified user defined reduction construct.
712 virtual void emitUserDefinedReduction(CodeGenFunction *CGF,
713 const OMPDeclareReductionDecl *D);
714 /// Get combiner/initializer for the specified user-defined reduction, if any.
715 virtual std::pair<llvm::Function *, llvm::Function *>
716 getUserDefinedReduction(const OMPDeclareReductionDecl *D);
717
718 /// Emit the function for the user defined mapper construct.
719 void emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
720 CodeGenFunction *CGF = nullptr);
721 /// Get the function for the specified user-defined mapper. If it does not
722 /// exist, create one.
723 llvm::Function *
724 getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D);
725
726 /// Emits outlined function for the specified OpenMP parallel directive
727 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
728 /// kmp_int32 BoundID, struct context_vars*).
729 /// \param CGF Reference to current CodeGenFunction.
730 /// \param D OpenMP directive.
731 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
732 /// \param InnermostKind Kind of innermost directive (for simple directives it
733 /// is a directive itself, for combined - its innermost directive).
734 /// \param CodeGen Code generation sequence for the \a D directive.
735 virtual llvm::Function *emitParallelOutlinedFunction(
736 CodeGenFunction &CGF, const OMPExecutableDirective &D,
737 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
738 const RegionCodeGenTy &CodeGen);
739
740 /// Emits outlined function for the specified OpenMP teams directive
741 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
742 /// kmp_int32 BoundID, struct context_vars*).
743 /// \param CGF Reference to current CodeGenFunction.
744 /// \param D OpenMP directive.
745 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
746 /// \param InnermostKind Kind of innermost directive (for simple directives it
747 /// is a directive itself, for combined - its innermost directive).
748 /// \param CodeGen Code generation sequence for the \a D directive.
749 virtual llvm::Function *emitTeamsOutlinedFunction(
750 CodeGenFunction &CGF, const OMPExecutableDirective &D,
751 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
752 const RegionCodeGenTy &CodeGen);
753
754 /// Emits outlined function for the OpenMP task directive \a D. This
755 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
756 /// TaskT).
757 /// \param D OpenMP directive.
758 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
759 /// \param PartIDVar Variable for partition id in the current OpenMP untied
760 /// task region.
761 /// \param TaskTVar Variable for task_t argument.
762 /// \param InnermostKind Kind of innermost directive (for simple directives it
763 /// is a directive itself, for combined - its innermost directive).
764 /// \param CodeGen Code generation sequence for the \a D directive.
765 /// \param Tied true if task is generated for tied task, false otherwise.
766 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
767 /// tasks.
768 ///
769 virtual llvm::Function *emitTaskOutlinedFunction(
770 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
771 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
772 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
773 bool Tied, unsigned &NumberOfParts);
774
775 /// Cleans up references to the objects in finished function.
776 ///
777 virtual void functionFinished(CodeGenFunction &CGF);
778
779 /// Emits code for parallel or serial call of the \a OutlinedFn with
780 /// variables captured in a record which address is stored in \a
781 /// CapturedStruct.
782 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
783 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
784 /// \param CapturedVars A pointer to the record with the references to
785 /// variables used in \a OutlinedFn function.
786 /// \param IfCond Condition in the associated 'if' clause, if it was
787 /// specified, nullptr otherwise.
788 /// \param NumThreads The value corresponding to the num_threads clause, if
789 /// any, or nullptr.
790 /// \param NumThreadsModifier The modifier of the num_threads clause, if
791 /// any, ignored otherwise.
792 /// \param Severity The severity corresponding to the num_threads clause, if
793 /// any, ignored otherwise.
794 /// \param Message The message string corresponding to the num_threads clause,
795 /// if any, or nullptr.
796 ///
797 virtual void
798 emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
799 llvm::Function *OutlinedFn,
800 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
801 llvm::Value *NumThreads,
802 OpenMPNumThreadsClauseModifier NumThreadsModifier =
803 OMPC_NUMTHREADS_unknown,
804 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
805 const Expr *Message = nullptr);
806
807 /// Emits a critical region.
808 /// \param CriticalName Name of the critical region.
809 /// \param CriticalOpGen Generator for the statement associated with the given
810 /// critical region.
811 /// \param Hint Value of the 'hint' clause (optional).
812 virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
813 const RegionCodeGenTy &CriticalOpGen,
814 SourceLocation Loc,
815 const Expr *Hint = nullptr);
816
817 /// Emits a master region.
818 /// \param MasterOpGen Generator for the statement associated with the given
819 /// master region.
820 virtual void emitMasterRegion(CodeGenFunction &CGF,
821 const RegionCodeGenTy &MasterOpGen,
822 SourceLocation Loc);
823
824 /// Emits a masked region.
825 /// \param MaskedOpGen Generator for the statement associated with the given
826 /// masked region.
827 virtual void emitMaskedRegion(CodeGenFunction &CGF,
828 const RegionCodeGenTy &MaskedOpGen,
829 SourceLocation Loc,
830 const Expr *Filter = nullptr);
831
832 /// Emits code for a taskyield directive.
833 virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);
834
835 /// Emit __kmpc_error call for error directive
836 /// extern void __kmpc_error(ident_t *loc, int severity, const char *message);
837 virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME,
838 bool IsFatal);
839
840 /// Emit a taskgroup region.
841 /// \param TaskgroupOpGen Generator for the statement associated with the
842 /// given taskgroup region.
843 virtual void emitTaskgroupRegion(CodeGenFunction &CGF,
844 const RegionCodeGenTy &TaskgroupOpGen,
845 SourceLocation Loc);
846
847 /// Emits a single region.
848 /// \param SingleOpGen Generator for the statement associated with the given
849 /// single region.
850 virtual void emitSingleRegion(CodeGenFunction &CGF,
851 const RegionCodeGenTy &SingleOpGen,
852 SourceLocation Loc,
853 ArrayRef<const Expr *> CopyprivateVars,
854 ArrayRef<const Expr *> DestExprs,
855 ArrayRef<const Expr *> SrcExprs,
856 ArrayRef<const Expr *> AssignmentOps);
857
858 /// Emit an ordered region.
859 /// \param OrderedOpGen Generator for the statement associated with the given
860 /// ordered region.
861 virtual void emitOrderedRegion(CodeGenFunction &CGF,
862 const RegionCodeGenTy &OrderedOpGen,
863 SourceLocation Loc, bool IsThreads);
864
865 /// Emit an implicit/explicit barrier for OpenMP threads.
866 /// \param Kind Directive for which this implicit barrier call must be
867 /// generated. Must be OMPD_barrier for explicit barrier generation.
868 /// \param EmitChecks true if need to emit checks for cancellation barriers.
869 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
870 /// runtime class decides which one to emit (simple or with cancellation
871 /// checks).
872 ///
873 virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
874 OpenMPDirectiveKind Kind,
875 bool EmitChecks = true,
876 bool ForceSimpleCall = false);
877
878 /// Check if the specified \a ScheduleKind is static non-chunked.
879 /// This kind of worksharing directive is emitted without outer loop.
880 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
881 /// \param Chunked True if chunk is specified in the clause.
882 ///
883 virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
884 bool Chunked) const;
885
886 /// Check if the specified \a ScheduleKind is static non-chunked.
887 /// This kind of distribute directive is emitted without outer loop.
888 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
889 /// \param Chunked True if chunk is specified in the clause.
890 ///
891 virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind,
892 bool Chunked) const;
893
894 /// Check if the specified \a ScheduleKind is static chunked.
895 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
896 /// \param Chunked True if chunk is specified in the clause.
897 ///
898 virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
899 bool Chunked) const;
900
901 /// Check if the specified \a ScheduleKind is static non-chunked.
902 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
903 /// \param Chunked True if chunk is specified in the clause.
904 ///
905 virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind,
906 bool Chunked) const;
907
908 /// Check if the specified \a ScheduleKind is dynamic.
909 /// This kind of worksharing directive is emitted without outer loop.
910 /// \param ScheduleKind Schedule Kind specified in the 'schedule' clause.
911 ///
912 virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const;
913
914 /// struct with the values to be passed to the dispatch runtime function
915 struct DispatchRTInput {
916 /// Loop lower bound
917 llvm::Value *LB = nullptr;
918 /// Loop upper bound
919 llvm::Value *UB = nullptr;
920 /// Chunk size specified using 'schedule' clause (nullptr if chunk
921 /// was not specified)
922 llvm::Value *Chunk = nullptr;
923 DispatchRTInput() = default;
924 DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk)
925 : LB(LB), UB(UB), Chunk(Chunk) {}
926 };
927
928 /// Call the appropriate runtime routine to initialize it before start
929 /// of loop.
930
931 /// This is used for non static scheduled types and when the ordered
932 /// clause is present on the loop construct.
933 /// Depending on the loop schedule, it is necessary to call some runtime
934 /// routine before start of the OpenMP loop to get the loop upper / lower
935 /// bounds \a LB and \a UB and stride \a ST.
936 ///
937 /// \param CGF Reference to current CodeGenFunction.
938 /// \param Loc Clang source location.
939 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
940 /// \param IVSize Size of the iteration variable in bits.
941 /// \param IVSigned Sign of the iteration variable.
942 /// \param Ordered true if loop is ordered, false otherwise.
943 /// \param DispatchValues struct containing llvm values for lower bound, upper
944 /// bound, and chunk expression.
945 /// For the default (nullptr) value, the chunk 1 will be used.
946 ///
947 virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
948 const OpenMPScheduleTy &ScheduleKind,
949 unsigned IVSize, bool IVSigned, bool Ordered,
950 const DispatchRTInput &DispatchValues);
951
952 /// This is used for non static scheduled types and when the ordered
953 /// clause is present on the loop construct.
954 ///
955 /// \param CGF Reference to current CodeGenFunction.
956 /// \param Loc Clang source location.
957 ///
958 virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc);
959
960 /// Struct with the values to be passed to the static runtime function
961 struct StaticRTInput {
962 /// Size of the iteration variable in bits.
963 unsigned IVSize = 0;
964 /// Sign of the iteration variable.
965 bool IVSigned = false;
966 /// true if loop is ordered, false otherwise.
967 bool Ordered = false;
968 /// Address of the output variable in which the flag of the last iteration
969 /// is returned.
970 Address IL = Address::invalid();
971 /// Address of the output variable in which the lower iteration number is
972 /// returned.
973 Address LB = Address::invalid();
974 /// Address of the output variable in which the upper iteration number is
975 /// returned.
976 Address UB = Address::invalid();
977 /// Address of the output variable in which the stride value is returned
978 /// necessary to generated the static_chunked scheduled loop.
979 Address ST = Address::invalid();
980 /// Value of the chunk for the static_chunked scheduled loop. For the
981 /// default (nullptr) value, the chunk 1 will be used.
982 llvm::Value *Chunk = nullptr;
983 StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL,
984 Address LB, Address UB, Address ST,
985 llvm::Value *Chunk = nullptr)
986 : IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB),
987 UB(UB), ST(ST), Chunk(Chunk) {}
988 };
989 /// Call the appropriate runtime routine to initialize it before start
990 /// of loop.
991 ///
992 /// This is used only in case of static schedule, when the user did not
993 /// specify a ordered clause on the loop construct.
994 /// Depending on the loop schedule, it is necessary to call some runtime
995 /// routine before start of the OpenMP loop to get the loop upper / lower
996 /// bounds LB and UB and stride ST.
997 ///
998 /// \param CGF Reference to current CodeGenFunction.
999 /// \param Loc Clang source location.
1000 /// \param DKind Kind of the directive.
1001 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1002 /// \param Values Input arguments for the construct.
1003 ///
1004 virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1005 OpenMPDirectiveKind DKind,
1006 const OpenMPScheduleTy &ScheduleKind,
1007 const StaticRTInput &Values);
1008
1009 ///
1010 /// \param CGF Reference to current CodeGenFunction.
1011 /// \param Loc Clang source location.
1012 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1013 /// \param Values Input arguments for the construct.
1014 ///
1015 virtual void emitDistributeStaticInit(CodeGenFunction &CGF,
1016 SourceLocation Loc,
1017 OpenMPDistScheduleClauseKind SchedKind,
1018 const StaticRTInput &Values);
1019
1020 /// Call the appropriate runtime routine to notify that we finished
1021 /// iteration of the ordered loop with the dynamic scheduling.
1022 ///
1023 /// \param CGF Reference to current CodeGenFunction.
1024 /// \param Loc Clang source location.
1025 /// \param IVSize Size of the iteration variable in bits.
1026 /// \param IVSigned Sign of the iteration variable.
1027 ///
1028 virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF,
1029 SourceLocation Loc, unsigned IVSize,
1030 bool IVSigned);
1031
1032 /// Call the appropriate runtime routine to notify that we finished
1033 /// all the work with current loop.
1034 ///
1035 /// \param CGF Reference to current CodeGenFunction.
1036 /// \param Loc Clang source location.
1037 /// \param DKind Kind of the directive for which the static finish is emitted.
1038 ///
1039 virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1040 OpenMPDirectiveKind DKind);
1041
1042 /// Call __kmpc_dispatch_next(
1043 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1044 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1045 /// kmp_int[32|64] *p_stride);
1046 /// \param IVSize Size of the iteration variable in bits.
1047 /// \param IVSigned Sign of the iteration variable.
1048 /// \param IL Address of the output variable in which the flag of the
1049 /// last iteration is returned.
1050 /// \param LB Address of the output variable in which the lower iteration
1051 /// number is returned.
1052 /// \param UB Address of the output variable in which the upper iteration
1053 /// number is returned.
1054 /// \param ST Address of the output variable in which the stride value is
1055 /// returned.
1056 virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1057 unsigned IVSize, bool IVSigned,
1058 Address IL, Address LB,
1059 Address UB, Address ST);
1060
1061 virtual llvm::Value *emitMessageClause(CodeGenFunction &CGF,
1062 const Expr *Message,
1063 SourceLocation Loc);
1064
1065 virtual llvm::Value *emitSeverityClause(OpenMPSeverityClauseKind Severity,
1066 SourceLocation Loc);
1067
1068 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1069 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1070 /// clause.
1071 /// If the modifier 'strict' is given:
1072 /// Emits call to void __kmpc_push_num_threads_strict(ident_t *loc, kmp_int32
1073 /// global_tid, kmp_int32 num_threads, int severity, const char *message) to
1074 /// generate code for 'num_threads' clause with 'strict' modifier.
1075 /// \param NumThreads An integer value of threads.
1076 virtual void emitNumThreadsClause(
1077 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
1078 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown,
1079 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
1080 SourceLocation SeverityLoc = SourceLocation(),
1081 const Expr *Message = nullptr,
1082 SourceLocation MessageLoc = SourceLocation());
1083
1084 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1085 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1086 virtual void emitProcBindClause(CodeGenFunction &CGF,
1087 llvm::omp::ProcBindKind ProcBind,
1088 SourceLocation Loc);
1089
1090 /// Returns address of the threadprivate variable for the current
1091 /// thread.
1092 /// \param VD Threadprivate variable.
1093 /// \param VDAddr Address of the global variable \a VD.
1094 /// \param Loc Location of the reference to threadprivate var.
1095 /// \return Address of the threadprivate variable for the current thread.
1096 virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1097 const VarDecl *VD, Address VDAddr,
1098 SourceLocation Loc);
1099
1100 /// Returns the address of the variable marked as declare target with link
1101 /// clause OR as declare target with to clause and unified memory.
1102 virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD);
1103
1104 /// Emit a code for initialization of threadprivate variable. It emits
1105 /// a call to runtime library which adds initial value to the newly created
1106 /// threadprivate variable (if it is not constant) and registers destructor
1107 /// for the variable (if any).
1108 /// \param VD Threadprivate variable.
1109 /// \param VDAddr Address of the global variable \a VD.
1110 /// \param Loc Location of threadprivate declaration.
1111 /// \param PerformInit true if initialization expression is not constant.
1112 virtual llvm::Function *
1113 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1114 SourceLocation Loc, bool PerformInit,
1115 CodeGenFunction *CGF = nullptr);
1116
1117 /// Emit code for handling declare target functions in the runtime.
1118 /// \param FD Declare target function.
1119 /// \param Addr Address of the global \a FD.
1120 /// \param PerformInit true if initialization expression is not constant.
1121 virtual void emitDeclareTargetFunction(const FunctionDecl *FD,
1122 llvm::GlobalValue *GV);
1123
1124 /// Register VTable to OpenMP offload entry.
1125 /// \param VTable VTable of the C++ class.
1126 /// \param RD C++ class decl.
1127 virtual void registerVTableOffloadEntry(llvm::GlobalVariable *VTable,
1128 const VarDecl *VD);
1129 /// Emit code for registering vtable by scanning through map clause
1130 /// in OpenMP target region.
1131 /// \param D OpenMP target directive.
1132 virtual void registerVTable(const OMPExecutableDirective &D);
1133
1134 /// Emit and register VTable for the C++ class in OpenMP offload entry.
1135 /// \param CXXRecord C++ class decl.
1136 /// \param VD Variable decl which holds VTable.
1137 virtual void emitAndRegisterVTable(CodeGenModule &CGM,
1138 CXXRecordDecl *CXXRecord,
1139 const VarDecl *VD);
1140
1141 /// Creates artificial threadprivate variable with name \p Name and type \p
1142 /// VarType.
1143 /// \param VarType Type of the artificial threadprivate variable.
1144 /// \param Name Name of the artificial threadprivate variable.
1145 virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1146 QualType VarType,
1147 StringRef Name);
1148
1149 /// Emit flush of the variables specified in 'omp flush' directive.
1150 /// \param Vars List of variables to flush.
1151 virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1152 SourceLocation Loc, llvm::AtomicOrdering AO);
1153
1154 /// Emit task region for the task directive. The task region is
1155 /// emitted in several steps:
1156 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1157 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1158 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1159 /// function:
1160 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1161 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1162 /// return 0;
1163 /// }
1164 /// 2. Copy a list of shared variables to field shareds of the resulting
1165 /// structure kmp_task_t returned by the previous call (if any).
1166 /// 3. Copy a pointer to destructions function to field destructions of the
1167 /// resulting structure kmp_task_t.
1168 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1169 /// kmp_task_t *new_task), where new_task is a resulting structure from
1170 /// previous items.
1171 /// \param D Current task directive.
1172 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1173 /// /*part_id*/, captured_struct */*__context*/);
1174 /// \param SharedsTy A type which contains references the shared variables.
1175 /// \param Shareds Context with the list of shared variables from the \p
1176 /// TaskFunction.
1177 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1178 /// otherwise.
1179 /// \param Data Additional data for task generation like tiednsee, final
1180 /// state, list of privates etc.
1181 virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1182 const OMPExecutableDirective &D,
1183 llvm::Function *TaskFunction, QualType SharedsTy,
1184 Address Shareds, const Expr *IfCond,
1185 const OMPTaskDataTy &Data);
1186
1187 /// Emit task region for the taskloop directive. The taskloop region is
1188 /// emitted in several steps:
1189 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1190 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1191 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1192 /// function:
1193 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1194 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1195 /// return 0;
1196 /// }
1197 /// 2. Copy a list of shared variables to field shareds of the resulting
1198 /// structure kmp_task_t returned by the previous call (if any).
1199 /// 3. Copy a pointer to destructions function to field destructions of the
1200 /// resulting structure kmp_task_t.
1201 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
1202 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
1203 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
1204 /// is a resulting structure from
1205 /// previous items.
1206 /// \param D Current task directive.
1207 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1208 /// /*part_id*/, captured_struct */*__context*/);
1209 /// \param SharedsTy A type which contains references the shared variables.
1210 /// \param Shareds Context with the list of shared variables from the \p
1211 /// TaskFunction.
1212 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1213 /// otherwise.
1214 /// \param Data Additional data for task generation like tiednsee, final
1215 /// state, list of privates etc.
1216 virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
1217 const OMPLoopDirective &D,
1218 llvm::Function *TaskFunction,
1219 QualType SharedsTy, Address Shareds,
1220 const Expr *IfCond, const OMPTaskDataTy &Data);
1221
1222 /// Emit code for the directive that does not require outlining.
1223 ///
1224 /// \param InnermostKind Kind of innermost directive (for simple directives it
1225 /// is a directive itself, for combined - its innermost directive).
1226 /// \param CodeGen Code generation sequence for the \a D directive.
1227 /// \param HasCancel true if region has inner cancel directive, false
1228 /// otherwise.
1229 virtual void emitInlinedDirective(CodeGenFunction &CGF,
1230 OpenMPDirectiveKind InnermostKind,
1231 const RegionCodeGenTy &CodeGen,
1232 bool HasCancel = false);
1233
1234 /// Emits reduction function.
1235 /// \param ReducerName Name of the function calling the reduction.
1236 /// \param ArgsElemType Array type containing pointers to reduction variables.
1237 /// \param Privates List of private copies for original reduction arguments.
1238 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1239 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1240 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1241 /// or 'operator binop(LHS, RHS)'.
1242 llvm::Function *emitReductionFunction(
1243 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
1244 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
1245 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps);
1246
1247 /// Emits single reduction combiner
1248 void emitSingleReductionCombiner(CodeGenFunction &CGF,
1249 const Expr *ReductionOp,
1250 const Expr *PrivateRef,
1251 const DeclRefExpr *LHS,
1252 const DeclRefExpr *RHS);
1253
1254 struct ReductionOptionsTy {
1255 bool WithNowait;
1256 bool SimpleReduction;
1257 llvm::SmallVector<bool, 8> IsPrivateVarReduction;
1258 OpenMPDirectiveKind ReductionKind;
1259 };
1260
1261 /// Emits code for private variable reduction
1262 /// \param Privates List of private copies for original reduction arguments.
1263 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1264 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1265 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1266 /// or 'operator binop(LHS, RHS)'.
1267 void emitPrivateReduction(CodeGenFunction &CGF, SourceLocation Loc,
1268 const Expr *Privates, const Expr *LHSExprs,
1269 const Expr *RHSExprs, const Expr *ReductionOps);
1270
1271 /// Emit a code for reduction clause. Next code should be emitted for
1272 /// reduction:
1273 /// \code
1274 ///
1275 /// static kmp_critical_name lock = { 0 };
1276 ///
1277 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
1278 /// ...
1279 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
1280 /// ...
1281 /// }
1282 ///
1283 /// ...
1284 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
1285 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
1286 /// RedList, reduce_func, &<lock>)) {
1287 /// case 1:
1288 /// ...
1289 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
1290 /// ...
1291 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
1292 /// break;
1293 /// case 2:
1294 /// ...
1295 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
1296 /// ...
1297 /// break;
1298 /// default:;
1299 /// }
1300 /// \endcode
1301 ///
1302 /// \param Privates List of private copies for original reduction arguments.
1303 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1304 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1305 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1306 /// or 'operator binop(LHS, RHS)'.
1307 /// \param Options List of options for reduction codegen:
1308 /// WithNowait true if parent directive has also nowait clause, false
1309 /// otherwise.
1310 /// SimpleReduction Emit reduction operation only. Used for omp simd
1311 /// directive on the host.
1312 /// ReductionKind The kind of reduction to perform.
1313 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
1314 ArrayRef<const Expr *> Privates,
1315 ArrayRef<const Expr *> LHSExprs,
1316 ArrayRef<const Expr *> RHSExprs,
1317 ArrayRef<const Expr *> ReductionOps,
1318 ReductionOptionsTy Options);
1319
1320 /// Emit a code for initialization of task reduction clause. Next code
1321 /// should be emitted for reduction:
1322 /// \code
1323 ///
1324 /// _taskred_item_t red_data[n];
1325 /// ...
1326 /// red_data[i].shar = &shareds[i];
1327 /// red_data[i].orig = &origs[i];
1328 /// red_data[i].size = sizeof(origs[i]);
1329 /// red_data[i].f_init = (void*)RedInit<i>;
1330 /// red_data[i].f_fini = (void*)RedDest<i>;
1331 /// red_data[i].f_comb = (void*)RedOp<i>;
1332 /// red_data[i].flags = <Flag_i>;
1333 /// ...
1334 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
1335 /// \endcode
1336 /// For reduction clause with task modifier it emits the next call:
1337 /// \code
1338 ///
1339 /// _taskred_item_t red_data[n];
1340 /// ...
1341 /// red_data[i].shar = &shareds[i];
1342 /// red_data[i].orig = &origs[i];
1343 /// red_data[i].size = sizeof(origs[i]);
1344 /// red_data[i].f_init = (void*)RedInit<i>;
1345 /// red_data[i].f_fini = (void*)RedDest<i>;
1346 /// red_data[i].f_comb = (void*)RedOp<i>;
1347 /// red_data[i].flags = <Flag_i>;
1348 /// ...
1349 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
1350 /// red_data);
1351 /// \endcode
1352 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
1353 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
1354 /// \param Data Additional data for task generation like tiedness, final
1355 /// state, list of privates, reductions etc.
1356 virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF,
1357 SourceLocation Loc,
1358 ArrayRef<const Expr *> LHSExprs,
1359 ArrayRef<const Expr *> RHSExprs,
1360 const OMPTaskDataTy &Data);
1361
1362 /// Emits the following code for reduction clause with task modifier:
1363 /// \code
1364 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
1365 /// \endcode
1366 virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
1367 bool IsWorksharingReduction);
1368
1369 /// Required to resolve existing problems in the runtime. Emits threadprivate
1370 /// variables to store the size of the VLAs/array sections for
1371 /// initializer/combiner/finalizer functions.
1372 /// \param RCG Allows to reuse an existing data for the reductions.
1373 /// \param N Reduction item for which fixups must be emitted.
1374 virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
1375 ReductionCodeGen &RCG, unsigned N);
1376
1377 /// Get the address of `void *` type of the privatue copy of the reduction
1378 /// item specified by the \p SharedLVal.
1379 /// \param ReductionsPtr Pointer to the reduction data returned by the
1380 /// emitTaskReductionInit function.
1381 /// \param SharedLVal Address of the original reduction item.
1382 virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
1383 llvm::Value *ReductionsPtr,
1384 LValue SharedLVal);
1385
1386 /// Emit code for 'taskwait' directive.
1387 virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
1388 const OMPTaskDataTy &Data);
1389
1390 /// Emit code for 'cancellation point' construct.
1391 /// \param CancelRegion Region kind for which the cancellation point must be
1392 /// emitted.
1393 ///
1394 virtual void emitCancellationPointCall(CodeGenFunction &CGF,
1395 SourceLocation Loc,
1396 OpenMPDirectiveKind CancelRegion);
1397
1398 /// Emit code for 'cancel' construct.
1399 /// \param IfCond Condition in the associated 'if' clause, if it was
1400 /// specified, nullptr otherwise.
1401 /// \param CancelRegion Region kind for which the cancel must be emitted.
1402 ///
1403 virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
1404 const Expr *IfCond,
1405 OpenMPDirectiveKind CancelRegion);
1406
1407 /// Emit the '<kernel>_kernel_environment' global for a target region that
1408 /// is compiled for a non-GPU (host-style) offload target, which does not
1409 /// go through the GPU device-side runtime init/deinit sequence.
1410 void emitHostKernelEnvironment(const OMPExecutableDirective &D,
1411 CodeGenFunction &CGF);
1412
1413 /// Emit outilined function for 'target' directive.
1414 /// \param D Directive to emit.
1415 /// \param ParentName Name of the function that encloses the target region.
1416 /// \param OutlinedFn Outlined function value to be defined by this call.
1417 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
1418 /// \param IsOffloadEntry True if the outlined function is an offload entry.
1419 /// \param CodeGen Code generation sequence for the \a D directive.
1420 /// An outlined function may not be an entry if, e.g. the if clause always
1421 /// evaluates to false.
1422 virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
1423 StringRef ParentName,
1424 llvm::Function *&OutlinedFn,
1425 llvm::Constant *&OutlinedFnID,
1426 bool IsOffloadEntry,
1427 const RegionCodeGenTy &CodeGen);
1428
1429 /// Returns the parent name of the target regions in \a FD: its enclosing
1430 /// non-closure function if \a FD is a closure, otherwise \a FD itself;
1431 /// constructors and destructors use their complete variant.
1432 StringRef getTargetRegionParentName(const FunctionDecl *FD) const;
1433
1434 /// Emit the target offloading code associated with \a D. The emitted
1435 /// code attempts offloading the execution to the device, an the event of
1436 /// a failure it executes the host version outlined in \a OutlinedFn.
1437 /// \param D Directive to emit.
1438 /// \param OutlinedFn Host version of the code to be offloaded.
1439 /// \param OutlinedFnID ID of host version of the code to be offloaded.
1440 /// \param IfCond Expression evaluated in if clause associated with the target
1441 /// directive, or null if no if clause is used.
1442 /// \param Device Expression evaluated in device clause associated with the
1443 /// target directive, or null if no device clause is used and device modifier.
1444 /// \param SizeEmitter Callback to emit number of iterations for loop-based
1445 /// directives.
1446 virtual void emitTargetCall(
1447 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1448 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
1449 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
1450 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
1451 const OMPLoopDirective &D)>
1452 SizeEmitter);
1453
1454 /// Emit the target regions enclosed in \a GD function definition or
1455 /// the function itself in case it is a valid device function. Returns true if
1456 /// \a GD was dealt with successfully.
1457 /// \param GD Function to scan.
1458 virtual bool emitTargetFunctions(GlobalDecl GD);
1459
1460 /// Emit the global variable if it is a valid device global variable.
1461 /// Returns true if \a GD was dealt with successfully.
1462 /// \param GD Variable declaration to emit.
1463 virtual bool emitTargetGlobalVariable(GlobalDecl GD);
1464
1465 /// Checks if the provided global decl \a GD is a declare target variable and
1466 /// registers it when emitting code for the host.
1467 virtual void registerTargetGlobalVariable(const VarDecl *VD,
1468 llvm::Constant *Addr);
1469
1470 /// Emit the global \a GD if it is meaningful for the target. Returns
1471 /// if it was emitted successfully.
1472 /// \param GD Global to scan.
1473 virtual bool emitTargetGlobal(GlobalDecl GD);
1474
1475 /// Creates all the offload entries in the current compilation unit
1476 /// along with the associated metadata.
1477 void createOffloadEntriesAndInfoMetadata();
1478
1479 /// Emits code for teams call of the \a OutlinedFn with
1480 /// variables captured in a record which address is stored in \a
1481 /// CapturedStruct.
1482 /// \param OutlinedFn Outlined function to be run by team masters. Type of
1483 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1484 /// \param CapturedVars A pointer to the record with the references to
1485 /// variables used in \a OutlinedFn function.
1486 ///
1487 virtual void emitTeamsCall(CodeGenFunction &CGF,
1488 const OMPExecutableDirective &D,
1489 SourceLocation Loc, llvm::Function *OutlinedFn,
1490 ArrayRef<llvm::Value *> CapturedVars);
1491
1492 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
1493 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
1494 /// for num_teams clause.
1495 /// \param NumTeams An integer expression of teams.
1496 /// \param ThreadLimit An integer expression of threads.
1497 virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
1498 const Expr *ThreadLimit, SourceLocation Loc);
1499
1500 /// Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32
1501 /// global_tid, kmp_int32 thread_limit) to generate code for
1502 /// thread_limit clause on target directive
1503 /// \param ThreadLimit An integer expression of threads.
1504 virtual void emitThreadLimitClause(CodeGenFunction &CGF,
1505 const Expr *ThreadLimit,
1506 SourceLocation Loc);
1507
1508 /// Struct that keeps all the relevant information that should be kept
1509 /// throughout a 'target data' region.
1510 class TargetDataInfo : public llvm::OpenMPIRBuilder::TargetDataInfo {
1511 public:
1512 explicit TargetDataInfo() : llvm::OpenMPIRBuilder::TargetDataInfo() {}
1513 explicit TargetDataInfo(bool RequiresDevicePointerInfo,
1514 bool SeparateBeginEndCalls)
1515 : llvm::OpenMPIRBuilder::TargetDataInfo(RequiresDevicePointerInfo,
1516 SeparateBeginEndCalls) {}
1517 /// Map between the a declaration of a capture and the corresponding new
1518 /// llvm address where the runtime returns the device pointers.
1519 llvm::DenseMap<const ValueDecl *, llvm::Value *> CaptureDeviceAddrMap;
1520 };
1521
1522 /// Emit the target data mapping code associated with \a D.
1523 /// \param D Directive to emit.
1524 /// \param IfCond Expression evaluated in if clause associated with the
1525 /// target directive, or null if no device clause is used.
1526 /// \param Device Expression evaluated in device clause associated with the
1527 /// target directive, or null if no device clause is used.
1528 /// \param Info A record used to store information that needs to be preserved
1529 /// until the region is closed.
1530 virtual void emitTargetDataCalls(CodeGenFunction &CGF,
1531 const OMPExecutableDirective &D,
1532 const Expr *IfCond, const Expr *Device,
1533 const RegionCodeGenTy &CodeGen,
1534 CGOpenMPRuntime::TargetDataInfo &Info);
1535
1536 /// Emit the data mapping/movement code associated with the directive
1537 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
1538 /// \param D Directive to emit.
1539 /// \param IfCond Expression evaluated in if clause associated with the target
1540 /// directive, or null if no if clause is used.
1541 /// \param Device Expression evaluated in device clause associated with the
1542 /// target directive, or null if no device clause is used.
1543 virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
1544 const OMPExecutableDirective &D,
1545 const Expr *IfCond,
1546 const Expr *Device);
1547
1548 /// Marks function \a Fn with properly mangled versions of vector functions.
1549 /// \param FD Function marked as 'declare simd'.
1550 /// \param Fn LLVM function that must be marked with 'declare simd'
1551 /// attributes.
1552 virtual void emitDeclareSimdFunction(const FunctionDecl *FD,
1553 llvm::Function *Fn);
1554
1555 /// Emit initialization for doacross loop nesting support.
1556 /// \param D Loop-based construct used in doacross nesting construct.
1557 virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
1558 ArrayRef<Expr *> NumIterations);
1559
1560 /// Emit code for doacross ordered directive with 'depend' clause.
1561 /// \param C 'depend' clause with 'sink|source' dependency kind.
1562 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1563 const OMPDependClause *C);
1564
1565 /// Emit code for doacross ordered directive with 'doacross' clause.
1566 /// \param C 'doacross' clause with 'sink|source' dependence type.
1567 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1568 const OMPDoacrossClause *C);
1569
1570 /// Translates the native parameter of outlined function if this is required
1571 /// for target.
1572 /// \param FD Field decl from captured record for the parameter.
1573 /// \param NativeParam Parameter itself.
1574 virtual const VarDecl *translateParameter(const FieldDecl *FD,
1575 const VarDecl *NativeParam) const {
1576 return NativeParam;
1577 }
1578
1579 /// Gets the address of the native argument basing on the address of the
1580 /// target-specific parameter.
1581 /// \param NativeParam Parameter itself.
1582 /// \param TargetParam Corresponding target-specific parameter.
1583 virtual Address getParameterAddress(CodeGenFunction &CGF,
1584 const VarDecl *NativeParam,
1585 const VarDecl *TargetParam) const;
1586
1587 /// Choose default schedule type and chunk value for the
1588 /// dist_schedule clause.
1589 virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
1590 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
1591 llvm::Value *&Chunk) const {}
1592
1593 /// Choose default schedule type and chunk value for the
1594 /// schedule clause.
1595 virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
1596 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
1597 const Expr *&ChunkExpr) const;
1598
1599 /// Emits call of the outlined function with the provided arguments,
1600 /// translating these arguments to correct target-specific arguments.
1601 virtual void
1602 emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
1603 llvm::FunctionCallee OutlinedFn,
1604 ArrayRef<llvm::Value *> Args = {}) const;
1605
1606 /// Emits OpenMP-specific function prolog.
1607 /// Required for device constructs.
1608 virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D);
1609
1610 /// Gets the OpenMP-specific address of the local variable.
1611 virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1612 const VarDecl *VD);
1613
1614 /// Marks the declaration as already emitted for the device code and returns
1615 /// true, if it was marked already, and false, otherwise.
1616 bool markAsGlobalTarget(GlobalDecl GD);
1617
1618 /// Emit deferred declare target variables marked for deferred emission.
1619 void emitDeferredTargetDecls() const;
1620
1621 /// Adjust some parameters for the target-based directives, like addresses of
1622 /// the variables captured by reference in lambdas.
1623 virtual void
1624 adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF,
1625 const OMPExecutableDirective &D) const;
1626
1627 /// Perform check on requires decl to ensure that target architecture
1628 /// supports unified addressing
1629 virtual void processRequiresDirective(const OMPRequiresDecl *D);
1630
1631 /// Gets default memory ordering as specified in requires directive.
1632 llvm::AtomicOrdering getDefaultMemoryOrdering() const;
1633
1634 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with
1635 /// the predefined allocator and translates it into the corresponding address
1636 /// space.
1637 virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS);
1638
1639 /// Return whether the unified_shared_memory has been specified.
1640 bool hasRequiresUnifiedSharedMemory() const;
1641
1642 /// Checks if the \p VD variable is marked as nontemporal declaration in
1643 /// current context.
1644 bool isNontemporalDecl(const ValueDecl *VD) const;
1645
1646 /// Create specialized alloca to handle lastprivate conditionals.
1647 Address emitLastprivateConditionalInit(CodeGenFunction &CGF,
1648 const VarDecl *VD);
1649
1650 /// Checks if the provided \p LVal is lastprivate conditional and emits the
1651 /// code to update the value of the original variable.
1652 /// \code
1653 /// lastprivate(conditional: a)
1654 /// ...
1655 /// <type> a;
1656 /// lp_a = ...;
1657 /// #pragma omp critical(a)
1658 /// if (last_iv_a <= iv) {
1659 /// last_iv_a = iv;
1660 /// global_a = lp_a;
1661 /// }
1662 /// \endcode
1663 virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
1664 const Expr *LHS);
1665
1666 /// Checks if the lastprivate conditional was updated in inner region and
1667 /// writes the value.
1668 /// \code
1669 /// lastprivate(conditional: a)
1670 /// ...
1671 /// <type> a;bool Fired = false;
1672 /// #pragma omp ... shared(a)
1673 /// {
1674 /// lp_a = ...;
1675 /// Fired = true;
1676 /// }
1677 /// if (Fired) {
1678 /// #pragma omp critical(a)
1679 /// if (last_iv_a <= iv) {
1680 /// last_iv_a = iv;
1681 /// global_a = lp_a;
1682 /// }
1683 /// Fired = false;
1684 /// }
1685 /// \endcode
1686 virtual void checkAndEmitSharedLastprivateConditional(
1687 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1688 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls);
1689
1690 /// Gets the address of the global copy used for lastprivate conditional
1691 /// update, if any.
1692 /// \param PrivLVal LValue for the private copy.
1693 /// \param VD Original lastprivate declaration.
1694 virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF,
1695 LValue PrivLVal,
1696 const VarDecl *VD,
1697 SourceLocation Loc);
1698
1699 /// Emits list of dependecies based on the provided data (array of
1700 /// dependence/expression pairs).
1701 /// \returns Pointer to the first element of the array casted to VoidPtr type.
1702 std::pair<llvm::Value *, Address>
1703 emitDependClause(CodeGenFunction &CGF,
1704 ArrayRef<OMPTaskDataTy::DependData> Dependencies,
1705 SourceLocation Loc);
1706
1707 /// Emits list of dependecies based on the provided data (array of
1708 /// dependence/expression pairs) for depobj construct. In this case, the
1709 /// variable is allocated in dynamically. \returns Pointer to the first
1710 /// element of the array casted to VoidPtr type.
1711 Address emitDepobjDependClause(CodeGenFunction &CGF,
1712 const OMPTaskDataTy::DependData &Dependencies,
1713 SourceLocation Loc);
1714
1715 /// Emits the code to destroy the dependency object provided in depobj
1716 /// directive.
1717 void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
1718 SourceLocation Loc);
1719
1720 /// Updates the dependency kind in the specified depobj object.
1721 /// \param DepobjLVal LValue for the main depobj object.
1722 /// \param NewDepKind New dependency kind.
1723 void emitUpdateDependObjectsClause(CodeGenFunction &CGF, LValue DepobjLVal,
1724 OpenMPDependClauseKind NewDepKind,
1725 SourceLocation Loc);
1726
1727 /// Initializes user defined allocators specified in the uses_allocators
1728 /// clauses.
1729 void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator,
1730 const Expr *AllocatorTraits);
1731
1732 /// Destroys user defined allocators specified in the uses_allocators clause.
1733 void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator);
1734
1735 /// Returns true if the variable is a local variable in untied task.
1736 bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const;
1737};
1738
1739/// Class supports emissionof SIMD-only code.
1740class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
1741public:
1742 explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {}
1743 ~CGOpenMPSIMDRuntime() override {}
1744
1745 /// Emits outlined function for the specified OpenMP parallel directive
1746 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1747 /// kmp_int32 BoundID, struct context_vars*).
1748 /// \param CGF Reference to current CodeGenFunction.
1749 /// \param D OpenMP directive.
1750 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1751 /// \param InnermostKind Kind of innermost directive (for simple directives it
1752 /// is a directive itself, for combined - its innermost directive).
1753 /// \param CodeGen Code generation sequence for the \a D directive.
1754 llvm::Function *emitParallelOutlinedFunction(
1755 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1756 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1757 const RegionCodeGenTy &CodeGen) override;
1758
1759 /// Emits outlined function for the specified OpenMP teams directive
1760 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1761 /// kmp_int32 BoundID, struct context_vars*).
1762 /// \param CGF Reference to current CodeGenFunction.
1763 /// \param D OpenMP directive.
1764 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1765 /// \param InnermostKind Kind of innermost directive (for simple directives it
1766 /// is a directive itself, for combined - its innermost directive).
1767 /// \param CodeGen Code generation sequence for the \a D directive.
1768 llvm::Function *emitTeamsOutlinedFunction(
1769 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1770 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1771 const RegionCodeGenTy &CodeGen) override;
1772
1773 /// Emits outlined function for the OpenMP task directive \a D. This
1774 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
1775 /// TaskT).
1776 /// \param D OpenMP directive.
1777 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1778 /// \param PartIDVar Variable for partition id in the current OpenMP untied
1779 /// task region.
1780 /// \param TaskTVar Variable for task_t argument.
1781 /// \param InnermostKind Kind of innermost directive (for simple directives it
1782 /// is a directive itself, for combined - its innermost directive).
1783 /// \param CodeGen Code generation sequence for the \a D directive.
1784 /// \param Tied true if task is generated for tied task, false otherwise.
1785 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
1786 /// tasks.
1787 ///
1788 llvm::Function *emitTaskOutlinedFunction(
1789 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1790 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1791 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1792 bool Tied, unsigned &NumberOfParts) override;
1793
1794 /// Emits code for parallel or serial call of the \a OutlinedFn with
1795 /// variables captured in a record which address is stored in \a
1796 /// CapturedStruct.
1797 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
1798 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1799 /// \param CapturedVars A pointer to the record with the references to
1800 /// variables used in \a OutlinedFn function.
1801 /// \param IfCond Condition in the associated 'if' clause, if it was
1802 /// specified, nullptr otherwise.
1803 /// \param NumThreads The value corresponding to the num_threads clause, if
1804 /// any, or nullptr.
1805 /// \param NumThreadsModifier The modifier of the num_threads clause, if
1806 /// any, ignored otherwise.
1807 /// \param Severity The severity corresponding to the num_threads clause, if
1808 /// any, ignored otherwise.
1809 /// \param Message The message string corresponding to the num_threads clause,
1810 /// if any, or nullptr.
1811 ///
1812 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1813 llvm::Function *OutlinedFn,
1814 ArrayRef<llvm::Value *> CapturedVars,
1815 const Expr *IfCond, llvm::Value *NumThreads,
1816 OpenMPNumThreadsClauseModifier NumThreadsModifier =
1817 OMPC_NUMTHREADS_unknown,
1818 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
1819 const Expr *Message = nullptr) override;
1820
1821 /// Emits a critical region.
1822 /// \param CriticalName Name of the critical region.
1823 /// \param CriticalOpGen Generator for the statement associated with the given
1824 /// critical region.
1825 /// \param Hint Value of the 'hint' clause (optional).
1826 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
1827 const RegionCodeGenTy &CriticalOpGen,
1828 SourceLocation Loc,
1829 const Expr *Hint = nullptr) override;
1830
1831 /// Emits a master region.
1832 /// \param MasterOpGen Generator for the statement associated with the given
1833 /// master region.
1834 void emitMasterRegion(CodeGenFunction &CGF,
1835 const RegionCodeGenTy &MasterOpGen,
1836 SourceLocation Loc) override;
1837
1838 /// Emits a masked region.
1839 /// \param MaskedOpGen Generator for the statement associated with the given
1840 /// masked region.
1841 void emitMaskedRegion(CodeGenFunction &CGF,
1842 const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc,
1843 const Expr *Filter = nullptr) override;
1844
1845 /// Emits a masked region.
1846 /// \param MaskedOpGen Generator for the statement associated with the given
1847 /// masked region.
1848
1849 /// Emits code for a taskyield directive.
1850 void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override;
1851
1852 /// Emit a taskgroup region.
1853 /// \param TaskgroupOpGen Generator for the statement associated with the
1854 /// given taskgroup region.
1855 void emitTaskgroupRegion(CodeGenFunction &CGF,
1856 const RegionCodeGenTy &TaskgroupOpGen,
1857 SourceLocation Loc) override;
1858
1859 /// Emits a single region.
1860 /// \param SingleOpGen Generator for the statement associated with the given
1861 /// single region.
1862 void emitSingleRegion(CodeGenFunction &CGF,
1863 const RegionCodeGenTy &SingleOpGen, SourceLocation Loc,
1864 ArrayRef<const Expr *> CopyprivateVars,
1865 ArrayRef<const Expr *> DestExprs,
1866 ArrayRef<const Expr *> SrcExprs,
1867 ArrayRef<const Expr *> AssignmentOps) override;
1868
1869 /// Emit an ordered region.
1870 /// \param OrderedOpGen Generator for the statement associated with the given
1871 /// ordered region.
1872 void emitOrderedRegion(CodeGenFunction &CGF,
1873 const RegionCodeGenTy &OrderedOpGen,
1874 SourceLocation Loc, bool IsThreads) override;
1875
1876 /// Emit an implicit/explicit barrier for OpenMP threads.
1877 /// \param Kind Directive for which this implicit barrier call must be
1878 /// generated. Must be OMPD_barrier for explicit barrier generation.
1879 /// \param EmitChecks true if need to emit checks for cancellation barriers.
1880 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
1881 /// runtime class decides which one to emit (simple or with cancellation
1882 /// checks).
1883 ///
1884 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
1885 OpenMPDirectiveKind Kind, bool EmitChecks = true,
1886 bool ForceSimpleCall = false) override;
1887
1888 /// This is used for non static scheduled types and when the ordered
1889 /// clause is present on the loop construct.
1890 /// Depending on the loop schedule, it is necessary to call some runtime
1891 /// routine before start of the OpenMP loop to get the loop upper / lower
1892 /// bounds \a LB and \a UB and stride \a ST.
1893 ///
1894 /// \param CGF Reference to current CodeGenFunction.
1895 /// \param Loc Clang source location.
1896 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1897 /// \param IVSize Size of the iteration variable in bits.
1898 /// \param IVSigned Sign of the iteration variable.
1899 /// \param Ordered true if loop is ordered, false otherwise.
1900 /// \param DispatchValues struct containing llvm values for lower bound, upper
1901 /// bound, and chunk expression.
1902 /// For the default (nullptr) value, the chunk 1 will be used.
1903 ///
1904 void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
1905 const OpenMPScheduleTy &ScheduleKind,
1906 unsigned IVSize, bool IVSigned, bool Ordered,
1907 const DispatchRTInput &DispatchValues) override;
1908
1909 /// This is used for non static scheduled types and when the ordered
1910 /// clause is present on the loop construct.
1911 ///
1912 /// \param CGF Reference to current CodeGenFunction.
1913 /// \param Loc Clang source location.
1914 ///
1915 void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override;
1916
1917 /// Call the appropriate runtime routine to initialize it before start
1918 /// of loop.
1919 ///
1920 /// This is used only in case of static schedule, when the user did not
1921 /// specify a ordered clause on the loop construct.
1922 /// Depending on the loop schedule, it is necessary to call some runtime
1923 /// routine before start of the OpenMP loop to get the loop upper / lower
1924 /// bounds LB and UB and stride ST.
1925 ///
1926 /// \param CGF Reference to current CodeGenFunction.
1927 /// \param Loc Clang source location.
1928 /// \param DKind Kind of the directive.
1929 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1930 /// \param Values Input arguments for the construct.
1931 ///
1932 void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1933 OpenMPDirectiveKind DKind,
1934 const OpenMPScheduleTy &ScheduleKind,
1935 const StaticRTInput &Values) override;
1936
1937 ///
1938 /// \param CGF Reference to current CodeGenFunction.
1939 /// \param Loc Clang source location.
1940 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1941 /// \param Values Input arguments for the construct.
1942 ///
1943 void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1944 OpenMPDistScheduleClauseKind SchedKind,
1945 const StaticRTInput &Values) override;
1946
1947 /// Call the appropriate runtime routine to notify that we finished
1948 /// iteration of the ordered loop with the dynamic scheduling.
1949 ///
1950 /// \param CGF Reference to current CodeGenFunction.
1951 /// \param Loc Clang source location.
1952 /// \param IVSize Size of the iteration variable in bits.
1953 /// \param IVSigned Sign of the iteration variable.
1954 ///
1955 void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc,
1956 unsigned IVSize, bool IVSigned) override;
1957
1958 /// Call the appropriate runtime routine to notify that we finished
1959 /// all the work with current loop.
1960 ///
1961 /// \param CGF Reference to current CodeGenFunction.
1962 /// \param Loc Clang source location.
1963 /// \param DKind Kind of the directive for which the static finish is emitted.
1964 ///
1965 void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1966 OpenMPDirectiveKind DKind) override;
1967
1968 /// Call __kmpc_dispatch_next(
1969 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1970 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1971 /// kmp_int[32|64] *p_stride);
1972 /// \param IVSize Size of the iteration variable in bits.
1973 /// \param IVSigned Sign of the iteration variable.
1974 /// \param IL Address of the output variable in which the flag of the
1975 /// last iteration is returned.
1976 /// \param LB Address of the output variable in which the lower iteration
1977 /// number is returned.
1978 /// \param UB Address of the output variable in which the upper iteration
1979 /// number is returned.
1980 /// \param ST Address of the output variable in which the stride value is
1981 /// returned.
1982 llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1983 unsigned IVSize, bool IVSigned, Address IL,
1984 Address LB, Address UB, Address ST) override;
1985
1986 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1987 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1988 /// clause.
1989 /// If the modifier 'strict' is given:
1990 /// Emits call to void __kmpc_push_num_threads_strict(ident_t *loc, kmp_int32
1991 /// global_tid, kmp_int32 num_threads, int severity, const char *message) to
1992 /// generate code for 'num_threads' clause with 'strict' modifier.
1993 /// \param NumThreads An integer value of threads.
1994 void emitNumThreadsClause(
1995 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
1996 OpenMPNumThreadsClauseModifier Modifier = OMPC_NUMTHREADS_unknown,
1997 OpenMPSeverityClauseKind Severity = OMPC_SEVERITY_fatal,
1998 SourceLocation SeverityLoc = SourceLocation(),
1999 const Expr *Message = nullptr,
2000 SourceLocation MessageLoc = SourceLocation()) override;
2001
2002 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
2003 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
2004 void emitProcBindClause(CodeGenFunction &CGF,
2005 llvm::omp::ProcBindKind ProcBind,
2006 SourceLocation Loc) override;
2007
2008 /// Returns address of the threadprivate variable for the current
2009 /// thread.
2010 /// \param VD Threadprivate variable.
2011 /// \param VDAddr Address of the global variable \a VD.
2012 /// \param Loc Location of the reference to threadprivate var.
2013 /// \return Address of the threadprivate variable for the current thread.
2014 Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD,
2015 Address VDAddr, SourceLocation Loc) override;
2016
2017 /// Emit a code for initialization of threadprivate variable. It emits
2018 /// a call to runtime library which adds initial value to the newly created
2019 /// threadprivate variable (if it is not constant) and registers destructor
2020 /// for the variable (if any).
2021 /// \param VD Threadprivate variable.
2022 /// \param VDAddr Address of the global variable \a VD.
2023 /// \param Loc Location of threadprivate declaration.
2024 /// \param PerformInit true if initialization expression is not constant.
2025 llvm::Function *
2026 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
2027 SourceLocation Loc, bool PerformInit,
2028 CodeGenFunction *CGF = nullptr) override;
2029
2030 /// Creates artificial threadprivate variable with name \p Name and type \p
2031 /// VarType.
2032 /// \param VarType Type of the artificial threadprivate variable.
2033 /// \param Name Name of the artificial threadprivate variable.
2034 Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
2035 QualType VarType,
2036 StringRef Name) override;
2037
2038 /// Emit flush of the variables specified in 'omp flush' directive.
2039 /// \param Vars List of variables to flush.
2040 void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
2041 SourceLocation Loc, llvm::AtomicOrdering AO) override;
2042
2043 /// Emit task region for the task directive. The task region is
2044 /// emitted in several steps:
2045 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
2046 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2047 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
2048 /// function:
2049 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
2050 /// TaskFunction(gtid, tt->part_id, tt->shareds);
2051 /// return 0;
2052 /// }
2053 /// 2. Copy a list of shared variables to field shareds of the resulting
2054 /// structure kmp_task_t returned by the previous call (if any).
2055 /// 3. Copy a pointer to destructions function to field destructions of the
2056 /// resulting structure kmp_task_t.
2057 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
2058 /// kmp_task_t *new_task), where new_task is a resulting structure from
2059 /// previous items.
2060 /// \param D Current task directive.
2061 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
2062 /// /*part_id*/, captured_struct */*__context*/);
2063 /// \param SharedsTy A type which contains references the shared variables.
2064 /// \param Shareds Context with the list of shared variables from the \p
2065 /// TaskFunction.
2066 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
2067 /// otherwise.
2068 /// \param Data Additional data for task generation like tiednsee, final
2069 /// state, list of privates etc.
2070 void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
2071 const OMPExecutableDirective &D,
2072 llvm::Function *TaskFunction, QualType SharedsTy,
2073 Address Shareds, const Expr *IfCond,
2074 const OMPTaskDataTy &Data) override;
2075
2076 /// Emit task region for the taskloop directive. The taskloop region is
2077 /// emitted in several steps:
2078 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
2079 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2080 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
2081 /// function:
2082 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
2083 /// TaskFunction(gtid, tt->part_id, tt->shareds);
2084 /// return 0;
2085 /// }
2086 /// 2. Copy a list of shared variables to field shareds of the resulting
2087 /// structure kmp_task_t returned by the previous call (if any).
2088 /// 3. Copy a pointer to destructions function to field destructions of the
2089 /// resulting structure kmp_task_t.
2090 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
2091 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
2092 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
2093 /// is a resulting structure from
2094 /// previous items.
2095 /// \param D Current task directive.
2096 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
2097 /// /*part_id*/, captured_struct */*__context*/);
2098 /// \param SharedsTy A type which contains references the shared variables.
2099 /// \param Shareds Context with the list of shared variables from the \p
2100 /// TaskFunction.
2101 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
2102 /// otherwise.
2103 /// \param Data Additional data for task generation like tiednsee, final
2104 /// state, list of privates etc.
2105 void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
2106 const OMPLoopDirective &D, llvm::Function *TaskFunction,
2107 QualType SharedsTy, Address Shareds, const Expr *IfCond,
2108 const OMPTaskDataTy &Data) override;
2109
2110 /// Emit a code for reduction clause. Next code should be emitted for
2111 /// reduction:
2112 /// \code
2113 ///
2114 /// static kmp_critical_name lock = { 0 };
2115 ///
2116 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
2117 /// ...
2118 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
2119 /// ...
2120 /// }
2121 ///
2122 /// ...
2123 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
2124 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
2125 /// RedList, reduce_func, &<lock>)) {
2126 /// case 1:
2127 /// ...
2128 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2129 /// ...
2130 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2131 /// break;
2132 /// case 2:
2133 /// ...
2134 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
2135 /// ...
2136 /// break;
2137 /// default:;
2138 /// }
2139 /// \endcode
2140 ///
2141 /// \param Privates List of private copies for original reduction arguments.
2142 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
2143 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
2144 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
2145 /// or 'operator binop(LHS, RHS)'.
2146 /// \param Options List of options for reduction codegen:
2147 /// WithNowait true if parent directive has also nowait clause, false
2148 /// otherwise.
2149 /// SimpleReduction Emit reduction operation only. Used for omp simd
2150 /// directive on the host.
2151 /// ReductionKind The kind of reduction to perform.
2152 void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
2153 ArrayRef<const Expr *> Privates,
2154 ArrayRef<const Expr *> LHSExprs,
2155 ArrayRef<const Expr *> RHSExprs,
2156 ArrayRef<const Expr *> ReductionOps,
2157 ReductionOptionsTy Options) override;
2158
2159 /// Emit a code for initialization of task reduction clause. Next code
2160 /// should be emitted for reduction:
2161 /// \code
2162 ///
2163 /// _taskred_item_t red_data[n];
2164 /// ...
2165 /// red_data[i].shar = &shareds[i];
2166 /// red_data[i].orig = &origs[i];
2167 /// red_data[i].size = sizeof(origs[i]);
2168 /// red_data[i].f_init = (void*)RedInit<i>;
2169 /// red_data[i].f_fini = (void*)RedDest<i>;
2170 /// red_data[i].f_comb = (void*)RedOp<i>;
2171 /// red_data[i].flags = <Flag_i>;
2172 /// ...
2173 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
2174 /// \endcode
2175 /// For reduction clause with task modifier it emits the next call:
2176 /// \code
2177 ///
2178 /// _taskred_item_t red_data[n];
2179 /// ...
2180 /// red_data[i].shar = &shareds[i];
2181 /// red_data[i].orig = &origs[i];
2182 /// red_data[i].size = sizeof(origs[i]);
2183 /// red_data[i].f_init = (void*)RedInit<i>;
2184 /// red_data[i].f_fini = (void*)RedDest<i>;
2185 /// red_data[i].f_comb = (void*)RedOp<i>;
2186 /// red_data[i].flags = <Flag_i>;
2187 /// ...
2188 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
2189 /// red_data);
2190 /// \endcode
2191 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
2192 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
2193 /// \param Data Additional data for task generation like tiedness, final
2194 /// state, list of privates, reductions etc.
2195 llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc,
2196 ArrayRef<const Expr *> LHSExprs,
2197 ArrayRef<const Expr *> RHSExprs,
2198 const OMPTaskDataTy &Data) override;
2199
2200 /// Emits the following code for reduction clause with task modifier:
2201 /// \code
2202 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
2203 /// \endcode
2204 void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
2205 bool IsWorksharingReduction) override;
2206
2207 /// Required to resolve existing problems in the runtime. Emits threadprivate
2208 /// variables to store the size of the VLAs/array sections for
2209 /// initializer/combiner/finalizer functions + emits threadprivate variable to
2210 /// store the pointer to the original reduction item for the custom
2211 /// initializer defined by declare reduction construct.
2212 /// \param RCG Allows to reuse an existing data for the reductions.
2213 /// \param N Reduction item for which fixups must be emitted.
2214 void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
2215 ReductionCodeGen &RCG, unsigned N) override;
2216
2217 /// Get the address of `void *` type of the privatue copy of the reduction
2218 /// item specified by the \p SharedLVal.
2219 /// \param ReductionsPtr Pointer to the reduction data returned by the
2220 /// emitTaskReductionInit function.
2221 /// \param SharedLVal Address of the original reduction item.
2222 Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
2223 llvm::Value *ReductionsPtr,
2224 LValue SharedLVal) override;
2225
2226 /// Emit code for 'taskwait' directive.
2227 void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
2228 const OMPTaskDataTy &Data) override;
2229
2230 /// Emit code for 'cancellation point' construct.
2231 /// \param CancelRegion Region kind for which the cancellation point must be
2232 /// emitted.
2233 ///
2234 void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc,
2235 OpenMPDirectiveKind CancelRegion) override;
2236
2237 /// Emit code for 'cancel' construct.
2238 /// \param IfCond Condition in the associated 'if' clause, if it was
2239 /// specified, nullptr otherwise.
2240 /// \param CancelRegion Region kind for which the cancel must be emitted.
2241 ///
2242 void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
2243 const Expr *IfCond,
2244 OpenMPDirectiveKind CancelRegion) override;
2245
2246 /// Emit outilined function for 'target' directive.
2247 /// \param D Directive to emit.
2248 /// \param ParentName Name of the function that encloses the target region.
2249 /// \param OutlinedFn Outlined function value to be defined by this call.
2250 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
2251 /// \param IsOffloadEntry True if the outlined function is an offload entry.
2252 /// \param CodeGen Code generation sequence for the \a D directive.
2253 /// An outlined function may not be an entry if, e.g. the if clause always
2254 /// evaluates to false.
2255 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
2256 StringRef ParentName,
2257 llvm::Function *&OutlinedFn,
2258 llvm::Constant *&OutlinedFnID,
2259 bool IsOffloadEntry,
2260 const RegionCodeGenTy &CodeGen) override;
2261
2262 /// Emit the target offloading code associated with \a D. The emitted
2263 /// code attempts offloading the execution to the device, an the event of
2264 /// a failure it executes the host version outlined in \a OutlinedFn.
2265 /// \param D Directive to emit.
2266 /// \param OutlinedFn Host version of the code to be offloaded.
2267 /// \param OutlinedFnID ID of host version of the code to be offloaded.
2268 /// \param IfCond Expression evaluated in if clause associated with the target
2269 /// directive, or null if no if clause is used.
2270 /// \param Device Expression evaluated in device clause associated with the
2271 /// target directive, or null if no device clause is used and device modifier.
2272 void emitTargetCall(
2273 CodeGenFunction &CGF, const OMPExecutableDirective &D,
2274 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
2275 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
2276 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
2277 const OMPLoopDirective &D)>
2278 SizeEmitter) override;
2279
2280 /// Emit the target regions enclosed in \a GD function definition or
2281 /// the function itself in case it is a valid device function. Returns true if
2282 /// \a GD was dealt with successfully.
2283 /// \param GD Function to scan.
2284 bool emitTargetFunctions(GlobalDecl GD) override;
2285
2286 /// Emit the global variable if it is a valid device global variable.
2287 /// Returns true if \a GD was dealt with successfully.
2288 /// \param GD Variable declaration to emit.
2289 bool emitTargetGlobalVariable(GlobalDecl GD) override;
2290
2291 /// Emit the global \a GD if it is meaningful for the target. Returns
2292 /// if it was emitted successfully.
2293 /// \param GD Global to scan.
2294 bool emitTargetGlobal(GlobalDecl GD) override;
2295
2296 /// Emits code for teams call of the \a OutlinedFn with
2297 /// variables captured in a record which address is stored in \a
2298 /// CapturedStruct.
2299 /// \param OutlinedFn Outlined function to be run by team masters. Type of
2300 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
2301 /// \param CapturedVars A pointer to the record with the references to
2302 /// variables used in \a OutlinedFn function.
2303 ///
2304 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
2305 SourceLocation Loc, llvm::Function *OutlinedFn,
2306 ArrayRef<llvm::Value *> CapturedVars) override;
2307
2308 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
2309 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
2310 /// for num_teams clause.
2311 /// \param NumTeams An integer expression of teams.
2312 /// \param ThreadLimit An integer expression of threads.
2313 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
2314 const Expr *ThreadLimit, SourceLocation Loc) override;
2315
2316 /// Emit the target data mapping code associated with \a D.
2317 /// \param D Directive to emit.
2318 /// \param IfCond Expression evaluated in if clause associated with the
2319 /// target directive, or null if no device clause is used.
2320 /// \param Device Expression evaluated in device clause associated with the
2321 /// target directive, or null if no device clause is used.
2322 /// \param Info A record used to store information that needs to be preserved
2323 /// until the region is closed.
2324 void emitTargetDataCalls(CodeGenFunction &CGF,
2325 const OMPExecutableDirective &D, const Expr *IfCond,
2326 const Expr *Device, const RegionCodeGenTy &CodeGen,
2327 CGOpenMPRuntime::TargetDataInfo &Info) override;
2328
2329 /// Emit the data mapping/movement code associated with the directive
2330 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
2331 /// \param D Directive to emit.
2332 /// \param IfCond Expression evaluated in if clause associated with the target
2333 /// directive, or null if no if clause is used.
2334 /// \param Device Expression evaluated in device clause associated with the
2335 /// target directive, or null if no device clause is used.
2336 void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
2337 const OMPExecutableDirective &D,
2338 const Expr *IfCond,
2339 const Expr *Device) override;
2340
2341 /// Emit initialization for doacross loop nesting support.
2342 /// \param D Loop-based construct used in doacross nesting construct.
2343 void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
2344 ArrayRef<Expr *> NumIterations) override;
2345
2346 /// Emit code for doacross ordered directive with 'depend' clause.
2347 /// \param C 'depend' clause with 'sink|source' dependency kind.
2348 void emitDoacrossOrdered(CodeGenFunction &CGF,
2349 const OMPDependClause *C) override;
2350
2351 /// Emit code for doacross ordered directive with 'doacross' clause.
2352 /// \param C 'doacross' clause with 'sink|source' dependence type.
2353 void emitDoacrossOrdered(CodeGenFunction &CGF,
2354 const OMPDoacrossClause *C) override;
2355
2356 /// Translates the native parameter of outlined function if this is required
2357 /// for target.
2358 /// \param FD Field decl from captured record for the parameter.
2359 /// \param NativeParam Parameter itself.
2360 const VarDecl *translateParameter(const FieldDecl *FD,
2361 const VarDecl *NativeParam) const override;
2362
2363 /// Gets the address of the native argument basing on the address of the
2364 /// target-specific parameter.
2365 /// \param NativeParam Parameter itself.
2366 /// \param TargetParam Corresponding target-specific parameter.
2367 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
2368 const VarDecl *TargetParam) const override;
2369
2370 /// Gets the OpenMP-specific address of the local variable.
2371 Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2372 const VarDecl *VD) override {
2373 return Address::invalid();
2374 }
2375};
2376
2377} // namespace CodeGen
2378// Utility for openmp doacross clause kind
2379namespace {
2380template <typename T> class OMPDoacrossKind {
2381public:
2382 bool isSink(const T *) { return false; }
2383 bool isSource(const T *) { return false; }
2384};
2385template <> class OMPDoacrossKind<OMPDependClause> {
2386public:
2387 bool isSink(const OMPDependClause *C) {
2388 return C->getDependencyKind() == OMPC_DEPEND_sink;
2389 }
2390 bool isSource(const OMPDependClause *C) {
2391 return C->getDependencyKind() == OMPC_DEPEND_source;
2392 }
2393};
2394template <> class OMPDoacrossKind<OMPDoacrossClause> {
2395public:
2396 bool isSource(const OMPDoacrossClause *C) {
2397 return C->getDependenceType() == OMPC_DOACROSS_source ||
2398 C->getDependenceType() == OMPC_DOACROSS_source_omp_cur_iteration;
2399 }
2400 bool isSink(const OMPDoacrossClause *C) {
2401 return C->getDependenceType() == OMPC_DOACROSS_sink ||
2402 C->getDependenceType() == OMPC_DOACROSS_sink_omp_cur_iteration;
2403 }
2404};
2405} // namespace
2406} // namespace clang
2407
2408#endif
2409