1//===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
14#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
15
16#include "CGValue.h"
17#include "clang/AST/DeclOpenMP.h"
18#include "clang/AST/GlobalDecl.h"
19#include "clang/AST/Type.h"
20#include "clang/Basic/OpenMPKinds.h"
21#include "clang/Basic/SourceLocation.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/PointerIntPair.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/Frontend/OpenMP/OMPConstants.h"
28#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/ValueHandle.h"
31#include "llvm/Support/AtomicOrdering.h"
32
33namespace llvm {
34class ArrayType;
35class Constant;
36class FunctionType;
37class GlobalVariable;
38class Type;
39class Value;
40class OpenMPIRBuilder;
41} // namespace llvm
42
43namespace clang {
44class Expr;
45class OMPDependClause;
46class OMPExecutableDirective;
47class OMPLoopDirective;
48class VarDecl;
49class OMPDeclareReductionDecl;
50
51namespace CodeGen {
52class Address;
53class CodeGenFunction;
54class CodeGenModule;
55
56/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
57/// region.
58class PrePostActionTy {
59public:
60 explicit PrePostActionTy() {}
61 virtual void Enter(CodeGenFunction &CGF) {}
62 virtual void Exit(CodeGenFunction &CGF) {}
63 virtual ~PrePostActionTy() {}
64};
65
66/// Class provides a way to call simple version of codegen for OpenMP region, or
67/// an advanced with possible pre|post-actions in codegen.
68class RegionCodeGenTy final {
69 intptr_t CodeGen;
70 typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &);
71 CodeGenTy Callback;
72 mutable PrePostActionTy *PrePostAction;
73 RegionCodeGenTy() = delete;
74 template <typename Callable>
75 static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF,
76 PrePostActionTy &Action) {
77 return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action);
78 }
79
80public:
81 template <typename Callable>
82 RegionCodeGenTy(
83 Callable &&CodeGen,
84 std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
85 RegionCodeGenTy>::value> * = nullptr)
86 : CodeGen(reinterpret_cast<intptr_t>(&CodeGen)),
87 Callback(CallbackFn<std::remove_reference_t<Callable>>),
88 PrePostAction(nullptr) {}
89 void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; }
90 void operator()(CodeGenFunction &CGF) const;
91};
92
93struct OMPTaskDataTy final {
94 SmallVector<const Expr *, 4> PrivateVars;
95 SmallVector<const Expr *, 4> PrivateCopies;
96 SmallVector<const Expr *, 4> FirstprivateVars;
97 SmallVector<const Expr *, 4> FirstprivateCopies;
98 SmallVector<const Expr *, 4> FirstprivateInits;
99 SmallVector<const Expr *, 4> LastprivateVars;
100 SmallVector<const Expr *, 4> LastprivateCopies;
101 SmallVector<const Expr *, 4> ReductionVars;
102 SmallVector<const Expr *, 4> ReductionOrigs;
103 SmallVector<const Expr *, 4> ReductionCopies;
104 SmallVector<const Expr *, 4> ReductionOps;
105 SmallVector<CanonicalDeclPtr<const VarDecl>, 4> PrivateLocals;
106 struct DependData {
107 OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown;
108 const Expr *IteratorExpr = nullptr;
109 SmallVector<const Expr *, 4> DepExprs;
110 explicit DependData() = default;
111 DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr)
112 : DepKind(DepKind), IteratorExpr(IteratorExpr) {}
113 };
114 SmallVector<DependData, 4> Dependences;
115 llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
116 llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule;
117 llvm::PointerIntPair<llvm::Value *, 1, bool> Priority;
118 llvm::Value *Reductions = nullptr;
119 unsigned NumberOfParts = 0;
120 bool Tied = true;
121 bool Nogroup = false;
122 bool IsReductionWithTaskMod = false;
123 bool IsWorksharingReduction = false;
124 bool HasNowaitClause = false;
125 bool HasModifier = false;
126};
127
128/// Class intended to support codegen of all kind of the reduction clauses.
129class ReductionCodeGen {
130private:
131 /// Data required for codegen of reduction clauses.
132 struct ReductionData {
133 /// Reference to the item shared between tasks to reduce into.
134 const Expr *Shared = nullptr;
135 /// Reference to the original item.
136 const Expr *Ref = nullptr;
137 /// Helper expression for generation of private copy.
138 const Expr *Private = nullptr;
139 /// Helper expression for generation reduction operation.
140 const Expr *ReductionOp = nullptr;
141 ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private,
142 const Expr *ReductionOp)
143 : Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) {
144 }
145 };
146 /// List of reduction-based clauses.
147 SmallVector<ReductionData, 4> ClausesData;
148
149 /// List of addresses of shared variables/expressions.
150 SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses;
151 /// List of addresses of original variables/expressions.
152 SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses;
153 /// Sizes of the reduction items in chars.
154 SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes;
155 /// Base declarations for the reduction items.
156 SmallVector<const VarDecl *, 4> BaseDecls;
157
158 /// Emits lvalue for shared expression.
159 LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E);
160 /// Emits upper bound for shared expression (if array section).
161 LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E);
162 /// Performs aggregate initialization.
163 /// \param N Number of reduction item in the common list.
164 /// \param PrivateAddr Address of the corresponding private item.
165 /// \param SharedAddr Address of the original shared variable.
166 /// \param DRD Declare reduction construct used for reduction item.
167 void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N,
168 Address PrivateAddr, Address SharedAddr,
169 const OMPDeclareReductionDecl *DRD);
170
171public:
172 ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs,
173 ArrayRef<const Expr *> Privates,
174 ArrayRef<const Expr *> ReductionOps);
175 /// Emits lvalue for the shared and original reduction item.
176 /// \param N Number of the reduction item.
177 void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N);
178 /// Emits the code for the variable-modified type, if required.
179 /// \param N Number of the reduction item.
180 void emitAggregateType(CodeGenFunction &CGF, unsigned N);
181 /// Emits the code for the variable-modified type, if required.
182 /// \param N Number of the reduction item.
183 /// \param Size Size of the type in chars.
184 void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size);
185 /// Performs initialization of the private copy for the reduction item.
186 /// \param N Number of the reduction item.
187 /// \param PrivateAddr Address of the corresponding private item.
188 /// \param DefaultInit Default initialization sequence that should be
189 /// performed if no reduction specific initialization is found.
190 /// \param SharedAddr Address of the original shared variable.
191 void
192 emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr,
193 Address SharedAddr,
194 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit);
195 /// Returns true if the private copy requires cleanups.
196 bool needCleanups(unsigned N);
197 /// Emits cleanup code for the reduction item.
198 /// \param N Number of the reduction item.
199 /// \param PrivateAddr Address of the corresponding private item.
200 void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr);
201 /// Adjusts \p PrivatedAddr for using instead of the original variable
202 /// address in normal operations.
203 /// \param N Number of the reduction item.
204 /// \param PrivateAddr Address of the corresponding private item.
205 Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
206 Address PrivateAddr);
207 /// Returns LValue for the reduction item.
208 LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; }
209 /// Returns LValue for the original reduction item.
210 LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; }
211 /// Returns the size of the reduction item (in chars and total number of
212 /// elements in the item), or nullptr, if the size is a constant.
213 std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const {
214 return Sizes[N];
215 }
216 /// Returns the base declaration of the reduction item.
217 const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; }
218 /// Returns the base declaration of the reduction item.
219 const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; }
220 /// Returns true if the initialization of the reduction item uses initializer
221 /// from declare reduction construct.
222 bool usesReductionInitializer(unsigned N) const;
223 /// Return the type of the private item.
224 QualType getPrivateType(unsigned N) const {
225 return cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl())
226 ->getType();
227 }
228};
229
230class CGOpenMPRuntime {
231public:
232 /// Allows to disable automatic handling of functions used in target regions
233 /// as those marked as `omp declare target`.
234 class DisableAutoDeclareTargetRAII {
235 CodeGenModule &CGM;
236 bool SavedShouldMarkAsGlobal = false;
237
238 public:
239 DisableAutoDeclareTargetRAII(CodeGenModule &CGM);
240 ~DisableAutoDeclareTargetRAII();
241 };
242
243 /// Manages list of nontemporal decls for the specified directive.
244 class NontemporalDeclsRAII {
245 CodeGenModule &CGM;
246 const bool NeedToPush;
247
248 public:
249 NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S);
250 ~NontemporalDeclsRAII();
251 };
252
253 /// Manages list of nontemporal decls for the specified directive.
254 class UntiedTaskLocalDeclsRAII {
255 CodeGenModule &CGM;
256 const bool NeedToPush;
257
258 public:
259 UntiedTaskLocalDeclsRAII(
260 CodeGenFunction &CGF,
261 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
262 std::pair<Address, Address>> &LocalVars);
263 ~UntiedTaskLocalDeclsRAII();
264 };
265
266 /// Maps the expression for the lastprivate variable to the global copy used
267 /// to store new value because original variables are not mapped in inner
268 /// parallel regions. Only private copies are captured but we need also to
269 /// store private copy in shared address.
270 /// Also, stores the expression for the private loop counter and it
271 /// threaprivate name.
272 struct LastprivateConditionalData {
273 llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>>
274 DeclToUniqueName;
275 LValue IVLVal;
276 llvm::Function *Fn = nullptr;
277 bool Disabled = false;
278 };
279 /// Manages list of lastprivate conditional decls for the specified directive.
280 class LastprivateConditionalRAII {
281 enum class ActionToDo {
282 DoNotPush,
283 PushAsLastprivateConditional,
284 DisableLastprivateConditional,
285 };
286 CodeGenModule &CGM;
287 ActionToDo Action = ActionToDo::DoNotPush;
288
289 /// Check and try to disable analysis of inner regions for changes in
290 /// lastprivate conditional.
291 void tryToDisableInnerAnalysis(const OMPExecutableDirective &S,
292 llvm::DenseSet<CanonicalDeclPtr<const Decl>>
293 &NeedToAddForLPCsAsDisabled) const;
294
295 LastprivateConditionalRAII(CodeGenFunction &CGF,
296 const OMPExecutableDirective &S);
297
298 public:
299 explicit LastprivateConditionalRAII(CodeGenFunction &CGF,
300 const OMPExecutableDirective &S,
301 LValue IVLVal);
302 static LastprivateConditionalRAII disable(CodeGenFunction &CGF,
303 const OMPExecutableDirective &S);
304 ~LastprivateConditionalRAII();
305 };
306
307 llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; }
308
309protected:
310 CodeGenModule &CGM;
311
312 /// An OpenMP-IR-Builder instance.
313 llvm::OpenMPIRBuilder OMPBuilder;
314
315 /// Helper to determine the min/max number of threads/teams for \p D.
316 void computeMinAndMaxThreadsAndTeams(
317 const OMPExecutableDirective &D, CodeGenFunction &CGF,
318 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs);
319
320 /// Helper to emit outlined function for 'target' directive.
321 /// \param D Directive to emit.
322 /// \param ParentName Name of the function that encloses the target region.
323 /// \param OutlinedFn Outlined function value to be defined by this call.
324 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
325 /// \param IsOffloadEntry True if the outlined function is an offload entry.
326 /// \param CodeGen Lambda codegen specific to an accelerator device.
327 /// An outlined function may not be an entry if, e.g. the if clause always
328 /// evaluates to false.
329 virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D,
330 StringRef ParentName,
331 llvm::Function *&OutlinedFn,
332 llvm::Constant *&OutlinedFnID,
333 bool IsOffloadEntry,
334 const RegionCodeGenTy &CodeGen);
335
336 /// Returns pointer to ident_t type.
337 llvm::Type *getIdentTyPointerTy();
338
339 /// Gets thread id value for the current thread.
340 ///
341 llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc);
342
343 /// Get the function name of an outlined region.
344 std::string getOutlinedHelperName(StringRef Name) const;
345 std::string getOutlinedHelperName(CodeGenFunction &CGF) const;
346
347 /// Get the function name of a reduction function.
348 std::string getReductionFuncName(StringRef Name) const;
349
350 /// Emits \p Callee function call with arguments \p Args with location \p Loc.
351 void emitCall(CodeGenFunction &CGF, SourceLocation Loc,
352 llvm::FunctionCallee Callee,
353 ArrayRef<llvm::Value *> Args = {}) const;
354
355 /// Emits address of the word in a memory where current thread id is
356 /// stored.
357 virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc);
358
359 void setLocThreadIdInsertPt(CodeGenFunction &CGF,
360 bool AtCurrentPoint = false);
361 void clearLocThreadIdInsertPt(CodeGenFunction &CGF);
362
363 /// Check if the default location must be constant.
364 /// Default is false to support OMPT/OMPD.
365 virtual bool isDefaultLocationConstant() const { return false; }
366
367 /// Returns additional flags that can be stored in reserved_2 field of the
368 /// default location.
369 virtual unsigned getDefaultLocationReserved2Flags() const { return 0; }
370
371 /// Returns default flags for the barriers depending on the directive, for
372 /// which this barier is going to be emitted.
373 static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind);
374
375 /// Get the LLVM type for the critical name.
376 llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;}
377
378 /// Returns corresponding lock object for the specified critical region
379 /// name. If the lock object does not exist it is created, otherwise the
380 /// reference to the existing copy is returned.
381 /// \param CriticalName Name of the critical region.
382 ///
383 llvm::Value *getCriticalRegionLock(StringRef CriticalName);
384
385protected:
386 /// Map for SourceLocation and OpenMP runtime library debug locations.
387 typedef llvm::DenseMap<SourceLocation, llvm::Value *> OpenMPDebugLocMapTy;
388 OpenMPDebugLocMapTy OpenMPDebugLocMap;
389 /// Stores debug location and ThreadID for the function.
390 struct DebugLocThreadIdTy {
391 llvm::Value *DebugLoc;
392 llvm::Value *ThreadID;
393 /// Insert point for the service instructions.
394 llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr;
395 };
396 /// Map of local debug location, ThreadId and functions.
397 typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy>
398 OpenMPLocThreadIDMapTy;
399 OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap;
400 /// Map of UDRs and corresponding combiner/initializer.
401 typedef llvm::DenseMap<const OMPDeclareReductionDecl *,
402 std::pair<llvm::Function *, llvm::Function *>>
403 UDRMapTy;
404 UDRMapTy UDRMap;
405 /// Map of functions and locally defined UDRs.
406 typedef llvm::DenseMap<llvm::Function *,
407 SmallVector<const OMPDeclareReductionDecl *, 4>>
408 FunctionUDRMapTy;
409 FunctionUDRMapTy FunctionUDRMap;
410 /// Map from the user-defined mapper declaration to its corresponding
411 /// functions.
412 llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap;
413 /// Map of functions and their local user-defined mappers.
414 using FunctionUDMMapTy =
415 llvm::DenseMap<llvm::Function *,
416 SmallVector<const OMPDeclareMapperDecl *, 4>>;
417 FunctionUDMMapTy FunctionUDMMap;
418 /// Maps local variables marked as lastprivate conditional to their internal
419 /// types.
420 llvm::DenseMap<llvm::Function *,
421 llvm::DenseMap<CanonicalDeclPtr<const Decl>,
422 std::tuple<QualType, const FieldDecl *,
423 const FieldDecl *, LValue>>>
424 LastprivateConditionalToTypes;
425 /// Maps function to the position of the untied task locals stack.
426 llvm::DenseMap<llvm::Function *, unsigned> FunctionToUntiedTaskStackMap;
427 /// Type kmp_critical_name, originally defined as typedef kmp_int32
428 /// kmp_critical_name[8];
429 llvm::ArrayType *KmpCriticalNameTy;
430 /// An ordered map of auto-generated variables to their unique names.
431 /// It stores variables with the following names: 1) ".gomp_critical_user_" +
432 /// <critical_section_name> + ".var" for "omp critical" directives; 2)
433 /// <mangled_name_for_global_var> + ".cache." for cache for threadprivate
434 /// variables.
435 llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>,
436 llvm::BumpPtrAllocator> InternalVars;
437 /// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);
438 llvm::Type *KmpRoutineEntryPtrTy = nullptr;
439 QualType KmpRoutineEntryPtrQTy;
440 /// Type typedef struct kmp_task {
441 /// void * shareds; /**< pointer to block of pointers to
442 /// shared vars */
443 /// kmp_routine_entry_t routine; /**< pointer to routine to call for
444 /// executing task */
445 /// kmp_int32 part_id; /**< part id for the task */
446 /// kmp_routine_entry_t destructors; /* pointer to function to invoke
447 /// deconstructors of firstprivate C++ objects */
448 /// } kmp_task_t;
449 QualType KmpTaskTQTy;
450 /// Saved kmp_task_t for task directive.
451 QualType SavedKmpTaskTQTy;
452 /// Saved kmp_task_t for taskloop-based directive.
453 QualType SavedKmpTaskloopTQTy;
454 /// Type typedef struct kmp_depend_info {
455 /// kmp_intptr_t base_addr;
456 /// size_t len;
457 /// struct {
458 /// bool in:1;
459 /// bool out:1;
460 /// } flags;
461 /// } kmp_depend_info_t;
462 QualType KmpDependInfoTy;
463 /// Type typedef struct kmp_task_affinity_info {
464 /// kmp_intptr_t base_addr;
465 /// size_t len;
466 /// struct {
467 /// bool flag1 : 1;
468 /// bool flag2 : 1;
469 /// kmp_int32 reserved : 30;
470 /// } flags;
471 /// } kmp_task_affinity_info_t;
472 QualType KmpTaskAffinityInfoTy;
473 /// struct kmp_dim { // loop bounds info casted to kmp_int64
474 /// kmp_int64 lo; // lower
475 /// kmp_int64 up; // upper
476 /// kmp_int64 st; // stride
477 /// };
478 QualType KmpDimTy;
479
480 bool ShouldMarkAsGlobal = true;
481 /// List of the emitted declarations.
482 llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls;
483 /// List of the global variables with their addresses that should not be
484 /// emitted for the target.
485 llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables;
486
487 /// List of variables that can become declare target implicitly and, thus,
488 /// must be emitted.
489 llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;
490
491 using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>;
492 /// Stack for list of declarations in current context marked as nontemporal.
493 /// The set is the union of all current stack elements.
494 llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack;
495
496 using UntiedLocalVarsAddressesMap =
497 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
498 std::pair<Address, Address>>;
499 llvm::SmallVector<UntiedLocalVarsAddressesMap, 4> UntiedLocalVarsStack;
500
501 /// Stack for list of addresses of declarations in current context marked as
502 /// lastprivate conditional. The set is the union of all current stack
503 /// elements.
504 llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack;
505
506 /// Flag for keeping track of weather a requires unified_shared_memory
507 /// directive is present.
508 bool HasRequiresUnifiedSharedMemory = false;
509
510 /// Atomic ordering from the omp requires directive.
511 llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
512
513 /// Flag for keeping track of weather a target region has been emitted.
514 bool HasEmittedTargetRegion = false;
515
516 /// Flag for keeping track of weather a device routine has been emitted.
517 /// Device routines are specific to the
518 bool HasEmittedDeclareTargetRegion = false;
519
520 /// Start scanning from statement \a S and emit all target regions
521 /// found along the way.
522 /// \param S Starting statement.
523 /// \param ParentName Name of the function declaration that is being scanned.
524 void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName);
525
526 /// Build type kmp_routine_entry_t (if not built yet).
527 void emitKmpRoutineEntryT(QualType KmpInt32Ty);
528
529 /// If the specified mangled name is not in the module, create and
530 /// return threadprivate cache object. This object is a pointer's worth of
531 /// storage that's reserved for use by the OpenMP runtime.
532 /// \param VD Threadprivate variable.
533 /// \return Cache variable for the specified threadprivate.
534 llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD);
535
536 /// Set of threadprivate variables with the generated initializer.
537 llvm::StringSet<> ThreadPrivateWithDefinition;
538
539 /// Set of declare target variables with the generated initializer.
540 llvm::StringSet<> DeclareTargetWithDefinition;
541
542 /// Emits initialization code for the threadprivate variables.
543 /// \param VDAddr Address of the global variable \a VD.
544 /// \param Ctor Pointer to a global init function for \a VD.
545 /// \param CopyCtor Pointer to a global copy function for \a VD.
546 /// \param Dtor Pointer to a global destructor function for \a VD.
547 /// \param Loc Location of threadprivate declaration.
548 void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr,
549 llvm::Value *Ctor, llvm::Value *CopyCtor,
550 llvm::Value *Dtor, SourceLocation Loc);
551
552 struct TaskResultTy {
553 llvm::Value *NewTask = nullptr;
554 llvm::Function *TaskEntry = nullptr;
555 llvm::Value *NewTaskNewTaskTTy = nullptr;
556 LValue TDBase;
557 const RecordDecl *KmpTaskTQTyRD = nullptr;
558 llvm::Value *TaskDupFn = nullptr;
559 };
560 /// Emit task region for the task directive. The task region is emitted in
561 /// several steps:
562 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
563 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
564 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
565 /// function:
566 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
567 /// TaskFunction(gtid, tt->part_id, tt->shareds);
568 /// return 0;
569 /// }
570 /// 2. Copy a list of shared variables to field shareds of the resulting
571 /// structure kmp_task_t returned by the previous call (if any).
572 /// 3. Copy a pointer to destructions function to field destructions of the
573 /// resulting structure kmp_task_t.
574 /// \param D Current task directive.
575 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
576 /// /*part_id*/, captured_struct */*__context*/);
577 /// \param SharedsTy A type which contains references the shared variables.
578 /// \param Shareds Context with the list of shared variables from the \p
579 /// TaskFunction.
580 /// \param Data Additional data for task generation like tiednsee, final
581 /// state, list of privates etc.
582 TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
583 const OMPExecutableDirective &D,
584 llvm::Function *TaskFunction, QualType SharedsTy,
585 Address Shareds, const OMPTaskDataTy &Data);
586
587 /// Emit update for lastprivate conditional data.
588 void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,
589 StringRef UniqueDeclName, LValue LVal,
590 SourceLocation Loc);
591
592 /// Returns the number of the elements and the address of the depobj
593 /// dependency array.
594 /// \return Number of elements in depobj array and the pointer to the array of
595 /// dependencies.
596 std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF,
597 LValue DepobjLVal,
598 SourceLocation Loc);
599
600 SmallVector<llvm::Value *, 4>
601 emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
602 const OMPTaskDataTy::DependData &Data);
603
604 void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
605 LValue PosLVal, const OMPTaskDataTy::DependData &Data,
606 Address DependenciesArray);
607
608public:
609 explicit CGOpenMPRuntime(CodeGenModule &CGM);
610 virtual ~CGOpenMPRuntime() {}
611 virtual void clear();
612
613 /// Emits object of ident_t type with info for source location.
614 /// \param Flags Flags for OpenMP location.
615 /// \param EmitLoc emit source location with debug-info is off.
616 ///
617 llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc,
618 unsigned Flags = 0, bool EmitLoc = false);
619
620 /// Emit the number of teams for a target directive. Inspect the num_teams
621 /// clause associated with a teams construct combined or closely nested
622 /// with the target directive.
623 ///
624 /// Emit a team of size one for directives such as 'target parallel' that
625 /// have no associated teams construct.
626 ///
627 /// Otherwise, return nullptr.
628 const Expr *getNumTeamsExprForTargetDirective(CodeGenFunction &CGF,
629 const OMPExecutableDirective &D,
630 int32_t &MinTeamsVal,
631 int32_t &MaxTeamsVal);
632 llvm::Value *emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
633 const OMPExecutableDirective &D);
634
635 /// Check for a number of threads upper bound constant value (stored in \p
636 /// UpperBound), or expression (returned). If the value is conditional (via an
637 /// if-clause), store the condition in \p CondExpr. Similarly, a potential
638 /// thread limit expression is stored in \p ThreadLimitExpr. If \p
639 /// UpperBoundOnly is true, no expression evaluation is perfomed.
640 const Expr *getNumThreadsExprForTargetDirective(
641 CodeGenFunction &CGF, const OMPExecutableDirective &D,
642 int32_t &UpperBound, bool UpperBoundOnly,
643 llvm::Value **CondExpr = nullptr, const Expr **ThreadLimitExpr = nullptr);
644
645 /// Emit an expression that denotes the number of threads a target region
646 /// shall use. Will generate "i32 0" to allow the runtime to choose.
647 llvm::Value *
648 emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
649 const OMPExecutableDirective &D);
650
651 /// Return the trip count of loops associated with constructs / 'target teams
652 /// distribute' and 'teams distribute parallel for'. \param SizeEmitter Emits
653 /// the int64 value for the number of iterations of the associated loop.
654 llvm::Value *emitTargetNumIterationsCall(
655 CodeGenFunction &CGF, const OMPExecutableDirective &D,
656 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
657 const OMPLoopDirective &D)>
658 SizeEmitter);
659
660 /// Returns true if the current target is a GPU.
661 virtual bool isGPU() const { return false; }
662
663 /// Check if the variable length declaration is delayed:
664 virtual bool isDelayedVariableLengthDecl(CodeGenFunction &CGF,
665 const VarDecl *VD) const {
666 return false;
667 };
668
669 /// Get call to __kmpc_alloc_shared
670 virtual std::pair<llvm::Value *, llvm::Value *>
671 getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) {
672 llvm_unreachable("not implemented");
673 }
674
675 /// Get call to __kmpc_free_shared
676 virtual void getKmpcFreeShared(
677 CodeGenFunction &CGF,
678 const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) {
679 llvm_unreachable("not implemented");
680 }
681
682 /// Emits code for OpenMP 'if' clause using specified \a CodeGen
683 /// function. Here is the logic:
684 /// if (Cond) {
685 /// ThenGen();
686 /// } else {
687 /// ElseGen();
688 /// }
689 void emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
690 const RegionCodeGenTy &ThenGen,
691 const RegionCodeGenTy &ElseGen);
692
693 /// Checks if the \p Body is the \a CompoundStmt and returns its child
694 /// statement iff there is only one that is not evaluatable at the compile
695 /// time.
696 static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body);
697
698 /// Get the platform-specific name separator.
699 std::string getName(ArrayRef<StringRef> Parts) const;
700
701 /// Emit code for the specified user defined reduction construct.
702 virtual void emitUserDefinedReduction(CodeGenFunction *CGF,
703 const OMPDeclareReductionDecl *D);
704 /// Get combiner/initializer for the specified user-defined reduction, if any.
705 virtual std::pair<llvm::Function *, llvm::Function *>
706 getUserDefinedReduction(const OMPDeclareReductionDecl *D);
707
708 /// Emit the function for the user defined mapper construct.
709 void emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
710 CodeGenFunction *CGF = nullptr);
711 /// Get the function for the specified user-defined mapper. If it does not
712 /// exist, create one.
713 llvm::Function *
714 getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D);
715
716 /// Emits outlined function for the specified OpenMP parallel directive
717 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
718 /// kmp_int32 BoundID, struct context_vars*).
719 /// \param CGF Reference to current CodeGenFunction.
720 /// \param D OpenMP directive.
721 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
722 /// \param InnermostKind Kind of innermost directive (for simple directives it
723 /// is a directive itself, for combined - its innermost directive).
724 /// \param CodeGen Code generation sequence for the \a D directive.
725 virtual llvm::Function *emitParallelOutlinedFunction(
726 CodeGenFunction &CGF, const OMPExecutableDirective &D,
727 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
728 const RegionCodeGenTy &CodeGen);
729
730 /// Emits outlined function for the specified OpenMP teams directive
731 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
732 /// kmp_int32 BoundID, struct context_vars*).
733 /// \param CGF Reference to current CodeGenFunction.
734 /// \param D OpenMP directive.
735 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
736 /// \param InnermostKind Kind of innermost directive (for simple directives it
737 /// is a directive itself, for combined - its innermost directive).
738 /// \param CodeGen Code generation sequence for the \a D directive.
739 virtual llvm::Function *emitTeamsOutlinedFunction(
740 CodeGenFunction &CGF, const OMPExecutableDirective &D,
741 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
742 const RegionCodeGenTy &CodeGen);
743
744 /// Emits outlined function for the OpenMP task directive \a D. This
745 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
746 /// TaskT).
747 /// \param D OpenMP directive.
748 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
749 /// \param PartIDVar Variable for partition id in the current OpenMP untied
750 /// task region.
751 /// \param TaskTVar Variable for task_t argument.
752 /// \param InnermostKind Kind of innermost directive (for simple directives it
753 /// is a directive itself, for combined - its innermost directive).
754 /// \param CodeGen Code generation sequence for the \a D directive.
755 /// \param Tied true if task is generated for tied task, false otherwise.
756 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
757 /// tasks.
758 ///
759 virtual llvm::Function *emitTaskOutlinedFunction(
760 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
761 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
762 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
763 bool Tied, unsigned &NumberOfParts);
764
765 /// Cleans up references to the objects in finished function.
766 ///
767 virtual void functionFinished(CodeGenFunction &CGF);
768
769 /// Emits code for parallel or serial call of the \a OutlinedFn with
770 /// variables captured in a record which address is stored in \a
771 /// CapturedStruct.
772 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
773 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
774 /// \param CapturedVars A pointer to the record with the references to
775 /// variables used in \a OutlinedFn function.
776 /// \param IfCond Condition in the associated 'if' clause, if it was
777 /// specified, nullptr otherwise.
778 /// \param NumThreads The value corresponding to the num_threads clause, if
779 /// any, or nullptr.
780 ///
781 virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
782 llvm::Function *OutlinedFn,
783 ArrayRef<llvm::Value *> CapturedVars,
784 const Expr *IfCond, llvm::Value *NumThreads);
785
786 /// Emits a critical region.
787 /// \param CriticalName Name of the critical region.
788 /// \param CriticalOpGen Generator for the statement associated with the given
789 /// critical region.
790 /// \param Hint Value of the 'hint' clause (optional).
791 virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
792 const RegionCodeGenTy &CriticalOpGen,
793 SourceLocation Loc,
794 const Expr *Hint = nullptr);
795
796 /// Emits a master region.
797 /// \param MasterOpGen Generator for the statement associated with the given
798 /// master region.
799 virtual void emitMasterRegion(CodeGenFunction &CGF,
800 const RegionCodeGenTy &MasterOpGen,
801 SourceLocation Loc);
802
803 /// Emits a masked region.
804 /// \param MaskedOpGen Generator for the statement associated with the given
805 /// masked region.
806 virtual void emitMaskedRegion(CodeGenFunction &CGF,
807 const RegionCodeGenTy &MaskedOpGen,
808 SourceLocation Loc,
809 const Expr *Filter = nullptr);
810
811 /// Emits code for a taskyield directive.
812 virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);
813
814 /// Emit __kmpc_error call for error directive
815 /// extern void __kmpc_error(ident_t *loc, int severity, const char *message);
816 virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME,
817 bool IsFatal);
818
819 /// Emit a taskgroup region.
820 /// \param TaskgroupOpGen Generator for the statement associated with the
821 /// given taskgroup region.
822 virtual void emitTaskgroupRegion(CodeGenFunction &CGF,
823 const RegionCodeGenTy &TaskgroupOpGen,
824 SourceLocation Loc);
825
826 /// Emits a single region.
827 /// \param SingleOpGen Generator for the statement associated with the given
828 /// single region.
829 virtual void emitSingleRegion(CodeGenFunction &CGF,
830 const RegionCodeGenTy &SingleOpGen,
831 SourceLocation Loc,
832 ArrayRef<const Expr *> CopyprivateVars,
833 ArrayRef<const Expr *> DestExprs,
834 ArrayRef<const Expr *> SrcExprs,
835 ArrayRef<const Expr *> AssignmentOps);
836
837 /// Emit an ordered region.
838 /// \param OrderedOpGen Generator for the statement associated with the given
839 /// ordered region.
840 virtual void emitOrderedRegion(CodeGenFunction &CGF,
841 const RegionCodeGenTy &OrderedOpGen,
842 SourceLocation Loc, bool IsThreads);
843
844 /// Emit an implicit/explicit barrier for OpenMP threads.
845 /// \param Kind Directive for which this implicit barrier call must be
846 /// generated. Must be OMPD_barrier for explicit barrier generation.
847 /// \param EmitChecks true if need to emit checks for cancellation barriers.
848 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
849 /// runtime class decides which one to emit (simple or with cancellation
850 /// checks).
851 ///
852 virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
853 OpenMPDirectiveKind Kind,
854 bool EmitChecks = true,
855 bool ForceSimpleCall = false);
856
857 /// Check if the specified \a ScheduleKind is static non-chunked.
858 /// This kind of worksharing directive is emitted without outer loop.
859 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
860 /// \param Chunked True if chunk is specified in the clause.
861 ///
862 virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
863 bool Chunked) const;
864
865 /// Check if the specified \a ScheduleKind is static non-chunked.
866 /// This kind of distribute directive is emitted without outer loop.
867 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
868 /// \param Chunked True if chunk is specified in the clause.
869 ///
870 virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind,
871 bool Chunked) const;
872
873 /// Check if the specified \a ScheduleKind is static chunked.
874 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
875 /// \param Chunked True if chunk is specified in the clause.
876 ///
877 virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
878 bool Chunked) const;
879
880 /// Check if the specified \a ScheduleKind is static non-chunked.
881 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
882 /// \param Chunked True if chunk is specified in the clause.
883 ///
884 virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind,
885 bool Chunked) const;
886
887 /// Check if the specified \a ScheduleKind is dynamic.
888 /// This kind of worksharing directive is emitted without outer loop.
889 /// \param ScheduleKind Schedule Kind specified in the 'schedule' clause.
890 ///
891 virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const;
892
893 /// struct with the values to be passed to the dispatch runtime function
894 struct DispatchRTInput {
895 /// Loop lower bound
896 llvm::Value *LB = nullptr;
897 /// Loop upper bound
898 llvm::Value *UB = nullptr;
899 /// Chunk size specified using 'schedule' clause (nullptr if chunk
900 /// was not specified)
901 llvm::Value *Chunk = nullptr;
902 DispatchRTInput() = default;
903 DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk)
904 : LB(LB), UB(UB), Chunk(Chunk) {}
905 };
906
907 /// Call the appropriate runtime routine to initialize it before start
908 /// of loop.
909
910 /// This is used for non static scheduled types and when the ordered
911 /// clause is present on the loop construct.
912 /// Depending on the loop schedule, it is necessary to call some runtime
913 /// routine before start of the OpenMP loop to get the loop upper / lower
914 /// bounds \a LB and \a UB and stride \a ST.
915 ///
916 /// \param CGF Reference to current CodeGenFunction.
917 /// \param Loc Clang source location.
918 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
919 /// \param IVSize Size of the iteration variable in bits.
920 /// \param IVSigned Sign of the iteration variable.
921 /// \param Ordered true if loop is ordered, false otherwise.
922 /// \param DispatchValues struct containing llvm values for lower bound, upper
923 /// bound, and chunk expression.
924 /// For the default (nullptr) value, the chunk 1 will be used.
925 ///
926 virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
927 const OpenMPScheduleTy &ScheduleKind,
928 unsigned IVSize, bool IVSigned, bool Ordered,
929 const DispatchRTInput &DispatchValues);
930
931 /// This is used for non static scheduled types and when the ordered
932 /// clause is present on the loop construct.
933 ///
934 /// \param CGF Reference to current CodeGenFunction.
935 /// \param Loc Clang source location.
936 ///
937 virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc);
938
939 /// Struct with the values to be passed to the static runtime function
940 struct StaticRTInput {
941 /// Size of the iteration variable in bits.
942 unsigned IVSize = 0;
943 /// Sign of the iteration variable.
944 bool IVSigned = false;
945 /// true if loop is ordered, false otherwise.
946 bool Ordered = false;
947 /// Address of the output variable in which the flag of the last iteration
948 /// is returned.
949 Address IL = Address::invalid();
950 /// Address of the output variable in which the lower iteration number is
951 /// returned.
952 Address LB = Address::invalid();
953 /// Address of the output variable in which the upper iteration number is
954 /// returned.
955 Address UB = Address::invalid();
956 /// Address of the output variable in which the stride value is returned
957 /// necessary to generated the static_chunked scheduled loop.
958 Address ST = Address::invalid();
959 /// Value of the chunk for the static_chunked scheduled loop. For the
960 /// default (nullptr) value, the chunk 1 will be used.
961 llvm::Value *Chunk = nullptr;
962 StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL,
963 Address LB, Address UB, Address ST,
964 llvm::Value *Chunk = nullptr)
965 : IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB),
966 UB(UB), ST(ST), Chunk(Chunk) {}
967 };
968 /// Call the appropriate runtime routine to initialize it before start
969 /// of loop.
970 ///
971 /// This is used only in case of static schedule, when the user did not
972 /// specify a ordered clause on the loop construct.
973 /// Depending on the loop schedule, it is necessary to call some runtime
974 /// routine before start of the OpenMP loop to get the loop upper / lower
975 /// bounds LB and UB and stride ST.
976 ///
977 /// \param CGF Reference to current CodeGenFunction.
978 /// \param Loc Clang source location.
979 /// \param DKind Kind of the directive.
980 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
981 /// \param Values Input arguments for the construct.
982 ///
983 virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
984 OpenMPDirectiveKind DKind,
985 const OpenMPScheduleTy &ScheduleKind,
986 const StaticRTInput &Values);
987
988 ///
989 /// \param CGF Reference to current CodeGenFunction.
990 /// \param Loc Clang source location.
991 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
992 /// \param Values Input arguments for the construct.
993 ///
994 virtual void emitDistributeStaticInit(CodeGenFunction &CGF,
995 SourceLocation Loc,
996 OpenMPDistScheduleClauseKind SchedKind,
997 const StaticRTInput &Values);
998
999 /// Call the appropriate runtime routine to notify that we finished
1000 /// iteration of the ordered loop with the dynamic scheduling.
1001 ///
1002 /// \param CGF Reference to current CodeGenFunction.
1003 /// \param Loc Clang source location.
1004 /// \param IVSize Size of the iteration variable in bits.
1005 /// \param IVSigned Sign of the iteration variable.
1006 ///
1007 virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF,
1008 SourceLocation Loc, unsigned IVSize,
1009 bool IVSigned);
1010
1011 /// Call the appropriate runtime routine to notify that we finished
1012 /// all the work with current loop.
1013 ///
1014 /// \param CGF Reference to current CodeGenFunction.
1015 /// \param Loc Clang source location.
1016 /// \param DKind Kind of the directive for which the static finish is emitted.
1017 ///
1018 virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1019 OpenMPDirectiveKind DKind);
1020
1021 /// Call __kmpc_dispatch_next(
1022 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1023 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1024 /// kmp_int[32|64] *p_stride);
1025 /// \param IVSize Size of the iteration variable in bits.
1026 /// \param IVSigned Sign of the iteration variable.
1027 /// \param IL Address of the output variable in which the flag of the
1028 /// last iteration is returned.
1029 /// \param LB Address of the output variable in which the lower iteration
1030 /// number is returned.
1031 /// \param UB Address of the output variable in which the upper iteration
1032 /// number is returned.
1033 /// \param ST Address of the output variable in which the stride value is
1034 /// returned.
1035 virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1036 unsigned IVSize, bool IVSigned,
1037 Address IL, Address LB,
1038 Address UB, Address ST);
1039
1040 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1041 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1042 /// clause.
1043 /// \param NumThreads An integer value of threads.
1044 virtual void emitNumThreadsClause(CodeGenFunction &CGF,
1045 llvm::Value *NumThreads,
1046 SourceLocation Loc);
1047
1048 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1049 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1050 virtual void emitProcBindClause(CodeGenFunction &CGF,
1051 llvm::omp::ProcBindKind ProcBind,
1052 SourceLocation Loc);
1053
1054 /// Returns address of the threadprivate variable for the current
1055 /// thread.
1056 /// \param VD Threadprivate variable.
1057 /// \param VDAddr Address of the global variable \a VD.
1058 /// \param Loc Location of the reference to threadprivate var.
1059 /// \return Address of the threadprivate variable for the current thread.
1060 virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1061 const VarDecl *VD, Address VDAddr,
1062 SourceLocation Loc);
1063
1064 /// Returns the address of the variable marked as declare target with link
1065 /// clause OR as declare target with to clause and unified memory.
1066 virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD);
1067
1068 /// Emit a code for initialization of threadprivate variable. It emits
1069 /// a call to runtime library which adds initial value to the newly created
1070 /// threadprivate variable (if it is not constant) and registers destructor
1071 /// for the variable (if any).
1072 /// \param VD Threadprivate variable.
1073 /// \param VDAddr Address of the global variable \a VD.
1074 /// \param Loc Location of threadprivate declaration.
1075 /// \param PerformInit true if initialization expression is not constant.
1076 virtual llvm::Function *
1077 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1078 SourceLocation Loc, bool PerformInit,
1079 CodeGenFunction *CGF = nullptr);
1080
1081 /// Emit code for handling declare target functions in the runtime.
1082 /// \param FD Declare target function.
1083 /// \param Addr Address of the global \a FD.
1084 /// \param PerformInit true if initialization expression is not constant.
1085 virtual void emitDeclareTargetFunction(const FunctionDecl *FD,
1086 llvm::GlobalValue *GV);
1087
1088 /// Creates artificial threadprivate variable with name \p Name and type \p
1089 /// VarType.
1090 /// \param VarType Type of the artificial threadprivate variable.
1091 /// \param Name Name of the artificial threadprivate variable.
1092 virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1093 QualType VarType,
1094 StringRef Name);
1095
1096 /// Emit flush of the variables specified in 'omp flush' directive.
1097 /// \param Vars List of variables to flush.
1098 virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1099 SourceLocation Loc, llvm::AtomicOrdering AO);
1100
1101 /// Emit task region for the task directive. The task region is
1102 /// emitted in several steps:
1103 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1104 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1105 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1106 /// function:
1107 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1108 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1109 /// return 0;
1110 /// }
1111 /// 2. Copy a list of shared variables to field shareds of the resulting
1112 /// structure kmp_task_t returned by the previous call (if any).
1113 /// 3. Copy a pointer to destructions function to field destructions of the
1114 /// resulting structure kmp_task_t.
1115 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1116 /// kmp_task_t *new_task), where new_task is a resulting structure from
1117 /// previous items.
1118 /// \param D Current task directive.
1119 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1120 /// /*part_id*/, captured_struct */*__context*/);
1121 /// \param SharedsTy A type which contains references the shared variables.
1122 /// \param Shareds Context with the list of shared variables from the \p
1123 /// TaskFunction.
1124 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1125 /// otherwise.
1126 /// \param Data Additional data for task generation like tiednsee, final
1127 /// state, list of privates etc.
1128 virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1129 const OMPExecutableDirective &D,
1130 llvm::Function *TaskFunction, QualType SharedsTy,
1131 Address Shareds, const Expr *IfCond,
1132 const OMPTaskDataTy &Data);
1133
1134 /// Emit task region for the taskloop directive. The taskloop region is
1135 /// emitted in several steps:
1136 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1137 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1138 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1139 /// function:
1140 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1141 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1142 /// return 0;
1143 /// }
1144 /// 2. Copy a list of shared variables to field shareds of the resulting
1145 /// structure kmp_task_t returned by the previous call (if any).
1146 /// 3. Copy a pointer to destructions function to field destructions of the
1147 /// resulting structure kmp_task_t.
1148 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
1149 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
1150 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
1151 /// is a resulting structure from
1152 /// previous items.
1153 /// \param D Current task directive.
1154 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1155 /// /*part_id*/, captured_struct */*__context*/);
1156 /// \param SharedsTy A type which contains references the shared variables.
1157 /// \param Shareds Context with the list of shared variables from the \p
1158 /// TaskFunction.
1159 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1160 /// otherwise.
1161 /// \param Data Additional data for task generation like tiednsee, final
1162 /// state, list of privates etc.
1163 virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
1164 const OMPLoopDirective &D,
1165 llvm::Function *TaskFunction,
1166 QualType SharedsTy, Address Shareds,
1167 const Expr *IfCond, const OMPTaskDataTy &Data);
1168
1169 /// Emit code for the directive that does not require outlining.
1170 ///
1171 /// \param InnermostKind Kind of innermost directive (for simple directives it
1172 /// is a directive itself, for combined - its innermost directive).
1173 /// \param CodeGen Code generation sequence for the \a D directive.
1174 /// \param HasCancel true if region has inner cancel directive, false
1175 /// otherwise.
1176 virtual void emitInlinedDirective(CodeGenFunction &CGF,
1177 OpenMPDirectiveKind InnermostKind,
1178 const RegionCodeGenTy &CodeGen,
1179 bool HasCancel = false);
1180
1181 /// Emits reduction function.
1182 /// \param ReducerName Name of the function calling the reduction.
1183 /// \param ArgsElemType Array type containing pointers to reduction variables.
1184 /// \param Privates List of private copies for original reduction arguments.
1185 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1186 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1187 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1188 /// or 'operator binop(LHS, RHS)'.
1189 llvm::Function *emitReductionFunction(
1190 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
1191 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
1192 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps);
1193
1194 /// Emits single reduction combiner
1195 void emitSingleReductionCombiner(CodeGenFunction &CGF,
1196 const Expr *ReductionOp,
1197 const Expr *PrivateRef,
1198 const DeclRefExpr *LHS,
1199 const DeclRefExpr *RHS);
1200
1201 struct ReductionOptionsTy {
1202 bool WithNowait;
1203 bool SimpleReduction;
1204 llvm::SmallVector<bool, 8> IsPrivateVarReduction;
1205 OpenMPDirectiveKind ReductionKind;
1206 };
1207
1208 /// Emits code for private variable reduction
1209 /// \param Privates List of private copies for original reduction arguments.
1210 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1211 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1212 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1213 /// or 'operator binop(LHS, RHS)'.
1214 void emitPrivateReduction(CodeGenFunction &CGF, SourceLocation Loc,
1215 const Expr *Privates, const Expr *LHSExprs,
1216 const Expr *RHSExprs, const Expr *ReductionOps);
1217
1218 /// Emit a code for reduction clause. Next code should be emitted for
1219 /// reduction:
1220 /// \code
1221 ///
1222 /// static kmp_critical_name lock = { 0 };
1223 ///
1224 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
1225 /// ...
1226 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
1227 /// ...
1228 /// }
1229 ///
1230 /// ...
1231 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
1232 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
1233 /// RedList, reduce_func, &<lock>)) {
1234 /// case 1:
1235 /// ...
1236 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
1237 /// ...
1238 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
1239 /// break;
1240 /// case 2:
1241 /// ...
1242 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
1243 /// ...
1244 /// break;
1245 /// default:;
1246 /// }
1247 /// \endcode
1248 ///
1249 /// \param Privates List of private copies for original reduction arguments.
1250 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1251 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1252 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1253 /// or 'operator binop(LHS, RHS)'.
1254 /// \param Options List of options for reduction codegen:
1255 /// WithNowait true if parent directive has also nowait clause, false
1256 /// otherwise.
1257 /// SimpleReduction Emit reduction operation only. Used for omp simd
1258 /// directive on the host.
1259 /// ReductionKind The kind of reduction to perform.
1260 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
1261 ArrayRef<const Expr *> Privates,
1262 ArrayRef<const Expr *> LHSExprs,
1263 ArrayRef<const Expr *> RHSExprs,
1264 ArrayRef<const Expr *> ReductionOps,
1265 ReductionOptionsTy Options);
1266
1267 /// Emit a code for initialization of task reduction clause. Next code
1268 /// should be emitted for reduction:
1269 /// \code
1270 ///
1271 /// _taskred_item_t red_data[n];
1272 /// ...
1273 /// red_data[i].shar = &shareds[i];
1274 /// red_data[i].orig = &origs[i];
1275 /// red_data[i].size = sizeof(origs[i]);
1276 /// red_data[i].f_init = (void*)RedInit<i>;
1277 /// red_data[i].f_fini = (void*)RedDest<i>;
1278 /// red_data[i].f_comb = (void*)RedOp<i>;
1279 /// red_data[i].flags = <Flag_i>;
1280 /// ...
1281 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
1282 /// \endcode
1283 /// For reduction clause with task modifier it emits the next call:
1284 /// \code
1285 ///
1286 /// _taskred_item_t red_data[n];
1287 /// ...
1288 /// red_data[i].shar = &shareds[i];
1289 /// red_data[i].orig = &origs[i];
1290 /// red_data[i].size = sizeof(origs[i]);
1291 /// red_data[i].f_init = (void*)RedInit<i>;
1292 /// red_data[i].f_fini = (void*)RedDest<i>;
1293 /// red_data[i].f_comb = (void*)RedOp<i>;
1294 /// red_data[i].flags = <Flag_i>;
1295 /// ...
1296 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
1297 /// red_data);
1298 /// \endcode
1299 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
1300 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
1301 /// \param Data Additional data for task generation like tiedness, final
1302 /// state, list of privates, reductions etc.
1303 virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF,
1304 SourceLocation Loc,
1305 ArrayRef<const Expr *> LHSExprs,
1306 ArrayRef<const Expr *> RHSExprs,
1307 const OMPTaskDataTy &Data);
1308
1309 /// Emits the following code for reduction clause with task modifier:
1310 /// \code
1311 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
1312 /// \endcode
1313 virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
1314 bool IsWorksharingReduction);
1315
1316 /// Required to resolve existing problems in the runtime. Emits threadprivate
1317 /// variables to store the size of the VLAs/array sections for
1318 /// initializer/combiner/finalizer functions.
1319 /// \param RCG Allows to reuse an existing data for the reductions.
1320 /// \param N Reduction item for which fixups must be emitted.
1321 virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
1322 ReductionCodeGen &RCG, unsigned N);
1323
1324 /// Get the address of `void *` type of the privatue copy of the reduction
1325 /// item specified by the \p SharedLVal.
1326 /// \param ReductionsPtr Pointer to the reduction data returned by the
1327 /// emitTaskReductionInit function.
1328 /// \param SharedLVal Address of the original reduction item.
1329 virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
1330 llvm::Value *ReductionsPtr,
1331 LValue SharedLVal);
1332
1333 /// Emit code for 'taskwait' directive.
1334 virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
1335 const OMPTaskDataTy &Data);
1336
1337 /// Emit code for 'cancellation point' construct.
1338 /// \param CancelRegion Region kind for which the cancellation point must be
1339 /// emitted.
1340 ///
1341 virtual void emitCancellationPointCall(CodeGenFunction &CGF,
1342 SourceLocation Loc,
1343 OpenMPDirectiveKind CancelRegion);
1344
1345 /// Emit code for 'cancel' construct.
1346 /// \param IfCond Condition in the associated 'if' clause, if it was
1347 /// specified, nullptr otherwise.
1348 /// \param CancelRegion Region kind for which the cancel must be emitted.
1349 ///
1350 virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
1351 const Expr *IfCond,
1352 OpenMPDirectiveKind CancelRegion);
1353
1354 /// Emit outilined function for 'target' directive.
1355 /// \param D Directive to emit.
1356 /// \param ParentName Name of the function that encloses the target region.
1357 /// \param OutlinedFn Outlined function value to be defined by this call.
1358 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
1359 /// \param IsOffloadEntry True if the outlined function is an offload entry.
1360 /// \param CodeGen Code generation sequence for the \a D directive.
1361 /// An outlined function may not be an entry if, e.g. the if clause always
1362 /// evaluates to false.
1363 virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
1364 StringRef ParentName,
1365 llvm::Function *&OutlinedFn,
1366 llvm::Constant *&OutlinedFnID,
1367 bool IsOffloadEntry,
1368 const RegionCodeGenTy &CodeGen);
1369
1370 /// Emit the target offloading code associated with \a D. The emitted
1371 /// code attempts offloading the execution to the device, an the event of
1372 /// a failure it executes the host version outlined in \a OutlinedFn.
1373 /// \param D Directive to emit.
1374 /// \param OutlinedFn Host version of the code to be offloaded.
1375 /// \param OutlinedFnID ID of host version of the code to be offloaded.
1376 /// \param IfCond Expression evaluated in if clause associated with the target
1377 /// directive, or null if no if clause is used.
1378 /// \param Device Expression evaluated in device clause associated with the
1379 /// target directive, or null if no device clause is used and device modifier.
1380 /// \param SizeEmitter Callback to emit number of iterations for loop-based
1381 /// directives.
1382 virtual void emitTargetCall(
1383 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1384 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
1385 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
1386 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
1387 const OMPLoopDirective &D)>
1388 SizeEmitter);
1389
1390 /// Emit the target regions enclosed in \a GD function definition or
1391 /// the function itself in case it is a valid device function. Returns true if
1392 /// \a GD was dealt with successfully.
1393 /// \param GD Function to scan.
1394 virtual bool emitTargetFunctions(GlobalDecl GD);
1395
1396 /// Emit the global variable if it is a valid device global variable.
1397 /// Returns true if \a GD was dealt with successfully.
1398 /// \param GD Variable declaration to emit.
1399 virtual bool emitTargetGlobalVariable(GlobalDecl GD);
1400
1401 /// Checks if the provided global decl \a GD is a declare target variable and
1402 /// registers it when emitting code for the host.
1403 virtual void registerTargetGlobalVariable(const VarDecl *VD,
1404 llvm::Constant *Addr);
1405
1406 /// Emit the global \a GD if it is meaningful for the target. Returns
1407 /// if it was emitted successfully.
1408 /// \param GD Global to scan.
1409 virtual bool emitTargetGlobal(GlobalDecl GD);
1410
1411 /// Creates all the offload entries in the current compilation unit
1412 /// along with the associated metadata.
1413 void createOffloadEntriesAndInfoMetadata();
1414
1415 /// Emits code for teams call of the \a OutlinedFn with
1416 /// variables captured in a record which address is stored in \a
1417 /// CapturedStruct.
1418 /// \param OutlinedFn Outlined function to be run by team masters. Type of
1419 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1420 /// \param CapturedVars A pointer to the record with the references to
1421 /// variables used in \a OutlinedFn function.
1422 ///
1423 virtual void emitTeamsCall(CodeGenFunction &CGF,
1424 const OMPExecutableDirective &D,
1425 SourceLocation Loc, llvm::Function *OutlinedFn,
1426 ArrayRef<llvm::Value *> CapturedVars);
1427
1428 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
1429 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
1430 /// for num_teams clause.
1431 /// \param NumTeams An integer expression of teams.
1432 /// \param ThreadLimit An integer expression of threads.
1433 virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
1434 const Expr *ThreadLimit, SourceLocation Loc);
1435
1436 /// Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32
1437 /// global_tid, kmp_int32 thread_limit) to generate code for
1438 /// thread_limit clause on target directive
1439 /// \param ThreadLimit An integer expression of threads.
1440 virtual void emitThreadLimitClause(CodeGenFunction &CGF,
1441 const Expr *ThreadLimit,
1442 SourceLocation Loc);
1443
1444 /// Struct that keeps all the relevant information that should be kept
1445 /// throughout a 'target data' region.
1446 class TargetDataInfo : public llvm::OpenMPIRBuilder::TargetDataInfo {
1447 public:
1448 explicit TargetDataInfo() : llvm::OpenMPIRBuilder::TargetDataInfo() {}
1449 explicit TargetDataInfo(bool RequiresDevicePointerInfo,
1450 bool SeparateBeginEndCalls)
1451 : llvm::OpenMPIRBuilder::TargetDataInfo(RequiresDevicePointerInfo,
1452 SeparateBeginEndCalls) {}
1453 /// Map between the a declaration of a capture and the corresponding new
1454 /// llvm address where the runtime returns the device pointers.
1455 llvm::DenseMap<const ValueDecl *, llvm::Value *> CaptureDeviceAddrMap;
1456 };
1457
1458 /// Emit the target data mapping code associated with \a D.
1459 /// \param D Directive to emit.
1460 /// \param IfCond Expression evaluated in if clause associated with the
1461 /// target directive, or null if no device clause is used.
1462 /// \param Device Expression evaluated in device clause associated with the
1463 /// target directive, or null if no device clause is used.
1464 /// \param Info A record used to store information that needs to be preserved
1465 /// until the region is closed.
1466 virtual void emitTargetDataCalls(CodeGenFunction &CGF,
1467 const OMPExecutableDirective &D,
1468 const Expr *IfCond, const Expr *Device,
1469 const RegionCodeGenTy &CodeGen,
1470 CGOpenMPRuntime::TargetDataInfo &Info);
1471
1472 /// Emit the data mapping/movement code associated with the directive
1473 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
1474 /// \param D Directive to emit.
1475 /// \param IfCond Expression evaluated in if clause associated with the target
1476 /// directive, or null if no if clause is used.
1477 /// \param Device Expression evaluated in device clause associated with the
1478 /// target directive, or null if no device clause is used.
1479 virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
1480 const OMPExecutableDirective &D,
1481 const Expr *IfCond,
1482 const Expr *Device);
1483
1484 /// Marks function \a Fn with properly mangled versions of vector functions.
1485 /// \param FD Function marked as 'declare simd'.
1486 /// \param Fn LLVM function that must be marked with 'declare simd'
1487 /// attributes.
1488 virtual void emitDeclareSimdFunction(const FunctionDecl *FD,
1489 llvm::Function *Fn);
1490
1491 /// Emit initialization for doacross loop nesting support.
1492 /// \param D Loop-based construct used in doacross nesting construct.
1493 virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
1494 ArrayRef<Expr *> NumIterations);
1495
1496 /// Emit code for doacross ordered directive with 'depend' clause.
1497 /// \param C 'depend' clause with 'sink|source' dependency kind.
1498 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1499 const OMPDependClause *C);
1500
1501 /// Emit code for doacross ordered directive with 'doacross' clause.
1502 /// \param C 'doacross' clause with 'sink|source' dependence type.
1503 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1504 const OMPDoacrossClause *C);
1505
1506 /// Translates the native parameter of outlined function if this is required
1507 /// for target.
1508 /// \param FD Field decl from captured record for the parameter.
1509 /// \param NativeParam Parameter itself.
1510 virtual const VarDecl *translateParameter(const FieldDecl *FD,
1511 const VarDecl *NativeParam) const {
1512 return NativeParam;
1513 }
1514
1515 /// Gets the address of the native argument basing on the address of the
1516 /// target-specific parameter.
1517 /// \param NativeParam Parameter itself.
1518 /// \param TargetParam Corresponding target-specific parameter.
1519 virtual Address getParameterAddress(CodeGenFunction &CGF,
1520 const VarDecl *NativeParam,
1521 const VarDecl *TargetParam) const;
1522
1523 /// Choose default schedule type and chunk value for the
1524 /// dist_schedule clause.
1525 virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
1526 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
1527 llvm::Value *&Chunk) const {}
1528
1529 /// Choose default schedule type and chunk value for the
1530 /// schedule clause.
1531 virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
1532 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
1533 const Expr *&ChunkExpr) const;
1534
1535 /// Emits call of the outlined function with the provided arguments,
1536 /// translating these arguments to correct target-specific arguments.
1537 virtual void
1538 emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
1539 llvm::FunctionCallee OutlinedFn,
1540 ArrayRef<llvm::Value *> Args = {}) const;
1541
1542 /// Emits OpenMP-specific function prolog.
1543 /// Required for device constructs.
1544 virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D);
1545
1546 /// Gets the OpenMP-specific address of the local variable.
1547 virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1548 const VarDecl *VD);
1549
1550 /// Marks the declaration as already emitted for the device code and returns
1551 /// true, if it was marked already, and false, otherwise.
1552 bool markAsGlobalTarget(GlobalDecl GD);
1553
1554 /// Emit deferred declare target variables marked for deferred emission.
1555 void emitDeferredTargetDecls() const;
1556
1557 /// Adjust some parameters for the target-based directives, like addresses of
1558 /// the variables captured by reference in lambdas.
1559 virtual void
1560 adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF,
1561 const OMPExecutableDirective &D) const;
1562
1563 /// Perform check on requires decl to ensure that target architecture
1564 /// supports unified addressing
1565 virtual void processRequiresDirective(const OMPRequiresDecl *D);
1566
1567 /// Gets default memory ordering as specified in requires directive.
1568 llvm::AtomicOrdering getDefaultMemoryOrdering() const;
1569
1570 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with
1571 /// the predefined allocator and translates it into the corresponding address
1572 /// space.
1573 virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS);
1574
1575 /// Return whether the unified_shared_memory has been specified.
1576 bool hasRequiresUnifiedSharedMemory() const;
1577
1578 /// Checks if the \p VD variable is marked as nontemporal declaration in
1579 /// current context.
1580 bool isNontemporalDecl(const ValueDecl *VD) const;
1581
1582 /// Create specialized alloca to handle lastprivate conditionals.
1583 Address emitLastprivateConditionalInit(CodeGenFunction &CGF,
1584 const VarDecl *VD);
1585
1586 /// Checks if the provided \p LVal is lastprivate conditional and emits the
1587 /// code to update the value of the original variable.
1588 /// \code
1589 /// lastprivate(conditional: a)
1590 /// ...
1591 /// <type> a;
1592 /// lp_a = ...;
1593 /// #pragma omp critical(a)
1594 /// if (last_iv_a <= iv) {
1595 /// last_iv_a = iv;
1596 /// global_a = lp_a;
1597 /// }
1598 /// \endcode
1599 virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
1600 const Expr *LHS);
1601
1602 /// Checks if the lastprivate conditional was updated in inner region and
1603 /// writes the value.
1604 /// \code
1605 /// lastprivate(conditional: a)
1606 /// ...
1607 /// <type> a;bool Fired = false;
1608 /// #pragma omp ... shared(a)
1609 /// {
1610 /// lp_a = ...;
1611 /// Fired = true;
1612 /// }
1613 /// if (Fired) {
1614 /// #pragma omp critical(a)
1615 /// if (last_iv_a <= iv) {
1616 /// last_iv_a = iv;
1617 /// global_a = lp_a;
1618 /// }
1619 /// Fired = false;
1620 /// }
1621 /// \endcode
1622 virtual void checkAndEmitSharedLastprivateConditional(
1623 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1624 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls);
1625
1626 /// Gets the address of the global copy used for lastprivate conditional
1627 /// update, if any.
1628 /// \param PrivLVal LValue for the private copy.
1629 /// \param VD Original lastprivate declaration.
1630 virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF,
1631 LValue PrivLVal,
1632 const VarDecl *VD,
1633 SourceLocation Loc);
1634
1635 /// Emits list of dependecies based on the provided data (array of
1636 /// dependence/expression pairs).
1637 /// \returns Pointer to the first element of the array casted to VoidPtr type.
1638 std::pair<llvm::Value *, Address>
1639 emitDependClause(CodeGenFunction &CGF,
1640 ArrayRef<OMPTaskDataTy::DependData> Dependencies,
1641 SourceLocation Loc);
1642
1643 /// Emits list of dependecies based on the provided data (array of
1644 /// dependence/expression pairs) for depobj construct. In this case, the
1645 /// variable is allocated in dynamically. \returns Pointer to the first
1646 /// element of the array casted to VoidPtr type.
1647 Address emitDepobjDependClause(CodeGenFunction &CGF,
1648 const OMPTaskDataTy::DependData &Dependencies,
1649 SourceLocation Loc);
1650
1651 /// Emits the code to destroy the dependency object provided in depobj
1652 /// directive.
1653 void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
1654 SourceLocation Loc);
1655
1656 /// Updates the dependency kind in the specified depobj object.
1657 /// \param DepobjLVal LValue for the main depobj object.
1658 /// \param NewDepKind New dependency kind.
1659 void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
1660 OpenMPDependClauseKind NewDepKind, SourceLocation Loc);
1661
1662 /// Initializes user defined allocators specified in the uses_allocators
1663 /// clauses.
1664 void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator,
1665 const Expr *AllocatorTraits);
1666
1667 /// Destroys user defined allocators specified in the uses_allocators clause.
1668 void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator);
1669
1670 /// Returns true if the variable is a local variable in untied task.
1671 bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const;
1672};
1673
1674/// Class supports emissionof SIMD-only code.
1675class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
1676public:
1677 explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {}
1678 ~CGOpenMPSIMDRuntime() override {}
1679
1680 /// Emits outlined function for the specified OpenMP parallel directive
1681 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1682 /// kmp_int32 BoundID, struct context_vars*).
1683 /// \param CGF Reference to current CodeGenFunction.
1684 /// \param D OpenMP directive.
1685 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1686 /// \param InnermostKind Kind of innermost directive (for simple directives it
1687 /// is a directive itself, for combined - its innermost directive).
1688 /// \param CodeGen Code generation sequence for the \a D directive.
1689 llvm::Function *emitParallelOutlinedFunction(
1690 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1691 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1692 const RegionCodeGenTy &CodeGen) override;
1693
1694 /// Emits outlined function for the specified OpenMP teams directive
1695 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1696 /// kmp_int32 BoundID, struct context_vars*).
1697 /// \param CGF Reference to current CodeGenFunction.
1698 /// \param D OpenMP directive.
1699 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1700 /// \param InnermostKind Kind of innermost directive (for simple directives it
1701 /// is a directive itself, for combined - its innermost directive).
1702 /// \param CodeGen Code generation sequence for the \a D directive.
1703 llvm::Function *emitTeamsOutlinedFunction(
1704 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1705 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1706 const RegionCodeGenTy &CodeGen) override;
1707
1708 /// Emits outlined function for the OpenMP task directive \a D. This
1709 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
1710 /// TaskT).
1711 /// \param D OpenMP directive.
1712 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1713 /// \param PartIDVar Variable for partition id in the current OpenMP untied
1714 /// task region.
1715 /// \param TaskTVar Variable for task_t argument.
1716 /// \param InnermostKind Kind of innermost directive (for simple directives it
1717 /// is a directive itself, for combined - its innermost directive).
1718 /// \param CodeGen Code generation sequence for the \a D directive.
1719 /// \param Tied true if task is generated for tied task, false otherwise.
1720 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
1721 /// tasks.
1722 ///
1723 llvm::Function *emitTaskOutlinedFunction(
1724 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1725 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1726 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1727 bool Tied, unsigned &NumberOfParts) override;
1728
1729 /// Emits code for parallel or serial call of the \a OutlinedFn with
1730 /// variables captured in a record which address is stored in \a
1731 /// CapturedStruct.
1732 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
1733 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1734 /// \param CapturedVars A pointer to the record with the references to
1735 /// variables used in \a OutlinedFn function.
1736 /// \param IfCond Condition in the associated 'if' clause, if it was
1737 /// specified, nullptr otherwise.
1738 /// \param NumThreads The value corresponding to the num_threads clause, if
1739 /// any, or nullptr.
1740 ///
1741 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1742 llvm::Function *OutlinedFn,
1743 ArrayRef<llvm::Value *> CapturedVars,
1744 const Expr *IfCond, llvm::Value *NumThreads) override;
1745
1746 /// Emits a critical region.
1747 /// \param CriticalName Name of the critical region.
1748 /// \param CriticalOpGen Generator for the statement associated with the given
1749 /// critical region.
1750 /// \param Hint Value of the 'hint' clause (optional).
1751 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
1752 const RegionCodeGenTy &CriticalOpGen,
1753 SourceLocation Loc,
1754 const Expr *Hint = nullptr) override;
1755
1756 /// Emits a master region.
1757 /// \param MasterOpGen Generator for the statement associated with the given
1758 /// master region.
1759 void emitMasterRegion(CodeGenFunction &CGF,
1760 const RegionCodeGenTy &MasterOpGen,
1761 SourceLocation Loc) override;
1762
1763 /// Emits a masked region.
1764 /// \param MaskedOpGen Generator for the statement associated with the given
1765 /// masked region.
1766 void emitMaskedRegion(CodeGenFunction &CGF,
1767 const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc,
1768 const Expr *Filter = nullptr) override;
1769
1770 /// Emits a masked region.
1771 /// \param MaskedOpGen Generator for the statement associated with the given
1772 /// masked region.
1773
1774 /// Emits code for a taskyield directive.
1775 void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override;
1776
1777 /// Emit a taskgroup region.
1778 /// \param TaskgroupOpGen Generator for the statement associated with the
1779 /// given taskgroup region.
1780 void emitTaskgroupRegion(CodeGenFunction &CGF,
1781 const RegionCodeGenTy &TaskgroupOpGen,
1782 SourceLocation Loc) override;
1783
1784 /// Emits a single region.
1785 /// \param SingleOpGen Generator for the statement associated with the given
1786 /// single region.
1787 void emitSingleRegion(CodeGenFunction &CGF,
1788 const RegionCodeGenTy &SingleOpGen, SourceLocation Loc,
1789 ArrayRef<const Expr *> CopyprivateVars,
1790 ArrayRef<const Expr *> DestExprs,
1791 ArrayRef<const Expr *> SrcExprs,
1792 ArrayRef<const Expr *> AssignmentOps) override;
1793
1794 /// Emit an ordered region.
1795 /// \param OrderedOpGen Generator for the statement associated with the given
1796 /// ordered region.
1797 void emitOrderedRegion(CodeGenFunction &CGF,
1798 const RegionCodeGenTy &OrderedOpGen,
1799 SourceLocation Loc, bool IsThreads) override;
1800
1801 /// Emit an implicit/explicit barrier for OpenMP threads.
1802 /// \param Kind Directive for which this implicit barrier call must be
1803 /// generated. Must be OMPD_barrier for explicit barrier generation.
1804 /// \param EmitChecks true if need to emit checks for cancellation barriers.
1805 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
1806 /// runtime class decides which one to emit (simple or with cancellation
1807 /// checks).
1808 ///
1809 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
1810 OpenMPDirectiveKind Kind, bool EmitChecks = true,
1811 bool ForceSimpleCall = false) override;
1812
1813 /// This is used for non static scheduled types and when the ordered
1814 /// clause is present on the loop construct.
1815 /// Depending on the loop schedule, it is necessary to call some runtime
1816 /// routine before start of the OpenMP loop to get the loop upper / lower
1817 /// bounds \a LB and \a UB and stride \a ST.
1818 ///
1819 /// \param CGF Reference to current CodeGenFunction.
1820 /// \param Loc Clang source location.
1821 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1822 /// \param IVSize Size of the iteration variable in bits.
1823 /// \param IVSigned Sign of the iteration variable.
1824 /// \param Ordered true if loop is ordered, false otherwise.
1825 /// \param DispatchValues struct containing llvm values for lower bound, upper
1826 /// bound, and chunk expression.
1827 /// For the default (nullptr) value, the chunk 1 will be used.
1828 ///
1829 void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
1830 const OpenMPScheduleTy &ScheduleKind,
1831 unsigned IVSize, bool IVSigned, bool Ordered,
1832 const DispatchRTInput &DispatchValues) override;
1833
1834 /// This is used for non static scheduled types and when the ordered
1835 /// clause is present on the loop construct.
1836 ///
1837 /// \param CGF Reference to current CodeGenFunction.
1838 /// \param Loc Clang source location.
1839 ///
1840 void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override;
1841
1842 /// Call the appropriate runtime routine to initialize it before start
1843 /// of loop.
1844 ///
1845 /// This is used only in case of static schedule, when the user did not
1846 /// specify a ordered clause on the loop construct.
1847 /// Depending on the loop schedule, it is necessary to call some runtime
1848 /// routine before start of the OpenMP loop to get the loop upper / lower
1849 /// bounds LB and UB and stride ST.
1850 ///
1851 /// \param CGF Reference to current CodeGenFunction.
1852 /// \param Loc Clang source location.
1853 /// \param DKind Kind of the directive.
1854 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1855 /// \param Values Input arguments for the construct.
1856 ///
1857 void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1858 OpenMPDirectiveKind DKind,
1859 const OpenMPScheduleTy &ScheduleKind,
1860 const StaticRTInput &Values) override;
1861
1862 ///
1863 /// \param CGF Reference to current CodeGenFunction.
1864 /// \param Loc Clang source location.
1865 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1866 /// \param Values Input arguments for the construct.
1867 ///
1868 void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1869 OpenMPDistScheduleClauseKind SchedKind,
1870 const StaticRTInput &Values) override;
1871
1872 /// Call the appropriate runtime routine to notify that we finished
1873 /// iteration of the ordered loop with the dynamic scheduling.
1874 ///
1875 /// \param CGF Reference to current CodeGenFunction.
1876 /// \param Loc Clang source location.
1877 /// \param IVSize Size of the iteration variable in bits.
1878 /// \param IVSigned Sign of the iteration variable.
1879 ///
1880 void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc,
1881 unsigned IVSize, bool IVSigned) override;
1882
1883 /// Call the appropriate runtime routine to notify that we finished
1884 /// all the work with current loop.
1885 ///
1886 /// \param CGF Reference to current CodeGenFunction.
1887 /// \param Loc Clang source location.
1888 /// \param DKind Kind of the directive for which the static finish is emitted.
1889 ///
1890 void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1891 OpenMPDirectiveKind DKind) override;
1892
1893 /// Call __kmpc_dispatch_next(
1894 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1895 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1896 /// kmp_int[32|64] *p_stride);
1897 /// \param IVSize Size of the iteration variable in bits.
1898 /// \param IVSigned Sign of the iteration variable.
1899 /// \param IL Address of the output variable in which the flag of the
1900 /// last iteration is returned.
1901 /// \param LB Address of the output variable in which the lower iteration
1902 /// number is returned.
1903 /// \param UB Address of the output variable in which the upper iteration
1904 /// number is returned.
1905 /// \param ST Address of the output variable in which the stride value is
1906 /// returned.
1907 llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1908 unsigned IVSize, bool IVSigned, Address IL,
1909 Address LB, Address UB, Address ST) override;
1910
1911 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1912 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1913 /// clause.
1914 /// \param NumThreads An integer value of threads.
1915 void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads,
1916 SourceLocation Loc) override;
1917
1918 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1919 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1920 void emitProcBindClause(CodeGenFunction &CGF,
1921 llvm::omp::ProcBindKind ProcBind,
1922 SourceLocation Loc) override;
1923
1924 /// Returns address of the threadprivate variable for the current
1925 /// thread.
1926 /// \param VD Threadprivate variable.
1927 /// \param VDAddr Address of the global variable \a VD.
1928 /// \param Loc Location of the reference to threadprivate var.
1929 /// \return Address of the threadprivate variable for the current thread.
1930 Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD,
1931 Address VDAddr, SourceLocation Loc) override;
1932
1933 /// Emit a code for initialization of threadprivate variable. It emits
1934 /// a call to runtime library which adds initial value to the newly created
1935 /// threadprivate variable (if it is not constant) and registers destructor
1936 /// for the variable (if any).
1937 /// \param VD Threadprivate variable.
1938 /// \param VDAddr Address of the global variable \a VD.
1939 /// \param Loc Location of threadprivate declaration.
1940 /// \param PerformInit true if initialization expression is not constant.
1941 llvm::Function *
1942 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1943 SourceLocation Loc, bool PerformInit,
1944 CodeGenFunction *CGF = nullptr) override;
1945
1946 /// Creates artificial threadprivate variable with name \p Name and type \p
1947 /// VarType.
1948 /// \param VarType Type of the artificial threadprivate variable.
1949 /// \param Name Name of the artificial threadprivate variable.
1950 Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1951 QualType VarType,
1952 StringRef Name) override;
1953
1954 /// Emit flush of the variables specified in 'omp flush' directive.
1955 /// \param Vars List of variables to flush.
1956 void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1957 SourceLocation Loc, llvm::AtomicOrdering AO) override;
1958
1959 /// Emit task region for the task directive. The task region is
1960 /// emitted in several steps:
1961 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1962 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1963 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1964 /// function:
1965 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1966 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1967 /// return 0;
1968 /// }
1969 /// 2. Copy a list of shared variables to field shareds of the resulting
1970 /// structure kmp_task_t returned by the previous call (if any).
1971 /// 3. Copy a pointer to destructions function to field destructions of the
1972 /// resulting structure kmp_task_t.
1973 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1974 /// kmp_task_t *new_task), where new_task is a resulting structure from
1975 /// previous items.
1976 /// \param D Current task directive.
1977 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1978 /// /*part_id*/, captured_struct */*__context*/);
1979 /// \param SharedsTy A type which contains references the shared variables.
1980 /// \param Shareds Context with the list of shared variables from the \p
1981 /// TaskFunction.
1982 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1983 /// otherwise.
1984 /// \param Data Additional data for task generation like tiednsee, final
1985 /// state, list of privates etc.
1986 void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1987 const OMPExecutableDirective &D,
1988 llvm::Function *TaskFunction, QualType SharedsTy,
1989 Address Shareds, const Expr *IfCond,
1990 const OMPTaskDataTy &Data) override;
1991
1992 /// Emit task region for the taskloop directive. The taskloop region is
1993 /// emitted in several steps:
1994 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1995 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1996 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1997 /// function:
1998 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1999 /// TaskFunction(gtid, tt->part_id, tt->shareds);
2000 /// return 0;
2001 /// }
2002 /// 2. Copy a list of shared variables to field shareds of the resulting
2003 /// structure kmp_task_t returned by the previous call (if any).
2004 /// 3. Copy a pointer to destructions function to field destructions of the
2005 /// resulting structure kmp_task_t.
2006 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
2007 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
2008 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
2009 /// is a resulting structure from
2010 /// previous items.
2011 /// \param D Current task directive.
2012 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
2013 /// /*part_id*/, captured_struct */*__context*/);
2014 /// \param SharedsTy A type which contains references the shared variables.
2015 /// \param Shareds Context with the list of shared variables from the \p
2016 /// TaskFunction.
2017 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
2018 /// otherwise.
2019 /// \param Data Additional data for task generation like tiednsee, final
2020 /// state, list of privates etc.
2021 void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
2022 const OMPLoopDirective &D, llvm::Function *TaskFunction,
2023 QualType SharedsTy, Address Shareds, const Expr *IfCond,
2024 const OMPTaskDataTy &Data) override;
2025
2026 /// Emit a code for reduction clause. Next code should be emitted for
2027 /// reduction:
2028 /// \code
2029 ///
2030 /// static kmp_critical_name lock = { 0 };
2031 ///
2032 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
2033 /// ...
2034 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
2035 /// ...
2036 /// }
2037 ///
2038 /// ...
2039 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
2040 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
2041 /// RedList, reduce_func, &<lock>)) {
2042 /// case 1:
2043 /// ...
2044 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2045 /// ...
2046 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2047 /// break;
2048 /// case 2:
2049 /// ...
2050 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
2051 /// ...
2052 /// break;
2053 /// default:;
2054 /// }
2055 /// \endcode
2056 ///
2057 /// \param Privates List of private copies for original reduction arguments.
2058 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
2059 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
2060 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
2061 /// or 'operator binop(LHS, RHS)'.
2062 /// \param Options List of options for reduction codegen:
2063 /// WithNowait true if parent directive has also nowait clause, false
2064 /// otherwise.
2065 /// SimpleReduction Emit reduction operation only. Used for omp simd
2066 /// directive on the host.
2067 /// ReductionKind The kind of reduction to perform.
2068 void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
2069 ArrayRef<const Expr *> Privates,
2070 ArrayRef<const Expr *> LHSExprs,
2071 ArrayRef<const Expr *> RHSExprs,
2072 ArrayRef<const Expr *> ReductionOps,
2073 ReductionOptionsTy Options) override;
2074
2075 /// Emit a code for initialization of task reduction clause. Next code
2076 /// should be emitted for reduction:
2077 /// \code
2078 ///
2079 /// _taskred_item_t red_data[n];
2080 /// ...
2081 /// red_data[i].shar = &shareds[i];
2082 /// red_data[i].orig = &origs[i];
2083 /// red_data[i].size = sizeof(origs[i]);
2084 /// red_data[i].f_init = (void*)RedInit<i>;
2085 /// red_data[i].f_fini = (void*)RedDest<i>;
2086 /// red_data[i].f_comb = (void*)RedOp<i>;
2087 /// red_data[i].flags = <Flag_i>;
2088 /// ...
2089 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
2090 /// \endcode
2091 /// For reduction clause with task modifier it emits the next call:
2092 /// \code
2093 ///
2094 /// _taskred_item_t red_data[n];
2095 /// ...
2096 /// red_data[i].shar = &shareds[i];
2097 /// red_data[i].orig = &origs[i];
2098 /// red_data[i].size = sizeof(origs[i]);
2099 /// red_data[i].f_init = (void*)RedInit<i>;
2100 /// red_data[i].f_fini = (void*)RedDest<i>;
2101 /// red_data[i].f_comb = (void*)RedOp<i>;
2102 /// red_data[i].flags = <Flag_i>;
2103 /// ...
2104 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
2105 /// red_data);
2106 /// \endcode
2107 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
2108 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
2109 /// \param Data Additional data for task generation like tiedness, final
2110 /// state, list of privates, reductions etc.
2111 llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc,
2112 ArrayRef<const Expr *> LHSExprs,
2113 ArrayRef<const Expr *> RHSExprs,
2114 const OMPTaskDataTy &Data) override;
2115
2116 /// Emits the following code for reduction clause with task modifier:
2117 /// \code
2118 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
2119 /// \endcode
2120 void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
2121 bool IsWorksharingReduction) override;
2122
2123 /// Required to resolve existing problems in the runtime. Emits threadprivate
2124 /// variables to store the size of the VLAs/array sections for
2125 /// initializer/combiner/finalizer functions + emits threadprivate variable to
2126 /// store the pointer to the original reduction item for the custom
2127 /// initializer defined by declare reduction construct.
2128 /// \param RCG Allows to reuse an existing data for the reductions.
2129 /// \param N Reduction item for which fixups must be emitted.
2130 void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
2131 ReductionCodeGen &RCG, unsigned N) override;
2132
2133 /// Get the address of `void *` type of the privatue copy of the reduction
2134 /// item specified by the \p SharedLVal.
2135 /// \param ReductionsPtr Pointer to the reduction data returned by the
2136 /// emitTaskReductionInit function.
2137 /// \param SharedLVal Address of the original reduction item.
2138 Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
2139 llvm::Value *ReductionsPtr,
2140 LValue SharedLVal) override;
2141
2142 /// Emit code for 'taskwait' directive.
2143 void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
2144 const OMPTaskDataTy &Data) override;
2145
2146 /// Emit code for 'cancellation point' construct.
2147 /// \param CancelRegion Region kind for which the cancellation point must be
2148 /// emitted.
2149 ///
2150 void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc,
2151 OpenMPDirectiveKind CancelRegion) override;
2152
2153 /// Emit code for 'cancel' construct.
2154 /// \param IfCond Condition in the associated 'if' clause, if it was
2155 /// specified, nullptr otherwise.
2156 /// \param CancelRegion Region kind for which the cancel must be emitted.
2157 ///
2158 void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
2159 const Expr *IfCond,
2160 OpenMPDirectiveKind CancelRegion) override;
2161
2162 /// Emit outilined function for 'target' directive.
2163 /// \param D Directive to emit.
2164 /// \param ParentName Name of the function that encloses the target region.
2165 /// \param OutlinedFn Outlined function value to be defined by this call.
2166 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
2167 /// \param IsOffloadEntry True if the outlined function is an offload entry.
2168 /// \param CodeGen Code generation sequence for the \a D directive.
2169 /// An outlined function may not be an entry if, e.g. the if clause always
2170 /// evaluates to false.
2171 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
2172 StringRef ParentName,
2173 llvm::Function *&OutlinedFn,
2174 llvm::Constant *&OutlinedFnID,
2175 bool IsOffloadEntry,
2176 const RegionCodeGenTy &CodeGen) override;
2177
2178 /// Emit the target offloading code associated with \a D. The emitted
2179 /// code attempts offloading the execution to the device, an the event of
2180 /// a failure it executes the host version outlined in \a OutlinedFn.
2181 /// \param D Directive to emit.
2182 /// \param OutlinedFn Host version of the code to be offloaded.
2183 /// \param OutlinedFnID ID of host version of the code to be offloaded.
2184 /// \param IfCond Expression evaluated in if clause associated with the target
2185 /// directive, or null if no if clause is used.
2186 /// \param Device Expression evaluated in device clause associated with the
2187 /// target directive, or null if no device clause is used and device modifier.
2188 void emitTargetCall(
2189 CodeGenFunction &CGF, const OMPExecutableDirective &D,
2190 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
2191 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
2192 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
2193 const OMPLoopDirective &D)>
2194 SizeEmitter) override;
2195
2196 /// Emit the target regions enclosed in \a GD function definition or
2197 /// the function itself in case it is a valid device function. Returns true if
2198 /// \a GD was dealt with successfully.
2199 /// \param GD Function to scan.
2200 bool emitTargetFunctions(GlobalDecl GD) override;
2201
2202 /// Emit the global variable if it is a valid device global variable.
2203 /// Returns true if \a GD was dealt with successfully.
2204 /// \param GD Variable declaration to emit.
2205 bool emitTargetGlobalVariable(GlobalDecl GD) override;
2206
2207 /// Emit the global \a GD if it is meaningful for the target. Returns
2208 /// if it was emitted successfully.
2209 /// \param GD Global to scan.
2210 bool emitTargetGlobal(GlobalDecl GD) override;
2211
2212 /// Emits code for teams call of the \a OutlinedFn with
2213 /// variables captured in a record which address is stored in \a
2214 /// CapturedStruct.
2215 /// \param OutlinedFn Outlined function to be run by team masters. Type of
2216 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
2217 /// \param CapturedVars A pointer to the record with the references to
2218 /// variables used in \a OutlinedFn function.
2219 ///
2220 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
2221 SourceLocation Loc, llvm::Function *OutlinedFn,
2222 ArrayRef<llvm::Value *> CapturedVars) override;
2223
2224 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
2225 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
2226 /// for num_teams clause.
2227 /// \param NumTeams An integer expression of teams.
2228 /// \param ThreadLimit An integer expression of threads.
2229 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
2230 const Expr *ThreadLimit, SourceLocation Loc) override;
2231
2232 /// Emit the target data mapping code associated with \a D.
2233 /// \param D Directive to emit.
2234 /// \param IfCond Expression evaluated in if clause associated with the
2235 /// target directive, or null if no device clause is used.
2236 /// \param Device Expression evaluated in device clause associated with the
2237 /// target directive, or null if no device clause is used.
2238 /// \param Info A record used to store information that needs to be preserved
2239 /// until the region is closed.
2240 void emitTargetDataCalls(CodeGenFunction &CGF,
2241 const OMPExecutableDirective &D, const Expr *IfCond,
2242 const Expr *Device, const RegionCodeGenTy &CodeGen,
2243 CGOpenMPRuntime::TargetDataInfo &Info) override;
2244
2245 /// Emit the data mapping/movement code associated with the directive
2246 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
2247 /// \param D Directive to emit.
2248 /// \param IfCond Expression evaluated in if clause associated with the target
2249 /// directive, or null if no if clause is used.
2250 /// \param Device Expression evaluated in device clause associated with the
2251 /// target directive, or null if no device clause is used.
2252 void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
2253 const OMPExecutableDirective &D,
2254 const Expr *IfCond,
2255 const Expr *Device) override;
2256
2257 /// Emit initialization for doacross loop nesting support.
2258 /// \param D Loop-based construct used in doacross nesting construct.
2259 void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
2260 ArrayRef<Expr *> NumIterations) override;
2261
2262 /// Emit code for doacross ordered directive with 'depend' clause.
2263 /// \param C 'depend' clause with 'sink|source' dependency kind.
2264 void emitDoacrossOrdered(CodeGenFunction &CGF,
2265 const OMPDependClause *C) override;
2266
2267 /// Emit code for doacross ordered directive with 'doacross' clause.
2268 /// \param C 'doacross' clause with 'sink|source' dependence type.
2269 void emitDoacrossOrdered(CodeGenFunction &CGF,
2270 const OMPDoacrossClause *C) override;
2271
2272 /// Translates the native parameter of outlined function if this is required
2273 /// for target.
2274 /// \param FD Field decl from captured record for the parameter.
2275 /// \param NativeParam Parameter itself.
2276 const VarDecl *translateParameter(const FieldDecl *FD,
2277 const VarDecl *NativeParam) const override;
2278
2279 /// Gets the address of the native argument basing on the address of the
2280 /// target-specific parameter.
2281 /// \param NativeParam Parameter itself.
2282 /// \param TargetParam Corresponding target-specific parameter.
2283 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
2284 const VarDecl *TargetParam) const override;
2285
2286 /// Gets the OpenMP-specific address of the local variable.
2287 Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2288 const VarDecl *VD) override {
2289 return Address::invalid();
2290 }
2291};
2292
2293} // namespace CodeGen
2294// Utility for openmp doacross clause kind
2295namespace {
2296template <typename T> class OMPDoacrossKind {
2297public:
2298 bool isSink(const T *) { return false; }
2299 bool isSource(const T *) { return false; }
2300};
2301template <> class OMPDoacrossKind<OMPDependClause> {
2302public:
2303 bool isSink(const OMPDependClause *C) {
2304 return C->getDependencyKind() == OMPC_DEPEND_sink;
2305 }
2306 bool isSource(const OMPDependClause *C) {
2307 return C->getDependencyKind() == OMPC_DEPEND_source;
2308 }
2309};
2310template <> class OMPDoacrossKind<OMPDoacrossClause> {
2311public:
2312 bool isSource(const OMPDoacrossClause *C) {
2313 return C->getDependenceType() == OMPC_DOACROSS_source ||
2314 C->getDependenceType() == OMPC_DOACROSS_source_omp_cur_iteration;
2315 }
2316 bool isSink(const OMPDoacrossClause *C) {
2317 return C->getDependenceType() == OMPC_DOACROSS_sink ||
2318 C->getDependenceType() == OMPC_DOACROSS_sink_omp_cur_iteration;
2319 }
2320};
2321} // namespace
2322} // namespace clang
2323
2324#endif
2325