1//===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
14#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
15
16#include "CGValue.h"
17#include "clang/AST/DeclOpenMP.h"
18#include "clang/AST/GlobalDecl.h"
19#include "clang/AST/Type.h"
20#include "clang/Basic/OpenMPKinds.h"
21#include "clang/Basic/SourceLocation.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/PointerIntPair.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/Frontend/OpenMP/OMPConstants.h"
28#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/ValueHandle.h"
31#include "llvm/Support/AtomicOrdering.h"
32
33namespace llvm {
34class ArrayType;
35class Constant;
36class FunctionType;
37class GlobalVariable;
38class Type;
39class Value;
40class OpenMPIRBuilder;
41} // namespace llvm
42
43namespace clang {
44class Expr;
45class OMPDependClause;
46class OMPExecutableDirective;
47class OMPLoopDirective;
48class VarDecl;
49class OMPDeclareReductionDecl;
50
51namespace CodeGen {
52class Address;
53class CodeGenFunction;
54class CodeGenModule;
55
56/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
57/// region.
58class PrePostActionTy {
59public:
60 explicit PrePostActionTy() {}
61 virtual void Enter(CodeGenFunction &CGF) {}
62 virtual void Exit(CodeGenFunction &CGF) {}
63 virtual ~PrePostActionTy() {}
64};
65
66/// Class provides a way to call simple version of codegen for OpenMP region, or
67/// an advanced with possible pre|post-actions in codegen.
68class RegionCodeGenTy final {
69 intptr_t CodeGen;
70 typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &);
71 CodeGenTy Callback;
72 mutable PrePostActionTy *PrePostAction;
73 RegionCodeGenTy() = delete;
74 template <typename Callable>
75 static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF,
76 PrePostActionTy &Action) {
77 return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action);
78 }
79
80public:
81 template <typename Callable>
82 RegionCodeGenTy(
83 Callable &&CodeGen,
84 std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
85 RegionCodeGenTy>::value> * = nullptr)
86 : CodeGen(reinterpret_cast<intptr_t>(&CodeGen)),
87 Callback(CallbackFn<std::remove_reference_t<Callable>>),
88 PrePostAction(nullptr) {}
89 void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; }
90 void operator()(CodeGenFunction &CGF) const;
91};
92
93struct OMPTaskDataTy final {
94 SmallVector<const Expr *, 4> PrivateVars;
95 SmallVector<const Expr *, 4> PrivateCopies;
96 SmallVector<const Expr *, 4> FirstprivateVars;
97 SmallVector<const Expr *, 4> FirstprivateCopies;
98 SmallVector<const Expr *, 4> FirstprivateInits;
99 SmallVector<const Expr *, 4> LastprivateVars;
100 SmallVector<const Expr *, 4> LastprivateCopies;
101 SmallVector<const Expr *, 4> ReductionVars;
102 SmallVector<const Expr *, 4> ReductionOrigs;
103 SmallVector<const Expr *, 4> ReductionCopies;
104 SmallVector<const Expr *, 4> ReductionOps;
105 SmallVector<CanonicalDeclPtr<const VarDecl>, 4> PrivateLocals;
106 struct DependData {
107 OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown;
108 const Expr *IteratorExpr = nullptr;
109 SmallVector<const Expr *, 4> DepExprs;
110 explicit DependData() = default;
111 DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr)
112 : DepKind(DepKind), IteratorExpr(IteratorExpr) {}
113 };
114 SmallVector<DependData, 4> Dependences;
115 llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
116 llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule;
117 llvm::PointerIntPair<llvm::Value *, 1, bool> Priority;
118 llvm::Value *Reductions = nullptr;
119 unsigned NumberOfParts = 0;
120 bool Tied = true;
121 bool Nogroup = false;
122 bool IsReductionWithTaskMod = false;
123 bool IsWorksharingReduction = false;
124 bool HasNowaitClause = false;
125};
126
127/// Class intended to support codegen of all kind of the reduction clauses.
128class ReductionCodeGen {
129private:
130 /// Data required for codegen of reduction clauses.
131 struct ReductionData {
132 /// Reference to the item shared between tasks to reduce into.
133 const Expr *Shared = nullptr;
134 /// Reference to the original item.
135 const Expr *Ref = nullptr;
136 /// Helper expression for generation of private copy.
137 const Expr *Private = nullptr;
138 /// Helper expression for generation reduction operation.
139 const Expr *ReductionOp = nullptr;
140 ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private,
141 const Expr *ReductionOp)
142 : Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) {
143 }
144 };
145 /// List of reduction-based clauses.
146 SmallVector<ReductionData, 4> ClausesData;
147
148 /// List of addresses of shared variables/expressions.
149 SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses;
150 /// List of addresses of original variables/expressions.
151 SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses;
152 /// Sizes of the reduction items in chars.
153 SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes;
154 /// Base declarations for the reduction items.
155 SmallVector<const VarDecl *, 4> BaseDecls;
156
157 /// Emits lvalue for shared expression.
158 LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E);
159 /// Emits upper bound for shared expression (if array section).
160 LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E);
161 /// Performs aggregate initialization.
162 /// \param N Number of reduction item in the common list.
163 /// \param PrivateAddr Address of the corresponding private item.
164 /// \param SharedAddr Address of the original shared variable.
165 /// \param DRD Declare reduction construct used for reduction item.
166 void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N,
167 Address PrivateAddr, Address SharedAddr,
168 const OMPDeclareReductionDecl *DRD);
169
170public:
171 ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs,
172 ArrayRef<const Expr *> Privates,
173 ArrayRef<const Expr *> ReductionOps);
174 /// Emits lvalue for the shared and original reduction item.
175 /// \param N Number of the reduction item.
176 void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N);
177 /// Emits the code for the variable-modified type, if required.
178 /// \param N Number of the reduction item.
179 void emitAggregateType(CodeGenFunction &CGF, unsigned N);
180 /// Emits the code for the variable-modified type, if required.
181 /// \param N Number of the reduction item.
182 /// \param Size Size of the type in chars.
183 void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size);
184 /// Performs initialization of the private copy for the reduction item.
185 /// \param N Number of the reduction item.
186 /// \param PrivateAddr Address of the corresponding private item.
187 /// \param DefaultInit Default initialization sequence that should be
188 /// performed if no reduction specific initialization is found.
189 /// \param SharedAddr Address of the original shared variable.
190 void
191 emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr,
192 Address SharedAddr,
193 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit);
194 /// Returns true if the private copy requires cleanups.
195 bool needCleanups(unsigned N);
196 /// Emits cleanup code for the reduction item.
197 /// \param N Number of the reduction item.
198 /// \param PrivateAddr Address of the corresponding private item.
199 void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr);
200 /// Adjusts \p PrivatedAddr for using instead of the original variable
201 /// address in normal operations.
202 /// \param N Number of the reduction item.
203 /// \param PrivateAddr Address of the corresponding private item.
204 Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
205 Address PrivateAddr);
206 /// Returns LValue for the reduction item.
207 LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; }
208 /// Returns LValue for the original reduction item.
209 LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; }
210 /// Returns the size of the reduction item (in chars and total number of
211 /// elements in the item), or nullptr, if the size is a constant.
212 std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const {
213 return Sizes[N];
214 }
215 /// Returns the base declaration of the reduction item.
216 const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; }
217 /// Returns the base declaration of the reduction item.
218 const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; }
219 /// Returns true if the initialization of the reduction item uses initializer
220 /// from declare reduction construct.
221 bool usesReductionInitializer(unsigned N) const;
222 /// Return the type of the private item.
223 QualType getPrivateType(unsigned N) const {
224 return cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl())
225 ->getType();
226 }
227};
228
229class CGOpenMPRuntime {
230public:
231 /// Allows to disable automatic handling of functions used in target regions
232 /// as those marked as `omp declare target`.
233 class DisableAutoDeclareTargetRAII {
234 CodeGenModule &CGM;
235 bool SavedShouldMarkAsGlobal = false;
236
237 public:
238 DisableAutoDeclareTargetRAII(CodeGenModule &CGM);
239 ~DisableAutoDeclareTargetRAII();
240 };
241
242 /// Manages list of nontemporal decls for the specified directive.
243 class NontemporalDeclsRAII {
244 CodeGenModule &CGM;
245 const bool NeedToPush;
246
247 public:
248 NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S);
249 ~NontemporalDeclsRAII();
250 };
251
252 /// Manages list of nontemporal decls for the specified directive.
253 class UntiedTaskLocalDeclsRAII {
254 CodeGenModule &CGM;
255 const bool NeedToPush;
256
257 public:
258 UntiedTaskLocalDeclsRAII(
259 CodeGenFunction &CGF,
260 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
261 std::pair<Address, Address>> &LocalVars);
262 ~UntiedTaskLocalDeclsRAII();
263 };
264
265 /// Maps the expression for the lastprivate variable to the global copy used
266 /// to store new value because original variables are not mapped in inner
267 /// parallel regions. Only private copies are captured but we need also to
268 /// store private copy in shared address.
269 /// Also, stores the expression for the private loop counter and it
270 /// threaprivate name.
271 struct LastprivateConditionalData {
272 llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>>
273 DeclToUniqueName;
274 LValue IVLVal;
275 llvm::Function *Fn = nullptr;
276 bool Disabled = false;
277 };
278 /// Manages list of lastprivate conditional decls for the specified directive.
279 class LastprivateConditionalRAII {
280 enum class ActionToDo {
281 DoNotPush,
282 PushAsLastprivateConditional,
283 DisableLastprivateConditional,
284 };
285 CodeGenModule &CGM;
286 ActionToDo Action = ActionToDo::DoNotPush;
287
288 /// Check and try to disable analysis of inner regions for changes in
289 /// lastprivate conditional.
290 void tryToDisableInnerAnalysis(const OMPExecutableDirective &S,
291 llvm::DenseSet<CanonicalDeclPtr<const Decl>>
292 &NeedToAddForLPCsAsDisabled) const;
293
294 LastprivateConditionalRAII(CodeGenFunction &CGF,
295 const OMPExecutableDirective &S);
296
297 public:
298 explicit LastprivateConditionalRAII(CodeGenFunction &CGF,
299 const OMPExecutableDirective &S,
300 LValue IVLVal);
301 static LastprivateConditionalRAII disable(CodeGenFunction &CGF,
302 const OMPExecutableDirective &S);
303 ~LastprivateConditionalRAII();
304 };
305
306 llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; }
307
308protected:
309 CodeGenModule &CGM;
310
311 /// An OpenMP-IR-Builder instance.
312 llvm::OpenMPIRBuilder OMPBuilder;
313
314 /// Helper to determine the min/max number of threads/teams for \p D.
315 void computeMinAndMaxThreadsAndTeams(const OMPExecutableDirective &D,
316 CodeGenFunction &CGF,
317 int32_t &MinThreadsVal,
318 int32_t &MaxThreadsVal,
319 int32_t &MinTeamsVal,
320 int32_t &MaxTeamsVal);
321
322 /// Helper to emit outlined function for 'target' directive.
323 /// \param D Directive to emit.
324 /// \param ParentName Name of the function that encloses the target region.
325 /// \param OutlinedFn Outlined function value to be defined by this call.
326 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
327 /// \param IsOffloadEntry True if the outlined function is an offload entry.
328 /// \param CodeGen Lambda codegen specific to an accelerator device.
329 /// An outlined function may not be an entry if, e.g. the if clause always
330 /// evaluates to false.
331 virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D,
332 StringRef ParentName,
333 llvm::Function *&OutlinedFn,
334 llvm::Constant *&OutlinedFnID,
335 bool IsOffloadEntry,
336 const RegionCodeGenTy &CodeGen);
337
338 /// Returns pointer to ident_t type.
339 llvm::Type *getIdentTyPointerTy();
340
341 /// Gets thread id value for the current thread.
342 ///
343 llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc);
344
345 /// Get the function name of an outlined region.
346 std::string getOutlinedHelperName(StringRef Name) const;
347 std::string getOutlinedHelperName(CodeGenFunction &CGF) const;
348
349 /// Get the function name of a reduction function.
350 std::string getReductionFuncName(StringRef Name) const;
351
352 /// Emits \p Callee function call with arguments \p Args with location \p Loc.
353 void emitCall(CodeGenFunction &CGF, SourceLocation Loc,
354 llvm::FunctionCallee Callee,
355 ArrayRef<llvm::Value *> Args = std::nullopt) const;
356
357 /// Emits address of the word in a memory where current thread id is
358 /// stored.
359 virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc);
360
361 void setLocThreadIdInsertPt(CodeGenFunction &CGF,
362 bool AtCurrentPoint = false);
363 void clearLocThreadIdInsertPt(CodeGenFunction &CGF);
364
365 /// Check if the default location must be constant.
366 /// Default is false to support OMPT/OMPD.
367 virtual bool isDefaultLocationConstant() const { return false; }
368
369 /// Returns additional flags that can be stored in reserved_2 field of the
370 /// default location.
371 virtual unsigned getDefaultLocationReserved2Flags() const { return 0; }
372
373 /// Returns default flags for the barriers depending on the directive, for
374 /// which this barier is going to be emitted.
375 static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind);
376
377 /// Get the LLVM type for the critical name.
378 llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;}
379
380 /// Returns corresponding lock object for the specified critical region
381 /// name. If the lock object does not exist it is created, otherwise the
382 /// reference to the existing copy is returned.
383 /// \param CriticalName Name of the critical region.
384 ///
385 llvm::Value *getCriticalRegionLock(StringRef CriticalName);
386
387protected:
388 /// Map for SourceLocation and OpenMP runtime library debug locations.
389 typedef llvm::DenseMap<SourceLocation, llvm::Value *> OpenMPDebugLocMapTy;
390 OpenMPDebugLocMapTy OpenMPDebugLocMap;
391 /// The type for a microtask which gets passed to __kmpc_fork_call().
392 /// Original representation is:
393 /// typedef void (kmpc_micro)(kmp_int32 global_tid, kmp_int32 bound_tid,...);
394 llvm::FunctionType *Kmpc_MicroTy = nullptr;
395 /// Stores debug location and ThreadID for the function.
396 struct DebugLocThreadIdTy {
397 llvm::Value *DebugLoc;
398 llvm::Value *ThreadID;
399 /// Insert point for the service instructions.
400 llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr;
401 };
402 /// Map of local debug location, ThreadId and functions.
403 typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy>
404 OpenMPLocThreadIDMapTy;
405 OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap;
406 /// Map of UDRs and corresponding combiner/initializer.
407 typedef llvm::DenseMap<const OMPDeclareReductionDecl *,
408 std::pair<llvm::Function *, llvm::Function *>>
409 UDRMapTy;
410 UDRMapTy UDRMap;
411 /// Map of functions and locally defined UDRs.
412 typedef llvm::DenseMap<llvm::Function *,
413 SmallVector<const OMPDeclareReductionDecl *, 4>>
414 FunctionUDRMapTy;
415 FunctionUDRMapTy FunctionUDRMap;
416 /// Map from the user-defined mapper declaration to its corresponding
417 /// functions.
418 llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap;
419 /// Map of functions and their local user-defined mappers.
420 using FunctionUDMMapTy =
421 llvm::DenseMap<llvm::Function *,
422 SmallVector<const OMPDeclareMapperDecl *, 4>>;
423 FunctionUDMMapTy FunctionUDMMap;
424 /// Maps local variables marked as lastprivate conditional to their internal
425 /// types.
426 llvm::DenseMap<llvm::Function *,
427 llvm::DenseMap<CanonicalDeclPtr<const Decl>,
428 std::tuple<QualType, const FieldDecl *,
429 const FieldDecl *, LValue>>>
430 LastprivateConditionalToTypes;
431 /// Maps function to the position of the untied task locals stack.
432 llvm::DenseMap<llvm::Function *, unsigned> FunctionToUntiedTaskStackMap;
433 /// Type kmp_critical_name, originally defined as typedef kmp_int32
434 /// kmp_critical_name[8];
435 llvm::ArrayType *KmpCriticalNameTy;
436 /// An ordered map of auto-generated variables to their unique names.
437 /// It stores variables with the following names: 1) ".gomp_critical_user_" +
438 /// <critical_section_name> + ".var" for "omp critical" directives; 2)
439 /// <mangled_name_for_global_var> + ".cache." for cache for threadprivate
440 /// variables.
441 llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>,
442 llvm::BumpPtrAllocator> InternalVars;
443 /// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);
444 llvm::Type *KmpRoutineEntryPtrTy = nullptr;
445 QualType KmpRoutineEntryPtrQTy;
446 /// Type typedef struct kmp_task {
447 /// void * shareds; /**< pointer to block of pointers to
448 /// shared vars */
449 /// kmp_routine_entry_t routine; /**< pointer to routine to call for
450 /// executing task */
451 /// kmp_int32 part_id; /**< part id for the task */
452 /// kmp_routine_entry_t destructors; /* pointer to function to invoke
453 /// deconstructors of firstprivate C++ objects */
454 /// } kmp_task_t;
455 QualType KmpTaskTQTy;
456 /// Saved kmp_task_t for task directive.
457 QualType SavedKmpTaskTQTy;
458 /// Saved kmp_task_t for taskloop-based directive.
459 QualType SavedKmpTaskloopTQTy;
460 /// Type typedef struct kmp_depend_info {
461 /// kmp_intptr_t base_addr;
462 /// size_t len;
463 /// struct {
464 /// bool in:1;
465 /// bool out:1;
466 /// } flags;
467 /// } kmp_depend_info_t;
468 QualType KmpDependInfoTy;
469 /// Type typedef struct kmp_task_affinity_info {
470 /// kmp_intptr_t base_addr;
471 /// size_t len;
472 /// struct {
473 /// bool flag1 : 1;
474 /// bool flag2 : 1;
475 /// kmp_int32 reserved : 30;
476 /// } flags;
477 /// } kmp_task_affinity_info_t;
478 QualType KmpTaskAffinityInfoTy;
479 /// struct kmp_dim { // loop bounds info casted to kmp_int64
480 /// kmp_int64 lo; // lower
481 /// kmp_int64 up; // upper
482 /// kmp_int64 st; // stride
483 /// };
484 QualType KmpDimTy;
485
486 bool ShouldMarkAsGlobal = true;
487 /// List of the emitted declarations.
488 llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls;
489 /// List of the global variables with their addresses that should not be
490 /// emitted for the target.
491 llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables;
492
493 /// List of variables that can become declare target implicitly and, thus,
494 /// must be emitted.
495 llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;
496
497 using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>;
498 /// Stack for list of declarations in current context marked as nontemporal.
499 /// The set is the union of all current stack elements.
500 llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack;
501
502 using UntiedLocalVarsAddressesMap =
503 llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
504 std::pair<Address, Address>>;
505 llvm::SmallVector<UntiedLocalVarsAddressesMap, 4> UntiedLocalVarsStack;
506
507 /// Stack for list of addresses of declarations in current context marked as
508 /// lastprivate conditional. The set is the union of all current stack
509 /// elements.
510 llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack;
511
512 /// Flag for keeping track of weather a requires unified_shared_memory
513 /// directive is present.
514 bool HasRequiresUnifiedSharedMemory = false;
515
516 /// Atomic ordering from the omp requires directive.
517 llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
518
519 /// Flag for keeping track of weather a target region has been emitted.
520 bool HasEmittedTargetRegion = false;
521
522 /// Flag for keeping track of weather a device routine has been emitted.
523 /// Device routines are specific to the
524 bool HasEmittedDeclareTargetRegion = false;
525
526 /// Start scanning from statement \a S and emit all target regions
527 /// found along the way.
528 /// \param S Starting statement.
529 /// \param ParentName Name of the function declaration that is being scanned.
530 void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName);
531
532 /// Build type kmp_routine_entry_t (if not built yet).
533 void emitKmpRoutineEntryT(QualType KmpInt32Ty);
534
535 /// Returns pointer to kmpc_micro type.
536 llvm::Type *getKmpc_MicroPointerTy();
537
538 /// If the specified mangled name is not in the module, create and
539 /// return threadprivate cache object. This object is a pointer's worth of
540 /// storage that's reserved for use by the OpenMP runtime.
541 /// \param VD Threadprivate variable.
542 /// \return Cache variable for the specified threadprivate.
543 llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD);
544
545 /// Set of threadprivate variables with the generated initializer.
546 llvm::StringSet<> ThreadPrivateWithDefinition;
547
548 /// Set of declare target variables with the generated initializer.
549 llvm::StringSet<> DeclareTargetWithDefinition;
550
551 /// Emits initialization code for the threadprivate variables.
552 /// \param VDAddr Address of the global variable \a VD.
553 /// \param Ctor Pointer to a global init function for \a VD.
554 /// \param CopyCtor Pointer to a global copy function for \a VD.
555 /// \param Dtor Pointer to a global destructor function for \a VD.
556 /// \param Loc Location of threadprivate declaration.
557 void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr,
558 llvm::Value *Ctor, llvm::Value *CopyCtor,
559 llvm::Value *Dtor, SourceLocation Loc);
560
561 /// Emit the array initialization or deletion portion for user-defined mapper
562 /// code generation.
563 void emitUDMapperArrayInitOrDel(CodeGenFunction &MapperCGF,
564 llvm::Value *Handle, llvm::Value *BasePtr,
565 llvm::Value *Ptr, llvm::Value *Size,
566 llvm::Value *MapType, llvm::Value *MapName,
567 CharUnits ElementSize,
568 llvm::BasicBlock *ExitBB, bool IsInit);
569
570 struct TaskResultTy {
571 llvm::Value *NewTask = nullptr;
572 llvm::Function *TaskEntry = nullptr;
573 llvm::Value *NewTaskNewTaskTTy = nullptr;
574 LValue TDBase;
575 const RecordDecl *KmpTaskTQTyRD = nullptr;
576 llvm::Value *TaskDupFn = nullptr;
577 };
578 /// Emit task region for the task directive. The task region is emitted in
579 /// several steps:
580 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
581 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
582 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
583 /// function:
584 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
585 /// TaskFunction(gtid, tt->part_id, tt->shareds);
586 /// return 0;
587 /// }
588 /// 2. Copy a list of shared variables to field shareds of the resulting
589 /// structure kmp_task_t returned by the previous call (if any).
590 /// 3. Copy a pointer to destructions function to field destructions of the
591 /// resulting structure kmp_task_t.
592 /// \param D Current task directive.
593 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
594 /// /*part_id*/, captured_struct */*__context*/);
595 /// \param SharedsTy A type which contains references the shared variables.
596 /// \param Shareds Context with the list of shared variables from the \p
597 /// TaskFunction.
598 /// \param Data Additional data for task generation like tiednsee, final
599 /// state, list of privates etc.
600 TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
601 const OMPExecutableDirective &D,
602 llvm::Function *TaskFunction, QualType SharedsTy,
603 Address Shareds, const OMPTaskDataTy &Data);
604
605 /// Emit update for lastprivate conditional data.
606 void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,
607 StringRef UniqueDeclName, LValue LVal,
608 SourceLocation Loc);
609
610 /// Returns the number of the elements and the address of the depobj
611 /// dependency array.
612 /// \return Number of elements in depobj array and the pointer to the array of
613 /// dependencies.
614 std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF,
615 LValue DepobjLVal,
616 SourceLocation Loc);
617
618 SmallVector<llvm::Value *, 4>
619 emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
620 const OMPTaskDataTy::DependData &Data);
621
622 void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
623 LValue PosLVal, const OMPTaskDataTy::DependData &Data,
624 Address DependenciesArray);
625
626public:
627 explicit CGOpenMPRuntime(CodeGenModule &CGM);
628 virtual ~CGOpenMPRuntime() {}
629 virtual void clear();
630
631 /// Emits object of ident_t type with info for source location.
632 /// \param Flags Flags for OpenMP location.
633 /// \param EmitLoc emit source location with debug-info is off.
634 ///
635 llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc,
636 unsigned Flags = 0, bool EmitLoc = false);
637
638 /// Emit the number of teams for a target directive. Inspect the num_teams
639 /// clause associated with a teams construct combined or closely nested
640 /// with the target directive.
641 ///
642 /// Emit a team of size one for directives such as 'target parallel' that
643 /// have no associated teams construct.
644 ///
645 /// Otherwise, return nullptr.
646 const Expr *getNumTeamsExprForTargetDirective(CodeGenFunction &CGF,
647 const OMPExecutableDirective &D,
648 int32_t &MinTeamsVal,
649 int32_t &MaxTeamsVal);
650 llvm::Value *emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
651 const OMPExecutableDirective &D);
652
653 /// Check for a number of threads upper bound constant value (stored in \p
654 /// UpperBound), or expression (returned). If the value is conditional (via an
655 /// if-clause), store the condition in \p CondExpr. Similarly, a potential
656 /// thread limit expression is stored in \p ThreadLimitExpr. If \p
657 /// UpperBoundOnly is true, no expression evaluation is perfomed.
658 const Expr *getNumThreadsExprForTargetDirective(
659 CodeGenFunction &CGF, const OMPExecutableDirective &D,
660 int32_t &UpperBound, bool UpperBoundOnly,
661 llvm::Value **CondExpr = nullptr, const Expr **ThreadLimitExpr = nullptr);
662
663 /// Emit an expression that denotes the number of threads a target region
664 /// shall use. Will generate "i32 0" to allow the runtime to choose.
665 llvm::Value *
666 emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
667 const OMPExecutableDirective &D);
668
669 /// Return the trip count of loops associated with constructs / 'target teams
670 /// distribute' and 'teams distribute parallel for'. \param SizeEmitter Emits
671 /// the int64 value for the number of iterations of the associated loop.
672 llvm::Value *emitTargetNumIterationsCall(
673 CodeGenFunction &CGF, const OMPExecutableDirective &D,
674 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
675 const OMPLoopDirective &D)>
676 SizeEmitter);
677
678 /// Returns true if the current target is a GPU.
679 virtual bool isGPU() const { return false; }
680
681 /// Check if the variable length declaration is delayed:
682 virtual bool isDelayedVariableLengthDecl(CodeGenFunction &CGF,
683 const VarDecl *VD) const {
684 return false;
685 };
686
687 /// Get call to __kmpc_alloc_shared
688 virtual std::pair<llvm::Value *, llvm::Value *>
689 getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) {
690 llvm_unreachable("not implemented");
691 }
692
693 /// Get call to __kmpc_free_shared
694 virtual void getKmpcFreeShared(
695 CodeGenFunction &CGF,
696 const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) {
697 llvm_unreachable("not implemented");
698 }
699
700 /// Emits code for OpenMP 'if' clause using specified \a CodeGen
701 /// function. Here is the logic:
702 /// if (Cond) {
703 /// ThenGen();
704 /// } else {
705 /// ElseGen();
706 /// }
707 void emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
708 const RegionCodeGenTy &ThenGen,
709 const RegionCodeGenTy &ElseGen);
710
711 /// Checks if the \p Body is the \a CompoundStmt and returns its child
712 /// statement iff there is only one that is not evaluatable at the compile
713 /// time.
714 static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body);
715
716 /// Get the platform-specific name separator.
717 std::string getName(ArrayRef<StringRef> Parts) const;
718
719 /// Emit code for the specified user defined reduction construct.
720 virtual void emitUserDefinedReduction(CodeGenFunction *CGF,
721 const OMPDeclareReductionDecl *D);
722 /// Get combiner/initializer for the specified user-defined reduction, if any.
723 virtual std::pair<llvm::Function *, llvm::Function *>
724 getUserDefinedReduction(const OMPDeclareReductionDecl *D);
725
726 /// Emit the function for the user defined mapper construct.
727 void emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
728 CodeGenFunction *CGF = nullptr);
729 /// Get the function for the specified user-defined mapper. If it does not
730 /// exist, create one.
731 llvm::Function *
732 getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D);
733
734 /// Emits outlined function for the specified OpenMP parallel directive
735 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
736 /// kmp_int32 BoundID, struct context_vars*).
737 /// \param CGF Reference to current CodeGenFunction.
738 /// \param D OpenMP directive.
739 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
740 /// \param InnermostKind Kind of innermost directive (for simple directives it
741 /// is a directive itself, for combined - its innermost directive).
742 /// \param CodeGen Code generation sequence for the \a D directive.
743 virtual llvm::Function *emitParallelOutlinedFunction(
744 CodeGenFunction &CGF, const OMPExecutableDirective &D,
745 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
746 const RegionCodeGenTy &CodeGen);
747
748 /// Emits outlined function for the specified OpenMP teams directive
749 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
750 /// kmp_int32 BoundID, struct context_vars*).
751 /// \param CGF Reference to current CodeGenFunction.
752 /// \param D OpenMP directive.
753 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
754 /// \param InnermostKind Kind of innermost directive (for simple directives it
755 /// is a directive itself, for combined - its innermost directive).
756 /// \param CodeGen Code generation sequence for the \a D directive.
757 virtual llvm::Function *emitTeamsOutlinedFunction(
758 CodeGenFunction &CGF, const OMPExecutableDirective &D,
759 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
760 const RegionCodeGenTy &CodeGen);
761
762 /// Emits outlined function for the OpenMP task directive \a D. This
763 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
764 /// TaskT).
765 /// \param D OpenMP directive.
766 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
767 /// \param PartIDVar Variable for partition id in the current OpenMP untied
768 /// task region.
769 /// \param TaskTVar Variable for task_t argument.
770 /// \param InnermostKind Kind of innermost directive (for simple directives it
771 /// is a directive itself, for combined - its innermost directive).
772 /// \param CodeGen Code generation sequence for the \a D directive.
773 /// \param Tied true if task is generated for tied task, false otherwise.
774 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
775 /// tasks.
776 ///
777 virtual llvm::Function *emitTaskOutlinedFunction(
778 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
779 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
780 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
781 bool Tied, unsigned &NumberOfParts);
782
783 /// Cleans up references to the objects in finished function.
784 ///
785 virtual void functionFinished(CodeGenFunction &CGF);
786
787 /// Emits code for parallel or serial call of the \a OutlinedFn with
788 /// variables captured in a record which address is stored in \a
789 /// CapturedStruct.
790 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
791 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
792 /// \param CapturedVars A pointer to the record with the references to
793 /// variables used in \a OutlinedFn function.
794 /// \param IfCond Condition in the associated 'if' clause, if it was
795 /// specified, nullptr otherwise.
796 /// \param NumThreads The value corresponding to the num_threads clause, if
797 /// any, or nullptr.
798 ///
799 virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
800 llvm::Function *OutlinedFn,
801 ArrayRef<llvm::Value *> CapturedVars,
802 const Expr *IfCond, llvm::Value *NumThreads);
803
804 /// Emits a critical region.
805 /// \param CriticalName Name of the critical region.
806 /// \param CriticalOpGen Generator for the statement associated with the given
807 /// critical region.
808 /// \param Hint Value of the 'hint' clause (optional).
809 virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
810 const RegionCodeGenTy &CriticalOpGen,
811 SourceLocation Loc,
812 const Expr *Hint = nullptr);
813
814 /// Emits a master region.
815 /// \param MasterOpGen Generator for the statement associated with the given
816 /// master region.
817 virtual void emitMasterRegion(CodeGenFunction &CGF,
818 const RegionCodeGenTy &MasterOpGen,
819 SourceLocation Loc);
820
821 /// Emits a masked region.
822 /// \param MaskedOpGen Generator for the statement associated with the given
823 /// masked region.
824 virtual void emitMaskedRegion(CodeGenFunction &CGF,
825 const RegionCodeGenTy &MaskedOpGen,
826 SourceLocation Loc,
827 const Expr *Filter = nullptr);
828
829 /// Emits code for a taskyield directive.
830 virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);
831
832 /// Emit __kmpc_error call for error directive
833 /// extern void __kmpc_error(ident_t *loc, int severity, const char *message);
834 virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME,
835 bool IsFatal);
836
837 /// Emit a taskgroup region.
838 /// \param TaskgroupOpGen Generator for the statement associated with the
839 /// given taskgroup region.
840 virtual void emitTaskgroupRegion(CodeGenFunction &CGF,
841 const RegionCodeGenTy &TaskgroupOpGen,
842 SourceLocation Loc);
843
844 /// Emits a single region.
845 /// \param SingleOpGen Generator for the statement associated with the given
846 /// single region.
847 virtual void emitSingleRegion(CodeGenFunction &CGF,
848 const RegionCodeGenTy &SingleOpGen,
849 SourceLocation Loc,
850 ArrayRef<const Expr *> CopyprivateVars,
851 ArrayRef<const Expr *> DestExprs,
852 ArrayRef<const Expr *> SrcExprs,
853 ArrayRef<const Expr *> AssignmentOps);
854
855 /// Emit an ordered region.
856 /// \param OrderedOpGen Generator for the statement associated with the given
857 /// ordered region.
858 virtual void emitOrderedRegion(CodeGenFunction &CGF,
859 const RegionCodeGenTy &OrderedOpGen,
860 SourceLocation Loc, bool IsThreads);
861
862 /// Emit an implicit/explicit barrier for OpenMP threads.
863 /// \param Kind Directive for which this implicit barrier call must be
864 /// generated. Must be OMPD_barrier for explicit barrier generation.
865 /// \param EmitChecks true if need to emit checks for cancellation barriers.
866 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
867 /// runtime class decides which one to emit (simple or with cancellation
868 /// checks).
869 ///
870 virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
871 OpenMPDirectiveKind Kind,
872 bool EmitChecks = true,
873 bool ForceSimpleCall = false);
874
875 /// Check if the specified \a ScheduleKind is static non-chunked.
876 /// This kind of worksharing directive is emitted without outer loop.
877 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
878 /// \param Chunked True if chunk is specified in the clause.
879 ///
880 virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
881 bool Chunked) const;
882
883 /// Check if the specified \a ScheduleKind is static non-chunked.
884 /// This kind of distribute directive is emitted without outer loop.
885 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
886 /// \param Chunked True if chunk is specified in the clause.
887 ///
888 virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind,
889 bool Chunked) const;
890
891 /// Check if the specified \a ScheduleKind is static chunked.
892 /// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
893 /// \param Chunked True if chunk is specified in the clause.
894 ///
895 virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
896 bool Chunked) const;
897
898 /// Check if the specified \a ScheduleKind is static non-chunked.
899 /// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
900 /// \param Chunked True if chunk is specified in the clause.
901 ///
902 virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind,
903 bool Chunked) const;
904
905 /// Check if the specified \a ScheduleKind is dynamic.
906 /// This kind of worksharing directive is emitted without outer loop.
907 /// \param ScheduleKind Schedule Kind specified in the 'schedule' clause.
908 ///
909 virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const;
910
911 /// struct with the values to be passed to the dispatch runtime function
912 struct DispatchRTInput {
913 /// Loop lower bound
914 llvm::Value *LB = nullptr;
915 /// Loop upper bound
916 llvm::Value *UB = nullptr;
917 /// Chunk size specified using 'schedule' clause (nullptr if chunk
918 /// was not specified)
919 llvm::Value *Chunk = nullptr;
920 DispatchRTInput() = default;
921 DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk)
922 : LB(LB), UB(UB), Chunk(Chunk) {}
923 };
924
925 /// Call the appropriate runtime routine to initialize it before start
926 /// of loop.
927
928 /// This is used for non static scheduled types and when the ordered
929 /// clause is present on the loop construct.
930 /// Depending on the loop schedule, it is necessary to call some runtime
931 /// routine before start of the OpenMP loop to get the loop upper / lower
932 /// bounds \a LB and \a UB and stride \a ST.
933 ///
934 /// \param CGF Reference to current CodeGenFunction.
935 /// \param Loc Clang source location.
936 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
937 /// \param IVSize Size of the iteration variable in bits.
938 /// \param IVSigned Sign of the iteration variable.
939 /// \param Ordered true if loop is ordered, false otherwise.
940 /// \param DispatchValues struct containing llvm values for lower bound, upper
941 /// bound, and chunk expression.
942 /// For the default (nullptr) value, the chunk 1 will be used.
943 ///
944 virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
945 const OpenMPScheduleTy &ScheduleKind,
946 unsigned IVSize, bool IVSigned, bool Ordered,
947 const DispatchRTInput &DispatchValues);
948
949 /// This is used for non static scheduled types and when the ordered
950 /// clause is present on the loop construct.
951 ///
952 /// \param CGF Reference to current CodeGenFunction.
953 /// \param Loc Clang source location.
954 ///
955 virtual void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc);
956
957 /// Struct with the values to be passed to the static runtime function
958 struct StaticRTInput {
959 /// Size of the iteration variable in bits.
960 unsigned IVSize = 0;
961 /// Sign of the iteration variable.
962 bool IVSigned = false;
963 /// true if loop is ordered, false otherwise.
964 bool Ordered = false;
965 /// Address of the output variable in which the flag of the last iteration
966 /// is returned.
967 Address IL = Address::invalid();
968 /// Address of the output variable in which the lower iteration number is
969 /// returned.
970 Address LB = Address::invalid();
971 /// Address of the output variable in which the upper iteration number is
972 /// returned.
973 Address UB = Address::invalid();
974 /// Address of the output variable in which the stride value is returned
975 /// necessary to generated the static_chunked scheduled loop.
976 Address ST = Address::invalid();
977 /// Value of the chunk for the static_chunked scheduled loop. For the
978 /// default (nullptr) value, the chunk 1 will be used.
979 llvm::Value *Chunk = nullptr;
980 StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL,
981 Address LB, Address UB, Address ST,
982 llvm::Value *Chunk = nullptr)
983 : IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB),
984 UB(UB), ST(ST), Chunk(Chunk) {}
985 };
986 /// Call the appropriate runtime routine to initialize it before start
987 /// of loop.
988 ///
989 /// This is used only in case of static schedule, when the user did not
990 /// specify a ordered clause on the loop construct.
991 /// Depending on the loop schedule, it is necessary to call some runtime
992 /// routine before start of the OpenMP loop to get the loop upper / lower
993 /// bounds LB and UB and stride ST.
994 ///
995 /// \param CGF Reference to current CodeGenFunction.
996 /// \param Loc Clang source location.
997 /// \param DKind Kind of the directive.
998 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
999 /// \param Values Input arguments for the construct.
1000 ///
1001 virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1002 OpenMPDirectiveKind DKind,
1003 const OpenMPScheduleTy &ScheduleKind,
1004 const StaticRTInput &Values);
1005
1006 ///
1007 /// \param CGF Reference to current CodeGenFunction.
1008 /// \param Loc Clang source location.
1009 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1010 /// \param Values Input arguments for the construct.
1011 ///
1012 virtual void emitDistributeStaticInit(CodeGenFunction &CGF,
1013 SourceLocation Loc,
1014 OpenMPDistScheduleClauseKind SchedKind,
1015 const StaticRTInput &Values);
1016
1017 /// Call the appropriate runtime routine to notify that we finished
1018 /// iteration of the ordered loop with the dynamic scheduling.
1019 ///
1020 /// \param CGF Reference to current CodeGenFunction.
1021 /// \param Loc Clang source location.
1022 /// \param IVSize Size of the iteration variable in bits.
1023 /// \param IVSigned Sign of the iteration variable.
1024 ///
1025 virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF,
1026 SourceLocation Loc, unsigned IVSize,
1027 bool IVSigned);
1028
1029 /// Call the appropriate runtime routine to notify that we finished
1030 /// all the work with current loop.
1031 ///
1032 /// \param CGF Reference to current CodeGenFunction.
1033 /// \param Loc Clang source location.
1034 /// \param DKind Kind of the directive for which the static finish is emitted.
1035 ///
1036 virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1037 OpenMPDirectiveKind DKind);
1038
1039 /// Call __kmpc_dispatch_next(
1040 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1041 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1042 /// kmp_int[32|64] *p_stride);
1043 /// \param IVSize Size of the iteration variable in bits.
1044 /// \param IVSigned Sign of the iteration variable.
1045 /// \param IL Address of the output variable in which the flag of the
1046 /// last iteration is returned.
1047 /// \param LB Address of the output variable in which the lower iteration
1048 /// number is returned.
1049 /// \param UB Address of the output variable in which the upper iteration
1050 /// number is returned.
1051 /// \param ST Address of the output variable in which the stride value is
1052 /// returned.
1053 virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1054 unsigned IVSize, bool IVSigned,
1055 Address IL, Address LB,
1056 Address UB, Address ST);
1057
1058 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1059 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1060 /// clause.
1061 /// \param NumThreads An integer value of threads.
1062 virtual void emitNumThreadsClause(CodeGenFunction &CGF,
1063 llvm::Value *NumThreads,
1064 SourceLocation Loc);
1065
1066 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1067 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1068 virtual void emitProcBindClause(CodeGenFunction &CGF,
1069 llvm::omp::ProcBindKind ProcBind,
1070 SourceLocation Loc);
1071
1072 /// Returns address of the threadprivate variable for the current
1073 /// thread.
1074 /// \param VD Threadprivate variable.
1075 /// \param VDAddr Address of the global variable \a VD.
1076 /// \param Loc Location of the reference to threadprivate var.
1077 /// \return Address of the threadprivate variable for the current thread.
1078 virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1079 const VarDecl *VD, Address VDAddr,
1080 SourceLocation Loc);
1081
1082 /// Returns the address of the variable marked as declare target with link
1083 /// clause OR as declare target with to clause and unified memory.
1084 virtual ConstantAddress getAddrOfDeclareTargetVar(const VarDecl *VD);
1085
1086 /// Emit a code for initialization of threadprivate variable. It emits
1087 /// a call to runtime library which adds initial value to the newly created
1088 /// threadprivate variable (if it is not constant) and registers destructor
1089 /// for the variable (if any).
1090 /// \param VD Threadprivate variable.
1091 /// \param VDAddr Address of the global variable \a VD.
1092 /// \param Loc Location of threadprivate declaration.
1093 /// \param PerformInit true if initialization expression is not constant.
1094 virtual llvm::Function *
1095 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1096 SourceLocation Loc, bool PerformInit,
1097 CodeGenFunction *CGF = nullptr);
1098
1099 /// Emit code for handling declare target functions in the runtime.
1100 /// \param FD Declare target function.
1101 /// \param Addr Address of the global \a FD.
1102 /// \param PerformInit true if initialization expression is not constant.
1103 virtual void emitDeclareTargetFunction(const FunctionDecl *FD,
1104 llvm::GlobalValue *GV);
1105
1106 /// Creates artificial threadprivate variable with name \p Name and type \p
1107 /// VarType.
1108 /// \param VarType Type of the artificial threadprivate variable.
1109 /// \param Name Name of the artificial threadprivate variable.
1110 virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1111 QualType VarType,
1112 StringRef Name);
1113
1114 /// Emit flush of the variables specified in 'omp flush' directive.
1115 /// \param Vars List of variables to flush.
1116 virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1117 SourceLocation Loc, llvm::AtomicOrdering AO);
1118
1119 /// Emit task region for the task directive. The task region is
1120 /// emitted in several steps:
1121 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1122 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1123 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1124 /// function:
1125 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1126 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1127 /// return 0;
1128 /// }
1129 /// 2. Copy a list of shared variables to field shareds of the resulting
1130 /// structure kmp_task_t returned by the previous call (if any).
1131 /// 3. Copy a pointer to destructions function to field destructions of the
1132 /// resulting structure kmp_task_t.
1133 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1134 /// kmp_task_t *new_task), where new_task is a resulting structure from
1135 /// previous items.
1136 /// \param D Current task directive.
1137 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1138 /// /*part_id*/, captured_struct */*__context*/);
1139 /// \param SharedsTy A type which contains references the shared variables.
1140 /// \param Shareds Context with the list of shared variables from the \p
1141 /// TaskFunction.
1142 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1143 /// otherwise.
1144 /// \param Data Additional data for task generation like tiednsee, final
1145 /// state, list of privates etc.
1146 virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1147 const OMPExecutableDirective &D,
1148 llvm::Function *TaskFunction, QualType SharedsTy,
1149 Address Shareds, const Expr *IfCond,
1150 const OMPTaskDataTy &Data);
1151
1152 /// Emit task region for the taskloop directive. The taskloop region is
1153 /// emitted in several steps:
1154 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1155 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1156 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1157 /// function:
1158 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1159 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1160 /// return 0;
1161 /// }
1162 /// 2. Copy a list of shared variables to field shareds of the resulting
1163 /// structure kmp_task_t returned by the previous call (if any).
1164 /// 3. Copy a pointer to destructions function to field destructions of the
1165 /// resulting structure kmp_task_t.
1166 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
1167 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
1168 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
1169 /// is a resulting structure from
1170 /// previous items.
1171 /// \param D Current task directive.
1172 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1173 /// /*part_id*/, captured_struct */*__context*/);
1174 /// \param SharedsTy A type which contains references the shared variables.
1175 /// \param Shareds Context with the list of shared variables from the \p
1176 /// TaskFunction.
1177 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1178 /// otherwise.
1179 /// \param Data Additional data for task generation like tiednsee, final
1180 /// state, list of privates etc.
1181 virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
1182 const OMPLoopDirective &D,
1183 llvm::Function *TaskFunction,
1184 QualType SharedsTy, Address Shareds,
1185 const Expr *IfCond, const OMPTaskDataTy &Data);
1186
1187 /// Emit code for the directive that does not require outlining.
1188 ///
1189 /// \param InnermostKind Kind of innermost directive (for simple directives it
1190 /// is a directive itself, for combined - its innermost directive).
1191 /// \param CodeGen Code generation sequence for the \a D directive.
1192 /// \param HasCancel true if region has inner cancel directive, false
1193 /// otherwise.
1194 virtual void emitInlinedDirective(CodeGenFunction &CGF,
1195 OpenMPDirectiveKind InnermostKind,
1196 const RegionCodeGenTy &CodeGen,
1197 bool HasCancel = false);
1198
1199 /// Emits reduction function.
1200 /// \param ReducerName Name of the function calling the reduction.
1201 /// \param ArgsElemType Array type containing pointers to reduction variables.
1202 /// \param Privates List of private copies for original reduction arguments.
1203 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1204 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1205 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1206 /// or 'operator binop(LHS, RHS)'.
1207 llvm::Function *emitReductionFunction(
1208 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
1209 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
1210 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps);
1211
1212 /// Emits single reduction combiner
1213 void emitSingleReductionCombiner(CodeGenFunction &CGF,
1214 const Expr *ReductionOp,
1215 const Expr *PrivateRef,
1216 const DeclRefExpr *LHS,
1217 const DeclRefExpr *RHS);
1218
1219 struct ReductionOptionsTy {
1220 bool WithNowait;
1221 bool SimpleReduction;
1222 OpenMPDirectiveKind ReductionKind;
1223 };
1224 /// Emit a code for reduction clause. Next code should be emitted for
1225 /// reduction:
1226 /// \code
1227 ///
1228 /// static kmp_critical_name lock = { 0 };
1229 ///
1230 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
1231 /// ...
1232 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
1233 /// ...
1234 /// }
1235 ///
1236 /// ...
1237 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
1238 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
1239 /// RedList, reduce_func, &<lock>)) {
1240 /// case 1:
1241 /// ...
1242 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
1243 /// ...
1244 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
1245 /// break;
1246 /// case 2:
1247 /// ...
1248 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
1249 /// ...
1250 /// break;
1251 /// default:;
1252 /// }
1253 /// \endcode
1254 ///
1255 /// \param Privates List of private copies for original reduction arguments.
1256 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1257 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1258 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1259 /// or 'operator binop(LHS, RHS)'.
1260 /// \param Options List of options for reduction codegen:
1261 /// WithNowait true if parent directive has also nowait clause, false
1262 /// otherwise.
1263 /// SimpleReduction Emit reduction operation only. Used for omp simd
1264 /// directive on the host.
1265 /// ReductionKind The kind of reduction to perform.
1266 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
1267 ArrayRef<const Expr *> Privates,
1268 ArrayRef<const Expr *> LHSExprs,
1269 ArrayRef<const Expr *> RHSExprs,
1270 ArrayRef<const Expr *> ReductionOps,
1271 ReductionOptionsTy Options);
1272
1273 /// Emit a code for initialization of task reduction clause. Next code
1274 /// should be emitted for reduction:
1275 /// \code
1276 ///
1277 /// _taskred_item_t red_data[n];
1278 /// ...
1279 /// red_data[i].shar = &shareds[i];
1280 /// red_data[i].orig = &origs[i];
1281 /// red_data[i].size = sizeof(origs[i]);
1282 /// red_data[i].f_init = (void*)RedInit<i>;
1283 /// red_data[i].f_fini = (void*)RedDest<i>;
1284 /// red_data[i].f_comb = (void*)RedOp<i>;
1285 /// red_data[i].flags = <Flag_i>;
1286 /// ...
1287 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
1288 /// \endcode
1289 /// For reduction clause with task modifier it emits the next call:
1290 /// \code
1291 ///
1292 /// _taskred_item_t red_data[n];
1293 /// ...
1294 /// red_data[i].shar = &shareds[i];
1295 /// red_data[i].orig = &origs[i];
1296 /// red_data[i].size = sizeof(origs[i]);
1297 /// red_data[i].f_init = (void*)RedInit<i>;
1298 /// red_data[i].f_fini = (void*)RedDest<i>;
1299 /// red_data[i].f_comb = (void*)RedOp<i>;
1300 /// red_data[i].flags = <Flag_i>;
1301 /// ...
1302 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
1303 /// red_data);
1304 /// \endcode
1305 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
1306 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
1307 /// \param Data Additional data for task generation like tiedness, final
1308 /// state, list of privates, reductions etc.
1309 virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF,
1310 SourceLocation Loc,
1311 ArrayRef<const Expr *> LHSExprs,
1312 ArrayRef<const Expr *> RHSExprs,
1313 const OMPTaskDataTy &Data);
1314
1315 /// Emits the following code for reduction clause with task modifier:
1316 /// \code
1317 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
1318 /// \endcode
1319 virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
1320 bool IsWorksharingReduction);
1321
1322 /// Required to resolve existing problems in the runtime. Emits threadprivate
1323 /// variables to store the size of the VLAs/array sections for
1324 /// initializer/combiner/finalizer functions.
1325 /// \param RCG Allows to reuse an existing data for the reductions.
1326 /// \param N Reduction item for which fixups must be emitted.
1327 virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
1328 ReductionCodeGen &RCG, unsigned N);
1329
1330 /// Get the address of `void *` type of the privatue copy of the reduction
1331 /// item specified by the \p SharedLVal.
1332 /// \param ReductionsPtr Pointer to the reduction data returned by the
1333 /// emitTaskReductionInit function.
1334 /// \param SharedLVal Address of the original reduction item.
1335 virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
1336 llvm::Value *ReductionsPtr,
1337 LValue SharedLVal);
1338
1339 /// Emit code for 'taskwait' directive.
1340 virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
1341 const OMPTaskDataTy &Data);
1342
1343 /// Emit code for 'cancellation point' construct.
1344 /// \param CancelRegion Region kind for which the cancellation point must be
1345 /// emitted.
1346 ///
1347 virtual void emitCancellationPointCall(CodeGenFunction &CGF,
1348 SourceLocation Loc,
1349 OpenMPDirectiveKind CancelRegion);
1350
1351 /// Emit code for 'cancel' construct.
1352 /// \param IfCond Condition in the associated 'if' clause, if it was
1353 /// specified, nullptr otherwise.
1354 /// \param CancelRegion Region kind for which the cancel must be emitted.
1355 ///
1356 virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
1357 const Expr *IfCond,
1358 OpenMPDirectiveKind CancelRegion);
1359
1360 /// Emit outilined function for 'target' directive.
1361 /// \param D Directive to emit.
1362 /// \param ParentName Name of the function that encloses the target region.
1363 /// \param OutlinedFn Outlined function value to be defined by this call.
1364 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
1365 /// \param IsOffloadEntry True if the outlined function is an offload entry.
1366 /// \param CodeGen Code generation sequence for the \a D directive.
1367 /// An outlined function may not be an entry if, e.g. the if clause always
1368 /// evaluates to false.
1369 virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
1370 StringRef ParentName,
1371 llvm::Function *&OutlinedFn,
1372 llvm::Constant *&OutlinedFnID,
1373 bool IsOffloadEntry,
1374 const RegionCodeGenTy &CodeGen);
1375
1376 /// Emit the target offloading code associated with \a D. The emitted
1377 /// code attempts offloading the execution to the device, an the event of
1378 /// a failure it executes the host version outlined in \a OutlinedFn.
1379 /// \param D Directive to emit.
1380 /// \param OutlinedFn Host version of the code to be offloaded.
1381 /// \param OutlinedFnID ID of host version of the code to be offloaded.
1382 /// \param IfCond Expression evaluated in if clause associated with the target
1383 /// directive, or null if no if clause is used.
1384 /// \param Device Expression evaluated in device clause associated with the
1385 /// target directive, or null if no device clause is used and device modifier.
1386 /// \param SizeEmitter Callback to emit number of iterations for loop-based
1387 /// directives.
1388 virtual void emitTargetCall(
1389 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1390 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
1391 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
1392 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
1393 const OMPLoopDirective &D)>
1394 SizeEmitter);
1395
1396 /// Emit the target regions enclosed in \a GD function definition or
1397 /// the function itself in case it is a valid device function. Returns true if
1398 /// \a GD was dealt with successfully.
1399 /// \param GD Function to scan.
1400 virtual bool emitTargetFunctions(GlobalDecl GD);
1401
1402 /// Emit the global variable if it is a valid device global variable.
1403 /// Returns true if \a GD was dealt with successfully.
1404 /// \param GD Variable declaration to emit.
1405 virtual bool emitTargetGlobalVariable(GlobalDecl GD);
1406
1407 /// Checks if the provided global decl \a GD is a declare target variable and
1408 /// registers it when emitting code for the host.
1409 virtual void registerTargetGlobalVariable(const VarDecl *VD,
1410 llvm::Constant *Addr);
1411
1412 /// Emit the global \a GD if it is meaningful for the target. Returns
1413 /// if it was emitted successfully.
1414 /// \param GD Global to scan.
1415 virtual bool emitTargetGlobal(GlobalDecl GD);
1416
1417 /// Creates all the offload entries in the current compilation unit
1418 /// along with the associated metadata.
1419 void createOffloadEntriesAndInfoMetadata();
1420
1421 /// Emits code for teams call of the \a OutlinedFn with
1422 /// variables captured in a record which address is stored in \a
1423 /// CapturedStruct.
1424 /// \param OutlinedFn Outlined function to be run by team masters. Type of
1425 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1426 /// \param CapturedVars A pointer to the record with the references to
1427 /// variables used in \a OutlinedFn function.
1428 ///
1429 virtual void emitTeamsCall(CodeGenFunction &CGF,
1430 const OMPExecutableDirective &D,
1431 SourceLocation Loc, llvm::Function *OutlinedFn,
1432 ArrayRef<llvm::Value *> CapturedVars);
1433
1434 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
1435 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
1436 /// for num_teams clause.
1437 /// \param NumTeams An integer expression of teams.
1438 /// \param ThreadLimit An integer expression of threads.
1439 virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
1440 const Expr *ThreadLimit, SourceLocation Loc);
1441
1442 /// Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32
1443 /// global_tid, kmp_int32 thread_limit) to generate code for
1444 /// thread_limit clause on target directive
1445 /// \param ThreadLimit An integer expression of threads.
1446 virtual void emitThreadLimitClause(CodeGenFunction &CGF,
1447 const Expr *ThreadLimit,
1448 SourceLocation Loc);
1449
1450 /// Struct that keeps all the relevant information that should be kept
1451 /// throughout a 'target data' region.
1452 class TargetDataInfo : public llvm::OpenMPIRBuilder::TargetDataInfo {
1453 public:
1454 explicit TargetDataInfo() : llvm::OpenMPIRBuilder::TargetDataInfo() {}
1455 explicit TargetDataInfo(bool RequiresDevicePointerInfo,
1456 bool SeparateBeginEndCalls)
1457 : llvm::OpenMPIRBuilder::TargetDataInfo(RequiresDevicePointerInfo,
1458 SeparateBeginEndCalls) {}
1459 /// Map between the a declaration of a capture and the corresponding new
1460 /// llvm address where the runtime returns the device pointers.
1461 llvm::DenseMap<const ValueDecl *, llvm::Value *> CaptureDeviceAddrMap;
1462 };
1463
1464 /// Emit the target data mapping code associated with \a D.
1465 /// \param D Directive to emit.
1466 /// \param IfCond Expression evaluated in if clause associated with the
1467 /// target directive, or null if no device clause is used.
1468 /// \param Device Expression evaluated in device clause associated with the
1469 /// target directive, or null if no device clause is used.
1470 /// \param Info A record used to store information that needs to be preserved
1471 /// until the region is closed.
1472 virtual void emitTargetDataCalls(CodeGenFunction &CGF,
1473 const OMPExecutableDirective &D,
1474 const Expr *IfCond, const Expr *Device,
1475 const RegionCodeGenTy &CodeGen,
1476 CGOpenMPRuntime::TargetDataInfo &Info);
1477
1478 /// Emit the data mapping/movement code associated with the directive
1479 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
1480 /// \param D Directive to emit.
1481 /// \param IfCond Expression evaluated in if clause associated with the target
1482 /// directive, or null if no if clause is used.
1483 /// \param Device Expression evaluated in device clause associated with the
1484 /// target directive, or null if no device clause is used.
1485 virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
1486 const OMPExecutableDirective &D,
1487 const Expr *IfCond,
1488 const Expr *Device);
1489
1490 /// Marks function \a Fn with properly mangled versions of vector functions.
1491 /// \param FD Function marked as 'declare simd'.
1492 /// \param Fn LLVM function that must be marked with 'declare simd'
1493 /// attributes.
1494 virtual void emitDeclareSimdFunction(const FunctionDecl *FD,
1495 llvm::Function *Fn);
1496
1497 /// Emit initialization for doacross loop nesting support.
1498 /// \param D Loop-based construct used in doacross nesting construct.
1499 virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
1500 ArrayRef<Expr *> NumIterations);
1501
1502 /// Emit code for doacross ordered directive with 'depend' clause.
1503 /// \param C 'depend' clause with 'sink|source' dependency kind.
1504 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1505 const OMPDependClause *C);
1506
1507 /// Emit code for doacross ordered directive with 'doacross' clause.
1508 /// \param C 'doacross' clause with 'sink|source' dependence type.
1509 virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1510 const OMPDoacrossClause *C);
1511
1512 /// Translates the native parameter of outlined function if this is required
1513 /// for target.
1514 /// \param FD Field decl from captured record for the parameter.
1515 /// \param NativeParam Parameter itself.
1516 virtual const VarDecl *translateParameter(const FieldDecl *FD,
1517 const VarDecl *NativeParam) const {
1518 return NativeParam;
1519 }
1520
1521 /// Gets the address of the native argument basing on the address of the
1522 /// target-specific parameter.
1523 /// \param NativeParam Parameter itself.
1524 /// \param TargetParam Corresponding target-specific parameter.
1525 virtual Address getParameterAddress(CodeGenFunction &CGF,
1526 const VarDecl *NativeParam,
1527 const VarDecl *TargetParam) const;
1528
1529 /// Choose default schedule type and chunk value for the
1530 /// dist_schedule clause.
1531 virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
1532 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
1533 llvm::Value *&Chunk) const {}
1534
1535 /// Choose default schedule type and chunk value for the
1536 /// schedule clause.
1537 virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
1538 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
1539 const Expr *&ChunkExpr) const;
1540
1541 /// Emits call of the outlined function with the provided arguments,
1542 /// translating these arguments to correct target-specific arguments.
1543 virtual void
1544 emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
1545 llvm::FunctionCallee OutlinedFn,
1546 ArrayRef<llvm::Value *> Args = std::nullopt) const;
1547
1548 /// Emits OpenMP-specific function prolog.
1549 /// Required for device constructs.
1550 virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D);
1551
1552 /// Gets the OpenMP-specific address of the local variable.
1553 virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1554 const VarDecl *VD);
1555
1556 /// Marks the declaration as already emitted for the device code and returns
1557 /// true, if it was marked already, and false, otherwise.
1558 bool markAsGlobalTarget(GlobalDecl GD);
1559
1560 /// Emit deferred declare target variables marked for deferred emission.
1561 void emitDeferredTargetDecls() const;
1562
1563 /// Adjust some parameters for the target-based directives, like addresses of
1564 /// the variables captured by reference in lambdas.
1565 virtual void
1566 adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF,
1567 const OMPExecutableDirective &D) const;
1568
1569 /// Perform check on requires decl to ensure that target architecture
1570 /// supports unified addressing
1571 virtual void processRequiresDirective(const OMPRequiresDecl *D);
1572
1573 /// Gets default memory ordering as specified in requires directive.
1574 llvm::AtomicOrdering getDefaultMemoryOrdering() const;
1575
1576 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with
1577 /// the predefined allocator and translates it into the corresponding address
1578 /// space.
1579 virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS);
1580
1581 /// Return whether the unified_shared_memory has been specified.
1582 bool hasRequiresUnifiedSharedMemory() const;
1583
1584 /// Checks if the \p VD variable is marked as nontemporal declaration in
1585 /// current context.
1586 bool isNontemporalDecl(const ValueDecl *VD) const;
1587
1588 /// Create specialized alloca to handle lastprivate conditionals.
1589 Address emitLastprivateConditionalInit(CodeGenFunction &CGF,
1590 const VarDecl *VD);
1591
1592 /// Checks if the provided \p LVal is lastprivate conditional and emits the
1593 /// code to update the value of the original variable.
1594 /// \code
1595 /// lastprivate(conditional: a)
1596 /// ...
1597 /// <type> a;
1598 /// lp_a = ...;
1599 /// #pragma omp critical(a)
1600 /// if (last_iv_a <= iv) {
1601 /// last_iv_a = iv;
1602 /// global_a = lp_a;
1603 /// }
1604 /// \endcode
1605 virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
1606 const Expr *LHS);
1607
1608 /// Checks if the lastprivate conditional was updated in inner region and
1609 /// writes the value.
1610 /// \code
1611 /// lastprivate(conditional: a)
1612 /// ...
1613 /// <type> a;bool Fired = false;
1614 /// #pragma omp ... shared(a)
1615 /// {
1616 /// lp_a = ...;
1617 /// Fired = true;
1618 /// }
1619 /// if (Fired) {
1620 /// #pragma omp critical(a)
1621 /// if (last_iv_a <= iv) {
1622 /// last_iv_a = iv;
1623 /// global_a = lp_a;
1624 /// }
1625 /// Fired = false;
1626 /// }
1627 /// \endcode
1628 virtual void checkAndEmitSharedLastprivateConditional(
1629 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1630 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls);
1631
1632 /// Gets the address of the global copy used for lastprivate conditional
1633 /// update, if any.
1634 /// \param PrivLVal LValue for the private copy.
1635 /// \param VD Original lastprivate declaration.
1636 virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF,
1637 LValue PrivLVal,
1638 const VarDecl *VD,
1639 SourceLocation Loc);
1640
1641 /// Emits list of dependecies based on the provided data (array of
1642 /// dependence/expression pairs).
1643 /// \returns Pointer to the first element of the array casted to VoidPtr type.
1644 std::pair<llvm::Value *, Address>
1645 emitDependClause(CodeGenFunction &CGF,
1646 ArrayRef<OMPTaskDataTy::DependData> Dependencies,
1647 SourceLocation Loc);
1648
1649 /// Emits list of dependecies based on the provided data (array of
1650 /// dependence/expression pairs) for depobj construct. In this case, the
1651 /// variable is allocated in dynamically. \returns Pointer to the first
1652 /// element of the array casted to VoidPtr type.
1653 Address emitDepobjDependClause(CodeGenFunction &CGF,
1654 const OMPTaskDataTy::DependData &Dependencies,
1655 SourceLocation Loc);
1656
1657 /// Emits the code to destroy the dependency object provided in depobj
1658 /// directive.
1659 void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
1660 SourceLocation Loc);
1661
1662 /// Updates the dependency kind in the specified depobj object.
1663 /// \param DepobjLVal LValue for the main depobj object.
1664 /// \param NewDepKind New dependency kind.
1665 void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
1666 OpenMPDependClauseKind NewDepKind, SourceLocation Loc);
1667
1668 /// Initializes user defined allocators specified in the uses_allocators
1669 /// clauses.
1670 void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator,
1671 const Expr *AllocatorTraits);
1672
1673 /// Destroys user defined allocators specified in the uses_allocators clause.
1674 void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator);
1675
1676 /// Returns true if the variable is a local variable in untied task.
1677 bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const;
1678};
1679
1680/// Class supports emissionof SIMD-only code.
1681class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
1682public:
1683 explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {}
1684 ~CGOpenMPSIMDRuntime() override {}
1685
1686 /// Emits outlined function for the specified OpenMP parallel directive
1687 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1688 /// kmp_int32 BoundID, struct context_vars*).
1689 /// \param CGF Reference to current CodeGenFunction.
1690 /// \param D OpenMP directive.
1691 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1692 /// \param InnermostKind Kind of innermost directive (for simple directives it
1693 /// is a directive itself, for combined - its innermost directive).
1694 /// \param CodeGen Code generation sequence for the \a D directive.
1695 llvm::Function *emitParallelOutlinedFunction(
1696 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1697 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1698 const RegionCodeGenTy &CodeGen) override;
1699
1700 /// Emits outlined function for the specified OpenMP teams directive
1701 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1702 /// kmp_int32 BoundID, struct context_vars*).
1703 /// \param CGF Reference to current CodeGenFunction.
1704 /// \param D OpenMP directive.
1705 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1706 /// \param InnermostKind Kind of innermost directive (for simple directives it
1707 /// is a directive itself, for combined - its innermost directive).
1708 /// \param CodeGen Code generation sequence for the \a D directive.
1709 llvm::Function *emitTeamsOutlinedFunction(
1710 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1711 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1712 const RegionCodeGenTy &CodeGen) override;
1713
1714 /// Emits outlined function for the OpenMP task directive \a D. This
1715 /// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
1716 /// TaskT).
1717 /// \param D OpenMP directive.
1718 /// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1719 /// \param PartIDVar Variable for partition id in the current OpenMP untied
1720 /// task region.
1721 /// \param TaskTVar Variable for task_t argument.
1722 /// \param InnermostKind Kind of innermost directive (for simple directives it
1723 /// is a directive itself, for combined - its innermost directive).
1724 /// \param CodeGen Code generation sequence for the \a D directive.
1725 /// \param Tied true if task is generated for tied task, false otherwise.
1726 /// \param NumberOfParts Number of parts in untied task. Ignored for tied
1727 /// tasks.
1728 ///
1729 llvm::Function *emitTaskOutlinedFunction(
1730 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1731 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1732 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1733 bool Tied, unsigned &NumberOfParts) override;
1734
1735 /// Emits code for parallel or serial call of the \a OutlinedFn with
1736 /// variables captured in a record which address is stored in \a
1737 /// CapturedStruct.
1738 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of
1739 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1740 /// \param CapturedVars A pointer to the record with the references to
1741 /// variables used in \a OutlinedFn function.
1742 /// \param IfCond Condition in the associated 'if' clause, if it was
1743 /// specified, nullptr otherwise.
1744 /// \param NumThreads The value corresponding to the num_threads clause, if
1745 /// any, or nullptr.
1746 ///
1747 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1748 llvm::Function *OutlinedFn,
1749 ArrayRef<llvm::Value *> CapturedVars,
1750 const Expr *IfCond, llvm::Value *NumThreads) override;
1751
1752 /// Emits a critical region.
1753 /// \param CriticalName Name of the critical region.
1754 /// \param CriticalOpGen Generator for the statement associated with the given
1755 /// critical region.
1756 /// \param Hint Value of the 'hint' clause (optional).
1757 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
1758 const RegionCodeGenTy &CriticalOpGen,
1759 SourceLocation Loc,
1760 const Expr *Hint = nullptr) override;
1761
1762 /// Emits a master region.
1763 /// \param MasterOpGen Generator for the statement associated with the given
1764 /// master region.
1765 void emitMasterRegion(CodeGenFunction &CGF,
1766 const RegionCodeGenTy &MasterOpGen,
1767 SourceLocation Loc) override;
1768
1769 /// Emits a masked region.
1770 /// \param MaskedOpGen Generator for the statement associated with the given
1771 /// masked region.
1772 void emitMaskedRegion(CodeGenFunction &CGF,
1773 const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc,
1774 const Expr *Filter = nullptr) override;
1775
1776 /// Emits a masked region.
1777 /// \param MaskedOpGen Generator for the statement associated with the given
1778 /// masked region.
1779
1780 /// Emits code for a taskyield directive.
1781 void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override;
1782
1783 /// Emit a taskgroup region.
1784 /// \param TaskgroupOpGen Generator for the statement associated with the
1785 /// given taskgroup region.
1786 void emitTaskgroupRegion(CodeGenFunction &CGF,
1787 const RegionCodeGenTy &TaskgroupOpGen,
1788 SourceLocation Loc) override;
1789
1790 /// Emits a single region.
1791 /// \param SingleOpGen Generator for the statement associated with the given
1792 /// single region.
1793 void emitSingleRegion(CodeGenFunction &CGF,
1794 const RegionCodeGenTy &SingleOpGen, SourceLocation Loc,
1795 ArrayRef<const Expr *> CopyprivateVars,
1796 ArrayRef<const Expr *> DestExprs,
1797 ArrayRef<const Expr *> SrcExprs,
1798 ArrayRef<const Expr *> AssignmentOps) override;
1799
1800 /// Emit an ordered region.
1801 /// \param OrderedOpGen Generator for the statement associated with the given
1802 /// ordered region.
1803 void emitOrderedRegion(CodeGenFunction &CGF,
1804 const RegionCodeGenTy &OrderedOpGen,
1805 SourceLocation Loc, bool IsThreads) override;
1806
1807 /// Emit an implicit/explicit barrier for OpenMP threads.
1808 /// \param Kind Directive for which this implicit barrier call must be
1809 /// generated. Must be OMPD_barrier for explicit barrier generation.
1810 /// \param EmitChecks true if need to emit checks for cancellation barriers.
1811 /// \param ForceSimpleCall true simple barrier call must be emitted, false if
1812 /// runtime class decides which one to emit (simple or with cancellation
1813 /// checks).
1814 ///
1815 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
1816 OpenMPDirectiveKind Kind, bool EmitChecks = true,
1817 bool ForceSimpleCall = false) override;
1818
1819 /// This is used for non static scheduled types and when the ordered
1820 /// clause is present on the loop construct.
1821 /// Depending on the loop schedule, it is necessary to call some runtime
1822 /// routine before start of the OpenMP loop to get the loop upper / lower
1823 /// bounds \a LB and \a UB and stride \a ST.
1824 ///
1825 /// \param CGF Reference to current CodeGenFunction.
1826 /// \param Loc Clang source location.
1827 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1828 /// \param IVSize Size of the iteration variable in bits.
1829 /// \param IVSigned Sign of the iteration variable.
1830 /// \param Ordered true if loop is ordered, false otherwise.
1831 /// \param DispatchValues struct containing llvm values for lower bound, upper
1832 /// bound, and chunk expression.
1833 /// For the default (nullptr) value, the chunk 1 will be used.
1834 ///
1835 void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
1836 const OpenMPScheduleTy &ScheduleKind,
1837 unsigned IVSize, bool IVSigned, bool Ordered,
1838 const DispatchRTInput &DispatchValues) override;
1839
1840 /// This is used for non static scheduled types and when the ordered
1841 /// clause is present on the loop construct.
1842 ///
1843 /// \param CGF Reference to current CodeGenFunction.
1844 /// \param Loc Clang source location.
1845 ///
1846 void emitForDispatchDeinit(CodeGenFunction &CGF, SourceLocation Loc) override;
1847
1848 /// Call the appropriate runtime routine to initialize it before start
1849 /// of loop.
1850 ///
1851 /// This is used only in case of static schedule, when the user did not
1852 /// specify a ordered clause on the loop construct.
1853 /// Depending on the loop schedule, it is necessary to call some runtime
1854 /// routine before start of the OpenMP loop to get the loop upper / lower
1855 /// bounds LB and UB and stride ST.
1856 ///
1857 /// \param CGF Reference to current CodeGenFunction.
1858 /// \param Loc Clang source location.
1859 /// \param DKind Kind of the directive.
1860 /// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1861 /// \param Values Input arguments for the construct.
1862 ///
1863 void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1864 OpenMPDirectiveKind DKind,
1865 const OpenMPScheduleTy &ScheduleKind,
1866 const StaticRTInput &Values) override;
1867
1868 ///
1869 /// \param CGF Reference to current CodeGenFunction.
1870 /// \param Loc Clang source location.
1871 /// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1872 /// \param Values Input arguments for the construct.
1873 ///
1874 void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1875 OpenMPDistScheduleClauseKind SchedKind,
1876 const StaticRTInput &Values) override;
1877
1878 /// Call the appropriate runtime routine to notify that we finished
1879 /// iteration of the ordered loop with the dynamic scheduling.
1880 ///
1881 /// \param CGF Reference to current CodeGenFunction.
1882 /// \param Loc Clang source location.
1883 /// \param IVSize Size of the iteration variable in bits.
1884 /// \param IVSigned Sign of the iteration variable.
1885 ///
1886 void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc,
1887 unsigned IVSize, bool IVSigned) override;
1888
1889 /// Call the appropriate runtime routine to notify that we finished
1890 /// all the work with current loop.
1891 ///
1892 /// \param CGF Reference to current CodeGenFunction.
1893 /// \param Loc Clang source location.
1894 /// \param DKind Kind of the directive for which the static finish is emitted.
1895 ///
1896 void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1897 OpenMPDirectiveKind DKind) override;
1898
1899 /// Call __kmpc_dispatch_next(
1900 /// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1901 /// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1902 /// kmp_int[32|64] *p_stride);
1903 /// \param IVSize Size of the iteration variable in bits.
1904 /// \param IVSigned Sign of the iteration variable.
1905 /// \param IL Address of the output variable in which the flag of the
1906 /// last iteration is returned.
1907 /// \param LB Address of the output variable in which the lower iteration
1908 /// number is returned.
1909 /// \param UB Address of the output variable in which the upper iteration
1910 /// number is returned.
1911 /// \param ST Address of the output variable in which the stride value is
1912 /// returned.
1913 llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1914 unsigned IVSize, bool IVSigned, Address IL,
1915 Address LB, Address UB, Address ST) override;
1916
1917 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1918 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1919 /// clause.
1920 /// \param NumThreads An integer value of threads.
1921 void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads,
1922 SourceLocation Loc) override;
1923
1924 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1925 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1926 void emitProcBindClause(CodeGenFunction &CGF,
1927 llvm::omp::ProcBindKind ProcBind,
1928 SourceLocation Loc) override;
1929
1930 /// Returns address of the threadprivate variable for the current
1931 /// thread.
1932 /// \param VD Threadprivate variable.
1933 /// \param VDAddr Address of the global variable \a VD.
1934 /// \param Loc Location of the reference to threadprivate var.
1935 /// \return Address of the threadprivate variable for the current thread.
1936 Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD,
1937 Address VDAddr, SourceLocation Loc) override;
1938
1939 /// Emit a code for initialization of threadprivate variable. It emits
1940 /// a call to runtime library which adds initial value to the newly created
1941 /// threadprivate variable (if it is not constant) and registers destructor
1942 /// for the variable (if any).
1943 /// \param VD Threadprivate variable.
1944 /// \param VDAddr Address of the global variable \a VD.
1945 /// \param Loc Location of threadprivate declaration.
1946 /// \param PerformInit true if initialization expression is not constant.
1947 llvm::Function *
1948 emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1949 SourceLocation Loc, bool PerformInit,
1950 CodeGenFunction *CGF = nullptr) override;
1951
1952 /// Creates artificial threadprivate variable with name \p Name and type \p
1953 /// VarType.
1954 /// \param VarType Type of the artificial threadprivate variable.
1955 /// \param Name Name of the artificial threadprivate variable.
1956 Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1957 QualType VarType,
1958 StringRef Name) override;
1959
1960 /// Emit flush of the variables specified in 'omp flush' directive.
1961 /// \param Vars List of variables to flush.
1962 void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1963 SourceLocation Loc, llvm::AtomicOrdering AO) override;
1964
1965 /// Emit task region for the task directive. The task region is
1966 /// emitted in several steps:
1967 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1968 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1969 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1970 /// function:
1971 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1972 /// TaskFunction(gtid, tt->part_id, tt->shareds);
1973 /// return 0;
1974 /// }
1975 /// 2. Copy a list of shared variables to field shareds of the resulting
1976 /// structure kmp_task_t returned by the previous call (if any).
1977 /// 3. Copy a pointer to destructions function to field destructions of the
1978 /// resulting structure kmp_task_t.
1979 /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1980 /// kmp_task_t *new_task), where new_task is a resulting structure from
1981 /// previous items.
1982 /// \param D Current task directive.
1983 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1984 /// /*part_id*/, captured_struct */*__context*/);
1985 /// \param SharedsTy A type which contains references the shared variables.
1986 /// \param Shareds Context with the list of shared variables from the \p
1987 /// TaskFunction.
1988 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1989 /// otherwise.
1990 /// \param Data Additional data for task generation like tiednsee, final
1991 /// state, list of privates etc.
1992 void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1993 const OMPExecutableDirective &D,
1994 llvm::Function *TaskFunction, QualType SharedsTy,
1995 Address Shareds, const Expr *IfCond,
1996 const OMPTaskDataTy &Data) override;
1997
1998 /// Emit task region for the taskloop directive. The taskloop region is
1999 /// emitted in several steps:
2000 /// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
2001 /// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
2002 /// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
2003 /// function:
2004 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
2005 /// TaskFunction(gtid, tt->part_id, tt->shareds);
2006 /// return 0;
2007 /// }
2008 /// 2. Copy a list of shared variables to field shareds of the resulting
2009 /// structure kmp_task_t returned by the previous call (if any).
2010 /// 3. Copy a pointer to destructions function to field destructions of the
2011 /// resulting structure kmp_task_t.
2012 /// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
2013 /// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
2014 /// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
2015 /// is a resulting structure from
2016 /// previous items.
2017 /// \param D Current task directive.
2018 /// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
2019 /// /*part_id*/, captured_struct */*__context*/);
2020 /// \param SharedsTy A type which contains references the shared variables.
2021 /// \param Shareds Context with the list of shared variables from the \p
2022 /// TaskFunction.
2023 /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
2024 /// otherwise.
2025 /// \param Data Additional data for task generation like tiednsee, final
2026 /// state, list of privates etc.
2027 void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
2028 const OMPLoopDirective &D, llvm::Function *TaskFunction,
2029 QualType SharedsTy, Address Shareds, const Expr *IfCond,
2030 const OMPTaskDataTy &Data) override;
2031
2032 /// Emit a code for reduction clause. Next code should be emitted for
2033 /// reduction:
2034 /// \code
2035 ///
2036 /// static kmp_critical_name lock = { 0 };
2037 ///
2038 /// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
2039 /// ...
2040 /// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
2041 /// ...
2042 /// }
2043 ///
2044 /// ...
2045 /// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
2046 /// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
2047 /// RedList, reduce_func, &<lock>)) {
2048 /// case 1:
2049 /// ...
2050 /// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2051 /// ...
2052 /// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2053 /// break;
2054 /// case 2:
2055 /// ...
2056 /// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
2057 /// ...
2058 /// break;
2059 /// default:;
2060 /// }
2061 /// \endcode
2062 ///
2063 /// \param Privates List of private copies for original reduction arguments.
2064 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
2065 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
2066 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
2067 /// or 'operator binop(LHS, RHS)'.
2068 /// \param Options List of options for reduction codegen:
2069 /// WithNowait true if parent directive has also nowait clause, false
2070 /// otherwise.
2071 /// SimpleReduction Emit reduction operation only. Used for omp simd
2072 /// directive on the host.
2073 /// ReductionKind The kind of reduction to perform.
2074 void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
2075 ArrayRef<const Expr *> Privates,
2076 ArrayRef<const Expr *> LHSExprs,
2077 ArrayRef<const Expr *> RHSExprs,
2078 ArrayRef<const Expr *> ReductionOps,
2079 ReductionOptionsTy Options) override;
2080
2081 /// Emit a code for initialization of task reduction clause. Next code
2082 /// should be emitted for reduction:
2083 /// \code
2084 ///
2085 /// _taskred_item_t red_data[n];
2086 /// ...
2087 /// red_data[i].shar = &shareds[i];
2088 /// red_data[i].orig = &origs[i];
2089 /// red_data[i].size = sizeof(origs[i]);
2090 /// red_data[i].f_init = (void*)RedInit<i>;
2091 /// red_data[i].f_fini = (void*)RedDest<i>;
2092 /// red_data[i].f_comb = (void*)RedOp<i>;
2093 /// red_data[i].flags = <Flag_i>;
2094 /// ...
2095 /// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
2096 /// \endcode
2097 /// For reduction clause with task modifier it emits the next call:
2098 /// \code
2099 ///
2100 /// _taskred_item_t red_data[n];
2101 /// ...
2102 /// red_data[i].shar = &shareds[i];
2103 /// red_data[i].orig = &origs[i];
2104 /// red_data[i].size = sizeof(origs[i]);
2105 /// red_data[i].f_init = (void*)RedInit<i>;
2106 /// red_data[i].f_fini = (void*)RedDest<i>;
2107 /// red_data[i].f_comb = (void*)RedOp<i>;
2108 /// red_data[i].flags = <Flag_i>;
2109 /// ...
2110 /// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
2111 /// red_data);
2112 /// \endcode
2113 /// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
2114 /// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
2115 /// \param Data Additional data for task generation like tiedness, final
2116 /// state, list of privates, reductions etc.
2117 llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc,
2118 ArrayRef<const Expr *> LHSExprs,
2119 ArrayRef<const Expr *> RHSExprs,
2120 const OMPTaskDataTy &Data) override;
2121
2122 /// Emits the following code for reduction clause with task modifier:
2123 /// \code
2124 /// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
2125 /// \endcode
2126 void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
2127 bool IsWorksharingReduction) override;
2128
2129 /// Required to resolve existing problems in the runtime. Emits threadprivate
2130 /// variables to store the size of the VLAs/array sections for
2131 /// initializer/combiner/finalizer functions + emits threadprivate variable to
2132 /// store the pointer to the original reduction item for the custom
2133 /// initializer defined by declare reduction construct.
2134 /// \param RCG Allows to reuse an existing data for the reductions.
2135 /// \param N Reduction item for which fixups must be emitted.
2136 void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
2137 ReductionCodeGen &RCG, unsigned N) override;
2138
2139 /// Get the address of `void *` type of the privatue copy of the reduction
2140 /// item specified by the \p SharedLVal.
2141 /// \param ReductionsPtr Pointer to the reduction data returned by the
2142 /// emitTaskReductionInit function.
2143 /// \param SharedLVal Address of the original reduction item.
2144 Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
2145 llvm::Value *ReductionsPtr,
2146 LValue SharedLVal) override;
2147
2148 /// Emit code for 'taskwait' directive.
2149 void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
2150 const OMPTaskDataTy &Data) override;
2151
2152 /// Emit code for 'cancellation point' construct.
2153 /// \param CancelRegion Region kind for which the cancellation point must be
2154 /// emitted.
2155 ///
2156 void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc,
2157 OpenMPDirectiveKind CancelRegion) override;
2158
2159 /// Emit code for 'cancel' construct.
2160 /// \param IfCond Condition in the associated 'if' clause, if it was
2161 /// specified, nullptr otherwise.
2162 /// \param CancelRegion Region kind for which the cancel must be emitted.
2163 ///
2164 void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
2165 const Expr *IfCond,
2166 OpenMPDirectiveKind CancelRegion) override;
2167
2168 /// Emit outilined function for 'target' directive.
2169 /// \param D Directive to emit.
2170 /// \param ParentName Name of the function that encloses the target region.
2171 /// \param OutlinedFn Outlined function value to be defined by this call.
2172 /// \param OutlinedFnID Outlined function ID value to be defined by this call.
2173 /// \param IsOffloadEntry True if the outlined function is an offload entry.
2174 /// \param CodeGen Code generation sequence for the \a D directive.
2175 /// An outlined function may not be an entry if, e.g. the if clause always
2176 /// evaluates to false.
2177 void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
2178 StringRef ParentName,
2179 llvm::Function *&OutlinedFn,
2180 llvm::Constant *&OutlinedFnID,
2181 bool IsOffloadEntry,
2182 const RegionCodeGenTy &CodeGen) override;
2183
2184 /// Emit the target offloading code associated with \a D. The emitted
2185 /// code attempts offloading the execution to the device, an the event of
2186 /// a failure it executes the host version outlined in \a OutlinedFn.
2187 /// \param D Directive to emit.
2188 /// \param OutlinedFn Host version of the code to be offloaded.
2189 /// \param OutlinedFnID ID of host version of the code to be offloaded.
2190 /// \param IfCond Expression evaluated in if clause associated with the target
2191 /// directive, or null if no if clause is used.
2192 /// \param Device Expression evaluated in device clause associated with the
2193 /// target directive, or null if no device clause is used and device modifier.
2194 void emitTargetCall(
2195 CodeGenFunction &CGF, const OMPExecutableDirective &D,
2196 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
2197 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
2198 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
2199 const OMPLoopDirective &D)>
2200 SizeEmitter) override;
2201
2202 /// Emit the target regions enclosed in \a GD function definition or
2203 /// the function itself in case it is a valid device function. Returns true if
2204 /// \a GD was dealt with successfully.
2205 /// \param GD Function to scan.
2206 bool emitTargetFunctions(GlobalDecl GD) override;
2207
2208 /// Emit the global variable if it is a valid device global variable.
2209 /// Returns true if \a GD was dealt with successfully.
2210 /// \param GD Variable declaration to emit.
2211 bool emitTargetGlobalVariable(GlobalDecl GD) override;
2212
2213 /// Emit the global \a GD if it is meaningful for the target. Returns
2214 /// if it was emitted successfully.
2215 /// \param GD Global to scan.
2216 bool emitTargetGlobal(GlobalDecl GD) override;
2217
2218 /// Emits code for teams call of the \a OutlinedFn with
2219 /// variables captured in a record which address is stored in \a
2220 /// CapturedStruct.
2221 /// \param OutlinedFn Outlined function to be run by team masters. Type of
2222 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
2223 /// \param CapturedVars A pointer to the record with the references to
2224 /// variables used in \a OutlinedFn function.
2225 ///
2226 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
2227 SourceLocation Loc, llvm::Function *OutlinedFn,
2228 ArrayRef<llvm::Value *> CapturedVars) override;
2229
2230 /// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
2231 /// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
2232 /// for num_teams clause.
2233 /// \param NumTeams An integer expression of teams.
2234 /// \param ThreadLimit An integer expression of threads.
2235 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
2236 const Expr *ThreadLimit, SourceLocation Loc) override;
2237
2238 /// Emit the target data mapping code associated with \a D.
2239 /// \param D Directive to emit.
2240 /// \param IfCond Expression evaluated in if clause associated with the
2241 /// target directive, or null if no device clause is used.
2242 /// \param Device Expression evaluated in device clause associated with the
2243 /// target directive, or null if no device clause is used.
2244 /// \param Info A record used to store information that needs to be preserved
2245 /// until the region is closed.
2246 void emitTargetDataCalls(CodeGenFunction &CGF,
2247 const OMPExecutableDirective &D, const Expr *IfCond,
2248 const Expr *Device, const RegionCodeGenTy &CodeGen,
2249 CGOpenMPRuntime::TargetDataInfo &Info) override;
2250
2251 /// Emit the data mapping/movement code associated with the directive
2252 /// \a D that should be of the form 'target [{enter|exit} data | update]'.
2253 /// \param D Directive to emit.
2254 /// \param IfCond Expression evaluated in if clause associated with the target
2255 /// directive, or null if no if clause is used.
2256 /// \param Device Expression evaluated in device clause associated with the
2257 /// target directive, or null if no device clause is used.
2258 void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
2259 const OMPExecutableDirective &D,
2260 const Expr *IfCond,
2261 const Expr *Device) override;
2262
2263 /// Emit initialization for doacross loop nesting support.
2264 /// \param D Loop-based construct used in doacross nesting construct.
2265 void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
2266 ArrayRef<Expr *> NumIterations) override;
2267
2268 /// Emit code for doacross ordered directive with 'depend' clause.
2269 /// \param C 'depend' clause with 'sink|source' dependency kind.
2270 void emitDoacrossOrdered(CodeGenFunction &CGF,
2271 const OMPDependClause *C) override;
2272
2273 /// Emit code for doacross ordered directive with 'doacross' clause.
2274 /// \param C 'doacross' clause with 'sink|source' dependence type.
2275 void emitDoacrossOrdered(CodeGenFunction &CGF,
2276 const OMPDoacrossClause *C) override;
2277
2278 /// Translates the native parameter of outlined function if this is required
2279 /// for target.
2280 /// \param FD Field decl from captured record for the parameter.
2281 /// \param NativeParam Parameter itself.
2282 const VarDecl *translateParameter(const FieldDecl *FD,
2283 const VarDecl *NativeParam) const override;
2284
2285 /// Gets the address of the native argument basing on the address of the
2286 /// target-specific parameter.
2287 /// \param NativeParam Parameter itself.
2288 /// \param TargetParam Corresponding target-specific parameter.
2289 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
2290 const VarDecl *TargetParam) const override;
2291
2292 /// Gets the OpenMP-specific address of the local variable.
2293 Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2294 const VarDecl *VD) override {
2295 return Address::invalid();
2296 }
2297};
2298
2299} // namespace CodeGen
2300// Utility for openmp doacross clause kind
2301namespace {
2302template <typename T> class OMPDoacrossKind {
2303public:
2304 bool isSink(const T *) { return false; }
2305 bool isSource(const T *) { return false; }
2306};
2307template <> class OMPDoacrossKind<OMPDependClause> {
2308public:
2309 bool isSink(const OMPDependClause *C) {
2310 return C->getDependencyKind() == OMPC_DEPEND_sink;
2311 }
2312 bool isSource(const OMPDependClause *C) {
2313 return C->getDependencyKind() == OMPC_DEPEND_source;
2314 }
2315};
2316template <> class OMPDoacrossKind<OMPDoacrossClause> {
2317public:
2318 bool isSource(const OMPDoacrossClause *C) {
2319 return C->getDependenceType() == OMPC_DOACROSS_source ||
2320 C->getDependenceType() == OMPC_DOACROSS_source_omp_cur_iteration;
2321 }
2322 bool isSink(const OMPDoacrossClause *C) {
2323 return C->getDependenceType() == OMPC_DOACROSS_sink ||
2324 C->getDependenceType() == OMPC_DOACROSS_sink_omp_cur_iteration;
2325 }
2326};
2327} // namespace
2328} // namespace clang
2329
2330#endif
2331