1//===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a class for OpenMP runtime code generation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGOpenMPRuntime.h"
14#include "ABIInfoImpl.h"
15#include "CGCXXABI.h"
16#include "CGCleanup.h"
17#include "CGDebugInfo.h"
18#include "CGRecordLayout.h"
19#include "CodeGenFunction.h"
20#include "TargetInfo.h"
21#include "clang/AST/APValue.h"
22#include "clang/AST/Attr.h"
23#include "clang/AST/Decl.h"
24#include "clang/AST/OpenMPClause.h"
25#include "clang/AST/StmtOpenMP.h"
26#include "clang/AST/StmtVisitor.h"
27#include "clang/Basic/DiagnosticFrontend.h"
28#include "clang/Basic/OpenMPKinds.h"
29#include "clang/Basic/SourceManager.h"
30#include "clang/CodeGen/ConstantInitBuilder.h"
31#include "llvm/ADT/ArrayRef.h"
32#include "llvm/ADT/SmallSet.h"
33#include "llvm/ADT/SmallVector.h"
34#include "llvm/ADT/StringExtras.h"
35#include "llvm/Bitcode/BitcodeReader.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DerivedTypes.h"
38#include "llvm/IR/GlobalValue.h"
39#include "llvm/IR/InstrTypes.h"
40#include "llvm/IR/Value.h"
41#include "llvm/Support/AtomicOrdering.h"
42#include "llvm/Support/VirtualFileSystem.h"
43#include "llvm/Support/raw_ostream.h"
44#include <cassert>
45#include <cstdint>
46#include <numeric>
47#include <optional>
48
49using namespace clang;
50using namespace CodeGen;
51using namespace llvm::omp;
52
53namespace {
54/// Base class for handling code generation inside OpenMP regions.
55class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
56public:
57 /// Kinds of OpenMP regions used in codegen.
58 enum CGOpenMPRegionKind {
59 /// Region with outlined function for standalone 'parallel'
60 /// directive.
61 ParallelOutlinedRegion,
62 /// Region with outlined function for standalone 'task' directive.
63 TaskOutlinedRegion,
64 /// Region for constructs that do not require function outlining,
65 /// like 'for', 'sections', 'atomic' etc. directives.
66 InlinedRegion,
67 /// Region with outlined function for standalone 'target' directive.
68 TargetRegion,
69 };
70
71 CGOpenMPRegionInfo(const CapturedStmt &CS,
72 const CGOpenMPRegionKind RegionKind,
73 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
74 bool HasCancel)
75 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind),
76 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
77
78 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind,
79 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind,
80 bool HasCancel)
81 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
82 Kind(Kind), HasCancel(HasCancel) {}
83
84 /// Get a variable or parameter for storing global thread id
85 /// inside OpenMP construct.
86 virtual const VarDecl *getThreadIDVariable() const = 0;
87
88 /// Emit the captured statement body.
89 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override;
90
91 /// Get an LValue for the current ThreadID variable.
92 /// \return LValue for thread id variable. This LValue always has type int32*.
93 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
94
95 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {}
96
97 CGOpenMPRegionKind getRegionKind() const { return RegionKind; }
98
99 OpenMPDirectiveKind getDirectiveKind() const { return Kind; }
100
101 bool hasCancel() const { return HasCancel; }
102
103 static bool classof(const CGCapturedStmtInfo *Info) {
104 return Info->getKind() == CR_OpenMP;
105 }
106
107 ~CGOpenMPRegionInfo() override = default;
108
109protected:
110 CGOpenMPRegionKind RegionKind;
111 RegionCodeGenTy CodeGen;
112 OpenMPDirectiveKind Kind;
113 bool HasCancel;
114};
115
116/// API for captured statement code generation in OpenMP constructs.
117class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo {
118public:
119 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar,
120 const RegionCodeGenTy &CodeGen,
121 OpenMPDirectiveKind Kind, bool HasCancel,
122 StringRef HelperName)
123 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
124 HasCancel),
125 ThreadIDVar(ThreadIDVar), HelperName(HelperName) {
126 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
127 }
128
129 /// Get a variable or parameter for storing global thread id
130 /// inside OpenMP construct.
131 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
132
133 /// Get the name of the capture helper.
134 StringRef getHelperName() const override { return HelperName; }
135
136 static bool classof(const CGCapturedStmtInfo *Info) {
137 return CGOpenMPRegionInfo::classof(Info) &&
138 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() ==
139 ParallelOutlinedRegion;
140 }
141
142private:
143 /// A variable or parameter storing global thread id for OpenMP
144 /// constructs.
145 const VarDecl *ThreadIDVar;
146 StringRef HelperName;
147};
148
149/// API for captured statement code generation in OpenMP constructs.
150class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo {
151public:
152 class UntiedTaskActionTy final : public PrePostActionTy {
153 bool Untied;
154 const VarDecl *PartIDVar;
155 const RegionCodeGenTy UntiedCodeGen;
156 llvm::SwitchInst *UntiedSwitch = nullptr;
157
158 public:
159 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar,
160 const RegionCodeGenTy &UntiedCodeGen)
161 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
162 void Enter(CodeGenFunction &CGF) override {
163 if (Untied) {
164 // Emit task switching point.
165 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
166 Ptr: CGF.GetAddrOfLocalVar(VD: PartIDVar),
167 PtrTy: PartIDVar->getType()->castAs<PointerType>());
168 llvm::Value *Res =
169 CGF.EmitLoadOfScalar(lvalue: PartIdLVal, Loc: PartIDVar->getLocation());
170 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: ".untied.done.");
171 UntiedSwitch = CGF.Builder.CreateSwitch(V: Res, Dest: DoneBB);
172 CGF.EmitBlock(BB: DoneBB);
173 CGF.EmitBranchThroughCleanup(Dest: CGF.ReturnBlock);
174 CGF.EmitBlock(BB: CGF.createBasicBlock(name: ".untied.jmp."));
175 UntiedSwitch->addCase(OnVal: CGF.Builder.getInt32(C: 0),
176 Dest: CGF.Builder.GetInsertBlock());
177 emitUntiedSwitch(CGF);
178 }
179 }
180 void emitUntiedSwitch(CodeGenFunction &CGF) const {
181 if (Untied) {
182 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue(
183 Ptr: CGF.GetAddrOfLocalVar(VD: PartIDVar),
184 PtrTy: PartIDVar->getType()->castAs<PointerType>());
185 CGF.EmitStoreOfScalar(value: CGF.Builder.getInt32(C: UntiedSwitch->getNumCases()),
186 lvalue: PartIdLVal);
187 UntiedCodeGen(CGF);
188 CodeGenFunction::JumpDest CurPoint =
189 CGF.getJumpDestInCurrentScope(Name: ".untied.next.");
190 CGF.EmitBranch(Block: CGF.ReturnBlock.getBlock());
191 CGF.EmitBlock(BB: CGF.createBasicBlock(name: ".untied.jmp."));
192 UntiedSwitch->addCase(OnVal: CGF.Builder.getInt32(C: UntiedSwitch->getNumCases()),
193 Dest: CGF.Builder.GetInsertBlock());
194 CGF.EmitBranchThroughCleanup(Dest: CurPoint);
195 CGF.EmitBlock(BB: CurPoint.getBlock());
196 }
197 }
198 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); }
199 };
200 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS,
201 const VarDecl *ThreadIDVar,
202 const RegionCodeGenTy &CodeGen,
203 OpenMPDirectiveKind Kind, bool HasCancel,
204 const UntiedTaskActionTy &Action)
205 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
206 ThreadIDVar(ThreadIDVar), Action(Action) {
207 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
208 }
209
210 /// Get a variable or parameter for storing global thread id
211 /// inside OpenMP construct.
212 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; }
213
214 /// Get an LValue for the current ThreadID variable.
215 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override;
216
217 /// Get the name of the capture helper.
218 StringRef getHelperName() const override { return ".omp_outlined."; }
219
220 void emitUntiedSwitch(CodeGenFunction &CGF) override {
221 Action.emitUntiedSwitch(CGF);
222 }
223
224 static bool classof(const CGCapturedStmtInfo *Info) {
225 return CGOpenMPRegionInfo::classof(Info) &&
226 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() ==
227 TaskOutlinedRegion;
228 }
229
230private:
231 /// A variable or parameter storing global thread id for OpenMP
232 /// constructs.
233 const VarDecl *ThreadIDVar;
234 /// Action for emitting code for untied tasks.
235 const UntiedTaskActionTy &Action;
236};
237
238/// API for inlined captured statement code generation in OpenMP
239/// constructs.
240class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo {
241public:
242 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,
243 const RegionCodeGenTy &CodeGen,
244 OpenMPDirectiveKind Kind, bool HasCancel)
245 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
246 OldCSI(OldCSI),
247 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(Val: OldCSI)) {}
248
249 // Retrieve the value of the context parameter.
250 llvm::Value *getContextValue() const override {
251 if (OuterRegionInfo)
252 return OuterRegionInfo->getContextValue();
253 llvm_unreachable("No context value for inlined OpenMP region");
254 }
255
256 void setContextValue(llvm::Value *V) override {
257 if (OuterRegionInfo) {
258 OuterRegionInfo->setContextValue(V);
259 return;
260 }
261 llvm_unreachable("No context value for inlined OpenMP region");
262 }
263
264 /// Lookup the captured field decl for a variable.
265 const FieldDecl *lookup(const VarDecl *VD) const override {
266 if (OuterRegionInfo)
267 return OuterRegionInfo->lookup(VD);
268 // If there is no outer outlined region,no need to lookup in a list of
269 // captured variables, we can use the original one.
270 return nullptr;
271 }
272
273 FieldDecl *getThisFieldDecl() const override {
274 if (OuterRegionInfo)
275 return OuterRegionInfo->getThisFieldDecl();
276 return nullptr;
277 }
278
279 /// Get a variable or parameter for storing global thread id
280 /// inside OpenMP construct.
281 const VarDecl *getThreadIDVariable() const override {
282 if (OuterRegionInfo)
283 return OuterRegionInfo->getThreadIDVariable();
284 return nullptr;
285 }
286
287 /// Get an LValue for the current ThreadID variable.
288 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override {
289 if (OuterRegionInfo)
290 return OuterRegionInfo->getThreadIDVariableLValue(CGF);
291 llvm_unreachable("No LValue for inlined OpenMP construct");
292 }
293
294 /// Get the name of the capture helper.
295 StringRef getHelperName() const override {
296 if (auto *OuterRegionInfo = getOldCSI())
297 return OuterRegionInfo->getHelperName();
298 llvm_unreachable("No helper name for inlined OpenMP construct");
299 }
300
301 void emitUntiedSwitch(CodeGenFunction &CGF) override {
302 if (OuterRegionInfo)
303 OuterRegionInfo->emitUntiedSwitch(CGF);
304 }
305
306 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; }
307
308 static bool classof(const CGCapturedStmtInfo *Info) {
309 return CGOpenMPRegionInfo::classof(Info) &&
310 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() == InlinedRegion;
311 }
312
313 ~CGOpenMPInlinedRegionInfo() override = default;
314
315private:
316 /// CodeGen info about outer OpenMP region.
317 CodeGenFunction::CGCapturedStmtInfo *OldCSI;
318 CGOpenMPRegionInfo *OuterRegionInfo;
319};
320
321/// API for captured statement code generation in OpenMP target
322/// constructs. For this captures, implicit parameters are used instead of the
323/// captured fields. The name of the target region has to be unique in a given
324/// application so it is provided by the client, because only the client has
325/// the information to generate that.
326class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
327public:
328 CGOpenMPTargetRegionInfo(const CapturedStmt &CS,
329 const RegionCodeGenTy &CodeGen, StringRef HelperName)
330 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
331 /*HasCancel=*/false),
332 HelperName(HelperName) {}
333
334 /// This is unused for target regions because each starts executing
335 /// with a single thread.
336 const VarDecl *getThreadIDVariable() const override { return nullptr; }
337
338 /// Get the name of the capture helper.
339 StringRef getHelperName() const override { return HelperName; }
340
341 static bool classof(const CGCapturedStmtInfo *Info) {
342 return CGOpenMPRegionInfo::classof(Info) &&
343 cast<CGOpenMPRegionInfo>(Val: Info)->getRegionKind() == TargetRegion;
344 }
345
346private:
347 StringRef HelperName;
348};
349
350static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
351 llvm_unreachable("No codegen for expressions");
352}
353/// API for generation of expressions captured in a innermost OpenMP
354/// region.
355class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo {
356public:
357 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS)
358 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
359 OMPD_unknown,
360 /*HasCancel=*/false),
361 PrivScope(CGF) {
362 // Make sure the globals captured in the provided statement are local by
363 // using the privatization logic. We assume the same variable is not
364 // captured more than once.
365 for (const auto &C : CS.captures()) {
366 if (!C.capturesVariable() && !C.capturesVariableByCopy())
367 continue;
368
369 const VarDecl *VD = C.getCapturedVar();
370 if (VD->isLocalVarDeclOrParm())
371 continue;
372
373 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD),
374 /*RefersToEnclosingVariableOrCapture=*/false,
375 VD->getType().getNonReferenceType(), VK_LValue,
376 C.getLocation());
377 PrivScope.addPrivate(LocalVD: VD, Addr: CGF.EmitLValue(E: &DRE).getAddress());
378 }
379 (void)PrivScope.Privatize();
380 }
381
382 /// Lookup the captured field decl for a variable.
383 const FieldDecl *lookup(const VarDecl *VD) const override {
384 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
385 return FD;
386 return nullptr;
387 }
388
389 /// Emit the captured statement body.
390 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override {
391 llvm_unreachable("No body for expressions");
392 }
393
394 /// Get a variable or parameter for storing global thread id
395 /// inside OpenMP construct.
396 const VarDecl *getThreadIDVariable() const override {
397 llvm_unreachable("No thread id for expressions");
398 }
399
400 /// Get the name of the capture helper.
401 StringRef getHelperName() const override {
402 llvm_unreachable("No helper name for expressions");
403 }
404
405 static bool classof(const CGCapturedStmtInfo *Info) { return false; }
406
407private:
408 /// Private scope to capture global variables.
409 CodeGenFunction::OMPPrivateScope PrivScope;
410};
411
412/// RAII for emitting code of OpenMP constructs.
413class InlinedOpenMPRegionRAII {
414 CodeGenFunction &CGF;
415 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;
416 FieldDecl *LambdaThisCaptureField = nullptr;
417 const CodeGen::CGBlockInfo *BlockInfo = nullptr;
418 bool NoInheritance = false;
419
420public:
421 /// Constructs region for combined constructs.
422 /// \param CodeGen Code generation sequence for combined directives. Includes
423 /// a list of functions used for code generation of implicitly inlined
424 /// regions.
425 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen,
426 OpenMPDirectiveKind Kind, bool HasCancel,
427 bool NoInheritance = true)
428 : CGF(CGF), NoInheritance(NoInheritance) {
429 // Start emission for the construct.
430 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo(
431 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel);
432 if (NoInheritance) {
433 std::swap(a&: CGF.LambdaCaptureFields, b&: LambdaCaptureFields);
434 LambdaThisCaptureField = CGF.LambdaThisCaptureField;
435 CGF.LambdaThisCaptureField = nullptr;
436 BlockInfo = CGF.BlockInfo;
437 CGF.BlockInfo = nullptr;
438 }
439 }
440
441 ~InlinedOpenMPRegionRAII() {
442 // Restore original CapturedStmtInfo only if we're done with code emission.
443 auto *OldCSI =
444 cast<CGOpenMPInlinedRegionInfo>(Val: CGF.CapturedStmtInfo)->getOldCSI();
445 delete CGF.CapturedStmtInfo;
446 CGF.CapturedStmtInfo = OldCSI;
447 if (NoInheritance) {
448 std::swap(a&: CGF.LambdaCaptureFields, b&: LambdaCaptureFields);
449 CGF.LambdaThisCaptureField = LambdaThisCaptureField;
450 CGF.BlockInfo = BlockInfo;
451 }
452 }
453};
454
455/// Values for bit flags used in the ident_t to describe the fields.
456/// All enumeric elements are named and described in accordance with the code
457/// from https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
458enum OpenMPLocationFlags : unsigned {
459 /// Use trampoline for internal microtask.
460 OMP_IDENT_IMD = 0x01,
461 /// Use c-style ident structure.
462 OMP_IDENT_KMPC = 0x02,
463 /// Atomic reduction option for kmpc_reduce.
464 OMP_ATOMIC_REDUCE = 0x10,
465 /// Explicit 'barrier' directive.
466 OMP_IDENT_BARRIER_EXPL = 0x20,
467 /// Implicit barrier in code.
468 OMP_IDENT_BARRIER_IMPL = 0x40,
469 /// Implicit barrier in 'for' directive.
470 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
471 /// Implicit barrier in 'sections' directive.
472 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
473 /// Implicit barrier in 'single' directive.
474 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,
475 /// Call of __kmp_for_static_init for static loop.
476 OMP_IDENT_WORK_LOOP = 0x200,
477 /// Call of __kmp_for_static_init for sections.
478 OMP_IDENT_WORK_SECTIONS = 0x400,
479 /// Call of __kmp_for_static_init for distribute.
480 OMP_IDENT_WORK_DISTRIBUTE = 0x800,
481 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE)
482};
483
484/// Describes ident structure that describes a source location.
485/// All descriptions are taken from
486/// https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h
487/// Original structure:
488/// typedef struct ident {
489/// kmp_int32 reserved_1; /**< might be used in Fortran;
490/// see above */
491/// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags;
492/// KMP_IDENT_KMPC identifies this union
493/// member */
494/// kmp_int32 reserved_2; /**< not really used in Fortran any more;
495/// see above */
496///#if USE_ITT_BUILD
497/// /* but currently used for storing
498/// region-specific ITT */
499/// /* contextual information. */
500///#endif /* USE_ITT_BUILD */
501/// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for
502/// C++ */
503/// char const *psource; /**< String describing the source location.
504/// The string is composed of semi-colon separated
505// fields which describe the source file,
506/// the function and a pair of line numbers that
507/// delimit the construct.
508/// */
509/// } ident_t;
510enum IdentFieldIndex {
511 /// might be used in Fortran
512 IdentField_Reserved_1,
513 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
514 IdentField_Flags,
515 /// Not really used in Fortran any more
516 IdentField_Reserved_2,
517 /// Source[4] in Fortran, do not use for C++
518 IdentField_Reserved_3,
519 /// String describing the source location. The string is composed of
520 /// semi-colon separated fields which describe the source file, the function
521 /// and a pair of line numbers that delimit the construct.
522 IdentField_PSource
523};
524
525/// Schedule types for 'omp for' loops (these enumerators are taken from
526/// the enum sched_type in kmp.h).
527enum OpenMPSchedType {
528 /// Lower bound for default (unordered) versions.
529 OMP_sch_lower = 32,
530 OMP_sch_static_chunked = 33,
531 OMP_sch_static = 34,
532 OMP_sch_dynamic_chunked = 35,
533 OMP_sch_guided_chunked = 36,
534 OMP_sch_runtime = 37,
535 OMP_sch_auto = 38,
536 /// static with chunk adjustment (e.g., simd)
537 OMP_sch_static_balanced_chunked = 45,
538 /// Lower bound for 'ordered' versions.
539 OMP_ord_lower = 64,
540 OMP_ord_static_chunked = 65,
541 OMP_ord_static = 66,
542 OMP_ord_dynamic_chunked = 67,
543 OMP_ord_guided_chunked = 68,
544 OMP_ord_runtime = 69,
545 OMP_ord_auto = 70,
546 OMP_sch_default = OMP_sch_static,
547 /// dist_schedule types
548 OMP_dist_sch_static_chunked = 91,
549 OMP_dist_sch_static = 92,
550 /// Fused distribute+for static schedule (entityId = team*nthreads + tid,
551 /// num_entities = nteams*nthreads). One for_static_init call, no
552 /// surrounding distribute_static_init. Matches
553 /// kmp_sched_distr_static_chunk_sched_static_chunkone in the device RTL
554 /// (openmp/device/include/DeviceTypes.h).
555 OMP_dist_sch_static_chunked_sch_static_chunkone = 93,
556 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
557 /// Set if the monotonic schedule modifier was present.
558 OMP_sch_modifier_monotonic = (1 << 29),
559 /// Set if the nonmonotonic schedule modifier was present.
560 OMP_sch_modifier_nonmonotonic = (1 << 30),
561};
562
563/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
564/// region.
565class CleanupTy final : public EHScopeStack::Cleanup {
566 PrePostActionTy *Action;
567
568public:
569 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {}
570 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
571 if (!CGF.HaveInsertPoint())
572 return;
573 Action->Exit(CGF);
574 }
575};
576
577} // anonymous namespace
578
579void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const {
580 CodeGenFunction::RunCleanupsScope Scope(CGF);
581 if (PrePostAction) {
582 CGF.EHStack.pushCleanup<CleanupTy>(Kind: NormalAndEHCleanup, A: PrePostAction);
583 Callback(CodeGen, CGF, *PrePostAction);
584 } else {
585 PrePostActionTy Action;
586 Callback(CodeGen, CGF, Action);
587 }
588}
589
590/// Check if the combiner is a call to UDR combiner and if it is so return the
591/// UDR decl used for reduction.
592static const OMPDeclareReductionDecl *
593getReductionInit(const Expr *ReductionOp) {
594 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp))
595 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Val: CE->getCallee()))
596 if (const auto *DRE =
597 dyn_cast<DeclRefExpr>(Val: OVE->getSourceExpr()->IgnoreImpCasts()))
598 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(Val: DRE->getDecl()))
599 return DRD;
600 return nullptr;
601}
602
603static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
604 const OMPDeclareReductionDecl *DRD,
605 const Expr *InitOp,
606 Address Private, Address Original,
607 QualType Ty) {
608 if (DRD->getInitializer()) {
609 std::pair<llvm::Function *, llvm::Function *> Reduction =
610 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(D: DRD);
611 const auto *CE = cast<CallExpr>(Val: InitOp);
612 const auto *OVE = cast<OpaqueValueExpr>(Val: CE->getCallee());
613 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
614 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
615 const auto *LHSDRE =
616 cast<DeclRefExpr>(Val: cast<UnaryOperator>(Val: LHS)->getSubExpr());
617 const auto *RHSDRE =
618 cast<DeclRefExpr>(Val: cast<UnaryOperator>(Val: RHS)->getSubExpr());
619 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
620 PrivateScope.addPrivate(LocalVD: cast<VarDecl>(Val: LHSDRE->getDecl()), Addr: Private);
621 PrivateScope.addPrivate(LocalVD: cast<VarDecl>(Val: RHSDRE->getDecl()), Addr: Original);
622 (void)PrivateScope.Privatize();
623 RValue Func = RValue::get(V: Reduction.second);
624 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
625 CGF.EmitIgnoredExpr(E: InitOp);
626 } else {
627 llvm::Constant *Init = CGF.CGM.EmitNullConstant(T: Ty);
628 std::string Name = CGF.CGM.getOpenMPRuntime().getName(Parts: {"init"});
629 auto *GV = new llvm::GlobalVariable(
630 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
631 llvm::GlobalValue::PrivateLinkage, Init, Name);
632 LValue LV = CGF.MakeNaturalAlignRawAddrLValue(V: GV, T: Ty);
633 RValue InitRVal;
634 switch (CGF.getEvaluationKind(T: Ty)) {
635 case TEK_Scalar:
636 InitRVal = CGF.EmitLoadOfLValue(V: LV, Loc: DRD->getLocation());
637 break;
638 case TEK_Complex:
639 InitRVal =
640 RValue::getComplex(C: CGF.EmitLoadOfComplex(src: LV, loc: DRD->getLocation()));
641 break;
642 case TEK_Aggregate: {
643 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_LValue);
644 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);
645 CGF.EmitAnyExprToMem(E: &OVE, Location: Private, Quals: Ty.getQualifiers(),
646 /*IsInitializer=*/false);
647 return;
648 }
649 }
650 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_PRValue);
651 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
652 CGF.EmitAnyExprToMem(E: &OVE, Location: Private, Quals: Ty.getQualifiers(),
653 /*IsInitializer=*/false);
654 }
655}
656
657/// Emit initialization of arrays of complex types.
658/// \param DestAddr Address of the array.
659/// \param Type Type of array.
660/// \param Init Initial expression of array.
661/// \param SrcAddr Address of the original array.
662static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
663 QualType Type, bool EmitDeclareReductionInit,
664 const Expr *Init,
665 const OMPDeclareReductionDecl *DRD,
666 Address SrcAddr = Address::invalid()) {
667 // Perform element-by-element initialization.
668 QualType ElementTy;
669
670 // Drill down to the base element type on both arrays.
671 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
672 llvm::Value *NumElements = CGF.emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: DestAddr);
673 if (DRD)
674 SrcAddr = SrcAddr.withElementType(ElemTy: DestAddr.getElementType());
675
676 llvm::Value *SrcBegin = nullptr;
677 if (DRD)
678 SrcBegin = SrcAddr.emitRawPointer(CGF);
679 llvm::Value *DestBegin = DestAddr.emitRawPointer(CGF);
680 // Cast from pointer to array type to pointer to single element.
681 llvm::Value *DestEnd =
682 CGF.Builder.CreateGEP(Ty: DestAddr.getElementType(), Ptr: DestBegin, IdxList: NumElements);
683 // The basic structure here is a while-do loop.
684 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.arrayinit.body");
685 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.arrayinit.done");
686 llvm::Value *IsEmpty =
687 CGF.Builder.CreateICmpEQ(LHS: DestBegin, RHS: DestEnd, Name: "omp.arrayinit.isempty");
688 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
689
690 // Enter the loop body, making that address the current address.
691 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
692 CGF.EmitBlock(BB: BodyBB);
693
694 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(T: ElementTy);
695
696 llvm::PHINode *SrcElementPHI = nullptr;
697 Address SrcElementCurrent = Address::invalid();
698 if (DRD) {
699 SrcElementPHI = CGF.Builder.CreatePHI(Ty: SrcBegin->getType(), NumReservedValues: 2,
700 Name: "omp.arraycpy.srcElementPast");
701 SrcElementPHI->addIncoming(V: SrcBegin, BB: EntryBB);
702 SrcElementCurrent =
703 Address(SrcElementPHI, SrcAddr.getElementType(),
704 SrcAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
705 }
706 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
707 Ty: DestBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
708 DestElementPHI->addIncoming(V: DestBegin, BB: EntryBB);
709 Address DestElementCurrent =
710 Address(DestElementPHI, DestAddr.getElementType(),
711 DestAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
712
713 // Emit copy.
714 {
715 CodeGenFunction::RunCleanupsScope InitScope(CGF);
716 if (EmitDeclareReductionInit) {
717 emitInitWithReductionInitializer(CGF, DRD, InitOp: Init, Private: DestElementCurrent,
718 Original: SrcElementCurrent, Ty: ElementTy);
719 } else
720 CGF.EmitAnyExprToMem(E: Init, Location: DestElementCurrent, Quals: ElementTy.getQualifiers(),
721 /*IsInitializer=*/false);
722 }
723
724 if (DRD) {
725 // Shift the address forward by one element.
726 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32(
727 Ty: SrcAddr.getElementType(), Ptr: SrcElementPHI, /*Idx0=*/1,
728 Name: "omp.arraycpy.dest.element");
729 SrcElementPHI->addIncoming(V: SrcElementNext, BB: CGF.Builder.GetInsertBlock());
730 }
731
732 // Shift the address forward by one element.
733 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32(
734 Ty: DestAddr.getElementType(), Ptr: DestElementPHI, /*Idx0=*/1,
735 Name: "omp.arraycpy.dest.element");
736 // Check whether we've reached the end.
737 llvm::Value *Done =
738 CGF.Builder.CreateICmpEQ(LHS: DestElementNext, RHS: DestEnd, Name: "omp.arraycpy.done");
739 CGF.Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
740 DestElementPHI->addIncoming(V: DestElementNext, BB: CGF.Builder.GetInsertBlock());
741
742 // Done.
743 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
744}
745
746LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) {
747 return CGF.EmitOMPSharedLValue(E);
748}
749
750LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF,
751 const Expr *E) {
752 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E))
753 return CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false);
754 return LValue();
755}
756
757void ReductionCodeGen::emitAggregateInitialization(
758 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
759 const OMPDeclareReductionDecl *DRD) {
760 // Emit VarDecl with copy init for arrays.
761 // Get the address of the original variable captured in current
762 // captured region.
763 const auto *PrivateVD =
764 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl());
765 bool EmitDeclareReductionInit =
766 DRD && (DRD->getInitializer() || !PrivateVD->hasInit());
767 EmitOMPAggregateInit(CGF, DestAddr: PrivateAddr, Type: PrivateVD->getType(),
768 EmitDeclareReductionInit,
769 Init: EmitDeclareReductionInit ? ClausesData[N].ReductionOp
770 : PrivateVD->getInit(),
771 DRD, SrcAddr: SharedAddr);
772}
773
774ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds,
775 ArrayRef<const Expr *> Origs,
776 ArrayRef<const Expr *> Privates,
777 ArrayRef<const Expr *> ReductionOps) {
778 ClausesData.reserve(N: Shareds.size());
779 SharedAddresses.reserve(N: Shareds.size());
780 Sizes.reserve(N: Shareds.size());
781 BaseDecls.reserve(N: Shareds.size());
782 const auto *IOrig = Origs.begin();
783 const auto *IPriv = Privates.begin();
784 const auto *IRed = ReductionOps.begin();
785 for (const Expr *Ref : Shareds) {
786 ClausesData.emplace_back(Args&: Ref, Args: *IOrig, Args: *IPriv, Args: *IRed);
787 std::advance(i&: IOrig, n: 1);
788 std::advance(i&: IPriv, n: 1);
789 std::advance(i&: IRed, n: 1);
790 }
791}
792
793void ReductionCodeGen::emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N) {
794 assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&
795 "Number of generated lvalues must be exactly N.");
796 LValue First = emitSharedLValue(CGF, E: ClausesData[N].Shared);
797 LValue Second = emitSharedLValueUB(CGF, E: ClausesData[N].Shared);
798 SharedAddresses.emplace_back(Args&: First, Args&: Second);
799 if (ClausesData[N].Shared == ClausesData[N].Ref) {
800 OrigAddresses.emplace_back(Args&: First, Args&: Second);
801 } else {
802 LValue First = emitSharedLValue(CGF, E: ClausesData[N].Ref);
803 LValue Second = emitSharedLValueUB(CGF, E: ClausesData[N].Ref);
804 OrigAddresses.emplace_back(Args&: First, Args&: Second);
805 }
806}
807
808void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) {
809 QualType PrivateType = getPrivateType(N);
810 bool AsArraySection = isa<ArraySectionExpr>(Val: ClausesData[N].Ref);
811 if (!PrivateType->isVariablyModifiedType()) {
812 Sizes.emplace_back(
813 Args: CGF.getTypeSize(Ty: OrigAddresses[N].first.getType().getNonReferenceType()),
814 Args: nullptr);
815 return;
816 }
817 llvm::Value *Size;
818 llvm::Value *SizeInChars;
819 auto *ElemType = OrigAddresses[N].first.getAddress().getElementType();
820 auto *ElemSizeOf = llvm::ConstantInt::get(
821 Ty: CGF.SizeTy, V: CGF.CGM.getDataLayout().getTypeAllocSize(Ty: ElemType));
822 if (AsArraySection) {
823 SizeInChars =
824 CGF.Builder.CreatePtrDiff(LHS: OrigAddresses[N].second.getPointer(CGF),
825 RHS: OrigAddresses[N].first.getPointer(CGF));
826 SizeInChars = CGF.Builder.CreateNUWAdd(LHS: SizeInChars, RHS: ElemSizeOf);
827 } else {
828 SizeInChars =
829 CGF.getTypeSize(Ty: OrigAddresses[N].first.getType().getNonReferenceType());
830 }
831 Size = ElemSizeOf->isOne()
832 ? SizeInChars
833 : CGF.Builder.CreateExactUDiv(LHS: SizeInChars, RHS: ElemSizeOf);
834 Sizes.emplace_back(Args&: SizeInChars, Args&: Size);
835 CodeGenFunction::OpaqueValueMapping OpaqueMap(
836 CGF,
837 cast<OpaqueValueExpr>(
838 Val: CGF.getContext().getAsVariableArrayType(T: PrivateType)->getSizeExpr()),
839 RValue::get(V: Size));
840 CGF.EmitVariablyModifiedType(Ty: PrivateType);
841}
842
843void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N,
844 llvm::Value *Size) {
845 QualType PrivateType = getPrivateType(N);
846 if (!PrivateType->isVariablyModifiedType()) {
847 assert(!Size && !Sizes[N].second &&
848 "Size should be nullptr for non-variably modified reduction "
849 "items.");
850 return;
851 }
852 CodeGenFunction::OpaqueValueMapping OpaqueMap(
853 CGF,
854 cast<OpaqueValueExpr>(
855 Val: CGF.getContext().getAsVariableArrayType(T: PrivateType)->getSizeExpr()),
856 RValue::get(V: Size));
857 CGF.EmitVariablyModifiedType(Ty: PrivateType);
858}
859
860void ReductionCodeGen::emitInitialization(
861 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, Address SharedAddr,
862 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) {
863 assert(SharedAddresses.size() > N && "No variable was generated");
864 const auto *PrivateVD =
865 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl());
866 const OMPDeclareReductionDecl *DRD =
867 getReductionInit(ReductionOp: ClausesData[N].ReductionOp);
868 if (CGF.getContext().getAsArrayType(T: PrivateVD->getType())) {
869 if (DRD && DRD->getInitializer())
870 (void)DefaultInit(CGF);
871 emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);
872 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
873 (void)DefaultInit(CGF);
874 QualType SharedType = SharedAddresses[N].first.getType();
875 emitInitWithReductionInitializer(CGF, DRD, InitOp: ClausesData[N].ReductionOp,
876 Private: PrivateAddr, Original: SharedAddr, Ty: SharedType);
877 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() &&
878 !CGF.isTrivialInitializer(Init: PrivateVD->getInit())) {
879 CGF.EmitAnyExprToMem(E: PrivateVD->getInit(), Location: PrivateAddr,
880 Quals: PrivateVD->getType().getQualifiers(),
881 /*IsInitializer=*/false);
882 }
883}
884
885bool ReductionCodeGen::needCleanups(unsigned N) {
886 QualType PrivateType = getPrivateType(N);
887 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
888 return DTorKind != QualType::DK_none;
889}
890
891void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N,
892 Address PrivateAddr) {
893 QualType PrivateType = getPrivateType(N);
894 QualType::DestructionKind DTorKind = PrivateType.isDestructedType();
895 if (needCleanups(N)) {
896 PrivateAddr =
897 PrivateAddr.withElementType(ElemTy: CGF.ConvertTypeForMem(T: PrivateType));
898 CGF.pushDestroy(dtorKind: DTorKind, addr: PrivateAddr, type: PrivateType);
899 }
900}
901
902static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
903 LValue BaseLV) {
904 BaseTy = BaseTy.getNonReferenceType();
905 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
906 !CGF.getContext().hasSameType(T1: BaseTy, T2: ElTy)) {
907 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) {
908 BaseLV = CGF.EmitLoadOfPointerLValue(Ptr: BaseLV.getAddress(), PtrTy);
909 } else {
910 LValue RefLVal = CGF.MakeAddrLValue(Addr: BaseLV.getAddress(), T: BaseTy);
911 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal);
912 }
913 BaseTy = BaseTy->getPointeeType();
914 }
915 return CGF.MakeAddrLValue(
916 Addr: BaseLV.getAddress().withElementType(ElemTy: CGF.ConvertTypeForMem(T: ElTy)),
917 T: BaseLV.getType(), BaseInfo: BaseLV.getBaseInfo(),
918 TBAAInfo: CGF.CGM.getTBAAInfoForSubobject(Base: BaseLV, AccessType: BaseLV.getType()));
919}
920
921static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy,
922 Address OriginalBaseAddress, llvm::Value *Addr) {
923 RawAddress Tmp = RawAddress::invalid();
924 Address TopTmp = Address::invalid();
925 Address MostTopTmp = Address::invalid();
926 BaseTy = BaseTy.getNonReferenceType();
927 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) &&
928 !CGF.getContext().hasSameType(T1: BaseTy, T2: ElTy)) {
929 Tmp = CGF.CreateMemTempWithoutCast(T: BaseTy);
930 if (TopTmp.isValid())
931 CGF.Builder.CreateStore(Val: Tmp.getPointer(), Addr: TopTmp);
932 else
933 MostTopTmp = Tmp;
934 TopTmp = Tmp;
935 BaseTy = BaseTy->getPointeeType();
936 }
937
938 if (Tmp.isValid()) {
939 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
940 V: Addr, DestTy: Tmp.getElementType());
941 CGF.Builder.CreateStore(Val: Addr, Addr: Tmp);
942 return MostTopTmp;
943 }
944
945 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
946 V: Addr, DestTy: OriginalBaseAddress.getType());
947 return OriginalBaseAddress.withPointer(NewPointer: Addr, IsKnownNonNull: NotKnownNonNull);
948}
949
950static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) {
951 const VarDecl *OrigVD = nullptr;
952 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: Ref)) {
953 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
954 while (const auto *TempOASE = dyn_cast<ArraySectionExpr>(Val: Base))
955 Base = TempOASE->getBase()->IgnoreParenImpCasts();
956 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
957 Base = TempASE->getBase()->IgnoreParenImpCasts();
958 DE = cast<DeclRefExpr>(Val: Base);
959 OrigVD = cast<VarDecl>(Val: DE->getDecl());
960 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: Ref)) {
961 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
962 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Val: Base))
963 Base = TempASE->getBase()->IgnoreParenImpCasts();
964 DE = cast<DeclRefExpr>(Val: Base);
965 OrigVD = cast<VarDecl>(Val: DE->getDecl());
966 }
967 return OrigVD;
968}
969
970Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
971 Address PrivateAddr) {
972 const DeclRefExpr *DE;
973 if (const VarDecl *OrigVD = ::getBaseDecl(Ref: ClausesData[N].Ref, DE)) {
974 BaseDecls.emplace_back(Args&: OrigVD);
975 LValue OriginalBaseLValue = CGF.EmitLValue(E: DE);
976 LValue BaseLValue =
977 loadToBegin(CGF, BaseTy: OrigVD->getType(), ElTy: SharedAddresses[N].first.getType(),
978 BaseLV: OriginalBaseLValue);
979 Address SharedAddr = SharedAddresses[N].first.getAddress();
980 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff(
981 ElemTy: SharedAddr.getElementType(), LHS: BaseLValue.getPointer(CGF),
982 RHS: SharedAddr.emitRawPointer(CGF));
983 llvm::Value *PrivatePointer =
984 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
985 V: PrivateAddr.emitRawPointer(CGF), DestTy: SharedAddr.getType());
986 llvm::Value *Ptr = CGF.Builder.CreateGEP(
987 Ty: SharedAddr.getElementType(), Ptr: PrivatePointer, IdxList: Adjustment);
988 return castToBase(CGF, BaseTy: OrigVD->getType(),
989 ElTy: SharedAddresses[N].first.getType(),
990 OriginalBaseAddress: OriginalBaseLValue.getAddress(), Addr: Ptr);
991 }
992 BaseDecls.emplace_back(
993 Args: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Ref)->getDecl()));
994 return PrivateAddr;
995}
996
997bool ReductionCodeGen::usesReductionInitializer(unsigned N) const {
998 const OMPDeclareReductionDecl *DRD =
999 getReductionInit(ReductionOp: ClausesData[N].ReductionOp);
1000 return DRD && DRD->getInitializer();
1001}
1002
1003LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
1004 return CGF.EmitLoadOfPointerLValue(
1005 Ptr: CGF.GetAddrOfLocalVar(VD: getThreadIDVariable()),
1006 PtrTy: getThreadIDVariable()->getType()->castAs<PointerType>());
1007}
1008
1009void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt *S) {
1010 if (!CGF.HaveInsertPoint())
1011 return;
1012 // 1.2.2 OpenMP Language Terminology
1013 // Structured block - An executable statement with a single entry at the
1014 // top and a single exit at the bottom.
1015 // The point of exit cannot be a branch out of the structured block.
1016 // longjmp() and throw() must not violate the entry/exit criteria.
1017 CGF.EHStack.pushTerminate();
1018 if (S)
1019 CGF.incrementProfileCounter(S);
1020 CodeGen(CGF);
1021 CGF.EHStack.popTerminate();
1022}
1023
1024LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
1025 CodeGenFunction &CGF) {
1026 return CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: getThreadIDVariable()),
1027 T: getThreadIDVariable()->getType(),
1028 Source: AlignmentSource::Decl);
1029}
1030
1031static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
1032 QualType FieldTy) {
1033 auto *Field = FieldDecl::Create(
1034 C, DC, StartLoc: SourceLocation(), IdLoc: SourceLocation(), /*Id=*/nullptr, T: FieldTy,
1035 TInfo: C.getTrivialTypeSourceInfo(T: FieldTy, Loc: SourceLocation()),
1036 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit);
1037 Field->setAccess(AS_public);
1038 DC->addDecl(D: Field);
1039 return Field;
1040}
1041
1042CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
1043 : CGM(CGM), OMPBuilder(CGM.getModule()) {
1044 KmpCriticalNameTy = llvm::ArrayType::get(ElementType: CGM.Int32Ty, /*NumElements*/ 8);
1045 llvm::OpenMPIRBuilderConfig Config(
1046 CGM.getLangOpts().OpenMPIsTargetDevice, isGPU(),
1047 CGM.getLangOpts().OpenMPOffloadMandatory,
1048 /*HasRequiresReverseOffload*/ false, /*HasRequiresUnifiedAddress*/ false,
1049 hasRequiresUnifiedSharedMemory(), /*HasRequiresDynamicAllocators*/ false);
1050 Config.setDefaultTargetAS(
1051 CGM.getContext().getTargetInfo().getTargetAddressSpace(AS: LangAS::Default));
1052 Config.setRuntimeCC(CGM.getRuntimeCC());
1053
1054 OMPBuilder.setConfig(Config);
1055 OMPBuilder.initialize();
1056 OMPBuilder.loadOffloadInfoMetadata(VFS&: *CGM.getFileSystem(),
1057 HostFilePath: CGM.getLangOpts().OpenMPIsTargetDevice
1058 ? CGM.getLangOpts().OMPHostIRFile
1059 : StringRef{});
1060
1061 // The user forces the compiler to behave as if omp requires
1062 // unified_shared_memory was given.
1063 if (CGM.getLangOpts().OpenMPForceUSM) {
1064 HasRequiresUnifiedSharedMemory = true;
1065 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
1066 }
1067}
1068
1069void CGOpenMPRuntime::clear() {
1070 InternalVars.clear();
1071 // Clean non-target variable declarations possibly used only in debug info.
1072 for (const auto &Data : EmittedNonTargetVariables) {
1073 if (!Data.getValue().pointsToAliveValue())
1074 continue;
1075 auto *GV = dyn_cast<llvm::GlobalVariable>(Val: Data.getValue());
1076 if (!GV)
1077 continue;
1078 if (!GV->isDeclaration() || GV->getNumUses() > 0)
1079 continue;
1080 GV->eraseFromParent();
1081 }
1082}
1083
1084std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
1085 return OMPBuilder.createPlatformSpecificName(Parts);
1086}
1087
1088static llvm::Function *
1089emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
1090 const Expr *CombinerInitializer, const VarDecl *In,
1091 const VarDecl *Out, bool IsCombiner) {
1092 // void .omp_combiner.(Ty *in, Ty *out);
1093 ASTContext &C = CGM.getContext();
1094 QualType PtrTy = C.getPointerType(T: Ty).withRestrict();
1095 auto *OmpOutParm = ImplicitParamDecl::Create(
1096 C, /*DC=*/nullptr, IdLoc: Out->getLocation(),
1097 /*Id=*/nullptr, T: PtrTy, ParamKind: ImplicitParamKind::Other);
1098 auto *OmpInParm = ImplicitParamDecl::Create(
1099 C, /*DC=*/nullptr, IdLoc: In->getLocation(),
1100 /*Id=*/nullptr, T: PtrTy, ParamKind: ImplicitParamKind::Other);
1101 FunctionArgList Args{OmpOutParm, OmpInParm};
1102 const CGFunctionInfo &FnInfo =
1103 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
1104 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
1105 std::string Name = CGM.getOpenMPRuntime().getName(
1106 Parts: {IsCombiner ? "omp_combiner" : "omp_initializer", ""});
1107 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
1108 N: Name, M: &CGM.getModule());
1109 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
1110 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
1111 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
1112 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
1113 Fn->removeFnAttr(Kind: llvm::Attribute::NoInline);
1114 Fn->removeFnAttr(Kind: llvm::Attribute::OptimizeNone);
1115 Fn->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
1116 }
1117 CodeGenFunction CGF(CGM);
1118 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
1119 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
1120 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc: In->getLocation(),
1121 StartLoc: Out->getLocation());
1122 CodeGenFunction::OMPPrivateScope Scope(CGF);
1123 Address AddrIn = CGF.GetAddrOfLocalVar(VD: OmpInParm);
1124 Scope.addPrivate(
1125 LocalVD: In, Addr: CGF.EmitLoadOfPointerLValue(Ptr: AddrIn, PtrTy: PtrTy->castAs<PointerType>())
1126 .getAddress());
1127 Address AddrOut = CGF.GetAddrOfLocalVar(VD: OmpOutParm);
1128 Scope.addPrivate(
1129 LocalVD: Out, Addr: CGF.EmitLoadOfPointerLValue(Ptr: AddrOut, PtrTy: PtrTy->castAs<PointerType>())
1130 .getAddress());
1131 (void)Scope.Privatize();
1132 if (!IsCombiner && Out->hasInit() &&
1133 !CGF.isTrivialInitializer(Init: Out->getInit())) {
1134 CGF.EmitAnyExprToMem(E: Out->getInit(), Location: CGF.GetAddrOfLocalVar(VD: Out),
1135 Quals: Out->getType().getQualifiers(),
1136 /*IsInitializer=*/true);
1137 }
1138 if (CombinerInitializer)
1139 CGF.EmitIgnoredExpr(E: CombinerInitializer);
1140 Scope.ForceCleanup();
1141 CGF.FinishFunction();
1142 return Fn;
1143}
1144
1145void CGOpenMPRuntime::emitUserDefinedReduction(
1146 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) {
1147 if (UDRMap.count(Val: D) > 0)
1148 return;
1149 llvm::Function *Combiner = emitCombinerOrInitializer(
1150 CGM, Ty: D->getType(), CombinerInitializer: D->getCombiner(),
1151 In: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getCombinerIn())->getDecl()),
1152 Out: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getCombinerOut())->getDecl()),
1153 /*IsCombiner=*/true);
1154 llvm::Function *Initializer = nullptr;
1155 if (const Expr *Init = D->getInitializer()) {
1156 Initializer = emitCombinerOrInitializer(
1157 CGM, Ty: D->getType(),
1158 CombinerInitializer: D->getInitializerKind() == OMPDeclareReductionInitKind::Call ? Init
1159 : nullptr,
1160 In: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getInitOrig())->getDecl()),
1161 Out: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getInitPriv())->getDecl()),
1162 /*IsCombiner=*/false);
1163 }
1164 UDRMap.try_emplace(Key: D, Args&: Combiner, Args&: Initializer);
1165 if (CGF)
1166 FunctionUDRMap[CGF->CurFn].push_back(Elt: D);
1167}
1168
1169std::pair<llvm::Function *, llvm::Function *>
1170CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
1171 auto I = UDRMap.find(Val: D);
1172 if (I != UDRMap.end())
1173 return I->second;
1174 emitUserDefinedReduction(/*CGF=*/nullptr, D);
1175 return UDRMap.lookup(Val: D);
1176}
1177
1178namespace {
1179// Temporary RAII solution to perform a push/pop stack event on the OpenMP IR
1180// Builder if one is present.
1181struct PushAndPopStackRAII {
1182 PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder, CodeGenFunction &CGF,
1183 bool HasCancel, llvm::omp::Directive Kind)
1184 : OMPBuilder(OMPBuilder) {
1185 if (!OMPBuilder)
1186 return;
1187
1188 // The following callback is the crucial part of clangs cleanup process.
1189 //
1190 // NOTE:
1191 // Once the OpenMPIRBuilder is used to create parallel regions (and
1192 // similar), the cancellation destination (Dest below) is determined via
1193 // IP. That means if we have variables to finalize we split the block at IP,
1194 // use the new block (=BB) as destination to build a JumpDest (via
1195 // getJumpDestInCurrentScope(BB)) which then is fed to
1196 // EmitBranchThroughCleanup. Furthermore, there will not be the need
1197 // to push & pop an FinalizationInfo object.
1198 // The FiniCB will still be needed but at the point where the
1199 // OpenMPIRBuilder is asked to construct a parallel (or similar) construct.
1200 auto FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {
1201 assert(IP.getBlock()->end() == IP.getPoint() &&
1202 "Clang CG should cause non-terminated block!");
1203 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1204 CGF.Builder.restoreIP(IP);
1205 CodeGenFunction::JumpDest Dest =
1206 CGF.getOMPCancelDestination(Kind: OMPD_parallel);
1207 CGF.EmitBranchThroughCleanup(Dest);
1208 return llvm::Error::success();
1209 };
1210
1211 // TODO: Remove this once we emit parallel regions through the
1212 // OpenMPIRBuilder as it can do this setup internally.
1213 llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB, Kind, HasCancel});
1214 OMPBuilder->pushFinalizationCB(FI: std::move(FI));
1215 }
1216 ~PushAndPopStackRAII() {
1217 if (OMPBuilder)
1218 OMPBuilder->popFinalizationCB();
1219 }
1220 llvm::OpenMPIRBuilder *OMPBuilder;
1221};
1222} // namespace
1223
1224static llvm::Function *emitParallelOrTeamsOutlinedFunction(
1225 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS,
1226 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1227 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) {
1228 assert(ThreadIDVar->getType()->isPointerType() &&
1229 "thread id variable must be of type kmp_int32 *");
1230 CodeGenFunction CGF(CGM, true);
1231 bool HasCancel = false;
1232 if (const auto *OPD = dyn_cast<OMPParallelDirective>(Val: &D))
1233 HasCancel = OPD->hasCancel();
1234 else if (const auto *OPD = dyn_cast<OMPTargetParallelDirective>(Val: &D))
1235 HasCancel = OPD->hasCancel();
1236 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(Val: &D))
1237 HasCancel = OPSD->hasCancel();
1238 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(Val: &D))
1239 HasCancel = OPFD->hasCancel();
1240 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(Val: &D))
1241 HasCancel = OPFD->hasCancel();
1242 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(Val: &D))
1243 HasCancel = OPFD->hasCancel();
1244 else if (const auto *OPFD =
1245 dyn_cast<OMPTeamsDistributeParallelForDirective>(Val: &D))
1246 HasCancel = OPFD->hasCancel();
1247 else if (const auto *OPFD =
1248 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(Val: &D))
1249 HasCancel = OPFD->hasCancel();
1250
1251 // TODO: Temporarily inform the OpenMPIRBuilder, if any, about the new
1252 // parallel region to make cancellation barriers work properly.
1253 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1254 PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);
1255 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
1256 HasCancel, OutlinedHelperName);
1257 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1258 return CGF.GenerateOpenMPCapturedStmtFunction(S: *CS, D);
1259}
1260
1261std::string CGOpenMPRuntime::getOutlinedHelperName(StringRef Name) const {
1262 std::string Suffix = getName(Parts: {"omp_outlined"});
1263 return (Name + Suffix).str();
1264}
1265
1266std::string CGOpenMPRuntime::getOutlinedHelperName(CodeGenFunction &CGF) const {
1267 return getOutlinedHelperName(Name: CGF.CurFn->getName());
1268}
1269
1270std::string CGOpenMPRuntime::getReductionFuncName(StringRef Name) const {
1271 std::string Suffix = getName(Parts: {"omp", "reduction", "reduction_func"});
1272 return (Name + Suffix).str();
1273}
1274
1275llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction(
1276 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1277 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1278 const RegionCodeGenTy &CodeGen) {
1279 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: OMPD_parallel);
1280 return emitParallelOrTeamsOutlinedFunction(
1281 CGM, D, CS, ThreadIDVar, InnermostKind, OutlinedHelperName: getOutlinedHelperName(CGF),
1282 CodeGen);
1283}
1284
1285llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
1286 CodeGenFunction &CGF, const OMPExecutableDirective &D,
1287 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1288 const RegionCodeGenTy &CodeGen) {
1289 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: OMPD_teams);
1290 llvm::Function *OutlinedFn = emitParallelOrTeamsOutlinedFunction(
1291 CGM, D, CS, ThreadIDVar, InnermostKind, OutlinedHelperName: getOutlinedHelperName(CGF),
1292 CodeGen);
1293 // A teams body is called once per team and is not handed back to the runtime
1294 // as a callback, so unlike a parallel body it cannot be re-entered while a
1295 // call to it is live.
1296 OutlinedFn->setDoesNotRecurse();
1297 return OutlinedFn;
1298}
1299
1300llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
1301 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1302 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1303 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1304 bool Tied, unsigned &NumberOfParts) {
1305 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF,
1306 PrePostActionTy &) {
1307 llvm::Value *ThreadID = getThreadID(CGF, Loc: D.getBeginLoc());
1308 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
1309 llvm::Value *TaskArgs[] = {
1310 UpLoc, ThreadID,
1311 CGF.EmitLoadOfPointerLValue(Ptr: CGF.GetAddrOfLocalVar(VD: TaskTVar),
1312 PtrTy: TaskTVar->getType()->castAs<PointerType>())
1313 .getPointer(CGF)};
1314 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1315 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task),
1316 args: TaskArgs);
1317 };
1318 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,
1319 UntiedCodeGen);
1320 CodeGen.setAction(Action);
1321 assert(!ThreadIDVar->getType()->isPointerType() &&
1322 "thread id variable must be of type kmp_int32 for tasks");
1323 const OpenMPDirectiveKind Region =
1324 isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind()) ? OMPD_taskloop
1325 : OMPD_task;
1326 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: Region);
1327 bool HasCancel = false;
1328 if (const auto *TD = dyn_cast<OMPTaskDirective>(Val: &D))
1329 HasCancel = TD->hasCancel();
1330 else if (const auto *TD = dyn_cast<OMPTaskLoopDirective>(Val: &D))
1331 HasCancel = TD->hasCancel();
1332 else if (const auto *TD = dyn_cast<OMPMasterTaskLoopDirective>(Val: &D))
1333 HasCancel = TD->hasCancel();
1334 else if (const auto *TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(Val: &D))
1335 HasCancel = TD->hasCancel();
1336
1337 CodeGenFunction CGF(CGM, true);
1338 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
1339 InnermostKind, HasCancel, Action);
1340 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
1341 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(S: *CS);
1342 if (!Tied)
1343 NumberOfParts = Action.getNumberOfParts();
1344 return Res;
1345}
1346
1347void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF,
1348 bool AtCurrentPoint) {
1349 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1350 assert(!Elem.ServiceInsertPt && "Insert point is set already.");
1351
1352 llvm::Value *Undef = llvm::UndefValue::get(T: CGF.Int32Ty);
1353 if (AtCurrentPoint) {
1354 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt",
1355 CGF.Builder.GetInsertBlock());
1356 } else {
1357 Elem.ServiceInsertPt = new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt");
1358 Elem.ServiceInsertPt->insertAfter(InsertPos: CGF.AllocaInsertPt->getIterator());
1359 }
1360}
1361
1362void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) {
1363 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1364 if (Elem.ServiceInsertPt) {
1365 llvm::Instruction *Ptr = Elem.ServiceInsertPt;
1366 Elem.ServiceInsertPt = nullptr;
1367 Ptr->eraseFromParent();
1368 }
1369}
1370
1371static StringRef getIdentStringFromSourceLocation(CodeGenFunction &CGF,
1372 SourceLocation Loc,
1373 SmallString<128> &Buffer) {
1374 llvm::raw_svector_ostream OS(Buffer);
1375 // Build debug location
1376 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1377 OS << ";";
1378 if (auto *DbgInfo = CGF.getDebugInfo())
1379 OS << DbgInfo->remapDIPath(PLoc.getFilename());
1380 else
1381 OS << PLoc.getFilename();
1382 OS << ";";
1383 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CGF.CurFuncDecl))
1384 OS << FD->getQualifiedNameAsString();
1385 OS << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
1386 return OS.str();
1387}
1388
1389llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF,
1390 SourceLocation Loc,
1391 unsigned Flags, bool EmitLoc) {
1392 uint32_t SrcLocStrSize;
1393 llvm::Constant *SrcLocStr;
1394 if ((!EmitLoc && CGM.getCodeGenOpts().getDebugInfo() ==
1395 llvm::codegenoptions::NoDebugInfo) ||
1396 Loc.isInvalid()) {
1397 SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
1398 } else {
1399 std::string FunctionName;
1400 std::string FileName;
1401 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(Val: CGF.CurFuncDecl))
1402 FunctionName = FD->getQualifiedNameAsString();
1403 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
1404 if (auto *DbgInfo = CGF.getDebugInfo())
1405 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
1406 else
1407 FileName = PLoc.getFilename();
1408 unsigned Line = PLoc.getLine();
1409 unsigned Column = PLoc.getColumn();
1410 SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(FunctionName, FileName, Line,
1411 Column, SrcLocStrSize);
1412 }
1413 unsigned Reserved2Flags = getDefaultLocationReserved2Flags();
1414 return OMPBuilder.getOrCreateIdent(
1415 SrcLocStr, SrcLocStrSize, Flags: llvm::omp::IdentFlag(Flags), Reserve2Flags: Reserved2Flags);
1416}
1417
1418llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF,
1419 SourceLocation Loc) {
1420 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1421 // If the OpenMPIRBuilder is used we need to use it for all thread id calls as
1422 // the clang invariants used below might be broken.
1423 if (CGM.getLangOpts().OpenMPIRBuilder) {
1424 SmallString<128> Buffer;
1425 OMPBuilder.updateToLocation(Loc: CGF.Builder);
1426 uint32_t SrcLocStrSize;
1427 auto *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(
1428 LocStr: getIdentStringFromSourceLocation(CGF, Loc, Buffer), SrcLocStrSize);
1429 return OMPBuilder.getOrCreateThreadID(
1430 Ident: OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize));
1431 }
1432
1433 llvm::Value *ThreadID = nullptr;
1434 // Check whether we've already cached a load of the thread id in this
1435 // function.
1436 auto I = OpenMPLocThreadIDMap.find(Val: CGF.CurFn);
1437 if (I != OpenMPLocThreadIDMap.end()) {
1438 ThreadID = I->second.ThreadID;
1439 if (ThreadID != nullptr)
1440 return ThreadID;
1441 }
1442 // If exceptions are enabled, do not use parameter to avoid possible crash.
1443 if (auto *OMPRegionInfo =
1444 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
1445 if (OMPRegionInfo->getThreadIDVariable()) {
1446 // Check if this an outlined function with thread id passed as argument.
1447 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1448 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent();
1449 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions ||
1450 !CGF.getLangOpts().CXXExceptions ||
1451 CGF.Builder.GetInsertBlock() == TopBlock ||
1452 !isa<llvm::Instruction>(Val: LVal.getPointer(CGF)) ||
1453 cast<llvm::Instruction>(Val: LVal.getPointer(CGF))->getParent() ==
1454 TopBlock ||
1455 cast<llvm::Instruction>(Val: LVal.getPointer(CGF))->getParent() ==
1456 CGF.Builder.GetInsertBlock()) {
1457 ThreadID = CGF.EmitLoadOfScalar(lvalue: LVal, Loc);
1458 // If value loaded in entry block, cache it and use it everywhere in
1459 // function.
1460 if (CGF.Builder.GetInsertBlock() == TopBlock)
1461 OpenMPLocThreadIDMap[CGF.CurFn].ThreadID = ThreadID;
1462 return ThreadID;
1463 }
1464 }
1465 }
1466
1467 // This is not an outlined function region - need to call __kmpc_int32
1468 // kmpc_global_thread_num(ident_t *loc).
1469 // Generate thread id value and cache this value for use across the
1470 // function.
1471 auto &Elem = OpenMPLocThreadIDMap[CGF.CurFn];
1472 if (!Elem.ServiceInsertPt)
1473 setLocThreadIdInsertPt(CGF);
1474 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
1475 CGF.Builder.SetInsertPoint(Elem.ServiceInsertPt);
1476 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
1477 llvm::CallInst *Call = CGF.Builder.CreateCall(
1478 Callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
1479 FnID: OMPRTL___kmpc_global_thread_num),
1480 Args: emitUpdateLocation(CGF, Loc));
1481 Call->setCallingConv(CGF.getRuntimeCC());
1482 Elem.ThreadID = Call;
1483 return Call;
1484}
1485
1486void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) {
1487 assert(CGF.CurFn && "No function in current CodeGenFunction.");
1488 if (OpenMPLocThreadIDMap.count(Val: CGF.CurFn)) {
1489 clearLocThreadIdInsertPt(CGF);
1490 OpenMPLocThreadIDMap.erase(Val: CGF.CurFn);
1491 }
1492 if (auto I = FunctionUDRMap.find(Val: CGF.CurFn); I != FunctionUDRMap.end()) {
1493 for (const auto *D : I->second)
1494 UDRMap.erase(Val: D);
1495 FunctionUDRMap.erase(I);
1496 }
1497 if (auto I = FunctionUDMMap.find(Val: CGF.CurFn); I != FunctionUDMMap.end()) {
1498 for (const auto *D : I->second)
1499 UDMMap.erase(Val: D);
1500 FunctionUDMMap.erase(I);
1501 }
1502 LastprivateConditionalToTypes.erase(Val: CGF.CurFn);
1503 FunctionToUntiedTaskStackMap.erase(Val: CGF.CurFn);
1504}
1505
1506llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
1507 return OMPBuilder.IdentPtr;
1508}
1509
1510static llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind
1511convertDeviceClause(const VarDecl *VD) {
1512 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
1513 OMPDeclareTargetDeclAttr::getDeviceType(VD);
1514 if (!DevTy)
1515 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1516
1517 switch ((int)*DevTy) { // Avoid -Wcovered-switch-default
1518 case OMPDeclareTargetDeclAttr::DT_Host:
1519 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;
1520 break;
1521 case OMPDeclareTargetDeclAttr::DT_NoHost:
1522 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;
1523 break;
1524 case OMPDeclareTargetDeclAttr::DT_Any:
1525 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;
1526 break;
1527 default:
1528 return llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;
1529 break;
1530 }
1531}
1532
1533static llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind
1534convertCaptureClause(const VarDecl *VD) {
1535 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =
1536 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
1537 if (!MapType)
1538 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1539 switch ((int)*MapType) { // Avoid -Wcovered-switch-default
1540 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:
1541 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
1542 break;
1543 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:
1544 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;
1545 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:
1546 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
1547 break;
1548 case OMPDeclareTargetDeclAttr::MapTypeTy::MT_Local:
1549 // MT_Local variables don't need offload entry (device-local).
1550 llvm_unreachable("MT_Local should not reach convertCaptureClause");
1551 break;
1552 default:
1553 return llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;
1554 break;
1555 }
1556}
1557
1558static llvm::TargetRegionEntryInfo getEntryInfoFromPresumedLoc(
1559 CodeGenModule &CGM, llvm::OpenMPIRBuilder &OMPBuilder,
1560 SourceLocation BeginLoc, llvm::StringRef ParentName = "") {
1561
1562 auto FileInfoCallBack = [&]() {
1563 SourceManager &SM = CGM.getContext().getSourceManager();
1564 PresumedLoc PLoc = SM.getPresumedLoc(Loc: BeginLoc);
1565
1566 if (!CGM.getFileSystem()->exists(Path: PLoc.getFilename()))
1567 PLoc = SM.getPresumedLoc(Loc: BeginLoc, /*UseLineDirectives=*/false);
1568
1569 return std::pair<std::string, uint64_t>(PLoc.getFilename(), PLoc.getLine());
1570 };
1571
1572 return OMPBuilder.getTargetEntryUniqueInfo(CallBack: FileInfoCallBack,
1573 VFS&: *CGM.getFileSystem(), ParentName);
1574}
1575
1576ConstantAddress CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) {
1577 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(GD: VD); };
1578
1579 auto LinkageForVariable = [&VD, this]() {
1580 return CGM.getLLVMLinkageVarDefinition(VD);
1581 };
1582
1583 std::vector<llvm::GlobalVariable *> GeneratedRefs;
1584
1585 llvm::Type *LlvmPtrTy = CGM.getTypes().ConvertTypeForMem(
1586 T: CGM.getContext().getPointerType(T: VD->getType()));
1587 llvm::Constant *addr = OMPBuilder.getAddrOfDeclareTargetVar(
1588 CaptureClause: convertCaptureClause(VD), DeviceClause: convertDeviceClause(VD),
1589 IsDeclaration: VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
1590 IsExternallyVisible: VD->isExternallyVisible(),
1591 EntryInfo: getEntryInfoFromPresumedLoc(CGM, OMPBuilder,
1592 BeginLoc: VD->getCanonicalDecl()->getBeginLoc()),
1593 MangledName: CGM.getMangledName(GD: VD), GeneratedRefs, OpenMPSIMD: CGM.getLangOpts().OpenMPSimd,
1594 TargetTriple: CGM.getLangOpts().OMPTargetTriples, LlvmPtrTy, GlobalInitializer: AddrOfGlobal,
1595 VariableLinkage: LinkageForVariable);
1596
1597 if (!addr)
1598 return ConstantAddress::invalid();
1599 return ConstantAddress(addr, LlvmPtrTy, CGM.getContext().getDeclAlign(D: VD));
1600}
1601
1602llvm::Constant *
1603CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
1604 assert(!CGM.getLangOpts().OpenMPUseTLS ||
1605 !CGM.getContext().getTargetInfo().isTLSSupported());
1606 // Lookup the entry, lazily creating it if necessary.
1607 std::string Suffix = getName(Parts: {"cache", ""});
1608 return OMPBuilder.getOrCreateInternalVariable(
1609 Ty: CGM.Int8PtrPtrTy, Name: Twine(CGM.getMangledName(GD: VD)).concat(Suffix).str());
1610}
1611
1612Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
1613 const VarDecl *VD,
1614 Address VDAddr,
1615 SourceLocation Loc) {
1616 if (CGM.getLangOpts().OpenMPUseTLS &&
1617 CGM.getContext().getTargetInfo().isTLSSupported())
1618 return VDAddr;
1619
1620 llvm::Type *VarTy = VDAddr.getElementType();
1621 llvm::Value *Args[] = {
1622 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
1623 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.Int8PtrTy),
1624 CGM.getSize(numChars: CGM.GetTargetTypeStoreSize(Ty: VarTy)),
1625 getOrCreateThreadPrivateCache(VD)};
1626 return Address(
1627 CGF.EmitRuntimeCall(
1628 callee: OMPBuilder.getOrCreateRuntimeFunction(
1629 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_cached),
1630 args: Args),
1631 CGF.Int8Ty, VDAddr.getAlignment());
1632}
1633
1634void CGOpenMPRuntime::emitThreadPrivateVarInit(
1635 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor,
1636 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
1637 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
1638 // library.
1639 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc);
1640 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
1641 M&: CGM.getModule(), FnID: OMPRTL___kmpc_global_thread_num),
1642 args: OMPLoc);
1643 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
1644 // to register constructor/destructor for variable.
1645 llvm::Value *Args[] = {
1646 OMPLoc,
1647 CGF.Builder.CreatePointerCast(V: VDAddr.emitRawPointer(CGF), DestTy: CGM.VoidPtrTy),
1648 Ctor, CopyCtor, Dtor};
1649 CGF.EmitRuntimeCall(
1650 callee: OMPBuilder.getOrCreateRuntimeFunction(
1651 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_register),
1652 args: Args);
1653}
1654
1655llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
1656 const VarDecl *VD, Address VDAddr, SourceLocation Loc,
1657 bool PerformInit, CodeGenFunction *CGF) {
1658 if (CGM.getLangOpts().OpenMPUseTLS &&
1659 CGM.getContext().getTargetInfo().isTLSSupported())
1660 return nullptr;
1661
1662 VD = VD->getDefinition(C&: CGM.getContext());
1663 if (VD && ThreadPrivateWithDefinition.insert(key: CGM.getMangledName(GD: VD)).second) {
1664 QualType ASTTy = VD->getType();
1665
1666 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
1667 const Expr *Init = VD->getAnyInitializer();
1668 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1669 // Generate function that re-emits the declaration's initializer into the
1670 // threadprivate copy of the variable VD
1671 CodeGenFunction CtorCGF(CGM);
1672 auto *Dst = ImplicitParamDecl::Create(
1673 C&: CGM.getContext(), /*DC=*/nullptr, IdLoc: Loc,
1674 /*Id=*/nullptr, T: CGM.getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1675
1676 FunctionArgList Args{Dst};
1677 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1678 resultType: CGM.getContext().VoidPtrTy, args: Args);
1679 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1680 std::string Name = getName(Parts: {"__kmpc_global_ctor_", ""});
1681 llvm::Function *Fn =
1682 CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: Name, FI, Loc);
1683 CtorCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidPtrTy, Fn, FnInfo: FI,
1684 Args, Loc, StartLoc: Loc);
1685 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar(
1686 Addr: CtorCGF.GetAddrOfLocalVar(VD: Dst), /*Volatile=*/false,
1687 Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1688 Address Arg(ArgVal, CtorCGF.ConvertTypeForMem(T: ASTTy),
1689 VDAddr.getAlignment());
1690 CtorCGF.EmitAnyExprToMem(E: Init, Location: Arg, Quals: Init->getType().getQualifiers(),
1691 /*IsInitializer=*/true);
1692 ArgVal = CtorCGF.EmitLoadOfScalar(
1693 Addr: CtorCGF.GetAddrOfLocalVar(VD: Dst), /*Volatile=*/false,
1694 Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1695 CtorCGF.Builder.CreateStore(Val: ArgVal, Addr: CtorCGF.ReturnValue);
1696 CtorCGF.FinishFunction();
1697 Ctor = Fn;
1698 }
1699 if (VD->getType().isDestructedType() != QualType::DK_none) {
1700 // Generate function that emits destructor call for the threadprivate copy
1701 // of the variable VD
1702 CodeGenFunction DtorCGF(CGM);
1703 auto *Dst = ImplicitParamDecl::Create(
1704 C&: CGM.getContext(), /*DC=*/nullptr, IdLoc: Loc,
1705 /*Id=*/nullptr, T: CGM.getContext().VoidPtrTy, ParamKind: ImplicitParamKind::Other);
1706
1707 FunctionArgList Args{Dst};
1708 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1709 resultType: CGM.getContext().VoidTy, args: Args);
1710 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(Info: FI);
1711 std::string Name = getName(Parts: {"__kmpc_global_dtor_", ""});
1712 llvm::Function *Fn =
1713 CGM.CreateGlobalInitOrCleanUpFunction(ty: FTy, name: Name, FI, Loc);
1714 auto NL = ApplyDebugLocation::CreateEmpty(CGF&: DtorCGF);
1715 DtorCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn, FnInfo: FI, Args,
1716 Loc, StartLoc: Loc);
1717 // Create a scope with an artificial location for the body of this function.
1718 auto AL = ApplyDebugLocation::CreateArtificial(CGF&: DtorCGF);
1719 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar(
1720 Addr: DtorCGF.GetAddrOfLocalVar(VD: Dst),
1721 /*Volatile=*/false, Ty: CGM.getContext().VoidPtrTy, Loc: Dst->getLocation());
1722 DtorCGF.emitDestroy(
1723 addr: Address(ArgVal, DtorCGF.Int8Ty, VDAddr.getAlignment()), type: ASTTy,
1724 destroyer: DtorCGF.getDestroyer(destructionKind: ASTTy.isDestructedType()),
1725 useEHCleanupForArray: DtorCGF.needsEHCleanup(kind: ASTTy.isDestructedType()));
1726 DtorCGF.FinishFunction();
1727 Dtor = Fn;
1728 }
1729 // Do not emit init function if it is not required.
1730 if (!Ctor && !Dtor)
1731 return nullptr;
1732
1733 // Copying constructor for the threadprivate variable.
1734 // Must be NULL - reserved by runtime, but currently it requires that this
1735 // parameter is always NULL. Otherwise it fires assertion.
1736 CopyCtor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1737 if (Ctor == nullptr) {
1738 Ctor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1739 }
1740 if (Dtor == nullptr) {
1741 Dtor = llvm::Constant::getNullValue(Ty: CGM.DefaultPtrTy);
1742 }
1743 if (!CGF) {
1744 auto *InitFunctionTy =
1745 llvm::FunctionType::get(Result: CGM.VoidTy, /*isVarArg*/ false);
1746 std::string Name = getName(Parts: {"__omp_threadprivate_init_", ""});
1747 llvm::Function *InitFunction = CGM.CreateGlobalInitOrCleanUpFunction(
1748 ty: InitFunctionTy, name: Name, FI: CGM.getTypes().arrangeNullaryFunction());
1749 CodeGenFunction InitCGF(CGM);
1750 FunctionArgList ArgList;
1751 InitCGF.StartFunction(GD: GlobalDecl(), RetTy: CGM.getContext().VoidTy, Fn: InitFunction,
1752 FnInfo: CGM.getTypes().arrangeNullaryFunction(), Args: ArgList,
1753 Loc, StartLoc: Loc);
1754 emitThreadPrivateVarInit(CGF&: InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1755 InitCGF.FinishFunction();
1756 return InitFunction;
1757 }
1758 emitThreadPrivateVarInit(CGF&: *CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
1759 }
1760 return nullptr;
1761}
1762
1763void CGOpenMPRuntime::emitDeclareTargetFunction(const FunctionDecl *FD,
1764 llvm::GlobalValue *GV) {
1765 std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
1766 OMPDeclareTargetDeclAttr::getActiveAttr(VD: FD);
1767
1768 // We only need to handle active 'indirect' declare target functions.
1769 if (!ActiveAttr || !(*ActiveAttr)->getIndirect())
1770 return;
1771
1772 // Get a mangled name to store the new device global in.
1773 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1774 CGM, OMPBuilder, BeginLoc: FD->getCanonicalDecl()->getBeginLoc(), ParentName: FD->getName());
1775 SmallString<128> Name;
1776 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);
1777
1778 // We need to generate a new global to hold the address of the indirectly
1779 // called device function. Doing this allows us to keep the visibility and
1780 // linkage of the associated function unchanged while allowing the runtime to
1781 // access its value.
1782 llvm::GlobalValue *Addr = GV;
1783 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1784 llvm::PointerType *FnPtrTy = llvm::PointerType::get(
1785 C&: CGM.getLLVMContext(),
1786 AddressSpace: CGM.getModule().getDataLayout().getProgramAddressSpace());
1787 Addr = new llvm::GlobalVariable(
1788 CGM.getModule(), FnPtrTy,
1789 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, GV, Name,
1790 nullptr, llvm::GlobalValue::NotThreadLocal,
1791 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1792 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1793 }
1794
1795 // Register the indirect Vtable:
1796 // This is similar to OMPTargetGlobalVarEntryIndirect, except that the
1797 // size field refers to the size of memory pointed to, not the size of
1798 // the pointer symbol itself (which is implicitly the size of a pointer).
1799 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1800 VarName: Name, Addr, VarSize: CGM.GetTargetTypeStoreSize(Ty: CGM.VoidPtrTy).getQuantity(),
1801 Flags: llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,
1802 Linkage: llvm::GlobalValue::WeakODRLinkage);
1803}
1804
1805void CGOpenMPRuntime::registerVTableOffloadEntry(llvm::GlobalVariable *VTable,
1806 const VarDecl *VD) {
1807 // TODO: add logic to avoid duplicate vtable registrations per
1808 // translation unit; though for external linkage, this should no
1809 // longer be an issue - or at least we can avoid the issue by
1810 // checking for an existing offloading entry. But, perhaps the
1811 // better approach is to defer emission of the vtables and offload
1812 // entries until later (by tracking a list of items that need to be
1813 // emitted).
1814
1815 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
1816
1817 // Generate a new externally visible global to point to the
1818 // internally visible vtable. Doing this allows us to keep the
1819 // visibility and linkage of the associated vtable unchanged while
1820 // allowing the runtime to access its value. The externally
1821 // visible global var needs to be emitted with a unique mangled
1822 // name that won't conflict with similarly named (internal)
1823 // vtables in other translation units.
1824
1825 // Register vtable with source location of dynamic object in map
1826 // clause.
1827 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
1828 CGM, OMPBuilder, BeginLoc: VD->getCanonicalDecl()->getBeginLoc(),
1829 ParentName: VTable->getName());
1830
1831 llvm::GlobalVariable *Addr = VTable;
1832 SmallString<128> AddrName;
1833 OMPBuilder.OffloadInfoManager.getTargetRegionEntryFnName(Name&: AddrName, EntryInfo);
1834 AddrName.append(RHS: "addr");
1835
1836 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
1837 Addr = new llvm::GlobalVariable(
1838 CGM.getModule(), VTable->getType(),
1839 /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, VTable,
1840 AddrName,
1841 /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
1842 CGM.getModule().getDataLayout().getDefaultGlobalsAddressSpace());
1843 Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1844 }
1845 OMPBuilder.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(
1846 VarName: AddrName, Addr: VTable,
1847 VarSize: CGM.getDataLayout().getTypeAllocSize(Ty: VTable->getInitializer()->getType()),
1848 Flags: llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirectVTable,
1849 Linkage: llvm::GlobalValue::WeakODRLinkage);
1850}
1851
1852void CGOpenMPRuntime::emitAndRegisterVTable(CodeGenModule &CGM,
1853 CXXRecordDecl *CXXRecord,
1854 const VarDecl *VD) {
1855 // Register C++ VTable to OpenMP Offload Entry if it's a new
1856 // CXXRecordDecl.
1857 if (CXXRecord && CXXRecord->isDynamicClass() &&
1858 !CGM.getOpenMPRuntime().VTableDeclMap.contains(Val: CXXRecord)) {
1859 auto Res = CGM.getOpenMPRuntime().VTableDeclMap.try_emplace(Key: CXXRecord, Args&: VD);
1860 if (Res.second) {
1861 CGM.EmitVTable(Class: CXXRecord);
1862 CodeGenVTables VTables = CGM.getVTables();
1863 llvm::GlobalVariable *VTablesAddr = VTables.GetAddrOfVTable(RD: CXXRecord);
1864 assert(VTablesAddr && "Expected non-null VTable address");
1865 // Must set VTables to weak since we're emitting them in multiple TUs now
1866 if (VTablesAddr->hasExternalLinkage())
1867 VTablesAddr->setLinkage(llvm::GlobalValue::WeakODRLinkage);
1868 CGM.getOpenMPRuntime().registerVTableOffloadEntry(VTable: VTablesAddr, VD);
1869 // Emit VTable for all the fields containing dynamic CXXRecord
1870 for (const FieldDecl *Field : CXXRecord->fields()) {
1871 if (CXXRecordDecl *RecordDecl = Field->getType()->getAsCXXRecordDecl())
1872 emitAndRegisterVTable(CGM, CXXRecord: RecordDecl, VD);
1873 }
1874 // Emit VTable for all dynamic parent class
1875 for (CXXBaseSpecifier &Base : CXXRecord->bases()) {
1876 if (CXXRecordDecl *BaseDecl = Base.getType()->getAsCXXRecordDecl())
1877 emitAndRegisterVTable(CGM, CXXRecord: BaseDecl, VD);
1878 }
1879 }
1880 }
1881}
1882
1883void CGOpenMPRuntime::registerVTable(const OMPExecutableDirective &D) {
1884 // Register VTable by scanning through the map clause of OpenMP target region.
1885 // Get CXXRecordDecl and VarDecl from Expr.
1886 auto GetVTableDecl = [](const Expr *E) {
1887 QualType VDTy = E->getType();
1888 CXXRecordDecl *CXXRecord = nullptr;
1889 if (const auto *RefType = VDTy->getAs<LValueReferenceType>())
1890 VDTy = RefType->getPointeeType();
1891 if (VDTy->isPointerType())
1892 CXXRecord = VDTy->getPointeeType()->getAsCXXRecordDecl();
1893 else
1894 CXXRecord = VDTy->getAsCXXRecordDecl();
1895
1896 const VarDecl *VD = nullptr;
1897 if (auto *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
1898 VD = cast<VarDecl>(Val: DRE->getDecl());
1899 } else if (auto *MRE = dyn_cast<MemberExpr>(Val: E)) {
1900 if (auto *BaseDRE = dyn_cast<DeclRefExpr>(Val: MRE->getBase())) {
1901 if (auto *BaseVD = dyn_cast<VarDecl>(Val: BaseDRE->getDecl()))
1902 VD = BaseVD;
1903 }
1904 }
1905 return std::pair<CXXRecordDecl *, const VarDecl *>(CXXRecord, VD);
1906 };
1907 // Collect VTable from OpenMP map clause.
1908 for (const auto *C : D.getClausesOfKind<OMPMapClause>()) {
1909 for (const auto *E : C->varlist()) {
1910 auto DeclPair = GetVTableDecl(E);
1911 // Ensure VD is not null
1912 if (DeclPair.second)
1913 emitAndRegisterVTable(CGM, CXXRecord: DeclPair.first, VD: DeclPair.second);
1914 }
1915 }
1916}
1917
1918Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1919 QualType VarType,
1920 StringRef Name) {
1921 std::string Suffix = getName(Parts: {"artificial", ""});
1922 llvm::Type *VarLVType = CGF.ConvertTypeForMem(T: VarType);
1923 llvm::GlobalVariable *GAddr = OMPBuilder.getOrCreateInternalVariable(
1924 Ty: VarLVType, Name: Twine(Name).concat(Suffix).str());
1925 if (CGM.getLangOpts().OpenMP && CGM.getLangOpts().OpenMPUseTLS &&
1926 CGM.getTarget().isTLSSupported()) {
1927 GAddr->setThreadLocal(/*Val=*/true);
1928 return Address(GAddr, GAddr->getValueType(),
1929 CGM.getContext().getTypeAlignInChars(T: VarType));
1930 }
1931 std::string CacheSuffix = getName(Parts: {"cache", ""});
1932 llvm::Value *Args[] = {
1933 emitUpdateLocation(CGF, Loc: SourceLocation()),
1934 getThreadID(CGF, Loc: SourceLocation()),
1935 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: GAddr, DestTy: CGM.VoidPtrTy),
1936 CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: VarType), DestTy: CGM.SizeTy,
1937 /*isSigned=*/false),
1938 OMPBuilder.getOrCreateInternalVariable(
1939 Ty: CGM.VoidPtrPtrTy,
1940 Name: Twine(Name).concat(Suffix).concat(Suffix: CacheSuffix).str())};
1941 return Address(
1942 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
1943 V: CGF.EmitRuntimeCall(
1944 callee: OMPBuilder.getOrCreateRuntimeFunction(
1945 M&: CGM.getModule(), FnID: OMPRTL___kmpc_threadprivate_cached),
1946 args: Args),
1947 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
1948 VarLVType, CGM.getContext().getTypeAlignInChars(T: VarType));
1949}
1950
1951void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
1952 const RegionCodeGenTy &ThenGen,
1953 const RegionCodeGenTy &ElseGen) {
1954 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange());
1955
1956 // If the condition constant folds and can be elided, try to avoid emitting
1957 // the condition and the dead arm of the if/else.
1958 bool CondConstant;
1959 if (CGF.ConstantFoldsToSimpleInteger(Cond, Result&: CondConstant)) {
1960 if (CondConstant)
1961 ThenGen(CGF);
1962 else
1963 ElseGen(CGF);
1964 return;
1965 }
1966
1967 // Otherwise, the condition did not fold, or we couldn't elide it. Just
1968 // emit the conditional branch.
1969 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock(name: "omp_if.then");
1970 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock(name: "omp_if.else");
1971 llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "omp_if.end");
1972 CGF.EmitBranchOnBoolExpr(Cond, TrueBlock: ThenBlock, FalseBlock: ElseBlock, /*TrueCount=*/0);
1973
1974 // Emit the 'then' code.
1975 CGF.EmitBlock(BB: ThenBlock);
1976 ThenGen(CGF);
1977 CGF.EmitBranch(Block: ContBlock);
1978 // Emit the 'else' code if present.
1979 // There is no need to emit line number for unconditional branch.
1980 (void)ApplyDebugLocation::CreateEmpty(CGF);
1981 CGF.EmitBlock(BB: ElseBlock);
1982 ElseGen(CGF);
1983 // There is no need to emit line number for unconditional branch.
1984 (void)ApplyDebugLocation::CreateEmpty(CGF);
1985 CGF.EmitBranch(Block: ContBlock);
1986 // Emit the continuation block for code after the if.
1987 CGF.EmitBlock(BB: ContBlock, /*IsFinished=*/true);
1988}
1989
1990void CGOpenMPRuntime::emitParallelCall(
1991 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
1992 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
1993 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
1994 OpenMPSeverityClauseKind Severity, const Expr *Message) {
1995 if (!CGF.HaveInsertPoint())
1996 return;
1997 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
1998 auto &M = CGM.getModule();
1999 auto &&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,
2000 this](CodeGenFunction &CGF, PrePostActionTy &) {
2001 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn);
2002 llvm::Value *Args[] = {
2003 RTLoc,
2004 CGF.Builder.getInt32(C: CapturedVars.size()), // Number of captured vars
2005 OutlinedFn};
2006 llvm::SmallVector<llvm::Value *, 16> RealArgs;
2007 RealArgs.append(in_start: std::begin(arr&: Args), in_end: std::end(arr&: Args));
2008 RealArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
2009
2010 llvm::FunctionCallee RTLFn =
2011 OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_fork_call);
2012 CGF.EmitRuntimeCall(callee: RTLFn, args: RealArgs);
2013 };
2014 auto &&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc, Loc,
2015 this](CodeGenFunction &CGF, PrePostActionTy &) {
2016 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
2017 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc);
2018 // Build calls:
2019 // __kmpc_serialized_parallel(&Loc, GTid);
2020 llvm::Value *Args[] = {RTLoc, ThreadID};
2021 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2022 M, FnID: OMPRTL___kmpc_serialized_parallel),
2023 args: Args);
2024
2025 // OutlinedFn(&GTid, &zero_bound, CapturedStruct);
2026 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
2027 RawAddress ZeroAddrBound =
2028 CGF.CreateDefaultAlignTempAlloca(Ty: CGF.Int32Ty,
2029 /*Name=*/".bound.zero.addr");
2030 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(/*C*/ 0), Addr: ZeroAddrBound);
2031 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2032 // ThreadId for serialized parallels is 0.
2033 OutlinedFnArgs.push_back(Elt: ThreadIDAddr.emitRawPointer(CGF));
2034 OutlinedFnArgs.push_back(Elt: ZeroAddrBound.getPointer());
2035 OutlinedFnArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
2036
2037 // Ensure we do not inline the function. This is trivially true for the ones
2038 // passed to __kmpc_fork_call but the ones called in serialized regions
2039 // could be inlined. This is not a perfect but it is closer to the invariant
2040 // we want, namely, every data environment starts with a new function.
2041 // TODO: We should pass the if condition to the runtime function and do the
2042 // handling there. Much cleaner code.
2043 OutlinedFn->removeFnAttr(Kind: llvm::Attribute::AlwaysInline);
2044 OutlinedFn->addFnAttr(Kind: llvm::Attribute::NoInline);
2045 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, Args: OutlinedFnArgs);
2046
2047 // __kmpc_end_serialized_parallel(&Loc, GTid);
2048 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
2049 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2050 M, FnID: OMPRTL___kmpc_end_serialized_parallel),
2051 args: EndArgs);
2052 };
2053 if (IfCond) {
2054 emitIfClause(CGF, Cond: IfCond, ThenGen, ElseGen);
2055 } else {
2056 RegionCodeGenTy ThenRCG(ThenGen);
2057 ThenRCG(CGF);
2058 }
2059}
2060
2061// If we're inside an (outlined) parallel region, use the region info's
2062// thread-ID variable (it is passed in a first argument of the outlined function
2063// as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
2064// regular serial code region, get thread ID by calling kmp_int32
2065// kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
2066// return the address of that temp.
2067Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF,
2068 SourceLocation Loc) {
2069 if (auto *OMPRegionInfo =
2070 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
2071 if (OMPRegionInfo->getThreadIDVariable())
2072 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2073
2074 llvm::Value *ThreadID = getThreadID(CGF, Loc);
2075 QualType Int32Ty =
2076 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
2077 Address ThreadIDTemp =
2078 CGF.CreateMemTempWithoutCast(T: Int32Ty, /*Name*/ ".threadid_temp.");
2079 CGF.EmitStoreOfScalar(value: ThreadID,
2080 lvalue: CGF.MakeAddrLValue(Addr: ThreadIDTemp, T: Int32Ty));
2081
2082 return ThreadIDTemp;
2083}
2084
2085llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) {
2086 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
2087 std::string Name = getName(Parts: {Prefix, "var"});
2088 llvm::GlobalVariable *GV =
2089 OMPBuilder.getOrCreateInternalVariable(Ty: KmpCriticalNameTy, Name);
2090 CGM.setDSOLocal(GV);
2091 return GV;
2092}
2093
2094namespace {
2095/// Common pre(post)-action for different OpenMP constructs.
2096class CommonActionTy final : public PrePostActionTy {
2097 llvm::FunctionCallee EnterCallee;
2098 ArrayRef<llvm::Value *> EnterArgs;
2099 llvm::FunctionCallee ExitCallee;
2100 ArrayRef<llvm::Value *> ExitArgs;
2101 bool Conditional;
2102 llvm::BasicBlock *ContBlock = nullptr;
2103
2104public:
2105 CommonActionTy(llvm::FunctionCallee EnterCallee,
2106 ArrayRef<llvm::Value *> EnterArgs,
2107 llvm::FunctionCallee ExitCallee,
2108 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
2109 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2110 ExitArgs(ExitArgs), Conditional(Conditional) {}
2111 void Enter(CodeGenFunction &CGF) override {
2112 llvm::Value *EnterRes = CGF.EmitRuntimeCall(callee: EnterCallee, args: EnterArgs);
2113 if (Conditional) {
2114 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(Arg: EnterRes);
2115 auto *ThenBlock = CGF.createBasicBlock(name: "omp_if.then");
2116 ContBlock = CGF.createBasicBlock(name: "omp_if.end");
2117 // Generate the branch (If-stmt)
2118 CGF.Builder.CreateCondBr(Cond: CallBool, True: ThenBlock, False: ContBlock);
2119 CGF.EmitBlock(BB: ThenBlock);
2120 }
2121 }
2122 void Done(CodeGenFunction &CGF) {
2123 // Emit the rest of blocks/branches
2124 CGF.EmitBranch(Block: ContBlock);
2125 CGF.EmitBlock(BB: ContBlock, IsFinished: true);
2126 }
2127 void Exit(CodeGenFunction &CGF) override {
2128 CGF.EmitRuntimeCall(callee: ExitCallee, args: ExitArgs);
2129 }
2130};
2131} // anonymous namespace
2132
2133void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF,
2134 StringRef CriticalName,
2135 const RegionCodeGenTy &CriticalOpGen,
2136 SourceLocation Loc, const Expr *Hint) {
2137 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]);
2138 // CriticalOpGen();
2139 // __kmpc_end_critical(ident_t *, gtid, Lock);
2140 // Prepare arguments and build a call to __kmpc_critical
2141 if (!CGF.HaveInsertPoint())
2142 return;
2143 llvm::FunctionCallee RuntimeFcn = OMPBuilder.getOrCreateRuntimeFunction(
2144 M&: CGM.getModule(),
2145 FnID: Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical);
2146 llvm::Value *LockVar = getCriticalRegionLock(CriticalName);
2147 unsigned LockVarArgIdx = 2;
2148 if (cast<llvm::GlobalVariable>(Val: LockVar)->getAddressSpace() !=
2149 RuntimeFcn.getFunctionType()
2150 ->getParamType(i: LockVarArgIdx)
2151 ->getPointerAddressSpace())
2152 LockVar = CGF.Builder.CreateAddrSpaceCast(
2153 V: LockVar, DestTy: RuntimeFcn.getFunctionType()->getParamType(i: LockVarArgIdx));
2154 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2155 LockVar};
2156 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(arr&: Args),
2157 std::end(arr&: Args));
2158 if (Hint) {
2159 EnterArgs.push_back(Elt: CGF.Builder.CreateIntCast(
2160 V: CGF.EmitScalarExpr(E: Hint), DestTy: CGM.Int32Ty, /*isSigned=*/false));
2161 }
2162 CommonActionTy Action(RuntimeFcn, EnterArgs,
2163 OMPBuilder.getOrCreateRuntimeFunction(
2164 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_critical),
2165 Args);
2166 CriticalOpGen.setAction(Action);
2167 emitInlinedDirective(CGF, InnermostKind: OMPD_critical, CodeGen: CriticalOpGen);
2168}
2169
2170void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF,
2171 const RegionCodeGenTy &MasterOpGen,
2172 SourceLocation Loc) {
2173 if (!CGF.HaveInsertPoint())
2174 return;
2175 // if(__kmpc_master(ident_t *, gtid)) {
2176 // MasterOpGen();
2177 // __kmpc_end_master(ident_t *, gtid);
2178 // }
2179 // Prepare arguments and build a call to __kmpc_master
2180 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2181 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2182 M&: CGM.getModule(), FnID: OMPRTL___kmpc_master),
2183 Args,
2184 OMPBuilder.getOrCreateRuntimeFunction(
2185 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_master),
2186 Args,
2187 /*Conditional=*/true);
2188 MasterOpGen.setAction(Action);
2189 emitInlinedDirective(CGF, InnermostKind: OMPD_master, CodeGen: MasterOpGen);
2190 Action.Done(CGF);
2191}
2192
2193void CGOpenMPRuntime::emitMaskedRegion(CodeGenFunction &CGF,
2194 const RegionCodeGenTy &MaskedOpGen,
2195 SourceLocation Loc, const Expr *Filter) {
2196 if (!CGF.HaveInsertPoint())
2197 return;
2198 // if(__kmpc_masked(ident_t *, gtid, filter)) {
2199 // MaskedOpGen();
2200 // __kmpc_end_masked(iden_t *, gtid);
2201 // }
2202 // Prepare arguments and build a call to __kmpc_masked
2203 llvm::Value *FilterVal = Filter
2204 ? CGF.EmitScalarExpr(E: Filter, IgnoreResultAssign: CGF.Int32Ty)
2205 : llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/0);
2206 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2207 FilterVal};
2208 llvm::Value *ArgsEnd[] = {emitUpdateLocation(CGF, Loc),
2209 getThreadID(CGF, Loc)};
2210 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2211 M&: CGM.getModule(), FnID: OMPRTL___kmpc_masked),
2212 Args,
2213 OMPBuilder.getOrCreateRuntimeFunction(
2214 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_masked),
2215 ArgsEnd,
2216 /*Conditional=*/true);
2217 MaskedOpGen.setAction(Action);
2218 emitInlinedDirective(CGF, InnermostKind: OMPD_masked, CodeGen: MaskedOpGen);
2219 Action.Done(CGF);
2220}
2221
2222void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
2223 SourceLocation Loc) {
2224 if (!CGF.HaveInsertPoint())
2225 return;
2226 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2227 OMPBuilder.createTaskyield(Loc: CGF.Builder);
2228 } else {
2229 // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
2230 llvm::Value *Args[] = {
2231 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2232 llvm::ConstantInt::get(Ty: CGM.IntTy, /*V=*/0, /*isSigned=*/IsSigned: true)};
2233 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2234 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_taskyield),
2235 args: Args);
2236 }
2237
2238 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
2239 Region->emitUntiedSwitch(CGF);
2240}
2241
2242void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
2243 const RegionCodeGenTy &TaskgroupOpGen,
2244 SourceLocation Loc) {
2245 if (!CGF.HaveInsertPoint())
2246 return;
2247 // __kmpc_taskgroup(ident_t *, gtid);
2248 // TaskgroupOpGen();
2249 // __kmpc_end_taskgroup(ident_t *, gtid);
2250 // Prepare arguments and build a call to __kmpc_taskgroup
2251 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2252 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2253 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskgroup),
2254 Args,
2255 OMPBuilder.getOrCreateRuntimeFunction(
2256 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_taskgroup),
2257 Args);
2258 TaskgroupOpGen.setAction(Action);
2259 emitInlinedDirective(CGF, InnermostKind: OMPD_taskgroup, CodeGen: TaskgroupOpGen);
2260}
2261
2262/// Given an array of pointers to variables, project the address of a
2263/// given variable.
2264static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array,
2265 unsigned Index, const VarDecl *Var) {
2266 // Pull out the pointer to the variable.
2267 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Addr: Array, Index);
2268 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: PtrAddr);
2269
2270 llvm::Type *ElemTy = CGF.ConvertTypeForMem(T: Var->getType());
2271 return Address(Ptr, ElemTy, CGF.getContext().getDeclAlign(D: Var));
2272}
2273
2274static llvm::Value *emitCopyprivateCopyFunction(
2275 CodeGenModule &CGM, llvm::Type *ArgsElemType,
2276 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2277 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps,
2278 SourceLocation Loc) {
2279 ASTContext &C = CGM.getContext();
2280 // void copy_func(void *LHSArg, void *RHSArg);
2281
2282 auto *LHSArg =
2283 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
2284 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2285 auto *RHSArg =
2286 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
2287 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
2288 FunctionArgList Args{LHSArg, RHSArg};
2289 const auto &CGFI =
2290 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
2291 std::string Name =
2292 CGM.getOpenMPRuntime().getName(Parts: {"omp", "copyprivate", "copy_func"});
2293 auto *Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: CGFI),
2294 Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
2295 M: &CGM.getModule());
2296 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: CGFI);
2297 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
2298 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
2299 Fn->setDoesNotRecurse();
2300 CodeGenFunction CGF(CGM);
2301 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo: CGFI, Args, Loc, StartLoc: Loc);
2302 // Dest = (void*[n])(LHSArg);
2303 // Src = (void*[n])(RHSArg);
2304 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2305 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: LHSArg)),
2306 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
2307 ArgsElemType, CGF.getPointerAlign());
2308 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2309 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: RHSArg)),
2310 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
2311 ArgsElemType, CGF.getPointerAlign());
2312 // *(Type0*)Dst[0] = *(Type0*)Src[0];
2313 // *(Type1*)Dst[1] = *(Type1*)Src[1];
2314 // ...
2315 // *(Typen*)Dst[n] = *(Typen*)Src[n];
2316 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) {
2317 const auto *DestVar =
2318 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: DestExprs[I])->getDecl());
2319 Address DestAddr = emitAddrOfVarFromArray(CGF, Array: LHS, Index: I, Var: DestVar);
2320
2321 const auto *SrcVar =
2322 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: SrcExprs[I])->getDecl());
2323 Address SrcAddr = emitAddrOfVarFromArray(CGF, Array: RHS, Index: I, Var: SrcVar);
2324
2325 const auto *VD = cast<DeclRefExpr>(Val: CopyprivateVars[I])->getDecl();
2326 QualType Type = VD->getType();
2327 CGF.EmitOMPCopy(OriginalType: Type, DestAddr, SrcAddr, DestVD: DestVar, SrcVD: SrcVar, Copy: AssignmentOps[I]);
2328 }
2329 CGF.FinishFunction();
2330 return Fn;
2331}
2332
2333void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF,
2334 const RegionCodeGenTy &SingleOpGen,
2335 SourceLocation Loc,
2336 ArrayRef<const Expr *> CopyprivateVars,
2337 ArrayRef<const Expr *> SrcExprs,
2338 ArrayRef<const Expr *> DstExprs,
2339 ArrayRef<const Expr *> AssignmentOps) {
2340 if (!CGF.HaveInsertPoint())
2341 return;
2342 assert(CopyprivateVars.size() == SrcExprs.size() &&
2343 CopyprivateVars.size() == DstExprs.size() &&
2344 CopyprivateVars.size() == AssignmentOps.size());
2345 ASTContext &C = CGM.getContext();
2346 // int32 did_it = 0;
2347 // if(__kmpc_single(ident_t *, gtid)) {
2348 // SingleOpGen();
2349 // __kmpc_end_single(ident_t *, gtid);
2350 // did_it = 1;
2351 // }
2352 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2353 // <copy_func>, did_it);
2354
2355 Address DidIt = Address::invalid();
2356 if (!CopyprivateVars.empty()) {
2357 // int32 did_it = 0;
2358 QualType KmpInt32Ty =
2359 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
2360 DidIt = CGF.CreateMemTempWithoutCast(T: KmpInt32Ty, Name: ".omp.copyprivate.did_it");
2361 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(C: 0), Addr: DidIt);
2362 }
2363 // Prepare arguments and build a call to __kmpc_single
2364 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2365 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2366 M&: CGM.getModule(), FnID: OMPRTL___kmpc_single),
2367 Args,
2368 OMPBuilder.getOrCreateRuntimeFunction(
2369 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_single),
2370 Args,
2371 /*Conditional=*/true);
2372 SingleOpGen.setAction(Action);
2373 emitInlinedDirective(CGF, InnermostKind: OMPD_single, CodeGen: SingleOpGen);
2374 if (DidIt.isValid()) {
2375 // did_it = 1;
2376 CGF.Builder.CreateStore(Val: CGF.Builder.getInt32(C: 1), Addr: DidIt);
2377 }
2378 Action.Done(CGF);
2379 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>,
2380 // <copy_func>, did_it);
2381 if (DidIt.isValid()) {
2382 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size());
2383 QualType CopyprivateArrayTy = C.getConstantArrayType(
2384 EltTy: C.VoidPtrTy, ArySize: ArraySize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
2385 /*IndexTypeQuals=*/0);
2386 // Create a list of all private variables for copyprivate.
2387 Address CopyprivateList = CGF.CreateMemTempWithoutCast(
2388 T: CopyprivateArrayTy, Name: ".omp.copyprivate.cpr_list");
2389 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) {
2390 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: CopyprivateList, Index: I);
2391 CGF.Builder.CreateStore(
2392 Val: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2393 V: CGF.EmitLValue(E: CopyprivateVars[I]).getPointer(CGF),
2394 DestTy: CGF.VoidPtrTy),
2395 Addr: Elem);
2396 }
2397 // Build function that copies private values from single region to all other
2398 // threads in the corresponding parallel region.
2399 llvm::Value *CpyFn = emitCopyprivateCopyFunction(
2400 CGM, ArgsElemType: CGF.ConvertTypeForMem(T: CopyprivateArrayTy), CopyprivateVars,
2401 DestExprs: SrcExprs, SrcExprs: DstExprs, AssignmentOps, Loc);
2402 llvm::Value *BufSize = CGF.getTypeSize(Ty: CopyprivateArrayTy);
2403 Address CL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
2404 Addr: CopyprivateList, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty);
2405 llvm::Value *DidItVal = CGF.Builder.CreateLoad(Addr: DidIt);
2406 llvm::Value *Args[] = {
2407 emitUpdateLocation(CGF, Loc), // ident_t *<loc>
2408 getThreadID(CGF, Loc), // i32 <gtid>
2409 BufSize, // size_t <buf_size>
2410 CL.emitRawPointer(CGF), // void *<copyprivate list>
2411 CpyFn, // void (*) (void *, void *) <copy_func>
2412 DidItVal // i32 did_it
2413 };
2414 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2415 M&: CGM.getModule(), FnID: OMPRTL___kmpc_copyprivate),
2416 args: Args);
2417 }
2418}
2419
2420void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF,
2421 const RegionCodeGenTy &OrderedOpGen,
2422 SourceLocation Loc, bool IsThreads) {
2423 if (!CGF.HaveInsertPoint())
2424 return;
2425 // __kmpc_ordered(ident_t *, gtid);
2426 // OrderedOpGen();
2427 // __kmpc_end_ordered(ident_t *, gtid);
2428 // Prepare arguments and build a call to __kmpc_ordered
2429 if (IsThreads) {
2430 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2431 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
2432 M&: CGM.getModule(), FnID: OMPRTL___kmpc_ordered),
2433 Args,
2434 OMPBuilder.getOrCreateRuntimeFunction(
2435 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_ordered),
2436 Args);
2437 OrderedOpGen.setAction(Action);
2438 emitInlinedDirective(CGF, InnermostKind: OMPD_ordered_blockassoc, CodeGen: OrderedOpGen);
2439 return;
2440 }
2441 emitInlinedDirective(CGF, InnermostKind: OMPD_ordered_blockassoc, CodeGen: OrderedOpGen);
2442}
2443
2444unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) {
2445 unsigned Flags;
2446 if (Kind == OMPD_for)
2447 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2448 else if (Kind == OMPD_sections)
2449 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2450 else if (Kind == OMPD_single)
2451 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2452 else if (Kind == OMPD_barrier)
2453 Flags = OMP_IDENT_BARRIER_EXPL;
2454 else
2455 Flags = OMP_IDENT_BARRIER_IMPL;
2456 return Flags;
2457}
2458
2459void CGOpenMPRuntime::getDefaultScheduleAndChunk(
2460 CodeGenFunction &CGF, const OMPLoopDirective &S,
2461 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const {
2462 // Check if the loop directive is actually a doacross loop directive. In this
2463 // case choose static, 1 schedule.
2464 if (llvm::any_of(
2465 Range: S.getClausesOfKind<OMPOrderedClause>(),
2466 P: [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) {
2467 ScheduleKind = OMPC_SCHEDULE_static;
2468 // Chunk size is 1 in this case.
2469 llvm::APInt ChunkSize(32, 1);
2470 ChunkExpr = IntegerLiteral::Create(
2471 C: CGF.getContext(), V: ChunkSize,
2472 type: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/0),
2473 l: SourceLocation());
2474 }
2475}
2476
2477void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
2478 OpenMPDirectiveKind Kind, bool EmitChecks,
2479 bool ForceSimpleCall) {
2480 // Check if we should use the OMPBuilder
2481 auto *OMPRegionInfo =
2482 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo);
2483 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2484 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
2485 cantFail(ValOrErr: OMPBuilder.createBarrier(Loc: CGF.Builder, Kind, ForceSimpleCall,
2486 CheckCancelFlag: EmitChecks));
2487 CGF.Builder.restoreIP(IP: AfterIP);
2488 return;
2489 }
2490
2491 if (!CGF.HaveInsertPoint())
2492 return;
2493 // Build call __kmpc_cancel_barrier(loc, thread_id);
2494 // Build call __kmpc_barrier(loc, thread_id);
2495 unsigned Flags = getDefaultFlagsForBarriers(Kind);
2496 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc,
2497 // thread_id);
2498 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags),
2499 getThreadID(CGF, Loc)};
2500 if (OMPRegionInfo) {
2501 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2502 llvm::Value *Result = CGF.EmitRuntimeCall(
2503 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
2504 FnID: OMPRTL___kmpc_cancel_barrier),
2505 args: Args);
2506 if (EmitChecks) {
2507 // if (__kmpc_cancel_barrier()) {
2508 // exit from construct;
2509 // }
2510 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
2511 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
2512 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
2513 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
2514 CGF.EmitBlock(BB: ExitBB);
2515 // exit from construct;
2516 CodeGenFunction::JumpDest CancelDestination =
2517 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
2518 CGF.EmitBranchThroughCleanup(Dest: CancelDestination);
2519 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
2520 }
2521 return;
2522 }
2523 }
2524 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2525 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
2526 args: Args);
2527}
2528
2529void CGOpenMPRuntime::emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc,
2530 Expr *ME, bool IsFatal) {
2531 llvm::Value *MVL = ME ? CGF.EmitScalarExpr(E: ME)
2532 : llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
2533 // Build call void __kmpc_error(ident_t *loc, int severity, const char
2534 // *message)
2535 llvm::Value *Args[] = {
2536 emitUpdateLocation(CGF, Loc, /*Flags=*/0, /*GenLoc=*/EmitLoc: true),
2537 llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: IsFatal ? 2 : 1),
2538 CGF.Builder.CreatePointerCast(V: MVL, DestTy: CGM.Int8PtrTy)};
2539 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2540 M&: CGM.getModule(), FnID: OMPRTL___kmpc_error),
2541 args: Args);
2542}
2543
2544/// Map the OpenMP loop schedule to the runtime enumeration.
2545static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind,
2546 bool Chunked, bool Ordered) {
2547 switch (ScheduleKind) {
2548 case OMPC_SCHEDULE_static:
2549 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)
2550 : (Ordered ? OMP_ord_static : OMP_sch_static);
2551 case OMPC_SCHEDULE_dynamic:
2552 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;
2553 case OMPC_SCHEDULE_guided:
2554 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;
2555 case OMPC_SCHEDULE_runtime:
2556 return Ordered ? OMP_ord_runtime : OMP_sch_runtime;
2557 case OMPC_SCHEDULE_auto:
2558 return Ordered ? OMP_ord_auto : OMP_sch_auto;
2559 case OMPC_SCHEDULE_unknown:
2560 assert(!Chunked && "chunk was specified but schedule kind not known");
2561 return Ordered ? OMP_ord_static : OMP_sch_static;
2562 }
2563 llvm_unreachable("Unexpected runtime schedule");
2564}
2565
2566/// Map the OpenMP distribute schedule to the runtime enumeration.
2567static OpenMPSchedType
2568getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) {
2569 // only static is allowed for dist_schedule
2570 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;
2571}
2572
2573bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
2574 bool Chunked) const {
2575 OpenMPSchedType Schedule =
2576 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2577 return Schedule == OMP_sch_static;
2578}
2579
2580bool CGOpenMPRuntime::isStaticNonchunked(
2581 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2582 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2583 return Schedule == OMP_dist_sch_static;
2584}
2585
2586bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
2587 bool Chunked) const {
2588 OpenMPSchedType Schedule =
2589 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false);
2590 return Schedule == OMP_sch_static_chunked;
2591}
2592
2593bool CGOpenMPRuntime::isStaticChunked(
2594 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const {
2595 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked);
2596 return Schedule == OMP_dist_sch_static_chunked;
2597}
2598
2599bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const {
2600 OpenMPSchedType Schedule =
2601 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false);
2602 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here");
2603 return Schedule != OMP_sch_static;
2604}
2605
2606static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule,
2607 OpenMPScheduleClauseModifier M1,
2608 OpenMPScheduleClauseModifier M2) {
2609 int Modifier = 0;
2610 switch (M1) {
2611 case OMPC_SCHEDULE_MODIFIER_monotonic:
2612 Modifier = OMP_sch_modifier_monotonic;
2613 break;
2614 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2615 Modifier = OMP_sch_modifier_nonmonotonic;
2616 break;
2617 case OMPC_SCHEDULE_MODIFIER_simd:
2618 if (Schedule == OMP_sch_static_chunked)
2619 Schedule = OMP_sch_static_balanced_chunked;
2620 break;
2621 case OMPC_SCHEDULE_MODIFIER_last:
2622 case OMPC_SCHEDULE_MODIFIER_unknown:
2623 break;
2624 }
2625 switch (M2) {
2626 case OMPC_SCHEDULE_MODIFIER_monotonic:
2627 Modifier = OMP_sch_modifier_monotonic;
2628 break;
2629 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2630 Modifier = OMP_sch_modifier_nonmonotonic;
2631 break;
2632 case OMPC_SCHEDULE_MODIFIER_simd:
2633 if (Schedule == OMP_sch_static_chunked)
2634 Schedule = OMP_sch_static_balanced_chunked;
2635 break;
2636 case OMPC_SCHEDULE_MODIFIER_last:
2637 case OMPC_SCHEDULE_MODIFIER_unknown:
2638 break;
2639 }
2640 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription.
2641 // If the static schedule kind is specified or if the ordered clause is
2642 // specified, and if the nonmonotonic modifier is not specified, the effect is
2643 // as if the monotonic modifier is specified. Otherwise, unless the monotonic
2644 // modifier is specified, the effect is as if the nonmonotonic modifier is
2645 // specified.
2646 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) {
2647 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||
2648 Schedule == OMP_sch_static_balanced_chunked ||
2649 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||
2650 Schedule == OMP_dist_sch_static_chunked ||
2651 Schedule == OMP_dist_sch_static ||
2652 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone))
2653 Modifier = OMP_sch_modifier_nonmonotonic;
2654 }
2655 return Schedule | Modifier;
2656}
2657
2658void CGOpenMPRuntime::emitForDispatchInit(
2659 CodeGenFunction &CGF, SourceLocation Loc,
2660 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
2661 bool Ordered, const DispatchRTInput &DispatchValues) {
2662 if (!CGF.HaveInsertPoint())
2663 return;
2664 OpenMPSchedType Schedule = getRuntimeSchedule(
2665 ScheduleKind: ScheduleKind.Schedule, Chunked: DispatchValues.Chunk != nullptr, Ordered);
2666 assert(Ordered ||
2667 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&
2668 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&
2669 Schedule != OMP_sch_static_balanced_chunked));
2670 // Call __kmpc_dispatch_init(
2671 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule,
2672 // kmp_int[32|64] lower, kmp_int[32|64] upper,
2673 // kmp_int[32|64] stride, kmp_int[32|64] chunk);
2674
2675 // If the Chunk was not specified in the clause - use default value 1.
2676 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk
2677 : CGF.Builder.getIntN(N: IVSize, C: 1);
2678 llvm::Value *Args[] = {
2679 emitUpdateLocation(CGF, Loc),
2680 getThreadID(CGF, Loc),
2681 CGF.Builder.getInt32(C: addMonoNonMonoModifier(
2682 CGM, Schedule, M1: ScheduleKind.M1, M2: ScheduleKind.M2)), // Schedule type
2683 DispatchValues.LB, // Lower
2684 DispatchValues.UB, // Upper
2685 CGF.Builder.getIntN(N: IVSize, C: 1), // Stride
2686 Chunk // Chunk
2687 };
2688 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchInitFunction(IVSize, IVSigned),
2689 args: Args);
2690}
2691
2692void CGOpenMPRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
2693 SourceLocation Loc) {
2694 if (!CGF.HaveInsertPoint())
2695 return;
2696 // Call __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 tid);
2697 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2698 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchDeinitFunction(), args: Args);
2699}
2700
2701static void emitForStaticInitCall(
2702 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,
2703 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,
2704 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
2705 const CGOpenMPRuntime::StaticRTInput &Values) {
2706 if (!CGF.HaveInsertPoint())
2707 return;
2708
2709 assert(!Values.Ordered);
2710 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||
2711 Schedule == OMP_sch_static_balanced_chunked ||
2712 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||
2713 Schedule == OMP_dist_sch_static ||
2714 Schedule == OMP_dist_sch_static_chunked ||
2715 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone);
2716
2717 // Call __kmpc_for_static_init(
2718 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype,
2719 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower,
2720 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride,
2721 // kmp_int[32|64] incr, kmp_int[32|64] chunk);
2722 llvm::Value *Chunk = Values.Chunk;
2723 if (Chunk == nullptr) {
2724 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||
2725 Schedule == OMP_dist_sch_static) &&
2726 "expected static non-chunked schedule");
2727 // If the Chunk was not specified in the clause - use default value 1.
2728 Chunk = CGF.Builder.getIntN(N: Values.IVSize, C: 1);
2729 } else {
2730 assert((Schedule == OMP_sch_static_chunked ||
2731 Schedule == OMP_sch_static_balanced_chunked ||
2732 Schedule == OMP_ord_static_chunked ||
2733 Schedule == OMP_dist_sch_static_chunked ||
2734 Schedule == OMP_dist_sch_static_chunked_sch_static_chunkone) &&
2735 "expected static chunked schedule");
2736 }
2737 llvm::Value *Args[] = {
2738 UpdateLocation,
2739 ThreadId,
2740 CGF.Builder.getInt32(C: addMonoNonMonoModifier(CGM&: CGF.CGM, Schedule, M1,
2741 M2)), // Schedule type
2742 Values.IL.emitRawPointer(CGF), // &isLastIter
2743 Values.LB.emitRawPointer(CGF), // &LB
2744 Values.UB.emitRawPointer(CGF), // &UB
2745 Values.ST.emitRawPointer(CGF), // &Stride
2746 CGF.Builder.getIntN(N: Values.IVSize, C: 1), // Incr
2747 Chunk // Chunk
2748 };
2749 CGF.EmitRuntimeCall(callee: ForStaticInitFunction, args: Args);
2750}
2751
2752void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
2753 SourceLocation Loc,
2754 OpenMPDirectiveKind DKind,
2755 const OpenMPScheduleTy &ScheduleKind,
2756 const StaticRTInput &Values) {
2757 OpenMPSchedType ScheduleNum =
2758 ScheduleKind.UseFusedDistChunkSchedule
2759 ? OMP_dist_sch_static_chunked_sch_static_chunkone
2760 : getRuntimeSchedule(ScheduleKind: ScheduleKind.Schedule, Chunked: Values.Chunk != nullptr,
2761 Ordered: Values.Ordered);
2762 assert((isOpenMPWorksharingDirective(DKind) || (DKind == OMPD_loop)) &&
2763 "Expected loop-based or sections-based directive.");
2764 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
2765 Flags: isOpenMPLoopDirective(DKind)
2766 ? OMP_IDENT_WORK_LOOP
2767 : OMP_IDENT_WORK_SECTIONS);
2768 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2769 llvm::FunctionCallee StaticInitFunction =
2770 OMPBuilder.createForStaticInitFunction(IVSize: Values.IVSize, IVSigned: Values.IVSigned,
2771 IsGPUDistribute: false);
2772 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
2773 emitForStaticInitCall(CGF, UpdateLocation: UpdatedLocation, ThreadId, ForStaticInitFunction: StaticInitFunction,
2774 Schedule: ScheduleNum, M1: ScheduleKind.M1, M2: ScheduleKind.M2, Values);
2775}
2776
2777void CGOpenMPRuntime::emitDistributeStaticInit(
2778 CodeGenFunction &CGF, SourceLocation Loc,
2779 OpenMPDistScheduleClauseKind SchedKind,
2780 const CGOpenMPRuntime::StaticRTInput &Values) {
2781 OpenMPSchedType ScheduleNum =
2782 getRuntimeSchedule(ScheduleKind: SchedKind, Chunked: Values.Chunk != nullptr);
2783 llvm::Value *UpdatedLocation =
2784 emitUpdateLocation(CGF, Loc, Flags: OMP_IDENT_WORK_DISTRIBUTE);
2785 llvm::Value *ThreadId = getThreadID(CGF, Loc);
2786 llvm::FunctionCallee StaticInitFunction;
2787 bool isGPUDistribute =
2788 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU();
2789 StaticInitFunction = OMPBuilder.createForStaticInitFunction(
2790 IVSize: Values.IVSize, IVSigned: Values.IVSigned, IsGPUDistribute: isGPUDistribute);
2791
2792 emitForStaticInitCall(CGF, UpdateLocation: UpdatedLocation, ThreadId, ForStaticInitFunction: StaticInitFunction,
2793 Schedule: ScheduleNum, M1: OMPC_SCHEDULE_MODIFIER_unknown,
2794 M2: OMPC_SCHEDULE_MODIFIER_unknown, Values);
2795}
2796
2797void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
2798 SourceLocation Loc,
2799 OpenMPDirectiveKind DKind) {
2800 assert((DKind == OMPD_distribute || DKind == OMPD_for ||
2801 DKind == OMPD_sections) &&
2802 "Expected distribute, for, or sections directive kind");
2803 if (!CGF.HaveInsertPoint())
2804 return;
2805 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid);
2806 llvm::Value *Args[] = {
2807 emitUpdateLocation(CGF, Loc,
2808 Flags: isOpenMPDistributeDirective(DKind) ||
2809 (DKind == OMPD_target_teams_loop)
2810 ? OMP_IDENT_WORK_DISTRIBUTE
2811 : isOpenMPLoopDirective(DKind)
2812 ? OMP_IDENT_WORK_LOOP
2813 : OMP_IDENT_WORK_SECTIONS),
2814 getThreadID(CGF, Loc)};
2815 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
2816 if (isOpenMPDistributeDirective(DKind) &&
2817 CGM.getLangOpts().OpenMPIsTargetDevice && CGM.getTriple().isGPU())
2818 CGF.EmitRuntimeCall(
2819 callee: OMPBuilder.getOrCreateRuntimeFunction(
2820 M&: CGM.getModule(), FnID: OMPRTL___kmpc_distribute_static_fini),
2821 args: Args);
2822 else
2823 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2824 M&: CGM.getModule(), FnID: OMPRTL___kmpc_for_static_fini),
2825 args: Args);
2826}
2827
2828void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
2829 SourceLocation Loc,
2830 unsigned IVSize,
2831 bool IVSigned) {
2832 if (!CGF.HaveInsertPoint())
2833 return;
2834 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid);
2835 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)};
2836 CGF.EmitRuntimeCall(callee: OMPBuilder.createDispatchFiniFunction(IVSize, IVSigned),
2837 args: Args);
2838}
2839
2840llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF,
2841 SourceLocation Loc, unsigned IVSize,
2842 bool IVSigned, Address IL,
2843 Address LB, Address UB,
2844 Address ST) {
2845 // Call __kmpc_dispatch_next(
2846 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
2847 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
2848 // kmp_int[32|64] *p_stride);
2849 llvm::Value *Args[] = {
2850 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2851 IL.emitRawPointer(CGF), // &isLastIter
2852 LB.emitRawPointer(CGF), // &Lower
2853 UB.emitRawPointer(CGF), // &Upper
2854 ST.emitRawPointer(CGF) // &Stride
2855 };
2856 llvm::Value *Call = CGF.EmitRuntimeCall(
2857 callee: OMPBuilder.createDispatchNextFunction(IVSize, IVSigned), args: Args);
2858 return CGF.EmitScalarConversion(
2859 Src: Call, SrcTy: CGF.getContext().getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/1),
2860 DstTy: CGF.getContext().BoolTy, Loc);
2861}
2862
2863llvm::Value *CGOpenMPRuntime::emitMessageClause(CodeGenFunction &CGF,
2864 const Expr *Message,
2865 SourceLocation Loc) {
2866 if (!Message)
2867 return llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
2868 return CGF.EmitScalarExpr(E: Message);
2869}
2870
2871llvm::Value *
2872CGOpenMPRuntime::emitSeverityClause(OpenMPSeverityClauseKind Severity,
2873 SourceLocation Loc) {
2874 // OpenMP 6.0, 10.4: "If no severity clause is specified then the effect is
2875 // as if sev-level is fatal."
2876 return llvm::ConstantInt::get(Ty: CGM.Int32Ty,
2877 V: Severity == OMPC_SEVERITY_warning ? 1 : 2);
2878}
2879
2880void CGOpenMPRuntime::emitNumThreadsClause(
2881 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
2882 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,
2883 SourceLocation SeverityLoc, const Expr *Message,
2884 SourceLocation MessageLoc) {
2885 if (!CGF.HaveInsertPoint())
2886 return;
2887 llvm::SmallVector<llvm::Value *, 4> Args(
2888 {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2889 CGF.Builder.CreateIntCast(V: NumThreads, DestTy: CGF.Int32Ty, /*isSigned*/ true)});
2890 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
2891 // or __kmpc_push_num_threads_strict(&loc, global_tid, num_threads, severity,
2892 // messsage) if strict modifier is used.
2893 RuntimeFunction FnID = OMPRTL___kmpc_push_num_threads;
2894 if (Modifier == OMPC_NUMTHREADS_strict) {
2895 FnID = OMPRTL___kmpc_push_num_threads_strict;
2896 Args.push_back(Elt: emitSeverityClause(Severity, Loc: SeverityLoc));
2897 Args.push_back(Elt: emitMessageClause(CGF, Message, Loc: MessageLoc));
2898 }
2899 CGF.EmitRuntimeCall(
2900 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID), args: Args);
2901}
2902
2903void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF,
2904 ProcBindKind ProcBind,
2905 SourceLocation Loc) {
2906 if (!CGF.HaveInsertPoint())
2907 return;
2908 assert(ProcBind != OMP_PROC_BIND_unknown && "Unsupported proc_bind value.");
2909 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind)
2910 llvm::Value *Args[] = {
2911 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
2912 llvm::ConstantInt::get(Ty: CGM.IntTy, V: unsigned(ProcBind), /*isSigned=*/IsSigned: true)};
2913 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2914 M&: CGM.getModule(), FnID: OMPRTL___kmpc_push_proc_bind),
2915 args: Args);
2916}
2917
2918void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
2919 SourceLocation Loc, llvm::AtomicOrdering AO) {
2920 if (CGF.CGM.getLangOpts().OpenMPIRBuilder) {
2921 OMPBuilder.createFlush(Loc: CGF.Builder);
2922 } else {
2923 if (!CGF.HaveInsertPoint())
2924 return;
2925 // Build call void __kmpc_flush(ident_t *loc)
2926 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
2927 M&: CGM.getModule(), FnID: OMPRTL___kmpc_flush),
2928 args: emitUpdateLocation(CGF, Loc));
2929 }
2930}
2931
2932namespace {
2933/// Indexes of fields for type kmp_task_t.
2934enum KmpTaskTFields {
2935 /// List of shared variables.
2936 KmpTaskTShareds,
2937 /// Task routine.
2938 KmpTaskTRoutine,
2939 /// Partition id for the untied tasks.
2940 KmpTaskTPartId,
2941 /// Function with call of destructors for private variables.
2942 Data1,
2943 /// Task priority.
2944 Data2,
2945 /// (Taskloops only) Lower bound.
2946 KmpTaskTLowerBound,
2947 /// (Taskloops only) Upper bound.
2948 KmpTaskTUpperBound,
2949 /// (Taskloops only) Stride.
2950 KmpTaskTStride,
2951 /// (Taskloops only) Is last iteration flag.
2952 KmpTaskTLastIter,
2953 /// (Taskloops only) Reduction data.
2954 KmpTaskTReductions,
2955};
2956} // anonymous namespace
2957
2958void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
2959 // If we are in simd mode or there are no entries, we don't need to do
2960 // anything.
2961 if (CGM.getLangOpts().OpenMPSimd || OMPBuilder.OffloadInfoManager.empty())
2962 return;
2963
2964 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
2965 [this](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
2966 const llvm::TargetRegionEntryInfo &EntryInfo) -> void {
2967 SourceLocation Loc;
2968 if (Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {
2969 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(),
2970 E = CGM.getContext().getSourceManager().fileinfo_end();
2971 I != E; ++I) {
2972 if (I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&
2973 I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {
2974 Loc = CGM.getContext().getSourceManager().translateFileLineCol(
2975 SourceFile: I->getFirst(), Line: EntryInfo.Line, Col: 1);
2976 break;
2977 }
2978 }
2979 }
2980 switch (Kind) {
2981 case llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {
2982 CGM.getDiags().Report(Loc,
2983 DiagID: diag::err_target_region_offloading_entry_incorrect)
2984 << EntryInfo.ParentName;
2985 } break;
2986 case llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {
2987 CGM.getDiags().Report(
2988 Loc, DiagID: diag::err_target_var_offloading_entry_incorrect_with_parent)
2989 << EntryInfo.ParentName;
2990 } break;
2991 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {
2992 CGM.getDiags().Report(DiagID: diag::err_target_var_offloading_entry_incorrect);
2993 } break;
2994 case llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_INDIRECT_ERROR: {
2995 unsigned DiagID = CGM.getDiags().getCustomDiagID(
2996 L: DiagnosticsEngine::Error, FormatString: "Offloading entry for indirect declare "
2997 "target variable is incorrect: the "
2998 "address is invalid.");
2999 CGM.getDiags().Report(DiagID);
3000 } break;
3001 }
3002 };
3003
3004 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportFn);
3005}
3006
3007void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) {
3008 if (!KmpRoutineEntryPtrTy) {
3009 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type.
3010 ASTContext &C = CGM.getContext();
3011 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3012 FunctionProtoType::ExtProtoInfo EPI;
3013 KmpRoutineEntryPtrQTy = C.getPointerType(
3014 T: C.getFunctionType(ResultTy: KmpInt32Ty, Args: KmpRoutineEntryTyArgs, EPI));
3015 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(T: KmpRoutineEntryPtrQTy);
3016 }
3017}
3018
3019namespace {
3020struct PrivateHelpersTy {
3021 PrivateHelpersTy(const Expr *OriginalRef, const VarDecl *Original,
3022 const VarDecl *PrivateCopy, const VarDecl *PrivateElemInit)
3023 : OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),
3024 PrivateElemInit(PrivateElemInit) {}
3025 PrivateHelpersTy(const VarDecl *Original) : Original(Original) {}
3026 const Expr *OriginalRef = nullptr;
3027 const VarDecl *Original = nullptr;
3028 const VarDecl *PrivateCopy = nullptr;
3029 const VarDecl *PrivateElemInit = nullptr;
3030 bool isLocalPrivate() const {
3031 return !OriginalRef && !PrivateCopy && !PrivateElemInit;
3032 }
3033};
3034typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy;
3035} // anonymous namespace
3036
3037static bool isAllocatableDecl(const VarDecl *VD) {
3038 const VarDecl *CVD = VD->getCanonicalDecl();
3039 if (!CVD->hasAttr<OMPAllocateDeclAttr>())
3040 return false;
3041 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
3042 // Use the default allocation.
3043 return !(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&
3044 !AA->getAllocator());
3045}
3046
3047static RecordDecl *
3048createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
3049 if (!Privates.empty()) {
3050 ASTContext &C = CGM.getContext();
3051 // Build struct .kmp_privates_t. {
3052 // /* private vars */
3053 // };
3054 RecordDecl *RD = C.buildImplicitRecord(Name: ".kmp_privates.t");
3055 RD->startDefinition();
3056 for (const auto &Pair : Privates) {
3057 const VarDecl *VD = Pair.second.Original;
3058 QualType Type = VD->getType().getNonReferenceType();
3059 // If the private variable is a local variable with lvalue ref type,
3060 // allocate the pointer instead of the pointee type.
3061 if (Pair.second.isLocalPrivate()) {
3062 if (VD->getType()->isLValueReferenceType())
3063 Type = C.getPointerType(T: Type);
3064 if (isAllocatableDecl(VD))
3065 Type = C.getPointerType(T: Type);
3066 }
3067 FieldDecl *FD = addFieldToRecordDecl(C, DC: RD, FieldTy: Type);
3068 if (VD->hasAttrs()) {
3069 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
3070 E(VD->getAttrs().end());
3071 I != E; ++I)
3072 FD->addAttr(A: *I);
3073 }
3074 }
3075 RD->completeDefinition();
3076 return RD;
3077 }
3078 return nullptr;
3079}
3080
3081static RecordDecl *
3082createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
3083 QualType KmpInt32Ty,
3084 QualType KmpRoutineEntryPointerQTy) {
3085 ASTContext &C = CGM.getContext();
3086 // Build struct kmp_task_t {
3087 // void * shareds;
3088 // kmp_routine_entry_t routine;
3089 // kmp_int32 part_id;
3090 // kmp_cmplrdata_t data1;
3091 // kmp_cmplrdata_t data2;
3092 // For taskloops additional fields:
3093 // kmp_uint64 lb;
3094 // kmp_uint64 ub;
3095 // kmp_int64 st;
3096 // kmp_int32 liter;
3097 // void * reductions;
3098 // };
3099 RecordDecl *UD = C.buildImplicitRecord(Name: "kmp_cmplrdata_t", TK: TagTypeKind::Union);
3100 UD->startDefinition();
3101 addFieldToRecordDecl(C, DC: UD, FieldTy: KmpInt32Ty);
3102 addFieldToRecordDecl(C, DC: UD, FieldTy: KmpRoutineEntryPointerQTy);
3103 UD->completeDefinition();
3104 CanQualType KmpCmplrdataTy = C.getCanonicalTagType(TD: UD);
3105 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_task_t");
3106 RD->startDefinition();
3107 addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
3108 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpRoutineEntryPointerQTy);
3109 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt32Ty);
3110 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpCmplrdataTy);
3111 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpCmplrdataTy);
3112 if (isOpenMPTaskLoopDirective(DKind: Kind)) {
3113 QualType KmpUInt64Ty =
3114 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0);
3115 QualType KmpInt64Ty =
3116 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
3117 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpUInt64Ty);
3118 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpUInt64Ty);
3119 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt64Ty);
3120 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpInt32Ty);
3121 addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
3122 }
3123 RD->completeDefinition();
3124 return RD;
3125}
3126
3127static RecordDecl *
3128createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
3129 ArrayRef<PrivateDataTy> Privates) {
3130 ASTContext &C = CGM.getContext();
3131 // Build struct kmp_task_t_with_privates {
3132 // kmp_task_t task_data;
3133 // .kmp_privates_t. privates;
3134 // };
3135 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_task_t_with_privates");
3136 RD->startDefinition();
3137 addFieldToRecordDecl(C, DC: RD, FieldTy: KmpTaskTQTy);
3138 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
3139 addFieldToRecordDecl(C, DC: RD, FieldTy: C.getCanonicalTagType(TD: PrivateRD));
3140 RD->completeDefinition();
3141 return RD;
3142}
3143
3144/// Emit a proxy function which accepts kmp_task_t as the second
3145/// argument.
3146/// \code
3147/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
3148/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt,
3149/// For taskloops:
3150/// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3151/// tt->reductions, tt->shareds);
3152/// return 0;
3153/// }
3154/// \endcode
3155static llvm::Function *
3156emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
3157 OpenMPDirectiveKind Kind, QualType KmpInt32Ty,
3158 QualType KmpTaskTWithPrivatesPtrQTy,
3159 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
3160 QualType SharedsPtrTy, llvm::Function *TaskFunction,
3161 llvm::Value *TaskPrivatesMap) {
3162 ASTContext &C = CGM.getContext();
3163 auto *GtidArg =
3164 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3165 T: KmpInt32Ty, ParamKind: ImplicitParamKind::Other);
3166 auto *TaskTypeArg = ImplicitParamDecl::Create(
3167 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3168 T: KmpTaskTWithPrivatesPtrQTy.withRestrict(), ParamKind: ImplicitParamKind::Other);
3169 FunctionArgList Args{GtidArg, TaskTypeArg};
3170 const auto &TaskEntryFnInfo =
3171 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: KmpInt32Ty, args: Args);
3172 llvm::FunctionType *TaskEntryTy =
3173 CGM.getTypes().GetFunctionType(Info: TaskEntryFnInfo);
3174 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"omp_task_entry", ""});
3175 auto *TaskEntry = llvm::Function::Create(
3176 Ty: TaskEntryTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &CGM.getModule());
3177 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskEntry, FI: TaskEntryFnInfo);
3178 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3179 TaskEntry->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3180 TaskEntry->setDoesNotRecurse();
3181 CodeGenFunction CGF(CGM);
3182 CGF.StartFunction(GD: GlobalDecl(), RetTy: KmpInt32Ty, Fn: TaskEntry, FnInfo: TaskEntryFnInfo, Args,
3183 Loc, StartLoc: Loc);
3184
3185 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
3186 // tt,
3187 // For taskloops:
3188 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter,
3189 // tt->task_data.shareds);
3190 llvm::Value *GtidParam = CGF.EmitLoadOfScalar(
3191 Addr: CGF.GetAddrOfLocalVar(VD: GtidArg), /*Volatile=*/false, Ty: KmpInt32Ty, Loc);
3192 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3193 Ptr: CGF.GetAddrOfLocalVar(VD: TaskTypeArg),
3194 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3195 const auto *KmpTaskTWithPrivatesQTyRD =
3196 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3197 LValue Base =
3198 CGF.EmitLValueForField(Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3199 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
3200 auto PartIdFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTPartId);
3201 LValue PartIdLVal = CGF.EmitLValueForField(Base, Field: *PartIdFI);
3202 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF);
3203
3204 auto SharedsFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTShareds);
3205 LValue SharedsLVal = CGF.EmitLValueForField(Base, Field: *SharedsFI);
3206 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3207 V: CGF.EmitLoadOfScalar(lvalue: SharedsLVal, Loc),
3208 DestTy: CGF.ConvertTypeForMem(T: SharedsPtrTy));
3209
3210 auto PrivatesFI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin(), n: 1);
3211 llvm::Value *PrivatesParam;
3212 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3213 LValue PrivatesLVal = CGF.EmitLValueForField(Base: TDBase, Field: *PrivatesFI);
3214 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3215 V: PrivatesLVal.getPointer(CGF), DestTy: CGF.VoidPtrTy);
3216 } else {
3217 PrivatesParam = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
3218 }
3219
3220 llvm::Value *CommonArgs[] = {
3221 GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,
3222 CGF.Builder
3223 .CreatePointerBitCastOrAddrSpaceCast(Addr: TDBase.getAddress(),
3224 Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty)
3225 .emitRawPointer(CGF)};
3226 SmallVector<llvm::Value *, 16> CallArgs(std::begin(arr&: CommonArgs),
3227 std::end(arr&: CommonArgs));
3228 if (isOpenMPTaskLoopDirective(DKind: Kind)) {
3229 auto LBFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLowerBound);
3230 LValue LBLVal = CGF.EmitLValueForField(Base, Field: *LBFI);
3231 llvm::Value *LBParam = CGF.EmitLoadOfScalar(lvalue: LBLVal, Loc);
3232 auto UBFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTUpperBound);
3233 LValue UBLVal = CGF.EmitLValueForField(Base, Field: *UBFI);
3234 llvm::Value *UBParam = CGF.EmitLoadOfScalar(lvalue: UBLVal, Loc);
3235 auto StFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTStride);
3236 LValue StLVal = CGF.EmitLValueForField(Base, Field: *StFI);
3237 llvm::Value *StParam = CGF.EmitLoadOfScalar(lvalue: StLVal, Loc);
3238 auto LIFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLastIter);
3239 LValue LILVal = CGF.EmitLValueForField(Base, Field: *LIFI);
3240 llvm::Value *LIParam = CGF.EmitLoadOfScalar(lvalue: LILVal, Loc);
3241 auto RFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTReductions);
3242 LValue RLVal = CGF.EmitLValueForField(Base, Field: *RFI);
3243 llvm::Value *RParam = CGF.EmitLoadOfScalar(lvalue: RLVal, Loc);
3244 CallArgs.push_back(Elt: LBParam);
3245 CallArgs.push_back(Elt: UBParam);
3246 CallArgs.push_back(Elt: StParam);
3247 CallArgs.push_back(Elt: LIParam);
3248 CallArgs.push_back(Elt: RParam);
3249 }
3250 CallArgs.push_back(Elt: SharedsParam);
3251
3252 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn: TaskFunction,
3253 Args: CallArgs);
3254 CGF.EmitStoreThroughLValue(Src: RValue::get(V: CGF.Builder.getInt32(/*C=*/0)),
3255 Dst: CGF.MakeAddrLValue(Addr: CGF.ReturnValue, T: KmpInt32Ty));
3256 CGF.FinishFunction();
3257 return TaskEntry;
3258}
3259
3260static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
3261 SourceLocation Loc,
3262 QualType KmpInt32Ty,
3263 QualType KmpTaskTWithPrivatesPtrQTy,
3264 QualType KmpTaskTWithPrivatesQTy) {
3265 ASTContext &C = CGM.getContext();
3266 auto *GtidArg =
3267 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3268 T: KmpInt32Ty, ParamKind: ImplicitParamKind::Other);
3269 auto *TaskTypeArg = ImplicitParamDecl::Create(
3270 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3271 T: KmpTaskTWithPrivatesPtrQTy.withRestrict(), ParamKind: ImplicitParamKind::Other);
3272 FunctionArgList Args{GtidArg, TaskTypeArg};
3273 const auto &DestructorFnInfo =
3274 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: KmpInt32Ty, args: Args);
3275 llvm::FunctionType *DestructorFnTy =
3276 CGM.getTypes().GetFunctionType(Info: DestructorFnInfo);
3277 std::string Name =
3278 CGM.getOpenMPRuntime().getName(Parts: {"omp_task_destructor", ""});
3279 auto *DestructorFn =
3280 llvm::Function::Create(Ty: DestructorFnTy, Linkage: llvm::GlobalValue::InternalLinkage,
3281 N: Name, M: &CGM.getModule());
3282 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: DestructorFn,
3283 FI: DestructorFnInfo);
3284 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3285 DestructorFn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3286 DestructorFn->setDoesNotRecurse();
3287 CodeGenFunction CGF(CGM);
3288 CGF.StartFunction(GD: GlobalDecl(), RetTy: KmpInt32Ty, Fn: DestructorFn, FnInfo: DestructorFnInfo,
3289 Args, Loc, StartLoc: Loc);
3290
3291 LValue Base = CGF.EmitLoadOfPointerLValue(
3292 Ptr: CGF.GetAddrOfLocalVar(VD: TaskTypeArg),
3293 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3294 const auto *KmpTaskTWithPrivatesQTyRD =
3295 KmpTaskTWithPrivatesQTy->castAsRecordDecl();
3296 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3297 Base = CGF.EmitLValueForField(Base, Field: *FI);
3298 for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
3299 if (QualType::DestructionKind DtorKind =
3300 Field->getType().isDestructedType()) {
3301 LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
3302 CGF.pushDestroy(dtorKind: DtorKind, addr: FieldLValue.getAddress(), type: Field->getType());
3303 }
3304 }
3305 CGF.FinishFunction();
3306 return DestructorFn;
3307}
3308
3309/// Emit a privates mapping function for correct handling of private and
3310/// firstprivate variables.
3311/// \code
3312/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
3313/// **noalias priv1,..., <tyn> **noalias privn) {
3314/// *priv1 = &.privates.priv1;
3315/// ...;
3316/// *privn = &.privates.privn;
3317/// }
3318/// \endcode
3319static llvm::Value *
3320emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
3321 const OMPTaskDataTy &Data, QualType PrivatesQTy,
3322 ArrayRef<PrivateDataTy> Privates) {
3323 ASTContext &C = CGM.getContext();
3324 FunctionArgList Args;
3325 auto *TaskPrivatesArg = ImplicitParamDecl::Create(
3326 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3327 T: C.getPointerType(T: PrivatesQTy).withConst().withRestrict(),
3328 ParamKind: ImplicitParamKind::Other);
3329 Args.push_back(Elt: TaskPrivatesArg);
3330 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, unsigned> PrivateVarsPos;
3331 unsigned Counter = 1;
3332 for (const Expr *E : Data.PrivateVars) {
3333 Args.push_back(Elt: ImplicitParamDecl::Create(
3334 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3335 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3336 .withConst()
3337 .withRestrict(),
3338 ParamKind: ImplicitParamKind::Other));
3339 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3340 PrivateVarsPos[VD] = Counter;
3341 ++Counter;
3342 }
3343 for (const Expr *E : Data.FirstprivateVars) {
3344 Args.push_back(Elt: ImplicitParamDecl::Create(
3345 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3346 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3347 .withConst()
3348 .withRestrict(),
3349 ParamKind: ImplicitParamKind::Other));
3350 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3351 PrivateVarsPos[VD] = Counter;
3352 ++Counter;
3353 }
3354 for (const Expr *E : Data.LastprivateVars) {
3355 Args.push_back(Elt: ImplicitParamDecl::Create(
3356 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3357 T: C.getPointerType(T: C.getPointerType(T: E->getType()))
3358 .withConst()
3359 .withRestrict(),
3360 ParamKind: ImplicitParamKind::Other));
3361 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3362 PrivateVarsPos[VD] = Counter;
3363 ++Counter;
3364 }
3365 for (const VarDecl *VD : Data.PrivateLocals) {
3366 QualType Ty = VD->getType().getNonReferenceType();
3367 if (VD->getType()->isLValueReferenceType())
3368 Ty = C.getPointerType(T: Ty);
3369 if (isAllocatableDecl(VD))
3370 Ty = C.getPointerType(T: Ty);
3371 Args.push_back(Elt: ImplicitParamDecl::Create(
3372 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
3373 T: C.getPointerType(T: C.getPointerType(T: Ty)).withConst().withRestrict(),
3374 ParamKind: ImplicitParamKind::Other));
3375 PrivateVarsPos[VD] = Counter;
3376 ++Counter;
3377 }
3378 const auto &TaskPrivatesMapFnInfo =
3379 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
3380 llvm::FunctionType *TaskPrivatesMapTy =
3381 CGM.getTypes().GetFunctionType(Info: TaskPrivatesMapFnInfo);
3382 std::string Name =
3383 CGM.getOpenMPRuntime().getName(Parts: {"omp_task_privates_map", ""});
3384 auto *TaskPrivatesMap = llvm::Function::Create(
3385 Ty: TaskPrivatesMapTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
3386 M: &CGM.getModule());
3387 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskPrivatesMap,
3388 FI: TaskPrivatesMapFnInfo);
3389 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3390 TaskPrivatesMap->addFnAttr(Kind: "sample-profile-suffix-elision-policy",
3391 Val: "selected");
3392 if (CGM.getCodeGenOpts().OptimizationLevel != 0) {
3393 TaskPrivatesMap->removeFnAttr(Kind: llvm::Attribute::NoInline);
3394 TaskPrivatesMap->removeFnAttr(Kind: llvm::Attribute::OptimizeNone);
3395 TaskPrivatesMap->addFnAttr(Kind: llvm::Attribute::AlwaysInline);
3396 }
3397 CodeGenFunction CGF(CGM);
3398 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn: TaskPrivatesMap,
3399 FnInfo: TaskPrivatesMapFnInfo, Args, Loc, StartLoc: Loc);
3400
3401 // *privi = &.privates.privi;
3402 LValue Base = CGF.EmitLoadOfPointerLValue(
3403 Ptr: CGF.GetAddrOfLocalVar(VD: TaskPrivatesArg),
3404 PtrTy: TaskPrivatesArg->getType()->castAs<PointerType>());
3405 const auto *PrivatesQTyRD = PrivatesQTy->castAsRecordDecl();
3406 Counter = 0;
3407 for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
3408 LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
3409 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3410 LValue RefLVal =
3411 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD), T: VD->getType());
3412 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue(
3413 Ptr: RefLVal.getAddress(), PtrTy: RefLVal.getType()->castAs<PointerType>());
3414 CGF.EmitStoreOfScalar(value: FieldLVal.getPointer(CGF), lvalue: RefLoadLVal);
3415 ++Counter;
3416 }
3417 CGF.FinishFunction();
3418 return TaskPrivatesMap;
3419}
3420
3421/// Emit initialization for private variables in task-based directives.
3422static void emitPrivatesInit(CodeGenFunction &CGF,
3423 const OMPExecutableDirective &D,
3424 Address KmpTaskSharedsPtr, LValue TDBase,
3425 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3426 QualType SharedsTy, QualType SharedsPtrTy,
3427 const OMPTaskDataTy &Data,
3428 ArrayRef<PrivateDataTy> Privates, bool ForDup) {
3429 ASTContext &C = CGF.getContext();
3430 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3431 LValue PrivatesBase = CGF.EmitLValueForField(Base: TDBase, Field: *FI);
3432 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind())
3433 ? OMPD_taskloop
3434 : OMPD_task;
3435 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: Kind);
3436 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
3437 LValue SrcBase;
3438 bool IsTargetTask =
3439 isOpenMPTargetDataManagementDirective(DKind: D.getDirectiveKind()) ||
3440 isOpenMPTargetExecutionDirective(DKind: D.getDirectiveKind());
3441 // For target-based directives skip 4 firstprivate arrays BasePointersArray,
3442 // PointersArray, SizesArray, and MappersArray. The original variables for
3443 // these arrays are not captured and we get their addresses explicitly.
3444 if ((!IsTargetTask && !Data.FirstprivateVars.empty() && ForDup) ||
3445 (IsTargetTask && KmpTaskSharedsPtr.isValid())) {
3446 SrcBase = CGF.MakeAddrLValue(
3447 Addr: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3448 Addr: KmpTaskSharedsPtr, Ty: CGF.ConvertTypeForMem(T: SharedsPtrTy),
3449 ElementTy: CGF.ConvertTypeForMem(T: SharedsTy)),
3450 T: SharedsTy);
3451 }
3452 FI = FI->getType()->castAsRecordDecl()->field_begin();
3453 for (const PrivateDataTy &Pair : Privates) {
3454 // Do not initialize private locals.
3455 if (Pair.second.isLocalPrivate()) {
3456 ++FI;
3457 continue;
3458 }
3459 const VarDecl *VD = Pair.second.PrivateCopy;
3460 const Expr *Init = VD->getAnyInitializer();
3461 if (Init && (!ForDup || (isa<CXXConstructExpr>(Val: Init) &&
3462 !CGF.isTrivialInitializer(Init)))) {
3463 LValue PrivateLValue = CGF.EmitLValueForField(Base: PrivatesBase, Field: *FI);
3464 if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
3465 const VarDecl *OriginalVD = Pair.second.Original;
3466 // Check if the variable is the target-based BasePointersArray,
3467 // PointersArray, SizesArray, or MappersArray.
3468 LValue SharedRefLValue;
3469 QualType Type = PrivateLValue.getType();
3470 const FieldDecl *SharedField = CapturesInfo.lookup(VD: OriginalVD);
3471 if (IsTargetTask && !SharedField) {
3472 assert(isa<ImplicitParamDecl>(OriginalVD) &&
3473 isa<CapturedDecl>(OriginalVD->getDeclContext()) &&
3474 cast<CapturedDecl>(OriginalVD->getDeclContext())
3475 ->getNumParams() == 0 &&
3476 isa<TranslationUnitDecl>(
3477 cast<CapturedDecl>(OriginalVD->getDeclContext())
3478 ->getDeclContext()) &&
3479 "Expected artificial target data variable.");
3480 SharedRefLValue =
3481 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: OriginalVD), T: Type);
3482 } else if (ForDup) {
3483 SharedRefLValue = CGF.EmitLValueForField(Base: SrcBase, Field: SharedField);
3484 SharedRefLValue = CGF.MakeAddrLValue(
3485 Addr: SharedRefLValue.getAddress().withAlignment(
3486 NewAlignment: C.getDeclAlign(D: OriginalVD)),
3487 T: SharedRefLValue.getType(), BaseInfo: LValueBaseInfo(AlignmentSource::Decl),
3488 TBAAInfo: SharedRefLValue.getTBAAInfo());
3489 } else if (CGF.LambdaCaptureFields.count(
3490 Val: Pair.second.Original->getCanonicalDecl()) > 0 ||
3491 isa_and_nonnull<BlockDecl>(Val: CGF.CurCodeDecl)) {
3492 SharedRefLValue = CGF.EmitLValue(E: Pair.second.OriginalRef);
3493 } else {
3494 // Processing for implicitly captured variables.
3495 InlinedOpenMPRegionRAII Region(
3496 CGF, [](CodeGenFunction &, PrePostActionTy &) {}, OMPD_unknown,
3497 /*HasCancel=*/false, /*NoInheritance=*/true);
3498 SharedRefLValue = CGF.EmitLValue(E: Pair.second.OriginalRef);
3499 }
3500 if (Type->isArrayType()) {
3501 // Initialize firstprivate array.
3502 if (!isa<CXXConstructExpr>(Val: Init) || CGF.isTrivialInitializer(Init)) {
3503 // Perform simple memcpy.
3504 CGF.EmitAggregateAssign(Dest: PrivateLValue, Src: SharedRefLValue, EltTy: Type);
3505 } else {
3506 // Initialize firstprivate array using element-by-element
3507 // initialization.
3508 CGF.EmitOMPAggregateAssign(
3509 DestAddr: PrivateLValue.getAddress(), SrcAddr: SharedRefLValue.getAddress(), OriginalType: Type,
3510 CopyGen: [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
3511 Address SrcElement) {
3512 // Clean up any temporaries needed by the initialization.
3513 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3514 InitScope.addPrivate(LocalVD: Elem, Addr: SrcElement);
3515 (void)InitScope.Privatize();
3516 // Emit initialization for single element.
3517 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
3518 CGF, &CapturesInfo);
3519 CGF.EmitAnyExprToMem(E: Init, Location: DestElement,
3520 Quals: Init->getType().getQualifiers(),
3521 /*IsInitializer=*/false);
3522 });
3523 }
3524 } else {
3525 CodeGenFunction::OMPPrivateScope InitScope(CGF);
3526 InitScope.addPrivate(LocalVD: Elem, Addr: SharedRefLValue.getAddress());
3527 (void)InitScope.Privatize();
3528 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
3529 CGF.EmitExprAsInit(init: Init, D: VD, lvalue: PrivateLValue,
3530 /*capturedByInit=*/false);
3531 }
3532 } else {
3533 CGF.EmitExprAsInit(init: Init, D: VD, lvalue: PrivateLValue, /*capturedByInit=*/false);
3534 }
3535 }
3536 ++FI;
3537 }
3538}
3539
3540/// Check if duplication function is required for taskloops.
3541static bool checkInitIsRequired(CodeGenFunction &CGF,
3542 ArrayRef<PrivateDataTy> Privates) {
3543 bool InitRequired = false;
3544 for (const PrivateDataTy &Pair : Privates) {
3545 if (Pair.second.isLocalPrivate())
3546 continue;
3547 const VarDecl *VD = Pair.second.PrivateCopy;
3548 const Expr *Init = VD->getAnyInitializer();
3549 InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(Val: Init) &&
3550 !CGF.isTrivialInitializer(Init));
3551 if (InitRequired)
3552 break;
3553 }
3554 return InitRequired;
3555}
3556
3557
3558/// Emit task_dup function (for initialization of
3559/// private/firstprivate/lastprivate vars and last_iter flag)
3560/// \code
3561/// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int
3562/// lastpriv) {
3563/// // setup lastprivate flag
3564/// task_dst->last = lastpriv;
3565/// // could be constructor calls here...
3566/// }
3567/// \endcode
3568static llvm::Value *
3569emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
3570 const OMPExecutableDirective &D,
3571 QualType KmpTaskTWithPrivatesPtrQTy,
3572 const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3573 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy,
3574 QualType SharedsPtrTy, const OMPTaskDataTy &Data,
3575 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) {
3576 ASTContext &C = CGM.getContext();
3577 auto *DstArg = ImplicitParamDecl::Create(
3578 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: KmpTaskTWithPrivatesPtrQTy,
3579 ParamKind: ImplicitParamKind::Other);
3580 auto *SrcArg = ImplicitParamDecl::Create(
3581 C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: KmpTaskTWithPrivatesPtrQTy,
3582 ParamKind: ImplicitParamKind::Other);
3583 auto *LastprivArg =
3584 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr, T: C.IntTy,
3585 ParamKind: ImplicitParamKind::Other);
3586 FunctionArgList Args{DstArg, SrcArg, LastprivArg};
3587 const auto &TaskDupFnInfo =
3588 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
3589 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(Info: TaskDupFnInfo);
3590 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"omp_task_dup", ""});
3591 auto *TaskDup = llvm::Function::Create(
3592 Ty: TaskDupTy, Linkage: llvm::GlobalValue::InternalLinkage, N: Name, M: &CGM.getModule());
3593 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: TaskDup, FI: TaskDupFnInfo);
3594 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
3595 TaskDup->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
3596 TaskDup->setDoesNotRecurse();
3597 CodeGenFunction CGF(CGM);
3598 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn: TaskDup, FnInfo: TaskDupFnInfo, Args, Loc,
3599 StartLoc: Loc);
3600
3601 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3602 Ptr: CGF.GetAddrOfLocalVar(VD: DstArg),
3603 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3604 // task_dst->liter = lastpriv;
3605 if (WithLastIter) {
3606 auto LIFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTLastIter);
3607 LValue Base = CGF.EmitLValueForField(
3608 Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3609 LValue LILVal = CGF.EmitLValueForField(Base, Field: *LIFI);
3610 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar(
3611 Addr: CGF.GetAddrOfLocalVar(VD: LastprivArg), /*Volatile=*/false, Ty: C.IntTy, Loc);
3612 CGF.EmitStoreOfScalar(value: Lastpriv, lvalue: LILVal);
3613 }
3614
3615 // Emit initial values for private copies (if any).
3616 assert(!Privates.empty());
3617 Address KmpTaskSharedsPtr = Address::invalid();
3618 if (!Data.FirstprivateVars.empty()) {
3619 LValue TDBase = CGF.EmitLoadOfPointerLValue(
3620 Ptr: CGF.GetAddrOfLocalVar(VD: SrcArg),
3621 PtrTy: KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
3622 LValue Base = CGF.EmitLValueForField(
3623 Base: TDBase, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
3624 KmpTaskSharedsPtr = Address(
3625 CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValueForField(
3626 Base, Field: *std::next(x: KmpTaskTQTyRD->field_begin(),
3627 n: KmpTaskTShareds)),
3628 Loc),
3629 CGF.Int8Ty, CGM.getNaturalTypeAlignment(T: SharedsTy));
3630 }
3631 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3632 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
3633 CGF.FinishFunction();
3634 return TaskDup;
3635}
3636
3637/// Checks if destructor function is required to be generated.
3638/// \return true if cleanups are required, false otherwise.
3639static bool
3640checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,
3641 ArrayRef<PrivateDataTy> Privates) {
3642 for (const PrivateDataTy &P : Privates) {
3643 if (P.second.isLocalPrivate())
3644 continue;
3645 QualType Ty = P.second.Original->getType().getNonReferenceType();
3646 if (Ty.isDestructedType())
3647 return true;
3648 }
3649 return false;
3650}
3651
3652namespace {
3653/// Loop generator for OpenMP iterator expression.
3654class OMPIteratorGeneratorScope final
3655 : public CodeGenFunction::OMPPrivateScope {
3656 CodeGenFunction &CGF;
3657 const OMPIteratorExpr *E = nullptr;
3658 SmallVector<CodeGenFunction::JumpDest, 4> ContDests;
3659 SmallVector<CodeGenFunction::JumpDest, 4> ExitDests;
3660 OMPIteratorGeneratorScope() = delete;
3661 OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) = delete;
3662
3663public:
3664 OMPIteratorGeneratorScope(CodeGenFunction &CGF, const OMPIteratorExpr *E)
3665 : CodeGenFunction::OMPPrivateScope(CGF), CGF(CGF), E(E) {
3666 if (!E)
3667 return;
3668 SmallVector<llvm::Value *, 4> Uppers;
3669 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3670 Uppers.push_back(Elt: CGF.EmitScalarExpr(E: E->getHelper(I).Upper));
3671 const auto *VD = cast<VarDecl>(Val: E->getIteratorDecl(I));
3672 addPrivate(LocalVD: VD, Addr: CGF.CreateMemTemp(T: VD->getType(), Name: VD->getName()));
3673 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3674 addPrivate(
3675 LocalVD: HelperData.CounterVD,
3676 Addr: CGF.CreateMemTemp(T: HelperData.CounterVD->getType(), Name: "counter.addr"));
3677 }
3678 Privatize();
3679
3680 for (unsigned I = 0, End = E->numOfIterators(); I < End; ++I) {
3681 const OMPIteratorHelperData &HelperData = E->getHelper(I);
3682 LValue CLVal =
3683 CGF.MakeAddrLValue(Addr: CGF.GetAddrOfLocalVar(VD: HelperData.CounterVD),
3684 T: HelperData.CounterVD->getType());
3685 // Counter = 0;
3686 CGF.EmitStoreOfScalar(
3687 value: llvm::ConstantInt::get(Ty: CLVal.getAddress().getElementType(), V: 0),
3688 lvalue: CLVal);
3689 CodeGenFunction::JumpDest &ContDest =
3690 ContDests.emplace_back(Args: CGF.getJumpDestInCurrentScope(Name: "iter.cont"));
3691 CodeGenFunction::JumpDest &ExitDest =
3692 ExitDests.emplace_back(Args: CGF.getJumpDestInCurrentScope(Name: "iter.exit"));
3693 // N = <number-of_iterations>;
3694 llvm::Value *N = Uppers[I];
3695 // cont:
3696 // if (Counter < N) goto body; else goto exit;
3697 CGF.EmitBlock(BB: ContDest.getBlock());
3698 auto *CVal =
3699 CGF.EmitLoadOfScalar(lvalue: CLVal, Loc: HelperData.CounterVD->getLocation());
3700 llvm::Value *Cmp =
3701 HelperData.CounterVD->getType()->isSignedIntegerOrEnumerationType()
3702 ? CGF.Builder.CreateICmpSLT(LHS: CVal, RHS: N)
3703 : CGF.Builder.CreateICmpULT(LHS: CVal, RHS: N);
3704 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "iter.body");
3705 CGF.Builder.CreateCondBr(Cond: Cmp, True: BodyBB, False: ExitDest.getBlock());
3706 // body:
3707 CGF.EmitBlock(BB: BodyBB);
3708 // Iteri = Begini + Counter * Stepi;
3709 CGF.EmitIgnoredExpr(E: HelperData.Update);
3710 }
3711 }
3712 ~OMPIteratorGeneratorScope() {
3713 if (!E)
3714 return;
3715 for (unsigned I = E->numOfIterators(); I > 0; --I) {
3716 // Counter = Counter + 1;
3717 const OMPIteratorHelperData &HelperData = E->getHelper(I: I - 1);
3718 CGF.EmitIgnoredExpr(E: HelperData.CounterUpdate);
3719 // goto cont;
3720 CGF.EmitBranchThroughCleanup(Dest: ContDests[I - 1]);
3721 // exit:
3722 CGF.EmitBlock(BB: ExitDests[I - 1].getBlock(), /*IsFinished=*/I == 1);
3723 }
3724 }
3725};
3726} // namespace
3727
3728static std::pair<llvm::Value *, llvm::Value *>
3729getPointerAndSize(CodeGenFunction &CGF, const Expr *E) {
3730 const auto *OASE = dyn_cast<OMPArrayShapingExpr>(Val: E);
3731 llvm::Value *Addr;
3732 if (OASE) {
3733 const Expr *Base = OASE->getBase();
3734 Addr = CGF.EmitScalarExpr(E: Base);
3735 } else {
3736 Addr = CGF.EmitLValue(E).getPointer(CGF);
3737 }
3738 llvm::Value *SizeVal;
3739 QualType Ty = E->getType();
3740 if (OASE) {
3741 SizeVal = CGF.getTypeSize(Ty: OASE->getBase()->getType()->getPointeeType());
3742 for (const Expr *SE : OASE->getDimensions()) {
3743 llvm::Value *Sz = CGF.EmitScalarExpr(E: SE);
3744 Sz = CGF.EmitScalarConversion(
3745 Src: Sz, SrcTy: SE->getType(), DstTy: CGF.getContext().getSizeType(), Loc: SE->getExprLoc());
3746 SizeVal = CGF.Builder.CreateNUWMul(LHS: SizeVal, RHS: Sz);
3747 }
3748 } else if (const auto *ASE =
3749 dyn_cast<ArraySectionExpr>(Val: E->IgnoreParenImpCasts())) {
3750 LValue UpAddrLVal = CGF.EmitArraySectionExpr(E: ASE, /*IsLowerBound=*/false);
3751 Address UpAddrAddress = UpAddrLVal.getAddress();
3752 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32(
3753 Ty: UpAddrAddress.getElementType(), Ptr: UpAddrAddress.emitRawPointer(CGF),
3754 /*Idx0=*/1);
3755 SizeVal = CGF.Builder.CreatePtrDiff(LHS: UpAddr, RHS: Addr, Name: "", /*IsNUW=*/true);
3756 } else {
3757 SizeVal = CGF.getTypeSize(Ty);
3758 }
3759 return std::make_pair(x&: Addr, y&: SizeVal);
3760}
3761
3762/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
3763static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
3764 QualType FlagsTy = C.getIntTypeForBitwidth(DestWidth: 32, /*Signed=*/false);
3765 if (KmpTaskAffinityInfoTy.isNull()) {
3766 RecordDecl *KmpAffinityInfoRD =
3767 C.buildImplicitRecord(Name: "kmp_task_affinity_info_t");
3768 KmpAffinityInfoRD->startDefinition();
3769 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: C.getIntPtrType());
3770 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: C.getSizeType());
3771 addFieldToRecordDecl(C, DC: KmpAffinityInfoRD, FieldTy: FlagsTy);
3772 KmpAffinityInfoRD->completeDefinition();
3773 KmpTaskAffinityInfoTy = C.getCanonicalTagType(TD: KmpAffinityInfoRD);
3774 }
3775}
3776
3777CGOpenMPRuntime::TaskResultTy
3778CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
3779 const OMPExecutableDirective &D,
3780 llvm::Function *TaskFunction, QualType SharedsTy,
3781 Address Shareds, const OMPTaskDataTy &Data) {
3782 ASTContext &C = CGM.getContext();
3783 llvm::SmallVector<PrivateDataTy, 4> Privates;
3784 // Aggregate privates and sort them by the alignment.
3785 const auto *I = Data.PrivateCopies.begin();
3786 for (const Expr *E : Data.PrivateVars) {
3787 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3788 Privates.emplace_back(
3789 Args: C.getDeclAlign(D: VD),
3790 Args: PrivateHelpersTy(E, VD, cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl()),
3791 /*PrivateElemInit=*/nullptr));
3792 ++I;
3793 }
3794 I = Data.FirstprivateCopies.begin();
3795 const auto *IElemInitRef = Data.FirstprivateInits.begin();
3796 for (const Expr *E : Data.FirstprivateVars) {
3797 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3798 Privates.emplace_back(
3799 Args: C.getDeclAlign(D: VD),
3800 Args: PrivateHelpersTy(
3801 E, VD, cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl()),
3802 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IElemInitRef)->getDecl())));
3803 ++I;
3804 ++IElemInitRef;
3805 }
3806 I = Data.LastprivateCopies.begin();
3807 for (const Expr *E : Data.LastprivateVars) {
3808 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl());
3809 Privates.emplace_back(
3810 Args: C.getDeclAlign(D: VD),
3811 Args: PrivateHelpersTy(E, VD, cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *I)->getDecl()),
3812 /*PrivateElemInit=*/nullptr));
3813 ++I;
3814 }
3815 for (const VarDecl *VD : Data.PrivateLocals) {
3816 if (isAllocatableDecl(VD))
3817 Privates.emplace_back(Args: CGM.getPointerAlign(), Args: PrivateHelpersTy(VD));
3818 else
3819 Privates.emplace_back(Args: C.getDeclAlign(D: VD), Args: PrivateHelpersTy(VD));
3820 }
3821 llvm::stable_sort(Range&: Privates,
3822 C: [](const PrivateDataTy &L, const PrivateDataTy &R) {
3823 return L.first > R.first;
3824 });
3825 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
3826 // Build type kmp_routine_entry_t (if not built yet).
3827 emitKmpRoutineEntryT(KmpInt32Ty);
3828 // Build type kmp_task_t (if not built yet).
3829 if (isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind())) {
3830 if (SavedKmpTaskloopTQTy.isNull()) {
3831 SavedKmpTaskloopTQTy = C.getCanonicalTagType(TD: createKmpTaskTRecordDecl(
3832 CGM, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPointerQTy: KmpRoutineEntryPtrQTy));
3833 }
3834 KmpTaskTQTy = SavedKmpTaskloopTQTy;
3835 } else {
3836 assert((D.getDirectiveKind() == OMPD_task ||
3837 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) ||
3838 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
3839 "Expected taskloop, task or target directive");
3840 if (SavedKmpTaskTQTy.isNull()) {
3841 SavedKmpTaskTQTy = C.getCanonicalTagType(TD: createKmpTaskTRecordDecl(
3842 CGM, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPointerQTy: KmpRoutineEntryPtrQTy));
3843 }
3844 KmpTaskTQTy = SavedKmpTaskTQTy;
3845 }
3846 const auto *KmpTaskTQTyRD = KmpTaskTQTy->castAsRecordDecl();
3847 // Build particular struct kmp_task_t for the given task.
3848 const RecordDecl *KmpTaskTWithPrivatesQTyRD =
3849 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
3850 CanQualType KmpTaskTWithPrivatesQTy =
3851 C.getCanonicalTagType(TD: KmpTaskTWithPrivatesQTyRD);
3852 QualType KmpTaskTWithPrivatesPtrQTy =
3853 C.getPointerType(T: KmpTaskTWithPrivatesQTy);
3854 llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.Builder.getPtrTy(AddrSpace: 0);
3855 llvm::Value *KmpTaskTWithPrivatesTySize =
3856 CGF.getTypeSize(Ty: KmpTaskTWithPrivatesQTy);
3857 QualType SharedsPtrTy = C.getPointerType(T: SharedsTy);
3858
3859 // Emit initial values for private copies (if any).
3860 llvm::Value *TaskPrivatesMap = nullptr;
3861 llvm::Type *TaskPrivatesMapTy =
3862 std::next(x: TaskFunction->arg_begin(), n: 3)->getType();
3863 if (!Privates.empty()) {
3864 auto FI = std::next(x: KmpTaskTWithPrivatesQTyRD->field_begin());
3865 TaskPrivatesMap =
3866 emitTaskPrivateMappingFunction(CGM, Loc, Data, PrivatesQTy: FI->getType(), Privates);
3867 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3868 V: TaskPrivatesMap, DestTy: TaskPrivatesMapTy);
3869 } else {
3870 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3871 T: cast<llvm::PointerType>(Val: TaskPrivatesMapTy));
3872 }
3873 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
3874 // kmp_task_t *tt);
3875 llvm::Function *TaskEntry = emitProxyTaskFunction(
3876 CGM, Loc, Kind: D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3877 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3878 TaskPrivatesMap);
3879
3880 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
3881 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
3882 // kmp_routine_entry_t *task_entry);
3883 // Task flags. Format is taken from
3884 // https://github.com/llvm/llvm-project/blob/main/openmp/runtime/src/kmp.h,
3885 // description of kmp_tasking_flags struct.
3886 enum {
3887 TiedFlag = 0x1,
3888 FinalFlag = 0x2,
3889 DestructorsFlag = 0x8,
3890 PriorityFlag = 0x20,
3891 DetachableFlag = 0x40,
3892 FreeAgentFlag = 0x80,
3893 TransparentFlag = 0x100,
3894 };
3895 unsigned Flags = Data.Tied ? TiedFlag : 0;
3896 bool NeedsCleanup = false;
3897 if (!Privates.empty()) {
3898 NeedsCleanup =
3899 checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);
3900 if (NeedsCleanup)
3901 Flags = Flags | DestructorsFlag;
3902 }
3903 if (const auto *Clause = D.getSingleClause<OMPThreadsetClause>()) {
3904 OpenMPThreadsetKind Kind = Clause->getThreadsetKind();
3905 if (Kind == OMPC_THREADSET_omp_pool)
3906 Flags = Flags | FreeAgentFlag;
3907 }
3908 if (D.getSingleClause<OMPTransparentClause>())
3909 Flags |= TransparentFlag;
3910
3911 if (Data.Priority.getInt())
3912 Flags = Flags | PriorityFlag;
3913 if (D.hasClausesOfKind<OMPDetachClause>())
3914 Flags = Flags | DetachableFlag;
3915 llvm::Value *TaskFlags =
3916 Data.Final.getPointer()
3917 ? CGF.Builder.CreateSelect(C: Data.Final.getPointer(),
3918 True: CGF.Builder.getInt32(C: FinalFlag),
3919 False: CGF.Builder.getInt32(/*C=*/0))
3920 : CGF.Builder.getInt32(C: Data.Final.getInt() ? FinalFlag : 0);
3921 TaskFlags = CGF.Builder.CreateOr(LHS: TaskFlags, RHS: CGF.Builder.getInt32(C: Flags));
3922 llvm::Value *SharedsSize = CGM.getSize(numChars: C.getTypeSizeInChars(T: SharedsTy));
3923 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc),
3924 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize,
3925 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
3926 V: TaskEntry, DestTy: KmpRoutineEntryPtrTy)};
3927 llvm::Value *NewTask;
3928 if (D.hasClausesOfKind<OMPNowaitClause>()) {
3929 // Check if we have any device clause associated with the directive.
3930 const Expr *Device = nullptr;
3931 if (auto *C = D.getSingleClause<OMPDeviceClause>())
3932 Device = C->getDevice();
3933 // Emit device ID if any otherwise use default value.
3934 llvm::Value *DeviceID;
3935 if (Device)
3936 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
3937 DestTy: CGF.Int64Ty, /*isSigned=*/true);
3938 else
3939 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
3940 AllocArgs.push_back(Elt: DeviceID);
3941 NewTask = CGF.EmitRuntimeCall(
3942 callee: OMPBuilder.getOrCreateRuntimeFunction(
3943 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_target_task_alloc),
3944 args: AllocArgs);
3945 } else {
3946 NewTask =
3947 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
3948 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task_alloc),
3949 args: AllocArgs);
3950 }
3951 // Emit detach clause initialization.
3952 // evt = (typeof(evt))__kmpc_task_allow_completion_event(loc, tid,
3953 // task_descriptor);
3954 if (const auto *DC = D.getSingleClause<OMPDetachClause>()) {
3955 const Expr *Evt = DC->getEventHandler()->IgnoreParenImpCasts();
3956 LValue EvtLVal = CGF.EmitLValue(E: Evt);
3957
3958 // Build kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
3959 // int gtid, kmp_task_t *task);
3960 llvm::Value *Loc = emitUpdateLocation(CGF, Loc: DC->getBeginLoc());
3961 llvm::Value *Tid = getThreadID(CGF, Loc: DC->getBeginLoc());
3962 Tid = CGF.Builder.CreateIntCast(V: Tid, DestTy: CGF.IntTy, /*isSigned=*/false);
3963 llvm::Value *EvtVal = CGF.EmitRuntimeCall(
3964 callee: OMPBuilder.getOrCreateRuntimeFunction(
3965 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_allow_completion_event),
3966 args: {Loc, Tid, NewTask});
3967 EvtVal = CGF.EmitScalarConversion(Src: EvtVal, SrcTy: C.VoidPtrTy, DstTy: Evt->getType(),
3968 Loc: Evt->getExprLoc());
3969 CGF.EmitStoreOfScalar(value: EvtVal, lvalue: EvtLVal);
3970 }
3971 // Process affinity clauses.
3972 if (D.hasClausesOfKind<OMPAffinityClause>()) {
3973 // Process list of affinity data.
3974 ASTContext &C = CGM.getContext();
3975 Address AffinitiesArray = Address::invalid();
3976 // Calculate number of elements to form the array of affinity data.
3977 llvm::Value *NumOfElements = nullptr;
3978 unsigned NumAffinities = 0;
3979 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
3980 if (const Expr *Modifier = C->getModifier()) {
3981 const auto *IE = cast<OMPIteratorExpr>(Val: Modifier->IgnoreParenImpCasts());
3982 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
3983 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
3984 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.SizeTy, /*isSigned=*/false);
3985 NumOfElements =
3986 NumOfElements ? CGF.Builder.CreateNUWMul(LHS: NumOfElements, RHS: Sz) : Sz;
3987 }
3988 } else {
3989 NumAffinities += C->varlist_size();
3990 }
3991 }
3992 getKmpAffinityType(C&: CGM.getContext(), KmpTaskAffinityInfoTy);
3993 // Fields ids in kmp_task_affinity_info record.
3994 enum RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };
3995
3996 QualType KmpTaskAffinityInfoArrayTy;
3997 if (NumOfElements) {
3998 NumOfElements = CGF.Builder.CreateNUWAdd(
3999 LHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumAffinities), RHS: NumOfElements);
4000 auto *OVE = new (C) OpaqueValueExpr(
4001 Loc,
4002 C.getIntTypeForBitwidth(DestWidth: C.getTypeSize(T: C.getSizeType()), /*Signed=*/0),
4003 VK_PRValue);
4004 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4005 RValue::get(V: NumOfElements));
4006 KmpTaskAffinityInfoArrayTy = C.getVariableArrayType(
4007 EltTy: KmpTaskAffinityInfoTy, NumElts: OVE, ASM: ArraySizeModifier::Normal,
4008 /*IndexTypeQuals=*/0);
4009 // Properly emit variable-sized array.
4010 auto *PD = ImplicitParamDecl::Create(C, T: KmpTaskAffinityInfoArrayTy,
4011 ParamKind: ImplicitParamKind::Other);
4012 CGF.EmitVarDecl(D: *PD);
4013 AffinitiesArray = CGF.GetAddrOfLocalVar(VD: PD);
4014 NumOfElements = CGF.Builder.CreateIntCast(V: NumOfElements, DestTy: CGF.Int32Ty,
4015 /*isSigned=*/false);
4016 } else {
4017 KmpTaskAffinityInfoArrayTy = C.getConstantArrayType(
4018 EltTy: KmpTaskAffinityInfoTy,
4019 ArySize: llvm::APInt(C.getTypeSize(T: C.getSizeType()), NumAffinities), SizeExpr: nullptr,
4020 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4021 AffinitiesArray = CGF.CreateMemTempWithoutCast(T: KmpTaskAffinityInfoArrayTy,
4022 Name: ".affs.arr.addr");
4023 AffinitiesArray = CGF.Builder.CreateConstArrayGEP(Addr: AffinitiesArray, Index: 0);
4024 NumOfElements = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: NumAffinities,
4025 /*isSigned=*/IsSigned: false);
4026 }
4027
4028 const auto *KmpAffinityInfoRD = KmpTaskAffinityInfoTy->getAsRecordDecl();
4029 // Fill array by elements without iterators.
4030 unsigned Pos = 0;
4031 bool HasIterator = false;
4032 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4033 if (C->getModifier()) {
4034 HasIterator = true;
4035 continue;
4036 }
4037 for (const Expr *E : C->varlist()) {
4038 llvm::Value *Addr;
4039 llvm::Value *Size;
4040 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4041 LValue Base =
4042 CGF.MakeAddrLValue(Addr: CGF.Builder.CreateConstGEP(Addr: AffinitiesArray, Index: Pos),
4043 T: KmpTaskAffinityInfoTy);
4044 // affs[i].base_addr = &<Affinities[i].second>;
4045 LValue BaseAddrLVal = CGF.EmitLValueForField(
4046 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: BaseAddr));
4047 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy),
4048 lvalue: BaseAddrLVal);
4049 // affs[i].len = sizeof(<Affinities[i].second>);
4050 LValue LenLVal = CGF.EmitLValueForField(
4051 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: Len));
4052 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4053 ++Pos;
4054 }
4055 }
4056 LValue PosLVal;
4057 if (HasIterator) {
4058 PosLVal = CGF.MakeAddrLValue(
4059 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "affs.counter.addr"),
4060 T: C.getSizeType());
4061 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Pos), lvalue: PosLVal);
4062 }
4063 // Process elements with iterators.
4064 for (const auto *C : D.getClausesOfKind<OMPAffinityClause>()) {
4065 const Expr *Modifier = C->getModifier();
4066 if (!Modifier)
4067 continue;
4068 OMPIteratorGeneratorScope IteratorScope(
4069 CGF, cast_or_null<OMPIteratorExpr>(Val: Modifier->IgnoreParenImpCasts()));
4070 for (const Expr *E : C->varlist()) {
4071 llvm::Value *Addr;
4072 llvm::Value *Size;
4073 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4074 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4075 LValue Base =
4076 CGF.MakeAddrLValue(Addr: CGF.Builder.CreateGEP(CGF, Addr: AffinitiesArray, Index: Idx),
4077 T: KmpTaskAffinityInfoTy);
4078 // affs[i].base_addr = &<Affinities[i].second>;
4079 LValue BaseAddrLVal = CGF.EmitLValueForField(
4080 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: BaseAddr));
4081 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy),
4082 lvalue: BaseAddrLVal);
4083 // affs[i].len = sizeof(<Affinities[i].second>);
4084 LValue LenLVal = CGF.EmitLValueForField(
4085 Base, Field: *std::next(x: KmpAffinityInfoRD->field_begin(), n: Len));
4086 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4087 Idx = CGF.Builder.CreateNUWAdd(
4088 LHS: Idx, RHS: llvm::ConstantInt::get(Ty: Idx->getType(), V: 1));
4089 CGF.EmitStoreOfScalar(value: Idx, lvalue: PosLVal);
4090 }
4091 }
4092 // Call to kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref,
4093 // kmp_int32 gtid, kmp_task_t *new_task, kmp_int32
4094 // naffins, kmp_task_affinity_info_t *affin_list);
4095 llvm::Value *LocRef = emitUpdateLocation(CGF, Loc);
4096 llvm::Value *GTid = getThreadID(CGF, Loc);
4097 llvm::Value *AffinListPtr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4098 V: AffinitiesArray.emitRawPointer(CGF), DestTy: CGM.VoidPtrTy);
4099 // FIXME: Emit the function and ignore its result for now unless the
4100 // runtime function is properly implemented.
4101 (void)CGF.EmitRuntimeCall(
4102 callee: OMPBuilder.getOrCreateRuntimeFunction(
4103 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_reg_task_with_affinity),
4104 args: {LocRef, GTid, NewTask, NumOfElements, AffinListPtr});
4105 }
4106 llvm::Value *NewTaskNewTaskTTy =
4107 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4108 V: NewTask, DestTy: KmpTaskTWithPrivatesPtrTy);
4109 LValue Base = CGF.MakeNaturalAlignRawAddrLValue(V: NewTaskNewTaskTTy,
4110 T: KmpTaskTWithPrivatesQTy);
4111 LValue TDBase =
4112 CGF.EmitLValueForField(Base, Field: *KmpTaskTWithPrivatesQTyRD->field_begin());
4113 // Fill the data in the resulting kmp_task_t record.
4114 // Copy shareds if there are any.
4115 Address KmpTaskSharedsPtr = Address::invalid();
4116 if (!SharedsTy->castAsRecordDecl()->field_empty()) {
4117 KmpTaskSharedsPtr = Address(
4118 CGF.EmitLoadOfScalar(
4119 lvalue: CGF.EmitLValueForField(
4120 Base: TDBase,
4121 Field: *std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTShareds)),
4122 Loc),
4123 CGF.Int8Ty, CGM.getNaturalTypeAlignment(T: SharedsTy));
4124 LValue Dest = CGF.MakeAddrLValue(Addr: KmpTaskSharedsPtr, T: SharedsTy);
4125 LValue Src = CGF.MakeAddrLValue(Addr: Shareds, T: SharedsTy);
4126 CGF.EmitAggregateCopy(Dest, Src, EltTy: SharedsTy, MayOverlap: AggValueSlot::DoesNotOverlap);
4127 }
4128 // Emit initial values for private copies (if any).
4129 TaskResultTy Result;
4130 if (!Privates.empty()) {
4131 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase: Base, KmpTaskTWithPrivatesQTyRD,
4132 SharedsTy, SharedsPtrTy, Data, Privates,
4133 /*ForDup=*/false);
4134 if (isOpenMPTaskLoopDirective(DKind: D.getDirectiveKind()) &&
4135 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
4136 Result.TaskDupFn = emitTaskDupFunction(
4137 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
4138 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
4139 /*WithLastIter=*/!Data.LastprivateVars.empty());
4140 }
4141 }
4142 // Fields of union "kmp_cmplrdata_t" for destructors and priority.
4143 enum { Priority = 0, Destructors = 1 };
4144 // Provide pointer to function with destructors for privates.
4145 auto FI = std::next(x: KmpTaskTQTyRD->field_begin(), n: Data1);
4146 const auto *KmpCmplrdataUD = (*FI)->getType()->castAsRecordDecl();
4147 assert(KmpCmplrdataUD->isUnion());
4148 if (NeedsCleanup) {
4149 llvm::Value *DestructorFn = emitDestructorsFunction(
4150 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
4151 KmpTaskTWithPrivatesQTy);
4152 LValue Data1LV = CGF.EmitLValueForField(Base: TDBase, Field: *FI);
4153 LValue DestructorsLV = CGF.EmitLValueForField(
4154 Base: Data1LV, Field: *std::next(x: KmpCmplrdataUD->field_begin(), n: Destructors));
4155 CGF.EmitStoreOfScalar(value: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4156 V: DestructorFn, DestTy: KmpRoutineEntryPtrTy),
4157 lvalue: DestructorsLV);
4158 }
4159 // Set priority.
4160 if (Data.Priority.getInt()) {
4161 LValue Data2LV = CGF.EmitLValueForField(
4162 Base: TDBase, Field: *std::next(x: KmpTaskTQTyRD->field_begin(), n: Data2));
4163 LValue PriorityLV = CGF.EmitLValueForField(
4164 Base: Data2LV, Field: *std::next(x: KmpCmplrdataUD->field_begin(), n: Priority));
4165 CGF.EmitStoreOfScalar(value: Data.Priority.getPointer(), lvalue: PriorityLV);
4166 }
4167 Result.NewTask = NewTask;
4168 Result.TaskEntry = TaskEntry;
4169 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy;
4170 Result.TDBase = TDBase;
4171 Result.KmpTaskTQTyRD = KmpTaskTQTyRD;
4172 return Result;
4173}
4174
4175/// Translates internal dependency kind into the runtime kind.
4176static RTLDependenceKindTy translateDependencyKind(OpenMPDependClauseKind K) {
4177 RTLDependenceKindTy DepKind;
4178 switch (K) {
4179 case OMPC_DEPEND_in:
4180 DepKind = RTLDependenceKindTy::DepIn;
4181 break;
4182 // Out and InOut dependencies must use the same code.
4183 case OMPC_DEPEND_out:
4184 case OMPC_DEPEND_inout:
4185 DepKind = RTLDependenceKindTy::DepInOut;
4186 break;
4187 case OMPC_DEPEND_mutexinoutset:
4188 DepKind = RTLDependenceKindTy::DepMutexInOutSet;
4189 break;
4190 case OMPC_DEPEND_inoutset:
4191 DepKind = RTLDependenceKindTy::DepInOutSet;
4192 break;
4193 case OMPC_DEPEND_outallmemory:
4194 DepKind = RTLDependenceKindTy::DepOmpAllMem;
4195 break;
4196 case OMPC_DEPEND_source:
4197 case OMPC_DEPEND_sink:
4198 case OMPC_DEPEND_depobj:
4199 case OMPC_DEPEND_inoutallmemory:
4200 case OMPC_DEPEND_unknown:
4201 llvm_unreachable("Unknown task dependence type");
4202 }
4203 return DepKind;
4204}
4205
4206/// Builds kmp_depend_info, if it is not built yet, and builds flags type.
4207static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
4208 QualType &FlagsTy) {
4209 FlagsTy = C.getIntTypeForBitwidth(DestWidth: C.getTypeSize(T: C.BoolTy), /*Signed=*/false);
4210 if (KmpDependInfoTy.isNull()) {
4211 RecordDecl *KmpDependInfoRD = C.buildImplicitRecord(Name: "kmp_depend_info");
4212 KmpDependInfoRD->startDefinition();
4213 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: C.getIntPtrType());
4214 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: C.getSizeType());
4215 addFieldToRecordDecl(C, DC: KmpDependInfoRD, FieldTy: FlagsTy);
4216 KmpDependInfoRD->completeDefinition();
4217 KmpDependInfoTy = C.getCanonicalTagType(TD: KmpDependInfoRD);
4218 }
4219}
4220
4221std::pair<llvm::Value *, LValue>
4222CGOpenMPRuntime::getDepobjElements(CodeGenFunction &CGF, LValue DepobjLVal,
4223 SourceLocation Loc) {
4224 ASTContext &C = CGM.getContext();
4225 QualType FlagsTy;
4226 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4227 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4228 QualType KmpDependInfoPtrTy = C.getPointerType(T: KmpDependInfoTy);
4229 LValue Base = CGF.EmitLoadOfPointerLValue(
4230 Ptr: DepobjLVal.getAddress().withElementType(
4231 ElemTy: CGF.ConvertTypeForMem(T: KmpDependInfoPtrTy)),
4232 PtrTy: KmpDependInfoPtrTy->castAs<PointerType>());
4233 Address DepObjAddr = CGF.Builder.CreateGEP(
4234 CGF, Addr: Base.getAddress(),
4235 Index: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: -1, /*isSigned=*/IsSigned: true));
4236 LValue NumDepsBase = CGF.MakeAddrLValue(
4237 Addr: DepObjAddr, T: KmpDependInfoTy, BaseInfo: Base.getBaseInfo(), TBAAInfo: Base.getTBAAInfo());
4238 // NumDeps = deps[i].base_addr;
4239 LValue BaseAddrLVal = CGF.EmitLValueForField(
4240 Base: NumDepsBase,
4241 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4242 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4243 llvm::Value *NumDeps = CGF.EmitLoadOfScalar(lvalue: BaseAddrLVal, Loc);
4244 return std::make_pair(x&: NumDeps, y&: Base);
4245}
4246
4247static void emitDependData(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4248 llvm::PointerUnion<unsigned *, LValue *> Pos,
4249 const OMPTaskDataTy::DependData &Data,
4250 Address DependenciesArray) {
4251 CodeGenModule &CGM = CGF.CGM;
4252 ASTContext &C = CGM.getContext();
4253 QualType FlagsTy;
4254 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4255 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4256 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(T: FlagsTy);
4257
4258 OMPIteratorGeneratorScope IteratorScope(
4259 CGF, cast_or_null<OMPIteratorExpr>(
4260 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4261 : nullptr));
4262 for (const Expr *E : Data.DepExprs) {
4263 llvm::Value *Addr;
4264 llvm::Value *Size;
4265
4266 // The expression will be a nullptr in the 'omp_all_memory' case.
4267 if (E) {
4268 std::tie(args&: Addr, args&: Size) = getPointerAndSize(CGF, E);
4269 Addr = CGF.Builder.CreatePtrToInt(V: Addr, DestTy: CGF.IntPtrTy);
4270 } else {
4271 Addr = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4272 Size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
4273 }
4274 LValue Base;
4275 if (unsigned *P = dyn_cast<unsigned *>(Val&: Pos)) {
4276 Base = CGF.MakeAddrLValue(
4277 Addr: CGF.Builder.CreateConstGEP(Addr: DependenciesArray, Index: *P), T: KmpDependInfoTy);
4278 } else {
4279 assert(E && "Expected a non-null expression");
4280 LValue &PosLVal = *cast<LValue *>(Val&: Pos);
4281 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4282 Base = CGF.MakeAddrLValue(
4283 Addr: CGF.Builder.CreateGEP(CGF, Addr: DependenciesArray, Index: Idx), T: KmpDependInfoTy);
4284 }
4285 // deps[i].base_addr = &<Dependencies[i].second>;
4286 LValue BaseAddrLVal = CGF.EmitLValueForField(
4287 Base,
4288 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4289 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4290 CGF.EmitStoreOfScalar(value: Addr, lvalue: BaseAddrLVal);
4291 // deps[i].len = sizeof(<Dependencies[i].second>);
4292 LValue LenLVal = CGF.EmitLValueForField(
4293 Base, Field: *std::next(x: KmpDependInfoRD->field_begin(),
4294 n: static_cast<unsigned int>(RTLDependInfoFields::Len)));
4295 CGF.EmitStoreOfScalar(value: Size, lvalue: LenLVal);
4296 // deps[i].flags = <Dependencies[i].first>;
4297 RTLDependenceKindTy DepKind = translateDependencyKind(K: Data.DepKind);
4298 LValue FlagsLVal = CGF.EmitLValueForField(
4299 Base,
4300 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4301 n: static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4302 CGF.EmitStoreOfScalar(
4303 value: llvm::ConstantInt::get(Ty: LLVMFlagsTy, V: static_cast<unsigned int>(DepKind)),
4304 lvalue: FlagsLVal);
4305 if (unsigned *P = dyn_cast<unsigned *>(Val&: Pos)) {
4306 ++(*P);
4307 } else {
4308 LValue &PosLVal = *cast<LValue *>(Val&: Pos);
4309 llvm::Value *Idx = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4310 Idx = CGF.Builder.CreateNUWAdd(LHS: Idx,
4311 RHS: llvm::ConstantInt::get(Ty: Idx->getType(), V: 1));
4312 CGF.EmitStoreOfScalar(value: Idx, lvalue: PosLVal);
4313 }
4314 }
4315}
4316
4317SmallVector<llvm::Value *, 4> CGOpenMPRuntime::emitDepobjElementsSizes(
4318 CodeGenFunction &CGF, QualType &KmpDependInfoTy,
4319 const OMPTaskDataTy::DependData &Data) {
4320 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4321 "Expected depobj dependency kind.");
4322 SmallVector<llvm::Value *, 4> Sizes;
4323 SmallVector<LValue, 4> SizeLVals;
4324 ASTContext &C = CGF.getContext();
4325 {
4326 OMPIteratorGeneratorScope IteratorScope(
4327 CGF, cast_or_null<OMPIteratorExpr>(
4328 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4329 : nullptr));
4330 for (const Expr *E : Data.DepExprs) {
4331 llvm::Value *NumDeps;
4332 LValue Base;
4333 LValue DepobjLVal = CGF.EmitLValue(E: E->IgnoreParenImpCasts());
4334 std::tie(args&: NumDeps, args&: Base) =
4335 getDepobjElements(CGF, DepobjLVal, Loc: E->getExprLoc());
4336 LValue NumLVal = CGF.MakeAddrLValue(
4337 Addr: CGF.CreateMemTempWithoutCast(T: C.getUIntPtrType(), Name: "depobj.size.addr"),
4338 T: C.getUIntPtrType());
4339 CGF.Builder.CreateStore(Val: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0),
4340 Addr: NumLVal.getAddress());
4341 llvm::Value *PrevVal = CGF.EmitLoadOfScalar(lvalue: NumLVal, Loc: E->getExprLoc());
4342 llvm::Value *Add = CGF.Builder.CreateNUWAdd(LHS: PrevVal, RHS: NumDeps);
4343 CGF.EmitStoreOfScalar(value: Add, lvalue: NumLVal);
4344 SizeLVals.push_back(Elt: NumLVal);
4345 }
4346 }
4347 for (unsigned I = 0, E = SizeLVals.size(); I < E; ++I) {
4348 llvm::Value *Size =
4349 CGF.EmitLoadOfScalar(lvalue: SizeLVals[I], Loc: Data.DepExprs[I]->getExprLoc());
4350 Sizes.push_back(Elt: Size);
4351 }
4352 return Sizes;
4353}
4354
4355void CGOpenMPRuntime::emitDepobjElements(CodeGenFunction &CGF,
4356 QualType &KmpDependInfoTy,
4357 LValue PosLVal,
4358 const OMPTaskDataTy::DependData &Data,
4359 Address DependenciesArray) {
4360 assert(Data.DepKind == OMPC_DEPEND_depobj &&
4361 "Expected depobj dependency kind.");
4362 llvm::Value *ElSize = CGF.getTypeSize(Ty: KmpDependInfoTy);
4363 {
4364 OMPIteratorGeneratorScope IteratorScope(
4365 CGF, cast_or_null<OMPIteratorExpr>(
4366 Val: Data.IteratorExpr ? Data.IteratorExpr->IgnoreParenImpCasts()
4367 : nullptr));
4368 for (const Expr *E : Data.DepExprs) {
4369 llvm::Value *NumDeps;
4370 LValue Base;
4371 LValue DepobjLVal = CGF.EmitLValue(E: E->IgnoreParenImpCasts());
4372 std::tie(args&: NumDeps, args&: Base) =
4373 getDepobjElements(CGF, DepobjLVal, Loc: E->getExprLoc());
4374
4375 // memcopy dependency data.
4376 llvm::Value *Size = CGF.Builder.CreateNUWMul(
4377 LHS: ElSize,
4378 RHS: CGF.Builder.CreateIntCast(V: NumDeps, DestTy: CGF.SizeTy, /*isSigned=*/false));
4379 llvm::Value *Pos = CGF.EmitLoadOfScalar(lvalue: PosLVal, Loc: E->getExprLoc());
4380 Address DepAddr = CGF.Builder.CreateGEP(CGF, Addr: DependenciesArray, Index: Pos);
4381 CGF.Builder.CreateMemCpy(Dest: DepAddr, Src: Base.getAddress(), Size);
4382
4383 // Increase pos.
4384 // pos += size;
4385 llvm::Value *Add = CGF.Builder.CreateNUWAdd(LHS: Pos, RHS: NumDeps);
4386 CGF.EmitStoreOfScalar(value: Add, lvalue: PosLVal);
4387 }
4388 }
4389}
4390
4391std::pair<llvm::Value *, Address> CGOpenMPRuntime::emitDependClause(
4392 CodeGenFunction &CGF, ArrayRef<OMPTaskDataTy::DependData> Dependencies,
4393 SourceLocation Loc) {
4394 if (llvm::all_of(Range&: Dependencies, P: [](const OMPTaskDataTy::DependData &D) {
4395 return D.DepExprs.empty();
4396 }))
4397 return std::make_pair(x: nullptr, y: Address::invalid());
4398 // Process list of dependencies.
4399 ASTContext &C = CGM.getContext();
4400 Address DependenciesArray = Address::invalid();
4401 llvm::Value *NumOfElements = nullptr;
4402 unsigned NumDependencies = std::accumulate(
4403 first: Dependencies.begin(), last: Dependencies.end(), init: 0,
4404 binary_op: [](unsigned V, const OMPTaskDataTy::DependData &D) {
4405 return D.DepKind == OMPC_DEPEND_depobj
4406 ? V
4407 : (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));
4408 });
4409 QualType FlagsTy;
4410 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4411 bool HasDepobjDeps = false;
4412 bool HasRegularWithIterators = false;
4413 llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4414 llvm::Value *NumOfRegularWithIterators =
4415 llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 0);
4416 // Calculate number of depobj dependencies and regular deps with the
4417 // iterators.
4418 for (const OMPTaskDataTy::DependData &D : Dependencies) {
4419 if (D.DepKind == OMPC_DEPEND_depobj) {
4420 SmallVector<llvm::Value *, 4> Sizes =
4421 emitDepobjElementsSizes(CGF, KmpDependInfoTy, Data: D);
4422 for (llvm::Value *Size : Sizes) {
4423 NumOfDepobjElements =
4424 CGF.Builder.CreateNUWAdd(LHS: NumOfDepobjElements, RHS: Size);
4425 }
4426 HasDepobjDeps = true;
4427 continue;
4428 }
4429 // Include number of iterations, if any.
4430
4431 if (const auto *IE = cast_or_null<OMPIteratorExpr>(Val: D.IteratorExpr)) {
4432 llvm::Value *ClauseIteratorSpace =
4433 llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: 1);
4434 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4435 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4436 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.IntPtrTy, /*isSigned=*/false);
4437 ClauseIteratorSpace = CGF.Builder.CreateNUWMul(LHS: Sz, RHS: ClauseIteratorSpace);
4438 }
4439 llvm::Value *NumClauseDeps = CGF.Builder.CreateNUWMul(
4440 LHS: ClauseIteratorSpace,
4441 RHS: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: D.DepExprs.size()));
4442 NumOfRegularWithIterators =
4443 CGF.Builder.CreateNUWAdd(LHS: NumOfRegularWithIterators, RHS: NumClauseDeps);
4444 HasRegularWithIterators = true;
4445 continue;
4446 }
4447 }
4448
4449 QualType KmpDependInfoArrayTy;
4450 if (HasDepobjDeps || HasRegularWithIterators) {
4451 NumOfElements = llvm::ConstantInt::get(Ty: CGM.IntPtrTy, V: NumDependencies,
4452 /*isSigned=*/IsSigned: false);
4453 if (HasDepobjDeps) {
4454 NumOfElements =
4455 CGF.Builder.CreateNUWAdd(LHS: NumOfDepobjElements, RHS: NumOfElements);
4456 }
4457 if (HasRegularWithIterators) {
4458 NumOfElements =
4459 CGF.Builder.CreateNUWAdd(LHS: NumOfRegularWithIterators, RHS: NumOfElements);
4460 }
4461 auto *OVE = new (C) OpaqueValueExpr(
4462 Loc, C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0),
4463 VK_PRValue);
4464 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, OVE,
4465 RValue::get(V: NumOfElements));
4466 KmpDependInfoArrayTy =
4467 C.getVariableArrayType(EltTy: KmpDependInfoTy, NumElts: OVE, ASM: ArraySizeModifier::Normal,
4468 /*IndexTypeQuals=*/0);
4469 // CGF.EmitVariablyModifiedType(KmpDependInfoArrayTy);
4470 // Properly emit variable-sized array.
4471 auto *PD = ImplicitParamDecl::Create(C, T: KmpDependInfoArrayTy,
4472 ParamKind: ImplicitParamKind::Other);
4473 CGF.EmitVarDecl(D: *PD);
4474 DependenciesArray = CGF.GetAddrOfLocalVar(VD: PD);
4475 NumOfElements = CGF.Builder.CreateIntCast(V: NumOfElements, DestTy: CGF.Int32Ty,
4476 /*isSigned=*/false);
4477 } else {
4478 KmpDependInfoArrayTy = C.getConstantArrayType(
4479 EltTy: KmpDependInfoTy, ArySize: llvm::APInt(/*numBits=*/64, NumDependencies), SizeExpr: nullptr,
4480 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4481 DependenciesArray =
4482 CGF.CreateMemTempWithoutCast(T: KmpDependInfoArrayTy, Name: ".dep.arr.addr");
4483 DependenciesArray = CGF.Builder.CreateConstArrayGEP(Addr: DependenciesArray, Index: 0);
4484 NumOfElements = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: NumDependencies,
4485 /*isSigned=*/IsSigned: false);
4486 }
4487 unsigned Pos = 0;
4488 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4489 if (Dep.DepKind == OMPC_DEPEND_depobj || Dep.IteratorExpr)
4490 continue;
4491 emitDependData(CGF, KmpDependInfoTy, Pos: &Pos, Data: Dep, DependenciesArray);
4492 }
4493 // Copy regular dependencies with iterators.
4494 LValue PosLVal = CGF.MakeAddrLValue(
4495 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "dep.counter.addr"),
4496 T: C.getSizeType());
4497 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Pos), lvalue: PosLVal);
4498 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4499 if (Dep.DepKind == OMPC_DEPEND_depobj || !Dep.IteratorExpr)
4500 continue;
4501 emitDependData(CGF, KmpDependInfoTy, Pos: &PosLVal, Data: Dep, DependenciesArray);
4502 }
4503 // Copy final depobj arrays without iterators.
4504 if (HasDepobjDeps) {
4505 for (const OMPTaskDataTy::DependData &Dep : Dependencies) {
4506 if (Dep.DepKind != OMPC_DEPEND_depobj)
4507 continue;
4508 emitDepobjElements(CGF, KmpDependInfoTy, PosLVal, Data: Dep, DependenciesArray);
4509 }
4510 }
4511 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4512 Addr: DependenciesArray, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty);
4513 return std::make_pair(x&: NumOfElements, y&: DependenciesArray);
4514}
4515
4516Address CGOpenMPRuntime::emitDepobjDependClause(
4517 CodeGenFunction &CGF, const OMPTaskDataTy::DependData &Dependencies,
4518 SourceLocation Loc) {
4519 if (Dependencies.DepExprs.empty())
4520 return Address::invalid();
4521 // Process list of dependencies.
4522 ASTContext &C = CGM.getContext();
4523 Address DependenciesArray = Address::invalid();
4524 unsigned NumDependencies = Dependencies.DepExprs.size();
4525 QualType FlagsTy;
4526 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4527 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4528
4529 llvm::Value *Size;
4530 // Define type kmp_depend_info[<Dependencies.size()>];
4531 // For depobj reserve one extra element to store the number of elements.
4532 // It is required to handle depobj(x) update(in) construct.
4533 // kmp_depend_info[<Dependencies.size()>] deps;
4534 llvm::Value *NumDepsVal;
4535 CharUnits Align = C.getTypeAlignInChars(T: KmpDependInfoTy);
4536 if (const auto *IE =
4537 cast_or_null<OMPIteratorExpr>(Val: Dependencies.IteratorExpr)) {
4538 NumDepsVal = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1);
4539 for (unsigned I = 0, E = IE->numOfIterators(); I < E; ++I) {
4540 llvm::Value *Sz = CGF.EmitScalarExpr(E: IE->getHelper(I).Upper);
4541 Sz = CGF.Builder.CreateIntCast(V: Sz, DestTy: CGF.SizeTy, /*isSigned=*/false);
4542 NumDepsVal = CGF.Builder.CreateNUWMul(LHS: NumDepsVal, RHS: Sz);
4543 }
4544 Size = CGF.Builder.CreateNUWAdd(LHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1),
4545 RHS: NumDepsVal);
4546 CharUnits SizeInBytes =
4547 C.getTypeSizeInChars(T: KmpDependInfoTy).alignTo(Align);
4548 llvm::Value *RecSize = CGM.getSize(numChars: SizeInBytes);
4549 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: RecSize);
4550 NumDepsVal =
4551 CGF.Builder.CreateIntCast(V: NumDepsVal, DestTy: CGF.IntPtrTy, /*isSigned=*/false);
4552 } else {
4553 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
4554 EltTy: KmpDependInfoTy, ArySize: llvm::APInt(/*numBits=*/64, NumDependencies + 1),
4555 SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
4556 CharUnits Sz = C.getTypeSizeInChars(T: KmpDependInfoArrayTy);
4557 Size = CGM.getSize(numChars: Sz.alignTo(Align));
4558 NumDepsVal = llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: NumDependencies);
4559 }
4560 // Need to allocate on the dynamic memory.
4561 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4562 // Use default allocator.
4563 llvm::Value *Allocator = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4564 llvm::Value *Args[] = {ThreadID, Size, Allocator};
4565
4566 llvm::Value *Addr =
4567 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4568 M&: CGM.getModule(), FnID: OMPRTL___kmpc_alloc),
4569 args: Args, name: ".dep.arr.addr");
4570 llvm::Type *KmpDependInfoLlvmTy = CGF.ConvertTypeForMem(T: KmpDependInfoTy);
4571 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4572 V: Addr, DestTy: CGF.Builder.getPtrTy(AddrSpace: 0));
4573 DependenciesArray = Address(Addr, KmpDependInfoLlvmTy, Align);
4574 // Write number of elements in the first element of array for depobj.
4575 LValue Base = CGF.MakeAddrLValue(Addr: DependenciesArray, T: KmpDependInfoTy);
4576 // deps[i].base_addr = NumDependencies;
4577 LValue BaseAddrLVal = CGF.EmitLValueForField(
4578 Base,
4579 Field: *std::next(x: KmpDependInfoRD->field_begin(),
4580 n: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr)));
4581 CGF.EmitStoreOfScalar(value: NumDepsVal, lvalue: BaseAddrLVal);
4582 llvm::PointerUnion<unsigned *, LValue *> Pos;
4583 unsigned Idx = 1;
4584 LValue PosLVal;
4585 if (Dependencies.IteratorExpr) {
4586 PosLVal = CGF.MakeAddrLValue(
4587 Addr: CGF.CreateMemTempWithoutCast(T: C.getSizeType(), Name: "iterator.counter.addr"),
4588 T: C.getSizeType());
4589 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: Idx), lvalue: PosLVal,
4590 /*IsInit=*/isInit: true);
4591 Pos = &PosLVal;
4592 } else {
4593 Pos = &Idx;
4594 }
4595 emitDependData(CGF, KmpDependInfoTy, Pos, Data: Dependencies, DependenciesArray);
4596 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4597 Addr: CGF.Builder.CreateConstGEP(Addr: DependenciesArray, Index: 1), Ty: CGF.VoidPtrTy,
4598 ElementTy: CGF.Int8Ty);
4599 return DependenciesArray;
4600}
4601
4602void CGOpenMPRuntime::emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
4603 SourceLocation Loc) {
4604 ASTContext &C = CGM.getContext();
4605 QualType FlagsTy;
4606 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4607 LValue Base = CGF.EmitLoadOfPointerLValue(Ptr: DepobjLVal.getAddress(),
4608 PtrTy: C.VoidPtrTy.castAs<PointerType>());
4609 QualType KmpDependInfoPtrTy = C.getPointerType(T: KmpDependInfoTy);
4610 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4611 Addr: Base.getAddress(), Ty: CGF.ConvertTypeForMem(T: KmpDependInfoPtrTy),
4612 ElementTy: CGF.ConvertTypeForMem(T: KmpDependInfoTy));
4613 llvm::Value *DepObjAddr = CGF.Builder.CreateGEP(
4614 Ty: Addr.getElementType(), Ptr: Addr.emitRawPointer(CGF),
4615 IdxList: llvm::ConstantInt::get(Ty: CGF.IntPtrTy, V: -1, /*isSigned=*/IsSigned: true));
4616 DepObjAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: DepObjAddr,
4617 DestTy: CGF.VoidPtrTy);
4618 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4619 // Use default allocator.
4620 llvm::Value *Allocator = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4621 llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};
4622
4623 // _kmpc_free(gtid, addr, nullptr);
4624 (void)CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4625 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free),
4626 args: Args);
4627}
4628
4629void CGOpenMPRuntime::emitUpdateDependObjectsClause(
4630 CodeGenFunction &CGF, LValue DepobjLVal, OpenMPDependClauseKind NewDepKind,
4631 SourceLocation Loc) {
4632 ASTContext &C = CGM.getContext();
4633 QualType FlagsTy;
4634 getDependTypes(C, KmpDependInfoTy, FlagsTy);
4635 auto *KmpDependInfoRD = KmpDependInfoTy->castAsRecordDecl();
4636 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(T: FlagsTy);
4637 llvm::Value *NumDeps;
4638 LValue Base;
4639 std::tie(args&: NumDeps, args&: Base) = getDepobjElements(CGF, DepobjLVal, Loc);
4640
4641 Address Begin = Base.getAddress();
4642 // Cast from pointer to array type to pointer to single element.
4643 llvm::Value *End = CGF.Builder.CreateGEP(Ty: Begin.getElementType(),
4644 Ptr: Begin.emitRawPointer(CGF), IdxList: NumDeps);
4645 // The basic structure here is a while-do loop.
4646 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.body");
4647 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.done");
4648 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4649 CGF.EmitBlock(BB: BodyBB);
4650 llvm::PHINode *ElementPHI =
4651 CGF.Builder.CreatePHI(Ty: Begin.getType(), NumReservedValues: 2, Name: "omp.elementPast");
4652 ElementPHI->addIncoming(V: Begin.emitRawPointer(CGF), BB: EntryBB);
4653 Begin = Begin.withPointer(NewPointer: ElementPHI, IsKnownNonNull: KnownNonNull);
4654 Base = CGF.MakeAddrLValue(Addr: Begin, T: KmpDependInfoTy, BaseInfo: Base.getBaseInfo(),
4655 TBAAInfo: Base.getTBAAInfo());
4656 // deps[i].flags = NewDepKind;
4657 RTLDependenceKindTy DepKind = translateDependencyKind(K: NewDepKind);
4658 LValue FlagsLVal = CGF.EmitLValueForField(
4659 Base, Field: *std::next(x: KmpDependInfoRD->field_begin(),
4660 n: static_cast<unsigned int>(RTLDependInfoFields::Flags)));
4661 CGF.EmitStoreOfScalar(
4662 value: llvm::ConstantInt::get(Ty: LLVMFlagsTy, V: static_cast<unsigned int>(DepKind)),
4663 lvalue: FlagsLVal);
4664
4665 // Shift the address forward by one element.
4666 llvm::Value *ElementNext =
4667 CGF.Builder.CreateConstGEP(Addr: Begin, /*Index=*/1, Name: "omp.elementNext")
4668 .emitRawPointer(CGF);
4669 ElementPHI->addIncoming(V: ElementNext, BB: CGF.Builder.GetInsertBlock());
4670 llvm::Value *IsEmpty =
4671 CGF.Builder.CreateICmpEQ(LHS: ElementNext, RHS: End, Name: "omp.isempty");
4672 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
4673 // Done.
4674 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
4675}
4676
4677void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
4678 const OMPExecutableDirective &D,
4679 llvm::Function *TaskFunction,
4680 QualType SharedsTy, Address Shareds,
4681 const Expr *IfCond,
4682 const OMPTaskDataTy &Data) {
4683 if (!CGF.HaveInsertPoint())
4684 return;
4685
4686 TaskResultTy Result =
4687 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4688 llvm::Value *NewTask = Result.NewTask;
4689 llvm::Function *TaskEntry = Result.TaskEntry;
4690 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
4691 LValue TDBase = Result.TDBase;
4692 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
4693 // Process list of dependences.
4694 Address DependenciesArray = Address::invalid();
4695 llvm::Value *NumOfElements;
4696 std::tie(args&: NumOfElements, args&: DependenciesArray) =
4697 emitDependClause(CGF, Dependencies: Data.Dependences, Loc);
4698
4699 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4700 // libcall.
4701 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
4702 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
4703 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
4704 // list is not empty
4705 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4706 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4707 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
4708 llvm::Value *DepTaskArgs[7];
4709 if (!Data.Dependences.empty()) {
4710 DepTaskArgs[0] = UpLoc;
4711 DepTaskArgs[1] = ThreadID;
4712 DepTaskArgs[2] = NewTask;
4713 DepTaskArgs[3] = NumOfElements;
4714 DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
4715 DepTaskArgs[5] = CGF.Builder.getInt32(C: 0);
4716 DepTaskArgs[6] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4717 }
4718 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
4719 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
4720 if (!Data.Tied) {
4721 auto PartIdFI = std::next(x: KmpTaskTQTyRD->field_begin(), n: KmpTaskTPartId);
4722 LValue PartIdLVal = CGF.EmitLValueForField(Base: TDBase, Field: *PartIdFI);
4723 CGF.EmitStoreOfScalar(value: CGF.Builder.getInt32(C: 0), lvalue: PartIdLVal);
4724 }
4725 if (!Data.Dependences.empty()) {
4726 CGF.EmitRuntimeCall(
4727 callee: OMPBuilder.getOrCreateRuntimeFunction(
4728 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task_with_deps),
4729 args: DepTaskArgs);
4730 } else {
4731 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4732 M&: CGM.getModule(), FnID: OMPRTL___kmpc_omp_task),
4733 args: TaskArgs);
4734 }
4735 // Check if parent region is untied and build return for untied task;
4736 if (auto *Region =
4737 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
4738 Region->emitUntiedSwitch(CGF);
4739 };
4740
4741 llvm::Value *DepWaitTaskArgs[7];
4742 if (!Data.Dependences.empty()) {
4743 DepWaitTaskArgs[0] = UpLoc;
4744 DepWaitTaskArgs[1] = ThreadID;
4745 DepWaitTaskArgs[2] = NumOfElements;
4746 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
4747 DepWaitTaskArgs[4] = CGF.Builder.getInt32(C: 0);
4748 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
4749 DepWaitTaskArgs[6] =
4750 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Data.HasNowaitClause);
4751 }
4752 auto &M = CGM.getModule();
4753 auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
4754 TaskEntry, &Data, &DepWaitTaskArgs,
4755 Loc](CodeGenFunction &CGF, PrePostActionTy &) {
4756 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
4757 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
4758 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
4759 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
4760 // is specified.
4761 if (!Data.Dependences.empty())
4762 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4763 M, FnID: OMPRTL___kmpc_omp_taskwait_deps_51),
4764 args: DepWaitTaskArgs);
4765 // Call proxy_task_entry(gtid, new_task);
4766 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
4767 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
4768 Action.Enter(CGF);
4769 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4770 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn: TaskEntry,
4771 Args: OutlinedFnArgs);
4772 };
4773
4774 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
4775 // kmp_task_t *new_task);
4776 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
4777 // kmp_task_t *new_task);
4778 RegionCodeGenTy RCG(CodeGen);
4779 CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
4780 M, FnID: OMPRTL___kmpc_omp_task_begin_if0),
4781 TaskArgs,
4782 OMPBuilder.getOrCreateRuntimeFunction(
4783 M, FnID: OMPRTL___kmpc_omp_task_complete_if0),
4784 TaskArgs);
4785 RCG.setAction(Action);
4786 RCG(CGF);
4787 };
4788
4789 if (IfCond) {
4790 emitIfClause(CGF, Cond: IfCond, ThenGen: ThenCodeGen, ElseGen: ElseCodeGen);
4791 } else {
4792 RegionCodeGenTy ThenRCG(ThenCodeGen);
4793 ThenRCG(CGF);
4794 }
4795}
4796
4797void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
4798 const OMPLoopDirective &D,
4799 llvm::Function *TaskFunction,
4800 QualType SharedsTy, Address Shareds,
4801 const Expr *IfCond,
4802 const OMPTaskDataTy &Data) {
4803 if (!CGF.HaveInsertPoint())
4804 return;
4805 TaskResultTy Result =
4806 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4807 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
4808 // libcall.
4809 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
4810 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
4811 // sched, kmp_uint64 grainsize, void *task_dup);
4812 llvm::Value *ThreadID = getThreadID(CGF, Loc);
4813 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
4814 llvm::Value *IfVal;
4815 if (IfCond) {
4816 IfVal = CGF.Builder.CreateIntCast(V: CGF.EvaluateExprAsBool(E: IfCond), DestTy: CGF.IntTy,
4817 /*isSigned=*/true);
4818 } else {
4819 IfVal = llvm::ConstantInt::getSigned(Ty: CGF.IntTy, /*V=*/1);
4820 }
4821
4822 LValue LBLVal = CGF.EmitLValueForField(
4823 Base: Result.TDBase,
4824 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTLowerBound));
4825 const auto *LBVar =
4826 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getLowerBoundVariable())->getDecl());
4827 CGF.EmitAnyExprToMem(E: LBVar->getInit(), Location: LBLVal.getAddress(), Quals: LBLVal.getQuals(),
4828 /*IsInitializer=*/true);
4829 LValue UBLVal = CGF.EmitLValueForField(
4830 Base: Result.TDBase,
4831 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTUpperBound));
4832 const auto *UBVar =
4833 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getUpperBoundVariable())->getDecl());
4834 CGF.EmitAnyExprToMem(E: UBVar->getInit(), Location: UBLVal.getAddress(), Quals: UBLVal.getQuals(),
4835 /*IsInitializer=*/true);
4836 LValue StLVal = CGF.EmitLValueForField(
4837 Base: Result.TDBase,
4838 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTStride));
4839 const auto *StVar =
4840 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D.getStrideVariable())->getDecl());
4841 CGF.EmitAnyExprToMem(E: StVar->getInit(), Location: StLVal.getAddress(), Quals: StLVal.getQuals(),
4842 /*IsInitializer=*/true);
4843 // Store reductions address.
4844 LValue RedLVal = CGF.EmitLValueForField(
4845 Base: Result.TDBase,
4846 Field: *std::next(x: Result.KmpTaskTQTyRD->field_begin(), n: KmpTaskTReductions));
4847 if (Data.Reductions) {
4848 CGF.EmitStoreOfScalar(value: Data.Reductions, lvalue: RedLVal);
4849 } else {
4850 CGF.EmitNullInitialization(DestPtr: RedLVal.getAddress(),
4851 Ty: CGF.getContext().VoidPtrTy);
4852 }
4853 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
4854 llvm::SmallVector<llvm::Value *, 12> TaskArgs{
4855 UpLoc,
4856 ThreadID,
4857 Result.NewTask,
4858 IfVal,
4859 LBLVal.getPointer(CGF),
4860 UBLVal.getPointer(CGF),
4861 CGF.EmitLoadOfScalar(lvalue: StLVal, Loc),
4862 llvm::ConstantInt::getSigned(
4863 Ty: CGF.IntTy, V: 1), // Always 1 because taskgroup emitted by the compiler
4864 llvm::ConstantInt::getSigned(
4865 Ty: CGF.IntTy, V: Data.Schedule.getPointer()
4866 ? Data.Schedule.getInt() ? NumTasks : Grainsize
4867 : NoSchedule),
4868 Data.Schedule.getPointer()
4869 ? CGF.Builder.CreateIntCast(V: Data.Schedule.getPointer(), DestTy: CGF.Int64Ty,
4870 /*isSigned=*/false)
4871 : llvm::ConstantInt::get(Ty: CGF.Int64Ty, /*V=*/0)};
4872 if (Data.HasModifier)
4873 TaskArgs.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: 1));
4874
4875 TaskArgs.push_back(Elt: Result.TaskDupFn
4876 ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
4877 V: Result.TaskDupFn, DestTy: CGF.VoidPtrTy)
4878 : llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy));
4879 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
4880 M&: CGM.getModule(), FnID: Data.HasModifier
4881 ? OMPRTL___kmpc_taskloop_5
4882 : OMPRTL___kmpc_taskloop),
4883 args: TaskArgs);
4884}
4885
4886/// Emit reduction operation for each element of array (required for
4887/// array sections) LHS op = RHS.
4888/// \param Type Type of array.
4889/// \param LHSVar Variable on the left side of the reduction operation
4890/// (references element of array in original variable).
4891/// \param RHSVar Variable on the right side of the reduction operation
4892/// (references element of array in original variable).
4893/// \param RedOpGen Generator of reduction operation with use of LHSVar and
4894/// RHSVar.
4895static void EmitOMPAggregateReduction(
4896 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar,
4897 const VarDecl *RHSVar,
4898 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *,
4899 const Expr *, const Expr *)> &RedOpGen,
4900 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr,
4901 const Expr *UpExpr = nullptr) {
4902 // Perform element-by-element initialization.
4903 QualType ElementTy;
4904 Address LHSAddr = CGF.GetAddrOfLocalVar(VD: LHSVar);
4905 Address RHSAddr = CGF.GetAddrOfLocalVar(VD: RHSVar);
4906
4907 // Drill down to the base element type on both arrays.
4908 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe();
4909 llvm::Value *NumElements = CGF.emitArrayLength(arrayType: ArrayTy, baseType&: ElementTy, addr&: LHSAddr);
4910
4911 llvm::Value *RHSBegin = RHSAddr.emitRawPointer(CGF);
4912 llvm::Value *LHSBegin = LHSAddr.emitRawPointer(CGF);
4913 // Cast from pointer to array type to pointer to single element.
4914 llvm::Value *LHSEnd =
4915 CGF.Builder.CreateGEP(Ty: LHSAddr.getElementType(), Ptr: LHSBegin, IdxList: NumElements);
4916 // The basic structure here is a while-do loop.
4917 llvm::BasicBlock *BodyBB = CGF.createBasicBlock(name: "omp.arraycpy.body");
4918 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "omp.arraycpy.done");
4919 llvm::Value *IsEmpty =
4920 CGF.Builder.CreateICmpEQ(LHS: LHSBegin, RHS: LHSEnd, Name: "omp.arraycpy.isempty");
4921 CGF.Builder.CreateCondBr(Cond: IsEmpty, True: DoneBB, False: BodyBB);
4922
4923 // Enter the loop body, making that address the current address.
4924 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock();
4925 CGF.EmitBlock(BB: BodyBB);
4926
4927 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(T: ElementTy);
4928
4929 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI(
4930 Ty: RHSBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.srcElementPast");
4931 RHSElementPHI->addIncoming(V: RHSBegin, BB: EntryBB);
4932 Address RHSElementCurrent(
4933 RHSElementPHI, RHSAddr.getElementType(),
4934 RHSAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
4935
4936 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI(
4937 Ty: LHSBegin->getType(), NumReservedValues: 2, Name: "omp.arraycpy.destElementPast");
4938 LHSElementPHI->addIncoming(V: LHSBegin, BB: EntryBB);
4939 Address LHSElementCurrent(
4940 LHSElementPHI, LHSAddr.getElementType(),
4941 LHSAddr.getAlignment().alignmentOfArrayElement(elementSize: ElementSize));
4942
4943 // Emit copy.
4944 CodeGenFunction::OMPPrivateScope Scope(CGF);
4945 Scope.addPrivate(LocalVD: LHSVar, Addr: LHSElementCurrent);
4946 Scope.addPrivate(LocalVD: RHSVar, Addr: RHSElementCurrent);
4947 Scope.Privatize();
4948 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4949 Scope.ForceCleanup();
4950
4951 // Shift the address forward by one element.
4952 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32(
4953 Ty: LHSAddr.getElementType(), Ptr: LHSElementPHI, /*Idx0=*/1,
4954 Name: "omp.arraycpy.dest.element");
4955 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32(
4956 Ty: RHSAddr.getElementType(), Ptr: RHSElementPHI, /*Idx0=*/1,
4957 Name: "omp.arraycpy.src.element");
4958 // Check whether we've reached the end.
4959 llvm::Value *Done =
4960 CGF.Builder.CreateICmpEQ(LHS: LHSElementNext, RHS: LHSEnd, Name: "omp.arraycpy.done");
4961 CGF.Builder.CreateCondBr(Cond: Done, True: DoneBB, False: BodyBB);
4962 LHSElementPHI->addIncoming(V: LHSElementNext, BB: CGF.Builder.GetInsertBlock());
4963 RHSElementPHI->addIncoming(V: RHSElementNext, BB: CGF.Builder.GetInsertBlock());
4964
4965 // Done.
4966 CGF.EmitBlock(BB: DoneBB, /*IsFinished=*/true);
4967}
4968
4969/// Emit reduction combiner. If the combiner is a simple expression emit it as
4970/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
4971/// UDR combiner function.
4972static void emitReductionCombiner(CodeGenFunction &CGF,
4973 const Expr *ReductionOp) {
4974 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp))
4975 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Val: CE->getCallee()))
4976 if (const auto *DRE =
4977 dyn_cast<DeclRefExpr>(Val: OVE->getSourceExpr()->IgnoreImpCasts()))
4978 if (const auto *DRD =
4979 dyn_cast<OMPDeclareReductionDecl>(Val: DRE->getDecl())) {
4980 std::pair<llvm::Function *, llvm::Function *> Reduction =
4981 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(D: DRD);
4982 RValue Func = RValue::get(V: Reduction.first);
4983 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
4984 CGF.EmitIgnoredExpr(E: ReductionOp);
4985 return;
4986 }
4987 CGF.EmitIgnoredExpr(E: ReductionOp);
4988}
4989
4990llvm::Function *CGOpenMPRuntime::emitReductionFunction(
4991 StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
4992 ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
4993 ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps) {
4994 ASTContext &C = CGM.getContext();
4995
4996 // void reduction_func(void *LHSArg, void *RHSArg);
4997 auto *LHSArg =
4998 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
4999 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5000 auto *RHSArg =
5001 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5002 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5003 FunctionArgList Args{LHSArg, RHSArg};
5004 const auto &CGFI =
5005 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5006 std::string Name = getReductionFuncName(Name: ReducerName);
5007 auto *Fn = llvm::Function::Create(Ty: CGM.getTypes().GetFunctionType(Info: CGFI),
5008 Linkage: llvm::GlobalValue::InternalLinkage, N: Name,
5009 M: &CGM.getModule());
5010 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: CGFI);
5011 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5012 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5013 Fn->setDoesNotRecurse();
5014 CodeGenFunction CGF(CGM);
5015 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo: CGFI, Args, Loc, StartLoc: Loc);
5016
5017 // Dst = (void*[n])(LHSArg);
5018 // Src = (void*[n])(RHSArg);
5019 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5020 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: LHSArg)),
5021 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5022 ArgsElemType, CGF.getPointerAlign());
5023 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5024 V: CGF.Builder.CreateLoad(Addr: CGF.GetAddrOfLocalVar(VD: RHSArg)),
5025 DestTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5026 ArgsElemType, CGF.getPointerAlign());
5027
5028 // ...
5029 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
5030 // ...
5031 CodeGenFunction::OMPPrivateScope Scope(CGF);
5032 const auto *IPriv = Privates.begin();
5033 unsigned Idx = 0;
5034 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) {
5035 const auto *RHSVar =
5036 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHSExprs[I])->getDecl());
5037 Scope.addPrivate(LocalVD: RHSVar, Addr: emitAddrOfVarFromArray(CGF, Array: RHS, Index: Idx, Var: RHSVar));
5038 const auto *LHSVar =
5039 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHSExprs[I])->getDecl());
5040 Scope.addPrivate(LocalVD: LHSVar, Addr: emitAddrOfVarFromArray(CGF, Array: LHS, Index: Idx, Var: LHSVar));
5041 QualType PrivTy = (*IPriv)->getType();
5042 if (PrivTy->isVariablyModifiedType()) {
5043 // Get array size and emit VLA type.
5044 ++Idx;
5045 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: LHS, Index: Idx);
5046 llvm::Value *Ptr = CGF.Builder.CreateLoad(Addr: Elem);
5047 const VariableArrayType *VLA =
5048 CGF.getContext().getAsVariableArrayType(T: PrivTy);
5049 const auto *OVE = cast<OpaqueValueExpr>(Val: VLA->getSizeExpr());
5050 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5051 CGF, OVE, RValue::get(V: CGF.Builder.CreatePtrToInt(V: Ptr, DestTy: CGF.SizeTy)));
5052 CGF.EmitVariablyModifiedType(Ty: PrivTy);
5053 }
5054 }
5055 Scope.Privatize();
5056 IPriv = Privates.begin();
5057 const auto *ILHS = LHSExprs.begin();
5058 const auto *IRHS = RHSExprs.begin();
5059 for (const Expr *E : ReductionOps) {
5060 if ((*IPriv)->getType()->isArrayType()) {
5061 // Emit reduction for array section.
5062 const auto *LHSVar = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5063 const auto *RHSVar = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5064 EmitOMPAggregateReduction(
5065 CGF, Type: (*IPriv)->getType(), LHSVar, RHSVar,
5066 RedOpGen: [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5067 emitReductionCombiner(CGF, ReductionOp: E);
5068 });
5069 } else {
5070 // Emit reduction for array subscript or single variable.
5071 emitReductionCombiner(CGF, ReductionOp: E);
5072 }
5073 ++IPriv;
5074 ++ILHS;
5075 ++IRHS;
5076 }
5077 Scope.ForceCleanup();
5078 CGF.FinishFunction();
5079 return Fn;
5080}
5081
5082void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF,
5083 const Expr *ReductionOp,
5084 const Expr *PrivateRef,
5085 const DeclRefExpr *LHS,
5086 const DeclRefExpr *RHS) {
5087 if (PrivateRef->getType()->isArrayType()) {
5088 // Emit reduction for array section.
5089 const auto *LHSVar = cast<VarDecl>(Val: LHS->getDecl());
5090 const auto *RHSVar = cast<VarDecl>(Val: RHS->getDecl());
5091 EmitOMPAggregateReduction(
5092 CGF, Type: PrivateRef->getType(), LHSVar, RHSVar,
5093 RedOpGen: [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
5094 emitReductionCombiner(CGF, ReductionOp);
5095 });
5096 } else {
5097 // Emit reduction for array subscript or single variable.
5098 emitReductionCombiner(CGF, ReductionOp);
5099 }
5100}
5101
5102static std::string generateUniqueName(CodeGenModule &CGM,
5103 llvm::StringRef Prefix, const Expr *Ref);
5104
5105void CGOpenMPRuntime::emitPrivateReduction(
5106 CodeGenFunction &CGF, SourceLocation Loc, const Expr *Privates,
5107 const Expr *LHSExprs, const Expr *RHSExprs, const Expr *ReductionOps) {
5108
5109 // Create a shared global variable (__shared_reduction_var) to accumulate the
5110 // final result.
5111 //
5112 // Call __kmpc_barrier to synchronize threads before initialization.
5113 //
5114 // The master thread (thread_id == 0) initializes __shared_reduction_var
5115 // with the identity value or initializer.
5116 //
5117 // Call __kmpc_barrier to synchronize before combining.
5118 // For each i:
5119 // - Thread enters critical section.
5120 // - Reads its private value from LHSExprs[i].
5121 // - Updates __shared_reduction_var[i] = RedOp_i(__shared_reduction_var[i],
5122 // Privates[i]).
5123 // - Exits critical section.
5124 //
5125 // Call __kmpc_barrier after combining.
5126 //
5127 // Each thread copies __shared_reduction_var[i] back to RHSExprs[i].
5128 //
5129 // Final __kmpc_barrier to synchronize after broadcasting
5130 QualType PrivateType = Privates->getType();
5131 llvm::Type *LLVMType = CGF.ConvertTypeForMem(T: PrivateType);
5132
5133 const OMPDeclareReductionDecl *UDR = getReductionInit(ReductionOp: ReductionOps);
5134 std::string ReductionVarNameStr;
5135 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Privates->IgnoreParenCasts()))
5136 ReductionVarNameStr =
5137 generateUniqueName(CGM, Prefix: DRE->getDecl()->getNameAsString(), Ref: Privates);
5138 else
5139 ReductionVarNameStr = "unnamed_priv_var";
5140
5141 // Create an internal shared variable
5142 std::string SharedName =
5143 CGM.getOpenMPRuntime().getName(Parts: {"internal_pivate_", ReductionVarNameStr});
5144 llvm::GlobalVariable *SharedVar = OMPBuilder.getOrCreateInternalVariable(
5145 Ty: LLVMType, Name: ".omp.reduction." + SharedName);
5146
5147 SharedVar->setAlignment(
5148 llvm::MaybeAlign(CGF.getContext().getTypeAlign(T: PrivateType) / 8));
5149
5150 Address SharedResult =
5151 CGF.MakeNaturalAlignRawAddrLValue(V: SharedVar, T: PrivateType).getAddress();
5152
5153 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5154 llvm::Value *BarrierLoc = emitUpdateLocation(CGF, Loc, Flags: OMP_ATOMIC_REDUCE);
5155 llvm::Value *BarrierArgs[] = {BarrierLoc, ThreadId};
5156
5157 llvm::BasicBlock *InitBB = CGF.createBasicBlock(name: "init");
5158 llvm::BasicBlock *InitEndBB = CGF.createBasicBlock(name: "init.end");
5159
5160 llvm::Value *IsWorker = CGF.Builder.CreateICmpEQ(
5161 LHS: ThreadId, RHS: llvm::ConstantInt::get(Ty: ThreadId->getType(), V: 0));
5162 CGF.Builder.CreateCondBr(Cond: IsWorker, True: InitBB, False: InitEndBB);
5163
5164 CGF.EmitBlock(BB: InitBB);
5165
5166 auto EmitSharedInit = [&]() {
5167 if (UDR) { // Check if it's a User-Defined Reduction
5168 if (const Expr *UDRInitExpr = UDR->getInitializer()) {
5169 std::pair<llvm::Function *, llvm::Function *> FnPair =
5170 getUserDefinedReduction(D: UDR);
5171 llvm::Function *InitializerFn = FnPair.second;
5172 if (InitializerFn) {
5173 if (const auto *CE =
5174 dyn_cast<CallExpr>(Val: UDRInitExpr->IgnoreParenImpCasts())) {
5175 const auto *OutDRE = cast<DeclRefExpr>(
5176 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 0)->IgnoreParenImpCasts())
5177 ->getSubExpr());
5178 const VarDecl *OutVD = cast<VarDecl>(Val: OutDRE->getDecl());
5179
5180 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5181 LocalScope.addPrivate(LocalVD: OutVD, Addr: SharedResult);
5182
5183 (void)LocalScope.Privatize();
5184 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(
5185 Val: CE->getCallee()->IgnoreParenImpCasts())) {
5186 CodeGenFunction::OpaqueValueMapping OpaqueMap(
5187 CGF, OVE, RValue::get(V: InitializerFn));
5188 CGF.EmitIgnoredExpr(E: CE);
5189 } else {
5190 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5191 Quals: PrivateType.getQualifiers(),
5192 /*IsInitializer=*/true);
5193 }
5194 } else {
5195 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5196 Quals: PrivateType.getQualifiers(),
5197 /*IsInitializer=*/true);
5198 }
5199 } else {
5200 CGF.EmitAnyExprToMem(E: UDRInitExpr, Location: SharedResult,
5201 Quals: PrivateType.getQualifiers(),
5202 /*IsInitializer=*/true);
5203 }
5204 } else {
5205 // EmitNullInitialization handles default construction for C++ classes
5206 // and zeroing for scalars, which is a reasonable default.
5207 CGF.EmitNullInitialization(DestPtr: SharedResult, Ty: PrivateType);
5208 }
5209 return; // UDR initialization handled
5210 }
5211 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Privates)) {
5212 if (const auto *VD = dyn_cast<VarDecl>(Val: DRE->getDecl())) {
5213 if (const Expr *InitExpr = VD->getInit()) {
5214 CGF.EmitAnyExprToMem(E: InitExpr, Location: SharedResult,
5215 Quals: PrivateType.getQualifiers(), IsInitializer: true);
5216 return;
5217 }
5218 }
5219 }
5220 CGF.EmitNullInitialization(DestPtr: SharedResult, Ty: PrivateType);
5221 };
5222 EmitSharedInit();
5223 CGF.Builder.CreateBr(Dest: InitEndBB);
5224 CGF.EmitBlock(BB: InitEndBB);
5225
5226 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5227 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5228 args: BarrierArgs);
5229
5230 const Expr *ReductionOp = ReductionOps;
5231 const OMPDeclareReductionDecl *CurrentUDR = getReductionInit(ReductionOp);
5232 LValue SharedLV = CGF.MakeAddrLValue(Addr: SharedResult, T: PrivateType);
5233 LValue LHSLV = CGF.EmitLValue(E: Privates);
5234
5235 auto EmitCriticalReduction = [&](auto ReductionGen) {
5236 std::string CriticalName = getName(Parts: {"reduction_critical"});
5237 emitCriticalRegion(CGF, CriticalName, CriticalOpGen: ReductionGen, Loc);
5238 };
5239
5240 if (CurrentUDR) {
5241 // Handle user-defined reduction.
5242 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5243 Action.Enter(CGF);
5244 std::pair<llvm::Function *, llvm::Function *> FnPair =
5245 getUserDefinedReduction(D: CurrentUDR);
5246 if (FnPair.first) {
5247 if (const auto *CE = dyn_cast<CallExpr>(Val: ReductionOp)) {
5248 const auto *OutDRE = cast<DeclRefExpr>(
5249 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 0)->IgnoreParenImpCasts())
5250 ->getSubExpr());
5251 const auto *InDRE = cast<DeclRefExpr>(
5252 Val: cast<UnaryOperator>(Val: CE->getArg(Arg: 1)->IgnoreParenImpCasts())
5253 ->getSubExpr());
5254 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5255 LocalScope.addPrivate(LocalVD: cast<VarDecl>(Val: OutDRE->getDecl()),
5256 Addr: SharedLV.getAddress());
5257 LocalScope.addPrivate(LocalVD: cast<VarDecl>(Val: InDRE->getDecl()),
5258 Addr: LHSLV.getAddress());
5259 (void)LocalScope.Privatize();
5260 emitReductionCombiner(CGF, ReductionOp);
5261 }
5262 }
5263 };
5264 EmitCriticalReduction(ReductionGen);
5265 } else {
5266 // Handle built-in reduction operations.
5267#ifndef NDEBUG
5268 const Expr *ReductionClauseExpr = ReductionOp->IgnoreParenCasts();
5269 if (const auto *Cleanup = dyn_cast<ExprWithCleanups>(ReductionClauseExpr))
5270 ReductionClauseExpr = Cleanup->getSubExpr()->IgnoreParenCasts();
5271
5272 const Expr *AssignRHS = nullptr;
5273 if (const auto *BinOp = dyn_cast<BinaryOperator>(ReductionClauseExpr)) {
5274 if (BinOp->getOpcode() == BO_Assign)
5275 AssignRHS = BinOp->getRHS();
5276 } else if (const auto *OpCall =
5277 dyn_cast<CXXOperatorCallExpr>(ReductionClauseExpr)) {
5278 if (OpCall->getOperator() == OO_Equal)
5279 AssignRHS = OpCall->getArg(1);
5280 }
5281
5282 assert(AssignRHS &&
5283 "Private Variable Reduction : Invalid ReductionOp expression");
5284#endif
5285
5286 auto ReductionGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
5287 Action.Enter(CGF);
5288 const auto *OmpOutDRE =
5289 dyn_cast<DeclRefExpr>(Val: LHSExprs->IgnoreParenImpCasts());
5290 const auto *OmpInDRE =
5291 dyn_cast<DeclRefExpr>(Val: RHSExprs->IgnoreParenImpCasts());
5292 assert(
5293 OmpOutDRE && OmpInDRE &&
5294 "Private Variable Reduction : LHSExpr/RHSExpr must be DeclRefExprs");
5295 const VarDecl *OmpOutVD = cast<VarDecl>(Val: OmpOutDRE->getDecl());
5296 const VarDecl *OmpInVD = cast<VarDecl>(Val: OmpInDRE->getDecl());
5297 CodeGenFunction::OMPPrivateScope LocalScope(CGF);
5298 LocalScope.addPrivate(LocalVD: OmpOutVD, Addr: SharedLV.getAddress());
5299 LocalScope.addPrivate(LocalVD: OmpInVD, Addr: LHSLV.getAddress());
5300 (void)LocalScope.Privatize();
5301 // Emit the actual reduction operation
5302 CGF.EmitIgnoredExpr(E: ReductionOp);
5303 };
5304 EmitCriticalReduction(ReductionGen);
5305 }
5306
5307 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5308 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5309 args: BarrierArgs);
5310
5311 // Broadcast final result
5312 bool IsAggregate = PrivateType->isAggregateType();
5313 LValue SharedLV1 = CGF.MakeAddrLValue(Addr: SharedResult, T: PrivateType);
5314 llvm::Value *FinalResultVal = nullptr;
5315 Address FinalResultAddr = Address::invalid();
5316
5317 if (IsAggregate)
5318 FinalResultAddr = SharedResult;
5319 else
5320 FinalResultVal = CGF.EmitLoadOfScalar(lvalue: SharedLV1, Loc);
5321
5322 LValue TargetLHSLV = CGF.EmitLValue(E: RHSExprs);
5323 if (IsAggregate) {
5324 CGF.EmitAggregateCopy(Dest: TargetLHSLV,
5325 Src: CGF.MakeAddrLValue(Addr: FinalResultAddr, T: PrivateType),
5326 EltTy: PrivateType, MayOverlap: AggValueSlot::DoesNotOverlap, isVolatile: false);
5327 } else {
5328 CGF.EmitStoreOfScalar(value: FinalResultVal, lvalue: TargetLHSLV);
5329 }
5330 // Final synchronization barrier
5331 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
5332 M&: CGM.getModule(), FnID: OMPRTL___kmpc_barrier),
5333 args: BarrierArgs);
5334
5335 // Combiner with original list item
5336 auto OriginalListCombiner = [&](CodeGenFunction &CGF,
5337 PrePostActionTy &Action) {
5338 Action.Enter(CGF);
5339 emitSingleReductionCombiner(CGF, ReductionOp: ReductionOps, PrivateRef: Privates,
5340 LHS: cast<DeclRefExpr>(Val: LHSExprs),
5341 RHS: cast<DeclRefExpr>(Val: RHSExprs));
5342 };
5343 EmitCriticalReduction(OriginalListCombiner);
5344}
5345
5346void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
5347 ArrayRef<const Expr *> OrgPrivates,
5348 ArrayRef<const Expr *> OrgLHSExprs,
5349 ArrayRef<const Expr *> OrgRHSExprs,
5350 ArrayRef<const Expr *> OrgReductionOps,
5351 ReductionOptionsTy Options) {
5352 if (!CGF.HaveInsertPoint())
5353 return;
5354
5355 bool WithNowait = Options.WithNowait;
5356 bool SimpleReduction = Options.SimpleReduction;
5357
5358 // Next code should be emitted for reduction:
5359 //
5360 // static kmp_critical_name lock = { 0 };
5361 //
5362 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
5363 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]);
5364 // ...
5365 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1],
5366 // *(Type<n>-1*)rhs[<n>-1]);
5367 // }
5368 //
5369 // ...
5370 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
5371 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5372 // RedList, reduce_func, &<lock>)) {
5373 // case 1:
5374 // ...
5375 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5376 // ...
5377 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5378 // break;
5379 // case 2:
5380 // ...
5381 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5382 // ...
5383 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);]
5384 // break;
5385 // default:;
5386 // }
5387 //
5388 // if SimpleReduction is true, only the next code is generated:
5389 // ...
5390 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5391 // ...
5392
5393 ASTContext &C = CGM.getContext();
5394
5395 if (SimpleReduction) {
5396 CodeGenFunction::RunCleanupsScope Scope(CGF);
5397 const auto *IPriv = OrgPrivates.begin();
5398 const auto *ILHS = OrgLHSExprs.begin();
5399 const auto *IRHS = OrgRHSExprs.begin();
5400 for (const Expr *E : OrgReductionOps) {
5401 emitSingleReductionCombiner(CGF, ReductionOp: E, PrivateRef: *IPriv, LHS: cast<DeclRefExpr>(Val: *ILHS),
5402 RHS: cast<DeclRefExpr>(Val: *IRHS));
5403 ++IPriv;
5404 ++ILHS;
5405 ++IRHS;
5406 }
5407 return;
5408 }
5409
5410 // Filter out shared reduction variables based on IsPrivateVarReduction flag.
5411 // Only keep entries where the corresponding variable is not private.
5412 SmallVector<const Expr *> FilteredPrivates, FilteredLHSExprs,
5413 FilteredRHSExprs, FilteredReductionOps;
5414 for (unsigned I : llvm::seq<unsigned>(
5415 Size: std::min(a: OrgReductionOps.size(), b: OrgLHSExprs.size()))) {
5416 if (!Options.IsPrivateVarReduction[I]) {
5417 FilteredPrivates.emplace_back(Args: OrgPrivates[I]);
5418 FilteredLHSExprs.emplace_back(Args: OrgLHSExprs[I]);
5419 FilteredRHSExprs.emplace_back(Args: OrgRHSExprs[I]);
5420 FilteredReductionOps.emplace_back(Args: OrgReductionOps[I]);
5421 }
5422 }
5423 // Wrap filtered vectors in ArrayRef for downstream shared reduction
5424 // processing.
5425 ArrayRef<const Expr *> Privates = FilteredPrivates;
5426 ArrayRef<const Expr *> LHSExprs = FilteredLHSExprs;
5427 ArrayRef<const Expr *> RHSExprs = FilteredRHSExprs;
5428 ArrayRef<const Expr *> ReductionOps = FilteredReductionOps;
5429
5430 // 1. Build a list of reduction variables.
5431 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]};
5432 auto Size = RHSExprs.size();
5433 for (const Expr *E : Privates) {
5434 if (E->getType()->isVariablyModifiedType())
5435 // Reserve place for array size.
5436 ++Size;
5437 }
5438 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size);
5439 QualType ReductionArrayTy = C.getConstantArrayType(
5440 EltTy: C.VoidPtrTy, ArySize: ArraySize, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal,
5441 /*IndexTypeQuals=*/0);
5442 RawAddress ReductionList =
5443 CGF.CreateMemTemp(T: ReductionArrayTy, Name: ".omp.reduction.red_list");
5444 const auto *IPriv = Privates.begin();
5445 unsigned Idx = 0;
5446 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) {
5447 Address Elem = CGF.Builder.CreateConstArrayGEP(Addr: ReductionList, Index: Idx);
5448 CGF.Builder.CreateStore(
5449 Val: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5450 V: CGF.EmitLValue(E: RHSExprs[I]).getPointer(CGF), DestTy: CGF.VoidPtrTy),
5451 Addr: Elem);
5452 if ((*IPriv)->getType()->isVariablyModifiedType()) {
5453 // Store array size.
5454 ++Idx;
5455 Elem = CGF.Builder.CreateConstArrayGEP(Addr: ReductionList, Index: Idx);
5456 llvm::Value *Size = CGF.Builder.CreateIntCast(
5457 V: CGF.getVLASize(
5458 vla: CGF.getContext().getAsVariableArrayType(T: (*IPriv)->getType()))
5459 .NumElts,
5460 DestTy: CGF.SizeTy, /*isSigned=*/false);
5461 CGF.Builder.CreateStore(Val: CGF.Builder.CreateIntToPtr(V: Size, DestTy: CGF.VoidPtrTy),
5462 Addr: Elem);
5463 }
5464 }
5465
5466 // 2. Emit reduce_func().
5467 llvm::Function *ReductionFn = emitReductionFunction(
5468 ReducerName: CGF.CurFn->getName(), Loc, ArgsElemType: CGF.ConvertTypeForMem(T: ReductionArrayTy),
5469 Privates, LHSExprs, RHSExprs, ReductionOps);
5470
5471 // 3. Create static kmp_critical_name lock = { 0 };
5472 std::string Name = getName(Parts: {"reduction"});
5473 llvm::Value *Lock = getCriticalRegionLock(CriticalName: Name);
5474
5475 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
5476 // RedList, reduce_func, &<lock>);
5477 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, Flags: OMP_ATOMIC_REDUCE);
5478 llvm::Value *ThreadId = getThreadID(CGF, Loc);
5479 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(Ty: ReductionArrayTy);
5480 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
5481 V: ReductionList.getPointer(), DestTy: CGF.VoidPtrTy);
5482 llvm::Value *Args[] = {
5483 IdentTLoc, // ident_t *<loc>
5484 ThreadId, // i32 <gtid>
5485 CGF.Builder.getInt32(C: RHSExprs.size()), // i32 <n>
5486 ReductionArrayTySize, // size_type sizeof(RedList)
5487 RL, // void *RedList
5488 ReductionFn, // void (*) (void *, void *) <reduce_func>
5489 Lock // kmp_critical_name *&<lock>
5490 };
5491 llvm::Value *Res = CGF.EmitRuntimeCall(
5492 callee: OMPBuilder.getOrCreateRuntimeFunction(
5493 M&: CGM.getModule(),
5494 FnID: WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),
5495 args: Args);
5496
5497 // 5. Build switch(res)
5498 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(name: ".omp.reduction.default");
5499 llvm::SwitchInst *SwInst =
5500 CGF.Builder.CreateSwitch(V: Res, Dest: DefaultBB, /*NumCases=*/2);
5501
5502 // 6. Build case 1:
5503 // ...
5504 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
5505 // ...
5506 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5507 // break;
5508 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(name: ".omp.reduction.case1");
5509 SwInst->addCase(OnVal: CGF.Builder.getInt32(C: 1), Dest: Case1BB);
5510 CGF.EmitBlock(BB: Case1BB);
5511
5512 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
5513 llvm::Value *EndArgs[] = {
5514 IdentTLoc, // ident_t *<loc>
5515 ThreadId, // i32 <gtid>
5516 Lock // kmp_critical_name *&<lock>
5517 };
5518 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](
5519 CodeGenFunction &CGF, PrePostActionTy &Action) {
5520 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5521 const auto *IPriv = Privates.begin();
5522 const auto *ILHS = LHSExprs.begin();
5523 const auto *IRHS = RHSExprs.begin();
5524 for (const Expr *E : ReductionOps) {
5525 RT.emitSingleReductionCombiner(CGF, ReductionOp: E, PrivateRef: *IPriv, LHS: cast<DeclRefExpr>(Val: *ILHS),
5526 RHS: cast<DeclRefExpr>(Val: *IRHS));
5527 ++IPriv;
5528 ++ILHS;
5529 ++IRHS;
5530 }
5531 };
5532 RegionCodeGenTy RCG(CodeGen);
5533 CommonActionTy Action(
5534 nullptr, {},
5535 OMPBuilder.getOrCreateRuntimeFunction(
5536 M&: CGM.getModule(), FnID: WithNowait ? OMPRTL___kmpc_end_reduce_nowait
5537 : OMPRTL___kmpc_end_reduce),
5538 EndArgs);
5539 RCG.setAction(Action);
5540 RCG(CGF);
5541
5542 CGF.EmitBranch(Block: DefaultBB);
5543
5544 // 7. Build case 2:
5545 // ...
5546 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
5547 // ...
5548 // break;
5549 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(name: ".omp.reduction.case2");
5550 SwInst->addCase(OnVal: CGF.Builder.getInt32(C: 2), Dest: Case2BB);
5551 CGF.EmitBlock(BB: Case2BB);
5552
5553 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps](
5554 CodeGenFunction &CGF, PrePostActionTy &Action) {
5555 const auto *ILHS = LHSExprs.begin();
5556 const auto *IRHS = RHSExprs.begin();
5557 const auto *IPriv = Privates.begin();
5558 for (const Expr *E : ReductionOps) {
5559 const Expr *XExpr = nullptr;
5560 const Expr *EExpr = nullptr;
5561 const Expr *UpExpr = nullptr;
5562 BinaryOperatorKind BO = BO_Comma;
5563 if (const auto *BO = dyn_cast<BinaryOperator>(Val: E)) {
5564 if (BO->getOpcode() == BO_Assign) {
5565 XExpr = BO->getLHS();
5566 UpExpr = BO->getRHS();
5567 }
5568 }
5569 // Try to emit update expression as a simple atomic.
5570 const Expr *RHSExpr = UpExpr;
5571 if (RHSExpr) {
5572 // Analyze RHS part of the whole expression.
5573 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(
5574 Val: RHSExpr->IgnoreParenImpCasts())) {
5575 // If this is a conditional operator, analyze its condition for
5576 // min/max reduction operator.
5577 RHSExpr = ACO->getCond();
5578 }
5579 if (const auto *BORHS =
5580 dyn_cast<BinaryOperator>(Val: RHSExpr->IgnoreParenImpCasts())) {
5581 EExpr = BORHS->getRHS();
5582 BO = BORHS->getOpcode();
5583 }
5584 }
5585 if (XExpr) {
5586 const auto *VD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5587 auto &&AtomicRedGen = [BO, VD,
5588 Loc](CodeGenFunction &CGF, const Expr *XExpr,
5589 const Expr *EExpr, const Expr *UpExpr) {
5590 LValue X = CGF.EmitLValue(E: XExpr);
5591 RValue E;
5592 if (EExpr)
5593 E = CGF.EmitAnyExpr(E: EExpr);
5594 CGF.EmitOMPAtomicSimpleUpdateExpr(
5595 X, E, BO, /*IsXLHSInRHSPart=*/true,
5596 AO: llvm::AtomicOrdering::Monotonic, Loc,
5597 CommonGen: [&CGF, UpExpr, VD, Loc](RValue XRValue) {
5598 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5599 Address LHSTemp = CGF.CreateMemTemp(T: VD->getType());
5600 CGF.emitOMPSimpleStore(
5601 LVal: CGF.MakeAddrLValue(Addr: LHSTemp, T: VD->getType()), RVal: XRValue,
5602 RValTy: VD->getType().getNonReferenceType(), Loc);
5603 PrivateScope.addPrivate(LocalVD: VD, Addr: LHSTemp);
5604 (void)PrivateScope.Privatize();
5605 return CGF.EmitAnyExpr(E: UpExpr);
5606 });
5607 };
5608 if ((*IPriv)->getType()->isArrayType()) {
5609 // Emit atomic reduction for array section.
5610 const auto *RHSVar =
5611 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5612 EmitOMPAggregateReduction(CGF, Type: (*IPriv)->getType(), LHSVar: VD, RHSVar,
5613 RedOpGen: AtomicRedGen, XExpr, EExpr, UpExpr);
5614 } else {
5615 // Emit atomic reduction for array subscript or single variable.
5616 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
5617 }
5618 } else {
5619 // Emit as a critical region.
5620 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *,
5621 const Expr *, const Expr *) {
5622 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
5623 std::string Name = RT.getName(Parts: {"atomic_reduction"});
5624 RT.emitCriticalRegion(
5625 CGF, CriticalName: Name,
5626 CriticalOpGen: [=](CodeGenFunction &CGF, PrePostActionTy &Action) {
5627 Action.Enter(CGF);
5628 emitReductionCombiner(CGF, ReductionOp: E);
5629 },
5630 Loc);
5631 };
5632 if ((*IPriv)->getType()->isArrayType()) {
5633 const auto *LHSVar =
5634 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *ILHS)->getDecl());
5635 const auto *RHSVar =
5636 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: *IRHS)->getDecl());
5637 EmitOMPAggregateReduction(CGF, Type: (*IPriv)->getType(), LHSVar, RHSVar,
5638 RedOpGen: CritRedGen);
5639 } else {
5640 CritRedGen(CGF, nullptr, nullptr, nullptr);
5641 }
5642 }
5643 ++ILHS;
5644 ++IRHS;
5645 ++IPriv;
5646 }
5647 };
5648 RegionCodeGenTy AtomicRCG(AtomicCodeGen);
5649 if (!WithNowait) {
5650 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>);
5651 llvm::Value *EndArgs[] = {
5652 IdentTLoc, // ident_t *<loc>
5653 ThreadId, // i32 <gtid>
5654 Lock // kmp_critical_name *&<lock>
5655 };
5656 CommonActionTy Action(nullptr, {},
5657 OMPBuilder.getOrCreateRuntimeFunction(
5658 M&: CGM.getModule(), FnID: OMPRTL___kmpc_end_reduce),
5659 EndArgs);
5660 AtomicRCG.setAction(Action);
5661 AtomicRCG(CGF);
5662 } else {
5663 AtomicRCG(CGF);
5664 }
5665
5666 CGF.EmitBranch(Block: DefaultBB);
5667 CGF.EmitBlock(BB: DefaultBB, /*IsFinished=*/true);
5668 assert(OrgLHSExprs.size() == OrgPrivates.size() &&
5669 "PrivateVarReduction: Privates size mismatch");
5670 assert(OrgLHSExprs.size() == OrgReductionOps.size() &&
5671 "PrivateVarReduction: ReductionOps size mismatch");
5672 for (unsigned I : llvm::seq<unsigned>(
5673 Size: std::min(a: OrgReductionOps.size(), b: OrgLHSExprs.size()))) {
5674 if (Options.IsPrivateVarReduction[I])
5675 emitPrivateReduction(CGF, Loc, Privates: OrgPrivates[I], LHSExprs: OrgLHSExprs[I],
5676 RHSExprs: OrgRHSExprs[I], ReductionOps: OrgReductionOps[I]);
5677 }
5678}
5679
5680/// Generates unique name for artificial threadprivate variables.
5681/// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>"
5682static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix,
5683 const Expr *Ref) {
5684 SmallString<256> Buffer;
5685 llvm::raw_svector_ostream Out(Buffer);
5686 const clang::DeclRefExpr *DE;
5687 const VarDecl *D = ::getBaseDecl(Ref, DE);
5688 if (!D)
5689 D = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: Ref)->getDecl());
5690 D = D->getCanonicalDecl();
5691 std::string Name = CGM.getOpenMPRuntime().getName(
5692 Parts: {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(GD: D)});
5693 Out << Prefix << Name << "_"
5694 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding();
5695 return std::string(Out.str());
5696}
5697
5698/// Emits reduction initializer function:
5699/// \code
5700/// void @.red_init(void* %arg, void* %orig) {
5701/// %0 = bitcast void* %arg to <type>*
5702/// store <type> <init>, <type>* %0
5703/// ret void
5704/// }
5705/// \endcode
5706static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM,
5707 SourceLocation Loc,
5708 ReductionCodeGen &RCG, unsigned N) {
5709 ASTContext &C = CGM.getContext();
5710 QualType VoidPtrTy = C.VoidPtrTy;
5711 VoidPtrTy.addRestrict();
5712 FunctionArgList Args;
5713 auto *Param =
5714 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5715 T: VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5716 auto *ParamOrig =
5717 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5718 T: VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5719 Args.emplace_back(Args&: Param);
5720 Args.emplace_back(Args&: ParamOrig);
5721 const auto &FnInfo =
5722 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5723 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5724 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_init", ""});
5725 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5726 N: Name, M: &CGM.getModule());
5727 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5728 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5729 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5730 Fn->setDoesNotRecurse();
5731 CodeGenFunction CGF(CGM);
5732 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5733 QualType PrivateType = RCG.getPrivateType(N);
5734 Address PrivateAddr = CGF.EmitLoadOfPointer(
5735 Ptr: CGF.GetAddrOfLocalVar(VD: Param).withElementType(ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5736 PtrTy: C.getPointerType(T: PrivateType)->castAs<PointerType>());
5737 llvm::Value *Size = nullptr;
5738 // If the size of the reduction item is non-constant, load it from global
5739 // threadprivate variable.
5740 if (RCG.getSizes(N).second) {
5741 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5742 CGF, VarType: CGM.getContext().getSizeType(),
5743 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5744 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
5745 Ty: CGM.getContext().getSizeType(), Loc);
5746 }
5747 RCG.emitAggregateType(CGF, N, Size);
5748 Address OrigAddr = Address::invalid();
5749 // If initializer uses initializer from declare reduction construct, emit a
5750 // pointer to the address of the original reduction item (reuired by reduction
5751 // initializer)
5752 if (RCG.usesReductionInitializer(N)) {
5753 Address SharedAddr = CGF.GetAddrOfLocalVar(VD: ParamOrig);
5754 OrigAddr = CGF.EmitLoadOfPointer(
5755 Ptr: SharedAddr,
5756 PtrTy: CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr());
5757 }
5758 // Emit the initializer:
5759 // %0 = bitcast void* %arg to <type>*
5760 // store <type> <init>, <type>* %0
5761 RCG.emitInitialization(CGF, N, PrivateAddr, SharedAddr: OrigAddr,
5762 DefaultInit: [](CodeGenFunction &) { return false; });
5763 CGF.FinishFunction();
5764 return Fn;
5765}
5766
5767/// Emits reduction combiner function:
5768/// \code
5769/// void @.red_comb(void* %arg0, void* %arg1) {
5770/// %lhs = bitcast void* %arg0 to <type>*
5771/// %rhs = bitcast void* %arg1 to <type>*
5772/// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs)
5773/// store <type> %2, <type>* %lhs
5774/// ret void
5775/// }
5776/// \endcode
5777static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM,
5778 SourceLocation Loc,
5779 ReductionCodeGen &RCG, unsigned N,
5780 const Expr *ReductionOp,
5781 const Expr *LHS, const Expr *RHS,
5782 const Expr *PrivateRef) {
5783 ASTContext &C = CGM.getContext();
5784 const auto *LHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: LHS)->getDecl());
5785 const auto *RHSVD = cast<VarDecl>(Val: cast<DeclRefExpr>(Val: RHS)->getDecl());
5786 FunctionArgList Args;
5787 auto *ParamInOut =
5788 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5789 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5790 auto *ParamIn =
5791 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5792 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5793 Args.emplace_back(Args&: ParamInOut);
5794 Args.emplace_back(Args&: ParamIn);
5795 const auto &FnInfo =
5796 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5797 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5798 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_comb", ""});
5799 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5800 N: Name, M: &CGM.getModule());
5801 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5802 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5803 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5804 Fn->setDoesNotRecurse();
5805 CodeGenFunction CGF(CGM);
5806 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5807 llvm::Value *Size = nullptr;
5808 // If the size of the reduction item is non-constant, load it from global
5809 // threadprivate variable.
5810 if (RCG.getSizes(N).second) {
5811 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5812 CGF, VarType: CGM.getContext().getSizeType(),
5813 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5814 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
5815 Ty: CGM.getContext().getSizeType(), Loc);
5816 }
5817 RCG.emitAggregateType(CGF, N, Size);
5818 // Remap lhs and rhs variables to the addresses of the function arguments.
5819 // %lhs = bitcast void* %arg0 to <type>*
5820 // %rhs = bitcast void* %arg1 to <type>*
5821 CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
5822 PrivateScope.addPrivate(
5823 LocalVD: LHSVD,
5824 // Pull out the pointer to the variable.
5825 Addr: CGF.EmitLoadOfPointer(
5826 Ptr: CGF.GetAddrOfLocalVar(VD: ParamInOut)
5827 .withElementType(ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5828 PtrTy: C.getPointerType(T: LHSVD->getType())->castAs<PointerType>()));
5829 PrivateScope.addPrivate(
5830 LocalVD: RHSVD,
5831 // Pull out the pointer to the variable.
5832 Addr: CGF.EmitLoadOfPointer(
5833 Ptr: CGF.GetAddrOfLocalVar(VD: ParamIn).withElementType(
5834 ElemTy: CGF.Builder.getPtrTy(AddrSpace: 0)),
5835 PtrTy: C.getPointerType(T: RHSVD->getType())->castAs<PointerType>()));
5836 PrivateScope.Privatize();
5837 // Emit the combiner body:
5838 // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs)
5839 // store <type> %2, <type>* %lhs
5840 CGM.getOpenMPRuntime().emitSingleReductionCombiner(
5841 CGF, ReductionOp, PrivateRef, LHS: cast<DeclRefExpr>(Val: LHS),
5842 RHS: cast<DeclRefExpr>(Val: RHS));
5843 CGF.FinishFunction();
5844 return Fn;
5845}
5846
5847/// Emits reduction finalizer function:
5848/// \code
5849/// void @.red_fini(void* %arg) {
5850/// %0 = bitcast void* %arg to <type>*
5851/// <destroy>(<type>* %0)
5852/// ret void
5853/// }
5854/// \endcode
5855static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM,
5856 SourceLocation Loc,
5857 ReductionCodeGen &RCG, unsigned N) {
5858 if (!RCG.needCleanups(N))
5859 return nullptr;
5860 ASTContext &C = CGM.getContext();
5861 FunctionArgList Args;
5862 auto *Param =
5863 ImplicitParamDecl::Create(C, /*DC=*/nullptr, IdLoc: Loc, /*Id=*/nullptr,
5864 T: C.VoidPtrTy, ParamKind: ImplicitParamKind::Other);
5865 Args.emplace_back(Args&: Param);
5866 const auto &FnInfo =
5867 CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: C.VoidTy, args: Args);
5868 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(Info: FnInfo);
5869 std::string Name = CGM.getOpenMPRuntime().getName(Parts: {"red_fini", ""});
5870 auto *Fn = llvm::Function::Create(Ty: FnTy, Linkage: llvm::GlobalValue::InternalLinkage,
5871 N: Name, M: &CGM.getModule());
5872 CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI: FnInfo);
5873 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
5874 Fn->addFnAttr(Kind: "sample-profile-suffix-elision-policy", Val: "selected");
5875 Fn->setDoesNotRecurse();
5876 CodeGenFunction CGF(CGM);
5877 CGF.StartFunction(GD: GlobalDecl(), RetTy: C.VoidTy, Fn, FnInfo, Args, Loc, StartLoc: Loc);
5878 Address PrivateAddr = CGF.EmitLoadOfPointer(
5879 Ptr: CGF.GetAddrOfLocalVar(VD: Param), PtrTy: C.VoidPtrTy.castAs<PointerType>());
5880 llvm::Value *Size = nullptr;
5881 // If the size of the reduction item is non-constant, load it from global
5882 // threadprivate variable.
5883 if (RCG.getSizes(N).second) {
5884 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate(
5885 CGF, VarType: CGM.getContext().getSizeType(),
5886 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
5887 Size = CGF.EmitLoadOfScalar(Addr: SizeAddr, /*Volatile=*/false,
5888 Ty: CGM.getContext().getSizeType(), Loc);
5889 }
5890 RCG.emitAggregateType(CGF, N, Size);
5891 // Emit the finalizer body:
5892 // <destroy>(<type>* %0)
5893 RCG.emitCleanups(CGF, N, PrivateAddr);
5894 CGF.FinishFunction(EndLoc: Loc);
5895 return Fn;
5896}
5897
5898llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
5899 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
5900 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
5901 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty())
5902 return nullptr;
5903
5904 // Build typedef struct:
5905 // kmp_taskred_input {
5906 // void *reduce_shar; // shared reduction item
5907 // void *reduce_orig; // original reduction item used for initialization
5908 // size_t reduce_size; // size of data item
5909 // void *reduce_init; // data initialization routine
5910 // void *reduce_fini; // data finalization routine
5911 // void *reduce_comb; // data combiner routine
5912 // kmp_task_red_flags_t flags; // flags for additional info from compiler
5913 // } kmp_taskred_input_t;
5914 ASTContext &C = CGM.getContext();
5915 RecordDecl *RD = C.buildImplicitRecord(Name: "kmp_taskred_input_t");
5916 RD->startDefinition();
5917 const FieldDecl *SharedFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5918 const FieldDecl *OrigFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5919 const FieldDecl *SizeFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.getSizeType());
5920 const FieldDecl *InitFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5921 const FieldDecl *FiniFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5922 const FieldDecl *CombFD = addFieldToRecordDecl(C, DC: RD, FieldTy: C.VoidPtrTy);
5923 const FieldDecl *FlagsFD = addFieldToRecordDecl(
5924 C, DC: RD, FieldTy: C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
5925 RD->completeDefinition();
5926 CanQualType RDType = C.getCanonicalTagType(TD: RD);
5927 unsigned Size = Data.ReductionVars.size();
5928 llvm::APInt ArraySize(/*numBits=*/64, Size);
5929 QualType ArrayRDType =
5930 C.getConstantArrayType(EltTy: RDType, ArySize: ArraySize, SizeExpr: nullptr,
5931 ASM: ArraySizeModifier::Normal, /*IndexTypeQuals=*/0);
5932 // kmp_task_red_input_t .rd_input.[Size];
5933 RawAddress TaskRedInput = CGF.CreateMemTemp(T: ArrayRDType, Name: ".rd_input.");
5934 ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionOrigs,
5935 Data.ReductionCopies, Data.ReductionOps);
5936 for (unsigned Cnt = 0; Cnt < Size; ++Cnt) {
5937 // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt];
5938 llvm::Value *Idxs[] = {llvm::ConstantInt::get(Ty: CGM.SizeTy, /*V=*/0),
5939 llvm::ConstantInt::get(Ty: CGM.SizeTy, V: Cnt)};
5940 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP(
5941 ElemTy: TaskRedInput.getElementType(), Ptr: TaskRedInput.getPointer(), IdxList: Idxs,
5942 /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc,
5943 Name: ".rd_input.gep.");
5944 LValue ElemLVal = CGF.MakeNaturalAlignRawAddrLValue(V: GEP, T: RDType);
5945 // ElemLVal.reduce_shar = &Shareds[Cnt];
5946 LValue SharedLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: SharedFD);
5947 RCG.emitSharedOrigLValue(CGF, N: Cnt);
5948 llvm::Value *Shared = RCG.getSharedLValue(N: Cnt).getPointer(CGF);
5949 CGF.EmitStoreOfScalar(value: Shared, lvalue: SharedLVal);
5950 // ElemLVal.reduce_orig = &Origs[Cnt];
5951 LValue OrigLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: OrigFD);
5952 llvm::Value *Orig = RCG.getOrigLValue(N: Cnt).getPointer(CGF);
5953 CGF.EmitStoreOfScalar(value: Orig, lvalue: OrigLVal);
5954 RCG.emitAggregateType(CGF, N: Cnt);
5955 llvm::Value *SizeValInChars;
5956 llvm::Value *SizeVal;
5957 std::tie(args&: SizeValInChars, args&: SizeVal) = RCG.getSizes(N: Cnt);
5958 // We use delayed creation/initialization for VLAs and array sections. It is
5959 // required because runtime does not provide the way to pass the sizes of
5960 // VLAs/array sections to initializer/combiner/finalizer functions. Instead
5961 // threadprivate global variables are used to store these values and use
5962 // them in the functions.
5963 bool DelayedCreation = !!SizeVal;
5964 SizeValInChars = CGF.Builder.CreateIntCast(V: SizeValInChars, DestTy: CGM.SizeTy,
5965 /*isSigned=*/false);
5966 LValue SizeLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: SizeFD);
5967 CGF.EmitStoreOfScalar(value: SizeValInChars, lvalue: SizeLVal);
5968 // ElemLVal.reduce_init = init;
5969 LValue InitLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: InitFD);
5970 llvm::Value *InitAddr = emitReduceInitFunction(CGM, Loc, RCG, N: Cnt);
5971 CGF.EmitStoreOfScalar(value: InitAddr, lvalue: InitLVal);
5972 // ElemLVal.reduce_fini = fini;
5973 LValue FiniLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: FiniFD);
5974 llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, N: Cnt);
5975 llvm::Value *FiniAddr =
5976 Fini ? Fini : llvm::ConstantPointerNull::get(T: CGM.VoidPtrTy);
5977 CGF.EmitStoreOfScalar(value: FiniAddr, lvalue: FiniLVal);
5978 // ElemLVal.reduce_comb = comb;
5979 LValue CombLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: CombFD);
5980 llvm::Value *CombAddr = emitReduceCombFunction(
5981 CGM, Loc, RCG, N: Cnt, ReductionOp: Data.ReductionOps[Cnt], LHS: LHSExprs[Cnt],
5982 RHS: RHSExprs[Cnt], PrivateRef: Data.ReductionCopies[Cnt]);
5983 CGF.EmitStoreOfScalar(value: CombAddr, lvalue: CombLVal);
5984 // ElemLVal.flags = 0;
5985 LValue FlagsLVal = CGF.EmitLValueForField(Base: ElemLVal, Field: FlagsFD);
5986 if (DelayedCreation) {
5987 CGF.EmitStoreOfScalar(
5988 value: llvm::ConstantInt::get(Ty: CGM.Int32Ty, /*V=*/1, /*isSigned=*/IsSigned: true),
5989 lvalue: FlagsLVal);
5990 } else
5991 CGF.EmitNullInitialization(DestPtr: FlagsLVal.getAddress(), Ty: FlagsLVal.getType());
5992 }
5993 if (Data.IsReductionWithTaskMod) {
5994 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
5995 // is_ws, int num, void *data);
5996 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
5997 llvm::Value *GTid = CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
5998 DestTy: CGM.IntTy, /*isSigned=*/true);
5999 llvm::Value *Args[] = {
6000 IdentTLoc, GTid,
6001 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Data.IsWorksharingReduction ? 1 : 0,
6002 /*isSigned=*/IsSigned: true),
6003 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size, /*isSigned=*/IsSigned: true),
6004 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6005 V: TaskRedInput.getPointer(), DestTy: CGM.VoidPtrTy)};
6006 return CGF.EmitRuntimeCall(
6007 callee: OMPBuilder.getOrCreateRuntimeFunction(
6008 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskred_modifier_init),
6009 args: Args);
6010 }
6011 // Build call void *__kmpc_taskred_init(int gtid, int num_data, void *data);
6012 llvm::Value *Args[] = {
6013 CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc), DestTy: CGM.IntTy,
6014 /*isSigned=*/true),
6015 llvm::ConstantInt::get(Ty: CGM.IntTy, V: Size, /*isSigned=*/IsSigned: true),
6016 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(V: TaskRedInput.getPointer(),
6017 DestTy: CGM.VoidPtrTy)};
6018 return CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6019 M&: CGM.getModule(), FnID: OMPRTL___kmpc_taskred_init),
6020 args: Args);
6021}
6022
6023void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
6024 SourceLocation Loc,
6025 bool IsWorksharingReduction) {
6026 // Build call void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int
6027 // is_ws, int num, void *data);
6028 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc);
6029 llvm::Value *GTid = CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6030 DestTy: CGM.IntTy, /*isSigned=*/true);
6031 llvm::Value *Args[] = {IdentTLoc, GTid,
6032 llvm::ConstantInt::get(Ty: CGM.IntTy,
6033 V: IsWorksharingReduction ? 1 : 0,
6034 /*isSigned=*/IsSigned: true)};
6035 (void)CGF.EmitRuntimeCall(
6036 callee: OMPBuilder.getOrCreateRuntimeFunction(
6037 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_reduction_modifier_fini),
6038 args: Args);
6039}
6040
6041void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
6042 SourceLocation Loc,
6043 ReductionCodeGen &RCG,
6044 unsigned N) {
6045 auto Sizes = RCG.getSizes(N);
6046 // Emit threadprivate global variable if the type is non-constant
6047 // (Sizes.second = nullptr).
6048 if (Sizes.second) {
6049 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(V: Sizes.second, DestTy: CGM.SizeTy,
6050 /*isSigned=*/false);
6051 Address SizeAddr = getAddrOfArtificialThreadPrivate(
6052 CGF, VarType: CGM.getContext().getSizeType(),
6053 Name: generateUniqueName(CGM, Prefix: "reduction_size", Ref: RCG.getRefExpr(N)));
6054 CGF.Builder.CreateStore(Val: SizeVal, Addr: SizeAddr, /*IsVolatile=*/false);
6055 }
6056}
6057
6058Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
6059 SourceLocation Loc,
6060 llvm::Value *ReductionsPtr,
6061 LValue SharedLVal) {
6062 // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void
6063 // *d);
6064 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(V: getThreadID(CGF, Loc),
6065 DestTy: CGM.IntTy,
6066 /*isSigned=*/true),
6067 ReductionsPtr,
6068 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6069 V: SharedLVal.getPointer(CGF), DestTy: CGM.VoidPtrTy)};
6070 return Address(
6071 CGF.EmitRuntimeCall(
6072 callee: OMPBuilder.getOrCreateRuntimeFunction(
6073 M&: CGM.getModule(), FnID: OMPRTL___kmpc_task_reduction_get_th_data),
6074 args: Args),
6075 CGF.Int8Ty, SharedLVal.getAlignment());
6076}
6077
6078void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
6079 const OMPTaskDataTy &Data) {
6080 if (!CGF.HaveInsertPoint())
6081 return;
6082
6083 if (CGF.CGM.getLangOpts().OpenMPIRBuilder && Data.Dependences.empty()) {
6084 // TODO: Need to support taskwait with dependences in the OpenMPIRBuilder.
6085 OMPBuilder.createTaskwait(Loc: CGF.Builder);
6086 } else {
6087 llvm::Value *ThreadID = getThreadID(CGF, Loc);
6088 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
6089 auto &M = CGM.getModule();
6090 Address DependenciesArray = Address::invalid();
6091 llvm::Value *NumOfElements;
6092 std::tie(args&: NumOfElements, args&: DependenciesArray) =
6093 emitDependClause(CGF, Dependencies: Data.Dependences, Loc);
6094 if (!Data.Dependences.empty()) {
6095 llvm::Value *DepWaitTaskArgs[7];
6096 DepWaitTaskArgs[0] = UpLoc;
6097 DepWaitTaskArgs[1] = ThreadID;
6098 DepWaitTaskArgs[2] = NumOfElements;
6099 DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
6100 DepWaitTaskArgs[4] = CGF.Builder.getInt32(C: 0);
6101 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6102 DepWaitTaskArgs[6] =
6103 llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: Data.HasNowaitClause);
6104
6105 CodeGenFunction::RunCleanupsScope LocalScope(CGF);
6106
6107 // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid,
6108 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
6109 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list,
6110 // kmp_int32 has_no_wait); if dependence info is specified.
6111 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6112 M, FnID: OMPRTL___kmpc_omp_taskwait_deps_51),
6113 args: DepWaitTaskArgs);
6114
6115 } else {
6116
6117 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
6118 // global_tid);
6119 llvm::Value *Args[] = {UpLoc, ThreadID};
6120 // Ignore return result until untied tasks are supported.
6121 CGF.EmitRuntimeCall(
6122 callee: OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_omp_taskwait),
6123 args: Args);
6124 }
6125 }
6126
6127 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
6128 Region->emitUntiedSwitch(CGF);
6129}
6130
6131void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF,
6132 OpenMPDirectiveKind InnerKind,
6133 const RegionCodeGenTy &CodeGen,
6134 bool HasCancel) {
6135 if (!CGF.HaveInsertPoint())
6136 return;
6137 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,
6138 InnerKind != OMPD_critical &&
6139 InnerKind != OMPD_master &&
6140 InnerKind != OMPD_masked);
6141 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr);
6142}
6143
6144namespace {
6145enum RTCancelKind {
6146 CancelNoreq = 0,
6147 CancelParallel = 1,
6148 CancelLoop = 2,
6149 CancelSections = 3,
6150 CancelTaskgroup = 4
6151};
6152} // anonymous namespace
6153
6154static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) {
6155 RTCancelKind CancelKind = CancelNoreq;
6156 if (CancelRegion == OMPD_parallel)
6157 CancelKind = CancelParallel;
6158 else if (CancelRegion == OMPD_for)
6159 CancelKind = CancelLoop;
6160 else if (CancelRegion == OMPD_sections)
6161 CancelKind = CancelSections;
6162 else {
6163 assert(CancelRegion == OMPD_taskgroup);
6164 CancelKind = CancelTaskgroup;
6165 }
6166 return CancelKind;
6167}
6168
6169void CGOpenMPRuntime::emitCancellationPointCall(
6170 CodeGenFunction &CGF, SourceLocation Loc,
6171 OpenMPDirectiveKind CancelRegion) {
6172 if (!CGF.HaveInsertPoint())
6173 return;
6174 // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32
6175 // global_tid, kmp_int32 cncl_kind);
6176 if (auto *OMPRegionInfo =
6177 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
6178 // For 'cancellation point taskgroup', the task region info may not have a
6179 // cancel. This may instead happen in another adjacent task.
6180 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {
6181 llvm::Value *Args[] = {
6182 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc),
6183 CGF.Builder.getInt32(C: getCancellationKind(CancelRegion))};
6184 // Ignore return result until untied tasks are supported.
6185 llvm::Value *Result = CGF.EmitRuntimeCall(
6186 callee: OMPBuilder.getOrCreateRuntimeFunction(
6187 M&: CGM.getModule(), FnID: OMPRTL___kmpc_cancellationpoint),
6188 args: Args);
6189 // if (__kmpc_cancellationpoint()) {
6190 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6191 // exit from construct;
6192 // }
6193 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
6194 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
6195 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
6196 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
6197 CGF.EmitBlock(BB: ExitBB);
6198 if (CancelRegion == OMPD_parallel)
6199 emitBarrierCall(CGF, Loc, Kind: OMPD_unknown, /*EmitChecks=*/false);
6200 // exit from construct;
6201 CodeGenFunction::JumpDest CancelDest =
6202 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
6203 CGF.EmitBranchThroughCleanup(Dest: CancelDest);
6204 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
6205 }
6206 }
6207}
6208
6209void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
6210 const Expr *IfCond,
6211 OpenMPDirectiveKind CancelRegion) {
6212 if (!CGF.HaveInsertPoint())
6213 return;
6214 // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid,
6215 // kmp_int32 cncl_kind);
6216 auto &M = CGM.getModule();
6217 if (auto *OMPRegionInfo =
6218 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo)) {
6219 auto &&ThenGen = [this, &M, Loc, CancelRegion,
6220 OMPRegionInfo](CodeGenFunction &CGF, PrePostActionTy &) {
6221 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime();
6222 llvm::Value *Args[] = {
6223 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
6224 CGF.Builder.getInt32(C: getCancellationKind(CancelRegion))};
6225 // Ignore return result until untied tasks are supported.
6226 llvm::Value *Result = CGF.EmitRuntimeCall(
6227 callee: OMPBuilder.getOrCreateRuntimeFunction(M, FnID: OMPRTL___kmpc_cancel), args: Args);
6228 // if (__kmpc_cancel()) {
6229 // call i32 @__kmpc_cancel_barrier( // for parallel cancellation only
6230 // exit from construct;
6231 // }
6232 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: ".cancel.exit");
6233 llvm::BasicBlock *ContBB = CGF.createBasicBlock(name: ".cancel.continue");
6234 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Result);
6235 CGF.Builder.CreateCondBr(Cond: Cmp, True: ExitBB, False: ContBB);
6236 CGF.EmitBlock(BB: ExitBB);
6237 if (CancelRegion == OMPD_parallel)
6238 RT.emitBarrierCall(CGF, Loc, Kind: OMPD_unknown, /*EmitChecks=*/false);
6239 // exit from construct;
6240 CodeGenFunction::JumpDest CancelDest =
6241 CGF.getOMPCancelDestination(Kind: OMPRegionInfo->getDirectiveKind());
6242 CGF.EmitBranchThroughCleanup(Dest: CancelDest);
6243 CGF.EmitBlock(BB: ContBB, /*IsFinished=*/true);
6244 };
6245 if (IfCond) {
6246 emitIfClause(CGF, Cond: IfCond, ThenGen,
6247 ElseGen: [](CodeGenFunction &, PrePostActionTy &) {});
6248 } else {
6249 RegionCodeGenTy ThenRCG(ThenGen);
6250 ThenRCG(CGF);
6251 }
6252 }
6253}
6254
6255namespace {
6256/// Cleanup action for uses_allocators support.
6257class OMPUsesAllocatorsActionTy final : public PrePostActionTy {
6258 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators;
6259
6260public:
6261 OMPUsesAllocatorsActionTy(
6262 ArrayRef<std::pair<const Expr *, const Expr *>> Allocators)
6263 : Allocators(Allocators) {}
6264 void Enter(CodeGenFunction &CGF) override {
6265 if (!CGF.HaveInsertPoint())
6266 return;
6267 for (const auto &AllocatorData : Allocators) {
6268 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsInit(
6269 CGF, Allocator: AllocatorData.first, AllocatorTraits: AllocatorData.second);
6270 }
6271 }
6272 void Exit(CodeGenFunction &CGF) override {
6273 if (!CGF.HaveInsertPoint())
6274 return;
6275 for (const auto &AllocatorData : Allocators) {
6276 CGF.CGM.getOpenMPRuntime().emitUsesAllocatorsFini(CGF,
6277 Allocator: AllocatorData.first);
6278 }
6279 }
6280};
6281} // namespace
6282
6283void CGOpenMPRuntime::emitTargetOutlinedFunction(
6284 const OMPExecutableDirective &D, StringRef ParentName,
6285 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6286 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6287 assert(!ParentName.empty() && "Invalid target entry parent name!");
6288 HasEmittedTargetRegion = true;
6289 SmallVector<std::pair<const Expr *, const Expr *>, 4> Allocators;
6290 for (const auto *C : D.getClausesOfKind<OMPUsesAllocatorsClause>()) {
6291 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
6292 const OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
6293 if (!D.AllocatorTraits)
6294 continue;
6295 Allocators.emplace_back(Args: D.Allocator, Args: D.AllocatorTraits);
6296 }
6297 }
6298 OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators);
6299 CodeGen.setAction(UsesAllocatorAction);
6300 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
6301 IsOffloadEntry, CodeGen);
6302}
6303
6304void CGOpenMPRuntime::emitUsesAllocatorsInit(CodeGenFunction &CGF,
6305 const Expr *Allocator,
6306 const Expr *AllocatorTraits) {
6307 llvm::Value *ThreadId = getThreadID(CGF, Loc: Allocator->getExprLoc());
6308 ThreadId = CGF.Builder.CreateIntCast(V: ThreadId, DestTy: CGF.IntTy, /*isSigned=*/true);
6309 // Use default memspace handle.
6310 llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(T: CGF.VoidPtrTy);
6311 llvm::Value *NumTraits = llvm::ConstantInt::get(
6312 Ty: CGF.IntTy, V: cast<ConstantArrayType>(
6313 Val: AllocatorTraits->getType()->getAsArrayTypeUnsafe())
6314 ->getSize()
6315 .getLimitedValue());
6316 LValue AllocatorTraitsLVal = CGF.EmitLValue(E: AllocatorTraits);
6317 Address Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
6318 Addr: AllocatorTraitsLVal.getAddress(), Ty: CGF.VoidPtrPtrTy, ElementTy: CGF.VoidPtrTy);
6319 AllocatorTraitsLVal = CGF.MakeAddrLValue(Addr, T: CGF.getContext().VoidPtrTy,
6320 BaseInfo: AllocatorTraitsLVal.getBaseInfo(),
6321 TBAAInfo: AllocatorTraitsLVal.getTBAAInfo());
6322 llvm::Value *Traits = Addr.emitRawPointer(CGF);
6323
6324 llvm::Value *AllocatorVal =
6325 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
6326 M&: CGM.getModule(), FnID: OMPRTL___kmpc_init_allocator),
6327 args: {ThreadId, MemSpaceHandle, NumTraits, Traits});
6328 // Store to allocator.
6329 CGF.EmitAutoVarAlloca(var: *cast<VarDecl>(
6330 Val: cast<DeclRefExpr>(Val: Allocator->IgnoreParenImpCasts())->getDecl()));
6331 LValue AllocatorLVal = CGF.EmitLValue(E: Allocator->IgnoreParenImpCasts());
6332 AllocatorVal =
6333 CGF.EmitScalarConversion(Src: AllocatorVal, SrcTy: CGF.getContext().VoidPtrTy,
6334 DstTy: Allocator->getType(), Loc: Allocator->getExprLoc());
6335 CGF.EmitStoreOfScalar(value: AllocatorVal, lvalue: AllocatorLVal);
6336}
6337
6338void CGOpenMPRuntime::emitUsesAllocatorsFini(CodeGenFunction &CGF,
6339 const Expr *Allocator) {
6340 llvm::Value *ThreadId = getThreadID(CGF, Loc: Allocator->getExprLoc());
6341 ThreadId = CGF.Builder.CreateIntCast(V: ThreadId, DestTy: CGF.IntTy, /*isSigned=*/true);
6342 LValue AllocatorLVal = CGF.EmitLValue(E: Allocator->IgnoreParenImpCasts());
6343 llvm::Value *AllocatorVal =
6344 CGF.EmitLoadOfScalar(lvalue: AllocatorLVal, Loc: Allocator->getExprLoc());
6345 AllocatorVal = CGF.EmitScalarConversion(Src: AllocatorVal, SrcTy: Allocator->getType(),
6346 DstTy: CGF.getContext().VoidPtrTy,
6347 Loc: Allocator->getExprLoc());
6348 (void)CGF.EmitRuntimeCall(
6349 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
6350 FnID: OMPRTL___kmpc_destroy_allocator),
6351 args: {ThreadId, AllocatorVal});
6352}
6353
6354void CGOpenMPRuntime::computeMinAndMaxThreadsAndTeams(
6355 const OMPExecutableDirective &D, CodeGenFunction &CGF,
6356 llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {
6357 assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 &&
6358 "invalid default attrs structure");
6359 int32_t &MaxTeamsVal = Attrs.MaxTeams.front();
6360 int32_t &MaxThreadsVal = Attrs.MaxThreads.front();
6361
6362 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal&: Attrs.MinTeams.front(),
6363 MaxTeamsVal);
6364 getNumThreadsExprForTargetDirective(CGF, D, UpperBound&: MaxThreadsVal,
6365 /*UpperBoundOnly=*/true);
6366
6367 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6368 for (auto *A : C->getAttrs()) {
6369 int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;
6370 int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;
6371 if (auto *Attr = dyn_cast<CUDALaunchBoundsAttr>(Val: A))
6372 CGM.handleCUDALaunchBoundsAttr(F: nullptr, A: Attr, MaxThreadsVal: &AttrMaxThreadsVal,
6373 MinBlocksVal: &AttrMinBlocksVal, MaxClusterRankVal: &AttrMaxBlocksVal);
6374 else if (auto *Attr = dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(Val: A))
6375 CGM.handleAMDGPUFlatWorkGroupSizeAttr(
6376 F: nullptr, A: Attr, /*ReqdWGS=*/nullptr, MinThreadsVal: &AttrMinThreadsVal,
6377 MaxThreadsVal: &AttrMaxThreadsVal);
6378 else
6379 continue;
6380
6381 Attrs.MinThreads.front() =
6382 std::max(a: Attrs.MinThreads.front(), b: AttrMinThreadsVal);
6383 if (AttrMaxThreadsVal > 0)
6384 MaxThreadsVal = MaxThreadsVal > 0
6385 ? std::min(a: MaxThreadsVal, b: AttrMaxThreadsVal)
6386 : AttrMaxThreadsVal;
6387 Attrs.MinTeams.front() =
6388 std::max(a: Attrs.MinTeams.front(), b: AttrMinBlocksVal);
6389 if (AttrMaxBlocksVal > 0)
6390 MaxTeamsVal = MaxTeamsVal > 0 ? std::min(a: MaxTeamsVal, b: AttrMaxBlocksVal)
6391 : AttrMaxBlocksVal;
6392 }
6393 }
6394}
6395
6396void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
6397 const OMPExecutableDirective &D, StringRef ParentName,
6398 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
6399 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
6400
6401 llvm::TargetRegionEntryInfo EntryInfo =
6402 getEntryInfoFromPresumedLoc(CGM, OMPBuilder, BeginLoc: D.getBeginLoc(), ParentName);
6403
6404 CodeGenFunction CGF(CGM, true);
6405 llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
6406 [&CGF, &D, &CodeGen, this](StringRef EntryFnName) {
6407 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: OMPD_target);
6408
6409 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
6410 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6411 if (CGM.getLangOpts().OpenMPIsTargetDevice && !isGPU())
6412 return CGF.GenerateOpenMPCapturedStmtFunctionAggregate(S: CS, D);
6413 return CGF.GenerateOpenMPCapturedStmtFunction(S: CS, D);
6414 };
6415
6416 cantFail(Err: OMPBuilder.emitTargetRegionFunction(
6417 EntryInfo, GenerateFunctionCallback&: GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,
6418 OutlinedFnID));
6419
6420 if (!OutlinedFn)
6421 return;
6422
6423 // A target body is entered once, from the kernel, and never re-entered by
6424 // the runtime, so it cannot occur in a cycle.
6425 OutlinedFn->setDoesNotRecurse();
6426
6427 CGM.getTargetCodeGenInfo().setTargetAttributes(D: nullptr, GV: OutlinedFn, M&: CGM);
6428
6429 for (auto *C : D.getClausesOfKind<OMPXAttributeClause>()) {
6430 for (auto *A : C->getAttrs()) {
6431 if (auto *Attr = dyn_cast<AMDGPUWavesPerEUAttr>(Val: A))
6432 CGM.handleAMDGPUWavesPerEUAttr(F: OutlinedFn, A: Attr);
6433 }
6434 }
6435 registerVTable(D);
6436}
6437
6438/// Checks if the expression is constant or does not have non-trivial function
6439/// calls.
6440static bool isTrivial(ASTContext &Ctx, const Expr * E) {
6441 // We can skip constant expressions.
6442 // We can skip expressions with trivial calls or simple expressions.
6443 return (E->isEvaluatable(Ctx, AllowSideEffects: Expr::SE_AllowUndefinedBehavior) ||
6444 !E->hasNonTrivialCall(Ctx)) &&
6445 !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true);
6446}
6447
6448const Stmt *CGOpenMPRuntime::getSingleCompoundChild(ASTContext &Ctx,
6449 const Stmt *Body) {
6450 const Stmt *Child = Body->IgnoreContainers();
6451 while (const auto *C = dyn_cast_or_null<CompoundStmt>(Val: Child)) {
6452 Child = nullptr;
6453 for (const Stmt *S : C->body()) {
6454 if (const auto *E = dyn_cast<Expr>(Val: S)) {
6455 if (isTrivial(Ctx, E))
6456 continue;
6457 }
6458 // Some of the statements can be ignored.
6459 if (isa<AsmStmt>(Val: S) || isa<NullStmt>(Val: S) || isa<OMPFlushDirective>(Val: S) ||
6460 isa<OMPBarrierDirective>(Val: S) || isa<OMPTaskyieldDirective>(Val: S))
6461 continue;
6462 // Analyze declarations.
6463 if (const auto *DS = dyn_cast<DeclStmt>(Val: S)) {
6464 if (llvm::all_of(Range: DS->decls(), P: [](const Decl *D) {
6465 if (isa<EmptyDecl>(Val: D) || isa<DeclContext>(Val: D) ||
6466 isa<TypeDecl>(Val: D) || isa<PragmaCommentDecl>(Val: D) ||
6467 isa<PragmaDetectMismatchDecl>(Val: D) || isa<UsingDecl>(Val: D) ||
6468 isa<UsingDirectiveDecl>(Val: D) ||
6469 isa<OMPDeclareReductionDecl>(Val: D) ||
6470 isa<OMPThreadPrivateDecl>(Val: D) || isa<OMPAllocateDecl>(Val: D))
6471 return true;
6472 const auto *VD = dyn_cast<VarDecl>(Val: D);
6473 if (!VD)
6474 return false;
6475 return VD->hasGlobalStorage() || !VD->isUsed();
6476 }))
6477 continue;
6478 }
6479 // Found multiple children - cannot get the one child only.
6480 if (Child)
6481 return nullptr;
6482 Child = S;
6483 }
6484 if (Child)
6485 Child = Child->IgnoreContainers();
6486 }
6487 return Child;
6488}
6489
6490const Expr *CGOpenMPRuntime::getNumTeamsExprForTargetDirective(
6491 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &MinTeamsVal,
6492 int32_t &MaxTeamsVal) {
6493
6494 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6495 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6496 "Expected target-based executable directive.");
6497 switch (DirectiveKind) {
6498 case OMPD_target: {
6499 const auto *CS = D.getInnermostCapturedStmt();
6500 const auto *Body =
6501 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
6502 const Stmt *ChildStmt =
6503 CGOpenMPRuntime::getSingleCompoundChild(Ctx&: CGF.getContext(), Body);
6504 if (const auto *NestedDir =
6505 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
6506 if (isOpenMPTeamsDirective(DKind: NestedDir->getDirectiveKind())) {
6507 if (NestedDir->hasClausesOfKind<OMPNumTeamsClause>()) {
6508 const Expr *NumTeams = NestedDir->getSingleClause<OMPNumTeamsClause>()
6509 ->getNumTeams()
6510 .front();
6511 if (NumTeams->isIntegerConstantExpr(Ctx: CGF.getContext()))
6512 if (auto Constant =
6513 NumTeams->getIntegerConstantExpr(Ctx: CGF.getContext()))
6514 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6515 return NumTeams;
6516 }
6517 MinTeamsVal = MaxTeamsVal = 0;
6518 return nullptr;
6519 }
6520 MinTeamsVal = MaxTeamsVal = 1;
6521 return nullptr;
6522 }
6523 // A value of -1 is used to check if we need to emit no teams region
6524 MinTeamsVal = MaxTeamsVal = -1;
6525 return nullptr;
6526 }
6527 case OMPD_target_teams_loop:
6528 case OMPD_target_teams:
6529 case OMPD_target_teams_distribute:
6530 case OMPD_target_teams_distribute_simd:
6531 case OMPD_target_teams_distribute_parallel_for:
6532 case OMPD_target_teams_distribute_parallel_for_simd: {
6533 if (D.hasClausesOfKind<OMPNumTeamsClause>()) {
6534 const Expr *NumTeams =
6535 D.getSingleClause<OMPNumTeamsClause>()->getNumTeams().front();
6536 if (NumTeams->isIntegerConstantExpr(Ctx: CGF.getContext()))
6537 if (auto Constant = NumTeams->getIntegerConstantExpr(Ctx: CGF.getContext()))
6538 MinTeamsVal = MaxTeamsVal = Constant->getExtValue();
6539 return NumTeams;
6540 }
6541 MinTeamsVal = MaxTeamsVal = 0;
6542 return nullptr;
6543 }
6544 case OMPD_target_parallel:
6545 case OMPD_target_parallel_for:
6546 case OMPD_target_parallel_for_simd:
6547 case OMPD_target_parallel_loop:
6548 case OMPD_target_simd:
6549 MinTeamsVal = MaxTeamsVal = 1;
6550 return nullptr;
6551 case OMPD_parallel:
6552 case OMPD_for:
6553 case OMPD_parallel_for:
6554 case OMPD_parallel_loop:
6555 case OMPD_parallel_master:
6556 case OMPD_parallel_sections:
6557 case OMPD_for_simd:
6558 case OMPD_parallel_for_simd:
6559 case OMPD_cancel:
6560 case OMPD_cancellation_point:
6561 case OMPD_ordered_standalone:
6562 case OMPD_ordered_blockassoc:
6563 case OMPD_threadprivate:
6564 case OMPD_allocate:
6565 case OMPD_task:
6566 case OMPD_simd:
6567 case OMPD_tile:
6568 case OMPD_unroll:
6569 case OMPD_sections:
6570 case OMPD_section:
6571 case OMPD_single:
6572 case OMPD_master:
6573 case OMPD_critical:
6574 case OMPD_taskyield:
6575 case OMPD_barrier:
6576 case OMPD_taskwait:
6577 case OMPD_taskgroup:
6578 case OMPD_atomic:
6579 case OMPD_flush:
6580 case OMPD_depobj:
6581 case OMPD_scan:
6582 case OMPD_teams:
6583 case OMPD_target_data:
6584 case OMPD_target_exit_data:
6585 case OMPD_target_enter_data:
6586 case OMPD_distribute:
6587 case OMPD_distribute_simd:
6588 case OMPD_distribute_parallel_for:
6589 case OMPD_distribute_parallel_for_simd:
6590 case OMPD_teams_distribute:
6591 case OMPD_teams_distribute_simd:
6592 case OMPD_teams_distribute_parallel_for:
6593 case OMPD_teams_distribute_parallel_for_simd:
6594 case OMPD_target_update:
6595 case OMPD_declare_simd:
6596 case OMPD_declare_variant:
6597 case OMPD_begin_declare_variant:
6598 case OMPD_end_declare_variant:
6599 case OMPD_declare_target:
6600 case OMPD_end_declare_target:
6601 case OMPD_declare_reduction:
6602 case OMPD_declare_mapper:
6603 case OMPD_taskloop:
6604 case OMPD_taskloop_simd:
6605 case OMPD_master_taskloop:
6606 case OMPD_master_taskloop_simd:
6607 case OMPD_parallel_master_taskloop:
6608 case OMPD_parallel_master_taskloop_simd:
6609 case OMPD_requires:
6610 case OMPD_metadirective:
6611 case OMPD_unknown:
6612 break;
6613 default:
6614 break;
6615 }
6616 llvm_unreachable("Unexpected directive kind.");
6617}
6618
6619llvm::Value *CGOpenMPRuntime::emitNumTeamsForTargetDirective(
6620 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6621 assert(!CGF.getLangOpts().OpenMPIsTargetDevice &&
6622 "Clauses associated with the teams directive expected to be emitted "
6623 "only for the host!");
6624 CGBuilderTy &Bld = CGF.Builder;
6625 int32_t MinNT = -1, MaxNT = -1;
6626 const Expr *NumTeams =
6627 getNumTeamsExprForTargetDirective(CGF, D, MinTeamsVal&: MinNT, MaxTeamsVal&: MaxNT);
6628 if (NumTeams != nullptr) {
6629 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6630
6631 switch (DirectiveKind) {
6632 case OMPD_target: {
6633 const auto *CS = D.getInnermostCapturedStmt();
6634 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6635 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6636 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(E: NumTeams,
6637 /*IgnoreResultAssign*/ true);
6638 return Bld.CreateIntCast(V: NumTeamsVal, DestTy: CGF.Int32Ty,
6639 /*isSigned=*/true);
6640 }
6641 case OMPD_target_teams:
6642 case OMPD_target_teams_distribute:
6643 case OMPD_target_teams_distribute_simd:
6644 case OMPD_target_teams_distribute_parallel_for:
6645 case OMPD_target_teams_distribute_parallel_for_simd: {
6646 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
6647 llvm::Value *NumTeamsVal = CGF.EmitScalarExpr(E: NumTeams,
6648 /*IgnoreResultAssign*/ true);
6649 return Bld.CreateIntCast(V: NumTeamsVal, DestTy: CGF.Int32Ty,
6650 /*isSigned=*/true);
6651 }
6652 default:
6653 break;
6654 }
6655 }
6656
6657 assert(MinNT == MaxNT && "Num threads ranges require handling here.");
6658 return llvm::ConstantInt::getSigned(Ty: CGF.Int32Ty, V: MinNT);
6659}
6660
6661/// Merge the thread count upper bound \p Val into \p UpperBound.
6662///
6663/// \p UpperBound is -1 while no thread limiting clause has been seen, 0 once
6664/// one has been seen whose value is not known at compile time, and otherwise
6665/// the smallest constant bound found so far.
6666///
6667/// Thread limiting clauses compose by taking the minimum, so a constant bound
6668/// stays valid whatever the clauses that are not compile time constants
6669/// evaluate to. That makes it correct to replace the 0 marker with \p Val, and
6670/// necessary to keep a clause from raising a smaller bound found earlier.
6671static void mergeThreadCountUpperBound(int32_t &UpperBound, int32_t Val) {
6672 UpperBound = UpperBound > 0 ? std::min(a: UpperBound, b: Val) : Val;
6673}
6674
6675/// Check for a num threads constant value (stored in \p DefaultVal), or
6676/// expression (stored in \p E). If the value is conditional (via an if-clause),
6677/// store the condition in \p CondVal. If \p E, and \p CondVal respectively, are
6678/// nullptr, no expression evaluation is perfomed.
6679static void getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS,
6680 const Expr **E, int32_t &UpperBound,
6681 bool UpperBoundOnly, llvm::Value **CondVal) {
6682 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6683 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6684 const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child);
6685 if (!Dir)
6686 return;
6687
6688 if (isOpenMPParallelDirective(DKind: Dir->getDirectiveKind())) {
6689 // Handle if clause. If if clause present, the number of threads is
6690 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6691 if (CondVal && Dir->hasClausesOfKind<OMPIfClause>()) {
6692 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6693 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6694 const OMPIfClause *IfClause = nullptr;
6695 for (const auto *C : Dir->getClausesOfKind<OMPIfClause>()) {
6696 if (C->getNameModifier() == OMPD_unknown ||
6697 C->getNameModifier() == OMPD_parallel) {
6698 IfClause = C;
6699 break;
6700 }
6701 }
6702 if (IfClause) {
6703 const Expr *CondExpr = IfClause->getCondition();
6704 bool Result;
6705 if (CondExpr->EvaluateAsBooleanCondition(Result, Ctx: CGF.getContext())) {
6706 if (!Result) {
6707 UpperBound = 1;
6708 return;
6709 }
6710 } else {
6711 CodeGenFunction::LexicalScope Scope(CGF, CondExpr->getSourceRange());
6712 if (const auto *PreInit =
6713 cast_or_null<DeclStmt>(Val: IfClause->getPreInitStmt())) {
6714 for (const auto *I : PreInit->decls()) {
6715 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6716 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6717 } else {
6718 CodeGenFunction::AutoVarEmission Emission =
6719 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6720 CGF.EmitAutoVarCleanups(emission: Emission);
6721 }
6722 }
6723 *CondVal = CGF.EvaluateExprAsBool(E: CondExpr);
6724 }
6725 }
6726 }
6727 }
6728 // Check the value of num_threads clause iff if clause was not specified
6729 // or is not evaluated to false.
6730 if (Dir->hasClausesOfKind<OMPNumThreadsClause>()) {
6731 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6732 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6733 const auto *NumThreadsClause =
6734 Dir->getSingleClause<OMPNumThreadsClause>();
6735 const Expr *NTExpr = NumThreadsClause->getNumThreads().front();
6736 if (NTExpr->isIntegerConstantExpr(Ctx: CGF.getContext()))
6737 if (auto Constant = NTExpr->getIntegerConstantExpr(Ctx: CGF.getContext()))
6738 mergeThreadCountUpperBound(
6739 UpperBound, Val: static_cast<int32_t>(Constant->getZExtValue()));
6740 // If we haven't found a upper bound, remember we saw a thread limiting
6741 // clause.
6742 if (UpperBound == -1)
6743 UpperBound = 0;
6744 if (!E)
6745 return;
6746 CodeGenFunction::LexicalScope Scope(CGF, NTExpr->getSourceRange());
6747 if (const auto *PreInit =
6748 cast_or_null<DeclStmt>(Val: NumThreadsClause->getPreInitStmt())) {
6749 for (const auto *I : PreInit->decls()) {
6750 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6751 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6752 } else {
6753 CodeGenFunction::AutoVarEmission Emission =
6754 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6755 CGF.EmitAutoVarCleanups(emission: Emission);
6756 }
6757 }
6758 }
6759 *E = NTExpr;
6760 }
6761 return;
6762 }
6763 if (isOpenMPSimdDirective(DKind: Dir->getDirectiveKind()))
6764 UpperBound = 1;
6765}
6766
6767const Expr *CGOpenMPRuntime::getNumThreadsExprForTargetDirective(
6768 CodeGenFunction &CGF, const OMPExecutableDirective &D, int32_t &UpperBound,
6769 bool UpperBoundOnly, llvm::Value **CondVal, const Expr **ThreadLimitExpr) {
6770 assert((!CGF.getLangOpts().OpenMPIsTargetDevice || UpperBoundOnly) &&
6771 "Clauses associated with the teams directive expected to be emitted "
6772 "only for the host!");
6773 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
6774 assert(isOpenMPTargetExecutionDirective(DirectiveKind) &&
6775 "Expected target-based executable directive.");
6776
6777 const Expr *NT = nullptr;
6778 const Expr **NTPtr = UpperBoundOnly ? nullptr : &NT;
6779
6780 auto CheckForConstExpr = [&](const Expr *E, const Expr **EPtr) {
6781 if (E->isIntegerConstantExpr(Ctx: CGF.getContext())) {
6782 if (auto Constant = E->getIntegerConstantExpr(Ctx: CGF.getContext()))
6783 mergeThreadCountUpperBound(
6784 UpperBound, Val: static_cast<int32_t>(Constant->getZExtValue()));
6785 }
6786 // If we haven't found a upper bound, remember we saw a thread limiting
6787 // clause.
6788 if (UpperBound == -1)
6789 UpperBound = 0;
6790 if (EPtr)
6791 *EPtr = E;
6792 };
6793
6794 auto ReturnSequential = [&]() {
6795 UpperBound = 1;
6796 return NT;
6797 };
6798
6799 switch (DirectiveKind) {
6800 case OMPD_target: {
6801 const CapturedStmt *CS = D.getInnermostCapturedStmt();
6802 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6803 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6804 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6805 // TODO: The standard is not clear how to resolve two thread limit clauses,
6806 // let's pick the teams one if it's present, otherwise the target one.
6807 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6808 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
6809 if (const auto *TLC = Dir->getSingleClause<OMPThreadLimitClause>()) {
6810 ThreadLimitClause = TLC;
6811 if (ThreadLimitExpr) {
6812 CGOpenMPInnerExprInfo CGInfo(CGF, *CS);
6813 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);
6814 CodeGenFunction::LexicalScope Scope(
6815 CGF,
6816 ThreadLimitClause->getThreadLimit().front()->getSourceRange());
6817 if (const auto *PreInit =
6818 cast_or_null<DeclStmt>(Val: ThreadLimitClause->getPreInitStmt())) {
6819 for (const auto *I : PreInit->decls()) {
6820 if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
6821 CGF.EmitVarDecl(D: cast<VarDecl>(Val: *I));
6822 } else {
6823 CodeGenFunction::AutoVarEmission Emission =
6824 CGF.EmitAutoVarAlloca(var: cast<VarDecl>(Val: *I));
6825 CGF.EmitAutoVarCleanups(emission: Emission);
6826 }
6827 }
6828 }
6829 }
6830 }
6831 }
6832 if (ThreadLimitClause)
6833 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6834 ThreadLimitExpr);
6835 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
6836 if (isOpenMPTeamsDirective(DKind: Dir->getDirectiveKind()) &&
6837 !isOpenMPDistributeDirective(DKind: Dir->getDirectiveKind())) {
6838 CS = Dir->getInnermostCapturedStmt();
6839 // Now that the 'teams' level has been peeled off, the remainder is
6840 // shaped like a 'target teams' region, so pick up the num_threads of
6841 // the directive nested in it the same way the OMPD_target_teams case
6842 // below does. Without this the upper bound of a construct written as
6843 // 'target' / 'teams' / 'distribute parallel for' would stay at the
6844 // default, while every combined spelling of the same construct honors
6845 // the clause. Only the bound is taken here: passing null for the
6846 // expression and the condition keeps this from emitting anything, so
6847 // the value the host passes to the kernel launch is left as it was.
6848 getNumThreads(CGF, CS, /*E=*/nullptr, UpperBound, UpperBoundOnly,
6849 /*CondVal=*/nullptr);
6850 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6851 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6852 Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child);
6853 }
6854 if (Dir && isOpenMPParallelDirective(DKind: Dir->getDirectiveKind())) {
6855 CS = Dir->getInnermostCapturedStmt();
6856 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6857 } else if (Dir && isOpenMPSimdDirective(DKind: Dir->getDirectiveKind()))
6858 return ReturnSequential();
6859 }
6860 return NT;
6861 }
6862 case OMPD_target_teams: {
6863 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6864 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6865 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6866 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6867 ThreadLimitExpr);
6868 }
6869 const CapturedStmt *CS = D.getInnermostCapturedStmt();
6870 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6871 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild(
6872 Ctx&: CGF.getContext(), Body: CS->getCapturedStmt());
6873 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Val: Child)) {
6874 if (Dir->getDirectiveKind() == OMPD_distribute) {
6875 CS = Dir->getInnermostCapturedStmt();
6876 getNumThreads(CGF, CS, E: NTPtr, UpperBound, UpperBoundOnly, CondVal);
6877 }
6878 }
6879 return NT;
6880 }
6881 case OMPD_target_teams_distribute:
6882 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6883 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6884 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6885 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6886 ThreadLimitExpr);
6887 }
6888 getNumThreads(CGF, CS: D.getInnermostCapturedStmt(), E: NTPtr, UpperBound,
6889 UpperBoundOnly, CondVal);
6890 return NT;
6891 case OMPD_target_teams_loop:
6892 case OMPD_target_parallel_loop:
6893 case OMPD_target_parallel:
6894 case OMPD_target_parallel_for:
6895 case OMPD_target_parallel_for_simd:
6896 case OMPD_target_teams_distribute_parallel_for:
6897 case OMPD_target_teams_distribute_parallel_for_simd: {
6898 if (CondVal && D.hasClausesOfKind<OMPIfClause>()) {
6899 const OMPIfClause *IfClause = nullptr;
6900 for (const auto *C : D.getClausesOfKind<OMPIfClause>()) {
6901 if (C->getNameModifier() == OMPD_unknown ||
6902 C->getNameModifier() == OMPD_parallel) {
6903 IfClause = C;
6904 break;
6905 }
6906 }
6907 if (IfClause) {
6908 const Expr *Cond = IfClause->getCondition();
6909 bool Result;
6910 if (Cond->EvaluateAsBooleanCondition(Result, Ctx: CGF.getContext())) {
6911 if (!Result)
6912 return ReturnSequential();
6913 } else {
6914 CodeGenFunction::RunCleanupsScope Scope(CGF);
6915 *CondVal = CGF.EvaluateExprAsBool(E: Cond);
6916 }
6917 }
6918 }
6919 if (D.hasClausesOfKind<OMPThreadLimitClause>()) {
6920 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF);
6921 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>();
6922 CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),
6923 ThreadLimitExpr);
6924 }
6925 if (D.hasClausesOfKind<OMPNumThreadsClause>()) {
6926 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
6927 const auto *NumThreadsClause = D.getSingleClause<OMPNumThreadsClause>();
6928 CheckForConstExpr(NumThreadsClause->getNumThreads().front(), nullptr);
6929 return NumThreadsClause->getNumThreads().front();
6930 }
6931 return NT;
6932 }
6933 case OMPD_target_teams_distribute_simd:
6934 case OMPD_target_simd:
6935 return ReturnSequential();
6936 default:
6937 break;
6938 }
6939 llvm_unreachable("Unsupported directive kind.");
6940}
6941
6942llvm::Value *CGOpenMPRuntime::emitNumThreadsForTargetDirective(
6943 CodeGenFunction &CGF, const OMPExecutableDirective &D) {
6944 llvm::Value *NumThreadsVal = nullptr;
6945 llvm::Value *CondVal = nullptr;
6946 llvm::Value *ThreadLimitVal = nullptr;
6947 const Expr *ThreadLimitExpr = nullptr;
6948 int32_t UpperBound = -1;
6949
6950 const Expr *NT = getNumThreadsExprForTargetDirective(
6951 CGF, D, UpperBound, /* UpperBoundOnly */ false, CondVal: &CondVal,
6952 ThreadLimitExpr: &ThreadLimitExpr);
6953
6954 // Thread limit expressions are used below, emit them.
6955 if (ThreadLimitExpr) {
6956 ThreadLimitVal =
6957 CGF.EmitScalarExpr(E: ThreadLimitExpr, /*IgnoreResultAssign=*/true);
6958 ThreadLimitVal = CGF.Builder.CreateIntCast(V: ThreadLimitVal, DestTy: CGF.Int32Ty,
6959 /*isSigned=*/false);
6960 }
6961
6962 // Generate the num teams expression.
6963 if (UpperBound == 1) {
6964 NumThreadsVal = CGF.Builder.getInt32(C: UpperBound);
6965 } else if (NT) {
6966 NumThreadsVal = CGF.EmitScalarExpr(E: NT, /*IgnoreResultAssign=*/true);
6967 NumThreadsVal = CGF.Builder.CreateIntCast(V: NumThreadsVal, DestTy: CGF.Int32Ty,
6968 /*isSigned=*/false);
6969 } else if (ThreadLimitVal) {
6970 // If we do not have a num threads value but a thread limit, replace the
6971 // former with the latter. We know handled the thread limit expression.
6972 NumThreadsVal = ThreadLimitVal;
6973 ThreadLimitVal = nullptr;
6974 } else {
6975 // Default to "0" which means runtime choice.
6976 assert(!ThreadLimitVal && "Default not applicable with thread limit value");
6977 NumThreadsVal = CGF.Builder.getInt32(C: 0);
6978 }
6979
6980 // Handle if clause. If if clause present, the number of threads is
6981 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1.
6982 if (CondVal) {
6983 CodeGenFunction::RunCleanupsScope Scope(CGF);
6984 NumThreadsVal = CGF.Builder.CreateSelect(C: CondVal, True: NumThreadsVal,
6985 False: CGF.Builder.getInt32(C: 1));
6986 }
6987
6988 // If the thread limit and num teams expression were present, take the
6989 // minimum.
6990 if (ThreadLimitVal) {
6991 NumThreadsVal = CGF.Builder.CreateSelect(
6992 C: CGF.Builder.CreateICmpULT(LHS: ThreadLimitVal, RHS: NumThreadsVal),
6993 True: ThreadLimitVal, False: NumThreadsVal);
6994 }
6995
6996 return NumThreadsVal;
6997}
6998
6999namespace {
7000LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
7001
7002// Utility to handle information from clauses associated with a given
7003// construct that use mappable expressions (e.g. 'map' clause, 'to' clause).
7004// It provides a convenient interface to obtain the information and generate
7005// code for that information.
7006class MappableExprsHandler {
7007public:
7008 /// Custom comparator for attach-pointer expressions that compares them by
7009 /// complexity (i.e. their component-depth) first, then by the order in which
7010 /// they were computed by collectAttachPtrExprInfo(), if they are semantically
7011 /// different.
7012 struct AttachPtrExprComparator {
7013 const MappableExprsHandler &Handler;
7014 // Cache of previous equality comparison results.
7015 mutable llvm::DenseMap<std::pair<const Expr *, const Expr *>, bool>
7016 CachedEqualityComparisons;
7017
7018 AttachPtrExprComparator(const MappableExprsHandler &H) : Handler(H) {}
7019 AttachPtrExprComparator() = delete;
7020
7021 // Return true iff LHS is "less than" RHS.
7022 bool operator()(const Expr *LHS, const Expr *RHS) const {
7023 if (LHS == RHS)
7024 return false;
7025
7026 // First, compare by complexity (depth)
7027 const auto ItLHS = Handler.AttachPtrComponentDepthMap.find(Val: LHS);
7028 const auto ItRHS = Handler.AttachPtrComponentDepthMap.find(Val: RHS);
7029
7030 std::optional<size_t> DepthLHS =
7031 (ItLHS != Handler.AttachPtrComponentDepthMap.end()) ? ItLHS->second
7032 : std::nullopt;
7033 std::optional<size_t> DepthRHS =
7034 (ItRHS != Handler.AttachPtrComponentDepthMap.end()) ? ItRHS->second
7035 : std::nullopt;
7036
7037 // std::nullopt (no attach pointer) has lowest complexity
7038 if (!DepthLHS.has_value() && !DepthRHS.has_value()) {
7039 // Both have same complexity, now check semantic equality
7040 if (areEqual(LHS, RHS))
7041 return false;
7042 // Different semantically, compare by computation order
7043 return wasComputedBefore(LHS, RHS);
7044 }
7045 if (!DepthLHS.has_value())
7046 return true; // LHS has lower complexity
7047 if (!DepthRHS.has_value())
7048 return false; // RHS has lower complexity
7049
7050 // Both have values, compare by depth (lower depth = lower complexity)
7051 if (DepthLHS.value() != DepthRHS.value())
7052 return DepthLHS.value() < DepthRHS.value();
7053
7054 // Same complexity, now check semantic equality
7055 if (areEqual(LHS, RHS))
7056 return false;
7057 // Different semantically, compare by computation order
7058 return wasComputedBefore(LHS, RHS);
7059 }
7060
7061 public:
7062 /// Return true if \p LHS and \p RHS are semantically equal. Uses pre-cached
7063 /// results, if available, otherwise does a recursive semantic comparison.
7064 bool areEqual(const Expr *LHS, const Expr *RHS) const {
7065 // Check cache first for faster lookup
7066 const auto CachedResultIt = CachedEqualityComparisons.find(Val: {LHS, RHS});
7067 if (CachedResultIt != CachedEqualityComparisons.end())
7068 return CachedResultIt->second;
7069
7070 bool ComparisonResult = areSemanticallyEqual(LHS, RHS);
7071
7072 // Cache the result for future lookups (both orders since semantic
7073 // equality is commutative)
7074 CachedEqualityComparisons[{LHS, RHS}] = ComparisonResult;
7075 CachedEqualityComparisons[{RHS, LHS}] = ComparisonResult;
7076 return ComparisonResult;
7077 }
7078
7079 /// Compare the two attach-ptr expressions by their computation order.
7080 /// Returns true iff LHS was computed before RHS by
7081 /// collectAttachPtrExprInfo().
7082 bool wasComputedBefore(const Expr *LHS, const Expr *RHS) const {
7083 const size_t &OrderLHS = Handler.AttachPtrComputationOrderMap.at(Val: LHS);
7084 const size_t &OrderRHS = Handler.AttachPtrComputationOrderMap.at(Val: RHS);
7085
7086 return OrderLHS < OrderRHS;
7087 }
7088
7089 private:
7090 /// Helper function to compare attach-pointer expressions semantically.
7091 /// This function handles various expression types that can be part of an
7092 /// attach-pointer.
7093 /// TODO: Not urgent, but we should ideally return true when comparing
7094 /// `p[10]`, `*(p + 10)`, `*(p + 5 + 5)`, `p[10:1]` etc.
7095 bool areSemanticallyEqual(const Expr *LHS, const Expr *RHS) const {
7096 if (LHS == RHS)
7097 return true;
7098
7099 // If only one is null, they aren't equal
7100 if (!LHS || !RHS)
7101 return false;
7102
7103 ASTContext &Ctx = Handler.CGF.getContext();
7104 // Strip away parentheses and no-op casts to get to the core expression
7105 LHS = LHS->IgnoreParenNoopCasts(Ctx);
7106 RHS = RHS->IgnoreParenNoopCasts(Ctx);
7107
7108 // Direct pointer comparison of the underlying expressions
7109 if (LHS == RHS)
7110 return true;
7111
7112 // Check if the expression classes match
7113 if (LHS->getStmtClass() != RHS->getStmtClass())
7114 return false;
7115
7116 // Handle DeclRefExpr (variable references)
7117 if (const auto *LD = dyn_cast<DeclRefExpr>(Val: LHS)) {
7118 const auto *RD = dyn_cast<DeclRefExpr>(Val: RHS);
7119 if (!RD)
7120 return false;
7121 return LD->getDecl()->getCanonicalDecl() ==
7122 RD->getDecl()->getCanonicalDecl();
7123 }
7124
7125 // Handle ArraySubscriptExpr (array indexing like a[i])
7126 if (const auto *LA = dyn_cast<ArraySubscriptExpr>(Val: LHS)) {
7127 const auto *RA = dyn_cast<ArraySubscriptExpr>(Val: RHS);
7128 if (!RA)
7129 return false;
7130 return areSemanticallyEqual(LHS: LA->getBase(), RHS: RA->getBase()) &&
7131 areSemanticallyEqual(LHS: LA->getIdx(), RHS: RA->getIdx());
7132 }
7133
7134 // Handle MemberExpr (member access like s.m or p->m)
7135 if (const auto *LM = dyn_cast<MemberExpr>(Val: LHS)) {
7136 const auto *RM = dyn_cast<MemberExpr>(Val: RHS);
7137 if (!RM)
7138 return false;
7139 if (LM->getMemberDecl()->getCanonicalDecl() !=
7140 RM->getMemberDecl()->getCanonicalDecl())
7141 return false;
7142 return areSemanticallyEqual(LHS: LM->getBase(), RHS: RM->getBase());
7143 }
7144
7145 // Handle UnaryOperator (unary operations like *p, &x, etc.)
7146 if (const auto *LU = dyn_cast<UnaryOperator>(Val: LHS)) {
7147 const auto *RU = dyn_cast<UnaryOperator>(Val: RHS);
7148 if (!RU)
7149 return false;
7150 if (LU->getOpcode() != RU->getOpcode())
7151 return false;
7152 return areSemanticallyEqual(LHS: LU->getSubExpr(), RHS: RU->getSubExpr());
7153 }
7154
7155 // Handle BinaryOperator (binary operations like p + offset)
7156 if (const auto *LB = dyn_cast<BinaryOperator>(Val: LHS)) {
7157 const auto *RB = dyn_cast<BinaryOperator>(Val: RHS);
7158 if (!RB)
7159 return false;
7160 if (LB->getOpcode() != RB->getOpcode())
7161 return false;
7162 return areSemanticallyEqual(LHS: LB->getLHS(), RHS: RB->getLHS()) &&
7163 areSemanticallyEqual(LHS: LB->getRHS(), RHS: RB->getRHS());
7164 }
7165
7166 // Handle ArraySectionExpr (array sections like a[0:1])
7167 // Attach pointers should not contain array-sections, but currently we
7168 // don't emit an error.
7169 if (const auto *LAS = dyn_cast<ArraySectionExpr>(Val: LHS)) {
7170 const auto *RAS = dyn_cast<ArraySectionExpr>(Val: RHS);
7171 if (!RAS)
7172 return false;
7173 return areSemanticallyEqual(LHS: LAS->getBase(), RHS: RAS->getBase()) &&
7174 areSemanticallyEqual(LHS: LAS->getLowerBound(),
7175 RHS: RAS->getLowerBound()) &&
7176 areSemanticallyEqual(LHS: LAS->getLength(), RHS: RAS->getLength());
7177 }
7178
7179 // Handle CastExpr (explicit casts)
7180 if (const auto *LC = dyn_cast<CastExpr>(Val: LHS)) {
7181 const auto *RC = dyn_cast<CastExpr>(Val: RHS);
7182 if (!RC)
7183 return false;
7184 if (LC->getCastKind() != RC->getCastKind())
7185 return false;
7186 return areSemanticallyEqual(LHS: LC->getSubExpr(), RHS: RC->getSubExpr());
7187 }
7188
7189 // Handle CXXThisExpr (this pointer)
7190 if (isa<CXXThisExpr>(Val: LHS) && isa<CXXThisExpr>(Val: RHS))
7191 return true;
7192
7193 // Handle IntegerLiteral (integer constants)
7194 if (const auto *LI = dyn_cast<IntegerLiteral>(Val: LHS)) {
7195 const auto *RI = dyn_cast<IntegerLiteral>(Val: RHS);
7196 if (!RI)
7197 return false;
7198 return LI->getValue() == RI->getValue();
7199 }
7200
7201 // Handle CharacterLiteral (character constants)
7202 if (const auto *LC = dyn_cast<CharacterLiteral>(Val: LHS)) {
7203 const auto *RC = dyn_cast<CharacterLiteral>(Val: RHS);
7204 if (!RC)
7205 return false;
7206 return LC->getValue() == RC->getValue();
7207 }
7208
7209 // Handle FloatingLiteral (floating point constants)
7210 if (const auto *LF = dyn_cast<FloatingLiteral>(Val: LHS)) {
7211 const auto *RF = dyn_cast<FloatingLiteral>(Val: RHS);
7212 if (!RF)
7213 return false;
7214 // Use bitwise comparison for floating point literals
7215 return LF->getValue().bitwiseIsEqual(RHS: RF->getValue());
7216 }
7217
7218 // Handle StringLiteral (string constants)
7219 if (const auto *LS = dyn_cast<StringLiteral>(Val: LHS)) {
7220 const auto *RS = dyn_cast<StringLiteral>(Val: RHS);
7221 if (!RS)
7222 return false;
7223 return LS->getString() == RS->getString();
7224 }
7225
7226 // Handle CXXNullPtrLiteralExpr (nullptr)
7227 if (isa<CXXNullPtrLiteralExpr>(Val: LHS) && isa<CXXNullPtrLiteralExpr>(Val: RHS))
7228 return true;
7229
7230 // Handle CXXBoolLiteralExpr (true/false)
7231 if (const auto *LB = dyn_cast<CXXBoolLiteralExpr>(Val: LHS)) {
7232 const auto *RB = dyn_cast<CXXBoolLiteralExpr>(Val: RHS);
7233 if (!RB)
7234 return false;
7235 return LB->getValue() == RB->getValue();
7236 }
7237
7238 // Fallback for other forms - use the existing comparison method
7239 return Expr::isSameComparisonOperand(E1: LHS, E2: RHS);
7240 }
7241 };
7242
7243 /// Get the offset of the OMP_MAP_MEMBER_OF field.
7244 static unsigned getFlagMemberOffset() {
7245 unsigned Offset = 0;
7246 for (uint64_t Remain =
7247 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
7248 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
7249 !(Remain & 1); Remain = Remain >> 1)
7250 Offset++;
7251 return Offset;
7252 }
7253
7254 /// Class that holds debugging information for a data mapping to be passed to
7255 /// the runtime library.
7256 class MappingExprInfo {
7257 /// The variable declaration used for the data mapping.
7258 const ValueDecl *MapDecl = nullptr;
7259 /// The original expression used in the map clause, or null if there is
7260 /// none.
7261 const Expr *MapExpr = nullptr;
7262
7263 public:
7264 MappingExprInfo(const ValueDecl *MapDecl, const Expr *MapExpr = nullptr)
7265 : MapDecl(MapDecl), MapExpr(MapExpr) {}
7266
7267 const ValueDecl *getMapDecl() const { return MapDecl; }
7268 const Expr *getMapExpr() const { return MapExpr; }
7269 };
7270
7271 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
7272 using MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7273 using MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;
7274 using MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;
7275 using MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;
7276 using MapNonContiguousArrayTy =
7277 llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;
7278 using MapExprsArrayTy = SmallVector<MappingExprInfo, 4>;
7279 using MapValueDeclsArrayTy = SmallVector<const ValueDecl *, 4>;
7280 using MapData =
7281 std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef,
7282 OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>,
7283 bool /*IsImplicit*/, const ValueDecl *, const Expr *>;
7284 using MapDataArrayTy = SmallVector<MapData, 4>;
7285
7286 /// This structure contains combined information generated for mappable
7287 /// clauses, including base pointers, pointers, sizes, map types, user-defined
7288 /// mappers, and non-contiguous information.
7289 struct MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {
7290 MapExprsArrayTy Exprs;
7291 MapValueDeclsArrayTy Mappers;
7292 MapValueDeclsArrayTy DevicePtrDecls;
7293
7294 /// Append arrays in \a CurInfo.
7295 void append(MapCombinedInfoTy &CurInfo) {
7296 Exprs.append(in_start: CurInfo.Exprs.begin(), in_end: CurInfo.Exprs.end());
7297 DevicePtrDecls.append(in_start: CurInfo.DevicePtrDecls.begin(),
7298 in_end: CurInfo.DevicePtrDecls.end());
7299 Mappers.append(in_start: CurInfo.Mappers.begin(), in_end: CurInfo.Mappers.end());
7300 llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);
7301 }
7302 };
7303
7304 /// Map between a struct and the its lowest & highest elements which have been
7305 /// mapped.
7306 /// [ValueDecl *] --> {LE(FieldIndex, Pointer),
7307 /// HE(FieldIndex, Pointer)}
7308 struct StructRangeInfoTy {
7309 MapCombinedInfoTy PreliminaryMapData;
7310 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = {
7311 0, Address::invalid()};
7312 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = {
7313 0, Address::invalid()};
7314 Address Base = Address::invalid();
7315 Address LB = Address::invalid();
7316 bool IsArraySection = false;
7317 bool HasCompleteRecord = false;
7318 };
7319
7320 /// A struct to store the attach pointer and pointee information, to be used
7321 /// when emitting an attach entry.
7322 struct AttachInfoTy {
7323 Address AttachPtrAddr = Address::invalid();
7324 Address AttachPteeAddr = Address::invalid();
7325 const ValueDecl *AttachPtrDecl = nullptr;
7326 const Expr *AttachMapExpr = nullptr;
7327
7328 bool isValid() const {
7329 return AttachPtrAddr.isValid() && AttachPteeAddr.isValid();
7330 }
7331 };
7332
7333 /// Check if there's any component list where the attach pointer expression
7334 /// matches the given captured variable.
7335 bool hasAttachEntryForCapturedVar(const ValueDecl *VD) const {
7336 for (const auto &AttachEntry : AttachPtrExprMap) {
7337 if (AttachEntry.second) {
7338 // Check if the attach pointer expression is a DeclRefExpr that
7339 // references the captured variable
7340 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: AttachEntry.second))
7341 if (DRE->getDecl() == VD)
7342 return true;
7343 }
7344 }
7345 return false;
7346 }
7347
7348 /// Get the previously-cached attach pointer for a component list, if-any.
7349 const Expr *getAttachPtrExpr(
7350 OMPClauseMappableExprCommon::MappableExprComponentListRef Components)
7351 const {
7352 const auto It = AttachPtrExprMap.find(Val: Components);
7353 if (It != AttachPtrExprMap.end())
7354 return It->second;
7355
7356 return nullptr;
7357 }
7358
7359private:
7360 /// Kind that defines how a device pointer has to be returned.
7361 struct MapInfo {
7362 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
7363 OpenMPMapClauseKind MapType = OMPC_MAP_unknown;
7364 ArrayRef<OpenMPMapModifierKind> MapModifiers;
7365 ArrayRef<OpenMPMotionModifierKind> MotionModifiers;
7366 bool ReturnDevicePointer = false;
7367 bool IsImplicit = false;
7368 const ValueDecl *Mapper = nullptr;
7369 const Expr *VarRef = nullptr;
7370 bool ForDeviceAddr = false;
7371 bool HasUdpFbNullify = false;
7372
7373 MapInfo() = default;
7374 MapInfo(
7375 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7376 OpenMPMapClauseKind MapType,
7377 ArrayRef<OpenMPMapModifierKind> MapModifiers,
7378 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7379 bool ReturnDevicePointer, bool IsImplicit,
7380 const ValueDecl *Mapper = nullptr, const Expr *VarRef = nullptr,
7381 bool ForDeviceAddr = false, bool HasUdpFbNullify = false)
7382 : Components(Components), MapType(MapType), MapModifiers(MapModifiers),
7383 MotionModifiers(MotionModifiers),
7384 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),
7385 Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr),
7386 HasUdpFbNullify(HasUdpFbNullify) {}
7387 };
7388
7389 /// The target directive from where the mappable clauses were extracted. It
7390 /// is either a executable directive or a user-defined mapper directive.
7391 llvm::PointerUnion<const OMPExecutableDirective *,
7392 const OMPDeclareMapperDecl *>
7393 CurDir;
7394
7395 /// Function the directive is being generated for.
7396 CodeGenFunction &CGF;
7397
7398 /// Set of all first private variables in the current directive.
7399 /// bool data is set to true if the variable is implicitly marked as
7400 /// firstprivate, false otherwise.
7401 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls;
7402
7403 /// Set of defaultmap clause kinds that use firstprivate behavior.
7404 llvm::SmallSet<OpenMPDefaultmapClauseKind, 4> DefaultmapFirstprivateKinds;
7405
7406 /// Map between device pointer declarations and their expression components.
7407 /// The key value for declarations in 'this' is null.
7408 llvm::DenseMap<
7409 const ValueDecl *,
7410 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7411 DevPointersMap;
7412
7413 /// Map between device addr declarations and their expression components.
7414 /// The key value for declarations in 'this' is null.
7415 llvm::DenseMap<
7416 const ValueDecl *,
7417 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>>
7418 HasDevAddrsMap;
7419
7420 /// Map between lambda declarations and their map type.
7421 llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;
7422
7423 /// Map from component lists to their attach pointer expressions.
7424 llvm::DenseMap<OMPClauseMappableExprCommon::MappableExprComponentListRef,
7425 const Expr *>
7426 AttachPtrExprMap;
7427
7428 /// Map from attach pointer expressions to their component depth.
7429 /// nullptr key has std::nullopt depth. This can be used to order attach-ptr
7430 /// expressions with increasing/decreasing depth.
7431 /// The component-depth of `nullptr` (i.e. no attach-ptr) is `std::nullopt`.
7432 /// TODO: Not urgent, but we should ideally use the number of pointer
7433 /// dereferences in an expr as an indicator of its complexity, instead of the
7434 /// component-depth. That would be needed for us to treat `p[1]`, `*(p + 10)`,
7435 /// `*(p + 5 + 5)` together.
7436 llvm::DenseMap<const Expr *, std::optional<size_t>>
7437 AttachPtrComponentDepthMap = {{nullptr, std::nullopt}};
7438
7439 /// Map from attach pointer expressions to the order they were computed in, in
7440 /// collectAttachPtrExprInfo().
7441 llvm::DenseMap<const Expr *, size_t> AttachPtrComputationOrderMap = {
7442 {nullptr, 0}};
7443
7444 /// An instance of attach-ptr-expr comparator that can be used throughout the
7445 /// lifetime of this handler.
7446 AttachPtrExprComparator AttachPtrComparator;
7447
7448 llvm::Value *getExprTypeSize(const Expr *E) const {
7449 QualType ExprTy = E->getType().getCanonicalType();
7450
7451 // Calculate the size for array shaping expression.
7452 if (const auto *OAE = dyn_cast<OMPArrayShapingExpr>(Val: E)) {
7453 llvm::Value *Size =
7454 CGF.getTypeSize(Ty: OAE->getBase()->getType()->getPointeeType());
7455 for (const Expr *SE : OAE->getDimensions()) {
7456 llvm::Value *Sz = CGF.EmitScalarExpr(E: SE);
7457 Sz = CGF.EmitScalarConversion(Src: Sz, SrcTy: SE->getType(),
7458 DstTy: CGF.getContext().getSizeType(),
7459 Loc: SE->getExprLoc());
7460 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: Sz);
7461 }
7462 return Size;
7463 }
7464
7465 // Reference types are ignored for mapping purposes.
7466 if (const auto *RefTy = ExprTy->getAs<ReferenceType>())
7467 ExprTy = RefTy->getPointeeType().getCanonicalType();
7468
7469 // Given that an array section is considered a built-in type, we need to
7470 // do the calculation based on the length of the section instead of relying
7471 // on CGF.getTypeSize(E->getType()).
7472 if (const auto *OAE = dyn_cast<ArraySectionExpr>(Val: E)) {
7473 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(
7474 Base: OAE->getBase()->IgnoreParenImpCasts())
7475 .getCanonicalType();
7476
7477 // If there is no length associated with the expression and lower bound is
7478 // not specified too, that means we are using the whole length of the
7479 // base.
7480 if (!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7481 !OAE->getLowerBound())
7482 return CGF.getTypeSize(Ty: BaseTy);
7483
7484 llvm::Value *ElemSize;
7485 if (const auto *PTy = BaseTy->getAs<PointerType>()) {
7486 ElemSize = CGF.getTypeSize(Ty: PTy->getPointeeType().getCanonicalType());
7487 } else {
7488 const auto *ATy = cast<ArrayType>(Val: BaseTy.getTypePtr());
7489 assert(ATy && "Expecting array type if not a pointer type.");
7490 ElemSize = CGF.getTypeSize(Ty: ATy->getElementType().getCanonicalType());
7491 }
7492
7493 // If we don't have a length at this point, that is because we have an
7494 // array section with a single element.
7495 if (!OAE->getLength() && OAE->getColonLocFirst().isInvalid())
7496 return ElemSize;
7497
7498 if (const Expr *LenExpr = OAE->getLength()) {
7499 llvm::Value *LengthVal = CGF.EmitScalarExpr(E: LenExpr);
7500 LengthVal = CGF.EmitScalarConversion(Src: LengthVal, SrcTy: LenExpr->getType(),
7501 DstTy: CGF.getContext().getSizeType(),
7502 Loc: LenExpr->getExprLoc());
7503 return CGF.Builder.CreateNUWMul(LHS: LengthVal, RHS: ElemSize);
7504 }
7505 assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&
7506 OAE->getLowerBound() && "expected array_section[lb:].");
7507 // Size = sizetype - lb * elemtype;
7508 llvm::Value *LengthVal = CGF.getTypeSize(Ty: BaseTy);
7509 llvm::Value *LBVal = CGF.EmitScalarExpr(E: OAE->getLowerBound());
7510 LBVal = CGF.EmitScalarConversion(Src: LBVal, SrcTy: OAE->getLowerBound()->getType(),
7511 DstTy: CGF.getContext().getSizeType(),
7512 Loc: OAE->getLowerBound()->getExprLoc());
7513 LBVal = CGF.Builder.CreateNUWMul(LHS: LBVal, RHS: ElemSize);
7514 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LHS: LengthVal, RHS: LBVal);
7515 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LHS: LengthVal, RHS: LBVal);
7516 LengthVal = CGF.Builder.CreateSelect(
7517 C: Cmp, True: TrueVal, False: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0));
7518 return LengthVal;
7519 }
7520 return CGF.getTypeSize(Ty: ExprTy);
7521 }
7522
7523 /// Return the corresponding bits for a given map clause modifier. Add
7524 /// a flag marking the map as a pointer if requested. Add a flag marking the
7525 /// map as the first one of a series of maps that relate to the same map
7526 /// expression.
7527 OpenMPOffloadMappingFlags getMapTypeBits(
7528 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7529 ArrayRef<OpenMPMotionModifierKind> MotionModifiers, bool IsImplicit,
7530 bool AddPtrFlag, bool AddIsTargetParamFlag, bool IsNonContiguous) const {
7531 OpenMPOffloadMappingFlags Bits =
7532 IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT
7533 : OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7534 switch (MapType) {
7535 case OMPC_MAP_alloc:
7536 case OMPC_MAP_release:
7537 // alloc and release is the default behavior in the runtime library, i.e.
7538 // if we don't pass any bits alloc/release that is what the runtime is
7539 // going to do. Therefore, we don't need to signal anything for these two
7540 // type modifiers.
7541 break;
7542 case OMPC_MAP_to:
7543 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;
7544 break;
7545 case OMPC_MAP_from:
7546 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7547 break;
7548 case OMPC_MAP_tofrom:
7549 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |
7550 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
7551 break;
7552 case OMPC_MAP_delete:
7553 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;
7554 break;
7555 case OMPC_MAP_unknown:
7556 llvm_unreachable("Unexpected map type!");
7557 }
7558 if (AddPtrFlag)
7559 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
7560 if (AddIsTargetParamFlag)
7561 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
7562 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_always))
7563 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;
7564 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_close))
7565 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;
7566 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_present) ||
7567 llvm::is_contained(Range&: MotionModifiers, Element: OMPC_MOTION_MODIFIER_present))
7568 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
7569 if (llvm::is_contained(Range&: MapModifiers, Element: OMPC_MAP_MODIFIER_ompx_hold))
7570 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
7571 if (IsNonContiguous)
7572 Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;
7573 return Bits;
7574 }
7575
7576 /// Return true if the provided expression is a final array section. A
7577 /// final array section, is one whose length can't be proved to be one.
7578 bool isFinalArraySectionExpression(const Expr *E) const {
7579 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E);
7580
7581 // It is not an array section and therefore not a unity-size one.
7582 if (!OASE)
7583 return false;
7584
7585 // An array section with no colon always refer to a single element.
7586 if (OASE->getColonLocFirst().isInvalid())
7587 return false;
7588
7589 const Expr *Length = OASE->getLength();
7590
7591 // If we don't have a length we have to check if the array has size 1
7592 // for this dimension. Also, we should always expect a length if the
7593 // base type is pointer.
7594 if (!Length) {
7595 QualType BaseQTy = ArraySectionExpr::getBaseOriginalType(
7596 Base: OASE->getBase()->IgnoreParenImpCasts())
7597 .getCanonicalType();
7598 if (const auto *ATy = dyn_cast<ConstantArrayType>(Val: BaseQTy.getTypePtr()))
7599 return ATy->getSExtSize() != 1;
7600 // If we don't have a constant dimension length, we have to consider
7601 // the current section as having any size, so it is not necessarily
7602 // unitary. If it happen to be unity size, that's user fault.
7603 return true;
7604 }
7605
7606 // Check if the length evaluates to 1.
7607 Expr::EvalResult Result;
7608 if (!Length->EvaluateAsInt(Result, Ctx: CGF.getContext()))
7609 return true; // Can have more that size 1.
7610
7611 llvm::APSInt ConstLength = Result.Val.getInt();
7612 return ConstLength.getSExtValue() != 1;
7613 }
7614
7615 /// Emit an attach entry into \p CombinedInfo, using the information from \p
7616 /// AttachInfo. For example, for a map of form `int *p; ... map(p[1:10])`,
7617 /// an attach entry has the following form:
7618 /// &p, &p[1], sizeof(void*), ATTACH
7619 void emitAttachEntry(CodeGenFunction &CGF, MapCombinedInfoTy &CombinedInfo,
7620 const AttachInfoTy &AttachInfo) const {
7621 assert(AttachInfo.isValid() &&
7622 "Expected valid attach pointer/pointee information!");
7623
7624 // Size is the size of the pointer itself - use pointer size, not BaseDecl
7625 // size
7626 llvm::Value *PointerSize = CGF.Builder.CreateIntCast(
7627 V: llvm::ConstantInt::get(
7628 Ty: CGF.CGM.SizeTy, V: CGF.getContext()
7629 .getTypeSizeInChars(T: CGF.getContext().VoidPtrTy)
7630 .getQuantity()),
7631 DestTy: CGF.Int64Ty, /*isSigned=*/true);
7632
7633 CombinedInfo.Exprs.emplace_back(Args: AttachInfo.AttachPtrDecl,
7634 Args: AttachInfo.AttachMapExpr);
7635 CombinedInfo.BasePointers.push_back(
7636 Elt: AttachInfo.AttachPtrAddr.emitRawPointer(CGF));
7637 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
7638 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
7639 CombinedInfo.Pointers.push_back(
7640 Elt: AttachInfo.AttachPteeAddr.emitRawPointer(CGF));
7641 CombinedInfo.Sizes.push_back(Elt: PointerSize);
7642 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
7643 // ATTACH entries themselves don't "have" a base attach-ptr.
7644 CombinedInfo.HasAttachPtr.push_back(Elt: false);
7645 CombinedInfo.Mappers.push_back(Elt: nullptr);
7646 CombinedInfo.NonContigInfo.Dims.push_back(Elt: 1);
7647 }
7648
7649 /// A helper class to copy structures with overlapped elements, i.e. those
7650 /// which have mappings of both "s" and "s.mem". Consecutive elements that
7651 /// are not explicitly copied have mapping nodes synthesized for them,
7652 /// taking care to avoid generating zero-sized copies.
7653 class CopyOverlappedEntryGaps {
7654 CodeGenFunction &CGF;
7655 MapCombinedInfoTy &CombinedInfo;
7656 OpenMPOffloadMappingFlags Flags = OpenMPOffloadMappingFlags::OMP_MAP_NONE;
7657 const ValueDecl *MapDecl = nullptr;
7658 const Expr *MapExpr = nullptr;
7659 Address BP = Address::invalid();
7660 bool IsNonContiguous = false;
7661 uint64_t DimSize = 0;
7662 // These elements track the position as the struct is iterated over
7663 // (in order of increasing element address).
7664 const RecordDecl *LastParent = nullptr;
7665 uint64_t Cursor = 0;
7666 unsigned LastIndex = -1u;
7667 Address LB = Address::invalid();
7668
7669 public:
7670 CopyOverlappedEntryGaps(CodeGenFunction &CGF,
7671 MapCombinedInfoTy &CombinedInfo,
7672 OpenMPOffloadMappingFlags Flags,
7673 const ValueDecl *MapDecl, const Expr *MapExpr,
7674 Address BP, Address LB, bool IsNonContiguous,
7675 uint64_t DimSize)
7676 : CGF(CGF), CombinedInfo(CombinedInfo), Flags(Flags), MapDecl(MapDecl),
7677 MapExpr(MapExpr), BP(BP), IsNonContiguous(IsNonContiguous),
7678 DimSize(DimSize), LB(LB) {}
7679
7680 void processField(
7681 const OMPClauseMappableExprCommon::MappableComponent &MC,
7682 const FieldDecl *FD,
7683 llvm::function_ref<LValue(CodeGenFunction &, const MemberExpr *)>
7684 EmitMemberExprBase) {
7685 const RecordDecl *RD = FD->getParent();
7686 const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(D: RD);
7687 uint64_t FieldOffset = RL.getFieldOffset(FieldNo: FD->getFieldIndex());
7688 uint64_t FieldSize =
7689 CGF.getContext().getTypeSize(T: FD->getType().getCanonicalType());
7690 Address ComponentLB = Address::invalid();
7691
7692 if (FD->getType()->isLValueReferenceType()) {
7693 const auto *ME = cast<MemberExpr>(Val: MC.getAssociatedExpression());
7694 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
7695 ComponentLB =
7696 CGF.EmitLValueForFieldInitialization(Base: BaseLVal, Field: FD).getAddress();
7697 } else {
7698 ComponentLB =
7699 CGF.EmitOMPSharedLValue(E: MC.getAssociatedExpression()).getAddress();
7700 }
7701
7702 if (!LastParent)
7703 LastParent = RD;
7704 if (FD->getParent() == LastParent) {
7705 if (FD->getFieldIndex() != LastIndex + 1)
7706 copyUntilField(FD, ComponentLB);
7707 } else {
7708 LastParent = FD->getParent();
7709 if (((int64_t)FieldOffset - (int64_t)Cursor) > 0)
7710 copyUntilField(FD, ComponentLB);
7711 }
7712 Cursor = FieldOffset + FieldSize;
7713 LastIndex = FD->getFieldIndex();
7714 LB = CGF.Builder.CreateConstGEP(Addr: ComponentLB, Index: 1);
7715 }
7716
7717 void copyUntilField(const FieldDecl *FD, Address ComponentLB) {
7718 llvm::Value *ComponentLBPtr = ComponentLB.emitRawPointer(CGF);
7719 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7720 llvm::Value *Size = CGF.Builder.CreatePtrDiff(LHS: ComponentLBPtr, RHS: LBPtr);
7721 copySizedChunk(Base: LBPtr, Size);
7722 }
7723
7724 void copyUntilEnd(Address HB) {
7725 if (LastParent) {
7726 const ASTRecordLayout &RL =
7727 CGF.getContext().getASTRecordLayout(D: LastParent);
7728 if ((uint64_t)CGF.getContext().toBits(CharSize: RL.getSize()) <= Cursor)
7729 return;
7730 }
7731 llvm::Value *LBPtr = LB.emitRawPointer(CGF);
7732 llvm::Value *Size = CGF.Builder.CreatePtrDiff(
7733 LHS: CGF.Builder.CreateConstGEP(Addr: HB, Index: 1).emitRawPointer(CGF), RHS: LBPtr);
7734 copySizedChunk(Base: LBPtr, Size);
7735 }
7736
7737 void copySizedChunk(llvm::Value *Base, llvm::Value *Size) {
7738 CombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
7739 CombinedInfo.BasePointers.push_back(Elt: BP.emitRawPointer(CGF));
7740 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
7741 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
7742 CombinedInfo.Pointers.push_back(Elt: Base);
7743 CombinedInfo.Sizes.push_back(
7744 Elt: CGF.Builder.CreateIntCast(V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/false));
7745 CombinedInfo.Types.push_back(Elt: Flags);
7746 CombinedInfo.HasAttachPtr.push_back(Elt: false);
7747 CombinedInfo.Mappers.push_back(Elt: nullptr);
7748 CombinedInfo.NonContigInfo.Dims.push_back(Elt: IsNonContiguous ? DimSize : 1);
7749 }
7750 };
7751
7752 /// Generate the base pointers, section pointers, sizes, map type bits, and
7753 /// user-defined mappers (all included in \a CombinedInfo) for the provided
7754 /// map type, map or motion modifiers, and expression components.
7755 /// \a IsFirstComponent should be set to true if the provided set of
7756 /// components is the first associated with a capture.
7757 void generateInfoForComponentList(
7758 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers,
7759 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
7760 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
7761 MapCombinedInfoTy &CombinedInfo,
7762 MapCombinedInfoTy &StructBaseCombinedInfo,
7763 StructRangeInfoTy &PartialStruct, AttachInfoTy &AttachInfo,
7764 bool IsFirstComponentList, bool IsImplicit,
7765 bool GenerateAllInfoForClauses, const ValueDecl *Mapper = nullptr,
7766 bool ForDeviceAddr = false, const ValueDecl *BaseDecl = nullptr,
7767 const Expr *MapExpr = nullptr,
7768 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
7769 OverlappedElements = {}) const {
7770
7771 // The following summarizes what has to be generated for each map and the
7772 // types below. The generated information is expressed in this order:
7773 // base pointer, section pointer, size, flags
7774 // (to add to the ones that come from the map type and modifier).
7775 // Entries annotated with (+) are only generated for "target" constructs,
7776 // and only if the variable at the beginning of the expression is used in
7777 // the region.
7778 //
7779 // double d;
7780 // int i[100];
7781 // float *p;
7782 // int **a = &i;
7783 //
7784 // struct S1 {
7785 // int i;
7786 // float f[50];
7787 // }
7788 // struct S2 {
7789 // int i;
7790 // float f[50];
7791 // S1 s;
7792 // double *p;
7793 // double *&pref;
7794 // struct S2 *ps;
7795 // int &ref;
7796 // }
7797 // S2 s;
7798 // S2 *ps;
7799 //
7800 // map(d)
7801 // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM
7802 //
7803 // map(i)
7804 // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM
7805 //
7806 // map(i[1:23])
7807 // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM
7808 //
7809 // map(p)
7810 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
7811 //
7812 // map(p[1:24])
7813 // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM // map pointee
7814 // &p, &p[1], sizeof(void*), ATTACH // attach pointer/pointee, if both
7815 // // are present, and either is new
7816 //
7817 // map(([22])p)
7818 // p, p, 22*sizeof(float), TARGET_PARAM | TO | FROM
7819 // &p, p, sizeof(void*), ATTACH
7820 //
7821 // map((*a)[0:3])
7822 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
7823 // (*a)[0], &(*a)[0], 3 * sizeof(int), TO | FROM
7824 // &(*a), &(*a)[0], sizeof(void*), ATTACH
7825 // (+) Only on target, if a is used in the region
7826 // Note: Since the attach base-pointer is `*a`, which is not a scalar
7827 // variable, it doesn't determine the clause on `a`. `a` is mapped using
7828 // a zero-length-array-section map by generateDefaultMapInfo, if it is
7829 // referenced in the target region, because it is a pointer.
7830 //
7831 // map(**a)
7832 // a, a, 0, TARGET_PARAM | IMPLICIT // (+)
7833 // &(*a)[0], &(*a)[0], sizeof(int), TO | FROM
7834 // &(*a), &(*a)[0], sizeof(void*), ATTACH
7835 // (+) Only on target, if a is used in the region
7836 //
7837 // map(s)
7838 // FIXME: This needs to also imply map(ref_ptr_ptee: s.ref), since the
7839 // effect is supposed to be same as if the user had a map for every element
7840 // of the struct. We currently do a shallow-map of s.
7841 // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM
7842 //
7843 // map(s.i)
7844 // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM
7845 //
7846 // map(s.s.f)
7847 // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
7848 //
7849 // map(s.p)
7850 // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM
7851 //
7852 // map(to: s.p[:22])
7853 // &s, &(s.p), sizeof(double*), TARGET_PARAM | IMPLICIT // (+)
7854 // &(s.p[0]), &(s.p[0]), 22 * sizeof(double*), TO | FROM
7855 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
7856 //
7857 // map(to: s.ref)
7858 // &s, &(ptr(s.ref)), sizeof(int*), TARGET_PARAM (*)
7859 // &s, &(ptee(s.ref)), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | TO (***)
7860 // (*) alloc space for struct members, only this is a target parameter.
7861 // (**) map the pointer (nothing to be mapped in this example) (the compiler
7862 // optimizes this entry out, same in the examples below)
7863 // (***) map the pointee (map: to)
7864 // Note: ptr(s.ref) represents the referring pointer of s.ref
7865 // ptee(s.ref) represents the referenced pointee of s.ref
7866 //
7867 // map(to: s.pref)
7868 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM
7869 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO
7870 //
7871 // map(to: s.pref[:22])
7872 // &s, &(ptr(s.pref)), sizeof(double**), TARGET_PARAM | IMPLICIT // (+)
7873 // &s, &(ptee(s.pref)), sizeof(double*), MEMBER_OF(1) | PTR_AND_OBJ | TO |
7874 // FROM | IMPLICIT // (+)
7875 // &(ptee(s.pref)[0]), &(ptee(s.pref)[0]), 22 * sizeof(double), TO
7876 // &(ptee(s.pref)), &(ptee(s.pref)[0]), sizeof(void*), ATTACH
7877 //
7878 // map(s.ps)
7879 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM
7880 //
7881 // map(from: s.ps->s.i)
7882 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7883 // &(s.ps[0]), &(s.ps->s.i), sizeof(int), FROM
7884 // &(s.ps), &(s.ps->s.i), sizeof(void*), ATTACH
7885 //
7886 // map(to: s.ps->ps)
7887 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7888 // &(s.ps[0]), &(s.ps->ps), sizeof(S2*), TO
7889 // &(s.ps), &(s.ps->ps), sizeof(void*), ATTACH
7890 //
7891 // map(s.ps->ps->ps)
7892 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7893 // &(s.ps->ps[0]), &(s.ps->ps->ps), sizeof(S2*), TO
7894 // &(s.ps->ps), &(s.ps->ps->ps), sizeof(void*), ATTACH
7895 //
7896 // map(to: s.ps->ps->s.f[:22])
7897 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM | IMPLICIT // (+)
7898 // &(s.ps->ps[0]), &(s.ps->ps->s.f[0]), 22*sizeof(float), TO
7899 // &(s.ps->ps), &(s.ps->ps->s.f[0]), sizeof(void*), ATTACH
7900 //
7901 // map(ps)
7902 // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM
7903 //
7904 // map(ps->i)
7905 // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM
7906 // &ps, &(ps->i), sizeof(void*), ATTACH
7907 //
7908 // map(ps->s.f)
7909 // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM
7910 // &ps, &(ps->s.f[0]), sizeof(ps), ATTACH
7911 //
7912 // map(from: ps->p)
7913 // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM
7914 // &ps, &(ps->p), sizeof(ps), ATTACH
7915 //
7916 // map(to: ps->p[:22])
7917 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
7918 // &(ps->p[0]), &(ps->p[0]), 22*sizeof(double), TO
7919 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
7920 //
7921 // map(ps->ps)
7922 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM
7923 // &ps, &(ps->ps), sizeof(ps), ATTACH
7924 //
7925 // map(from: ps->ps->s.i)
7926 // ps, &(ps[0]), 0, TARGET_PARAM | IMPLICIT // (+)
7927 // &(ps->ps[0]), &(ps->ps->s.i), sizeof(int), FROM
7928 // &(ps->ps), &(ps->ps->s.i), sizeof(void*), ATTACH
7929 //
7930 // map(from: ps->ps->ps)
7931 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7932 // &(ps->ps[0]), &(ps->ps->ps), sizeof(S2*), FROM
7933 // &(ps->ps), &(ps->ps->ps), sizeof(void*), ATTACH
7934 //
7935 // map(ps->ps->ps->ps)
7936 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7937 // &(ps->ps->ps[0]), &(ps->ps->ps->ps), sizeof(S2*), FROM
7938 // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(void*), ATTACH
7939 //
7940 // map(to: ps->ps->ps->s.f[:22])
7941 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7942 // &(ps->ps->ps[0]), &(ps->ps->ps->s.f[0]), 22*sizeof(float), TO
7943 // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), sizeof(void*), ATTACH
7944 //
7945 // map(to: s.f[:22]) map(from: s.p[:33])
7946 // On target, and if s is used in the region:
7947 //
7948 // &s, &(s.f[0]), 50*sizeof(float) +
7949 // sizeof(struct S1) +
7950 // sizeof(double*) (**), TARGET_PARAM
7951 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO
7952 // &s, &(s.p), sizeof(double*), MEMBER_OF(1) | TO |
7953 // FROM | IMPLICIT
7954 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
7955 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
7956 // (**) allocate contiguous space needed to fit all mapped members even if
7957 // we allocate space for members not mapped (in this example,
7958 // s.f[22..49] and s.s are not mapped, yet we must allocate space for
7959 // them as well because they fall between &s.f[0] and &s.p)
7960 //
7961 // On other constructs, and, if s is not used in the region, on target:
7962 // &s, &(s.f[0]), 22*sizeof(float), TO
7963 // &(s.p[0]), &(s.p[0]), 33*sizeof(double), FROM
7964 // &(s.p), &(s.p[0]), sizeof(void*), ATTACH
7965 //
7966 // map(from: s.f[:22]) map(to: ps->p[:33])
7967 // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM
7968 // &ps[0], &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7969 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
7970 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
7971 //
7972 // map(from: s.f[:22], s.s) map(to: ps->p[:33])
7973 // &s, &(s.f[0]), 50*sizeof(float) +
7974 // sizeof(struct S1), TARGET_PARAM
7975 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM
7976 // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM
7977 // ps, &ps[0], 0, TARGET_PARAM | IMPLICIT // (+)
7978 // &(ps->p[0]), &(ps->p[0]), 33*sizeof(double), TO
7979 // &(ps->p), &(ps->p[0]), sizeof(void*), ATTACH
7980 //
7981 // map(p[:100], p)
7982 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM
7983 // p, &p[0], 100*sizeof(float), TO | FROM
7984 // &p, &p[0], sizeof(float*), ATTACH
7985
7986 // Track if the map information being generated is the first for a capture.
7987 bool IsCaptureFirstInfo = IsFirstComponentList;
7988 // When the variable is on a declare target link or in a to clause with
7989 // unified memory, a reference is needed to hold the host/device address
7990 // of the variable.
7991 bool RequiresReference = false;
7992
7993 // Scan the components from the base to the complete expression.
7994 auto CI = Components.rbegin();
7995 auto CE = Components.rend();
7996 auto I = CI;
7997
7998 // Track if the map information being generated is the first for a list of
7999 // components.
8000 bool IsExpressionFirstInfo = true;
8001 bool FirstPointerInComplexData = false;
8002 Address BP = Address::invalid();
8003 Address FinalLowestElem = Address::invalid();
8004 const Expr *AssocExpr = I->getAssociatedExpression();
8005 const auto *AE = dyn_cast<ArraySubscriptExpr>(Val: AssocExpr);
8006 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8007 const auto *OAShE = dyn_cast<OMPArrayShapingExpr>(Val: AssocExpr);
8008
8009 // Get the pointer-attachment base-pointer for the given list, if any.
8010 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
8011 auto [AttachPtrAddr, AttachPteeBaseAddr] =
8012 getAttachPtrAddrAndPteeBaseAddr(AttachPtrExpr, CGF);
8013
8014 bool HasAttachPtr = AttachPtrExpr != nullptr;
8015 bool FirstComponentIsForAttachPtr = AssocExpr == AttachPtrExpr;
8016 bool SeenAttachPtr = FirstComponentIsForAttachPtr;
8017
8018 if (FirstComponentIsForAttachPtr) {
8019 // No need to process AttachPtr here. It will be processed at the end
8020 // after we have computed the pointee's address.
8021 ++I;
8022 } else if (isa<MemberExpr>(Val: AssocExpr)) {
8023 // The base is the 'this' pointer. The content of the pointer is going
8024 // to be the base of the field being mapped.
8025 BP = CGF.LoadCXXThisAddress();
8026 } else if ((AE && isa<CXXThisExpr>(Val: AE->getBase()->IgnoreParenImpCasts())) ||
8027 (OASE &&
8028 isa<CXXThisExpr>(Val: OASE->getBase()->IgnoreParenImpCasts()))) {
8029 BP = CGF.EmitOMPSharedLValue(E: AssocExpr).getAddress();
8030 } else if (OAShE &&
8031 isa<CXXThisExpr>(Val: OAShE->getBase()->IgnoreParenCasts())) {
8032 BP = Address(
8033 CGF.EmitScalarExpr(E: OAShE->getBase()),
8034 CGF.ConvertTypeForMem(T: OAShE->getBase()->getType()->getPointeeType()),
8035 CGF.getContext().getTypeAlignInChars(T: OAShE->getBase()->getType()));
8036 } else {
8037 // The base is the reference to the variable.
8038 // BP = &Var.
8039 BP = CGF.EmitOMPSharedLValue(E: AssocExpr).getAddress();
8040 if (const auto *VD =
8041 dyn_cast_or_null<VarDecl>(Val: I->getAssociatedDeclaration())) {
8042 if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
8043 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
8044 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
8045 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
8046 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
8047 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) {
8048 RequiresReference = true;
8049 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
8050 }
8051 }
8052 }
8053
8054 // If the variable is a pointer and is being dereferenced (i.e. is not
8055 // the last component), the base has to be the pointer itself, not its
8056 // reference. References are ignored for mapping purposes.
8057 QualType Ty =
8058 I->getAssociatedDeclaration()->getType().getNonReferenceType();
8059 if (Ty->isAnyPointerType() && std::next(x: I) != CE) {
8060 // No need to generate individual map information for the pointer, it
8061 // can be associated with the combined storage if shared memory mode is
8062 // active or the base declaration is not global variable.
8063 const auto *VD = dyn_cast<VarDecl>(Val: I->getAssociatedDeclaration());
8064 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
8065 !VD || VD->hasLocalStorage() || HasAttachPtr)
8066 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8067 else
8068 FirstPointerInComplexData = true;
8069 ++I;
8070 }
8071 }
8072
8073 // Track whether a component of the list should be marked as MEMBER_OF some
8074 // combined entry (for partial structs). Only the first PTR_AND_OBJ entry
8075 // in a component list should be marked as MEMBER_OF, all subsequent entries
8076 // do not belong to the base struct. E.g.
8077 // struct S2 s;
8078 // s.ps->ps->ps->f[:]
8079 // (1) (2) (3) (4)
8080 // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a
8081 // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3)
8082 // is the pointee of ps(2) which is not member of struct s, so it should not
8083 // be marked as such (it is still PTR_AND_OBJ).
8084 // The variable is initialized to false so that PTR_AND_OBJ entries which
8085 // are not struct members are not considered (e.g. array of pointers to
8086 // data).
8087 bool ShouldBeMemberOf = false;
8088
8089 // Variable keeping track of whether or not we have encountered a component
8090 // in the component list which is a member expression. Useful when we have a
8091 // pointer or a final array section, in which case it is the previous
8092 // component in the list which tells us whether we have a member expression.
8093 // E.g. X.f[:]
8094 // While processing the final array section "[:]" it is "f" which tells us
8095 // whether we are dealing with a member of a declared struct.
8096 const MemberExpr *EncounteredME = nullptr;
8097
8098 // Track for the total number of dimension. Start from one for the dummy
8099 // dimension.
8100 uint64_t DimSize = 1;
8101
8102 // Detects non-contiguous updates due to strided accesses.
8103 // Sets the 'IsNonContiguous' flag so that the 'MapType' bits are set
8104 // correctly when generating information to be passed to the runtime. The
8105 // flag is set to true if any array section has a stride not equal to 1, or
8106 // if the stride is not a constant expression (conservatively assumed
8107 // non-contiguous).
8108 bool IsNonContiguous =
8109 CombinedInfo.NonContigInfo.IsNonContiguous ||
8110 any_of(Range&: Components, P: [&](const auto &Component) {
8111 const auto *OASE =
8112 dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression());
8113 if (!OASE)
8114 return false;
8115
8116 const Expr *StrideExpr = OASE->getStride();
8117 if (!StrideExpr)
8118 return false;
8119
8120 assert(StrideExpr->getType()->isIntegerType() &&
8121 "Stride expression must be of integer type");
8122
8123 // If stride is not evaluatable as a constant, treat as
8124 // non-contiguous.
8125 const auto Constant =
8126 StrideExpr->getIntegerConstantExpr(Ctx: CGF.getContext());
8127 if (!Constant)
8128 return true;
8129
8130 // Treat non-unitary strides as non-contiguous.
8131 return !Constant->isOne();
8132 });
8133
8134 bool IsPrevMemberReference = false;
8135
8136 bool IsPartialMapped =
8137 !PartialStruct.PreliminaryMapData.BasePointers.empty();
8138
8139 // We need to check if we will be encountering any MEs. If we do not
8140 // encounter any ME expression it means we will be mapping the whole struct.
8141 // In that case we need to skip adding an entry for the struct to the
8142 // CombinedInfo list and instead add an entry to the StructBaseCombinedInfo
8143 // list only when generating all info for clauses.
8144 bool IsMappingWholeStruct = true;
8145 if (!GenerateAllInfoForClauses) {
8146 IsMappingWholeStruct = false;
8147 } else {
8148 for (auto TempI = I; TempI != CE; ++TempI) {
8149 const MemberExpr *PossibleME =
8150 dyn_cast<MemberExpr>(Val: TempI->getAssociatedExpression());
8151 if (PossibleME) {
8152 IsMappingWholeStruct = false;
8153 break;
8154 }
8155 }
8156 }
8157
8158 bool SeenFirstNonBinOpExprAfterAttachPtr = false;
8159 for (; I != CE; ++I) {
8160 // If we have a valid attach-ptr, we skip processing all components until
8161 // after the attach-ptr.
8162 if (HasAttachPtr && !SeenAttachPtr) {
8163 SeenAttachPtr = I->getAssociatedExpression() == AttachPtrExpr;
8164 continue;
8165 }
8166
8167 // After finding the attach pointer, skip binary-ops, to skip past
8168 // expressions like (p + 10), for a map like map(*(p + 10)), where p is
8169 // the attach-ptr.
8170 if (HasAttachPtr && !SeenFirstNonBinOpExprAfterAttachPtr) {
8171 const auto *BO = dyn_cast<BinaryOperator>(Val: I->getAssociatedExpression());
8172 if (BO)
8173 continue;
8174
8175 // Found the first non-binary-operator component after attach
8176 SeenFirstNonBinOpExprAfterAttachPtr = true;
8177 BP = AttachPteeBaseAddr;
8178 }
8179
8180 // If the current component is member of a struct (parent struct) mark it.
8181 if (!EncounteredME) {
8182 EncounteredME = dyn_cast<MemberExpr>(Val: I->getAssociatedExpression());
8183 // If we encounter a PTR_AND_OBJ entry from now on it should be marked
8184 // as MEMBER_OF the parent struct.
8185 if (EncounteredME) {
8186 ShouldBeMemberOf = true;
8187 // Do not emit as complex pointer if this is actually not array-like
8188 // expression.
8189 if (FirstPointerInComplexData) {
8190 QualType Ty = std::prev(x: I)
8191 ->getAssociatedDeclaration()
8192 ->getType()
8193 .getNonReferenceType();
8194 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8195 FirstPointerInComplexData = false;
8196 }
8197 }
8198 }
8199
8200 auto Next = std::next(x: I);
8201
8202 // We need to generate the addresses and sizes if this is the last
8203 // component, if the component is a pointer or if it is an array section
8204 // whose length can't be proved to be one. If this is a pointer, it
8205 // becomes the base address for the following components.
8206
8207 // A final array section, is one whose length can't be proved to be one.
8208 // If the map item is non-contiguous then we don't treat any array section
8209 // as final array section.
8210 bool IsFinalArraySection =
8211 !IsNonContiguous &&
8212 isFinalArraySectionExpression(E: I->getAssociatedExpression());
8213
8214 // If we have a declaration for the mapping use that, otherwise use
8215 // the base declaration of the map clause.
8216 const ValueDecl *MapDecl = (I->getAssociatedDeclaration())
8217 ? I->getAssociatedDeclaration()
8218 : BaseDecl;
8219 MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()
8220 : MapExpr;
8221
8222 // Get information on whether the element is a pointer. Have to do a
8223 // special treatment for array sections given that they are built-in
8224 // types.
8225 const auto *OASE =
8226 dyn_cast<ArraySectionExpr>(Val: I->getAssociatedExpression());
8227 const auto *OAShE =
8228 dyn_cast<OMPArrayShapingExpr>(Val: I->getAssociatedExpression());
8229 const auto *UO = dyn_cast<UnaryOperator>(Val: I->getAssociatedExpression());
8230 const auto *BO = dyn_cast<BinaryOperator>(Val: I->getAssociatedExpression());
8231 bool IsPointer =
8232 OAShE ||
8233 (OASE && ArraySectionExpr::getBaseOriginalType(Base: OASE)
8234 .getCanonicalType()
8235 ->isAnyPointerType()) ||
8236 I->getAssociatedExpression()->getType()->isAnyPointerType();
8237 bool IsMemberReference = isa<MemberExpr>(Val: I->getAssociatedExpression()) &&
8238 MapDecl &&
8239 MapDecl->getType()->isLValueReferenceType();
8240 bool IsNonDerefPointer = IsPointer &&
8241 !(UO && UO->getOpcode() != UO_Deref) && !BO &&
8242 !IsNonContiguous;
8243
8244 if (OASE)
8245 ++DimSize;
8246
8247 if (Next == CE || IsMemberReference || IsNonDerefPointer ||
8248 IsFinalArraySection) {
8249 // If this is not the last component, we expect the pointer to be
8250 // associated with an array expression or member expression.
8251 assert((Next == CE ||
8252 isa<MemberExpr>(Next->getAssociatedExpression()) ||
8253 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||
8254 isa<ArraySectionExpr>(Next->getAssociatedExpression()) ||
8255 isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||
8256 isa<UnaryOperator>(Next->getAssociatedExpression()) ||
8257 isa<BinaryOperator>(Next->getAssociatedExpression())) &&
8258 "Unexpected expression");
8259
8260 Address LB = Address::invalid();
8261 Address LowestElem = Address::invalid();
8262 auto &&EmitMemberExprBase = [](CodeGenFunction &CGF,
8263 const MemberExpr *E) {
8264 const Expr *BaseExpr = E->getBase();
8265 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a
8266 // scalar.
8267 LValue BaseLV;
8268 if (E->isArrow()) {
8269 LValueBaseInfo BaseInfo;
8270 TBAAAccessInfo TBAAInfo;
8271 Address Addr =
8272 CGF.EmitPointerWithAlignment(Addr: BaseExpr, BaseInfo: &BaseInfo, TBAAInfo: &TBAAInfo);
8273 QualType PtrTy = BaseExpr->getType()->getPointeeType();
8274 BaseLV = CGF.MakeAddrLValue(Addr, T: PtrTy, BaseInfo, TBAAInfo);
8275 } else {
8276 BaseLV = CGF.EmitOMPSharedLValue(E: BaseExpr);
8277 }
8278 return BaseLV;
8279 };
8280 if (OAShE) {
8281 LowestElem = LB =
8282 Address(CGF.EmitScalarExpr(E: OAShE->getBase()),
8283 CGF.ConvertTypeForMem(
8284 T: OAShE->getBase()->getType()->getPointeeType()),
8285 CGF.getContext().getTypeAlignInChars(
8286 T: OAShE->getBase()->getType()));
8287 } else if (IsMemberReference) {
8288 const auto *ME = cast<MemberExpr>(Val: I->getAssociatedExpression());
8289 LValue BaseLVal = EmitMemberExprBase(CGF, ME);
8290 LowestElem = CGF.EmitLValueForFieldInitialization(
8291 Base: BaseLVal, Field: cast<FieldDecl>(Val: MapDecl))
8292 .getAddress();
8293 LB = CGF.EmitLoadOfReferenceLValue(RefAddr: LowestElem, RefTy: MapDecl->getType())
8294 .getAddress();
8295 } else {
8296 LowestElem = LB =
8297 CGF.EmitOMPSharedLValue(E: I->getAssociatedExpression())
8298 .getAddress();
8299 }
8300
8301 // Save the final LowestElem, to use it as the pointee in attach maps,
8302 // if emitted.
8303 if (Next == CE)
8304 FinalLowestElem = LowestElem;
8305
8306 // If this component is a pointer inside the base struct then we don't
8307 // need to create any entry for it - it will be combined with the object
8308 // it is pointing to into a single PTR_AND_OBJ entry.
8309 bool IsMemberPointerOrAddr =
8310 EncounteredME &&
8311 (((IsPointer || ForDeviceAddr) &&
8312 I->getAssociatedExpression() == EncounteredME) ||
8313 (IsPrevMemberReference && !IsPointer) ||
8314 (IsMemberReference && Next != CE &&
8315 !Next->getAssociatedExpression()->getType()->isPointerType()));
8316 if (!OverlappedElements.empty() && Next == CE) {
8317 // Handle base element with the info for overlapped elements.
8318 assert(!PartialStruct.Base.isValid() && "The base element is set.");
8319 assert(!IsPointer &&
8320 "Unexpected base element with the pointer type.");
8321 // Mark the whole struct as the struct that requires allocation on the
8322 // device.
8323 PartialStruct.LowestElem = {0, LowestElem};
8324 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars(
8325 T: I->getAssociatedExpression()->getType());
8326 Address HB = CGF.Builder.CreateConstGEP(
8327 Addr: CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
8328 Addr: LowestElem, Ty: CGF.VoidPtrTy, ElementTy: CGF.Int8Ty),
8329 Index: TypeSize.getQuantity() - 1);
8330 PartialStruct.HighestElem = {
8331 std::numeric_limits<decltype(
8332 PartialStruct.HighestElem.first)>::max(),
8333 HB};
8334 PartialStruct.Base = BP;
8335 PartialStruct.LB = LB;
8336 assert(
8337 PartialStruct.PreliminaryMapData.BasePointers.empty() &&
8338 "Overlapped elements must be used only once for the variable.");
8339 std::swap(a&: PartialStruct.PreliminaryMapData, b&: CombinedInfo);
8340 // Emit data for non-overlapped data.
8341 OpenMPOffloadMappingFlags Flags =
8342 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
8343 getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,
8344 /*AddPtrFlag=*/false,
8345 /*AddIsTargetParamFlag=*/false, IsNonContiguous);
8346 CopyOverlappedEntryGaps CopyGaps(CGF, CombinedInfo, Flags, MapDecl,
8347 MapExpr, BP, LB, IsNonContiguous,
8348 DimSize);
8349 // Do bitcopy of all non-overlapped structure elements.
8350 for (OMPClauseMappableExprCommon::MappableExprComponentListRef
8351 Component : OverlappedElements) {
8352 for (const OMPClauseMappableExprCommon::MappableComponent &MC :
8353 Component) {
8354 if (const ValueDecl *VD = MC.getAssociatedDeclaration()) {
8355 if (const auto *FD = dyn_cast<FieldDecl>(Val: VD)) {
8356 CopyGaps.processField(MC, FD, EmitMemberExprBase);
8357 }
8358 }
8359 }
8360 }
8361 CopyGaps.copyUntilEnd(HB);
8362 break;
8363 }
8364 llvm::Value *Size = getExprTypeSize(E: I->getAssociatedExpression());
8365 // Skip adding an entry in the CurInfo of this combined entry if the
8366 // whole struct is currently being mapped. The struct needs to be added
8367 // in the first position before any data internal to the struct is being
8368 // mapped.
8369 // Skip adding an entry in the CurInfo of this combined entry if the
8370 // PartialStruct.PreliminaryMapData.BasePointers has been mapped.
8371 if ((!IsMemberPointerOrAddr && !IsPartialMapped) ||
8372 (Next == CE && MapType != OMPC_MAP_unknown)) {
8373 if (!IsMappingWholeStruct) {
8374 CombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
8375 CombinedInfo.BasePointers.push_back(Elt: BP.emitRawPointer(CGF));
8376 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
8377 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
8378 CombinedInfo.Pointers.push_back(Elt: LB.emitRawPointer(CGF));
8379 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
8380 V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/true));
8381 CombinedInfo.NonContigInfo.Dims.push_back(Elt: IsNonContiguous ? DimSize
8382 : 1);
8383 } else {
8384 StructBaseCombinedInfo.Exprs.emplace_back(Args&: MapDecl, Args&: MapExpr);
8385 StructBaseCombinedInfo.BasePointers.push_back(
8386 Elt: BP.emitRawPointer(CGF));
8387 StructBaseCombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
8388 StructBaseCombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
8389 StructBaseCombinedInfo.Pointers.push_back(Elt: LB.emitRawPointer(CGF));
8390 StructBaseCombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
8391 V: Size, DestTy: CGF.Int64Ty, /*isSigned=*/true));
8392 StructBaseCombinedInfo.NonContigInfo.Dims.push_back(
8393 Elt: IsNonContiguous ? DimSize : 1);
8394 }
8395
8396 // If Mapper is valid, the last component inherits the mapper.
8397 bool HasMapper = Mapper && Next == CE;
8398 if (!IsMappingWholeStruct)
8399 CombinedInfo.Mappers.push_back(Elt: HasMapper ? Mapper : nullptr);
8400 else
8401 StructBaseCombinedInfo.Mappers.push_back(Elt: HasMapper ? Mapper
8402 : nullptr);
8403
8404 // We need to add a pointer flag for each map that comes from the
8405 // same expression except for the first one. We also need to signal
8406 // this map is the first one that relates with the current capture
8407 // (there is a set of entries for each capture).
8408 OpenMPOffloadMappingFlags Flags = getMapTypeBits(
8409 MapType, MapModifiers, MotionModifiers, IsImplicit,
8410 AddPtrFlag: !IsExpressionFirstInfo || RequiresReference ||
8411 FirstPointerInComplexData || IsMemberReference,
8412 AddIsTargetParamFlag: IsCaptureFirstInfo && !RequiresReference, IsNonContiguous);
8413
8414 if (!IsExpressionFirstInfo || IsMemberReference) {
8415 // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well,
8416 // then we reset the TO/FROM/ALWAYS/DELETE/CLOSE flags.
8417 if (IsPointer || (IsMemberReference && Next != CE))
8418 Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |
8419 OpenMPOffloadMappingFlags::OMP_MAP_FROM |
8420 OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |
8421 OpenMPOffloadMappingFlags::OMP_MAP_DELETE |
8422 OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
8423
8424 if (ShouldBeMemberOf) {
8425 // Set placeholder value MEMBER_OF=FFFF to indicate that the flag
8426 // should be later updated with the correct value of MEMBER_OF.
8427 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
8428 // From now on, all subsequent PTR_AND_OBJ entries should not be
8429 // marked as MEMBER_OF.
8430 ShouldBeMemberOf = false;
8431 }
8432 }
8433
8434 if (!IsMappingWholeStruct) {
8435 CombinedInfo.Types.push_back(Elt: Flags);
8436 // HasAttachPtr marks pointee entries, which have a base attach-ptr.
8437 CombinedInfo.HasAttachPtr.push_back(Elt: HasAttachPtr);
8438 } else {
8439 StructBaseCombinedInfo.Types.push_back(Elt: Flags);
8440 StructBaseCombinedInfo.HasAttachPtr.push_back(Elt: HasAttachPtr);
8441 }
8442 }
8443
8444 // If we have encountered a member expression so far, keep track of the
8445 // mapped member. If the parent is "*this", then the value declaration
8446 // is nullptr.
8447 if (EncounteredME) {
8448 const auto *FD = cast<FieldDecl>(Val: EncounteredME->getMemberDecl());
8449 unsigned FieldIndex = FD->getFieldIndex();
8450
8451 // Update info about the lowest and highest elements for this struct
8452 if (!PartialStruct.Base.isValid()) {
8453 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8454 if (IsFinalArraySection && OASE) {
8455 Address HB =
8456 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false)
8457 .getAddress();
8458 PartialStruct.HighestElem = {FieldIndex, HB};
8459 } else {
8460 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8461 }
8462 PartialStruct.Base = BP;
8463 PartialStruct.LB = BP;
8464 } else if (FieldIndex < PartialStruct.LowestElem.first) {
8465 PartialStruct.LowestElem = {FieldIndex, LowestElem};
8466 } else if (FieldIndex > PartialStruct.HighestElem.first) {
8467 if (IsFinalArraySection && OASE) {
8468 Address HB =
8469 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/false)
8470 .getAddress();
8471 PartialStruct.HighestElem = {FieldIndex, HB};
8472 } else {
8473 PartialStruct.HighestElem = {FieldIndex, LowestElem};
8474 }
8475 }
8476 }
8477
8478 // Need to emit combined struct for array sections.
8479 if (IsFinalArraySection || IsNonContiguous)
8480 PartialStruct.IsArraySection = true;
8481
8482 // If we have a final array section, we are done with this expression.
8483 if (IsFinalArraySection)
8484 break;
8485
8486 // The pointer becomes the base for the next element.
8487 if (Next != CE)
8488 BP = IsMemberReference ? LowestElem : LB;
8489 if (!IsPartialMapped)
8490 IsExpressionFirstInfo = false;
8491 IsCaptureFirstInfo = false;
8492 FirstPointerInComplexData = false;
8493 IsPrevMemberReference = IsMemberReference;
8494 } else if (FirstPointerInComplexData) {
8495 QualType Ty = Components.rbegin()
8496 ->getAssociatedDeclaration()
8497 ->getType()
8498 .getNonReferenceType();
8499 BP = CGF.EmitLoadOfPointer(Ptr: BP, PtrTy: Ty->castAs<PointerType>());
8500 FirstPointerInComplexData = false;
8501 }
8502 }
8503 // If ran into the whole component - allocate the space for the whole
8504 // record.
8505 if (!EncounteredME)
8506 PartialStruct.HasCompleteRecord = true;
8507
8508 // Populate ATTACH information for later processing by emitAttachEntry.
8509 if (shouldEmitAttachEntry(PointerExpr: AttachPtrExpr, MapBaseDecl: BaseDecl, CGF, CurDir)) {
8510 AttachInfo.AttachPtrAddr = AttachPtrAddr;
8511 AttachInfo.AttachPteeAddr = FinalLowestElem;
8512 AttachInfo.AttachPtrDecl = BaseDecl;
8513 AttachInfo.AttachMapExpr = MapExpr;
8514 }
8515
8516 if (!IsNonContiguous)
8517 return;
8518
8519 const ASTContext &Context = CGF.getContext();
8520
8521 // For supporting stride in array section, we need to initialize the first
8522 // dimension size as 1, first offset as 0, and first count as 1
8523 MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 0)};
8524 MapValuesArrayTy CurCounts;
8525 MapValuesArrayTy CurStrides = {llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 1)};
8526 MapValuesArrayTy DimSizes{llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: 1)};
8527 uint64_t ElementTypeSize;
8528
8529 // Collect Size information for each dimension and get the element size as
8530 // the first Stride. For example, for `int arr[10][10]`, the DimSizes
8531 // should be [10, 10] and the first stride is 4 btyes.
8532 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8533 Components) {
8534 const Expr *AssocExpr = Component.getAssociatedExpression();
8535 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8536
8537 if (!OASE)
8538 continue;
8539
8540 QualType Ty = ArraySectionExpr::getBaseOriginalType(Base: OASE->getBase());
8541 auto *CAT = Context.getAsConstantArrayType(T: Ty);
8542 auto *VAT = Context.getAsVariableArrayType(T: Ty);
8543
8544 // We need all the dimension size except for the last dimension.
8545 assert((VAT || CAT || &Component == &*Components.begin()) &&
8546 "Should be either ConstantArray or VariableArray if not the "
8547 "first Component");
8548
8549 // Get element size if CurCounts is empty.
8550 if (CurCounts.empty()) {
8551 const Type *ElementType = nullptr;
8552 if (CAT)
8553 ElementType = CAT->getElementType().getTypePtr();
8554 else if (VAT)
8555 ElementType = VAT->getElementType().getTypePtr();
8556 else if (&Component == &*Components.begin()) {
8557 // If the base is a raw pointer (e.g. T *data with data[a:b:c]),
8558 // there was no earlier CAT/VAT/array handling to establish
8559 // ElementType. Capture the pointee type now so that subsequent
8560 // components (offset/length/stride) have a concrete element type to
8561 // work with. This makes pointer-backed sections behave consistently
8562 // with CAT/VAT/array bases.
8563 if (const auto *PtrType = Ty->getAs<PointerType>())
8564 ElementType = PtrType->getPointeeType().getTypePtr();
8565 } else {
8566 // Any component after the first should never have a raw pointer type;
8567 // by this point. ElementType must already be known (set above or in
8568 // prior array / CAT / VAT handling).
8569 assert(!Ty->isPointerType() &&
8570 "Non-first components should not be raw pointers");
8571 }
8572
8573 // At this stage, if ElementType was a base pointer and we are in the
8574 // first iteration, it has been computed.
8575 if (ElementType) {
8576 // For the case that having pointer as base, we need to remove one
8577 // level of indirection.
8578 if (&Component != &*Components.begin())
8579 ElementType = ElementType->getPointeeOrArrayElementType();
8580 ElementTypeSize =
8581 Context.getTypeSizeInChars(T: ElementType).getQuantity();
8582 CurCounts.push_back(
8583 Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: ElementTypeSize));
8584 }
8585 }
8586 // Get dimension value except for the last dimension since we don't need
8587 // it.
8588 if (DimSizes.size() < Components.size() - 1) {
8589 if (CAT)
8590 DimSizes.push_back(
8591 Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: CAT->getZExtSize()));
8592 else if (VAT)
8593 DimSizes.push_back(Elt: CGF.Builder.CreateIntCast(
8594 V: CGF.EmitScalarExpr(E: VAT->getSizeExpr()), DestTy: CGF.Int64Ty,
8595 /*IsSigned=*/isSigned: false));
8596 }
8597 }
8598
8599 // Skip the dummy dimension since we have already have its information.
8600 auto *DI = DimSizes.begin() + 1;
8601 // Product of dimension.
8602 llvm::Value *DimProd =
8603 llvm::ConstantInt::get(Ty: CGF.CGM.Int64Ty, V: ElementTypeSize);
8604
8605 // Collect info for non-contiguous. Notice that offset, count, and stride
8606 // are only meaningful for array-section, so we insert a null for anything
8607 // other than array-section.
8608 // Also, the size of offset, count, and stride are not the same as
8609 // pointers, base_pointers, sizes, or dims. Instead, the size of offset,
8610 // count, and stride are the same as the number of non-contiguous
8611 // declaration in target update to/from clause.
8612 for (const OMPClauseMappableExprCommon::MappableComponent &Component :
8613 Components) {
8614 const Expr *AssocExpr = Component.getAssociatedExpression();
8615
8616 if (const auto *AE = dyn_cast<ArraySubscriptExpr>(Val: AssocExpr)) {
8617 llvm::Value *Offset = CGF.Builder.CreateIntCast(
8618 V: CGF.EmitScalarExpr(E: AE->getIdx()), DestTy: CGF.Int64Ty,
8619 /*isSigned=*/false);
8620 CurOffsets.push_back(Elt: Offset);
8621 CurCounts.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, /*V=*/1));
8622 CurStrides.push_back(Elt: CurStrides.back());
8623 continue;
8624 }
8625
8626 const auto *OASE = dyn_cast<ArraySectionExpr>(Val: AssocExpr);
8627
8628 if (!OASE)
8629 continue;
8630
8631 // Offset
8632 const Expr *OffsetExpr = OASE->getLowerBound();
8633 llvm::Value *Offset = nullptr;
8634 if (!OffsetExpr) {
8635 // If offset is absent, then we just set it to zero.
8636 Offset = llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
8637 } else {
8638 Offset = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: OffsetExpr),
8639 DestTy: CGF.Int64Ty,
8640 /*isSigned=*/false);
8641 }
8642
8643 // Count
8644 const Expr *CountExpr = OASE->getLength();
8645 llvm::Value *Count = nullptr;
8646 if (!CountExpr) {
8647 // In Clang, once a high dimension is an array section, we construct all
8648 // the lower dimension as array section, however, for case like
8649 // arr[0:2][2], Clang construct the inner dimension as an array section
8650 // but it actually is not in an array section form according to spec.
8651 if (!OASE->getColonLocFirst().isValid() &&
8652 !OASE->getColonLocSecond().isValid()) {
8653 Count = llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 1);
8654 } else {
8655 // OpenMP 5.0, 2.1.5 Array Sections, Description.
8656 // When the length is absent it defaults to ⌈(size −
8657 // lower-bound)/stride⌉, where size is the size of the array
8658 // dimension.
8659 const Expr *StrideExpr = OASE->getStride();
8660 llvm::Value *Stride =
8661 StrideExpr
8662 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: StrideExpr),
8663 DestTy: CGF.Int64Ty, /*isSigned=*/false)
8664 : nullptr;
8665 if (Stride)
8666 Count = CGF.Builder.CreateUDiv(
8667 LHS: CGF.Builder.CreateNUWSub(LHS: *DI, RHS: Offset), RHS: Stride);
8668 else
8669 Count = CGF.Builder.CreateNUWSub(LHS: *DI, RHS: Offset);
8670 }
8671 } else {
8672 Count = CGF.EmitScalarExpr(E: CountExpr);
8673 }
8674 Count = CGF.Builder.CreateIntCast(V: Count, DestTy: CGF.Int64Ty, /*isSigned=*/false);
8675 CurCounts.push_back(Elt: Count);
8676
8677 // Stride_n' = Stride_n * (D_0 * D_1 ... * D_n-1) * Unit size
8678 // Offset_n' = Offset_n * (D_0 * D_1 ... * D_n-1) * Unit size
8679 // Take `int arr[5][5][5]` and `arr[0:2:2][1:2:1][0:2:2]` as an example:
8680 // Offset Count Stride
8681 // D0 0 4 1 (int) <- dummy dimension
8682 // D1 0 2 8 (2 * (1) * 4)
8683 // D2 100 2 20 (1 * (1 * 5) * 4)
8684 // D3 0 2 200 (2 * (1 * 5 * 4) * 4)
8685 const Expr *StrideExpr = OASE->getStride();
8686 llvm::Value *Stride =
8687 StrideExpr
8688 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: StrideExpr),
8689 DestTy: CGF.Int64Ty, /*isSigned=*/false)
8690 : nullptr;
8691 DimProd = CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: *(DI - 1));
8692 if (Stride)
8693 CurStrides.push_back(Elt: CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: Stride));
8694 else
8695 CurStrides.push_back(Elt: DimProd);
8696
8697 Offset = CGF.Builder.CreateNUWMul(LHS: DimProd, RHS: Offset);
8698 CurOffsets.push_back(Elt: Offset);
8699
8700 if (DI != DimSizes.end())
8701 ++DI;
8702 }
8703
8704 CombinedInfo.NonContigInfo.Offsets.push_back(Elt: CurOffsets);
8705 CombinedInfo.NonContigInfo.Counts.push_back(Elt: CurCounts);
8706 CombinedInfo.NonContigInfo.Strides.push_back(Elt: CurStrides);
8707 }
8708
8709 /// Return the adjusted map modifiers if the declaration a capture refers to
8710 /// appears in a first-private clause. This is expected to be used only with
8711 /// directives that start with 'target'.
8712 OpenMPOffloadMappingFlags
8713 getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const {
8714 assert(Cap.capturesVariable() && "Expected capture by reference only!");
8715
8716 // A first private variable captured by reference will use only the
8717 // 'private ptr' and 'map to' flag. Return the right flags if the captured
8718 // declaration is known as first-private in this handler.
8719 if (FirstPrivateDecls.count(Val: Cap.getCapturedVar())) {
8720 if (Cap.getCapturedVar()->getType()->isAnyPointerType())
8721 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8722 OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
8723 return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
8724 OpenMPOffloadMappingFlags::OMP_MAP_TO;
8725 }
8726 auto I = LambdasMap.find(Val: Cap.getCapturedVar()->getCanonicalDecl());
8727 if (I != LambdasMap.end())
8728 // for map(to: lambda): using user specified map type.
8729 return getMapTypeBits(
8730 MapType: I->getSecond()->getMapType(), MapModifiers: I->getSecond()->getMapTypeModifiers(),
8731 /*MotionModifiers=*/{}, IsImplicit: I->getSecond()->isImplicit(),
8732 /*AddPtrFlag=*/false,
8733 /*AddIsTargetParamFlag=*/false,
8734 /*isNonContiguous=*/IsNonContiguous: false);
8735 return OpenMPOffloadMappingFlags::OMP_MAP_TO |
8736 OpenMPOffloadMappingFlags::OMP_MAP_FROM;
8737 }
8738
8739 void getPlainLayout(const CXXRecordDecl *RD,
8740 llvm::SmallVectorImpl<const FieldDecl *> &Layout,
8741 bool AsBase) const {
8742 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD);
8743
8744 llvm::StructType *St =
8745 AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType();
8746
8747 unsigned NumElements = St->getNumElements();
8748 llvm::SmallVector<
8749 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>
8750 RecordLayout(NumElements);
8751
8752 // Fill bases.
8753 for (const auto &I : RD->bases()) {
8754 if (I.isVirtual())
8755 continue;
8756
8757 QualType BaseTy = I.getType();
8758 const auto *Base = BaseTy->getAsCXXRecordDecl();
8759 // Ignore empty bases.
8760 if (isEmptyRecordForLayout(Context: CGF.getContext(), T: BaseTy) ||
8761 CGF.getContext()
8762 .getASTRecordLayout(D: Base)
8763 .getNonVirtualSize()
8764 .isZero())
8765 continue;
8766
8767 unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(RD: Base);
8768 RecordLayout[FieldIndex] = Base;
8769 }
8770 // Fill in virtual bases.
8771 for (const auto &I : RD->vbases()) {
8772 QualType BaseTy = I.getType();
8773 // Ignore empty bases.
8774 if (isEmptyRecordForLayout(Context: CGF.getContext(), T: BaseTy))
8775 continue;
8776
8777 const auto *Base = BaseTy->getAsCXXRecordDecl();
8778 unsigned FieldIndex = RL.getVirtualBaseIndex(base: Base);
8779 if (RecordLayout[FieldIndex])
8780 continue;
8781 RecordLayout[FieldIndex] = Base;
8782 }
8783 // Fill in all the fields.
8784 assert(!RD->isUnion() && "Unexpected union.");
8785 for (const auto *Field : RD->fields()) {
8786 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we
8787 // will fill in later.)
8788 if (!Field->isBitField() &&
8789 !isEmptyFieldForLayout(Context: CGF.getContext(), FD: Field)) {
8790 unsigned FieldIndex = RL.getLLVMFieldNo(FD: Field);
8791 RecordLayout[FieldIndex] = Field;
8792 }
8793 }
8794 for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>
8795 &Data : RecordLayout) {
8796 if (Data.isNull())
8797 continue;
8798 if (const auto *Base = dyn_cast<const CXXRecordDecl *>(Val: Data))
8799 getPlainLayout(RD: Base, Layout, /*AsBase=*/true);
8800 else
8801 Layout.push_back(Elt: cast<const FieldDecl *>(Val: Data));
8802 }
8803 }
8804
8805 /// Returns the address corresponding to \p PointerExpr.
8806 static Address getAttachPtrAddr(const Expr *PointerExpr,
8807 CodeGenFunction &CGF) {
8808 assert(PointerExpr && "Cannot get addr from null attach-ptr expr");
8809 Address AttachPtrAddr = Address::invalid();
8810
8811 if (auto *DRE = dyn_cast<DeclRefExpr>(Val: PointerExpr)) {
8812 // If the pointer is a variable, we can use its address directly.
8813 AttachPtrAddr = CGF.EmitLValue(E: DRE).getAddress();
8814 } else if (auto *OASE = dyn_cast<ArraySectionExpr>(Val: PointerExpr)) {
8815 AttachPtrAddr =
8816 CGF.EmitArraySectionExpr(E: OASE, /*IsLowerBound=*/true).getAddress();
8817 } else if (auto *ASE = dyn_cast<ArraySubscriptExpr>(Val: PointerExpr)) {
8818 AttachPtrAddr = CGF.EmitLValue(E: ASE).getAddress();
8819 } else if (auto *ME = dyn_cast<MemberExpr>(Val: PointerExpr)) {
8820 AttachPtrAddr = CGF.EmitMemberExpr(E: ME).getAddress();
8821 } else if (auto *UO = dyn_cast<UnaryOperator>(Val: PointerExpr)) {
8822 assert(UO->getOpcode() == UO_Deref &&
8823 "Unexpected unary-operator on attach-ptr-expr");
8824 AttachPtrAddr = CGF.EmitLValue(E: UO).getAddress();
8825 }
8826 assert(AttachPtrAddr.isValid() &&
8827 "Failed to get address for attach pointer expression");
8828 return AttachPtrAddr;
8829 }
8830
8831 /// Get the address of the attach pointer, and a load from it, to get the
8832 /// pointee base address.
8833 /// \return A pair containing AttachPtrAddr and AttachPteeBaseAddr. The pair
8834 /// contains invalid addresses if \p AttachPtrExpr is null.
8835 static std::pair<Address, Address>
8836 getAttachPtrAddrAndPteeBaseAddr(const Expr *AttachPtrExpr,
8837 CodeGenFunction &CGF) {
8838
8839 if (!AttachPtrExpr)
8840 return {Address::invalid(), Address::invalid()};
8841
8842 Address AttachPtrAddr = getAttachPtrAddr(PointerExpr: AttachPtrExpr, CGF);
8843 assert(AttachPtrAddr.isValid() && "Invalid attach pointer addr");
8844
8845 QualType AttachPtrType =
8846 OMPClauseMappableExprCommon::getComponentExprElementType(Exp: AttachPtrExpr)
8847 .getCanonicalType();
8848
8849 Address AttachPteeBaseAddr = CGF.EmitLoadOfPointer(
8850 Ptr: AttachPtrAddr, PtrTy: AttachPtrType->castAs<PointerType>());
8851 assert(AttachPteeBaseAddr.isValid() && "Invalid attach pointee base addr");
8852
8853 return {AttachPtrAddr, AttachPteeBaseAddr};
8854 }
8855
8856 /// Returns whether an attach entry should be emitted for a map on
8857 /// \p MapBaseDecl on the directive \p CurDir.
8858 static bool
8859 shouldEmitAttachEntry(const Expr *PointerExpr, const ValueDecl *MapBaseDecl,
8860 CodeGenFunction &CGF,
8861 llvm::PointerUnion<const OMPExecutableDirective *,
8862 const OMPDeclareMapperDecl *>
8863 CurDir) {
8864 if (!PointerExpr)
8865 return false;
8866
8867 // Pointer attachment is needed at map-entering time or for declare
8868 // mappers.
8869 return isa<const OMPDeclareMapperDecl *>(Val: CurDir) ||
8870 isOpenMPTargetMapEnteringDirective(
8871 DKind: cast<const OMPExecutableDirective *>(Val&: CurDir)
8872 ->getDirectiveKind());
8873 }
8874
8875 /// Computes the attach-ptr expr for \p Components, and updates various maps
8876 /// with the information.
8877 /// It internally calls OMPClauseMappableExprCommon::findAttachPtrExpr()
8878 /// with the OpenMPDirectiveKind extracted from \p CurDir.
8879 /// It updates AttachPtrComputationOrderMap, AttachPtrComponentDepthMap, and
8880 /// AttachPtrExprMap.
8881 void collectAttachPtrExprInfo(
8882 OMPClauseMappableExprCommon::MappableExprComponentListRef Components,
8883 llvm::PointerUnion<const OMPExecutableDirective *,
8884 const OMPDeclareMapperDecl *>
8885 CurDir) {
8886
8887 OpenMPDirectiveKind CurDirectiveID =
8888 isa<const OMPDeclareMapperDecl *>(Val: CurDir)
8889 ? OMPD_declare_mapper
8890 : cast<const OMPExecutableDirective *>(Val&: CurDir)->getDirectiveKind();
8891
8892 const auto &[AttachPtrExpr, Depth] =
8893 OMPClauseMappableExprCommon::findAttachPtrExpr(Components,
8894 CurDirKind: CurDirectiveID);
8895
8896 AttachPtrComputationOrderMap.try_emplace(
8897 Key: AttachPtrExpr, Args: AttachPtrComputationOrderMap.size());
8898 AttachPtrComponentDepthMap.try_emplace(Key: AttachPtrExpr, Args: Depth);
8899 AttachPtrExprMap.try_emplace(Key: Components, Args: AttachPtrExpr);
8900 }
8901
8902 /// Generate all the base pointers, section pointers, sizes, map types, and
8903 /// mappers for the extracted mappable expressions (all included in \a
8904 /// CombinedInfo). Also, for each item that relates with a device pointer, a
8905 /// pair of the relevant declaration and index where it occurs is appended to
8906 /// the device pointers info array.
8907 void generateAllInfoForClauses(
8908 ArrayRef<const OMPClause *> Clauses, MapCombinedInfoTy &CombinedInfo,
8909 llvm::OpenMPIRBuilder &OMPBuilder,
8910 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
8911 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
8912 // We have to process the component lists that relate with the same
8913 // declaration in a single chunk so that we can generate the map flags
8914 // correctly. Therefore, we organize all lists in a map.
8915 enum MapKind { Present, Allocs, Other, Total };
8916 llvm::MapVector<CanonicalDeclPtr<const Decl>,
8917 SmallVector<SmallVector<MapInfo, 8>, 4>>
8918 Info;
8919
8920 // Helper function to fill the information map for the different supported
8921 // clauses.
8922 auto &&InfoGen =
8923 [&Info, &SkipVarSet](
8924 const ValueDecl *D, MapKind Kind,
8925 OMPClauseMappableExprCommon::MappableExprComponentListRef L,
8926 OpenMPMapClauseKind MapType,
8927 ArrayRef<OpenMPMapModifierKind> MapModifiers,
8928 ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
8929 bool ReturnDevicePointer, bool IsImplicit, const ValueDecl *Mapper,
8930 const Expr *VarRef = nullptr, bool ForDeviceAddr = false) {
8931 if (SkipVarSet.contains(V: D))
8932 return;
8933 auto It = Info.try_emplace(Key: D, Args: Total).first;
8934 It->second[Kind].emplace_back(
8935 Args&: L, Args&: MapType, Args&: MapModifiers, Args&: MotionModifiers, Args&: ReturnDevicePointer,
8936 Args&: IsImplicit, Args&: Mapper, Args&: VarRef, Args&: ForDeviceAddr);
8937 };
8938
8939 for (const auto *Cl : Clauses) {
8940 const auto *C = dyn_cast<OMPMapClause>(Val: Cl);
8941 if (!C)
8942 continue;
8943 MapKind Kind = Other;
8944 if (llvm::is_contained(Range: C->getMapTypeModifiers(),
8945 Element: OMPC_MAP_MODIFIER_present))
8946 Kind = Present;
8947 else if (C->getMapType() == OMPC_MAP_alloc)
8948 Kind = Allocs;
8949 const auto *EI = C->getVarRefs().begin();
8950 for (const auto L : C->component_lists()) {
8951 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
8952 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), C->getMapType(),
8953 C->getMapTypeModifiers(), {},
8954 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(t: L),
8955 E);
8956 ++EI;
8957 }
8958 }
8959 for (const auto *Cl : Clauses) {
8960 const auto *C = dyn_cast<OMPToClause>(Val: Cl);
8961 if (!C)
8962 continue;
8963 MapKind Kind = Other;
8964 if (llvm::is_contained(Range: C->getMotionModifiers(),
8965 Element: OMPC_MOTION_MODIFIER_present))
8966 Kind = Present;
8967 if (llvm::is_contained(Range: C->getMotionModifiers(),
8968 Element: OMPC_MOTION_MODIFIER_iterator)) {
8969 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
8970 Val: C->getIteratorModifier()->IgnoreParenImpCasts())) {
8971 const auto *VD = cast<VarDecl>(Val: IteratorExpr->getIteratorDecl(I: 0));
8972 CGF.EmitVarDecl(D: *VD);
8973 }
8974 }
8975
8976 const auto *EI = C->getVarRefs().begin();
8977 for (const auto L : C->component_lists()) {
8978 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), OMPC_MAP_to, {},
8979 C->getMotionModifiers(), /*ReturnDevicePointer=*/false,
8980 C->isImplicit(), std::get<2>(t: L), *EI);
8981 ++EI;
8982 }
8983 }
8984 for (const auto *Cl : Clauses) {
8985 const auto *C = dyn_cast<OMPFromClause>(Val: Cl);
8986 if (!C)
8987 continue;
8988 MapKind Kind = Other;
8989 if (llvm::is_contained(Range: C->getMotionModifiers(),
8990 Element: OMPC_MOTION_MODIFIER_present))
8991 Kind = Present;
8992 if (llvm::is_contained(Range: C->getMotionModifiers(),
8993 Element: OMPC_MOTION_MODIFIER_iterator)) {
8994 if (auto *IteratorExpr = dyn_cast<OMPIteratorExpr>(
8995 Val: C->getIteratorModifier()->IgnoreParenImpCasts())) {
8996 const auto *VD = cast<VarDecl>(Val: IteratorExpr->getIteratorDecl(I: 0));
8997 CGF.EmitVarDecl(D: *VD);
8998 }
8999 }
9000
9001 const auto *EI = C->getVarRefs().begin();
9002 for (const auto L : C->component_lists()) {
9003 InfoGen(std::get<0>(t: L), Kind, std::get<1>(t: L), OMPC_MAP_from, {},
9004 C->getMotionModifiers(),
9005 /*ReturnDevicePointer=*/false, C->isImplicit(), std::get<2>(t: L),
9006 *EI);
9007 ++EI;
9008 }
9009 }
9010
9011 // Look at the use_device_ptr and use_device_addr clauses information and
9012 // mark the existing map entries as such. If there is no map information for
9013 // an entry in the use_device_ptr and use_device_addr list, we create one
9014 // with map type 'return_param' and zero size section. It is the user's
9015 // fault if that was not mapped before. If there is no map information, then
9016 // we defer the emission of that entry until all the maps for the same VD
9017 // have been handled.
9018 MapCombinedInfoTy UseDeviceDataCombinedInfo;
9019
9020 auto &&UseDeviceDataCombinedInfoGen =
9021 [&UseDeviceDataCombinedInfo](const ValueDecl *VD, llvm::Value *Ptr,
9022 CodeGenFunction &CGF, bool IsDevAddr,
9023 bool HasUdpFbNullify = false) {
9024 UseDeviceDataCombinedInfo.Exprs.push_back(Elt: VD);
9025 UseDeviceDataCombinedInfo.BasePointers.emplace_back(Args&: Ptr);
9026 UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(Args&: VD);
9027 UseDeviceDataCombinedInfo.DevicePointers.emplace_back(
9028 Args: IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);
9029 // FIXME: For use_device_addr on array-sections, this should
9030 // be the starting address of the section.
9031 // e.g. int *p;
9032 // ... use_device_addr(p[3])
9033 // &p[0], &p[3], /*size=*/0, RETURN_PARAM
9034 UseDeviceDataCombinedInfo.Pointers.push_back(Elt: Ptr);
9035 UseDeviceDataCombinedInfo.Sizes.push_back(
9036 Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
9037 OpenMPOffloadMappingFlags Flags =
9038 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9039 if (HasUdpFbNullify)
9040 Flags |= OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9041 UseDeviceDataCombinedInfo.Types.push_back(Elt: Flags);
9042 UseDeviceDataCombinedInfo.HasAttachPtr.push_back(Elt: false);
9043 UseDeviceDataCombinedInfo.Mappers.push_back(Elt: nullptr);
9044 };
9045
9046 auto &&MapInfoGen =
9047 [&UseDeviceDataCombinedInfoGen](
9048 CodeGenFunction &CGF, const Expr *IE, const ValueDecl *VD,
9049 OMPClauseMappableExprCommon::MappableExprComponentListRef
9050 Components,
9051 bool IsDevAddr, bool IEIsAttachPtrForDevAddr = false,
9052 bool HasUdpFbNullify = false) {
9053 // We didn't find any match in our map information - generate a zero
9054 // size array section.
9055 llvm::Value *Ptr;
9056 if (IsDevAddr && !IEIsAttachPtrForDevAddr) {
9057 if (IE->isGLValue())
9058 Ptr = CGF.EmitLValue(E: IE).getPointer(CGF);
9059 else
9060 Ptr = CGF.EmitScalarExpr(E: IE);
9061 } else {
9062 Ptr = CGF.EmitLoadOfScalar(lvalue: CGF.EmitLValue(E: IE), Loc: IE->getExprLoc());
9063 }
9064 bool TreatDevAddrAsDevPtr = IEIsAttachPtrForDevAddr;
9065 // For the purpose of address-translation, treat something like the
9066 // following:
9067 // int *p;
9068 // ... use_device_addr(p[1])
9069 // equivalent to
9070 // ... use_device_ptr(p)
9071 UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, /*IsDevAddr=*/IsDevAddr &&
9072 !TreatDevAddrAsDevPtr,
9073 HasUdpFbNullify);
9074 };
9075
9076 auto &&IsMapInfoExist =
9077 [&Info, this](CodeGenFunction &CGF, const ValueDecl *VD, const Expr *IE,
9078 const Expr *DesiredAttachPtrExpr, bool IsDevAddr,
9079 bool HasUdpFbNullify = false) -> bool {
9080 // We potentially have map information for this declaration already.
9081 // Look for the first set of components that refer to it. If found,
9082 // return true.
9083 // If the first component is a member expression, we have to look into
9084 // 'this', which maps to null in the map of map information. Otherwise
9085 // look directly for the information.
9086 auto It = Info.find(Key: isa<MemberExpr>(Val: IE) ? nullptr : VD);
9087 if (It != Info.end()) {
9088 bool Found = false;
9089 for (auto &Data : It->second) {
9090 MapInfo *CI = nullptr;
9091 // We potentially have multiple maps for the same decl. We need to
9092 // only consider those for which the attach-ptr matches the desired
9093 // attach-ptr.
9094 auto *It = llvm::find_if(Range&: Data, P: [&](const MapInfo &MI) {
9095 if (MI.Components.back().getAssociatedDeclaration() != VD)
9096 return false;
9097
9098 const Expr *MapAttachPtr = getAttachPtrExpr(Components: MI.Components);
9099 bool Match = AttachPtrComparator.areEqual(LHS: MapAttachPtr,
9100 RHS: DesiredAttachPtrExpr);
9101 return Match;
9102 });
9103
9104 if (It != Data.end())
9105 CI = &*It;
9106
9107 if (CI) {
9108 if (IsDevAddr) {
9109 CI->ForDeviceAddr = true;
9110 CI->ReturnDevicePointer = true;
9111 CI->HasUdpFbNullify = HasUdpFbNullify;
9112 Found = true;
9113 break;
9114 } else {
9115 auto PrevCI = std::next(x: CI->Components.rbegin());
9116 const auto *VarD = dyn_cast<VarDecl>(Val: VD);
9117 const Expr *AttachPtrExpr = getAttachPtrExpr(Components: CI->Components);
9118 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() ||
9119 isa<MemberExpr>(Val: IE) ||
9120 !VD->getType().getNonReferenceType()->isPointerType() ||
9121 PrevCI == CI->Components.rend() ||
9122 isa<MemberExpr>(Val: PrevCI->getAssociatedExpression()) || !VarD ||
9123 VarD->hasLocalStorage() ||
9124 (isa_and_nonnull<DeclRefExpr>(Val: AttachPtrExpr) &&
9125 VD == cast<DeclRefExpr>(Val: AttachPtrExpr)->getDecl())) {
9126 CI->ForDeviceAddr = IsDevAddr;
9127 CI->ReturnDevicePointer = true;
9128 CI->HasUdpFbNullify = HasUdpFbNullify;
9129 Found = true;
9130 break;
9131 }
9132 }
9133 }
9134 }
9135 return Found;
9136 }
9137 return false;
9138 };
9139
9140 // Look at the use_device_ptr clause information and mark the existing map
9141 // entries as such. If there is no map information for an entry in the
9142 // use_device_ptr list, we create one with map type 'alloc' and zero size
9143 // section. It is the user fault if that was not mapped before. If there is
9144 // no map information and the pointer is a struct member, then we defer the
9145 // emission of that entry until the whole struct has been processed.
9146 for (const auto *Cl : Clauses) {
9147 const auto *C = dyn_cast<OMPUseDevicePtrClause>(Val: Cl);
9148 if (!C)
9149 continue;
9150 bool HasUdpFbNullify =
9151 C->getFallbackModifier() == OMPC_USE_DEVICE_PTR_FALLBACK_fb_nullify;
9152 for (const auto L : C->component_lists()) {
9153 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9154 std::get<1>(t: L);
9155 assert(!Components.empty() &&
9156 "Not expecting empty list of components!");
9157 const ValueDecl *VD = Components.back().getAssociatedDeclaration();
9158 VD = cast<ValueDecl>(Val: VD->getCanonicalDecl());
9159 const Expr *IE = Components.back().getAssociatedExpression();
9160 // For use_device_ptr, we match an existing map clause if its attach-ptr
9161 // is same as the use_device_ptr operand. e.g.
9162 // map expr | use_device_ptr expr | current behavior
9163 // ---------|---------------------|-----------------
9164 // p[1] | p | match
9165 // ps->a | ps | match
9166 // p | p | no match
9167 const Expr *UDPOperandExpr =
9168 Components.front().getAssociatedExpression();
9169 if (IsMapInfoExist(CGF, VD, IE,
9170 /*DesiredAttachPtrExpr=*/UDPOperandExpr,
9171 /*IsDevAddr=*/false, HasUdpFbNullify))
9172 continue;
9173 MapInfoGen(CGF, IE, VD, Components, /*IsDevAddr=*/false,
9174 /*IEIsAttachPtrForDevAddr=*/false, HasUdpFbNullify);
9175 }
9176 }
9177
9178 llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;
9179 for (const auto *Cl : Clauses) {
9180 const auto *C = dyn_cast<OMPUseDeviceAddrClause>(Val: Cl);
9181 if (!C)
9182 continue;
9183 for (const auto L : C->component_lists()) {
9184 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9185 std::get<1>(t: L);
9186 assert(!std::get<1>(L).empty() &&
9187 "Not expecting empty list of components!");
9188 const ValueDecl *VD = std::get<1>(t: L).back().getAssociatedDeclaration();
9189 if (!Processed.insert(V: VD).second)
9190 continue;
9191 VD = cast<ValueDecl>(Val: VD->getCanonicalDecl());
9192 // For use_device_addr, we match an existing map clause if the
9193 // use_device_addr operand's attach-ptr matches the map operand's
9194 // attach-ptr.
9195 // We chould also restrict to only match cases when there is a full
9196 // match between the map/use_device_addr clause exprs, but that may be
9197 // unnecessary.
9198 //
9199 // map expr | use_device_addr expr | current | possible restrictive/
9200 // | | behavior | safer behavior
9201 // ---------|----------------------|-----------|-----------------------
9202 // p | p | match | match
9203 // p[0] | p[0] | match | match
9204 // p[0:1] | p[0] | match | no match
9205 // p[0:1] | p[2:1] | match | no match
9206 // p[1] | p[0] | match | no match
9207 // ps->a | ps->b | match | no match
9208 // p | p[0] | no match | no match
9209 // pp | pp[0][0] | no match | no match
9210 const Expr *UDAAttachPtrExpr = getAttachPtrExpr(Components);
9211 const Expr *IE = std::get<1>(t: L).back().getAssociatedExpression();
9212 assert((!UDAAttachPtrExpr || UDAAttachPtrExpr == IE) &&
9213 "use_device_addr operand has an attach-ptr, but does not match "
9214 "last component's expr.");
9215 if (IsMapInfoExist(CGF, VD, IE,
9216 /*DesiredAttachPtrExpr=*/UDAAttachPtrExpr,
9217 /*IsDevAddr=*/true))
9218 continue;
9219 MapInfoGen(CGF, IE, VD, Components,
9220 /*IsDevAddr=*/true,
9221 /*IEIsAttachPtrForDevAddr=*/UDAAttachPtrExpr != nullptr);
9222 }
9223 }
9224
9225 for (const auto &Data : Info) {
9226 MapCombinedInfoTy CurInfo;
9227 const Decl *D = Data.first;
9228 const ValueDecl *VD = cast_or_null<ValueDecl>(Val: D);
9229 // Group component lists by their AttachPtrExpr and process them in order
9230 // of increasing complexity (nullptr first, then simple expressions like
9231 // p, then more complex ones like p[0], etc.)
9232 //
9233 // This is similar to how generateInfoForCaptureFromClauseInfo handles
9234 // grouping for target constructs.
9235 SmallVector<std::pair<const Expr *, MapInfo>, 16> AttachPtrMapInfoPairs;
9236
9237 // First, collect all MapData entries with their attach-ptr exprs.
9238 for (const auto &M : Data.second) {
9239 for (const MapInfo &L : M) {
9240 assert(!L.Components.empty() &&
9241 "Not expecting declaration with no component lists.");
9242
9243 const Expr *AttachPtrExpr = getAttachPtrExpr(Components: L.Components);
9244 AttachPtrMapInfoPairs.emplace_back(Args&: AttachPtrExpr, Args: L);
9245 }
9246 }
9247
9248 // Next, sort by increasing order of their complexity.
9249 llvm::stable_sort(Range&: AttachPtrMapInfoPairs,
9250 C: [this](const auto &LHS, const auto &RHS) {
9251 return AttachPtrComparator(LHS.first, RHS.first);
9252 });
9253
9254 // And finally, process them all in order, grouping those with
9255 // equivalent attach-ptr exprs together.
9256 auto *It = AttachPtrMapInfoPairs.begin();
9257 while (It != AttachPtrMapInfoPairs.end()) {
9258 const Expr *AttachPtrExpr = It->first;
9259
9260 SmallVector<MapInfo, 8> GroupLists;
9261 while (It != AttachPtrMapInfoPairs.end() &&
9262 (It->first == AttachPtrExpr ||
9263 AttachPtrComparator.areEqual(LHS: It->first, RHS: AttachPtrExpr))) {
9264 GroupLists.push_back(Elt: It->second);
9265 ++It;
9266 }
9267 assert(!GroupLists.empty() && "GroupLists should not be empty");
9268
9269 StructRangeInfoTy PartialStruct;
9270 AttachInfoTy AttachInfo;
9271 MapCombinedInfoTy GroupCurInfo;
9272 // Current group's struct base information:
9273 MapCombinedInfoTy GroupStructBaseCurInfo;
9274 for (const MapInfo &L : GroupLists) {
9275 // Remember the current base pointer index.
9276 unsigned CurrentBasePointersIdx = GroupCurInfo.BasePointers.size();
9277 unsigned StructBasePointersIdx =
9278 GroupStructBaseCurInfo.BasePointers.size();
9279
9280 GroupCurInfo.NonContigInfo.IsNonContiguous =
9281 L.Components.back().isNonContiguous();
9282 generateInfoForComponentList(
9283 MapType: L.MapType, MapModifiers: L.MapModifiers, MotionModifiers: L.MotionModifiers, Components: L.Components,
9284 CombinedInfo&: GroupCurInfo, StructBaseCombinedInfo&: GroupStructBaseCurInfo, PartialStruct, AttachInfo,
9285 /*IsFirstComponentList=*/false, IsImplicit: L.IsImplicit,
9286 /*GenerateAllInfoForClauses*/ true, Mapper: L.Mapper, ForDeviceAddr: L.ForDeviceAddr, BaseDecl: VD,
9287 MapExpr: L.VarRef, /*OverlappedElements*/ {});
9288
9289 // If this entry relates to a device pointer, set the relevant
9290 // declaration and add the 'return pointer' flag.
9291 if (L.ReturnDevicePointer) {
9292 // Check whether a value was added to either GroupCurInfo or
9293 // GroupStructBaseCurInfo and error if no value was added to either
9294 // of them:
9295 assert((CurrentBasePointersIdx < GroupCurInfo.BasePointers.size() ||
9296 StructBasePointersIdx <
9297 GroupStructBaseCurInfo.BasePointers.size()) &&
9298 "Unexpected number of mapped base pointers.");
9299
9300 // Choose a base pointer index which is always valid:
9301 const ValueDecl *RelevantVD =
9302 L.Components.back().getAssociatedDeclaration();
9303 assert(RelevantVD &&
9304 "No relevant declaration related with device pointer??");
9305
9306 // If GroupStructBaseCurInfo has been updated this iteration then
9307 // work on the first new entry added to it i.e. make sure that when
9308 // multiple values are added to any of the lists, the first value
9309 // added is being modified by the assignments below (not the last
9310 // value added).
9311 auto SetDevicePointerInfo = [&](MapCombinedInfoTy &Info,
9312 unsigned Idx) {
9313 Info.DevicePtrDecls[Idx] = RelevantVD;
9314 Info.DevicePointers[Idx] = L.ForDeviceAddr
9315 ? DeviceInfoTy::Address
9316 : DeviceInfoTy::Pointer;
9317 Info.Types[Idx] |=
9318 OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
9319 if (L.HasUdpFbNullify)
9320 Info.Types[Idx] |=
9321 OpenMPOffloadMappingFlags::OMP_MAP_FB_NULLIFY;
9322 };
9323
9324 if (StructBasePointersIdx <
9325 GroupStructBaseCurInfo.BasePointers.size())
9326 SetDevicePointerInfo(GroupStructBaseCurInfo,
9327 StructBasePointersIdx);
9328 else
9329 SetDevicePointerInfo(GroupCurInfo, CurrentBasePointersIdx);
9330 }
9331 }
9332
9333 // Unify entries in one list making sure the struct mapping precedes the
9334 // individual fields:
9335 MapCombinedInfoTy GroupUnionCurInfo;
9336 GroupUnionCurInfo.append(CurInfo&: GroupStructBaseCurInfo);
9337 GroupUnionCurInfo.append(CurInfo&: GroupCurInfo);
9338
9339 // If there is an entry in PartialStruct it means we have a struct with
9340 // individual members mapped. Emit an extra combined entry.
9341 if (PartialStruct.Base.isValid()) {
9342 // Prepend a synthetic dimension of length 1 to represent the
9343 // aggregated struct object. Using 1 (not 0, as 0 produced an
9344 // incorrect non-contiguous descriptor (DimSize==1), causing the
9345 // non-contiguous motion clause path to be skipped.) is important:
9346 // * It preserves the correct rank so targetDataUpdate() computes
9347 // DimSize == 2 for cases like strided array sections originating
9348 // from user-defined mappers (e.g. test with s.data[0:8:2]).
9349 GroupUnionCurInfo.NonContigInfo.Dims.insert(
9350 I: GroupUnionCurInfo.NonContigInfo.Dims.begin(), Elt: 1);
9351 emitCombinedEntry(
9352 CombinedInfo&: CurInfo, CurTypes&: GroupUnionCurInfo.Types, PartialStruct, AttachInfo,
9353 /*IsMapThis=*/!VD, OMPBuilder, VD,
9354 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size(),
9355 /*NotTargetParams=*/true);
9356 }
9357
9358 // Append this group's results to the overall CurInfo in the correct
9359 // order: combined-entry -> original-field-entries -> attach-entry
9360 CurInfo.append(CurInfo&: GroupUnionCurInfo);
9361 if (AttachInfo.isValid())
9362 emitAttachEntry(CGF, CombinedInfo&: CurInfo, AttachInfo);
9363 }
9364
9365 // We need to append the results of this capture to what we already have.
9366 CombinedInfo.append(CurInfo);
9367 }
9368 // Append data for use_device_ptr/addr clauses.
9369 CombinedInfo.append(CurInfo&: UseDeviceDataCombinedInfo);
9370 }
9371
9372public:
9373 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
9374 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9375 // Extract firstprivate clause information.
9376 for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
9377 for (const auto *D : C->varlist())
9378 FirstPrivateDecls.try_emplace(
9379 Key: cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D)->getDecl()), Args: C->isImplicit());
9380 // Extract implicit firstprivates from uses_allocators clauses.
9381 for (const auto *C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) {
9382 for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
9383 OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
9384 if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(Val: D.AllocatorTraits))
9385 FirstPrivateDecls.try_emplace(Key: cast<VarDecl>(Val: DRE->getDecl()),
9386 /*Implicit=*/Args: true);
9387 else if (const auto *VD = dyn_cast<VarDecl>(
9388 Val: cast<DeclRefExpr>(Val: D.Allocator->IgnoreParenImpCasts())
9389 ->getDecl()))
9390 FirstPrivateDecls.try_emplace(Key: VD, /*Implicit=*/Args: true);
9391 }
9392 }
9393 // Extract defaultmap clause information.
9394 for (const auto *C : Dir.getClausesOfKind<OMPDefaultmapClause>())
9395 if (C->getDefaultmapModifier() == OMPC_DEFAULTMAP_MODIFIER_firstprivate)
9396 DefaultmapFirstprivateKinds.insert(V: C->getDefaultmapKind());
9397 // Extract device pointer clause information.
9398 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9399 for (auto L : C->component_lists())
9400 DevPointersMap[std::get<0>(t&: L)].push_back(Elt: std::get<1>(t&: L));
9401 // Extract device addr clause information.
9402 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9403 for (auto L : C->component_lists())
9404 HasDevAddrsMap[std::get<0>(t&: L)].push_back(Elt: std::get<1>(t&: L));
9405 // Extract map information.
9406 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>()) {
9407 if (C->getMapType() != OMPC_MAP_to)
9408 continue;
9409 for (auto L : C->component_lists()) {
9410 const ValueDecl *VD = std::get<0>(t&: L);
9411 const auto *RD = VD ? VD->getType()
9412 .getCanonicalType()
9413 .getNonReferenceType()
9414 ->getAsCXXRecordDecl()
9415 : nullptr;
9416 if (RD && RD->isLambda())
9417 LambdasMap.try_emplace(Key: std::get<0>(t&: L), Args&: C);
9418 }
9419 }
9420
9421 auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) {
9422 for (auto L : C->component_lists()) {
9423 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9424 std::get<1>(L);
9425 if (!Components.empty())
9426 collectAttachPtrExprInfo(Components, CurDir);
9427 }
9428 };
9429
9430 // Populate the AttachPtrExprMap for all component lists from map-related
9431 // clauses.
9432 for (const auto *C : Dir.getClausesOfKind<OMPMapClause>())
9433 CollectAttachPtrExprsForClauseComponents(C);
9434 for (const auto *C : Dir.getClausesOfKind<OMPToClause>())
9435 CollectAttachPtrExprsForClauseComponents(C);
9436 for (const auto *C : Dir.getClausesOfKind<OMPFromClause>())
9437 CollectAttachPtrExprsForClauseComponents(C);
9438 for (const auto *C : Dir.getClausesOfKind<OMPUseDevicePtrClause>())
9439 CollectAttachPtrExprsForClauseComponents(C);
9440 for (const auto *C : Dir.getClausesOfKind<OMPUseDeviceAddrClause>())
9441 CollectAttachPtrExprsForClauseComponents(C);
9442 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>())
9443 CollectAttachPtrExprsForClauseComponents(C);
9444 for (const auto *C : Dir.getClausesOfKind<OMPHasDeviceAddrClause>())
9445 CollectAttachPtrExprsForClauseComponents(C);
9446 }
9447
9448 /// Constructor for the declare mapper directive.
9449 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF)
9450 : CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
9451 auto CollectAttachPtrExprsForClauseComponents = [this](const auto *C) {
9452 for (auto L : C->component_lists()) {
9453 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
9454 std::get<1>(L);
9455 if (!Components.empty())
9456 collectAttachPtrExprInfo(Components, CurDir);
9457 }
9458 };
9459
9460 // Populate the AttachPtrExprMap for all component lists from map-related
9461 // clauses in the declare mapper directive, to enable attach-style mapping
9462 // for mappers.
9463 for (const auto *Cl : Dir.clauses()) {
9464 if (const auto *C = dyn_cast<OMPMapClause>(Val: Cl))
9465 CollectAttachPtrExprsForClauseComponents(C);
9466 else if (const auto *C = dyn_cast<OMPToClause>(Val: Cl))
9467 CollectAttachPtrExprsForClauseComponents(C);
9468 else if (const auto *C = dyn_cast<OMPFromClause>(Val: Cl))
9469 CollectAttachPtrExprsForClauseComponents(C);
9470 }
9471 }
9472
9473 /// Generate code for the combined entry if we have a partially mapped struct
9474 /// and take care of the mapping flags of the arguments corresponding to
9475 /// individual struct members.
9476 /// If a valid \p AttachInfo exists, its pointee addr will be updated to point
9477 /// to the combined-entry's begin address, if emitted.
9478 /// \p PartialStruct contains attach base-pointer information.
9479 /// \returns The index of the combined entry if one was added, std::nullopt
9480 /// otherwise.
9481 void emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,
9482 MapFlagsArrayTy &CurTypes,
9483 const StructRangeInfoTy &PartialStruct,
9484 AttachInfoTy &AttachInfo, bool IsMapThis,
9485 llvm::OpenMPIRBuilder &OMPBuilder, const ValueDecl *VD,
9486 unsigned OffsetForMemberOfFlag,
9487 bool NotTargetParams) const {
9488 if (CurTypes.size() == 1 &&
9489 ((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
9490 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&
9491 !PartialStruct.IsArraySection)
9492 return;
9493 Address LBAddr = PartialStruct.LowestElem.second;
9494 Address HBAddr = PartialStruct.HighestElem.second;
9495 if (PartialStruct.HasCompleteRecord) {
9496 LBAddr = PartialStruct.LB;
9497 HBAddr = PartialStruct.LB;
9498 }
9499 CombinedInfo.Exprs.push_back(Elt: VD);
9500 // Base is the base of the struct
9501 CombinedInfo.BasePointers.push_back(Elt: PartialStruct.Base.emitRawPointer(CGF));
9502 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9503 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9504 // Pointer is the address of the lowest element
9505 llvm::Value *LB = LBAddr.emitRawPointer(CGF);
9506 const CXXMethodDecl *MD =
9507 CGF.CurFuncDecl ? dyn_cast<CXXMethodDecl>(Val: CGF.CurFuncDecl) : nullptr;
9508 const CXXRecordDecl *RD = MD ? MD->getParent() : nullptr;
9509 bool HasBaseClass = RD && IsMapThis ? RD->getNumBases() > 0 : false;
9510 // There should not be a mapper for a combined entry.
9511 if (HasBaseClass) {
9512 // OpenMP 5.2 148:21:
9513 // If the target construct is within a class non-static member function,
9514 // and a variable is an accessible data member of the object for which the
9515 // non-static data member function is invoked, the variable is treated as
9516 // if the this[:1] expression had appeared in a map clause with a map-type
9517 // of tofrom.
9518 // Emit this[:1]
9519 CombinedInfo.Pointers.push_back(Elt: PartialStruct.Base.emitRawPointer(CGF));
9520 QualType Ty = MD->getFunctionObjectParameterType();
9521 llvm::Value *Size =
9522 CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty), DestTy: CGF.Int64Ty,
9523 /*isSigned=*/true);
9524 CombinedInfo.Sizes.push_back(Elt: Size);
9525 } else {
9526 CombinedInfo.Pointers.push_back(Elt: LB);
9527 // Size is (addr of {highest+1} element) - (addr of lowest element)
9528 llvm::Value *HB = HBAddr.emitRawPointer(CGF);
9529 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(
9530 Ty: HBAddr.getElementType(), Ptr: HB, /*Idx0=*/1);
9531 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(V: LB, DestTy: CGF.VoidPtrTy);
9532 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(V: HAddr, DestTy: CGF.VoidPtrTy);
9533 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(LHS: CHAddr, RHS: CLAddr);
9534 llvm::Value *Size = CGF.Builder.CreateIntCast(V: Diff, DestTy: CGF.Int64Ty,
9535 /*isSigned=*/false);
9536 CombinedInfo.Sizes.push_back(Elt: Size);
9537 }
9538 CombinedInfo.Mappers.push_back(Elt: nullptr);
9539 // Map type is always TARGET_PARAM, if generate info for captures.
9540 CombinedInfo.Types.push_back(
9541 Elt: NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE
9542 : !PartialStruct.PreliminaryMapData.BasePointers.empty()
9543 ? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ
9544 : OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
9545 // A combined entry has a base attach-ptr if its constituents do. e.g.:
9546 // map(s2.s1p->x, s2.s1p->y)
9547 // combined entry:
9548 // s2.s1p[0], s2.s1p->x, sizeof(s1p->x..y), ALLOC
9549 // here s2.s1p is the attach-ptr for the combined entry.
9550 // See the inline comments in emitUserDefinedMapper's definition for how
9551 // entries with an attach-ptr are treated.
9552 CombinedInfo.HasAttachPtr.push_back(Elt: AttachInfo.isValid());
9553 // If any element has the present modifier, then make sure the runtime
9554 // doesn't attempt to allocate the struct.
9555 if (CurTypes.end() !=
9556 llvm::find_if(Range&: CurTypes, P: [](OpenMPOffloadMappingFlags Type) {
9557 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9558 Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
9559 }))
9560 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;
9561 // Remove TARGET_PARAM flag from the first element
9562 (*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
9563 // If any element has the ompx_hold modifier, then make sure the runtime
9564 // uses the hold reference count for the struct as a whole so that it won't
9565 // be unmapped by an extra dynamic reference count decrement. Add it to all
9566 // elements as well so the runtime knows which reference count to check
9567 // when determining whether it's time for device-to-host transfers of
9568 // individual elements.
9569 if (CurTypes.end() !=
9570 llvm::find_if(Range&: CurTypes, P: [](OpenMPOffloadMappingFlags Type) {
9571 return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
9572 Type & OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);
9573 })) {
9574 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9575 for (auto &M : CurTypes)
9576 M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;
9577 }
9578
9579 // All other current entries will be MEMBER_OF the combined entry
9580 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9581 // 0xFFFF in the MEMBER_OF field, or ATTACH entries since they are expected
9582 // to be handled by themselves, after all other maps).
9583 OpenMPOffloadMappingFlags MemberOfFlag = OMPBuilder.getMemberOfFlag(
9584 Position: OffsetForMemberOfFlag + CombinedInfo.BasePointers.size() - 1);
9585 for (auto &M : CurTypes)
9586 OMPBuilder.setCorrectMemberOfFlag(Flags&: M, MemberOfFlag);
9587
9588 // When we are emitting a combined entry. If there were any pending
9589 // attachments to be done, we do them to the begin address of the combined
9590 // entry. Note that this means only one attachment per combined-entry will
9591 // be done. So, for instance, if we have:
9592 // S *ps;
9593 // ... map(ps->a, ps->b)
9594 // When we are emitting a combined entry. If AttachInfo is valid,
9595 // update the pointee address to point to the begin address of the combined
9596 // entry. This ensures that if we have multiple maps like:
9597 // `map(ps->a, ps->b)`, we still get a single ATTACH entry, like:
9598 //
9599 // &ps[0], &ps->a, sizeof(ps->a to ps->b), ALLOC // combined-entry
9600 // &ps[0], &ps->a, sizeof(ps->a), TO | FROM
9601 // &ps[0], &ps->b, sizeof(ps->b), TO | FROM
9602 // &ps, &ps->a, sizeof(void*), ATTACH // Use combined-entry's LB
9603 if (AttachInfo.isValid())
9604 AttachInfo.AttachPteeAddr = LBAddr;
9605 }
9606
9607 /// Generate all the base pointers, section pointers, sizes, map types, and
9608 /// mappers for the extracted mappable expressions (all included in \a
9609 /// CombinedInfo). Also, for each item that relates with a device pointer, a
9610 /// pair of the relevant declaration and index where it occurs is appended to
9611 /// the device pointers info array.
9612 void generateAllInfo(
9613 MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9614 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkipVarSet =
9615 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) const {
9616 assert(isa<const OMPExecutableDirective *>(CurDir) &&
9617 "Expect a executable directive");
9618 const auto *CurExecDir = cast<const OMPExecutableDirective *>(Val: CurDir);
9619 generateAllInfoForClauses(Clauses: CurExecDir->clauses(), CombinedInfo, OMPBuilder,
9620 SkipVarSet);
9621 }
9622
9623 /// Generate all the base pointers, section pointers, sizes, map types, and
9624 /// mappers for the extracted map clauses of user-defined mapper (all included
9625 /// in \a CombinedInfo).
9626 void generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,
9627 llvm::OpenMPIRBuilder &OMPBuilder) const {
9628 assert(isa<const OMPDeclareMapperDecl *>(CurDir) &&
9629 "Expect a declare mapper directive");
9630 const auto *CurMapperDir = cast<const OMPDeclareMapperDecl *>(Val: CurDir);
9631 generateAllInfoForClauses(Clauses: CurMapperDir->clauses(), CombinedInfo,
9632 OMPBuilder);
9633 }
9634
9635 /// Emit capture info for lambdas for variables captured by reference.
9636 void generateInfoForLambdaCaptures(
9637 const ValueDecl *VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,
9638 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const {
9639 QualType VDType = VD->getType().getCanonicalType().getNonReferenceType();
9640 const auto *RD = VDType->getAsCXXRecordDecl();
9641 if (!RD || !RD->isLambda())
9642 return;
9643 Address VDAddr(Arg, CGF.ConvertTypeForMem(T: VDType),
9644 CGF.getContext().getDeclAlign(D: VD));
9645 LValue VDLVal = CGF.MakeAddrLValue(Addr: VDAddr, T: VDType);
9646 llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;
9647 FieldDecl *ThisCapture = nullptr;
9648 RD->getCaptureFields(Captures, ThisCapture);
9649 if (ThisCapture) {
9650 LValue ThisLVal =
9651 CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: ThisCapture);
9652 LValue ThisLValVal = CGF.EmitLValueForField(Base: VDLVal, Field: ThisCapture);
9653 LambdaPointers.try_emplace(Key: ThisLVal.getPointer(CGF),
9654 Args: VDLVal.getPointer(CGF));
9655 CombinedInfo.Exprs.push_back(Elt: VD);
9656 CombinedInfo.BasePointers.push_back(Elt: ThisLVal.getPointer(CGF));
9657 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9658 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9659 CombinedInfo.Pointers.push_back(Elt: ThisLValVal.getPointer(CGF));
9660 CombinedInfo.Sizes.push_back(
9661 Elt: CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: CGF.getContext().VoidPtrTy),
9662 DestTy: CGF.Int64Ty, /*isSigned=*/true));
9663 CombinedInfo.Types.push_back(
9664 Elt: OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9665 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9666 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9667 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9668 CombinedInfo.HasAttachPtr.push_back(Elt: false);
9669 CombinedInfo.Mappers.push_back(Elt: nullptr);
9670 }
9671 for (const LambdaCapture &LC : RD->captures()) {
9672 if (!LC.capturesVariable())
9673 continue;
9674 const VarDecl *VD = cast<VarDecl>(Val: LC.getCapturedVar());
9675 if (LC.getCaptureKind() != LCK_ByRef && !VD->getType()->isPointerType())
9676 continue;
9677 auto It = Captures.find(Val: VD);
9678 assert(It != Captures.end() && "Found lambda capture without field.");
9679 LValue VarLVal = CGF.EmitLValueForFieldInitialization(Base: VDLVal, Field: It->second);
9680 if (LC.getCaptureKind() == LCK_ByRef) {
9681 LValue VarLValVal = CGF.EmitLValueForField(Base: VDLVal, Field: It->second);
9682 LambdaPointers.try_emplace(Key: VarLVal.getPointer(CGF),
9683 Args: VDLVal.getPointer(CGF));
9684 CombinedInfo.Exprs.push_back(Elt: VD);
9685 CombinedInfo.BasePointers.push_back(Elt: VarLVal.getPointer(CGF));
9686 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9687 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9688 CombinedInfo.Pointers.push_back(Elt: VarLValVal.getPointer(CGF));
9689 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
9690 V: CGF.getTypeSize(
9691 Ty: VD->getType().getCanonicalType().getNonReferenceType()),
9692 DestTy: CGF.Int64Ty, /*isSigned=*/true));
9693 } else {
9694 RValue VarRVal = CGF.EmitLoadOfLValue(V: VarLVal, Loc: RD->getLocation());
9695 LambdaPointers.try_emplace(Key: VarLVal.getPointer(CGF),
9696 Args: VDLVal.getPointer(CGF));
9697 CombinedInfo.Exprs.push_back(Elt: VD);
9698 CombinedInfo.BasePointers.push_back(Elt: VarLVal.getPointer(CGF));
9699 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
9700 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
9701 CombinedInfo.Pointers.push_back(Elt: VarRVal.getScalarVal());
9702 CombinedInfo.Sizes.push_back(Elt: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0));
9703 }
9704 CombinedInfo.Types.push_back(
9705 Elt: OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9706 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9707 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9708 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
9709 CombinedInfo.HasAttachPtr.push_back(Elt: false);
9710 CombinedInfo.Mappers.push_back(Elt: nullptr);
9711 }
9712 }
9713
9714 /// Set correct indices for lambdas captures.
9715 void adjustMemberOfForLambdaCaptures(
9716 llvm::OpenMPIRBuilder &OMPBuilder,
9717 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,
9718 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
9719 MapFlagsArrayTy &Types) const {
9720 for (unsigned I = 0, E = Types.size(); I < E; ++I) {
9721 // Set correct member_of idx for all implicit lambda captures.
9722 if (Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
9723 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
9724 OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
9725 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))
9726 continue;
9727 llvm::Value *BasePtr = LambdaPointers.lookup(Val: BasePointers[I]);
9728 assert(BasePtr && "Unable to find base lambda address.");
9729 int TgtIdx = -1;
9730 for (unsigned J = I; J > 0; --J) {
9731 unsigned Idx = J - 1;
9732 if (Pointers[Idx] != BasePtr)
9733 continue;
9734 TgtIdx = Idx;
9735 break;
9736 }
9737 assert(TgtIdx != -1 && "Unable to find parent lambda.");
9738 // All other current entries will be MEMBER_OF the combined entry
9739 // (except for PTR_AND_OBJ entries which do not have a placeholder value
9740 // 0xFFFF in the MEMBER_OF field).
9741 OpenMPOffloadMappingFlags MemberOfFlag =
9742 OMPBuilder.getMemberOfFlag(Position: TgtIdx);
9743 OMPBuilder.setCorrectMemberOfFlag(Flags&: Types[I], MemberOfFlag);
9744 }
9745 }
9746
9747 /// Populate component lists for non-lambda captured variables from map,
9748 /// is_device_ptr and has_device_addr clause info.
9749 void populateComponentListsForNonLambdaCaptureFromClauses(
9750 const ValueDecl *VD, MapDataArrayTy &DeclComponentLists,
9751 SmallVectorImpl<
9752 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
9753 &StorageForImplicitlyAddedComponentLists) const {
9754 if (VD && LambdasMap.count(Val: VD))
9755 return;
9756
9757 // For member fields list in is_device_ptr, store it in
9758 // DeclComponentLists for generating components info.
9759 static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;
9760 auto It = DevPointersMap.find(Val: VD);
9761 if (It != DevPointersMap.end())
9762 for (const auto &MCL : It->second)
9763 DeclComponentLists.emplace_back(Args: MCL, Args: OMPC_MAP_to, Args: Unknown,
9764 /*IsImpicit = */ Args: true, Args: nullptr,
9765 Args: nullptr);
9766 auto I = HasDevAddrsMap.find(Val: VD);
9767 if (I != HasDevAddrsMap.end())
9768 for (const auto &MCL : I->second)
9769 DeclComponentLists.emplace_back(Args: MCL, Args: OMPC_MAP_tofrom, Args: Unknown,
9770 /*IsImpicit = */ Args: true, Args: nullptr,
9771 Args: nullptr);
9772 assert(isa<const OMPExecutableDirective *>(CurDir) &&
9773 "Expect a executable directive");
9774 const auto *CurExecDir = cast<const OMPExecutableDirective *>(Val: CurDir);
9775 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) {
9776 const auto *EI = C->getVarRefs().begin();
9777 for (const auto L : C->decl_component_lists(VD)) {
9778 const ValueDecl *VDecl, *Mapper;
9779 // The Expression is not correct if the mapping is implicit
9780 const Expr *E = (C->getMapLoc().isValid()) ? *EI : nullptr;
9781 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
9782 std::tie(args&: VDecl, args&: Components, args&: Mapper) = L;
9783 assert(VDecl == VD && "We got information for the wrong declaration??");
9784 assert(!Components.empty() &&
9785 "Not expecting declaration with no component lists.");
9786 DeclComponentLists.emplace_back(Args&: Components, Args: C->getMapType(),
9787 Args: C->getMapTypeModifiers(),
9788 Args: C->isImplicit(), Args&: Mapper, Args&: E);
9789 ++EI;
9790 }
9791 }
9792
9793 // For the target construct, if there's a map with a base-pointer that's
9794 // a member of an implicitly captured struct, of the current class,
9795 // we need to emit an implicit map on the pointer.
9796 if (isOpenMPTargetExecutionDirective(DKind: CurExecDir->getDirectiveKind()))
9797 addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
9798 CapturedVD: VD, DeclComponentLists, ComponentVectorStorage&: StorageForImplicitlyAddedComponentLists);
9799
9800 llvm::stable_sort(Range&: DeclComponentLists, C: [](const MapData &LHS,
9801 const MapData &RHS) {
9802 ArrayRef<OpenMPMapModifierKind> MapModifiers = std::get<2>(t: LHS);
9803 OpenMPMapClauseKind MapType = std::get<1>(t: RHS);
9804 bool HasPresent =
9805 llvm::is_contained(Range&: MapModifiers, Element: clang::OMPC_MAP_MODIFIER_present);
9806 bool HasAllocs = MapType == OMPC_MAP_alloc;
9807 MapModifiers = std::get<2>(t: RHS);
9808 MapType = std::get<1>(t: LHS);
9809 bool HasPresentR =
9810 llvm::is_contained(Range&: MapModifiers, Element: clang::OMPC_MAP_MODIFIER_present);
9811 bool HasAllocsR = MapType == OMPC_MAP_alloc;
9812 return (HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);
9813 });
9814 }
9815
9816 /// On a target construct, if there's an implicit map on a struct, or that of
9817 /// this[:], and an explicit map with a member of that struct/class as the
9818 /// base-pointer, we need to make sure that base-pointer is implicitly mapped,
9819 /// to make sure we don't map the full struct/class. For example:
9820 ///
9821 /// \code
9822 /// struct S {
9823 /// int dummy[10000];
9824 /// int *p;
9825 /// void f1() {
9826 /// #pragma omp target map(p[0:1])
9827 /// (void)this;
9828 /// }
9829 /// }; S s;
9830 ///
9831 /// void f2() {
9832 /// #pragma omp target map(s.p[0:10])
9833 /// (void)s;
9834 /// }
9835 /// \endcode
9836 ///
9837 /// Only `this-p` and `s.p` should be mapped in the two cases above.
9838 //
9839 // OpenMP 6.0: 7.9.6 map clause, pg 285
9840 // If a list item with an implicitly determined data-mapping attribute does
9841 // not have any corresponding storage in the device data environment prior to
9842 // a task encountering the construct associated with the map clause, and one
9843 // or more contiguous parts of the original storage are either list items or
9844 // base pointers to list items that are explicitly mapped on the construct,
9845 // only those parts of the original storage will have corresponding storage in
9846 // the device data environment as a result of the map clauses on the
9847 // construct.
9848 void addImplicitMapForAttachPtrBaseIfMemberOfCapturedVD(
9849 const ValueDecl *CapturedVD, MapDataArrayTy &DeclComponentLists,
9850 SmallVectorImpl<
9851 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>>
9852 &ComponentVectorStorage) const {
9853 bool IsThisCapture = CapturedVD == nullptr;
9854
9855 for (const auto &ComponentsAndAttachPtr : AttachPtrExprMap) {
9856 OMPClauseMappableExprCommon::MappableExprComponentListRef
9857 ComponentsWithAttachPtr = ComponentsAndAttachPtr.first;
9858 const Expr *AttachPtrExpr = ComponentsAndAttachPtr.second;
9859 if (!AttachPtrExpr)
9860 continue;
9861
9862 const auto *ME = dyn_cast<MemberExpr>(Val: AttachPtrExpr);
9863 if (!ME)
9864 continue;
9865
9866 const Expr *Base = ME->getBase()->IgnoreParenImpCasts();
9867
9868 // If we are handling a "this" capture, then we are looking for
9869 // attach-ptrs of form `this->p`, either explicitly or implicitly.
9870 if (IsThisCapture && !ME->isImplicitCXXThis() && !isa<CXXThisExpr>(Val: Base))
9871 continue;
9872
9873 if (!IsThisCapture && (!isa<DeclRefExpr>(Val: Base) ||
9874 cast<DeclRefExpr>(Val: Base)->getDecl() != CapturedVD))
9875 continue;
9876
9877 // For non-this captures, we are looking for attach-ptrs of form
9878 // `s.p`.
9879 // For non-this captures, we are looking for attach-ptrs like `s.p`.
9880 if (!IsThisCapture && (ME->isArrow() || !isa<DeclRefExpr>(Val: Base) ||
9881 cast<DeclRefExpr>(Val: Base)->getDecl() != CapturedVD))
9882 continue;
9883
9884 // Check if we have an existing map on either:
9885 // this[:], s, this->p, or s.p, in which case, we don't need to add
9886 // an implicit one for the attach-ptr s.p/this->p.
9887 bool FoundExistingMap = false;
9888 for (const MapData &ExistingL : DeclComponentLists) {
9889 OMPClauseMappableExprCommon::MappableExprComponentListRef
9890 ExistingComponents = std::get<0>(t: ExistingL);
9891
9892 if (ExistingComponents.empty())
9893 continue;
9894
9895 // First check if we have a map like map(this->p) or map(s.p).
9896 const auto &FirstComponent = ExistingComponents.front();
9897 const Expr *FirstExpr = FirstComponent.getAssociatedExpression();
9898
9899 if (!FirstExpr)
9900 continue;
9901
9902 // First check if we have a map like map(this->p) or map(s.p).
9903 if (AttachPtrComparator.areEqual(LHS: FirstExpr, RHS: AttachPtrExpr)) {
9904 FoundExistingMap = true;
9905 break;
9906 }
9907
9908 // Check if we have a map like this[0:1]
9909 if (IsThisCapture) {
9910 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: FirstExpr)) {
9911 if (isa<CXXThisExpr>(Val: OASE->getBase()->IgnoreParenImpCasts())) {
9912 FoundExistingMap = true;
9913 break;
9914 }
9915 }
9916 continue;
9917 }
9918
9919 // When the attach-ptr is something like `s.p`, check if
9920 // `s` itself is mapped explicitly.
9921 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: FirstExpr)) {
9922 if (DRE->getDecl() == CapturedVD) {
9923 FoundExistingMap = true;
9924 break;
9925 }
9926 }
9927 }
9928
9929 if (FoundExistingMap)
9930 continue;
9931
9932 // If no base map is found, we need to create an implicit map for the
9933 // attach-pointer expr.
9934
9935 ComponentVectorStorage.emplace_back();
9936 auto &AttachPtrComponents = ComponentVectorStorage.back();
9937
9938 static const OpenMPMapModifierKind Unknown = OMPC_MAP_MODIFIER_unknown;
9939 bool SeenAttachPtrComponent = false;
9940 // For creating a map on the attach-ptr `s.p/this->p`, we copy all
9941 // components from the component-list which has `s.p/this->p`
9942 // as the attach-ptr, starting from the component which matches
9943 // `s.p/this->p`. This way, we'll have component-lists of
9944 // `s.p` -> `s`, and `this->p` -> `this`.
9945 for (size_t i = 0; i < ComponentsWithAttachPtr.size(); ++i) {
9946 const auto &Component = ComponentsWithAttachPtr[i];
9947 const Expr *ComponentExpr = Component.getAssociatedExpression();
9948
9949 if (!SeenAttachPtrComponent && ComponentExpr != AttachPtrExpr)
9950 continue;
9951 SeenAttachPtrComponent = true;
9952
9953 AttachPtrComponents.emplace_back(Args: Component.getAssociatedExpression(),
9954 Args: Component.getAssociatedDeclaration(),
9955 Args: Component.isNonContiguous());
9956 }
9957 assert(!AttachPtrComponents.empty() &&
9958 "Could not populate component-lists for mapping attach-ptr");
9959
9960 DeclComponentLists.emplace_back(
9961 Args&: AttachPtrComponents, Args: OMPC_MAP_tofrom, Args: Unknown,
9962 /*IsImplicit=*/Args: true, /*mapper=*/Args: nullptr, Args&: AttachPtrExpr);
9963 }
9964 }
9965
9966 /// For a capture that has an associated clause, generate the base pointers,
9967 /// section pointers, sizes, map types, and mappers (all included in
9968 /// \a CurCaptureVarInfo).
9969 void generateInfoForCaptureFromClauseInfo(
9970 const MapDataArrayTy &DeclComponentListsFromClauses,
9971 const CapturedStmt::Capture *Cap, llvm::Value *Arg,
9972 MapCombinedInfoTy &CurCaptureVarInfo, llvm::OpenMPIRBuilder &OMPBuilder,
9973 unsigned OffsetForMemberOfFlag) const {
9974 assert(!Cap->capturesVariableArrayType() &&
9975 "Not expecting to generate map info for a variable array type!");
9976
9977 // We need to know when we generating information for the first component
9978 const ValueDecl *VD = Cap->capturesThis()
9979 ? nullptr
9980 : Cap->getCapturedVar()->getCanonicalDecl();
9981
9982 // for map(to: lambda): skip here, processing it in
9983 // generateDefaultMapInfo
9984 if (LambdasMap.count(Val: VD))
9985 return;
9986
9987 // If this declaration appears in a is_device_ptr clause we just have to
9988 // pass the pointer by value. If it is a reference to a declaration, we just
9989 // pass its value.
9990 if (VD && (DevPointersMap.count(Val: VD) || HasDevAddrsMap.count(Val: VD))) {
9991 CurCaptureVarInfo.Exprs.push_back(Elt: VD);
9992 CurCaptureVarInfo.BasePointers.emplace_back(Args&: Arg);
9993 CurCaptureVarInfo.DevicePtrDecls.emplace_back(Args&: VD);
9994 CurCaptureVarInfo.DevicePointers.emplace_back(Args: DeviceInfoTy::Pointer);
9995 CurCaptureVarInfo.Pointers.push_back(Elt: Arg);
9996 CurCaptureVarInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
9997 V: CGF.getTypeSize(Ty: CGF.getContext().VoidPtrTy), DestTy: CGF.Int64Ty,
9998 /*isSigned=*/true));
9999 CurCaptureVarInfo.Types.push_back(
10000 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
10001 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);
10002 CurCaptureVarInfo.HasAttachPtr.push_back(Elt: false);
10003 CurCaptureVarInfo.Mappers.push_back(Elt: nullptr);
10004 return;
10005 }
10006
10007 auto GenerateInfoForComponentLists =
10008 [&](ArrayRef<MapData> DeclComponentListsFromClauses,
10009 bool IsEligibleForTargetParamFlag) {
10010 MapCombinedInfoTy CurInfoForComponentLists;
10011 StructRangeInfoTy PartialStruct;
10012 AttachInfoTy AttachInfo;
10013
10014 if (DeclComponentListsFromClauses.empty())
10015 return;
10016
10017 generateInfoForCaptureFromComponentLists(
10018 VD, DeclComponentLists: DeclComponentListsFromClauses, CurComponentListInfo&: CurInfoForComponentLists,
10019 PartialStruct, AttachInfo, IsListEligibleForTargetParamFlag: IsEligibleForTargetParamFlag);
10020
10021 // If there is an entry in PartialStruct it means we have a
10022 // struct with individual members mapped. Emit an extra combined
10023 // entry.
10024 if (PartialStruct.Base.isValid()) {
10025 CurCaptureVarInfo.append(CurInfo&: PartialStruct.PreliminaryMapData);
10026 emitCombinedEntry(
10027 CombinedInfo&: CurCaptureVarInfo, CurTypes&: CurInfoForComponentLists.Types,
10028 PartialStruct, AttachInfo, IsMapThis: Cap->capturesThis(), OMPBuilder,
10029 /*VD=*/nullptr, OffsetForMemberOfFlag,
10030 /*NotTargetParams*/ !IsEligibleForTargetParamFlag);
10031 }
10032
10033 // We do the appends to get the entries in the following order:
10034 // combined-entry -> individual-field-entries -> attach-entry,
10035 CurCaptureVarInfo.append(CurInfo&: CurInfoForComponentLists);
10036 if (AttachInfo.isValid())
10037 emitAttachEntry(CGF, CombinedInfo&: CurCaptureVarInfo, AttachInfo);
10038 };
10039
10040 // Group component lists by their AttachPtrExpr and process them in order
10041 // of increasing complexity (nullptr first, then simple expressions like p,
10042 // then more complex ones like p[0], etc.)
10043 //
10044 // This ensure that we:
10045 // * handle maps that can contribute towards setting the kernel argument,
10046 // (e.g. map(ps), or map(ps[0])), before any that cannot (e.g. ps->pt->d).
10047 // * allocate a single contiguous storage for all exprs with the same
10048 // captured var and having the same attach-ptr.
10049 //
10050 // Example: The map clauses below should be handled grouped together based
10051 // on their attachable-base-pointers:
10052 // map-clause | attachable-base-pointer
10053 // --------------------------+------------------------
10054 // map(p, ps) | nullptr
10055 // map(p[0]) | p
10056 // map(p[0]->b, p[0]->c) | p[0]
10057 // map(ps->d, ps->e, ps->pt) | ps
10058 // map(ps->pt->d, ps->pt->e) | ps->pt
10059
10060 // First, collect all MapData entries with their attach-ptr exprs.
10061 SmallVector<std::pair<const Expr *, MapData>, 16> AttachPtrMapDataPairs;
10062
10063 for (const MapData &L : DeclComponentListsFromClauses) {
10064 OMPClauseMappableExprCommon::MappableExprComponentListRef Components =
10065 std::get<0>(t: L);
10066 const Expr *AttachPtrExpr = getAttachPtrExpr(Components);
10067 AttachPtrMapDataPairs.emplace_back(Args&: AttachPtrExpr, Args: L);
10068 }
10069
10070 // Next, sort by increasing order of their complexity.
10071 llvm::stable_sort(Range&: AttachPtrMapDataPairs,
10072 C: [this](const auto &LHS, const auto &RHS) {
10073 return AttachPtrComparator(LHS.first, RHS.first);
10074 });
10075
10076 bool NoDefaultMappingDoneForVD = CurCaptureVarInfo.BasePointers.empty();
10077 bool IsFirstGroup = true;
10078
10079 // And finally, process them all in order, grouping those with
10080 // equivalent attach-ptr exprs together.
10081 auto *It = AttachPtrMapDataPairs.begin();
10082 while (It != AttachPtrMapDataPairs.end()) {
10083 const Expr *AttachPtrExpr = It->first;
10084
10085 MapDataArrayTy GroupLists;
10086 while (It != AttachPtrMapDataPairs.end() &&
10087 (It->first == AttachPtrExpr ||
10088 AttachPtrComparator.areEqual(LHS: It->first, RHS: AttachPtrExpr))) {
10089 GroupLists.push_back(Elt: It->second);
10090 ++It;
10091 }
10092 assert(!GroupLists.empty() && "GroupLists should not be empty");
10093
10094 // Determine if this group of component-lists is eligible for TARGET_PARAM
10095 // flag. Only the first group processed should be eligible, and only if no
10096 // default mapping was done.
10097 bool IsEligibleForTargetParamFlag =
10098 IsFirstGroup && NoDefaultMappingDoneForVD;
10099
10100 GenerateInfoForComponentLists(GroupLists, IsEligibleForTargetParamFlag);
10101 IsFirstGroup = false;
10102 }
10103 }
10104
10105 /// Generate the base pointers, section pointers, sizes, map types, and
10106 /// mappers associated to \a DeclComponentLists for a given capture
10107 /// \a VD (all included in \a CurComponentListInfo).
10108 void generateInfoForCaptureFromComponentLists(
10109 const ValueDecl *VD, ArrayRef<MapData> DeclComponentLists,
10110 MapCombinedInfoTy &CurComponentListInfo, StructRangeInfoTy &PartialStruct,
10111 AttachInfoTy &AttachInfo, bool IsListEligibleForTargetParamFlag) const {
10112 // Find overlapping elements (including the offset from the base element).
10113 llvm::SmallDenseMap<
10114 const MapData *,
10115 llvm::SmallVector<
10116 OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>,
10117 4>
10118 OverlappedData;
10119 size_t Count = 0;
10120 for (const MapData &L : DeclComponentLists) {
10121 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10122 OpenMPMapClauseKind MapType;
10123 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10124 bool IsImplicit;
10125 const ValueDecl *Mapper;
10126 const Expr *VarRef;
10127 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10128 L;
10129 ++Count;
10130 for (const MapData &L1 : ArrayRef(DeclComponentLists).slice(N: Count)) {
10131 OMPClauseMappableExprCommon::MappableExprComponentListRef Components1;
10132 std::tie(args&: Components1, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper,
10133 args&: VarRef) = L1;
10134 auto CI = Components.rbegin();
10135 auto CE = Components.rend();
10136 auto SI = Components1.rbegin();
10137 auto SE = Components1.rend();
10138 for (; CI != CE && SI != SE; ++CI, ++SI) {
10139 if (CI->getAssociatedExpression()->getStmtClass() !=
10140 SI->getAssociatedExpression()->getStmtClass())
10141 break;
10142 // Are we dealing with different variables/fields?
10143 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())
10144 break;
10145 }
10146 // Found overlapping if, at least for one component, reached the head
10147 // of the components list.
10148 if (CI == CE || SI == SE) {
10149 // Ignore it if it is the same component.
10150 if (CI == CE && SI == SE)
10151 continue;
10152 const auto It = (SI == SE) ? CI : SI;
10153 // If one component is a pointer and another one is a kind of
10154 // dereference of this pointer (array subscript, section, dereference,
10155 // etc.), it is not an overlapping.
10156 // Same, if one component is a base and another component is a
10157 // dereferenced pointer memberexpr with the same base.
10158 if (!isa<MemberExpr>(Val: It->getAssociatedExpression()) ||
10159 (std::prev(x: It)->getAssociatedDeclaration() &&
10160 std::prev(x: It)
10161 ->getAssociatedDeclaration()
10162 ->getType()
10163 ->isPointerType()) ||
10164 (It->getAssociatedDeclaration() &&
10165 It->getAssociatedDeclaration()->getType()->isPointerType() &&
10166 std::next(x: It) != CE && std::next(x: It) != SE))
10167 continue;
10168 const MapData &BaseData = CI == CE ? L : L1;
10169 OMPClauseMappableExprCommon::MappableExprComponentListRef SubData =
10170 SI == SE ? Components : Components1;
10171 OverlappedData[&BaseData].push_back(Elt: SubData);
10172 }
10173 }
10174 }
10175 // Sort the overlapped elements for each item.
10176 llvm::SmallVector<const FieldDecl *, 4> Layout;
10177 if (!OverlappedData.empty()) {
10178 const Type *BaseType = VD->getType().getCanonicalType().getTypePtr();
10179 const Type *OrigType = BaseType->getPointeeOrArrayElementType();
10180 while (BaseType != OrigType) {
10181 BaseType = OrigType->getCanonicalTypeInternal().getTypePtr();
10182 OrigType = BaseType->getPointeeOrArrayElementType();
10183 }
10184
10185 if (const auto *CRD = BaseType->getAsCXXRecordDecl())
10186 getPlainLayout(RD: CRD, Layout, /*AsBase=*/false);
10187 else {
10188 const auto *RD = BaseType->getAsRecordDecl();
10189 Layout.append(in_start: RD->field_begin(), in_end: RD->field_end());
10190 }
10191 }
10192 for (auto &Pair : OverlappedData) {
10193 llvm::stable_sort(
10194 Range&: Pair.getSecond(),
10195 C: [&Layout](
10196 OMPClauseMappableExprCommon::MappableExprComponentListRef First,
10197 OMPClauseMappableExprCommon::MappableExprComponentListRef
10198 Second) {
10199 auto CI = First.rbegin();
10200 auto CE = First.rend();
10201 auto SI = Second.rbegin();
10202 auto SE = Second.rend();
10203 for (; CI != CE && SI != SE; ++CI, ++SI) {
10204 if (CI->getAssociatedExpression()->getStmtClass() !=
10205 SI->getAssociatedExpression()->getStmtClass())
10206 break;
10207 // Are we dealing with different variables/fields?
10208 if (CI->getAssociatedDeclaration() !=
10209 SI->getAssociatedDeclaration())
10210 break;
10211 }
10212
10213 // Lists contain the same elements.
10214 if (CI == CE && SI == SE)
10215 return false;
10216
10217 // List with less elements is less than list with more elements.
10218 if (CI == CE || SI == SE)
10219 return CI == CE;
10220
10221 const auto *FD1 = cast<FieldDecl>(Val: CI->getAssociatedDeclaration());
10222 const auto *FD2 = cast<FieldDecl>(Val: SI->getAssociatedDeclaration());
10223 if (FD1->getParent() == FD2->getParent())
10224 return FD1->getFieldIndex() < FD2->getFieldIndex();
10225 const auto *It =
10226 llvm::find_if(Range&: Layout, P: [FD1, FD2](const FieldDecl *FD) {
10227 return FD == FD1 || FD == FD2;
10228 });
10229 return *It == FD1;
10230 });
10231 }
10232
10233 // Associated with a capture, because the mapping flags depend on it.
10234 // Go through all of the elements with the overlapped elements.
10235 bool AddTargetParamFlag = IsListEligibleForTargetParamFlag;
10236 MapCombinedInfoTy StructBaseCombinedInfo;
10237 for (const auto &Pair : OverlappedData) {
10238 const MapData &L = *Pair.getFirst();
10239 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10240 OpenMPMapClauseKind MapType;
10241 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10242 bool IsImplicit;
10243 const ValueDecl *Mapper;
10244 const Expr *VarRef;
10245 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10246 L;
10247 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef>
10248 OverlappedComponents = Pair.getSecond();
10249 generateInfoForComponentList(
10250 MapType, MapModifiers, MotionModifiers: {}, Components, CombinedInfo&: CurComponentListInfo,
10251 StructBaseCombinedInfo, PartialStruct, AttachInfo, IsFirstComponentList: AddTargetParamFlag,
10252 IsImplicit, /*GenerateAllInfoForClauses*/ false, Mapper,
10253 /*ForDeviceAddr=*/false, BaseDecl: VD, MapExpr: VarRef, OverlappedElements: OverlappedComponents);
10254 AddTargetParamFlag = false;
10255 }
10256 // Go through other elements without overlapped elements.
10257 for (const MapData &L : DeclComponentLists) {
10258 OMPClauseMappableExprCommon::MappableExprComponentListRef Components;
10259 OpenMPMapClauseKind MapType;
10260 ArrayRef<OpenMPMapModifierKind> MapModifiers;
10261 bool IsImplicit;
10262 const ValueDecl *Mapper;
10263 const Expr *VarRef;
10264 std::tie(args&: Components, args&: MapType, args&: MapModifiers, args&: IsImplicit, args&: Mapper, args&: VarRef) =
10265 L;
10266 auto It = OverlappedData.find(Val: &L);
10267 if (It == OverlappedData.end())
10268 generateInfoForComponentList(
10269 MapType, MapModifiers, MotionModifiers: {}, Components, CombinedInfo&: CurComponentListInfo,
10270 StructBaseCombinedInfo, PartialStruct, AttachInfo,
10271 IsFirstComponentList: AddTargetParamFlag, IsImplicit, /*GenerateAllInfoForClauses*/ false,
10272 Mapper, /*ForDeviceAddr=*/false, BaseDecl: VD, MapExpr: VarRef,
10273 /*OverlappedElements*/ {});
10274 AddTargetParamFlag = false;
10275 }
10276 }
10277
10278 /// Check if a variable should be treated as firstprivate due to explicit
10279 /// firstprivate clause or defaultmap(firstprivate:...).
10280 bool isEffectivelyFirstprivate(const VarDecl *VD, QualType Type) const {
10281 // Check explicit firstprivate clauses (not implicit from defaultmap)
10282 auto I = FirstPrivateDecls.find(Val: VD);
10283 if (I != FirstPrivateDecls.end() && !I->getSecond())
10284 return true; // Explicit firstprivate only
10285
10286 // Check defaultmap(firstprivate:scalar) for scalar types
10287 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_scalar)) {
10288 if (Type->isScalarType())
10289 return true;
10290 }
10291
10292 // Check defaultmap(firstprivate:pointer) for pointer types
10293 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_pointer)) {
10294 if (Type->isAnyPointerType())
10295 return true;
10296 }
10297
10298 // Check defaultmap(firstprivate:aggregate) for aggregate types
10299 if (DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_aggregate)) {
10300 if (Type->isAggregateType())
10301 return true;
10302 }
10303
10304 // Check defaultmap(firstprivate:all) for all types
10305 return DefaultmapFirstprivateKinds.count(V: OMPC_DEFAULTMAP_all);
10306 }
10307
10308 /// Generate the default map information for a given capture \a CI,
10309 /// record field declaration \a RI and captured value \a CV.
10310 void generateDefaultMapInfo(const CapturedStmt::Capture &CI,
10311 const FieldDecl &RI, llvm::Value *CV,
10312 MapCombinedInfoTy &CombinedInfo) const {
10313 bool IsImplicit = true;
10314 // Do the default mapping.
10315 if (CI.capturesThis()) {
10316 CombinedInfo.Exprs.push_back(Elt: nullptr);
10317 CombinedInfo.BasePointers.push_back(Elt: CV);
10318 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10319 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10320 CombinedInfo.Pointers.push_back(Elt: CV);
10321 const auto *PtrTy = cast<PointerType>(Val: RI.getType().getTypePtr());
10322 CombinedInfo.Sizes.push_back(
10323 Elt: CGF.Builder.CreateIntCast(V: CGF.getTypeSize(Ty: PtrTy->getPointeeType()),
10324 DestTy: CGF.Int64Ty, /*isSigned=*/true));
10325 // Default map type.
10326 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_TO |
10327 OpenMPOffloadMappingFlags::OMP_MAP_FROM);
10328 } else if (CI.capturesVariableByCopy()) {
10329 const VarDecl *VD = CI.getCapturedVar();
10330 CombinedInfo.Exprs.push_back(Elt: VD->getCanonicalDecl());
10331 CombinedInfo.BasePointers.push_back(Elt: CV);
10332 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10333 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10334 CombinedInfo.Pointers.push_back(Elt: CV);
10335 bool IsFirstprivate =
10336 isEffectivelyFirstprivate(VD, Type: RI.getType().getNonReferenceType());
10337
10338 if (!RI.getType()->isAnyPointerType()) {
10339 // We have to signal to the runtime captures passed by value that are
10340 // not pointers.
10341 CombinedInfo.Types.push_back(
10342 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10343 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10344 V: CGF.getTypeSize(Ty: RI.getType()), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10345 } else if (IsFirstprivate) {
10346 // Firstprivate pointers should be passed by value (as literals)
10347 // without performing a present table lookup at runtime.
10348 CombinedInfo.Types.push_back(
10349 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10350 // Use zero size for pointer literals (just passing the pointer value)
10351 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10352 } else {
10353 // Pointers are implicitly mapped with a zero size and no flags
10354 // (other than first map that is added for all implicit maps).
10355 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_NONE);
10356 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10357 }
10358 auto I = FirstPrivateDecls.find(Val: VD);
10359 if (I != FirstPrivateDecls.end())
10360 IsImplicit = I->getSecond();
10361 } else {
10362 assert(CI.capturesVariable() && "Expected captured reference.");
10363 const auto *PtrTy = cast<ReferenceType>(Val: RI.getType().getTypePtr());
10364 QualType ElementType = PtrTy->getPointeeType();
10365 const VarDecl *VD = CI.getCapturedVar();
10366 bool IsFirstprivate = isEffectivelyFirstprivate(VD, Type: ElementType);
10367 CombinedInfo.Exprs.push_back(Elt: VD->getCanonicalDecl());
10368 CombinedInfo.BasePointers.push_back(Elt: CV);
10369 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
10370 CombinedInfo.DevicePointers.push_back(Elt: DeviceInfoTy::None);
10371
10372 // For firstprivate pointers, pass by value instead of dereferencing
10373 if (IsFirstprivate && ElementType->isAnyPointerType()) {
10374 // Treat as a literal value (pass the pointer value itself)
10375 CombinedInfo.Pointers.push_back(Elt: CV);
10376 // Use zero size for pointer literals
10377 CombinedInfo.Sizes.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int64Ty));
10378 CombinedInfo.Types.push_back(
10379 Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
10380 } else {
10381 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10382 V: CGF.getTypeSize(Ty: ElementType), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10383 // The default map type for a scalar/complex type is 'to' because by
10384 // default the value doesn't have to be retrieved. For an aggregate
10385 // type, the default is 'tofrom'.
10386 CombinedInfo.Types.push_back(Elt: getMapModifiersForPrivateClauses(Cap: CI));
10387 CombinedInfo.Pointers.push_back(Elt: CV);
10388 }
10389 auto I = FirstPrivateDecls.find(Val: VD);
10390 if (I != FirstPrivateDecls.end())
10391 IsImplicit = I->getSecond();
10392 }
10393 // Every default map produces a single argument which is a target parameter.
10394 CombinedInfo.Types.back() |=
10395 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;
10396
10397 // Add flag stating this is an implicit map.
10398 if (IsImplicit)
10399 CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;
10400
10401 CombinedInfo.HasAttachPtr.push_back(Elt: false);
10402 // No user-defined mapper for default mapping.
10403 CombinedInfo.Mappers.push_back(Elt: nullptr);
10404 }
10405};
10406} // anonymous namespace
10407
10408// Try to extract the base declaration from a `this->x` expression if possible.
10409static ValueDecl *getDeclFromThisExpr(const Expr *E) {
10410 if (!E)
10411 return nullptr;
10412
10413 if (const auto *OASE = dyn_cast<ArraySectionExpr>(Val: E->IgnoreParenCasts()))
10414 if (const MemberExpr *ME =
10415 dyn_cast<MemberExpr>(Val: OASE->getBase()->IgnoreParenImpCasts()))
10416 return ME->getMemberDecl();
10417 return nullptr;
10418}
10419
10420/// Emit a string constant containing the names of the values mapped to the
10421/// offloading runtime library.
10422static llvm::Constant *
10423emitMappingInformation(CodeGenFunction &CGF, llvm::OpenMPIRBuilder &OMPBuilder,
10424 MappableExprsHandler::MappingExprInfo &MapExprs) {
10425
10426 uint32_t SrcLocStrSize;
10427 if (!MapExprs.getMapDecl() && !MapExprs.getMapExpr())
10428 return OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
10429
10430 SourceLocation Loc;
10431 if (!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {
10432 if (const ValueDecl *VD = getDeclFromThisExpr(E: MapExprs.getMapExpr()))
10433 Loc = VD->getLocation();
10434 else
10435 Loc = MapExprs.getMapExpr()->getExprLoc();
10436 } else {
10437 Loc = MapExprs.getMapDecl()->getLocation();
10438 }
10439
10440 std::string ExprName;
10441 if (MapExprs.getMapExpr()) {
10442 PrintingPolicy P(CGF.getContext().getLangOpts());
10443 llvm::raw_string_ostream OS(ExprName);
10444 MapExprs.getMapExpr()->printPretty(OS, Helper: nullptr, Policy: P);
10445 } else {
10446 ExprName = MapExprs.getMapDecl()->getNameAsString();
10447 }
10448
10449 std::string FileName;
10450 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
10451 if (auto *DbgInfo = CGF.getDebugInfo())
10452 FileName = DbgInfo->remapDIPath(PLoc.getFilename());
10453 else
10454 FileName = PLoc.getFilename();
10455 return OMPBuilder.getOrCreateSrcLocStr(FunctionName: FileName, FileName: ExprName, Line: PLoc.getLine(),
10456 Column: PLoc.getColumn(), SrcLocStrSize);
10457}
10458/// Emit the arrays used to pass the captures and map information to the
10459/// offloading runtime library. If there is no map or capture information,
10460/// return nullptr by reference.
10461static void emitOffloadingArraysAndArgs(
10462 CodeGenFunction &CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
10463 CGOpenMPRuntime::TargetDataInfo &Info, llvm::OpenMPIRBuilder &OMPBuilder,
10464 bool IsNonContiguous = false, bool ForEndCall = false) {
10465 CodeGenModule &CGM = CGF.CGM;
10466
10467 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
10468 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
10469 CGF.AllocaInsertPt->getIterator());
10470 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
10471 CGF.Builder.GetInsertPoint());
10472
10473 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
10474 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
10475 Info.CaptureDeviceAddrMap.try_emplace(Key: DevVD, Args&: NewDecl);
10476 }
10477 };
10478
10479 auto CustomMapperCB = [&](unsigned int I) {
10480 llvm::Function *MFunc = nullptr;
10481 if (CombinedInfo.Mappers[I]) {
10482 Info.HasMapper = true;
10483 MFunc = CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
10484 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
10485 }
10486 return MFunc;
10487 };
10488 cantFail(Err: OMPBuilder.emitOffloadingArraysAndArgs(
10489 AllocaIP, CodeGenIP, Info, RTArgs&: Info.RTArgs, CombinedInfo, CustomMapperCB,
10490 IsNonContiguous, ForEndCall, DeviceAddrCB));
10491}
10492
10493/// Check for inner distribute directive.
10494static const OMPExecutableDirective *
10495getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
10496 const auto *CS = D.getInnermostCapturedStmt();
10497 const auto *Body =
10498 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
10499 const Stmt *ChildStmt =
10500 CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
10501
10502 if (const auto *NestedDir =
10503 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
10504 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
10505 switch (D.getDirectiveKind()) {
10506 case OMPD_target:
10507 // For now, treat 'target' with nested 'teams loop' as if it's
10508 // distributed (target teams distribute).
10509 if (isOpenMPDistributeDirective(DKind) || DKind == OMPD_teams_loop)
10510 return NestedDir;
10511 if (DKind == OMPD_teams) {
10512 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
10513 /*IgnoreCaptured=*/true);
10514 if (!Body)
10515 return nullptr;
10516 ChildStmt = CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body);
10517 if (const auto *NND =
10518 dyn_cast_or_null<OMPExecutableDirective>(Val: ChildStmt)) {
10519 DKind = NND->getDirectiveKind();
10520 if (isOpenMPDistributeDirective(DKind))
10521 return NND;
10522 }
10523 }
10524 return nullptr;
10525 case OMPD_target_teams:
10526 if (isOpenMPDistributeDirective(DKind))
10527 return NestedDir;
10528 return nullptr;
10529 case OMPD_target_parallel:
10530 case OMPD_target_simd:
10531 case OMPD_target_parallel_for:
10532 case OMPD_target_parallel_for_simd:
10533 return nullptr;
10534 case OMPD_target_teams_distribute:
10535 case OMPD_target_teams_distribute_simd:
10536 case OMPD_target_teams_distribute_parallel_for:
10537 case OMPD_target_teams_distribute_parallel_for_simd:
10538 case OMPD_parallel:
10539 case OMPD_for:
10540 case OMPD_parallel_for:
10541 case OMPD_parallel_master:
10542 case OMPD_parallel_sections:
10543 case OMPD_for_simd:
10544 case OMPD_parallel_for_simd:
10545 case OMPD_cancel:
10546 case OMPD_cancellation_point:
10547 case OMPD_ordered_standalone:
10548 case OMPD_ordered_blockassoc:
10549 case OMPD_threadprivate:
10550 case OMPD_allocate:
10551 case OMPD_task:
10552 case OMPD_simd:
10553 case OMPD_tile:
10554 case OMPD_unroll:
10555 case OMPD_sections:
10556 case OMPD_section:
10557 case OMPD_single:
10558 case OMPD_master:
10559 case OMPD_critical:
10560 case OMPD_taskyield:
10561 case OMPD_barrier:
10562 case OMPD_taskwait:
10563 case OMPD_taskgroup:
10564 case OMPD_atomic:
10565 case OMPD_flush:
10566 case OMPD_depobj:
10567 case OMPD_scan:
10568 case OMPD_teams:
10569 case OMPD_target_data:
10570 case OMPD_target_exit_data:
10571 case OMPD_target_enter_data:
10572 case OMPD_distribute:
10573 case OMPD_distribute_simd:
10574 case OMPD_distribute_parallel_for:
10575 case OMPD_distribute_parallel_for_simd:
10576 case OMPD_teams_distribute:
10577 case OMPD_teams_distribute_simd:
10578 case OMPD_teams_distribute_parallel_for:
10579 case OMPD_teams_distribute_parallel_for_simd:
10580 case OMPD_target_update:
10581 case OMPD_declare_simd:
10582 case OMPD_declare_variant:
10583 case OMPD_begin_declare_variant:
10584 case OMPD_end_declare_variant:
10585 case OMPD_declare_target:
10586 case OMPD_end_declare_target:
10587 case OMPD_declare_reduction:
10588 case OMPD_declare_mapper:
10589 case OMPD_taskloop:
10590 case OMPD_taskloop_simd:
10591 case OMPD_master_taskloop:
10592 case OMPD_master_taskloop_simd:
10593 case OMPD_parallel_master_taskloop:
10594 case OMPD_parallel_master_taskloop_simd:
10595 case OMPD_requires:
10596 case OMPD_metadirective:
10597 case OMPD_unknown:
10598 default:
10599 llvm_unreachable("Unexpected directive.");
10600 }
10601 }
10602
10603 return nullptr;
10604}
10605
10606/// Emit the user-defined mapper function. The code generation follows the
10607/// pattern in the example below.
10608/// \code
10609/// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle,
10610/// void *base, void *begin,
10611/// int64_t size, int64_t type,
10612/// void *name = nullptr) {
10613/// // Allocate space for an array section first.
10614/// if ((size > 1 || (base != begin)) && !maptype.IsDelete)
10615/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10616/// size*sizeof(Ty), clearToFromMember(type));
10617/// // Map members.
10618/// for (unsigned i = 0; i < size; i++) {
10619/// N = __tgt_mapper_num_components(rt_mapper_handle);
10620/// // For each component specified by this mapper:
10621/// for (auto c : begin[i]->all_components) {
10622/// // MEMBER_OF grouping: tie this component to the current array element
10623/// // (component N) by adding N<<48. Exceptions:
10624/// // - ATTACH entries are not members of any struct storage range.
10625/// // - Pointee entries (reached via a pointer member) occupy separate
10626/// // storage; their inner MEMBER_OF bits are shifted by N instead.
10627/// if (c.isAttach() || c.isPointee())
10628/// member_type = c.arg_type + (c.hasInnerMemberOf() ? N<<48 : 0);
10629/// else
10630/// member_type = c.arg_type + N<<48;
10631/// // Map-type-modifying bits (ALWAYS, DELETE, CLOSE) from the outer map
10632/// // clause are propagated to each component, except ATTACH entries
10633/// // (ATTACH|ALWAYS is reserved for attach(always), and other modifier
10634/// // bits have no meaning for ATTACH). PRESENT is additionally
10635/// // propagated to components with HasAttachPtr (the pointee data) at
10636/// // OpenMP >= 6.0.
10637/// present_bit = (v60 && c.hasAttachPtr()) ? PRESENT : 0;
10638/// imported_modifier_bits =
10639/// type & (ALWAYS | DELETE | CLOSE | present_bit);
10640/// effective_type = c.isAttach() ? member_type
10641/// : member_type | imported_modifier_bits;
10642/// if (c.hasMapper())
10643/// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size,
10644/// effective_type, c.arg_name);
10645/// else
10646/// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base,
10647/// c.arg_begin, c.arg_size, effective_type,
10648/// c.arg_name);
10649/// }
10650/// }
10651/// // Delete the array section.
10652/// if (size > 1 && maptype.IsDelete)
10653/// __tgt_push_mapper_component(rt_mapper_handle, base, begin,
10654/// size*sizeof(Ty), clearToFromMember(type));
10655/// }
10656/// \endcode
10657void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
10658 CodeGenFunction *CGF) {
10659 if (UDMMap.count(Val: D) > 0)
10660 return;
10661 ASTContext &C = CGM.getContext();
10662 QualType Ty = D->getType();
10663 auto *MapperVarDecl =
10664 cast<VarDecl>(Val: cast<DeclRefExpr>(Val: D->getMapperVarRef())->getDecl());
10665 CharUnits ElementSize = C.getTypeSizeInChars(T: Ty);
10666 llvm::Type *ElemTy = CGM.getTypes().ConvertTypeForMem(T: Ty);
10667
10668 CodeGenFunction MapperCGF(CGM);
10669 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
10670 auto PrivatizeAndGenMapInfoCB =
10671 [&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI,
10672 llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & {
10673 MapperCGF.Builder.restoreIP(IP: CodeGenIP);
10674
10675 // Privatize the declared variable of mapper to be the current array
10676 // element.
10677 Address PtrCurrent(
10678 PtrPHI, ElemTy,
10679 Address(BeginArg, MapperCGF.VoidPtrTy, CGM.getPointerAlign())
10680 .getAlignment()
10681 .alignmentOfArrayElement(elementSize: ElementSize));
10682 CodeGenFunction::OMPPrivateScope Scope(MapperCGF);
10683 Scope.addPrivate(LocalVD: MapperVarDecl, Addr: PtrCurrent);
10684 (void)Scope.Privatize();
10685
10686 // Get map clause information.
10687 MappableExprsHandler MEHandler(*D, MapperCGF);
10688 MEHandler.generateAllInfoForMapper(CombinedInfo, OMPBuilder);
10689
10690 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10691 return emitMappingInformation(CGF&: MapperCGF, OMPBuilder, MapExprs&: MapExpr);
10692 };
10693 if (CGM.getCodeGenOpts().getDebugInfo() !=
10694 llvm::codegenoptions::NoDebugInfo) {
10695 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
10696 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
10697 F: FillInfoMap);
10698 }
10699
10700 return CombinedInfo;
10701 };
10702
10703 auto CustomMapperCB = [&](unsigned I) {
10704 llvm::Function *MapperFunc = nullptr;
10705 if (CombinedInfo.Mappers[I]) {
10706 // Call the corresponding mapper function.
10707 MapperFunc = getOrCreateUserDefinedMapperFunc(
10708 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
10709 assert(MapperFunc && "Expect a valid mapper function is available.");
10710 }
10711 return MapperFunc;
10712 };
10713
10714 SmallString<64> TyStr;
10715 llvm::raw_svector_ostream Out(TyStr);
10716 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: Ty, Out);
10717 std::string Name = getName(Parts: {"omp_mapper", TyStr, D->getName()});
10718
10719 // Propagate the PRESENT modifier to the pointee entries (those with
10720 // HasAttachPtr) only for OpenMP >= 6.0; before 6.0 the present modifier does
10721 // not apply to the pointee (see the OpenMP 6.0 erratum on the present motion
10722 // vs. map-type modifier divergence).
10723 bool PropagatePresentToPointee = CGM.getLangOpts().OpenMP >= 60;
10724 llvm::Function *NewFn = cantFail(ValOrErr: OMPBuilder.emitUserDefinedMapper(
10725 PrivAndGenMapInfoCB: PrivatizeAndGenMapInfoCB, ElemTy, FuncName: Name, CustomMapperCB,
10726 /*PreserveMemberOfFlags=*/false, PropagatePresentToPointee));
10727 UDMMap.try_emplace(Key: D, Args&: NewFn);
10728 if (CGF)
10729 FunctionUDMMap[CGF->CurFn].push_back(Elt: D);
10730}
10731
10732llvm::Function *CGOpenMPRuntime::getOrCreateUserDefinedMapperFunc(
10733 const OMPDeclareMapperDecl *D) {
10734 auto I = UDMMap.find(Val: D);
10735 if (I != UDMMap.end())
10736 return I->second;
10737 emitUserDefinedMapper(D);
10738 return UDMMap.lookup(Val: D);
10739}
10740
10741llvm::Value *CGOpenMPRuntime::emitTargetNumIterationsCall(
10742 CodeGenFunction &CGF, const OMPExecutableDirective &D,
10743 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
10744 const OMPLoopDirective &D)>
10745 SizeEmitter) {
10746 OpenMPDirectiveKind Kind = D.getDirectiveKind();
10747 const OMPExecutableDirective *TD = &D;
10748 // Get nested teams distribute kind directive, if any. For now, treat
10749 // 'target_teams_loop' as if it's really a target_teams_distribute.
10750 if ((!isOpenMPDistributeDirective(DKind: Kind) || !isOpenMPTeamsDirective(DKind: Kind)) &&
10751 Kind != OMPD_target_teams_loop)
10752 TD = getNestedDistributeDirective(Ctx&: CGM.getContext(), D);
10753 if (!TD)
10754 return llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
10755
10756 const auto *LD = cast<OMPLoopDirective>(Val: TD);
10757 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD))
10758 return NumIterations;
10759 return llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: 0);
10760}
10761
10762static void
10763emitTargetCallFallback(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
10764 const OMPExecutableDirective &D,
10765 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
10766 bool RequiresOuterTask, const CapturedStmt &CS,
10767 bool OffloadingMandatory, CodeGenFunction &CGF) {
10768 if (OffloadingMandatory) {
10769 CGF.Builder.CreateUnreachable();
10770 } else {
10771 if (RequiresOuterTask) {
10772 CapturedVars.clear();
10773 CGF.GenerateOpenMPCapturedVars(S: CS, CapturedVars);
10774 }
10775 llvm::SmallVector<llvm::Value *, 16> Args(CapturedVars.begin(),
10776 CapturedVars.end());
10777 Args.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Builder.getPtrTy()));
10778 OMPRuntime->emitOutlinedFunctionCall(CGF, Loc: D.getBeginLoc(), OutlinedFn,
10779 Args);
10780 }
10781}
10782
10783static llvm::Value *emitDeviceID(
10784 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
10785 CodeGenFunction &CGF) {
10786 // Emit device ID if any.
10787 llvm::Value *DeviceID;
10788 if (Device.getPointer()) {
10789 assert((Device.getInt() == OMPC_DEVICE_unknown ||
10790 Device.getInt() == OMPC_DEVICE_device_num) &&
10791 "Expected device_num modifier.");
10792 llvm::Value *DevVal = CGF.EmitScalarExpr(E: Device.getPointer());
10793 DeviceID =
10794 CGF.Builder.CreateIntCast(V: DevVal, DestTy: CGF.Int64Ty, /*isSigned=*/true);
10795 } else {
10796 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
10797 }
10798 return DeviceID;
10799}
10800
10801static std::pair<llvm::Value *, OMPDynGroupprivateFallbackType>
10802emitDynCGroupMem(const OMPExecutableDirective &D, CodeGenFunction &CGF) {
10803 llvm::Value *DynGP = CGF.Builder.getInt32(C: 0);
10804 auto DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
10805
10806 if (auto *DynGPClause = D.getSingleClause<OMPDynGroupprivateClause>()) {
10807 CodeGenFunction::RunCleanupsScope DynGPScope(CGF);
10808 llvm::Value *DynGPVal =
10809 CGF.EmitScalarExpr(E: DynGPClause->getSize(), /*IgnoreResultAssign=*/true);
10810 DynGP = CGF.Builder.CreateIntCast(V: DynGPVal, DestTy: CGF.Int32Ty,
10811 /*isSigned=*/false);
10812 auto FallbackModifier = DynGPClause->getDynGroupprivateFallbackModifier();
10813 switch (FallbackModifier) {
10814 case OMPC_DYN_GROUPPRIVATE_FALLBACK_abort:
10815 DynGPFallback = OMPDynGroupprivateFallbackType::Abort;
10816 break;
10817 case OMPC_DYN_GROUPPRIVATE_FALLBACK_null:
10818 DynGPFallback = OMPDynGroupprivateFallbackType::Null;
10819 break;
10820 case OMPC_DYN_GROUPPRIVATE_FALLBACK_default_mem:
10821 case OMPC_DYN_GROUPPRIVATE_FALLBACK_unknown:
10822 // This is the default for dyn_groupprivate.
10823 DynGPFallback = OMPDynGroupprivateFallbackType::DefaultMem;
10824 break;
10825 default:
10826 llvm_unreachable("Unknown fallback modifier for OpenMP dyn_groupprivate");
10827 }
10828 } else if (auto *OMPXDynCGClause =
10829 D.getSingleClause<OMPXDynCGroupMemClause>()) {
10830 CodeGenFunction::RunCleanupsScope DynCGMemScope(CGF);
10831 llvm::Value *DynCGMemVal = CGF.EmitScalarExpr(E: OMPXDynCGClause->getSize(),
10832 /*IgnoreResultAssign=*/true);
10833 DynGP = CGF.Builder.CreateIntCast(V: DynCGMemVal, DestTy: CGF.Int32Ty,
10834 /*isSigned=*/false);
10835 }
10836 return {DynGP, DynGPFallback};
10837}
10838
10839static void genMapInfoForCaptures(
10840 MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
10841 const CapturedStmt &CS, llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
10842 llvm::OpenMPIRBuilder &OMPBuilder,
10843 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &MappedVarSet,
10844 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
10845
10846 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;
10847 auto RI = CS.getCapturedRecordDecl()->field_begin();
10848 auto *CV = CapturedVars.begin();
10849 for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(),
10850 CE = CS.capture_end();
10851 CI != CE; ++CI, ++RI, ++CV) {
10852 MappableExprsHandler::MapCombinedInfoTy CurInfo;
10853
10854 // VLA sizes are passed to the outlined region by copy and do not have map
10855 // information associated.
10856 if (CI->capturesVariableArrayType()) {
10857 CurInfo.Exprs.push_back(Elt: nullptr);
10858 CurInfo.BasePointers.push_back(Elt: *CV);
10859 CurInfo.DevicePtrDecls.push_back(Elt: nullptr);
10860 CurInfo.DevicePointers.push_back(
10861 Elt: MappableExprsHandler::DeviceInfoTy::None);
10862 CurInfo.Pointers.push_back(Elt: *CV);
10863 CurInfo.Sizes.push_back(Elt: CGF.Builder.CreateIntCast(
10864 V: CGF.getTypeSize(Ty: RI->getType()), DestTy: CGF.Int64Ty, /*isSigned=*/true));
10865 // Copy to the device as an argument. No need to retrieve it.
10866 CurInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |
10867 OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
10868 OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
10869 CurInfo.HasAttachPtr.push_back(Elt: false);
10870 CurInfo.Mappers.push_back(Elt: nullptr);
10871 } else {
10872 const ValueDecl *CapturedVD =
10873 CI->capturesThis() ? nullptr
10874 : CI->getCapturedVar()->getCanonicalDecl();
10875 bool HasEntryWithCVAsAttachPtr = false;
10876 if (CapturedVD)
10877 HasEntryWithCVAsAttachPtr =
10878 MEHandler.hasAttachEntryForCapturedVar(VD: CapturedVD);
10879
10880 // Populate component lists for the captured variable from clauses.
10881 MappableExprsHandler::MapDataArrayTy DeclComponentLists;
10882 SmallVector<
10883 SmallVector<OMPClauseMappableExprCommon::MappableComponent, 8>, 4>
10884 StorageForImplicitlyAddedComponentLists;
10885 MEHandler.populateComponentListsForNonLambdaCaptureFromClauses(
10886 VD: CapturedVD, DeclComponentLists,
10887 StorageForImplicitlyAddedComponentLists);
10888
10889 // OpenMP 6.0, 15.8, target construct, restrictions:
10890 // * A list item in a map clause that is specified on a target construct
10891 // must have a base variable or base pointer.
10892 //
10893 // Map clauses on a target construct must either have a base pointer, or a
10894 // base-variable. So, if we don't have a base-pointer, that means that it
10895 // must have a base-variable, i.e. we have a map like `map(s)`, `map(s.x)`
10896 // etc. In such cases, we do not need to handle default map generation
10897 // for `s`.
10898 bool HasEntryWithoutAttachPtr =
10899 llvm::any_of(Range&: DeclComponentLists, P: [&](const auto &MapData) {
10900 OMPClauseMappableExprCommon::MappableExprComponentListRef
10901 Components = std::get<0>(MapData);
10902 return !MEHandler.getAttachPtrExpr(Components);
10903 });
10904
10905 // Generate default map info first if there's no direct map with CV as
10906 // the base-variable, or attach pointer.
10907 if (DeclComponentLists.empty() ||
10908 (!HasEntryWithCVAsAttachPtr && !HasEntryWithoutAttachPtr))
10909 MEHandler.generateDefaultMapInfo(CI: *CI, RI: **RI, CV: *CV, CombinedInfo&: CurInfo);
10910
10911 // If we have any information in the map clause, we use it, otherwise we
10912 // just do a default mapping.
10913 MEHandler.generateInfoForCaptureFromClauseInfo(
10914 DeclComponentListsFromClauses: DeclComponentLists, Cap: CI, Arg: *CV, CurCaptureVarInfo&: CurInfo, OMPBuilder,
10915 /*OffsetForMemberOfFlag=*/CombinedInfo.BasePointers.size());
10916
10917 if (!CI->capturesThis())
10918 MappedVarSet.insert(V: CI->getCapturedVar());
10919 else
10920 MappedVarSet.insert(V: nullptr);
10921
10922 // Generate correct mapping for variables captured by reference in
10923 // lambdas.
10924 if (CI->capturesVariable())
10925 MEHandler.generateInfoForLambdaCaptures(VD: CI->getCapturedVar(), Arg: *CV,
10926 CombinedInfo&: CurInfo, LambdaPointers);
10927 }
10928 // We expect to have at least an element of information for this capture.
10929 assert(!CurInfo.BasePointers.empty() &&
10930 "Non-existing map pointer for capture!");
10931 assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&
10932 CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&
10933 CurInfo.BasePointers.size() == CurInfo.Types.size() &&
10934 CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&
10935 "Inconsistent map information sizes!");
10936
10937 // We need to append the results of this capture to what we already have.
10938 CombinedInfo.append(CurInfo);
10939 }
10940 // Adjust MEMBER_OF flags for the lambdas captures.
10941 MEHandler.adjustMemberOfForLambdaCaptures(
10942 OMPBuilder, LambdaPointers, BasePointers&: CombinedInfo.BasePointers,
10943 Pointers&: CombinedInfo.Pointers, Types&: CombinedInfo.Types);
10944}
10945static void
10946genMapInfo(MappableExprsHandler &MEHandler, CodeGenFunction &CGF,
10947 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
10948 llvm::OpenMPIRBuilder &OMPBuilder,
10949 const llvm::DenseSet<CanonicalDeclPtr<const Decl>> &SkippedVarSet =
10950 llvm::DenseSet<CanonicalDeclPtr<const Decl>>()) {
10951
10952 CodeGenModule &CGM = CGF.CGM;
10953 // Map any list items in a map clause that were not captures because they
10954 // weren't referenced within the construct.
10955 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkipVarSet: SkippedVarSet);
10956
10957 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
10958 return emitMappingInformation(CGF, OMPBuilder, MapExprs&: MapExpr);
10959 };
10960 if (CGM.getCodeGenOpts().getDebugInfo() !=
10961 llvm::codegenoptions::NoDebugInfo) {
10962 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
10963 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
10964 F: FillInfoMap);
10965 }
10966}
10967
10968static void genMapInfo(const OMPExecutableDirective &D, CodeGenFunction &CGF,
10969 const CapturedStmt &CS,
10970 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
10971 llvm::OpenMPIRBuilder &OMPBuilder,
10972 MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {
10973 // Get mappable expression information.
10974 MappableExprsHandler MEHandler(D, CGF);
10975 llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet;
10976
10977 genMapInfoForCaptures(MEHandler, CGF, CS, CapturedVars, OMPBuilder,
10978 MappedVarSet, CombinedInfo);
10979 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, SkippedVarSet: MappedVarSet);
10980}
10981
10982template <typename ClauseTy>
10983static void
10984emitClauseForBareTargetDirective(CodeGenFunction &CGF,
10985 const OMPExecutableDirective &D,
10986 llvm::SmallVectorImpl<llvm::Value *> &Values) {
10987 const auto *C = D.getSingleClause<ClauseTy>();
10988 assert(!C->varlist_empty() &&
10989 "ompx_bare requires explicit num_teams and thread_limit");
10990 CodeGenFunction::RunCleanupsScope Scope(CGF);
10991 for (auto *E : C->varlist()) {
10992 llvm::Value *V = CGF.EmitScalarExpr(E);
10993 Values.push_back(
10994 Elt: CGF.Builder.CreateIntCast(V, DestTy: CGF.Int32Ty, /*isSigned=*/true));
10995 }
10996}
10997
10998static void emitTargetCallKernelLaunch(
10999 CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
11000 const OMPExecutableDirective &D,
11001 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars, bool RequiresOuterTask,
11002 const CapturedStmt &CS, bool OffloadingMandatory,
11003 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11004 llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,
11005 llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,
11006 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11007 const OMPLoopDirective &D)>
11008 SizeEmitter,
11009 CodeGenFunction &CGF, CodeGenModule &CGM) {
11010 llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->getOMPBuilder();
11011
11012 // Fill up the arrays with all the captured variables.
11013 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11014 CGOpenMPRuntime::TargetDataInfo Info;
11015 genMapInfo(D, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);
11016
11017 // Append a null entry for the implicit dyn_ptr argument.
11018 using OpenMPOffloadMappingFlags = llvm::omp::OpenMPOffloadMappingFlags;
11019 auto *NullPtr = llvm::Constant::getNullValue(Ty: CGF.Builder.getPtrTy());
11020 CombinedInfo.BasePointers.push_back(Elt: NullPtr);
11021 CombinedInfo.Pointers.push_back(Elt: NullPtr);
11022 CombinedInfo.DevicePointers.push_back(
11023 Elt: llvm::OpenMPIRBuilder::DeviceInfoTy::None);
11024 CombinedInfo.Sizes.push_back(Elt: CGF.Builder.getInt64(C: 0));
11025 CombinedInfo.Types.push_back(Elt: OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |
11026 OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);
11027 CombinedInfo.HasAttachPtr.push_back(Elt: false);
11028 if (!CombinedInfo.Names.empty())
11029 CombinedInfo.Names.push_back(Elt: NullPtr);
11030 CombinedInfo.Exprs.push_back(Elt: nullptr);
11031 CombinedInfo.Mappers.push_back(Elt: nullptr);
11032 CombinedInfo.DevicePtrDecls.push_back(Elt: nullptr);
11033
11034 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
11035 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
11036
11037 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
11038 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
11039 CGF.VoidPtrTy, CGM.getPointerAlign());
11040 InputInfo.PointersArray =
11041 Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11042 InputInfo.SizesArray =
11043 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
11044 InputInfo.MappersArray =
11045 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11046 MapTypesArray = Info.RTArgs.MapTypesArray;
11047 MapNamesArray = Info.RTArgs.MapNamesArray;
11048
11049 auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
11050 RequiresOuterTask, &CS, OffloadingMandatory, Device,
11051 OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
11052 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
11053 bool IsReverseOffloading = Device.getInt() == OMPC_DEVICE_ancestor;
11054
11055 if (IsReverseOffloading) {
11056 // Reverse offloading is not supported, so just execute on the host.
11057 // FIXME: This fallback solution is incorrect since it ignores the
11058 // OMP_TARGET_OFFLOAD environment variable. Instead it would be better to
11059 // assert here and ensure SEMA emits an error.
11060 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11061 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11062 return;
11063 }
11064
11065 bool HasNoWait = D.hasClausesOfKind<OMPNowaitClause>();
11066 unsigned NumTargetItems = InputInfo.NumberOfTargetItems;
11067
11068 llvm::Value *BasePointersArray =
11069 InputInfo.BasePointersArray.emitRawPointer(CGF);
11070 llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);
11071 llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);
11072 llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);
11073
11074 auto &&EmitTargetCallFallbackCB =
11075 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11076 OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)
11077 -> llvm::OpenMPIRBuilder::InsertPointTy {
11078 CGF.Builder.restoreIP(IP);
11079 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11080 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11081 return CGF.Builder.saveIP();
11082 };
11083
11084 bool IsBare = D.hasClausesOfKind<OMPXBareClause>();
11085 SmallVector<llvm::Value *, 3> NumTeams;
11086 SmallVector<llvm::Value *, 3> NumThreads;
11087 if (IsBare) {
11088 emitClauseForBareTargetDirective<OMPNumTeamsClause>(CGF, D, Values&: NumTeams);
11089 emitClauseForBareTargetDirective<OMPThreadLimitClause>(CGF, D,
11090 Values&: NumThreads);
11091 } else {
11092 NumTeams.push_back(Elt: OMPRuntime->emitNumTeamsForTargetDirective(CGF, D));
11093 NumThreads.push_back(
11094 Elt: OMPRuntime->emitNumThreadsForTargetDirective(CGF, D));
11095 }
11096
11097 llvm::Value *DeviceID = emitDeviceID(Device, CGF);
11098 llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, Loc: D.getBeginLoc());
11099 llvm::Value *NumIterations =
11100 OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter);
11101 auto [DynCGroupMem, DynCGroupMemFallback] = emitDynCGroupMem(D, CGF);
11102 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(
11103 CGF.AllocaInsertPt->getParent(), CGF.AllocaInsertPt->getIterator());
11104
11105 llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(
11106 BasePointersArray, PointersArray, SizesArray, MapTypesArray,
11107 nullptr /* MapTypesArrayEnd */, MappersArray, MapNamesArray);
11108
11109 llvm::OpenMPIRBuilder::TargetKernelArgs Args(
11110 NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,
11111 DynCGroupMem, HasNoWait, /*StrictBlocks=*/IsBare,
11112 /*StrictThreads=*/IsBare, DynCGroupMemFallback);
11113
11114 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
11115 cantFail(ValOrErr: OMPRuntime->getOMPBuilder().emitKernelLaunch(
11116 Loc: CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
11117 RTLoc, AllocaIP));
11118 CGF.Builder.restoreIP(IP: AfterIP);
11119 };
11120
11121 if (RequiresOuterTask)
11122 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ThenGen, InputInfo);
11123 else
11124 OMPRuntime->emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ThenGen);
11125}
11126
11127static void
11128emitTargetCallElse(CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
11129 const OMPExecutableDirective &D,
11130 llvm::SmallVectorImpl<llvm::Value *> &CapturedVars,
11131 bool RequiresOuterTask, const CapturedStmt &CS,
11132 bool OffloadingMandatory, CodeGenFunction &CGF) {
11133
11134 // Notify that the host version must be executed.
11135 auto &&ElseGen =
11136 [&OMPRuntime, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11137 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11138 emitTargetCallFallback(OMPRuntime, OutlinedFn, D, CapturedVars,
11139 RequiresOuterTask, CS, OffloadingMandatory, CGF);
11140 };
11141
11142 if (RequiresOuterTask) {
11143 CodeGenFunction::OMPTargetDataInfo InputInfo;
11144 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ElseGen, InputInfo);
11145 } else {
11146 OMPRuntime->emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ElseGen);
11147 }
11148}
11149
11150void CGOpenMPRuntime::emitTargetCall(
11151 CodeGenFunction &CGF, const OMPExecutableDirective &D,
11152 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
11153 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
11154 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
11155 const OMPLoopDirective &D)>
11156 SizeEmitter) {
11157 if (!CGF.HaveInsertPoint())
11158 return;
11159
11160 const bool OffloadingMandatory = !CGM.getLangOpts().OpenMPIsTargetDevice &&
11161 CGM.getLangOpts().OpenMPOffloadMandatory;
11162
11163 assert((OffloadingMandatory || OutlinedFn) && "Invalid outlined function!");
11164
11165 const bool RequiresOuterTask =
11166 D.hasClausesOfKind<OMPDependClause>() ||
11167 D.hasClausesOfKind<OMPNowaitClause>() ||
11168 D.hasClausesOfKind<OMPInReductionClause>() ||
11169 (CGM.getLangOpts().OpenMP >= 51 &&
11170 needsTaskBasedThreadLimit(DKind: D.getDirectiveKind()) &&
11171 D.hasClausesOfKind<OMPThreadLimitClause>());
11172 llvm::SmallVector<llvm::Value *, 16> CapturedVars;
11173 const CapturedStmt &CS = *D.getCapturedStmt(RegionKind: OMPD_target);
11174 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF,
11175 PrePostActionTy &) {
11176 CGF.GenerateOpenMPCapturedVars(S: CS, CapturedVars);
11177 };
11178 emitInlinedDirective(CGF, InnerKind: OMPD_unknown, CodeGen: ArgsCodegen);
11179
11180 CodeGenFunction::OMPTargetDataInfo InputInfo;
11181 llvm::Value *MapTypesArray = nullptr;
11182 llvm::Value *MapNamesArray = nullptr;
11183
11184 auto &&TargetThenGen = [this, OutlinedFn, &D, &CapturedVars,
11185 RequiresOuterTask, &CS, OffloadingMandatory, Device,
11186 OutlinedFnID, &InputInfo, &MapTypesArray,
11187 &MapNamesArray, SizeEmitter](CodeGenFunction &CGF,
11188 PrePostActionTy &) {
11189 emitTargetCallKernelLaunch(OMPRuntime: this, OutlinedFn, D, CapturedVars,
11190 RequiresOuterTask, CS, OffloadingMandatory,
11191 Device, OutlinedFnID, InputInfo, MapTypesArray,
11192 MapNamesArray, SizeEmitter, CGF, CGM);
11193 };
11194
11195 auto &&TargetElseGen =
11196 [this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
11197 OffloadingMandatory](CodeGenFunction &CGF, PrePostActionTy &) {
11198 emitTargetCallElse(OMPRuntime: this, OutlinedFn, D, CapturedVars, RequiresOuterTask,
11199 CS, OffloadingMandatory, CGF);
11200 };
11201
11202 // If we have a target function ID it means that we need to support
11203 // offloading, otherwise, just execute on the host. We need to execute on host
11204 // regardless of the conditional in the if clause if, e.g., the user do not
11205 // specify target triples.
11206 if (OutlinedFnID) {
11207 if (IfCond) {
11208 emitIfClause(CGF, Cond: IfCond, ThenGen: TargetThenGen, ElseGen: TargetElseGen);
11209 } else {
11210 RegionCodeGenTy ThenRCG(TargetThenGen);
11211 ThenRCG(CGF);
11212 }
11213 } else {
11214 RegionCodeGenTy ElseRCG(TargetElseGen);
11215 ElseRCG(CGF);
11216 }
11217}
11218
11219void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
11220 StringRef ParentName) {
11221 if (!S)
11222 return;
11223
11224 // Register vtable from device for target data and target directives.
11225 // Add this block here since scanForTargetRegionsFunctions ignores
11226 // target data by checking if S is a executable directive (target).
11227 if (auto *E = dyn_cast<OMPExecutableDirective>(Val: S);
11228 E && isOpenMPTargetDataManagementDirective(DKind: E->getDirectiveKind())) {
11229 // Don't need to check if it's device compile
11230 // since scanForTargetRegionsFunctions currently only called
11231 // in device compilation.
11232 registerVTable(D: *E);
11233 }
11234
11235 // Codegen OMP target directives that offload compute to the device.
11236 bool RequiresDeviceCodegen =
11237 isa<OMPExecutableDirective>(Val: S) &&
11238 isOpenMPTargetExecutionDirective(
11239 DKind: cast<OMPExecutableDirective>(Val: S)->getDirectiveKind());
11240
11241 if (RequiresDeviceCodegen) {
11242 const auto &E = *cast<OMPExecutableDirective>(Val: S);
11243
11244 llvm::TargetRegionEntryInfo EntryInfo = getEntryInfoFromPresumedLoc(
11245 CGM, OMPBuilder, BeginLoc: E.getBeginLoc(), ParentName);
11246
11247 // Is this a target region that should not be emitted as an entry point? If
11248 // so just signal we are done with this target region.
11249 if (!OMPBuilder.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))
11250 return;
11251
11252 switch (E.getDirectiveKind()) {
11253 case OMPD_target:
11254 CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName,
11255 S: cast<OMPTargetDirective>(Val: E));
11256 break;
11257 case OMPD_target_parallel:
11258 CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
11259 CGM, ParentName, S: cast<OMPTargetParallelDirective>(Val: E));
11260 break;
11261 case OMPD_target_teams:
11262 CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
11263 CGM, ParentName, S: cast<OMPTargetTeamsDirective>(Val: E));
11264 break;
11265 case OMPD_target_teams_distribute:
11266 CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
11267 CGM, ParentName, S: cast<OMPTargetTeamsDistributeDirective>(Val: E));
11268 break;
11269 case OMPD_target_teams_distribute_simd:
11270 CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
11271 CGM, ParentName, S: cast<OMPTargetTeamsDistributeSimdDirective>(Val: E));
11272 break;
11273 case OMPD_target_parallel_for:
11274 CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
11275 CGM, ParentName, S: cast<OMPTargetParallelForDirective>(Val: E));
11276 break;
11277 case OMPD_target_parallel_for_simd:
11278 CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
11279 CGM, ParentName, S: cast<OMPTargetParallelForSimdDirective>(Val: E));
11280 break;
11281 case OMPD_target_simd:
11282 CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
11283 CGM, ParentName, S: cast<OMPTargetSimdDirective>(Val: E));
11284 break;
11285 case OMPD_target_teams_distribute_parallel_for:
11286 CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
11287 CGM, ParentName,
11288 S: cast<OMPTargetTeamsDistributeParallelForDirective>(Val: E));
11289 break;
11290 case OMPD_target_teams_distribute_parallel_for_simd:
11291 CodeGenFunction::
11292 EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
11293 CGM, ParentName,
11294 S: cast<OMPTargetTeamsDistributeParallelForSimdDirective>(Val: E));
11295 break;
11296 case OMPD_target_teams_loop:
11297 CodeGenFunction::EmitOMPTargetTeamsGenericLoopDeviceFunction(
11298 CGM, ParentName, S: cast<OMPTargetTeamsGenericLoopDirective>(Val: E));
11299 break;
11300 case OMPD_target_parallel_loop:
11301 CodeGenFunction::EmitOMPTargetParallelGenericLoopDeviceFunction(
11302 CGM, ParentName, S: cast<OMPTargetParallelGenericLoopDirective>(Val: E));
11303 break;
11304 case OMPD_parallel:
11305 case OMPD_for:
11306 case OMPD_parallel_for:
11307 case OMPD_parallel_master:
11308 case OMPD_parallel_sections:
11309 case OMPD_for_simd:
11310 case OMPD_parallel_for_simd:
11311 case OMPD_cancel:
11312 case OMPD_cancellation_point:
11313 case OMPD_ordered_standalone:
11314 case OMPD_ordered_blockassoc:
11315 case OMPD_threadprivate:
11316 case OMPD_allocate:
11317 case OMPD_task:
11318 case OMPD_simd:
11319 case OMPD_tile:
11320 case OMPD_unroll:
11321 case OMPD_sections:
11322 case OMPD_section:
11323 case OMPD_single:
11324 case OMPD_master:
11325 case OMPD_critical:
11326 case OMPD_taskyield:
11327 case OMPD_barrier:
11328 case OMPD_taskwait:
11329 case OMPD_taskgroup:
11330 case OMPD_atomic:
11331 case OMPD_flush:
11332 case OMPD_depobj:
11333 case OMPD_scan:
11334 case OMPD_teams:
11335 case OMPD_target_data:
11336 case OMPD_target_exit_data:
11337 case OMPD_target_enter_data:
11338 case OMPD_distribute:
11339 case OMPD_distribute_simd:
11340 case OMPD_distribute_parallel_for:
11341 case OMPD_distribute_parallel_for_simd:
11342 case OMPD_teams_distribute:
11343 case OMPD_teams_distribute_simd:
11344 case OMPD_teams_distribute_parallel_for:
11345 case OMPD_teams_distribute_parallel_for_simd:
11346 case OMPD_target_update:
11347 case OMPD_declare_simd:
11348 case OMPD_declare_variant:
11349 case OMPD_begin_declare_variant:
11350 case OMPD_end_declare_variant:
11351 case OMPD_declare_target:
11352 case OMPD_end_declare_target:
11353 case OMPD_declare_reduction:
11354 case OMPD_declare_mapper:
11355 case OMPD_taskloop:
11356 case OMPD_taskloop_simd:
11357 case OMPD_master_taskloop:
11358 case OMPD_master_taskloop_simd:
11359 case OMPD_parallel_master_taskloop:
11360 case OMPD_parallel_master_taskloop_simd:
11361 case OMPD_requires:
11362 case OMPD_metadirective:
11363 case OMPD_unknown:
11364 default:
11365 llvm_unreachable("Unknown target directive for OpenMP device codegen.");
11366 }
11367 return;
11368 }
11369
11370 if (const auto *E = dyn_cast<OMPExecutableDirective>(Val: S)) {
11371 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt())
11372 return;
11373
11374 scanForTargetRegionsFunctions(S: E->getRawStmt(), ParentName);
11375 return;
11376 }
11377
11378 // If this is a lambda function, look into its body.
11379 if (const auto *L = dyn_cast<LambdaExpr>(Val: S))
11380 S = L->getBody();
11381
11382 // Keep looking for target regions recursively.
11383 for (const Stmt *II : S->children())
11384 scanForTargetRegionsFunctions(S: II, ParentName);
11385}
11386
11387static bool isAssumedToBeNotEmitted(const ValueDecl *VD, bool IsDevice) {
11388 std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
11389 OMPDeclareTargetDeclAttr::getDeviceType(VD);
11390 if (!DevTy)
11391 return false;
11392 // Do not emit device_type(nohost) functions for the host.
11393 if (!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
11394 return true;
11395 // Do not emit device_type(host) functions for the device.
11396 if (IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)
11397 return true;
11398 return false;
11399}
11400
11401bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) {
11402 // If emitting code for the host, we do not process FD here. Instead we do
11403 // the normal code generation.
11404 if (!CGM.getLangOpts().OpenMPIsTargetDevice) {
11405 if (const auto *FD = dyn_cast<FunctionDecl>(Val: GD.getDecl()))
11406 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: FD),
11407 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11408 return true;
11409 return false;
11410 }
11411
11412 const ValueDecl *VD = cast<ValueDecl>(Val: GD.getDecl());
11413 // Try to detect target regions in the function.
11414 if (const auto *FD = dyn_cast<FunctionDecl>(Val: VD)) {
11415 StringRef Name = CGM.getMangledName(GD);
11416 scanForTargetRegionsFunctions(S: FD->getBody(), ParentName: Name);
11417 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: FD),
11418 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11419 return true;
11420 }
11421
11422 // Do not emit function if it is not marked as declare target.
11423 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&
11424 AlreadyEmittedTargetDecls.count(V: VD) == 0;
11425}
11426
11427bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
11428 if (isAssumedToBeNotEmitted(VD: cast<ValueDecl>(Val: GD.getDecl()),
11429 IsDevice: CGM.getLangOpts().OpenMPIsTargetDevice))
11430 return true;
11431
11432 if (!CGM.getLangOpts().OpenMPIsTargetDevice)
11433 return false;
11434
11435 // Check if there are Ctors/Dtors in this declaration and look for target
11436 // regions in it. We use the complete variant to produce the kernel name
11437 // mangling.
11438 QualType RDTy = cast<VarDecl>(Val: GD.getDecl())->getType();
11439 if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) {
11440 for (const CXXConstructorDecl *Ctor : RD->ctors()) {
11441 StringRef ParentName =
11442 CGM.getMangledName(GD: GlobalDecl(Ctor, Ctor_Complete));
11443 scanForTargetRegionsFunctions(S: Ctor->getBody(), ParentName);
11444 }
11445 if (const CXXDestructorDecl *Dtor = RD->getDestructor()) {
11446 StringRef ParentName =
11447 CGM.getMangledName(GD: GlobalDecl(Dtor, Dtor_Complete));
11448 scanForTargetRegionsFunctions(S: Dtor->getBody(), ParentName);
11449 }
11450 }
11451
11452 // Do not emit variable if it is not marked as declare target.
11453 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11454 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(
11455 VD: cast<VarDecl>(Val: GD.getDecl()));
11456 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
11457 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11458 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11459 HasRequiresUnifiedSharedMemory)) {
11460 DeferredGlobalVariables.insert(V: cast<VarDecl>(Val: GD.getDecl()));
11461 return true;
11462 }
11463 return false;
11464}
11465
11466void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,
11467 llvm::Constant *Addr) {
11468 if (CGM.getLangOpts().OMPTargetTriples.empty() &&
11469 !CGM.getLangOpts().OpenMPIsTargetDevice)
11470 return;
11471
11472 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11473 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11474
11475 // If this is an 'extern' declaration we defer to the canonical definition and
11476 // do not emit an offloading entry.
11477 if (Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&
11478 VD->hasExternalStorage())
11479 return;
11480
11481 // MT_Local variables use direct access with no host-device mapping.
11482 // No offload entry needed — the device global keeps its own initializer.
11483 if (Res && *Res == OMPDeclareTargetDeclAttr::MT_Local)
11484 return;
11485
11486 if (!Res) {
11487 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11488 // Register non-target variables being emitted in device code (debug info
11489 // may cause this).
11490 StringRef VarName = CGM.getMangledName(GD: VD);
11491 EmittedNonTargetVariables.try_emplace(Key: VarName, Args&: Addr);
11492 }
11493 return;
11494 }
11495
11496 auto AddrOfGlobal = [&VD, this]() { return CGM.GetAddrOfGlobal(GD: VD); };
11497 auto LinkageForVariable = [&VD, this]() {
11498 return CGM.getLLVMLinkageVarDefinition(VD);
11499 };
11500
11501 std::vector<llvm::GlobalVariable *> GeneratedRefs;
11502 OMPBuilder.registerTargetGlobalVariable(
11503 CaptureClause: convertCaptureClause(VD), DeviceClause: convertDeviceClause(VD),
11504 IsDeclaration: VD->hasDefinition(CGM.getContext()) == VarDecl::DeclarationOnly,
11505 IsExternallyVisible: VD->isExternallyVisible(),
11506 EntryInfo: getEntryInfoFromPresumedLoc(CGM, OMPBuilder,
11507 BeginLoc: VD->getCanonicalDecl()->getBeginLoc()),
11508 MangledName: CGM.getMangledName(GD: VD), GeneratedRefs, OpenMPSIMD: CGM.getLangOpts().OpenMPSimd,
11509 TargetTriple: CGM.getLangOpts().OMPTargetTriples, GlobalInitializer: AddrOfGlobal, VariableLinkage: LinkageForVariable,
11510 LlvmPtrTy: CGM.getTypes().ConvertTypeForMem(
11511 T: CGM.getContext().getPointerType(T: VD->getType())),
11512 Addr);
11513
11514 for (auto *ref : GeneratedRefs)
11515 CGM.addCompilerUsedGlobal(GV: ref);
11516}
11517
11518bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {
11519 if (isa<FunctionDecl>(Val: GD.getDecl()) ||
11520 isa<OMPDeclareReductionDecl>(Val: GD.getDecl()))
11521 return emitTargetFunctions(GD);
11522
11523 return emitTargetGlobalVariable(GD);
11524}
11525
11526void CGOpenMPRuntime::emitDeferredTargetDecls() const {
11527 for (const VarDecl *VD : DeferredGlobalVariables) {
11528 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
11529 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
11530 if (!Res)
11531 continue;
11532 // MT_Local and MT_To/MT_Enter without USM are always emitted.
11533 if (*Res == OMPDeclareTargetDeclAttr::MT_Local ||
11534 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11535 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
11536 !HasRequiresUnifiedSharedMemory)) {
11537 CGM.EmitGlobal(D: VD);
11538 } else {
11539 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||
11540 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
11541 *Res == OMPDeclareTargetDeclAttr::MT_Enter ||
11542 *Res == OMPDeclareTargetDeclAttr::MT_Local) &&
11543 HasRequiresUnifiedSharedMemory)) &&
11544 "Expected link clause or to clause with unified memory.");
11545 (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
11546 }
11547 }
11548}
11549
11550void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas(
11551 CodeGenFunction &CGF, const OMPExecutableDirective &D) const {
11552 assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) &&
11553 " Expected target-based directive.");
11554}
11555
11556void CGOpenMPRuntime::processRequiresDirective(const OMPRequiresDecl *D) {
11557 for (const OMPClause *Clause : D->clauselists()) {
11558 if (Clause->getClauseKind() == OMPC_unified_shared_memory) {
11559 HasRequiresUnifiedSharedMemory = true;
11560 OMPBuilder.Config.setHasRequiresUnifiedSharedMemory(true);
11561 } else if (const auto *AC =
11562 dyn_cast<OMPAtomicDefaultMemOrderClause>(Val: Clause)) {
11563 switch (AC->getAtomicDefaultMemOrderKind()) {
11564 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:
11565 RequiresAtomicOrdering = llvm::AtomicOrdering::AcquireRelease;
11566 break;
11567 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:
11568 RequiresAtomicOrdering = llvm::AtomicOrdering::SequentiallyConsistent;
11569 break;
11570 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:
11571 RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
11572 break;
11573 case OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown:
11574 break;
11575 }
11576 }
11577 }
11578}
11579
11580llvm::AtomicOrdering CGOpenMPRuntime::getDefaultMemoryOrdering() const {
11581 return RequiresAtomicOrdering;
11582}
11583
11584bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD,
11585 LangAS &AS) {
11586 if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>())
11587 return false;
11588 const auto *A = VD->getAttr<OMPAllocateDeclAttr>();
11589 switch(A->getAllocatorType()) {
11590 case OMPAllocateDeclAttr::OMPNullMemAlloc:
11591 case OMPAllocateDeclAttr::OMPDefaultMemAlloc:
11592 // Not supported, fallback to the default mem space.
11593 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc:
11594 case OMPAllocateDeclAttr::OMPCGroupMemAlloc:
11595 case OMPAllocateDeclAttr::OMPHighBWMemAlloc:
11596 case OMPAllocateDeclAttr::OMPLowLatMemAlloc:
11597 case OMPAllocateDeclAttr::OMPThreadMemAlloc:
11598 case OMPAllocateDeclAttr::OMPConstMemAlloc:
11599 case OMPAllocateDeclAttr::OMPPTeamMemAlloc:
11600 AS = LangAS::Default;
11601 return true;
11602 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:
11603 llvm_unreachable("Expected predefined allocator for the variables with the "
11604 "static storage.");
11605 }
11606 return false;
11607}
11608
11609bool CGOpenMPRuntime::hasRequiresUnifiedSharedMemory() const {
11610 return HasRequiresUnifiedSharedMemory;
11611}
11612
11613CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII(
11614 CodeGenModule &CGM)
11615 : CGM(CGM) {
11616 if (CGM.getLangOpts().OpenMPIsTargetDevice) {
11617 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal;
11618 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false;
11619 }
11620}
11621
11622CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() {
11623 if (CGM.getLangOpts().OpenMPIsTargetDevice)
11624 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal;
11625}
11626
11627bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) {
11628 if (!CGM.getLangOpts().OpenMPIsTargetDevice || !ShouldMarkAsGlobal)
11629 return true;
11630
11631 const auto *D = cast<FunctionDecl>(Val: GD.getDecl());
11632 // Do not emit function if it is marked as declare target as it was already
11633 // emitted.
11634 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: D)) {
11635 if (D->hasBody() && AlreadyEmittedTargetDecls.count(V: D) == 0) {
11636 if (auto *F = dyn_cast_or_null<llvm::Function>(
11637 Val: CGM.GetGlobalValue(Ref: CGM.getMangledName(GD))))
11638 return !F->isDeclaration();
11639 return false;
11640 }
11641 return true;
11642 }
11643
11644 return !AlreadyEmittedTargetDecls.insert(V: D).second;
11645}
11646
11647void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF,
11648 const OMPExecutableDirective &D,
11649 SourceLocation Loc,
11650 llvm::Function *OutlinedFn,
11651 ArrayRef<llvm::Value *> CapturedVars) {
11652 if (!CGF.HaveInsertPoint())
11653 return;
11654
11655 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11656 CodeGenFunction::RunCleanupsScope Scope(CGF);
11657
11658 // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn);
11659 llvm::Value *Args[] = {
11660 RTLoc,
11661 CGF.Builder.getInt32(C: CapturedVars.size()), // Number of captured vars
11662 OutlinedFn};
11663 llvm::SmallVector<llvm::Value *, 16> RealArgs;
11664 RealArgs.append(in_start: std::begin(arr&: Args), in_end: std::end(arr&: Args));
11665 RealArgs.append(in_start: CapturedVars.begin(), in_end: CapturedVars.end());
11666
11667 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
11668 M&: CGM.getModule(), FnID: OMPRTL___kmpc_fork_teams);
11669 CGF.EmitRuntimeCall(callee: RTLFn, args: RealArgs);
11670}
11671
11672void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
11673 const Expr *NumTeams,
11674 const Expr *ThreadLimit,
11675 SourceLocation Loc) {
11676 if (!CGF.HaveInsertPoint())
11677 return;
11678
11679 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11680
11681 llvm::Value *NumTeamsVal =
11682 NumTeams
11683 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: NumTeams),
11684 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11685 : CGF.Builder.getInt32(C: 0);
11686
11687 llvm::Value *ThreadLimitVal =
11688 ThreadLimit
11689 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: ThreadLimit),
11690 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11691 : CGF.Builder.getInt32(C: 0);
11692
11693 // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit)
11694 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal,
11695 ThreadLimitVal};
11696 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
11697 M&: CGM.getModule(), FnID: OMPRTL___kmpc_push_num_teams),
11698 args: PushNumTeamsArgs);
11699}
11700
11701void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF,
11702 const Expr *ThreadLimit,
11703 SourceLocation Loc) {
11704 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
11705 llvm::Value *ThreadLimitVal =
11706 ThreadLimit
11707 ? CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: ThreadLimit),
11708 DestTy: CGF.CGM.Int32Ty, /* isSigned = */ true)
11709 : CGF.Builder.getInt32(C: 0);
11710
11711 // Build call __kmpc_set_thread_limit(&loc, global_tid, thread_limit)
11712 llvm::Value *ThreadLimitArgs[] = {RTLoc, getThreadID(CGF, Loc),
11713 ThreadLimitVal};
11714 CGF.EmitRuntimeCall(callee: OMPBuilder.getOrCreateRuntimeFunction(
11715 M&: CGM.getModule(), FnID: OMPRTL___kmpc_set_thread_limit),
11716 args: ThreadLimitArgs);
11717}
11718
11719void CGOpenMPRuntime::emitTargetDataCalls(
11720 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11721 const Expr *Device, const RegionCodeGenTy &CodeGen,
11722 CGOpenMPRuntime::TargetDataInfo &Info) {
11723 if (!CGF.HaveInsertPoint())
11724 return;
11725
11726 // Action used to replace the default codegen action and turn privatization
11727 // off.
11728 PrePostActionTy NoPrivAction;
11729
11730 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
11731
11732 llvm::Value *IfCondVal = nullptr;
11733 if (IfCond)
11734 IfCondVal = CGF.EvaluateExprAsBool(E: IfCond);
11735
11736 // Emit device ID if any.
11737 llvm::Value *DeviceID = nullptr;
11738 if (Device) {
11739 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
11740 DestTy: CGF.Int64Ty, /*isSigned=*/true);
11741 } else {
11742 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
11743 }
11744
11745 // Fill up the arrays with all the mapped variables.
11746 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11747 auto GenMapInfoCB =
11748 [&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
11749 CGF.Builder.restoreIP(IP: CodeGenIP);
11750 // Get map clause information.
11751 MappableExprsHandler MEHandler(D, CGF);
11752 MEHandler.generateAllInfo(CombinedInfo, OMPBuilder);
11753
11754 auto FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {
11755 return emitMappingInformation(CGF, OMPBuilder, MapExprs&: MapExpr);
11756 };
11757 if (CGM.getCodeGenOpts().getDebugInfo() !=
11758 llvm::codegenoptions::NoDebugInfo) {
11759 CombinedInfo.Names.resize(N: CombinedInfo.Exprs.size());
11760 llvm::transform(Range&: CombinedInfo.Exprs, d_first: CombinedInfo.Names.begin(),
11761 F: FillInfoMap);
11762 }
11763
11764 return CombinedInfo;
11765 };
11766 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
11767 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
11768 CGF.Builder.restoreIP(IP: CodeGenIP);
11769 switch (BodyGenType) {
11770 case BodyGenTy::Priv:
11771 if (!Info.CaptureDeviceAddrMap.empty())
11772 CodeGen(CGF);
11773 break;
11774 case BodyGenTy::DupNoPriv:
11775 if (!Info.CaptureDeviceAddrMap.empty()) {
11776 CodeGen.setAction(NoPrivAction);
11777 CodeGen(CGF);
11778 }
11779 break;
11780 case BodyGenTy::NoPriv:
11781 if (Info.CaptureDeviceAddrMap.empty()) {
11782 CodeGen.setAction(NoPrivAction);
11783 CodeGen(CGF);
11784 }
11785 break;
11786 }
11787 return InsertPointTy(CGF.Builder.GetInsertBlock(),
11788 CGF.Builder.GetInsertPoint());
11789 };
11790
11791 auto DeviceAddrCB = [&](unsigned int I, llvm::Value *NewDecl) {
11792 if (const ValueDecl *DevVD = CombinedInfo.DevicePtrDecls[I]) {
11793 Info.CaptureDeviceAddrMap.try_emplace(Key: DevVD, Args&: NewDecl);
11794 }
11795 };
11796
11797 auto CustomMapperCB = [&](unsigned int I) {
11798 llvm::Function *MFunc = nullptr;
11799 if (CombinedInfo.Mappers[I]) {
11800 Info.HasMapper = true;
11801 MFunc = CGF.CGM.getOpenMPRuntime().getOrCreateUserDefinedMapperFunc(
11802 D: cast<OMPDeclareMapperDecl>(Val: CombinedInfo.Mappers[I]));
11803 }
11804 return MFunc;
11805 };
11806
11807 // Source location for the ident struct
11808 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
11809
11810 InsertPointTy AllocaIP(CGF.AllocaInsertPt->getParent(),
11811 CGF.AllocaInsertPt->getIterator());
11812 InsertPointTy CodeGenIP(CGF.Builder.GetInsertBlock(),
11813 CGF.Builder.GetInsertPoint());
11814 llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CGF.Builder);
11815 llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
11816 cantFail(ValOrErr: OMPBuilder.createTargetData(
11817 Loc: OmpLoc, AllocaIP, CodeGenIP, /*DeallocBlocks=*/{}, DeviceID,
11818 IfCond: IfCondVal, Info, GenMapInfoCB, CustomMapperCB,
11819 /*MapperFunc=*/nullptr, BodyGenCB: BodyCB, DeviceAddrCB, SrcLocInfo: RTLoc));
11820 CGF.Builder.restoreIP(IP: AfterIP);
11821}
11822
11823void CGOpenMPRuntime::emitTargetDataStandAloneCall(
11824 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
11825 const Expr *Device) {
11826 if (!CGF.HaveInsertPoint())
11827 return;
11828
11829 assert((isa<OMPTargetEnterDataDirective>(D) ||
11830 isa<OMPTargetExitDataDirective>(D) ||
11831 isa<OMPTargetUpdateDirective>(D)) &&
11832 "Expecting either target enter, exit data, or update directives.");
11833
11834 CodeGenFunction::OMPTargetDataInfo InputInfo;
11835 llvm::Value *MapTypesArray = nullptr;
11836 llvm::Value *MapNamesArray = nullptr;
11837 // Generate the code for the opening of the data environment.
11838 auto &&ThenGen = [this, &D, Device, &InputInfo, &MapTypesArray,
11839 &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) {
11840 // Emit device ID if any.
11841 llvm::Value *DeviceID = nullptr;
11842 if (Device) {
11843 DeviceID = CGF.Builder.CreateIntCast(V: CGF.EmitScalarExpr(E: Device),
11844 DestTy: CGF.Int64Ty, /*isSigned=*/true);
11845 } else {
11846 DeviceID = CGF.Builder.getInt64(C: OMP_DEVICEID_UNDEF);
11847 }
11848
11849 // Emit the number of elements in the offloading arrays.
11850 llvm::Constant *PointerNum =
11851 CGF.Builder.getInt32(C: InputInfo.NumberOfTargetItems);
11852
11853 // Source location for the ident struct
11854 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc: D.getBeginLoc());
11855
11856 SmallVector<llvm::Value *, 13> OffloadingArgs(
11857 {RTLoc, DeviceID, PointerNum,
11858 InputInfo.BasePointersArray.emitRawPointer(CGF),
11859 InputInfo.PointersArray.emitRawPointer(CGF),
11860 InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray,
11861 InputInfo.MappersArray.emitRawPointer(CGF)});
11862
11863 // Select the right runtime function call for each standalone
11864 // directive.
11865 const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
11866 RuntimeFunction RTLFn;
11867 switch (D.getDirectiveKind()) {
11868 case OMPD_target_enter_data:
11869 RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
11870 : OMPRTL___tgt_target_data_begin_mapper;
11871 break;
11872 case OMPD_target_exit_data:
11873 RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
11874 : OMPRTL___tgt_target_data_end_mapper;
11875 break;
11876 case OMPD_target_update:
11877 RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
11878 : OMPRTL___tgt_target_data_update_mapper;
11879 break;
11880 case OMPD_parallel:
11881 case OMPD_for:
11882 case OMPD_parallel_for:
11883 case OMPD_parallel_master:
11884 case OMPD_parallel_sections:
11885 case OMPD_for_simd:
11886 case OMPD_parallel_for_simd:
11887 case OMPD_cancel:
11888 case OMPD_cancellation_point:
11889 case OMPD_ordered_standalone:
11890 case OMPD_ordered_blockassoc:
11891 case OMPD_threadprivate:
11892 case OMPD_allocate:
11893 case OMPD_task:
11894 case OMPD_simd:
11895 case OMPD_tile:
11896 case OMPD_unroll:
11897 case OMPD_sections:
11898 case OMPD_section:
11899 case OMPD_single:
11900 case OMPD_master:
11901 case OMPD_critical:
11902 case OMPD_taskyield:
11903 case OMPD_barrier:
11904 case OMPD_taskwait:
11905 case OMPD_taskgroup:
11906 case OMPD_atomic:
11907 case OMPD_flush:
11908 case OMPD_depobj:
11909 case OMPD_scan:
11910 case OMPD_teams:
11911 case OMPD_target_data:
11912 case OMPD_distribute:
11913 case OMPD_distribute_simd:
11914 case OMPD_distribute_parallel_for:
11915 case OMPD_distribute_parallel_for_simd:
11916 case OMPD_teams_distribute:
11917 case OMPD_teams_distribute_simd:
11918 case OMPD_teams_distribute_parallel_for:
11919 case OMPD_teams_distribute_parallel_for_simd:
11920 case OMPD_declare_simd:
11921 case OMPD_declare_variant:
11922 case OMPD_begin_declare_variant:
11923 case OMPD_end_declare_variant:
11924 case OMPD_declare_target:
11925 case OMPD_end_declare_target:
11926 case OMPD_declare_reduction:
11927 case OMPD_declare_mapper:
11928 case OMPD_taskloop:
11929 case OMPD_taskloop_simd:
11930 case OMPD_master_taskloop:
11931 case OMPD_master_taskloop_simd:
11932 case OMPD_parallel_master_taskloop:
11933 case OMPD_parallel_master_taskloop_simd:
11934 case OMPD_target:
11935 case OMPD_target_simd:
11936 case OMPD_target_teams_distribute:
11937 case OMPD_target_teams_distribute_simd:
11938 case OMPD_target_teams_distribute_parallel_for:
11939 case OMPD_target_teams_distribute_parallel_for_simd:
11940 case OMPD_target_teams:
11941 case OMPD_target_parallel:
11942 case OMPD_target_parallel_for:
11943 case OMPD_target_parallel_for_simd:
11944 case OMPD_requires:
11945 case OMPD_metadirective:
11946 case OMPD_unknown:
11947 default:
11948 llvm_unreachable("Unexpected standalone target data directive.");
11949 break;
11950 }
11951 if (HasNowait) {
11952 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int32Ty));
11953 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.VoidPtrTy));
11954 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.Int32Ty));
11955 OffloadingArgs.push_back(Elt: llvm::Constant::getNullValue(Ty: CGF.VoidPtrTy));
11956 }
11957 CGF.EmitRuntimeCall(
11958 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID: RTLFn),
11959 args: OffloadingArgs);
11960 };
11961
11962 auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
11963 &MapNamesArray](CodeGenFunction &CGF,
11964 PrePostActionTy &) {
11965 // Fill up the arrays with all the mapped variables.
11966 MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
11967 CGOpenMPRuntime::TargetDataInfo Info;
11968 MappableExprsHandler MEHandler(D, CGF);
11969 genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder);
11970 emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
11971 /*IsNonContiguous=*/true, /*ForEndCall=*/false);
11972
11973 bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>() ||
11974 D.hasClausesOfKind<OMPNowaitClause>();
11975
11976 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
11977 InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
11978 CGF.VoidPtrTy, CGM.getPointerAlign());
11979 InputInfo.PointersArray = Address(Info.RTArgs.PointersArray, CGF.VoidPtrTy,
11980 CGM.getPointerAlign());
11981 InputInfo.SizesArray =
11982 Address(Info.RTArgs.SizesArray, CGF.Int64Ty, CGM.getPointerAlign());
11983 InputInfo.MappersArray =
11984 Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
11985 MapTypesArray = Info.RTArgs.MapTypesArray;
11986 MapNamesArray = Info.RTArgs.MapNamesArray;
11987 if (RequiresOuterTask)
11988 CGF.EmitOMPTargetTaskBasedDirective(S: D, BodyGen: ThenGen, InputInfo);
11989 else
11990 emitInlinedDirective(CGF, InnerKind: D.getDirectiveKind(), CodeGen: ThenGen);
11991 };
11992
11993 if (IfCond) {
11994 emitIfClause(CGF, Cond: IfCond, ThenGen: TargetThenGen,
11995 ElseGen: [](CodeGenFunction &CGF, PrePostActionTy &) {});
11996 } else {
11997 RegionCodeGenTy ThenRCG(TargetThenGen);
11998 ThenRCG(CGF);
11999 }
12000}
12001
12002static unsigned
12003evaluateCDTSize(const FunctionDecl *FD,
12004 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12005 // Every vector variant of a SIMD-enabled function has a vector length (VLEN).
12006 // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument
12007 // of that clause. The VLEN value must be power of 2.
12008 // In other case the notion of the function`s "characteristic data type" (CDT)
12009 // is used to compute the vector length.
12010 // CDT is defined in the following order:
12011 // a) For non-void function, the CDT is the return type.
12012 // b) If the function has any non-uniform, non-linear parameters, then the
12013 // CDT is the type of the first such parameter.
12014 // c) If the CDT determined by a) or b) above is struct, union, or class
12015 // type which is pass-by-value (except for the type that maps to the
12016 // built-in complex data type), the characteristic data type is int.
12017 // d) If none of the above three cases is applicable, the CDT is int.
12018 // The VLEN is then determined based on the CDT and the size of vector
12019 // register of that ISA for which current vector version is generated. The
12020 // VLEN is computed using the formula below:
12021 // VLEN = sizeof(vector_register) / sizeof(CDT),
12022 // where vector register size specified in section 3.2.1 Registers and the
12023 // Stack Frame of original AMD64 ABI document.
12024 QualType RetType = FD->getReturnType();
12025 if (RetType.isNull())
12026 return 0;
12027 ASTContext &C = FD->getASTContext();
12028 QualType CDT;
12029 if (!RetType.isNull() && !RetType->isVoidType()) {
12030 CDT = RetType;
12031 } else {
12032 unsigned Offset = 0;
12033 if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FD)) {
12034 if (ParamAttrs[Offset].Kind ==
12035 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector)
12036 CDT = C.getPointerType(T: C.getCanonicalTagType(TD: MD->getParent()));
12037 ++Offset;
12038 }
12039 if (CDT.isNull()) {
12040 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
12041 if (ParamAttrs[I + Offset].Kind ==
12042 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector) {
12043 CDT = FD->getParamDecl(i: I)->getType();
12044 break;
12045 }
12046 }
12047 }
12048 }
12049 if (CDT.isNull())
12050 CDT = C.IntTy;
12051 CDT = CDT->getCanonicalTypeUnqualified();
12052 if (CDT->isRecordType() || CDT->isUnionType())
12053 CDT = C.IntTy;
12054 return C.getTypeSize(T: CDT);
12055}
12056
12057// This are the Functions that are needed to mangle the name of the
12058// vector functions generated by the compiler, according to the rules
12059// defined in the "Vector Function ABI specifications for AArch64",
12060// available at
12061// https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi.
12062
12063/// Maps To Vector (MTV), as defined in 4.1.1 of the AAVFABI (2021Q1).
12064static bool getAArch64MTV(QualType QT,
12065 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind) {
12066 QT = QT.getCanonicalType();
12067
12068 if (QT->isVoidType())
12069 return false;
12070
12071 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform)
12072 return false;
12073
12074 if (Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal ||
12075 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef)
12076 return false;
12077
12078 if ((Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12079 Kind == llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal) &&
12080 !QT->isReferenceType())
12081 return false;
12082
12083 return true;
12084}
12085
12086/// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI.
12087static bool getAArch64PBV(QualType QT, ASTContext &C) {
12088 QT = QT.getCanonicalType();
12089 unsigned Size = C.getTypeSize(T: QT);
12090
12091 // Only scalars and complex within 16 bytes wide set PVB to true.
12092 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)
12093 return false;
12094
12095 if (QT->isFloatingType())
12096 return true;
12097
12098 if (QT->isIntegerType())
12099 return true;
12100
12101 if (QT->isPointerType())
12102 return true;
12103
12104 // TODO: Add support for complex types (section 3.1.2, item 2).
12105
12106 return false;
12107}
12108
12109/// Computes the lane size (LS) of a return type or of an input parameter,
12110/// as defined by `LS(P)` in 3.2.1 of the AAVFABI.
12111/// TODO: Add support for references, section 3.2.1, item 1.
12112static unsigned getAArch64LS(QualType QT,
12113 llvm::OpenMPIRBuilder::DeclareSimdKindTy Kind,
12114 ASTContext &C) {
12115 if (!getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) {
12116 QualType PTy = QT.getCanonicalType()->getPointeeType();
12117 if (getAArch64PBV(QT: PTy, C))
12118 return C.getTypeSize(T: PTy);
12119 }
12120 if (getAArch64PBV(QT, C))
12121 return C.getTypeSize(T: QT);
12122
12123 return C.getTypeSize(T: C.getUIntPtrType());
12124}
12125
12126// Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the
12127// signature of the scalar function, as defined in 3.2.2 of the
12128// AAVFABI.
12129static std::tuple<unsigned, unsigned, bool>
12130getNDSWDS(const FunctionDecl *FD,
12131 ArrayRef<llvm::OpenMPIRBuilder::DeclareSimdAttrTy> ParamAttrs) {
12132 QualType RetType = FD->getReturnType().getCanonicalType();
12133
12134 ASTContext &C = FD->getASTContext();
12135
12136 bool OutputBecomesInput = false;
12137
12138 llvm::SmallVector<unsigned, 8> Sizes;
12139 if (!RetType->isVoidType()) {
12140 Sizes.push_back(Elt: getAArch64LS(
12141 QT: RetType, Kind: llvm::OpenMPIRBuilder::DeclareSimdKindTy::Vector, C));
12142 if (!getAArch64PBV(QT: RetType, C) && getAArch64MTV(QT: RetType, Kind: {}))
12143 OutputBecomesInput = true;
12144 }
12145 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) {
12146 QualType QT = FD->getParamDecl(i: I)->getType().getCanonicalType();
12147 Sizes.push_back(Elt: getAArch64LS(QT, Kind: ParamAttrs[I].Kind, C));
12148 }
12149
12150 assert(!Sizes.empty() && "Unable to determine NDS and WDS.");
12151 // The LS of a function parameter / return value can only be a power
12152 // of 2, starting from 8 bits, up to 128.
12153 assert(llvm::all_of(Sizes,
12154 [](unsigned Size) {
12155 return Size == 8 || Size == 16 || Size == 32 ||
12156 Size == 64 || Size == 128;
12157 }) &&
12158 "Invalid size");
12159
12160 return std::make_tuple(args&: *llvm::min_element(Range&: Sizes), args&: *llvm::max_element(Range&: Sizes),
12161 args&: OutputBecomesInput);
12162}
12163
12164static llvm::OpenMPIRBuilder::DeclareSimdBranch
12165convertDeclareSimdBranch(OMPDeclareSimdDeclAttr::BranchStateTy State) {
12166 switch (State) {
12167 case OMPDeclareSimdDeclAttr::BS_Undefined:
12168 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Undefined;
12169 case OMPDeclareSimdDeclAttr::BS_Inbranch:
12170 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Inbranch;
12171 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
12172 return llvm::OpenMPIRBuilder::DeclareSimdBranch::Notinbranch;
12173 }
12174 llvm_unreachable("unexpected declare simd branch state");
12175}
12176
12177// Check the values provided via `simdlen` by the user.
12178static bool validateAArch64Simdlen(CodeGenModule &CGM, SourceLocation SLoc,
12179 unsigned UserVLEN, unsigned WDS, char ISA) {
12180 // 1. A `simdlen(1)` doesn't produce vector signatures.
12181 if (UserVLEN == 1) {
12182 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_1_no_effect);
12183 return false;
12184 }
12185
12186 // 2. Section 3.3.1, item 1: user input must be a power of 2 for Advanced
12187 // SIMD.
12188 if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(Value: UserVLEN)) {
12189 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_requires_power_of_2);
12190 return false;
12191 }
12192
12193 // 3. Section 3.4.1: SVE fixed length must obey the architectural limits.
12194 if (ISA == 's' && UserVLEN != 0 &&
12195 ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0))) {
12196 CGM.getDiags().Report(Loc: SLoc, DiagID: diag::warn_simdlen_must_fit_lanes) << WDS;
12197 return false;
12198 }
12199
12200 return true;
12201}
12202
12203void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD,
12204 llvm::Function *Fn) {
12205 ASTContext &C = CGM.getContext();
12206 FD = FD->getMostRecentDecl();
12207 while (FD) {
12208 // Map params to their positions in function decl.
12209 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
12210 if (isa<CXXMethodDecl>(Val: FD))
12211 ParamPositions.try_emplace(Key: FD, Args: 0);
12212 unsigned ParamPos = ParamPositions.size();
12213 for (const ParmVarDecl *P : FD->parameters()) {
12214 ParamPositions.try_emplace(Key: P->getCanonicalDecl(), Args&: ParamPos);
12215 ++ParamPos;
12216 }
12217 for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
12218 llvm::SmallVector<llvm::OpenMPIRBuilder::DeclareSimdAttrTy, 8> ParamAttrs(
12219 ParamPositions.size());
12220 // Mark uniform parameters.
12221 for (const Expr *E : Attr->uniforms()) {
12222 E = E->IgnoreParenImpCasts();
12223 unsigned Pos;
12224 if (isa<CXXThisExpr>(Val: E)) {
12225 Pos = ParamPositions[FD];
12226 } else {
12227 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12228 ->getCanonicalDecl();
12229 auto It = ParamPositions.find(Val: PVD);
12230 assert(It != ParamPositions.end() && "Function parameter not found");
12231 Pos = It->second;
12232 }
12233 ParamAttrs[Pos].Kind =
12234 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Uniform;
12235 }
12236 // Get alignment info.
12237 auto *NI = Attr->alignments_begin();
12238 for (const Expr *E : Attr->aligneds()) {
12239 E = E->IgnoreParenImpCasts();
12240 unsigned Pos;
12241 QualType ParmTy;
12242 if (isa<CXXThisExpr>(Val: E)) {
12243 Pos = ParamPositions[FD];
12244 ParmTy = E->getType();
12245 } else {
12246 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12247 ->getCanonicalDecl();
12248 auto It = ParamPositions.find(Val: PVD);
12249 assert(It != ParamPositions.end() && "Function parameter not found");
12250 Pos = It->second;
12251 ParmTy = PVD->getType();
12252 }
12253 ParamAttrs[Pos].Alignment =
12254 (*NI)
12255 ? (*NI)->EvaluateKnownConstInt(Ctx: C)
12256 : llvm::APSInt::getUnsigned(
12257 X: C.toCharUnitsFromBits(BitSize: C.getOpenMPDefaultSimdAlign(T: ParmTy))
12258 .getQuantity());
12259 ++NI;
12260 }
12261 // Mark linear parameters.
12262 auto *SI = Attr->steps_begin();
12263 auto *MI = Attr->modifiers_begin();
12264 for (const Expr *E : Attr->linears()) {
12265 E = E->IgnoreParenImpCasts();
12266 unsigned Pos;
12267 bool IsReferenceType = false;
12268 // Rescaling factor needed to compute the linear parameter
12269 // value in the mangled name.
12270 unsigned PtrRescalingFactor = 1;
12271 if (isa<CXXThisExpr>(Val: E)) {
12272 Pos = ParamPositions[FD];
12273 auto *P = cast<PointerType>(Val: E->getType());
12274 PtrRescalingFactor = CGM.getContext()
12275 .getTypeSizeInChars(T: P->getPointeeType())
12276 .getQuantity();
12277 } else {
12278 const auto *PVD = cast<ParmVarDecl>(Val: cast<DeclRefExpr>(Val: E)->getDecl())
12279 ->getCanonicalDecl();
12280 auto It = ParamPositions.find(Val: PVD);
12281 assert(It != ParamPositions.end() && "Function parameter not found");
12282 Pos = It->second;
12283 if (auto *P = dyn_cast<PointerType>(Val: PVD->getType()))
12284 PtrRescalingFactor = CGM.getContext()
12285 .getTypeSizeInChars(T: P->getPointeeType())
12286 .getQuantity();
12287 else if (PVD->getType()->isReferenceType()) {
12288 IsReferenceType = true;
12289 PtrRescalingFactor =
12290 CGM.getContext()
12291 .getTypeSizeInChars(T: PVD->getType().getNonReferenceType())
12292 .getQuantity();
12293 }
12294 }
12295 llvm::OpenMPIRBuilder::DeclareSimdAttrTy &ParamAttr = ParamAttrs[Pos];
12296 if (*MI == OMPC_LINEAR_ref)
12297 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef;
12298 else if (*MI == OMPC_LINEAR_uval)
12299 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearUVal;
12300 else if (IsReferenceType)
12301 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearVal;
12302 else
12303 ParamAttr.Kind = llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear;
12304 // Assuming a stride of 1, for `linear` without modifiers.
12305 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(X: 1);
12306 if (*SI) {
12307 Expr::EvalResult Result;
12308 if (!(*SI)->EvaluateAsInt(Result, Ctx: C, AllowSideEffects: Expr::SE_AllowSideEffects)) {
12309 if (const auto *DRE =
12310 cast<DeclRefExpr>(Val: (*SI)->IgnoreParenImpCasts())) {
12311 if (const auto *StridePVD =
12312 dyn_cast<ParmVarDecl>(Val: DRE->getDecl())) {
12313 ParamAttr.HasVarStride = true;
12314 auto It = ParamPositions.find(Val: StridePVD->getCanonicalDecl());
12315 assert(It != ParamPositions.end() &&
12316 "Function parameter not found");
12317 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(X: It->second);
12318 }
12319 }
12320 } else {
12321 ParamAttr.StrideOrArg = Result.Val.getInt();
12322 }
12323 }
12324 // If we are using a linear clause on a pointer, we need to
12325 // rescale the value of linear_step with the byte size of the
12326 // pointee type.
12327 if (!ParamAttr.HasVarStride &&
12328 (ParamAttr.Kind ==
12329 llvm::OpenMPIRBuilder::DeclareSimdKindTy::Linear ||
12330 ParamAttr.Kind ==
12331 llvm::OpenMPIRBuilder::DeclareSimdKindTy::LinearRef))
12332 ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;
12333 ++SI;
12334 ++MI;
12335 }
12336 llvm::APSInt VLENVal;
12337 SourceLocation ExprLoc;
12338 const Expr *VLENExpr = Attr->getSimdlen();
12339 if (VLENExpr) {
12340 VLENVal = VLENExpr->EvaluateKnownConstInt(Ctx: C);
12341 ExprLoc = VLENExpr->getExprLoc();
12342 }
12343 llvm::OpenMPIRBuilder::DeclareSimdBranch State =
12344 convertDeclareSimdBranch(State: Attr->getBranchState());
12345 if (CGM.getTriple().isX86()) {
12346 unsigned NumElts = evaluateCDTSize(FD, ParamAttrs);
12347 assert(NumElts && "Non-zero simdlen/cdtsize expected");
12348 OMPBuilder.emitX86DeclareSimdFunction(Fn, NumElements: NumElts, VLENVal, ParamAttrs,
12349 Branch: State);
12350 } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) {
12351 unsigned VLEN = VLENVal.getExtValue();
12352 // Get basic data for building the vector signature.
12353 const auto Data = getNDSWDS(FD, ParamAttrs);
12354 const unsigned NDS = std::get<0>(t: Data);
12355 const unsigned WDS = std::get<1>(t: Data);
12356 const bool OutputBecomesInput = std::get<2>(t: Data);
12357 if (CGM.getTarget().hasFeature(Feature: "sve")) {
12358 if (validateAArch64Simdlen(CGM, SLoc: ExprLoc, UserVLEN: VLEN, WDS, ISA: 's'))
12359 OMPBuilder.emitAArch64DeclareSimdFunction(
12360 Fn, VLENVal: VLEN, ParamAttrs, Branch: State, ISA: 's', NarrowestDataSize: NDS, OutputBecomesInput);
12361 } else if (CGM.getTarget().hasFeature(Feature: "neon")) {
12362 if (validateAArch64Simdlen(CGM, SLoc: ExprLoc, UserVLEN: VLEN, WDS, ISA: 'n'))
12363 OMPBuilder.emitAArch64DeclareSimdFunction(
12364 Fn, VLENVal: VLEN, ParamAttrs, Branch: State, ISA: 'n', NarrowestDataSize: NDS, OutputBecomesInput);
12365 }
12366 }
12367 }
12368 FD = FD->getPreviousDecl();
12369 }
12370}
12371
12372namespace {
12373/// Cleanup action for doacross support.
12374class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
12375public:
12376 static const int DoacrossFinArgs = 2;
12377
12378private:
12379 llvm::FunctionCallee RTLFn;
12380 llvm::Value *Args[DoacrossFinArgs];
12381
12382public:
12383 DoacrossCleanupTy(llvm::FunctionCallee RTLFn,
12384 ArrayRef<llvm::Value *> CallArgs)
12385 : RTLFn(RTLFn) {
12386 assert(CallArgs.size() == DoacrossFinArgs);
12387 std::copy(first: CallArgs.begin(), last: CallArgs.end(), result: std::begin(arr&: Args));
12388 }
12389 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12390 if (!CGF.HaveInsertPoint())
12391 return;
12392 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12393 }
12394};
12395} // namespace
12396
12397void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
12398 const OMPLoopDirective &D,
12399 ArrayRef<Expr *> NumIterations) {
12400 if (!CGF.HaveInsertPoint())
12401 return;
12402
12403 ASTContext &C = CGM.getContext();
12404 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true);
12405 RecordDecl *RD;
12406 if (KmpDimTy.isNull()) {
12407 // Build struct kmp_dim { // loop bounds info casted to kmp_int64
12408 // kmp_int64 lo; // lower
12409 // kmp_int64 up; // upper
12410 // kmp_int64 st; // stride
12411 // };
12412 RD = C.buildImplicitRecord(Name: "kmp_dim");
12413 RD->startDefinition();
12414 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12415 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12416 addFieldToRecordDecl(C, DC: RD, FieldTy: Int64Ty);
12417 RD->completeDefinition();
12418 KmpDimTy = C.getCanonicalTagType(TD: RD);
12419 } else {
12420 RD = KmpDimTy->castAsRecordDecl();
12421 }
12422 llvm::APInt Size(/*numBits=*/32, NumIterations.size());
12423 QualType ArrayTy = C.getConstantArrayType(EltTy: KmpDimTy, ArySize: Size, SizeExpr: nullptr,
12424 ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
12425
12426 Address DimsAddr = CGF.CreateMemTemp(T: ArrayTy, Name: "dims");
12427 CGF.EmitNullInitialization(DestPtr: DimsAddr, Ty: ArrayTy);
12428 enum { LowerFD = 0, UpperFD, StrideFD };
12429 // Fill dims with data.
12430 for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) {
12431 LValue DimsLVal = CGF.MakeAddrLValue(
12432 Addr: CGF.Builder.CreateConstArrayGEP(Addr: DimsAddr, Index: I), T: KmpDimTy);
12433 // dims.upper = num_iterations;
12434 LValue UpperLVal = CGF.EmitLValueForField(
12435 Base: DimsLVal, Field: *std::next(x: RD->field_begin(), n: UpperFD));
12436 llvm::Value *NumIterVal = CGF.EmitScalarConversion(
12437 Src: CGF.EmitScalarExpr(E: NumIterations[I]), SrcTy: NumIterations[I]->getType(),
12438 DstTy: Int64Ty, Loc: NumIterations[I]->getExprLoc());
12439 CGF.EmitStoreOfScalar(value: NumIterVal, lvalue: UpperLVal);
12440 // dims.stride = 1;
12441 LValue StrideLVal = CGF.EmitLValueForField(
12442 Base: DimsLVal, Field: *std::next(x: RD->field_begin(), n: StrideFD));
12443 CGF.EmitStoreOfScalar(value: llvm::ConstantInt::getSigned(Ty: CGM.Int64Ty, /*V=*/1),
12444 lvalue: StrideLVal);
12445 }
12446
12447 // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,
12448 // kmp_int32 num_dims, struct kmp_dim * dims);
12449 llvm::Value *Args[] = {
12450 emitUpdateLocation(CGF, Loc: D.getBeginLoc()),
12451 getThreadID(CGF, Loc: D.getBeginLoc()),
12452 llvm::ConstantInt::getSigned(Ty: CGM.Int32Ty, V: NumIterations.size()),
12453 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12454 V: CGF.Builder.CreateConstArrayGEP(Addr: DimsAddr, Index: 0).emitRawPointer(CGF),
12455 DestTy: CGM.VoidPtrTy)};
12456
12457 llvm::FunctionCallee RTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12458 M&: CGM.getModule(), FnID: OMPRTL___kmpc_doacross_init);
12459 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12460 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
12461 emitUpdateLocation(CGF, Loc: D.getEndLoc()), getThreadID(CGF, Loc: D.getEndLoc())};
12462 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12463 M&: CGM.getModule(), FnID: OMPRTL___kmpc_doacross_fini);
12464 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(Kind: NormalAndEHCleanup, A: FiniRTLFn,
12465 A: llvm::ArrayRef(FiniArgs));
12466}
12467
12468template <typename T>
12469static void EmitDoacrossOrdered(CodeGenFunction &CGF, CodeGenModule &CGM,
12470 const T *C, llvm::Value *ULoc,
12471 llvm::Value *ThreadID) {
12472 QualType Int64Ty =
12473 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1);
12474 llvm::APInt Size(/*numBits=*/32, C->getNumLoops());
12475 QualType ArrayTy = CGM.getContext().getConstantArrayType(
12476 EltTy: Int64Ty, ArySize: Size, SizeExpr: nullptr, ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
12477 Address CntAddr = CGF.CreateMemTemp(T: ArrayTy, Name: ".cnt.addr");
12478 for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) {
12479 const Expr *CounterVal = C->getLoopData(I);
12480 assert(CounterVal);
12481 llvm::Value *CntVal = CGF.EmitScalarConversion(
12482 Src: CGF.EmitScalarExpr(E: CounterVal), SrcTy: CounterVal->getType(), DstTy: Int64Ty,
12483 Loc: CounterVal->getExprLoc());
12484 CGF.EmitStoreOfScalar(Value: CntVal, Addr: CGF.Builder.CreateConstArrayGEP(Addr: CntAddr, Index: I),
12485 /*Volatile=*/false, Ty: Int64Ty);
12486 }
12487 llvm::Value *Args[] = {
12488 ULoc, ThreadID,
12489 CGF.Builder.CreateConstArrayGEP(Addr: CntAddr, Index: 0).emitRawPointer(CGF)};
12490 llvm::FunctionCallee RTLFn;
12491 llvm::OpenMPIRBuilder &OMPBuilder = CGM.getOpenMPRuntime().getOMPBuilder();
12492 OMPDoacrossKind<T> ODK;
12493 if (ODK.isSource(C)) {
12494 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
12495 FnID: OMPRTL___kmpc_doacross_post);
12496 } else {
12497 assert(ODK.isSink(C) && "Expect sink modifier.");
12498 RTLFn = OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(),
12499 FnID: OMPRTL___kmpc_doacross_wait);
12500 }
12501 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12502}
12503
12504void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12505 const OMPDependClause *C) {
12506 return EmitDoacrossOrdered<OMPDependClause>(
12507 CGF, CGM, C, ULoc: emitUpdateLocation(CGF, Loc: C->getBeginLoc()),
12508 ThreadID: getThreadID(CGF, Loc: C->getBeginLoc()));
12509}
12510
12511void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
12512 const OMPDoacrossClause *C) {
12513 return EmitDoacrossOrdered<OMPDoacrossClause>(
12514 CGF, CGM, C, ULoc: emitUpdateLocation(CGF, Loc: C->getBeginLoc()),
12515 ThreadID: getThreadID(CGF, Loc: C->getBeginLoc()));
12516}
12517
12518void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc,
12519 llvm::FunctionCallee Callee,
12520 ArrayRef<llvm::Value *> Args) const {
12521 assert(Loc.isValid() && "Outlined function call location must be valid.");
12522 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, TemporaryLocation: Loc);
12523
12524 if (auto *Fn = dyn_cast<llvm::Function>(Val: Callee.getCallee())) {
12525 if (Fn->doesNotThrow()) {
12526 CGF.EmitNounwindRuntimeCall(callee: Fn, args: Args);
12527 return;
12528 }
12529 }
12530 CGF.EmitRuntimeCall(callee: Callee, args: Args);
12531}
12532
12533void CGOpenMPRuntime::emitOutlinedFunctionCall(
12534 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn,
12535 ArrayRef<llvm::Value *> Args) const {
12536 emitCall(CGF, Loc, Callee: OutlinedFn, Args);
12537}
12538
12539void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) {
12540 if (const auto *FD = dyn_cast<FunctionDecl>(Val: D))
12541 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD: FD))
12542 HasEmittedDeclareTargetRegion = true;
12543}
12544
12545Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
12546 const VarDecl *NativeParam,
12547 const VarDecl *TargetParam) const {
12548 return CGF.GetAddrOfLocalVar(VD: NativeParam);
12549}
12550
12551/// Return allocator value from expression, or return a null allocator (default
12552/// when no allocator specified).
12553static llvm::Value *getAllocatorVal(CodeGenFunction &CGF,
12554 const Expr *Allocator) {
12555 llvm::Value *AllocVal;
12556 if (Allocator) {
12557 AllocVal = CGF.EmitScalarExpr(E: Allocator);
12558 // According to the standard, the original allocator type is a enum
12559 // (integer). Convert to pointer type, if required.
12560 AllocVal = CGF.EmitScalarConversion(Src: AllocVal, SrcTy: Allocator->getType(),
12561 DstTy: CGF.getContext().VoidPtrTy,
12562 Loc: Allocator->getExprLoc());
12563 } else {
12564 // If no allocator specified, it defaults to the null allocator.
12565 AllocVal = llvm::Constant::getNullValue(
12566 Ty: CGF.CGM.getTypes().ConvertType(T: CGF.getContext().VoidPtrTy));
12567 }
12568 return AllocVal;
12569}
12570
12571/// Return the alignment from an allocate directive if present.
12572static llvm::Value *getAlignmentValue(CodeGenModule &CGM, const VarDecl *VD) {
12573 std::optional<CharUnits> AllocateAlignment = CGM.getOMPAllocateAlignment(VD);
12574
12575 if (!AllocateAlignment)
12576 return nullptr;
12577
12578 return llvm::ConstantInt::get(Ty: CGM.SizeTy, V: AllocateAlignment->getQuantity());
12579}
12580
12581Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
12582 const VarDecl *VD) {
12583 if (!VD)
12584 return Address::invalid();
12585 Address UntiedAddr = Address::invalid();
12586 Address UntiedRealAddr = Address::invalid();
12587 auto It = FunctionToUntiedTaskStackMap.find(Val: CGF.CurFn);
12588 if (It != FunctionToUntiedTaskStackMap.end()) {
12589 const UntiedLocalVarsAddressesMap &UntiedData =
12590 UntiedLocalVarsStack[It->second];
12591 auto I = UntiedData.find(Key: VD);
12592 if (I != UntiedData.end()) {
12593 UntiedAddr = I->second.first;
12594 UntiedRealAddr = I->second.second;
12595 }
12596 }
12597 const VarDecl *CVD = VD->getCanonicalDecl();
12598 if (CVD->hasAttr<OMPAllocateDeclAttr>()) {
12599 // Use the default allocation.
12600 if (!isAllocatableDecl(VD))
12601 return UntiedAddr;
12602 llvm::Value *Size;
12603 CharUnits Align = CGM.getContext().getDeclAlign(D: CVD);
12604 if (CVD->getType()->isVariablyModifiedType()) {
12605 Size = CGF.getTypeSize(Ty: CVD->getType());
12606 // Align the size: ((size + align - 1) / align) * align
12607 Size = CGF.Builder.CreateNUWAdd(
12608 LHS: Size, RHS: CGM.getSize(numChars: Align - CharUnits::fromQuantity(Quantity: 1)));
12609 Size = CGF.Builder.CreateUDiv(LHS: Size, RHS: CGM.getSize(numChars: Align));
12610 Size = CGF.Builder.CreateNUWMul(LHS: Size, RHS: CGM.getSize(numChars: Align));
12611 } else {
12612 CharUnits Sz = CGM.getContext().getTypeSizeInChars(T: CVD->getType());
12613 Size = CGM.getSize(numChars: Sz.alignTo(Align));
12614 }
12615 llvm::Value *ThreadID = getThreadID(CGF, Loc: CVD->getBeginLoc());
12616 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>();
12617 const Expr *Allocator = AA->getAllocator();
12618 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator);
12619 llvm::Value *Alignment = getAlignmentValue(CGM, VD: CVD);
12620 SmallVector<llvm::Value *, 4> Args;
12621 Args.push_back(Elt: ThreadID);
12622 if (Alignment)
12623 Args.push_back(Elt: Alignment);
12624 Args.push_back(Elt: Size);
12625 Args.push_back(Elt: AllocVal);
12626 llvm::omp::RuntimeFunction FnID =
12627 Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;
12628 llvm::Value *Addr = CGF.EmitRuntimeCall(
12629 callee: OMPBuilder.getOrCreateRuntimeFunction(M&: CGM.getModule(), FnID), args: Args,
12630 name: getName(Parts: {CVD->getName(), ".void.addr"}));
12631 llvm::FunctionCallee FiniRTLFn = OMPBuilder.getOrCreateRuntimeFunction(
12632 M&: CGM.getModule(), FnID: OMPRTL___kmpc_free);
12633 QualType Ty = CGM.getContext().getPointerType(T: CVD->getType());
12634 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12635 V: Addr, DestTy: CGF.ConvertTypeForMem(T: Ty), Name: getName(Parts: {CVD->getName(), ".addr"}));
12636 if (UntiedAddr.isValid())
12637 CGF.EmitStoreOfScalar(Value: Addr, Addr: UntiedAddr, /*Volatile=*/false, Ty);
12638
12639 // Cleanup action for allocate support.
12640 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {
12641 llvm::FunctionCallee RTLFn;
12642 SourceLocation::UIntTy LocEncoding;
12643 Address Addr;
12644 const Expr *AllocExpr;
12645
12646 public:
12647 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,
12648 SourceLocation::UIntTy LocEncoding, Address Addr,
12649 const Expr *AllocExpr)
12650 : RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),
12651 AllocExpr(AllocExpr) {}
12652 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {
12653 if (!CGF.HaveInsertPoint())
12654 return;
12655 llvm::Value *Args[3];
12656 Args[0] = CGF.CGM.getOpenMPRuntime().getThreadID(
12657 CGF, Loc: SourceLocation::getFromRawEncoding(Encoding: LocEncoding));
12658 Args[1] = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
12659 V: Addr.emitRawPointer(CGF), DestTy: CGF.VoidPtrTy);
12660 llvm::Value *AllocVal = getAllocatorVal(CGF, Allocator: AllocExpr);
12661 Args[2] = AllocVal;
12662 CGF.EmitRuntimeCall(callee: RTLFn, args: Args);
12663 }
12664 };
12665 Address VDAddr =
12666 UntiedRealAddr.isValid()
12667 ? UntiedRealAddr
12668 : Address(Addr, CGF.ConvertTypeForMem(T: CVD->getType()), Align);
12669 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(
12670 Kind: NormalAndEHCleanup, A: FiniRTLFn, A: CVD->getLocation().getRawEncoding(),
12671 A: VDAddr, A: Allocator);
12672 if (UntiedRealAddr.isValid())
12673 if (auto *Region =
12674 dyn_cast_or_null<CGOpenMPRegionInfo>(Val: CGF.CapturedStmtInfo))
12675 Region->emitUntiedSwitch(CGF);
12676 return VDAddr;
12677 }
12678 return UntiedAddr;
12679}
12680
12681bool CGOpenMPRuntime::isLocalVarInUntiedTask(CodeGenFunction &CGF,
12682 const VarDecl *VD) const {
12683 auto It = FunctionToUntiedTaskStackMap.find(Val: CGF.CurFn);
12684 if (It == FunctionToUntiedTaskStackMap.end())
12685 return false;
12686 return UntiedLocalVarsStack[It->second].count(Key: VD) > 0;
12687}
12688
12689CGOpenMPRuntime::NontemporalDeclsRAII::NontemporalDeclsRAII(
12690 CodeGenModule &CGM, const OMPLoopDirective &S)
12691 : CGM(CGM), NeedToPush(S.hasClausesOfKind<OMPNontemporalClause>()) {
12692 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12693 if (!NeedToPush)
12694 return;
12695 NontemporalDeclsSet &DS =
12696 CGM.getOpenMPRuntime().NontemporalDeclsStack.emplace_back();
12697 for (const auto *C : S.getClausesOfKind<OMPNontemporalClause>()) {
12698 for (const Stmt *Ref : C->private_refs()) {
12699 const auto *SimpleRefExpr = cast<Expr>(Val: Ref)->IgnoreParenImpCasts();
12700 const ValueDecl *VD;
12701 if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: SimpleRefExpr)) {
12702 VD = DRE->getDecl();
12703 } else {
12704 const auto *ME = cast<MemberExpr>(Val: SimpleRefExpr);
12705 assert((ME->isImplicitCXXThis() ||
12706 isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&
12707 "Expected member of current class.");
12708 VD = ME->getMemberDecl();
12709 }
12710 DS.insert(V: VD);
12711 }
12712 }
12713}
12714
12715CGOpenMPRuntime::NontemporalDeclsRAII::~NontemporalDeclsRAII() {
12716 if (!NeedToPush)
12717 return;
12718 CGM.getOpenMPRuntime().NontemporalDeclsStack.pop_back();
12719}
12720
12721CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::UntiedTaskLocalDeclsRAII(
12722 CodeGenFunction &CGF,
12723 const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
12724 std::pair<Address, Address>> &LocalVars)
12725 : CGM(CGF.CGM), NeedToPush(!LocalVars.empty()) {
12726 if (!NeedToPush)
12727 return;
12728 CGM.getOpenMPRuntime().FunctionToUntiedTaskStackMap.try_emplace(
12729 Key: CGF.CurFn, Args: CGM.getOpenMPRuntime().UntiedLocalVarsStack.size());
12730 CGM.getOpenMPRuntime().UntiedLocalVarsStack.push_back(Elt: LocalVars);
12731}
12732
12733CGOpenMPRuntime::UntiedTaskLocalDeclsRAII::~UntiedTaskLocalDeclsRAII() {
12734 if (!NeedToPush)
12735 return;
12736 CGM.getOpenMPRuntime().UntiedLocalVarsStack.pop_back();
12737}
12738
12739bool CGOpenMPRuntime::isNontemporalDecl(const ValueDecl *VD) const {
12740 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12741
12742 return llvm::any_of(
12743 Range&: CGM.getOpenMPRuntime().NontemporalDeclsStack,
12744 P: [VD](const NontemporalDeclsSet &Set) { return Set.contains(V: VD); });
12745}
12746
12747void CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(
12748 const OMPExecutableDirective &S,
12749 llvm::DenseSet<CanonicalDeclPtr<const Decl>> &NeedToAddForLPCsAsDisabled)
12750 const {
12751 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs;
12752 // Vars in target/task regions must be excluded completely.
12753 if (isOpenMPTargetExecutionDirective(DKind: S.getDirectiveKind()) ||
12754 isOpenMPTaskingDirective(Kind: S.getDirectiveKind())) {
12755 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
12756 getOpenMPCaptureRegions(CaptureRegions, DKind: S.getDirectiveKind());
12757 const CapturedStmt *CS = S.getCapturedStmt(RegionKind: CaptureRegions.front());
12758 for (const CapturedStmt::Capture &Cap : CS->captures()) {
12759 if (Cap.capturesVariable() || Cap.capturesVariableByCopy())
12760 NeedToCheckForLPCs.insert(V: Cap.getCapturedVar());
12761 }
12762 }
12763 // Exclude vars in private clauses.
12764 for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
12765 for (const Expr *Ref : C->varlist()) {
12766 if (!Ref->getType()->isScalarType())
12767 continue;
12768 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12769 if (!DRE)
12770 continue;
12771 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12772 }
12773 }
12774 for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
12775 for (const Expr *Ref : C->varlist()) {
12776 if (!Ref->getType()->isScalarType())
12777 continue;
12778 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12779 if (!DRE)
12780 continue;
12781 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12782 }
12783 }
12784 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
12785 for (const Expr *Ref : C->varlist()) {
12786 if (!Ref->getType()->isScalarType())
12787 continue;
12788 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12789 if (!DRE)
12790 continue;
12791 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12792 }
12793 }
12794 for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
12795 for (const Expr *Ref : C->varlist()) {
12796 if (!Ref->getType()->isScalarType())
12797 continue;
12798 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12799 if (!DRE)
12800 continue;
12801 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12802 }
12803 }
12804 for (const auto *C : S.getClausesOfKind<OMPLinearClause>()) {
12805 for (const Expr *Ref : C->varlist()) {
12806 if (!Ref->getType()->isScalarType())
12807 continue;
12808 const auto *DRE = dyn_cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts());
12809 if (!DRE)
12810 continue;
12811 NeedToCheckForLPCs.insert(V: DRE->getDecl());
12812 }
12813 }
12814 for (const Decl *VD : NeedToCheckForLPCs) {
12815 for (const LastprivateConditionalData &Data :
12816 llvm::reverse(C&: CGM.getOpenMPRuntime().LastprivateConditionalStack)) {
12817 if (Data.DeclToUniqueName.count(Key: VD) > 0) {
12818 if (!Data.Disabled)
12819 NeedToAddForLPCsAsDisabled.insert(V: VD);
12820 break;
12821 }
12822 }
12823 }
12824}
12825
12826CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
12827 CodeGenFunction &CGF, const OMPExecutableDirective &S, LValue IVLVal)
12828 : CGM(CGF.CGM),
12829 Action((CGM.getLangOpts().OpenMP >= 50 &&
12830 llvm::any_of(Range: S.getClausesOfKind<OMPLastprivateClause>(),
12831 P: [](const OMPLastprivateClause *C) {
12832 return C->getKind() ==
12833 OMPC_LASTPRIVATE_conditional;
12834 }))
12835 ? ActionToDo::PushAsLastprivateConditional
12836 : ActionToDo::DoNotPush) {
12837 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12838 if (CGM.getLangOpts().OpenMP < 50 || Action == ActionToDo::DoNotPush)
12839 return;
12840 assert(Action == ActionToDo::PushAsLastprivateConditional &&
12841 "Expected a push action.");
12842 LastprivateConditionalData &Data =
12843 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
12844 for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
12845 if (C->getKind() != OMPC_LASTPRIVATE_conditional)
12846 continue;
12847
12848 for (const Expr *Ref : C->varlist()) {
12849 Data.DeclToUniqueName.insert(KV: std::make_pair(
12850 x: cast<DeclRefExpr>(Val: Ref->IgnoreParenImpCasts())->getDecl(),
12851 y: SmallString<16>(generateUniqueName(CGM, Prefix: "pl_cond", Ref))));
12852 }
12853 }
12854 Data.IVLVal = IVLVal;
12855 Data.Fn = CGF.CurFn;
12856}
12857
12858CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(
12859 CodeGenFunction &CGF, const OMPExecutableDirective &S)
12860 : CGM(CGF.CGM), Action(ActionToDo::DoNotPush) {
12861 assert(CGM.getLangOpts().OpenMP && "Not in OpenMP mode.");
12862 if (CGM.getLangOpts().OpenMP < 50)
12863 return;
12864 llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled;
12865 tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);
12866 if (!NeedToAddForLPCsAsDisabled.empty()) {
12867 Action = ActionToDo::DisableLastprivateConditional;
12868 LastprivateConditionalData &Data =
12869 CGM.getOpenMPRuntime().LastprivateConditionalStack.emplace_back();
12870 for (const Decl *VD : NeedToAddForLPCsAsDisabled)
12871 Data.DeclToUniqueName.try_emplace(Key: VD);
12872 Data.Fn = CGF.CurFn;
12873 Data.Disabled = true;
12874 }
12875}
12876
12877CGOpenMPRuntime::LastprivateConditionalRAII
12878CGOpenMPRuntime::LastprivateConditionalRAII::disable(
12879 CodeGenFunction &CGF, const OMPExecutableDirective &S) {
12880 return LastprivateConditionalRAII(CGF, S);
12881}
12882
12883CGOpenMPRuntime::LastprivateConditionalRAII::~LastprivateConditionalRAII() {
12884 if (CGM.getLangOpts().OpenMP < 50)
12885 return;
12886 if (Action == ActionToDo::DisableLastprivateConditional) {
12887 assert(CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
12888 "Expected list of disabled private vars.");
12889 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
12890 }
12891 if (Action == ActionToDo::PushAsLastprivateConditional) {
12892 assert(
12893 !CGM.getOpenMPRuntime().LastprivateConditionalStack.back().Disabled &&
12894 "Expected list of lastprivate conditional vars.");
12895 CGM.getOpenMPRuntime().LastprivateConditionalStack.pop_back();
12896 }
12897}
12898
12899Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF,
12900 const VarDecl *VD) {
12901 ASTContext &C = CGM.getContext();
12902 auto I = LastprivateConditionalToTypes.try_emplace(Key: CGF.CurFn).first;
12903 QualType NewType;
12904 const FieldDecl *VDField;
12905 const FieldDecl *FiredField;
12906 LValue BaseLVal;
12907 auto VI = I->getSecond().find(Val: VD);
12908 if (VI == I->getSecond().end()) {
12909 RecordDecl *RD = C.buildImplicitRecord(Name: "lasprivate.conditional");
12910 RD->startDefinition();
12911 VDField = addFieldToRecordDecl(C, DC: RD, FieldTy: VD->getType().getNonReferenceType());
12912 FiredField = addFieldToRecordDecl(C, DC: RD, FieldTy: C.CharTy);
12913 RD->completeDefinition();
12914 NewType = C.getCanonicalTagType(TD: RD);
12915 Address Addr = CGF.CreateMemTemp(T: NewType, Align: C.getDeclAlign(D: VD), Name: VD->getName());
12916 BaseLVal = CGF.MakeAddrLValue(Addr, T: NewType, Source: AlignmentSource::Decl);
12917 I->getSecond().try_emplace(Key: VD, Args&: NewType, Args&: VDField, Args&: FiredField, Args&: BaseLVal);
12918 } else {
12919 NewType = std::get<0>(t&: VI->getSecond());
12920 VDField = std::get<1>(t&: VI->getSecond());
12921 FiredField = std::get<2>(t&: VI->getSecond());
12922 BaseLVal = std::get<3>(t&: VI->getSecond());
12923 }
12924 LValue FiredLVal =
12925 CGF.EmitLValueForField(Base: BaseLVal, Field: FiredField);
12926 CGF.EmitStoreOfScalar(
12927 value: llvm::ConstantInt::getNullValue(Ty: CGF.ConvertTypeForMem(T: C.CharTy)),
12928 lvalue: FiredLVal);
12929 return CGF.EmitLValueForField(Base: BaseLVal, Field: VDField).getAddress();
12930}
12931
12932namespace {
12933/// Checks if the lastprivate conditional variable is referenced in LHS.
12934class LastprivateConditionalRefChecker final
12935 : public ConstStmtVisitor<LastprivateConditionalRefChecker, bool> {
12936 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM;
12937 const Expr *FoundE = nullptr;
12938 const Decl *FoundD = nullptr;
12939 StringRef UniqueDeclName;
12940 LValue IVLVal;
12941 llvm::Function *FoundFn = nullptr;
12942 SourceLocation Loc;
12943
12944public:
12945 bool VisitDeclRefExpr(const DeclRefExpr *E) {
12946 for (const CGOpenMPRuntime::LastprivateConditionalData &D :
12947 llvm::reverse(C&: LPM)) {
12948 auto It = D.DeclToUniqueName.find(Key: E->getDecl());
12949 if (It == D.DeclToUniqueName.end())
12950 continue;
12951 if (D.Disabled)
12952 return false;
12953 FoundE = E;
12954 FoundD = E->getDecl()->getCanonicalDecl();
12955 UniqueDeclName = It->second;
12956 IVLVal = D.IVLVal;
12957 FoundFn = D.Fn;
12958 break;
12959 }
12960 return FoundE == E;
12961 }
12962 bool VisitMemberExpr(const MemberExpr *E) {
12963 if (!CodeGenFunction::IsWrappedCXXThis(E: E->getBase()))
12964 return false;
12965 for (const CGOpenMPRuntime::LastprivateConditionalData &D :
12966 llvm::reverse(C&: LPM)) {
12967 auto It = D.DeclToUniqueName.find(Key: E->getMemberDecl());
12968 if (It == D.DeclToUniqueName.end())
12969 continue;
12970 if (D.Disabled)
12971 return false;
12972 FoundE = E;
12973 FoundD = E->getMemberDecl()->getCanonicalDecl();
12974 UniqueDeclName = It->second;
12975 IVLVal = D.IVLVal;
12976 FoundFn = D.Fn;
12977 break;
12978 }
12979 return FoundE == E;
12980 }
12981 bool VisitStmt(const Stmt *S) {
12982 for (const Stmt *Child : S->children()) {
12983 if (!Child)
12984 continue;
12985 if (const auto *E = dyn_cast<Expr>(Val: Child))
12986 if (!E->isGLValue())
12987 continue;
12988 if (Visit(S: Child))
12989 return true;
12990 }
12991 return false;
12992 }
12993 explicit LastprivateConditionalRefChecker(
12994 ArrayRef<CGOpenMPRuntime::LastprivateConditionalData> LPM)
12995 : LPM(LPM) {}
12996 std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>
12997 getFoundData() const {
12998 return std::make_tuple(args: FoundE, args: FoundD, args: UniqueDeclName, args: IVLVal, args: FoundFn);
12999 }
13000};
13001} // namespace
13002
13003void CGOpenMPRuntime::emitLastprivateConditionalUpdate(CodeGenFunction &CGF,
13004 LValue IVLVal,
13005 StringRef UniqueDeclName,
13006 LValue LVal,
13007 SourceLocation Loc) {
13008 // Last updated loop counter for the lastprivate conditional var.
13009 // int<xx> last_iv = 0;
13010 llvm::Type *LLIVTy = CGF.ConvertTypeForMem(T: IVLVal.getType());
13011 llvm::Constant *LastIV = OMPBuilder.getOrCreateInternalVariable(
13012 Ty: LLIVTy, Name: getName(Parts: {UniqueDeclName, "iv"}));
13013 cast<llvm::GlobalVariable>(Val: LastIV)->setAlignment(
13014 IVLVal.getAlignment().getAsAlign());
13015 LValue LastIVLVal =
13016 CGF.MakeNaturalAlignRawAddrLValue(V: LastIV, T: IVLVal.getType());
13017
13018 // Last value of the lastprivate conditional.
13019 // decltype(priv_a) last_a;
13020 llvm::GlobalVariable *Last = OMPBuilder.getOrCreateInternalVariable(
13021 Ty: CGF.ConvertTypeForMem(T: LVal.getType()), Name: UniqueDeclName);
13022 cast<llvm::GlobalVariable>(Val: Last)->setAlignment(
13023 LVal.getAlignment().getAsAlign());
13024 LValue LastLVal =
13025 CGF.MakeRawAddrLValue(V: Last, T: LVal.getType(), Alignment: LVal.getAlignment());
13026
13027 // Global loop counter. Required to handle inner parallel-for regions.
13028 // iv
13029 llvm::Value *IVVal = CGF.EmitLoadOfScalar(lvalue: IVLVal, Loc);
13030
13031 // #pragma omp critical(a)
13032 // if (last_iv <= iv) {
13033 // last_iv = iv;
13034 // last_a = priv_a;
13035 // }
13036 auto &&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,
13037 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
13038 Action.Enter(CGF);
13039 llvm::Value *LastIVVal = CGF.EmitLoadOfScalar(lvalue: LastIVLVal, Loc);
13040 // (last_iv <= iv) ? Check if the variable is updated and store new
13041 // value in global var.
13042 llvm::Value *CmpRes;
13043 if (IVLVal.getType()->isSignedIntegerType()) {
13044 CmpRes = CGF.Builder.CreateICmpSLE(LHS: LastIVVal, RHS: IVVal);
13045 } else {
13046 assert(IVLVal.getType()->isUnsignedIntegerType() &&
13047 "Loop iteration variable must be integer.");
13048 CmpRes = CGF.Builder.CreateICmpULE(LHS: LastIVVal, RHS: IVVal);
13049 }
13050 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: "lp_cond_then");
13051 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(name: "lp_cond_exit");
13052 CGF.Builder.CreateCondBr(Cond: CmpRes, True: ThenBB, False: ExitBB);
13053 // {
13054 CGF.EmitBlock(BB: ThenBB);
13055
13056 // last_iv = iv;
13057 CGF.EmitStoreOfScalar(value: IVVal, lvalue: LastIVLVal);
13058
13059 // last_a = priv_a;
13060 switch (CGF.getEvaluationKind(T: LVal.getType())) {
13061 case TEK_Scalar: {
13062 llvm::Value *PrivVal = CGF.EmitLoadOfScalar(lvalue: LVal, Loc);
13063 CGF.EmitStoreOfScalar(value: PrivVal, lvalue: LastLVal);
13064 break;
13065 }
13066 case TEK_Complex: {
13067 CodeGenFunction::ComplexPairTy PrivVal = CGF.EmitLoadOfComplex(src: LVal, loc: Loc);
13068 CGF.EmitStoreOfComplex(V: PrivVal, dest: LastLVal, /*isInit=*/false);
13069 break;
13070 }
13071 case TEK_Aggregate:
13072 llvm_unreachable(
13073 "Aggregates are not supported in lastprivate conditional.");
13074 }
13075 // }
13076 CGF.EmitBranch(Block: ExitBB);
13077 // There is no need to emit line number for unconditional branch.
13078 (void)ApplyDebugLocation::CreateEmpty(CGF);
13079 CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
13080 };
13081
13082 if (CGM.getLangOpts().OpenMPSimd) {
13083 // Do not emit as a critical region as no parallel region could be emitted.
13084 RegionCodeGenTy ThenRCG(CodeGen);
13085 ThenRCG(CGF);
13086 } else {
13087 emitCriticalRegion(CGF, CriticalName: UniqueDeclName, CriticalOpGen: CodeGen, Loc);
13088 }
13089}
13090
13091void CGOpenMPRuntime::checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
13092 const Expr *LHS) {
13093 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
13094 return;
13095 LastprivateConditionalRefChecker Checker(LastprivateConditionalStack);
13096 if (!Checker.Visit(S: LHS))
13097 return;
13098 const Expr *FoundE;
13099 const Decl *FoundD;
13100 StringRef UniqueDeclName;
13101 LValue IVLVal;
13102 llvm::Function *FoundFn;
13103 std::tie(args&: FoundE, args&: FoundD, args&: UniqueDeclName, args&: IVLVal, args&: FoundFn) =
13104 Checker.getFoundData();
13105 if (FoundFn != CGF.CurFn) {
13106 // Special codegen for inner parallel regions.
13107 // ((struct.lastprivate.conditional*)&priv_a)->Fired = 1;
13108 auto It = LastprivateConditionalToTypes[FoundFn].find(Val: FoundD);
13109 assert(It != LastprivateConditionalToTypes[FoundFn].end() &&
13110 "Lastprivate conditional is not found in outer region.");
13111 QualType StructTy = std::get<0>(t&: It->getSecond());
13112 const FieldDecl* FiredDecl = std::get<2>(t&: It->getSecond());
13113 LValue PrivLVal = CGF.EmitLValue(E: FoundE);
13114 Address StructAddr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
13115 Addr: PrivLVal.getAddress(),
13116 Ty: CGF.ConvertTypeForMem(T: CGF.getContext().getPointerType(T: StructTy)),
13117 ElementTy: CGF.ConvertTypeForMem(T: StructTy));
13118 LValue BaseLVal =
13119 CGF.MakeAddrLValue(Addr: StructAddr, T: StructTy, Source: AlignmentSource::Decl);
13120 LValue FiredLVal = CGF.EmitLValueForField(Base: BaseLVal, Field: FiredDecl);
13121 CGF.EmitAtomicStore(rvalue: RValue::get(V: llvm::ConstantInt::get(
13122 Ty: CGF.ConvertTypeForMem(T: FiredDecl->getType()), V: 1)),
13123 lvalue: FiredLVal, AO: llvm::AtomicOrdering::Unordered,
13124 /*IsVolatile=*/true, /*isInit=*/false);
13125 return;
13126 }
13127
13128 // Private address of the lastprivate conditional in the current context.
13129 // priv_a
13130 LValue LVal = CGF.EmitLValue(E: FoundE);
13131 emitLastprivateConditionalUpdate(CGF, IVLVal, UniqueDeclName, LVal,
13132 Loc: FoundE->getExprLoc());
13133}
13134
13135void CGOpenMPRuntime::checkAndEmitSharedLastprivateConditional(
13136 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13137 const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls) {
13138 if (CGF.getLangOpts().OpenMP < 50 || LastprivateConditionalStack.empty())
13139 return;
13140 auto Range = llvm::reverse(C&: LastprivateConditionalStack);
13141 auto It = llvm::find_if(
13142 Range, P: [](const LastprivateConditionalData &D) { return !D.Disabled; });
13143 if (It == Range.end() || It->Fn != CGF.CurFn)
13144 return;
13145 auto LPCI = LastprivateConditionalToTypes.find(Val: It->Fn);
13146 assert(LPCI != LastprivateConditionalToTypes.end() &&
13147 "Lastprivates must be registered already.");
13148 SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
13149 getOpenMPCaptureRegions(CaptureRegions, DKind: D.getDirectiveKind());
13150 const CapturedStmt *CS = D.getCapturedStmt(RegionKind: CaptureRegions.back());
13151 for (const auto &Pair : It->DeclToUniqueName) {
13152 const auto *VD = cast<VarDecl>(Val: Pair.first->getCanonicalDecl());
13153 if (!CS->capturesVariable(Var: VD) || IgnoredDecls.contains(V: VD))
13154 continue;
13155 auto I = LPCI->getSecond().find(Val: Pair.first);
13156 assert(I != LPCI->getSecond().end() &&
13157 "Lastprivate must be rehistered already.");
13158 // bool Cmp = priv_a.Fired != 0;
13159 LValue BaseLVal = std::get<3>(t&: I->getSecond());
13160 LValue FiredLVal =
13161 CGF.EmitLValueForField(Base: BaseLVal, Field: std::get<2>(t&: I->getSecond()));
13162 llvm::Value *Res = CGF.EmitLoadOfScalar(lvalue: FiredLVal, Loc: D.getBeginLoc());
13163 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Arg: Res);
13164 llvm::BasicBlock *ThenBB = CGF.createBasicBlock(name: "lpc.then");
13165 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(name: "lpc.done");
13166 // if (Cmp) {
13167 CGF.Builder.CreateCondBr(Cond: Cmp, True: ThenBB, False: DoneBB);
13168 CGF.EmitBlock(BB: ThenBB);
13169 Address Addr = CGF.GetAddrOfLocalVar(VD);
13170 LValue LVal;
13171 if (VD->getType()->isReferenceType())
13172 LVal = CGF.EmitLoadOfReferenceLValue(RefAddr: Addr, RefTy: VD->getType(),
13173 Source: AlignmentSource::Decl);
13174 else
13175 LVal = CGF.MakeAddrLValue(Addr, T: VD->getType().getNonReferenceType(),
13176 Source: AlignmentSource::Decl);
13177 emitLastprivateConditionalUpdate(CGF, IVLVal: It->IVLVal, UniqueDeclName: Pair.second, LVal,
13178 Loc: D.getBeginLoc());
13179 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
13180 CGF.EmitBlock(BB: DoneBB, /*IsFinal=*/IsFinished: true);
13181 // }
13182 }
13183}
13184
13185void CGOpenMPRuntime::emitLastprivateConditionalFinalUpdate(
13186 CodeGenFunction &CGF, LValue PrivLVal, const VarDecl *VD,
13187 SourceLocation Loc) {
13188 if (CGF.getLangOpts().OpenMP < 50)
13189 return;
13190 auto It = LastprivateConditionalStack.back().DeclToUniqueName.find(Key: VD);
13191 assert(It != LastprivateConditionalStack.back().DeclToUniqueName.end() &&
13192 "Unknown lastprivate conditional variable.");
13193 StringRef UniqueName = It->second;
13194 llvm::GlobalVariable *GV = CGM.getModule().getNamedGlobal(Name: UniqueName);
13195 // The variable was not updated in the region - exit.
13196 if (!GV)
13197 return;
13198 LValue LPLVal = CGF.MakeRawAddrLValue(
13199 V: GV, T: PrivLVal.getType().getNonReferenceType(), Alignment: PrivLVal.getAlignment());
13200 llvm::Value *Res = CGF.EmitLoadOfScalar(lvalue: LPLVal, Loc);
13201 CGF.EmitStoreOfScalar(value: Res, lvalue: PrivLVal);
13202}
13203
13204llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
13205 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13206 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13207 const RegionCodeGenTy &CodeGen) {
13208 llvm_unreachable("Not supported in SIMD-only mode");
13209}
13210
13211llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction(
13212 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13213 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
13214 const RegionCodeGenTy &CodeGen) {
13215 llvm_unreachable("Not supported in SIMD-only mode");
13216}
13217
13218llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction(
13219 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
13220 const VarDecl *PartIDVar, const VarDecl *TaskTVar,
13221 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
13222 bool Tied, unsigned &NumberOfParts) {
13223 llvm_unreachable("Not supported in SIMD-only mode");
13224}
13225
13226void CGOpenMPSIMDRuntime::emitParallelCall(
13227 CodeGenFunction &CGF, SourceLocation Loc, llvm::Function *OutlinedFn,
13228 ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond,
13229 llvm::Value *NumThreads, OpenMPNumThreadsClauseModifier NumThreadsModifier,
13230 OpenMPSeverityClauseKind Severity, const Expr *Message) {
13231 llvm_unreachable("Not supported in SIMD-only mode");
13232}
13233
13234void CGOpenMPSIMDRuntime::emitCriticalRegion(
13235 CodeGenFunction &CGF, StringRef CriticalName,
13236 const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc,
13237 const Expr *Hint) {
13238 llvm_unreachable("Not supported in SIMD-only mode");
13239}
13240
13241void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF,
13242 const RegionCodeGenTy &MasterOpGen,
13243 SourceLocation Loc) {
13244 llvm_unreachable("Not supported in SIMD-only mode");
13245}
13246
13247void CGOpenMPSIMDRuntime::emitMaskedRegion(CodeGenFunction &CGF,
13248 const RegionCodeGenTy &MasterOpGen,
13249 SourceLocation Loc,
13250 const Expr *Filter) {
13251 llvm_unreachable("Not supported in SIMD-only mode");
13252}
13253
13254void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
13255 SourceLocation Loc) {
13256 llvm_unreachable("Not supported in SIMD-only mode");
13257}
13258
13259void CGOpenMPSIMDRuntime::emitTaskgroupRegion(
13260 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
13261 SourceLocation Loc) {
13262 llvm_unreachable("Not supported in SIMD-only mode");
13263}
13264
13265void CGOpenMPSIMDRuntime::emitSingleRegion(
13266 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen,
13267 SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars,
13268 ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs,
13269 ArrayRef<const Expr *> AssignmentOps) {
13270 llvm_unreachable("Not supported in SIMD-only mode");
13271}
13272
13273void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF,
13274 const RegionCodeGenTy &OrderedOpGen,
13275 SourceLocation Loc,
13276 bool IsThreads) {
13277 llvm_unreachable("Not supported in SIMD-only mode");
13278}
13279
13280void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF,
13281 SourceLocation Loc,
13282 OpenMPDirectiveKind Kind,
13283 bool EmitChecks,
13284 bool ForceSimpleCall) {
13285 llvm_unreachable("Not supported in SIMD-only mode");
13286}
13287
13288void CGOpenMPSIMDRuntime::emitForDispatchInit(
13289 CodeGenFunction &CGF, SourceLocation Loc,
13290 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned,
13291 bool Ordered, const DispatchRTInput &DispatchValues) {
13292 llvm_unreachable("Not supported in SIMD-only mode");
13293}
13294
13295void CGOpenMPSIMDRuntime::emitForDispatchDeinit(CodeGenFunction &CGF,
13296 SourceLocation Loc) {
13297 llvm_unreachable("Not supported in SIMD-only mode");
13298}
13299
13300void CGOpenMPSIMDRuntime::emitForStaticInit(
13301 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind,
13302 const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) {
13303 llvm_unreachable("Not supported in SIMD-only mode");
13304}
13305
13306void CGOpenMPSIMDRuntime::emitDistributeStaticInit(
13307 CodeGenFunction &CGF, SourceLocation Loc,
13308 OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) {
13309 llvm_unreachable("Not supported in SIMD-only mode");
13310}
13311
13312void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF,
13313 SourceLocation Loc,
13314 unsigned IVSize,
13315 bool IVSigned) {
13316 llvm_unreachable("Not supported in SIMD-only mode");
13317}
13318
13319void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF,
13320 SourceLocation Loc,
13321 OpenMPDirectiveKind DKind) {
13322 llvm_unreachable("Not supported in SIMD-only mode");
13323}
13324
13325llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF,
13326 SourceLocation Loc,
13327 unsigned IVSize, bool IVSigned,
13328 Address IL, Address LB,
13329 Address UB, Address ST) {
13330 llvm_unreachable("Not supported in SIMD-only mode");
13331}
13332
13333void CGOpenMPSIMDRuntime::emitNumThreadsClause(
13334 CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc,
13335 OpenMPNumThreadsClauseModifier Modifier, OpenMPSeverityClauseKind Severity,
13336 SourceLocation SeverityLoc, const Expr *Message,
13337 SourceLocation MessageLoc) {
13338 llvm_unreachable("Not supported in SIMD-only mode");
13339}
13340
13341void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF,
13342 ProcBindKind ProcBind,
13343 SourceLocation Loc) {
13344 llvm_unreachable("Not supported in SIMD-only mode");
13345}
13346
13347Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF,
13348 const VarDecl *VD,
13349 Address VDAddr,
13350 SourceLocation Loc) {
13351 llvm_unreachable("Not supported in SIMD-only mode");
13352}
13353
13354llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition(
13355 const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit,
13356 CodeGenFunction *CGF) {
13357 llvm_unreachable("Not supported in SIMD-only mode");
13358}
13359
13360Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate(
13361 CodeGenFunction &CGF, QualType VarType, StringRef Name) {
13362 llvm_unreachable("Not supported in SIMD-only mode");
13363}
13364
13365void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
13366 ArrayRef<const Expr *> Vars,
13367 SourceLocation Loc,
13368 llvm::AtomicOrdering AO) {
13369 llvm_unreachable("Not supported in SIMD-only mode");
13370}
13371
13372void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
13373 const OMPExecutableDirective &D,
13374 llvm::Function *TaskFunction,
13375 QualType SharedsTy, Address Shareds,
13376 const Expr *IfCond,
13377 const OMPTaskDataTy &Data) {
13378 llvm_unreachable("Not supported in SIMD-only mode");
13379}
13380
13381void CGOpenMPSIMDRuntime::emitTaskLoopCall(
13382 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
13383 llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
13384 const Expr *IfCond, const OMPTaskDataTy &Data) {
13385 llvm_unreachable("Not supported in SIMD-only mode");
13386}
13387
13388void CGOpenMPSIMDRuntime::emitReduction(
13389 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates,
13390 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs,
13391 ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) {
13392 assert(Options.SimpleReduction && "Only simple reduction is expected.");
13393 CGOpenMPRuntime::emitReduction(CGF, Loc, OrgPrivates: Privates, OrgLHSExprs: LHSExprs, OrgRHSExprs: RHSExprs,
13394 OrgReductionOps: ReductionOps, Options);
13395}
13396
13397llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit(
13398 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs,
13399 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) {
13400 llvm_unreachable("Not supported in SIMD-only mode");
13401}
13402
13403void CGOpenMPSIMDRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
13404 SourceLocation Loc,
13405 bool IsWorksharingReduction) {
13406 llvm_unreachable("Not supported in SIMD-only mode");
13407}
13408
13409void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF,
13410 SourceLocation Loc,
13411 ReductionCodeGen &RCG,
13412 unsigned N) {
13413 llvm_unreachable("Not supported in SIMD-only mode");
13414}
13415
13416Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
13417 SourceLocation Loc,
13418 llvm::Value *ReductionsPtr,
13419 LValue SharedLVal) {
13420 llvm_unreachable("Not supported in SIMD-only mode");
13421}
13422
13423void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
13424 SourceLocation Loc,
13425 const OMPTaskDataTy &Data) {
13426 llvm_unreachable("Not supported in SIMD-only mode");
13427}
13428
13429void CGOpenMPSIMDRuntime::emitCancellationPointCall(
13430 CodeGenFunction &CGF, SourceLocation Loc,
13431 OpenMPDirectiveKind CancelRegion) {
13432 llvm_unreachable("Not supported in SIMD-only mode");
13433}
13434
13435void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF,
13436 SourceLocation Loc, const Expr *IfCond,
13437 OpenMPDirectiveKind CancelRegion) {
13438 llvm_unreachable("Not supported in SIMD-only mode");
13439}
13440
13441void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(
13442 const OMPExecutableDirective &D, StringRef ParentName,
13443 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
13444 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
13445 llvm_unreachable("Not supported in SIMD-only mode");
13446}
13447
13448void CGOpenMPSIMDRuntime::emitTargetCall(
13449 CodeGenFunction &CGF, const OMPExecutableDirective &D,
13450 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
13451 llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
13452 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
13453 const OMPLoopDirective &D)>
13454 SizeEmitter) {
13455 llvm_unreachable("Not supported in SIMD-only mode");
13456}
13457
13458bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) {
13459 llvm_unreachable("Not supported in SIMD-only mode");
13460}
13461
13462bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) {
13463 llvm_unreachable("Not supported in SIMD-only mode");
13464}
13465
13466bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) {
13467 return false;
13468}
13469
13470void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF,
13471 const OMPExecutableDirective &D,
13472 SourceLocation Loc,
13473 llvm::Function *OutlinedFn,
13474 ArrayRef<llvm::Value *> CapturedVars) {
13475 llvm_unreachable("Not supported in SIMD-only mode");
13476}
13477
13478void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
13479 const Expr *NumTeams,
13480 const Expr *ThreadLimit,
13481 SourceLocation Loc) {
13482 llvm_unreachable("Not supported in SIMD-only mode");
13483}
13484
13485void CGOpenMPSIMDRuntime::emitTargetDataCalls(
13486 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13487 const Expr *Device, const RegionCodeGenTy &CodeGen,
13488 CGOpenMPRuntime::TargetDataInfo &Info) {
13489 llvm_unreachable("Not supported in SIMD-only mode");
13490}
13491
13492void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall(
13493 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
13494 const Expr *Device) {
13495 llvm_unreachable("Not supported in SIMD-only mode");
13496}
13497
13498void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF,
13499 const OMPLoopDirective &D,
13500 ArrayRef<Expr *> NumIterations) {
13501 llvm_unreachable("Not supported in SIMD-only mode");
13502}
13503
13504void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
13505 const OMPDependClause *C) {
13506 llvm_unreachable("Not supported in SIMD-only mode");
13507}
13508
13509void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF,
13510 const OMPDoacrossClause *C) {
13511 llvm_unreachable("Not supported in SIMD-only mode");
13512}
13513
13514const VarDecl *
13515CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD,
13516 const VarDecl *NativeParam) const {
13517 llvm_unreachable("Not supported in SIMD-only mode");
13518}
13519
13520Address
13521CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF,
13522 const VarDecl *NativeParam,
13523 const VarDecl *TargetParam) const {
13524 llvm_unreachable("Not supported in SIMD-only mode");
13525}
13526